diff options
author | thegeorg <thegeorg@yandex-team.com> | 2022-08-19 15:00:44 +0300 |
---|---|---|
committer | thegeorg <thegeorg@yandex-team.com> | 2022-08-19 15:00:44 +0300 |
commit | ad2a1b622d2bf6cf025982846153d9c4c791af2c (patch) | |
tree | 8906addc18a494ece9dff28b2701a37ef4b52bf8 | |
parent | 7b61b052f3baa7e43edca48c373f95b5e5f1c845 (diff) | |
download | ydb-ad2a1b622d2bf6cf025982846153d9c4c791af2c.tar.gz |
Let cmake export determine which build/scripts are mandatory
30 files changed, 0 insertions, 3120 deletions
diff --git a/build/scripts/cgo1_wrapper.py b/build/scripts/cgo1_wrapper.py deleted file mode 100644 index 986082f7e9..0000000000 --- a/build/scripts/cgo1_wrapper.py +++ /dev/null @@ -1,45 +0,0 @@ -import argparse -import shutil -import subprocess -import sys - - -CGO1_SUFFIX='.cgo1.go' - - -def call(cmd, cwd, env=None): - # sys.stderr.write('{}\n'.format(' '.join(cmd))) - return subprocess.call(cmd, stdin=None, stderr=sys.stderr, stdout=sys.stdout, cwd=cwd, env=env) - - -def process_file(source_root, source_prefix, build_root, build_prefix, src_path, comment_prefix): - dst_path = '{}.tmp'.format(src_path) - with open(src_path, 'r') as src_file, open(dst_path, 'w') as dst_file: - for line in src_file: - if line.startswith(comment_prefix): - dst_file.write(line.replace(source_root, source_prefix).replace(build_root, build_prefix)) - else: - dst_file.write(line) - shutil.move(dst_path, src_path) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument('--build-prefix', default='__ARCADIA_BUILD_ROOT_PREFIX__') - parser.add_argument('--build-root', required=True) - parser.add_argument('--cgo1-files', nargs='+', required=True) - parser.add_argument('--cgo2-files', nargs='+', required=True) - parser.add_argument('--source-prefix', default='__ARCADIA_SOURCE_ROOT_PREFIX__') - parser.add_argument('--source-root', required=True) - parser.add_argument('cgo1_cmd', nargs='*') - args = parser.parse_args() - - exit_code = call(args.cgo1_cmd, args.source_root) - if exit_code != 0: - sys.exit(exit_code) - - for src_path in args.cgo1_files: - process_file(args.source_root, args.source_prefix, args.build_root, args.build_prefix, src_path, '//') - - for src_path in args.cgo2_files: - process_file(args.source_root, args.source_prefix, args.build_root, args.build_prefix, src_path, '#line') diff --git a/build/scripts/check_config_h.py b/build/scripts/check_config_h.py deleted file mode 100644 index 07bc12e230..0000000000 --- a/build/scripts/check_config_h.py +++ /dev/null @@ -1,89 +0,0 @@ -import sys - -data = """ -#if defined(SIZEOF_LONG) -static_assert(sizeof(long) == SIZEOF_LONG, "fixme 1"); -#endif - -#if defined(SIZEOF_PTHREAD_T) -#include <pthread.h> - -static_assert(sizeof(pthread_t) == SIZEOF_PTHREAD_T, "fixme 2"); -#endif - -#if defined(SIZEOF_SIZE_T) -#include <stddef.h> - -static_assert(sizeof(size_t) == SIZEOF_SIZE_T, "fixme 3"); -#endif - -#if defined(SIZEOF_TIME_T) -#include <time.h> - -static_assert(sizeof(time_t) == SIZEOF_TIME_T, "fixme 4"); -#endif - -#if defined(SIZEOF_UINTPTR_T) -#include <stdint.h> - -static_assert(sizeof(uintptr_t) == SIZEOF_UINTPTR_T, "fixme 5"); -#endif - -#if defined(SIZEOF_VOID_P) -static_assert(sizeof(void*) == SIZEOF_VOID_P, "fixme 6"); -#endif - -#if defined(SIZEOF_FPOS_T) -#include <stdio.h> - -static_assert(sizeof(fpos_t) == SIZEOF_FPOS_T, "fixme 7"); -#endif - -#if defined(SIZEOF_DOUBLE) -static_assert(sizeof(double) == SIZEOF_DOUBLE, "fixme 8"); -#endif - -#if defined(SIZEOF_LONG_DOUBLE) -static_assert(sizeof(long double) == SIZEOF_LONG_DOUBLE, "fixme 9"); -#endif - -#if defined(SIZEOF_FLOAT) -static_assert(sizeof(float) == SIZEOF_FLOAT, "fixme 10"); -#endif - -#if defined(SIZEOF_INT) -static_assert(sizeof(int) == SIZEOF_INT, "fixme 11"); -#endif - -#if defined(SIZEOF_LONG_LONG) -static_assert(sizeof(long long) == SIZEOF_LONG_LONG, "fixme 12"); -#endif - -#if defined(SIZEOF_OFF_T) -#include <stdio.h> - -static_assert(sizeof(off_t) == SIZEOF_OFF_T, "fixme 13"); -#endif - -#if defined(SIZEOF_PID_T) -#include <unistd.h> - -static_assert(sizeof(pid_t) == SIZEOF_PID_T, "fixme 14"); -#endif - -#if defined(SIZEOF_SHORT) -static_assert(sizeof(short) == SIZEOF_SHORT, "fixme 15"); -#endif - -#if defined(SIZEOF_WCHAR_T) -static_assert(sizeof(wchar_t) == SIZEOF_WCHAR_T, "fixme 16"); -#endif - -#if defined(SIZEOF__BOOL) -//TODO -#endif -""" -if __name__ == '__main__': - with open(sys.argv[2], 'w') as f: - f.write('#include <' + sys.argv[1] + '>\n\n') - f.write(data) diff --git a/build/scripts/clang_wrapper.py b/build/scripts/clang_wrapper.py deleted file mode 100644 index af3869f789..0000000000 --- a/build/scripts/clang_wrapper.py +++ /dev/null @@ -1,53 +0,0 @@ -import subprocess -import sys - - -def fix(s): - # disable dbg DEVTOOLS-2744 - if s == '-g': - return None - if s == '/Z7' or s == '/Od' or s == '/Ob0' or s == '/D_DEBUG': - return None - - # disable sanitizers for generated code - if s.startswith('-fsanitize') or s == '-Dmemory_sanitizer_enabled' or s.startswith('-fsanitize-blacklist'): - return None - - # strip gcc toolchain flags (appear when crosscompiling) - if s.startswith('-fabi-version'): - return None - - # remove arguments unknown to clang-cl - if s == '-fcase-insensitive-paths': # or s == '-fno-lto': # DEVTOOLSSUPPORT-3966 - return None - - # Paths under .ya/tools/v3/.../msvc/include are divided with '\' - return s.replace('\\', '/') - - -def fix_path(p): - try: - i = p.rfind('/bin/clang') - p = p[:i] + '/bin/clang-cl' - except ValueError: - pass - return p - - -if __name__ == '__main__': - is_on_win = sys.argv[1] == 'yes' - path = sys.argv[2] - args = filter(None, [fix(s) for s in sys.argv[3:]]) - if is_on_win: - path = fix_path(path) - try: - i = args.index('-emit-llvm') - args[i:i+1] = ['-Xclang', '-emit-llvm'] - except ValueError: - pass - args.append('-fms-compatibility-version=19') - - cmd = [path] + args - - rc = subprocess.call(cmd, shell=False, stderr=sys.stderr, stdout=sys.stdout) - sys.exit(rc) diff --git a/build/scripts/compile_java.py b/build/scripts/compile_java.py deleted file mode 100644 index e95869e853..0000000000 --- a/build/scripts/compile_java.py +++ /dev/null @@ -1,102 +0,0 @@ -import optparse -import contextlib -import os -import shutil -import subprocess as sp -import tarfile -import zipfile -import sys - - -def parse_args(args): - parser = optparse.OptionParser() - parser.add_option('--javac-bin') - parser.add_option('--jar-bin') - parser.add_option('--vcs-mf') - parser.add_option('--package-prefix') - parser.add_option('--jar-output') - parser.add_option('--srcs-jar-output') - return parser.parse_args(args) - - -def mkdir_p(directory): - if not os.path.exists(directory): - os.makedirs(directory) - - -def split_cmd_by_delim(cmd, delim='DELIM'): - result = [[]] - for arg in cmd: - if arg == delim: - result.append([]) - else: - result[-1].append(arg) - return result - - -def main(): - cmd_parts = split_cmd_by_delim(sys.argv) - assert len(cmd_parts) == 3 - args, javac_opts, peers = cmd_parts - opts, jsrcs = parse_args(args) - - jsrcs += list(filter(lambda x: x.endswith('.jsrc'), peers)) - peers = list(filter(lambda x: not x.endswith('.jsrc'), peers)) - - sources_dir = 'src' - mkdir_p(sources_dir) - for s in jsrcs: - if s.endswith('.jsrc'): - with contextlib.closing(tarfile.open(s, 'r')) as tf: - tf.extractall(sources_dir) - - srcs = [] - for r, _, files in os.walk(sources_dir): - for f in files: - srcs.append(os.path.join(r, f)) - srcs += jsrcs - srcs = list(filter(lambda x: x.endswith('.java'), srcs)) - - classes_dir = 'cls' - mkdir_p(classes_dir) - classpath = os.pathsep.join(peers) - - if srcs: - temp_sources_file = 'temp.sources.list' - with open(temp_sources_file, 'w') as ts: - ts.write(' '.join(srcs)) - sp.check_call([opts.javac_bin, '-nowarn', '-g', '-classpath', classpath, '-encoding', 'UTF-8', '-d', classes_dir] + javac_opts + ['@' + temp_sources_file]) - - for s in jsrcs: - if s.endswith('-sources.jar'): - with zipfile.ZipFile(s) as zf: - zf.extractall(sources_dir) - - elif s.endswith('.jar'): - with zipfile.ZipFile(s) as zf: - zf.extractall(classes_dir) - - if opts.vcs_mf: - sp.check_call([opts.jar_bin, 'cfm', opts.jar_output, opts.vcs_mf, os.curdir], cwd=classes_dir) - else: - sp.check_call([opts.jar_bin, 'cfM', opts.jar_output, os.curdir], cwd=classes_dir) - - if opts.srcs_jar_output: - for s in jsrcs: - if s.endswith('.java'): - if opts.package_prefix: - d = os.path.join(sources_dir, *(opts.package_prefix.split('.') + [os.path.basename(s)])) - - else: - d = os.path.join(sources_dir, os.path.basename(s)) - - shutil.copyfile(s, d) - - if opts.vcs_mf: - sp.check_call([opts.jar_bin, 'cfm', opts.srcs_jar_output, opts.vcs_mf, os.curdir], cwd=sources_dir) - else: - sp.check_call([opts.jar_bin, 'cfM', opts.srcs_jar_output, os.curdir], cwd=sources_dir) - - -if __name__ == '__main__': - main() diff --git a/build/scripts/configure_file.py b/build/scripts/configure_file.py deleted file mode 100755 index 1873ed70eb..0000000000 --- a/build/scripts/configure_file.py +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env python2.7 - -import sys -import os.path -import re - -cmakeDef01 = "#cmakedefine01" -cmakeDef = "#cmakedefine" - - -def replaceLine(l, varDict, define): - words = l.split() - if words: - if words[0] == cmakeDef: - sPos = l.find(cmakeDef) - ePos = sPos + len(cmakeDef) - l = l[:sPos] + define + l[ePos:] + '\n' - if words[0] == cmakeDef01: - var = words[1] - cmakeValue = varDict.get(var) - if cmakeValue == 'yes': - val = '1' - else: - val = '0' - sPos = l.find(cmakeDef01) - ePos = l.find(var) + len(var) - l = l[:sPos] + define + ' ' + var + ' ' + val + l[ePos + 1:] + '\n' - - finder = re.compile(".*?(@[a-zA-Z0-9_]+@).*") - while True: - re_result = finder.match(l) - if not re_result: - return l - key = re_result.group(1)[1:-1] - l = l[:re_result.start(1)] + varDict.get(key, '') + l[re_result.end(1):] - - -def main(inputPath, outputPath, varDict): - define = '#define' if os.path.splitext(outputPath)[1] != '.asm' else '%define' - with open(outputPath, 'w') as output: - with open(inputPath, 'r') as input: - for l in input: - output.write(replaceLine(l, varDict, define)) - - -def usage(): - print "usage: configure_file.py inputPath outputPath key1=value1 ..." - exit(1) - - -if __name__ == "__main__": - if len(sys.argv) < 3: - usage() - varDict = {} - for x in sys.argv[3:]: - key, value = str(x).split('=', 1) - varDict[key] = value - - main(sys.argv[1], sys.argv[2], varDict) diff --git a/build/scripts/copy_docs_files_to_dir.py b/build/scripts/copy_docs_files_to_dir.py deleted file mode 100644 index 27fd171ee6..0000000000 --- a/build/scripts/copy_docs_files_to_dir.py +++ /dev/null @@ -1,138 +0,0 @@ -import argparse -import errno -import os -import process_command_files as pcf -import shutil -import sys - - -def parse_args(): - parser = argparse.ArgumentParser() - parser.add_argument('--bin-dir', nargs='*') - parser.add_argument('--build-root', required=True) - parser.add_argument('--dest-dir', required=True) - parser.add_argument('--docs-dir', action='append', nargs=2, dest='docs_dirs', default=None) - parser.add_argument('--existing', choices=('skip', 'overwrite'), default='overwrite') - parser.add_argument('--source-root', required=True) - parser.add_argument('--src-dir', action='append', nargs='*', dest='src_dirs', default=None) - parser.add_argument('files', nargs='*') - return parser.parse_args(pcf.get_args(sys.argv[1:])) - - -def makedirs(dirname): - try: - os.makedirs(dirname) - except OSError as e: - if e.errno == errno.EEXIST and os.path.isdir(dirname): - pass - else: - raise - - -def copy_file(src, dst, overwrite=False, orig_path=None, generated=False): - if os.path.exists(dst) and not overwrite: - return - - makedirs(os.path.dirname(dst)) - - with open(src, 'r') as fsrc, open(dst, 'w') as fdst: - # if (orig_path or generated) and src.endswith('.md'): - # fdst.write('---\n{}\n\n---\n'.format('generated: true' if generated else 'vcsPath: {}'.format(orig_path))) - shutil.copyfileobj(fsrc, fdst) - - -def main(): - args = parse_args() - - dest_dir = os.path.normpath(args.dest_dir) - makedirs(dest_dir) - - source_root = os.path.normpath(args.source_root) + os.path.sep - build_root = os.path.normpath(args.build_root) + os.path.sep - - is_overwrite_existing = args.existing == 'overwrite' - - if args.docs_dirs: - for item in args.docs_dirs: - assert len(item) == 2 - docs_dir, nm = item[0], item[1] - assert not os.path.isabs(docs_dir) - if nm and nm != '.': - assert not os.path.isabs(nm) - dst = os.path.join(dest_dir, nm) - else: - dst = dest_dir - - abs_docs_dir = os.path.join(args.source_root, docs_dir) - - for root, _, files in os.walk(abs_docs_dir): - for f in files: - if os.path.islink(os.path.join(root, f)): - continue - file_src = os.path.join(root, f) - assert file_src.startswith(source_root) - file_dst = os.path.join(dst, os.path.relpath(root, abs_docs_dir), f) - copy_file(file_src, file_dst, overwrite=is_overwrite_existing, orig_path=file_src[len(source_root):]) - - if args.src_dirs: - for item in args.src_dirs: - assert len(item) > 1 - src_dir, nm = os.path.normpath(item[0]), item[1] - assert os.path.isabs(src_dir) - if nm and nm != '.': - assert not os.path.isabs(nm) - dst = os.path.join(dest_dir, nm) - else: - dst = dest_dir - - if src_dir.startswith(source_root): - root = source_root - is_from_source_root = True - else: - assert src_dir.startswith(build_root) - root = build_root - is_from_source_root = False - - for f in item[2:]: - file_src = os.path.normpath(f) - assert file_src.startswith(root) - rel_path = file_src[len(root):] if is_from_source_root else None - file_dst = os.path.join(dst, file_src[len(src_dir):]) - copy_file(file_src, file_dst, overwrite=is_overwrite_existing, orig_path=rel_path) - - if args.bin_dir: - assert len(args.bin_dir) > 1 - bin_dir, bin_dir_namespace = os.path.normpath(args.bin_dir[0]) + os.path.sep, args.bin_dir[1] - assert bin_dir.startswith(build_root) - if bin_dir_namespace and bin_dir_namespace != '.': - assert not os.path.isabs(bin_dir_namespace) - dst = os.path.join(dest_dir, bin_dir_namespace) - else: - dst = dest_dir - - for file_src in args.bin_dir[2:]: - assert os.path.isfile(file_src) - assert file_src.startswith(bin_dir) - file_dst = os.path.join(dst, file_src[len(bin_dir):]) - copy_file(file_src, file_dst, overwrite=is_overwrite_existing, orig_path=None) - - for src in args.files: - generated = False - file_src = os.path.normpath(src) - assert os.path.isfile(file_src), 'File [{}] does not exist...'.format(file_src) - rel_path = file_src - if file_src.startswith(source_root): - rel_path = file_src[len(source_root):] - elif file_src.startswith(build_root): - # generated = True - # rel_path = file_src[len(build_root):] - rel_path = None - else: - raise Exception('Unexpected file path [{}].'.format(file_src)) - assert not os.path.isabs(rel_path) - file_dst = os.path.join(args.dest_dir, rel_path) - copy_file(file_src, file_dst, is_overwrite_existing, rel_path, generated) - - -if __name__ == '__main__': - main() diff --git a/build/scripts/extract_docs.py b/build/scripts/extract_docs.py deleted file mode 100644 index 20e8311346..0000000000 --- a/build/scripts/extract_docs.py +++ /dev/null @@ -1,43 +0,0 @@ -import argparse -import os -import process_command_files as pcf -import tarfile -import sys - - -def parse_args(): - parser = argparse.ArgumentParser() - parser.add_argument('--dest-dir', required=True) - parser.add_argument('--skip-prefix', dest='skip_prefixes', action='append', default=[]) - parser.add_argument('docs', nargs='*') - return parser.parse_args(pcf.get_args(sys.argv[1:])) - - -def main(): - args = parse_args() - - prefixes = ['{}{}'.format(os.path.normpath(p), os.path.sep) for p in args.skip_prefixes] - - def _valid_docslib(path): - base = os.path.basename(path) - return base.endswith(('.docslib', '.docslib.fake')) or base == 'preprocessed.tar.gz' - - for src in [p for p in args.docs if _valid_docslib(p)]: - if src == 'preprocessed.tar.gz': - rel_dst = os.path.dirname(os.path.normpath(src)) - for prefix in prefixes: - if src.startswith(prefix): - rel_dst = rel_dst[len(prefix):] - continue - assert not os.path.isabs(rel_dst) - dest_dir = os.path.join(args.dest_dir, rel_dst) - else: - dest_dir = args.dest_dir - if not os.path.exists(dest_dir): - os.makedirs(dest_dir) - with tarfile.open(src, 'r') as tar_file: - tar_file.extractall(dest_dir) - - -if __name__ == '__main__': - main() diff --git a/build/scripts/fetch_from.py b/build/scripts/fetch_from.py deleted file mode 100755 index aa04f816b5..0000000000 --- a/build/scripts/fetch_from.py +++ /dev/null @@ -1,375 +0,0 @@ -import datetime as dt -import errno -import hashlib -import json -import logging -import os -import platform -import random -import shutil -import socket -import string -import sys -import tarfile -import urllib2 - -import retry - - -def make_user_agent(): - return 'fetch_from: {host}'.format(host=socket.gethostname()) - - -def add_common_arguments(parser): - parser.add_argument('--copy-to') # used by jbuild in fetch_resource - parser.add_argument('--rename-to') # used by test_node in inject_mds_resource_to_graph - parser.add_argument('--copy-to-dir') - parser.add_argument('--untar-to') - parser.add_argument('--rename', action='append', default=[], metavar='FILE', help='rename FILE to the corresponding output') - parser.add_argument('--executable', action='store_true', help='make outputs executable') - parser.add_argument('--log-path') - parser.add_argument('-v', '--verbose', action='store_true', default=os.environ.get('YA_VERBOSE_FETCHER'), help='increase stderr verbosity') - parser.add_argument('outputs', nargs='*', default=[]) - - -def ensure_dir(path): - if not (path == '' or os.path.isdir(path)): - os.makedirs(path) - - -# Reference code: library/python/fs/__init__.py -def hardlink_or_copy(src, dst): - ensure_dir(os.path.dirname(dst)) - - if os.name == 'nt': - shutil.copy(src, dst) - else: - try: - os.link(src, dst) - except OSError as e: - if e.errno == errno.EEXIST: - return - elif e.errno in (errno.EXDEV, errno.EMLINK, errno.EINVAL, errno.EACCES): - sys.stderr.write("Can't make hardlink (errno={}) - fallback to copy: {} -> {}\n".format(e.errno, src, dst)) - shutil.copy(src, dst) - else: - raise - - -def rename_or_copy_and_remove(src, dst): - ensure_dir(os.path.dirname(dst)) - - try: - os.rename(src, dst) - except OSError: - shutil.copy(src, dst) - os.remove(src) - - -class BadChecksumFetchError(Exception): - pass - - -class IncompleteFetchError(Exception): - pass - - -class ResourceUnpackingError(Exception): - pass - - -class ResourceIsDirectoryError(Exception): - pass - - -class OutputIsDirectoryError(Exception): - pass - - -class OutputNotExistError(Exception): - pass - - -def setup_logging(args, base_name): - def makedirs(path): - try: - os.makedirs(path) - except OSError: - pass - - if args.log_path: - log_file_name = args.log_path - else: - log_file_name = base_name + ".log" - - args.abs_log_path = os.path.abspath(log_file_name) - makedirs(os.path.dirname(args.abs_log_path)) - logging.basicConfig(filename=args.abs_log_path, level=logging.DEBUG) - if args.verbose: - logging.getLogger().addHandler(logging.StreamHandler(sys.stderr)) - - -def is_temporary(e): - - def is_broken(e): - return isinstance(e, urllib2.HTTPError) and e.code in (410, 404) - - if is_broken(e): - return False - - if isinstance(e, (BadChecksumFetchError, IncompleteFetchError, urllib2.URLError, socket.error)): - return True - - import error - - return error.is_temporary_error(e) - - -def uniq_string_generator(size=6, chars=string.ascii_lowercase + string.digits): - return ''.join(random.choice(chars) for _ in range(size)) - - -def report_to_snowden(value): - def inner(): - body = { - 'namespace': 'ygg', - 'key': 'fetch-from-sandbox', - 'value': json.dumps(value), - } - - urllib2.urlopen( - 'https://back-snowden.qloud.yandex-team.ru/report/add', - json.dumps([body, ]), - timeout=5, - ) - - try: - inner() - except Exception as e: - logging.warning('report_to_snowden failed: %s', e) - - -def copy_stream(read, *writers, **kwargs): - chunk_size = kwargs.get('size', 1024*1024) - while True: - data = read(chunk_size) - if not data: - break - for write in writers: - write(data) - - -def md5file(fname): - res = hashlib.md5() - with open(fname, 'rb') as f: - copy_stream(f.read, res.update) - return res.hexdigest() - - -def git_like_hash_with_size(filepath): - """ - Calculate git like hash for path - """ - sha = hashlib.sha1() - - file_size = 0 - - with open(filepath, 'rb') as f: - while True: - block = f.read(2 ** 16) - - if not block: - break - - file_size += len(block) - sha.update(block) - - sha.update('\0') - sha.update(str(file_size)) - - return sha.hexdigest(), file_size - - -def size_printer(display_name, size): - sz = [0] - last_stamp = [dt.datetime.now()] - - def printer(chunk): - sz[0] += len(chunk) - now = dt.datetime.now() - if last_stamp[0] + dt.timedelta(seconds=10) < now: - if size: - print >>sys.stderr, "##status##{} - [[imp]]{:.1f}%[[rst]]".format(display_name, 100.0 * sz[0] / size if size else 0) - last_stamp[0] = now - - return printer - - -def fetch_url(url, unpack, resource_file_name, expected_md5=None, expected_sha1=None, tries=10, writers=None): - logging.info('Downloading from url %s name %s and expected md5 %s', url, resource_file_name, expected_md5) - tmp_file_name = uniq_string_generator() - - request = urllib2.Request(url, headers={'User-Agent': make_user_agent()}) - req = retry.retry_func(lambda: urllib2.urlopen(request, timeout=30), tries=tries, delay=5, backoff=1.57079) - logging.debug('Headers: %s', req.headers.headers) - expected_file_size = int(req.headers.get('Content-Length', 0)) - real_md5 = hashlib.md5() - real_sha1 = hashlib.sha1() - - with open(tmp_file_name, 'wb') as fp: - copy_stream( - req.read, - fp.write, - real_md5.update, - real_sha1.update, - size_printer(resource_file_name, expected_file_size), - *([] if writers is None else writers) - ) - - real_md5 = real_md5.hexdigest() - real_file_size = os.path.getsize(tmp_file_name) - real_sha1.update('\0') - real_sha1.update(str(real_file_size)) - real_sha1 = real_sha1.hexdigest() - - if unpack: - tmp_dir = tmp_file_name + '.dir' - os.makedirs(tmp_dir) - with tarfile.open(tmp_file_name, mode="r|gz") as tar: - tar.extractall(tmp_dir) - tmp_file_name = os.path.join(tmp_dir, resource_file_name) - real_md5 = md5file(tmp_file_name) - - logging.info('File size %s (expected %s)', real_file_size, expected_file_size or "UNKNOWN") - logging.info('File md5 %s (expected %s)', real_md5, expected_md5) - logging.info('File sha1 %s (expected %s)', real_sha1, expected_sha1) - - if expected_md5 and real_md5 != expected_md5: - report_to_snowden( - { - 'headers': req.headers.headers, - 'expected_md5': expected_md5, - 'real_md5': real_md5 - } - ) - - raise BadChecksumFetchError( - 'Downloaded {}, but expected {} for {}'.format( - real_md5, - expected_md5, - url, - ) - ) - - if expected_sha1 and real_sha1 != expected_sha1: - report_to_snowden( - { - 'headers': req.headers.headers, - 'expected_sha1': expected_sha1, - 'real_sha1': real_sha1 - } - ) - - raise BadChecksumFetchError( - 'Downloaded {}, but expected {} for {}'.format( - real_sha1, - expected_sha1, - url, - ) - ) - - if expected_file_size and expected_file_size != real_file_size: - report_to_snowden({'headers': req.headers.headers, 'file_size': real_file_size}) - - raise IncompleteFetchError( - 'Downloaded {}, but expected {} for {}'.format( - real_file_size, - expected_file_size, - url, - ) - ) - - return tmp_file_name - - -def chmod(filename, mode): - if platform.system().lower() == 'windows': - # https://docs.microsoft.com/en-us/windows/win32/fileio/hard-links-and-junctions: - # hard to reset read-only attribute for removal if there are multiple hardlinks - return - stat = os.stat(filename) - if stat.st_mode & 0o777 != mode: - try: - os.chmod(filename, mode) - except OSError: - import pwd - sys.stderr.write("{} st_mode: {} pwuid: {}\n".format(filename, stat.st_mode, pwd.getpwuid(os.stat(filename).st_uid))) - raise - - -def process(fetched_file, file_name, args, remove=True): - assert len(args.rename) <= len(args.outputs), ( - 'too few outputs to rename', args.rename, 'into', args.outputs) - - # Forbid changes to the loaded resource - chmod(fetched_file, 0o444) - - if not os.path.isfile(fetched_file): - raise ResourceIsDirectoryError('Resource must be a file, not a directory: %s' % fetched_file) - - if args.copy_to: - hardlink_or_copy(fetched_file, args.copy_to) - if not args.outputs: - args.outputs = [args.copy_to] - - if args.rename_to: - args.rename.append(fetched_file) - if not args.outputs: - args.outputs = [args.rename_to] - - if args.copy_to_dir: - hardlink_or_copy(fetched_file, os.path.join(args.copy_to_dir, file_name)) - - if args.untar_to: - ensure_dir(args.untar_to) - # Extract only requested files - try: - with tarfile.open(fetched_file, mode='r:*') as tar: - inputs = set(map(os.path.normpath, args.rename + args.outputs[len(args.rename):])) - members = [entry for entry in tar if os.path.normpath(os.path.join(args.untar_to, entry.name)) in inputs] - tar.extractall(args.untar_to, members=members) - # Forbid changes to the loaded resource data - for root, _, files in os.walk(args.untar_to): - for filename in files: - chmod(os.path.join(root, filename), 0o444) - except tarfile.ReadError as e: - logging.exception(e) - raise ResourceUnpackingError('File {} cannot be untared'.format(fetched_file)) - - for src, dst in zip(args.rename, args.outputs): - if src == 'RESOURCE': - src = fetched_file - if os.path.abspath(src) == os.path.abspath(fetched_file): - logging.info('Copying %s to %s', src, dst) - hardlink_or_copy(src, dst) - else: - logging.info('Renaming %s to %s', src, dst) - if os.path.exists(dst): - raise ResourceUnpackingError("Target file already exists ({} -> {})".format(src, dst)) - if remove: - rename_or_copy_and_remove(src, dst) - else: - hardlink_or_copy(src, dst) - - for path in args.outputs: - if not os.path.exists(path): - raise OutputNotExistError('Output does not exist: %s' % os.path.abspath(path)) - if not os.path.isfile(path): - raise OutputIsDirectoryError('Output must be a file, not a directory: %s' % os.path.abspath(path)) - if args.executable: - chmod(path, os.stat(path).st_mode | 0o111) - if os.path.abspath(path) == os.path.abspath(fetched_file): - remove = False - - if remove: - os.remove(fetched_file) diff --git a/build/scripts/fetch_from_sandbox.py b/build/scripts/fetch_from_sandbox.py deleted file mode 100755 index a99542e174..0000000000 --- a/build/scripts/fetch_from_sandbox.py +++ /dev/null @@ -1,269 +0,0 @@ -import itertools -import json -import logging -import argparse -import os -import random -import subprocess -import sys -import time -import urllib2 -import uuid - -import fetch_from - - -ORIGIN_SUFFIX = '?origin=fetch-from-sandbox' -MDS_PREFIX = 'http://storage-int.mds.yandex.net/get-sandbox/' -TEMPORARY_ERROR_CODES = (429, 500, 503, 504) - - -def parse_args(): - parser = argparse.ArgumentParser() - fetch_from.add_common_arguments(parser) - parser.add_argument('--resource-id', type=int, required=True) - parser.add_argument('--custom-fetcher') - parser.add_argument('--resource-file') - return parser.parse_args() - - -class ResourceInfoError(Exception): - pass - - -class UnsupportedProtocolException(Exception): - pass - - -def _sky_path(): - return "/usr/local/bin/sky" - - -def _is_skynet_avaliable(): - if not os.path.exists(_sky_path()): - return False - try: - subprocess.check_output([_sky_path(), "--version"]) - return True - except subprocess.CalledProcessError: - return False - except OSError: - return False - - -def download_by_skynet(resource_info, file_name): - def sky_get(skynet_id, target_dir, timeout=None): - cmd_args = [_sky_path(), 'get', "-N", "Backbone", "--user", "--wait", "--dir", target_dir, skynet_id] - if timeout is not None: - cmd_args += ["--timeout", str(timeout)] - logging.info('Call skynet with args: %s', cmd_args) - stdout = subprocess.check_output(cmd_args).strip() - logging.debug('Skynet call with args %s is finished, result is %s', cmd_args, stdout) - return stdout - - if not _is_skynet_avaliable(): - raise UnsupportedProtocolException("Skynet is not available") - - skynet_id = resource_info.get("skynet_id") - if not skynet_id: - raise ValueError("Resource does not have skynet_id") - - temp_dir = os.path.abspath(fetch_from.uniq_string_generator()) - os.mkdir(temp_dir) - sky_get(skynet_id, temp_dir) - return os.path.join(temp_dir, file_name) - - -def _urlopen(url, data=None, headers=None): - n = 10 - tout = 30 - started = time.time() - reqid = uuid.uuid4() - - request = urllib2.Request(url, data=data, headers=headers or {}) - request.add_header('X-Request-Timeout', str(tout)) - request.add_header('X-Request-Id', str(reqid)) - request.add_header('User-Agent', 'fetch_from_sandbox.py') - for i in xrange(n): - retry_after = i - try: - request.add_header('X-Request-Duration', str(int(time.time() - started))) - return urllib2.urlopen(request, timeout=tout).read() - - except urllib2.HTTPError as e: - logging.warning('failed to fetch URL %s with HTTP code %d: %s', url, e.code, e) - retry_after = int(e.headers.get('Retry-After', str(retry_after))) - - if e.code not in TEMPORARY_ERROR_CODES: - raise - - except Exception as e: - logging.warning('failed to fetch URL %s: %s', url, e) - - if i + 1 == n: - raise e - - time.sleep(retry_after) - - -def _query(url): - return json.loads(_urlopen(url)) - - -_SANDBOX_BASE_URL = 'https://sandbox.yandex-team.ru/api/v1.0' - - -def get_resource_info(resource_id, touch=False, no_links=False): - url = ''.join((_SANDBOX_BASE_URL, '/resource/', str(resource_id))) - headers = {} - if touch: - headers.update({'X-Touch-Resource': '1'}) - if no_links: - headers.update({'X-No-Links': '1'}) - return _query(url) - - -def get_resource_http_links(resource_id): - url = ''.join((_SANDBOX_BASE_URL, '/resource/', str(resource_id), '/data/http')) - return [r['url'] + ORIGIN_SUFFIX for r in _query(url)] - - -def fetch_via_script(script, resource_id): - return subprocess.check_output([script, str(resource_id)]).rstrip() - - -def fetch(resource_id, custom_fetcher): - try: - resource_info = get_resource_info(resource_id, touch=True, no_links=True) - except Exception as e: - sys.stderr.write( - "Failed to fetch resource {}: {}\n".format(resource_id, str(e)) - ) - raise - - if resource_info.get('state', 'DELETED') != 'READY': - raise ResourceInfoError("Resource {} is not READY".format(resource_id)) - - logging.info('Resource %s info %s', str(resource_id), json.dumps(resource_info)) - - resource_file_name = os.path.basename(resource_info["file_name"]) - expected_md5 = resource_info.get('md5') - - proxy_link = resource_info['http']['proxy'] + ORIGIN_SUFFIX - - mds_id = resource_info.get('attributes', {}).get('mds') - mds_link = MDS_PREFIX + mds_id if mds_id else None - - def get_storage_links(): - storage_links = get_resource_http_links(resource_id) - random.shuffle(storage_links) - return storage_links - - skynet = _is_skynet_avaliable() - - if not skynet: - logging.info("Skynet is not available, will try other protocols") - - def iter_tries(): - if skynet: - yield lambda: download_by_skynet(resource_info, resource_file_name) - - if custom_fetcher: - yield lambda: fetch_via_script(custom_fetcher, resource_id) - - # Don't try too hard here: we will get back to proxy later on - yield lambda: fetch_from.fetch_url(proxy_link, False, resource_file_name, expected_md5, tries=2) - for x in get_storage_links(): - # Don't spend too much time connecting single host - yield lambda: fetch_from.fetch_url(x, False, resource_file_name, expected_md5, tries=1) - if mds_link is not None: - # Don't try too hard here: we will get back to MDS later on - yield lambda: fetch_from.fetch_url(mds_link, True, resource_file_name, expected_md5, tries=2) - yield lambda: fetch_from.fetch_url(proxy_link, False, resource_file_name, expected_md5) - if mds_link is not None: - yield lambda: fetch_from.fetch_url(mds_link, True, resource_file_name, expected_md5) - - if resource_info.get('attributes', {}).get('ttl') != 'inf': - sys.stderr.write('WARNING: resource {} ttl is not "inf".\n'.format(resource_id)) - - exc_info = None - for i, action in enumerate(itertools.islice(iter_tries(), 0, 10)): - try: - fetched_file = action() - break - except UnsupportedProtocolException: - pass - except subprocess.CalledProcessError as e: - logging.warning('failed to fetch resource %s with subprocess: %s', resource_id, e) - time.sleep(i) - except urllib2.HTTPError as e: - logging.warning('failed to fetch resource %s with HTTP code %d: %s', resource_id, e.code, e) - if e.code not in TEMPORARY_ERROR_CODES: - exc_info = exc_info or sys.exc_info() - time.sleep(i) - except Exception as e: - logging.exception(e) - exc_info = exc_info or sys.exc_info() - time.sleep(i) - else: - if exc_info: - raise exc_info[0], exc_info[1], exc_info[2] - else: - raise Exception("No available protocol and/or server to fetch resource") - - return fetched_file, resource_info['file_name'] - - -def _get_resource_info_from_file(resource_file): - if resource_file is None or not os.path.exists(resource_file): - return None - - RESOURCE_INFO_JSON = "resource_info.json" - RESOURCE_CONTENT_FILE_NAME = "resource" - - resource_dir, resource_file = os.path.split(resource_file) - if resource_file != RESOURCE_CONTENT_FILE_NAME: - return None - - resource_json = os.path.join(resource_dir, RESOURCE_INFO_JSON) - if not os.path.isfile(resource_json): - return None - - try: - with open(resource_json, 'r') as j: - resource_info = json.load(j) - resource_info['file_name'] # check consistency - return resource_info - except: - logging.debug('Invalid %s in %s', RESOURCE_INFO_JSON, resource_dir) - - return None - - -def main(args): - custom_fetcher = os.environ.get('YA_CUSTOM_FETCHER') - - resource_info = _get_resource_info_from_file(args.resource_file) - if resource_info: - fetched_file = args.resource_file - file_name = resource_info['file_name'] - else: - # This code should be merged to ya and removed. - fetched_file, file_name = fetch(args.resource_id, custom_fetcher) - - fetch_from.process(fetched_file, file_name, args, remove=not custom_fetcher and not resource_info) - - -if __name__ == '__main__': - args = parse_args() - fetch_from.setup_logging(args, os.path.basename(__file__)) - - try: - main(args) - except Exception as e: - logging.exception(e) - print >>sys.stderr, open(args.abs_log_path).read() - sys.stderr.flush() - - import error - sys.exit(error.ExitCodes.INFRASTRUCTURE_ERROR if fetch_from.is_temporary(e) else 1) diff --git a/build/scripts/fs_tools.py b/build/scripts/fs_tools.py deleted file mode 100644 index dec4c349c8..0000000000 --- a/build/scripts/fs_tools.py +++ /dev/null @@ -1,104 +0,0 @@ -from __future__ import print_function - -import os -import platform -import sys -import shutil -import errno - -import process_command_files as pcf - - -def link_or_copy(src, dst): - try: - if platform.system().lower() == 'windows': - shutil.copy(src, dst) - else: - os.link(src, dst) - except OSError as e: - if e.errno == errno.EEXIST: - print('link_or_copy: destination file already exists: {}'.format(dst), file=sys.stderr) - if e.errno == errno.ENOENT: - print('link_or_copy: source file doesn\'t exists: {}'.format(src), file=sys.stderr) - raise - - -if __name__ == '__main__': - mode = sys.argv[1] - args = pcf.get_args(sys.argv[2:]) - - if mode == 'copy': - shutil.copy(args[0], args[1]) - elif mode == 'copy_tree_no_link': - dst = args[1] - shutil.copytree(args[0], dst, ignore=lambda dirname, names: [n for n in names if os.path.islink(os.path.join(dirname, n))]) - elif mode == 'copy_files': - src = args[0] - dst = args[1] - files = open(args[2]).read().strip().split() - for f in files: - s = os.path.join(src, f) - d = os.path.join(dst, f) - if os.path.exists(d): - continue - try: - os.makedirs(os.path.dirname(d)) - except OSError: - pass - shutil.copy(s, d) - elif mode == 'copy_all_files': - src = args[0] - dst = args[1] - for root, _, files in os.walk(src): - for f in files: - if os.path.islink(os.path.join(root, f)): - continue - file_dst = os.path.join(dst, os.path.relpath(root, src), f) - if os.path.exists(file_dst): - continue - try: - os.makedirs(os.path.dirname(file_dst)) - except OSError: - pass - shutil.copy(os.path.join(root, f), file_dst) - elif mode == 'rename_if_exists': - if os.path.exists(args[0]): - shutil.move(args[0], args[1]) - elif mode == 'rename': - targetdir = os.path.dirname(args[1]) - if targetdir and not os.path.exists(targetdir): - os.makedirs(os.path.dirname(args[1])) - shutil.move(args[0], args[1]) - elif mode == 'remove': - for f in args: - try: - if os.path.isfile(f) or os.path.islink(f): - os.remove(f) - else: - shutil.rmtree(f) - except OSError: - pass - elif mode == 'link_or_copy': - link_or_copy(args[0], args[1]) - elif mode == 'link_or_copy_to_dir': - assert len(args) > 1 - start = 0 - if args[0] == '--no-check': - if args == 2: - sys.exit() - start = 1 - dst = args[-1] - for src in args[start:-1]: - link_or_copy(src, os.path.join(dst, os.path.basename(src))) - elif mode == 'cat': - with open(args[0], 'w') as dst: - for input_name in args[1:]: - with open(input_name) as src: - dst.write(src.read()) - elif mode == 'md': - try: - os.makedirs(args[0]) - except OSError: - pass - else: - raise Exception('unsupported tool %s' % mode) diff --git a/build/scripts/gen_py3_reg.py b/build/scripts/gen_py3_reg.py deleted file mode 100644 index 149c094898..0000000000 --- a/build/scripts/gen_py3_reg.py +++ /dev/null @@ -1,34 +0,0 @@ -import sys - -template = ''' -struct PyObject; -extern "C" int PyImport_AppendInittab(const char* name, PyObject* (*initfunc)()); -extern "C" PyObject* {1}(); - -namespace { - struct TRegistrar { - inline TRegistrar() { - // TODO Collect all modules and call PyImport_ExtendInittab once - PyImport_AppendInittab("{0}", {1}); - } - } REG; -} -''' - - -def mangle(name): - if '.' not in name: - return name - return ''.join('{}{}'.format(len(s), s) for s in name.split('.')) - -if __name__ == '__main__': - if len(sys.argv) != 3: - print >>sys.stderr, 'Usage: <path/to/gen_py_reg.py> <python_module_name> <output_file>' - print >>sys.stderr, 'Passed: ' + ' '.join(sys.argv) - sys.exit(1) - - with open(sys.argv[2], 'w') as f: - modname = sys.argv[1] - initname = 'PyInit_' + mangle(modname) - code = template.replace('{0}', modname).replace('{1}', initname) - f.write(code) diff --git a/build/scripts/gen_py_protos.py b/build/scripts/gen_py_protos.py deleted file mode 100644 index 08397472f9..0000000000 --- a/build/scripts/gen_py_protos.py +++ /dev/null @@ -1,67 +0,0 @@ -import os -from os import path -import shutil -import subprocess -import sys -import tempfile -import argparse -import re - - -OUT_DIR_ARG = '--python_out=' - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("--suffixes", nargs="*", default=[]) - parser.add_argument("protoc_args", nargs=argparse.REMAINDER) - script_args = parser.parse_args() - - args = script_args.protoc_args - - if args[0] == "--": - args = args[1:] - - out_dir_orig = None - out_dir_temp = None - plugin_out_dirs_orig = {} - for i in range(len(args)): - if args[i].startswith(OUT_DIR_ARG): - assert not out_dir_orig, 'Duplicate "{0}" param'.format(OUT_DIR_ARG) - out_dir_orig = args[i][len(OUT_DIR_ARG):] - out_dir_temp = tempfile.mkdtemp(dir=out_dir_orig) - args[i] = OUT_DIR_ARG + out_dir_temp - continue - - match = re.match(r"^(--(\w+)_out=).*", args[i]) - if match: - plugin_out_dir_arg = match.group(1) - plugin = match.group(2) - assert plugin not in plugin_out_dirs_orig, 'Duplicate "{0}" param'.format(plugin_out_dir_arg) - plugin_out_dirs_orig[plugin] = args[i][len(plugin_out_dir_arg):] - assert plugin_out_dirs_orig[plugin] == out_dir_orig, 'Params "{0}" and "{1}" expected to have the same value'.format(OUT_DIR_ARG, plugin_out_dir_arg) - args[i] = plugin_out_dir_arg + out_dir_temp - - assert out_dir_temp, 'Param "{0}" not found'.format(OUT_DIR_ARG) - - retcode = subprocess.call(args) - assert not retcode, 'Protoc failed for command {}'.format(' '.join(args)) - - for root_temp, dirs, files in os.walk(out_dir_temp): - sub_dir = path.relpath(root_temp, out_dir_temp) - root_orig = path.join(out_dir_orig, sub_dir) - for d in dirs: - d_orig = path.join(root_orig, d) - if not path.exists(d_orig): - os.mkdir(d_orig) - for f in files: - f_orig = f - for suf in script_args.suffixes: - if f.endswith(suf): - f_orig = f[:-len(suf)] + "__int__" + suf - break - os.rename(path.join(root_temp, f), path.join(root_orig, f_orig)) - shutil.rmtree(out_dir_temp) - - -if __name__ == '__main__': - main() diff --git a/build/scripts/gen_py_reg.py b/build/scripts/gen_py_reg.py deleted file mode 100644 index 1560135ae8..0000000000 --- a/build/scripts/gen_py_reg.py +++ /dev/null @@ -1,32 +0,0 @@ -import sys - -template = ''' -extern "C" void PyImport_AppendInittab(const char* name, void (*fn)(void)); -extern "C" void {1}(); - -namespace { - struct TRegistrar { - inline TRegistrar() { - PyImport_AppendInittab("{0}", {1}); - } - } REG; -} -''' - - -def mangle(name): - if '.' not in name: - return name - return ''.join('{}{}'.format(len(s), s) for s in name.split('.')) - -if __name__ == '__main__': - if len(sys.argv) != 3: - print >>sys.stderr, 'Usage: <path/to/gen_py_reg.py> <python_module_name> <output_file>' - print >>sys.stderr, 'Passed: ' + ' '.join(sys.argv) - sys.exit(1) - - with open(sys.argv[2], 'w') as f: - modname = sys.argv[1] - initname = 'init' + mangle(modname) - code = template.replace('{0}', modname).replace('{1}', initname) - f.write(code) diff --git a/build/scripts/go_fake_include/go_asm.h b/build/scripts/go_fake_include/go_asm.h deleted file mode 100644 index e69de29bb2..0000000000 --- a/build/scripts/go_fake_include/go_asm.h +++ /dev/null diff --git a/build/scripts/go_tool.py b/build/scripts/go_tool.py deleted file mode 100644 index 7c51e65a2e..0000000000 --- a/build/scripts/go_tool.py +++ /dev/null @@ -1,867 +0,0 @@ -from __future__ import absolute_import, unicode_literals -import argparse -import copy -import json -import os -import re -import shutil -import subprocess -import sys -import tarfile -import tempfile -import threading -import six -from functools import reduce - -import process_command_files as pcf -import process_whole_archive_option as pwa - -arc_project_prefix = 'a.yandex-team.ru/' -std_lib_prefix = 'contrib/go/_std_1.18/src/' -vendor_prefix = 'vendor/' -vet_info_ext = '.vet.out' -vet_report_ext = '.vet.txt' - -FIXED_CGO1_SUFFIX='.fixed.cgo1.go' - -COMPILE_OPTIMIZATION_FLAGS=('-N',) - - -def get_trimpath_args(args): - return ['-trimpath', args.trimpath] if args.trimpath else [] - - -def preprocess_cgo1(src_path, dst_path, source_root): - with open(src_path, 'r') as f: - content = f.read() - content = content.replace('__ARCADIA_SOURCE_ROOT_PREFIX__', source_root) - with open(dst_path, 'w') as f: - f.write(content) - - -def preprocess_args(args): - # Temporary work around for noauto - if args.cgo_srcs and len(args.cgo_srcs) > 0: - cgo_srcs_set = set(args.cgo_srcs) - args.srcs = [x for x in args.srcs if x not in cgo_srcs_set] - - args.pkg_root = os.path.join(args.toolchain_root, 'pkg') - toolchain_tool_root = os.path.join(args.pkg_root, 'tool', '{}_{}'.format(args.host_os, args.host_arch)) - args.go_compile = os.path.join(toolchain_tool_root, 'compile') - args.go_cgo = os.path.join(toolchain_tool_root, 'cgo') - args.go_link = os.path.join(toolchain_tool_root, 'link') - args.go_asm = os.path.join(toolchain_tool_root, 'asm') - args.go_pack = os.path.join(toolchain_tool_root, 'pack') - args.go_vet = os.path.join(toolchain_tool_root, 'vet') if args.vet is True else args.vet - args.output = os.path.normpath(args.output) - args.vet_report_output = vet_report_output_name(args.output, args.vet_report_ext) - args.trimpath = None - if args.debug_root_map: - roots = {'build': args.build_root, 'source': args.source_root, 'tools': args.tools_root} - replaces = [] - for root in args.debug_root_map.split(';'): - src, dst = root.split('=', 1) - assert src in roots - replaces.append('{}=>{}'.format(roots[src], dst)) - del roots[src] - assert len(replaces) > 0 - args.trimpath = ';'.join(replaces) - args.build_root = os.path.normpath(args.build_root) - args.build_root_dir = args.build_root + os.path.sep - args.source_root = os.path.normpath(args.source_root) - args.source_root_dir = args.source_root + os.path.sep - args.output_root = os.path.normpath(args.output_root) - args.import_map = {} - args.module_map = {} - if args.cgo_peers: - args.cgo_peers = [x for x in args.cgo_peers if not x.endswith('.fake.pkg')] - - srcs = [] - for f in args.srcs: - if f.endswith('.gosrc'): - with tarfile.open(f, 'r') as tar: - srcs.extend(os.path.join(args.output_root, src) for src in tar.getnames()) - tar.extractall(path=args.output_root) - else: - srcs.append(f) - args.srcs = srcs - - assert args.mode == 'test' or args.test_srcs is None and args.xtest_srcs is None - # add lexical oreder by basename for go sources - args.srcs.sort(key=lambda x: os.path.basename(x)) - if args.test_srcs: - args.srcs += sorted(args.test_srcs, key=lambda x: os.path.basename(x)) - del args.test_srcs - if args.xtest_srcs: - args.xtest_srcs.sort(key=lambda x: os.path.basename(x)) - - # compute root relative module dir path - assert args.output is None or args.output_root == os.path.dirname(args.output) - assert args.output_root.startswith(args.build_root_dir) - args.module_path = args.output_root[len(args.build_root_dir):] - args.source_module_dir = os.path.join(args.source_root, args.test_import_path or args.module_path) + os.path.sep - assert len(args.module_path) > 0 - args.import_path, args.is_std = get_import_path(args.module_path) - - assert args.asmhdr is None or args.word == 'go' - - srcs = [] - for f in args.srcs: - if f.endswith(FIXED_CGO1_SUFFIX) and f.startswith(args.build_root_dir): - path = os.path.join(args.output_root, '{}.cgo1.go'.format(os.path.basename(f[:-len(FIXED_CGO1_SUFFIX)]))) - srcs.append(path) - preprocess_cgo1(f, path, args.source_root) - else: - srcs.append(f) - args.srcs = srcs - - if args.extldflags: - args.extldflags = pwa.ProcessWholeArchiveOption(args.targ_os).construct_cmd(args.extldflags) - - classify_srcs(args.srcs, args) - - -def compare_versions(version1, version2): - def last_index(version): - index = version.find('beta') - return len(version) if index < 0 else index - - v1 = tuple(x.zfill(8) for x in version1[:last_index(version1)].split('.')) - v2 = tuple(x.zfill(8) for x in version2[:last_index(version2)].split('.')) - if v1 == v2: - return 0 - return 1 if v1 < v2 else -1 - - -def get_symlink_or_copyfile(): - os_symlink = getattr(os, 'symlink', None) - if os_symlink is None or os.name == 'nt': - os_symlink = shutil.copyfile - return os_symlink - - -def copy_args(args): - return copy.copy(args) - - -def get_vendor_index(import_path): - index = import_path.rfind('/' + vendor_prefix) - if index < 0: - index = 0 if import_path.startswith(vendor_prefix) else index - else: - index = index + 1 - return index - - -def get_import_path(module_path): - assert len(module_path) > 0 - import_path = module_path.replace('\\', '/') - is_std_module = import_path.startswith(std_lib_prefix) - if is_std_module: - import_path = import_path[len(std_lib_prefix):] - elif import_path.startswith(vendor_prefix): - import_path = import_path[len(vendor_prefix):] - else: - import_path = arc_project_prefix + import_path - assert len(import_path) > 0 - return import_path, is_std_module - - -def call(cmd, cwd, env=None): - # sys.stderr.write('{}\n'.format(' '.join(cmd))) - return subprocess.check_output(cmd, stdin=None, stderr=subprocess.STDOUT, cwd=cwd, env=env, text=True) - - -def classify_srcs(srcs, args): - args.go_srcs = [x for x in srcs if x.endswith('.go')] - args.asm_srcs = [x for x in srcs if x.endswith('.s')] - args.objects = [x for x in srcs if x.endswith('.o') or x.endswith('.obj')] - args.symabis = [x for x in srcs if x.endswith('.symabis')] - args.sysos = [x for x in srcs if x.endswith('.syso')] - - -def get_import_config_info(peers, gen_importmap, import_map={}, module_map={}): - info = {'importmap': [], 'packagefile': [], 'standard': {}} - if gen_importmap: - for key, value in six.iteritems(import_map): - info['importmap'].append((key, value)) - for peer in peers: - peer_import_path, is_std = get_import_path(os.path.dirname(peer)) - if gen_importmap: - index = get_vendor_index(peer_import_path) - if index >= 0: - index += len(vendor_prefix) - info['importmap'].append((peer_import_path[index:], peer_import_path)) - info['packagefile'].append((peer_import_path, os.path.join(args.build_root, peer))) - if is_std: - info['standard'][peer_import_path] = True - for key, value in six.iteritems(module_map): - info['packagefile'].append((key, value)) - return info - - -def create_import_config(peers, gen_importmap, import_map={}, module_map={}): - lines = [] - info = get_import_config_info(peers, gen_importmap, import_map, module_map) - for key in ('importmap', 'packagefile'): - for item in info[key]: - lines.append('{} {}={}'.format(key, *item)) - if len(lines) > 0: - lines.append('') - content = '\n'.join(lines) - # sys.stderr.writelines('{}\n'.format(l) for l in lines) - with tempfile.NamedTemporaryFile(delete=False) as f: - f.write(content.encode('UTF-8')) - return f.name - return None - - -def create_embed_config(args): - data = { - 'Patterns': {}, - 'Files': {}, - } - for info in args.embed: - pattern = info[0] - if pattern.endswith('/**/*'): - pattern = pattern[:-3] - files = {os.path.relpath(f, args.source_module_dir).replace('\\', '/'): f for f in info[1:]} - data['Patterns'][pattern] = list(files.keys()) - data['Files'].update(files) - # sys.stderr.write('{}\n'.format(json.dumps(data, indent=4))) - with tempfile.NamedTemporaryFile(delete=False, suffix='.embedcfg') as f: - f.write(json.dumps(data).encode('UTF-8')) - return f.name - - -def vet_info_output_name(path, ext=None): - return '{}{}'.format(path, ext or vet_info_ext) - - -def vet_report_output_name(path, ext=None): - return '{}{}'.format(path, ext or vet_report_ext) - - -def get_source_path(args): - return args.test_import_path or args.module_path - - -def gen_vet_info(args): - import_path = args.real_import_path if hasattr(args, 'real_import_path') else args.import_path - info = get_import_config_info(args.peers, True, args.import_map, args.module_map) - - import_map = dict(info['importmap']) - # FIXME(snermolaev): it seems that adding import map for 'fake' package - # does't make any harm (it needs to be revised later) - import_map['unsafe'] = 'unsafe' - - for (key, _) in info['packagefile']: - if key not in import_map: - import_map[key] = key - - data = { - 'ID': import_path, - 'Compiler': 'gc', - 'Dir': os.path.join(args.source_root, get_source_path(args)), - 'ImportPath': import_path, - 'GoFiles': [x for x in args.go_srcs if x.endswith('.go')], - 'NonGoFiles': [x for x in args.go_srcs if not x.endswith('.go')], - 'ImportMap': import_map, - 'PackageFile': dict(info['packagefile']), - 'Standard': dict(info['standard']), - 'PackageVetx': dict((key, vet_info_output_name(value)) for key, value in info['packagefile']), - 'VetxOnly': False, - 'VetxOutput': vet_info_output_name(args.output), - 'SucceedOnTypecheckFailure': False - } - # sys.stderr.write('{}\n'.format(json.dumps(data, indent=4))) - return data - - -def create_vet_config(args, info): - with tempfile.NamedTemporaryFile(delete=False, suffix='.cfg') as f: - f.write(json.dumps(info).encode('UTF-8')) - return f.name - - -def decode_vet_report(json_report): - report = '' - if json_report: - try: - full_diags = json.JSONDecoder().decode(json_report.decode('UTF-8')) - except ValueError: - report = json_report - else: - messages = [] - for _, module_diags in six.iteritems(full_diags): - for _, type_diags in six.iteritems(module_diags): - for diag in type_diags: - messages.append('{}: {}'.format(diag['posn'], json.dumps(diag['message']))) - report = '\n'.join(messages) - - return report - - -def dump_vet_report(args, report): - if report: - report = report.replace(args.build_root, '$B') - report = report.replace(args.source_root, '$S') - with open(args.vet_report_output, 'w') as f: - f.write(report) - - -def read_vet_report(args): - assert args - report = '' - if os.path.exists(args.vet_report_output): - with open(args.vet_report_output, 'r') as f: - report += f.read() - return report - - -def dump_vet_report_for_tests(args, *test_args_list): - dump_vet_report(args, reduce(lambda x, y: x + read_vet_report(y), [_f for _f in test_args_list if _f], '')) - - -def do_vet(args): - assert args.vet - info = gen_vet_info(args) - vet_config = create_vet_config(args, info) - cmd = [args.go_vet, '-json'] - if args.vet_flags: - cmd.extend(args.vet_flags) - cmd.append(vet_config) - # sys.stderr.write('>>>> [{}]\n'.format(' '.join(cmd))) - p_vet = subprocess.Popen(cmd, stdin=None, stderr=subprocess.PIPE, stdout=subprocess.PIPE, cwd=args.source_root) - vet_out, vet_err = p_vet.communicate() - report = decode_vet_report(vet_out) if vet_out else '' - dump_vet_report(args, report) - if p_vet.returncode: - raise subprocess.CalledProcessError(returncode=p_vet.returncode, cmd=cmd, output=vet_err) - - -def _do_compile_go(args): - import_path, is_std_module = args.import_path, args.is_std - cmd = [ - args.go_compile, - '-o', - args.output, - '-p', - import_path, - '-D', - '""', - '-goversion', - 'go{}'.format(args.goversion) - ] - if args.lang: - cmd.append('-lang=go{}'.format(args.lang)) - cmd.extend(get_trimpath_args(args)) - compiling_runtime = False - if is_std_module: - cmd.append('-std') - if import_path in ('runtime', 'internal/abi', 'internal/bytealg', 'internal/cpu') or import_path.startswith('runtime/internal/'): - cmd.append('-+') - compiling_runtime = True - import_config_name = create_import_config(args.peers, True, args.import_map, args.module_map) - if import_config_name: - cmd += ['-importcfg', import_config_name] - else: - if import_path == 'unsafe' or len(args.objects) > 0 or args.asmhdr: - pass - else: - cmd.append('-complete') - # if compare_versions('1.16', args.goversion) >= 0: - if args.embed: - embed_config_name = create_embed_config(args) - cmd.extend(['-embedcfg', embed_config_name]) - if args.asmhdr: - cmd += ['-asmhdr', args.asmhdr] - # Use .symabis (starting from 1.12 version) - if args.symabis: - cmd += ['-symabis'] + args.symabis - # If 1.12 <= version < 1.13 we have to pass -allabis for 'runtime' and 'runtime/internal/atomic' - # if compare_versions('1.13', args.goversion) >= 0: - # pass - # elif import_path in ('runtime', 'runtime/internal/atomic'): - # cmd.append('-allabis') - compile_workers = '4' - if args.compile_flags: - if compiling_runtime: - cmd.extend(x for x in args.compile_flags if x not in COMPILE_OPTIMIZATION_FLAGS) - else: - cmd.extend(args.compile_flags) - if any([x in ('-race', '-shared') for x in args.compile_flags]): - compile_workers = '1' - cmd += ['-pack', '-c={}'.format(compile_workers)] - cmd += args.go_srcs - call(cmd, args.build_root) - - -class VetThread(threading.Thread): - - def __init__(self, target, args): - super(VetThread, self).__init__(target=target, args=args) - self.exc_info = None - - def run(self): - try: - super(VetThread, self).run() - except: - self.exc_info = sys.exc_info() - - def join_with_exception(self, reraise_exception): - self.join() - if reraise_exception and self.exc_info: - six.reraise(self.exc_info[0], self.exc_info[1], self.exc_info[2]) - - -def do_compile_go(args): - raise_exception_from_vet = False - if args.vet: - run_vet = VetThread(target=do_vet, args=(args,)) - run_vet.start() - try: - _do_compile_go(args) - raise_exception_from_vet = True - finally: - if args.vet: - run_vet.join_with_exception(raise_exception_from_vet) - - -def do_compile_asm(args): - def need_compiling_runtime(import_path): - return import_path in ('runtime', 'reflect', 'syscall') or \ - import_path.startswith('runtime/internal/') or \ - compare_versions('1.17', args.goversion) >= 0 and import_path == 'internal/bytealg' - - assert(len(args.srcs) == 1 and len(args.asm_srcs) == 1) - cmd = [args.go_asm] - cmd += get_trimpath_args(args) - cmd += ['-I', args.output_root, '-I', os.path.join(args.pkg_root, 'include')] - cmd += ['-D', 'GOOS_' + args.targ_os, '-D', 'GOARCH_' + args.targ_arch, '-o', args.output] - - # if compare_versions('1.16', args.goversion) >= 0: - cmd += ['-p', args.import_path] - if need_compiling_runtime(args.import_path): - cmd += ['-compiling-runtime'] - - if args.asm_flags: - cmd += args.asm_flags - cmd += args.asm_srcs - call(cmd, args.build_root) - - -def do_link_lib(args): - if len(args.asm_srcs) > 0: - asmargs = copy_args(args) - asmargs.asmhdr = os.path.join(asmargs.output_root, 'go_asm.h') - do_compile_go(asmargs) - for src in asmargs.asm_srcs: - asmargs.srcs = [src] - asmargs.asm_srcs = [src] - asmargs.output = os.path.join(asmargs.output_root, os.path.basename(src) + '.o') - do_compile_asm(asmargs) - args.objects.append(asmargs.output) - else: - do_compile_go(args) - if args.objects or args.sysos: - cmd = [args.go_pack, 'r', args.output] + args.objects + args.sysos - call(cmd, args.build_root) - - -def do_link_exe(args): - assert args.extld is not None - assert args.non_local_peers is not None - compile_args = copy_args(args) - compile_args.output = os.path.join(args.output_root, 'main.a') - compile_args.real_import_path = compile_args.import_path - compile_args.import_path = 'main' - - if args.vcs and os.path.isfile(compile_args.vcs): - build_info = os.path.join('library', 'go', 'core', 'buildinfo') - if any([x.startswith(build_info) for x in compile_args.peers]): - compile_args.go_srcs.append(compile_args.vcs) - - do_link_lib(compile_args) - cmd = [args.go_link, '-o', args.output] - import_config_name = create_import_config(args.peers + args.non_local_peers, False, args.import_map, args.module_map) - if import_config_name: - cmd += ['-importcfg', import_config_name] - if args.link_flags: - cmd += args.link_flags - - if args.mode in ('exe', 'test'): - cmd.append('-buildmode=exe') - elif args.mode == 'dll': - cmd.append('-buildmode=c-shared') - else: - assert False, 'Unexpected mode: {}'.format(args.mode) - cmd.append('-extld={}'.format(args.extld)) - - extldflags = [] - if args.extldflags is not None: - filter_musl = bool - if args.musl: - cmd.append('-linkmode=external') - extldflags.append('-static') - filter_musl = lambda x: x not in ('-lc', '-ldl', '-lm', '-lpthread', '-lrt') - extldflags += [x for x in args.extldflags if filter_musl(x)] - cgo_peers = [] - if args.cgo_peers is not None and len(args.cgo_peers) > 0: - is_group = args.targ_os == 'linux' - if is_group: - cgo_peers.append('-Wl,--start-group') - cgo_peers.extend(args.cgo_peers) - if is_group: - cgo_peers.append('-Wl,--end-group') - try: - index = extldflags.index('--cgo-peers') - extldflags = extldflags[:index] + cgo_peers + extldflags[index+1:] - except ValueError: - extldflags.extend(cgo_peers) - if len(extldflags) > 0: - cmd.append('-extldflags={}'.format(' '.join(extldflags))) - cmd.append(compile_args.output) - call(cmd, args.build_root) - - -def gen_cover_info(args): - lines = [] - lines.extend([ - """ -var ( - coverCounters = make(map[string][]uint32) - coverBlocks = make(map[string][]testing.CoverBlock) -) - """, - 'func init() {', - ]) - for var, file in (x.split(':') for x in args.cover_info): - lines.append(' coverRegisterFile("{file}", _cover0.{var}.Count[:], _cover0.{var}.Pos[:], _cover0.{var}.NumStmt[:])'.format(file=file, var=var)) - lines.extend([ - '}', - """ -func coverRegisterFile(fileName string, counter []uint32, pos []uint32, numStmts []uint16) { - if 3*len(counter) != len(pos) || len(counter) != len(numStmts) { - panic("coverage: mismatched sizes") - } - if coverCounters[fileName] != nil { - // Already registered. - return - } - coverCounters[fileName] = counter - block := make([]testing.CoverBlock, len(counter)) - for i := range counter { - block[i] = testing.CoverBlock{ - Line0: pos[3*i+0], - Col0: uint16(pos[3*i+2]), - Line1: pos[3*i+1], - Col1: uint16(pos[3*i+2]>>16), - Stmts: numStmts[i], - } - } - coverBlocks[fileName] = block -} - """, - ]) - return lines - - -def filter_out_skip_tests(tests, skip_tests): - skip_set = set() - star_skip_set = set() - for t in skip_tests: - work_set = star_skip_set if '*' in t else skip_set - work_set.add(t) - - re_star_tests = None - if len(star_skip_set) > 0: - re_star_tests = re.compile(re.sub(r'(\*)+', r'.\1', '^({})$'.format('|'.join(star_skip_set)))) - - return [x for x in tests if not (x in skip_tests or re_star_tests and re_star_tests.match(x))] - - -def gen_test_main(args, test_lib_args, xtest_lib_args): - assert args and (test_lib_args or xtest_lib_args) - test_miner = args.test_miner - test_module_path = test_lib_args.import_path if test_lib_args else xtest_lib_args.import_path - is_cover = args.cover_info and len(args.cover_info) > 0 - - # Prepare GOPATH - # $BINDIR - # |- __go__ - # |- src - # |- pkg - # |- ${TARGET_OS}_${TARGET_ARCH} - go_path_root = os.path.join(args.output_root, '__go__') - test_src_dir = os.path.join(go_path_root, 'src') - target_os_arch = '_'.join([args.targ_os, args.targ_arch]) - test_pkg_dir = os.path.join(go_path_root, 'pkg', target_os_arch, os.path.dirname(test_module_path)) - os.makedirs(test_pkg_dir) - - my_env = os.environ.copy() - my_env['GOROOT'] = '' - my_env['GOPATH'] = go_path_root - my_env['GOARCH'] = args.targ_arch - my_env['GOOS'] = args.targ_os - - tests = [] - xtests = [] - os_symlink = get_symlink_or_copyfile() - - # Get the list of "internal" tests - if test_lib_args: - os.makedirs(os.path.join(test_src_dir, test_module_path)) - os_symlink(test_lib_args.output, os.path.join(test_pkg_dir, os.path.basename(test_module_path) + '.a')) - cmd = [test_miner, '-benchmarks', '-tests', test_module_path] - tests = [x for x in (call(cmd, test_lib_args.output_root, my_env) or '').strip().split('\n') if len(x) > 0] - if args.skip_tests: - tests = filter_out_skip_tests(tests, args.skip_tests) - test_main_found = '#TestMain' in tests - - # Get the list of "external" tests - if xtest_lib_args: - xtest_module_path = xtest_lib_args.import_path - os.makedirs(os.path.join(test_src_dir, xtest_module_path)) - os_symlink(xtest_lib_args.output, os.path.join(test_pkg_dir, os.path.basename(xtest_module_path) + '.a')) - cmd = [test_miner, '-benchmarks', '-tests', xtest_module_path] - xtests = [x for x in (call(cmd, xtest_lib_args.output_root, my_env) or '').strip().split('\n') if len(x) > 0] - if args.skip_tests: - xtests = filter_out_skip_tests(xtests, args.skip_tests) - xtest_main_found = '#TestMain' in xtests - - test_main_package = None - if test_main_found and xtest_main_found: - assert False, 'multiple definition of TestMain' - elif test_main_found: - test_main_package = '_test' - elif xtest_main_found: - test_main_package = '_xtest' - - shutil.rmtree(go_path_root) - - lines = ['package main', '', 'import ('] - if test_main_package is None: - lines.append(' "os"') - lines.extend([' "testing"', ' "testing/internal/testdeps"']) - - if len(tests) > 0: - lines.append(' _test "{}"'.format(test_module_path)) - elif test_lib_args: - lines.append(' _ "{}"'.format(test_module_path)) - - if len(xtests) > 0: - lines.append(' _xtest "{}"'.format(xtest_module_path)) - elif xtest_lib_args: - lines.append(' _ "{}"'.format(xtest_module_path)) - - if is_cover: - lines.append(' _cover0 "{}"'.format(test_module_path)) - lines.extend([')', '']) - - if compare_versions('1.18', args.goversion) < 0: - kinds = ['Test', 'Benchmark', 'Example'] - else: - kinds = ['Test', 'Benchmark', 'FuzzTarget', 'Example'] - - var_names = [] - for kind in kinds: - var_name = '{}s'.format(kind.lower()) - var_names.append(var_name) - lines.append('var {} = []testing.Internal{}{{'.format(var_name, kind)) - for test in [x for x in tests if x.startswith(kind)]: - lines.append(' {{"{test}", _test.{test}}},'.format(test=test)) - for test in [x for x in xtests if x.startswith(kind)]: - lines.append(' {{"{test}", _xtest.{test}}},'.format(test=test)) - lines.extend(['}', '']) - - if is_cover: - lines.extend(gen_cover_info(args)) - - lines.append('func main() {') - if is_cover: - lines.extend([ - ' testing.RegisterCover(testing.Cover{', - ' Mode: "set",', - ' Counters: coverCounters,', - ' Blocks: coverBlocks,', - ' CoveredPackages: "",', - ' })', - ]) - lines.extend([ - ' m := testing.MainStart(testdeps.TestDeps{{}}, {})'.format(', '.join(var_names)), - '', - ]) - - if test_main_package: - lines.append(' {}.TestMain(m)'.format(test_main_package)) - else: - lines.append(' os.Exit(m.Run())') - lines.extend(['}', '']) - - content = '\n'.join(lines) - # sys.stderr.write('{}\n'.format(content)) - return content - - -def do_link_test(args): - assert args.srcs or args.xtest_srcs - assert args.test_miner is not None - - test_module_path = get_source_path(args) - test_import_path, _ = get_import_path(test_module_path) - - test_lib_args = copy_args(args) if args.srcs else None - xtest_lib_args = copy_args(args) if args.xtest_srcs else None - if xtest_lib_args is not None: - xtest_lib_args.embed = args.embed_xtest if args.embed_xtest else None - - ydx_file_name = None - xtest_ydx_file_name = None - need_append_ydx = test_lib_args and xtest_lib_args and args.ydx_file and args.vet_flags - if need_append_ydx: - def find_ydx_file_name(name, flags): - for i, elem in enumerate(flags): - if elem.endswith(name): - return (i, elem) - assert False, 'Unreachable code' - - idx, ydx_file_name = find_ydx_file_name(xtest_lib_args.ydx_file, xtest_lib_args.vet_flags) - xtest_ydx_file_name = '{}_xtest'.format(ydx_file_name) - xtest_lib_args.vet_flags = copy.copy(xtest_lib_args.vet_flags) - xtest_lib_args.vet_flags[idx] = xtest_ydx_file_name - - if test_lib_args: - test_lib_args.output = os.path.join(args.output_root, 'test.a') - test_lib_args.vet_report_output = vet_report_output_name(test_lib_args.output) - test_lib_args.module_path = test_module_path - test_lib_args.import_path = test_import_path - do_link_lib(test_lib_args) - - if xtest_lib_args: - xtest_lib_args.srcs = xtest_lib_args.xtest_srcs - classify_srcs(xtest_lib_args.srcs, xtest_lib_args) - xtest_lib_args.output = os.path.join(args.output_root, 'xtest.a') - xtest_lib_args.vet_report_output = vet_report_output_name(xtest_lib_args.output) - xtest_lib_args.module_path = test_module_path + '_test' - xtest_lib_args.import_path = test_import_path + '_test' - if test_lib_args: - xtest_lib_args.module_map[test_import_path] = test_lib_args.output - need_append_ydx = args.ydx_file and args.srcs and args.vet_flags - do_link_lib(xtest_lib_args) - - if need_append_ydx: - with open(os.path.join(args.build_root, ydx_file_name), 'ab') as dst_file: - with open(os.path.join(args.build_root, xtest_ydx_file_name), 'rb') as src_file: - dst_file.write(src_file.read()) - - test_main_content = gen_test_main(args, test_lib_args, xtest_lib_args) - test_main_name = os.path.join(args.output_root, '_test_main.go') - with open(test_main_name, "w") as f: - f.write(test_main_content) - test_args = copy_args(args) - test_args.embed = None - test_args.srcs = [test_main_name] - if test_args.test_import_path is None: - # it seems that we can do it unconditionally, but this kind - # of mangling doesn't really looks good to me and we leave it - # for pure GO_TEST module - test_args.module_path = test_args.module_path + '___test_main__' - test_args.import_path = test_args.import_path + '___test_main__' - classify_srcs(test_args.srcs, test_args) - if test_lib_args: - test_args.module_map[test_lib_args.import_path] = test_lib_args.output - if xtest_lib_args: - test_args.module_map[xtest_lib_args.import_path] = xtest_lib_args.output - - if args.vet: - dump_vet_report_for_tests(test_args, test_lib_args, xtest_lib_args) - test_args.vet = False - - do_link_exe(test_args) - - -if __name__ == '__main__': - args = pcf.get_args(sys.argv[1:]) - - parser = argparse.ArgumentParser(prefix_chars='+') - parser.add_argument('++mode', choices=['dll', 'exe', 'lib', 'test'], required=True) - parser.add_argument('++srcs', nargs='*', required=True) - parser.add_argument('++cgo-srcs', nargs='*') - parser.add_argument('++test_srcs', nargs='*') - parser.add_argument('++xtest_srcs', nargs='*') - parser.add_argument('++cover_info', nargs='*') - parser.add_argument('++output', nargs='?', default=None) - parser.add_argument('++source-root', default=None) - parser.add_argument('++build-root', required=True) - parser.add_argument('++tools-root', default=None) - parser.add_argument('++output-root', required=True) - parser.add_argument('++toolchain-root', required=True) - parser.add_argument('++host-os', choices=['linux', 'darwin', 'windows'], required=True) - parser.add_argument('++host-arch', choices=['amd64', 'arm64'], required=True) - parser.add_argument('++targ-os', choices=['linux', 'darwin', 'windows'], required=True) - parser.add_argument('++targ-arch', choices=['amd64', 'x86', 'arm64'], required=True) - parser.add_argument('++peers', nargs='*') - parser.add_argument('++non-local-peers', nargs='*') - parser.add_argument('++cgo-peers', nargs='*') - parser.add_argument('++asmhdr', nargs='?', default=None) - parser.add_argument('++test-import-path', nargs='?') - parser.add_argument('++test-miner', nargs='?') - parser.add_argument('++arc-project-prefix', nargs='?', default=arc_project_prefix) - parser.add_argument('++std-lib-prefix', nargs='?', default=std_lib_prefix) - parser.add_argument('++vendor-prefix', nargs='?', default=vendor_prefix) - parser.add_argument('++extld', nargs='?', default=None) - parser.add_argument('++extldflags', nargs='+', default=None) - parser.add_argument('++goversion', required=True) - parser.add_argument('++lang', nargs='?', default=None) - parser.add_argument('++asm-flags', nargs='*') - parser.add_argument('++compile-flags', nargs='*') - parser.add_argument('++link-flags', nargs='*') - parser.add_argument('++vcs', nargs='?', default=None) - parser.add_argument('++vet', nargs='?', const=True, default=False) - parser.add_argument('++vet-flags', nargs='*', default=None) - parser.add_argument('++vet-info-ext', default=vet_info_ext) - parser.add_argument('++vet-report-ext', default=vet_report_ext) - parser.add_argument('++musl', action='store_true') - parser.add_argument('++skip-tests', nargs='*', default=None) - parser.add_argument('++ydx-file', default='') - parser.add_argument('++debug-root-map', default=None) - parser.add_argument('++embed', action='append', nargs='*') - parser.add_argument('++embed_xtest', action='append', nargs='*') - args = parser.parse_args(args) - - arc_project_prefix = args.arc_project_prefix - std_lib_prefix = args.std_lib_prefix - vendor_prefix = args.vendor_prefix - vet_info_ext = args.vet_info_ext - vet_report_ext = args.vet_report_ext - - preprocess_args(args) - - try: - os.unlink(args.output) - except OSError: - pass - - # We are going to support only 'lib', 'exe' and 'cgo' build modes currently - # and as a result we are going to generate only one build node per module - # (or program) - dispatch = { - 'exe': do_link_exe, - 'dll': do_link_exe, - 'lib': do_link_lib, - 'test': do_link_test - } - - exit_code = 1 - try: - dispatch[args.mode](args) - exit_code = 0 - except KeyError: - sys.stderr.write('Unknown build mode [{}]...\n'.format(args.mode)) - except subprocess.CalledProcessError as e: - sys.stderr.write('{} returned non-zero exit code {}.\n{}\n'.format(' '.join(e.cmd), e.returncode, e.output)) - exit_code = e.returncode - except Exception as e: - sys.stderr.write('Unhandled exception [{}]...\n'.format(str(e))) - sys.exit(exit_code) diff --git a/build/scripts/link_dyn_lib.py b/build/scripts/link_dyn_lib.py deleted file mode 100644 index 918c23293c..0000000000 --- a/build/scripts/link_dyn_lib.py +++ /dev/null @@ -1,335 +0,0 @@ -import sys -import os -import subprocess -import tempfile -import collections -import optparse -import pipes - -from process_whole_archive_option import ProcessWholeArchiveOption - - -def shlex_join(cmd): - # equivalent to shlex.join() in python 3 - return ' '.join( - pipes.quote(part) - for part in cmd - ) - - -def parse_export_file(p): - with open(p, 'r') as f: - for l in f: - l = l.strip() - - if l and '#' not in l: - words = l.split() - if len(words) == 2 and words[0] == 'linux_version': - yield {'linux_version': words[1]} - elif len(words) == 2: - yield {'lang': words[0], 'sym': words[1]} - elif len(words) == 1: - yield {'lang': 'C', 'sym': words[0]} - else: - raise Exception('unsupported exports line: ' + l) - - -def to_c(sym): - symbols = collections.deque(sym.split('::')) - c_prefixes = [ # demangle prefixes for c++ symbols - '_ZN', # namespace - '_ZTIN', # typeinfo for - '_ZTSN', # typeinfo name for - '_ZTTN', # VTT for - '_ZTVN', # vtable for - '_ZNK', # const methods - ] - c_sym = '' - while symbols: - s = symbols.popleft() - if s == '*': - c_sym += '*' - break - if '*' in s and len(s) > 1: - raise Exception('Unsupported format, cannot guess length of symbol: ' + s) - c_sym += str(len(s)) + s - if symbols: - raise Exception('Unsupported format: ' + sym) - if c_sym[-1] != '*': - c_sym += 'E*' - return ['{prefix}{sym}'.format(prefix=prefix, sym=c_sym) for prefix in c_prefixes] - - -def fix_darwin_param(ex): - for item in ex: - if item.get('linux_version'): - continue - - if item['lang'] == 'C': - yield '-Wl,-exported_symbol,_' + item['sym'] - elif item['lang'] == 'C++': - for sym in to_c(item['sym']): - yield '-Wl,-exported_symbol,_' + sym - else: - raise Exception('unsupported lang: ' + item['lang']) - - -def fix_gnu_param(arch, ex): - d = collections.defaultdict(list) - version = None - for item in ex: - if item.get('linux_version'): - if not version: - version = item.get('linux_version') - else: - raise Exception('More than one linux_version defined') - elif item['lang'] == 'C++': - d['C'].extend(to_c(item['sym'])) - else: - d[item['lang']].append(item['sym']) - - with tempfile.NamedTemporaryFile(mode='wt', delete=False) as f: - if version: - f.write('{} {{\nglobal:\n'.format(version)) - else: - f.write('{\nglobal:\n') - - for k, v in d.items(): - f.write(' extern "' + k + '" {\n') - - for x in v: - f.write(' ' + x + ';\n') - - f.write(' };\n') - - f.write('local: *;\n};\n') - - ret = ['-Wl,--version-script=' + f.name] - - if arch == 'ANDROID': - ret += ['-Wl,--export-dynamic'] - - return ret - - -def fix_windows_param(ex): - with tempfile.NamedTemporaryFile(delete=False) as def_file: - exports = [] - for item in ex: - if item.get('lang') == 'C': - exports.append(item.get('sym')) - def_file.write('EXPORTS\n') - for export in exports: - def_file.write(' {}\n'.format(export)) - return ['/DEF:{}'.format(def_file.name)] - - -MUSL_LIBS = '-lc', '-lcrypt', '-ldl', '-lm', '-lpthread', '-lrt', '-lutil' - -CUDA_LIBRARIES = { - '-lcublas_static': '-lcublas', - '-lcublasLt_static': '-lcublasLt', - '-lcudart_static': '-lcudart', - '-lcudnn_static': '-lcudnn', - '-lcufft_static_nocallback': '-lcufft', - '-lcurand_static': '-lcurand', - '-lcusolver_static': '-lcusolver', - '-lcusparse_static': '-lcusparse', - '-lmyelin_compiler_static': '-lmyelin', - '-lmyelin_executor_static': '-lnvcaffe_parser', - '-lmyelin_pattern_library_static': '', - '-lmyelin_pattern_runtime_static': '', - '-lnvinfer_static': '-lnvinfer', - '-lnvinfer_plugin_static': '-lnvinfer_plugin', - '-lnvonnxparser_static': '-lnvonnxparser', - '-lnvparsers_static': '-lnvparsers' -} - - -def fix_cmd(arch, c): - if arch == 'WINDOWS': - prefix = '/DEF:' - f = fix_windows_param - else: - prefix = '-Wl,--version-script=' - if arch in ('DARWIN', 'IOS', 'IOSSIM'): - f = fix_darwin_param - else: - f = lambda x: fix_gnu_param(arch, x) - - def do_fix(p): - if p.startswith(prefix) and p.endswith('.exports'): - fname = p[len(prefix):] - - return list(f(list(parse_export_file(fname)))) - - if p.endswith('.supp'): - return [] - - if p.endswith('.pkg.fake'): - return [] - - return [p] - - return sum((do_fix(x) for x in c), []) - - -def fix_cmd_for_musl(cmd): - flags = [] - for flag in cmd: - if flag not in MUSL_LIBS: - flags.append(flag) - return flags - - -def fix_cmd_for_dynamic_cuda(cmd): - flags = [] - for flag in cmd: - if flag in CUDA_LIBRARIES: - flags.append(CUDA_LIBRARIES[flag]) - else: - flags.append(flag) - return flags - - -def parse_args(): - parser = optparse.OptionParser() - parser.disable_interspersed_args() - parser.add_option('--arch') - parser.add_option('--target') - parser.add_option('--soname') - parser.add_option('--fix-elf') - parser.add_option('--linker-output') - parser.add_option('--musl', action='store_true') - parser.add_option('--dynamic-cuda', action='store_true') - parser.add_option('--whole-archive-peers', action='append') - parser.add_option('--whole-archive-libs', action='append') - return parser.parse_args() - - -if __name__ == '__main__': - opts, args = parse_args() - - assert opts.arch - assert opts.target - - cmd = fix_cmd(opts.arch, args) - - if opts.musl: - cmd = fix_cmd_for_musl(cmd) - if opts.dynamic_cuda: - cmd = fix_cmd_for_dynamic_cuda(cmd) - - cmd = ProcessWholeArchiveOption(opts.arch, opts.whole_archive_peers, opts.whole_archive_libs).construct_cmd(cmd) - - if opts.linker_output: - stdout = open(opts.linker_output, 'w') - else: - stdout = sys.stdout - - proc = subprocess.Popen(cmd, shell=False, stderr=sys.stderr, stdout=stdout) - proc.communicate() - - if proc.returncode: - print >>sys.stderr, 'linker has failed with retcode:', proc.returncode - print >>sys.stderr, 'linker command:', shlex_join(cmd) - sys.exit(proc.returncode) - - if opts.fix_elf: - cmd = [opts.fix_elf, opts.target] - proc = subprocess.Popen(cmd, shell=False, stderr=sys.stderr, stdout=sys.stdout) - proc.communicate() - - if proc.returncode: - print >>sys.stderr, 'fix_elf has failed with retcode:', proc.returncode - print >>sys.stderr, 'fix_elf command:', shlex_join(cmd) - sys.exit(proc.returncode) - - if opts.soname and opts.soname != opts.target: - if os.path.exists(opts.soname): - os.unlink(opts.soname) - os.link(opts.target, opts.soname) - - -# -----------------Test---------------- # -def write_temp_file(content): - import yatest.common as yc - filename = yc.output_path('test.exports') - with open(filename, 'w') as f: - f.write(content) - return filename - - -def test_fix_cmd_darwin(): - export_file_content = """ -C++ geobase5::details::lookup_impl::* -C++ geobase5::hardcoded_service -""" - filename = write_temp_file(export_file_content) - args = ['-Wl,--version-script={}'.format(filename)] - assert fix_cmd('DARWIN', args) == [ - '-Wl,-exported_symbol,__ZN8geobase57details11lookup_impl*', - '-Wl,-exported_symbol,__ZTIN8geobase57details11lookup_impl*', - '-Wl,-exported_symbol,__ZTSN8geobase57details11lookup_impl*', - '-Wl,-exported_symbol,__ZTTN8geobase57details11lookup_impl*', - '-Wl,-exported_symbol,__ZTVN8geobase57details11lookup_impl*', - '-Wl,-exported_symbol,__ZNK8geobase57details11lookup_impl*', - '-Wl,-exported_symbol,__ZN8geobase517hardcoded_serviceE*', - '-Wl,-exported_symbol,__ZTIN8geobase517hardcoded_serviceE*', - '-Wl,-exported_symbol,__ZTSN8geobase517hardcoded_serviceE*', - '-Wl,-exported_symbol,__ZTTN8geobase517hardcoded_serviceE*', - '-Wl,-exported_symbol,__ZTVN8geobase517hardcoded_serviceE*', - '-Wl,-exported_symbol,__ZNK8geobase517hardcoded_serviceE*', - ] - - -def run_fix_gnu_param(export_file_content): - filename = write_temp_file(export_file_content) - result = fix_gnu_param('LINUX', list(parse_export_file(filename)))[0] - version_script_path = result[len('-Wl,--version-script='):] - with open(version_script_path) as f: - content = f.read() - return content - - -def test_fix_gnu_param(): - export_file_content = """ -C++ geobase5::details::lookup_impl::* -C getFactoryMap -""" - assert run_fix_gnu_param(export_file_content) == """{ -global: - extern "C" { - _ZN8geobase57details11lookup_impl*; - _ZTIN8geobase57details11lookup_impl*; - _ZTSN8geobase57details11lookup_impl*; - _ZTTN8geobase57details11lookup_impl*; - _ZTVN8geobase57details11lookup_impl*; - _ZNK8geobase57details11lookup_impl*; - getFactoryMap; - }; -local: *; -}; -""" - - -def test_fix_gnu_param_with_linux_version(): - export_file_content = """ -C++ geobase5::details::lookup_impl::* -linux_version ver1.0 -C getFactoryMap -""" - assert run_fix_gnu_param(export_file_content) == """ver1.0 { -global: - extern "C" { - _ZN8geobase57details11lookup_impl*; - _ZTIN8geobase57details11lookup_impl*; - _ZTSN8geobase57details11lookup_impl*; - _ZTTN8geobase57details11lookup_impl*; - _ZTVN8geobase57details11lookup_impl*; - _ZNK8geobase57details11lookup_impl*; - getFactoryMap; - }; -local: *; -}; -""" diff --git a/build/scripts/link_exe.py b/build/scripts/link_exe.py deleted file mode 100644 index aa96818851..0000000000 --- a/build/scripts/link_exe.py +++ /dev/null @@ -1,130 +0,0 @@ -import sys -import subprocess -import optparse - -from process_whole_archive_option import ProcessWholeArchiveOption - - -def get_leaks_suppressions(cmd): - supp, newcmd = [], [] - for arg in cmd: - if arg.endswith(".supp"): - supp.append(arg) - else: - newcmd.append(arg) - return supp, newcmd - - -MUSL_LIBS = '-lc', '-lcrypt', '-ldl', '-lm', '-lpthread', '-lrt', '-lutil' - - -CUDA_LIBRARIES = { - '-lcublas_static': '-lcublas', - '-lcublasLt_static': '-lcublasLt', - '-lcudart_static': '-lcudart', - '-lcudnn_static': '-lcudnn', - '-lcufft_static_nocallback': '-lcufft', - '-lcurand_static': '-lcurand', - '-lcusolver_static': '-lcusolver', - '-lcusparse_static': '-lcusparse', - '-lmyelin_compiler_static': '-lmyelin', - '-lmyelin_executor_static': '-lnvcaffe_parser', - '-lmyelin_pattern_library_static': '', - '-lmyelin_pattern_runtime_static': '', - '-lnvinfer_static': '-lnvinfer', - '-lnvinfer_plugin_static': '-lnvinfer_plugin', - '-lnvonnxparser_static': '-lnvonnxparser', - '-lnvparsers_static': '-lnvparsers' -} - - -def remove_excessive_flags(cmd): - flags = [] - for flag in cmd: - if not flag.endswith('.ios.interface') and not flag.endswith('.pkg.fake'): - flags.append(flag) - return flags - - -def fix_cmd_for_musl(cmd): - flags = [] - for flag in cmd: - if flag not in MUSL_LIBS: - flags.append(flag) - return flags - - -def fix_cmd_for_dynamic_cuda(cmd): - flags = [] - for flag in cmd: - if flag in CUDA_LIBRARIES: - flags.append(CUDA_LIBRARIES[flag]) - else: - flags.append(flag) - return flags - - -def gen_default_suppressions(inputs, output, source_root): - import collections - import os - - supp_map = collections.defaultdict(set) - for filename in inputs: - sanitizer = os.path.basename(filename).split('.', 1)[0] - with open(os.path.join(source_root, filename)) as src: - for line in src: - line = line.strip() - if not line or line.startswith('#'): - continue - supp_map[sanitizer].add(line) - - with open(output, "wb") as dst: - for supp_type, supps in supp_map.items(): - dst.write('extern "C" const char *__%s_default_suppressions() {\n' % supp_type) - dst.write(' return "{}";\n'.format('\\n'.join(sorted(supps)))) - dst.write('}\n') - - -def parse_args(): - parser = optparse.OptionParser() - parser.disable_interspersed_args() - parser.add_option('--musl', action='store_true') - parser.add_option('--custom-step') - parser.add_option('--python') - parser.add_option('--source-root') - parser.add_option('--dynamic-cuda', action='store_true') - parser.add_option('--arch') - parser.add_option('--linker-output') - parser.add_option('--whole-archive-peers', action='append') - parser.add_option('--whole-archive-libs', action='append') - return parser.parse_args() - - -if __name__ == '__main__': - opts, args = parse_args() - - cmd = remove_excessive_flags(args) - if opts.musl: - cmd = fix_cmd_for_musl(cmd) - - if opts.dynamic_cuda: - cmd = fix_cmd_for_dynamic_cuda(cmd) - cmd = ProcessWholeArchiveOption(opts.arch, opts.whole_archive_peers, opts.whole_archive_libs).construct_cmd(cmd) - - if opts.custom_step: - assert opts.python - subprocess.check_call([opts.python] + [opts.custom_step] + args) - - supp, cmd = get_leaks_suppressions(cmd) - if supp: - src_file = "default_suppressions.cpp" - gen_default_suppressions(supp, src_file, opts.source_root) - cmd += [src_file] - - if opts.linker_output: - stdout = open(opts.linker_output, 'w') - else: - stdout = sys.stdout - - rc = subprocess.call(cmd, shell=False, stderr=sys.stderr, stdout=stdout) - sys.exit(rc) diff --git a/build/scripts/link_lib.py b/build/scripts/link_lib.py deleted file mode 100644 index 5ca50ef844..0000000000 --- a/build/scripts/link_lib.py +++ /dev/null @@ -1,85 +0,0 @@ -import sys -import subprocess -import tempfile -import os - - -class Opts(object): - def __init__(self, args): - self.archiver = args[0] - self.arch_type = args[1] - self.llvm_ar_format = args[2] - self.build_root = args[3] - self.plugin = args[4] - self.output = args[5] - auto_input = args[6:] - - if self.arch_type == 'GNU_AR': - self.create_flags = ['rcs'] - self.modify_flags = ['-M'] - elif self.arch_type == 'LLVM_AR': - self.create_flags = ['rcs', '--format=%s' % self.llvm_ar_format] - self.modify_flags = ['-M'] - elif self.arch_type == 'LIBTOOL': - self.create_flags = ['-static', '-o'] - self.modify_flags = [] - - need_modify = self.arch_type != 'LIBTOOL' and any(item.endswith('.a') for item in auto_input) - if need_modify: - self.objs = filter(lambda x: x.endswith('.o'), auto_input) - self.libs = filter(lambda x: x.endswith('.a'), auto_input) - else: - self.objs = auto_input - self.libs = [] - - self.plugin_flags = ['--plugin', self.plugin] if self.plugin != 'None' else [] - - -def get_opts(args): - return Opts(args) - - -if __name__ == "__main__": - opts = get_opts(sys.argv[1:]) - - # There is a bug in llvm-ar. Some files with size slightly greater 2^32 - # still have GNU format instead of GNU64 and cause link problems. - # Workaround just lowers llvm-ar's GNU64 threshold to 2^31. - if opts.arch_type == 'LLVM_AR': - os.environ['SYM64_THRESHOLD'] = '31' - - def call(): - try: - p = subprocess.Popen(cmd, stdin=stdin, cwd=opts.build_root) - rc = p.wait() - return rc - except OSError as e: - raise Exception('while running %s: %s' % (' '.join(cmd), e)) - - try: - os.unlink(opts.output) - except OSError: - pass - - if not opts.libs: - cmd = [opts.archiver] + opts.create_flags + opts.plugin_flags + [opts.output] + opts.objs - stdin = None - exit_code = call() - else: - temp = tempfile.NamedTemporaryFile(dir=os.path.dirname(opts.output), delete=False) - - with open(temp.name, 'w') as tmp: - tmp.write('CREATE {0}\n'.format(opts.output)) - for lib in opts.libs: - tmp.write('ADDLIB {0}\n'.format(lib)) - for obj in opts.objs: - tmp.write('ADDMOD {0}\n'.format(obj)) - tmp.write('SAVE\n') - tmp.write('END\n') - cmd = [opts.archiver] + opts.modify_flags + opts.plugin_flags - stdin = open(temp.name) - exit_code = call() - os.remove(temp.name) - - if exit_code != 0: - raise Exception('{0} returned non-zero exit code {1}. Stop.'.format(' '.join(cmd), exit_code)) diff --git a/build/scripts/llvm_opt_wrapper.py b/build/scripts/llvm_opt_wrapper.py deleted file mode 100644 index 38ca3004af..0000000000 --- a/build/scripts/llvm_opt_wrapper.py +++ /dev/null @@ -1,18 +0,0 @@ -import subprocess -import sys - - -def fix(s): - # we use '#' instead of ',' because ymake always splits args by comma - if 'internalize' in s: - return s.replace('#', ',') - - return s - - -if __name__ == '__main__': - path = sys.argv[1] - args = [fix(s) for s in [path] + sys.argv[2:]] - - rc = subprocess.call(args, shell=False, stderr=sys.stderr, stdout=sys.stdout) - sys.exit(rc) diff --git a/build/scripts/merge_files.py b/build/scripts/merge_files.py deleted file mode 100644 index d42d6a2139..0000000000 --- a/build/scripts/merge_files.py +++ /dev/null @@ -1,8 +0,0 @@ -import sys - - -if __name__ == "__main__": - with open(sys.argv[1], "w") as f: - for appended in sys.argv[2:]: - with open(appended) as a: - f.write(a.read()) diff --git a/build/scripts/preprocess.py b/build/scripts/preprocess.py deleted file mode 100644 index 4657bef732..0000000000 --- a/build/scripts/preprocess.py +++ /dev/null @@ -1,48 +0,0 @@ -import sys -import os - - -def load_file(p): - with open(p, 'r') as f: - return f.read() - - -def step(base, data, hh): - def flt(): - for l in data.split('\n'): - if l in hh: - pp = os.path.join(base, hh[l]) - - yield '\n\n' + load_file(pp) + '\n\n' - - os.unlink(pp) - else: - yield l - - return '\n'.join(flt()) - - -def subst_headers(path, headers): - hh = dict() - - for h in headers: - hh['# include "' + h + '"'] = h - - data = load_file(path) - prev = data - - while True: - ret = step(os.path.dirname(path), prev, hh) - - if ret == prev: - break - - prev = ret - - if data != prev: - with open(path, 'w') as f: - f.write(prev) - - -if __name__ == '__main__': - subst_headers(sys.argv[1], ['stack.hh', 'position.hh', 'location.hh']) diff --git a/build/scripts/py_compile.py b/build/scripts/py_compile.py deleted file mode 100755 index 936dbe8816..0000000000 --- a/build/scripts/py_compile.py +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from __future__ import print_function, absolute_import, division - -import marshal -import sys - - -def main(): - srcpathx, in_fname, out_fname = sys.argv[1:] - srcpath = srcpathx[:-1] - - with open(in_fname, 'r') as in_file: - source = in_file.read() - - code = compile(source, srcpath, 'exec', dont_inherit=True) - - with open(out_fname, 'wb') as out_file: - marshal.dump(code, out_file) - - -if __name__ == "__main__": - main() diff --git a/build/scripts/rodata2asm.py b/build/scripts/rodata2asm.py deleted file mode 100644 index 555639499f..0000000000 --- a/build/scripts/rodata2asm.py +++ /dev/null @@ -1,31 +0,0 @@ -import os -import argparse - - -def main(): - parser = argparse.ArgumentParser(description='Convert rodata into asm source with embedded file content') - parser.add_argument('symbol', help='symvol name exported from generated filr') - parser.add_argument('rodata', help='input .rodata file path') - parser.add_argument('asm', type=argparse.FileType('w', encoding='UTF-8'), help='destination .asm file path') - parser.add_argument('--elf', action='store_true') - - args = parser.parse_args() - - file_size = os.path.getsize(args.rodata) - - args.asm.write('global ' + args.symbol + '\n') - args.asm.write('global ' + args.symbol + 'Size' + '\n') - args.asm.write('SECTION .rodata ALIGN=16\n') - args.asm.write(args.symbol + ':\nincbin "' + args.rodata + '"\n') - args.asm.write('align 4, db 0\n') - args.asm.write(args.symbol + 'Size:\ndd ' + str(file_size) + '\n') - - if args.elf: - args.asm.write('size ' + args.symbol + ' ' + str(file_size) + '\n') - args.asm.write('size ' + args.symbol + 'Size 4\n') - - args.asm.close() - - -if __name__ == '__main__': - main() diff --git a/build/scripts/run_llvm_dsymutil.py b/build/scripts/run_llvm_dsymutil.py deleted file mode 100644 index 4f43362ad9..0000000000 --- a/build/scripts/run_llvm_dsymutil.py +++ /dev/null @@ -1,11 +0,0 @@ -import os -import sys -import subprocess - - -if __name__ == '__main__': - with open(os.devnull, 'w') as fnull: - p = subprocess.Popen(sys.argv[1:], shell=False, stderr=fnull, stdout=sys.stdout) - - p.communicate() - sys.exit(p.returncode) diff --git a/build/scripts/stdout2stderr.py b/build/scripts/stdout2stderr.py deleted file mode 100644 index d7861fdda3..0000000000 --- a/build/scripts/stdout2stderr.py +++ /dev/null @@ -1,6 +0,0 @@ -import subprocess -import sys - -if __name__ == '__main__': - assert len(sys.argv) > 1 - sys.exit(subprocess.Popen(sys.argv[1:], stdout=sys.stderr).wait()) diff --git a/build/scripts/tar_sources.py b/build/scripts/tar_sources.py deleted file mode 100644 index 54e2839a69..0000000000 --- a/build/scripts/tar_sources.py +++ /dev/null @@ -1,41 +0,0 @@ -import argparse -import os -import tarfile - - -def parse_args(): - parser = argparse.ArgumentParser() - parser.add_argument('--exts', nargs='*', default=None) - parser.add_argument('--flat', action='store_true') - parser.add_argument('--input', required=True) - parser.add_argument('--output', required=True) - parser.add_argument('--prefix', default=None) - - return parser.parse_args() - - -def main(): - args = parse_args() - - py_srcs = [] - for root, _, files in os.walk(args.input): - for f in files: - if not args.exts or f.endswith(tuple(args.exts)): - py_srcs.append(os.path.join(root, f)) - - compression_mode = '' - if args.output.endswith(('.tar.gz', '.tgz')): - compression_mode = 'gz' - elif args.output.endswith('.bzip2'): - compression_mode = 'bz2' - - with tarfile.open(args.output, 'w:{}'.format(compression_mode)) as out: - for f in py_srcs: - arcname = os.path.basename(f) if args.flat else os.path.relpath(f, args.input) - if args.prefix: - arcname = os.path.join(args.prefix, arcname) - out.add(f, arcname=arcname) - - -if __name__ == '__main__': - main() diff --git a/build/scripts/tared_protoc.py b/build/scripts/tared_protoc.py deleted file mode 100644 index 7643e1dbfe..0000000000 --- a/build/scripts/tared_protoc.py +++ /dev/null @@ -1,31 +0,0 @@ -import os -import optparse -import tarfile -import contextlib -import subprocess as sp - - -def parse_args(): - parser = optparse.OptionParser() - parser.disable_interspersed_args() - parser.add_option('--tar-output') - parser.add_option('--protoc-out-dir') - return parser.parse_args() - - -def main(): - opts, args = parse_args() - assert opts.tar_output - assert opts.protoc_out_dir - - if not os.path.exists(opts.protoc_out_dir): - os.makedirs(opts.protoc_out_dir) - - sp.check_call(args) - - with contextlib.closing(tarfile.open(opts.tar_output, 'w')) as tf: - tf.add(opts.protoc_out_dir, arcname='') - - -if __name__ == '__main__': - main() diff --git a/build/scripts/touch.py b/build/scripts/touch.py deleted file mode 100755 index e01ba7f86b..0000000000 --- a/build/scripts/touch.py +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env python - -import optparse -import os -import sys -import time - - -def main(argv): - parser = optparse.OptionParser(add_help_option=False) - parser.disable_interspersed_args() - - parser.add_option('-?', '--help', dest='help', - action='store_true', default=None, help='print help') - parser.add_option('-t', dest='t', action='store', default=None) - - opts, argv_rest = parser.parse_args(argv) - if getattr(opts, 'help', False): - parser.print_help() - return 0 - - tspec = opts.t - if tspec is None: - times = None - else: - head, sep, tail = tspec.partition('.') - if 8 > len(head): - raise Exception("time spec must follow format [[CC]YY]MMDDhhmm[.SS]: " + tspec + '; ' + head) - tfmt = '' - if 12 == len(head): - tfmt += '%Y' - elif 10 == len(head): - tfmt += '%y' - tfmt += '%m%d%H%M' - if 2 == len(tail): - tfmt += '.%S' - mtime = time.mktime(time.strptime(tspec, tfmt)) - times = (mtime, mtime) - - for file in argv_rest: - try: - os.utime(file, times) - except: - open(file, 'w').close() - if times is not None: - os.utime(file, times) - - -if __name__ == '__main__': - sys.exit(main(sys.argv[1:])) diff --git a/build/scripts/xargs.py b/build/scripts/xargs.py deleted file mode 100644 index 5d68929ecc..0000000000 --- a/build/scripts/xargs.py +++ /dev/null @@ -1,18 +0,0 @@ -import sys -import os -import subprocess - -if __name__ == '__main__': - pos = sys.argv.index('--') - fname = sys.argv[pos + 1] - cmd = sys.argv[pos + 2:] - - with open(fname, 'r') as f: - args = [x.strip() for x in f] - - os.remove(fname) - - p = subprocess.Popen(cmd + args, shell=False, stderr=sys.stderr, stdout=sys.stdout) - p.communicate() - - sys.exit(p.returncode) diff --git a/build/scripts/yield_line.py b/build/scripts/yield_line.py deleted file mode 100644 index 9c1c539146..0000000000 --- a/build/scripts/yield_line.py +++ /dev/null @@ -1,7 +0,0 @@ -import sys - -if __name__ == '__main__': - pos = sys.argv.index('--') - - with open(sys.argv[pos + 1], 'a') as f: - f.write(' '.join(sys.argv[pos + 2:]) + '\n') |