diff options
author | Anton Samokhvalov <pg83@yandex.ru> | 2022-02-10 16:45:15 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:45:15 +0300 |
commit | 72cb13b4aff9bc9cf22e49251bc8fd143f82538f (patch) | |
tree | da2c34829458c7d4e74bdfbdf85dff449e9e7fb8 /build/scripts | |
parent | 778e51ba091dc39e7b7fcab2b9cf4dbedfb6f2b5 (diff) | |
download | ydb-72cb13b4aff9bc9cf22e49251bc8fd143f82538f.tar.gz |
Restoring authorship annotation for Anton Samokhvalov <pg83@yandex.ru>. Commit 1 of 2.
Diffstat (limited to 'build/scripts')
-rw-r--r-- | build/scripts/check_config_h.py | 138 | ||||
-rw-r--r-- | build/scripts/f2c.py | 20 | ||||
-rwxr-xr-x | build/scripts/fetch_from_sandbox.py | 6 | ||||
-rw-r--r-- | build/scripts/fs_tools.py | 10 | ||||
-rw-r--r-- | build/scripts/gen_join_srcs.py | 24 | ||||
-rw-r--r-- | build/scripts/gen_mx_table.py | 112 | ||||
-rw-r--r-- | build/scripts/gen_py_reg.py | 22 | ||||
-rw-r--r-- | build/scripts/gen_ub.py | 172 | ||||
-rw-r--r-- | build/scripts/link_dyn_lib.py | 112 | ||||
-rw-r--r-- | build/scripts/link_lib.py | 22 | ||||
-rwxr-xr-x | build/scripts/mkver.py | 8 | ||||
-rw-r--r-- | build/scripts/perl_wrapper.py | 12 | ||||
-rw-r--r-- | build/scripts/preprocess.py | 88 | ||||
-rw-r--r-- | build/scripts/run_msvc_wine.py | 710 | ||||
-rwxr-xr-x | build/scripts/run_tool.py | 6 | ||||
-rw-r--r-- | build/scripts/xargs.py | 16 | ||||
-rw-r--r-- | build/scripts/yield_line.py | 6 | ||||
-rw-r--r-- | build/scripts/yndexer.py | 2 |
18 files changed, 743 insertions, 743 deletions
diff --git a/build/scripts/check_config_h.py b/build/scripts/check_config_h.py index 07bc12e230..a19a405d0a 100644 --- a/build/scripts/check_config_h.py +++ b/build/scripts/check_config_h.py @@ -1,88 +1,88 @@ -import sys - -data = """ -#if defined(SIZEOF_LONG) +import sys + +data = """ +#if defined(SIZEOF_LONG) static_assert(sizeof(long) == SIZEOF_LONG, "fixme 1"); -#endif - -#if defined(SIZEOF_PTHREAD_T) -#include <pthread.h> - +#endif + +#if defined(SIZEOF_PTHREAD_T) +#include <pthread.h> + static_assert(sizeof(pthread_t) == SIZEOF_PTHREAD_T, "fixme 2"); -#endif - -#if defined(SIZEOF_SIZE_T) -#include <stddef.h> - +#endif + +#if defined(SIZEOF_SIZE_T) +#include <stddef.h> + static_assert(sizeof(size_t) == SIZEOF_SIZE_T, "fixme 3"); -#endif - -#if defined(SIZEOF_TIME_T) -#include <time.h> - +#endif + +#if defined(SIZEOF_TIME_T) +#include <time.h> + static_assert(sizeof(time_t) == SIZEOF_TIME_T, "fixme 4"); -#endif - -#if defined(SIZEOF_UINTPTR_T) -#include <stdint.h> - +#endif + +#if defined(SIZEOF_UINTPTR_T) +#include <stdint.h> + static_assert(sizeof(uintptr_t) == SIZEOF_UINTPTR_T, "fixme 5"); -#endif - -#if defined(SIZEOF_VOID_P) +#endif + +#if defined(SIZEOF_VOID_P) static_assert(sizeof(void*) == SIZEOF_VOID_P, "fixme 6"); -#endif - -#if defined(SIZEOF_FPOS_T) -#include <stdio.h> - +#endif + +#if defined(SIZEOF_FPOS_T) +#include <stdio.h> + static_assert(sizeof(fpos_t) == SIZEOF_FPOS_T, "fixme 7"); -#endif - -#if defined(SIZEOF_DOUBLE) +#endif + +#if defined(SIZEOF_DOUBLE) static_assert(sizeof(double) == SIZEOF_DOUBLE, "fixme 8"); -#endif - -#if defined(SIZEOF_LONG_DOUBLE) +#endif + +#if defined(SIZEOF_LONG_DOUBLE) static_assert(sizeof(long double) == SIZEOF_LONG_DOUBLE, "fixme 9"); -#endif - -#if defined(SIZEOF_FLOAT) +#endif + +#if defined(SIZEOF_FLOAT) static_assert(sizeof(float) == SIZEOF_FLOAT, "fixme 10"); -#endif - -#if defined(SIZEOF_INT) +#endif + +#if defined(SIZEOF_INT) static_assert(sizeof(int) == SIZEOF_INT, "fixme 11"); -#endif - -#if defined(SIZEOF_LONG_LONG) +#endif + +#if defined(SIZEOF_LONG_LONG) static_assert(sizeof(long long) == SIZEOF_LONG_LONG, "fixme 12"); -#endif - -#if defined(SIZEOF_OFF_T) -#include <stdio.h> - +#endif + +#if defined(SIZEOF_OFF_T) +#include <stdio.h> + static_assert(sizeof(off_t) == SIZEOF_OFF_T, "fixme 13"); -#endif - -#if defined(SIZEOF_PID_T) -#include <unistd.h> - +#endif + +#if defined(SIZEOF_PID_T) +#include <unistd.h> + static_assert(sizeof(pid_t) == SIZEOF_PID_T, "fixme 14"); -#endif - -#if defined(SIZEOF_SHORT) +#endif + +#if defined(SIZEOF_SHORT) static_assert(sizeof(short) == SIZEOF_SHORT, "fixme 15"); -#endif - -#if defined(SIZEOF_WCHAR_T) +#endif + +#if defined(SIZEOF_WCHAR_T) static_assert(sizeof(wchar_t) == SIZEOF_WCHAR_T, "fixme 16"); -#endif - -#if defined(SIZEOF__BOOL) -//TODO -#endif -""" +#endif + +#if defined(SIZEOF__BOOL) +//TODO +#endif +""" if __name__ == '__main__': with open(sys.argv[2], 'w') as f: f.write('#include <' + sys.argv[1] + '>\n\n') diff --git a/build/scripts/f2c.py b/build/scripts/f2c.py index 7021e1391f..243d3965a4 100644 --- a/build/scripts/f2c.py +++ b/build/scripts/f2c.py @@ -1,8 +1,8 @@ -import sys -import subprocess -import argparse -import os - +import sys +import subprocess +import argparse +import os + header = '''\ #ifdef __GNUC__ @@ -30,11 +30,11 @@ def mkdir_p(directory): if __name__ == '__main__': parser = argparse.ArgumentParser() - + parser.add_argument('-t', '--tool') parser.add_argument('-c', '--input') parser.add_argument('-o', '--output') - + args = parser.parse_args() tmpdir = args.output + '.f2c' mkdir_p(tmpdir) @@ -44,14 +44,14 @@ if __name__ == '__main__': stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) stdout, stderr = p.communicate(input=open(args.input).read()) ret = p.wait() - + if ret: print >>sys.stderr, 'f2c failed: %s, %s' % (stderr, ret) sys.exit(ret) - + if 'Error' in stderr: print >>sys.stderr, stderr - + with open(args.output, 'w') as f: f.write(header) f.write(stdout) diff --git a/build/scripts/fetch_from_sandbox.py b/build/scripts/fetch_from_sandbox.py index a99542e174..bd84f7a4d2 100755 --- a/build/scripts/fetch_from_sandbox.py +++ b/build/scripts/fetch_from_sandbox.py @@ -6,12 +6,12 @@ import os import random import subprocess import sys -import time +import time import urllib2 import uuid - + import fetch_from - + ORIGIN_SUFFIX = '?origin=fetch-from-sandbox' MDS_PREFIX = 'http://storage-int.mds.yandex.net/get-sandbox/' diff --git a/build/scripts/fs_tools.py b/build/scripts/fs_tools.py index dec4c349c8..d420e85c50 100644 --- a/build/scripts/fs_tools.py +++ b/build/scripts/fs_tools.py @@ -1,11 +1,11 @@ from __future__ import print_function -import os +import os import platform -import sys -import shutil +import sys +import shutil import errno - + import process_command_files as pcf @@ -26,7 +26,7 @@ def link_or_copy(src, dst): if __name__ == '__main__': mode = sys.argv[1] args = pcf.get_args(sys.argv[2:]) - + if mode == 'copy': shutil.copy(args[0], args[1]) elif mode == 'copy_tree_no_link': diff --git a/build/scripts/gen_join_srcs.py b/build/scripts/gen_join_srcs.py index e0c2df161a..b4ddbb587e 100644 --- a/build/scripts/gen_join_srcs.py +++ b/build/scripts/gen_join_srcs.py @@ -1,16 +1,16 @@ -import sys - +import sys + import process_command_files as pcf + - -with open(sys.argv[1], 'w') as f: - f.write('#if defined(__GNUC__)\n') - f.write('#pragma GCC diagnostic ignored "-Wunknown-pragmas"\n') - f.write('#if defined(__clang__)\n') - f.write('#pragma GCC diagnostic ignored "-Wunknown-warning-option"\n') - f.write('#endif\n') - f.write('#pragma GCC diagnostic ignored "-Wsubobject-linkage"\n') - f.write('#endif\n\n') - +with open(sys.argv[1], 'w') as f: + f.write('#if defined(__GNUC__)\n') + f.write('#pragma GCC diagnostic ignored "-Wunknown-pragmas"\n') + f.write('#if defined(__clang__)\n') + f.write('#pragma GCC diagnostic ignored "-Wunknown-warning-option"\n') + f.write('#endif\n') + f.write('#pragma GCC diagnostic ignored "-Wsubobject-linkage"\n') + f.write('#endif\n\n') + for arg in pcf.iter_args(sys.argv[2:]): f.write('#include "' + arg + '"\n') diff --git a/build/scripts/gen_mx_table.py b/build/scripts/gen_mx_table.py index 187c21c539..cce69e5cfb 100644 --- a/build/scripts/gen_mx_table.py +++ b/build/scripts/gen_mx_table.py @@ -1,75 +1,75 @@ -import sys - -tmpl = """ -#include "yabs_mx_calc_table.h" - -#include <kernel/matrixnet/mn_sse.h> - +import sys + +tmpl = """ +#include "yabs_mx_calc_table.h" + +#include <kernel/matrixnet/mn_sse.h> + #include <library/cpp/archive/yarchive.h> - -#include <util/memory/blob.h> -#include <util/generic/hash.h> -#include <util/generic/ptr.h> -#include <util/generic/singleton.h> - -using namespace NMatrixnet; - -extern "C" { - extern const unsigned char MxFormulas[]; - extern const ui32 MxFormulasSize; -} - -namespace { - struct TFml: public TBlob, public TMnSseInfo { - inline TFml(const TBlob& b) - : TBlob(b) - , TMnSseInfo(Data(), Size()) - { - } - }; - + +#include <util/memory/blob.h> +#include <util/generic/hash.h> +#include <util/generic/ptr.h> +#include <util/generic/singleton.h> + +using namespace NMatrixnet; + +extern "C" { + extern const unsigned char MxFormulas[]; + extern const ui32 MxFormulasSize; +} + +namespace { + struct TFml: public TBlob, public TMnSseInfo { + inline TFml(const TBlob& b) + : TBlob(b) + , TMnSseInfo(Data(), Size()) + { + } + }; + struct TFormulas: public THashMap<size_t, TAutoPtr<TFml>> { - inline TFormulas() { - TBlob b = TBlob::NoCopy(MxFormulas, MxFormulasSize); - TArchiveReader ar(b); - %s - } - + inline TFormulas() { + TBlob b = TBlob::NoCopy(MxFormulas, MxFormulasSize); + TArchiveReader ar(b); + %s + } + inline const TMnSseInfo& at(size_t n) const noexcept { - return *find(n)->second; - } - }; - - %s - - static func_descr_t yabs_funcs[] = { - %s - }; -} - -yabs_mx_calc_table_t yabs_mx_calc_table = {YABS_MX_CALC_VERSION, 10000, 0, yabs_funcs}; -""" - + return *find(n)->second; + } + }; + + %s + + static func_descr_t yabs_funcs[] = { + %s + }; +} + +yabs_mx_calc_table_t yabs_mx_calc_table = {YABS_MX_CALC_VERSION, 10000, 0, yabs_funcs}; +""" + if __name__ == '__main__': init = [] body = [] defs = {} - + for i in sys.argv[1:]: name = i.replace('.', '_') num = long(name.split('_')[1]) - + init.append('(*this)[%s] = new TFml(ar.ObjectBlobByKey("%s"));' % (num, '/' + i)) - + f1 = 'static void yabs_%s(size_t count, const float** args, double* res) {Singleton<TFormulas>()->at(%s).DoCalcRelevs(args, res, count);}' % (name, num) f2 = 'static size_t yabs_%s_factor_count() {return Singleton<TFormulas>()->at(%s).MaxFactorIndex() + 1;}' % (name, num) - + body.append(f1) body.append(f2) - + d1 = 'yabs_%s' % name d2 = 'yabs_%s_factor_count' % name - + defs[num] = '{%s, %s}' % (d1, d2) - + print tmpl % ('\n'.join(init), '\n\n'.join(body), ',\n'.join((defs.get(i, '{nullptr, nullptr}') for i in range(0, 10000)))) diff --git a/build/scripts/gen_py_reg.py b/build/scripts/gen_py_reg.py index 1560135ae8..7526198395 100644 --- a/build/scripts/gen_py_reg.py +++ b/build/scripts/gen_py_reg.py @@ -1,19 +1,19 @@ import sys -template = ''' +template = ''' extern "C" void PyImport_AppendInittab(const char* name, void (*fn)(void)); extern "C" void {1}(); - -namespace { - struct TRegistrar { - inline TRegistrar() { + +namespace { + struct TRegistrar { + inline TRegistrar() { PyImport_AppendInittab("{0}", {1}); - } - } REG; -} -''' - - + } + } REG; +} +''' + + def mangle(name): if '.' not in name: return name diff --git a/build/scripts/gen_ub.py b/build/scripts/gen_ub.py index ad79cda926..cced92e99b 100644 --- a/build/scripts/gen_ub.py +++ b/build/scripts/gen_ub.py @@ -1,86 +1,86 @@ -import argparse -import os -import tarfile -import contextlib -import hashlib -import base64 -import io - - -stub = """#!/usr/bin/env python - -info = {info} -data = "{data}" - -import platform -import os -import sys -import tarfile -import contextlib -import io -import base64 - - -def current_platform(): - arch = platform.machine().upper() - - if arch == 'AMD64': - arch = 'X86_64' - - platf = platform.system().upper() - - if platf.startswith('WIN'): - platf = 'WIN' - - return (platf + '-' + arch).lower() - - -def extract_file(fname): - with contextlib.closing(tarfile.open(fileobj=io.BytesIO(base64.b64decode(data)))) as f: - return f.extractfile(fname).read() - - -fname = info[current_platform()] -my_path = os.path.realpath(os.path.abspath(__file__)) -tmp_path = my_path + '.tmp' - -with open(tmp_path, 'wb') as f: - f.write(extract_file(fname)) - -os.rename(tmp_path, my_path) -os.chmod(my_path, 0775) -os.execv(sys.argv[0], sys.argv) -""" - - -def gen_ub(output, data): - info = {} - binary = io.BytesIO() - - with contextlib.closing(tarfile.open(mode='w:bz2', fileobj=binary, dereference=True)) as f: - for pl, path in data: - fname = os.path.basename(path) - pl = pl.split('-') - pl = pl[1] + '-' + pl[2] - info[pl] = fname - f.add(path, arcname=fname) - - binary = binary.getvalue() - info['md5'] = hashlib.md5(binary).hexdigest() - - with open(output, 'w') as f: - f.write(stub.format(info=info, data=base64.b64encode(binary))) - - os.chmod(output, 0775) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - - parser.add_argument('--path', action='append') - parser.add_argument('--platform', action='append') - parser.add_argument('--output', action='store') - - args = parser.parse_args() - - gen_ub(args.output, zip(args.platform, args.path)) +import argparse +import os +import tarfile +import contextlib +import hashlib +import base64 +import io + + +stub = """#!/usr/bin/env python + +info = {info} +data = "{data}" + +import platform +import os +import sys +import tarfile +import contextlib +import io +import base64 + + +def current_platform(): + arch = platform.machine().upper() + + if arch == 'AMD64': + arch = 'X86_64' + + platf = platform.system().upper() + + if platf.startswith('WIN'): + platf = 'WIN' + + return (platf + '-' + arch).lower() + + +def extract_file(fname): + with contextlib.closing(tarfile.open(fileobj=io.BytesIO(base64.b64decode(data)))) as f: + return f.extractfile(fname).read() + + +fname = info[current_platform()] +my_path = os.path.realpath(os.path.abspath(__file__)) +tmp_path = my_path + '.tmp' + +with open(tmp_path, 'wb') as f: + f.write(extract_file(fname)) + +os.rename(tmp_path, my_path) +os.chmod(my_path, 0775) +os.execv(sys.argv[0], sys.argv) +""" + + +def gen_ub(output, data): + info = {} + binary = io.BytesIO() + + with contextlib.closing(tarfile.open(mode='w:bz2', fileobj=binary, dereference=True)) as f: + for pl, path in data: + fname = os.path.basename(path) + pl = pl.split('-') + pl = pl[1] + '-' + pl[2] + info[pl] = fname + f.add(path, arcname=fname) + + binary = binary.getvalue() + info['md5'] = hashlib.md5(binary).hexdigest() + + with open(output, 'w') as f: + f.write(stub.format(info=info, data=base64.b64encode(binary))) + + os.chmod(output, 0775) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + + parser.add_argument('--path', action='append') + parser.add_argument('--platform', action='append') + parser.add_argument('--output', action='store') + + args = parser.parse_args() + + gen_ub(args.output, zip(args.platform, args.path)) diff --git a/build/scripts/link_dyn_lib.py b/build/scripts/link_dyn_lib.py index 23487f5c1e..58ae45d101 100644 --- a/build/scripts/link_dyn_lib.py +++ b/build/scripts/link_dyn_lib.py @@ -1,13 +1,13 @@ -import sys -import os +import sys +import os import subprocess -import tempfile -import collections +import tempfile +import collections import optparse import pipes - + from process_whole_archive_option import ProcessWholeArchiveOption - + def shlex_join(cmd): # equivalent to shlex.join() in python 3 @@ -17,12 +17,12 @@ def shlex_join(cmd): ) -def parse_export_file(p): - with open(p, 'r') as f: +def parse_export_file(p): + with open(p, 'r') as f: for l in f: - l = l.strip() - - if l and '#' not in l: + l = l.strip() + + if l and '#' not in l: words = l.split() if len(words) == 2 and words[0] == 'linux_version': yield {'linux_version': words[1]} @@ -32,8 +32,8 @@ def parse_export_file(p): yield {'lang': 'C', 'sym': words[0]} else: raise Exception('unsupported exports line: ' + l) - - + + def to_c(sym): symbols = collections.deque(sym.split('::')) c_prefixes = [ # demangle prefixes for c++ symbols @@ -60,24 +60,24 @@ def to_c(sym): return ['{prefix}{sym}'.format(prefix=prefix, sym=c_sym) for prefix in c_prefixes] -def fix_darwin_param(ex): - for item in ex: +def fix_darwin_param(ex): + for item in ex: if item.get('linux_version'): continue - if item['lang'] == 'C': - yield '-Wl,-exported_symbol,_' + item['sym'] + if item['lang'] == 'C': + yield '-Wl,-exported_symbol,_' + item['sym'] elif item['lang'] == 'C++': for sym in to_c(item['sym']): yield '-Wl,-exported_symbol,_' + sym - else: - raise Exception('unsupported lang: ' + item['lang']) - - -def fix_gnu_param(arch, ex): - d = collections.defaultdict(list) + else: + raise Exception('unsupported lang: ' + item['lang']) + + +def fix_gnu_param(arch, ex): + d = collections.defaultdict(list) version = None - for item in ex: + for item in ex: if item.get('linux_version'): if not version: version = item.get('linux_version') @@ -87,31 +87,31 @@ def fix_gnu_param(arch, ex): d['C'].extend(to_c(item['sym'])) else: d[item['lang']].append(item['sym']) - + with tempfile.NamedTemporaryFile(mode='wt', delete=False) as f: if version: f.write('{} {{\nglobal:\n'.format(version)) else: f.write('{\nglobal:\n') - - for k, v in d.items(): - f.write(' extern "' + k + '" {\n') - - for x in v: + + for k, v in d.items(): + f.write(' extern "' + k + '" {\n') + + for x in v: f.write(' ' + x + ';\n') - - f.write(' };\n') - - f.write('local: *;\n};\n') - - ret = ['-Wl,--version-script=' + f.name] - - if arch == 'ANDROID': - ret += ['-Wl,--export-dynamic'] - - return ret - - + + f.write(' };\n') + + f.write('local: *;\n};\n') + + ret = ['-Wl,--version-script=' + f.name] + + if arch == 'ANDROID': + ret += ['-Wl,--export-dynamic'] + + return ret + + def fix_windows_param(ex): with tempfile.NamedTemporaryFile(delete=False) as def_file: exports = [] @@ -131,33 +131,33 @@ def fix_cmd(arch, musl, c): if arch == 'WINDOWS': prefix = '/DEF:' f = fix_windows_param - else: + else: prefix = '-Wl,--version-script=' if arch in ('DARWIN', 'IOS'): f = fix_darwin_param else: f = lambda x: fix_gnu_param(arch, x) - - def do_fix(p): + + def do_fix(p): if musl and p in musl_libs: return [] - if p.startswith(prefix) and p.endswith('.exports'): - fname = p[len(prefix):] - - return list(f(list(parse_export_file(fname)))) - + if p.startswith(prefix) and p.endswith('.exports'): + fname = p[len(prefix):] + + return list(f(list(parse_export_file(fname)))) + if p.endswith('.supp'): return [] if p.endswith('.pkg.fake'): return [] - return [p] - - return sum((do_fix(x) for x in c), []) - - + return [p] + + return sum((do_fix(x) for x in c), []) + + def parse_args(): parser = optparse.OptionParser() parser.disable_interspersed_args() diff --git a/build/scripts/link_lib.py b/build/scripts/link_lib.py index 344d50d4eb..c3ef7b9607 100644 --- a/build/scripts/link_lib.py +++ b/build/scripts/link_lib.py @@ -49,18 +49,18 @@ if __name__ == "__main__": os.environ['SYM64_THRESHOLD'] = '31' def call(): - try: - p = subprocess.Popen(cmd, stdin=stdin, cwd=opts.build_root) - rc = p.wait() - return rc - except OSError as e: - raise Exception('while running %s: %s' % (' '.join(cmd), e)) - - try: - os.unlink(opts.output) - except OSError: - pass + try: + p = subprocess.Popen(cmd, stdin=stdin, cwd=opts.build_root) + rc = p.wait() + return rc + except OSError as e: + raise Exception('while running %s: %s' % (' '.join(cmd), e)) + try: + os.unlink(opts.output) + except OSError: + pass + if not opts.libs: cmd = [opts.archiver] + opts.create_flags + opts.plugin_flags + [opts.output] + opts.objs stdin = None diff --git a/build/scripts/mkver.py b/build/scripts/mkver.py index 321cdaade1..6b4ac58bf7 100755 --- a/build/scripts/mkver.py +++ b/build/scripts/mkver.py @@ -1,12 +1,12 @@ -import sys - +import sys + if __name__ == '__main__': with open(sys.argv[1], 'r') as f: data = f.readline() - + beg = data.find('(') + 1 end = data.find(')') version = data[beg:end] - + print '#pragma once' print '#define DEBIAN_VERSION "%s"' % version diff --git a/build/scripts/perl_wrapper.py b/build/scripts/perl_wrapper.py index cb4027f1d3..cd456e3056 100644 --- a/build/scripts/perl_wrapper.py +++ b/build/scripts/perl_wrapper.py @@ -1,24 +1,24 @@ import os import sys -import shutil +import shutil if __name__ == '__main__': path = sys.argv[1] to = sys.argv[-1] fr = sys.argv[-2] to_dir = os.path.dirname(to) - + os.chdir(to_dir) - + f1 = os.path.basename(fr) fr_ = os.path.dirname(fr) f2 = os.path.basename(fr_) fr_ = os.path.dirname(fr_) - + os.makedirs(f2) shutil.copyfile(fr, os.path.join(f2, f1)) - + if path[0] != '/': path = os.path.join(os.path.dirname(__file__), path) - + os.execv(path, [path] + sys.argv[2:]) diff --git a/build/scripts/preprocess.py b/build/scripts/preprocess.py index 4657bef732..24846b4c23 100644 --- a/build/scripts/preprocess.py +++ b/build/scripts/preprocess.py @@ -1,48 +1,48 @@ import sys import os - - -def load_file(p): - with open(p, 'r') as f: - return f.read() - - -def step(base, data, hh): - def flt(): - for l in data.split('\n'): - if l in hh: - pp = os.path.join(base, hh[l]) - - yield '\n\n' + load_file(pp) + '\n\n' - - os.unlink(pp) - else: - yield l - - return '\n'.join(flt()) - - -def subst_headers(path, headers): - hh = dict() - - for h in headers: - hh['# include "' + h + '"'] = h - - data = load_file(path) - prev = data - - while True: - ret = step(os.path.dirname(path), prev, hh) - - if ret == prev: - break - - prev = ret - - if data != prev: - with open(path, 'w') as f: - f.write(prev) - - + + +def load_file(p): + with open(p, 'r') as f: + return f.read() + + +def step(base, data, hh): + def flt(): + for l in data.split('\n'): + if l in hh: + pp = os.path.join(base, hh[l]) + + yield '\n\n' + load_file(pp) + '\n\n' + + os.unlink(pp) + else: + yield l + + return '\n'.join(flt()) + + +def subst_headers(path, headers): + hh = dict() + + for h in headers: + hh['# include "' + h + '"'] = h + + data = load_file(path) + prev = data + + while True: + ret = step(os.path.dirname(path), prev, hh) + + if ret == prev: + break + + prev = ret + + if data != prev: + with open(path, 'w') as f: + f.write(prev) + + if __name__ == '__main__': subst_headers(sys.argv[1], ['stack.hh', 'position.hh', 'location.hh']) diff --git a/build/scripts/run_msvc_wine.py b/build/scripts/run_msvc_wine.py index 439d1f8831..4763a39f70 100644 --- a/build/scripts/run_msvc_wine.py +++ b/build/scripts/run_msvc_wine.py @@ -1,60 +1,60 @@ -import sys -import os +import sys +import os import re -import subprocess -import signal -import time -import json +import subprocess +import signal +import time +import json import argparse import errno - + import process_command_files as pcf import process_whole_archive_option as pwa + - -procs = [] -build_kekeke = 45 - - +procs = [] +build_kekeke = 45 + + def stringize(s): return s.encode('utf-8') if isinstance(s, unicode) else s -def run_subprocess(*args, **kwargs): +def run_subprocess(*args, **kwargs): if 'env' in kwargs: kwargs['env'] = {stringize(k): stringize(v) for k, v in kwargs['env'].iteritems()} - p = subprocess.Popen(*args, **kwargs) - - procs.append(p) - - return p - - -def terminate_slaves(): - for p in procs: - try: - p.terminate() - except Exception: - pass - - -def sig_term(sig, fr): - terminate_slaves() - sys.exit(sig) - - -def subst_path(l): - if len(l) > 3: - if l[:3].lower() in ('z:\\', 'z:/'): - return l[2:].replace('\\', '/') - - return l - - + p = subprocess.Popen(*args, **kwargs) + + procs.append(p) + + return p + + +def terminate_slaves(): + for p in procs: + try: + p.terminate() + except Exception: + pass + + +def sig_term(sig, fr): + terminate_slaves() + sys.exit(sig) + + +def subst_path(l): + if len(l) > 3: + if l[:3].lower() in ('z:\\', 'z:/'): + return l[2:].replace('\\', '/') + + return l + + def call_wine_cmd_once(wine, cmd, env, mode): p = run_subprocess(wine + cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=env, close_fds=True, shell=False) - + output = find_cmd_out(cmd) error = None if output is not None and os.path.exists(output): @@ -69,15 +69,15 @@ def call_wine_cmd_once(wine, cmd, env, mode): if error is not None: print >> sys.stderr, 'Output {} already exists and we have failed to remove it: {}'.format(output, error) - # print >>sys.stderr, cmd, env, wine - + # print >>sys.stderr, cmd, env, wine + stdout_and_stderr, _ = p.communicate() - + return_code = p.returncode if not stdout_and_stderr: if return_code != 0: raise Exception('wine did something strange') - + return return_code elif ' : fatal error ' in stdout_and_stderr: return_code = 1 @@ -85,27 +85,27 @@ def call_wine_cmd_once(wine, cmd, env, mode): return_code = 2 lines = [x.strip() for x in stdout_and_stderr.split('\n')] - - prefixes = [ - 'Microsoft (R)', - 'Copyright (C)', - 'Application tried to create a window', - 'The graphics driver is missing', - 'Could not load wine-gecko', - 'wine: configuration in', - 'wine: created the configuration directory', + + prefixes = [ + 'Microsoft (R)', + 'Copyright (C)', + 'Application tried to create a window', + 'The graphics driver is missing', + 'Could not load wine-gecko', + 'wine: configuration in', + 'wine: created the configuration directory', 'libpng warning:' - ] - - suffixes = [ - '.c', - '.cxx', - '.cc', - '.cpp', - '.masm', - ] - - substrs = [ + ] + + suffixes = [ + '.c', + '.cxx', + '.cc', + '.cpp', + '.masm', + ] + + substrs = [ 'Creating library Z:', 'err:heap', 'err:menubuilder:', @@ -113,211 +113,211 @@ def call_wine_cmd_once(wine, cmd, env, mode): 'err:ole:', 'err:wincodecs:', 'err:winediag:', - ] - - def good_line(l): - for x in prefixes: - if l.startswith(x): - return False - - for x in suffixes: - if l.endswith(x): - return False - - for x in substrs: - if x in l: - return False - - return True - - def filter_lines(): - for l in lines: - if good_line(l): - yield subst_path(l.strip()) - + ] + + def good_line(l): + for x in prefixes: + if l.startswith(x): + return False + + for x in suffixes: + if l.endswith(x): + return False + + for x in substrs: + if x in l: + return False + + return True + + def filter_lines(): + for l in lines: + if good_line(l): + yield subst_path(l.strip()) + stdout_and_stderr = '\n'.join(filter_lines()).strip() - + if stdout_and_stderr: print >>sys.stderr, stdout_and_stderr - + return return_code - - -def prepare_vc(fr, to): - for p in os.listdir(fr): - fr_p = os.path.join(fr, p) - to_p = os.path.join(to, p) - - if not os.path.exists(to_p): - print >>sys.stderr, 'install %s -> %s' % (fr_p, to_p) - - os.link(fr_p, to_p) - - -def run_slave(): + + +def prepare_vc(fr, to): + for p in os.listdir(fr): + fr_p = os.path.join(fr, p) + to_p = os.path.join(to, p) + + if not os.path.exists(to_p): + print >>sys.stderr, 'install %s -> %s' % (fr_p, to_p) + + os.link(fr_p, to_p) + + +def run_slave(): args = json.loads(sys.argv[3]) wine = sys.argv[1] - - signal.signal(signal.SIGTERM, sig_term) - - if args.get('tout', None): - signal.signal(signal.SIGALRM, sig_term) - signal.alarm(args['tout']) - - tout = 0.1 - - while True: - try: + + signal.signal(signal.SIGTERM, sig_term) + + if args.get('tout', None): + signal.signal(signal.SIGALRM, sig_term) + signal.alarm(args['tout']) + + tout = 0.1 + + while True: + try: return call_wine_cmd_once([wine], args['cmd'], args['env'], args['mode']) - except Exception as e: - print >>sys.stderr, '%s, will retry in %s' % (str(e), tout) - - time.sleep(tout) - tout = min(2 * tout, 4) - - -def find_cmd_out(args): - for arg in args: - if arg.startswith('/Fo'): - return arg[3:] - - if arg.startswith('/OUT:'): - return arg[5:] - - -def calc_zero_cnt(data): - zero_cnt = 0 - - for ch in data: - if ch == chr(0): - zero_cnt += 1 - - return zero_cnt - - -def is_good_file(p): - if not os.path.isfile(p): - return False - - if os.path.getsize(p) < 300: - return False - + except Exception as e: + print >>sys.stderr, '%s, will retry in %s' % (str(e), tout) + + time.sleep(tout) + tout = min(2 * tout, 4) + + +def find_cmd_out(args): + for arg in args: + if arg.startswith('/Fo'): + return arg[3:] + + if arg.startswith('/OUT:'): + return arg[5:] + + +def calc_zero_cnt(data): + zero_cnt = 0 + + for ch in data: + if ch == chr(0): + zero_cnt += 1 + + return zero_cnt + + +def is_good_file(p): + if not os.path.isfile(p): + return False + + if os.path.getsize(p) < 300: + return False + asm_pattern = re.compile('asm(\.\w+)?\.obj$') if asm_pattern.search(p): - pass - elif p.endswith('.obj'): - with open(p, 'rb') as f: - prefix = f.read(200) - - if ord(prefix[0]) != 0: - return False - - if ord(prefix[1]) != 0: - return False - - if ord(prefix[2]) != 0xFF: - return False - - if ord(prefix[3]) != 0xFF: - return False - - if calc_zero_cnt(prefix) > 195: - return False - - f.seek(-100, os.SEEK_END) - last = f.read(100) - - if calc_zero_cnt(last) > 95: - return False - - if last[-1] != chr(0): - return False - elif p.endswith('.lib'): - with open(p, 'rb') as f: - if f.read(7) != '!<arch>': - return False - - return True - - -RED = '\x1b[31;1m' -GRAY = '\x1b[30;1m' -RST = '\x1b[0m' -MGT = '\x1b[35m' -YEL = '\x1b[33m' -GRN = '\x1b[32m' -CYA = '\x1b[36m' - - -def colorize_strings(l): - p = l.find("'") - - if p >= 0: - yield l[:p] - - l = l[p + 1:] - - p = l.find("'") - - if p >= 0: - yield CYA + "'" + subst_path(l[:p]) + "'" + RST - - for x in colorize_strings(l[p + 1:]): - yield x - else: - yield "'" + l - else: - yield l - - -def colorize_line(l): - lll = l - - try: - parts = [] - - if l.startswith('(compiler file'): - return ''.join(colorize_strings(l)) - - if l.startswith('/'): - p = l.find('(') - parts.append(GRAY + l[:p] + RST) - l = l[p:] - - if l and l.startswith('('): - p = l.find(')') - parts.append(':' + MGT + l[1:p] + RST) - l = l[p + 1:] - - if l: - if l.startswith(' : '): - l = l[1:] - - if l.startswith(': error'): - parts.append(': ' + RED + 'error' + RST) - l = l[7:] - elif l.startswith(': warning'): - parts.append(': ' + YEL + 'warning' + RST) - l = l[9:] - elif l.startswith(': note'): - parts.append(': ' + GRN + 'note' + RST) - l = l[6:] - elif l.startswith('fatal error'): - parts.append(RED + 'fatal error' + RST) - l = l[11:] - - if l: - parts.extend(colorize_strings(l)) - - return ''.join(parts) + pass + elif p.endswith('.obj'): + with open(p, 'rb') as f: + prefix = f.read(200) + + if ord(prefix[0]) != 0: + return False + + if ord(prefix[1]) != 0: + return False + + if ord(prefix[2]) != 0xFF: + return False + + if ord(prefix[3]) != 0xFF: + return False + + if calc_zero_cnt(prefix) > 195: + return False + + f.seek(-100, os.SEEK_END) + last = f.read(100) + + if calc_zero_cnt(last) > 95: + return False + + if last[-1] != chr(0): + return False + elif p.endswith('.lib'): + with open(p, 'rb') as f: + if f.read(7) != '!<arch>': + return False + + return True + + +RED = '\x1b[31;1m' +GRAY = '\x1b[30;1m' +RST = '\x1b[0m' +MGT = '\x1b[35m' +YEL = '\x1b[33m' +GRN = '\x1b[32m' +CYA = '\x1b[36m' + + +def colorize_strings(l): + p = l.find("'") + + if p >= 0: + yield l[:p] + + l = l[p + 1:] + + p = l.find("'") + + if p >= 0: + yield CYA + "'" + subst_path(l[:p]) + "'" + RST + + for x in colorize_strings(l[p + 1:]): + yield x + else: + yield "'" + l + else: + yield l + + +def colorize_line(l): + lll = l + + try: + parts = [] + + if l.startswith('(compiler file'): + return ''.join(colorize_strings(l)) + + if l.startswith('/'): + p = l.find('(') + parts.append(GRAY + l[:p] + RST) + l = l[p:] + + if l and l.startswith('('): + p = l.find(')') + parts.append(':' + MGT + l[1:p] + RST) + l = l[p + 1:] + + if l: + if l.startswith(' : '): + l = l[1:] + + if l.startswith(': error'): + parts.append(': ' + RED + 'error' + RST) + l = l[7:] + elif l.startswith(': warning'): + parts.append(': ' + YEL + 'warning' + RST) + l = l[9:] + elif l.startswith(': note'): + parts.append(': ' + GRN + 'note' + RST) + l = l[6:] + elif l.startswith('fatal error'): + parts.append(RED + 'fatal error' + RST) + l = l[11:] + + if l: + parts.extend(colorize_strings(l)) + + return ''.join(parts) except Exception: - return lll - - -def colorize(out): - return '\n'.join(colorize_line(l) for l in out.split('\n')) - - + return lll + + +def colorize(out): + return '\n'.join(colorize_line(l) for l in out.split('\n')) + + def trim_path(path, winepath): p1 = run_subprocess([winepath, '-w', path], stdout=subprocess.PIPE, stderr=subprocess.PIPE) p1_stdout, p1_stderr = p1.communicate() @@ -376,7 +376,7 @@ def fix_path(p): if p.startswith('/Fo'): return '/Fo' + p[3:].replace('/', '\\') return p - + def process_free_args(args, wine, bld_root, mode): whole_archive_prefix = '/WHOLEARCHIVE:' short_names = {} @@ -385,16 +385,16 @@ def process_free_args(args, wine, bld_root, mode): # Slow for no benefit. # arc_root = args.arcadia_root # short_names[arc_root] = trim_path(arc_root, winepath) - + free_args, wa_peers, wa_libs = pwa.get_whole_archive_peers_and_libs(pcf.skip_markers(args)) - + process_link = lambda x: make_full_path_arg(x, bld_root, short_names[bld_root]) if mode in ('link', 'lib') else x def process_arg(arg): with_wa_prefix = arg.startswith(whole_archive_prefix) prefix = whole_archive_prefix if with_wa_prefix else '' without_prefix_arg = arg[len(prefix):] return prefix + fix_path(process_link(downsize_path(without_prefix_arg, short_names))) - + result = [] for arg in free_args: if pcf.is_cmdfile_arg(arg): @@ -430,51 +430,51 @@ def run_main(): free_args = args.free_args wine_dir = os.path.dirname(os.path.dirname(wine)) - bin_dir = os.path.dirname(binary) + bin_dir = os.path.dirname(binary) tc_dir = os.path.dirname(os.path.dirname(os.path.dirname(bin_dir))) if not incl_paths: incl_paths = [tc_dir + '/VC/include', tc_dir + '/include'] - + cmd_out = find_cmd_out(free_args) - env = os.environ.copy() - - env.pop('DISPLAY', None) - + env = os.environ.copy() + + env.pop('DISPLAY', None) + env['WINEDLLOVERRIDES'] = 'msvcr{}=n'.format(version) - env['WINEDEBUG'] = 'fixme-all' + env['WINEDEBUG'] = 'fixme-all' env['INCLUDE'] = ';'.join(fix_path(p) for p in incl_paths) - env['VSINSTALLDIR'] = fix_path(tc_dir) - env['VCINSTALLDIR'] = fix_path(tc_dir + '/VC') - env['WindowsSdkDir'] = fix_path(tc_dir) + env['VSINSTALLDIR'] = fix_path(tc_dir) + env['VCINSTALLDIR'] = fix_path(tc_dir + '/VC') + env['WindowsSdkDir'] = fix_path(tc_dir) env['LIBPATH'] = fix_path(tc_dir + '/VC/lib/amd64') env['LIB'] = fix_path(tc_dir + '/VC/lib/amd64') env['LD_LIBRARY_PATH'] = ':'.join(wine_dir + d for d in ['/lib', '/lib64', '/lib64/wine']) - + cmd = [binary] + process_free_args(free_args, wine, bld_root, mode) for x in ('/NOLOGO', '/nologo', '/FD'): - try: - cmd.remove(x) - except ValueError: - pass - - def run_process(sleep, tout): - if sleep: - time.sleep(sleep) - - args = { - 'cmd': cmd, - 'env': env, - 'mode': mode, - 'tout': tout - } - + try: + cmd.remove(x) + except ValueError: + pass + + def run_process(sleep, tout): + if sleep: + time.sleep(sleep) + + args = { + 'cmd': cmd, + 'env': env, + 'mode': mode, + 'tout': tout + } + slave_cmd = [sys.executable, sys.argv[0], wine, 'slave', json.dumps(args)] p = run_subprocess(slave_cmd, stderr=subprocess.STDOUT, stdout=subprocess.PIPE, shell=False) - out, _ = p.communicate() - return p.wait(), out - + out, _ = p.communicate() + return p.wait(), out + def print_err_log(log): if not log: return @@ -483,74 +483,74 @@ def run_main(): print >>sys.stderr, log tout = 200 - - while True: - rc, out = run_process(0, tout) - - if rc in (-signal.SIGALRM, signal.SIGALRM): + + while True: + rc, out = run_process(0, tout) + + if rc in (-signal.SIGALRM, signal.SIGALRM): print_err_log(out) - print >>sys.stderr, '##append_tag##time out' + print >>sys.stderr, '##append_tag##time out' elif out and ' stack overflow ' in out: - print >>sys.stderr, '##append_tag##stack overflow' - elif out and 'recvmsg: Connection reset by peer' in out: - print >>sys.stderr, '##append_tag##wine gone' - elif out and 'D8037' in out: - print >>sys.stderr, '##append_tag##repair wine' - - try: - os.unlink(os.path.join(os.environ['WINEPREFIX'], '.update-timestamp')) - except Exception as e: - print >>sys.stderr, e - - else: + print >>sys.stderr, '##append_tag##stack overflow' + elif out and 'recvmsg: Connection reset by peer' in out: + print >>sys.stderr, '##append_tag##wine gone' + elif out and 'D8037' in out: + print >>sys.stderr, '##append_tag##repair wine' + + try: + os.unlink(os.path.join(os.environ['WINEPREFIX'], '.update-timestamp')) + except Exception as e: + print >>sys.stderr, e + + else: print_err_log(out) - - # non-zero return code - bad, return it immediately - if rc: + + # non-zero return code - bad, return it immediately + if rc: print >>sys.stderr, '##win_cmd##' + ' '.join(cmd) print >>sys.stderr, '##args##' + ' '.join(free_args) - return rc - - # check for output existence(if we expect it!) and real length - if cmd_out: - if is_good_file(cmd_out): - return 0 - else: - # retry! - print >>sys.stderr, '##append_tag##no output' - else: - return 0 - - tout *= 3 - - -def main(): + return rc + + # check for output existence(if we expect it!) and real length + if cmd_out: + if is_good_file(cmd_out): + return 0 + else: + # retry! + print >>sys.stderr, '##append_tag##no output' + else: + return 0 + + tout *= 3 + + +def main(): prefix_suffix = os.environ.pop('WINEPREFIX_SUFFIX', None) if prefix_suffix is not None: prefix = os.environ.pop('WINEPREFIX', None) if prefix is not None: os.environ['WINEPREFIX'] = os.path.join(prefix, prefix_suffix) - # just in case - signal.alarm(2000) - - if sys.argv[2] == 'slave': - func = run_slave - else: - func = run_main - - try: - try: - sys.exit(func()) - finally: - terminate_slaves() - except KeyboardInterrupt: - sys.exit(4) - except Exception as e: - print >>sys.stderr, str(e) - - sys.exit(3) - - -if __name__ == '__main__': - main() + # just in case + signal.alarm(2000) + + if sys.argv[2] == 'slave': + func = run_slave + else: + func = run_main + + try: + try: + sys.exit(func()) + finally: + terminate_slaves() + except KeyboardInterrupt: + sys.exit(4) + except Exception as e: + print >>sys.stderr, str(e) + + sys.exit(3) + + +if __name__ == '__main__': + main() diff --git a/build/scripts/run_tool.py b/build/scripts/run_tool.py index 00e3ff6f1e..fe4b544617 100755 --- a/build/scripts/run_tool.py +++ b/build/scripts/run_tool.py @@ -1,7 +1,7 @@ -import sys -import subprocess +import sys +import subprocess import os - + if __name__ == '__main__': env = os.environ.copy() diff --git a/build/scripts/xargs.py b/build/scripts/xargs.py index 5d68929ecc..64295d594f 100644 --- a/build/scripts/xargs.py +++ b/build/scripts/xargs.py @@ -1,18 +1,18 @@ -import sys -import os -import subprocess - +import sys +import os +import subprocess + if __name__ == '__main__': pos = sys.argv.index('--') fname = sys.argv[pos + 1] cmd = sys.argv[pos + 2:] - + with open(fname, 'r') as f: args = [x.strip() for x in f] - + os.remove(fname) - + p = subprocess.Popen(cmd + args, shell=False, stderr=sys.stderr, stdout=sys.stdout) p.communicate() - + sys.exit(p.returncode) diff --git a/build/scripts/yield_line.py b/build/scripts/yield_line.py index 9c1c539146..e95ab5ec62 100644 --- a/build/scripts/yield_line.py +++ b/build/scripts/yield_line.py @@ -1,7 +1,7 @@ -import sys - +import sys + if __name__ == '__main__': pos = sys.argv.index('--') - + with open(sys.argv[pos + 1], 'a') as f: f.write(' '.join(sys.argv[pos + 2:]) + '\n') diff --git a/build/scripts/yndexer.py b/build/scripts/yndexer.py index a38e28ba99..82defb59f8 100644 --- a/build/scripts/yndexer.py +++ b/build/scripts/yndexer.py @@ -17,7 +17,7 @@ def _try_to_kill(process): def touch(path): if not os.path.exists(path): - with open(path, 'w'): + with open(path, 'w'): pass |