diff options
author | alexv-smirnov <alex@ydb.tech> | 2023-03-28 22:25:04 +0300 |
---|---|---|
committer | alexv-smirnov <alex@ydb.tech> | 2023-03-28 22:25:04 +0300 |
commit | b8a17f9b1c166d2e9a26b99348a4c29d972caf55 (patch) | |
tree | 1a2d881f1a9452b9c6103dbf69d73da7624e98e5 /build/scripts | |
parent | 25659221f18577ea38430a8ec3349836f5626b6a (diff) | |
download | ydb-b8a17f9b1c166d2e9a26b99348a4c29d972caf55.tar.gz |
Revert ymake build from ydb oss export
Diffstat (limited to 'build/scripts')
133 files changed, 0 insertions, 10991 deletions
diff --git a/build/scripts/_check_compiler.cpp b/build/scripts/_check_compiler.cpp deleted file mode 100644 index 53c5fdf179..0000000000 --- a/build/scripts/_check_compiler.cpp +++ /dev/null @@ -1 +0,0 @@ -#include <stdio.h> diff --git a/build/scripts/_fake_src.cpp b/build/scripts/_fake_src.cpp deleted file mode 100644 index 139597f9cb..0000000000 --- a/build/scripts/_fake_src.cpp +++ /dev/null @@ -1,2 +0,0 @@ - - diff --git a/build/scripts/append_file.py b/build/scripts/append_file.py deleted file mode 100644 index 6b5d53bc71..0000000000 --- a/build/scripts/append_file.py +++ /dev/null @@ -1,9 +0,0 @@ -import sys - - -if __name__ == "__main__": - - file_path = sys.argv[1] - with open(file_path, "a") as f: - for text in sys.argv[2:]: - print >>f, text diff --git a/build/scripts/autotar_gendirs.py b/build/scripts/autotar_gendirs.py deleted file mode 100644 index a1228108aa..0000000000 --- a/build/scripts/autotar_gendirs.py +++ /dev/null @@ -1,70 +0,0 @@ -from __future__ import print_function - -import os -import sys -import argparse -import tarfile -import subprocess - - -def is_exe(fpath): - return os.path.isfile(fpath) and os.access(fpath, os.X_OK) - - -def pack_dir(dir_path, dest_path): - dir_path = os.path.abspath(dir_path) - for tar_exe in ('/usr/bin/tar', '/bin/tar'): - if is_exe(tar_exe): - subprocess.check_call([tar_exe, '-cf', dest_path, '-C', os.path.dirname(dir_path), os.path.basename(dir_path)]) - break - else: - with tarfile.open(dest_path, 'w') as out: - out.add(dir_path, arcname=os.path.basename(dir_path)) - - -def unpack_dir(tared_dir, dest_path): - tared_dir = os.path.abspath(tared_dir) - if not os.path.exists(dest_path): - os.makedirs(dest_path) - for tar_exe in ('/usr/bin/tar', '/bin/tar'): - if is_exe(tar_exe): - subprocess.check_call([tar_exe, '-xf', tared_dir, '-C', dest_path]) - break - else: - with tarfile.open(tared_dir, 'r') as tar_file: - tar_file.extractall(dest_path) - - -# Must only be used to pack directories in build root -# Must silently accept empty list of dirs and do nothing in such case (workaround for ymake.core.conf limitations) -def main(args): - parser = argparse.ArgumentParser() - parser.add_argument('--pack', action='store_true', default=False) - parser.add_argument('--unpack', action='store_true', default=False) - parser.add_argument('--ext') - parser.add_argument('--outs', nargs='*', default=[]) - parser.add_argument('dirs', nargs='*') - args = parser.parse_args(args) - - if args.pack: - if len(args.dirs) != len(args.outs): - print("Number and oder of dirs to pack must match to the number and order of outs", file=sys.stderr) - return 1 - for dir, dest in zip(args.dirs, args.outs): - pack_dir(dir, dest) - elif args.unpack: - for tared_dir in args.dirs: - if not tared_dir.endswith(args.ext): - print("Requested to unpack '{}' which do not have required extension '{}'".format(tared_dir, args.ext), file=sys.stderr) - return 1 - dest = os.path.dirname(tared_dir) - unpack_dir(tared_dir, dest) - else: - print("Neither --pack nor --unpack specified. Don't know what to do.", file=sys.stderr) - return 1 - - return 0 - - -if __name__ == '__main__': - sys.exit(main(sys.argv[1:])) diff --git a/build/scripts/build_catboost.py b/build/scripts/build_catboost.py deleted file mode 100755 index 78334fc5f7..0000000000 --- a/build/scripts/build_catboost.py +++ /dev/null @@ -1,71 +0,0 @@ -import sys -import os -import shutil -import re -import subprocess - -def get_value(val): - dct = val.split('=', 1) - if len(dct) > 1: - return dct[1] - return '' - - -class BuildCbBase(object): - def run(self, cbmodel, cbname, cb_cpp_path): - - data_prefix = "CB_External_" - data = data_prefix + cbname - datasize = data + "Size" - - cbtype = "const NCatboostCalcer::TCatboostCalcer" - cbload = "(ReadModel({0}, {1}, EModelType::CatboostBinary))".format(data, datasize) - - cb_cpp_tmp_path = cb_cpp_path + ".tmp" - cb_cpp_tmp = open(cb_cpp_tmp_path, 'w') - - cb_cpp_tmp.write("#include <kernel/catboost/catboost_calcer.h>\n") - - ro_data_path = os.path.dirname(cb_cpp_path) + "/" + data_prefix + cbname + ".rodata" - cb_cpp_tmp.write("namespace{\n") - cb_cpp_tmp.write(" extern \"C\" {\n") - cb_cpp_tmp.write(" extern const unsigned char {1}{0}[];\n".format(cbname, data_prefix)) - cb_cpp_tmp.write(" extern const ui32 {1}{0}Size;\n".format(cbname, data_prefix)) - cb_cpp_tmp.write(" }\n") - cb_cpp_tmp.write("}\n") - archiverCall = subprocess.Popen([self.archiver, "-q", "-p", "-o", ro_data_path, cbmodel], stdout=None, stderr=subprocess.PIPE) - archiverCall.wait() - cb_cpp_tmp.write("extern {0} {1};\n".format(cbtype, cbname)) - cb_cpp_tmp.write("{0} {1}{2};".format(cbtype, cbname, cbload)) - cb_cpp_tmp.close() - shutil.move(cb_cpp_tmp_path, cb_cpp_path) - -class BuildCb(BuildCbBase): - def run(self, argv): - if len(argv) < 5: - print >>sys.stderr, "BuildCb.Run(<ARCADIA_ROOT> <archiver> <mninfo> <mnname> <cppOutput> [params...])" - sys.exit(1) - - self.SrcRoot = argv[0] - self.archiver = argv[1] - cbmodel = argv[2] - cbname = argv[3] - cb_cpp_path = argv[4] - - super(BuildCb, self).run(cbmodel, cbname, cb_cpp_path) - - -def build_cb_f(argv): - build_cb = BuildCb() - build_cb.run(argv) - - -if __name__ == '__main__': - if len(sys.argv) < 2: - print >>sys.stderr, "Usage: build_cb.py <funcName> <args...>" - sys.exit(1) - - if (sys.argv[2:]): - globals()[sys.argv[1]](sys.argv[2:]) - else: - globals()[sys.argv[1]]() diff --git a/build/scripts/build_dll_and_java.py b/build/scripts/build_dll_and_java.py deleted file mode 100644 index b9d8aff4df..0000000000 --- a/build/scripts/build_dll_and_java.py +++ /dev/null @@ -1,47 +0,0 @@ -import os -import subprocess -import sys - - -def just_do_it(argv): - delim = argv[0] - args = [] - for item in argv: - if item == delim: - args.append([]) - else: - args[-1].append(item) - dll_cmd, java_cmd, inputs, dll_out, java_out, jsrs_out, roots = args - dll_out, java_out, jsrs_out, build_root, source_root = dll_out[0], java_out[0], jsrs_out[0], roots[0], roots[1] - for inp in inputs: - origin_inp = inp - if os.path.isabs(inp): - if os.path.relpath(inp, build_root).startswith('..'): - inp = os.path.relpath(inp, source_root) - else: - inp = os.path.relpath(inp, build_root) - ext = os.path.splitext(inp)[1] - if ext in ('.o', '.obj'): - if os.path.join(build_root, inp) in java_cmd: - inp = os.path.join(build_root, inp) - if sys.platform == 'win32': - inp = inp.replace('\\', '/') - if inp not in java_cmd: - inp = build_root + '/' + inp - java_cmd.remove(inp) - if ext in ('.java', '.jsrc'): - if origin_inp in dll_cmd: - inp = origin_inp - elif os.path.join(build_root, inp) in dll_cmd: - inp = os.path.join(build_root, inp) - if sys.platform == 'win32': - inp = inp.replace('\\', '/') - dll_cmd.remove(inp) - java_cmd.insert(java_cmd.index(dll_out), java_out) - java_cmd.remove(dll_out) - subprocess.check_call(java_cmd) - subprocess.check_call(dll_cmd) - - -if __name__ == '__main__': - just_do_it(sys.argv[1:]) diff --git a/build/scripts/build_java_codenav_index.py b/build/scripts/build_java_codenav_index.py deleted file mode 100644 index d7ac4f3213..0000000000 --- a/build/scripts/build_java_codenav_index.py +++ /dev/null @@ -1,49 +0,0 @@ -import sys -import re -import os -import subprocess - -FAKE_ARCADIA_ROOT = 'fake_arcadia_root' -FAKE_BUILD_ROOT = 'fake_build_root' - - -def modify_sources_file(origin, target, source_roots_map): - def _cut_source_root(src): - for pref, fake_root in source_roots_map.items(): - if src.startswith(pref): - return os.path.join(fake_root, os.path.relpath(src, pref)) - return src - - with open(origin) as o: - srcs = [i for line in o for i in re.split('\\s+', line) if i] - new_srcs = map(_cut_source_root, srcs) - with open(target, 'w') as t: - t.write(' '.join(new_srcs)) - - -def just_do_it(argv): - corpus_name, build_root, arcadia_root, sources_file, javac_tail_cmd = argv[0], argv[1], argv[2], argv[3], argv[4:] - fake_arcadia_root = os.path.join(build_root, FAKE_ARCADIA_ROOT) - fake_build_root = os.path.join(build_root, FAKE_BUILD_ROOT) - fake_source_roots = { - arcadia_root: fake_arcadia_root, - build_root: fake_build_root, - } - modify_sources_file(sources_file, os.path.join(os.path.dirname(sources_file), '_' + os.path.basename(sources_file)), fake_source_roots) - kindex_data_root = '{}/kindex'.format(os.path.join(build_root, os.path.dirname(corpus_name))) - if not os.path.exists(kindex_data_root): - os.makedirs(kindex_data_root) - env = os.environ.copy() - env['KYTHE_ROOT_DIRECTORY'] = build_root - env['KYTHE_OUTPUT_DIRECTORY'] = kindex_data_root - env['KYTHE_CORPUS'] = os.path.relpath(corpus_name, build_root) - os.symlink(arcadia_root, fake_arcadia_root) - os.symlink(build_root, fake_build_root) - try: - subprocess.check_call(javac_tail_cmd, env=env) - finally: - os.unlink(fake_arcadia_root) - os.unlink(fake_build_root) - -if __name__ == '__main__': - just_do_it(sys.argv[1:]) diff --git a/build/scripts/build_java_with_error_prone.py b/build/scripts/build_java_with_error_prone.py deleted file mode 100644 index 910443552e..0000000000 --- a/build/scripts/build_java_with_error_prone.py +++ /dev/null @@ -1,36 +0,0 @@ -import sys -import os - -ERROR_PRONE_FLAGS = [ - '-Xep:FunctionalInterfaceMethodChanged:WARN', - '-Xep:ReturnValueIgnored:WARN', -] - -JAVA10_EXPORTS = [ - '--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED', - '--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED', - '--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED', - '--add-exports=jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED', - '--add-exports=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED', - '--add-exports=jdk.compiler/com.sun.tools.javac.processing=ALL-UNNAMED', - '--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED', - '--add-exports=jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED' -] - - -def just_do_it(argv): - java, error_prone_tool, javac_cmd = argv[0], argv[1], argv[2:] - if java.endswith('javac') or java.endswith('javac.exe'): - for f in javac_cmd: - if f.startswith('-Xep'): - ERROR_PRONE_FLAGS.append(f) - for f in ERROR_PRONE_FLAGS: - if f in javac_cmd: - javac_cmd.remove(f) - os.execv(java, [java] + JAVA10_EXPORTS + ['-processorpath', error_prone_tool, '-XDcompilePolicy=byfile'] + [(' '.join(['-Xplugin:ErrorProne'] + ERROR_PRONE_FLAGS))] + javac_cmd) - else: - os.execv(java, [java, '-Xbootclasspath/p:' + error_prone_tool, 'com.google.errorprone.ErrorProneCompiler'] + ERROR_PRONE_FLAGS + javac_cmd) - - -if __name__ == '__main__': - just_do_it(sys.argv[1:]) diff --git a/build/scripts/build_java_with_error_prone2.py b/build/scripts/build_java_with_error_prone2.py deleted file mode 100644 index ddf1ccbfc1..0000000000 --- a/build/scripts/build_java_with_error_prone2.py +++ /dev/null @@ -1,87 +0,0 @@ -import sys -import os -import re -import subprocess -import platform - - -ERROR_PRONE_FLAGS = [ - '-Xep:FunctionalInterfaceMethodChanged:WARN', - '-Xep:ReturnValueIgnored:WARN', -] - -JAVA10_EXPORTS = [ - '--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED', - '--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED', - '--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED', - '--add-exports=jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED', - '--add-exports=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED', - '--add-exports=jdk.compiler/com.sun.tools.javac.processing=ALL-UNNAMED', - '--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED', - '--add-exports=jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED', -] - - -def get_java_version(exe): - p = subprocess.Popen([exe, '-version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, err = p.communicate() - for line in (out or '').strip().split("\n") + (err or '').strip().split("\n"): - m = re.match(r'java version "(.+)"', line) - if m: - parts = m.groups()[0].split(".") - return parts[1] if parts[0] == "1" else parts[0] - m = re.match(r'openjdk version "(\d+).*"', line) - if m: - parts = m.groups()[0].split(".") - return parts[0] - return None - - -def get_classpath(cmd): - for i, part in enumerate(cmd): - if part == '-classpath': - i += 1 - if i < len(cmd): - return cmd[i] - else: - return None - return None - - -def parse_args(argv): - parsed = [] - for i in range(len(argv)): - if not argv[i].startswith('-'): - parsed.append(argv[i]) - if len(parsed) >= 3: - break - return parsed + [argv[i + 1:]] - - -def just_do_it(argv): - java, javac, error_prone_tool, javac_cmd = parse_args(argv) - ver = get_java_version(java) - if not ver: - raise Exception("Can't determine java version") - if int(ver) >= 10: - for f in javac_cmd: - if f.startswith('-Xep'): - ERROR_PRONE_FLAGS.append(f) - for f in ERROR_PRONE_FLAGS: - if f in javac_cmd: - javac_cmd.remove(f) - if '-processor' in javac_cmd: - classpath = get_classpath(javac_cmd) - if classpath: - error_prone_tool = error_prone_tool + os.pathsep + classpath - cmd = [javac] + JAVA10_EXPORTS + ['-processorpath', error_prone_tool, '-XDcompilePolicy=byfile'] + [(' '.join(['-Xplugin:ErrorProne'] + ERROR_PRONE_FLAGS))] + javac_cmd - else: - cmd = [java, '-Xbootclasspath/p:' + error_prone_tool, 'com.google.errorprone.ErrorProneCompiler'] + ERROR_PRONE_FLAGS + javac_cmd - if platform.system() == 'Windows': - sys.exit(subprocess.Popen(cmd).wait()) - else: - os.execv(cmd[0], cmd) - - -if __name__ == '__main__': - just_do_it(sys.argv[1:]) diff --git a/build/scripts/build_mn.py b/build/scripts/build_mn.py deleted file mode 100755 index 5bb03c247c..0000000000 --- a/build/scripts/build_mn.py +++ /dev/null @@ -1,330 +0,0 @@ -#!/usr/bin/env python -# Ymake MatrixNet support - -import sys -import os -import shutil -import re -import subprocess - - -def get_value(val): - dct = val.split('=', 1) - if len(dct) > 1: - return dct[1] - return '' - - -class BuildMnBase(object): - def Run(self, mninfo, mnname, mnrankingSuffix, mncppPath, check=False, ptr=False, multi=False): - self.mninfo = mninfo - self.mnname = mnname - self.mnrankingSuffix = mnrankingSuffix - self.mncppPath = mncppPath - self.check = check - self.ptr = ptr - self.multi = multi - dataprefix = "MN_External_" - mninfoName = os.path.basename(self.mninfo) - data = dataprefix + mnname - datasize = data + "Size" - - if self.multi: - if self.ptr: - mntype = "const NMatrixnet::TMnMultiCategPtr" - mnload = "(new NMatrixnet::TMnMultiCateg( {1}, {2}, \"{0}\"))".format(mninfoName, data, datasize) - else: - mntype = "const NMatrixnet::TMnMultiCateg" - mnload = "({1}, {2}, \"{0}\")".format(mninfoName, data, datasize) - else: - if self.ptr: - mntype = "const NMatrixnet::TMnSsePtr" - mnload = "(new NMatrixnet::TMnSseInfo({1}, {2}, \"{0}\"))".format(mninfoName, data, datasize) - else: - mntype = "const NMatrixnet::TMnSseInfo" - mnload = "({1}, {2}, \"{0}\")".format(mninfoName, data, datasize) - - if self.check: - self.CheckMn() - - mncpptmpPath = self.mncppPath + ".tmp" - mncpptmp = open(mncpptmpPath, 'w') - - if self.multi: - mncpptmp.write("#include <kernel/matrixnet/mn_multi_categ.h>\n") - else: - mncpptmp.write("#include <kernel/matrixnet/mn_sse.h>\n") - - rodatapath = os.path.dirname(self.mncppPath) + "/" + dataprefix + self.mnname + ".rodata" - mncpptmp.write("namespace{\n") - mncpptmp.write(" extern \"C\" {\n") - mncpptmp.write(" extern const unsigned char {1}{0}[];\n".format(self.mnname, dataprefix)) - mncpptmp.write(" extern const ui32 {1}{0}Size;\n".format(self.mnname, dataprefix)) - mncpptmp.write(" }\n") - mncpptmp.write("}\n") - archiverCall = subprocess.Popen([self.archiver, "-q", "-p", "-o", rodatapath, self.mninfo], stdout=None, stderr=subprocess.PIPE) - archiverCall.wait() - mncpptmp.write("extern {0} {1};\n".format(mntype, self.mnname)) - mncpptmp.write("{0} {1}{2};".format(mntype, self.mnname, mnload)) - mncpptmp.close() - shutil.move(mncpptmpPath, self.mncppPath) - - def CheckMn(self): - if not self.fml_unused_tool: - print >>sys.stderr, "fml_unused_tool undefined!" - failed_msg = "fml_unused_tool failed: {0} -A {1} -e -r {2}".format(self.fml_unused_tool, self.SrcRoot, self.mninfo) - assert not subprocess.call([self.fml_unused_tool, "-A", self.SrcRoot, "-e", "-r", self.mninfo]), failed_msg - - -class BuildMn(BuildMnBase): - def Run(self, argv): - if len(argv) < 6: - print >>sys.stderr, "BuildMn.Run(<ARCADIA_ROOT> <archiver> <mninfo> <mnname> <mnrankingSuffix> <cppOutput> [params...])" - sys.exit(1) - - self.SrcRoot = argv[0] - self.archiver = argv[1] - - mninfo = argv[2] - mnname = argv[3] - mnrankingSuffix = argv[4] - mncppPath = argv[5] - check = False - ptr = False - multi = False - self.fml_unused_tool = '' - for param in argv[6:]: - if param == "CHECK": - check = True - elif param == "PTR": - ptr = True - elif param == "MULTI": - multi = True - elif param.startswith('fml_tool='): - self.fml_unused_tool = get_value(param) - else: - print >>sys.stdout, "Unknown param: {0}".format(param) - super(BuildMn, self).Run(mninfo, mnname, mnrankingSuffix, mncppPath, check=check, ptr=ptr, multi=multi) - - -class BuildMns(BuildMnBase): - def InitBase(self, listname, mnrankingSuffix): - self.autogen = '// DO NOT EDIT THIS FILE DIRECTLY, AUTOGENERATED!\n' - self.mnrankingSuffix = mnrankingSuffix - self.mnlistname = listname + mnrankingSuffix - self.mnlistelem = "const NMatrixnet::TMnSsePtr*" - mnlisttype = "TMap< TString, {0} >".format(self.mnlistelem) - self.mnlist = "const {0} {1}".format(mnlisttype, self.mnlistname) - - self.mnmultilistname = "{0}{1}Multi".format(listname, self.mnrankingSuffix) - self.mnmultilistelem = "const NMatrixnet::TMnMultiCategPtr*" - mnmultilisttype = "TMap< TString, {0} >".format(self.mnmultilistelem) - self.mnmultilist = "const {0} {1}".format(mnmultilisttype, self.mnmultilistname) - - def InitForAll(self, argv): - if len(argv) < 8: - print >>sys.stderr, "BuildMns.InitForAll(<ARCADIA_ROOT> <BINDIR> <archiver> <listname> <mnranking_suffix> <hdrfile> <srcfile> <mninfos> [fml_tool=<fml_unused_tool> CHECK])" - sys.exit(1) - - bmns_args = [] - self.check = False - self.fml_unused_tool = '' - for arg in argv: - if arg == "CHECK": - self.check = True - elif arg.startswith('fml_tool='): - self.fml_unused_tool = get_value(arg) - else: - bmns_args.append(arg) - - self.SrcRoot = bmns_args[0] - self.BINDIR = bmns_args[1] - self.archiver = bmns_args[2] - self.listname = bmns_args[3] - self.mnrankingSuffix = get_value(bmns_args[4]) - self.hdrfile = bmns_args[5] - self.srcfile = bmns_args[6] - self.mninfos = bmns_args[7:] - - self.InitBase(self.listname, self.mnrankingSuffix) - - def InitForHeader(self, argv): - if len(argv) < 4: - print >>sys.stderr, "BuildMns.InitForHeader(<listname> <rankingSuffix> <hdrfile> <mninfos...>)" - sys.exit(1) - - self.listname = argv[0] - self.mnrankingSuffix = get_value(argv[1]) - self.hdrfile = argv[2] - self.mninfos = argv[3:] - - self.InitBase(self.listname, self.mnrankingSuffix) - - def InitForCpp(self, argv): - if len(argv) < 5: - print >>sys.stderr, "BuildMns.InitForCpp(<listname> <rankingSuffix> <hdrfile> <srcfile> <mninfos...>)" - sys.exit(1) - - self.listname = argv[0] - self.mnrankingSuffix = get_value(argv[1]) - self.hdrfile = argv[2] - self.srcfile = argv[3] - self.mninfos = argv[4:] - - self.InitBase(self.listname, self.mnrankingSuffix) - - def InitForFiles(self, argv): - if len(argv) < 7: - print >>sys.stderr, "BuildMns.InitForFiles(<ARCADIA_ROOT> <BINDIR> <archiver> <fml_unused_tool> <listname> <rankingSuffix> <mninfos...> [CHECK])" - sys.exit(1) - - bmns_args = [] - self.check = False - self.fml_unused_tool = '' - for arg in argv: - if arg == "CHECK": - self.check = True - elif arg.startswith('fml_tool='): - self.fml_unused_tool = get_value(arg) - else: - bmns_args.append(arg) - - self.SrcRoot = bmns_args[0] - self.BINDIR = bmns_args[1] - self.archiver = bmns_args[2] - self.listname = bmns_args[3] - self.mnrankingSuffix = get_value(bmns_args[4]) - self.mninfos = bmns_args[5:] - - def BuildMnsHeader(self): - if self.mninfos: - self.mninfos = sorted(set(self.mninfos)) - - tmpHdrPath = self.hdrfile + ".tmp" - tmpHdrFile = open(tmpHdrPath, 'w') - - tmpHdrFile.write(self.autogen) - tmpHdrFile.write("#include <kernel/matrixnet/mn_sse.h>\n") - tmpHdrFile.write("#include <kernel/matrixnet/mn_multi_categ.h>\n\n") - tmpHdrFile.write("extern {0};\n".format(self.mnlist)) - tmpHdrFile.write("extern {0};\n".format(self.mnmultilist)) - - for item in self.mninfos: - mnfilename = os.path.basename(item) - mnfilename, ext = os.path.splitext(mnfilename) - - mnname = re.sub("[^-a-zA-Z0-9_]", "_", mnfilename) - - if ext == ".info": - mnname = "staticMn{0}{1}Ptr".format(self.mnrankingSuffix, mnname) - tmpHdrFile.write("extern const NMatrixnet::TMnSsePtr {0};\n".format(mnname)) - elif ext == ".mnmc": - mnname = "staticMnMulti{0}{1}Ptr".format(self.mnrankingSuffix, mnname) - tmpHdrFile.write("extern const NMatrixnet::TMnMultiCategPtr {0};\n".format(mnname)) - - tmpHdrFile.close() - shutil.move(tmpHdrPath, self.hdrfile) - - def BuildMnFiles(self): - for item in self.mninfos: - mnfilename = os.path.basename(item) - mnfilename, ext = os.path.splitext(mnfilename) - - mnname = re.sub("[^-a-zA-Z0-9_]", "_", mnfilename) - - if ext == ".info": - mnname = "staticMn{0}{1}Ptr".format(self.mnrankingSuffix, mnname) - super(BuildMns, self).Run(item, mnname, self.mnrankingSuffix, self.BINDIR + "/mn.{0}.cpp".format(mnname), check=self.check, ptr=True, multi=False) - elif ext == ".mnmc": - mnname = "staticMnMulti{0}{1}Ptr".format(self.mnrankingSuffix, mnname) - # BUILD_MN_PTR_MULTI - super(BuildMns, self).Run(item, mnname, self.mnrankingSuffix, self.BINDIR + "/mnmulti.{0}.cpp".format(mnname), check=False, ptr=True, multi=True) - - def BuildMnsCpp(self): - if self.mninfos: - self.mninfos = sorted(set(self.mninfos)) - - tmpSrcPath = self.srcfile + ".tmp" - tmpSrcFile = open(tmpSrcPath, 'w') - hdrrel = os.path.basename(self.hdrfile) - - mnnames = [] - mnmultinames = [] - for item in self.mninfos: - mnfilename = os.path.basename(item) - mnfilename, ext = os.path.splitext(mnfilename) - - if ext == ".info": - mnnames.append(mnfilename) - elif ext == ".mnmc": - mnmultinames.append(mnfilename) - - tmpSrcFile.write(self.autogen) - tmpSrcFile.write("#include \"{0}\"\n\n".format(hdrrel)) - - if mnnames: - mndata = self.mnlistname + "_data" - tmpSrcFile.write("static const std::pair< TString, {0} > {1}[] = {{\n".format(self.mnlistelem, mndata)) - for item in mnnames: - mnname = re.sub("[^-a-zA-Z0-9_]", "_", item) - tmpSrcFile.write(" std::make_pair(TString(\"{0}\"), &staticMn{1}{2}Ptr),\n".format(item, self.mnrankingSuffix, mnname)) - tmpSrcFile.write("};\n") - tmpSrcFile.write("{0}({1},{1} + sizeof({1}) / sizeof({1}[0]));\n\n".format(self.mnlist, mndata)) - else: - tmpSrcFile.write("{0};\n\n".format(self.mnlist)) - - if mnmultinames: - mnmultidata = self.mnmultilistname + "_data" - tmpSrcFile.write("static const std::pair< TString, {0} > {1}[] = {{\n".format(self.mnmultilistelem, mnmultidata)) - for item in mnmultinames: - mnname = re.sub("[^-a-zA-Z0-9_]", "_", item) - tmpSrcFile.write(" std::make_pair(TString(\"{0}\"), &staticMnMulti{1}{2}Ptr),\n".format(item, self.mnrankingSuffix, mnname)) - tmpSrcFile.write("};\n") - tmpSrcFile.write("{0}({1},{1} + sizeof({1}) / sizeof({1}[0]));\n".format(self.mnmultilist, mnmultidata)) - else: - tmpSrcFile.write("{0};\n".format(self.mnmultilist)) - - tmpSrcFile.close() - shutil.move(tmpSrcPath, self.srcfile) - - -def BuildMnsAllF(argv): - bldMns = BuildMns() - bldMns.InitForAll(argv) - bldMns.BuildMnsCpp() - bldMns.BuildMnsHeader() - bldMns.BuildMnFiles() - - -def BuildMnsCppF(argv): - bldMns = BuildMns() - bldMns.InitForCpp(argv) - bldMns.BuildMnsCpp() - - -def BuildMnsHeaderF(argv): - bldMns = BuildMns() - bldMns.InitForHeader(argv) - bldMns.BuildMnsHeader() - - -def BuildMnsFilesF(argv): - bldMns = BuildMns() - bldMns.InitForFiles(argv) - bldMns.BuildMnFiles() - - -def BuildMnF(argv): - bldMn = BuildMn() - bldMn.Run(argv) - - -if __name__ == '__main__': - if len(sys.argv) < 2: - print >>sys.stderr, "Usage: build_mn.py <funcName> <args...>" - sys.exit(1) - - if (sys.argv[2:]): - globals()[sys.argv[1]](sys.argv[2:]) - else: - globals()[sys.argv[1]]() diff --git a/build/scripts/build_pln_header.py b/build/scripts/build_pln_header.py deleted file mode 100755 index c73693f444..0000000000 --- a/build/scripts/build_pln_header.py +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env python - -import sys -import os - - -def BuildPlnHeader(): - if len(sys.argv) < 2: - print >>sys.stderr, "Usage: build_pln_header.py <absolute/path/to/OutFile>" - sys.exit(1) - - print >>sys.stdout, "Build Pln Header..." - outPath = sys.argv[1] - tmpPath = outPath + '.tmp' - tmpFile = open(tmpPath, 'w') - - tmpFile.write('#include <library/cpp/sse/sse.h>\n') - tmpFile.write('#include <kernel/relevfml/relev_fml.h>\n') - for path in sys.argv[2:]: - name = os.path.basename(path).split(".")[0] # name without extensions - tmpFile.write('\nextern SRelevanceFormula fml{0};\n'.format(name)) - tmpFile.write('float {0}(const float* f);\n'.format(name)) - tmpFile.write('void {0}SSE(const float* const* factors, float* result);\n'.format(name)) - tmpFile.close() - try: - os.remove(outPath) - except: - pass - try: - os.rename(tmpPath, outPath) - except: - print >>sys.stdout, 'Error: Failed to rename ' + tmpPath + ' to ' + outPath - -if __name__ == '__main__': - BuildPlnHeader() diff --git a/build/scripts/c_templates/ya.make b/build/scripts/c_templates/ya.make deleted file mode 100644 index b395e11021..0000000000 --- a/build/scripts/c_templates/ya.make +++ /dev/null @@ -1,7 +0,0 @@ -OWNER(g:ymake) - -LIBRARY(dummy-vcs) -NO_PLATFORM() - -SRCS(svn_interface.c) -END() diff --git a/build/scripts/cat.py b/build/scripts/cat.py deleted file mode 100755 index 0c3f73d96f..0000000000 --- a/build/scripts/cat.py +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env python -import sys -from shutil import copyfileobj as copy -import os.path - -if __name__ == '__main__': - for filename in sys.argv[1:] or ["-"]: - if filename == "-": - copy(sys.stdin, sys.stdout) - else: - if os.path.exists(filename): - with open(filename, 'rb') as file: - copy(file, sys.stdout) - else: - sys.stderr.write('cat.py: {0}: No such file or directory\n'.format(filename)) diff --git a/build/scripts/cgo1_wrapper.py b/build/scripts/cgo1_wrapper.py deleted file mode 100644 index 986082f7e9..0000000000 --- a/build/scripts/cgo1_wrapper.py +++ /dev/null @@ -1,45 +0,0 @@ -import argparse -import shutil -import subprocess -import sys - - -CGO1_SUFFIX='.cgo1.go' - - -def call(cmd, cwd, env=None): - # sys.stderr.write('{}\n'.format(' '.join(cmd))) - return subprocess.call(cmd, stdin=None, stderr=sys.stderr, stdout=sys.stdout, cwd=cwd, env=env) - - -def process_file(source_root, source_prefix, build_root, build_prefix, src_path, comment_prefix): - dst_path = '{}.tmp'.format(src_path) - with open(src_path, 'r') as src_file, open(dst_path, 'w') as dst_file: - for line in src_file: - if line.startswith(comment_prefix): - dst_file.write(line.replace(source_root, source_prefix).replace(build_root, build_prefix)) - else: - dst_file.write(line) - shutil.move(dst_path, src_path) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument('--build-prefix', default='__ARCADIA_BUILD_ROOT_PREFIX__') - parser.add_argument('--build-root', required=True) - parser.add_argument('--cgo1-files', nargs='+', required=True) - parser.add_argument('--cgo2-files', nargs='+', required=True) - parser.add_argument('--source-prefix', default='__ARCADIA_SOURCE_ROOT_PREFIX__') - parser.add_argument('--source-root', required=True) - parser.add_argument('cgo1_cmd', nargs='*') - args = parser.parse_args() - - exit_code = call(args.cgo1_cmd, args.source_root) - if exit_code != 0: - sys.exit(exit_code) - - for src_path in args.cgo1_files: - process_file(args.source_root, args.source_prefix, args.build_root, args.build_prefix, src_path, '//') - - for src_path in args.cgo2_files: - process_file(args.source_root, args.source_prefix, args.build_root, args.build_prefix, src_path, '#line') diff --git a/build/scripts/check_config_h.py b/build/scripts/check_config_h.py deleted file mode 100644 index 07bc12e230..0000000000 --- a/build/scripts/check_config_h.py +++ /dev/null @@ -1,89 +0,0 @@ -import sys - -data = """ -#if defined(SIZEOF_LONG) -static_assert(sizeof(long) == SIZEOF_LONG, "fixme 1"); -#endif - -#if defined(SIZEOF_PTHREAD_T) -#include <pthread.h> - -static_assert(sizeof(pthread_t) == SIZEOF_PTHREAD_T, "fixme 2"); -#endif - -#if defined(SIZEOF_SIZE_T) -#include <stddef.h> - -static_assert(sizeof(size_t) == SIZEOF_SIZE_T, "fixme 3"); -#endif - -#if defined(SIZEOF_TIME_T) -#include <time.h> - -static_assert(sizeof(time_t) == SIZEOF_TIME_T, "fixme 4"); -#endif - -#if defined(SIZEOF_UINTPTR_T) -#include <stdint.h> - -static_assert(sizeof(uintptr_t) == SIZEOF_UINTPTR_T, "fixme 5"); -#endif - -#if defined(SIZEOF_VOID_P) -static_assert(sizeof(void*) == SIZEOF_VOID_P, "fixme 6"); -#endif - -#if defined(SIZEOF_FPOS_T) -#include <stdio.h> - -static_assert(sizeof(fpos_t) == SIZEOF_FPOS_T, "fixme 7"); -#endif - -#if defined(SIZEOF_DOUBLE) -static_assert(sizeof(double) == SIZEOF_DOUBLE, "fixme 8"); -#endif - -#if defined(SIZEOF_LONG_DOUBLE) -static_assert(sizeof(long double) == SIZEOF_LONG_DOUBLE, "fixme 9"); -#endif - -#if defined(SIZEOF_FLOAT) -static_assert(sizeof(float) == SIZEOF_FLOAT, "fixme 10"); -#endif - -#if defined(SIZEOF_INT) -static_assert(sizeof(int) == SIZEOF_INT, "fixme 11"); -#endif - -#if defined(SIZEOF_LONG_LONG) -static_assert(sizeof(long long) == SIZEOF_LONG_LONG, "fixme 12"); -#endif - -#if defined(SIZEOF_OFF_T) -#include <stdio.h> - -static_assert(sizeof(off_t) == SIZEOF_OFF_T, "fixme 13"); -#endif - -#if defined(SIZEOF_PID_T) -#include <unistd.h> - -static_assert(sizeof(pid_t) == SIZEOF_PID_T, "fixme 14"); -#endif - -#if defined(SIZEOF_SHORT) -static_assert(sizeof(short) == SIZEOF_SHORT, "fixme 15"); -#endif - -#if defined(SIZEOF_WCHAR_T) -static_assert(sizeof(wchar_t) == SIZEOF_WCHAR_T, "fixme 16"); -#endif - -#if defined(SIZEOF__BOOL) -//TODO -#endif -""" -if __name__ == '__main__': - with open(sys.argv[2], 'w') as f: - f.write('#include <' + sys.argv[1] + '>\n\n') - f.write(data) diff --git a/build/scripts/clang_tidy.py b/build/scripts/clang_tidy.py deleted file mode 100644 index 982e9015cb..0000000000 --- a/build/scripts/clang_tidy.py +++ /dev/null @@ -1,172 +0,0 @@ -import argparse -import json -import os -import re -import shutil -import sys - -import subprocess - -import yaml - - -def setup_script(args): - global tidy_config_validation - sys.path.append(os.path.dirname(args.config_validation_script)) - import tidy_config_validation - - -def parse_args(): - parser = argparse.ArgumentParser() - parser.add_argument("--testing-src", required=True) - parser.add_argument("--clang-tidy-bin", required=True) - parser.add_argument("--config-validation-script", required=True) - parser.add_argument("--ymake-python", required=True) - parser.add_argument("--tidy-json", required=True) - parser.add_argument("--source-root", required=True) - parser.add_argument("--build-root", required=True) - parser.add_argument("--default-config-file", required=True) - parser.add_argument("--project-config-file", required=True) - parser.add_argument("--export-fixes", required=True) - parser.add_argument("--checks", required=False, default="") - parser.add_argument("--header-filter", required=False, default=None) - return parser.parse_known_args() - - -def generate_compilation_database(clang_cmd, source_root, filename, path): - compile_database = [ - { - "file": filename, - "command": subprocess.list2cmdline(clang_cmd), - "directory": source_root, - } - ] - compilation_database_json = os.path.join(path, "compile_commands.json") - with open(compilation_database_json, "w") as afile: - json.dump(compile_database, afile) - return compilation_database_json - - -def load_profile(path): - if os.path.exists(path): - files = os.listdir(path) - if len(files) == 1: - with open(os.path.join(path, files[0])) as afile: - return json.load(afile)["profile"] - elif len(files) > 1: - return { - "error": "found several profile files: {}".format(files), - } - return { - "error": "profile file is missing", - } - - -def load_fixes(path): - if os.path.exists(path): - with open(path, 'r') as afile: - return afile.read() - else: - return "" - - -def is_generated(testing_src, build_root): - return testing_src.startswith(build_root) - - -def generate_outputs(output_json): - output_obj = os.path.splitext(output_json)[0] + ".o" - open(output_obj, "w").close() - open(output_json, "w").close() - - -def filter_configs(result_config, filtered_config): - with open(result_config, 'r') as afile: - input_config = yaml.safe_load(afile) - result_config = tidy_config_validation.filter_config(input_config) - with open(filtered_config, 'w') as afile: - yaml.safe_dump(result_config, afile) - - -def main(): - args, clang_cmd = parse_args() - setup_script(args) - clang_tidy_bin = args.clang_tidy_bin - output_json = args.tidy_json - generate_outputs(output_json) - if is_generated(args.testing_src, args.build_root): - return - if args.header_filter is None: - # .pb.h files will be excluded because they are not in source_root - header_filter = r"^" + re.escape(os.path.dirname(args.testing_src)) + r".*" - else: - header_filter = r"^(" + args.header_filter + r").*" - - def ensure_clean_dir(path): - path = os.path.join(args.build_root, path) - if os.path.exists(path): - shutil.rmtree(path) - os.makedirs(path) - return path - - profile_tmpdir = ensure_clean_dir("profile_tmpdir") - db_tmpdir = ensure_clean_dir("db_tmpdir") - fixes_file = "fixes.txt" - config_dir = ensure_clean_dir("config_dir") - result_config_file = args.default_config_file - if args.project_config_file != args.default_config_file: - result_config = os.path.join(config_dir, "result_tidy_config.yaml") - filtered_config = os.path.join(config_dir, "filtered_tidy_config.yaml") - filter_configs(args.project_config_file, filtered_config) - result_config_file = tidy_config_validation.merge_tidy_configs( - base_config_path=args.default_config_file, - additional_config_path=filtered_config, - result_config_path=result_config, - ) - compile_command_path = generate_compilation_database(clang_cmd, args.source_root, args.testing_src, db_tmpdir) - - cmd = [ - clang_tidy_bin, - args.testing_src, - "-p", - compile_command_path, - "--warnings-as-errors", - "*", - "--config-file", - result_config_file, - "--header-filter", - header_filter, - "--use-color", - "--enable-check-profile", - "--store-check-profile={}".format(profile_tmpdir), - ] - if args.export_fixes == "yes": - cmd += ["--export-fixes", fixes_file] - - if args.checks: - cmd += ["--checks", args.checks] - - print("cmd: {}".format(' '.join(cmd))) - res = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, err = res.communicate() - out = out.replace(args.source_root, "$(SOURCE_ROOT)") - profile = load_profile(profile_tmpdir) - testing_src = os.path.relpath(args.testing_src, args.source_root) - tidy_fixes = load_fixes(fixes_file) - - with open(output_json, "wb") as afile: - json.dump( - { - "file": testing_src, - "exit_code": res.returncode, - "profile": profile, - "stderr": err, - "stdout": out, - "fixes": tidy_fixes, - }, - afile, - ) - - -if __name__ == "__main__": - main() diff --git a/build/scripts/clang_tidy_arch.py b/build/scripts/clang_tidy_arch.py deleted file mode 100644 index 7caf623a3d..0000000000 --- a/build/scripts/clang_tidy_arch.py +++ /dev/null @@ -1,33 +0,0 @@ -import os -import argparse -import json - - -def parse_args(): - parser = argparse.ArgumentParser() - parser.add_argument("--output-file") - parser.add_argument("--build-root") - parser.add_argument("--source-root") - return parser.parse_known_args() - - -def main(): - args, unknown_args = parse_args() - inputs = unknown_args - result_json = {} - for inp in inputs: - if os.path.exists(inp) and inp.endswith("tidyjson"): - with open(inp, 'r') as afile: - file_content = afile.read().strip() - if not file_content: - continue - errors = json.loads(file_content) - testing_src = errors["file"] - result_json[testing_src] = errors - - with open(args.output_file, 'w') as afile: - json.dump(result_json, afile, indent=4) # TODO remove indent - - -if __name__ == "__main__": - main() diff --git a/build/scripts/clang_wrapper.py b/build/scripts/clang_wrapper.py deleted file mode 100644 index af3869f789..0000000000 --- a/build/scripts/clang_wrapper.py +++ /dev/null @@ -1,53 +0,0 @@ -import subprocess -import sys - - -def fix(s): - # disable dbg DEVTOOLS-2744 - if s == '-g': - return None - if s == '/Z7' or s == '/Od' or s == '/Ob0' or s == '/D_DEBUG': - return None - - # disable sanitizers for generated code - if s.startswith('-fsanitize') or s == '-Dmemory_sanitizer_enabled' or s.startswith('-fsanitize-blacklist'): - return None - - # strip gcc toolchain flags (appear when crosscompiling) - if s.startswith('-fabi-version'): - return None - - # remove arguments unknown to clang-cl - if s == '-fcase-insensitive-paths': # or s == '-fno-lto': # DEVTOOLSSUPPORT-3966 - return None - - # Paths under .ya/tools/v3/.../msvc/include are divided with '\' - return s.replace('\\', '/') - - -def fix_path(p): - try: - i = p.rfind('/bin/clang') - p = p[:i] + '/bin/clang-cl' - except ValueError: - pass - return p - - -if __name__ == '__main__': - is_on_win = sys.argv[1] == 'yes' - path = sys.argv[2] - args = filter(None, [fix(s) for s in sys.argv[3:]]) - if is_on_win: - path = fix_path(path) - try: - i = args.index('-emit-llvm') - args[i:i+1] = ['-Xclang', '-emit-llvm'] - except ValueError: - pass - args.append('-fms-compatibility-version=19') - - cmd = [path] + args - - rc = subprocess.call(cmd, shell=False, stderr=sys.stderr, stdout=sys.stdout) - sys.exit(rc) diff --git a/build/scripts/collect_java_srcs.py b/build/scripts/collect_java_srcs.py deleted file mode 100644 index 170002520a..0000000000 --- a/build/scripts/collect_java_srcs.py +++ /dev/null @@ -1,51 +0,0 @@ -import os -import sys -import contextlib -import tarfile -import zipfile - - -if __name__ == '__main__': - build_root = sys.argv[1] - root = os.path.normpath(sys.argv[2]) - dest = os.path.normpath(sys.argv[3]) - srcs = sys.argv[4:] - - for src in srcs: - src = os.path.normpath(src) - if src.endswith('.java') or src.endswith('.kt'): - src_rel_path = os.path.relpath(src, root) - - if os.path.join(root, src_rel_path) == src: - # Inside root - dst = os.path.join(dest, src_rel_path) - - else: - # Outside root - print>>sys.stderr, 'External src file "{}" is outside of srcdir {}, ignore'.format( - os.path.relpath(src, build_root), - os.path.relpath(root, build_root), - ) - continue - - if os.path.exists(dst): - print>>sys.stderr, 'Duplicate external src file {}, choice is undefined'.format( - os.path.relpath(dst, root) - ) - - else: - destdir = os.path.dirname(dst) - if destdir and not os.path.exists(destdir): - os.makedirs(destdir) - os.rename(src, dst) - - elif src.endswith('.jsr'): - with contextlib.closing(tarfile.open(src, 'r')) as tf: - tf.extractall(dst) - - elif src.endswith('-sources.jar'): - with zipfile.ZipFile(src) as zf: - zf.extractall(dst) - - else: - print>>sys.stderr, 'Unrecognized file type', os.path.relpath(src, build_root) diff --git a/build/scripts/compile_cuda.py b/build/scripts/compile_cuda.py deleted file mode 100644 index f8e1fa2b6d..0000000000 --- a/build/scripts/compile_cuda.py +++ /dev/null @@ -1,168 +0,0 @@ -import sys -import subprocess -import os -import collections -import re -import tempfile - - -def is_clang(command): - for word in command: - if '--compiler-bindir' in word and 'clang' in word: - return True - - return False - - -def main(): - try: - sys.argv.remove('--y_skip_nocxxinc') - skip_nocxxinc = True - except ValueError: - skip_nocxxinc = False - - spl = sys.argv.index('--cflags') - cmd = 1 - mtime0 = None - if sys.argv[1] == '--mtime': - mtime0 = sys.argv[2] - cmd = 3 - command = sys.argv[cmd: spl] - cflags = sys.argv[spl + 1:] - - dump_args = False - if '--y_dump_args' in command: - command.remove('--y_dump_args') - dump_args = True - - executable = command[0] - if not os.path.exists(executable): - print >> sys.stderr, '{} not found'.format(executable) - sys.exit(1) - - if is_clang(command): - # nvcc concatenates the sources for clang, and clang reports unused - # things from .h files as if they they were defined in a .cpp file. - cflags += ['-Wno-unused-function', '-Wno-unused-parameter'] - - if not is_clang(command) and '-fopenmp=libomp' in cflags: - cflags.append('-fopenmp') - cflags.remove('-fopenmp=libomp') - - skip_list = [ - '-gline-tables-only', - # clang coverage - '-fprofile-instr-generate', - '-fcoverage-mapping', - '/Zc:inline', # disable unreferenced functions (kernel registrators) remove - '-Wno-c++17-extensions', - '-flto', - '-faligned-allocation', - '-fsized-deallocation', - # While it might be reasonable to compile host part of .cu sources with these optimizations enabled, - # nvcc passes these options down towards cicc which lacks x86_64 extensions support. - '-msse2', - '-msse3', - '-mssse3', - '-msse4.1', - '-msse4.2', - ] - - if skip_nocxxinc: - skip_list.append('-nostdinc++') - - for flag in skip_list: - if flag in cflags: - cflags.remove(flag) - - skip_prefix_list = [ - '-fsanitize=', - '-fsanitize-coverage=', - '-fsanitize-blacklist=', - '--system-header-prefix', - ] - new_cflags = [] - for flag in cflags: - if all(not flag.startswith(skip_prefix) for skip_prefix in skip_prefix_list): - if flag.startswith('-fopenmp-version='): - new_cflags.append('-fopenmp-version=45') # Clang 11 only supports OpenMP 4.5, but the default is 5.0, so we need to forcefully redefine it. - else: - new_cflags.append(flag) - cflags = new_cflags - - if not is_clang(command): - def good(arg): - if arg.startswith('--target='): - return False - return True - cflags = filter(good, cflags) - - cpp_args = [] - compiler_args = [] - - # NVCC requires particular MSVC versions which may differ from the version - # used to compile regular C++ code. We have a separate MSVC in Arcadia for - # the CUDA builds and pass it's root in $Y_VC_Root. - # The separate MSVC for CUDA may absent in Yandex Open Source builds. - vc_root = os.environ.get('Y_VC_Root') - - cflags_queue = collections.deque(cflags) - while cflags_queue: - - arg = cflags_queue.popleft() - if arg == '-mllvm': - compiler_args.append(arg) - compiler_args.append(cflags_queue.popleft()) - continue - if arg[:2].upper() in ('-I', '/I', '-B'): - value = arg[2:] - if not value: - value = cflags_queue.popleft() - if arg[1] == 'I': - cpp_args.append('-I{}'.format(value)) - elif arg[1] == 'B': # todo: delete "B" flag check when cuda stop to use gcc - pass - continue - - match = re.match(r'[-/]D(.*)', arg) - if match: - define = match.group(1) - # We have C++ flags configured for the regular C++ build. - # There is Y_MSVC_INCLUDE define with a path to the VC header files. - # We need to change the path accordingly when using a separate MSVC for CUDA. - if vc_root and define.startswith('Y_MSVC_INCLUDE'): - define = os.path.expandvars('Y_MSVC_INCLUDE={}/include'.format(vc_root)) - cpp_args.append('-D' + define.replace('\\', '/')) - continue - - compiler_args.append(arg) - - command += cpp_args - if compiler_args: - command += ['--compiler-options', ','.join(compiler_args)] - - # --keep is necessary to prevent nvcc from embedding nvcc pid in generated - # symbols. It makes nvcc use the original file name as the prefix in the - # generated files (otherwise it also prepends tmpxft_{pid}_00000000-5), and - # cicc derives the module name from its {input}.cpp1.ii file name. - command += ['--keep', '--keep-dir', tempfile.mkdtemp(prefix='compile_cuda.py.')] - # nvcc generates symbols like __fatbinwrap_{len}_{basename}_{hash} where - # {basename} is {input}.cpp1.ii with non-C chars translated to _, {len} is - # {basename} length, and {hash} is the hash of first exported symbol in - # {input}.cpp1.ii if there is one, otherwise it is based on its modification - # time (converted to string in the local timezone) and the current working - # directory. To stabilize the names of these symbols we need to fix mtime, - # timezone, and cwd. - if mtime0: - os.environ['LD_PRELOAD'] = mtime0 - os.environ['TZ'] = 'UTC0' # POSIX fixed offset format. - os.environ['TZDIR'] = '/var/empty' # Against counterfeit /usr/share/zoneinfo/$TZ. - - if dump_args: - sys.stdout.write('\n'.join(command)) - else: - sys.exit(subprocess.Popen(command, stdout=sys.stderr, stderr=sys.stderr, cwd='/').wait()) - - -if __name__ == '__main__': - main() diff --git a/build/scripts/compile_java.py b/build/scripts/compile_java.py deleted file mode 100644 index 7b6455ebf0..0000000000 --- a/build/scripts/compile_java.py +++ /dev/null @@ -1,122 +0,0 @@ -import argparse -import contextlib -from distutils import dir_util -import os -import shutil -import subprocess as sp -import tarfile -import zipfile -import sys - - -def parse_args(args): - parser = argparse.ArgumentParser(description='Wrapper to invoke Java compilation from ya make build') - parser.add_argument('--javac-bin', help='path to javac') - parser.add_argument('--jar-bin', help='path to jar tool') - parser.add_argument('--java-bin', help='path to java binary') - parser.add_argument('--kotlin-compiler', help='path to kotlin compiler jar file') - parser.add_argument('--vcs-mf', help='path to VCS info manifest snippet') - parser.add_argument('--package-prefix', help='package prefix for resource files') - parser.add_argument('--jar-output', help='jar file with compiled classes destination path') - parser.add_argument('--srcs-jar-output', help='jar file with sources destination path') - parser.add_argument('srcs', nargs="*") - args = parser.parse_args(args) - return args, args.srcs - - -def mkdir_p(directory): - if not os.path.exists(directory): - os.makedirs(directory) - - -def split_cmd_by_delim(cmd, delim='DELIM'): - result = [[]] - for arg in cmd: - if arg == delim: - result.append([]) - else: - result[-1].append(arg) - return result - - -def main(): - cmd_parts = split_cmd_by_delim(sys.argv[1:]) - assert len(cmd_parts) == 4 - args, javac_opts, peers, ktc_opts = cmd_parts - opts, jsrcs = parse_args(args) - - jsrcs += list(filter(lambda x: x.endswith('.jsrc'), peers)) - peers = list(filter(lambda x: not x.endswith('.jsrc'), peers)) - - sources_dir = 'src' - mkdir_p(sources_dir) - for s in jsrcs: - if s.endswith('.jsrc'): - with contextlib.closing(tarfile.open(s, 'r')) as tf: - tf.extractall(sources_dir) - - srcs = [] - for r, _, files in os.walk(sources_dir): - for f in files: - srcs.append(os.path.join(r, f)) - srcs += jsrcs - ktsrcs = list(filter(lambda x: x.endswith('.kt'), srcs)) - srcs = list(filter(lambda x: x.endswith('.java'), srcs)) - - classes_dir = 'cls' - mkdir_p(classes_dir) - classpath = os.pathsep.join(peers) - - if srcs: - temp_sources_file = 'temp.sources.list' - with open(temp_sources_file, 'w') as ts: - ts.write(' '.join(srcs)) - - if ktsrcs: - temp_kt_sources_file = 'temp.kt.sources.list' - with open(temp_kt_sources_file, 'w') as ts: - ts.write(' '.join(ktsrcs + srcs)) - kt_classes_dir = 'kt_cls' - mkdir_p(kt_classes_dir) - sp.check_call([opts.java_bin, '-jar', opts.kotlin_compiler, '-classpath', classpath, '-d', kt_classes_dir] + ktc_opts + ['@' + temp_kt_sources_file]) - classpath = os.pathsep.join([kt_classes_dir, classpath]) - - if srcs: - sp.check_call([opts.javac_bin, '-nowarn', '-g', '-classpath', classpath, '-encoding', 'UTF-8', '-d', classes_dir] + javac_opts + ['@' + temp_sources_file]) - - for s in jsrcs: - if s.endswith('-sources.jar'): - with zipfile.ZipFile(s) as zf: - zf.extractall(sources_dir) - - elif s.endswith('.jar'): - with zipfile.ZipFile(s) as zf: - zf.extractall(classes_dir) - - if ktsrcs: - dir_util.copy_tree(kt_classes_dir, classes_dir) - - if opts.vcs_mf: - sp.check_call([opts.jar_bin, 'cfm', opts.jar_output, opts.vcs_mf, os.curdir], cwd=classes_dir) - else: - sp.check_call([opts.jar_bin, 'cfM', opts.jar_output, os.curdir], cwd=classes_dir) - - if opts.srcs_jar_output: - for s in jsrcs: - if s.endswith('.java'): - if opts.package_prefix: - d = os.path.join(sources_dir, *(opts.package_prefix.split('.') + [os.path.basename(s)])) - - else: - d = os.path.join(sources_dir, os.path.basename(s)) - - shutil.copyfile(s, d) - - if opts.vcs_mf: - sp.check_call([opts.jar_bin, 'cfm', opts.srcs_jar_output, opts.vcs_mf, os.curdir], cwd=sources_dir) - else: - sp.check_call([opts.jar_bin, 'cfM', opts.srcs_jar_output, os.curdir], cwd=sources_dir) - - -if __name__ == '__main__': - main() diff --git a/build/scripts/compile_jsrc.py b/build/scripts/compile_jsrc.py deleted file mode 100644 index 8760e5eee9..0000000000 --- a/build/scripts/compile_jsrc.py +++ /dev/null @@ -1,24 +0,0 @@ -import argparse -import os -import tarfile - - -def parse_args(): - parser = argparse.ArgumentParser() - parser.add_argument('--input', nargs='*', required=True) - parser.add_argument('--output', required=True) - parser.add_argument('--prefix', required=True) - - return parser.parse_args() - - -def main(): - args = parse_args() - - with tarfile.open(args.output, 'w') as out: - for f in args.input: - out.add(f, arcname=os.path.relpath(f, args.prefix)) - - -if __name__ == '__main__': - main() diff --git a/build/scripts/compile_pysrc.py b/build/scripts/compile_pysrc.py deleted file mode 100644 index e3637e18e2..0000000000 --- a/build/scripts/compile_pysrc.py +++ /dev/null @@ -1,101 +0,0 @@ -import argparse -import os -import shutil -import subprocess -import tarfile - - -LIMIT = 6000 - - -def parse_args(): - parser = argparse.ArgumentParser() - parser.add_argument('--input', required=True) - parser.add_argument('--output', required=True) - parser.add_argument('--rescompiler', required=True) - subparsers = parser.add_subparsers(dest='mode') - - parser_py2 = subparsers.add_parser('py2') - parser_py2.add_argument('--py_compile', required=True) - parser_py2.add_argument('--python', required=True) - - parser_py3 = subparsers.add_parser('py3') - parser_py3.add_argument('--pycc', required=True) - - return parser.parse_args() - - -def call(cmd, cwd=None, env=None): - return subprocess.check_output(cmd, stdin=None, stderr=subprocess.STDOUT, cwd=cwd, env=env) - - -def iterate_py2_resource_params(py_files): - for py in py_files: - mod = py[:-3].replace('/', '.') - key = '/py_modules/{}'.format(mod) - yield py, key - yield '-', 'resfs/src/{}={}'.format(key, py) - yield '{}.yapyc'.format(py), '/py_code/{}'.format(mod) - - -def iterate_py3_resource_params(py_files): - for py in py_files: - for ext in ('', '.yapyc3'): - path = '{}{}'.format(py, ext) - dest = 'py/{}'.format(path) - key = 'resfs/file/{}'.format(dest) - src = 'resfs/src/{}={}'.format(key, os.path.basename(path)) - yield '-', src - yield path, key - - -def main(): - args = parse_args() - - names = [] - with tarfile.open(args.input, 'r') as tar: - names = tar.getnames() - tar.extractall() - - if args.mode == 'py3': - pycc_cmd = [args.pycc] - pycc_ext = '.yapyc3' - iterate_resource_params = iterate_py3_resource_params - else: - pycc_cmd = [args.python, args.py_compile] - pycc_ext = '.yapyc' - iterate_resource_params = iterate_py2_resource_params - - py_files = sorted(names) - - for py in py_files: - cmd = pycc_cmd + ['{}-'.format(os.path.basename(py)), py, '{}{}'.format(py, pycc_ext)] - call(cmd) - - outputs = [] - cmd = [args.rescompiler, '{}.0'.format(args.output)] - size = 0 - for path, key in iterate_resource_params(py_files): - addendum = len(path) + len(key) - if size + addendum > LIMIT and len(cmd) > 2: - call(cmd) - outputs.append(cmd[1]) - cmd[1] = '{}.{}'.format(args.output, len(outputs)) - cmd = cmd[0:2] - size = 0 - cmd.extend([path, key]) - size += addendum - if len(outputs) == 0: - cmd[1] = args.output - call(cmd) - else: - call(cmd) - outputs.append(cmd[1]) - with open(args.output, 'w') as fout: - for fname in outputs: - with open(fname, 'r') as fin: - shutil.copyfileobj(fin, fout) - - -if __name__ == '__main__': - main() diff --git a/build/scripts/configure_file.py b/build/scripts/configure_file.py deleted file mode 100755 index 193ad7ec9e..0000000000 --- a/build/scripts/configure_file.py +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env python2.7 - -import sys -import os.path -import re - -cmakeDef01 = "#cmakedefine01" -cmakeDef = "#cmakedefine" - - -def replaceLine(l, varDict, define): - words = l.split() - if words: - if words[0] == cmakeDef: - sPos = l.find(cmakeDef) - ePos = sPos + len(cmakeDef) - l = l[:sPos] + define + l[ePos:] + '\n' - if words[0] == cmakeDef01: - var = words[1] - cmakeValue = varDict.get(var) - if cmakeValue == 'yes': - val = '1' - else: - val = '0' - sPos = l.find(cmakeDef01) - ePos = l.find(var) + len(var) - l = l[:sPos] + define + ' ' + var + ' ' + val + l[ePos + 1:] + '\n' - - finder = re.compile(".*?(@[a-zA-Z0-9_]+@).*") - while True: - re_result = finder.match(l) - if not re_result: - return l - key = re_result.group(1)[1:-1] - l = l[:re_result.start(1)] + varDict.get(key, '') + l[re_result.end(1):] - - -def main(inputPath, outputPath, varDict): - define = '#define' if os.path.splitext(outputPath)[1] != '.asm' else '%define' - with open(outputPath, 'w') as output: - with open(inputPath, 'r') as input: - for l in input: - output.write(replaceLine(l, varDict, define)) - - -def usage(): - print("usage: configure_file.py inputPath outputPath key1=value1 ...") - exit(1) - - -if __name__ == "__main__": - if len(sys.argv) < 3: - usage() - varDict = {} - for x in sys.argv[3:]: - key, value = str(x).split('=', 1) - varDict[key] = value - - main(sys.argv[1], sys.argv[2], varDict) diff --git a/build/scripts/container.py b/build/scripts/container.py deleted file mode 100644 index 27e6f921f3..0000000000 --- a/build/scripts/container.py +++ /dev/null @@ -1,30 +0,0 @@ -import subprocess -import os -import shutil - - -class ContainerError(Exception): - pass - - -def join_layers(input_paths, output_path, squashfs_path): - - if len(input_paths) == 1: - shutil.copy2(input_paths[0], output_path) - - else: - # We cannot use appending here as it doesn't allow replacing files - for input_path in input_paths: - unpack_cmd = [ os.path.join(squashfs_path, 'unsquashfs') ] - unpack_cmd.extend([ '-f', input_path ]) - subprocess.run(unpack_cmd) - - pack_cmd = [ os.path.join(squashfs_path, 'mksquashfs') ] - pack_cmd.append(os.path.join(os.curdir, 'squashfs-root')) - pack_cmd.append(output_path) - pack_cmd.append('-all-root') - subprocess.run(pack_cmd) - - shutil.rmtree(os.path.join(os.curdir, 'squashfs-root')) - - return 0 diff --git a/build/scripts/copy_docs_files.py b/build/scripts/copy_docs_files.py deleted file mode 100644 index c444dd509d..0000000000 --- a/build/scripts/copy_docs_files.py +++ /dev/null @@ -1,102 +0,0 @@ -import argparse -import codecs -import errno -import os -import process_command_files as pcf -import shutil -import sys - - -def parse_args(): - parser = argparse.ArgumentParser() - parser.add_argument('--build-root', required=True) - parser.add_argument('--dst-dir', required=True) - parser.add_argument('--existing', choices=('skip', 'overwrite'), default='overwrite') - parser.add_argument('--source-root', required=True) - parser.add_argument('--src-dir', required=None) - parser.add_argument('files', nargs='*') - return parser.parse_args(pcf.get_args(sys.argv[1:])) - - -def makedirs(dirname): - try: - os.makedirs(dirname) - except OSError as e: - if e.errno == errno.EEXIST and os.path.isdir(dirname): - pass - else: - raise - - -def copy_file(src, dst, overwrite=False, orig_path=None, generated=False): - if os.path.exists(dst) and not overwrite: - return - - makedirs(os.path.dirname(dst)) - - with open(src, 'rb') as fsrc, open(dst, 'wb') as fdst: - if (orig_path or generated) and src.endswith('.md'): - out = b'' - buf = fsrc.readline() - bom_length = len(codecs.BOM_UTF8) - if buf[:bom_length] == codecs.BOM_UTF8: - out += codecs.BOM_UTF8 - buf = buf[bom_length:] - info = 'generated: true\n' if generated else 'vcsPath: {}\n'.format(orig_path) - if buf.startswith(b'---') and b'\n' in buf[3:] and buf[3:].rstrip(b'\r\n') == b'': - content = b'' - found = False - while True: - line = fsrc.readline() - if len(line) == 0: - break - content += line - if line.startswith(b'---') and line[3:].rstrip(b'\r\n') == b'': - found = True - break - out += buf - if found: - out += info.encode('utf-8') - out += content - else: - out += '---\n{}---\n'.format(info).encode('utf-8') - out += buf - fdst.write(out) - shutil.copyfileobj(fsrc, fdst) - - -def main(): - args = parse_args() - - source_root = os.path.normpath(args.source_root) + os.path.sep - build_root = os.path.normpath(args.build_root) + os.path.sep - - dst_dir = os.path.normpath(args.dst_dir) - assert dst_dir.startswith(build_root) - makedirs(dst_dir) - - src_dir = os.path.normpath(args.src_dir) + os.path.sep - assert src_dir.startswith(source_root) - - if src_dir.startswith(source_root): - root = source_root - is_from_source_root = True - elif src_dir.startswith(build_root): - root = build_root - is_from_source_root = False - else: - assert False, 'src_dir [{}] should start with [{}] or [{}]'.format(src_dir, source_root, build_root) - - is_overwrite_existing = args.existing == 'overwrite' - - for f in [os.path.normpath(f) for f in args.files]: - src_file = os.path.join(src_dir, f) - dst_file = os.path.join(dst_dir, f) - if src_file == dst_file: - continue - rel_path = src_file[len(root):] if is_from_source_root else None - copy_file(src_file, dst_file, overwrite=is_overwrite_existing, orig_path=rel_path) - - -if __name__ == '__main__': - main() diff --git a/build/scripts/copy_docs_files_to_dir.py b/build/scripts/copy_docs_files_to_dir.py deleted file mode 100644 index bfd3c65698..0000000000 --- a/build/scripts/copy_docs_files_to_dir.py +++ /dev/null @@ -1,164 +0,0 @@ -import argparse -import codecs -import errno -import os -import process_command_files as pcf -import shutil -import sys - - -def parse_args(): - parser = argparse.ArgumentParser() - parser.add_argument('--bin-dir', nargs='*') - parser.add_argument('--build-root', required=True) - parser.add_argument('--dest-dir', required=True) - parser.add_argument('--docs-dir', action='append', nargs=2, dest='docs_dirs', default=None) - parser.add_argument('--existing', choices=('skip', 'overwrite'), default='overwrite') - parser.add_argument('--source-root', required=True) - parser.add_argument('--src-dir', action='append', nargs='*', dest='src_dirs', default=None) - parser.add_argument('files', nargs='*') - return parser.parse_args(pcf.get_args(sys.argv[1:])) - - -def makedirs(dirname): - try: - os.makedirs(dirname) - except OSError as e: - if e.errno == errno.EEXIST and os.path.isdir(dirname): - pass - else: - raise - - -def copy_file(src, dst, overwrite=False, orig_path=None): - if os.path.exists(dst) and not overwrite: - return - - makedirs(os.path.dirname(dst)) - - with open(src, 'rb') as fsrc, open(dst, 'wb') as fdst: - if orig_path and src.endswith('.md'): - out = b'' - buf = fsrc.readline() - bom_length = len(codecs.BOM_UTF8) - if buf[:bom_length] == codecs.BOM_UTF8: - out += codecs.BOM_UTF8 - buf = buf[bom_length:] - info = 'vcsPath: {}\n'.format(orig_path) - if buf.startswith(b'---') and b'\n' in buf[3:] and buf[3:].rstrip(b'\r\n') == b'': - content = b'' - found = False - while True: - line = fsrc.readline() - if len(line) == 0: - break - content += line - if line.startswith(b'---') and line[3:].rstrip(b'\r\n') == b'': - found = True - break - out += buf - if found: - out += info.encode('utf-8') - out += content - else: - out += '---\n{}---\n'.format(info).encode('utf-8') - out += buf - fdst.write(out) - shutil.copyfileobj(fsrc, fdst) - - -def main(): - args = parse_args() - - dest_dir = os.path.normpath(args.dest_dir) - makedirs(dest_dir) - - source_root = os.path.normpath(args.source_root) + os.path.sep - build_root = os.path.normpath(args.build_root) + os.path.sep - - is_overwrite_existing = args.existing == 'overwrite' - - if args.docs_dirs: - for item in args.docs_dirs: - assert len(item) == 2 - docs_dir, nm = item[0], item[1] - assert not os.path.isabs(docs_dir) - if nm and nm != '.': - assert not os.path.isabs(nm) - dst = os.path.join(dest_dir, nm) - else: - dst = dest_dir - - abs_docs_dir = os.path.join(args.source_root, docs_dir) - - for root, _, files in os.walk(abs_docs_dir): - for f in files: - if os.path.islink(os.path.join(root, f)): - continue - file_src = os.path.join(root, f) - assert file_src.startswith(source_root) - file_dst = os.path.join(dst, os.path.relpath(root, abs_docs_dir), f) - copy_file(file_src, file_dst, overwrite=is_overwrite_existing, orig_path=file_src[len(source_root):]) - - if args.src_dirs: - for item in args.src_dirs: - assert len(item) > 1 - src_dir, nm = os.path.normpath(item[0]), item[1] - assert os.path.isabs(src_dir) - if nm and nm != '.': - assert not os.path.isabs(nm) - dst = os.path.join(dest_dir, nm) - else: - dst = dest_dir - - if src_dir.startswith(source_root): - root = source_root - is_from_source_root = True - else: - assert src_dir.startswith(build_root) - root = build_root - is_from_source_root = False - - for f in item[2:]: - file_src = os.path.normpath(f) - assert file_src.startswith(root) - rel_path = file_src[len(root):] if is_from_source_root else None - file_dst = os.path.join(dst, file_src[len(src_dir):]) - copy_file(file_src, file_dst, overwrite=is_overwrite_existing, orig_path=rel_path) - - if args.bin_dir: - assert len(args.bin_dir) > 1 - bin_dir, bin_dir_namespace = os.path.normpath(args.bin_dir[0]) + os.path.sep, args.bin_dir[1] - assert bin_dir.startswith(build_root) - if bin_dir_namespace and bin_dir_namespace != '.': - assert not os.path.isabs(bin_dir_namespace) - dst = os.path.join(dest_dir, bin_dir_namespace) - else: - dst = dest_dir - - for file_src in args.bin_dir[2:]: - assert os.path.isfile(file_src) - assert file_src.startswith(bin_dir) - file_dst = os.path.join(dst, file_src[len(bin_dir):]) - copy_file(file_src, file_dst, overwrite=is_overwrite_existing, orig_path=None) - - for src in args.files: - file_src = os.path.normpath(src) - assert os.path.isfile(file_src), 'File [{}] does not exist...'.format(file_src) - rel_path = file_src - orig_path = None - if file_src.startswith(source_root): - rel_path = file_src[len(source_root):] - orig_path = rel_path - elif file_src.startswith(build_root): - rel_path = file_src[len(build_root):] - else: - raise Exception('Unexpected file path [{}].'.format(file_src)) - assert not os.path.isabs(rel_path) - file_dst = os.path.join(args.dest_dir, rel_path) - if file_dst != file_src: - copy_file(file_src, file_dst, is_overwrite_existing, orig_path) - - -if __name__ == '__main__': - main() diff --git a/build/scripts/copy_files_to_dir.py b/build/scripts/copy_files_to_dir.py deleted file mode 100644 index ead57ba16e..0000000000 --- a/build/scripts/copy_files_to_dir.py +++ /dev/null @@ -1,59 +0,0 @@ -import argparse -import errno -import os -import process_command_files as pcf -import shutil -import sys - - -def parse_args(): - parser = argparse.ArgumentParser() - parser.add_argument('--dest-dir', required=True) - parser.add_argument('--existing', choices=('skip', 'overwrite'), default='overwrite') - parser.add_argument('--flat', action='store_true') - parser.add_argument('--skip-prefix', dest='skip_prefixes', action='append', default=[]) - parser.add_argument('files', nargs='*') - return parser.parse_args(pcf.get_args(sys.argv[1:])) - - -def makedirs(dirname): - try: - os.makedirs(dirname) - except OSError as e: - if e.errno == errno.EEXIST and os.path.isdir(dirname): - pass - else: - raise - - -def main(): - args = parse_args() - - dest_dir = os.path.normpath(args.dest_dir) + os.pathsep - makedirs(dest_dir) - - prefixes = ['{}{}'.format(os.path.normpath(p), os.path.sep) for p in args.skip_prefixes] - - for src in args.files: - src = os.path.normpath(src) - assert os.path.isfile(src) - if args.flat: - rel_dst = os.path.basename(src) - else: - rel_dst = src - for prefix in prefixes: - if src.startswith(prefix): - rel_dst = src[len(prefix):] - break - assert not os.path.isabs(rel_dst) - dst = os.path.join(args.dest_dir, rel_dst) - if os.path.isfile(dst) and args.existing == 'skip': - break - - makedirs(os.path.dirname(dst)) - - shutil.copyfile(src, dst) - - -if __name__ == '__main__': - main() diff --git a/build/scripts/copy_to_dir.py b/build/scripts/copy_to_dir.py deleted file mode 100644 index 9baeb5ffac..0000000000 --- a/build/scripts/copy_to_dir.py +++ /dev/null @@ -1,75 +0,0 @@ -import errno -import sys -import os -import shutil -import optparse -import tarfile - - -def parse_args(): - parser = optparse.OptionParser() - parser.add_option('--build-root') - parser.add_option('--dest-dir') - parser.add_option('--dest-arch') - return parser.parse_args() - - -def ensure_dir_exists(path): - try: - os.makedirs(path) - except OSError as e: - if e.errno == errno.EEXIST and os.path.isdir(path): - pass - else: - raise - - -def hardlink_or_copy(src, dst): - if os.name == 'nt': - shutil.copy(src, dst) - else: - try: - os.link(src, dst) - except OSError as e: - if e.errno == errno.EEXIST: - return - elif e.errno == errno.EXDEV: - sys.stderr.write("Can't make cross-device hardlink - fallback to copy: {} -> {}\n".format(src, dst)) - shutil.copy(src, dst) - else: - raise - - -def main(): - opts, args = parse_args() - assert opts.build_root - assert opts.dest_dir - - dest_arch = None - if opts.dest_arch: - if opts.dest_arch.endswith('.tar'): - dest_arch = tarfile.open(opts.dest_arch, 'w', dereference=True) - elif opts.dest_arch.endswith('.tar.gz') or opts.dest_arch.endswith('.tgz'): - dest_arch = tarfile.open(opts.dest_arch, 'w:gz', dereference=True) - else: - # TODO: move check to graph generation stage - raise Exception('Unsopported archive type for {}. Use one of: tar, tar.gz, tgz.'.format(os.path.basename(opts.dest_arch))) - - for arg in args: - dst = arg - if dst.startswith(opts.build_root): - dst = dst[len(opts.build_root) + 1:] - - if dest_arch and not arg.endswith('.pkg.fake'): - dest_arch.add(arg, arcname=dst) - - dst = os.path.join(opts.dest_dir, dst) - ensure_dir_exists(os.path.dirname(dst)) - hardlink_or_copy(arg, dst) - - if dest_arch: - dest_arch.close() - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/build/scripts/coverage-info.py b/build/scripts/coverage-info.py deleted file mode 100644 index 94491d9256..0000000000 --- a/build/scripts/coverage-info.py +++ /dev/null @@ -1,282 +0,0 @@ -import argparse -import os -import sys -import tarfile -import collections -import subprocess -import re - - -GCDA_EXT = '.gcda' -GCNO_EXT = '.gcno' - - -def suffixes(path): - """ - >>> list(suffixes('/a/b/c')) - ['c', 'b/c', '/a/b/c'] - >>> list(suffixes('/a/b/c/')) - ['c', 'b/c', '/a/b/c'] - >>> list(suffixes('/a')) - ['/a'] - >>> list(suffixes('/a/')) - ['/a'] - >>> list(suffixes('/')) - [] - """ - path = os.path.normpath(path) - - def up_dirs(cur_path): - while os.path.dirname(cur_path) != cur_path: - cur_path = os.path.dirname(cur_path) - yield cur_path - - for x in up_dirs(path): - yield path.replace(x + os.path.sep, '') - - -def recast(in_file, out_file, probe_path, update_stat): - PREFIX = 'SF:' - - probed_path = None - - any_payload = False - - with open(in_file, 'r') as input, open(out_file, 'w') as output: - active = True - for line in input: - line = line.rstrip('\n') - if line.startswith('TN:'): - output.write(line + '\n') - elif line.startswith(PREFIX): - path = line[len(PREFIX):] - probed_path = probe_path(path) - if probed_path: - output.write(PREFIX + probed_path + '\n') - active = bool(probed_path) - else: - if active: - update_stat(probed_path, line) - output.write(line + '\n') - any_payload = True - - return any_payload - - -def print_stat(da, fnda, teamcity_stat_output): - lines_hit = sum(map(bool, da.values())) - lines_total = len(da.values()) - lines_coverage = 100.0 * lines_hit / lines_total if lines_total else 0 - - func_hit = sum(map(bool, fnda.values())) - func_total = len(fnda.values()) - func_coverage = 100.0 * func_hit / func_total if func_total else 0 - - print >>sys.stderr, '[[imp]]Lines[[rst]] {: >16} {: >16} {: >16.1f}%'.format(lines_hit, lines_total, lines_coverage) - print >>sys.stderr, '[[imp]]Functions[[rst]] {: >16} {: >16} {: >16.1f}%'.format(func_hit, func_total, func_coverage) - - if teamcity_stat_output: - with open(teamcity_stat_output, 'w') as tc_file: - tc_file.write("##teamcity[blockOpened name='Code Coverage Summary']\n") - tc_file.write("##teamcity[buildStatisticValue key=\'CodeCoverageAbsLTotal\' value='{}']\n".format(lines_total)) - tc_file.write("##teamcity[buildStatisticValue key=\'CodeCoverageAbsLCovered\' value='{}']\n".format(lines_hit)) - tc_file.write("##teamcity[buildStatisticValue key=\'CodeCoverageAbsMTotal\' value='{}']\n".format(func_total)) - tc_file.write("##teamcity[buildStatisticValue key=\'CodeCoverageAbsMCovered\' value='{}']\n".format(func_hit)) - tc_file.write("##teamcity[blockClosed name='Code Coverage Summary']\n") - - -def chunks(l, n): - """ - >>> list(chunks(range(10), 3)) - [[0, 1, 2], [3, 4, 5], [6, 7, 8], [9]] - >>> list(chunks(range(10), 5)) - [[0, 1, 2, 3, 4], [5, 6, 7, 8, 9]] - """ - for i in xrange(0, len(l), n): - yield l[i:i + n] - - -def combine_info_files(lcov, files, out_file): - chunk_size = 50 - files = list(set(files)) - - for chunk in chunks(files, chunk_size): - combine_cmd = [lcov] - if os.path.exists(out_file): - chunk.append(out_file) - for trace in chunk: - assert os.path.exists(trace), "Trace file does not exist: {} (cwd={})".format(trace, os.getcwd()) - combine_cmd += ["-a", os.path.abspath(trace)] - print >>sys.stderr, '## lcov', ' '.join(combine_cmd[1:]) - out_file_tmp = "combined.tmp" - with open(out_file_tmp, "w") as stdout: - subprocess.check_call(combine_cmd, stdout=stdout) - if os.path.exists(out_file): - os.remove(out_file) - os.rename(out_file_tmp, out_file) - - -def probe_path_global(path, source_root, prefix_filter, exclude_files): - if path.endswith('_ut.cpp'): - return None - - for suff in reversed(list(suffixes(path))): - if (not prefix_filter or suff.startswith(prefix_filter)) and (not exclude_files or not exclude_files.match(suff)): - full_path = source_root + os.sep + suff - if os.path.isfile(full_path): - return full_path - - return None - - -def update_stat_global(src_file, line, fnda, da): - if line.startswith("FNDA:"): - visits, func_name = line[len("FNDA:"):].split(',') - fnda[src_file + func_name] += int(visits) - - if line.startswith("DA"): - line_number, visits = line[len("DA:"):].split(',') - if visits == '=====': - visits = 0 - - da[src_file + line_number] += int(visits) - - -def gen_info_global(cmd, cov_info, probe_path, update_stat, lcov_args): - print >>sys.stderr, '## geninfo', ' '.join(cmd) - subprocess.check_call(cmd) - if recast(cov_info + '.tmp', cov_info, probe_path, update_stat): - lcov_args.append(cov_info) - - -def init_all_coverage_files(gcno_archive, fname2gcno, fname2info, geninfo_executable, gcov_tool, gen_info, prefix_filter, exclude_files): - with tarfile.open(gcno_archive) as gcno_tf: - for gcno_item in gcno_tf: - if gcno_item.isfile() and gcno_item.name.endswith(GCNO_EXT): - gcno_tf.extract(gcno_item) - - gcno_name = gcno_item.name - source_fname = gcno_name[:-len(GCNO_EXT)] - if prefix_filter and not source_fname.startswith(prefix_filter): - sys.stderr.write("Skipping {} (doesn't match prefix '{}')\n".format(source_fname, prefix_filter)) - continue - if exclude_files and exclude_files.search(source_fname): - sys.stderr.write("Skipping {} (matched exclude pattern '{}')\n".format(source_fname, exclude_files.pattern)) - continue - - fname2gcno[source_fname] = gcno_name - - if os.path.getsize(gcno_name) > 0: - coverage_info = source_fname + '.' + str(len(fname2info[source_fname])) + '.info' - fname2info[source_fname].append(coverage_info) - geninfo_cmd = [ - geninfo_executable, - '--gcov-tool', gcov_tool, - '-i', gcno_name, - '-o', coverage_info + '.tmp' - ] - gen_info(geninfo_cmd, coverage_info) - - -def process_all_coverage_files(gcda_archive, fname2gcno, fname2info, geninfo_executable, gcov_tool, gen_info): - with tarfile.open(gcda_archive) as gcda_tf: - for gcda_item in gcda_tf: - if gcda_item.isfile() and gcda_item.name.endswith(GCDA_EXT): - gcda_name = gcda_item.name - source_fname = gcda_name[:-len(GCDA_EXT)] - for suff in suffixes(source_fname): - if suff in fname2gcno: - gcda_new_name = suff + GCDA_EXT - gcda_item.name = gcda_new_name - gcda_tf.extract(gcda_item) - if os.path.getsize(gcda_new_name) > 0: - coverage_info = suff + '.' + str(len(fname2info[suff])) + '.info' - fname2info[suff].append(coverage_info) - geninfo_cmd = [ - geninfo_executable, - '--gcov-tool', gcov_tool, - gcda_new_name, - '-o', coverage_info + '.tmp' - ] - gen_info(geninfo_cmd, coverage_info) - - -def gen_cobertura(tool, output, combined_info): - cmd = [ - tool, - combined_info, - '-b', '#hamster#', - '-o', output - ] - p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, err = p.communicate() - if p.returncode: - raise Exception('lcov_cobertura failed with exit code {}\nstdout: {}\nstderr: {}'.format(p.returncode, out, err)) - - -def main(source_root, output, gcno_archive, gcda_archive, gcov_tool, prefix_filter, exclude_regexp, teamcity_stat_output, coverage_report_path, gcov_report, lcov_cobertura): - exclude_files = re.compile(exclude_regexp) if exclude_regexp else None - - fname2gcno = {} - fname2info = collections.defaultdict(list) - lcov_args = [] - geninfo_executable = os.path.join(source_root, 'devtools', 'lcov', 'geninfo') - - def probe_path(path): - return probe_path_global(path, source_root, prefix_filter, exclude_files) - - fnda = collections.defaultdict(int) - da = collections.defaultdict(int) - - def update_stat(src_file, line): - update_stat_global(src_file, line, da, fnda) - - def gen_info(cmd, cov_info): - gen_info_global(cmd, cov_info, probe_path, update_stat, lcov_args) - - init_all_coverage_files(gcno_archive, fname2gcno, fname2info, geninfo_executable, gcov_tool, gen_info, prefix_filter, exclude_files) - process_all_coverage_files(gcda_archive, fname2gcno, fname2info, geninfo_executable, gcov_tool, gen_info) - - if coverage_report_path: - output_dir = coverage_report_path - else: - output_dir = output + '.dir' - - if not os.path.exists(output_dir): - os.makedirs(output_dir) - - teamcity_stat_file = None - if teamcity_stat_output: - teamcity_stat_file = os.path.join(output_dir, 'teamcity.out') - print_stat(da, fnda, teamcity_stat_file) - - if lcov_args: - output_trace = "combined.info" - combine_info_files(os.path.join(source_root, 'devtools', 'lcov', 'lcov'), lcov_args, output_trace) - cmd = [os.path.join(source_root, 'devtools', 'lcov', 'genhtml'), '-p', source_root, '--ignore-errors', 'source', '-o', output_dir, output_trace] - print >>sys.stderr, '## genhtml', ' '.join(cmd) - subprocess.check_call(cmd) - if lcov_cobertura: - gen_cobertura(lcov_cobertura, gcov_report, output_trace) - - with tarfile.open(output, 'w') as tar: - tar.add(output_dir, arcname='.') - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - - parser.add_argument('--source-root', action='store') - parser.add_argument('--output', action='store') - parser.add_argument('--gcno-archive', action='store') - parser.add_argument('--gcda-archive', action='store') - parser.add_argument('--gcov-tool', action='store') - parser.add_argument('--prefix-filter', action='store') - parser.add_argument('--exclude-regexp', action='store') - parser.add_argument('--teamcity-stat-output', action='store_const', const=True) - parser.add_argument('--coverage-report-path', action='store') - parser.add_argument('--gcov-report', action='store') - parser.add_argument('--lcov-cobertura', action='store') - - args = parser.parse_args() - main(**vars(args)) diff --git a/build/scripts/cpp_flatc_wrapper.py b/build/scripts/cpp_flatc_wrapper.py deleted file mode 100644 index 9f74b65570..0000000000 --- a/build/scripts/cpp_flatc_wrapper.py +++ /dev/null @@ -1,30 +0,0 @@ -import os -import subprocess -import sys - - -def main(): - cmd = sys.argv[1:] - h_file = None - try: - index = cmd.index('-o') - h_file = cmd[index+1] - cmd[index+1] = os.path.dirname(h_file) - except (ValueError, IndexError): - pass - p = subprocess.run(cmd, capture_output=True, text=True) - if p.returncode: - if p.stdout: - sys.stderr.write('stdout:\n{}\n'.format(p.stdout)) - if p.stderr: - sys.stderr.write('stderr:\n{}\n'.format(p.stderr)) - sys.exit(p.returncode) - if h_file and h_file.endswith(('.fbs.h', '.fbs64.h')): - cpp_file = '{}.cpp'.format(h_file[:-2]) - with open(cpp_file, 'w') as f: - f.write('#include "{}"\n'.format(os.path.basename(h_file))) - sys.exit(0) - - -if __name__ == '__main__': - main() diff --git a/build/scripts/create_jcoverage_report.py b/build/scripts/create_jcoverage_report.py deleted file mode 100644 index 45083ff4f7..0000000000 --- a/build/scripts/create_jcoverage_report.py +++ /dev/null @@ -1,112 +0,0 @@ -import argparse -import tarfile -import zipfile -import os -import sys -import time -import subprocess - - -def mkdir_p(path): - try: - os.makedirs(path) - except OSError: - pass - - -class Timer(object): - - def __init__(self): - self.start = time.time() - - def step(self, msg): - sys.stderr.write("{} ({}s)\n".format(msg, int(time.time() - self.start))) - self.start = time.time() - - -def main(source, output, java, prefix_filter, exclude_filter, jars_list, output_format, tar_output, agent_disposition, runners_paths): - timer = Timer() - reports_dir = 'jacoco_reports_dir' - mkdir_p(reports_dir) - with tarfile.open(source) as tf: - tf.extractall(reports_dir) - timer.step("Coverage data extracted") - reports = [os.path.join(reports_dir, fname) for fname in os.listdir(reports_dir)] - - with open(jars_list) as f: - jars = f.read().strip().split() - if jars and runners_paths: - for r in runners_paths: - try: - jars.remove(r) - except ValueError: - pass - - src_dir = 'sources_dir' - cls_dir = 'classes_dir' - - mkdir_p(src_dir) - mkdir_p(cls_dir) - - for jar in jars: - if jar.endswith('devtools-jacoco-agent.jar'): - agent_disposition = jar - - # Skip java contrib - it's irrelevant coverage - if jar.startswith('contrib/java'): - continue - - with zipfile.ZipFile(jar) as jf: - for entry in jf.infolist(): - if entry.filename.endswith('.java'): - dest = src_dir - - elif entry.filename.endswith('.class'): - dest = cls_dir - - else: - continue - - entry.filename = entry.filename.encode('utf-8') - jf.extract(entry, dest) - timer.step("Jar files extracted") - - if not agent_disposition: - print>>sys.stderr, 'Can\'t find jacoco agent. Will not generate html report for java coverage.' - - if tar_output: - report_dir = 'java.report.temp' - else: - report_dir = output - mkdir_p(report_dir) - - if agent_disposition: - agent_cmd = [java, '-jar', agent_disposition, src_dir, cls_dir, prefix_filter or '.', exclude_filter or '__no_exclude__', report_dir, output_format] - agent_cmd += reports - subprocess.check_call(agent_cmd) - timer.step("Jacoco finished") - - if tar_output: - with tarfile.open(output, 'w') as outf: - outf.add(report_dir, arcname='.') - - -if __name__ == '__main__': - if 'LC_ALL' in os.environ: - if os.environ['LC_ALL'] == 'C': - os.environ['LC_ALL'] = 'en_GB.UTF-8' - - parser = argparse.ArgumentParser() - - parser.add_argument('--source', action='store') - parser.add_argument('--output', action='store') - parser.add_argument('--java', action='store') - parser.add_argument('--prefix-filter', action='store') - parser.add_argument('--exclude-filter', action='store') - parser.add_argument('--jars-list', action='store') - parser.add_argument('--output-format', action='store', default="html") - parser.add_argument('--raw-output', dest='tar_output', action='store_false', default=True) - parser.add_argument('--agent-disposition', action='store') - parser.add_argument('--runner-path', dest='runners_paths', action='append', default=[]) - args = parser.parse_args() - main(**vars(args)) diff --git a/build/scripts/custom_link_green_mysql.py b/build/scripts/custom_link_green_mysql.py deleted file mode 100644 index 13bb9e4ac7..0000000000 --- a/build/scripts/custom_link_green_mysql.py +++ /dev/null @@ -1,97 +0,0 @@ -import os -import shutil -import subprocess -import sys -import tempfile - - -SYMBOLS_TO_PATCH = ( - 'connect', - 'poll', - 'recv', - 'recvfrom', - 'send', - 'sendto', -) - -class Error(Exception): - pass - - -def find_compiler(args): - for arg in args: - if os.path.basename(arg) in ('clang', 'clang++'): - return arg - raise Error('No known compiler found in the command line') - - -def find_libraries(project, args): - if not project.endswith('/'): - project = project + '/' - - for arg in args: - if arg.startswith(project): - yield arg - - -def rename_symbol(symbol): - return 'green_{}'.format(symbol) - - -def patch_object(object_path, objcopy): - args = [objcopy] - for symbol in SYMBOLS_TO_PATCH: - args.extend(('--redefine-sym', '{}={}'.format(symbol, rename_symbol(symbol)))) - args.append(object_path) - subprocess.check_call(args) - - -def patch_library(library_path, ar, objcopy): - tmpdir = tempfile.mkdtemp(dir=os.path.dirname(library_path)) - try: - subprocess.check_call((ar, 'x', library_path), cwd=tmpdir) - names = os.listdir(tmpdir) - for name in names: - patch_object(os.path.join(tmpdir, name), objcopy=objcopy) - - new_library_path = os.path.join(tmpdir, 'library.a') - subprocess.check_call([ar, 'rcs', new_library_path] + names, cwd=tmpdir) - - os.rename(new_library_path, library_path) - - finally: - shutil.rmtree(tmpdir) - - -def main(): - try: - args = sys.argv[1:] - compiler = find_compiler(args) - compiler_dir = os.path.dirname(compiler) - - def get_tool(name): - path = os.path.join(compiler_dir, name) - if not os.path.exists(path): - raise Error('No {} found alongside the compiler'.format(name)) - return path - - ar = get_tool('llvm-ar') - objcopy = get_tool('llvm-objcopy') - - libraries = tuple(find_libraries('contrib/libs/libmysql_r', args)) - for library in libraries: - library_path = os.path.abspath(library) - if not os.path.exists(library_path): - raise Error('No {} file exists'.format(library)) - - patch_library(library_path, ar=ar, objcopy=objcopy) - - except Exception as error: - name = os.path.basename(sys.argv[0]) - command = ' '.join(args) - message = '{name} failed: {error}\nCommand line: {command}' - print >> sys.stderr, message.format(**locals()) - - -if __name__ == '__main__': - main() diff --git a/build/scripts/decimal_md5.py b/build/scripts/decimal_md5.py deleted file mode 100644 index e70ca80a09..0000000000 --- a/build/scripts/decimal_md5.py +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import hashlib -import struct -import sys -import os -import argparse - - -def print_code(checksum, func_name): - if len(func_name) == 0: # safe fallback for old ya.make files - func_name = "DecimalMD5" - print 'const char* ' + func_name + '() {return "' + checksum + '";}' - - -def ensure_paths_exist(paths): - bad_paths = sorted( - path for path in paths - if not os.path.exists(path) - ) - if bad_paths: - print >> sys.stderr, "decimal_md5 inputs do not exist:" - for path in bad_paths: - print >> sys.stderr, path - sys.exit(1) - - -def _update_digest_with_file_contents(digest, path, block_size=65535): - with open(path) as f: - while True: - block = f.read(block_size) - if not block: - break - digest.update(block) - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("--fixed-output", help="don not calculate md5, use this value instead") - parser.add_argument("--lower-bits", help="use specified count of lower bits", type=int, default=32) - parser.add_argument("--source-root", help="arcadia source root") - parser.add_argument("--func-name", help="custom function name to be defined", default="DecimalMD5") - parser.add_argument("targets", nargs='*', default=['.']) - - args = parser.parse_args() - - abs_paths = [ - os.path.join(args.source_root, target) - for target in args.targets - ] - ensure_paths_exist(abs_paths) - - if args.fixed_output: - try: - bitmask = (1 << args.lower_bits) - 1 - fmt = '{:0%dd}' % len(str(bitmask)) - checksum = fmt.format(int(args.fixed_output) & bitmask) - except ValueError: - raise ValueError("decimal_md5: bad value passed via --fixed-output: %s" % args.fixed_output) - print_code(str(checksum), func_name=args.func_name) - return - - md5 = hashlib.md5() - for path in abs_paths: - _update_digest_with_file_contents(md5, path) - - md5_parts = struct.unpack('IIII', md5.digest()) - md5_int = sum(part << (32 * n) for n, part in enumerate(md5_parts)) - bitmask = (1 << args.lower_bits) - 1 - fmt = '{:0%dd}' % len(str(bitmask)) - - checksum_str = fmt.format(md5_int & bitmask) - print_code(checksum_str, func_name=args.func_name) - - -if __name__ == "__main__": - main() - diff --git a/build/scripts/error.py b/build/scripts/error.py deleted file mode 100644 index f7d8ecb2cc..0000000000 --- a/build/scripts/error.py +++ /dev/null @@ -1,77 +0,0 @@ -# Sync content of this file with devtools/ya/core/error/__init__.py - -TEMPORARY_ERROR_MESSAGES = [ - 'Connection reset by peer', - 'Connection timed out', - 'Function not implemented', - 'I/O operation on closed file', - 'Internal Server Error', - 'Network connection closed unexpectedly', - 'Network is unreachable', - 'No route to host', - 'No space left on device', - 'Not enough space', - 'Temporary failure in name resolution', - 'The read operation timed out', - 'timeout: timed out', -] - - -# Node exit codes -class ExitCodes(object): - TEST_FAILED = 10 - COMPILATION_FAILED = 11 - INFRASTRUCTURE_ERROR = 12 - NOT_RETRIABLE_ERROR = 13 - YT_STORE_FETCH_ERROR = 14 - - -def merge_exit_codes(exit_codes): - return max(e if e >= 0 else 1 for e in exit_codes) if exit_codes else 0 - - -def is_temporary_error(exc): - import logging - logger = logging.getLogger(__name__) - - if getattr(exc, 'temporary', False): - logger.debug("Exception has temporary attribute: %s", exc) - return True - - import errno - err = getattr(exc, 'errno', None) - - if err == errno.ECONNREFUSED or err == errno.ENETUNREACH: - logger.debug("Exception has errno attribute: %s (errno=%s)", exc, err) - return True - - import socket - - if isinstance(exc, socket.timeout) or isinstance(getattr(exc, 'reason', None), socket.timeout): - logger.debug("Socket timeout exception: %s", exc) - return True - - if isinstance(exc, socket.gaierror): - logger.debug("Getaddrinfo exception: %s", exc) - return True - - import urllib2 - - if isinstance(exc, urllib2.HTTPError) and exc.code in (429, ): - logger.debug("urllib2.HTTPError: %s", exc) - return True - - import httplib - - if isinstance(exc, httplib.IncompleteRead): - logger.debug("IncompleteRead exception: %s", exc) - return True - - exc_str = str(exc) - - for message in TEMPORARY_ERROR_MESSAGES: - if message in exc_str: - logger.debug("Found temporary error pattern (%s): %s", message, exc_str) - return True - - return False diff --git a/build/scripts/extract_asrc.py b/build/scripts/extract_asrc.py deleted file mode 100644 index 89892ddf2d..0000000000 --- a/build/scripts/extract_asrc.py +++ /dev/null @@ -1,23 +0,0 @@ -import argparse -import os -import tarfile - - -def parse_args(): - parser = argparse.ArgumentParser() - parser.add_argument('--input', nargs='*', required=True) - parser.add_argument('--output', required=True) - - return parser.parse_args() - - -def main(): - args = parse_args() - - for asrc in filter(lambda x: x.endswith('.asrc') and os.path.exists(x), args.input): - with tarfile.open(asrc, 'r') as tar: - tar.extractall(path=args.output) - - -if __name__ == '__main__': - main() diff --git a/build/scripts/extract_docs.py b/build/scripts/extract_docs.py deleted file mode 100644 index 20e8311346..0000000000 --- a/build/scripts/extract_docs.py +++ /dev/null @@ -1,43 +0,0 @@ -import argparse -import os -import process_command_files as pcf -import tarfile -import sys - - -def parse_args(): - parser = argparse.ArgumentParser() - parser.add_argument('--dest-dir', required=True) - parser.add_argument('--skip-prefix', dest='skip_prefixes', action='append', default=[]) - parser.add_argument('docs', nargs='*') - return parser.parse_args(pcf.get_args(sys.argv[1:])) - - -def main(): - args = parse_args() - - prefixes = ['{}{}'.format(os.path.normpath(p), os.path.sep) for p in args.skip_prefixes] - - def _valid_docslib(path): - base = os.path.basename(path) - return base.endswith(('.docslib', '.docslib.fake')) or base == 'preprocessed.tar.gz' - - for src in [p for p in args.docs if _valid_docslib(p)]: - if src == 'preprocessed.tar.gz': - rel_dst = os.path.dirname(os.path.normpath(src)) - for prefix in prefixes: - if src.startswith(prefix): - rel_dst = rel_dst[len(prefix):] - continue - assert not os.path.isabs(rel_dst) - dest_dir = os.path.join(args.dest_dir, rel_dst) - else: - dest_dir = args.dest_dir - if not os.path.exists(dest_dir): - os.makedirs(dest_dir) - with tarfile.open(src, 'r') as tar_file: - tar_file.extractall(dest_dir) - - -if __name__ == '__main__': - main() diff --git a/build/scripts/extract_jacoco_report.py b/build/scripts/extract_jacoco_report.py deleted file mode 100644 index 02e4ba9f13..0000000000 --- a/build/scripts/extract_jacoco_report.py +++ /dev/null @@ -1,29 +0,0 @@ -import argparse -import os -import re -import tarfile - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - - parser.add_argument('--archive', action='store') - parser.add_argument('--source-re', action='store') - parser.add_argument('--destination', action='store') - - args = parser.parse_args() - - with tarfile.open(args.archive) as tf: - open(args.destination, 'wb').close() - extract_list = [] - matcher = re.compile(args.source_re) - temp_dir = os.path.join(os.path.dirname(args.destination), 'temp_profiles') - if not os.path.exists(temp_dir): - os.makedirs(temp_dir) - for f in [i for i in tf if matcher.match(i.name)]: - tf.extract(f, path=temp_dir) - for directory, _, srcs in os.walk(temp_dir): - for f in srcs: - with open(args.destination, 'ab') as dst: - with open(os.path.join(temp_dir, directory, f), 'rb') as src: - dst.write(src.read()) diff --git a/build/scripts/f2c.py b/build/scripts/f2c.py deleted file mode 100644 index 7021e1391f..0000000000 --- a/build/scripts/f2c.py +++ /dev/null @@ -1,58 +0,0 @@ -import sys -import subprocess -import argparse -import os - - -header = '''\ -#ifdef __GNUC__ -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-parameter" -#pragma GCC diagnostic ignored "-Wmissing-braces" -#pragma GCC diagnostic ignored "-Wuninitialized" -#pragma GCC diagnostic ignored "-Wreturn-type" -#pragma GCC diagnostic ignored "-Wmissing-field-initializers" -#endif - -''' - -footer = ''' -#ifdef __GNUC__ -#pragma GCC diagnostic pop -#endif -''' - - -def mkdir_p(directory): - if not os.path.exists(directory): - os.makedirs(directory) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - - parser.add_argument('-t', '--tool') - parser.add_argument('-c', '--input') - parser.add_argument('-o', '--output') - - args = parser.parse_args() - tmpdir = args.output + '.f2c' - mkdir_p(tmpdir) - # should parse includes, really - p = subprocess.Popen( - [args.tool, '-w', '-R', '-a', '-I' + os.path.dirname(args.input), '-T' + tmpdir], - stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) - stdout, stderr = p.communicate(input=open(args.input).read()) - ret = p.wait() - - if ret: - print >>sys.stderr, 'f2c failed: %s, %s' % (stderr, ret) - sys.exit(ret) - - if 'Error' in stderr: - print >>sys.stderr, stderr - - with open(args.output, 'w') as f: - f.write(header) - f.write(stdout) - f.write(footer) diff --git a/build/scripts/fail_module_cmd.py b/build/scripts/fail_module_cmd.py deleted file mode 100644 index fa14c0d851..0000000000 --- a/build/scripts/fail_module_cmd.py +++ /dev/null @@ -1,7 +0,0 @@ -import sys - - -if __name__ == '__main__': - assert len(sys.argv) == 2, 'Unexpected number of arguments...' - sys.stderr.write('Error: module command for target [[bad]]{}[[rst]] was not executed due to build graph configuration errors...\n'.format(sys.argv[1])) - sys.exit(1) diff --git a/build/scripts/fetch_from.py b/build/scripts/fetch_from.py deleted file mode 100755 index 78306d1794..0000000000 --- a/build/scripts/fetch_from.py +++ /dev/null @@ -1,393 +0,0 @@ -import datetime as dt -import errno -import hashlib -import json -import logging -import os -import platform -import random -import shutil -import socket -import string -import sys -import tarfile -import urllib2 - -import retry - - -def make_user_agent(): - return 'fetch_from: {host}'.format(host=socket.gethostname()) - - -def add_common_arguments(parser): - parser.add_argument('--copy-to') # used by jbuild in fetch_resource - parser.add_argument('--rename-to') # used by test_node in inject_mds_resource_to_graph - parser.add_argument('--copy-to-dir') - parser.add_argument('--untar-to') - parser.add_argument('--rename', action='append', default=[], metavar='FILE', help='rename FILE to the corresponding output') - parser.add_argument('--executable', action='store_true', help='make outputs executable') - parser.add_argument('--log-path') - parser.add_argument('-v', '--verbose', action='store_true', default=os.environ.get('YA_VERBOSE_FETCHER'), help='increase stderr verbosity') - parser.add_argument('outputs', nargs='*', default=[]) - - -def ensure_dir(path): - if not (path == '' or os.path.isdir(path)): - os.makedirs(path) - - -# Reference code: library/python/fs/__init__.py -def hardlink_or_copy(src, dst): - ensure_dir(os.path.dirname(dst)) - - if os.name == 'nt': - shutil.copy(src, dst) - else: - try: - os.link(src, dst) - except OSError as e: - if e.errno == errno.EEXIST: - return - elif e.errno in (errno.EXDEV, errno.EMLINK, errno.EINVAL, errno.EACCES): - sys.stderr.write("Can't make hardlink (errno={}) - fallback to copy: {} -> {}\n".format(e.errno, src, dst)) - shutil.copy(src, dst) - else: - raise - - -def rename_or_copy_and_remove(src, dst): - ensure_dir(os.path.dirname(dst)) - - try: - os.rename(src, dst) - except OSError: - shutil.copy(src, dst) - os.remove(src) - - -class BadChecksumFetchError(Exception): - pass - - -class IncompleteFetchError(Exception): - pass - - -class ResourceUnpackingError(Exception): - pass - - -class ResourceIsDirectoryError(Exception): - pass - - -class OutputIsDirectoryError(Exception): - pass - - -class OutputNotExistError(Exception): - pass - - -def setup_logging(args, base_name): - def makedirs(path): - try: - os.makedirs(path) - except OSError: - pass - - if args.log_path: - log_file_name = args.log_path - else: - log_file_name = base_name + ".log" - - args.abs_log_path = os.path.abspath(log_file_name) - makedirs(os.path.dirname(args.abs_log_path)) - logging.basicConfig(filename=args.abs_log_path, level=logging.DEBUG) - if args.verbose: - logging.getLogger().addHandler(logging.StreamHandler(sys.stderr)) - - -def is_temporary(e): - - def is_broken(e): - return isinstance(e, urllib2.HTTPError) and e.code in (410, 404) - - if is_broken(e): - return False - - if isinstance(e, (BadChecksumFetchError, IncompleteFetchError, urllib2.URLError, socket.error)): - return True - - import error - - return error.is_temporary_error(e) - - -def uniq_string_generator(size=6, chars=string.ascii_lowercase + string.digits): - return ''.join(random.choice(chars) for _ in range(size)) - - -def report_to_snowden(value): - def inner(): - body = { - 'namespace': 'ygg', - 'key': 'fetch-from-sandbox', - 'value': json.dumps(value), - } - - urllib2.urlopen( - 'https://back-snowden.qloud.yandex-team.ru/report/add', - json.dumps([body, ]), - timeout=5, - ) - - try: - inner() - except Exception as e: - logging.warning('report_to_snowden failed: %s', e) - - -def copy_stream(read, *writers, **kwargs): - chunk_size = kwargs.get('size', 1024*1024) - while True: - data = read(chunk_size) - if not data: - break - for write in writers: - write(data) - - -def md5file(fname): - res = hashlib.md5() - with open(fname, 'rb') as f: - copy_stream(f.read, res.update) - return res.hexdigest() - - -def git_like_hash_with_size(filepath): - """ - Calculate git like hash for path - """ - sha = hashlib.sha1() - - file_size = 0 - - with open(filepath, 'rb') as f: - while True: - block = f.read(2 ** 16) - - if not block: - break - - file_size += len(block) - sha.update(block) - - sha.update('\0') - sha.update(str(file_size)) - - return sha.hexdigest(), file_size - - -def size_printer(display_name, size): - sz = [0] - last_stamp = [dt.datetime.now()] - - def printer(chunk): - sz[0] += len(chunk) - now = dt.datetime.now() - if last_stamp[0] + dt.timedelta(seconds=10) < now: - if size: - print >>sys.stderr, "##status##{} - [[imp]]{:.1f}%[[rst]]".format(display_name, 100.0 * sz[0] / size if size else 0) - last_stamp[0] = now - - return printer - - -def fetch_url(url, unpack, resource_file_name, expected_md5=None, expected_sha1=None, tries=10, writers=None): - logging.info('Downloading from url %s name %s and expected md5 %s', url, resource_file_name, expected_md5) - tmp_file_name = uniq_string_generator() - - request = urllib2.Request(url, headers={'User-Agent': make_user_agent()}) - req = retry.retry_func(lambda: urllib2.urlopen(request, timeout=30), tries=tries, delay=5, backoff=1.57079) - logging.debug('Headers: %s', req.headers.headers) - expected_file_size = int(req.headers.get('Content-Length', 0)) - real_md5 = hashlib.md5() - real_sha1 = hashlib.sha1() - - with open(tmp_file_name, 'wb') as fp: - copy_stream( - req.read, - fp.write, - real_md5.update, - real_sha1.update, - size_printer(resource_file_name, expected_file_size), - *([] if writers is None else writers) - ) - - real_md5 = real_md5.hexdigest() - real_file_size = os.path.getsize(tmp_file_name) - real_sha1.update('\0') - real_sha1.update(str(real_file_size)) - real_sha1 = real_sha1.hexdigest() - - if unpack: - tmp_dir = tmp_file_name + '.dir' - os.makedirs(tmp_dir) - with tarfile.open(tmp_file_name, mode="r|gz") as tar: - tar.extractall(tmp_dir) - tmp_file_name = os.path.join(tmp_dir, resource_file_name) - real_md5 = md5file(tmp_file_name) - - logging.info('File size %s (expected %s)', real_file_size, expected_file_size or "UNKNOWN") - logging.info('File md5 %s (expected %s)', real_md5, expected_md5) - logging.info('File sha1 %s (expected %s)', real_sha1, expected_sha1) - - if expected_md5 and real_md5 != expected_md5: - report_to_snowden( - { - 'headers': req.headers.headers, - 'expected_md5': expected_md5, - 'real_md5': real_md5 - } - ) - - raise BadChecksumFetchError( - 'Downloaded {}, but expected {} for {}'.format( - real_md5, - expected_md5, - url, - ) - ) - - if expected_sha1 and real_sha1 != expected_sha1: - report_to_snowden( - { - 'headers': req.headers.headers, - 'expected_sha1': expected_sha1, - 'real_sha1': real_sha1 - } - ) - - raise BadChecksumFetchError( - 'Downloaded {}, but expected {} for {}'.format( - real_sha1, - expected_sha1, - url, - ) - ) - - if expected_file_size and expected_file_size != real_file_size: - report_to_snowden({'headers': req.headers.headers, 'file_size': real_file_size}) - - raise IncompleteFetchError( - 'Downloaded {}, but expected {} for {}'.format( - real_file_size, - expected_file_size, - url, - ) - ) - - return tmp_file_name - - -def chmod(filename, mode): - if platform.system().lower() == 'windows': - # https://docs.microsoft.com/en-us/windows/win32/fileio/hard-links-and-junctions: - # hard to reset read-only attribute for removal if there are multiple hardlinks - return - stat = os.stat(filename) - if stat.st_mode & 0o777 != mode: - try: - os.chmod(filename, mode) - except OSError: - import pwd - sys.stderr.write("{} st_mode: {} pwuid: {}\n".format(filename, stat.st_mode, pwd.getpwuid(os.stat(filename).st_uid))) - raise - - -def process(fetched_file, file_name, args, remove=True): - assert len(args.rename) <= len(args.outputs), ( - 'too few outputs to rename', args.rename, 'into', args.outputs) - - fetched_file_is_dir = os.path.isdir(fetched_file) - if fetched_file_is_dir and not args.untar_to: - raise ResourceIsDirectoryError('Resource may be directory only with untar_to option: ' + fetched_file) - - # make all read only - if fetched_file_is_dir: - for root, _, files in os.walk(fetched_file): - for filename in files: - chmod(os.path.join(root, filename), 0o444) - else: - chmod(fetched_file, 0o444) - - - if args.copy_to: - hardlink_or_copy(fetched_file, args.copy_to) - if not args.outputs: - args.outputs = [args.copy_to] - - if args.rename_to: - args.rename.append(fetched_file) - if not args.outputs: - args.outputs = [args.rename_to] - - if args.copy_to_dir: - hardlink_or_copy(fetched_file, os.path.join(args.copy_to_dir, file_name)) - - if args.untar_to: - ensure_dir(args.untar_to) - inputs = set(map(os.path.normpath, args.rename + args.outputs[len(args.rename):])) - if fetched_file_is_dir: - for member in inputs: - hardlink_or_copy( - os.path.normpath(os.path.join(fetched_file, member)), - os.path.normpath(os.path.join(args.untar_to, member))) - else: - # Extract only requested files - try: - with tarfile.open(fetched_file, mode='r:*') as tar: - members = [entry for entry in tar if os.path.normpath(os.path.join(args.untar_to, entry.name)) in inputs] - tar.extractall(args.untar_to, members=members) - except tarfile.ReadError as e: - logging.exception(e) - raise ResourceUnpackingError('File {} cannot be untared'.format(fetched_file)) - - # Forbid changes to the loaded resource data - for root, _, files in os.walk(args.untar_to): - for filename in files: - chmod(os.path.join(root, filename), 0o444) - - - for src, dst in zip(args.rename, args.outputs): - if src == 'RESOURCE': - src = fetched_file - if os.path.abspath(src) == os.path.abspath(fetched_file): - logging.info('Copying %s to %s', src, dst) - hardlink_or_copy(src, dst) - else: - logging.info('Renaming %s to %s', src, dst) - if os.path.exists(dst): - raise ResourceUnpackingError("Target file already exists ({} -> {})".format(src, dst)) - if remove: - rename_or_copy_and_remove(src, dst) - else: - hardlink_or_copy(src, dst) - - for path in args.outputs: - if not os.path.exists(path): - raise OutputNotExistError('Output does not exist: %s' % os.path.abspath(path)) - if not os.path.isfile(path): - raise OutputIsDirectoryError('Output must be a file, not a directory: %s' % os.path.abspath(path)) - if args.executable: - chmod(path, os.stat(path).st_mode | 0o111) - if os.path.abspath(path) == os.path.abspath(fetched_file): - remove = False - - if remove: - if fetched_file_is_dir: - shutil.rmtree(fetched_file) - else: - os.remove(fetched_file) diff --git a/build/scripts/fetch_from_archive.py b/build/scripts/fetch_from_archive.py deleted file mode 100644 index 57aff91b5e..0000000000 --- a/build/scripts/fetch_from_archive.py +++ /dev/null @@ -1,36 +0,0 @@ -import os -import sys -import logging -import argparse - -import fetch_from - - -def parse_args(): - parser = argparse.ArgumentParser() - fetch_from.add_common_arguments(parser) - parser.add_argument('--file-name', required=True) - parser.add_argument('--archive', required=True) - - return parser.parse_args() - -def main(args): - archive = args.archive - file_name = args.file_name.rstrip('-') - - fetch_from.process(archive, file_name, args, remove=False) - - -if __name__ == '__main__': - args = parse_args() - fetch_from.setup_logging(args, os.path.basename(__file__)) - - try: - main(args) - except Exception as e: - logging.exception(e) - print >>sys.stderr, open(args.abs_log_path).read() - sys.stderr.flush() - - import error - sys.exit(error.ExitCodes.INFRASTRUCTURE_ERROR if fetch_from.is_temporary(e) else 1) diff --git a/build/scripts/fetch_from_external.py b/build/scripts/fetch_from_external.py deleted file mode 100644 index d4ed6f4221..0000000000 --- a/build/scripts/fetch_from_external.py +++ /dev/null @@ -1,60 +0,0 @@ -import sys -import json -import os.path -import fetch_from -import argparse -import logging - - -def parse_args(): - parser = argparse.ArgumentParser() - fetch_from.add_common_arguments(parser) - parser.add_argument('--external-file', required=True) - parser.add_argument('--custom-fetcher') - parser.add_argument('--resource-file') - return parser.parse_args() - - -def main(args): - external_file = args.external_file.rstrip('.external') - if os.path.isfile(args.resource_file): - fetch_from.process(args.resource_file, os.path.basename(args.resource_file), args, False) - return - - error = None - try: - with open(args.external_file) as f: - js = json.load(f) - - if js['storage'] == 'SANDBOX': - import fetch_from_sandbox as ffsb - del args.external_file - args.resource_id = js['resource_id'] - ffsb.main(args) - elif js['storage'] == 'MDS': - import fetch_from_mds as fmds - del args.external_file - args.key = js['resource_id'] - fmds.main(args) - else: - error = 'Unsupported storage in {}'.format(external_file) - except: - logging.error('Invalid external file: {}'.format(external_file)) - raise - if error: - raise Exception(error) - - -if __name__ == '__main__': - args = parse_args() - fetch_from.setup_logging(args, os.path.basename(__file__)) - - try: - main(args) - except Exception as e: - logging.exception(e) - print >>sys.stderr, open(args.abs_log_path).read() - sys.stderr.flush() - - import error - sys.exit(error.ExitCodes.INFRASTRUCTURE_ERROR if fetch_from.is_temporary(e) else 1) diff --git a/build/scripts/fetch_from_mds.py b/build/scripts/fetch_from_mds.py deleted file mode 100644 index 5e4e656394..0000000000 --- a/build/scripts/fetch_from_mds.py +++ /dev/null @@ -1,50 +0,0 @@ -import os -import sys -import logging -import argparse - -import fetch_from - -MDS_PREFIX = "https://storage.yandex-team.ru/get-devtools/" - - -def parse_args(): - parser = argparse.ArgumentParser() - fetch_from.add_common_arguments(parser) - - parser.add_argument('--key', required=True) - - return parser.parse_args() - - -def fetch(key): - parts = key.split("/") - if len(parts) != 3: - raise ValueError("Invalid MDS key '{}'".format(key)) - - _, sha1, file_name = parts - - fetched_file = fetch_from.fetch_url(MDS_PREFIX + key, False, file_name, expected_sha1=sha1) - - return fetched_file, file_name - - -def main(args): - fetched_file, resource_file_name = fetch(args.key) - - fetch_from.process(fetched_file, resource_file_name, args) - - -if __name__ == '__main__': - args = parse_args() - fetch_from.setup_logging(args, os.path.basename(__file__)) - - try: - main(args) - except Exception as e: - logging.exception(e) - print >>sys.stderr, open(args.abs_log_path).read() - sys.stderr.flush() - - import error - sys.exit(error.ExitCodes.INFRASTRUCTURE_ERROR if fetch_from.is_temporary(e) else 1) diff --git a/build/scripts/fetch_from_npm.py b/build/scripts/fetch_from_npm.py deleted file mode 100644 index 3f87c1fefa..0000000000 --- a/build/scripts/fetch_from_npm.py +++ /dev/null @@ -1,109 +0,0 @@ -import os -import sys -import time -import logging -import argparse -import hashlib - -import sky -import fetch_from - - -NPM_BASEURL = "http://npm.yandex-team.ru/" - - -def parse_args(): - parser = argparse.ArgumentParser() - fetch_from.add_common_arguments(parser) - - parser.add_argument("--name", required=True) - parser.add_argument("--version", required=True) - parser.add_argument("--sky-id", required=True) - parser.add_argument("--integrity", required=True) - parser.add_argument("--integrity-algorithm", required=True) - - return parser.parse_args() - - -def fetch(name, version, sky_id, integrity, integrity_algorithm, file_name, tries=5): - """ - :param name: package name - :type name: str - :param version: package version - :type version: str - :param sky_id: sky id of tarball - :type sky_id: str - :param integrity: tarball integrity (hex) - :type integrity: str - :param integrity_algorithm: integrity algorithm (known for openssl) - :type integrity_algorithm: str - :param tries: tries count - :type tries: int - :return: path to fetched file - :rtype: str - """ - # if sky.is_avaliable() and 'NOTS_FETCH_FROM_HTTP_ONLY' not in os.environ: - # fetcher = lambda: sky.fetch(sky_id, file_name) - # else: - # Отключаем походы через скай - # TODO: https://st.yandex-team.ru/FBP-351 - if 'NOTS_FETCH_FROM_SKY' in os.environ and sky.is_avaliable(): - fetcher = lambda: sky.fetch(sky_id, file_name) - else: - fetcher = lambda: _fetch_via_http(name, version, integrity, integrity_algorithm, file_name) - - fetched_file = None - exc_info = None - - for i in range(0, tries): - try: - fetched_file = fetcher() - exc_info = None - break - except Exception as e: - logging.exception(e) - exc_info = exc_info or sys.exc_info() - time.sleep(i) - - if exc_info: - raise exc_info[0], exc_info[1], exc_info[2] - - return fetched_file - - -def _fetch_via_http(name, version, integrity, integrity_algorithm, file_name): - # Example: "http://npm.yandex-team.ru/@scope/name/-/name-0.0.1.tgz" for @scope/name v0.0.1. - url = NPM_BASEURL + "/".join([name, "-", "{}-{}.tgz".format(name.split("/").pop(), version)]) - - hashobj = hashlib.new(integrity_algorithm) - fetched_file = fetch_from.fetch_url(url, False, file_name, tries=1, writers=[hashobj.update]) - - if hashobj.hexdigest() != integrity: - raise fetch_from.BadChecksumFetchError("Expected {}, but got {} for {}".format( - integrity, - hashobj.hexdigest(), - file_name, - )) - - return fetched_file - - -def main(args): - file_name = os.path.basename(args.copy_to) - fetched_file = fetch(args.name, args.version, args.sky_id, args.integrity, args.integrity_algorithm, file_name) - fetch_from.process(fetched_file, file_name, args) - - -if __name__ == "__main__": - args = parse_args() - fetch_from.setup_logging(args, os.path.basename(__file__)) - - try: - main(args) - except Exception as e: - logging.exception(e) - print >>sys.stderr, open(args.abs_log_path).read() - sys.stderr.flush() - - import error - sys.exit(error.ExitCodes.INFRASTRUCTURE_ERROR if fetch_from.is_temporary(e) else 1) diff --git a/build/scripts/fetch_from_sandbox.py b/build/scripts/fetch_from_sandbox.py deleted file mode 100755 index a99542e174..0000000000 --- a/build/scripts/fetch_from_sandbox.py +++ /dev/null @@ -1,269 +0,0 @@ -import itertools -import json -import logging -import argparse -import os -import random -import subprocess -import sys -import time -import urllib2 -import uuid - -import fetch_from - - -ORIGIN_SUFFIX = '?origin=fetch-from-sandbox' -MDS_PREFIX = 'http://storage-int.mds.yandex.net/get-sandbox/' -TEMPORARY_ERROR_CODES = (429, 500, 503, 504) - - -def parse_args(): - parser = argparse.ArgumentParser() - fetch_from.add_common_arguments(parser) - parser.add_argument('--resource-id', type=int, required=True) - parser.add_argument('--custom-fetcher') - parser.add_argument('--resource-file') - return parser.parse_args() - - -class ResourceInfoError(Exception): - pass - - -class UnsupportedProtocolException(Exception): - pass - - -def _sky_path(): - return "/usr/local/bin/sky" - - -def _is_skynet_avaliable(): - if not os.path.exists(_sky_path()): - return False - try: - subprocess.check_output([_sky_path(), "--version"]) - return True - except subprocess.CalledProcessError: - return False - except OSError: - return False - - -def download_by_skynet(resource_info, file_name): - def sky_get(skynet_id, target_dir, timeout=None): - cmd_args = [_sky_path(), 'get', "-N", "Backbone", "--user", "--wait", "--dir", target_dir, skynet_id] - if timeout is not None: - cmd_args += ["--timeout", str(timeout)] - logging.info('Call skynet with args: %s', cmd_args) - stdout = subprocess.check_output(cmd_args).strip() - logging.debug('Skynet call with args %s is finished, result is %s', cmd_args, stdout) - return stdout - - if not _is_skynet_avaliable(): - raise UnsupportedProtocolException("Skynet is not available") - - skynet_id = resource_info.get("skynet_id") - if not skynet_id: - raise ValueError("Resource does not have skynet_id") - - temp_dir = os.path.abspath(fetch_from.uniq_string_generator()) - os.mkdir(temp_dir) - sky_get(skynet_id, temp_dir) - return os.path.join(temp_dir, file_name) - - -def _urlopen(url, data=None, headers=None): - n = 10 - tout = 30 - started = time.time() - reqid = uuid.uuid4() - - request = urllib2.Request(url, data=data, headers=headers or {}) - request.add_header('X-Request-Timeout', str(tout)) - request.add_header('X-Request-Id', str(reqid)) - request.add_header('User-Agent', 'fetch_from_sandbox.py') - for i in xrange(n): - retry_after = i - try: - request.add_header('X-Request-Duration', str(int(time.time() - started))) - return urllib2.urlopen(request, timeout=tout).read() - - except urllib2.HTTPError as e: - logging.warning('failed to fetch URL %s with HTTP code %d: %s', url, e.code, e) - retry_after = int(e.headers.get('Retry-After', str(retry_after))) - - if e.code not in TEMPORARY_ERROR_CODES: - raise - - except Exception as e: - logging.warning('failed to fetch URL %s: %s', url, e) - - if i + 1 == n: - raise e - - time.sleep(retry_after) - - -def _query(url): - return json.loads(_urlopen(url)) - - -_SANDBOX_BASE_URL = 'https://sandbox.yandex-team.ru/api/v1.0' - - -def get_resource_info(resource_id, touch=False, no_links=False): - url = ''.join((_SANDBOX_BASE_URL, '/resource/', str(resource_id))) - headers = {} - if touch: - headers.update({'X-Touch-Resource': '1'}) - if no_links: - headers.update({'X-No-Links': '1'}) - return _query(url) - - -def get_resource_http_links(resource_id): - url = ''.join((_SANDBOX_BASE_URL, '/resource/', str(resource_id), '/data/http')) - return [r['url'] + ORIGIN_SUFFIX for r in _query(url)] - - -def fetch_via_script(script, resource_id): - return subprocess.check_output([script, str(resource_id)]).rstrip() - - -def fetch(resource_id, custom_fetcher): - try: - resource_info = get_resource_info(resource_id, touch=True, no_links=True) - except Exception as e: - sys.stderr.write( - "Failed to fetch resource {}: {}\n".format(resource_id, str(e)) - ) - raise - - if resource_info.get('state', 'DELETED') != 'READY': - raise ResourceInfoError("Resource {} is not READY".format(resource_id)) - - logging.info('Resource %s info %s', str(resource_id), json.dumps(resource_info)) - - resource_file_name = os.path.basename(resource_info["file_name"]) - expected_md5 = resource_info.get('md5') - - proxy_link = resource_info['http']['proxy'] + ORIGIN_SUFFIX - - mds_id = resource_info.get('attributes', {}).get('mds') - mds_link = MDS_PREFIX + mds_id if mds_id else None - - def get_storage_links(): - storage_links = get_resource_http_links(resource_id) - random.shuffle(storage_links) - return storage_links - - skynet = _is_skynet_avaliable() - - if not skynet: - logging.info("Skynet is not available, will try other protocols") - - def iter_tries(): - if skynet: - yield lambda: download_by_skynet(resource_info, resource_file_name) - - if custom_fetcher: - yield lambda: fetch_via_script(custom_fetcher, resource_id) - - # Don't try too hard here: we will get back to proxy later on - yield lambda: fetch_from.fetch_url(proxy_link, False, resource_file_name, expected_md5, tries=2) - for x in get_storage_links(): - # Don't spend too much time connecting single host - yield lambda: fetch_from.fetch_url(x, False, resource_file_name, expected_md5, tries=1) - if mds_link is not None: - # Don't try too hard here: we will get back to MDS later on - yield lambda: fetch_from.fetch_url(mds_link, True, resource_file_name, expected_md5, tries=2) - yield lambda: fetch_from.fetch_url(proxy_link, False, resource_file_name, expected_md5) - if mds_link is not None: - yield lambda: fetch_from.fetch_url(mds_link, True, resource_file_name, expected_md5) - - if resource_info.get('attributes', {}).get('ttl') != 'inf': - sys.stderr.write('WARNING: resource {} ttl is not "inf".\n'.format(resource_id)) - - exc_info = None - for i, action in enumerate(itertools.islice(iter_tries(), 0, 10)): - try: - fetched_file = action() - break - except UnsupportedProtocolException: - pass - except subprocess.CalledProcessError as e: - logging.warning('failed to fetch resource %s with subprocess: %s', resource_id, e) - time.sleep(i) - except urllib2.HTTPError as e: - logging.warning('failed to fetch resource %s with HTTP code %d: %s', resource_id, e.code, e) - if e.code not in TEMPORARY_ERROR_CODES: - exc_info = exc_info or sys.exc_info() - time.sleep(i) - except Exception as e: - logging.exception(e) - exc_info = exc_info or sys.exc_info() - time.sleep(i) - else: - if exc_info: - raise exc_info[0], exc_info[1], exc_info[2] - else: - raise Exception("No available protocol and/or server to fetch resource") - - return fetched_file, resource_info['file_name'] - - -def _get_resource_info_from_file(resource_file): - if resource_file is None or not os.path.exists(resource_file): - return None - - RESOURCE_INFO_JSON = "resource_info.json" - RESOURCE_CONTENT_FILE_NAME = "resource" - - resource_dir, resource_file = os.path.split(resource_file) - if resource_file != RESOURCE_CONTENT_FILE_NAME: - return None - - resource_json = os.path.join(resource_dir, RESOURCE_INFO_JSON) - if not os.path.isfile(resource_json): - return None - - try: - with open(resource_json, 'r') as j: - resource_info = json.load(j) - resource_info['file_name'] # check consistency - return resource_info - except: - logging.debug('Invalid %s in %s', RESOURCE_INFO_JSON, resource_dir) - - return None - - -def main(args): - custom_fetcher = os.environ.get('YA_CUSTOM_FETCHER') - - resource_info = _get_resource_info_from_file(args.resource_file) - if resource_info: - fetched_file = args.resource_file - file_name = resource_info['file_name'] - else: - # This code should be merged to ya and removed. - fetched_file, file_name = fetch(args.resource_id, custom_fetcher) - - fetch_from.process(fetched_file, file_name, args, remove=not custom_fetcher and not resource_info) - - -if __name__ == '__main__': - args = parse_args() - fetch_from.setup_logging(args, os.path.basename(__file__)) - - try: - main(args) - except Exception as e: - logging.exception(e) - print >>sys.stderr, open(args.abs_log_path).read() - sys.stderr.flush() - - import error - sys.exit(error.ExitCodes.INFRASTRUCTURE_ERROR if fetch_from.is_temporary(e) else 1) diff --git a/build/scripts/fetch_resource.py b/build/scripts/fetch_resource.py deleted file mode 100644 index d5af311e5d..0000000000 --- a/build/scripts/fetch_resource.py +++ /dev/null @@ -1,43 +0,0 @@ -import urllib2 -import argparse -import xmlrpclib - - -def parse_args(): - parser = argparse.ArgumentParser() - parser.add_argument('-r', '--resource-id', type=int, required=True) - parser.add_argument('-o', '--output', required=True) - return parser.parse_args() - - -def fetch(url, retries=4, timeout=5): - for i in xrange(retries): - try: - return urllib2.urlopen(url, timeout=timeout).read() - - except Exception: - if i + 1 < retries: - continue - - else: - raise - - -def fetch_resource(id_): - urls = xmlrpclib.ServerProxy("https://sandbox.yandex-team.ru/sandbox/xmlrpc").get_resource_http_links(id_) - - for u in urls: - try: - return fetch(u) - - except Exception: - continue - - raise Exception('Cannot fetch resource {}'.format(id_)) - - -if __name__ == '__main__': - args = parse_args() - - with open(args.output, 'wb') as f: - f.write(fetch_resource(int(args.resource_id))) diff --git a/build/scripts/filter_zip.py b/build/scripts/filter_zip.py deleted file mode 100644 index b2121b9c9e..0000000000 --- a/build/scripts/filter_zip.py +++ /dev/null @@ -1,71 +0,0 @@ -import argparse -import os -import re -import uuid -import zipfile - - -def pattern_to_regexp(p): - return re.compile( - '^' - + re.escape(p) - .replace(r'\*\*\/', '[_DIR_]') - .replace(r'\*', '[_FILE_]') - .replace('[_DIR_]', '(.*/)?') - .replace('[_FILE_]', '([^/]*)') - + '$' - ) - - -def is_deathman(positive_filter, negative_filter, candidate): - remove = positive_filter - for pf in positive_filter: - if pf.match(candidate): - remove = False - break - if not negative_filter or remove: - return remove - for nf in negative_filter: - if nf.match(candidate): - remove = True - break - return remove - - -def just_do_it(): - parser = argparse.ArgumentParser() - parser.add_argument('--positive', action='append', default=[]) - parser.add_argument('--negative', action='append', default=[]) - parser.add_argument('--file', action='store', required=True) - args = parser.parse_args() - if not args.positive and not args.negative: - return - pos = [pattern_to_regexp(i) for i in args.positive] - neg = [pattern_to_regexp(i) for i in args.negative] - temp_dirname = None - for _ in range(10): - candidate = '__unpacked_{}__'.format(uuid.uuid4()) - if not os.path.exists(candidate): - temp_dirname = candidate - os.makedirs(temp_dirname) - if not temp_dirname: - raise Exception("Can't generate name for temp dir") - - with zipfile.ZipFile(args.file, 'r') as zip_ref: - zip_ref.extractall(temp_dirname) - - for root, _, files in os.walk(temp_dirname): - for f in files: - candidate = os.path.join(root, f).replace('\\', '/') - if is_deathman(pos, neg, os.path.relpath(candidate, temp_dirname)): - os.remove(candidate) - - with zipfile.ZipFile(args.file, 'w') as zip_ref: - for root, _, files in os.walk(temp_dirname): - for f in files: - realname = os.path.join(root, f) - zip_ref.write(realname, os.path.sep.join(os.path.normpath(realname).split(os.path.sep, 2)[1:])) - - -if __name__ == '__main__': - just_do_it() diff --git a/build/scripts/find_and_tar.py b/build/scripts/find_and_tar.py deleted file mode 100644 index f251623c68..0000000000 --- a/build/scripts/find_and_tar.py +++ /dev/null @@ -1,22 +0,0 @@ -import os -import sys -import tarfile - - -def find_gcno(dirname, tail): - for cur, _dirs, files in os.walk(dirname): - for f in files: - if f.endswith(tail): - yield os.path.relpath(os.path.join(cur, f)) - - -def main(args): - output = args[0] - tail = args[1] if len(args) > 1 else '' - with tarfile.open(output, 'w:') as tf: - for f in find_gcno(os.getcwd(), tail): - tf.add(f) - - -if __name__ == '__main__': - main(sys.argv[1:]) diff --git a/build/scripts/find_time_trace.py b/build/scripts/find_time_trace.py deleted file mode 100644 index 954d203caa..0000000000 --- a/build/scripts/find_time_trace.py +++ /dev/null @@ -1,17 +0,0 @@ -import os -import sys - -# /scripts/find_time_trace.py <object_file> <destination> -# clang generates `-ftime-trace` output file path based on main output file path - - -def main(): - assert len(sys.argv) == 3 - obj_path = sys.argv[1] - trace_path = sys.argv[2] - orig_trace_path = obj_path.rpartition('.o')[0] + '.json' - os.rename(orig_trace_path, trace_path) - - -if __name__ == '__main__': - main() diff --git a/build/scripts/fix_java_command_file_cp.py b/build/scripts/fix_java_command_file_cp.py deleted file mode 100644 index fc87048c32..0000000000 --- a/build/scripts/fix_java_command_file_cp.py +++ /dev/null @@ -1,34 +0,0 @@ -import sys -import os -import argparse -import subprocess -import platform - - -def fix_files(args): - args = args[:] - parser = argparse.ArgumentParser() - parser.add_argument('--build-root', default=None) - args, tail = parser.parse_known_args(args) - for idx, arg in list(enumerate(tail)): - if arg.startswith('@') and os.path.isfile(arg[1:]): - with open(arg[1:]) as f: - fixed = [i.strip() for i in f] - if args.build_root: - fixed = [os.path.join(args.build_root, i) for ln in fixed for i in ln.split(os.path.pathsep)] - fixed = os.pathsep.join([i.strip() for i in fixed]) - fixed_name = list(os.path.splitext(arg)) - fixed_name[0] += '_fixed' - fixed_name = ''.join(fixed_name) - with open(fixed_name[1:], 'w') as f: - f.write(fixed) - tail[idx:idx + 1] = [fixed_name] - return tail - - -if __name__ == '__main__': - args = fix_files(sys.argv[1:]) - if platform.system() == 'Windows': - sys.exit(subprocess.Popen(args).wait()) - else: - os.execv(args[0], args) diff --git a/build/scripts/fix_msvc_output.py b/build/scripts/fix_msvc_output.py deleted file mode 100644 index 183a442e1f..0000000000 --- a/build/scripts/fix_msvc_output.py +++ /dev/null @@ -1,43 +0,0 @@ -import subprocess -import sys - -import process_command_files as pcf -import process_whole_archive_option as pwa - - -def out2err(cmd): - return subprocess.Popen(cmd, stdout=sys.stderr).wait() - - -def decoding_needed(strval): - if sys.version_info >= (3, 0, 0): - return isinstance(strval, bytes) - else: - return False - - -def out2err_cut_first_line(cmd): - p = subprocess.Popen(cmd, stdout=subprocess.PIPE) - first_line = True - while True: - line = p.stdout.readline() - line = line.decode('utf-8') if decoding_needed(line) else line - if not line: - break - if first_line: - sys.stdout.write(line) - first_line = False - else: - sys.stderr.write(line) - return p.wait() - - -if __name__ == '__main__': - mode = sys.argv[1] - args, wa_peers, wa_libs = pwa.get_whole_archive_peers_and_libs(pcf.skip_markers(sys.argv[2:])) - cmd = pwa.ProcessWholeArchiveOption('WINDOWS', wa_peers, wa_libs).construct_cmd(args) - run = out2err - if mode in ('cl', 'ml'): - # First line of cl.exe and ml64.exe stdout is useless: it prints input file - run = out2err_cut_first_line - sys.exit(run(cmd)) diff --git a/build/scripts/fs_tools.py b/build/scripts/fs_tools.py deleted file mode 100644 index dec4c349c8..0000000000 --- a/build/scripts/fs_tools.py +++ /dev/null @@ -1,104 +0,0 @@ -from __future__ import print_function - -import os -import platform -import sys -import shutil -import errno - -import process_command_files as pcf - - -def link_or_copy(src, dst): - try: - if platform.system().lower() == 'windows': - shutil.copy(src, dst) - else: - os.link(src, dst) - except OSError as e: - if e.errno == errno.EEXIST: - print('link_or_copy: destination file already exists: {}'.format(dst), file=sys.stderr) - if e.errno == errno.ENOENT: - print('link_or_copy: source file doesn\'t exists: {}'.format(src), file=sys.stderr) - raise - - -if __name__ == '__main__': - mode = sys.argv[1] - args = pcf.get_args(sys.argv[2:]) - - if mode == 'copy': - shutil.copy(args[0], args[1]) - elif mode == 'copy_tree_no_link': - dst = args[1] - shutil.copytree(args[0], dst, ignore=lambda dirname, names: [n for n in names if os.path.islink(os.path.join(dirname, n))]) - elif mode == 'copy_files': - src = args[0] - dst = args[1] - files = open(args[2]).read().strip().split() - for f in files: - s = os.path.join(src, f) - d = os.path.join(dst, f) - if os.path.exists(d): - continue - try: - os.makedirs(os.path.dirname(d)) - except OSError: - pass - shutil.copy(s, d) - elif mode == 'copy_all_files': - src = args[0] - dst = args[1] - for root, _, files in os.walk(src): - for f in files: - if os.path.islink(os.path.join(root, f)): - continue - file_dst = os.path.join(dst, os.path.relpath(root, src), f) - if os.path.exists(file_dst): - continue - try: - os.makedirs(os.path.dirname(file_dst)) - except OSError: - pass - shutil.copy(os.path.join(root, f), file_dst) - elif mode == 'rename_if_exists': - if os.path.exists(args[0]): - shutil.move(args[0], args[1]) - elif mode == 'rename': - targetdir = os.path.dirname(args[1]) - if targetdir and not os.path.exists(targetdir): - os.makedirs(os.path.dirname(args[1])) - shutil.move(args[0], args[1]) - elif mode == 'remove': - for f in args: - try: - if os.path.isfile(f) or os.path.islink(f): - os.remove(f) - else: - shutil.rmtree(f) - except OSError: - pass - elif mode == 'link_or_copy': - link_or_copy(args[0], args[1]) - elif mode == 'link_or_copy_to_dir': - assert len(args) > 1 - start = 0 - if args[0] == '--no-check': - if args == 2: - sys.exit() - start = 1 - dst = args[-1] - for src in args[start:-1]: - link_or_copy(src, os.path.join(dst, os.path.basename(src))) - elif mode == 'cat': - with open(args[0], 'w') as dst: - for input_name in args[1:]: - with open(input_name) as src: - dst.write(src.read()) - elif mode == 'md': - try: - os.makedirs(args[0]) - except OSError: - pass - else: - raise Exception('unsupported tool %s' % mode) diff --git a/build/scripts/gen_aar_gradle_script.py b/build/scripts/gen_aar_gradle_script.py deleted file mode 100644 index e59b1db7b5..0000000000 --- a/build/scripts/gen_aar_gradle_script.py +++ /dev/null @@ -1,371 +0,0 @@ -import argparse -import os -import tarfile - -FLAT_DIRS_REPO_TEMPLATE='flatDir {{ dirs {dirs} }}\n' -MAVEN_REPO_TEMPLATE='maven {{ url "{repo}" }}\n' -KEYSTORE_TEMLATE='signingConfigs {{ debug {{ storeFile file("{keystore}") }} }}\n' - -ENABLE_JAVADOC = 'tasks["bundle${suffix}Aar"].dependsOn packageJavadocTask' -DO_NOT_STRIP = '''\ - packagingOptions { - doNotStrip "*/arm64-v8a/*.so" - doNotStrip "*/armeabi-v7a/*.so" - doNotStrip "*/x86_64/*.so" - doNotStrip "*/x86/*.so" - } -''' - -AAR_TEMPLATE = """\ -ext.jniLibsDirs = [ - {jni_libs_dirs} -] - -ext.resDirs = [ - {res_dirs} -] - -ext.assetsDirs = [ - {assets_dirs} -] - -ext.javaDirs = [ - {java_dirs} -] - -def aidlDirs = [ - {aidl_dirs} -] - -ext.bundles = [ - {bundles} -] - -ext.androidArs = [ - {aars} -] - -ext.compileOnlyAndroidArs = [ - {compile_only_aars} -] - -def minVersion = 21 -def compileVersion = 30 -def targetVersion = 30 -def buildVersion = '30.0.3' - -import com.android.build.gradle.LibraryPlugin -import java.nio.file.Files -import java.nio.file.Paths -import java.util.regex.Matcher -import java.util.regex.Pattern -import java.util.zip.ZipFile - - -apply plugin: 'com.github.dcendents.android-maven' - -buildDir = "$projectDir/build" - -if (!ext.has("packageSuffix")) - ext.packageSuffix = "" - -buildscript {{ -// repositories {{ -// jcenter() -// mavenCentral() -// }} - - repositories {{ - {maven_repos} - }} - - dependencies {{ - classpath 'com.android.tools.build:gradle:4.0.2' - classpath 'com.github.dcendents:android-maven-gradle-plugin:1.5' - }} -}} - -apply plugin: LibraryPlugin - -repositories {{ -// flatDir {{ -// dirs System.env.PKG_ROOT + '/bundle' -// }} -// maven {{ -// url "http://maven.google.com/" -// }} -// maven {{ -// url "http://artifactory.yandex.net/artifactory/public/" -// }} - - {flat_dirs_repo} - - {maven_repos} -}} - -android {{ - {keystore} - - compileSdkVersion compileVersion - buildToolsVersion buildVersion - - defaultConfig {{ - minSdkVersion minVersion - targetSdkVersion targetVersion - consumerProguardFiles '{proguard_rules}' - }} - - sourceSets {{ - main {{ - manifest.srcFile '{manifest}' - jniLibs.srcDirs = jniLibsDirs - res.srcDirs = resDirs - assets.srcDirs = assetsDirs - java.srcDirs = javaDirs - aidl.srcDirs = aidlDirs - }} - // We don't use this feature, so we set it to nonexisting directory - androidTest.setRoot('bundle/tests') - }} - - {do_not_strip} - - dependencies {{ - for (bundle in bundles) - compile("$bundle") {{ - transitive = true - }} - for (bundle in androidArs) - compile(bundle) {{ - transitive = true - }} - for (bundle in compileOnlyAndroidArs) - compileOnly(bundle) - }} - - android.libraryVariants.all {{ variant -> - def suffix = variant.buildType.name.capitalize() - - def sourcesJarTask = project.tasks.create(name: "sourcesJar${{suffix}}", type: Jar) {{ - classifier = 'sources' - from android.sourceSets.main.java.srcDirs - include '**/*.java' - eachFile {{ fcd -> - def segments = fcd.relativePath.segments - if (segments[0] == 'impl') {{ - fcd.relativePath = new RelativePath(true, segments.drop(1)) - }} - }} - includeEmptyDirs = false - }} - - def manifestFile = android.sourceSets.main.manifest.srcFile - def manifestXml = new XmlParser().parse(manifestFile) - - def packageName = manifestXml['@package'] - def groupName = packageName.tokenize('.')[0..-2].join('.') - - def androidNs = new groovy.xml.Namespace("http://schemas.android.com/apk/res/android") - def packageVersion = manifestXml.attributes()[androidNs.versionName] - - def writePomTask = project.tasks.create(name: "writePom${{suffix}}") {{ - pom {{ - project {{ - groupId groupName - version packageVersion - packaging 'aar' - }} - }}.writeTo("$buildDir/${{rootProject.name}}$packageSuffix-pom.xml") - }} - - tasks["bundle${{suffix}}Aar"].dependsOn sourcesJarTask - tasks["bundle${{suffix}}Aar"].dependsOn writePomTask - }} - - android.libraryVariants.all {{ variant -> - def capitalizedVariantName = variant.name.capitalize() - def suffix = variant.buildType.name.capitalize() - - def javadocTask = project.tasks.create(name: "generate${{capitalizedVariantName}}Javadoc", type: Javadoc) {{ - group = "Javadoc" - description "Generates Javadoc for $capitalizedVariantName" - - title = "Yandex documentation" - - source = android.sourceSets.main.java.srcDirs - include "**/*/yandex/*/**" - // TODO: remove this when we support internal doc exclusion in IDL - // https://st.yandex-team.ru/MAPSMOBCORE-11364 - exclude "**/internal/**" - - ext.androidJar = "${{android.sdkDirectory.path}}/platforms/${{android.compileSdkVersion}}/android.jar" - classpath = - files(android.getBootClasspath().join(File.pathSeparator)) + - configurations.compile + - files(ext.androidJar) + - files(variant.javaCompile.outputs.files) - - destinationDir = file("$buildDir/${{rootProject.name}}-javadoc/$capitalizedVariantName/") - - options.encoding = "UTF-8" - - failOnError false - - afterEvaluate {{ - def dependencyTree = project.configurations.compile.getAsFileTree() - def aar_set = dependencyTree.matching{{include "**/*.aar"}}.getFiles() - def jar_tree = dependencyTree.matching{{include "**/*.jar"}} - - classpath += files(android.libraryVariants.collect {{ libraryVariant -> - libraryVariant.javaCompileProvider.get().classpath.files - }}) - - aar_set.each{{ aar -> - def outputPath = "$buildDir/tmp/aarJar/${{aar.name.replace('.aar', '.jar')}}" - classpath += files(outputPath) - - dependsOn task(name: "extract_${{aar.getAbsolutePath().replace(File.separatorChar, '_' as char)}}-${{capitalizedVariantName}}").doLast {{ - extractClassesJar(aar, outputPath) - }} - }} - }} - }} - - def packageJavadocTask = project.tasks.create(name: "package${{capitalizedVariantName}}Javadoc", type: Tar) {{ - description "Makes an archive from Javadoc output" - from "${{buildDir}}/${{rootProject.name}}-javadoc/$capitalizedVariantName/" - archiveFileName = "${{rootProject.name}}-javadoc.tar.gz" - destinationDirectory = new File("${{buildDir}}") - dependsOn javadocTask - }} - - {enable_javadoc} - }} - -}} - -private def extractClassesJar(aarPath, outputPath) {{ - if (!aarPath.exists()) {{ - throw new GradleException("AAR $aarPath not found") - }} - - def zip = new ZipFile(aarPath) - zip.entries().each {{ - if (it.name == "classes.jar") {{ - def path = Paths.get(outputPath) - if (!Files.exists(path)) {{ - Files.createDirectories(path.getParent()) - Files.copy(zip.getInputStream(it), path) - }} - }} - }} - zip.close() -}} - -""" - - -def gen_build_script(args): - - def wrap(items): - return ',\n '.join('"{}"'.format(x) for x in items) - - bundles = [] - bundles_dirs = set(args.flat_repos) - for bundle in args.bundles: - dir_name, base_name = os.path.split(bundle) - assert(len(dir_name) > 0 and len(base_name) > 0) - name, ext = os.path.splitext(base_name) - assert(len(name) > 0 and ext == '.aar') - bundles_dirs.add(dir_name) - bundles.append('com.yandex:{}@aar'.format(name)) - - if len(bundles_dirs) > 0: - flat_dirs_repo = FLAT_DIRS_REPO_TEMPLATE.format(dirs=wrap(bundles_dirs)) - else: - flat_dirs_repo = '' - - maven_repos = ''.join(MAVEN_REPO_TEMPLATE.format(repo=repo) for repo in args.maven_repos) - - if args.keystore: - keystore = KEYSTORE_TEMLATE.format(keystore=args.keystore) - else: - keystore = '' - - if args.generate_doc: - enable_javadoc = ENABLE_JAVADOC - else: - enable_javadoc = '' - - if args.do_not_strip: - do_not_strip = DO_NOT_STRIP - else: - do_not_strip = '' - - return AAR_TEMPLATE.format( - aars=wrap(args.aars), - compile_only_aars=wrap(args.compile_only_aars), - aidl_dirs=wrap(args.aidl_dirs), - assets_dirs=wrap(args.assets_dirs), - bundles=wrap(bundles), - do_not_strip=do_not_strip, - enable_javadoc=enable_javadoc, - flat_dirs_repo=flat_dirs_repo, - java_dirs=wrap(args.java_dirs), - jni_libs_dirs=wrap(args.jni_libs_dirs), - keystore=keystore, - manifest=args.manifest, - maven_repos=maven_repos, - proguard_rules=args.proguard_rules, - res_dirs=wrap(args.res_dirs), - ) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument('--aars', nargs='*', default=[]) - parser.add_argument('--compile-only-aars', nargs='*', default=[]) - parser.add_argument('--aidl-dirs', nargs='*', default=[]) - parser.add_argument('--assets-dirs', nargs='*', default=[]) - parser.add_argument('--bundle-name', nargs='?', default='default-bundle-name') - parser.add_argument('--bundles', nargs='*', default=[]) - parser.add_argument('--do-not-strip', action='store_true') - parser.add_argument('--flat-repos', nargs='*', default=[]) - parser.add_argument('--generate-doc', action='store_true') - parser.add_argument('--java-dirs', nargs='*', default=[]) - parser.add_argument('--jni-libs-dirs', nargs='*', default=[]) - parser.add_argument('--keystore', default=None) - parser.add_argument('--manifest', required=True) - parser.add_argument('--maven-repos', nargs='*', default=[]) - parser.add_argument('--output-dir', required=True) - parser.add_argument('--peers', nargs='*', default=[]) - parser.add_argument('--proguard-rules', nargs='?', default=None) - parser.add_argument('--res-dirs', nargs='*', default=[]) - args = parser.parse_args() - - if args.proguard_rules is None: - args.proguard_rules = os.path.join(args.output_dir, 'proguard-rules.txt') - with open(args.proguard_rules, 'w') as f: - pass - - for index, jsrc in enumerate(filter(lambda x: x.endswith('.jsrc'), args.peers)): - jsrc_dir = os.path.join(args.output_dir, 'jsrc_{}'.format(str(index))) - os.makedirs(jsrc_dir) - with tarfile.open(jsrc, 'r') as tar: - tar.extractall(path=jsrc_dir) - args.java_dirs.append(jsrc_dir) - - args.build_gradle = os.path.join(args.output_dir, 'build.gradle') - args.settings_gradle = os.path.join(args.output_dir, 'settings.gradle') - args.gradle_properties = os.path.join(args.output_dir, 'gradle.properties') - - content = gen_build_script(args) - with open(args.build_gradle, 'w') as f: - f.write(content) - - with open(args.gradle_properties, 'w') as f: - f.write('android.useAndroidX=true') - - if args.bundle_name: - with open(args.settings_gradle, 'w') as f: - f.write('rootProject.name = "{}"'.format(args.bundle_name)) diff --git a/build/scripts/gen_java_codenav_entry.py b/build/scripts/gen_java_codenav_entry.py deleted file mode 100644 index ff0a5c737d..0000000000 --- a/build/scripts/gen_java_codenav_entry.py +++ /dev/null @@ -1,57 +0,0 @@ -import argparse -import datetime -import os -import subprocess -import sys -import tarfile - - -def extract_kindexes(kindexes): - for kindex in kindexes: - with tarfile.TarFile(kindex) as tf: - for fname in [i for i in tf.getnames() if i.endswith('.kzip')]: - tf.extract(fname) - yield fname - - -def just_do_it(java, kythe, kythe_to_proto, out_name, binding_only, kindexes): - temp_out_name = out_name + '.temp' - kindex_inputs = list(extract_kindexes(kindexes)) - open(temp_out_name, 'w').close() - start = datetime.datetime.now() - for kindex in kindex_inputs: - print >> sys.stderr, '[INFO] Processing:', kindex - indexer_start = datetime.datetime.now() - p = subprocess.Popen([java, '-jar', os.path.join(kythe, 'indexers/java_indexer.jar'), kindex], stdout=subprocess.PIPE) - indexer_out, _ = p.communicate() - print >> sys.stderr, '[INFO] Indexer execution time:', (datetime.datetime.now() - indexer_start).total_seconds(), 'seconds' - if p.returncode: - raise Exception('java_indexer failed with exit code {}'.format(p.returncode)) - dedup_start = datetime.datetime.now() - p = subprocess.Popen([os.path.join(kythe, 'tools/dedup_stream')], stdin=subprocess.PIPE, stdout=subprocess.PIPE) - dedup_out, _ = p.communicate(indexer_out) - print >> sys.stderr, '[INFO] Dedup execution time:', (datetime.datetime.now() - dedup_start).total_seconds(), 'seconds' - if p.returncode: - raise Exception('dedup_stream failed with exit code {}'.format(p.returncode)) - entrystream_start = datetime.datetime.now() - p = subprocess.Popen([os.path.join(kythe, 'tools/entrystream'), '--write_json'], stdin=subprocess.PIPE, stdout=open(temp_out_name, 'a')) - p.communicate(dedup_out) - if p.returncode: - raise Exception('entrystream failed with exit code {}'.format(p.returncode)) - print >> sys.stderr, '[INFO] Entrystream execution time:', (datetime.datetime.now() - entrystream_start).total_seconds(), 'seconds' - preprocess_start = datetime.datetime.now() - subprocess.check_call([kythe_to_proto, '--preprocess-entry', '--entries', temp_out_name, '--out', out_name] + (['--only-binding-data'] if binding_only else [])) - print >> sys.stderr, '[INFO] Preprocessing execution time:', (datetime.datetime.now() - preprocess_start).total_seconds(), 'seconds' - print >> sys.stderr, '[INFO] Total execution time:', (datetime.datetime.now() - start).total_seconds(), 'seconds' - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument("--java", help="java path") - parser.add_argument("--kythe", help="kythe path") - parser.add_argument("--kythe-to-proto", help="kythe_to_proto tool path") - parser.add_argument("--out-name", help="entries json out name") - parser.add_argument("--binding-only", action="store_true", default=False, help="filter only binding data") - parser.add_argument("kindexes", nargs='*') - args = parser.parse_args() - just_do_it(args.java, args.kythe, args.kythe_to_proto, args.out_name, args.binding_only, args.kindexes) diff --git a/build/scripts/gen_java_codenav_protobuf.py b/build/scripts/gen_java_codenav_protobuf.py deleted file mode 100644 index aee8cfe6c3..0000000000 --- a/build/scripts/gen_java_codenav_protobuf.py +++ /dev/null @@ -1,22 +0,0 @@ -import argparse -import os - - -def just_do_it(kythe_to_proto, entries, out_name, build_file, source_root): - with open(build_file) as f: - classpath = os.pathsep.join([line.strip() for line in f]) - os.execv( - kythe_to_proto, - [kythe_to_proto, '--sources-rel-root', 'fake_arcadia_root', '--entries', entries, '--out', out_name, '--classpath', classpath, '--arcadia-root', source_root] - ) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument("--kythe-to-proto", help="kythe_to_proto tool path") - parser.add_argument("--entries", help="entries json path") - parser.add_argument("--out-name", help="protbuf out name") - parser.add_argument("--build-file", help="build file( containing classpath )") - parser.add_argument("--source-root", help="source root") - args = parser.parse_args() - just_do_it(args.kythe_to_proto, args.entries, args.out_name, args.build_file, args.source_root) diff --git a/build/scripts/gen_mx_table.py b/build/scripts/gen_mx_table.py deleted file mode 100644 index 187c21c539..0000000000 --- a/build/scripts/gen_mx_table.py +++ /dev/null @@ -1,75 +0,0 @@ -import sys - -tmpl = """ -#include "yabs_mx_calc_table.h" - -#include <kernel/matrixnet/mn_sse.h> - -#include <library/cpp/archive/yarchive.h> - -#include <util/memory/blob.h> -#include <util/generic/hash.h> -#include <util/generic/ptr.h> -#include <util/generic/singleton.h> - -using namespace NMatrixnet; - -extern "C" { - extern const unsigned char MxFormulas[]; - extern const ui32 MxFormulasSize; -} - -namespace { - struct TFml: public TBlob, public TMnSseInfo { - inline TFml(const TBlob& b) - : TBlob(b) - , TMnSseInfo(Data(), Size()) - { - } - }; - - struct TFormulas: public THashMap<size_t, TAutoPtr<TFml>> { - inline TFormulas() { - TBlob b = TBlob::NoCopy(MxFormulas, MxFormulasSize); - TArchiveReader ar(b); - %s - } - - inline const TMnSseInfo& at(size_t n) const noexcept { - return *find(n)->second; - } - }; - - %s - - static func_descr_t yabs_funcs[] = { - %s - }; -} - -yabs_mx_calc_table_t yabs_mx_calc_table = {YABS_MX_CALC_VERSION, 10000, 0, yabs_funcs}; -""" - -if __name__ == '__main__': - init = [] - body = [] - defs = {} - - for i in sys.argv[1:]: - name = i.replace('.', '_') - num = long(name.split('_')[1]) - - init.append('(*this)[%s] = new TFml(ar.ObjectBlobByKey("%s"));' % (num, '/' + i)) - - f1 = 'static void yabs_%s(size_t count, const float** args, double* res) {Singleton<TFormulas>()->at(%s).DoCalcRelevs(args, res, count);}' % (name, num) - f2 = 'static size_t yabs_%s_factor_count() {return Singleton<TFormulas>()->at(%s).MaxFactorIndex() + 1;}' % (name, num) - - body.append(f1) - body.append(f2) - - d1 = 'yabs_%s' % name - d2 = 'yabs_%s_factor_count' % name - - defs[num] = '{%s, %s}' % (d1, d2) - - print tmpl % ('\n'.join(init), '\n\n'.join(body), ',\n'.join((defs.get(i, '{nullptr, nullptr}') for i in range(0, 10000)))) diff --git a/build/scripts/gen_py3_reg.py b/build/scripts/gen_py3_reg.py deleted file mode 100644 index 149c094898..0000000000 --- a/build/scripts/gen_py3_reg.py +++ /dev/null @@ -1,34 +0,0 @@ -import sys - -template = ''' -struct PyObject; -extern "C" int PyImport_AppendInittab(const char* name, PyObject* (*initfunc)()); -extern "C" PyObject* {1}(); - -namespace { - struct TRegistrar { - inline TRegistrar() { - // TODO Collect all modules and call PyImport_ExtendInittab once - PyImport_AppendInittab("{0}", {1}); - } - } REG; -} -''' - - -def mangle(name): - if '.' not in name: - return name - return ''.join('{}{}'.format(len(s), s) for s in name.split('.')) - -if __name__ == '__main__': - if len(sys.argv) != 3: - print >>sys.stderr, 'Usage: <path/to/gen_py_reg.py> <python_module_name> <output_file>' - print >>sys.stderr, 'Passed: ' + ' '.join(sys.argv) - sys.exit(1) - - with open(sys.argv[2], 'w') as f: - modname = sys.argv[1] - initname = 'PyInit_' + mangle(modname) - code = template.replace('{0}', modname).replace('{1}', initname) - f.write(code) diff --git a/build/scripts/gen_py_protos.py b/build/scripts/gen_py_protos.py deleted file mode 100644 index 08397472f9..0000000000 --- a/build/scripts/gen_py_protos.py +++ /dev/null @@ -1,67 +0,0 @@ -import os -from os import path -import shutil -import subprocess -import sys -import tempfile -import argparse -import re - - -OUT_DIR_ARG = '--python_out=' - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("--suffixes", nargs="*", default=[]) - parser.add_argument("protoc_args", nargs=argparse.REMAINDER) - script_args = parser.parse_args() - - args = script_args.protoc_args - - if args[0] == "--": - args = args[1:] - - out_dir_orig = None - out_dir_temp = None - plugin_out_dirs_orig = {} - for i in range(len(args)): - if args[i].startswith(OUT_DIR_ARG): - assert not out_dir_orig, 'Duplicate "{0}" param'.format(OUT_DIR_ARG) - out_dir_orig = args[i][len(OUT_DIR_ARG):] - out_dir_temp = tempfile.mkdtemp(dir=out_dir_orig) - args[i] = OUT_DIR_ARG + out_dir_temp - continue - - match = re.match(r"^(--(\w+)_out=).*", args[i]) - if match: - plugin_out_dir_arg = match.group(1) - plugin = match.group(2) - assert plugin not in plugin_out_dirs_orig, 'Duplicate "{0}" param'.format(plugin_out_dir_arg) - plugin_out_dirs_orig[plugin] = args[i][len(plugin_out_dir_arg):] - assert plugin_out_dirs_orig[plugin] == out_dir_orig, 'Params "{0}" and "{1}" expected to have the same value'.format(OUT_DIR_ARG, plugin_out_dir_arg) - args[i] = plugin_out_dir_arg + out_dir_temp - - assert out_dir_temp, 'Param "{0}" not found'.format(OUT_DIR_ARG) - - retcode = subprocess.call(args) - assert not retcode, 'Protoc failed for command {}'.format(' '.join(args)) - - for root_temp, dirs, files in os.walk(out_dir_temp): - sub_dir = path.relpath(root_temp, out_dir_temp) - root_orig = path.join(out_dir_orig, sub_dir) - for d in dirs: - d_orig = path.join(root_orig, d) - if not path.exists(d_orig): - os.mkdir(d_orig) - for f in files: - f_orig = f - for suf in script_args.suffixes: - if f.endswith(suf): - f_orig = f[:-len(suf)] + "__int__" + suf - break - os.rename(path.join(root_temp, f), path.join(root_orig, f_orig)) - shutil.rmtree(out_dir_temp) - - -if __name__ == '__main__': - main() diff --git a/build/scripts/gen_py_reg.py b/build/scripts/gen_py_reg.py deleted file mode 100644 index 1560135ae8..0000000000 --- a/build/scripts/gen_py_reg.py +++ /dev/null @@ -1,32 +0,0 @@ -import sys - -template = ''' -extern "C" void PyImport_AppendInittab(const char* name, void (*fn)(void)); -extern "C" void {1}(); - -namespace { - struct TRegistrar { - inline TRegistrar() { - PyImport_AppendInittab("{0}", {1}); - } - } REG; -} -''' - - -def mangle(name): - if '.' not in name: - return name - return ''.join('{}{}'.format(len(s), s) for s in name.split('.')) - -if __name__ == '__main__': - if len(sys.argv) != 3: - print >>sys.stderr, 'Usage: <path/to/gen_py_reg.py> <python_module_name> <output_file>' - print >>sys.stderr, 'Passed: ' + ' '.join(sys.argv) - sys.exit(1) - - with open(sys.argv[2], 'w') as f: - modname = sys.argv[1] - initname = 'init' + mangle(modname) - code = template.replace('{0}', modname).replace('{1}', initname) - f.write(code) diff --git a/build/scripts/gen_swiftc_output_map.py b/build/scripts/gen_swiftc_output_map.py deleted file mode 100644 index 01ce85f256..0000000000 --- a/build/scripts/gen_swiftc_output_map.py +++ /dev/null @@ -1,15 +0,0 @@ -import json -import sys - - -def just_do_it(args): - source_root, build_root, out_file, srcs = args[0], args[1], args[2], args[3:] - assert(len(srcs)) - result_obj = {} - for src in srcs: - result_obj[src] = {'object': src.replace(source_root, build_root) + '.o'} - with open(out_file, 'w') as of: - of.write(json.dumps(result_obj)) - -if __name__ == '__main__': - just_do_it(sys.argv[1:]) diff --git a/build/scripts/gen_tasklet_reg.py b/build/scripts/gen_tasklet_reg.py deleted file mode 100644 index 0f7f66ad51..0000000000 --- a/build/scripts/gen_tasklet_reg.py +++ /dev/null @@ -1,51 +0,0 @@ -import argparse - -TEMPLATE = '''\ -{includes}\ -#include <tasklet/v1/runtime/lib/{language}_wrapper.h> -#include <tasklet/v1/runtime/lib/registry.h> - -static const NTasklet::TRegHelper REG( - "{name}", - new NTasklet::{wrapper} -); -''' - -WRAPPER = { - 'cpp': 'TCppWrapper<{impl}>()', - 'js': 'TJsWrapper("{impl}")', - 'go': 'TGoWrapper("{impl}")', - 'py': 'TPythonWrapper("{impl}")', - 'java': 'TJavaWrapper("{impl}", "{py_wrapper}")', -} - - -def parse_args(): - parser = argparse.ArgumentParser() - parser.add_argument('name') - parser.add_argument('output') - parser.add_argument('-l', '--lang', choices=WRAPPER, required=True) - parser.add_argument('-i', '--impl', required=True) - parser.add_argument('-w', '--wrapper', required=False) - parser.add_argument('includes', nargs='*') - - return parser.parse_args() - - -if __name__ == '__main__': - args = parse_args() - - includes = ''.join( - '#include <{}>\n'.format(include) - for include in args.includes - ) - - code = TEMPLATE.format( - includes=includes, - language=args.lang, - name=args.name, - wrapper=WRAPPER[args.lang].format(impl=args.impl, py_wrapper=args.wrapper), - ) - - with open(args.output, 'w') as f: - f.write(code) diff --git a/build/scripts/gen_test_apk_gradle_script.py b/build/scripts/gen_test_apk_gradle_script.py deleted file mode 100644 index b3a4f89e46..0000000000 --- a/build/scripts/gen_test_apk_gradle_script.py +++ /dev/null @@ -1,193 +0,0 @@ -import argparse -import os -import tarfile -import xml.etree.ElementTree as etree - -FLAT_DIRS_REPO_TEMPLATE='flatDir {{ dirs {dirs} }}\n' -MAVEN_REPO_TEMPLATE='maven {{ url "{repo}" }}\n' -KEYSTORE_TEMLATE='signingConfigs {{ debug {{ storeFile file("{keystore}") }} }}\n' - -TEST_APK_TEMPLATE = """\ -ext.jniLibsDirs = [ - {jni_libs_dirs} -] -ext.resDirs = [ - {res_dirs} -] -ext.javaDirs = [ - {java_dirs} -] -ext.bundles = [ - {bundles} -] - -buildscript {{ -// repositories {{ -// jcenter() -// }} - - repositories {{ - {maven_repos} - }} - - dependencies {{ - classpath 'com.android.tools.build:gradle:3.5.3' - }} -}} - -apply plugin: 'com.android.application' - -repositories {{ -// maven {{ -// url "http://maven.google.com/" -// }} -// maven {{ -// url "http://artifactory.yandex.net/artifactory/public/" -// }} -// flatDir {{ -// dirs System.env.PKG_ROOT + '/bundle' -// }} - - {flat_dirs_repo} - - {maven_repos} -}} - -dependencies {{ - for (bundle in bundles) {{ - compile("$bundle") - }} -}} - -android {{ - {keystore} - - compileSdkVersion 30 - buildToolsVersion "30.0.3" - - - defaultConfig {{ - minSdkVersion 21 - targetSdkVersion 30 - applicationId "{app_id}" - }} - - sourceSets {{ - main {{ - manifest.srcFile 'Manifest.xml' - jniLibs.srcDirs = jniLibsDirs - res.srcDirs = resDirs - java.srcDirs = javaDirs - }} - }} - - applicationVariants.all {{ variant -> - variant.outputs.each {{ output -> - def fileName = "$projectDir/output/{app_id}.apk" - output.outputFileName = new File(output.outputFile.parent, fileName).getName() - }} - }} - - dependencies {{ - implementation 'com.google.android.gms:play-services-location:21.0.1' - implementation 'com.google.android.gms:play-services-gcm:17.0.0' - implementation 'com.evernote:android-job:1.2.6' - implementation 'androidx.annotation:annotation:1.1.0' - implementation 'androidx.core:core:1.1.0' - }} -}} -""" - - -def create_native_properties(output_dir, library_name): - native_properties_file = os.path.join(output_dir, 'native_library_name.xml') - resources = etree.Element('resources') - name = etree.SubElement(resources, 'item', dict(name='native_library_name', type='string')) - name.text = library_name - etree.ElementTree(resources).write(native_properties_file, xml_declaration=True, encoding='utf-8') - - -def gen_build_script(args): - def wrap(items): - return ',\n '.join('"{}"'.format(x) for x in items) - - bundles = [] - bundles_dirs = set(args.flat_repos) - for bundle in args.bundles: - dir_name, base_name = os.path.split(bundle) - assert(len(dir_name) > 0 and len(base_name) > 0) - name, ext = os.path.splitext(base_name) - assert(len(name) > 0 and ext == '.aar') - bundles_dirs.add(dir_name) - bundles.append('com.yandex:{}@aar'.format(name)) - - if len(bundles_dirs) > 0: - flat_dirs_repo = FLAT_DIRS_REPO_TEMPLATE.format(dirs=wrap(bundles_dirs)) - else: - flat_dirs_repo = '' - - maven_repos = ''.join(MAVEN_REPO_TEMPLATE.format(repo=repo) for repo in args.maven_repos) - - if args.keystore: - keystore = KEYSTORE_TEMLATE.format(keystore=args.keystore) - else: - keystore = '' - - return TEST_APK_TEMPLATE.format( - app_id=args.app_id, - jni_libs_dirs=wrap(args.jni_libs_dirs), - res_dirs=wrap(args.res_dirs), - java_dirs=wrap(args.java_dirs), - maven_repos=maven_repos, - bundles=wrap(bundles), - flat_dirs_repo=flat_dirs_repo, - keystore=keystore, - ) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument('--aars', nargs='*', default=[]) - parser.add_argument('--app-id', required=True) - parser.add_argument('--assets-dirs', nargs='*', default=[]) - parser.add_argument('--bundles', nargs='*', default=[]) - parser.add_argument('--bundle-name', nargs='?', default=None) - parser.add_argument('--java-dirs', nargs='*', default=[]) - parser.add_argument('--jni-libs-dirs', nargs='*', default=[]) - parser.add_argument('--library-name', required=True) - parser.add_argument('--manifest', required=True) - parser.add_argument('--flat-repos', nargs='*', default=[]) - parser.add_argument('--maven-repos', nargs='*', default=[]) - parser.add_argument('--output-dir', required=True) - parser.add_argument('--peers', nargs='*', default=[]) - parser.add_argument('--keystore', default=None) - parser.add_argument('--res-dirs', nargs='*', default=[]) - args = parser.parse_args() - - for index, jsrc in enumerate(filter(lambda x: x.endswith('.jsrc'), args.peers)): - jsrc_dir = os.path.join(args.output_dir, 'jsrc_{}'.format(str(index))) - os.makedirs(jsrc_dir) - with tarfile.open(jsrc, 'r') as tar: - tar.extractall(path=jsrc_dir) - args.java_dirs.append(jsrc_dir) - - args.build_gradle = os.path.join(args.output_dir, 'build.gradle') - args.settings_gradle = os.path.join(args.output_dir, 'settings.gradle') - args.gradle_properties = os.path.join(args.output_dir, 'gradle.properties') - - content = gen_build_script(args) - with open(args.build_gradle, 'w') as f: - f.write(content) - - with open(args.gradle_properties, 'w') as f: - f.write('''android.enableJetifier=true - android.useAndroidX=true - org.gradle.jvmargs=-Xmx8192m -XX:MaxPermSize=512m''') - - if args.bundle_name: - with open(args.settings_gradle, 'w') as f: - f.write('rootProject.name = "{}"'.format(args.bundle_name)) - - values_dir = os.path.join(args.output_dir, 'res', 'values') - os.makedirs(values_dir) - create_native_properties(values_dir, args.library_name) diff --git a/build/scripts/gen_ub.py b/build/scripts/gen_ub.py deleted file mode 100644 index ad79cda926..0000000000 --- a/build/scripts/gen_ub.py +++ /dev/null @@ -1,86 +0,0 @@ -import argparse -import os -import tarfile -import contextlib -import hashlib -import base64 -import io - - -stub = """#!/usr/bin/env python - -info = {info} -data = "{data}" - -import platform -import os -import sys -import tarfile -import contextlib -import io -import base64 - - -def current_platform(): - arch = platform.machine().upper() - - if arch == 'AMD64': - arch = 'X86_64' - - platf = platform.system().upper() - - if platf.startswith('WIN'): - platf = 'WIN' - - return (platf + '-' + arch).lower() - - -def extract_file(fname): - with contextlib.closing(tarfile.open(fileobj=io.BytesIO(base64.b64decode(data)))) as f: - return f.extractfile(fname).read() - - -fname = info[current_platform()] -my_path = os.path.realpath(os.path.abspath(__file__)) -tmp_path = my_path + '.tmp' - -with open(tmp_path, 'wb') as f: - f.write(extract_file(fname)) - -os.rename(tmp_path, my_path) -os.chmod(my_path, 0775) -os.execv(sys.argv[0], sys.argv) -""" - - -def gen_ub(output, data): - info = {} - binary = io.BytesIO() - - with contextlib.closing(tarfile.open(mode='w:bz2', fileobj=binary, dereference=True)) as f: - for pl, path in data: - fname = os.path.basename(path) - pl = pl.split('-') - pl = pl[1] + '-' + pl[2] - info[pl] = fname - f.add(path, arcname=fname) - - binary = binary.getvalue() - info['md5'] = hashlib.md5(binary).hexdigest() - - with open(output, 'w') as f: - f.write(stub.format(info=info, data=base64.b64encode(binary))) - - os.chmod(output, 0775) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - - parser.add_argument('--path', action='append') - parser.add_argument('--platform', action='append') - parser.add_argument('--output', action='store') - - args = parser.parse_args() - - gen_ub(args.output, zip(args.platform, args.path)) diff --git a/build/scripts/gen_yql_python_udf.py b/build/scripts/gen_yql_python_udf.py deleted file mode 100644 index 13b5898117..0000000000 --- a/build/scripts/gen_yql_python_udf.py +++ /dev/null @@ -1,55 +0,0 @@ -import sys - -TEMPLATE=""" -#include <yql/udfs/common/python/python_udf/python_udf.h> - -#include <ydb/library/yql/public/udf/udf_registrator.h> - -#if @WITH_LIBRA@ -#include <yql/udfs/quality/libra/module/module.h> -#endif - -using namespace NKikimr::NUdf; - -#ifdef BUILD_UDF - -#if @WITH_LIBRA@ -LIBRA_MODULE(TLibraModule, "Libra@MODULE_NAME@"); -#endif - -extern "C" UDF_API void Register(IRegistrator& registrator, ui32 flags) { - RegisterYqlPythonUdf(registrator, flags, TStringBuf("@MODULE_NAME@"), TStringBuf("@PACKAGE_NAME@"), EPythonFlavor::@FLAVOR@); -#if @WITH_LIBRA@ - RegisterHelper<TLibraModule>(registrator); -#endif -} - -extern "C" UDF_API ui32 AbiVersion() { - return CurrentAbiVersion(); -} - -extern "C" UDF_API void SetBackTraceCallback(TBackTraceCallback callback) { - SetBackTraceCallbackImpl(callback); -} - -#endif -""" - - -def main(): - assert len(sys.argv) == 6 - flavor, module_name, package_name, path, libra_flag = sys.argv[1:] - with open(path, 'w') as f: - f.write( - TEMPLATE - .strip() - .replace('@MODULE_NAME@', module_name) - .replace('@PACKAGE_NAME@', package_name) - .replace('@FLAVOR@', flavor) - .replace('@WITH_LIBRA@', libra_flag) - ) - f.write('\n') - - -if __name__ == "__main__": - main() diff --git a/build/scripts/generate_mf.py b/build/scripts/generate_mf.py deleted file mode 100644 index a44a969980..0000000000 --- a/build/scripts/generate_mf.py +++ /dev/null @@ -1,113 +0,0 @@ -import json -import logging -import optparse -import os -import sys -import io - -import process_command_files as pcf - -class BadMfError(Exception): - pass - - -class GplNotAllowed(Exception): - pass - - -def process_quotes(s): - for quote_char in '\'"': - if s.startswith(quote_char) and s.endswith(quote_char): - return s[1:-1] - return s - - -def parse_args(): - args = pcf.get_args(sys.argv[1:]) - lics, peers, free_args, credits = [], [], [], [] - current_list = free_args - for a in args: - if a == '-Ya,lics': - current_list = lics - elif a == '-Ya,peers': - current_list = peers - elif a == '-Ya,credits': - current_list = credits - elif a and a.startswith('-'): - current_list = free_args - current_list.append(a) - else: - current_list.append(a) - - parser = optparse.OptionParser() - parser.add_option('--build-root') - parser.add_option('--module-name') - parser.add_option('-o', '--output') - parser.add_option('-c', '--credits-output') - parser.add_option('-t', '--type') - opts, _ = parser.parse_args(free_args) - return lics, peers, credits, opts, - - -def generate_header(meta): - return '-' * 20 + meta.get('path', 'Unknown module') + '-' * 20 - - -def generate_mf(): - lics, peers, credits, options = parse_args() - - meta = { - 'module_name': options.module_name, - 'path': os.path.dirname(options.output), - 'licenses': lics, - 'dependencies': [], - 'license_texts': '' - } - - build_root = options.build_root - file_name = os.path.join(build_root, options.output) - - if options.type != 'LIBRARY': - for rel_filename in peers: - with open(os.path.join(build_root, rel_filename + '.mf')) as peer_file: - peer_meta = json.load(peer_file) - meta['dependencies'].append(peer_meta) - - if credits: - union_texts = [] - for texts_file in credits: - with open(process_quotes(texts_file)) as f: - union_texts.append(f.read()) - meta['license_texts'] = '\n\n'.join(union_texts) - - if options.credits_output: - final_credits = [] - if meta['license_texts']: - final_credits.append(generate_header(meta) + '\n' + meta['license_texts']) - for peer in peers: - candidate = os.path.join(build_root, peer + '.mf') - with open(candidate) as src: - data = json.loads(src.read()) - texts = data.get('license_texts') - if texts: - candidate_text = generate_header(data) + '\n' + texts - if isinstance(candidate_text, unicode): - candidate_text = candidate_text.encode('utf-8') - final_credits.append(candidate_text) - - with io.open(options.credits_output, 'w', encoding='utf-8') as f: - data = '\n\n'.join(final_credits) - if isinstance(data, str): - data = data.decode('utf-8') - f.write(data) - - with open(file_name, 'w') as mf_file: - json.dump(meta, mf_file, indent=4) - - -if __name__ == '__main__': - try: - generate_mf() - except Exception as e: - sys.stderr.write(str(e) + '\n') - sys.exit(1) diff --git a/build/scripts/generate_pom.py b/build/scripts/generate_pom.py deleted file mode 100644 index d91bce6249..0000000000 --- a/build/scripts/generate_pom.py +++ /dev/null @@ -1,336 +0,0 @@ -from __future__ import print_function - -import sys -import xml.etree.ElementTree as et -import argparse -import os -import json -import base64 -import re - - -DEFAULT_YANDEX_GROUP_ID = 'ru.yandex' -DEFAULT_NAMESPACE = 'http://maven.apache.org/POM/4.0.0' -XSI_NAMESPACE = 'http://www.w3.org/2001/XMLSchema-instance' -SCHEMA_LOCATION = 'http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd' -MODEL_VERSION = '4.0.0' - -MAVEN_PLUGIN_GROUP_ID = 'org.apache.maven.plugins' -MAVEN_PLUGIN_ARTIFACT_ID = 'maven-compiler-plugin' -MAVEN_PLUGIN_VERSION = '3.3' -JAVA_LANGUAGE_LEVEL = '1.8' - -MAVEN_BUILD_HELPER_GROUP_ID = 'org.codehaus.mojo' -MAVEN_BUILD_HELPER_ARTIFACT_ID = 'build-helper-maven-plugin' -MAVEN_BUILD_HELPER_VERSION = '1.9.1' - -MAVEN_EXEC_GROUP_ID = 'org.codehaus.mojo' -MAVEN_EXEC_ARTIFACT_ID = 'exec-maven-plugin' -MAVEN_EXEC_VERSION = '1.5.0' - -MAVEN_SUREFIRE_GROUP_ID = 'org.apache.maven.plugins' -MAVEN_SUREFIRE_ARTIFACT_ID = 'maven-surefire-plugin' -MAVEN_SUREFIRE_VERSION = '2.12.2' - - -def target_from_contrib(target_path): - return target_path.startswith('contrib') - - -def split_artifacts(s): - m = re.match('^([^:]*:[^:]*:[^:]*:[^:]*)(.*)$', s) - if not m or not m.groups(): - return [] - if not m.groups()[1].startswith('::'): - return [m.groups()[0]] - return [m.groups()[0]] + m.groups()[1].split('::')[1:] - - -def parse_coord_file(target_coords): - deps = set() - excludes = set() - target = None - - with open(target_coords, 'r') as f: - for ln in f: - ln = ln.strip('\n') - if ln.startswith('D='): - if target is None: - target = ln[2:] - else: - group, artifact = ln[2:].split(':')[0:2] - deps.add('{}:{}'.format(group, artifact)) - elif ln.startswith('E='): - group, artifact = ln[2:].split(':')[0:2] - excludes.add('{}:{}'.format(group, artifact)) - return target, deps, excludes - - -def applied_excludes(deps, excludes): - return sorted(list(deps & excludes)) - - -def build_pom_and_export_to_maven(**kwargs): - target_path = kwargs.get('target_path') - target = kwargs.get('target') - pom_path = kwargs.get('pom_path') - source_dirs = kwargs.get('source_dirs') - output_dir = kwargs.get('output_dir') - final_name = kwargs.get('final_name') - packaging = kwargs.get('packaging') - test_target_dependencies = kwargs.get('test_target_dependencies') - test_target_dependencies_exclude = kwargs.get('test_target_dependencies_exclude') - modules_path = kwargs.get('modules_path') - base64_prop_vars = kwargs.get('properties') - prop_vars = kwargs.get('property') - external_jars = kwargs.get('external_jars') - resources = kwargs.get('resources') - run_java_programs = [json.loads(base64.b64decode(i)) for i in kwargs.get('run_java_programs')] - test_source_dirs = kwargs.get('test_source_dirs') - test_resource_dirs = kwargs.get('test_resource_dirs') - - from_coord_files = kwargs.get('from_coord_files') - deps_coords = kwargs.get('deps_coords') - target_coords = kwargs.get('target_coords') - if from_coord_files: - target, _, all_excludes = parse_coord_file(target_coords) - # TODO: ymake java -> jbuild java values format conversion must be removed - target = ':'.join(target.split(':')[:3]) - target_dependencies = [] - for dep in deps_coords: - dep_coord, dep_deps, _ = parse_coord_file(dep) - excluded = applied_excludes(dep_deps, all_excludes) - if len(excluded) > 0: - target_dependencies.append('{}::{}'.format(dep_coord, '::'.join(excluded))) - else: - target_dependencies.append(dep_coord) - else: - target_dependencies = kwargs.get('target_dependencies') - - if kwargs.get('vcs_info') is not None: - with open(kwargs.get('vcs_info'), 'r') as vcs_json: - vcs_revision = json.load(vcs_json).get('ARCADIA_SOURCE_LAST_CHANGE') - target = target.format(vcs_revision=vcs_revision) - target_dependencies = [dep.format(vcs_revision=vcs_revision) for dep in target_dependencies] - - modules = [] - - def _indent(elem, level=0): - ind = "\n" + level * " " - if len(elem): - if not elem.text or not elem.text.strip(): - elem.text = ind + " " - if not elem.tail or not elem.tail.strip(): - elem.tail = ind - for elem in elem: - _indent(elem, level + 1) - if not elem.tail or not elem.tail.strip(): - elem.tail = ind - else: - if level and (not elem.tail or not elem.tail.strip()): - elem.tail = ind - - project = et.Element( - '{}{}{}project'.format('{', DEFAULT_NAMESPACE, '}'), - attrib={'{}{}{}schemaLocation'.format('{', XSI_NAMESPACE, '}'): SCHEMA_LOCATION} - ) - - group_id, artifact_id, version = target.split(':') - - et.SubElement(project, 'modelVersion').text = MODEL_VERSION - et.SubElement(project, 'groupId').text = group_id - et.SubElement(project, 'artifactId').text = artifact_id - et.SubElement(project, 'version').text = version - et.SubElement(project, 'packaging').text = packaging - - properties = et.SubElement(project, 'properties') - et.SubElement(properties, 'project.build.sourceEncoding').text = 'UTF-8' - - if base64_prop_vars: - for property, value in json.loads(base64.b64decode(base64_prop_vars)).items(): - et.SubElement(properties, property).text = value - for rawprop in prop_vars: - property, sep, value = rawprop.partition('=') - if sep != '=': - print("Can't find propertyr name and property value in {}. No '=' symbol found".format(rawprop)) - sys.exit(1) - et.SubElement(properties, property).text = value - - if modules_path: - with open(modules_path) as f: - modules = [i.strip() for i in f if i.strip()] - - if modules: - modules_el = et.SubElement(project, 'modules') - for module in modules: - et.SubElement(modules_el, 'module').text = module - - build = et.SubElement(project, 'build') - if source_dirs: - et.SubElement(build, 'sourceDirectory').text = source_dirs[0] - source_dirs = source_dirs[1:] - if test_source_dirs: - et.SubElement(build, 'testSourceDirectory').text = test_source_dirs[0] - test_source_dirs = test_source_dirs[1:] - if output_dir: - et.SubElement(build, 'outputDirectory').text = output_dir - if final_name: - et.SubElement(build, 'finalName').text = final_name - if resources: - resource_element = et.SubElement(et.SubElement(build, 'resources'), 'resource') - et.SubElement(resource_element, 'directory').text = '${basedir}' - includes = et.SubElement(resource_element, 'includes') - for resource in resources: - et.SubElement(includes, 'include').text = resource - if test_resource_dirs: - test_resource_element = et.SubElement(build, 'testResources') - for test_resource_dir in test_resource_dirs: - et.SubElement(et.SubElement(test_resource_element, 'testResource'), 'directory').text = '${basedir}' + (('/' + test_resource_dir) if test_resource_dir != '.' else '') - - plugins = et.SubElement(build, 'plugins') - - if packaging != 'pom': - maven_plugin = et.SubElement(plugins, 'plugin') - et.SubElement(maven_plugin, 'groupId').text = MAVEN_PLUGIN_GROUP_ID - et.SubElement(maven_plugin, 'artifactId').text = MAVEN_PLUGIN_ARTIFACT_ID - et.SubElement(maven_plugin, 'version').text = MAVEN_PLUGIN_VERSION - configuration = et.SubElement(maven_plugin, 'configuration') - et.SubElement(configuration, 'source').text = JAVA_LANGUAGE_LEVEL - et.SubElement(configuration, 'target').text = JAVA_LANGUAGE_LEVEL - - if source_dirs or external_jars or test_source_dirs: - build_helper_plugin = et.SubElement(plugins, 'plugin') - et.SubElement(build_helper_plugin, 'groupId').text = MAVEN_BUILD_HELPER_GROUP_ID - et.SubElement(build_helper_plugin, 'artifactId').text = MAVEN_BUILD_HELPER_ARTIFACT_ID - et.SubElement(build_helper_plugin, 'version').text = MAVEN_BUILD_HELPER_VERSION - executions = et.SubElement(build_helper_plugin, 'executions') - if source_dirs: - execution = et.SubElement(executions, 'execution') - et.SubElement(execution, 'id').text = 'add-source' - et.SubElement(execution, 'phase').text = 'generate-sources' - et.SubElement(et.SubElement(execution, 'goals'), 'goal').text = 'add-source' - sources = et.SubElement(et.SubElement(execution, 'configuration'), 'sources') - for source_dir in source_dirs: - et.SubElement(sources, 'source').text = source_dir - if external_jars: - execution = et.SubElement(executions, 'execution') - et.SubElement(execution, 'id').text = 'attach-artifacts' - et.SubElement(execution, 'phase').text = 'generate-sources' - et.SubElement(et.SubElement(execution, 'goals'), 'goal').text = 'attach-artifact' - artifacts = et.SubElement(et.SubElement(execution, 'configuration'), 'artifacts') - for external_jar in external_jars: - external_artifact = et.SubElement(artifacts, 'artifact') - et.SubElement(external_artifact, 'file').text = '${basedir}/' + external_jar - et.SubElement(external_artifact, 'type').text = 'jar' - if test_source_dirs: - execution = et.SubElement(executions, 'execution') - et.SubElement(execution, 'id').text = 'add-test-source' - et.SubElement(execution, 'phase').text = 'generate-test-sources' - et.SubElement(et.SubElement(execution, 'goals'), 'goal').text = 'add-test-source' - sources = et.SubElement(et.SubElement(execution, 'configuration'), 'sources') - for source_dir in source_dirs: - et.SubElement(sources, 'source').text = source_dir - - if run_java_programs: - exec_plugin = et.SubElement(plugins, 'plugin') - et.SubElement(exec_plugin, 'groupId').text = MAVEN_EXEC_GROUP_ID - et.SubElement(exec_plugin, 'artifactId').text = MAVEN_EXEC_ARTIFACT_ID - et.SubElement(exec_plugin, 'version').text = MAVEN_EXEC_VERSION - jp_dependencies = et.SubElement(exec_plugin, 'dependencies') - executions = et.SubElement(exec_plugin, 'executions') - for java_program in run_java_programs: - execution = et.SubElement(executions, 'execution') - et.SubElement(execution, 'phase').text = 'generate-sources' - et.SubElement(et.SubElement(execution, 'goals'), 'goal').text = 'java' - jp_configuration = et.SubElement(execution, 'configuration') - main_cls, args = None, [] - for word in java_program['cmd']: - if not main_cls and not word.startswith('-'): - main_cls = word - else: - args.append(word) - et.SubElement(jp_configuration, 'mainClass').text = main_cls - et.SubElement(jp_configuration, 'includePluginDependencies').text = 'true' - et.SubElement(jp_configuration, 'includeProjectDependencies').text = 'false' - if args: - jp_arguments = et.SubElement(jp_configuration, 'arguments') - for arg in args: - et.SubElement(jp_arguments, 'argument').text = arg - if java_program['deps']: - for jp_dep in java_program['deps']: - jp_dependency = et.SubElement(jp_dependencies, 'dependency') - jp_g, jp_a, jp_v = jp_dep.split(':') - et.SubElement(jp_dependency, 'groupId').text = jp_g - et.SubElement(jp_dependency, 'artifactId').text = jp_a - et.SubElement(jp_dependency, 'version').text = jp_v - et.SubElement(jp_dependency, 'type').text = 'jar' - - if target_dependencies + test_target_dependencies: - dependencies = et.SubElement(project, 'dependencies') - for target_dependency in target_dependencies + test_target_dependencies: - dependency = et.SubElement(dependencies, 'dependency') - dependency_info = split_artifacts(target_dependency) - - group_id, artifact_id, version, classifier = dependency_info[0].split(':') - - et.SubElement(dependency, 'groupId').text = group_id - et.SubElement(dependency, 'artifactId').text = artifact_id - et.SubElement(dependency, 'version').text = version - if classifier: - et.SubElement(dependency, 'classifier').text = classifier - if target_dependency in test_target_dependencies: - et.SubElement(dependency, 'scope').text = 'test' - - if len(dependency_info) > 1: - exclusions = et.SubElement(dependency, 'exclusions') - for exclude in dependency_info[1:]: - group_id, artifact_id = exclude.split(':') - exclusion_el = et.SubElement(exclusions, 'exclusion') - et.SubElement(exclusion_el, 'groupId').text = group_id - et.SubElement(exclusion_el, 'artifactId').text = artifact_id - - if test_target_dependencies_exclude: - surefire_plugin = et.SubElement(plugins, 'plugin') - et.SubElement(surefire_plugin, 'groupId').text = MAVEN_SUREFIRE_GROUP_ID - et.SubElement(surefire_plugin, 'artifactId').text = MAVEN_SUREFIRE_ARTIFACT_ID - et.SubElement(surefire_plugin, 'version').text = MAVEN_SUREFIRE_VERSION - classpath_excludes = et.SubElement(et.SubElement(surefire_plugin, 'configuration'), 'classpathDependencyExcludes') - for classpath_exclude in test_target_dependencies_exclude: - et.SubElement(classpath_excludes, 'classpathDependencyExclude').text = classpath_exclude - - et.register_namespace('', DEFAULT_NAMESPACE) - et.register_namespace('xsi', XSI_NAMESPACE) - - _indent(project) - - et.ElementTree(project).write(pom_path) - sys.stderr.write("[MAVEN EXPORT] Generated {} file for target {}\n".format(os.path.basename(pom_path), target_path)) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument('--from-coord-files', action='store_true') - parser.add_argument('--deps-coords', action='append', default=[]) - parser.add_argument('--target-coords', action='store') - parser.add_argument('--target-path', action='store', default='') - parser.add_argument('--target', action='store') - parser.add_argument('--pom-path', action='store') - parser.add_argument('--source-dirs', action='append', default=[]) - parser.add_argument('--external-jars', action='append', default=[]) - parser.add_argument('--resources', action='append', default=[]) - parser.add_argument('--run-java-programs', action='append', default=[]) - parser.add_argument('--output-dir') - parser.add_argument('--final-name') - parser.add_argument('--packaging', default='jar') - parser.add_argument('--target-dependencies', action='append', default=[]) - parser.add_argument('--test-target-dependencies', action='append', default=[]) - parser.add_argument('--test-target-dependencies-exclude', action='append', default=[]) - parser.add_argument('--modules-path', action='store') - parser.add_argument('--properties') - parser.add_argument('--property', action='append', default=[]) - parser.add_argument('--test-source-dirs', action='append', default=[]) - parser.add_argument('--test-resource-dirs', action='append', default=[]) - parser.add_argument('--vcs-info', action='store', default=None) - args = parser.parse_args() - - build_pom_and_export_to_maven(**vars(args)) diff --git a/build/scripts/go_fake_include/go_asm.h b/build/scripts/go_fake_include/go_asm.h deleted file mode 100644 index e69de29bb2..0000000000 --- a/build/scripts/go_fake_include/go_asm.h +++ /dev/null diff --git a/build/scripts/go_proto_wrapper.py b/build/scripts/go_proto_wrapper.py deleted file mode 100644 index 4c530ee780..0000000000 --- a/build/scripts/go_proto_wrapper.py +++ /dev/null @@ -1,82 +0,0 @@ -from __future__ import absolute_import, unicode_literals -import os -import re -import shutil -import subprocess -import sys -import tempfile -from six.moves import range - - -OUT_DIR_FLAG_PATTERN = re.compile(r'^(--go(([-_]\w+))*_out=)') - - -def move_tree(src_root, dst_root): - for root, _, files in os.walk(src_root): - rel_dir = os.path.relpath(root, src_root) - dst_dir = os.path.join(dst_root, rel_dir) - if not os.path.exists(dst_dir): - os.mkdir(dst_dir) - for file in files: - os.rename(os.path.join(root, file), os.path.join(dst_dir, file)) - - -def main(arcadia_prefix, contrib_prefix, proto_namespace, args): - out_dir_orig = None - out_dir_temp = None - for i in range(len(args)): - m = re.match(OUT_DIR_FLAG_PATTERN, args[i]) - if m: - out_dir_flag = m.group(1) - index = max(len(out_dir_flag), args[i].rfind(':')+1) - out_dir = args[i][index:] - if out_dir_orig: - assert out_dir_orig == out_dir, 'Output directories do not match: [{}] and [{}]'.format(out_dir_orig, out_dir) - else: - out_dir_orig = out_dir - out_dir_temp = tempfile.mkdtemp(dir=out_dir_orig) - args[i] = (args[i][:index] + out_dir_temp).replace('|', ',') - assert out_dir_temp is not None, 'Output directory is not specified' - - try: - subprocess.check_output(args, stdin=None, stderr=subprocess.STDOUT) - except subprocess.CalledProcessError as e: - sys.stderr.write('{} returned non-zero exit code {}.\n{}\n'.format(' '.join(e.cmd), e.returncode, e.output.decode('utf-8'))) - return e.returncode - - # All Arcadia GO projects should have 'a.yandex-team.ru/' namespace prefix. - # If the namespace doesn't start with 'a.yandex-team.ru/' prefix then this - # project is from vendor directory under the root of Arcadia. - out_dir_src = os.path.normpath(os.path.join(out_dir_temp, arcadia_prefix, proto_namespace)) - out_dir_dst = out_dir_orig - is_from_contrib = False - if not os.path.isdir(out_dir_src): - is_from_contrib = True - out_dir_src = out_dir_temp - out_dir_dst = os.path.join(out_dir_orig, contrib_prefix) - - if not os.path.exists(out_dir_src) or is_from_contrib: - protos = [x for x in args if x.endswith('.proto')] - if not is_from_contrib or not all(x.startswith(contrib_prefix) for x in protos): - proto_list = [] - option_re = re.compile(r'^\s*option\s+go_package\s*=\s*') - for arg in protos: - with open(arg, 'r') as f: - if not any([re.match(option_re, line) for line in f]): - proto_list.append(arg) - if proto_list: - sys.stderr.write( - '\nError: Option go_package is not specified in the following proto files: {}\n' - '\nNOTE! You can find detailed description of how to properly set go_package ' - 'option here https://wiki.yandex-team.ru/devrules/Go/#protobufigrpc'.format(', '.join(proto_list))) - return 1 - - move_tree(out_dir_src, out_dir_dst) - - shutil.rmtree(out_dir_temp) - - return 0 - - -if __name__ == '__main__': - sys.exit(main(os.path.normpath(sys.argv[1]), os.path.normpath(sys.argv[2]), os.path.normpath(sys.argv[3]), sys.argv[4:])) diff --git a/build/scripts/go_tool.py b/build/scripts/go_tool.py deleted file mode 100644 index 86f84c56b3..0000000000 --- a/build/scripts/go_tool.py +++ /dev/null @@ -1,868 +0,0 @@ -from __future__ import absolute_import, unicode_literals -import argparse -import copy -import json -import os -import re -import shutil -import subprocess -import sys -import tarfile -import tempfile -import threading -import six -from functools import reduce - -import process_command_files as pcf -import process_whole_archive_option as pwa - -arc_project_prefix = 'a.yandex-team.ru/' -std_lib_prefix = 'contrib/go/_std_1.18/src/' -vendor_prefix = 'vendor/' -vet_info_ext = '.vet.out' -vet_report_ext = '.vet.txt' - -FIXED_CGO1_SUFFIX='.fixed.cgo1.go' - -COMPILE_OPTIMIZATION_FLAGS=('-N',) - - -def get_trimpath_args(args): - return ['-trimpath', args.trimpath] if args.trimpath else [] - - -def preprocess_cgo1(src_path, dst_path, source_root): - with open(src_path, 'r') as f: - content = f.read() - content = content.replace('__ARCADIA_SOURCE_ROOT_PREFIX__', source_root) - with open(dst_path, 'w') as f: - f.write(content) - - -def preprocess_args(args): - # Temporary work around for noauto - if args.cgo_srcs and len(args.cgo_srcs) > 0: - cgo_srcs_set = set(args.cgo_srcs) - args.srcs = [x for x in args.srcs if x not in cgo_srcs_set] - - args.pkg_root = os.path.join(args.toolchain_root, 'pkg') - toolchain_tool_root = os.path.join(args.pkg_root, 'tool', '{}_{}'.format(args.host_os, args.host_arch)) - args.go_compile = os.path.join(toolchain_tool_root, 'compile') - args.go_cgo = os.path.join(toolchain_tool_root, 'cgo') - args.go_link = os.path.join(toolchain_tool_root, 'link') - args.go_asm = os.path.join(toolchain_tool_root, 'asm') - args.go_pack = os.path.join(toolchain_tool_root, 'pack') - args.go_vet = os.path.join(toolchain_tool_root, 'vet') if args.vet is True else args.vet - args.output = os.path.normpath(args.output) - args.vet_report_output = vet_report_output_name(args.output, args.vet_report_ext) - args.trimpath = None - if args.debug_root_map: - roots = {'build': args.build_root, 'source': args.source_root, 'tools': args.tools_root} - replaces = [] - for root in args.debug_root_map.split(';'): - src, dst = root.split('=', 1) - assert src in roots - replaces.append('{}=>{}'.format(roots[src], dst)) - del roots[src] - assert len(replaces) > 0 - args.trimpath = ';'.join(replaces) - args.build_root = os.path.normpath(args.build_root) - args.build_root_dir = args.build_root + os.path.sep - args.source_root = os.path.normpath(args.source_root) - args.source_root_dir = args.source_root + os.path.sep - args.output_root = os.path.normpath(args.output_root) - args.import_map = {} - args.module_map = {} - if args.cgo_peers: - args.cgo_peers = [x for x in args.cgo_peers if not x.endswith('.fake.pkg')] - - srcs = [] - for f in args.srcs: - if f.endswith('.gosrc'): - with tarfile.open(f, 'r') as tar: - srcs.extend(os.path.join(args.output_root, src) for src in tar.getnames()) - tar.extractall(path=args.output_root) - else: - srcs.append(f) - args.srcs = srcs - - assert args.mode == 'test' or args.test_srcs is None and args.xtest_srcs is None - # add lexical oreder by basename for go sources - args.srcs.sort(key=lambda x: os.path.basename(x)) - if args.test_srcs: - args.srcs += sorted(args.test_srcs, key=lambda x: os.path.basename(x)) - del args.test_srcs - if args.xtest_srcs: - args.xtest_srcs.sort(key=lambda x: os.path.basename(x)) - - # compute root relative module dir path - assert args.output is None or args.output_root == os.path.dirname(args.output) - assert args.output_root.startswith(args.build_root_dir) - args.module_path = args.output_root[len(args.build_root_dir):] - args.source_module_dir = os.path.join(args.source_root, args.test_import_path or args.module_path) + os.path.sep - assert len(args.module_path) > 0 - args.import_path, args.is_std = get_import_path(args.module_path) - - assert args.asmhdr is None or args.word == 'go' - - srcs = [] - for f in args.srcs: - if f.endswith(FIXED_CGO1_SUFFIX) and f.startswith(args.build_root_dir): - path = os.path.join(args.output_root, '{}.cgo1.go'.format(os.path.basename(f[:-len(FIXED_CGO1_SUFFIX)]))) - srcs.append(path) - preprocess_cgo1(f, path, args.source_root) - else: - srcs.append(f) - args.srcs = srcs - - if args.extldflags: - args.extldflags = pwa.ProcessWholeArchiveOption(args.targ_os).construct_cmd(args.extldflags) - - classify_srcs(args.srcs, args) - - -def compare_versions(version1, version2): - def last_index(version): - index = version.find('beta') - return len(version) if index < 0 else index - - v1 = tuple(x.zfill(8) for x in version1[:last_index(version1)].split('.')) - v2 = tuple(x.zfill(8) for x in version2[:last_index(version2)].split('.')) - if v1 == v2: - return 0 - return 1 if v1 < v2 else -1 - - -def get_symlink_or_copyfile(): - os_symlink = getattr(os, 'symlink', None) - if os_symlink is None or os.name == 'nt': - os_symlink = shutil.copyfile - return os_symlink - - -def copy_args(args): - return copy.copy(args) - - -def get_vendor_index(import_path): - index = import_path.rfind('/' + vendor_prefix) - if index < 0: - index = 0 if import_path.startswith(vendor_prefix) else index - else: - index = index + 1 - return index - - -def get_import_path(module_path): - assert len(module_path) > 0 - import_path = module_path.replace('\\', '/') - is_std_module = import_path.startswith(std_lib_prefix) - if is_std_module: - import_path = import_path[len(std_lib_prefix):] - elif import_path.startswith(vendor_prefix): - import_path = import_path[len(vendor_prefix):] - else: - import_path = arc_project_prefix + import_path - assert len(import_path) > 0 - return import_path, is_std_module - - -def call(cmd, cwd, env=None): - # sys.stderr.write('{}\n'.format(' '.join(cmd))) - return subprocess.check_output(cmd, stdin=None, stderr=subprocess.STDOUT, cwd=cwd, env=env, text=True) - - -def classify_srcs(srcs, args): - args.go_srcs = [x for x in srcs if x.endswith('.go')] - args.asm_srcs = [x for x in srcs if x.endswith('.s')] - args.objects = [x for x in srcs if x.endswith('.o') or x.endswith('.obj')] - args.symabis = [x for x in srcs if x.endswith('.symabis')] - args.sysos = [x for x in srcs if x.endswith('.syso')] - - -def get_import_config_info(peers, gen_importmap, import_map={}, module_map={}): - info = {'importmap': [], 'packagefile': [], 'standard': {}} - if gen_importmap: - for key, value in six.iteritems(import_map): - info['importmap'].append((key, value)) - for peer in peers: - peer_import_path, is_std = get_import_path(os.path.dirname(peer)) - if gen_importmap: - index = get_vendor_index(peer_import_path) - if index >= 0: - index += len(vendor_prefix) - info['importmap'].append((peer_import_path[index:], peer_import_path)) - info['packagefile'].append((peer_import_path, os.path.join(args.build_root, peer))) - if is_std: - info['standard'][peer_import_path] = True - for key, value in six.iteritems(module_map): - info['packagefile'].append((key, value)) - return info - - -def create_import_config(peers, gen_importmap, import_map={}, module_map={}): - lines = [] - info = get_import_config_info(peers, gen_importmap, import_map, module_map) - for key in ('importmap', 'packagefile'): - for item in info[key]: - lines.append('{} {}={}'.format(key, *item)) - if len(lines) > 0: - lines.append('') - content = '\n'.join(lines) - # sys.stderr.writelines('{}\n'.format(l) for l in lines) - with tempfile.NamedTemporaryFile(delete=False) as f: - f.write(content.encode('UTF-8')) - return f.name - return None - - -def create_embed_config(args): - data = { - 'Patterns': {}, - 'Files': {}, - } - for info in args.embed: - pattern = info[0] - if pattern.endswith('/**/*'): - pattern = pattern[:-3] - files = {os.path.relpath(f, args.source_module_dir).replace('\\', '/'): f for f in info[1:]} - data['Patterns'][pattern] = list(files.keys()) - data['Files'].update(files) - # sys.stderr.write('{}\n'.format(json.dumps(data, indent=4))) - with tempfile.NamedTemporaryFile(delete=False, suffix='.embedcfg') as f: - f.write(json.dumps(data).encode('UTF-8')) - return f.name - - -def vet_info_output_name(path, ext=None): - return '{}{}'.format(path, ext or vet_info_ext) - - -def vet_report_output_name(path, ext=None): - return '{}{}'.format(path, ext or vet_report_ext) - - -def get_source_path(args): - return args.test_import_path or args.module_path - - -def gen_vet_info(args): - import_path = args.real_import_path if hasattr(args, 'real_import_path') else args.import_path - info = get_import_config_info(args.peers, True, args.import_map, args.module_map) - - import_map = dict(info['importmap']) - # FIXME(snermolaev): it seems that adding import map for 'fake' package - # does't make any harm (it needs to be revised later) - import_map['unsafe'] = 'unsafe' - - for (key, _) in info['packagefile']: - if key not in import_map: - import_map[key] = key - - data = { - 'ID': import_path, - 'Compiler': 'gc', - 'Dir': os.path.join(args.source_root, get_source_path(args)), - 'ImportPath': import_path, - 'GoFiles': [x for x in args.go_srcs if x.endswith('.go')], - 'NonGoFiles': [x for x in args.go_srcs if not x.endswith('.go')], - 'ImportMap': import_map, - 'PackageFile': dict(info['packagefile']), - 'Standard': dict(info['standard']), - 'PackageVetx': dict((key, vet_info_output_name(value)) for key, value in info['packagefile']), - 'VetxOnly': False, - 'VetxOutput': vet_info_output_name(args.output), - 'SucceedOnTypecheckFailure': False - } - # sys.stderr.write('{}\n'.format(json.dumps(data, indent=4))) - return data - - -def create_vet_config(args, info): - with tempfile.NamedTemporaryFile(delete=False, suffix='.cfg') as f: - f.write(json.dumps(info).encode('UTF-8')) - return f.name - - -def decode_vet_report(json_report): - report = '' - if json_report: - try: - full_diags = json.JSONDecoder().decode(json_report.decode('UTF-8')) - except ValueError: - report = json_report - else: - messages = [] - for _, module_diags in six.iteritems(full_diags): - for _, type_diags in six.iteritems(module_diags): - for diag in type_diags: - messages.append('{}: {}'.format(diag['posn'], json.dumps(diag['message']))) - report = '\n'.join(messages) - - return report - - -def dump_vet_report(args, report): - if report: - report = report.replace(args.build_root, '$B') - report = report.replace(args.source_root, '$S') - with open(args.vet_report_output, 'w') as f: - f.write(report) - - -def read_vet_report(args): - assert args - report = '' - if os.path.exists(args.vet_report_output): - with open(args.vet_report_output, 'r') as f: - report += f.read() - return report - - -def dump_vet_report_for_tests(args, *test_args_list): - dump_vet_report(args, reduce(lambda x, y: x + read_vet_report(y), [_f for _f in test_args_list if _f], '')) - - -def do_vet(args): - assert args.vet - info = gen_vet_info(args) - vet_config = create_vet_config(args, info) - cmd = [args.go_vet, '-json'] - if args.vet_flags: - cmd.extend(args.vet_flags) - cmd.append(vet_config) - # sys.stderr.write('>>>> [{}]\n'.format(' '.join(cmd))) - p_vet = subprocess.Popen(cmd, stdin=None, stderr=subprocess.PIPE, stdout=subprocess.PIPE, cwd=args.source_root) - vet_out, vet_err = p_vet.communicate() - report = decode_vet_report(vet_out) if vet_out else '' - dump_vet_report(args, report) - if p_vet.returncode: - raise subprocess.CalledProcessError(returncode=p_vet.returncode, cmd=cmd, output=vet_err) - - -def _do_compile_go(args): - import_path, is_std_module = args.import_path, args.is_std - cmd = [ - args.go_compile, - '-o', - args.output, - '-p', - import_path if import_path != "unsafe" else "", - '-D', - '""', - ] - if args.lang: - cmd.append('-lang=go{}'.format(args.lang)) - cmd.extend(get_trimpath_args(args)) - compiling_runtime = False - if is_std_module: - cmd.append('-std') - if import_path in ('runtime', 'internal/abi', 'internal/bytealg', 'internal/cpu') or import_path.startswith('runtime/internal/'): - cmd.append('-+') - compiling_runtime = True - import_config_name = create_import_config(args.peers, True, args.import_map, args.module_map) - if import_config_name: - cmd += ['-importcfg', import_config_name] - else: - if import_path == 'unsafe' or len(args.objects) > 0 or args.asmhdr: - pass - else: - cmd.append('-complete') - # if compare_versions('1.16', args.goversion) >= 0: - if args.embed: - embed_config_name = create_embed_config(args) - cmd.extend(['-embedcfg', embed_config_name]) - if args.asmhdr: - cmd += ['-asmhdr', args.asmhdr] - # Use .symabis (starting from 1.12 version) - if args.symabis: - cmd += ['-symabis'] + args.symabis - # If 1.12 <= version < 1.13 we have to pass -allabis for 'runtime' and 'runtime/internal/atomic' - # if compare_versions('1.13', args.goversion) >= 0: - # pass - # elif import_path in ('runtime', 'runtime/internal/atomic'): - # cmd.append('-allabis') - compile_workers = '4' - if args.compile_flags: - if compiling_runtime: - cmd.extend(x for x in args.compile_flags if x not in COMPILE_OPTIMIZATION_FLAGS) - else: - cmd.extend(args.compile_flags) - if any([x in ('-race', '-shared') for x in args.compile_flags]): - compile_workers = '1' - cmd += ['-pack', '-c={}'.format(compile_workers)] - cmd += args.go_srcs - call(cmd, args.build_root) - - -class VetThread(threading.Thread): - - def __init__(self, target, args): - super(VetThread, self).__init__(target=target, args=args) - self.exc_info = None - - def run(self): - try: - super(VetThread, self).run() - except: - self.exc_info = sys.exc_info() - - def join_with_exception(self, reraise_exception): - self.join() - if reraise_exception and self.exc_info: - six.reraise(self.exc_info[0], self.exc_info[1], self.exc_info[2]) - - -def do_compile_go(args): - raise_exception_from_vet = False - if args.vet: - run_vet = VetThread(target=do_vet, args=(args,)) - run_vet.start() - try: - _do_compile_go(args) - raise_exception_from_vet = True - finally: - if args.vet: - run_vet.join_with_exception(raise_exception_from_vet) - - -def do_compile_asm(args): - def need_compiling_runtime(import_path): - return import_path in ('runtime', 'reflect', 'syscall') or \ - import_path.startswith('runtime/internal/') or \ - compare_versions('1.17', args.goversion) >= 0 and import_path == 'internal/bytealg' - - assert(len(args.srcs) == 1 and len(args.asm_srcs) == 1) - cmd = [args.go_asm] - cmd += get_trimpath_args(args) - cmd += ['-I', args.output_root, '-I', os.path.join(args.pkg_root, 'include')] - cmd += ['-D', 'GOOS_' + args.targ_os, '-D', 'GOARCH_' + args.targ_arch, '-o', args.output] - - # if compare_versions('1.16', args.goversion) >= 0: - cmd += ['-p', args.import_path] - if need_compiling_runtime(args.import_path): - cmd += ['-compiling-runtime'] - - if args.asm_flags: - cmd += args.asm_flags - cmd += args.asm_srcs - call(cmd, args.build_root) - - -def do_link_lib(args): - if len(args.asm_srcs) > 0: - asmargs = copy_args(args) - asmargs.asmhdr = os.path.join(asmargs.output_root, 'go_asm.h') - do_compile_go(asmargs) - for src in asmargs.asm_srcs: - asmargs.srcs = [src] - asmargs.asm_srcs = [src] - asmargs.output = os.path.join(asmargs.output_root, os.path.basename(src) + '.o') - do_compile_asm(asmargs) - args.objects.append(asmargs.output) - else: - do_compile_go(args) - if args.objects or args.sysos: - cmd = [args.go_pack, 'r', args.output] + args.objects + args.sysos - call(cmd, args.build_root) - - -def do_link_exe(args): - assert args.extld is not None - assert args.non_local_peers is not None - compile_args = copy_args(args) - compile_args.output = os.path.join(args.output_root, 'main.a') - compile_args.real_import_path = compile_args.import_path - compile_args.import_path = 'main' - - if args.vcs and os.path.isfile(compile_args.vcs): - build_info = os.path.join('library', 'go', 'core', 'buildinfo') - if any([x.startswith(build_info) for x in compile_args.peers]): - compile_args.go_srcs.append(compile_args.vcs) - - do_link_lib(compile_args) - cmd = [args.go_link, '-o', args.output] - import_config_name = create_import_config(args.peers + args.non_local_peers, False, args.import_map, args.module_map) - if import_config_name: - cmd += ['-importcfg', import_config_name] - if args.link_flags: - cmd += args.link_flags - - if args.buildmode: - cmd.append('-buildmode={}'.format(args.buildmode)) - elif args.mode in ('exe', 'test'): - cmd.append('-buildmode=exe') - elif args.mode == 'dll': - cmd.append('-buildmode=c-shared') - else: - assert False, 'Unexpected mode: {}'.format(args.mode) - cmd.append('-extld={}'.format(args.extld)) - - extldflags = [] - if args.extldflags is not None: - filter_musl = bool - if args.musl: - cmd.append('-linkmode=external') - extldflags.append('-static') - filter_musl = lambda x: x not in ('-lc', '-ldl', '-lm', '-lpthread', '-lrt') - extldflags += [x for x in args.extldflags if filter_musl(x)] - cgo_peers = [] - if args.cgo_peers is not None and len(args.cgo_peers) > 0: - is_group = args.targ_os == 'linux' - if is_group: - cgo_peers.append('-Wl,--start-group') - cgo_peers.extend(args.cgo_peers) - if is_group: - cgo_peers.append('-Wl,--end-group') - try: - index = extldflags.index('--cgo-peers') - extldflags = extldflags[:index] + cgo_peers + extldflags[index+1:] - except ValueError: - extldflags.extend(cgo_peers) - if len(extldflags) > 0: - cmd.append('-extldflags={}'.format(' '.join(extldflags))) - cmd.append(compile_args.output) - call(cmd, args.build_root) - - -def gen_cover_info(args): - lines = [] - lines.extend([ - """ -var ( - coverCounters = make(map[string][]uint32) - coverBlocks = make(map[string][]testing.CoverBlock) -) - """, - 'func init() {', - ]) - for var, file in (x.split(':') for x in args.cover_info): - lines.append(' coverRegisterFile("{file}", _cover0.{var}.Count[:], _cover0.{var}.Pos[:], _cover0.{var}.NumStmt[:])'.format(file=file, var=var)) - lines.extend([ - '}', - """ -func coverRegisterFile(fileName string, counter []uint32, pos []uint32, numStmts []uint16) { - if 3*len(counter) != len(pos) || len(counter) != len(numStmts) { - panic("coverage: mismatched sizes") - } - if coverCounters[fileName] != nil { - // Already registered. - return - } - coverCounters[fileName] = counter - block := make([]testing.CoverBlock, len(counter)) - for i := range counter { - block[i] = testing.CoverBlock{ - Line0: pos[3*i+0], - Col0: uint16(pos[3*i+2]), - Line1: pos[3*i+1], - Col1: uint16(pos[3*i+2]>>16), - Stmts: numStmts[i], - } - } - coverBlocks[fileName] = block -} - """, - ]) - return lines - - -def filter_out_skip_tests(tests, skip_tests): - skip_set = set() - star_skip_set = set() - for t in skip_tests: - work_set = star_skip_set if '*' in t else skip_set - work_set.add(t) - - re_star_tests = None - if len(star_skip_set) > 0: - re_star_tests = re.compile(re.sub(r'(\*)+', r'.\1', '^({})$'.format('|'.join(star_skip_set)))) - - return [x for x in tests if not (x in skip_tests or re_star_tests and re_star_tests.match(x))] - - -def gen_test_main(args, test_lib_args, xtest_lib_args): - assert args and (test_lib_args or xtest_lib_args) - test_miner = args.test_miner - test_module_path = test_lib_args.import_path if test_lib_args else xtest_lib_args.import_path - is_cover = args.cover_info and len(args.cover_info) > 0 - - # Prepare GOPATH - # $BINDIR - # |- __go__ - # |- src - # |- pkg - # |- ${TARGET_OS}_${TARGET_ARCH} - go_path_root = os.path.join(args.output_root, '__go__') - test_src_dir = os.path.join(go_path_root, 'src') - target_os_arch = '_'.join([args.targ_os, args.targ_arch]) - test_pkg_dir = os.path.join(go_path_root, 'pkg', target_os_arch, os.path.dirname(test_module_path)) - os.makedirs(test_pkg_dir) - - my_env = os.environ.copy() - my_env['GOROOT'] = '' - my_env['GOPATH'] = go_path_root - my_env['GOARCH'] = args.targ_arch - my_env['GOOS'] = args.targ_os - - tests = [] - xtests = [] - os_symlink = get_symlink_or_copyfile() - - # Get the list of "internal" tests - if test_lib_args: - os.makedirs(os.path.join(test_src_dir, test_module_path)) - os_symlink(test_lib_args.output, os.path.join(test_pkg_dir, os.path.basename(test_module_path) + '.a')) - cmd = [test_miner, '-benchmarks', '-tests', test_module_path] - tests = [x for x in (call(cmd, test_lib_args.output_root, my_env) or '').strip().split('\n') if len(x) > 0] - if args.skip_tests: - tests = filter_out_skip_tests(tests, args.skip_tests) - test_main_found = '#TestMain' in tests - - # Get the list of "external" tests - if xtest_lib_args: - xtest_module_path = xtest_lib_args.import_path - os.makedirs(os.path.join(test_src_dir, xtest_module_path)) - os_symlink(xtest_lib_args.output, os.path.join(test_pkg_dir, os.path.basename(xtest_module_path) + '.a')) - cmd = [test_miner, '-benchmarks', '-tests', xtest_module_path] - xtests = [x for x in (call(cmd, xtest_lib_args.output_root, my_env) or '').strip().split('\n') if len(x) > 0] - if args.skip_tests: - xtests = filter_out_skip_tests(xtests, args.skip_tests) - xtest_main_found = '#TestMain' in xtests - - test_main_package = None - if test_main_found and xtest_main_found: - assert False, 'multiple definition of TestMain' - elif test_main_found: - test_main_package = '_test' - elif xtest_main_found: - test_main_package = '_xtest' - - shutil.rmtree(go_path_root) - - lines = ['package main', '', 'import ('] - if test_main_package is None: - lines.append(' "os"') - lines.extend([' "testing"', ' "testing/internal/testdeps"']) - - if len(tests) > 0: - lines.append(' _test "{}"'.format(test_module_path)) - elif test_lib_args: - lines.append(' _ "{}"'.format(test_module_path)) - - if len(xtests) > 0: - lines.append(' _xtest "{}"'.format(xtest_module_path)) - elif xtest_lib_args: - lines.append(' _ "{}"'.format(xtest_module_path)) - - if is_cover: - lines.append(' _cover0 "{}"'.format(test_module_path)) - lines.extend([')', '']) - - if compare_versions('1.18', args.goversion) < 0: - kinds = ['Test', 'Benchmark', 'Example'] - else: - kinds = ['Test', 'Benchmark', 'FuzzTarget', 'Example'] - - var_names = [] - for kind in kinds: - var_name = '{}s'.format(kind.lower()) - var_names.append(var_name) - lines.append('var {} = []testing.Internal{}{{'.format(var_name, kind)) - for test in [x for x in tests if x.startswith(kind)]: - lines.append(' {{"{test}", _test.{test}}},'.format(test=test)) - for test in [x for x in xtests if x.startswith(kind)]: - lines.append(' {{"{test}", _xtest.{test}}},'.format(test=test)) - lines.extend(['}', '']) - - if is_cover: - lines.extend(gen_cover_info(args)) - - lines.append('func main() {') - if is_cover: - lines.extend([ - ' testing.RegisterCover(testing.Cover{', - ' Mode: "set",', - ' Counters: coverCounters,', - ' Blocks: coverBlocks,', - ' CoveredPackages: "",', - ' })', - ]) - lines.extend([ - ' m := testing.MainStart(testdeps.TestDeps{{}}, {})'.format(', '.join(var_names)), - '', - ]) - - if test_main_package: - lines.append(' {}.TestMain(m)'.format(test_main_package)) - else: - lines.append(' os.Exit(m.Run())') - lines.extend(['}', '']) - - content = '\n'.join(lines) - # sys.stderr.write('{}\n'.format(content)) - return content - - -def do_link_test(args): - assert args.srcs or args.xtest_srcs - assert args.test_miner is not None - - test_module_path = get_source_path(args) - test_import_path, _ = get_import_path(test_module_path) - - test_lib_args = copy_args(args) if args.srcs else None - xtest_lib_args = copy_args(args) if args.xtest_srcs else None - if xtest_lib_args is not None: - xtest_lib_args.embed = args.embed_xtest if args.embed_xtest else None - - ydx_file_name = None - xtest_ydx_file_name = None - need_append_ydx = test_lib_args and xtest_lib_args and args.ydx_file and args.vet_flags - if need_append_ydx: - def find_ydx_file_name(name, flags): - for i, elem in enumerate(flags): - if elem.endswith(name): - return (i, elem) - assert False, 'Unreachable code' - - idx, ydx_file_name = find_ydx_file_name(xtest_lib_args.ydx_file, xtest_lib_args.vet_flags) - xtest_ydx_file_name = '{}_xtest'.format(ydx_file_name) - xtest_lib_args.vet_flags = copy.copy(xtest_lib_args.vet_flags) - xtest_lib_args.vet_flags[idx] = xtest_ydx_file_name - - if test_lib_args: - test_lib_args.output = os.path.join(args.output_root, 'test.a') - test_lib_args.vet_report_output = vet_report_output_name(test_lib_args.output) - test_lib_args.module_path = test_module_path - test_lib_args.import_path = test_import_path - do_link_lib(test_lib_args) - - if xtest_lib_args: - xtest_lib_args.srcs = xtest_lib_args.xtest_srcs - classify_srcs(xtest_lib_args.srcs, xtest_lib_args) - xtest_lib_args.output = os.path.join(args.output_root, 'xtest.a') - xtest_lib_args.vet_report_output = vet_report_output_name(xtest_lib_args.output) - xtest_lib_args.module_path = test_module_path + '_test' - xtest_lib_args.import_path = test_import_path + '_test' - if test_lib_args: - xtest_lib_args.module_map[test_import_path] = test_lib_args.output - need_append_ydx = args.ydx_file and args.srcs and args.vet_flags - do_link_lib(xtest_lib_args) - - if need_append_ydx: - with open(os.path.join(args.build_root, ydx_file_name), 'ab') as dst_file: - with open(os.path.join(args.build_root, xtest_ydx_file_name), 'rb') as src_file: - dst_file.write(src_file.read()) - - test_main_content = gen_test_main(args, test_lib_args, xtest_lib_args) - test_main_name = os.path.join(args.output_root, '_test_main.go') - with open(test_main_name, "w") as f: - f.write(test_main_content) - test_args = copy_args(args) - test_args.embed = None - test_args.srcs = [test_main_name] - if test_args.test_import_path is None: - # it seems that we can do it unconditionally, but this kind - # of mangling doesn't really looks good to me and we leave it - # for pure GO_TEST module - test_args.module_path = test_args.module_path + '___test_main__' - test_args.import_path = test_args.import_path + '___test_main__' - classify_srcs(test_args.srcs, test_args) - if test_lib_args: - test_args.module_map[test_lib_args.import_path] = test_lib_args.output - if xtest_lib_args: - test_args.module_map[xtest_lib_args.import_path] = xtest_lib_args.output - - if args.vet: - dump_vet_report_for_tests(test_args, test_lib_args, xtest_lib_args) - test_args.vet = False - - do_link_exe(test_args) - - -if __name__ == '__main__': - args = pcf.get_args(sys.argv[1:]) - - parser = argparse.ArgumentParser(prefix_chars='+') - parser.add_argument('++mode', choices=['dll', 'exe', 'lib', 'test'], required=True) - parser.add_argument('++buildmode', choices=['c-shared', 'exe', 'pie']) - parser.add_argument('++srcs', nargs='*', required=True) - parser.add_argument('++cgo-srcs', nargs='*') - parser.add_argument('++test_srcs', nargs='*') - parser.add_argument('++xtest_srcs', nargs='*') - parser.add_argument('++cover_info', nargs='*') - parser.add_argument('++output', nargs='?', default=None) - parser.add_argument('++source-root', default=None) - parser.add_argument('++build-root', required=True) - parser.add_argument('++tools-root', default=None) - parser.add_argument('++output-root', required=True) - parser.add_argument('++toolchain-root', required=True) - parser.add_argument('++host-os', choices=['linux', 'darwin', 'windows'], required=True) - parser.add_argument('++host-arch', choices=['amd64', 'arm64'], required=True) - parser.add_argument('++targ-os', choices=['linux', 'darwin', 'windows'], required=True) - parser.add_argument('++targ-arch', choices=['amd64', 'x86', 'arm64'], required=True) - parser.add_argument('++peers', nargs='*') - parser.add_argument('++non-local-peers', nargs='*') - parser.add_argument('++cgo-peers', nargs='*') - parser.add_argument('++asmhdr', nargs='?', default=None) - parser.add_argument('++test-import-path', nargs='?') - parser.add_argument('++test-miner', nargs='?') - parser.add_argument('++arc-project-prefix', nargs='?', default=arc_project_prefix) - parser.add_argument('++std-lib-prefix', nargs='?', default=std_lib_prefix) - parser.add_argument('++vendor-prefix', nargs='?', default=vendor_prefix) - parser.add_argument('++extld', nargs='?', default=None) - parser.add_argument('++extldflags', nargs='+', default=None) - parser.add_argument('++goversion', required=True) - parser.add_argument('++lang', nargs='?', default=None) - parser.add_argument('++asm-flags', nargs='*') - parser.add_argument('++compile-flags', nargs='*') - parser.add_argument('++link-flags', nargs='*') - parser.add_argument('++vcs', nargs='?', default=None) - parser.add_argument('++vet', nargs='?', const=True, default=False) - parser.add_argument('++vet-flags', nargs='*', default=None) - parser.add_argument('++vet-info-ext', default=vet_info_ext) - parser.add_argument('++vet-report-ext', default=vet_report_ext) - parser.add_argument('++musl', action='store_true') - parser.add_argument('++skip-tests', nargs='*', default=None) - parser.add_argument('++ydx-file', default='') - parser.add_argument('++debug-root-map', default=None) - parser.add_argument('++embed', action='append', nargs='*') - parser.add_argument('++embed_xtest', action='append', nargs='*') - args = parser.parse_args(args) - - arc_project_prefix = args.arc_project_prefix - std_lib_prefix = args.std_lib_prefix - vendor_prefix = args.vendor_prefix - vet_info_ext = args.vet_info_ext - vet_report_ext = args.vet_report_ext - - preprocess_args(args) - - try: - os.unlink(args.output) - except OSError: - pass - - # We are going to support only 'lib', 'exe' and 'cgo' build modes currently - # and as a result we are going to generate only one build node per module - # (or program) - dispatch = { - 'exe': do_link_exe, - 'dll': do_link_exe, - 'lib': do_link_lib, - 'test': do_link_test - } - - exit_code = 1 - try: - dispatch[args.mode](args) - exit_code = 0 - except KeyError: - sys.stderr.write('Unknown build mode [{}]...\n'.format(args.mode)) - except subprocess.CalledProcessError as e: - sys.stderr.write('{} returned non-zero exit code {}.\n{}\n'.format(' '.join(e.cmd), e.returncode, e.output)) - exit_code = e.returncode - except Exception as e: - sys.stderr.write('Unhandled exception [{}]...\n'.format(str(e))) - sys.exit(exit_code) diff --git a/build/scripts/ios_wrapper.py b/build/scripts/ios_wrapper.py deleted file mode 100644 index d3aa48387a..0000000000 --- a/build/scripts/ios_wrapper.py +++ /dev/null @@ -1,180 +0,0 @@ -import errno -import json -import os -import shutil -import subprocess -import sys -import tarfile -import plistlib - - -def ensure_dir(path): - try: - os.makedirs(path) - except OSError as e: - if e.errno != errno.EEXIST or not os.path.isdir(path): - raise - - -def just_do_it(args): - if not args: - raise Exception('Not enough args!') - parts = [[]] - for arg in args: - if arg == '__DELIM__': - parts.append([]) - else: - parts[-1].append(arg) - if len(parts) != 3 or len(parts[0]) != 5: - raise Exception('Bad call') - bin_name, ibtool_path, main_out, app_name, module_dir = parts[0] - bin_name = os.path.basename(bin_name) - inputs, storyboard_user_flags = parts[1:] - plists, storyboards, signs, nibs, resources, signed_resources, plist_jsons, strings = [], [], [], [], [], [], [], [] - for i in inputs: - if i.endswith('.plist') or i.endswith('.partial_plist'): - plists.append(i) - elif i.endswith('.compiled_storyboard_tar'): - storyboards.append(i) - elif i.endswith('.xcent'): - signs.append(i) - elif i.endswith('.nib'): - nibs.append(i) - elif i.endswith('.resource_tar'): - resources.append(i) - elif i.endswith('.signed_resource_tar'): - signed_resources.append(i) - elif i.endswith('.plist_json'): - plist_jsons.append(i) - elif i.endswith('.strings_tar'): - strings.append(i) - else: - print >> sys.stderr, 'Unknown input:', i, 'ignoring' - if not plists: - raise Exception("Can't find plist files") - if not plists[0].endswith('.plist'): - print >> sys.stderr, "Main plist file can be defined incorretly" - if not storyboards: - print >> sys.stderr, "Storyboards list are empty" - if len(signs) > 1: - raise Exception("Too many .xcent files") - app_dir = os.path.join(module_dir, app_name + '.app') - ensure_dir(app_dir) - copy_nibs(nibs, module_dir, app_dir) - replaced_parameters = { - 'DEVELOPMENT_LANGUAGE': 'en', - 'EXECUTABLE_NAME': bin_name, - 'PRODUCT_BUNDLE_IDENTIFIER': 'Yandex.' + app_name, - 'PRODUCT_NAME': app_name, - } - replaced_templates = {} - for plist_json in plist_jsons: - with open(plist_json) as jsonfile: - for k, v in json.loads(jsonfile.read()).items(): - replaced_parameters[k] = v - for k, v in replaced_parameters.items(): - replaced_templates['$(' + k + ')'] = v - replaced_templates['${' + k + '}'] = v - make_main_plist(plists, os.path.join(app_dir, 'Info.plist'), replaced_templates) - link_storyboards(ibtool_path, storyboards, app_name, app_dir, storyboard_user_flags) - if resources: - extract_resources(resources, app_dir) - if signed_resources: - extract_resources(signed_resources, app_dir, sign=True) - if strings: - extract_resources(strings, app_dir, strings=True) - if not signs: - sign_file = os.path.join(module_dir, app_name + '.xcent') - with open(sign_file, 'w') as f: - f.write('''<?xml version="1.0" encoding="UTF-8"?> -<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> -<plist version="1.0"> -<dict> - <key>com.apple.security.get-task-allow</key> - <true/> -</dict> -</plist> - ''') - else: - sign_file = signs[0] - sign_application(sign_file, app_dir) - make_archive(app_dir, main_out) - - -def is_exe(fpath): - return os.path.isfile(fpath) and os.access(fpath, os.X_OK) - - -def copy_nibs(nibs, module_dir, app_dir): - for nib in nibs: - dst = os.path.join(app_dir, os.path.relpath(nib, module_dir)) - ensure_dir(os.path.dirname(dst)) - shutil.copyfile(nib, dst) - - -def make_main_plist(inputs, out, replaced_parameters): - united_data = {} - for i in inputs: - united_data.update(plistlib.readPlist(i)) - - def scan_n_replace(root): - if not isinstance(root, dict): - raise Exception('Invalid state') - for k in root: - if isinstance(root[k], list): - for i in xrange(len(root[k])): - if isinstance(root[k][i], dict): - scan_n_replace(root[k][i]) - elif root[k][i] in replaced_parameters: - root[k][i] = replaced_parameters[root[k][i]] - elif isinstance(root[k], dict): - scan_n_replace(root[k]) - else: - if root[k] in replaced_parameters: - root[k] = replaced_parameters[root[k]] - scan_n_replace(united_data) - plistlib.writePlist(united_data, out) - subprocess.check_call(['/usr/bin/plutil', '-convert', 'binary1', out]) - - -def link_storyboards(ibtool, archives, app_name, app_dir, flags): - unpacked = [] - for arc in archives: - unpacked.append(os.path.splitext(arc)[0] + 'c') - ensure_dir(unpacked[-1]) - with tarfile.open(arc) as a: - a.extractall(path=unpacked[-1]) - flags += [ - '--module', app_name, - '--link', app_dir, - ] - subprocess.check_call([ibtool] + flags + - ['--errors', '--warnings', '--notices', '--output-format', 'human-readable-text'] + - unpacked) - - -def sign_application(xcent, app_dir): - subprocess.check_call(['/usr/bin/codesign', '--force', '--sign', '-', '--entitlements', xcent, '--timestamp=none', app_dir]) - - -def extract_resources(resources, app_dir, strings=False, sign=False): - for res in resources: - with tarfile.open(res) as tf: - for tfinfo in tf: - tf.extract(tfinfo.name, app_dir) - if strings: - subprocess.check_call(['/usr/bin/plutil', '-convert', 'binary1', os.path.join(app_dir, tfinfo.name)]) - if sign: - subprocess.check_call(['/usr/bin/codesign', '--force', '--sign', '-', os.path.join(app_dir, tfinfo.name)]) - - -def make_archive(app_dir, output): - with tarfile.open(output, "w") as tar_handle: - for root, _, files in os.walk(app_dir): - for f in files: - tar_handle.add(os.path.join(root, f), arcname=os.path.join(os.path.basename(app_dir), - os.path.relpath(os.path.join(root, f), app_dir))) - - -if __name__ == '__main__': - just_do_it(sys.argv[1:]) diff --git a/build/scripts/java_pack_to_file.py b/build/scripts/java_pack_to_file.py deleted file mode 100644 index c8ab7c311b..0000000000 --- a/build/scripts/java_pack_to_file.py +++ /dev/null @@ -1,43 +0,0 @@ -import os -import re -import optparse - -JAVA_PACKAGE_REGEX = re.compile(r'^\s*package\s+(.*?);', flags=re.MULTILINE | re.DOTALL) -KOTLIN_PACKAGE_REGEX = re.compile(r'^\s*package\s+(.*?)^', flags=re.MULTILINE | re.DOTALL) - - -def parse_args(): - parser = optparse.OptionParser() - parser.add_option('-o', '--output') - parser.add_option('-a', '--source-root', dest='source_root') - return parser.parse_args() - - -def get_package_name(filename): - with open(filename) as afile: - content = afile.read() - if filename.endswith(".kt"): - match = KOTLIN_PACKAGE_REGEX.search(content) - if match: - return match.group(1).strip().replace('.', '/') - else: - match = JAVA_PACKAGE_REGEX.search(content) - if match: - return match.group(1).replace('\n\t ', '').replace('.', '/') - return '' - - -def write_coverage_sources(output, srcroot, files): - with open(output, 'w') as afile: - for filename in files: - pname = get_package_name(os.path.join(srcroot, filename)) - afile.write(os.path.join(pname, os.path.basename(filename)) + ':' + filename + '\n') - - -def main(): - opts, files = parse_args() - write_coverage_sources(opts.output, opts.source_root, files) - - -if __name__ == '__main__': - exit(main()) diff --git a/build/scripts/jni_swig.py b/build/scripts/jni_swig.py deleted file mode 100644 index 4b2220430b..0000000000 --- a/build/scripts/jni_swig.py +++ /dev/null @@ -1,46 +0,0 @@ -import argparse -import subprocess -import re -import os -import tarfile - -def parse_args(): - parser = argparse.ArgumentParser(description='Wrapper script to invoke swig.') - parser.add_argument('--swig', help='path to the swig executable') - parser.add_argument('--default-module', type=str, help='swig -module argument value for inputs without %module statement') - parser.add_argument('--package-by-file', help='path to file which dir must be converted to swig -package argument') - parser.add_argument('--jsrc', help='jsrc output archive filename') - parser.add_argument('--src', help='input .swg file path') - parser.add_argument('--out-header', help='header file which must exist even if it was not generated by swig') - parser.add_argument('args', nargs="*", help='regular swig arguments') - - return parser.parse_args() - - -def path2pkg(path): - return path.replace('/', '.').replace('-', '_') - - -def main(args): - package = path2pkg(os.path.dirname(args.package_by_file)) - outdir = None - if args.jsrc: - outdir = package.replace('.', '/') - outdir_abs = os.path.join(os.path.dirname(args.jsrc), outdir) - if not os.path.exists(outdir_abs): - os.makedirs(outdir_abs) - cmd = [args.swig, '-c++', '-java', '-package', package] + (['-outdir', outdir_abs] if outdir is not None else []) + args.args - if '-module' not in args.args and args.default_module: - with open(args.src, 'r') as f: - if not re.search(r'(?m)^%module\b', f.read()): - cmd += ['-module', args.default_module] - subprocess.check_call(cmd + [args.src]) - if args.out_header and not os.path.exists(args.out_header): - open(args.out_header, 'w').close() - if args.jsrc: - with tarfile.open(args.jsrc, 'a') as tf: - tf.add(outdir_abs, arcname=outdir) - - -if __name__ == '__main__': - main(parse_args()) diff --git a/build/scripts/kt_copy.py b/build/scripts/kt_copy.py deleted file mode 100644 index f833c24ef4..0000000000 --- a/build/scripts/kt_copy.py +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env python -import sys - -if __name__ == '__main__': - source = sys.argv[1] - destination = sys.argv[2] - source_root = sys.argv[3] - build_root = sys.argv[4] - with open(source, 'r') as afile: - src_content = afile.read() - src_content = src_content.replace(source_root + '/', "") - result_srcs = "" - for line in src_content.split("\n"): - if not line.startswith(build_root): - result_srcs += line + "\n" - with open(destination, 'w') as afile: - afile.write(result_srcs) diff --git a/build/scripts/link_asrc.py b/build/scripts/link_asrc.py deleted file mode 100644 index eec5fe09a8..0000000000 --- a/build/scripts/link_asrc.py +++ /dev/null @@ -1,84 +0,0 @@ -import argparse -import itertools -import os -import tarfile - - -DELIM_JAVA = '__DELIM_JAVA__' -DELIM_RES = '__DELIM_RES__' -DELIM_ASSETS = '__DELIM_ASSETS__' -DELIM_AIDL = '__DELIM_AIDL__' - -DELIMS = ( - DELIM_JAVA, - DELIM_RES, - DELIM_ASSETS, - DELIM_AIDL, -) - -DESTS = { - DELIM_JAVA: 'src', - DELIM_RES: 'res', - DELIM_ASSETS: 'assets', - DELIM_AIDL: 'aidl', -} - - -def parse_args(): - parser = argparse.ArgumentParser() - parser.add_argument('--asrcs', nargs='*') - parser.add_argument('--input', nargs='*') - parser.add_argument('--jsrcs', nargs='*') - parser.add_argument('--output', required=True) - parser.add_argument('--work', required=True) - - return parser.parse_args() - - -def main(): - args = parse_args() - - files = [] - parts = [] - - if args.input and len(args.input) > 0: - for x in args.input: - if x in DELIMS: - assert(len(parts) == 0 or len(parts[-1]) > 1) - parts.append([x]) - else: - assert(len(parts) > 0) - parts[-1].append(x) - assert(len(parts[-1]) > 1) - - if args.jsrcs and len(args.jsrcs): - src_dir = os.path.join(args.work, DESTS[DELIM_JAVA]) - os.makedirs(src_dir) - - for jsrc in filter(lambda x: x.endswith('.jsrc'), args.jsrcs): - with tarfile.open(jsrc, 'r') as tar: - names = tar.getnames() - if names and len(names) > 0: - parts.append([DELIM_JAVA, src_dir]) - parts[-1].extend(itertools.imap(lambda x: os.path.join(src_dir, x), names)) - tar.extractall(path=src_dir) - - if args.asrcs and len(args.asrcs): - for asrc in filter(lambda x: x.endswith('.asrc') and os.path.exists(x), args.asrcs): - with tarfile.open(asrc, 'r') as tar: - files.extend(tar.getnames()) - tar.extractall(path=args.work) - - with tarfile.open(args.output, 'w') as out: - for part in parts: - dest = DESTS[part[0]] - prefix = part[1] - for f in part[2:]: - out.add(f, arcname=os.path.join(dest, os.path.relpath(f, prefix))) - - for f in files: - out.add(os.path.join(args.work, f), arcname=f) - - -if __name__ == '__main__': - main() diff --git a/build/scripts/link_dyn_lib.py b/build/scripts/link_dyn_lib.py deleted file mode 100644 index 81e643f630..0000000000 --- a/build/scripts/link_dyn_lib.py +++ /dev/null @@ -1,341 +0,0 @@ -import sys -import os -import subprocess -import tempfile -import collections -import optparse -import pipes - -from process_whole_archive_option import ProcessWholeArchiveOption - - -def shlex_join(cmd): - # equivalent to shlex.join() in python 3 - return ' '.join( - pipes.quote(part) - for part in cmd - ) - - -def parse_export_file(p): - with open(p, 'r') as f: - for l in f: - l = l.strip() - - if l and '#' not in l: - words = l.split() - if len(words) == 2 and words[0] == 'linux_version': - yield {'linux_version': words[1]} - elif len(words) == 2: - yield {'lang': words[0], 'sym': words[1]} - elif len(words) == 1: - yield {'lang': 'C', 'sym': words[0]} - else: - raise Exception('unsupported exports line: ' + l) - - -def to_c(sym): - symbols = collections.deque(sym.split('::')) - c_prefixes = [ # demangle prefixes for c++ symbols - '_ZN', # namespace - '_ZTIN', # typeinfo for - '_ZTSN', # typeinfo name for - '_ZTTN', # VTT for - '_ZTVN', # vtable for - '_ZNK', # const methods - ] - c_sym = '' - while symbols: - s = symbols.popleft() - if s == '*': - c_sym += '*' - break - if '*' in s and len(s) > 1: - raise Exception('Unsupported format, cannot guess length of symbol: ' + s) - c_sym += str(len(s)) + s - if symbols: - raise Exception('Unsupported format: ' + sym) - if c_sym[-1] != '*': - c_sym += 'E*' - return ['{prefix}{sym}'.format(prefix=prefix, sym=c_sym) for prefix in c_prefixes] - - -def fix_darwin_param(ex): - for item in ex: - if item.get('linux_version'): - continue - - if item['lang'] == 'C': - yield '-Wl,-exported_symbol,_' + item['sym'] - elif item['lang'] == 'C++': - for sym in to_c(item['sym']): - yield '-Wl,-exported_symbol,_' + sym - else: - raise Exception('unsupported lang: ' + item['lang']) - - -def fix_gnu_param(arch, ex): - d = collections.defaultdict(list) - version = None - for item in ex: - if item.get('linux_version'): - if not version: - version = item.get('linux_version') - else: - raise Exception('More than one linux_version defined') - elif item['lang'] == 'C++': - d['C'].extend(to_c(item['sym'])) - else: - d[item['lang']].append(item['sym']) - - with tempfile.NamedTemporaryFile(mode='wt', delete=False) as f: - if version: - f.write('{} {{\nglobal:\n'.format(version)) - else: - f.write('{\nglobal:\n') - - for k, v in d.items(): - f.write(' extern "' + k + '" {\n') - - for x in v: - f.write(' ' + x + ';\n') - - f.write(' };\n') - - f.write('local: *;\n};\n') - - ret = ['-Wl,--version-script=' + f.name] - - if arch == 'ANDROID': - ret += ['-Wl,--export-dynamic'] - - return ret - - -def fix_windows_param(ex): - with tempfile.NamedTemporaryFile(delete=False) as def_file: - exports = [] - for item in ex: - if item.get('lang') == 'C': - exports.append(item.get('sym')) - def_file.write('EXPORTS\n') - for export in exports: - def_file.write(' {}\n'.format(export)) - return ['/DEF:{}'.format(def_file.name)] - - -MUSL_LIBS = '-lc', '-lcrypt', '-ldl', '-lm', '-lpthread', '-lrt', '-lutil' - -CUDA_LIBRARIES = { - '-lcublas_static': '-lcublas', - '-lcublasLt_static': '-lcublasLt', - '-lcudart_static': '-lcudart', - '-lcudnn_static': '-lcudnn', - '-lcufft_static_nocallback': '-lcufft', - '-lcurand_static': '-lcurand', - '-lcusolver_static': '-lcusolver', - '-lcusparse_static': '-lcusparse', - '-lmyelin_compiler_static': '-lmyelin', - '-lmyelin_executor_static': '-lnvcaffe_parser', - '-lmyelin_pattern_library_static': '', - '-lmyelin_pattern_runtime_static': '', - '-lnvinfer_static': '-lnvinfer', - '-lnvinfer_plugin_static': '-lnvinfer_plugin', - '-lnvonnxparser_static': '-lnvonnxparser', - '-lnvparsers_static': '-lnvparsers' -} - - -def fix_cmd(arch, c): - if arch == 'WINDOWS': - prefix = '/DEF:' - f = fix_windows_param - else: - prefix = '-Wl,--version-script=' - if arch in ('DARWIN', 'IOS', 'IOSSIM'): - f = fix_darwin_param - else: - f = lambda x: fix_gnu_param(arch, x) - - def do_fix(p): - if p.startswith(prefix) and p.endswith('.exports'): - fname = p[len(prefix):] - - return list(f(list(parse_export_file(fname)))) - - if p.endswith('.supp'): - return [] - - if p.endswith('.pkg.fake'): - return [] - - return [p] - - return sum((do_fix(x) for x in c), []) - - -def fix_cmd_for_musl(cmd): - flags = [] - for flag in cmd: - if flag not in MUSL_LIBS: - flags.append(flag) - return flags - - -def fix_cmd_for_dynamic_cuda(cmd): - flags = [] - for flag in cmd: - if flag in CUDA_LIBRARIES: - flags.append(CUDA_LIBRARIES[flag]) - else: - flags.append(flag) - return flags - - -def parse_args(): - parser = optparse.OptionParser() - parser.disable_interspersed_args() - parser.add_option('--arch') - parser.add_option('--target') - parser.add_option('--soname') - parser.add_option('--fix-elf') - parser.add_option('--linker-output') - parser.add_option('--musl', action='store_true') - parser.add_option('--dynamic-cuda', action='store_true') - parser.add_option('--whole-archive-peers', action='append') - parser.add_option('--whole-archive-libs', action='append') - parser.add_option('--custom-step') - parser.add_option('--python') - return parser.parse_args() - - -if __name__ == '__main__': - opts, args = parse_args() - - assert opts.arch - assert opts.target - - cmd = fix_cmd(opts.arch, args) - - if opts.musl: - cmd = fix_cmd_for_musl(cmd) - if opts.dynamic_cuda: - cmd = fix_cmd_for_dynamic_cuda(cmd) - - cmd = ProcessWholeArchiveOption(opts.arch, opts.whole_archive_peers, opts.whole_archive_libs).construct_cmd(cmd) - - if opts.custom_step: - assert opts.python - subprocess.check_call([opts.python] + [opts.custom_step] + cmd) - - if opts.linker_output: - stdout = open(opts.linker_output, 'w') - else: - stdout = sys.stdout - - proc = subprocess.Popen(cmd, shell=False, stderr=sys.stderr, stdout=stdout) - proc.communicate() - - if proc.returncode: - print >>sys.stderr, 'linker has failed with retcode:', proc.returncode - print >>sys.stderr, 'linker command:', shlex_join(cmd) - sys.exit(proc.returncode) - - if opts.fix_elf: - cmd = [opts.fix_elf, opts.target] - proc = subprocess.Popen(cmd, shell=False, stderr=sys.stderr, stdout=sys.stdout) - proc.communicate() - - if proc.returncode: - print >>sys.stderr, 'fix_elf has failed with retcode:', proc.returncode - print >>sys.stderr, 'fix_elf command:', shlex_join(cmd) - sys.exit(proc.returncode) - - if opts.soname and opts.soname != opts.target: - if os.path.exists(opts.soname): - os.unlink(opts.soname) - os.link(opts.target, opts.soname) - - -# -----------------Test---------------- # -def write_temp_file(content): - import yatest.common as yc - filename = yc.output_path('test.exports') - with open(filename, 'w') as f: - f.write(content) - return filename - - -def test_fix_cmd_darwin(): - export_file_content = """ -C++ geobase5::details::lookup_impl::* -C++ geobase5::hardcoded_service -""" - filename = write_temp_file(export_file_content) - args = ['-Wl,--version-script={}'.format(filename)] - assert fix_cmd('DARWIN', args) == [ - '-Wl,-exported_symbol,__ZN8geobase57details11lookup_impl*', - '-Wl,-exported_symbol,__ZTIN8geobase57details11lookup_impl*', - '-Wl,-exported_symbol,__ZTSN8geobase57details11lookup_impl*', - '-Wl,-exported_symbol,__ZTTN8geobase57details11lookup_impl*', - '-Wl,-exported_symbol,__ZTVN8geobase57details11lookup_impl*', - '-Wl,-exported_symbol,__ZNK8geobase57details11lookup_impl*', - '-Wl,-exported_symbol,__ZN8geobase517hardcoded_serviceE*', - '-Wl,-exported_symbol,__ZTIN8geobase517hardcoded_serviceE*', - '-Wl,-exported_symbol,__ZTSN8geobase517hardcoded_serviceE*', - '-Wl,-exported_symbol,__ZTTN8geobase517hardcoded_serviceE*', - '-Wl,-exported_symbol,__ZTVN8geobase517hardcoded_serviceE*', - '-Wl,-exported_symbol,__ZNK8geobase517hardcoded_serviceE*', - ] - - -def run_fix_gnu_param(export_file_content): - filename = write_temp_file(export_file_content) - result = fix_gnu_param('LINUX', list(parse_export_file(filename)))[0] - version_script_path = result[len('-Wl,--version-script='):] - with open(version_script_path) as f: - content = f.read() - return content - - -def test_fix_gnu_param(): - export_file_content = """ -C++ geobase5::details::lookup_impl::* -C getFactoryMap -""" - assert run_fix_gnu_param(export_file_content) == """{ -global: - extern "C" { - _ZN8geobase57details11lookup_impl*; - _ZTIN8geobase57details11lookup_impl*; - _ZTSN8geobase57details11lookup_impl*; - _ZTTN8geobase57details11lookup_impl*; - _ZTVN8geobase57details11lookup_impl*; - _ZNK8geobase57details11lookup_impl*; - getFactoryMap; - }; -local: *; -}; -""" - - -def test_fix_gnu_param_with_linux_version(): - export_file_content = """ -C++ geobase5::details::lookup_impl::* -linux_version ver1.0 -C getFactoryMap -""" - assert run_fix_gnu_param(export_file_content) == """ver1.0 { -global: - extern "C" { - _ZN8geobase57details11lookup_impl*; - _ZTIN8geobase57details11lookup_impl*; - _ZTSN8geobase57details11lookup_impl*; - _ZTTN8geobase57details11lookup_impl*; - _ZTVN8geobase57details11lookup_impl*; - _ZNK8geobase57details11lookup_impl*; - getFactoryMap; - }; -local: *; -}; -""" diff --git a/build/scripts/link_exe.py b/build/scripts/link_exe.py deleted file mode 100644 index d50557ebec..0000000000 --- a/build/scripts/link_exe.py +++ /dev/null @@ -1,174 +0,0 @@ -import sys -import subprocess -import optparse - -from process_whole_archive_option import ProcessWholeArchiveOption - - -def get_leaks_suppressions(cmd): - supp, newcmd = [], [] - for arg in cmd: - if arg.endswith(".supp"): - supp.append(arg) - else: - newcmd.append(arg) - return supp, newcmd - - -MUSL_LIBS = '-lc', '-lcrypt', '-ldl', '-lm', '-lpthread', '-lrt', '-lutil' - - -CUDA_LIBRARIES = { - '-lcublas_static': '-lcublas', - '-lcublasLt_static': '-lcublasLt', - '-lcudart_static': '-lcudart', - '-lcudnn_static': '-lcudnn', - '-lcufft_static_nocallback': '-lcufft', - '-lcurand_static': '-lcurand', - '-lcusolver_static': '-lcusolver', - '-lcusparse_static': '-lcusparse', - '-lmyelin_compiler_static': '-lmyelin', - '-lmyelin_executor_static': '-lnvcaffe_parser', - '-lmyelin_pattern_library_static': '', - '-lmyelin_pattern_runtime_static': '', - '-lnvinfer_static': '-lnvinfer', - '-lnvinfer_plugin_static': '-lnvinfer_plugin', - '-lnvonnxparser_static': '-lnvonnxparser', - '-lnvparsers_static': '-lnvparsers' -} - - -def remove_excessive_flags(cmd): - flags = [] - for flag in cmd: - if not flag.endswith('.ios.interface') and not flag.endswith('.pkg.fake'): - flags.append(flag) - return flags - - -def fix_sanitize_flag(cmd): - """ - Remove -fsanitize=address flag if sanitazers are linked explicitly for linux target. - """ - for flag in cmd: - if flag.startswith('--target') and 'linux' not in flag.lower(): - # use toolchained sanitize libraries - return cmd - - CLANG_RT = 'contrib/libs/clang14-rt/lib/' - sanitize_flags = { - '-fsanitize=address': CLANG_RT + 'asan', - '-fsanitize=memory': CLANG_RT + 'msan', - '-fsanitize=leak': CLANG_RT + 'lsan', - '-fsanitize=undefined': CLANG_RT + 'ubsan', - '-fsanitize=thread': CLANG_RT + 'tsan' - } - - used_sanitize_libs = [] - aux = [] - for flag in cmd: - if flag.startswith('-fsanitize-coverage='): - # do not link sanitizer libraries from clang - aux.append('-fno-sanitize-link-runtime') - if flag in sanitize_flags and any(s.startswith(sanitize_flags[flag]) for s in cmd): - # exclude '-fsanitize=' if appropriate library is linked explicitly - continue - if any(flag.startswith(lib) for lib in sanitize_flags.values()): - used_sanitize_libs.append(flag) - continue - aux.append(flag) - - # move sanitize libraries out of the repeatedly searched group of archives - flags = [] - for flag in aux: - if flag == '-Wl,--start-group': - flags += ['-Wl,--whole-archive'] + used_sanitize_libs + ['-Wl,--no-whole-archive'] - flags.append(flag) - - return flags - - -def fix_cmd_for_musl(cmd): - flags = [] - for flag in cmd: - if flag not in MUSL_LIBS: - flags.append(flag) - return flags - - -def fix_cmd_for_dynamic_cuda(cmd): - flags = [] - for flag in cmd: - if flag in CUDA_LIBRARIES: - flags.append(CUDA_LIBRARIES[flag]) - else: - flags.append(flag) - return flags - - -def gen_default_suppressions(inputs, output, source_root): - import collections - import os - - supp_map = collections.defaultdict(set) - for filename in inputs: - sanitizer = os.path.basename(filename).split('.', 1)[0] - with open(os.path.join(source_root, filename)) as src: - for line in src: - line = line.strip() - if not line or line.startswith('#'): - continue - supp_map[sanitizer].add(line) - - with open(output, "wb") as dst: - for supp_type, supps in supp_map.items(): - dst.write('extern "C" const char *__%s_default_suppressions() {\n' % supp_type) - dst.write(' return "{}";\n'.format('\\n'.join(sorted(supps)))) - dst.write('}\n') - - -def parse_args(): - parser = optparse.OptionParser() - parser.disable_interspersed_args() - parser.add_option('--musl', action='store_true') - parser.add_option('--custom-step') - parser.add_option('--python') - parser.add_option('--source-root') - parser.add_option('--dynamic-cuda', action='store_true') - parser.add_option('--arch') - parser.add_option('--linker-output') - parser.add_option('--whole-archive-peers', action='append') - parser.add_option('--whole-archive-libs', action='append') - return parser.parse_args() - - -if __name__ == '__main__': - opts, args = parse_args() - - cmd = remove_excessive_flags(args) - if opts.musl: - cmd = fix_cmd_for_musl(cmd) - - cmd = fix_sanitize_flag(cmd) - - if opts.dynamic_cuda: - cmd = fix_cmd_for_dynamic_cuda(cmd) - cmd = ProcessWholeArchiveOption(opts.arch, opts.whole_archive_peers, opts.whole_archive_libs).construct_cmd(cmd) - - if opts.custom_step: - assert opts.python - subprocess.check_call([opts.python] + [opts.custom_step] + args) - - supp, cmd = get_leaks_suppressions(cmd) - if supp: - src_file = "default_suppressions.cpp" - gen_default_suppressions(supp, src_file, opts.source_root) - cmd += [src_file] - - if opts.linker_output: - stdout = open(opts.linker_output, 'w') - else: - stdout = sys.stdout - - rc = subprocess.call(cmd, shell=False, stderr=sys.stderr, stdout=stdout) - sys.exit(rc) diff --git a/build/scripts/link_fat_obj.py b/build/scripts/link_fat_obj.py deleted file mode 100644 index c189668b9e..0000000000 --- a/build/scripts/link_fat_obj.py +++ /dev/null @@ -1,91 +0,0 @@ -import argparse -import subprocess -import sys - -from process_whole_archive_option import ProcessWholeArchiveOption - -YA_ARG_PREFIX = '-Ya,' - - -def get_args(): - parser = argparse.ArgumentParser() - parser.add_argument('--obj') - parser.add_argument('--globals-lib') - parser.add_argument('--lib', required=True) - parser.add_argument('--arch', required=True) - parser.add_argument('--build-root', default=None) - parser.add_argument('--with-own-obj', action='store_true', default=False) - parser.add_argument('--with-global-srcs', action='store_true', default=False) - - groups = {} - args_list = groups.setdefault('default', []) - for arg in sys.argv[1:]: - if arg == '--with-own-obj': - groups['default'].append(arg) - elif arg == '--globals-lib': - groups['default'].append(arg) - elif arg == '--with-global-srcs': - groups['default'].append(arg) - elif arg.startswith(YA_ARG_PREFIX): - group_name = arg[len(YA_ARG_PREFIX):] - args_list = groups.setdefault(group_name, []) - else: - args_list.append(arg) - - return parser.parse_args(groups['default']), groups - - -def strip_suppression_files(srcs): - return [s for s in srcs if not s.endswith('.supp')] - - -def main(): - args, groups = get_args() - - # Inputs - auto_input = groups['input'] - - # Outputs - lib_output = args.lib - obj_output = args.obj - - # Dependencies - global_srcs = groups['global_srcs'] - global_srcs = strip_suppression_files(global_srcs) - global_srcs = ProcessWholeArchiveOption(args.arch).construct_cmd(global_srcs) - peers = groups['peers'] - - # Tools - linker = groups['linker'] - archiver = groups['archiver'] - - do_link = linker + ['-o', obj_output, '-Wl,-r', '-nodefaultlibs', '-nostartfiles'] + global_srcs + auto_input - do_archive = archiver + [lib_output] + peers - do_globals = None - if args.globals_lib: - do_globals = archiver + [args.globals_lib] + auto_input + global_srcs - if args.with_own_obj: - do_archive += auto_input - if args.with_global_srcs: - do_archive += global_srcs - - def call(c): - proc = subprocess.Popen(c, shell=False, stderr=sys.stderr, stdout=sys.stdout, cwd=args.build_root) - proc.communicate() - return proc.returncode - - if obj_output: - link_res = call(do_link) - if link_res: - sys.exit(link_res) - - if do_globals: - glob_res = call(do_globals) - if glob_res: - sys.exit(glob_res) - - sys.exit(call(do_archive)) - - -if __name__ == '__main__': - main() diff --git a/build/scripts/link_jsrc.py b/build/scripts/link_jsrc.py deleted file mode 100644 index feae72fe4e..0000000000 --- a/build/scripts/link_jsrc.py +++ /dev/null @@ -1,27 +0,0 @@ -import argparse -import tarfile - - -def parse_args(): - parser = argparse.ArgumentParser() - parser.add_argument('--input', nargs='*') - parser.add_argument('--output', required=True) - - return parser.parse_args() - - -def main(): - args = parse_args() - - with tarfile.open(args.output, 'w') as dest: - for jsrc in [j for j in args.input if j.endswith('.jsrc')]: - with tarfile.open(jsrc, 'r') as src: - for item in [m for m in src.getmembers() if m.name != '']: - if item.isdir(): - dest.addfile(item) - else: - dest.addfile(item, src.extractfile(item)) - - -if __name__ == '__main__': - main() diff --git a/build/scripts/link_lib.py b/build/scripts/link_lib.py deleted file mode 100644 index e73c02027a..0000000000 --- a/build/scripts/link_lib.py +++ /dev/null @@ -1,101 +0,0 @@ -import sys -import subprocess -import tempfile -import os -import shutil - - -class Opts(object): - def __init__(self, args): - self.archiver = args[0] - self.arch_type = args[1] - self.llvm_ar_format = args[2] - self.build_root = args[3] - self.plugin = args[4] - self.output = args[5] - auto_input = args[6:] - - self.need_modify = False - self.extra_args = [] - - if self.arch_type.endswith('_AR'): - if self.arch_type == 'GNU_AR': - self.create_flags = ['rcs'] - self.modify_flags = ['-M'] - elif self.arch_type == 'LLVM_AR': - self.create_flags = ['rcs', '--format=%s' % self.llvm_ar_format] - self.modify_flags = ['-M'] - self.need_modify = any(item.endswith('.a') for item in auto_input) - if self.need_modify: - self.objs = list( filter(lambda x: x.endswith('.o'), auto_input) ) - self.libs = list( filter(lambda x: x.endswith('.a'), auto_input) ) - else: - self.objs = auto_input - self.libs = [] - self.output_opts = [self.output] - elif self.arch_type == 'LIBTOOL': - self.create_flags = ['-static'] - self.objs = auto_input - self.libs = [] - self.output_opts = ['-o', self.output] - elif self.arch_type == 'LIB': - self.create_flags = [] - self.extra_args = list( filter(lambda x: x.startswith('/'), auto_input) ) - self.objs = list( filter(lambda x: not x.startswith('/'), auto_input) ) - self.libs = [] - self.output_opts = ['/OUT:' + self.output] - - self.plugin_flags = ['--plugin', self.plugin] if self.plugin != 'None' else [] - - -def get_opts(args): - return Opts(args) - - -if __name__ == "__main__": - opts = get_opts(sys.argv[1:]) - - # There is a bug in llvm-ar. Some files with size slightly greater 2^32 - # still have GNU format instead of GNU64 and cause link problems. - # Workaround just lowers llvm-ar's GNU64 threshold to 2^31. - if opts.arch_type == 'LLVM_AR': - os.environ['SYM64_THRESHOLD'] = '31' - - def call(): - try: - p = subprocess.Popen(cmd, stdin=stdin, cwd=opts.build_root) - rc = p.wait() - return rc - except OSError as e: - raise Exception('while running %s: %s' % (' '.join(cmd), e)) - - try: - os.unlink(opts.output) - except OSError: - pass - - if not opts.need_modify: - cmd = [opts.archiver] + opts.create_flags + opts.plugin_flags + opts.extra_args + opts.output_opts + opts.objs - stdin = None - exit_code = call() - elif len(opts.objs) == 0 and len(opts.libs) == 1: - shutil.copy(opts.libs[0], opts.output) - exit_code = 0 - else: - temp = tempfile.NamedTemporaryFile(dir=os.path.dirname(opts.output), delete=False) - - with open(temp.name, 'w') as tmp: - tmp.write('CREATE {0}\n'.format(opts.output)) - for lib in opts.libs: - tmp.write('ADDLIB {0}\n'.format(lib)) - for obj in opts.objs: - tmp.write('ADDMOD {0}\n'.format(obj)) - tmp.write('SAVE\n') - tmp.write('END\n') - cmd = [opts.archiver] + opts.modify_flags + opts.plugin_flags - stdin = open(temp.name) - exit_code = call() - os.remove(temp.name) - - if exit_code != 0: - raise Exception('{0} returned non-zero exit code {1}. Stop.'.format(' '.join(cmd), exit_code)) diff --git a/build/scripts/list.py b/build/scripts/list.py deleted file mode 100644 index 7c3b2ae695..0000000000 --- a/build/scripts/list.py +++ /dev/null @@ -1,4 +0,0 @@ -import sys
-
-if __name__ == "__main__":
- print(' '.join(sys.argv[1:]))
diff --git a/build/scripts/llvm_opt_wrapper.py b/build/scripts/llvm_opt_wrapper.py deleted file mode 100644 index 38ca3004af..0000000000 --- a/build/scripts/llvm_opt_wrapper.py +++ /dev/null @@ -1,18 +0,0 @@ -import subprocess -import sys - - -def fix(s): - # we use '#' instead of ',' because ymake always splits args by comma - if 'internalize' in s: - return s.replace('#', ',') - - return s - - -if __name__ == '__main__': - path = sys.argv[1] - args = [fix(s) for s in [path] + sys.argv[2:]] - - rc = subprocess.call(args, shell=False, stderr=sys.stderr, stdout=sys.stdout) - sys.exit(rc) diff --git a/build/scripts/make_container.py b/build/scripts/make_container.py deleted file mode 100644 index a485baffdd..0000000000 --- a/build/scripts/make_container.py +++ /dev/null @@ -1,94 +0,0 @@ -import os -import shutil -import stat -import struct -import subprocess -import sys - -import container # 1 - - -def main(output_path, entry_path, input_paths, squashfs_path): - output_tmp_path = output_path + '.tmp' - shutil.copy2(entry_path, output_tmp_path) - st = os.stat(output_tmp_path) - os.chmod(output_tmp_path, st.st_mode | stat.S_IWUSR) - - layer_paths = [] - other_paths = [] - for input_path in input_paths: - (layer_paths if input_path.endswith('.container_layer') else other_paths).append(input_path) - - if len(other_paths) == 0: - raise Exception('No program in container dependencies') - - if len(other_paths) > 1: - raise Exception('Multiple non-layer inputs') - - program_path = other_paths[0] - program_container_path = os.path.basename(program_path) - - os.symlink(program_container_path, 'entry') - add_cmd = [ os.path.join(squashfs_path, 'mksquashfs') ] - add_cmd.extend([program_path, 'entry', 'program_layer']) - subprocess.run(add_cmd) - - layer_paths.append('program_layer') - - container.join_layers(layer_paths, 'container_data', squashfs_path) - - size = 0 - block_size = 1024 * 1024 - - with open(output_tmp_path, 'ab') as output: - with open('container_data', 'rb') as input_: - while True: - data = input_.read(block_size) - output.write(data) - size += len(data) - - if len(data) < block_size: - break - - with open(os.path.join(squashfs_path, 'unsquashfs'), 'rb') as input_: - while True: - data = input_.read(block_size) - output.write(data) - size += len(data) - - if len(data) == 0: - break - - - output.write(struct.pack('<Q', size)) - - os.rename(output_tmp_path, output_path) - - -def entry(): - import argparse - - parser = argparse.ArgumentParser() - parser.add_argument('-o', '--output', required=True) - parser.add_argument('-s', '--squashfs-path', required=True) - parser.add_argument('input', nargs='*') - - args = parser.parse_args() - - def is_container_entry(path): - return os.path.basename(path) == '_container_entry' - - input_paths = [] - entry_paths = [] - - for input_path in args.input: - (entry_paths if is_container_entry(input_path) else input_paths).append(input_path) - - if len(entry_paths) != 1: - raise Exception('Could not select container entry from {}'.format(entry_paths)) - - return main(args.output, entry_paths[0], input_paths, args.squashfs_path) - - -if __name__ == '__main__': - sys.exit(entry()) diff --git a/build/scripts/make_container_layer.py b/build/scripts/make_container_layer.py deleted file mode 100644 index 4f61f5a2e5..0000000000 --- a/build/scripts/make_container_layer.py +++ /dev/null @@ -1,24 +0,0 @@ -import sys - -import container # 1 - - -class UserError(Exception): - pass - - -def entry(): - import argparse - - parser = argparse.ArgumentParser() - parser.add_argument('-o', '--output', required=True) - parser.add_argument('-s', '--squashfs-path', required=True) - parser.add_argument('input', nargs='*') - - args = parser.parse_args() - - return container.join_layers(args.input, args.output, args.squashfs_path) - - -if __name__ == '__main__': - sys.exit(entry()) diff --git a/build/scripts/make_java_classpath_file.py b/build/scripts/make_java_classpath_file.py deleted file mode 100644 index c70a7876d7..0000000000 --- a/build/scripts/make_java_classpath_file.py +++ /dev/null @@ -1,26 +0,0 @@ -import os -import sys - -import process_command_files as pcf - - -def make_cp_file(args): - source = args[0] - destination = args[1] - with open(source) as src: - lines = [l.strip() for l in src if l.strip()] - with open(destination, 'w') as dst: - dst.write(os.pathsep.join(lines)) - -def make_cp_file_from_args(args): - destination = args[0] - with open(destination, 'w') as dst: - dst.write(os.pathsep.join(args[1:])) - - -if __name__ == '__main__': - args = pcf.get_args(sys.argv[1:]) - if sys.argv[1] != '--from-args': - make_cp_file(args) - else: - make_cp_file_from_args(args[1:]) diff --git a/build/scripts/make_java_srclists.py b/build/scripts/make_java_srclists.py deleted file mode 100644 index 65174bafd7..0000000000 --- a/build/scripts/make_java_srclists.py +++ /dev/null @@ -1,128 +0,0 @@ -import os -import sys -import argparse - -import process_command_files as pcf -import java_pack_to_file as jcov - - -def writelines(f, rng): - f.writelines(item + '\n' for item in rng) - - -def add_rel_src_to_coverage(coverage, src, source_root): - rel = os.path.relpath(src, source_root) - if not rel.startswith('..' + os.path.sep): - coverage.append(rel) - - -def main(): - args = pcf.get_args(sys.argv[1:]) - parser = argparse.ArgumentParser() - parser.add_argument('--moddir') - parser.add_argument('--java') - parser.add_argument('--groovy') - parser.add_argument('--kotlin') - parser.add_argument('--coverage') - parser.add_argument('--source-root') - args, remaining_args = parser.parse_known_args(args) - - java = [] - kotlin = [] - groovy = [] - coverage = [] - - cur_resources_list_file = None - cur_jsources_list_file = None - cur_srcdir = None - cur_resources = [] - cur_jsources = [] - - FILE_ARG = 1 - RESOURCES_DIR_ARG = 2 - SRCDIR_ARG = 3 - JSOURCES_DIR_ARG = 4 - - next_arg=FILE_ARG - - for src in remaining_args: - if next_arg == RESOURCES_DIR_ARG: - assert cur_resources_list_file is None - cur_resources_list_file = src - next_arg = FILE_ARG - continue - elif next_arg == JSOURCES_DIR_ARG: - assert cur_jsources_list_file is None - cur_jsources_list_file = src - next_arg = FILE_ARG - continue - elif next_arg == SRCDIR_ARG: - assert cur_srcdir is None - cur_srcdir = src if os.path.isabs(src) else os.path.join(args.moddir, src) - next_arg = FILE_ARG - continue - - if src.endswith(".java"): - java.append(src) - kotlin.append(src) - if args.coverage and args.source_root: - add_rel_src_to_coverage(coverage, src, args.source_root) - elif args.kotlin and src.endswith(".kt"): - kotlin.append(src) - if args.coverage and args.source_root: - add_rel_src_to_coverage(coverage, src, args.source_root) - elif args.groovy and src.endswith(".groovy"): - groovy.append(src) - else: - if src == '--resources': - if cur_resources_list_file is not None: - with open(cur_resources_list_file, 'w') as f: - writelines(f, cur_resources) - cur_resources_list_file = None - cur_srcdir = None - cur_resources = [] - next_arg = RESOURCES_DIR_ARG - continue - if src == '--jsources': - if cur_jsources_list_file is not None: - with open(cur_jsources_list_file, 'w') as f: - writelines(f, cur_jsources) - cur_jsources_list_file = None - cur_jsources = [] - next_arg = JSOURCES_DIR_ARG - continue - elif src == '--srcdir': - next_arg = SRCDIR_ARG - continue - else: - assert cur_srcdir is not None and cur_resources_list_file is not None - cur_resources.append(os.path.relpath(src, cur_srcdir)) - - if cur_jsources_list_file is not None: - assert cur_srcdir is not None - cur_jsources.append(os.path.relpath(src, cur_srcdir)) - - if cur_resources_list_file is not None: - with open(cur_resources_list_file, 'w') as f: - writelines(f, cur_resources) - if cur_jsources_list_file is not None: - with open(cur_jsources_list_file, 'w') as f: - writelines(f, cur_jsources) - - if args.java: - with open(args.java, 'w') as f: - writelines(f, java) - if args.kotlin: - with open(args.kotlin, 'w') as f: - writelines(f, kotlin) - if args.groovy: - with open(args.groovy, 'w') as f: - writelines(f, groovy) - if args.coverage: - jcov.write_coverage_sources(args.coverage, args.source_root, coverage) - - return 0 - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/build/scripts/make_manifest_from_bf.py b/build/scripts/make_manifest_from_bf.py deleted file mode 100644 index bfea3ba3de..0000000000 --- a/build/scripts/make_manifest_from_bf.py +++ /dev/null @@ -1,28 +0,0 @@ -import sys -import zipfile -import os -import re - - -def prepare_path(path): - return ('file:/' + path.lstrip('/')) if os.path.isabs(path) else path - - -def main(args): - bf, mf = args[0], args[1] - if not os.path.exists(os.path.dirname(mf)): - os.makedirs(os.path.dirname(mf)) - with open(bf) as f: - class_path = f.read().strip() - class_path = ' '.join(map(prepare_path, class_path.split('\n'))) - with zipfile.ZipFile(mf, 'w') as zf: - lines = [] - while class_path: - lines.append(class_path[:60]) - class_path = class_path[60:] - if lines: - zf.writestr('META-INF/MANIFEST.MF', 'Manifest-Version: 1.0\nClass-Path: \n ' + '\n '.join(lines) + ' \n\n') - - -if __name__ == '__main__': - main(sys.argv[1:]) diff --git a/build/scripts/mangle_typeinfo_names.py b/build/scripts/mangle_typeinfo_names.py deleted file mode 100755 index 8f30a46552..0000000000 --- a/build/scripts/mangle_typeinfo_names.py +++ /dev/null @@ -1,317 +0,0 @@ -#!/usr/bin/env python - -import base64 -import hashlib -import io -import os -import struct -import subprocess -import sys -from collections import namedtuple - - -""" -ELF-64 Object File Format: https://uclibc.org/docs/elf-64-gen.pdf -ELF-32: https://uclibc.org/docs/elf.pdf -""" - - -MANGLED_HASH_SIZE = 15 - -# len(base64(sha1(name)[:MANGLED_HASH_SIZE]) + '\x00') -MANGLED_NAME_SIZE = 21 - - -ArObject = namedtuple('ArObject', ['header', 'data']) - -ElfSection = namedtuple('ElfSection', ['header_offset', 'name', 'data_offset', 'size', 'link', 'entry_size']) - - -def find(it, pred): - return next(iter(filter(pred, it)), None) - - -def mangle_name(name_bytes): - sha1 = hashlib.sha1() - sha1.update(name_bytes) - dgst = sha1.digest() - return base64.b64encode(dgst[:MANGLED_HASH_SIZE]) - - -def unpack(format, buffer, offset=0): - return struct.unpack(format, buffer[offset : offset + struct.calcsize(format)]) - - -def unpack_section_header(buffer, offset, elf64): - # read sh_name, sh_offset, sh_size, sh_link, sh_entsize from section headers (Elf64_Shdr/Elf32_Shdr): - # - # typedef struct - # { - # Elf64_Word sh_name; /* Section name */ - # Elf64_Word sh_type; /* Section type */ - # Elf64_Xword sh_flags; /* Section attributes */ - # Elf64_Addr sh_addr; /* Virtual address in memory */ - # Elf64_Off sh_offset; /* Offset in file */ - # Elf64_Xword sh_size; /* Size of section */ - # Elf64_Word sh_link; /* Link to other section */ - # Elf64_Word sh_info; /* Miscellaneous information */ - # Elf64_Xword sh_addralign; /* Address alignment boundary */ - # Elf64_Xword sh_entsize; /* Size of entries, if section has table */ - # } Elf64_Shdr; - # - # typedef struct { - # Elf32_Word sh_name; - # Elf32_Word sh_type; - # Elf32_Word sh_flags; - # Elf32_Addr sh_addr; - # Elf32_Off sh_offset; - # Elf32_Word sh_size; - # Elf32_Word sh_link; - # Elf32_Word sh_info; - # Elf32_Word sh_addralign; - # Elf32_Word sh_entsize; - # } Elf32_Shdr; - - section_header_format = '< L 20x Q Q L 12x Q' if elf64 else '< L 12x L L L 8x L' - return ElfSection(offset, *unpack(section_header_format, buffer, offset)) - - -def read_elf_sections(elf_data, elf64): - # read e_shoff, e_shentsize, e_shnum, e_shstrndx from elf header (Elf64_Ehdr/Elf32_Ehdr): - # - # typedef struct - # { - # unsigned char e_ident[16]; /* ELF identification */ - # Elf64_Half e_type; /* Object file type */ - # Elf64_Half e_machine; /* Machine type */ - # Elf64_Word e_version; /* Object file version */ - # Elf64_Addr e_entry; /* Entry point address */ - # Elf64_Off e_phoff; /* Program header offset */ - # Elf64_Off e_shoff; /* Section header offset */ - # Elf64_Word e_flags; /* Processor-specific flags */ - # Elf64_Half e_ehsize; /* ELF header size */ - # Elf64_Half e_phentsize; /* Size of program header entry */ - # Elf64_Half e_phnum; /* Number of program header entries */ - # Elf64_Half e_shentsize; /* Size of section header entry */ - # Elf64_Half e_shnum; /* Number of section header entries */ - # Elf64_Half e_shstrndx; /* Section name string table index */ - # } Elf64_Ehdr; - # - # #define EI_NIDENT 16 - # - # typedef struct { - # unsigned char e_ident[EI_NIDENT]; - # Elf32_Half e_type; - # Elf32_Half e_machine; - # Elf32_Word e_version; - # Elf32_Addr e_entry; - # Elf32_Off e_phoff; - # Elf32_Off e_shoff; - # Elf32_Word e_flags; - # Elf32_Half e_ehsize; - # Elf32_Half e_phentsize; - # Elf32_Half e_phnum; - # Elf32_Half e_shentsize; - # Elf32_Half e_shnum; - # Elf32_Half e_shstrndx; - # } Elf32_Ehdr; - - section_header_offset, section_header_entry_size, section_header_entries_number,\ - section_name_string_table_index = unpack('< Q 10x 3H', elf_data, 40) if elf64 else unpack('< L 10x 3H', elf_data, 32) - - # https://refspecs.linuxfoundation.org/elf/gabi4+/ch4.sheader.html - # If the number of sections is greater than or equal to SHN_LORESERVE (0xff00), e_shnum has the value SHN_UNDEF (0) - # and the actual number of section header table entries is contained in the sh_size field of the section header - # at index 0 (otherwise, the sh_size member of the initial entry contains 0). - if section_header_entries_number == 0: - section_header_entries_number = unpack_section_header(elf_data, section_header_offset, elf64).size - - sections = [unpack_section_header(elf_data, section_header_offset + i * section_header_entry_size, elf64) - for i in range(section_header_entries_number)] - - # section names data - section_names_section = sections[section_name_string_table_index] - section_names_data = elf_data[section_names_section.data_offset : section_names_section.data_offset + section_names_section.size] - - # read section names - for i, section in enumerate(sections): - sections[i] = section._replace( - name=section_names_data[section.name : section_names_data.find(b'\x00', section.name)].decode()) - - return sections - - -def mangle_elf_typeinfo_names(elf_data, elf64, sections): - symbol_sizes = {} - - for sect_i, section in enumerate(sections): - if not section.name.startswith('.rodata._ZTS') or section.size <= MANGLED_NAME_SIZE: - continue - - typeinfo_name = elf_data[section.data_offset : section.data_offset + section.size] - mangled = mangle_name(typeinfo_name.rstrip(b'\x00')) + b'\x00' - if len(mangled) >= len(typeinfo_name): - continue - - # patch section data - elf_data[section.data_offset : section.data_offset + len(mangled)] = mangled - # patch section size (sh_size in Elf64_Shdr/Elf32_Shdr) - if elf64: - elf_data[section.header_offset + 32 : section.header_offset + 40] = struct.pack('< Q', len(mangled)) - else: - elf_data[section.header_offset + 20 : section.header_offset + 24] = struct.pack('< L', len(mangled)) - - symbol_sizes[section.name[len('.rodata.'):]] = len(mangled) - - return symbol_sizes - - -def patch_elf_symbol_sizes(elf_data, elf64, sections, symbol_sizes): - symtab = find(sections, lambda s: s.name == '.symtab') - if not symtab: - return - - for sym_i in range(symtab.size / symtab.entry_size): - symtab_entry_offset = symtab.data_offset + symtab.entry_size * sym_i - symtab_entry = elf_data[symtab_entry_offset : symtab_entry_offset + symtab.entry_size] - - # unpack symbol name offset in symbols name section (st_name) from Elf64_Sym/Elf32_Sym: - # - # typedef struct - # { - # Elf64_Word st_name; /* Symbol name */ - # unsigned char st_info; /* Type and Binding attributes */ - # unsigned char st_other; /* Reserved */ - # Elf64_Half st_shndx; /* Section table index */ - # Elf64_Addr st_value; /* Symbol value */ - # Elf64_Xword st_size; /* Size of object (e.g., common) */ - # } Elf64_Sym; - # - # typedef struct { - # Elf32_Word st_name; - # Elf32_Addr st_value; - # Elf32_Word st_size; - # unsigned char st_info; - # unsigned char st_other; - # Elf32_Half st_shndx; - # } Elf32_Sym; - symbol_name_offset = unpack('< L', symtab_entry)[0] - - # symbol name offset from start of elf file - global_name_offset = sections[symtab.link].data_offset + symbol_name_offset - - name = elf_data[global_name_offset : elf_data.find(b'\x00', global_name_offset)].decode() - symbol_size = symbol_sizes.get(name) - if symbol_size: - # patch st_size in Elf64_Sym/Elf32_Sym - if elf64: - elf_data[symtab_entry_offset + 16 : symtab_entry_offset + 24] = struct.pack('< Q', symbol_size) - else: - elf_data[symtab_entry_offset + 8 : symtab_entry_offset + 12] = struct.pack('< L', symbol_size) - - -def mangle_elf(elf_data): - elf_data = bytearray(elf_data) - - ei_mag, ei_class = unpack('4s B', elf_data) - assert ei_mag == b'\x7fELF' - if ei_class == 1: # ELFCLASS32 - elf64 = False - elif ei_class == 2: # ELFCLASS64 - elf64 = True - else: - raise Exception('unknown ei_class: ' + str(ei_class)) - - sections = read_elf_sections(elf_data, elf64) - - symbol_sizes = mangle_elf_typeinfo_names(elf_data, elf64, sections) - - if len(symbol_sizes) != 0: - patch_elf_symbol_sizes(elf_data, elf64, sections, symbol_sizes) - - return elf_data - - -def read_ar_object(ar): - # ar format: https://docs.oracle.com/cd/E36784_01/html/E36873/ar.h-3head.html - # - # #define ARFMAG "`\n" /* header trailer string */ - # - # struct ar_hdr /* file member header */ - # { - # char ar_name[16]; /* '/' terminated file member name */ - # char ar_date[12]; /* file member date */ - # char ar_uid[6] /* file member user identification */ - # char ar_gid[6] /* file member group identification */ - # char ar_mode[8] /* file member mode (octal) */ - # char ar_size[10]; /* file member size */ - # char ar_fmag[2]; /* header trailer string */ - # }; - - header = ar.read(60) - if len(header) == 0: - return None - assert header[58:] == b'`\n' - - size = int(bytes(header[48:58]).decode().rstrip(' ')) - data = ar.read(size) - return ArObject(header, data) - - -def is_elf_data(data): - return data[:4] == b'\x7fELF' - - -def mangle_ar_impl(ar, out): - ar_magic = ar.read(8) - if ar_magic != b'!<arch>\n': - raise Exception('bad ar magic: {}'.format(ar_magic)) - - out.write(ar_magic) - - string_table = None - - while True: - obj = read_ar_object(ar) - if not obj: - break - - data = mangle_elf(obj.data) if is_elf_data(obj.data) else obj.data - - out.write(obj.header) - out.write(data) - - -def mangle_ar(path): - out_path = path + '.mangled' - with open(path, 'rb') as ar: - try: - with open(out_path, 'wb') as out: - mangle_ar_impl(ar, out) - except: - os.unlink(out_path) - raise - - os.rename(out_path, path) - - -def main(): - for arg in sys.argv[1:]: - if not ((arg.endswith('.o') or arg.endswith('.a')) and os.path.exists(arg)): - continue - - if arg.endswith('.o'): - with open(arg, 'rb') as o: - data = o.read() - mangled = mangle_elf(data) if is_elf_data(data) else None - - if mangled: - os.unlink(arg) - with open(arg, 'wb') as o: - o.write(mangled) - elif arg.endswith('.a'): - mangle_ar(arg) - - -if __name__ == '__main__': - main() diff --git a/build/scripts/merge_coverage_data.py b/build/scripts/merge_coverage_data.py deleted file mode 100644 index b7fa3c6a86..0000000000 --- a/build/scripts/merge_coverage_data.py +++ /dev/null @@ -1,32 +0,0 @@ -import sys -import tarfile -import copy -import os -import uuid - - -def main(args): - output_file, args = args[0], args[1:] - # heretic@: Splits files on which could be merged( files ) and which should not be merged( expendables ) - # expendables will be in output_file in form {name}{ordinal number of archive in args[]}.{extension} - try: - split_i = args.index('-no-merge') - except ValueError: - split_i = len(args) - files, expendables = args[:split_i], args[split_i + 1:] - - with tarfile.open(output_file, 'w') as outf: - for x in files: - with tarfile.open(x) as tf: - for tarinfo in tf: - new_tarinfo = copy.deepcopy(tarinfo) - if new_tarinfo.name in expendables: - dirname, basename = os.path.split(new_tarinfo.name) - basename_parts = basename.split('.', 1) - new_basename = '.'.join([basename_parts[0] + str(uuid.uuid4())] + basename_parts[1:]) - new_tarinfo.name = os.path.join(dirname, new_basename) - outf.addfile(new_tarinfo, tf.extractfile(tarinfo)) - - -if __name__ == '__main__': - main(sys.argv[1:]) diff --git a/build/scripts/merge_files.py b/build/scripts/merge_files.py deleted file mode 100644 index d42d6a2139..0000000000 --- a/build/scripts/merge_files.py +++ /dev/null @@ -1,8 +0,0 @@ -import sys - - -if __name__ == "__main__": - with open(sys.argv[1], "w") as f: - for appended in sys.argv[2:]: - with open(appended) as a: - f.write(a.read()) diff --git a/build/scripts/mkdir.py b/build/scripts/mkdir.py deleted file mode 100755 index a326b29300..0000000000 --- a/build/scripts/mkdir.py +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env python -import os -import sys - - -def mkdir_p(directory): - if not os.path.exists(directory): - os.makedirs(directory) - -if __name__ == "__main__": - for directory in sys.argv[1:]: - mkdir_p(directory) diff --git a/build/scripts/mkdocs_builder_wrapper.py b/build/scripts/mkdocs_builder_wrapper.py deleted file mode 100644 index 7a0df04190..0000000000 --- a/build/scripts/mkdocs_builder_wrapper.py +++ /dev/null @@ -1,36 +0,0 @@ -from __future__ import unicode_literals -import os -import subprocess -import sys - - -def main(): - cmd = [] - build_root = sys.argv[1] - length = len(build_root) - is_dep = False - for arg in sys.argv[2:]: - if is_dep: - is_dep = False - if not arg.endswith('.tar.gz'): - continue - basename = os.path.basename(arg) - assert arg.startswith(build_root) and len(arg) > length + len(basename) and arg[length] in ('/', '\\') - cmd.extend([str('--dep'), str('{}:{}:{}'.format(build_root, os.path.dirname(arg[length+1:]), basename))]) - elif arg == '--dep': - is_dep = True - else: - cmd.append(arg) - assert not is_dep - p = subprocess.Popen(cmd, stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, err = p.communicate() - if p.returncode: - if out: - sys.stderr.write('stdout:\n{}\n'.format(out.decode('utf-8'))) - if err: - sys.stderr.write('stderr:\n{}\n'.format(err.decode('utf-8'))) - sys.exit(p.returncode) - - -if __name__ == '__main__': - main() diff --git a/build/scripts/mkver.py b/build/scripts/mkver.py deleted file mode 100755 index 321cdaade1..0000000000 --- a/build/scripts/mkver.py +++ /dev/null @@ -1,12 +0,0 @@ -import sys - -if __name__ == '__main__': - with open(sys.argv[1], 'r') as f: - data = f.readline() - - beg = data.find('(') + 1 - end = data.find(')') - version = data[beg:end] - - print '#pragma once' - print '#define DEBIAN_VERSION "%s"' % version diff --git a/build/scripts/move.py b/build/scripts/move.py deleted file mode 100644 index 3f611fbc2e..0000000000 --- a/build/scripts/move.py +++ /dev/null @@ -1,15 +0,0 @@ -import os -import sys - -# /script/move.py <src-1> <tgt-1> <src-2> <tgt-2> ... <src-n> <tgt-n> -# renames src-1 to tgt-1, src-2 to tgt-2, ..., src-n to tgt-n. - - -def main(): - assert len(sys.argv) % 2 == 1 - for index in range(1, len(sys.argv), 2): - os.rename(sys.argv[index], sys.argv[index + 1]) - - -if __name__ == '__main__': - main() diff --git a/build/scripts/pack_ios.py b/build/scripts/pack_ios.py deleted file mode 100644 index 37c36d1f95..0000000000 --- a/build/scripts/pack_ios.py +++ /dev/null @@ -1,48 +0,0 @@ -import argparse -import os -import shutil -import subprocess -import sys -import tarfile - - -def just_do_it(): - parser = argparse.ArgumentParser() - parser.add_argument("--binary", required=True, help="executable file") - parser.add_argument("--target", required=True, help="target archive path") - parser.add_argument("--temp-dir", required=True, help="temp dir") - parser.add_argument("peers", nargs='*') - args = parser.parse_args() - app_tar = [p for p in args.peers if p.endswith('.ios.interface')] - if not app_tar: - print >> sys.stderr, 'No one IOS_INTERFACE module found' - shutil.copyfile(args.binary, os.path.join(args.temp_dir, 'bin')) - if os.path.exists(args.target): - os.remove(args.target) - with tarfile.open(args.target, 'w') as tf: - tf.add(os.path.join(args.temp_dir, 'bin'), arcname=os.path.join(os.path.basename(args.binary) + '.app', 'bin')) - return - if len(app_tar) > 1: - app_tar = [p for p in args.peers if not p.endswith('.default.ios.interface')] - if len(app_tar) > 1: - print >> sys.stderr, 'Many IOS_INTERFACE modules found, {} will be used'.format(app_tar[-1]) - app_tar = app_tar[-1] - with tarfile.open(app_tar) as tf: - tf.extractall(args.temp_dir) - tar_suffix = '.default.ios.interface' if app_tar.endswith('.default.ios.interface') else '.ios.interface' - app_unpacked_path = os.path.join(args.temp_dir, os.path.basename(app_tar)[:-len(tar_suffix)] + '.app') - if not os.path.exists(app_unpacked_path): - raise Exception('Bad IOS_INTERFACE resource: {}'.format(app_tar)) - shutil.copyfile(args.binary, os.path.join(app_unpacked_path, 'bin')) - subprocess.check_call(['/usr/bin/codesign', '--force', '--sign', '-', app_unpacked_path]) - if os.path.exists(args.target): - os.remove(args.target) - binary_origin_name = os.path.basename(args.binary) - while os.path.splitext(binary_origin_name)[1]: - binary_origin_name = os.path.splitext(binary_origin_name)[0] - with tarfile.open(args.target, 'w') as tf: - tf.add(app_unpacked_path, arcname=binary_origin_name + '.app', recursive=True) - - -if __name__ == '__main__': - just_do_it() diff --git a/build/scripts/pack_jcoverage_resources.py b/build/scripts/pack_jcoverage_resources.py deleted file mode 100644 index f6e181067a..0000000000 --- a/build/scripts/pack_jcoverage_resources.py +++ /dev/null @@ -1,24 +0,0 @@ -import sys -import tarfile -import os -import subprocess - - -def main(args): - output_file = args[0] - report_file = args[1] - - res = subprocess.call(args[args.index('-end') + 1:]) - - if not os.path.exists(report_file): - print>>sys.stderr, 'Can\'t find jacoco exec file' - return res - - with tarfile.open(output_file, 'w') as outf: - outf.add(report_file, arcname=os.path.basename(report_file)) - - return res - - -if __name__ == '__main__': - sys.exit(main(sys.argv[1:])) diff --git a/build/scripts/perl_wrapper.py b/build/scripts/perl_wrapper.py deleted file mode 100644 index cb4027f1d3..0000000000 --- a/build/scripts/perl_wrapper.py +++ /dev/null @@ -1,24 +0,0 @@ -import os -import sys -import shutil - -if __name__ == '__main__': - path = sys.argv[1] - to = sys.argv[-1] - fr = sys.argv[-2] - to_dir = os.path.dirname(to) - - os.chdir(to_dir) - - f1 = os.path.basename(fr) - fr_ = os.path.dirname(fr) - f2 = os.path.basename(fr_) - fr_ = os.path.dirname(fr_) - - os.makedirs(f2) - shutil.copyfile(fr, os.path.join(f2, f1)) - - if path[0] != '/': - path = os.path.join(os.path.dirname(__file__), path) - - os.execv(path, [path] + sys.argv[2:]) diff --git a/build/scripts/postprocess_go_fbs.py b/build/scripts/postprocess_go_fbs.py deleted file mode 100644 index e6d5184a5e..0000000000 --- a/build/scripts/postprocess_go_fbs.py +++ /dev/null @@ -1,72 +0,0 @@ -import argparse -import re -import os - - -# very simple regexp to find go import statement in the source code -# NOTE! only one-line comments are somehow considered -IMPORT_DECL=re.compile(r''' - \bimport - ( - \s+((\.|\w+)\s+)?"[^"]+" ( \s+//[^\n]* )? - | \s* \( \s* ( ( \s+ ((\.|\w+)\s+)? "[^"]+" )? ( \s* //[^\n]* )? )* \s* \) - )''', re.MULTILINE | re.DOTALL | re.VERBOSE) - - -def parse_args(): - parser = argparse.ArgumentParser() - parser.add_argument('--input-dir', required=True) - parser.add_argument('--map', nargs='*', default=None) - - return parser.parse_args() - - -def process_go_file(file_name, import_map): - content = '' - with open(file_name, 'r') as f: - content = f.read() - - start = -1 - end = -1 - for it in IMPORT_DECL.finditer(content): - if start < 0: - start = it.start() - end = it.end() - - if start < 0: - return - - imports = content[start:end] - for namespace, path in import_map.items(): - ns = namespace.split('.') - name = '__'.join(ns) - import_path = '/'.join(ns) - imports = imports.replace('{} "{}"'.format(name, import_path), '{} "a.yandex-team.ru/{}"'.format(name, path)) - - if imports != content[start:end]: - with open(file_name, 'w') as f: - f.write(content[:start]) - f.write(imports) - f.write(content[end:]) - - -def main(): - args = parse_args() - - if not args.map: - return - - raw_import_map = sorted(set(args.map)) - import_map = dict(z.split('=', 1) for z in raw_import_map) - if len(raw_import_map) != len(import_map): - for k, v in (z.split('=', 1) for z in raw_import_map): - if v != import_map[k]: - raise Exception('import map [{}] contains different values for key [{}]: [{}] and [{}].'.format(args.map, k, v, import_map[k])) - - for root, _, files in os.walk(args.input_dir): - for src in (f for f in files if f.endswith('.go')): - process_go_file(os.path.join(root, src), import_map) - - -if __name__ == '__main__': - main() diff --git a/build/scripts/preprocess.py b/build/scripts/preprocess.py deleted file mode 100644 index 4657bef732..0000000000 --- a/build/scripts/preprocess.py +++ /dev/null @@ -1,48 +0,0 @@ -import sys -import os - - -def load_file(p): - with open(p, 'r') as f: - return f.read() - - -def step(base, data, hh): - def flt(): - for l in data.split('\n'): - if l in hh: - pp = os.path.join(base, hh[l]) - - yield '\n\n' + load_file(pp) + '\n\n' - - os.unlink(pp) - else: - yield l - - return '\n'.join(flt()) - - -def subst_headers(path, headers): - hh = dict() - - for h in headers: - hh['# include "' + h + '"'] = h - - data = load_file(path) - prev = data - - while True: - ret = step(os.path.dirname(path), prev, hh) - - if ret == prev: - break - - prev = ret - - if data != prev: - with open(path, 'w') as f: - f.write(prev) - - -if __name__ == '__main__': - subst_headers(sys.argv[1], ['stack.hh', 'position.hh', 'location.hh']) diff --git a/build/scripts/process_whole_archive_option.py b/build/scripts/process_whole_archive_option.py deleted file mode 100644 index a9c4ef676a..0000000000 --- a/build/scripts/process_whole_archive_option.py +++ /dev/null @@ -1,176 +0,0 @@ -import os -import sys - -import process_command_files as pcf - - -class ProcessWholeArchiveOption(): - def __init__(self, arch, peers=None, libs=None): - self.arch = arch.upper() - self.peers = { x : 0 for x in peers } if peers else None - self.libs = { x : 0 for x in libs } if libs else None - self.start_wa_marker = '--start-wa' - self.end_wa_marker = '--end-wa' - - def _match_peer_lib(self, arg, ext): - key = None - if arg.endswith(ext): - key = os.path.dirname(arg) - return key if key and self.peers and key in self.peers else None - - def _match_lib(self, arg): - return arg if self.libs and arg in self.libs else None - - def _process_arg(self, arg, ext='.a'): - peer_key = self._match_peer_lib(arg, ext) - lib_key = self._match_lib(arg) - if peer_key: - self.peers[peer_key] += 1 - if lib_key: - self.libs[lib_key] += 1 - return peer_key if peer_key else lib_key - - def _check_peers(self): - if self.peers: - for key, value in self.peers.items(): - assert value > 0, '"{}" specified in WHOLE_ARCHIVE() macro is not used on link command'.format(key) - - def _construct_cmd_apple(self, args): - force_load_flag = '-Wl,-force_load,' - is_inside_wa_markers = False - - cmd = [] - for arg in args: - if arg.startswith(force_load_flag): - cmd.append(arg) - elif arg == self.start_wa_marker: - is_inside_wa_markers = True - elif arg == self.end_wa_marker: - is_inside_wa_markers = False - elif is_inside_wa_markers: - cmd.append(force_load_flag + arg) - else: - key = self._process_arg(arg) - cmd.append(force_load_flag + arg if key else arg) - - self._check_peers() - - return cmd - - def _construct_cmd_win(self, args): - whole_archive_prefix = '/WHOLEARCHIVE:' - is_inside_wa_markers = False - - def add_prefix(arg, need_check_peers_and_libs): - key = self._process_arg(arg, '.lib') if need_check_peers_and_libs else arg - return whole_archive_prefix + arg if key else arg - - def add_whole_archive_prefix(arg, need_check_peers_and_libs): - if not pcf.is_cmdfile_arg(arg): - return add_prefix(arg, need_check_peers_and_libs) - - cmd_file_path = pcf.cmdfile_path(arg) - cf_args = pcf.read_from_command_file(cmd_file_path) - with open(cmd_file_path, 'w') as afile: - for cf_arg in cf_args: - afile.write(add_prefix(cf_arg, need_check_peers_and_libs) + "\n") - return arg - - cmd = [] - for arg in args: - if arg == self.start_wa_marker: - is_inside_wa_markers = True - elif arg == self.end_wa_marker: - is_inside_wa_markers = False - elif is_inside_wa_markers: - cmd.append(add_whole_archive_prefix(arg, False)) - continue - elif self.peers or self.libs: - cmd.append(add_whole_archive_prefix(arg, True)) - else: - cmd.append(arg) - - self._check_peers() - - return cmd - - def _construct_cmd_linux(self, args): - whole_archive_flag = '-Wl,--whole-archive' - no_whole_archive_flag = '-Wl,--no-whole-archive' - - def replace_markers(arg): - if arg == self.start_wa_marker: - return whole_archive_flag - if arg == self.end_wa_marker: - return no_whole_archive_flag - return arg - - args = [replace_markers(arg) for arg in args] - - cmd = [] - is_inside_whole_archive = False - is_whole_archive = False - # We are trying not to create excessive sequences of consecutive flags - # -Wl,--no-whole-archive -Wl,--whole-archive ('externally' specified - # flags -Wl,--[no-]whole-archive are not taken for consideration in this - # optimization intentionally) - for arg in args: - if arg == whole_archive_flag: - is_inside_whole_archive = True - is_whole_archive = False - elif arg == no_whole_archive_flag: - is_inside_whole_archive = False - is_whole_archive = False - else: - key = self._process_arg(arg) - if not is_inside_whole_archive: - if key: - if not is_whole_archive: - cmd.append(whole_archive_flag) - is_whole_archive = True - elif is_whole_archive: - cmd.append(no_whole_archive_flag) - is_whole_archive = False - - cmd.append(arg) - - if is_whole_archive: - cmd.append(no_whole_archive_flag) - - self._check_peers() - - return cmd - - def construct_cmd(self, args): - if self.arch in ('DARWIN', 'IOS', 'IOSSIM'): - return self._construct_cmd_apple(args) - - if self.arch == 'WINDOWS': - return self._construct_cmd_win(args) - - return self._construct_cmd_linux(args) - - -def get_whole_archive_peers_and_libs(args): - remaining_args = [] - peers = [] - libs = [] - peers_flag = '--whole-archive-peers' - libs_flag = '--whole-archive-libs' - - next_is_peer = False - next_is_lib = False - for arg in args: - if arg == peers_flag: - next_is_peer = True - elif arg == libs_flag: - next_is_lib = True - elif next_is_peer: - peers.append(arg) - next_is_peer = False - elif next_is_lib: - libs.append(arg) - next_is_lib = False - else: - remaining_args.append(arg) - return remaining_args, peers, libs diff --git a/build/scripts/py_compile.py b/build/scripts/py_compile.py deleted file mode 100755 index 936dbe8816..0000000000 --- a/build/scripts/py_compile.py +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from __future__ import print_function, absolute_import, division - -import marshal -import sys - - -def main(): - srcpathx, in_fname, out_fname = sys.argv[1:] - srcpath = srcpathx[:-1] - - with open(in_fname, 'r') as in_file: - source = in_file.read() - - code = compile(source, srcpath, 'exec', dont_inherit=True) - - with open(out_fname, 'wb') as out_file: - marshal.dump(code, out_file) - - -if __name__ == "__main__": - main() diff --git a/build/scripts/python_yndexer.py b/build/scripts/python_yndexer.py deleted file mode 100644 index 3180665387..0000000000 --- a/build/scripts/python_yndexer.py +++ /dev/null @@ -1,53 +0,0 @@ -import os -import sys -import threading -import subprocess - - -def _try_to_kill(process): - try: - process.kill() - except Exception: - pass - - -def touch(path): - if not os.path.exists(path): - with open(path, 'w') as _: - pass - - -class Process(object): - def __init__(self, args): - self._process = subprocess.Popen(args) - self._event = threading.Event() - self._result = None - thread = threading.Thread(target=self._run) - thread.setDaemon(True) - thread.start() - - def _run(self): - self._process.communicate() - self._result = self._process.returncode - self._event.set() - - def wait(self, timeout): - self._event.wait(timeout=timeout) - _try_to_kill(self._process) - return self._result - - -if __name__ == '__main__': - yndexer = sys.argv[1] - timeout = int(sys.argv[2]) - output_file = sys.argv[3] - input_file = sys.argv[4] - partition_count = sys.argv[5] - partition_index = sys.argv[6] - - process = Process([yndexer, '-f', input_file, '-y', output_file, '-c', partition_count, '-i', partition_index]) - result = process.wait(timeout=timeout) - - if result != 0: - print >> sys.stderr, 'Yndexing process finished with code', result - touch(output_file) diff --git a/build/scripts/resolve_java_srcs.py b/build/scripts/resolve_java_srcs.py deleted file mode 100644 index a2e6c20012..0000000000 --- a/build/scripts/resolve_java_srcs.py +++ /dev/null @@ -1,106 +0,0 @@ -import os -import argparse -import re -import sys - - -def list_all_files(directory, prefix='/', hidden_files=False): - result = [] - if os.path.exists(directory): - for i in os.listdir(directory): - abs_path = os.path.join(directory, i) - result += list_all_files(os.path.join(directory, abs_path), prefix + i + '/', hidden_files) \ - if os.path.isdir(abs_path) else ([prefix + i] if (hidden_files or not i.startswith('.')) else []) - return result - - -def pattern_to_regexp(p): - return '^' + \ - ('/' if not p.startswith('**') else '') + \ - re.escape(p).replace( - r'\*\*\/', '[_DIR_]' - ).replace( - r'\*', '[_FILE_]' - ).replace( - '[_DIR_]', '(.*/)?' - ).replace( - '[_FILE_]', '([^/]*)' - ) + '$' - - -def resolve_java_srcs(srcdir, include_patterns, exclude_patterns, all_resources, resolve_kotlin=False, resolve_groovy=False): - result = {'java': [], 'not_java': [], 'kotlin': [], 'groovy': []} - include_patterns_normal, include_patterns_hidden, exclude_patterns_normal, exclude_patterns_hidden = [], [], [], [] - for vis, hid, patterns in ((include_patterns_normal, include_patterns_hidden, include_patterns), (exclude_patterns_normal, exclude_patterns_hidden, exclude_patterns),): - for pattern in patterns: - if (pattern if pattern.find('/') == -1 else pattern.rsplit('/', 1)[1]).startswith('.'): - hid.append(pattern) - else: - vis.append(pattern) - re_patterns = map(pattern_to_regexp, vis + hid) - if sys.platform in ('win32', 'darwin'): - re_patterns = [re.compile(i, re.IGNORECASE) for i in re_patterns] - else: - re_patterns = [re.compile(i) for i in re_patterns] - vis[:], hid[:] = re_patterns[:len(vis)], re_patterns[len(vis):] - - for inc_patterns, exc_patterns, with_hidden_files in ( - (include_patterns_normal, exclude_patterns_normal, False), - (include_patterns_hidden, exclude_patterns_hidden, True), - ): - for f in list_all_files(srcdir, hidden_files=with_hidden_files): - excluded = False - - for exc_re in exc_patterns: - if exc_re.match(f): - excluded = True - break - - if excluded: - continue - - for inc_re in inc_patterns: - if inc_re.match(f): - s = os.path.normpath(f[1:]) - if all_resources or not (f.endswith('.java') or f.endswith('.kt') or f.endswith('.groovy')): - result['not_java'].append(s) - elif f.endswith('.java'): - result['java'].append(os.path.join(srcdir, s)) - elif f.endswith('.kt') and resolve_kotlin: - result['kotlin'].append(os.path.join(srcdir, s)) - elif f.endswith('.groovy') and resolve_groovy: - result['groovy'].append(os.path.join(srcdir, s)) - else: - result['not_java'].append(s) - break - - return sorted(result['java']), sorted(result['not_java']), sorted(result['kotlin']), sorted(result['groovy']) - - -def do_it(directory, sources_file, resources_file, kotlin_sources_file, groovy_sources_file, include_patterns, exclude_patterns, resolve_kotlin, resolve_groovy, append, all_resources): - j, r, k, g = resolve_java_srcs(directory, include_patterns, exclude_patterns, all_resources, resolve_kotlin, resolve_groovy) - mode = 'a' if append else 'w' - open(sources_file, mode).writelines(i + '\n' for i in j) - open(resources_file, mode).writelines(i + '\n' for i in r) - if kotlin_sources_file: - open(kotlin_sources_file, mode).writelines(i + '\n' for i in k + j) - if groovy_sources_file: - open(groovy_sources_file, mode).writelines(i + '\n' for i in g + j) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument('-d', '--directory', required=True) - parser.add_argument('-s', '--sources-file', required=True) - parser.add_argument('-r', '--resources-file', required=True) - parser.add_argument('-k', '--kotlin-sources-file', default=None) - parser.add_argument('-g', '--groovy-sources-file', default=None) - parser.add_argument('--append', action='store_true', default=False) - parser.add_argument('--all-resources', action='store_true', default=False) - parser.add_argument('--resolve-kotlin', action='store_true', default=False) - parser.add_argument('--resolve-groovy', action='store_true', default=False) - parser.add_argument('--include-patterns', nargs='*', default=[]) - parser.add_argument('--exclude-patterns', nargs='*', default=[]) - args = parser.parse_args() - - do_it(**vars(args)) diff --git a/build/scripts/retry.py b/build/scripts/retry.py deleted file mode 100644 index d14170bfec..0000000000 --- a/build/scripts/retry.py +++ /dev/null @@ -1,29 +0,0 @@ -import time -import functools - - -# Partly copy-pasted from contrib/python/retry -def retry_func(f, exceptions=Exception, tries=-1, delay=1, max_delay=None, backoff=1): - _tries, _delay = tries, delay - while _tries: - try: - return f() - except exceptions as e: - _tries -= 1 - if not _tries: - raise - - time.sleep(_delay) - _delay *= backoff - - if max_delay is not None: - _delay = min(_delay, max_delay) - - -def retry(**retry_kwargs): - def decorator(func): - @functools.wraps(func) - def wrapper(*args, **kwargs): - return retry_func(lambda: func(*args, **kwargs), **retry_kwargs) - return wrapper - return decorator diff --git a/build/scripts/rodata2asm.py b/build/scripts/rodata2asm.py deleted file mode 100644 index 555639499f..0000000000 --- a/build/scripts/rodata2asm.py +++ /dev/null @@ -1,31 +0,0 @@ -import os -import argparse - - -def main(): - parser = argparse.ArgumentParser(description='Convert rodata into asm source with embedded file content') - parser.add_argument('symbol', help='symvol name exported from generated filr') - parser.add_argument('rodata', help='input .rodata file path') - parser.add_argument('asm', type=argparse.FileType('w', encoding='UTF-8'), help='destination .asm file path') - parser.add_argument('--elf', action='store_true') - - args = parser.parse_args() - - file_size = os.path.getsize(args.rodata) - - args.asm.write('global ' + args.symbol + '\n') - args.asm.write('global ' + args.symbol + 'Size' + '\n') - args.asm.write('SECTION .rodata ALIGN=16\n') - args.asm.write(args.symbol + ':\nincbin "' + args.rodata + '"\n') - args.asm.write('align 4, db 0\n') - args.asm.write(args.symbol + 'Size:\ndd ' + str(file_size) + '\n') - - if args.elf: - args.asm.write('size ' + args.symbol + ' ' + str(file_size) + '\n') - args.asm.write('size ' + args.symbol + 'Size 4\n') - - args.asm.close() - - -if __name__ == '__main__': - main() diff --git a/build/scripts/run_ios_simulator.py b/build/scripts/run_ios_simulator.py deleted file mode 100644 index 052c855b77..0000000000 --- a/build/scripts/run_ios_simulator.py +++ /dev/null @@ -1,79 +0,0 @@ -import argparse -import json -import os -import subprocess -import sys - - -def just_do_it(): - parser = argparse.ArgumentParser() - parser.add_argument("--action", choices=["create", "spawn", "kill"]) - parser.add_argument("--simctl", help="simctl binary path") - parser.add_argument("--profiles", help="profiles path") - parser.add_argument("--device-dir", help="devices directory") - parser.add_argument("--device-name", help="temp device name") - args, tail = parser.parse_known_args() - if args.action == 'create': - action_create(args.simctl, args.profiles, args.device_dir, args.device_name, tail) - elif args.action == "spawn": - action_spawn(args.simctl, args.profiles, args.device_dir, args.device_name, tail) - elif args.action == "kill": - action_kill(args.simctl, args.profiles, args.device_dir, args.device_name) - - -def action_create(simctl, profiles, device_dir, name, args): - parser = argparse.ArgumentParser() - parser.add_argument("--device-type", default="com.apple.CoreSimulator.SimDeviceType.iPhone-X") - parser.add_argument("--device-runtime", default="com.apple.CoreSimulator.SimRuntime.iOS-12-1") - args = parser.parse_args(args) - all_devices = list(get_all_devices(simctl, profiles, device_dir)) - if filter(lambda x: x["name"] == name, all_devices): - raise Exception("Device named {} already exists".format(name)) - subprocess.check_call([simctl, "--profiles", profiles, "--set", device_dir, "create", name, args.device_type, args.device_runtime]) - created = filter(lambda x: x["name"] == name, get_all_devices(simctl, profiles, device_dir)) - if not created: - raise Exception("Creation error: temp device named {} not found".format(name)) - created = created[0] - if created["availability"] != "(available)": - raise Exception("Creation error: temp device {} status is {} ((available) expected)".format(name, created["availability"])) - - -def action_spawn(simctl, profiles, device_dir, name, args): - device = filter(lambda x: x["name"] == name, get_all_devices(simctl, profiles, device_dir)) - if not device: - raise Exception("Can't spawn process: device named {} not found".format(name)) - if len(device) > 1: - raise Exception("Can't spawn process: too many devices named {} found".format(name)) - device = device[0] - os.execv(simctl, [simctl, "--profiles", profiles, "--set", device_dir, "spawn", device["udid"]] + args) - - -def action_kill(simctl, profiles, device_dir, name): - device = filter(lambda x: x["name"] == name, get_all_devices(simctl, profiles, device_dir)) - if not device: - print >> sys.stderr, "Device named {} not found; do nothing".format(name) - return - if len(device) > 1: - raise Exception("Can't remove: too many devices named {}:\n{}".format(name, '\n'.join(i for i in device))) - device = device[0] - os.execv(simctl, [simctl, "--profiles", profiles, "--set", device_dir, "delete", device["udid"]]) - - -def get_all_devices(simctl, profiles, device_dir): - p = subprocess.Popen([simctl, "--profiles", profiles, "--set", device_dir, "list", "--json", "devices"], stdout=subprocess.PIPE) - out, _ = p.communicate() - rc = p.wait() - if rc: - raise Exception("Devices list command return code is {}\nstdout:\n{}".format(rc, out)) - raw_object = json.loads(out) - if "devices" not in raw_object: - raise Exception("Devices not found in\n{}".format(json.dumps(raw_object))) - raw_object = raw_object["devices"] - for os_name, devices in raw_object.items(): - for device in devices: - device["os_name"] = os_name - yield device - - -if __name__ == '__main__': - just_do_it() diff --git a/build/scripts/run_javac.py b/build/scripts/run_javac.py deleted file mode 100644 index c35546e0fe..0000000000 --- a/build/scripts/run_javac.py +++ /dev/null @@ -1,122 +0,0 @@ -import sys -import subprocess -import optparse -import re - - -def parse_args(): - parser = optparse.OptionParser() - parser.disable_interspersed_args() - parser.add_option('--sources-list') - parser.add_option('--verbose', default=False, action='store_true') - parser.add_option('--remove-notes', default=False, action='store_true') - parser.add_option('--ignore-errors', default=False, action='store_true') - parser.add_option('--kotlin', default=False, action='store_true') - return parser.parse_args() - - -COLORING = { - r'^(?P<path>.*):(?P<line>\d*): error: (?P<msg>.*)': lambda m: '[[unimp]]{path}[[rst]]:[[alt2]]{line}[[rst]]: [[c:light-red]]error[[rst]]: [[bad]]{msg}[[rst]]'.format( - path=m.group('path'), - line=m.group('line'), - msg=m.group('msg'), - ), - r'^(?P<path>.*):(?P<line>\d*): warning: (?P<msg>.*)': lambda m: '[[unimp]]{path}[[rst]]:[[alt2]]{line}[[rst]]: [[c:light-yellow]]warning[[rst]]: {msg}'.format( - path=m.group('path'), - line=m.group('line'), - msg=m.group('msg'), - ), - r'^warning: ': lambda m: '[[c:light-yellow]]warning[[rst]]: ', - r'^error: (?P<msg>.*)': lambda m: '[[c:light-red]]error[[rst]]: [[bad]]{msg}[[rst]]'.format(msg=m.group('msg')), - r'^Note: ': lambda m: '[[c:light-cyan]]Note[[rst]]: ', -} - - -def colorize(err): - for regex, sub in COLORING.iteritems(): - err = re.sub(regex, sub, err, flags=re.MULTILINE) - return err - - -def remove_notes(err): - return '\n'.join([line for line in err.split('\n') if not line.startswith('Note:')]) - - -def find_javac(cmd): - if not cmd: - return None - if cmd[0].endswith('javac') or cmd[0].endswith('javac.exe'): - return cmd[0] - if len(cmd) > 2 and cmd[1].endswith('build_java_with_error_prone.py'): - for javas in ('java', 'javac'): - if cmd[2].endswith(javas) or cmd[2].endswith(javas + '.exe'): - return cmd[2] - return None - - -# temporary, for jdk8/jdk9+ compatibility -def fix_cmd(cmd): - if not cmd: - return cmd - javac = find_javac(cmd) - if not javac: - return cmd - p = subprocess.Popen([javac, '-version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, err = p.communicate() - out, err = out.strip(), err.strip() - for prefix in ('javac 1.8', 'java version "1.8'): - for raw_out in ((out or ''), (err or '')): - for line in raw_out.split('\n'): - if line.startswith(prefix): - res = [] - i = 0 - while i < len(cmd): - for option in ('--add-exports', '--add-modules'): - if cmd[i] == option: - i += 1 - break - elif cmd[i].startswith(option + '='): - break - else: - res.append(cmd[i]) - i += 1 - return res - return cmd - - -def main(): - opts, cmd = parse_args() - - with open(opts.sources_list) as f: - input_files = f.read().strip().split() - - if opts.kotlin: - input_files = [i for i in input_files if i.endswith('.kt')] - - if not input_files: - if opts.verbose: - sys.stderr.write('No files to compile, javac is not launched.\n') - - else: - p = subprocess.Popen(fix_cmd(cmd), stderr=subprocess.PIPE) - _, err = p.communicate() - rc = p.wait() - - if opts.remove_notes: - err = remove_notes(err) - - try: - err = colorize(err) - - except Exception: - pass - - if opts.ignore_errors and rc: - sys.stderr.write('error: javac actually failed with exit code {}\n'.format(rc)) - rc = 0 - sys.stderr.write(err) - sys.exit(rc) - - -if __name__ == '__main__': - main() diff --git a/build/scripts/run_junit.py b/build/scripts/run_junit.py deleted file mode 100644 index 089f149f72..0000000000 --- a/build/scripts/run_junit.py +++ /dev/null @@ -1,65 +0,0 @@ -import os -import sys - -SHUTDOWN_SIGNAL = 'SIGUSR1' - - -class SignalInterruptionError(Exception): - pass - - -def on_shutdown(s, f): - raise SignalInterruptionError() - - -def main(): - args = sys.argv[1:] - - def execve(): - os.execve(args[0], args, os.environ) - - jar_binary = args[args.index('--jar-binary') + 1] - java_bin_dir = os.path.dirname(jar_binary) - jstack_binary = os.path.join(java_bin_dir, 'jstack') - - if not os.path.exists(jstack_binary): - sys.stderr.write("jstack is missing: {}\n".format(jstack_binary)) - execve() - - import signal - - signum = getattr(signal, SHUTDOWN_SIGNAL, None) - - if signum is None: - execve() - - import subprocess - - proc = subprocess.Popen(args) - signal.signal(signum, on_shutdown) - timeout = False - - try: - proc.wait() - except SignalInterruptionError: - sys.stderr.write("\nGot {} signal: going to shutdown junit\n".format(signum)) - # Dump stack traces - subprocess.call([jstack_binary, str(proc.pid)], stdout=sys.stderr) - # Kill junit - for more info see DEVTOOLS-7636 - os.kill(proc.pid, signal.SIGKILL) - proc.wait() - timeout = True - - if proc.returncode: - sys.stderr.write('java exit code: {}\n'.format(proc.returncode)) - if timeout: - # In case of timeout return specific exit code - # https://a.yandex-team.ru/arc/trunk/arcadia/devtools/ya/test/const/__init__.py?rev=r8578188#L301 - proc.returncode = 10 - sys.stderr.write('java exit code changed to {}\n'.format(proc.returncode)) - - return proc.returncode - - -if __name__ == '__main__': - exit(main()) diff --git a/build/scripts/run_llvm_dsymutil.py b/build/scripts/run_llvm_dsymutil.py deleted file mode 100644 index 4f43362ad9..0000000000 --- a/build/scripts/run_llvm_dsymutil.py +++ /dev/null @@ -1,11 +0,0 @@ -import os -import sys -import subprocess - - -if __name__ == '__main__': - with open(os.devnull, 'w') as fnull: - p = subprocess.Popen(sys.argv[1:], shell=False, stderr=fnull, stdout=sys.stdout) - - p.communicate() - sys.exit(p.returncode) diff --git a/build/scripts/run_msvc_wine.py b/build/scripts/run_msvc_wine.py deleted file mode 100644 index eb5ed3eba5..0000000000 --- a/build/scripts/run_msvc_wine.py +++ /dev/null @@ -1,586 +0,0 @@ -import sys -import os -import re -import subprocess -import signal -import time -import json -import argparse -import errno - -import process_command_files as pcf -import process_whole_archive_option as pwa - - -procs = [] -build_kekeke = 45 - - -def stringize(s): - return s.encode('utf-8') if isinstance(s, unicode) else s - - -def run_subprocess(*args, **kwargs): - if 'env' in kwargs: - kwargs['env'] = {stringize(k): stringize(v) for k, v in kwargs['env'].iteritems()} - - p = subprocess.Popen(*args, **kwargs) - - procs.append(p) - - return p - - -def run_subprocess_with_timeout(timeout, args): - attempts_remaining = 5 - delay = 1 - p = None - while True: - try: - p = run_subprocess(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - stdout, stderr = p.communicate(timeout=timeout) - return p, stdout, stderr - except subprocess.TimeoutExpired as e: - print >>sys.stderr, 'timeout running {0}, error {1}, delay {2} seconds'.format(args, str(e), delay) - if p is not None: - try: - p.kill() - p.wait(timeout=1) - except Exception: - pass - attempts_remaining -= 1 - if attempts_remaining == 0: - raise - time.sleep(delay) - delay = min(2 * delay, 4) - - -def terminate_slaves(): - for p in procs: - try: - p.terminate() - except Exception: - pass - - -def sig_term(sig, fr): - terminate_slaves() - sys.exit(sig) - - -def subst_path(l): - if len(l) > 3: - if l[:3].lower() in ('z:\\', 'z:/'): - return l[2:].replace('\\', '/') - - return l - - -def call_wine_cmd_once(wine, cmd, env, mode): - p = run_subprocess(wine + cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=env, close_fds=True, shell=False) - - output = find_cmd_out(cmd) - error = None - if output is not None and os.path.exists(output): - try: - os.remove(output) - except OSError as e: - if e.errno != errno.ENOENT: - error = e - except Exception as e: - error = e - - if error is not None: - print >> sys.stderr, 'Output {} already exists and we have failed to remove it: {}'.format(output, error) - - # print >>sys.stderr, cmd, env, wine - - stdout_and_stderr, _ = p.communicate() - - return_code = p.returncode - if not stdout_and_stderr: - if return_code != 0: - raise Exception('wine did something strange') - - return return_code - elif ' : fatal error ' in stdout_and_stderr: - return_code = 1 - elif ' : error ' in stdout_and_stderr: - return_code = 2 - - lines = [x.strip() for x in stdout_and_stderr.split('\n')] - - prefixes = [ - 'Microsoft (R)', - 'Copyright (C)', - 'Application tried to create a window', - 'The graphics driver is missing', - 'Could not load wine-gecko', - 'wine: configuration in', - 'wine: created the configuration directory', - 'libpng warning:' - ] - - suffixes = [ - '.c', - '.cxx', - '.cc', - '.cpp', - '.masm', - ] - - substrs = [ - 'Creating library Z:', - 'err:heap', - 'err:menubuilder:', - 'err:msvcrt', - 'err:ole:', - 'err:wincodecs:', - 'err:winediag:', - ] - - def good_line(l): - for x in prefixes: - if l.startswith(x): - return False - - for x in suffixes: - if l.endswith(x): - return False - - for x in substrs: - if x in l: - return False - - return True - - def filter_lines(): - for l in lines: - if good_line(l): - yield subst_path(l.strip()) - - stdout_and_stderr = '\n'.join(filter_lines()).strip() - - if stdout_and_stderr: - print >>sys.stderr, stdout_and_stderr - - return return_code - - -def prepare_vc(fr, to): - for p in os.listdir(fr): - fr_p = os.path.join(fr, p) - to_p = os.path.join(to, p) - - if not os.path.exists(to_p): - print >>sys.stderr, 'install %s -> %s' % (fr_p, to_p) - - os.link(fr_p, to_p) - - -def run_slave(): - args = json.loads(sys.argv[3]) - wine = sys.argv[1] - - signal.signal(signal.SIGTERM, sig_term) - - if args.get('tout', None): - signal.signal(signal.SIGALRM, sig_term) - signal.alarm(args['tout']) - - tout = 0.1 - - while True: - try: - return call_wine_cmd_once([wine], args['cmd'], args['env'], args['mode']) - except Exception as e: - print >>sys.stderr, '%s, will retry in %s' % (str(e), tout) - - time.sleep(tout) - tout = min(2 * tout, 4) - - -def find_cmd_out(args): - for arg in args: - if arg.startswith('/Fo'): - return arg[3:] - - if arg.startswith('/OUT:'): - return arg[5:] - - -def calc_zero_cnt(data): - zero_cnt = 0 - - for ch in data: - if ch == chr(0): - zero_cnt += 1 - - return zero_cnt - - -def is_good_file(p): - if not os.path.isfile(p): - return False - - if os.path.getsize(p) < 300: - return False - - asm_pattern = re.compile(r'asm(\.\w+)?\.obj$') - if asm_pattern.search(p): - pass - elif p.endswith('.obj'): - with open(p, 'rb') as f: - prefix = f.read(200) - - if ord(prefix[0]) != 0: - return False - - if ord(prefix[1]) != 0: - return False - - if ord(prefix[2]) != 0xFF: - return False - - if ord(prefix[3]) != 0xFF: - return False - - if calc_zero_cnt(prefix) > 195: - return False - - f.seek(-100, os.SEEK_END) - last = f.read(100) - - if calc_zero_cnt(last) > 95: - return False - - if last[-1] != chr(0): - return False - elif p.endswith('.lib'): - with open(p, 'rb') as f: - if f.read(7) != '!<arch>': - return False - - return True - - -RED = '\x1b[31;1m' -GRAY = '\x1b[30;1m' -RST = '\x1b[0m' -MGT = '\x1b[35m' -YEL = '\x1b[33m' -GRN = '\x1b[32m' -CYA = '\x1b[36m' - - -def colorize_strings(l): - p = l.find("'") - - if p >= 0: - yield l[:p] - - l = l[p + 1:] - - p = l.find("'") - - if p >= 0: - yield CYA + "'" + subst_path(l[:p]) + "'" + RST - - for x in colorize_strings(l[p + 1:]): - yield x - else: - yield "'" + l - else: - yield l - - -def colorize_line(l): - lll = l - - try: - parts = [] - - if l.startswith('(compiler file'): - return ''.join(colorize_strings(l)) - - if l.startswith('/'): - p = l.find('(') - parts.append(GRAY + l[:p] + RST) - l = l[p:] - - if l and l.startswith('('): - p = l.find(')') - parts.append(':' + MGT + l[1:p] + RST) - l = l[p + 1:] - - if l: - if l.startswith(' : '): - l = l[1:] - - if l.startswith(': error'): - parts.append(': ' + RED + 'error' + RST) - l = l[7:] - elif l.startswith(': warning'): - parts.append(': ' + YEL + 'warning' + RST) - l = l[9:] - elif l.startswith(': note'): - parts.append(': ' + GRN + 'note' + RST) - l = l[6:] - elif l.startswith('fatal error'): - parts.append(RED + 'fatal error' + RST) - l = l[11:] - - if l: - parts.extend(colorize_strings(l)) - - return ''.join(parts) - except Exception: - return lll - - -def colorize(out): - return '\n'.join(colorize_line(l) for l in out.split('\n')) - - -def trim_path(path, winepath): - p1, p1_stdout, p1_stderr = run_subprocess_with_timeout(60, [winepath, '-w', path]) - win_path = p1_stdout.strip() - - if p1.returncode != 0 or not win_path: - # Fall back to only winepath -s - win_path = path - - p2, p2_stdout, p2_stderr = run_subprocess_with_timeout(60, [winepath, '-s', win_path]) - short_path = p2_stdout.strip() - - check_path = short_path - if check_path.startswith(('Z:', 'z:')): - check_path = check_path[2:] - - if not check_path[1:].startswith((path[1:4], path[1:4].upper())): - raise Exception( - 'Cannot trim path {}; 1st winepath exit code: {}, stdout:\n{}\n stderr:\n{}\n 2nd winepath exit code: {}, stdout:\n{}\n stderr:\n{}'.format( - path, p1.returncode, p1_stdout, p1_stderr, p2.returncode, p2_stdout, p2_stderr - )) - - return short_path - - -def downsize_path(path, short_names): - flag = '' - if path.startswith('/Fo'): - flag = '/Fo' - path = path[3:] - - for full_name, short_name in short_names.items(): - if path.startswith(full_name): - path = path.replace(full_name, short_name) - - return flag + path - - -def make_full_path_arg(arg, bld_root, short_root): - if arg[0] != '/' and len(os.path.join(bld_root, arg)) > 250: - return os.path.join(short_root, arg) - return arg - - -def fix_path(p): - topdirs = ['/%s/' % d for d in os.listdir('/')] - - def abs_path_start(path, pos): - if pos < 0: - return False - return pos == 0 or path[pos - 1] == ':' - - pp = None - for pr in topdirs: - pp2 = p.find(pr) - if abs_path_start(p, pp2) and (pp is None or pp > pp2): - pp = pp2 - if pp is not None: - return p[:pp] + 'Z:' + p[pp:].replace('/', '\\') - if p.startswith('/Fo'): - return '/Fo' + p[3:].replace('/', '\\') - return p - - -def process_free_args(args, wine, bld_root, mode): - whole_archive_prefix = '/WHOLEARCHIVE:' - short_names = {} - winepath = os.path.join(os.path.dirname(wine), 'winepath') - short_names[bld_root] = trim_path(bld_root, winepath) - # Slow for no benefit. - # arc_root = args.arcadia_root - # short_names[arc_root] = trim_path(arc_root, winepath) - - free_args, wa_peers, wa_libs = pwa.get_whole_archive_peers_and_libs(pcf.skip_markers(args)) - - process_link = lambda x: make_full_path_arg(x, bld_root, short_names[bld_root]) if mode in ('link', 'lib') else x - - def process_arg(arg): - with_wa_prefix = arg.startswith(whole_archive_prefix) - prefix = whole_archive_prefix if with_wa_prefix else '' - without_prefix_arg = arg[len(prefix):] - return prefix + fix_path(process_link(downsize_path(without_prefix_arg, short_names))) - - result = [] - for arg in free_args: - if pcf.is_cmdfile_arg(arg): - cmd_file_path = pcf.cmdfile_path(arg) - cf_args = pcf.read_from_command_file(cmd_file_path) - with open(cmd_file_path, 'w') as afile: - for cf_arg in cf_args: - afile.write(process_arg(cf_arg) + "\n") - result.append(arg) - else: - result.append(process_arg(arg)) - return pwa.ProcessWholeArchiveOption('WINDOWS', wa_peers, wa_libs).construct_cmd(result) - - -def run_main(): - parser = argparse.ArgumentParser() - parser.add_argument('wine', action='store') - parser.add_argument('-v', action='store', dest='version', default='120') - parser.add_argument('-I', action='append', dest='incl_paths') - parser.add_argument('mode', action='store') - parser.add_argument('arcadia_root', action='store') - parser.add_argument('arcadia_build_root', action='store') - parser.add_argument('binary', action='store') - parser.add_argument('free_args', nargs=argparse.REMAINDER) - # By now just unpack. Ideally we should fix path and pack arguments back into command file - args = parser.parse_args() - - wine = args.wine - mode = args.mode - binary = args.binary - version = args.version - incl_paths = args.incl_paths - bld_root = args.arcadia_build_root - free_args = args.free_args - - wine_dir = os.path.dirname(os.path.dirname(wine)) - bin_dir = os.path.dirname(binary) - tc_dir = os.path.dirname(os.path.dirname(os.path.dirname(bin_dir))) - if not incl_paths: - incl_paths = [tc_dir + '/VC/include', tc_dir + '/include'] - - cmd_out = find_cmd_out(free_args) - - env = os.environ.copy() - - env.pop('DISPLAY', None) - - env['WINEDLLOVERRIDES'] = 'msvcr{}=n'.format(version) - env['WINEDEBUG'] = 'fixme-all' - env['INCLUDE'] = ';'.join(fix_path(p) for p in incl_paths) - env['VSINSTALLDIR'] = fix_path(tc_dir) - env['VCINSTALLDIR'] = fix_path(tc_dir + '/VC') - env['WindowsSdkDir'] = fix_path(tc_dir) - env['LIBPATH'] = fix_path(tc_dir + '/VC/lib/amd64') - env['LIB'] = fix_path(tc_dir + '/VC/lib/amd64') - env['LD_LIBRARY_PATH'] = ':'.join(wine_dir + d for d in ['/lib', '/lib64', '/lib64/wine']) - - cmd = [binary] + process_free_args(free_args, wine, bld_root, mode) - - for x in ('/NOLOGO', '/nologo', '/FD'): - try: - cmd.remove(x) - except ValueError: - pass - - def run_process(sleep, tout): - if sleep: - time.sleep(sleep) - - args = { - 'cmd': cmd, - 'env': env, - 'mode': mode, - 'tout': tout - } - - slave_cmd = [sys.executable, sys.argv[0], wine, 'slave', json.dumps(args)] - p = run_subprocess(slave_cmd, stderr=subprocess.STDOUT, stdout=subprocess.PIPE, shell=False) - out, _ = p.communicate() - return p.wait(), out - - def print_err_log(log): - if not log: - return - if mode == 'cxx': - log = colorize(log) - print >>sys.stderr, log - - tout = 200 - - while True: - rc, out = run_process(0, tout) - - if rc in (-signal.SIGALRM, signal.SIGALRM): - print_err_log(out) - print >>sys.stderr, '##append_tag##time out' - elif out and ' stack overflow ' in out: - print >>sys.stderr, '##append_tag##stack overflow' - elif out and 'recvmsg: Connection reset by peer' in out: - print >>sys.stderr, '##append_tag##wine gone' - elif out and 'D8037' in out: - print >>sys.stderr, '##append_tag##repair wine' - - try: - os.unlink(os.path.join(os.environ['WINEPREFIX'], '.update-timestamp')) - except Exception as e: - print >>sys.stderr, e - - else: - print_err_log(out) - - # non-zero return code - bad, return it immediately - if rc: - print >>sys.stderr, '##win_cmd##' + ' '.join(cmd) - print >>sys.stderr, '##args##' + ' '.join(free_args) - return rc - - # check for output existence(if we expect it!) and real length - if cmd_out: - if is_good_file(cmd_out): - return 0 - else: - # retry! - print >>sys.stderr, '##append_tag##no output' - else: - return 0 - - tout *= 3 - - -def main(): - prefix_suffix = os.environ.pop('WINEPREFIX_SUFFIX', None) - if prefix_suffix is not None: - prefix = os.environ.pop('WINEPREFIX', None) - if prefix is not None: - os.environ['WINEPREFIX'] = os.path.join(prefix, prefix_suffix) - - # just in case - signal.alarm(2000) - - if sys.argv[2] == 'slave': - func = run_slave - else: - func = run_main - - try: - try: - sys.exit(func()) - finally: - terminate_slaves() - except KeyboardInterrupt: - sys.exit(4) - except Exception as e: - print >>sys.stderr, str(e) - - sys.exit(3) - - -if __name__ == '__main__': - main() diff --git a/build/scripts/run_sonar.py b/build/scripts/run_sonar.py deleted file mode 100644 index 761cc34b78..0000000000 --- a/build/scripts/run_sonar.py +++ /dev/null @@ -1,121 +0,0 @@ -import os -import sys -import zipfile -import tarfile -import subprocess as sp -import optparse -import shutil -import xml.etree.ElementTree as et - - -def parse_args(): - parser = optparse.OptionParser() - parser.add_option( - '--classes-jar-path', - dest='classes_jar_paths', - action='append', - default=[], - ) - parser.add_option('--sources-jar-path') - parser.add_option('--sonar-scanner-jar-path') - parser.add_option('--sonar-scanner-main-class') - parser.add_option('--java-coverage-merged-tar') - parser.add_option('--java-binary-path') - parser.add_option('--log-path') - parser.add_option('--gcov-report-path') - parser.add_option('--source-root') - parser.add_option('--java-args', action='append', default=[]) - return parser.parse_args() - - -def extract_zip_file(zip_file_path, dest_dir): - with zipfile.ZipFile(zip_file_path) as arch: - arch.extractall(dest_dir) - - -def get_source_real_path(source_root, path): - parts = os.path.normpath(path).split(os.path.sep) - for i in xrange(len(parts)): - if os.path.exists(os.path.join(source_root, *parts[i:])): - return os.path.join(*parts[i:]) - return None - - -def collect_cpp_sources(report, source_root, destination): - sources = set() - with open(report) as f: - root = et.fromstring(f.read()) - for f in root.findall('.//class[@filename]'): - real_filename = get_source_real_path(source_root, f.attrib['filename']) - if real_filename: - f.attrib['filename'] = real_filename - sources.add(real_filename) - with open(report, 'w') as f: - pref = '''<?xml version="1.0" ?> -<!DOCTYPE coverage - SYSTEM 'http://cobertura.sourceforge.net/xml/coverage-03.dtd'>\n''' - f.write(pref + et.tostring(root, encoding='utf-8') + '\n\n') - for src in sources: - dst = os.path.join(destination, src) - src = os.path.join(source_root, src) - if os.path.isfile(src): - if not os.path.exists(os.path.dirname(dst)): - os.makedirs(os.path.dirname(dst)) - os.link(src, dst) - - -def main(opts, props_args): - sources_dir = os.path.abspath('src') - base_props_args = ['-Dsonar.sources=' + sources_dir] - os.mkdir(sources_dir) - if opts.sources_jar_path: - extract_zip_file(opts.sources_jar_path, sources_dir) - if opts.gcov_report_path: - collect_cpp_sources(opts.gcov_report_path, opts.source_root, sources_dir) - base_props_args += ['-Dsonar.projectBaseDir=' + sources_dir, '-Dsonar.cxx.coverage.reportPath=' + opts.gcov_report_path] - - if opts.classes_jar_paths: - classes_dir = os.path.abspath('cls') - os.mkdir(classes_dir) - - for classes_jar_path in opts.classes_jar_paths: - extract_zip_file(classes_jar_path, classes_dir) - - base_props_args.append('-Dsonar.java.binaries=' + classes_dir) - - if opts.java_coverage_merged_tar: - jacoco_report_path = os.path.abspath('jacoco.exec') - with open(jacoco_report_path, 'w') as dest: - with tarfile.open(opts.java_coverage_merged_tar) as tar: - for src in tar: - extracted = tar.extractfile(src) - if extracted is not None: - shutil.copyfileobj(extracted, dest) - - base_props_args += [ - '-Dsonar.core.codeCoveragePlugin=jacoco', - '-Dsonar.jacoco.reportPath=' + jacoco_report_path - ] - java_args = ['-{}'.format(i) for i in opts.java_args] + ['-Djava.net.preferIPv6Addresses=true', '-Djava.net.preferIPv4Addresses=false'] - - sonar_cmd = [ - opts.java_binary_path, - ] + java_args + [ - '-classpath', - opts.sonar_scanner_jar_path, - ] + base_props_args + props_args + [opts.sonar_scanner_main_class, '-X'] - - p = sp.Popen(sonar_cmd, stdout=sp.PIPE, stderr=sp.STDOUT) - out, _ = p.communicate() - - sys.stderr.write(out) - with open(opts.log_path, 'a') as f: - f.write(out) - - sys.exit(p.returncode) - - -if __name__ == '__main__': - opts, args = parse_args() - props_args = ['-D' + arg for arg in args] - main(opts, props_args) diff --git a/build/scripts/setup_java_tmpdir.py b/build/scripts/setup_java_tmpdir.py deleted file mode 100644 index e478d4aa96..0000000000 --- a/build/scripts/setup_java_tmpdir.py +++ /dev/null @@ -1,40 +0,0 @@ -import os -import sys -import platform -import subprocess - - -def fix_tmpdir(cmd): - if not cmd: - return cmd - java_id, option_name = None, None - for i, java in enumerate(cmd): - if java.endswith('java') or java.endswith('java.exe'): - java_id = i - option_name = '-Djava.io.tmpdir=' - break - if java.endswith('javac') or java.endswith('javac.exe'): - java_id = i - option_name = '-J-Djava.io.tmpdir=' - break - if java_id is None: - return cmd - for arg in cmd[java_id:]: - if arg.startswith(option_name): - return cmd - tmpdir = os.environ.get('TMPDIR') or os.environ.get('TEMPDIR') - if not tmpdir: - return cmd - return cmd[:java_id + 1] + ['{}{}'.format(option_name, tmpdir)] + cmd[java_id + 1:] - - -def just_do_it(): - args = fix_tmpdir(sys.argv[1:]) - if platform.system() == 'Windows': - sys.exit(subprocess.Popen(args).wait()) - else: - os.execv(args[0], args) - - -if __name__ == '__main__': - just_do_it() diff --git a/build/scripts/sky.py b/build/scripts/sky.py deleted file mode 100644 index b703af7ed1..0000000000 --- a/build/scripts/sky.py +++ /dev/null @@ -1,43 +0,0 @@ -import logging -import os -import subprocess - -import fetch_from - - -class UnsupportedProtocolException(Exception): - pass - - -def executable_path(): - return "/usr/local/bin/sky" - - -def is_avaliable(): - if not os.path.exists(executable_path()): - return False - try: - subprocess.check_output([executable_path(), "--version"]) - return True - except subprocess.CalledProcessError: - return False - except OSError: - return False - - -def fetch(skynet_id, file_name, timeout=None): - if not is_avaliable(): - raise UnsupportedProtocolException("Skynet is not available") - - target_dir = os.path.abspath(fetch_from.uniq_string_generator()) - os.mkdir(target_dir) - - cmd_args = [executable_path(), "get", "-N", "Backbone", "--user", "--wait", "--dir", target_dir, skynet_id] - if timeout is not None: - cmd_args += ["--timeout", str(timeout)] - - logging.info("Call skynet with args: %s", cmd_args) - stdout = subprocess.check_output(cmd_args).strip() - logging.debug("Skynet call with args %s is finished, result is %s", cmd_args, stdout) - - return os.path.join(target_dir, file_name) diff --git a/build/scripts/stderr2stdout.py b/build/scripts/stderr2stdout.py deleted file mode 100644 index 0e510da373..0000000000 --- a/build/scripts/stderr2stdout.py +++ /dev/null @@ -1,6 +0,0 @@ -import subprocess -import sys - -if __name__ == '__main__': - assert len(sys.argv) > 1 - sys.exit(subprocess.Popen(sys.argv[1:], stderr=sys.stdout).wait()) diff --git a/build/scripts/stdout2stderr.py b/build/scripts/stdout2stderr.py deleted file mode 100644 index d7861fdda3..0000000000 --- a/build/scripts/stdout2stderr.py +++ /dev/null @@ -1,6 +0,0 @@ -import subprocess -import sys - -if __name__ == '__main__': - assert len(sys.argv) > 1 - sys.exit(subprocess.Popen(sys.argv[1:], stdout=sys.stderr).wait()) diff --git a/build/scripts/symlink.py b/build/scripts/symlink.py deleted file mode 100755 index 9e30a25065..0000000000 --- a/build/scripts/symlink.py +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env python - -import sys -import os -import platform -from subprocess import call - - -def symlink(): - if len(sys.argv) < 3: - print >>sys.stderr, "Usage: symlink.py <source> <target>" - sys.exit(1) - - source = sys.argv[1] - target = sys.argv[2] - - print("Making a symbolic link from {0} to {1}".format(source, target)) - - sysName = platform.system() - if sysName == "Windows": # and not os.path.exists(target) - if os.path.isdir(source): - call(["mklink", "/D", target, source], shell=True) - else: - call(["mklink", target, source], shell=True) - else: - call(["ln", "-f", "-s", "-n", source, target]) - -if __name__ == '__main__': - symlink() diff --git a/build/scripts/tar_directory.py b/build/scripts/tar_directory.py deleted file mode 100644 index a91889fa22..0000000000 --- a/build/scripts/tar_directory.py +++ /dev/null @@ -1,45 +0,0 @@ -import os -import sys -import tarfile - - -def is_exe(fpath): - return os.path.isfile(fpath) and os.access(fpath, os.X_OK) - - -def main(args): - if len(args) < 2 or len(args) > 3: - raise Exception("Illegal usage: `tar_directory.py archive.tar directory [skip prefix]` or `tar_directory.py archive.tar output_directory --extract`") - tar, directory, prefix, extract = args[0], args[1], None, False - if len(args) == 3: - if args[2] == '--extract': - extract = True - else: - prefix = args[2] - for tar_exe in ('/usr/bin/tar', '/bin/tar'): - if not is_exe(tar_exe): - continue - if extract: - dest = os.path.abspath(directory) - if not os.path.exists(dest): - os.makedirs(dest) - os.execv(tar_exe, [tar_exe, '-xf', tar, '-C', dest]) - else: - source = os.path.relpath(directory, prefix) if prefix else directory - os.execv(tar_exe, [tar_exe, '-cf', tar] + (['-C', prefix] if prefix else []) + [source]) - break - else: - if extract: - dest = os.path.abspath(directory) - if not os.path.exists(dest): - os.makedirs(dest) - with tarfile.open(tar, 'r') as tar_file: - tar_file.extractall(dest) - else: - source = directory - with tarfile.open(tar, 'w') as out: - out.add(os.path.abspath(source), arcname=os.path.relpath(source, prefix) if prefix else source) - - -if __name__ == '__main__': - main(sys.argv[1:]) diff --git a/build/scripts/tar_sources.py b/build/scripts/tar_sources.py deleted file mode 100644 index 54e2839a69..0000000000 --- a/build/scripts/tar_sources.py +++ /dev/null @@ -1,41 +0,0 @@ -import argparse -import os -import tarfile - - -def parse_args(): - parser = argparse.ArgumentParser() - parser.add_argument('--exts', nargs='*', default=None) - parser.add_argument('--flat', action='store_true') - parser.add_argument('--input', required=True) - parser.add_argument('--output', required=True) - parser.add_argument('--prefix', default=None) - - return parser.parse_args() - - -def main(): - args = parse_args() - - py_srcs = [] - for root, _, files in os.walk(args.input): - for f in files: - if not args.exts or f.endswith(tuple(args.exts)): - py_srcs.append(os.path.join(root, f)) - - compression_mode = '' - if args.output.endswith(('.tar.gz', '.tgz')): - compression_mode = 'gz' - elif args.output.endswith('.bzip2'): - compression_mode = 'bz2' - - with tarfile.open(args.output, 'w:{}'.format(compression_mode)) as out: - for f in py_srcs: - arcname = os.path.basename(f) if args.flat else os.path.relpath(f, args.input) - if args.prefix: - arcname = os.path.join(args.prefix, arcname) - out.add(f, arcname=arcname) - - -if __name__ == '__main__': - main() diff --git a/build/scripts/tared_protoc.py b/build/scripts/tared_protoc.py deleted file mode 100644 index 7643e1dbfe..0000000000 --- a/build/scripts/tared_protoc.py +++ /dev/null @@ -1,31 +0,0 @@ -import os -import optparse -import tarfile -import contextlib -import subprocess as sp - - -def parse_args(): - parser = optparse.OptionParser() - parser.disable_interspersed_args() - parser.add_option('--tar-output') - parser.add_option('--protoc-out-dir') - return parser.parse_args() - - -def main(): - opts, args = parse_args() - assert opts.tar_output - assert opts.protoc_out_dir - - if not os.path.exists(opts.protoc_out_dir): - os.makedirs(opts.protoc_out_dir) - - sp.check_call(args) - - with contextlib.closing(tarfile.open(opts.tar_output, 'w')) as tf: - tf.add(opts.protoc_out_dir, arcname='') - - -if __name__ == '__main__': - main() diff --git a/build/scripts/touch.py b/build/scripts/touch.py deleted file mode 100755 index e01ba7f86b..0000000000 --- a/build/scripts/touch.py +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env python - -import optparse -import os -import sys -import time - - -def main(argv): - parser = optparse.OptionParser(add_help_option=False) - parser.disable_interspersed_args() - - parser.add_option('-?', '--help', dest='help', - action='store_true', default=None, help='print help') - parser.add_option('-t', dest='t', action='store', default=None) - - opts, argv_rest = parser.parse_args(argv) - if getattr(opts, 'help', False): - parser.print_help() - return 0 - - tspec = opts.t - if tspec is None: - times = None - else: - head, sep, tail = tspec.partition('.') - if 8 > len(head): - raise Exception("time spec must follow format [[CC]YY]MMDDhhmm[.SS]: " + tspec + '; ' + head) - tfmt = '' - if 12 == len(head): - tfmt += '%Y' - elif 10 == len(head): - tfmt += '%y' - tfmt += '%m%d%H%M' - if 2 == len(tail): - tfmt += '.%S' - mtime = time.mktime(time.strptime(tspec, tfmt)) - times = (mtime, mtime) - - for file in argv_rest: - try: - os.utime(file, times) - except: - open(file, 'w').close() - if times is not None: - os.utime(file, times) - - -if __name__ == '__main__': - sys.exit(main(sys.argv[1:])) diff --git a/build/scripts/unpacking_jtest_runner.py b/build/scripts/unpacking_jtest_runner.py deleted file mode 100644 index 9730dcd711..0000000000 --- a/build/scripts/unpacking_jtest_runner.py +++ /dev/null @@ -1,148 +0,0 @@ -import io -import json -import optparse -import os -import sys -import subprocess -import time -import zipfile -import platform - -# This script changes test run classpath by unpacking tests.jar -> tests-dir. The goal -# is to launch tests with the same classpath as maven does. - - -def parse_args(): - parser = optparse.OptionParser() - parser.disable_interspersed_args() - parser.add_option('--trace-file') - parser.add_option('--jar-binary') - parser.add_option('--tests-jar-path') - parser.add_option('--classpath-option-type', choices=('manifest', 'command_file', 'list'), default='manifest') - return parser.parse_args() - - -# temporary, for jdk8/jdk9+ compatibility -def fix_cmd(cmd): - if not cmd: - return cmd - java = cmd[0] - if not java.endswith('java') and not java.endswith('java.exe'): - return cmd - p = subprocess.Popen([java, '-version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, err = p.communicate() - out, err = out.strip(), err.strip() - if ((out or '').strip().startswith('java version "1.8') or (err or '').strip().startswith('java version "1.8')): - res = [] - i = 0 - while i < len(cmd): - for option in ('--add-exports', '--add-modules'): - if cmd[i] == option: - i += 1 - break - elif cmd[i].startswith(option + '='): - break - else: - res.append(cmd[i]) - i += 1 - return res - return cmd - - -def dump_event(etype, data, filename): - event = { - 'timestamp': time.time(), - 'value': data, - 'name': etype, - } - - with io.open(filename, 'a', encoding='utf8') as afile: - afile.write(unicode(json.dumps(event) + '\n')) - - -def dump_chunk_event(data, filename): - return dump_event('chunk-event', data, filename) - - -def extract_jars(dest, archive): - os.makedirs(dest) - with zipfile.ZipFile(archive) as zf: - zf.extractall(dest) - - -def make_bfg_from_cp(class_path, out): - class_path = ' '.join( - map(lambda path: ('file:/' + path.lstrip('/')) if os.path.isabs(path) else path, class_path) - ) - with zipfile.ZipFile(out, 'w') as zf: - lines = [] - while class_path: - lines.append(class_path[:60]) - class_path = class_path[60:] - if lines: - zf.writestr('META-INF/MANIFEST.MF', 'Manifest-Version: 1.0\nClass-Path: \n ' + '\n '.join(lines) + ' \n\n') - - -def make_command_file_from_cp(class_path, out): - with open(out, 'w') as cp_file: - cp_file.write(os.pathsep.join(class_path)) - - -def main(): - s = time.time() - opts, args = parse_args() - - # unpack tests jar - try: - build_root = args[args.index('--build-root') + 1] - dest = os.path.join(build_root, 'test-classes') - except Exception: - build_root = '' - dest = os.path.abspath('test-classes') - - extract_jars(dest, opts.tests_jar_path) - - metrics = { - 'suite_jtest_extract_jars_(seconds)': time.time() - s, - } - - s = time.time() - # fix java classpath - cp_idx = args.index('-classpath') - if args[cp_idx + 1].startswith('@'): - real_name = args[cp_idx + 1][1:] - mf = os.path.join(os.path.dirname(real_name), 'fixed.bfg.jar') - with open(real_name) as origin: - class_path = [os.path.join(build_root, i.strip()) for i in origin] - if opts.tests_jar_path in class_path: - class_path.remove(opts.tests_jar_path) - if opts.classpath_option_type == 'manifest': - make_bfg_from_cp(class_path, mf) - mf = os.pathsep.join([dest, mf]) - elif opts.classpath_option_type == 'command_file': - mf = os.path.splitext(mf)[0] + '.txt' - make_command_file_from_cp([dest] + class_path, mf) - mf = "@" + mf - elif opts.classpath_option_type == 'list': - mf = os.pathsep.join([dest] + class_path) - else: - raise Exception("Unexpected classpath option type: " + opts.classpath_option_type) - args = fix_cmd(args[:cp_idx + 1]) + [mf] + args[cp_idx + 2:] - else: - args[cp_idx + 1] = args[cp_idx + 1].replace(opts.tests_jar_path, dest) - args = fix_cmd(args[:cp_idx]) + args[cp_idx:] - - metrics['suite_jtest_fix_classpath_(seconds)'] = time.time() - s - - if opts.trace_file: - dump_chunk_event({'metrics': metrics}, opts.trace_file) - - # run java cmd - if platform.system() == 'Windows': - sys.exit(subprocess.Popen(args).wait()) - else: - os.execv(args[0], args) - - -if __name__ == '__main__': - main() diff --git a/build/scripts/with_coverage.py b/build/scripts/with_coverage.py deleted file mode 100644 index d62435c3b8..0000000000 --- a/build/scripts/with_coverage.py +++ /dev/null @@ -1,40 +0,0 @@ -# TODO prettyboy remove after ya-bin release - -import os -import sys -import subprocess -import tarfile -import random -import shutil - - -def mkdir_p(path): - try: - os.makedirs(path) - except OSError: - pass - - -def main(args): - coverage_path = os.path.abspath(args[0]) - coverage_dir = coverage_path + '.' + str(random.getrandbits(64)) - - mkdir_p(coverage_dir) - - env = os.environ.copy() - env['GCOV_PREFIX'] = coverage_dir - - subprocess.check_call(args[1:], env=env) - - arch_path = coverage_dir + '.archive' - - with tarfile.open(arch_path, 'w:') as tar: - tar.add(coverage_dir, arcname='.') - - os.rename(arch_path, coverage_path) - - shutil.rmtree(coverage_dir) - - -if __name__ == '__main__': - main(sys.argv[1:]) diff --git a/build/scripts/with_crash_on_timeout.py b/build/scripts/with_crash_on_timeout.py deleted file mode 100644 index bde864ed29..0000000000 --- a/build/scripts/with_crash_on_timeout.py +++ /dev/null @@ -1,22 +0,0 @@ -# TODO prettyboy remove after ya-bin release - -import os -import sys -import subprocess -import json - - -def main(args): - meta_path = os.path.abspath(args[0]) - timeout_code = int(args[1]) - subprocess.check_call(args[2:]) - with open(meta_path) as f: - meta_info = json.loads(f.read()) - if meta_info["exit_code"] == timeout_code: - print >> sys.stderr, meta_info["project"], 'crashed by timeout, use --test-disable-timeout option' - return 1 - return 0 - - -if __name__ == '__main__': - main(sys.argv[1:]) diff --git a/build/scripts/with_kapt_args.py b/build/scripts/with_kapt_args.py deleted file mode 100644 index eb7438a4c9..0000000000 --- a/build/scripts/with_kapt_args.py +++ /dev/null @@ -1,35 +0,0 @@ -import sys -import os -import subprocess -import platform -import argparse -import re - - -def parse_args(args): - parser = argparse.ArgumentParser() - parser.add_argument('--ap-classpath', nargs='*', type=str, dest='classpath') - cmd_start = args.index('--') - return parser.parse_args(args[:cmd_start]), args[cmd_start+1:] - - -def get_ap_classpath(directory): - jar_re = re.compile(r'.*(?<!-sources)\.jar') - found_jars = [os.path.join(address, name) for address, dirs, files in os.walk(directory) for name in files if jar_re.match(name)] - if len(found_jars) != 1: - raise Exception("found %d JAR files in directory %s" % (len(found_jars), directory)) - arg = 'plugin:org.jetbrains.kotlin.kapt3:apclasspath=' + found_jars[0] - return '-P', arg - - -def create_extra_args(args): - cp_opts = [arg for d in args.classpath for arg in get_ap_classpath(d)] - return cp_opts - -if __name__ == '__main__': - args, cmd = parse_args(sys.argv[1:]) - res = cmd + create_extra_args(args) - if platform.system() == 'Windows': - sys.exit(subprocess.Popen(res).wait()) - else: - os.execv(res[0], res) diff --git a/build/scripts/with_pathsep_resolve.py b/build/scripts/with_pathsep_resolve.py deleted file mode 100644 index 37c8c598ae..0000000000 --- a/build/scripts/with_pathsep_resolve.py +++ /dev/null @@ -1,23 +0,0 @@ -import sys -import os -import subprocess -import platform - - -def fix_args(args): - just_replace_it = False - for arg in args: - if arg == '--fix-path-sep': - just_replace_it = True - continue - if just_replace_it: - arg = arg.replace('::', os.pathsep) - just_replace_it = False - yield arg - -if __name__ == '__main__': - res = list(fix_args(sys.argv[1:])) - if platform.system() == 'Windows': - sys.exit(subprocess.Popen(res).wait()) - else: - os.execv(res[0], res) diff --git a/build/scripts/wrap_groovyc.py b/build/scripts/wrap_groovyc.py deleted file mode 100644 index 068b73fd87..0000000000 --- a/build/scripts/wrap_groovyc.py +++ /dev/null @@ -1,23 +0,0 @@ -import platform -import sys -import os -import subprocess - - -def fix_windows(args): - for arg in args: - if os.path.basename(arg) == 'groovyc' and os.path.basename(os.path.dirname(arg)) == 'bin': - yield arg + '.bat' - else: - yield arg - - -if __name__ == '__main__': - env = os.environ.copy() - jdk = sys.argv[1] - env['JAVA_HOME'] = jdk - args = sys.argv[2:] - if platform.system() == 'Windows': - sys.exit(subprocess.Popen(list(fix_windows(args)), env=env).wait()) - else: - os.execve(args[0], args, env) diff --git a/build/scripts/wrapper.py b/build/scripts/wrapper.py deleted file mode 100644 index 1e9d7955a5..0000000000 --- a/build/scripts/wrapper.py +++ /dev/null @@ -1,11 +0,0 @@ -import os -import sys - - -if __name__ == '__main__': - path = sys.argv[1] - - if path[0] != '/': - path = os.path.join(os.path.dirname(__file__), path) - - os.execv(path, [path] + sys.argv[2:]) diff --git a/build/scripts/write_file_size.py b/build/scripts/write_file_size.py deleted file mode 100644 index 880fb90644..0000000000 --- a/build/scripts/write_file_size.py +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env python -import sys -import os.path - -if __name__ == '__main__': - output = sys.argv[1] - size_sum = 0 - for filename in sys.argv[2:]: - if os.path.exists(filename): - size_sum += os.path.getsize(filename) - else: - sys.stderr.write('write_file_size.py: {0}: No such file or directory\n'.format(filename)) - sys.exit(1) - with open(output, 'w') as f: - f.write(str(size_sum)) diff --git a/build/scripts/writer.py b/build/scripts/writer.py deleted file mode 100644 index 21bb3006e5..0000000000 --- a/build/scripts/writer.py +++ /dev/null @@ -1,40 +0,0 @@ -import sys -import argparse - -import process_command_files as pcf - - -def parse_args(): - args = pcf.get_args(sys.argv[1:]) - parser = argparse.ArgumentParser() - parser.add_argument('-f', '--file', dest='file_path') - parser.add_argument('-a', '--append', action='store_true', default=False) - parser.add_argument('-Q', '--quote', action='store_true', default=False) - parser.add_argument('-s', '--addspace', action='store_true', default=False) - parser.add_argument('-c', '--content', action='append', dest='content') - parser.add_argument('-m', '--content-multiple', nargs='*', dest='content') - parser.add_argument('-P', '--path-list', action='store_true', default=False) - return parser.parse_args(args) - - -def smart_shell_quote(v): - if v is None: - return None - if ' ' in v or '"' in v or "'" in v: - return "\"{0}\"".format(v.replace('"', '\\"')) - return v - -if __name__ == '__main__': - args = parse_args() - open_type = 'a' if args.append else 'w' - - content = args.content - if args.quote: - content = [smart_shell_quote(ln) for ln in content] if content is not None else None - content = '\n'.join(content) - - with open(args.file_path, open_type) as f: - if args.addspace: - f.write(' ') - if content is not None: - f.write(content) diff --git a/build/scripts/xargs.py b/build/scripts/xargs.py deleted file mode 100644 index 5d68929ecc..0000000000 --- a/build/scripts/xargs.py +++ /dev/null @@ -1,18 +0,0 @@ -import sys -import os -import subprocess - -if __name__ == '__main__': - pos = sys.argv.index('--') - fname = sys.argv[pos + 1] - cmd = sys.argv[pos + 2:] - - with open(fname, 'r') as f: - args = [x.strip() for x in f] - - os.remove(fname) - - p = subprocess.Popen(cmd + args, shell=False, stderr=sys.stderr, stdout=sys.stdout) - p.communicate() - - sys.exit(p.returncode) diff --git a/build/scripts/ya.make b/build/scripts/ya.make deleted file mode 100644 index 837f0e0494..0000000000 --- a/build/scripts/ya.make +++ /dev/null @@ -1,110 +0,0 @@ -OWNER(g:ymake) - -PY23_TEST() - -IF (PY2) - TEST_SRCS( - build_dll_and_java.py - build_java_codenav_index.py - build_java_with_error_prone.py - build_java_with_error_prone2.py - build_mn.py - build_pln_header.py - cat.py - cgo1_wrapper.py - check_config_h.py - collect_java_srcs.py - compile_cuda.py - compile_java.py - compile_jsrc.py - compile_pysrc.py - configure_file.py - copy_docs_files.py - copy_docs_files_to_dir.py - copy_files_to_dir.py - copy_to_dir.py - coverage-info.py - cpp_flatc_wrapper.py - create_jcoverage_report.py - extract_asrc.py - extract_docs.py - extract_jacoco_report.py - f2c.py - fail_module_cmd.py - fetch_from.py - fetch_from_external.py - fetch_from_mds.py - fetch_from_npm.py - fetch_from_sandbox.py - fetch_resource.py - filter_zip.py - find_and_tar.py - fix_msvc_output.py - fs_tools.py - gen_aar_gradle_script.py - gen_java_codenav_entry.py - gen_java_codenav_protobuf.py - gen_mx_table.py - gen_py3_reg.py - gen_py_reg.py - gen_test_apk_gradle_script.py - gen_ub.py - generate_pom.py - go_proto_wrapper.py - go_tool.py - ios_wrapper.py - java_pack_to_file.py - link_asrc.py - link_dyn_lib.py - link_exe.py - link_fat_obj.py - link_lib.py - llvm_opt_wrapper.py - merge_coverage_data.py - merge_files.py - mkdir.py - mkdocs_builder_wrapper.py - mkver.py - pack_ios.py - pack_jcoverage_resources.py - perl_wrapper.py - postprocess_go_fbs.py - preprocess.py - py_compile.py - run_ios_simulator.py - run_javac.py - run_junit.py - run_llvm_dsymutil.py - run_msvc_wine.py - run_tool.py - sky.py - stdout2stderr.py - symlink.py - tar_directory.py - tar_sources.py - tared_protoc.py - touch.py - unpacking_jtest_runner.py - vcs_info.py - with_coverage.py - with_crash_on_timeout.py - with_pathsep_resolve.py - wrap_groovyc.py - wrapper.py - writer.py - write_file_size.py - xargs.py - yield_line.py - yndexer.py - ) -ELSEIF(PY3) - TEST_SRCS( - build_info_gen.py - ) -ENDIF() - -PEERDIR( - ydb/library/yql/public/udf -) - -END() diff --git a/build/scripts/yield_line.py b/build/scripts/yield_line.py deleted file mode 100644 index c7087e521e..0000000000 --- a/build/scripts/yield_line.py +++ /dev/null @@ -1,8 +0,0 @@ -import sys - - -if __name__ == '__main__': - pos = sys.argv.index('--') - - with open(sys.argv[pos + 1], 'a') as f: - f.write(' '.join(sys.argv[pos + 2:]) + '\n') diff --git a/build/scripts/yndexer.py b/build/scripts/yndexer.py deleted file mode 100644 index a38e28ba99..0000000000 --- a/build/scripts/yndexer.py +++ /dev/null @@ -1,79 +0,0 @@ -import sys -import subprocess -import threading -import os -import re - - -rx_resource_dir = re.compile(r'libraries: =([^:]*)') - - -def _try_to_kill(process): - try: - process.kill() - except Exception: - pass - - -def touch(path): - if not os.path.exists(path): - with open(path, 'w'): - pass - - -class Process(object): - def __init__(self, args): - self._process = subprocess.Popen(args) - self._event = threading.Event() - self._result = None - thread = threading.Thread(target=self._run) - thread.setDaemon(True) - thread.start() - - def _run(self): - self._process.communicate() - self._result = self._process.returncode - self._event.set() - - def wait(self, timeout): - self._event.wait(timeout=timeout) - _try_to_kill(self._process) - return self._result - - -if __name__ == '__main__': - args = sys.argv - - yndexer = args[1] - timeout = int(args[2]) - arc_root = args[3] - build_root = args[4] - input_file = args[5] - output_file = args[-1] - tail_args = args[6:-1] - - subprocess.check_call(tail_args) - - clang = tail_args[0] - out = subprocess.check_output([clang, '-print-search-dirs']) - resource_dir = rx_resource_dir.search(out).group(1) - - yndexer_args = [ - yndexer, input_file, - '-pb2', - '-i', 'arc::{}'.format(arc_root), - '-i', 'build::{}'.format(build_root), - '-i', '.IGNORE::/', - '-o', os.path.dirname(output_file), - '-n', os.path.basename(output_file).rsplit('.ydx.pb2', 1)[0], - '--' - ] + tail_args + [ - '-resource-dir', resource_dir, - ] - - process = Process(yndexer_args) - result = process.wait(timeout=timeout) - - if result != 0: - print >> sys.stderr, 'Yndexing process finished with code', result - touch(output_file) |