diff options
author | heretic <heretic@yandex-team.ru> | 2022-02-10 16:45:43 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:45:43 +0300 |
commit | 397cbe258b9e064f49c4ca575279f02f39fef76e (patch) | |
tree | a0b0eb3cca6a14e4e8ea715393637672fa651284 /build/scripts | |
parent | 43f5a35593ebc9f6bcea619bb170394ea7ae468e (diff) | |
download | ydb-397cbe258b9e064f49c4ca575279f02f39fef76e.tar.gz |
Restoring authorship annotation for <heretic@yandex-team.ru>. Commit 1 of 2.
Diffstat (limited to 'build/scripts')
40 files changed, 1667 insertions, 1667 deletions
diff --git a/build/scripts/build_dll_and_java.py b/build/scripts/build_dll_and_java.py index b9d8aff4df..41fb9413e5 100644 --- a/build/scripts/build_dll_and_java.py +++ b/build/scripts/build_dll_and_java.py @@ -1,47 +1,47 @@ -import os -import subprocess -import sys - - -def just_do_it(argv): - delim = argv[0] - args = [] - for item in argv: - if item == delim: - args.append([]) - else: - args[-1].append(item) - dll_cmd, java_cmd, inputs, dll_out, java_out, jsrs_out, roots = args - dll_out, java_out, jsrs_out, build_root, source_root = dll_out[0], java_out[0], jsrs_out[0], roots[0], roots[1] - for inp in inputs: - origin_inp = inp - if os.path.isabs(inp): - if os.path.relpath(inp, build_root).startswith('..'): - inp = os.path.relpath(inp, source_root) - else: - inp = os.path.relpath(inp, build_root) - ext = os.path.splitext(inp)[1] - if ext in ('.o', '.obj'): - if os.path.join(build_root, inp) in java_cmd: - inp = os.path.join(build_root, inp) - if sys.platform == 'win32': - inp = inp.replace('\\', '/') - if inp not in java_cmd: - inp = build_root + '/' + inp - java_cmd.remove(inp) - if ext in ('.java', '.jsrc'): - if origin_inp in dll_cmd: - inp = origin_inp - elif os.path.join(build_root, inp) in dll_cmd: - inp = os.path.join(build_root, inp) - if sys.platform == 'win32': - inp = inp.replace('\\', '/') +import os +import subprocess +import sys + + +def just_do_it(argv): + delim = argv[0] + args = [] + for item in argv: + if item == delim: + args.append([]) + else: + args[-1].append(item) + dll_cmd, java_cmd, inputs, dll_out, java_out, jsrs_out, roots = args + dll_out, java_out, jsrs_out, build_root, source_root = dll_out[0], java_out[0], jsrs_out[0], roots[0], roots[1] + for inp in inputs: + origin_inp = inp + if os.path.isabs(inp): + if os.path.relpath(inp, build_root).startswith('..'): + inp = os.path.relpath(inp, source_root) + else: + inp = os.path.relpath(inp, build_root) + ext = os.path.splitext(inp)[1] + if ext in ('.o', '.obj'): + if os.path.join(build_root, inp) in java_cmd: + inp = os.path.join(build_root, inp) + if sys.platform == 'win32': + inp = inp.replace('\\', '/') + if inp not in java_cmd: + inp = build_root + '/' + inp + java_cmd.remove(inp) + if ext in ('.java', '.jsrc'): + if origin_inp in dll_cmd: + inp = origin_inp + elif os.path.join(build_root, inp) in dll_cmd: + inp = os.path.join(build_root, inp) + if sys.platform == 'win32': + inp = inp.replace('\\', '/') dll_cmd.remove(inp) - java_cmd.insert(java_cmd.index(dll_out), java_out) - java_cmd.remove(dll_out) - subprocess.check_call(java_cmd) - subprocess.check_call(dll_cmd) - - -if __name__ == '__main__': - just_do_it(sys.argv[1:]) + java_cmd.insert(java_cmd.index(dll_out), java_out) + java_cmd.remove(dll_out) + subprocess.check_call(java_cmd) + subprocess.check_call(dll_cmd) + + +if __name__ == '__main__': + just_do_it(sys.argv[1:]) diff --git a/build/scripts/build_info_gen.py b/build/scripts/build_info_gen.py index 949e0a1636..9bc8475ded 100644 --- a/build/scripts/build_info_gen.py +++ b/build/scripts/build_info_gen.py @@ -1,63 +1,63 @@ -import subprocess -import os -import sys -import re - -indent = " " - - -def escape_special_symbols(strval): - retval = "" - for c in strval: - if c in ("\\", "\""): - retval += "\\" + c - elif ord(c) < 0x20: - retval += c.encode("string_escape") - else: - retval += c - return retval - - -def escape_line_feed(strval): - return re.sub(r'\\n', r'\\n"\\\n' + indent + '"', strval) - - -def escaped_define(strkey, strval): - return "#define " + strkey + " \"" + escape_line_feed(escape_special_symbols(strval)) + "\"" - - -def get_build_info(compiler, flags): - build_info = "Build info:\n" - build_info += indent + "Compiler: " + compiler + "\n" - build_info += indent + "Compiler version: \n" + get_compiler_info(compiler) + "\n" - build_info += indent + "Compile flags: " + (flags if flags else "no flags info") - return build_info - - -def get_compiler_info(compiler): - compiler_binary = os.path.basename(compiler).lower() - if len(compiler.split(' ')) > 1 or compiler_binary == "ymake" or compiler_binary == "ymake.exe": - compiler_ver_out = "Build by wrapper. No useful info here." - else: - compiler_ver_cmd = [compiler] - if compiler_binary not in ("cl", "cl.exe"): - compiler_ver_cmd.append('--version') - compiler_ver_out = subprocess.Popen(compiler_ver_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout.read() - return "\n".join(['{}{}'.format(indent * 2, line.strip()) for line in compiler_ver_out.splitlines() if line.strip()]) - - -def main(): - if len(sys.argv) != 4: +import subprocess +import os +import sys +import re + +indent = " " + + +def escape_special_symbols(strval): + retval = "" + for c in strval: + if c in ("\\", "\""): + retval += "\\" + c + elif ord(c) < 0x20: + retval += c.encode("string_escape") + else: + retval += c + return retval + + +def escape_line_feed(strval): + return re.sub(r'\\n', r'\\n"\\\n' + indent + '"', strval) + + +def escaped_define(strkey, strval): + return "#define " + strkey + " \"" + escape_line_feed(escape_special_symbols(strval)) + "\"" + + +def get_build_info(compiler, flags): + build_info = "Build info:\n" + build_info += indent + "Compiler: " + compiler + "\n" + build_info += indent + "Compiler version: \n" + get_compiler_info(compiler) + "\n" + build_info += indent + "Compile flags: " + (flags if flags else "no flags info") + return build_info + + +def get_compiler_info(compiler): + compiler_binary = os.path.basename(compiler).lower() + if len(compiler.split(' ')) > 1 or compiler_binary == "ymake" or compiler_binary == "ymake.exe": + compiler_ver_out = "Build by wrapper. No useful info here." + else: + compiler_ver_cmd = [compiler] + if compiler_binary not in ("cl", "cl.exe"): + compiler_ver_cmd.append('--version') + compiler_ver_out = subprocess.Popen(compiler_ver_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout.read() + return "\n".join(['{}{}'.format(indent * 2, line.strip()) for line in compiler_ver_out.splitlines() if line.strip()]) + + +def main(): + if len(sys.argv) != 4: print >>sys.stderr, "Usage: build_info_gen.py <output file> <CXX compiler> <CXX flags>" - sys.exit(1) - cxx_compiler = sys.argv[2] - cxx_flags = sys.argv[3] - with open(sys.argv[1], 'w') as result: - print >> result, "#pragma once\n" - print >> result, escaped_define("BUILD_INFO", get_build_info(cxx_compiler, cxx_flags)) - print >> result, escaped_define("BUILD_COMPILER", cxx_compiler) - print >> result, escaped_define("BUILD_COMPILER_VERSION", get_compiler_info(cxx_compiler)) - print >> result, escaped_define("BUILD_COMPILER_FLAGS", cxx_flags) - -if __name__ == "__main__": - main() + sys.exit(1) + cxx_compiler = sys.argv[2] + cxx_flags = sys.argv[3] + with open(sys.argv[1], 'w') as result: + print >> result, "#pragma once\n" + print >> result, escaped_define("BUILD_INFO", get_build_info(cxx_compiler, cxx_flags)) + print >> result, escaped_define("BUILD_COMPILER", cxx_compiler) + print >> result, escaped_define("BUILD_COMPILER_VERSION", get_compiler_info(cxx_compiler)) + print >> result, escaped_define("BUILD_COMPILER_FLAGS", cxx_flags) + +if __name__ == "__main__": + main() diff --git a/build/scripts/build_java_codenav_index.py b/build/scripts/build_java_codenav_index.py index d7ac4f3213..ad6752896b 100644 --- a/build/scripts/build_java_codenav_index.py +++ b/build/scripts/build_java_codenav_index.py @@ -1,49 +1,49 @@ -import sys -import re -import os -import subprocess - -FAKE_ARCADIA_ROOT = 'fake_arcadia_root' -FAKE_BUILD_ROOT = 'fake_build_root' - - -def modify_sources_file(origin, target, source_roots_map): - def _cut_source_root(src): - for pref, fake_root in source_roots_map.items(): - if src.startswith(pref): - return os.path.join(fake_root, os.path.relpath(src, pref)) - return src - - with open(origin) as o: - srcs = [i for line in o for i in re.split('\\s+', line) if i] - new_srcs = map(_cut_source_root, srcs) - with open(target, 'w') as t: - t.write(' '.join(new_srcs)) - - -def just_do_it(argv): +import sys +import re +import os +import subprocess + +FAKE_ARCADIA_ROOT = 'fake_arcadia_root' +FAKE_BUILD_ROOT = 'fake_build_root' + + +def modify_sources_file(origin, target, source_roots_map): + def _cut_source_root(src): + for pref, fake_root in source_roots_map.items(): + if src.startswith(pref): + return os.path.join(fake_root, os.path.relpath(src, pref)) + return src + + with open(origin) as o: + srcs = [i for line in o for i in re.split('\\s+', line) if i] + new_srcs = map(_cut_source_root, srcs) + with open(target, 'w') as t: + t.write(' '.join(new_srcs)) + + +def just_do_it(argv): corpus_name, build_root, arcadia_root, sources_file, javac_tail_cmd = argv[0], argv[1], argv[2], argv[3], argv[4:] - fake_arcadia_root = os.path.join(build_root, FAKE_ARCADIA_ROOT) - fake_build_root = os.path.join(build_root, FAKE_BUILD_ROOT) - fake_source_roots = { - arcadia_root: fake_arcadia_root, - build_root: fake_build_root, - } - modify_sources_file(sources_file, os.path.join(os.path.dirname(sources_file), '_' + os.path.basename(sources_file)), fake_source_roots) - kindex_data_root = '{}/kindex'.format(os.path.join(build_root, os.path.dirname(corpus_name))) - if not os.path.exists(kindex_data_root): - os.makedirs(kindex_data_root) - env = os.environ.copy() - env['KYTHE_ROOT_DIRECTORY'] = build_root - env['KYTHE_OUTPUT_DIRECTORY'] = kindex_data_root - env['KYTHE_CORPUS'] = os.path.relpath(corpus_name, build_root) - os.symlink(arcadia_root, fake_arcadia_root) - os.symlink(build_root, fake_build_root) - try: + fake_arcadia_root = os.path.join(build_root, FAKE_ARCADIA_ROOT) + fake_build_root = os.path.join(build_root, FAKE_BUILD_ROOT) + fake_source_roots = { + arcadia_root: fake_arcadia_root, + build_root: fake_build_root, + } + modify_sources_file(sources_file, os.path.join(os.path.dirname(sources_file), '_' + os.path.basename(sources_file)), fake_source_roots) + kindex_data_root = '{}/kindex'.format(os.path.join(build_root, os.path.dirname(corpus_name))) + if not os.path.exists(kindex_data_root): + os.makedirs(kindex_data_root) + env = os.environ.copy() + env['KYTHE_ROOT_DIRECTORY'] = build_root + env['KYTHE_OUTPUT_DIRECTORY'] = kindex_data_root + env['KYTHE_CORPUS'] = os.path.relpath(corpus_name, build_root) + os.symlink(arcadia_root, fake_arcadia_root) + os.symlink(build_root, fake_build_root) + try: subprocess.check_call(javac_tail_cmd, env=env) - finally: - os.unlink(fake_arcadia_root) - os.unlink(fake_build_root) - -if __name__ == '__main__': - just_do_it(sys.argv[1:]) + finally: + os.unlink(fake_arcadia_root) + os.unlink(fake_build_root) + +if __name__ == '__main__': + just_do_it(sys.argv[1:]) diff --git a/build/scripts/build_java_with_error_prone.py b/build/scripts/build_java_with_error_prone.py index 910443552e..19fcd37a64 100644 --- a/build/scripts/build_java_with_error_prone.py +++ b/build/scripts/build_java_with_error_prone.py @@ -1,36 +1,36 @@ -import sys -import os - -ERROR_PRONE_FLAGS = [ - '-Xep:FunctionalInterfaceMethodChanged:WARN', - '-Xep:ReturnValueIgnored:WARN', -] - -JAVA10_EXPORTS = [ - '--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED', - '--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED', - '--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED', - '--add-exports=jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED', - '--add-exports=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED', - '--add-exports=jdk.compiler/com.sun.tools.javac.processing=ALL-UNNAMED', - '--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED', +import sys +import os + +ERROR_PRONE_FLAGS = [ + '-Xep:FunctionalInterfaceMethodChanged:WARN', + '-Xep:ReturnValueIgnored:WARN', +] + +JAVA10_EXPORTS = [ + '--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED', + '--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED', + '--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED', + '--add-exports=jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED', + '--add-exports=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED', + '--add-exports=jdk.compiler/com.sun.tools.javac.processing=ALL-UNNAMED', + '--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED', '--add-exports=jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED' -] - - -def just_do_it(argv): - java, error_prone_tool, javac_cmd = argv[0], argv[1], argv[2:] - if java.endswith('javac') or java.endswith('javac.exe'): - for f in javac_cmd: +] + + +def just_do_it(argv): + java, error_prone_tool, javac_cmd = argv[0], argv[1], argv[2:] + if java.endswith('javac') or java.endswith('javac.exe'): + for f in javac_cmd: if f.startswith('-Xep'): - ERROR_PRONE_FLAGS.append(f) - for f in ERROR_PRONE_FLAGS: - if f in javac_cmd: - javac_cmd.remove(f) - os.execv(java, [java] + JAVA10_EXPORTS + ['-processorpath', error_prone_tool, '-XDcompilePolicy=byfile'] + [(' '.join(['-Xplugin:ErrorProne'] + ERROR_PRONE_FLAGS))] + javac_cmd) - else: - os.execv(java, [java, '-Xbootclasspath/p:' + error_prone_tool, 'com.google.errorprone.ErrorProneCompiler'] + ERROR_PRONE_FLAGS + javac_cmd) - - -if __name__ == '__main__': - just_do_it(sys.argv[1:]) + ERROR_PRONE_FLAGS.append(f) + for f in ERROR_PRONE_FLAGS: + if f in javac_cmd: + javac_cmd.remove(f) + os.execv(java, [java] + JAVA10_EXPORTS + ['-processorpath', error_prone_tool, '-XDcompilePolicy=byfile'] + [(' '.join(['-Xplugin:ErrorProne'] + ERROR_PRONE_FLAGS))] + javac_cmd) + else: + os.execv(java, [java, '-Xbootclasspath/p:' + error_prone_tool, 'com.google.errorprone.ErrorProneCompiler'] + ERROR_PRONE_FLAGS + javac_cmd) + + +if __name__ == '__main__': + just_do_it(sys.argv[1:]) diff --git a/build/scripts/build_java_with_error_prone2.py b/build/scripts/build_java_with_error_prone2.py index 4efc1e444a..94521f7c4b 100644 --- a/build/scripts/build_java_with_error_prone2.py +++ b/build/scripts/build_java_with_error_prone2.py @@ -1,87 +1,87 @@ -import sys -import os -import re -import subprocess -import platform - - -ERROR_PRONE_FLAGS = [ - '-Xep:FunctionalInterfaceMethodChanged:WARN', - '-Xep:ReturnValueIgnored:WARN', -] - -JAVA10_EXPORTS = [ - '--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED', - '--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED', - '--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED', - '--add-exports=jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED', - '--add-exports=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED', - '--add-exports=jdk.compiler/com.sun.tools.javac.processing=ALL-UNNAMED', - '--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED', +import sys +import os +import re +import subprocess +import platform + + +ERROR_PRONE_FLAGS = [ + '-Xep:FunctionalInterfaceMethodChanged:WARN', + '-Xep:ReturnValueIgnored:WARN', +] + +JAVA10_EXPORTS = [ + '--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED', + '--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED', + '--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED', + '--add-exports=jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED', + '--add-exports=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED', + '--add-exports=jdk.compiler/com.sun.tools.javac.processing=ALL-UNNAMED', + '--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED', '--add-exports=jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED', -] - - -def get_java_version(exe): - p = subprocess.Popen([exe, '-version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, err = p.communicate() - for line in ((out or '').strip() + (err or '').strip()).split("\n"): +] + + +def get_java_version(exe): + p = subprocess.Popen([exe, '-version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = p.communicate() + for line in ((out or '').strip() + (err or '').strip()).split("\n"): m = re.match(r'java version "(.+)"', line) - if m: - parts = m.groups()[0].split(".") - return parts[1] if parts[0] == "1" else parts[0] + if m: + parts = m.groups()[0].split(".") + return parts[1] if parts[0] == "1" else parts[0] m = re.match(r'openjdk version "(\d+).*"', line) - if m: - parts = m.groups()[0].split(".") - return parts[0] - return None - - -def get_classpath(cmd): - for i, part in enumerate(cmd): - if part == '-classpath': - i += 1 - if i < len(cmd): - return cmd[i] - else: - return None - return None - - -def parse_args(argv): - parsed = [] - for i in range(len(argv)): - if not argv[i].startswith('-'): - parsed.append(argv[i]) - if len(parsed) >= 3: - break - return parsed + [argv[i + 1:]] - - -def just_do_it(argv): - java, javac, error_prone_tool, javac_cmd = parse_args(argv) - ver = get_java_version(java) - if not ver: - raise Exception("Can't determine java version") - if int(ver) >= 10: - for f in javac_cmd: + if m: + parts = m.groups()[0].split(".") + return parts[0] + return None + + +def get_classpath(cmd): + for i, part in enumerate(cmd): + if part == '-classpath': + i += 1 + if i < len(cmd): + return cmd[i] + else: + return None + return None + + +def parse_args(argv): + parsed = [] + for i in range(len(argv)): + if not argv[i].startswith('-'): + parsed.append(argv[i]) + if len(parsed) >= 3: + break + return parsed + [argv[i + 1:]] + + +def just_do_it(argv): + java, javac, error_prone_tool, javac_cmd = parse_args(argv) + ver = get_java_version(java) + if not ver: + raise Exception("Can't determine java version") + if int(ver) >= 10: + for f in javac_cmd: if f.startswith('-Xep'): - ERROR_PRONE_FLAGS.append(f) - for f in ERROR_PRONE_FLAGS: - if f in javac_cmd: - javac_cmd.remove(f) - if '-processor' in javac_cmd: - classpath = get_classpath(javac_cmd) - if classpath: - error_prone_tool = error_prone_tool + os.pathsep + classpath - cmd = [javac] + JAVA10_EXPORTS + ['-processorpath', error_prone_tool, '-XDcompilePolicy=byfile'] + [(' '.join(['-Xplugin:ErrorProne'] + ERROR_PRONE_FLAGS))] + javac_cmd - else: - cmd = [java, '-Xbootclasspath/p:' + error_prone_tool, 'com.google.errorprone.ErrorProneCompiler'] + ERROR_PRONE_FLAGS + javac_cmd - if platform.system() == 'Windows': - sys.exit(subprocess.Popen(cmd).wait()) - else: - os.execv(cmd[0], cmd) - - -if __name__ == '__main__': - just_do_it(sys.argv[1:]) + ERROR_PRONE_FLAGS.append(f) + for f in ERROR_PRONE_FLAGS: + if f in javac_cmd: + javac_cmd.remove(f) + if '-processor' in javac_cmd: + classpath = get_classpath(javac_cmd) + if classpath: + error_prone_tool = error_prone_tool + os.pathsep + classpath + cmd = [javac] + JAVA10_EXPORTS + ['-processorpath', error_prone_tool, '-XDcompilePolicy=byfile'] + [(' '.join(['-Xplugin:ErrorProne'] + ERROR_PRONE_FLAGS))] + javac_cmd + else: + cmd = [java, '-Xbootclasspath/p:' + error_prone_tool, 'com.google.errorprone.ErrorProneCompiler'] + ERROR_PRONE_FLAGS + javac_cmd + if platform.system() == 'Windows': + sys.exit(subprocess.Popen(cmd).wait()) + else: + os.execv(cmd[0], cmd) + + +if __name__ == '__main__': + just_do_it(sys.argv[1:]) diff --git a/build/scripts/collect_java_srcs.py b/build/scripts/collect_java_srcs.py index 170002520a..457791b9a6 100644 --- a/build/scripts/collect_java_srcs.py +++ b/build/scripts/collect_java_srcs.py @@ -13,7 +13,7 @@ if __name__ == '__main__': for src in srcs: src = os.path.normpath(src) - if src.endswith('.java') or src.endswith('.kt'): + if src.endswith('.java') or src.endswith('.kt'): src_rel_path = os.path.relpath(src, root) if os.path.join(root, src_rel_path) == src: diff --git a/build/scripts/compile_cuda.py b/build/scripts/compile_cuda.py index c0bec50b2a..c54351eeb4 100644 --- a/build/scripts/compile_cuda.py +++ b/build/scripts/compile_cuda.py @@ -63,15 +63,15 @@ def main(): if flag in cflags: cflags.remove(flag) - skip_prefix_list = [ - '-fsanitize=', + skip_prefix_list = [ + '-fsanitize=', '-fsanitize-coverage=', - '-fsanitize-blacklist=', + '-fsanitize-blacklist=', '--system-header-prefix', - ] - for prefix in skip_prefix_list: - cflags = [i for i in cflags if not i.startswith(prefix)] - + ] + for prefix in skip_prefix_list: + cflags = [i for i in cflags if not i.startswith(prefix)] + if not is_clang(command): def good(arg): if arg.startswith('--target='): @@ -99,14 +99,14 @@ def main(): compiler_args.append(arg) compiler_args.append(cflags_queue.popleft()) continue - if arg[:2].upper() in ('-I', '/I', '-B'): + if arg[:2].upper() in ('-I', '/I', '-B'): value = arg[2:] if not value: value = cflags_queue.popleft() - if arg[1] == 'I': + if arg[1] == 'I': cpp_args.append('-I{}'.format(value)) - elif arg[1] == 'B': # todo: delete "B" flag check when cuda stop to use gcc - pass + elif arg[1] == 'B': # todo: delete "B" flag check when cuda stop to use gcc + pass continue match = re.match(r'[-/]D(.*)', arg) diff --git a/build/scripts/compile_java.py b/build/scripts/compile_java.py index e95869e853..34f5fde875 100644 --- a/build/scripts/compile_java.py +++ b/build/scripts/compile_java.py @@ -5,10 +5,10 @@ import shutil import subprocess as sp import tarfile import zipfile -import sys +import sys -def parse_args(args): +def parse_args(args): parser = optparse.OptionParser() parser.add_option('--javac-bin') parser.add_option('--jar-bin') @@ -16,7 +16,7 @@ def parse_args(args): parser.add_option('--package-prefix') parser.add_option('--jar-output') parser.add_option('--srcs-jar-output') - return parser.parse_args(args) + return parser.parse_args(args) def mkdir_p(directory): @@ -24,21 +24,21 @@ def mkdir_p(directory): os.makedirs(directory) -def split_cmd_by_delim(cmd, delim='DELIM'): - result = [[]] - for arg in cmd: - if arg == delim: - result.append([]) - else: - result[-1].append(arg) - return result - - +def split_cmd_by_delim(cmd, delim='DELIM'): + result = [[]] + for arg in cmd: + if arg == delim: + result.append([]) + else: + result[-1].append(arg) + return result + + def main(): - cmd_parts = split_cmd_by_delim(sys.argv) - assert len(cmd_parts) == 3 - args, javac_opts, peers = cmd_parts - opts, jsrcs = parse_args(args) + cmd_parts = split_cmd_by_delim(sys.argv) + assert len(cmd_parts) == 3 + args, javac_opts, peers = cmd_parts + opts, jsrcs = parse_args(args) jsrcs += list(filter(lambda x: x.endswith('.jsrc'), peers)) peers = list(filter(lambda x: not x.endswith('.jsrc'), peers)) @@ -54,18 +54,18 @@ def main(): for r, _, files in os.walk(sources_dir): for f in files: srcs.append(os.path.join(r, f)) - srcs += jsrcs - srcs = list(filter(lambda x: x.endswith('.java'), srcs)) + srcs += jsrcs + srcs = list(filter(lambda x: x.endswith('.java'), srcs)) classes_dir = 'cls' mkdir_p(classes_dir) - classpath = os.pathsep.join(peers) + classpath = os.pathsep.join(peers) if srcs: - temp_sources_file = 'temp.sources.list' - with open(temp_sources_file, 'w') as ts: - ts.write(' '.join(srcs)) - sp.check_call([opts.javac_bin, '-nowarn', '-g', '-classpath', classpath, '-encoding', 'UTF-8', '-d', classes_dir] + javac_opts + ['@' + temp_sources_file]) + temp_sources_file = 'temp.sources.list' + with open(temp_sources_file, 'w') as ts: + ts.write(' '.join(srcs)) + sp.check_call([opts.javac_bin, '-nowarn', '-g', '-classpath', classpath, '-encoding', 'UTF-8', '-d', classes_dir] + javac_opts + ['@' + temp_sources_file]) for s in jsrcs: if s.endswith('-sources.jar'): diff --git a/build/scripts/configure_file.py b/build/scripts/configure_file.py index 1873ed70eb..a943f89a7e 100755 --- a/build/scripts/configure_file.py +++ b/build/scripts/configure_file.py @@ -2,7 +2,7 @@ import sys import os.path -import re +import re cmakeDef01 = "#cmakedefine01" cmakeDef = "#cmakedefine" @@ -26,13 +26,13 @@ def replaceLine(l, varDict, define): ePos = l.find(var) + len(var) l = l[:sPos] + define + ' ' + var + ' ' + val + l[ePos + 1:] + '\n' - finder = re.compile(".*?(@[a-zA-Z0-9_]+@).*") + finder = re.compile(".*?(@[a-zA-Z0-9_]+@).*") while True: - re_result = finder.match(l) - if not re_result: + re_result = finder.match(l) + if not re_result: return l - key = re_result.group(1)[1:-1] - l = l[:re_result.start(1)] + varDict.get(key, '') + l[re_result.end(1):] + key = re_result.group(1)[1:-1] + l = l[:re_result.start(1)] + varDict.get(key, '') + l[re_result.end(1):] def main(inputPath, outputPath, varDict): diff --git a/build/scripts/coverage-info.py b/build/scripts/coverage-info.py index 94491d9256..365a97a368 100644 --- a/build/scripts/coverage-info.py +++ b/build/scripts/coverage-info.py @@ -201,20 +201,20 @@ def process_all_coverage_files(gcda_archive, fname2gcno, fname2info, geninfo_exe gen_info(geninfo_cmd, coverage_info) -def gen_cobertura(tool, output, combined_info): - cmd = [ - tool, - combined_info, - '-b', '#hamster#', - '-o', output - ] - p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, err = p.communicate() - if p.returncode: - raise Exception('lcov_cobertura failed with exit code {}\nstdout: {}\nstderr: {}'.format(p.returncode, out, err)) - - -def main(source_root, output, gcno_archive, gcda_archive, gcov_tool, prefix_filter, exclude_regexp, teamcity_stat_output, coverage_report_path, gcov_report, lcov_cobertura): +def gen_cobertura(tool, output, combined_info): + cmd = [ + tool, + combined_info, + '-b', '#hamster#', + '-o', output + ] + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = p.communicate() + if p.returncode: + raise Exception('lcov_cobertura failed with exit code {}\nstdout: {}\nstderr: {}'.format(p.returncode, out, err)) + + +def main(source_root, output, gcno_archive, gcda_archive, gcov_tool, prefix_filter, exclude_regexp, teamcity_stat_output, coverage_report_path, gcov_report, lcov_cobertura): exclude_files = re.compile(exclude_regexp) if exclude_regexp else None fname2gcno = {} @@ -256,8 +256,8 @@ def main(source_root, output, gcno_archive, gcda_archive, gcov_tool, prefix_filt cmd = [os.path.join(source_root, 'devtools', 'lcov', 'genhtml'), '-p', source_root, '--ignore-errors', 'source', '-o', output_dir, output_trace] print >>sys.stderr, '## genhtml', ' '.join(cmd) subprocess.check_call(cmd) - if lcov_cobertura: - gen_cobertura(lcov_cobertura, gcov_report, output_trace) + if lcov_cobertura: + gen_cobertura(lcov_cobertura, gcov_report, output_trace) with tarfile.open(output, 'w') as tar: tar.add(output_dir, arcname='.') @@ -275,8 +275,8 @@ if __name__ == '__main__': parser.add_argument('--exclude-regexp', action='store') parser.add_argument('--teamcity-stat-output', action='store_const', const=True) parser.add_argument('--coverage-report-path', action='store') - parser.add_argument('--gcov-report', action='store') - parser.add_argument('--lcov-cobertura', action='store') + parser.add_argument('--gcov-report', action='store') + parser.add_argument('--lcov-cobertura', action='store') args = parser.parse_args() main(**vars(args)) diff --git a/build/scripts/create_jcoverage_report.py b/build/scripts/create_jcoverage_report.py index 45083ff4f7..4a74f7677b 100644 --- a/build/scripts/create_jcoverage_report.py +++ b/build/scripts/create_jcoverage_report.py @@ -1,19 +1,19 @@ -import argparse -import tarfile +import argparse +import tarfile import zipfile -import os +import os import sys import time -import subprocess - - -def mkdir_p(path): - try: - os.makedirs(path) - except OSError: - pass - - +import subprocess + + +def mkdir_p(path): + try: + os.makedirs(path) + except OSError: + pass + + class Timer(object): def __init__(self): @@ -24,7 +24,7 @@ class Timer(object): self.start = time.time() -def main(source, output, java, prefix_filter, exclude_filter, jars_list, output_format, tar_output, agent_disposition, runners_paths): +def main(source, output, java, prefix_filter, exclude_filter, jars_list, output_format, tar_output, agent_disposition, runners_paths): timer = Timer() reports_dir = 'jacoco_reports_dir' mkdir_p(reports_dir) @@ -35,12 +35,12 @@ def main(source, output, java, prefix_filter, exclude_filter, jars_list, output_ with open(jars_list) as f: jars = f.read().strip().split() - if jars and runners_paths: - for r in runners_paths: - try: - jars.remove(r) - except ValueError: - pass + if jars and runners_paths: + for r in runners_paths: + try: + jars.remove(r) + except ValueError: + pass src_dir = 'sources_dir' cls_dir = 'classes_dir' @@ -78,8 +78,8 @@ def main(source, output, java, prefix_filter, exclude_filter, jars_list, output_ report_dir = 'java.report.temp' else: report_dir = output - mkdir_p(report_dir) - + mkdir_p(report_dir) + if agent_disposition: agent_cmd = [java, '-jar', agent_disposition, src_dir, cls_dir, prefix_filter or '.', exclude_filter or '__no_exclude__', report_dir, output_format] agent_cmd += reports @@ -89,24 +89,24 @@ def main(source, output, java, prefix_filter, exclude_filter, jars_list, output_ if tar_output: with tarfile.open(output, 'w') as outf: outf.add(report_dir, arcname='.') - - -if __name__ == '__main__': + + +if __name__ == '__main__': if 'LC_ALL' in os.environ: if os.environ['LC_ALL'] == 'C': os.environ['LC_ALL'] = 'en_GB.UTF-8' - parser = argparse.ArgumentParser() - - parser.add_argument('--source', action='store') - parser.add_argument('--output', action='store') - parser.add_argument('--java', action='store') - parser.add_argument('--prefix-filter', action='store') - parser.add_argument('--exclude-filter', action='store') + parser = argparse.ArgumentParser() + + parser.add_argument('--source', action='store') + parser.add_argument('--output', action='store') + parser.add_argument('--java', action='store') + parser.add_argument('--prefix-filter', action='store') + parser.add_argument('--exclude-filter', action='store') parser.add_argument('--jars-list', action='store') parser.add_argument('--output-format', action='store', default="html") parser.add_argument('--raw-output', dest='tar_output', action='store_false', default=True) parser.add_argument('--agent-disposition', action='store') - parser.add_argument('--runner-path', dest='runners_paths', action='append', default=[]) - args = parser.parse_args() - main(**vars(args)) + parser.add_argument('--runner-path', dest='runners_paths', action='append', default=[]) + args = parser.parse_args() + main(**vars(args)) diff --git a/build/scripts/extract_jacoco_report.py b/build/scripts/extract_jacoco_report.py index 02e4ba9f13..07c5cae13c 100644 --- a/build/scripts/extract_jacoco_report.py +++ b/build/scripts/extract_jacoco_report.py @@ -1,29 +1,29 @@ -import argparse -import os -import re -import tarfile - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - - parser.add_argument('--archive', action='store') - parser.add_argument('--source-re', action='store') - parser.add_argument('--destination', action='store') - - args = parser.parse_args() - - with tarfile.open(args.archive) as tf: - open(args.destination, 'wb').close() - extract_list = [] - matcher = re.compile(args.source_re) - temp_dir = os.path.join(os.path.dirname(args.destination), 'temp_profiles') - if not os.path.exists(temp_dir): - os.makedirs(temp_dir) - for f in [i for i in tf if matcher.match(i.name)]: - tf.extract(f, path=temp_dir) - for directory, _, srcs in os.walk(temp_dir): - for f in srcs: - with open(args.destination, 'ab') as dst: - with open(os.path.join(temp_dir, directory, f), 'rb') as src: - dst.write(src.read()) +import argparse +import os +import re +import tarfile + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + + parser.add_argument('--archive', action='store') + parser.add_argument('--source-re', action='store') + parser.add_argument('--destination', action='store') + + args = parser.parse_args() + + with tarfile.open(args.archive) as tf: + open(args.destination, 'wb').close() + extract_list = [] + matcher = re.compile(args.source_re) + temp_dir = os.path.join(os.path.dirname(args.destination), 'temp_profiles') + if not os.path.exists(temp_dir): + os.makedirs(temp_dir) + for f in [i for i in tf if matcher.match(i.name)]: + tf.extract(f, path=temp_dir) + for directory, _, srcs in os.walk(temp_dir): + for f in srcs: + with open(args.destination, 'ab') as dst: + with open(os.path.join(temp_dir, directory, f), 'rb') as src: + dst.write(src.read()) diff --git a/build/scripts/filter_zip.py b/build/scripts/filter_zip.py index b2121b9c9e..3ee15b07fb 100644 --- a/build/scripts/filter_zip.py +++ b/build/scripts/filter_zip.py @@ -1,11 +1,11 @@ -import argparse -import os -import re -import uuid -import zipfile - - -def pattern_to_regexp(p): +import argparse +import os +import re +import uuid +import zipfile + + +def pattern_to_regexp(p): return re.compile( '^' + re.escape(p) @@ -15,57 +15,57 @@ def pattern_to_regexp(p): .replace('[_FILE_]', '([^/]*)') + '$' ) - - -def is_deathman(positive_filter, negative_filter, candidate): - remove = positive_filter - for pf in positive_filter: - if pf.match(candidate): - remove = False - break - if not negative_filter or remove: - return remove - for nf in negative_filter: - if nf.match(candidate): - remove = True - break - return remove - - -def just_do_it(): - parser = argparse.ArgumentParser() - parser.add_argument('--positive', action='append', default=[]) - parser.add_argument('--negative', action='append', default=[]) - parser.add_argument('--file', action='store', required=True) - args = parser.parse_args() - if not args.positive and not args.negative: - return - pos = [pattern_to_regexp(i) for i in args.positive] - neg = [pattern_to_regexp(i) for i in args.negative] - temp_dirname = None - for _ in range(10): - candidate = '__unpacked_{}__'.format(uuid.uuid4()) - if not os.path.exists(candidate): - temp_dirname = candidate - os.makedirs(temp_dirname) - if not temp_dirname: - raise Exception("Can't generate name for temp dir") - - with zipfile.ZipFile(args.file, 'r') as zip_ref: - zip_ref.extractall(temp_dirname) - - for root, _, files in os.walk(temp_dirname): - for f in files: - candidate = os.path.join(root, f).replace('\\', '/') - if is_deathman(pos, neg, os.path.relpath(candidate, temp_dirname)): - os.remove(candidate) - - with zipfile.ZipFile(args.file, 'w') as zip_ref: - for root, _, files in os.walk(temp_dirname): - for f in files: - realname = os.path.join(root, f) - zip_ref.write(realname, os.path.sep.join(os.path.normpath(realname).split(os.path.sep, 2)[1:])) - - -if __name__ == '__main__': - just_do_it() + + +def is_deathman(positive_filter, negative_filter, candidate): + remove = positive_filter + for pf in positive_filter: + if pf.match(candidate): + remove = False + break + if not negative_filter or remove: + return remove + for nf in negative_filter: + if nf.match(candidate): + remove = True + break + return remove + + +def just_do_it(): + parser = argparse.ArgumentParser() + parser.add_argument('--positive', action='append', default=[]) + parser.add_argument('--negative', action='append', default=[]) + parser.add_argument('--file', action='store', required=True) + args = parser.parse_args() + if not args.positive and not args.negative: + return + pos = [pattern_to_regexp(i) for i in args.positive] + neg = [pattern_to_regexp(i) for i in args.negative] + temp_dirname = None + for _ in range(10): + candidate = '__unpacked_{}__'.format(uuid.uuid4()) + if not os.path.exists(candidate): + temp_dirname = candidate + os.makedirs(temp_dirname) + if not temp_dirname: + raise Exception("Can't generate name for temp dir") + + with zipfile.ZipFile(args.file, 'r') as zip_ref: + zip_ref.extractall(temp_dirname) + + for root, _, files in os.walk(temp_dirname): + for f in files: + candidate = os.path.join(root, f).replace('\\', '/') + if is_deathman(pos, neg, os.path.relpath(candidate, temp_dirname)): + os.remove(candidate) + + with zipfile.ZipFile(args.file, 'w') as zip_ref: + for root, _, files in os.walk(temp_dirname): + for f in files: + realname = os.path.join(root, f) + zip_ref.write(realname, os.path.sep.join(os.path.normpath(realname).split(os.path.sep, 2)[1:])) + + +if __name__ == '__main__': + just_do_it() diff --git a/build/scripts/fix_java_command_file_cp.py b/build/scripts/fix_java_command_file_cp.py index fc87048c32..6497733b73 100644 --- a/build/scripts/fix_java_command_file_cp.py +++ b/build/scripts/fix_java_command_file_cp.py @@ -1,34 +1,34 @@ -import sys -import os -import argparse -import subprocess -import platform - - -def fix_files(args): - args = args[:] - parser = argparse.ArgumentParser() - parser.add_argument('--build-root', default=None) - args, tail = parser.parse_known_args(args) - for idx, arg in list(enumerate(tail)): - if arg.startswith('@') and os.path.isfile(arg[1:]): - with open(arg[1:]) as f: - fixed = [i.strip() for i in f] - if args.build_root: +import sys +import os +import argparse +import subprocess +import platform + + +def fix_files(args): + args = args[:] + parser = argparse.ArgumentParser() + parser.add_argument('--build-root', default=None) + args, tail = parser.parse_known_args(args) + for idx, arg in list(enumerate(tail)): + if arg.startswith('@') and os.path.isfile(arg[1:]): + with open(arg[1:]) as f: + fixed = [i.strip() for i in f] + if args.build_root: fixed = [os.path.join(args.build_root, i) for ln in fixed for i in ln.split(os.path.pathsep)] - fixed = os.pathsep.join([i.strip() for i in fixed]) - fixed_name = list(os.path.splitext(arg)) - fixed_name[0] += '_fixed' - fixed_name = ''.join(fixed_name) - with open(fixed_name[1:], 'w') as f: - f.write(fixed) - tail[idx:idx + 1] = [fixed_name] - return tail - - -if __name__ == '__main__': - args = fix_files(sys.argv[1:]) - if platform.system() == 'Windows': - sys.exit(subprocess.Popen(args).wait()) - else: - os.execv(args[0], args) + fixed = os.pathsep.join([i.strip() for i in fixed]) + fixed_name = list(os.path.splitext(arg)) + fixed_name[0] += '_fixed' + fixed_name = ''.join(fixed_name) + with open(fixed_name[1:], 'w') as f: + f.write(fixed) + tail[idx:idx + 1] = [fixed_name] + return tail + + +if __name__ == '__main__': + args = fix_files(sys.argv[1:]) + if platform.system() == 'Windows': + sys.exit(subprocess.Popen(args).wait()) + else: + os.execv(args[0], args) diff --git a/build/scripts/fs_tools.py b/build/scripts/fs_tools.py index dec4c349c8..f6d81ca488 100644 --- a/build/scripts/fs_tools.py +++ b/build/scripts/fs_tools.py @@ -1,7 +1,7 @@ from __future__ import print_function import os -import platform +import platform import sys import shutil import errno @@ -46,29 +46,29 @@ if __name__ == '__main__': except OSError: pass shutil.copy(s, d) - elif mode == 'copy_all_files': - src = args[0] - dst = args[1] - for root, _, files in os.walk(src): - for f in files: - if os.path.islink(os.path.join(root, f)): - continue - file_dst = os.path.join(dst, os.path.relpath(root, src), f) - if os.path.exists(file_dst): - continue - try: - os.makedirs(os.path.dirname(file_dst)) - except OSError: - pass - shutil.copy(os.path.join(root, f), file_dst) + elif mode == 'copy_all_files': + src = args[0] + dst = args[1] + for root, _, files in os.walk(src): + for f in files: + if os.path.islink(os.path.join(root, f)): + continue + file_dst = os.path.join(dst, os.path.relpath(root, src), f) + if os.path.exists(file_dst): + continue + try: + os.makedirs(os.path.dirname(file_dst)) + except OSError: + pass + shutil.copy(os.path.join(root, f), file_dst) elif mode == 'rename_if_exists': if os.path.exists(args[0]): shutil.move(args[0], args[1]) elif mode == 'rename': - targetdir = os.path.dirname(args[1]) - if targetdir and not os.path.exists(targetdir): - os.makedirs(os.path.dirname(args[1])) - shutil.move(args[0], args[1]) + targetdir = os.path.dirname(args[1]) + if targetdir and not os.path.exists(targetdir): + os.makedirs(os.path.dirname(args[1])) + shutil.move(args[0], args[1]) elif mode == 'remove': for f in args: try: @@ -78,7 +78,7 @@ if __name__ == '__main__': shutil.rmtree(f) except OSError: pass - elif mode == 'link_or_copy': + elif mode == 'link_or_copy': link_or_copy(args[0], args[1]) elif mode == 'link_or_copy_to_dir': assert len(args) > 1 @@ -90,15 +90,15 @@ if __name__ == '__main__': dst = args[-1] for src in args[start:-1]: link_or_copy(src, os.path.join(dst, os.path.basename(src))) - elif mode == 'cat': - with open(args[0], 'w') as dst: - for input_name in args[1:]: - with open(input_name) as src: - dst.write(src.read()) - elif mode == 'md': - try: - os.makedirs(args[0]) - except OSError: - pass + elif mode == 'cat': + with open(args[0], 'w') as dst: + for input_name in args[1:]: + with open(input_name) as src: + dst.write(src.read()) + elif mode == 'md': + try: + os.makedirs(args[0]) + except OSError: + pass else: raise Exception('unsupported tool %s' % mode) diff --git a/build/scripts/gen_java_codenav_entry.py b/build/scripts/gen_java_codenav_entry.py index ff0a5c737d..a2eb9b87fd 100644 --- a/build/scripts/gen_java_codenav_entry.py +++ b/build/scripts/gen_java_codenav_entry.py @@ -1,57 +1,57 @@ -import argparse -import datetime -import os -import subprocess -import sys -import tarfile +import argparse +import datetime +import os +import subprocess +import sys +import tarfile + + +def extract_kindexes(kindexes): + for kindex in kindexes: + with tarfile.TarFile(kindex) as tf: + for fname in [i for i in tf.getnames() if i.endswith('.kzip')]: + tf.extract(fname) + yield fname + + +def just_do_it(java, kythe, kythe_to_proto, out_name, binding_only, kindexes): + temp_out_name = out_name + '.temp' + kindex_inputs = list(extract_kindexes(kindexes)) + open(temp_out_name, 'w').close() + start = datetime.datetime.now() + for kindex in kindex_inputs: + print >> sys.stderr, '[INFO] Processing:', kindex + indexer_start = datetime.datetime.now() + p = subprocess.Popen([java, '-jar', os.path.join(kythe, 'indexers/java_indexer.jar'), kindex], stdout=subprocess.PIPE) + indexer_out, _ = p.communicate() + print >> sys.stderr, '[INFO] Indexer execution time:', (datetime.datetime.now() - indexer_start).total_seconds(), 'seconds' + if p.returncode: + raise Exception('java_indexer failed with exit code {}'.format(p.returncode)) + dedup_start = datetime.datetime.now() + p = subprocess.Popen([os.path.join(kythe, 'tools/dedup_stream')], stdin=subprocess.PIPE, stdout=subprocess.PIPE) + dedup_out, _ = p.communicate(indexer_out) + print >> sys.stderr, '[INFO] Dedup execution time:', (datetime.datetime.now() - dedup_start).total_seconds(), 'seconds' + if p.returncode: + raise Exception('dedup_stream failed with exit code {}'.format(p.returncode)) + entrystream_start = datetime.datetime.now() + p = subprocess.Popen([os.path.join(kythe, 'tools/entrystream'), '--write_json'], stdin=subprocess.PIPE, stdout=open(temp_out_name, 'a')) + p.communicate(dedup_out) + if p.returncode: + raise Exception('entrystream failed with exit code {}'.format(p.returncode)) + print >> sys.stderr, '[INFO] Entrystream execution time:', (datetime.datetime.now() - entrystream_start).total_seconds(), 'seconds' + preprocess_start = datetime.datetime.now() + subprocess.check_call([kythe_to_proto, '--preprocess-entry', '--entries', temp_out_name, '--out', out_name] + (['--only-binding-data'] if binding_only else [])) + print >> sys.stderr, '[INFO] Preprocessing execution time:', (datetime.datetime.now() - preprocess_start).total_seconds(), 'seconds' + print >> sys.stderr, '[INFO] Total execution time:', (datetime.datetime.now() - start).total_seconds(), 'seconds' + - -def extract_kindexes(kindexes): - for kindex in kindexes: - with tarfile.TarFile(kindex) as tf: - for fname in [i for i in tf.getnames() if i.endswith('.kzip')]: - tf.extract(fname) - yield fname - - -def just_do_it(java, kythe, kythe_to_proto, out_name, binding_only, kindexes): - temp_out_name = out_name + '.temp' - kindex_inputs = list(extract_kindexes(kindexes)) - open(temp_out_name, 'w').close() - start = datetime.datetime.now() - for kindex in kindex_inputs: - print >> sys.stderr, '[INFO] Processing:', kindex - indexer_start = datetime.datetime.now() - p = subprocess.Popen([java, '-jar', os.path.join(kythe, 'indexers/java_indexer.jar'), kindex], stdout=subprocess.PIPE) - indexer_out, _ = p.communicate() - print >> sys.stderr, '[INFO] Indexer execution time:', (datetime.datetime.now() - indexer_start).total_seconds(), 'seconds' - if p.returncode: - raise Exception('java_indexer failed with exit code {}'.format(p.returncode)) - dedup_start = datetime.datetime.now() - p = subprocess.Popen([os.path.join(kythe, 'tools/dedup_stream')], stdin=subprocess.PIPE, stdout=subprocess.PIPE) - dedup_out, _ = p.communicate(indexer_out) - print >> sys.stderr, '[INFO] Dedup execution time:', (datetime.datetime.now() - dedup_start).total_seconds(), 'seconds' - if p.returncode: - raise Exception('dedup_stream failed with exit code {}'.format(p.returncode)) - entrystream_start = datetime.datetime.now() - p = subprocess.Popen([os.path.join(kythe, 'tools/entrystream'), '--write_json'], stdin=subprocess.PIPE, stdout=open(temp_out_name, 'a')) - p.communicate(dedup_out) - if p.returncode: - raise Exception('entrystream failed with exit code {}'.format(p.returncode)) - print >> sys.stderr, '[INFO] Entrystream execution time:', (datetime.datetime.now() - entrystream_start).total_seconds(), 'seconds' - preprocess_start = datetime.datetime.now() - subprocess.check_call([kythe_to_proto, '--preprocess-entry', '--entries', temp_out_name, '--out', out_name] + (['--only-binding-data'] if binding_only else [])) - print >> sys.stderr, '[INFO] Preprocessing execution time:', (datetime.datetime.now() - preprocess_start).total_seconds(), 'seconds' - print >> sys.stderr, '[INFO] Total execution time:', (datetime.datetime.now() - start).total_seconds(), 'seconds' - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument("--java", help="java path") - parser.add_argument("--kythe", help="kythe path") - parser.add_argument("--kythe-to-proto", help="kythe_to_proto tool path") - parser.add_argument("--out-name", help="entries json out name") - parser.add_argument("--binding-only", action="store_true", default=False, help="filter only binding data") - parser.add_argument("kindexes", nargs='*') - args = parser.parse_args() - just_do_it(args.java, args.kythe, args.kythe_to_proto, args.out_name, args.binding_only, args.kindexes) +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument("--java", help="java path") + parser.add_argument("--kythe", help="kythe path") + parser.add_argument("--kythe-to-proto", help="kythe_to_proto tool path") + parser.add_argument("--out-name", help="entries json out name") + parser.add_argument("--binding-only", action="store_true", default=False, help="filter only binding data") + parser.add_argument("kindexes", nargs='*') + args = parser.parse_args() + just_do_it(args.java, args.kythe, args.kythe_to_proto, args.out_name, args.binding_only, args.kindexes) diff --git a/build/scripts/gen_java_codenav_protobuf.py b/build/scripts/gen_java_codenav_protobuf.py index aee8cfe6c3..335c0fc888 100644 --- a/build/scripts/gen_java_codenav_protobuf.py +++ b/build/scripts/gen_java_codenav_protobuf.py @@ -1,22 +1,22 @@ -import argparse -import os - - -def just_do_it(kythe_to_proto, entries, out_name, build_file, source_root): - with open(build_file) as f: - classpath = os.pathsep.join([line.strip() for line in f]) - os.execv( - kythe_to_proto, - [kythe_to_proto, '--sources-rel-root', 'fake_arcadia_root', '--entries', entries, '--out', out_name, '--classpath', classpath, '--arcadia-root', source_root] - ) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument("--kythe-to-proto", help="kythe_to_proto tool path") - parser.add_argument("--entries", help="entries json path") - parser.add_argument("--out-name", help="protbuf out name") - parser.add_argument("--build-file", help="build file( containing classpath )") - parser.add_argument("--source-root", help="source root") - args = parser.parse_args() - just_do_it(args.kythe_to_proto, args.entries, args.out_name, args.build_file, args.source_root) +import argparse +import os + + +def just_do_it(kythe_to_proto, entries, out_name, build_file, source_root): + with open(build_file) as f: + classpath = os.pathsep.join([line.strip() for line in f]) + os.execv( + kythe_to_proto, + [kythe_to_proto, '--sources-rel-root', 'fake_arcadia_root', '--entries', entries, '--out', out_name, '--classpath', classpath, '--arcadia-root', source_root] + ) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument("--kythe-to-proto", help="kythe_to_proto tool path") + parser.add_argument("--entries", help="entries json path") + parser.add_argument("--out-name", help="protbuf out name") + parser.add_argument("--build-file", help="build file( containing classpath )") + parser.add_argument("--source-root", help="source root") + args = parser.parse_args() + just_do_it(args.kythe_to_proto, args.entries, args.out_name, args.build_file, args.source_root) diff --git a/build/scripts/gen_swiftc_output_map.py b/build/scripts/gen_swiftc_output_map.py index 01ce85f256..6162edea22 100644 --- a/build/scripts/gen_swiftc_output_map.py +++ b/build/scripts/gen_swiftc_output_map.py @@ -1,15 +1,15 @@ -import json -import sys - - -def just_do_it(args): - source_root, build_root, out_file, srcs = args[0], args[1], args[2], args[3:] - assert(len(srcs)) - result_obj = {} - for src in srcs: - result_obj[src] = {'object': src.replace(source_root, build_root) + '.o'} - with open(out_file, 'w') as of: - of.write(json.dumps(result_obj)) - -if __name__ == '__main__': - just_do_it(sys.argv[1:]) +import json +import sys + + +def just_do_it(args): + source_root, build_root, out_file, srcs = args[0], args[1], args[2], args[3:] + assert(len(srcs)) + result_obj = {} + for src in srcs: + result_obj[src] = {'object': src.replace(source_root, build_root) + '.o'} + with open(out_file, 'w') as of: + of.write(json.dumps(result_obj)) + +if __name__ == '__main__': + just_do_it(sys.argv[1:]) diff --git a/build/scripts/generate_mf.py b/build/scripts/generate_mf.py index a44a969980..aa28d0b349 100644 --- a/build/scripts/generate_mf.py +++ b/build/scripts/generate_mf.py @@ -3,7 +3,7 @@ import logging import optparse import os import sys -import io +import io import process_command_files as pcf @@ -24,15 +24,15 @@ def process_quotes(s): def parse_args(): args = pcf.get_args(sys.argv[1:]) - lics, peers, free_args, credits = [], [], [], [] + lics, peers, free_args, credits = [], [], [], [] current_list = free_args for a in args: if a == '-Ya,lics': current_list = lics elif a == '-Ya,peers': current_list = peers - elif a == '-Ya,credits': - current_list = credits + elif a == '-Ya,credits': + current_list = credits elif a and a.startswith('-'): current_list = free_args current_list.append(a) @@ -43,26 +43,26 @@ def parse_args(): parser.add_option('--build-root') parser.add_option('--module-name') parser.add_option('-o', '--output') - parser.add_option('-c', '--credits-output') + parser.add_option('-c', '--credits-output') parser.add_option('-t', '--type') opts, _ = parser.parse_args(free_args) - return lics, peers, credits, opts, - - -def generate_header(meta): - return '-' * 20 + meta.get('path', 'Unknown module') + '-' * 20 + return lics, peers, credits, opts, +def generate_header(meta): + return '-' * 20 + meta.get('path', 'Unknown module') + '-' * 20 + + def generate_mf(): - lics, peers, credits, options = parse_args() + lics, peers, credits, options = parse_args() - meta = { - 'module_name': options.module_name, - 'path': os.path.dirname(options.output), - 'licenses': lics, - 'dependencies': [], - 'license_texts': '' - } + meta = { + 'module_name': options.module_name, + 'path': os.path.dirname(options.output), + 'licenses': lics, + 'dependencies': [], + 'license_texts': '' + } build_root = options.build_root file_name = os.path.join(build_root, options.output) @@ -73,34 +73,34 @@ def generate_mf(): peer_meta = json.load(peer_file) meta['dependencies'].append(peer_meta) - if credits: - union_texts = [] - for texts_file in credits: + if credits: + union_texts = [] + for texts_file in credits: with open(process_quotes(texts_file)) as f: - union_texts.append(f.read()) - meta['license_texts'] = '\n\n'.join(union_texts) - - if options.credits_output: - final_credits = [] - if meta['license_texts']: - final_credits.append(generate_header(meta) + '\n' + meta['license_texts']) - for peer in peers: - candidate = os.path.join(build_root, peer + '.mf') - with open(candidate) as src: - data = json.loads(src.read()) - texts = data.get('license_texts') - if texts: - candidate_text = generate_header(data) + '\n' + texts - if isinstance(candidate_text, unicode): - candidate_text = candidate_text.encode('utf-8') - final_credits.append(candidate_text) - - with io.open(options.credits_output, 'w', encoding='utf-8') as f: - data = '\n\n'.join(final_credits) - if isinstance(data, str): - data = data.decode('utf-8') - f.write(data) - + union_texts.append(f.read()) + meta['license_texts'] = '\n\n'.join(union_texts) + + if options.credits_output: + final_credits = [] + if meta['license_texts']: + final_credits.append(generate_header(meta) + '\n' + meta['license_texts']) + for peer in peers: + candidate = os.path.join(build_root, peer + '.mf') + with open(candidate) as src: + data = json.loads(src.read()) + texts = data.get('license_texts') + if texts: + candidate_text = generate_header(data) + '\n' + texts + if isinstance(candidate_text, unicode): + candidate_text = candidate_text.encode('utf-8') + final_credits.append(candidate_text) + + with io.open(options.credits_output, 'w', encoding='utf-8') as f: + data = '\n\n'.join(final_credits) + if isinstance(data, str): + data = data.decode('utf-8') + f.write(data) + with open(file_name, 'w') as mf_file: json.dump(meta, mf_file, indent=4) diff --git a/build/scripts/generate_pom.py b/build/scripts/generate_pom.py index 200caebc0b..95d718b776 100644 --- a/build/scripts/generate_pom.py +++ b/build/scripts/generate_pom.py @@ -1,10 +1,10 @@ import sys import xml.etree.ElementTree as et import argparse -import os -import json -import base64 -import re +import os +import json +import base64 +import re DEFAULT_YANDEX_GROUP_ID = 'ru.yandex' @@ -18,53 +18,53 @@ MAVEN_PLUGIN_ARTIFACT_ID = 'maven-compiler-plugin' MAVEN_PLUGIN_VERSION = '3.3' JAVA_LANGUAGE_LEVEL = '1.8' -MAVEN_BUILD_HELPER_GROUP_ID = 'org.codehaus.mojo' -MAVEN_BUILD_HELPER_ARTIFACT_ID = 'build-helper-maven-plugin' -MAVEN_BUILD_HELPER_VERSION = '1.9.1' - -MAVEN_EXEC_GROUP_ID = 'org.codehaus.mojo' -MAVEN_EXEC_ARTIFACT_ID = 'exec-maven-plugin' -MAVEN_EXEC_VERSION = '1.5.0' - -MAVEN_SUREFIRE_GROUP_ID = 'org.apache.maven.plugins' -MAVEN_SUREFIRE_ARTIFACT_ID = 'maven-surefire-plugin' -MAVEN_SUREFIRE_VERSION = '2.12.2' - - +MAVEN_BUILD_HELPER_GROUP_ID = 'org.codehaus.mojo' +MAVEN_BUILD_HELPER_ARTIFACT_ID = 'build-helper-maven-plugin' +MAVEN_BUILD_HELPER_VERSION = '1.9.1' + +MAVEN_EXEC_GROUP_ID = 'org.codehaus.mojo' +MAVEN_EXEC_ARTIFACT_ID = 'exec-maven-plugin' +MAVEN_EXEC_VERSION = '1.5.0' + +MAVEN_SUREFIRE_GROUP_ID = 'org.apache.maven.plugins' +MAVEN_SUREFIRE_ARTIFACT_ID = 'maven-surefire-plugin' +MAVEN_SUREFIRE_VERSION = '2.12.2' + + def target_from_contrib(target_path): return target_path.startswith('contrib') -def split_artifacts(s): - m = re.match('^([^:]*:[^:]*:[^:]*:[^:]*)(.*)$', s) - if not m or not m.groups(): - return [] - if not m.groups()[1].startswith('::'): - return [m.groups()[0]] - return [m.groups()[0]] + m.groups()[1].split('::')[1:] - - -def build_pom_and_export_to_maven(**kwargs): - target_path = kwargs.get('target_path') - target = kwargs.get('target') - pom_path = kwargs.get('pom_path') - source_dirs = kwargs.get('source_dirs') - output_dir = kwargs.get('output_dir') - final_name = kwargs.get('final_name') - packaging = kwargs.get('packaging') - target_dependencies = kwargs.get('target_dependencies') - test_target_dependencies = kwargs.get('test_target_dependencies') - test_target_dependencies_exclude = kwargs.get('test_target_dependencies_exclude') - modules_path = kwargs.get('modules_path') - prop_vars = kwargs.get('properties') - external_jars = kwargs.get('external_jars') - resources = kwargs.get('resources') - run_java_programs = [json.loads(base64.b64decode(i)) for i in kwargs.get('run_java_programs')] - test_source_dirs = kwargs.get('test_source_dirs') - test_resource_dirs = kwargs.get('test_resource_dirs') - - modules = [] - +def split_artifacts(s): + m = re.match('^([^:]*:[^:]*:[^:]*:[^:]*)(.*)$', s) + if not m or not m.groups(): + return [] + if not m.groups()[1].startswith('::'): + return [m.groups()[0]] + return [m.groups()[0]] + m.groups()[1].split('::')[1:] + + +def build_pom_and_export_to_maven(**kwargs): + target_path = kwargs.get('target_path') + target = kwargs.get('target') + pom_path = kwargs.get('pom_path') + source_dirs = kwargs.get('source_dirs') + output_dir = kwargs.get('output_dir') + final_name = kwargs.get('final_name') + packaging = kwargs.get('packaging') + target_dependencies = kwargs.get('target_dependencies') + test_target_dependencies = kwargs.get('test_target_dependencies') + test_target_dependencies_exclude = kwargs.get('test_target_dependencies_exclude') + modules_path = kwargs.get('modules_path') + prop_vars = kwargs.get('properties') + external_jars = kwargs.get('external_jars') + resources = kwargs.get('resources') + run_java_programs = [json.loads(base64.b64decode(i)) for i in kwargs.get('run_java_programs')] + test_source_dirs = kwargs.get('test_source_dirs') + test_resource_dirs = kwargs.get('test_resource_dirs') + + modules = [] + def _indent(elem, level=0): ind = "\n" + level * " " if len(elem): @@ -91,49 +91,49 @@ def build_pom_and_export_to_maven(**kwargs): et.SubElement(project, 'groupId').text = group_id et.SubElement(project, 'artifactId').text = artifact_id et.SubElement(project, 'version').text = version - et.SubElement(project, 'packaging').text = packaging + et.SubElement(project, 'packaging').text = packaging properties = et.SubElement(project, 'properties') et.SubElement(properties, 'project.build.sourceEncoding').text = 'UTF-8' - if prop_vars: - for property, value in json.loads(base64.b64decode(prop_vars)).items(): - et.SubElement(properties, property).text = value - - if modules_path: - with open(modules_path) as f: - modules = [i.strip() for i in f if i.strip()] - - if modules: - modules_el = et.SubElement(project, 'modules') - for module in modules: - et.SubElement(modules_el, 'module').text = module - - build = et.SubElement(project, 'build') - if source_dirs: - et.SubElement(build, 'sourceDirectory').text = source_dirs[0] - source_dirs = source_dirs[1:] - if test_source_dirs: - et.SubElement(build, 'testSourceDirectory').text = test_source_dirs[0] - test_source_dirs = test_source_dirs[1:] - if output_dir: - et.SubElement(build, 'outputDirectory').text = output_dir - if final_name: - et.SubElement(build, 'finalName').text = final_name - if resources: - resource_element = et.SubElement(et.SubElement(build, 'resources'), 'resource') - et.SubElement(resource_element, 'directory').text = '${basedir}' - includes = et.SubElement(resource_element, 'includes') - for resource in resources: - et.SubElement(includes, 'include').text = resource - if test_resource_dirs: - test_resource_element = et.SubElement(build, 'testResources') - for test_resource_dir in test_resource_dirs: - et.SubElement(et.SubElement(test_resource_element, 'testResource'), 'directory').text = '${basedir}' + (('/' + test_resource_dir) if test_resource_dir != '.' else '') - - plugins = et.SubElement(build, 'plugins') - - if packaging != 'pom': + if prop_vars: + for property, value in json.loads(base64.b64decode(prop_vars)).items(): + et.SubElement(properties, property).text = value + + if modules_path: + with open(modules_path) as f: + modules = [i.strip() for i in f if i.strip()] + + if modules: + modules_el = et.SubElement(project, 'modules') + for module in modules: + et.SubElement(modules_el, 'module').text = module + + build = et.SubElement(project, 'build') + if source_dirs: + et.SubElement(build, 'sourceDirectory').text = source_dirs[0] + source_dirs = source_dirs[1:] + if test_source_dirs: + et.SubElement(build, 'testSourceDirectory').text = test_source_dirs[0] + test_source_dirs = test_source_dirs[1:] + if output_dir: + et.SubElement(build, 'outputDirectory').text = output_dir + if final_name: + et.SubElement(build, 'finalName').text = final_name + if resources: + resource_element = et.SubElement(et.SubElement(build, 'resources'), 'resource') + et.SubElement(resource_element, 'directory').text = '${basedir}' + includes = et.SubElement(resource_element, 'includes') + for resource in resources: + et.SubElement(includes, 'include').text = resource + if test_resource_dirs: + test_resource_element = et.SubElement(build, 'testResources') + for test_resource_dir in test_resource_dirs: + et.SubElement(et.SubElement(test_resource_element, 'testResource'), 'directory').text = '${basedir}' + (('/' + test_resource_dir) if test_resource_dir != '.' else '') + + plugins = et.SubElement(build, 'plugins') + + if packaging != 'pom': maven_plugin = et.SubElement(plugins, 'plugin') et.SubElement(maven_plugin, 'groupId').text = MAVEN_PLUGIN_GROUP_ID et.SubElement(maven_plugin, 'artifactId').text = MAVEN_PLUGIN_ARTIFACT_ID @@ -142,134 +142,134 @@ def build_pom_and_export_to_maven(**kwargs): et.SubElement(configuration, 'source').text = JAVA_LANGUAGE_LEVEL et.SubElement(configuration, 'target').text = JAVA_LANGUAGE_LEVEL - if source_dirs or external_jars or test_source_dirs: - build_helper_plugin = et.SubElement(plugins, 'plugin') - et.SubElement(build_helper_plugin, 'groupId').text = MAVEN_BUILD_HELPER_GROUP_ID - et.SubElement(build_helper_plugin, 'artifactId').text = MAVEN_BUILD_HELPER_ARTIFACT_ID - et.SubElement(build_helper_plugin, 'version').text = MAVEN_BUILD_HELPER_VERSION - executions = et.SubElement(build_helper_plugin, 'executions') - if source_dirs: - execution = et.SubElement(executions, 'execution') - et.SubElement(execution, 'id').text = 'add-source' - et.SubElement(execution, 'phase').text = 'generate-sources' - et.SubElement(et.SubElement(execution, 'goals'), 'goal').text = 'add-source' - sources = et.SubElement(et.SubElement(execution, 'configuration'), 'sources') - for source_dir in source_dirs: - et.SubElement(sources, 'source').text = source_dir - if external_jars: - execution = et.SubElement(executions, 'execution') - et.SubElement(execution, 'id').text = 'attach-artifacts' - et.SubElement(execution, 'phase').text = 'generate-sources' - et.SubElement(et.SubElement(execution, 'goals'), 'goal').text = 'attach-artifact' - artifacts = et.SubElement(et.SubElement(execution, 'configuration'), 'artifacts') - for external_jar in external_jars: - external_artifact = et.SubElement(artifacts, 'artifact') - et.SubElement(external_artifact, 'file').text = '${basedir}/' + external_jar - et.SubElement(external_artifact, 'type').text = 'jar' - if test_source_dirs: - execution = et.SubElement(executions, 'execution') - et.SubElement(execution, 'id').text = 'add-test-source' - et.SubElement(execution, 'phase').text = 'generate-test-sources' - et.SubElement(et.SubElement(execution, 'goals'), 'goal').text = 'add-test-source' - sources = et.SubElement(et.SubElement(execution, 'configuration'), 'sources') - for source_dir in source_dirs: - et.SubElement(sources, 'source').text = source_dir - - if run_java_programs: - exec_plugin = et.SubElement(plugins, 'plugin') - et.SubElement(exec_plugin, 'groupId').text = MAVEN_EXEC_GROUP_ID - et.SubElement(exec_plugin, 'artifactId').text = MAVEN_EXEC_ARTIFACT_ID - et.SubElement(exec_plugin, 'version').text = MAVEN_EXEC_VERSION - jp_dependencies = et.SubElement(exec_plugin, 'dependencies') - executions = et.SubElement(exec_plugin, 'executions') - for java_program in run_java_programs: - execution = et.SubElement(executions, 'execution') - et.SubElement(execution, 'phase').text = 'generate-sources' - et.SubElement(et.SubElement(execution, 'goals'), 'goal').text = 'java' - jp_configuration = et.SubElement(execution, 'configuration') - main_cls, args = None, [] - for word in java_program['cmd']: - if not main_cls and not word.startswith('-'): - main_cls = word - else: - args.append(word) - et.SubElement(jp_configuration, 'mainClass').text = main_cls - et.SubElement(jp_configuration, 'includePluginDependencies').text = 'true' - et.SubElement(jp_configuration, 'includeProjectDependencies').text = 'false' - if args: - jp_arguments = et.SubElement(jp_configuration, 'arguments') - for arg in args: - et.SubElement(jp_arguments, 'argument').text = arg - if java_program['deps']: - for jp_dep in java_program['deps']: - jp_dependency = et.SubElement(jp_dependencies, 'dependency') - jp_g, jp_a, jp_v = jp_dep.split(':') - et.SubElement(jp_dependency, 'groupId').text = jp_g - et.SubElement(jp_dependency, 'artifactId').text = jp_a - et.SubElement(jp_dependency, 'version').text = jp_v - et.SubElement(jp_dependency, 'type').text = 'jar' - - if target_dependencies + test_target_dependencies: - dependencies = et.SubElement(project, 'dependencies') - for target_dependency in target_dependencies + test_target_dependencies: - dependency = et.SubElement(dependencies, 'dependency') - dependency_info = split_artifacts(target_dependency) - - group_id, artifact_id, version, classifier = dependency_info[0].split(':') - - et.SubElement(dependency, 'groupId').text = group_id - et.SubElement(dependency, 'artifactId').text = artifact_id - et.SubElement(dependency, 'version').text = version - if classifier: - et.SubElement(dependency, 'classifier').text = classifier - if target_dependency in test_target_dependencies: - et.SubElement(dependency, 'scope').text = 'test' - - if len(dependency_info) > 1: - exclusions = et.SubElement(dependency, 'exclusions') - for exclude in dependency_info[1:]: - group_id, artifact_id = exclude.split(':') - exclusion_el = et.SubElement(exclusions, 'exclusion') - et.SubElement(exclusion_el, 'groupId').text = group_id - et.SubElement(exclusion_el, 'artifactId').text = artifact_id - - if test_target_dependencies_exclude: - surefire_plugin = et.SubElement(plugins, 'plugin') - et.SubElement(surefire_plugin, 'groupId').text = MAVEN_SUREFIRE_GROUP_ID - et.SubElement(surefire_plugin, 'artifactId').text = MAVEN_SUREFIRE_ARTIFACT_ID - et.SubElement(surefire_plugin, 'version').text = MAVEN_SUREFIRE_VERSION - classpath_excludes = et.SubElement(et.SubElement(surefire_plugin, 'configuration'), 'classpathDependencyExcludes') - for classpath_exclude in test_target_dependencies_exclude: - et.SubElement(classpath_excludes, 'classpathDependencyExclude').text = classpath_exclude - + if source_dirs or external_jars or test_source_dirs: + build_helper_plugin = et.SubElement(plugins, 'plugin') + et.SubElement(build_helper_plugin, 'groupId').text = MAVEN_BUILD_HELPER_GROUP_ID + et.SubElement(build_helper_plugin, 'artifactId').text = MAVEN_BUILD_HELPER_ARTIFACT_ID + et.SubElement(build_helper_plugin, 'version').text = MAVEN_BUILD_HELPER_VERSION + executions = et.SubElement(build_helper_plugin, 'executions') + if source_dirs: + execution = et.SubElement(executions, 'execution') + et.SubElement(execution, 'id').text = 'add-source' + et.SubElement(execution, 'phase').text = 'generate-sources' + et.SubElement(et.SubElement(execution, 'goals'), 'goal').text = 'add-source' + sources = et.SubElement(et.SubElement(execution, 'configuration'), 'sources') + for source_dir in source_dirs: + et.SubElement(sources, 'source').text = source_dir + if external_jars: + execution = et.SubElement(executions, 'execution') + et.SubElement(execution, 'id').text = 'attach-artifacts' + et.SubElement(execution, 'phase').text = 'generate-sources' + et.SubElement(et.SubElement(execution, 'goals'), 'goal').text = 'attach-artifact' + artifacts = et.SubElement(et.SubElement(execution, 'configuration'), 'artifacts') + for external_jar in external_jars: + external_artifact = et.SubElement(artifacts, 'artifact') + et.SubElement(external_artifact, 'file').text = '${basedir}/' + external_jar + et.SubElement(external_artifact, 'type').text = 'jar' + if test_source_dirs: + execution = et.SubElement(executions, 'execution') + et.SubElement(execution, 'id').text = 'add-test-source' + et.SubElement(execution, 'phase').text = 'generate-test-sources' + et.SubElement(et.SubElement(execution, 'goals'), 'goal').text = 'add-test-source' + sources = et.SubElement(et.SubElement(execution, 'configuration'), 'sources') + for source_dir in source_dirs: + et.SubElement(sources, 'source').text = source_dir + + if run_java_programs: + exec_plugin = et.SubElement(plugins, 'plugin') + et.SubElement(exec_plugin, 'groupId').text = MAVEN_EXEC_GROUP_ID + et.SubElement(exec_plugin, 'artifactId').text = MAVEN_EXEC_ARTIFACT_ID + et.SubElement(exec_plugin, 'version').text = MAVEN_EXEC_VERSION + jp_dependencies = et.SubElement(exec_plugin, 'dependencies') + executions = et.SubElement(exec_plugin, 'executions') + for java_program in run_java_programs: + execution = et.SubElement(executions, 'execution') + et.SubElement(execution, 'phase').text = 'generate-sources' + et.SubElement(et.SubElement(execution, 'goals'), 'goal').text = 'java' + jp_configuration = et.SubElement(execution, 'configuration') + main_cls, args = None, [] + for word in java_program['cmd']: + if not main_cls and not word.startswith('-'): + main_cls = word + else: + args.append(word) + et.SubElement(jp_configuration, 'mainClass').text = main_cls + et.SubElement(jp_configuration, 'includePluginDependencies').text = 'true' + et.SubElement(jp_configuration, 'includeProjectDependencies').text = 'false' + if args: + jp_arguments = et.SubElement(jp_configuration, 'arguments') + for arg in args: + et.SubElement(jp_arguments, 'argument').text = arg + if java_program['deps']: + for jp_dep in java_program['deps']: + jp_dependency = et.SubElement(jp_dependencies, 'dependency') + jp_g, jp_a, jp_v = jp_dep.split(':') + et.SubElement(jp_dependency, 'groupId').text = jp_g + et.SubElement(jp_dependency, 'artifactId').text = jp_a + et.SubElement(jp_dependency, 'version').text = jp_v + et.SubElement(jp_dependency, 'type').text = 'jar' + + if target_dependencies + test_target_dependencies: + dependencies = et.SubElement(project, 'dependencies') + for target_dependency in target_dependencies + test_target_dependencies: + dependency = et.SubElement(dependencies, 'dependency') + dependency_info = split_artifacts(target_dependency) + + group_id, artifact_id, version, classifier = dependency_info[0].split(':') + + et.SubElement(dependency, 'groupId').text = group_id + et.SubElement(dependency, 'artifactId').text = artifact_id + et.SubElement(dependency, 'version').text = version + if classifier: + et.SubElement(dependency, 'classifier').text = classifier + if target_dependency in test_target_dependencies: + et.SubElement(dependency, 'scope').text = 'test' + + if len(dependency_info) > 1: + exclusions = et.SubElement(dependency, 'exclusions') + for exclude in dependency_info[1:]: + group_id, artifact_id = exclude.split(':') + exclusion_el = et.SubElement(exclusions, 'exclusion') + et.SubElement(exclusion_el, 'groupId').text = group_id + et.SubElement(exclusion_el, 'artifactId').text = artifact_id + + if test_target_dependencies_exclude: + surefire_plugin = et.SubElement(plugins, 'plugin') + et.SubElement(surefire_plugin, 'groupId').text = MAVEN_SUREFIRE_GROUP_ID + et.SubElement(surefire_plugin, 'artifactId').text = MAVEN_SUREFIRE_ARTIFACT_ID + et.SubElement(surefire_plugin, 'version').text = MAVEN_SUREFIRE_VERSION + classpath_excludes = et.SubElement(et.SubElement(surefire_plugin, 'configuration'), 'classpathDependencyExcludes') + for classpath_exclude in test_target_dependencies_exclude: + et.SubElement(classpath_excludes, 'classpathDependencyExclude').text = classpath_exclude + et.register_namespace('', DEFAULT_NAMESPACE) et.register_namespace('xsi', XSI_NAMESPACE) _indent(project) et.ElementTree(project).write(pom_path) - sys.stderr.write("[MAVEN EXPORT] Generated {} file for target {}\n".format(os.path.basename(pom_path), target_path)) + sys.stderr.write("[MAVEN EXPORT] Generated {} file for target {}\n".format(os.path.basename(pom_path), target_path)) if __name__ == '__main__': parser = argparse.ArgumentParser() - parser.add_argument('--target-path', action='store', default='') - parser.add_argument('--target', action='store') - parser.add_argument('--pom-path', action='store') - parser.add_argument('--source-dirs', action='append', default=[]) - parser.add_argument('--external-jars', action='append', default=[]) - parser.add_argument('--resources', action='append', default=[]) - parser.add_argument('--run-java-programs', action='append', default=[]) - parser.add_argument('--output-dir') - parser.add_argument('--final-name') - parser.add_argument('--packaging', default='jar') - parser.add_argument('--target-dependencies', action='append', default=[]) - parser.add_argument('--test-target-dependencies', action='append', default=[]) - parser.add_argument('--test-target-dependencies-exclude', action='append', default=[]) - parser.add_argument('--modules-path', action='store') - parser.add_argument('--properties') - parser.add_argument('--test-source-dirs', action='append', default=[]) - parser.add_argument('--test-resource-dirs', action='append', default=[]) + parser.add_argument('--target-path', action='store', default='') + parser.add_argument('--target', action='store') + parser.add_argument('--pom-path', action='store') + parser.add_argument('--source-dirs', action='append', default=[]) + parser.add_argument('--external-jars', action='append', default=[]) + parser.add_argument('--resources', action='append', default=[]) + parser.add_argument('--run-java-programs', action='append', default=[]) + parser.add_argument('--output-dir') + parser.add_argument('--final-name') + parser.add_argument('--packaging', default='jar') + parser.add_argument('--target-dependencies', action='append', default=[]) + parser.add_argument('--test-target-dependencies', action='append', default=[]) + parser.add_argument('--test-target-dependencies-exclude', action='append', default=[]) + parser.add_argument('--modules-path', action='store') + parser.add_argument('--properties') + parser.add_argument('--test-source-dirs', action='append', default=[]) + parser.add_argument('--test-resource-dirs', action='append', default=[]) args = parser.parse_args() - build_pom_and_export_to_maven(**vars(args)) + build_pom_and_export_to_maven(**vars(args)) diff --git a/build/scripts/go_tool.py b/build/scripts/go_tool.py index c1e98b20c0..7ee1550241 100644 --- a/build/scripts/go_tool.py +++ b/build/scripts/go_tool.py @@ -796,9 +796,9 @@ if __name__ == '__main__': parser.add_argument('++output-root', required=True) parser.add_argument('++toolchain-root', required=True) parser.add_argument('++host-os', choices=['linux', 'darwin', 'windows'], required=True) - parser.add_argument('++host-arch', choices=['amd64', 'arm64'], required=True) + parser.add_argument('++host-arch', choices=['amd64', 'arm64'], required=True) parser.add_argument('++targ-os', choices=['linux', 'darwin', 'windows'], required=True) - parser.add_argument('++targ-arch', choices=['amd64', 'x86', 'arm64'], required=True) + parser.add_argument('++targ-arch', choices=['amd64', 'x86', 'arm64'], required=True) parser.add_argument('++peers', nargs='*') parser.add_argument('++non-local-peers', nargs='*') parser.add_argument('++cgo-peers', nargs='*') diff --git a/build/scripts/ios_wrapper.py b/build/scripts/ios_wrapper.py index d3aa48387a..0d6d1d231f 100644 --- a/build/scripts/ios_wrapper.py +++ b/build/scripts/ios_wrapper.py @@ -1,180 +1,180 @@ -import errno -import json -import os -import shutil -import subprocess -import sys -import tarfile -import plistlib - - -def ensure_dir(path): - try: - os.makedirs(path) - except OSError as e: - if e.errno != errno.EEXIST or not os.path.isdir(path): - raise - - -def just_do_it(args): - if not args: - raise Exception('Not enough args!') - parts = [[]] - for arg in args: - if arg == '__DELIM__': - parts.append([]) - else: - parts[-1].append(arg) - if len(parts) != 3 or len(parts[0]) != 5: - raise Exception('Bad call') +import errno +import json +import os +import shutil +import subprocess +import sys +import tarfile +import plistlib + + +def ensure_dir(path): + try: + os.makedirs(path) + except OSError as e: + if e.errno != errno.EEXIST or not os.path.isdir(path): + raise + + +def just_do_it(args): + if not args: + raise Exception('Not enough args!') + parts = [[]] + for arg in args: + if arg == '__DELIM__': + parts.append([]) + else: + parts[-1].append(arg) + if len(parts) != 3 or len(parts[0]) != 5: + raise Exception('Bad call') bin_name, ibtool_path, main_out, app_name, module_dir = parts[0] - bin_name = os.path.basename(bin_name) - inputs, storyboard_user_flags = parts[1:] - plists, storyboards, signs, nibs, resources, signed_resources, plist_jsons, strings = [], [], [], [], [], [], [], [] - for i in inputs: - if i.endswith('.plist') or i.endswith('.partial_plist'): - plists.append(i) - elif i.endswith('.compiled_storyboard_tar'): - storyboards.append(i) - elif i.endswith('.xcent'): - signs.append(i) - elif i.endswith('.nib'): - nibs.append(i) - elif i.endswith('.resource_tar'): - resources.append(i) - elif i.endswith('.signed_resource_tar'): - signed_resources.append(i) - elif i.endswith('.plist_json'): - plist_jsons.append(i) - elif i.endswith('.strings_tar'): - strings.append(i) - else: - print >> sys.stderr, 'Unknown input:', i, 'ignoring' - if not plists: - raise Exception("Can't find plist files") - if not plists[0].endswith('.plist'): - print >> sys.stderr, "Main plist file can be defined incorretly" - if not storyboards: - print >> sys.stderr, "Storyboards list are empty" - if len(signs) > 1: - raise Exception("Too many .xcent files") - app_dir = os.path.join(module_dir, app_name + '.app') - ensure_dir(app_dir) - copy_nibs(nibs, module_dir, app_dir) - replaced_parameters = { - 'DEVELOPMENT_LANGUAGE': 'en', - 'EXECUTABLE_NAME': bin_name, - 'PRODUCT_BUNDLE_IDENTIFIER': 'Yandex.' + app_name, - 'PRODUCT_NAME': app_name, - } - replaced_templates = {} - for plist_json in plist_jsons: - with open(plist_json) as jsonfile: - for k, v in json.loads(jsonfile.read()).items(): - replaced_parameters[k] = v - for k, v in replaced_parameters.items(): - replaced_templates['$(' + k + ')'] = v - replaced_templates['${' + k + '}'] = v - make_main_plist(plists, os.path.join(app_dir, 'Info.plist'), replaced_templates) + bin_name = os.path.basename(bin_name) + inputs, storyboard_user_flags = parts[1:] + plists, storyboards, signs, nibs, resources, signed_resources, plist_jsons, strings = [], [], [], [], [], [], [], [] + for i in inputs: + if i.endswith('.plist') or i.endswith('.partial_plist'): + plists.append(i) + elif i.endswith('.compiled_storyboard_tar'): + storyboards.append(i) + elif i.endswith('.xcent'): + signs.append(i) + elif i.endswith('.nib'): + nibs.append(i) + elif i.endswith('.resource_tar'): + resources.append(i) + elif i.endswith('.signed_resource_tar'): + signed_resources.append(i) + elif i.endswith('.plist_json'): + plist_jsons.append(i) + elif i.endswith('.strings_tar'): + strings.append(i) + else: + print >> sys.stderr, 'Unknown input:', i, 'ignoring' + if not plists: + raise Exception("Can't find plist files") + if not plists[0].endswith('.plist'): + print >> sys.stderr, "Main plist file can be defined incorretly" + if not storyboards: + print >> sys.stderr, "Storyboards list are empty" + if len(signs) > 1: + raise Exception("Too many .xcent files") + app_dir = os.path.join(module_dir, app_name + '.app') + ensure_dir(app_dir) + copy_nibs(nibs, module_dir, app_dir) + replaced_parameters = { + 'DEVELOPMENT_LANGUAGE': 'en', + 'EXECUTABLE_NAME': bin_name, + 'PRODUCT_BUNDLE_IDENTIFIER': 'Yandex.' + app_name, + 'PRODUCT_NAME': app_name, + } + replaced_templates = {} + for plist_json in plist_jsons: + with open(plist_json) as jsonfile: + for k, v in json.loads(jsonfile.read()).items(): + replaced_parameters[k] = v + for k, v in replaced_parameters.items(): + replaced_templates['$(' + k + ')'] = v + replaced_templates['${' + k + '}'] = v + make_main_plist(plists, os.path.join(app_dir, 'Info.plist'), replaced_templates) link_storyboards(ibtool_path, storyboards, app_name, app_dir, storyboard_user_flags) - if resources: - extract_resources(resources, app_dir) - if signed_resources: - extract_resources(signed_resources, app_dir, sign=True) - if strings: - extract_resources(strings, app_dir, strings=True) - if not signs: - sign_file = os.path.join(module_dir, app_name + '.xcent') - with open(sign_file, 'w') as f: - f.write('''<?xml version="1.0" encoding="UTF-8"?> -<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> -<plist version="1.0"> -<dict> - <key>com.apple.security.get-task-allow</key> - <true/> -</dict> -</plist> - ''') - else: - sign_file = signs[0] - sign_application(sign_file, app_dir) - make_archive(app_dir, main_out) - - -def is_exe(fpath): - return os.path.isfile(fpath) and os.access(fpath, os.X_OK) - - -def copy_nibs(nibs, module_dir, app_dir): - for nib in nibs: - dst = os.path.join(app_dir, os.path.relpath(nib, module_dir)) - ensure_dir(os.path.dirname(dst)) - shutil.copyfile(nib, dst) - - -def make_main_plist(inputs, out, replaced_parameters): - united_data = {} - for i in inputs: - united_data.update(plistlib.readPlist(i)) - - def scan_n_replace(root): - if not isinstance(root, dict): - raise Exception('Invalid state') - for k in root: - if isinstance(root[k], list): - for i in xrange(len(root[k])): - if isinstance(root[k][i], dict): - scan_n_replace(root[k][i]) - elif root[k][i] in replaced_parameters: - root[k][i] = replaced_parameters[root[k][i]] - elif isinstance(root[k], dict): - scan_n_replace(root[k]) - else: - if root[k] in replaced_parameters: - root[k] = replaced_parameters[root[k]] - scan_n_replace(united_data) - plistlib.writePlist(united_data, out) - subprocess.check_call(['/usr/bin/plutil', '-convert', 'binary1', out]) - - -def link_storyboards(ibtool, archives, app_name, app_dir, flags): - unpacked = [] - for arc in archives: - unpacked.append(os.path.splitext(arc)[0] + 'c') - ensure_dir(unpacked[-1]) - with tarfile.open(arc) as a: - a.extractall(path=unpacked[-1]) - flags += [ - '--module', app_name, - '--link', app_dir, - ] - subprocess.check_call([ibtool] + flags + - ['--errors', '--warnings', '--notices', '--output-format', 'human-readable-text'] + - unpacked) - - -def sign_application(xcent, app_dir): - subprocess.check_call(['/usr/bin/codesign', '--force', '--sign', '-', '--entitlements', xcent, '--timestamp=none', app_dir]) - - -def extract_resources(resources, app_dir, strings=False, sign=False): - for res in resources: - with tarfile.open(res) as tf: - for tfinfo in tf: - tf.extract(tfinfo.name, app_dir) - if strings: - subprocess.check_call(['/usr/bin/plutil', '-convert', 'binary1', os.path.join(app_dir, tfinfo.name)]) - if sign: - subprocess.check_call(['/usr/bin/codesign', '--force', '--sign', '-', os.path.join(app_dir, tfinfo.name)]) - - -def make_archive(app_dir, output): - with tarfile.open(output, "w") as tar_handle: - for root, _, files in os.walk(app_dir): - for f in files: - tar_handle.add(os.path.join(root, f), arcname=os.path.join(os.path.basename(app_dir), - os.path.relpath(os.path.join(root, f), app_dir))) - - -if __name__ == '__main__': - just_do_it(sys.argv[1:]) + if resources: + extract_resources(resources, app_dir) + if signed_resources: + extract_resources(signed_resources, app_dir, sign=True) + if strings: + extract_resources(strings, app_dir, strings=True) + if not signs: + sign_file = os.path.join(module_dir, app_name + '.xcent') + with open(sign_file, 'w') as f: + f.write('''<?xml version="1.0" encoding="UTF-8"?> +<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> +<plist version="1.0"> +<dict> + <key>com.apple.security.get-task-allow</key> + <true/> +</dict> +</plist> + ''') + else: + sign_file = signs[0] + sign_application(sign_file, app_dir) + make_archive(app_dir, main_out) + + +def is_exe(fpath): + return os.path.isfile(fpath) and os.access(fpath, os.X_OK) + + +def copy_nibs(nibs, module_dir, app_dir): + for nib in nibs: + dst = os.path.join(app_dir, os.path.relpath(nib, module_dir)) + ensure_dir(os.path.dirname(dst)) + shutil.copyfile(nib, dst) + + +def make_main_plist(inputs, out, replaced_parameters): + united_data = {} + for i in inputs: + united_data.update(plistlib.readPlist(i)) + + def scan_n_replace(root): + if not isinstance(root, dict): + raise Exception('Invalid state') + for k in root: + if isinstance(root[k], list): + for i in xrange(len(root[k])): + if isinstance(root[k][i], dict): + scan_n_replace(root[k][i]) + elif root[k][i] in replaced_parameters: + root[k][i] = replaced_parameters[root[k][i]] + elif isinstance(root[k], dict): + scan_n_replace(root[k]) + else: + if root[k] in replaced_parameters: + root[k] = replaced_parameters[root[k]] + scan_n_replace(united_data) + plistlib.writePlist(united_data, out) + subprocess.check_call(['/usr/bin/plutil', '-convert', 'binary1', out]) + + +def link_storyboards(ibtool, archives, app_name, app_dir, flags): + unpacked = [] + for arc in archives: + unpacked.append(os.path.splitext(arc)[0] + 'c') + ensure_dir(unpacked[-1]) + with tarfile.open(arc) as a: + a.extractall(path=unpacked[-1]) + flags += [ + '--module', app_name, + '--link', app_dir, + ] + subprocess.check_call([ibtool] + flags + + ['--errors', '--warnings', '--notices', '--output-format', 'human-readable-text'] + + unpacked) + + +def sign_application(xcent, app_dir): + subprocess.check_call(['/usr/bin/codesign', '--force', '--sign', '-', '--entitlements', xcent, '--timestamp=none', app_dir]) + + +def extract_resources(resources, app_dir, strings=False, sign=False): + for res in resources: + with tarfile.open(res) as tf: + for tfinfo in tf: + tf.extract(tfinfo.name, app_dir) + if strings: + subprocess.check_call(['/usr/bin/plutil', '-convert', 'binary1', os.path.join(app_dir, tfinfo.name)]) + if sign: + subprocess.check_call(['/usr/bin/codesign', '--force', '--sign', '-', os.path.join(app_dir, tfinfo.name)]) + + +def make_archive(app_dir, output): + with tarfile.open(output, "w") as tar_handle: + for root, _, files in os.walk(app_dir): + for f in files: + tar_handle.add(os.path.join(root, f), arcname=os.path.join(os.path.basename(app_dir), + os.path.relpath(os.path.join(root, f), app_dir))) + + +if __name__ == '__main__': + just_do_it(sys.argv[1:]) diff --git a/build/scripts/link_fat_obj.py b/build/scripts/link_fat_obj.py index c189668b9e..6646889d3a 100644 --- a/build/scripts/link_fat_obj.py +++ b/build/scripts/link_fat_obj.py @@ -9,24 +9,24 @@ YA_ARG_PREFIX = '-Ya,' def get_args(): parser = argparse.ArgumentParser() - parser.add_argument('--obj') - parser.add_argument('--globals-lib') + parser.add_argument('--obj') + parser.add_argument('--globals-lib') parser.add_argument('--lib', required=True) parser.add_argument('--arch', required=True) - parser.add_argument('--build-root', default=None) - parser.add_argument('--with-own-obj', action='store_true', default=False) - parser.add_argument('--with-global-srcs', action='store_true', default=False) + parser.add_argument('--build-root', default=None) + parser.add_argument('--with-own-obj', action='store_true', default=False) + parser.add_argument('--with-global-srcs', action='store_true', default=False) groups = {} args_list = groups.setdefault('default', []) for arg in sys.argv[1:]: - if arg == '--with-own-obj': - groups['default'].append(arg) - elif arg == '--globals-lib': - groups['default'].append(arg) - elif arg == '--with-global-srcs': - groups['default'].append(arg) - elif arg.startswith(YA_ARG_PREFIX): + if arg == '--with-own-obj': + groups['default'].append(arg) + elif arg == '--globals-lib': + groups['default'].append(arg) + elif arg == '--with-global-srcs': + groups['default'].append(arg) + elif arg.startswith(YA_ARG_PREFIX): group_name = arg[len(YA_ARG_PREFIX):] args_list = groups.setdefault(group_name, []) else: @@ -61,29 +61,29 @@ def main(): do_link = linker + ['-o', obj_output, '-Wl,-r', '-nodefaultlibs', '-nostartfiles'] + global_srcs + auto_input do_archive = archiver + [lib_output] + peers - do_globals = None - if args.globals_lib: - do_globals = archiver + [args.globals_lib] + auto_input + global_srcs - if args.with_own_obj: - do_archive += auto_input - if args.with_global_srcs: - do_archive += global_srcs + do_globals = None + if args.globals_lib: + do_globals = archiver + [args.globals_lib] + auto_input + global_srcs + if args.with_own_obj: + do_archive += auto_input + if args.with_global_srcs: + do_archive += global_srcs def call(c): - proc = subprocess.Popen(c, shell=False, stderr=sys.stderr, stdout=sys.stdout, cwd=args.build_root) + proc = subprocess.Popen(c, shell=False, stderr=sys.stderr, stdout=sys.stdout, cwd=args.build_root) proc.communicate() return proc.returncode - if obj_output: - link_res = call(do_link) - if link_res: - sys.exit(link_res) - - if do_globals: - glob_res = call(do_globals) - if glob_res: - sys.exit(glob_res) + if obj_output: + link_res = call(do_link) + if link_res: + sys.exit(link_res) + if do_globals: + glob_res = call(do_globals) + if glob_res: + sys.exit(glob_res) + sys.exit(call(do_archive)) diff --git a/build/scripts/make_java_classpath_file.py b/build/scripts/make_java_classpath_file.py index c70a7876d7..9b4eab5150 100644 --- a/build/scripts/make_java_classpath_file.py +++ b/build/scripts/make_java_classpath_file.py @@ -1,24 +1,24 @@ -import os -import sys - +import os +import sys + import process_command_files as pcf + - -def make_cp_file(args): - source = args[0] - destination = args[1] - with open(source) as src: - lines = [l.strip() for l in src if l.strip()] - with open(destination, 'w') as dst: - dst.write(os.pathsep.join(lines)) - +def make_cp_file(args): + source = args[0] + destination = args[1] + with open(source) as src: + lines = [l.strip() for l in src if l.strip()] + with open(destination, 'w') as dst: + dst.write(os.pathsep.join(lines)) + def make_cp_file_from_args(args): destination = args[0] with open(destination, 'w') as dst: dst.write(os.pathsep.join(args[1:])) + - -if __name__ == '__main__': +if __name__ == '__main__': args = pcf.get_args(sys.argv[1:]) if sys.argv[1] != '--from-args': make_cp_file(args) diff --git a/build/scripts/make_manifest_from_bf.py b/build/scripts/make_manifest_from_bf.py index bfea3ba3de..96e74d55e3 100644 --- a/build/scripts/make_manifest_from_bf.py +++ b/build/scripts/make_manifest_from_bf.py @@ -1,28 +1,28 @@ -import sys -import zipfile -import os -import re - - -def prepare_path(path): - return ('file:/' + path.lstrip('/')) if os.path.isabs(path) else path - - -def main(args): - bf, mf = args[0], args[1] - if not os.path.exists(os.path.dirname(mf)): - os.makedirs(os.path.dirname(mf)) - with open(bf) as f: - class_path = f.read().strip() - class_path = ' '.join(map(prepare_path, class_path.split('\n'))) - with zipfile.ZipFile(mf, 'w') as zf: - lines = [] - while class_path: - lines.append(class_path[:60]) - class_path = class_path[60:] - if lines: - zf.writestr('META-INF/MANIFEST.MF', 'Manifest-Version: 1.0\nClass-Path: \n ' + '\n '.join(lines) + ' \n\n') - - -if __name__ == '__main__': - main(sys.argv[1:]) +import sys +import zipfile +import os +import re + + +def prepare_path(path): + return ('file:/' + path.lstrip('/')) if os.path.isabs(path) else path + + +def main(args): + bf, mf = args[0], args[1] + if not os.path.exists(os.path.dirname(mf)): + os.makedirs(os.path.dirname(mf)) + with open(bf) as f: + class_path = f.read().strip() + class_path = ' '.join(map(prepare_path, class_path.split('\n'))) + with zipfile.ZipFile(mf, 'w') as zf: + lines = [] + while class_path: + lines.append(class_path[:60]) + class_path = class_path[60:] + if lines: + zf.writestr('META-INF/MANIFEST.MF', 'Manifest-Version: 1.0\nClass-Path: \n ' + '\n '.join(lines) + ' \n\n') + + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/build/scripts/merge_coverage_data.py b/build/scripts/merge_coverage_data.py index b7fa3c6a86..a2a0e0fa44 100644 --- a/build/scripts/merge_coverage_data.py +++ b/build/scripts/merge_coverage_data.py @@ -1,7 +1,7 @@ import sys import tarfile -import copy -import os +import copy +import os import uuid @@ -17,15 +17,15 @@ def main(args): with tarfile.open(output_file, 'w') as outf: for x in files: - with tarfile.open(x) as tf: - for tarinfo in tf: - new_tarinfo = copy.deepcopy(tarinfo) - if new_tarinfo.name in expendables: + with tarfile.open(x) as tf: + for tarinfo in tf: + new_tarinfo = copy.deepcopy(tarinfo) + if new_tarinfo.name in expendables: dirname, basename = os.path.split(new_tarinfo.name) basename_parts = basename.split('.', 1) new_basename = '.'.join([basename_parts[0] + str(uuid.uuid4())] + basename_parts[1:]) new_tarinfo.name = os.path.join(dirname, new_basename) - outf.addfile(new_tarinfo, tf.extractfile(tarinfo)) + outf.addfile(new_tarinfo, tf.extractfile(tarinfo)) if __name__ == '__main__': diff --git a/build/scripts/merge_files.py b/build/scripts/merge_files.py index d42d6a2139..25317d9e17 100644 --- a/build/scripts/merge_files.py +++ b/build/scripts/merge_files.py @@ -1,8 +1,8 @@ -import sys - - -if __name__ == "__main__": - with open(sys.argv[1], "w") as f: - for appended in sys.argv[2:]: - with open(appended) as a: - f.write(a.read()) +import sys + + +if __name__ == "__main__": + with open(sys.argv[1], "w") as f: + for appended in sys.argv[2:]: + with open(appended) as a: + f.write(a.read()) diff --git a/build/scripts/pack_ios.py b/build/scripts/pack_ios.py index 37c36d1f95..7f7fdf9074 100644 --- a/build/scripts/pack_ios.py +++ b/build/scripts/pack_ios.py @@ -1,48 +1,48 @@ -import argparse -import os -import shutil -import subprocess -import sys -import tarfile - - -def just_do_it(): - parser = argparse.ArgumentParser() - parser.add_argument("--binary", required=True, help="executable file") - parser.add_argument("--target", required=True, help="target archive path") - parser.add_argument("--temp-dir", required=True, help="temp dir") - parser.add_argument("peers", nargs='*') - args = parser.parse_args() - app_tar = [p for p in args.peers if p.endswith('.ios.interface')] - if not app_tar: - print >> sys.stderr, 'No one IOS_INTERFACE module found' - shutil.copyfile(args.binary, os.path.join(args.temp_dir, 'bin')) - if os.path.exists(args.target): - os.remove(args.target) - with tarfile.open(args.target, 'w') as tf: - tf.add(os.path.join(args.temp_dir, 'bin'), arcname=os.path.join(os.path.basename(args.binary) + '.app', 'bin')) - return - if len(app_tar) > 1: - app_tar = [p for p in args.peers if not p.endswith('.default.ios.interface')] - if len(app_tar) > 1: - print >> sys.stderr, 'Many IOS_INTERFACE modules found, {} will be used'.format(app_tar[-1]) - app_tar = app_tar[-1] - with tarfile.open(app_tar) as tf: - tf.extractall(args.temp_dir) - tar_suffix = '.default.ios.interface' if app_tar.endswith('.default.ios.interface') else '.ios.interface' - app_unpacked_path = os.path.join(args.temp_dir, os.path.basename(app_tar)[:-len(tar_suffix)] + '.app') - if not os.path.exists(app_unpacked_path): - raise Exception('Bad IOS_INTERFACE resource: {}'.format(app_tar)) - shutil.copyfile(args.binary, os.path.join(app_unpacked_path, 'bin')) - subprocess.check_call(['/usr/bin/codesign', '--force', '--sign', '-', app_unpacked_path]) - if os.path.exists(args.target): - os.remove(args.target) - binary_origin_name = os.path.basename(args.binary) - while os.path.splitext(binary_origin_name)[1]: - binary_origin_name = os.path.splitext(binary_origin_name)[0] - with tarfile.open(args.target, 'w') as tf: - tf.add(app_unpacked_path, arcname=binary_origin_name + '.app', recursive=True) - - -if __name__ == '__main__': - just_do_it() +import argparse +import os +import shutil +import subprocess +import sys +import tarfile + + +def just_do_it(): + parser = argparse.ArgumentParser() + parser.add_argument("--binary", required=True, help="executable file") + parser.add_argument("--target", required=True, help="target archive path") + parser.add_argument("--temp-dir", required=True, help="temp dir") + parser.add_argument("peers", nargs='*') + args = parser.parse_args() + app_tar = [p for p in args.peers if p.endswith('.ios.interface')] + if not app_tar: + print >> sys.stderr, 'No one IOS_INTERFACE module found' + shutil.copyfile(args.binary, os.path.join(args.temp_dir, 'bin')) + if os.path.exists(args.target): + os.remove(args.target) + with tarfile.open(args.target, 'w') as tf: + tf.add(os.path.join(args.temp_dir, 'bin'), arcname=os.path.join(os.path.basename(args.binary) + '.app', 'bin')) + return + if len(app_tar) > 1: + app_tar = [p for p in args.peers if not p.endswith('.default.ios.interface')] + if len(app_tar) > 1: + print >> sys.stderr, 'Many IOS_INTERFACE modules found, {} will be used'.format(app_tar[-1]) + app_tar = app_tar[-1] + with tarfile.open(app_tar) as tf: + tf.extractall(args.temp_dir) + tar_suffix = '.default.ios.interface' if app_tar.endswith('.default.ios.interface') else '.ios.interface' + app_unpacked_path = os.path.join(args.temp_dir, os.path.basename(app_tar)[:-len(tar_suffix)] + '.app') + if not os.path.exists(app_unpacked_path): + raise Exception('Bad IOS_INTERFACE resource: {}'.format(app_tar)) + shutil.copyfile(args.binary, os.path.join(app_unpacked_path, 'bin')) + subprocess.check_call(['/usr/bin/codesign', '--force', '--sign', '-', app_unpacked_path]) + if os.path.exists(args.target): + os.remove(args.target) + binary_origin_name = os.path.basename(args.binary) + while os.path.splitext(binary_origin_name)[1]: + binary_origin_name = os.path.splitext(binary_origin_name)[0] + with tarfile.open(args.target, 'w') as tf: + tf.add(app_unpacked_path, arcname=binary_origin_name + '.app', recursive=True) + + +if __name__ == '__main__': + just_do_it() diff --git a/build/scripts/pack_jcoverage_resources.py b/build/scripts/pack_jcoverage_resources.py index f6e181067a..a282512460 100644 --- a/build/scripts/pack_jcoverage_resources.py +++ b/build/scripts/pack_jcoverage_resources.py @@ -1,24 +1,24 @@ -import sys -import tarfile -import os -import subprocess - - -def main(args): - output_file = args[0] - report_file = args[1] - - res = subprocess.call(args[args.index('-end') + 1:]) +import sys +import tarfile +import os +import subprocess + + +def main(args): + output_file = args[0] + report_file = args[1] + + res = subprocess.call(args[args.index('-end') + 1:]) if not os.path.exists(report_file): print>>sys.stderr, 'Can\'t find jacoco exec file' return res - with tarfile.open(output_file, 'w') as outf: - outf.add(report_file, arcname=os.path.basename(report_file)) - - return res - + with tarfile.open(output_file, 'w') as outf: + outf.add(report_file, arcname=os.path.basename(report_file)) -if __name__ == '__main__': + return res + + +if __name__ == '__main__': sys.exit(main(sys.argv[1:])) diff --git a/build/scripts/resolve_java_srcs.py b/build/scripts/resolve_java_srcs.py index a2e6c20012..479fded4af 100644 --- a/build/scripts/resolve_java_srcs.py +++ b/build/scripts/resolve_java_srcs.py @@ -1,19 +1,19 @@ -import os -import argparse -import re -import sys - - -def list_all_files(directory, prefix='/', hidden_files=False): - result = [] - if os.path.exists(directory): - for i in os.listdir(directory): - abs_path = os.path.join(directory, i) - result += list_all_files(os.path.join(directory, abs_path), prefix + i + '/', hidden_files) \ - if os.path.isdir(abs_path) else ([prefix + i] if (hidden_files or not i.startswith('.')) else []) - return result - - +import os +import argparse +import re +import sys + + +def list_all_files(directory, prefix='/', hidden_files=False): + result = [] + if os.path.exists(directory): + for i in os.listdir(directory): + abs_path = os.path.join(directory, i) + result += list_all_files(os.path.join(directory, abs_path), prefix + i + '/', hidden_files) \ + if os.path.isdir(abs_path) else ([prefix + i] if (hidden_files or not i.startswith('.')) else []) + return result + + def pattern_to_regexp(p): return '^' + \ ('/' if not p.startswith('**') else '') + \ @@ -28,79 +28,79 @@ def pattern_to_regexp(p): ) + '$' -def resolve_java_srcs(srcdir, include_patterns, exclude_patterns, all_resources, resolve_kotlin=False, resolve_groovy=False): - result = {'java': [], 'not_java': [], 'kotlin': [], 'groovy': []} - include_patterns_normal, include_patterns_hidden, exclude_patterns_normal, exclude_patterns_hidden = [], [], [], [] - for vis, hid, patterns in ((include_patterns_normal, include_patterns_hidden, include_patterns), (exclude_patterns_normal, exclude_patterns_hidden, exclude_patterns),): - for pattern in patterns: - if (pattern if pattern.find('/') == -1 else pattern.rsplit('/', 1)[1]).startswith('.'): - hid.append(pattern) - else: - vis.append(pattern) - re_patterns = map(pattern_to_regexp, vis + hid) - if sys.platform in ('win32', 'darwin'): - re_patterns = [re.compile(i, re.IGNORECASE) for i in re_patterns] - else: - re_patterns = [re.compile(i) for i in re_patterns] - vis[:], hid[:] = re_patterns[:len(vis)], re_patterns[len(vis):] - - for inc_patterns, exc_patterns, with_hidden_files in ( - (include_patterns_normal, exclude_patterns_normal, False), - (include_patterns_hidden, exclude_patterns_hidden, True), - ): - for f in list_all_files(srcdir, hidden_files=with_hidden_files): - excluded = False - - for exc_re in exc_patterns: - if exc_re.match(f): - excluded = True - break - - if excluded: - continue - - for inc_re in inc_patterns: - if inc_re.match(f): - s = os.path.normpath(f[1:]) - if all_resources or not (f.endswith('.java') or f.endswith('.kt') or f.endswith('.groovy')): - result['not_java'].append(s) - elif f.endswith('.java'): +def resolve_java_srcs(srcdir, include_patterns, exclude_patterns, all_resources, resolve_kotlin=False, resolve_groovy=False): + result = {'java': [], 'not_java': [], 'kotlin': [], 'groovy': []} + include_patterns_normal, include_patterns_hidden, exclude_patterns_normal, exclude_patterns_hidden = [], [], [], [] + for vis, hid, patterns in ((include_patterns_normal, include_patterns_hidden, include_patterns), (exclude_patterns_normal, exclude_patterns_hidden, exclude_patterns),): + for pattern in patterns: + if (pattern if pattern.find('/') == -1 else pattern.rsplit('/', 1)[1]).startswith('.'): + hid.append(pattern) + else: + vis.append(pattern) + re_patterns = map(pattern_to_regexp, vis + hid) + if sys.platform in ('win32', 'darwin'): + re_patterns = [re.compile(i, re.IGNORECASE) for i in re_patterns] + else: + re_patterns = [re.compile(i) for i in re_patterns] + vis[:], hid[:] = re_patterns[:len(vis)], re_patterns[len(vis):] + + for inc_patterns, exc_patterns, with_hidden_files in ( + (include_patterns_normal, exclude_patterns_normal, False), + (include_patterns_hidden, exclude_patterns_hidden, True), + ): + for f in list_all_files(srcdir, hidden_files=with_hidden_files): + excluded = False + + for exc_re in exc_patterns: + if exc_re.match(f): + excluded = True + break + + if excluded: + continue + + for inc_re in inc_patterns: + if inc_re.match(f): + s = os.path.normpath(f[1:]) + if all_resources or not (f.endswith('.java') or f.endswith('.kt') or f.endswith('.groovy')): + result['not_java'].append(s) + elif f.endswith('.java'): result['java'].append(os.path.join(srcdir, s)) - elif f.endswith('.kt') and resolve_kotlin: + elif f.endswith('.kt') and resolve_kotlin: result['kotlin'].append(os.path.join(srcdir, s)) - elif f.endswith('.groovy') and resolve_groovy: + elif f.endswith('.groovy') and resolve_groovy: result['groovy'].append(os.path.join(srcdir, s)) - else: - result['not_java'].append(s) - break - - return sorted(result['java']), sorted(result['not_java']), sorted(result['kotlin']), sorted(result['groovy']) - + else: + result['not_java'].append(s) + break + return sorted(result['java']), sorted(result['not_java']), sorted(result['kotlin']), sorted(result['groovy']) + + def do_it(directory, sources_file, resources_file, kotlin_sources_file, groovy_sources_file, include_patterns, exclude_patterns, resolve_kotlin, resolve_groovy, append, all_resources): j, r, k, g = resolve_java_srcs(directory, include_patterns, exclude_patterns, all_resources, resolve_kotlin, resolve_groovy) mode = 'a' if append else 'w' open(sources_file, mode).writelines(i + '\n' for i in j) open(resources_file, mode).writelines(i + '\n' for i in r) - if kotlin_sources_file: + if kotlin_sources_file: open(kotlin_sources_file, mode).writelines(i + '\n' for i in k + j) - if groovy_sources_file: + if groovy_sources_file: open(groovy_sources_file, mode).writelines(i + '\n' for i in g + j) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument('-d', '--directory', required=True) - parser.add_argument('-s', '--sources-file', required=True) - parser.add_argument('-r', '--resources-file', required=True) - parser.add_argument('-k', '--kotlin-sources-file', default=None) - parser.add_argument('-g', '--groovy-sources-file', default=None) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('-d', '--directory', required=True) + parser.add_argument('-s', '--sources-file', required=True) + parser.add_argument('-r', '--resources-file', required=True) + parser.add_argument('-k', '--kotlin-sources-file', default=None) + parser.add_argument('-g', '--groovy-sources-file', default=None) parser.add_argument('--append', action='store_true', default=False) parser.add_argument('--all-resources', action='store_true', default=False) - parser.add_argument('--resolve-kotlin', action='store_true', default=False) - parser.add_argument('--resolve-groovy', action='store_true', default=False) + parser.add_argument('--resolve-kotlin', action='store_true', default=False) + parser.add_argument('--resolve-groovy', action='store_true', default=False) parser.add_argument('--include-patterns', nargs='*', default=[]) parser.add_argument('--exclude-patterns', nargs='*', default=[]) - args = parser.parse_args() - - do_it(**vars(args)) + args = parser.parse_args() + + do_it(**vars(args)) diff --git a/build/scripts/run_ios_simulator.py b/build/scripts/run_ios_simulator.py index 052c855b77..22bb27d4cc 100644 --- a/build/scripts/run_ios_simulator.py +++ b/build/scripts/run_ios_simulator.py @@ -1,79 +1,79 @@ -import argparse -import json -import os -import subprocess -import sys - - -def just_do_it(): - parser = argparse.ArgumentParser() - parser.add_argument("--action", choices=["create", "spawn", "kill"]) - parser.add_argument("--simctl", help="simctl binary path") - parser.add_argument("--profiles", help="profiles path") - parser.add_argument("--device-dir", help="devices directory") - parser.add_argument("--device-name", help="temp device name") - args, tail = parser.parse_known_args() - if args.action == 'create': - action_create(args.simctl, args.profiles, args.device_dir, args.device_name, tail) - elif args.action == "spawn": - action_spawn(args.simctl, args.profiles, args.device_dir, args.device_name, tail) - elif args.action == "kill": - action_kill(args.simctl, args.profiles, args.device_dir, args.device_name) - - -def action_create(simctl, profiles, device_dir, name, args): - parser = argparse.ArgumentParser() - parser.add_argument("--device-type", default="com.apple.CoreSimulator.SimDeviceType.iPhone-X") - parser.add_argument("--device-runtime", default="com.apple.CoreSimulator.SimRuntime.iOS-12-1") - args = parser.parse_args(args) - all_devices = list(get_all_devices(simctl, profiles, device_dir)) - if filter(lambda x: x["name"] == name, all_devices): - raise Exception("Device named {} already exists".format(name)) - subprocess.check_call([simctl, "--profiles", profiles, "--set", device_dir, "create", name, args.device_type, args.device_runtime]) - created = filter(lambda x: x["name"] == name, get_all_devices(simctl, profiles, device_dir)) - if not created: - raise Exception("Creation error: temp device named {} not found".format(name)) - created = created[0] - if created["availability"] != "(available)": - raise Exception("Creation error: temp device {} status is {} ((available) expected)".format(name, created["availability"])) - - -def action_spawn(simctl, profiles, device_dir, name, args): - device = filter(lambda x: x["name"] == name, get_all_devices(simctl, profiles, device_dir)) - if not device: - raise Exception("Can't spawn process: device named {} not found".format(name)) - if len(device) > 1: - raise Exception("Can't spawn process: too many devices named {} found".format(name)) - device = device[0] - os.execv(simctl, [simctl, "--profiles", profiles, "--set", device_dir, "spawn", device["udid"]] + args) - - -def action_kill(simctl, profiles, device_dir, name): - device = filter(lambda x: x["name"] == name, get_all_devices(simctl, profiles, device_dir)) - if not device: - print >> sys.stderr, "Device named {} not found; do nothing".format(name) - return - if len(device) > 1: - raise Exception("Can't remove: too many devices named {}:\n{}".format(name, '\n'.join(i for i in device))) - device = device[0] - os.execv(simctl, [simctl, "--profiles", profiles, "--set", device_dir, "delete", device["udid"]]) - - -def get_all_devices(simctl, profiles, device_dir): - p = subprocess.Popen([simctl, "--profiles", profiles, "--set", device_dir, "list", "--json", "devices"], stdout=subprocess.PIPE) - out, _ = p.communicate() - rc = p.wait() - if rc: - raise Exception("Devices list command return code is {}\nstdout:\n{}".format(rc, out)) - raw_object = json.loads(out) - if "devices" not in raw_object: - raise Exception("Devices not found in\n{}".format(json.dumps(raw_object))) - raw_object = raw_object["devices"] - for os_name, devices in raw_object.items(): - for device in devices: - device["os_name"] = os_name - yield device - - -if __name__ == '__main__': - just_do_it() +import argparse +import json +import os +import subprocess +import sys + + +def just_do_it(): + parser = argparse.ArgumentParser() + parser.add_argument("--action", choices=["create", "spawn", "kill"]) + parser.add_argument("--simctl", help="simctl binary path") + parser.add_argument("--profiles", help="profiles path") + parser.add_argument("--device-dir", help="devices directory") + parser.add_argument("--device-name", help="temp device name") + args, tail = parser.parse_known_args() + if args.action == 'create': + action_create(args.simctl, args.profiles, args.device_dir, args.device_name, tail) + elif args.action == "spawn": + action_spawn(args.simctl, args.profiles, args.device_dir, args.device_name, tail) + elif args.action == "kill": + action_kill(args.simctl, args.profiles, args.device_dir, args.device_name) + + +def action_create(simctl, profiles, device_dir, name, args): + parser = argparse.ArgumentParser() + parser.add_argument("--device-type", default="com.apple.CoreSimulator.SimDeviceType.iPhone-X") + parser.add_argument("--device-runtime", default="com.apple.CoreSimulator.SimRuntime.iOS-12-1") + args = parser.parse_args(args) + all_devices = list(get_all_devices(simctl, profiles, device_dir)) + if filter(lambda x: x["name"] == name, all_devices): + raise Exception("Device named {} already exists".format(name)) + subprocess.check_call([simctl, "--profiles", profiles, "--set", device_dir, "create", name, args.device_type, args.device_runtime]) + created = filter(lambda x: x["name"] == name, get_all_devices(simctl, profiles, device_dir)) + if not created: + raise Exception("Creation error: temp device named {} not found".format(name)) + created = created[0] + if created["availability"] != "(available)": + raise Exception("Creation error: temp device {} status is {} ((available) expected)".format(name, created["availability"])) + + +def action_spawn(simctl, profiles, device_dir, name, args): + device = filter(lambda x: x["name"] == name, get_all_devices(simctl, profiles, device_dir)) + if not device: + raise Exception("Can't spawn process: device named {} not found".format(name)) + if len(device) > 1: + raise Exception("Can't spawn process: too many devices named {} found".format(name)) + device = device[0] + os.execv(simctl, [simctl, "--profiles", profiles, "--set", device_dir, "spawn", device["udid"]] + args) + + +def action_kill(simctl, profiles, device_dir, name): + device = filter(lambda x: x["name"] == name, get_all_devices(simctl, profiles, device_dir)) + if not device: + print >> sys.stderr, "Device named {} not found; do nothing".format(name) + return + if len(device) > 1: + raise Exception("Can't remove: too many devices named {}:\n{}".format(name, '\n'.join(i for i in device))) + device = device[0] + os.execv(simctl, [simctl, "--profiles", profiles, "--set", device_dir, "delete", device["udid"]]) + + +def get_all_devices(simctl, profiles, device_dir): + p = subprocess.Popen([simctl, "--profiles", profiles, "--set", device_dir, "list", "--json", "devices"], stdout=subprocess.PIPE) + out, _ = p.communicate() + rc = p.wait() + if rc: + raise Exception("Devices list command return code is {}\nstdout:\n{}".format(rc, out)) + raw_object = json.loads(out) + if "devices" not in raw_object: + raise Exception("Devices not found in\n{}".format(json.dumps(raw_object))) + raw_object = raw_object["devices"] + for os_name, devices in raw_object.items(): + for device in devices: + device["os_name"] = os_name + yield device + + +if __name__ == '__main__': + just_do_it() diff --git a/build/scripts/run_javac.py b/build/scripts/run_javac.py index c35546e0fe..0adf42a2e0 100644 --- a/build/scripts/run_javac.py +++ b/build/scripts/run_javac.py @@ -10,8 +10,8 @@ def parse_args(): parser.add_option('--sources-list') parser.add_option('--verbose', default=False, action='store_true') parser.add_option('--remove-notes', default=False, action='store_true') - parser.add_option('--ignore-errors', default=False, action='store_true') - parser.add_option('--kotlin', default=False, action='store_true') + parser.add_option('--ignore-errors', default=False, action='store_true') + parser.add_option('--kotlin', default=False, action='store_true') return parser.parse_args() @@ -42,63 +42,63 @@ def remove_notes(err): return '\n'.join([line for line in err.split('\n') if not line.startswith('Note:')]) -def find_javac(cmd): - if not cmd: - return None - if cmd[0].endswith('javac') or cmd[0].endswith('javac.exe'): - return cmd[0] - if len(cmd) > 2 and cmd[1].endswith('build_java_with_error_prone.py'): - for javas in ('java', 'javac'): - if cmd[2].endswith(javas) or cmd[2].endswith(javas + '.exe'): - return cmd[2] - return None - - -# temporary, for jdk8/jdk9+ compatibility -def fix_cmd(cmd): - if not cmd: - return cmd - javac = find_javac(cmd) - if not javac: - return cmd - p = subprocess.Popen([javac, '-version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, err = p.communicate() - out, err = out.strip(), err.strip() - for prefix in ('javac 1.8', 'java version "1.8'): - for raw_out in ((out or ''), (err or '')): - for line in raw_out.split('\n'): - if line.startswith(prefix): - res = [] - i = 0 - while i < len(cmd): - for option in ('--add-exports', '--add-modules'): - if cmd[i] == option: - i += 1 - break - elif cmd[i].startswith(option + '='): - break - else: - res.append(cmd[i]) - i += 1 - return res - return cmd - - +def find_javac(cmd): + if not cmd: + return None + if cmd[0].endswith('javac') or cmd[0].endswith('javac.exe'): + return cmd[0] + if len(cmd) > 2 and cmd[1].endswith('build_java_with_error_prone.py'): + for javas in ('java', 'javac'): + if cmd[2].endswith(javas) or cmd[2].endswith(javas + '.exe'): + return cmd[2] + return None + + +# temporary, for jdk8/jdk9+ compatibility +def fix_cmd(cmd): + if not cmd: + return cmd + javac = find_javac(cmd) + if not javac: + return cmd + p = subprocess.Popen([javac, '-version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = p.communicate() + out, err = out.strip(), err.strip() + for prefix in ('javac 1.8', 'java version "1.8'): + for raw_out in ((out or ''), (err or '')): + for line in raw_out.split('\n'): + if line.startswith(prefix): + res = [] + i = 0 + while i < len(cmd): + for option in ('--add-exports', '--add-modules'): + if cmd[i] == option: + i += 1 + break + elif cmd[i].startswith(option + '='): + break + else: + res.append(cmd[i]) + i += 1 + return res + return cmd + + def main(): opts, cmd = parse_args() with open(opts.sources_list) as f: input_files = f.read().strip().split() - if opts.kotlin: - input_files = [i for i in input_files if i.endswith('.kt')] - + if opts.kotlin: + input_files = [i for i in input_files if i.endswith('.kt')] + if not input_files: if opts.verbose: sys.stderr.write('No files to compile, javac is not launched.\n') else: - p = subprocess.Popen(fix_cmd(cmd), stderr=subprocess.PIPE) + p = subprocess.Popen(fix_cmd(cmd), stderr=subprocess.PIPE) _, err = p.communicate() rc = p.wait() @@ -111,9 +111,9 @@ def main(): except Exception: pass - if opts.ignore_errors and rc: - sys.stderr.write('error: javac actually failed with exit code {}\n'.format(rc)) - rc = 0 + if opts.ignore_errors and rc: + sys.stderr.write('error: javac actually failed with exit code {}\n'.format(rc)) + rc = 0 sys.stderr.write(err) sys.exit(rc) diff --git a/build/scripts/run_sonar.py b/build/scripts/run_sonar.py index 761cc34b78..1609f2ba30 100644 --- a/build/scripts/run_sonar.py +++ b/build/scripts/run_sonar.py @@ -5,7 +5,7 @@ import tarfile import subprocess as sp import optparse import shutil -import xml.etree.ElementTree as et +import xml.etree.ElementTree as et def parse_args(): @@ -22,9 +22,9 @@ def parse_args(): parser.add_option('--java-coverage-merged-tar') parser.add_option('--java-binary-path') parser.add_option('--log-path') - parser.add_option('--gcov-report-path') - parser.add_option('--source-root') - parser.add_option('--java-args', action='append', default=[]) + parser.add_option('--gcov-report-path') + parser.add_option('--source-root') + parser.add_option('--java-args', action='append', default=[]) return parser.parse_args() @@ -33,46 +33,46 @@ def extract_zip_file(zip_file_path, dest_dir): arch.extractall(dest_dir) -def get_source_real_path(source_root, path): - parts = os.path.normpath(path).split(os.path.sep) - for i in xrange(len(parts)): - if os.path.exists(os.path.join(source_root, *parts[i:])): - return os.path.join(*parts[i:]) - return None - - -def collect_cpp_sources(report, source_root, destination): - sources = set() - with open(report) as f: - root = et.fromstring(f.read()) - for f in root.findall('.//class[@filename]'): - real_filename = get_source_real_path(source_root, f.attrib['filename']) - if real_filename: - f.attrib['filename'] = real_filename - sources.add(real_filename) - with open(report, 'w') as f: - pref = '''<?xml version="1.0" ?> -<!DOCTYPE coverage - SYSTEM 'http://cobertura.sourceforge.net/xml/coverage-03.dtd'>\n''' - f.write(pref + et.tostring(root, encoding='utf-8') + '\n\n') - for src in sources: - dst = os.path.join(destination, src) - src = os.path.join(source_root, src) - if os.path.isfile(src): - if not os.path.exists(os.path.dirname(dst)): - os.makedirs(os.path.dirname(dst)) - os.link(src, dst) - - +def get_source_real_path(source_root, path): + parts = os.path.normpath(path).split(os.path.sep) + for i in xrange(len(parts)): + if os.path.exists(os.path.join(source_root, *parts[i:])): + return os.path.join(*parts[i:]) + return None + + +def collect_cpp_sources(report, source_root, destination): + sources = set() + with open(report) as f: + root = et.fromstring(f.read()) + for f in root.findall('.//class[@filename]'): + real_filename = get_source_real_path(source_root, f.attrib['filename']) + if real_filename: + f.attrib['filename'] = real_filename + sources.add(real_filename) + with open(report, 'w') as f: + pref = '''<?xml version="1.0" ?> +<!DOCTYPE coverage + SYSTEM 'http://cobertura.sourceforge.net/xml/coverage-03.dtd'>\n''' + f.write(pref + et.tostring(root, encoding='utf-8') + '\n\n') + for src in sources: + dst = os.path.join(destination, src) + src = os.path.join(source_root, src) + if os.path.isfile(src): + if not os.path.exists(os.path.dirname(dst)): + os.makedirs(os.path.dirname(dst)) + os.link(src, dst) + + def main(opts, props_args): sources_dir = os.path.abspath('src') - base_props_args = ['-Dsonar.sources=' + sources_dir] + base_props_args = ['-Dsonar.sources=' + sources_dir] os.mkdir(sources_dir) - if opts.sources_jar_path: - extract_zip_file(opts.sources_jar_path, sources_dir) - if opts.gcov_report_path: - collect_cpp_sources(opts.gcov_report_path, opts.source_root, sources_dir) - base_props_args += ['-Dsonar.projectBaseDir=' + sources_dir, '-Dsonar.cxx.coverage.reportPath=' + opts.gcov_report_path] + if opts.sources_jar_path: + extract_zip_file(opts.sources_jar_path, sources_dir) + if opts.gcov_report_path: + collect_cpp_sources(opts.gcov_report_path, opts.source_root, sources_dir) + base_props_args += ['-Dsonar.projectBaseDir=' + sources_dir, '-Dsonar.cxx.coverage.reportPath=' + opts.gcov_report_path] if opts.classes_jar_paths: classes_dir = os.path.abspath('cls') @@ -88,28 +88,28 @@ def main(opts, props_args): with open(jacoco_report_path, 'w') as dest: with tarfile.open(opts.java_coverage_merged_tar) as tar: for src in tar: - extracted = tar.extractfile(src) - if extracted is not None: - shutil.copyfileobj(extracted, dest) + extracted = tar.extractfile(src) + if extracted is not None: + shutil.copyfileobj(extracted, dest) base_props_args += [ '-Dsonar.core.codeCoveragePlugin=jacoco', '-Dsonar.jacoco.reportPath=' + jacoco_report_path ] - java_args = ['-{}'.format(i) for i in opts.java_args] + ['-Djava.net.preferIPv6Addresses=true', '-Djava.net.preferIPv4Addresses=false'] + java_args = ['-{}'.format(i) for i in opts.java_args] + ['-Djava.net.preferIPv6Addresses=true', '-Djava.net.preferIPv4Addresses=false'] sonar_cmd = [ opts.java_binary_path, - ] + java_args + [ + ] + java_args + [ '-classpath', opts.sonar_scanner_jar_path, - ] + base_props_args + props_args + [opts.sonar_scanner_main_class, '-X'] + ] + base_props_args + props_args + [opts.sonar_scanner_main_class, '-X'] p = sp.Popen(sonar_cmd, stdout=sp.PIPE, stderr=sp.STDOUT) out, _ = p.communicate() sys.stderr.write(out) - with open(opts.log_path, 'a') as f: + with open(opts.log_path, 'a') as f: f.write(out) sys.exit(p.returncode) diff --git a/build/scripts/setup_java_tmpdir.py b/build/scripts/setup_java_tmpdir.py index e478d4aa96..65f59fc0ee 100644 --- a/build/scripts/setup_java_tmpdir.py +++ b/build/scripts/setup_java_tmpdir.py @@ -1,40 +1,40 @@ -import os -import sys -import platform -import subprocess - - -def fix_tmpdir(cmd): - if not cmd: - return cmd - java_id, option_name = None, None - for i, java in enumerate(cmd): - if java.endswith('java') or java.endswith('java.exe'): - java_id = i - option_name = '-Djava.io.tmpdir=' - break - if java.endswith('javac') or java.endswith('javac.exe'): - java_id = i - option_name = '-J-Djava.io.tmpdir=' - break - if java_id is None: - return cmd - for arg in cmd[java_id:]: - if arg.startswith(option_name): - return cmd - tmpdir = os.environ.get('TMPDIR') or os.environ.get('TEMPDIR') - if not tmpdir: - return cmd - return cmd[:java_id + 1] + ['{}{}'.format(option_name, tmpdir)] + cmd[java_id + 1:] - - -def just_do_it(): - args = fix_tmpdir(sys.argv[1:]) - if platform.system() == 'Windows': - sys.exit(subprocess.Popen(args).wait()) - else: - os.execv(args[0], args) - - -if __name__ == '__main__': - just_do_it() +import os +import sys +import platform +import subprocess + + +def fix_tmpdir(cmd): + if not cmd: + return cmd + java_id, option_name = None, None + for i, java in enumerate(cmd): + if java.endswith('java') or java.endswith('java.exe'): + java_id = i + option_name = '-Djava.io.tmpdir=' + break + if java.endswith('javac') or java.endswith('javac.exe'): + java_id = i + option_name = '-J-Djava.io.tmpdir=' + break + if java_id is None: + return cmd + for arg in cmd[java_id:]: + if arg.startswith(option_name): + return cmd + tmpdir = os.environ.get('TMPDIR') or os.environ.get('TEMPDIR') + if not tmpdir: + return cmd + return cmd[:java_id + 1] + ['{}{}'.format(option_name, tmpdir)] + cmd[java_id + 1:] + + +def just_do_it(): + args = fix_tmpdir(sys.argv[1:]) + if platform.system() == 'Windows': + sys.exit(subprocess.Popen(args).wait()) + else: + os.execv(args[0], args) + + +if __name__ == '__main__': + just_do_it() diff --git a/build/scripts/tar_directory.py b/build/scripts/tar_directory.py index a91889fa22..8bacdec3a9 100644 --- a/build/scripts/tar_directory.py +++ b/build/scripts/tar_directory.py @@ -1,22 +1,22 @@ -import os -import sys -import tarfile - - -def is_exe(fpath): - return os.path.isfile(fpath) and os.access(fpath, os.X_OK) - - -def main(args): - if len(args) < 2 or len(args) > 3: +import os +import sys +import tarfile + + +def is_exe(fpath): + return os.path.isfile(fpath) and os.access(fpath, os.X_OK) + + +def main(args): + if len(args) < 2 or len(args) > 3: raise Exception("Illegal usage: `tar_directory.py archive.tar directory [skip prefix]` or `tar_directory.py archive.tar output_directory --extract`") tar, directory, prefix, extract = args[0], args[1], None, False - if len(args) == 3: + if len(args) == 3: if args[2] == '--extract': extract = True else: prefix = args[2] - for tar_exe in ('/usr/bin/tar', '/bin/tar'): + for tar_exe in ('/usr/bin/tar', '/bin/tar'): if not is_exe(tar_exe): continue if extract: @@ -26,9 +26,9 @@ def main(args): os.execv(tar_exe, [tar_exe, '-xf', tar, '-C', dest]) else: source = os.path.relpath(directory, prefix) if prefix else directory - os.execv(tar_exe, [tar_exe, '-cf', tar] + (['-C', prefix] if prefix else []) + [source]) + os.execv(tar_exe, [tar_exe, '-cf', tar] + (['-C', prefix] if prefix else []) + [source]) break - else: + else: if extract: dest = os.path.abspath(directory) if not os.path.exists(dest): @@ -39,7 +39,7 @@ def main(args): source = directory with tarfile.open(tar, 'w') as out: out.add(os.path.abspath(source), arcname=os.path.relpath(source, prefix) if prefix else source) - - -if __name__ == '__main__': - main(sys.argv[1:]) + + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/build/scripts/unpacking_jtest_runner.py b/build/scripts/unpacking_jtest_runner.py index 9730dcd711..0d168fb2ce 100644 --- a/build/scripts/unpacking_jtest_runner.py +++ b/build/scripts/unpacking_jtest_runner.py @@ -2,11 +2,11 @@ import io import json import optparse import os -import sys +import sys import subprocess import time -import zipfile -import platform +import zipfile +import platform # This script changes test run classpath by unpacking tests.jar -> tests-dir. The goal # is to launch tests with the same classpath as maven does. @@ -18,37 +18,37 @@ def parse_args(): parser.add_option('--trace-file') parser.add_option('--jar-binary') parser.add_option('--tests-jar-path') - parser.add_option('--classpath-option-type', choices=('manifest', 'command_file', 'list'), default='manifest') + parser.add_option('--classpath-option-type', choices=('manifest', 'command_file', 'list'), default='manifest') return parser.parse_args() -# temporary, for jdk8/jdk9+ compatibility -def fix_cmd(cmd): - if not cmd: - return cmd - java = cmd[0] - if not java.endswith('java') and not java.endswith('java.exe'): - return cmd - p = subprocess.Popen([java, '-version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, err = p.communicate() - out, err = out.strip(), err.strip() - if ((out or '').strip().startswith('java version "1.8') or (err or '').strip().startswith('java version "1.8')): - res = [] - i = 0 - while i < len(cmd): - for option in ('--add-exports', '--add-modules'): - if cmd[i] == option: - i += 1 - break - elif cmd[i].startswith(option + '='): - break - else: - res.append(cmd[i]) - i += 1 - return res - return cmd - - +# temporary, for jdk8/jdk9+ compatibility +def fix_cmd(cmd): + if not cmd: + return cmd + java = cmd[0] + if not java.endswith('java') and not java.endswith('java.exe'): + return cmd + p = subprocess.Popen([java, '-version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = p.communicate() + out, err = out.strip(), err.strip() + if ((out or '').strip().startswith('java version "1.8') or (err or '').strip().startswith('java version "1.8')): + res = [] + i = 0 + while i < len(cmd): + for option in ('--add-exports', '--add-modules'): + if cmd[i] == option: + i += 1 + break + elif cmd[i].startswith(option + '='): + break + else: + res.append(cmd[i]) + i += 1 + return res + return cmd + + def dump_event(etype, data, filename): event = { 'timestamp': time.time(), @@ -70,34 +70,34 @@ def extract_jars(dest, archive): zf.extractall(dest) -def make_bfg_from_cp(class_path, out): - class_path = ' '.join( - map(lambda path: ('file:/' + path.lstrip('/')) if os.path.isabs(path) else path, class_path) - ) - with zipfile.ZipFile(out, 'w') as zf: - lines = [] - while class_path: - lines.append(class_path[:60]) - class_path = class_path[60:] - if lines: - zf.writestr('META-INF/MANIFEST.MF', 'Manifest-Version: 1.0\nClass-Path: \n ' + '\n '.join(lines) + ' \n\n') - - -def make_command_file_from_cp(class_path, out): - with open(out, 'w') as cp_file: - cp_file.write(os.pathsep.join(class_path)) - - +def make_bfg_from_cp(class_path, out): + class_path = ' '.join( + map(lambda path: ('file:/' + path.lstrip('/')) if os.path.isabs(path) else path, class_path) + ) + with zipfile.ZipFile(out, 'w') as zf: + lines = [] + while class_path: + lines.append(class_path[:60]) + class_path = class_path[60:] + if lines: + zf.writestr('META-INF/MANIFEST.MF', 'Manifest-Version: 1.0\nClass-Path: \n ' + '\n '.join(lines) + ' \n\n') + + +def make_command_file_from_cp(class_path, out): + with open(out, 'w') as cp_file: + cp_file.write(os.pathsep.join(class_path)) + + def main(): s = time.time() opts, args = parse_args() # unpack tests jar try: - build_root = args[args.index('--build-root') + 1] - dest = os.path.join(build_root, 'test-classes') + build_root = args[args.index('--build-root') + 1] + dest = os.path.join(build_root, 'test-classes') except Exception: - build_root = '' + build_root = '' dest = os.path.abspath('test-classes') extract_jars(dest, opts.tests_jar_path) @@ -108,29 +108,29 @@ def main(): s = time.time() # fix java classpath - cp_idx = args.index('-classpath') - if args[cp_idx + 1].startswith('@'): - real_name = args[cp_idx + 1][1:] - mf = os.path.join(os.path.dirname(real_name), 'fixed.bfg.jar') - with open(real_name) as origin: - class_path = [os.path.join(build_root, i.strip()) for i in origin] - if opts.tests_jar_path in class_path: - class_path.remove(opts.tests_jar_path) - if opts.classpath_option_type == 'manifest': - make_bfg_from_cp(class_path, mf) - mf = os.pathsep.join([dest, mf]) - elif opts.classpath_option_type == 'command_file': - mf = os.path.splitext(mf)[0] + '.txt' - make_command_file_from_cp([dest] + class_path, mf) - mf = "@" + mf - elif opts.classpath_option_type == 'list': - mf = os.pathsep.join([dest] + class_path) - else: - raise Exception("Unexpected classpath option type: " + opts.classpath_option_type) - args = fix_cmd(args[:cp_idx + 1]) + [mf] + args[cp_idx + 2:] - else: - args[cp_idx + 1] = args[cp_idx + 1].replace(opts.tests_jar_path, dest) - args = fix_cmd(args[:cp_idx]) + args[cp_idx:] + cp_idx = args.index('-classpath') + if args[cp_idx + 1].startswith('@'): + real_name = args[cp_idx + 1][1:] + mf = os.path.join(os.path.dirname(real_name), 'fixed.bfg.jar') + with open(real_name) as origin: + class_path = [os.path.join(build_root, i.strip()) for i in origin] + if opts.tests_jar_path in class_path: + class_path.remove(opts.tests_jar_path) + if opts.classpath_option_type == 'manifest': + make_bfg_from_cp(class_path, mf) + mf = os.pathsep.join([dest, mf]) + elif opts.classpath_option_type == 'command_file': + mf = os.path.splitext(mf)[0] + '.txt' + make_command_file_from_cp([dest] + class_path, mf) + mf = "@" + mf + elif opts.classpath_option_type == 'list': + mf = os.pathsep.join([dest] + class_path) + else: + raise Exception("Unexpected classpath option type: " + opts.classpath_option_type) + args = fix_cmd(args[:cp_idx + 1]) + [mf] + args[cp_idx + 2:] + else: + args[cp_idx + 1] = args[cp_idx + 1].replace(opts.tests_jar_path, dest) + args = fix_cmd(args[:cp_idx]) + args[cp_idx:] metrics['suite_jtest_fix_classpath_(seconds)'] = time.time() - s @@ -138,10 +138,10 @@ def main(): dump_chunk_event({'metrics': metrics}, opts.trace_file) # run java cmd - if platform.system() == 'Windows': - sys.exit(subprocess.Popen(args).wait()) - else: - os.execv(args[0], args) + if platform.system() == 'Windows': + sys.exit(subprocess.Popen(args).wait()) + else: + os.execv(args[0], args) if __name__ == '__main__': diff --git a/build/scripts/with_crash_on_timeout.py b/build/scripts/with_crash_on_timeout.py index bde864ed29..dc6df738d7 100644 --- a/build/scripts/with_crash_on_timeout.py +++ b/build/scripts/with_crash_on_timeout.py @@ -1,22 +1,22 @@ # TODO prettyboy remove after ya-bin release -import os -import sys -import subprocess -import json - - -def main(args): - meta_path = os.path.abspath(args[0]) - timeout_code = int(args[1]) - subprocess.check_call(args[2:]) - with open(meta_path) as f: - meta_info = json.loads(f.read()) - if meta_info["exit_code"] == timeout_code: - print >> sys.stderr, meta_info["project"], 'crashed by timeout, use --test-disable-timeout option' - return 1 - return 0 - - -if __name__ == '__main__': - main(sys.argv[1:]) +import os +import sys +import subprocess +import json + + +def main(args): + meta_path = os.path.abspath(args[0]) + timeout_code = int(args[1]) + subprocess.check_call(args[2:]) + with open(meta_path) as f: + meta_info = json.loads(f.read()) + if meta_info["exit_code"] == timeout_code: + print >> sys.stderr, meta_info["project"], 'crashed by timeout, use --test-disable-timeout option' + return 1 + return 0 + + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/build/scripts/with_pathsep_resolve.py b/build/scripts/with_pathsep_resolve.py index 37c8c598ae..986e757f49 100644 --- a/build/scripts/with_pathsep_resolve.py +++ b/build/scripts/with_pathsep_resolve.py @@ -1,23 +1,23 @@ -import sys -import os -import subprocess -import platform - - -def fix_args(args): - just_replace_it = False - for arg in args: - if arg == '--fix-path-sep': - just_replace_it = True - continue - if just_replace_it: - arg = arg.replace('::', os.pathsep) - just_replace_it = False - yield arg - -if __name__ == '__main__': - res = list(fix_args(sys.argv[1:])) - if platform.system() == 'Windows': - sys.exit(subprocess.Popen(res).wait()) - else: - os.execv(res[0], res) +import sys +import os +import subprocess +import platform + + +def fix_args(args): + just_replace_it = False + for arg in args: + if arg == '--fix-path-sep': + just_replace_it = True + continue + if just_replace_it: + arg = arg.replace('::', os.pathsep) + just_replace_it = False + yield arg + +if __name__ == '__main__': + res = list(fix_args(sys.argv[1:])) + if platform.system() == 'Windows': + sys.exit(subprocess.Popen(res).wait()) + else: + os.execv(res[0], res) diff --git a/build/scripts/wrap_groovyc.py b/build/scripts/wrap_groovyc.py index 068b73fd87..489ffa80bc 100644 --- a/build/scripts/wrap_groovyc.py +++ b/build/scripts/wrap_groovyc.py @@ -1,23 +1,23 @@ -import platform -import sys -import os -import subprocess - - -def fix_windows(args): - for arg in args: - if os.path.basename(arg) == 'groovyc' and os.path.basename(os.path.dirname(arg)) == 'bin': - yield arg + '.bat' - else: - yield arg - - -if __name__ == '__main__': - env = os.environ.copy() - jdk = sys.argv[1] - env['JAVA_HOME'] = jdk - args = sys.argv[2:] - if platform.system() == 'Windows': - sys.exit(subprocess.Popen(list(fix_windows(args)), env=env).wait()) - else: - os.execve(args[0], args, env) +import platform +import sys +import os +import subprocess + + +def fix_windows(args): + for arg in args: + if os.path.basename(arg) == 'groovyc' and os.path.basename(os.path.dirname(arg)) == 'bin': + yield arg + '.bat' + else: + yield arg + + +if __name__ == '__main__': + env = os.environ.copy() + jdk = sys.argv[1] + env['JAVA_HOME'] = jdk + args = sys.argv[2:] + if platform.system() == 'Windows': + sys.exit(subprocess.Popen(list(fix_windows(args)), env=env).wait()) + else: + os.execve(args[0], args, env) diff --git a/build/scripts/ya.make b/build/scripts/ya.make index 710165e40d..407aafb0d3 100644 --- a/build/scripts/ya.make +++ b/build/scripts/ya.make @@ -3,11 +3,11 @@ OWNER(g:ymake) PY2TEST() TEST_SRCS( - build_dll_and_java.py - build_info_gen.py - build_java_codenav_index.py - build_java_with_error_prone.py - build_java_with_error_prone2.py + build_dll_and_java.py + build_info_gen.py + build_java_codenav_index.py + build_java_with_error_prone.py + build_java_with_error_prone2.py build_mn.py build_pln_header.py cat.py @@ -26,7 +26,7 @@ TEST_SRCS( create_jcoverage_report.py extract_asrc.py extract_docs.py - extract_jacoco_report.py + extract_jacoco_report.py f2c.py fail_module_cmd.py fetch_from.py @@ -35,13 +35,13 @@ TEST_SRCS( fetch_from_npm.py fetch_from_sandbox.py fetch_resource.py - filter_zip.py + filter_zip.py find_and_tar.py fix_msvc_output.py fs_tools.py gen_aar_gradle_script.py gen_java_codenav_entry.py - gen_java_codenav_protobuf.py + gen_java_codenav_protobuf.py gen_mx_table.py gen_py3_reg.py gen_py_reg.py @@ -50,7 +50,7 @@ TEST_SRCS( generate_pom.py go_proto_wrapper.py go_tool.py - ios_wrapper.py + ios_wrapper.py java_pack_to_file.py link_asrc.py link_dyn_lib.py @@ -59,17 +59,17 @@ TEST_SRCS( link_lib.py llvm_opt_wrapper.py merge_coverage_data.py - merge_files.py + merge_files.py mkdir.py mkdocs_builder_wrapper.py mkver.py - pack_ios.py + pack_ios.py pack_jcoverage_resources.py perl_wrapper.py postprocess_go_fbs.py preprocess.py py_compile.py - run_ios_simulator.py + run_ios_simulator.py run_javac.py run_junit.py run_llvm_dsymutil.py |