diff options
author | snermolaev <snermolaev@yandex-team.ru> | 2022-02-10 16:45:53 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:45:53 +0300 |
commit | 7353a3fdea9c67c256980c00a2b3b67f09b23a27 (patch) | |
tree | 1a2c5ffcf89eb53ecd79dbc9bc0a195c27404d0c /build/scripts/compile_pysrc.py | |
parent | 2015790ac9fcc04caab83fccc23ab2460310a797 (diff) | |
download | ydb-7353a3fdea9c67c256980c00a2b3b67f09b23a27.tar.gz |
Restoring authorship annotation for <snermolaev@yandex-team.ru>. Commit 2 of 2.
Diffstat (limited to 'build/scripts/compile_pysrc.py')
-rw-r--r-- | build/scripts/compile_pysrc.py | 202 |
1 files changed, 101 insertions, 101 deletions
diff --git a/build/scripts/compile_pysrc.py b/build/scripts/compile_pysrc.py index 91107b530e..e3637e18e2 100644 --- a/build/scripts/compile_pysrc.py +++ b/build/scripts/compile_pysrc.py @@ -1,101 +1,101 @@ -import argparse -import os -import shutil -import subprocess -import tarfile - - -LIMIT = 6000 - - -def parse_args(): - parser = argparse.ArgumentParser() - parser.add_argument('--input', required=True) - parser.add_argument('--output', required=True) - parser.add_argument('--rescompiler', required=True) - subparsers = parser.add_subparsers(dest='mode') - - parser_py2 = subparsers.add_parser('py2') - parser_py2.add_argument('--py_compile', required=True) - parser_py2.add_argument('--python', required=True) - - parser_py3 = subparsers.add_parser('py3') - parser_py3.add_argument('--pycc', required=True) - - return parser.parse_args() - - -def call(cmd, cwd=None, env=None): - return subprocess.check_output(cmd, stdin=None, stderr=subprocess.STDOUT, cwd=cwd, env=env) - - -def iterate_py2_resource_params(py_files): - for py in py_files: - mod = py[:-3].replace('/', '.') - key = '/py_modules/{}'.format(mod) - yield py, key - yield '-', 'resfs/src/{}={}'.format(key, py) - yield '{}.yapyc'.format(py), '/py_code/{}'.format(mod) - - -def iterate_py3_resource_params(py_files): - for py in py_files: - for ext in ('', '.yapyc3'): - path = '{}{}'.format(py, ext) - dest = 'py/{}'.format(path) - key = 'resfs/file/{}'.format(dest) - src = 'resfs/src/{}={}'.format(key, os.path.basename(path)) - yield '-', src - yield path, key - - -def main(): - args = parse_args() - - names = [] - with tarfile.open(args.input, 'r') as tar: - names = tar.getnames() - tar.extractall() - - if args.mode == 'py3': - pycc_cmd = [args.pycc] - pycc_ext = '.yapyc3' - iterate_resource_params = iterate_py3_resource_params - else: - pycc_cmd = [args.python, args.py_compile] - pycc_ext = '.yapyc' - iterate_resource_params = iterate_py2_resource_params - - py_files = sorted(names) - - for py in py_files: - cmd = pycc_cmd + ['{}-'.format(os.path.basename(py)), py, '{}{}'.format(py, pycc_ext)] - call(cmd) - - outputs = [] - cmd = [args.rescompiler, '{}.0'.format(args.output)] - size = 0 - for path, key in iterate_resource_params(py_files): - addendum = len(path) + len(key) - if size + addendum > LIMIT and len(cmd) > 2: - call(cmd) - outputs.append(cmd[1]) - cmd[1] = '{}.{}'.format(args.output, len(outputs)) - cmd = cmd[0:2] - size = 0 - cmd.extend([path, key]) - size += addendum - if len(outputs) == 0: - cmd[1] = args.output - call(cmd) - else: - call(cmd) - outputs.append(cmd[1]) - with open(args.output, 'w') as fout: - for fname in outputs: - with open(fname, 'r') as fin: - shutil.copyfileobj(fin, fout) - - -if __name__ == '__main__': - main() +import argparse +import os +import shutil +import subprocess +import tarfile + + +LIMIT = 6000 + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument('--input', required=True) + parser.add_argument('--output', required=True) + parser.add_argument('--rescompiler', required=True) + subparsers = parser.add_subparsers(dest='mode') + + parser_py2 = subparsers.add_parser('py2') + parser_py2.add_argument('--py_compile', required=True) + parser_py2.add_argument('--python', required=True) + + parser_py3 = subparsers.add_parser('py3') + parser_py3.add_argument('--pycc', required=True) + + return parser.parse_args() + + +def call(cmd, cwd=None, env=None): + return subprocess.check_output(cmd, stdin=None, stderr=subprocess.STDOUT, cwd=cwd, env=env) + + +def iterate_py2_resource_params(py_files): + for py in py_files: + mod = py[:-3].replace('/', '.') + key = '/py_modules/{}'.format(mod) + yield py, key + yield '-', 'resfs/src/{}={}'.format(key, py) + yield '{}.yapyc'.format(py), '/py_code/{}'.format(mod) + + +def iterate_py3_resource_params(py_files): + for py in py_files: + for ext in ('', '.yapyc3'): + path = '{}{}'.format(py, ext) + dest = 'py/{}'.format(path) + key = 'resfs/file/{}'.format(dest) + src = 'resfs/src/{}={}'.format(key, os.path.basename(path)) + yield '-', src + yield path, key + + +def main(): + args = parse_args() + + names = [] + with tarfile.open(args.input, 'r') as tar: + names = tar.getnames() + tar.extractall() + + if args.mode == 'py3': + pycc_cmd = [args.pycc] + pycc_ext = '.yapyc3' + iterate_resource_params = iterate_py3_resource_params + else: + pycc_cmd = [args.python, args.py_compile] + pycc_ext = '.yapyc' + iterate_resource_params = iterate_py2_resource_params + + py_files = sorted(names) + + for py in py_files: + cmd = pycc_cmd + ['{}-'.format(os.path.basename(py)), py, '{}{}'.format(py, pycc_ext)] + call(cmd) + + outputs = [] + cmd = [args.rescompiler, '{}.0'.format(args.output)] + size = 0 + for path, key in iterate_resource_params(py_files): + addendum = len(path) + len(key) + if size + addendum > LIMIT and len(cmd) > 2: + call(cmd) + outputs.append(cmd[1]) + cmd[1] = '{}.{}'.format(args.output, len(outputs)) + cmd = cmd[0:2] + size = 0 + cmd.extend([path, key]) + size += addendum + if len(outputs) == 0: + cmd[1] = args.output + call(cmd) + else: + call(cmd) + outputs.append(cmd[1]) + with open(args.output, 'w') as fout: + for fname in outputs: + with open(fname, 'r') as fin: + shutil.copyfileobj(fin, fout) + + +if __name__ == '__main__': + main() |