diff options
author | spreis <spreis@yandex-team.com> | 2024-12-13 08:25:28 +0300 |
---|---|---|
committer | spreis <spreis@yandex-team.com> | 2024-12-13 09:07:25 +0300 |
commit | e90b83df32a421a93c021eaa91710c1193585f9a (patch) | |
tree | 1bcbc4aa0bcc3b28e8faa8a3b4617e02189faa73 | |
parent | 88c51d7b8433a730e5426a24cc86e21465ccab5b (diff) | |
download | ydb-e90b83df32a421a93c021eaa91710c1193585f9a.tar.gz |
Never compress python build resources to speedup python configuration and build
There is no runtime perf impact, but
- The change reduces build graph by removing 2 nodes out of 5 for python which accelerates both configuration and build
- surprisingly in all experiments there were positive impact on binary size about 2MiB
commit_hash:909fdeeb91b7f20c749177d638e97d5a469e422d
-rw-r--r-- | build/plugins/pybuild.py | 34 | ||||
-rw-r--r-- | build/plugins/ytest.py | 4 | ||||
-rw-r--r-- | library/python/runtime_py3/ya.make | 1 |
3 files changed, 20 insertions, 19 deletions
diff --git a/build/plugins/pybuild.py b/build/plugins/pybuild.py index 07ced44d044..744449d1098 100644 --- a/build/plugins/pybuild.py +++ b/build/plugins/pybuild.py @@ -5,7 +5,7 @@ import six from hashlib import md5 import ymake -from _common import stripext, rootrel_arc_src, listid, pathid, lazy, get_no_lint_value +from _common import stripext, rootrel_arc_src, listid, pathid, lazy, get_no_lint_value, ugly_conftest_exception YA_IDE_VENV_VAR = 'YA_IDE_VENV' @@ -15,12 +15,12 @@ DEFAULT_FLAKE8_FILE_PROCESSING_TIME = "1.5" # in seconds DEFAULT_BLACK_FILE_PROCESSING_TIME = "1.5" # in seconds -def _split_macro_call(macro_call, data, item_size, chunk_size=1024): +def _split_macro_call(macro_call, data, item_size, chunk_size=1024, compress=False): index = 0 length = len(data) offset = item_size * chunk_size while index + 1 < length: - macro_call(data[index : index + offset]) + macro_call(([] if compress else ['DONT_COMPRESS']) + data[index : index + offset]) index += offset @@ -507,10 +507,10 @@ def onpy_srcs(unit, *args): if py_files2res: # Compile original and generated sources into target for proper cython coverage calculation for files2res in (py_files2res, cpp_files2res): - unit.onresource_files([x for name, path in files2res for x in ('DEST', name, path)]) + unit.onresource_files(['DONT_COMPRESS'] + [x for name, path in files2res for x in ('DEST', name, path)]) if include_map: - data = [] + data = ['DONT_COMPRESS'] prefix = 'resfs/cython/include' for line in sorted( '{}/{}={}'.format(prefix, filename, ':'.join(sorted(files))) @@ -541,6 +541,7 @@ def onpy_srcs(unit, *args): if py3: mod_list_md5 = md5() + compress = False for path, mod in pys: mod_list_md5.update(six.ensure_binary(mod)) if not (venv and is_extended_source_search_enabled(path, unit)): @@ -552,17 +553,21 @@ def onpy_srcs(unit, *args): dst = path + uniq_suffix(path, unit) unit.on_py3_compile_bytecode([root_rel_path + '-', path, dst]) res += ['DEST', dest + '.yapyc3', dst + '.yapyc3'] + if not compress and ugly_conftest_exception(path): + compress = True if py_namespaces: # Note: Add md5 to key to prevent key collision if two or more PY_SRCS() used in the same ya.make - ns_res = [] + ns_res = ['DONT_COMPRESS'] for path, ns in sorted(py_namespaces.items()): key = '{}/{}/{}'.format(PY_NAMESPACE_PREFIX, mod_list_md5.hexdigest(), path) namespaces = ':'.join(sorted(ns)) ns_res += ['-', '{}="{}"'.format(key, namespaces)] unit.onresource(ns_res) - _split_macro_call(unit.onresource_files, res, (3 if with_py else 0) + (3 if with_pyc else 0)) + _split_macro_call( + unit.onresource_files, res, (3 if with_py else 0) + (3 if with_pyc else 0), compress=compress + ) add_python_lint_checks( unit, 3, [path for path, mod in pys] + unit.get(['_PY_EXTRA_LINT_FILES_VALUE']).split() ) @@ -571,12 +576,7 @@ def onpy_srcs(unit, *args): root_rel_path = rootrel_arc_src(path, unit) if with_py: key = '/py_modules/' + mod - res += [ - path, - key, - '-', - 'resfs/src/{}={}'.format(key, root_rel_path), - ] + res += [path, key, '-', 'resfs/src/{}=${{rootrel;input;context=TEXT:"{}"}}'.format(key, path)] if with_pyc: src = unit.resolve_arc_path(path) or path dst = path + uniq_suffix(path, unit) @@ -595,7 +595,7 @@ def onpy_srcs(unit, *args): pyis_dups = ', '.join(name for name in sorted(pyis_dups)) ymake.report_configure_error('Duplicate(s) is found in the PY_SRCS macro: {}'.format(pyis_dups)) - res = [] + res = ['DONT_COMPRESS'] for path, mod in pyis: dest = 'py/' + mod.replace('.', '/') + '.pyi' res += ['DEST', dest, path] @@ -671,7 +671,7 @@ def onpy_doctests(unit, *args): The packages should be part of a test (listed as sources of the test or its PEERDIRs). """ if unit.get('PY3TEST_BIN' if is_py3(unit) else 'PYTEST_BIN') != 'no': - unit.onresource(['-', 'PY_DOCTEST_PACKAGES="{}"'.format(' '.join(args))]) + unit.onresource(['DONT_COMPRESS', '-', 'PY_DOCTEST_PACKAGES="{}"'.format(' '.join(args))]) def py_register(unit, func, py3): @@ -716,7 +716,7 @@ def py_main(unit, arg): unit_needs_main = unit.get('MODULE_TYPE') in ('PROGRAM', 'DLL') if unit_needs_main: py_program(unit, is_py3(unit)) - unit.onresource(['-', 'PY_MAIN={}'.format(arg)]) + unit.onresource(['DONT_COMPRESS', '-', 'PY_MAIN={}'.format(arg)]) def onpy_main(unit, arg): @@ -748,7 +748,7 @@ def onpy_constructor(unit, arg): arg = arg + '=init' else: arg[arg.index(':')] = '=' - unit.onresource(['-', 'py/constructors/{}'.format(arg)]) + unit.onresource(['DONT_COMPRESS', '-', 'py/constructors/{}'.format(arg)]) def onpy_enums_serialization(unit, *args): diff --git a/build/plugins/ytest.py b/build/plugins/ytest.py index a8ef930a07d..49110348777 100644 --- a/build/plugins/ytest.py +++ b/build/plugins/ytest.py @@ -772,7 +772,7 @@ def onadd_check(unit, *args): def on_register_no_check_imports(unit): s = unit.get('NO_CHECK_IMPORTS_FOR_VALUE') if s not in ('', 'None'): - unit.onresource(['-', 'py/no_check_imports/{}="{}"'.format(_common.pathid(s), s)]) + unit.onresource(['DONT_COMPRESS', '-', 'py/no_check_imports/{}="{}"'.format(_common.pathid(s), s)]) @df.with_fields( @@ -936,7 +936,7 @@ def onjava_test_deps(fields, unit, *args): def onsetup_pytest_bin(unit, *args): use_arcadia_python = unit.get('USE_ARCADIA_PYTHON') == "yes" if use_arcadia_python: - unit.onresource(['-', 'PY_MAIN={}'.format("library.python.pytest.main:main")]) # XXX + unit.onresource(['DONT_COMPRESS', '-', 'PY_MAIN={}'.format("library.python.pytest.main:main")]) # XXX unit.onadd_pytest_bin(list(args)) diff --git a/library/python/runtime_py3/ya.make b/library/python/runtime_py3/ya.make index 05687fdd8fc..0ad7d3e13f7 100644 --- a/library/python/runtime_py3/ya.make +++ b/library/python/runtime_py3/ya.make @@ -29,6 +29,7 @@ IF (CYTHON_COVERAGE) # Let covarage support add all needed files to resources ELSE() RESOURCE_FILES( + DONT_COMPRESS PREFIX ${MODDIR}/ __res.pyx importer.pxi |