diff options
| author | spreis <[email protected]> | 2024-12-13 08:25:28 +0300 | 
|---|---|---|
| committer | spreis <[email protected]> | 2024-12-13 09:07:25 +0300 | 
| commit | e90b83df32a421a93c021eaa91710c1193585f9a (patch) | |
| tree | 1bcbc4aa0bcc3b28e8faa8a3b4617e02189faa73 | |
| parent | 88c51d7b8433a730e5426a24cc86e21465ccab5b (diff) | |
Never compress python build resources to speedup python configuration and build
There is no runtime perf impact, but
- The change reduces build graph by removing 2 nodes out of 5 for python which accelerates both configuration and build
- surprisingly in all experiments there were positive impact on binary size about 2MiB
commit_hash:909fdeeb91b7f20c749177d638e97d5a469e422d
| -rw-r--r-- | build/plugins/pybuild.py | 34 | ||||
| -rw-r--r-- | build/plugins/ytest.py | 4 | ||||
| -rw-r--r-- | library/python/runtime_py3/ya.make | 1 | 
3 files changed, 20 insertions, 19 deletions
| diff --git a/build/plugins/pybuild.py b/build/plugins/pybuild.py index 07ced44d044..744449d1098 100644 --- a/build/plugins/pybuild.py +++ b/build/plugins/pybuild.py @@ -5,7 +5,7 @@ import six  from hashlib import md5  import ymake -from _common import stripext, rootrel_arc_src, listid, pathid, lazy, get_no_lint_value +from _common import stripext, rootrel_arc_src, listid, pathid, lazy, get_no_lint_value, ugly_conftest_exception  YA_IDE_VENV_VAR = 'YA_IDE_VENV' @@ -15,12 +15,12 @@ DEFAULT_FLAKE8_FILE_PROCESSING_TIME = "1.5"  # in seconds  DEFAULT_BLACK_FILE_PROCESSING_TIME = "1.5"  # in seconds -def _split_macro_call(macro_call, data, item_size, chunk_size=1024): +def _split_macro_call(macro_call, data, item_size, chunk_size=1024, compress=False):      index = 0      length = len(data)      offset = item_size * chunk_size      while index + 1 < length: -        macro_call(data[index : index + offset]) +        macro_call(([] if compress else ['DONT_COMPRESS']) + data[index : index + offset])          index += offset @@ -507,10 +507,10 @@ def onpy_srcs(unit, *args):          if py_files2res:              # Compile original and generated sources into target for proper cython coverage calculation              for files2res in (py_files2res, cpp_files2res): -                unit.onresource_files([x for name, path in files2res for x in ('DEST', name, path)]) +                unit.onresource_files(['DONT_COMPRESS'] + [x for name, path in files2res for x in ('DEST', name, path)])          if include_map: -            data = [] +            data = ['DONT_COMPRESS']              prefix = 'resfs/cython/include'              for line in sorted(                  '{}/{}={}'.format(prefix, filename, ':'.join(sorted(files))) @@ -541,6 +541,7 @@ def onpy_srcs(unit, *args):          if py3:              mod_list_md5 = md5() +            compress = False              for path, mod in pys:                  mod_list_md5.update(six.ensure_binary(mod))                  if not (venv and is_extended_source_search_enabled(path, unit)): @@ -552,17 +553,21 @@ def onpy_srcs(unit, *args):                          dst = path + uniq_suffix(path, unit)                          unit.on_py3_compile_bytecode([root_rel_path + '-', path, dst])                          res += ['DEST', dest + '.yapyc3', dst + '.yapyc3'] +                    if not compress and ugly_conftest_exception(path): +                        compress = True              if py_namespaces:                  # Note: Add md5 to key to prevent key collision if two or more PY_SRCS() used in the same ya.make -                ns_res = [] +                ns_res = ['DONT_COMPRESS']                  for path, ns in sorted(py_namespaces.items()):                      key = '{}/{}/{}'.format(PY_NAMESPACE_PREFIX, mod_list_md5.hexdigest(), path)                      namespaces = ':'.join(sorted(ns))                      ns_res += ['-', '{}="{}"'.format(key, namespaces)]                  unit.onresource(ns_res) -            _split_macro_call(unit.onresource_files, res, (3 if with_py else 0) + (3 if with_pyc else 0)) +            _split_macro_call( +                unit.onresource_files, res, (3 if with_py else 0) + (3 if with_pyc else 0), compress=compress +            )              add_python_lint_checks(                  unit, 3, [path for path, mod in pys] + unit.get(['_PY_EXTRA_LINT_FILES_VALUE']).split()              ) @@ -571,12 +576,7 @@ def onpy_srcs(unit, *args):                  root_rel_path = rootrel_arc_src(path, unit)                  if with_py:                      key = '/py_modules/' + mod -                    res += [ -                        path, -                        key, -                        '-', -                        'resfs/src/{}={}'.format(key, root_rel_path), -                    ] +                    res += [path, key, '-', 'resfs/src/{}=${{rootrel;input;context=TEXT:"{}"}}'.format(key, path)]                  if with_pyc:                      src = unit.resolve_arc_path(path) or path                      dst = path + uniq_suffix(path, unit) @@ -595,7 +595,7 @@ def onpy_srcs(unit, *args):              pyis_dups = ', '.join(name for name in sorted(pyis_dups))              ymake.report_configure_error('Duplicate(s) is found in the PY_SRCS macro: {}'.format(pyis_dups)) -        res = [] +        res = ['DONT_COMPRESS']          for path, mod in pyis:              dest = 'py/' + mod.replace('.', '/') + '.pyi'              res += ['DEST', dest, path] @@ -671,7 +671,7 @@ def onpy_doctests(unit, *args):      The packages should be part of a test (listed as sources of the test or its PEERDIRs).      """      if unit.get('PY3TEST_BIN' if is_py3(unit) else 'PYTEST_BIN') != 'no': -        unit.onresource(['-', 'PY_DOCTEST_PACKAGES="{}"'.format(' '.join(args))]) +        unit.onresource(['DONT_COMPRESS', '-', 'PY_DOCTEST_PACKAGES="{}"'.format(' '.join(args))])  def py_register(unit, func, py3): @@ -716,7 +716,7 @@ def py_main(unit, arg):      unit_needs_main = unit.get('MODULE_TYPE') in ('PROGRAM', 'DLL')      if unit_needs_main:          py_program(unit, is_py3(unit)) -    unit.onresource(['-', 'PY_MAIN={}'.format(arg)]) +    unit.onresource(['DONT_COMPRESS', '-', 'PY_MAIN={}'.format(arg)])  def onpy_main(unit, arg): @@ -748,7 +748,7 @@ def onpy_constructor(unit, arg):          arg = arg + '=init'      else:          arg[arg.index(':')] = '=' -    unit.onresource(['-', 'py/constructors/{}'.format(arg)]) +    unit.onresource(['DONT_COMPRESS', '-', 'py/constructors/{}'.format(arg)])  def onpy_enums_serialization(unit, *args): diff --git a/build/plugins/ytest.py b/build/plugins/ytest.py index a8ef930a07d..49110348777 100644 --- a/build/plugins/ytest.py +++ b/build/plugins/ytest.py @@ -772,7 +772,7 @@ def onadd_check(unit, *args):  def on_register_no_check_imports(unit):      s = unit.get('NO_CHECK_IMPORTS_FOR_VALUE')      if s not in ('', 'None'): -        unit.onresource(['-', 'py/no_check_imports/{}="{}"'.format(_common.pathid(s), s)]) +        unit.onresource(['DONT_COMPRESS', '-', 'py/no_check_imports/{}="{}"'.format(_common.pathid(s), s)])  @df.with_fields( @@ -936,7 +936,7 @@ def onjava_test_deps(fields, unit, *args):  def onsetup_pytest_bin(unit, *args):      use_arcadia_python = unit.get('USE_ARCADIA_PYTHON') == "yes"      if use_arcadia_python: -        unit.onresource(['-', 'PY_MAIN={}'.format("library.python.pytest.main:main")])  # XXX +        unit.onresource(['DONT_COMPRESS', '-', 'PY_MAIN={}'.format("library.python.pytest.main:main")])  # XXX          unit.onadd_pytest_bin(list(args)) diff --git a/library/python/runtime_py3/ya.make b/library/python/runtime_py3/ya.make index 05687fdd8fc..0ad7d3e13f7 100644 --- a/library/python/runtime_py3/ya.make +++ b/library/python/runtime_py3/ya.make @@ -29,6 +29,7 @@ IF (CYTHON_COVERAGE)      # Let covarage support add all needed files to resources  ELSE()      RESOURCE_FILES( +        DONT_COMPRESS          PREFIX ${MODDIR}/          __res.pyx          importer.pxi | 
