diff options
| author | prettyboy <prettyboy@yandex-team.com> | 2025-01-16 21:01:45 +0300 |
|---|---|---|
| committer | prettyboy <prettyboy@yandex-team.com> | 2025-01-16 22:26:51 +0300 |
| commit | 2f3fd95aac24e27a3b0aa2badda49db82bf36cc5 (patch) | |
| tree | 55c8665476c55cd7564ac4f9a0adc2cb078f9c9e /build/plugins | |
| parent | 8f1bc40ec9764df3efecadb5e8a715ada81789d4 (diff) | |
| download | ydb-2f3fd95aac24e27a3b0aa2badda49db82bf36cc5.tar.gz | |
Prototype - external python files mode for tests
Режим позволяет перезапускать python тесты или интеграционные тесты на питон программы при внесении изменений в python файлы без перекомпиляции.
Сценарий:
- прогнали тесты
- внесли изменение в `library/python/func/__init__.py`
- перезапустили, смотрим на времена
На примере интеграционного devtools/ya/handlers/analyze_make/tests в codenv (base конфигурация).
#|
||
ya m -rA
|
ya m -rA --ext-py
||
||
real 10m25,928s
user 0m53,465s
sys 1m14,791s
```
Total time by type:
[935171 ms] [LD] [count: 3, ave time 311723.67 msec]
[ 13657 ms] [prepare:AC] [count: 4, ave time 3414.25 msec]
[ 10436 ms] [TM] [count: 1, ave time 10436.00 msec]
[ 10387 ms] [prepare:get from local cache] [count: 1521, ave time 6.83 msec]
[ 6480 ms] [TS] [count: 1, ave time 6480.00 msec]
[ 3823 ms] [prepare:put into local cache, clean build dir] [count: 7, ave time 546.14 msec]
[ 2279 ms] [CC] [count: 1, ave time 2279.00 msec]
[ 1251 ms] [PY] [count: 1, ave time 1251.00 msec]
[ 372 ms] [prepare:yt-store] [count: 2, ave time 186.00 msec]
[ 154 ms] [AR] [count: 1, ave time 154.00 msec]
...
```
|
real 0m22,094s
user 0m26,372s
sys 0m3,363s
```
Total time by type:
[13140 ms] [TM] [count: 1, ave time 13140.00 msec]
[ 351 ms] [prepare:yt-store] [count: 1, ave time 351.00 msec]
[ 81 ms] [prepare:tools] [count: 8, ave time 10.12 msec]
[ 32 ms] [prepare:AC] [count: 2, ave time 16.00 msec]
[ 31 ms] [prepare:get from local cache] [count: 7, ave time 4.43 msec]
[ 16 ms] [prepare:clean] [count: 3, ave time 5.33 msec]
```
||
|#
В codenv в base конфигуарции (4cpu) невероятно долго идёт линковка. Тот же самый сценарий, но на mous (54cpu)
#|
||
ya m -rA
|
ya m -rA --ext-py
||
||
real 0m25,435s
user 0m40,242s
sys 0m22,434s
```
Total time by type:
[30958 ms] [prepare:get from local cache] [count: 1521, ave time 20.35 msec]
[16293 ms] [prepare:AC] [count: 4, ave time 4073.25 msec]
[11161 ms] [LD] [count: 3, ave time 3720.33 msec]
[ 4256 ms] [TM] [count: 1, ave time 4256.00 msec]
[ 3529 ms] [TS] [count: 1, ave time 3529.00 msec]
[ 1542 ms] [CC] [count: 1, ave time 1542.00 msec]
[ 635 ms] [PY] [count: 1, ave time 635.00 msec]
...
```
|
real 0m18,321s
user 0m21,673s
sys 0m3,204s
```
Total time by type:
[10786 ms] [TM] [count: 1, ave time 10786.00 msec]
[ 1568 ms] [prepare:yt-store] [count: 1, ave time 1568.00 msec]
[ 151 ms] [prepare:tools] [count: 8, ave time 18.88 msec]
[ 61 ms] [prepare:get from local cache] [count: 7, ave time 8.71 msec]
[ 48 ms] [prepare:AC] [count: 2, ave time 24.00 msec]
[ 14 ms] [prepare:clean] [count: 3, ave time 4.67 msec]
```
||
|#
В новом режиме тест ожидаемо замедляется, так как все питон программы при каждом старте вынуждены читать файлы с фс (ещё через арк), а потом ещё и компилировать байткод, т.е. холодный запуск
commit_hash:24c5a46a8385d3c065abfb6fc5b40f7ad24bb1cb
Diffstat (limited to 'build/plugins')
| -rw-r--r-- | build/plugins/pybuild.py | 25 | ||||
| -rw-r--r-- | build/plugins/ytest.py | 8 |
2 files changed, 27 insertions, 6 deletions
diff --git a/build/plugins/pybuild.py b/build/plugins/pybuild.py index 744449d109..229d4b1665 100644 --- a/build/plugins/pybuild.py +++ b/build/plugins/pybuild.py @@ -8,7 +8,6 @@ import ymake from _common import stripext, rootrel_arc_src, listid, pathid, lazy, get_no_lint_value, ugly_conftest_exception -YA_IDE_VENV_VAR = 'YA_IDE_VENV' PY_NAMESPACE_PREFIX = 'py/namespace' BUILTIN_PROTO = 'builtin_proto' DEFAULT_FLAKE8_FILE_PROCESSING_TIME = "1.5" # in seconds @@ -37,8 +36,8 @@ def is_extended_source_search_enabled(path, unit): return False if unit.get('NO_EXTENDED_SOURCE_SEARCH') == 'yes': return False - # contrib is unfriendly to extended source search - if unit.resolve_arc_path(path).startswith('$S/contrib/'): + # Python itself and contrib/python are unfriendly to extended source search + if unit.resolve_arc_path(path).startswith(('$S/contrib/python', '$S/contrib/tools/python3')): return False return True @@ -252,7 +251,7 @@ def onpy_srcs(unit, *args): with_py = not unit.get('PYBUILD_NO_PY') with_pyc = not unit.get('PYBUILD_NO_PYC') in_proto_library = unit.get('PY_PROTO') or unit.get('PY3_PROTO') - venv = unit.get(YA_IDE_VENV_VAR) + external_py_files_mode = unit.get('EXTERNAL_PY_FILES') or unit.get('YA_IDE_VENV') need_gazetteer_peerdir = False trim = 0 @@ -542,10 +541,21 @@ def onpy_srcs(unit, *args): if py3: mod_list_md5 = md5() compress = False + resfs_mocks = [] + for path, mod in pys: mod_list_md5.update(six.ensure_binary(mod)) - if not (venv and is_extended_source_search_enabled(path, unit)): - dest = 'py/' + mod.replace('.', '/') + '.py' + dest = 'py/' + mod.replace('.', '/') + '.py' + # In external_py_files mode we want to build python binaries without embedded python files. + # The application will still be able to load them from the file system. + # However, the import hook still needs meta information about the file locations to work correctly. + # That's why we manually register the files in resfs/src/resfs/file, but don't provide any actual files. + # For more info see: + # - library/python/runtime_py3 + # - https://docs.yandex-team.ru/ya-make/manual/python/vars + if external_py_files_mode and is_extended_source_search_enabled(path, unit): + resfs_mocks += ['-', 'resfs/src/resfs/file/' + dest + '=' + rootrel_arc_src(path, unit)] + else: if with_py: res += ['DEST', dest, path] if with_pyc: @@ -556,6 +566,9 @@ def onpy_srcs(unit, *args): if not compress and ugly_conftest_exception(path): compress = True + if resfs_mocks: + unit.onresource(['DONT_COMPRESS'] + resfs_mocks) + if py_namespaces: # Note: Add md5 to key to prevent key collision if two or more PY_SRCS() used in the same ya.make ns_res = ['DONT_COMPRESS'] diff --git a/build/plugins/ytest.py b/build/plugins/ytest.py index 4911034877..ef47c32835 100644 --- a/build/plugins/ytest.py +++ b/build/plugins/ytest.py @@ -790,13 +790,21 @@ def onadd_check_py_imports(fields, unit, *args): if unit.get("TIDY") == "yes": # graph changed for clang_tidy tests return + if unit.get('NO_CHECK_IMPORTS_FOR_VALUE').strip() == "": return + unit.onpeerdir(['library/python/testing/import_test']) dart_record = create_dart_record(fields, unit, (), {}) dart_record[df.TestName.KEY] = 'pyimports' dart_record[df.ScriptRelPath.KEY] = 'py.imports' + # Import tests work correctly in this mode, but can slow down by 2-3 times, + # due to the fact that files need to be read from the file system. + # Therefore, we disable them, since the external-py-files mode is designed exclusively + # to speed up the short cycle of developing regular tests. + if unit.get('EXTERNAL_PY_FILES'): + dart_record[df.SkipTest.KEY] = 'Import tests disabled in external-py-files mode' data = dump_test(unit, dart_record) if data: |