Prototype - external python files mode for tests

Режим позволяет перезапускать python тесты или интеграционные тесты на питон программы при внесении изменений в python файлы без перекомпиляции.

Сценарий:
- прогнали тесты
- внесли изменение в `library/python/func/__init__.py`
- перезапустили, смотрим на времена

На примере интеграционного devtools/ya/handlers/analyze_make/tests в codenv (base конфигурация).

#|
||
ya m -rA
|
ya m -rA --ext-py
||
||
real    10m25,928s
user    0m53,465s
sys     1m14,791s

```
Total time by type:
[935171 ms] [LD] [count: 3, ave time 311723.67 msec]
[ 13657 ms] [prepare:AC] [count: 4, ave time 3414.25 msec]
[ 10436 ms] [TM] [count: 1, ave time 10436.00 msec]
[ 10387 ms] [prepare:get from local cache] [count: 1521, ave time 6.83 msec]
[  6480 ms] [TS] [count: 1, ave time 6480.00 msec]
[  3823 ms] [prepare:put into local cache, clean build dir] [count: 7, ave time 546.14 msec]
[  2279 ms] [CC] [count: 1, ave time 2279.00 msec]
[  1251 ms] [PY] [count: 1, ave time 1251.00 msec]
[   372 ms] [prepare:yt-store] [count: 2, ave time 186.00 msec]
[   154 ms] [AR] [count: 1, ave time 154.00 msec]
...
```

|

real    0m22,094s
user    0m26,372s
sys     0m3,363s

```
Total time by type:
[13140 ms] [TM] [count: 1, ave time 13140.00 msec]
[  351 ms] [prepare:yt-store] [count: 1, ave time 351.00 msec]
[   81 ms] [prepare:tools] [count: 8, ave time 10.12 msec]
[   32 ms] [prepare:AC] [count: 2, ave time 16.00 msec]
[   31 ms] [prepare:get from local cache] [count: 7, ave time 4.43 msec]
[   16 ms] [prepare:clean] [count: 3, ave time 5.33 msec]
```

||
|#

В codenv в base конфигуарции (4cpu) невероятно долго идёт линковка. Тот же самый сценарий, но на mous (54cpu)

#|
||
ya m -rA
|
ya m -rA --ext-py
||
||

real    0m25,435s
user    0m40,242s
sys     0m22,434s

```
Total time by type:
[30958 ms] [prepare:get from local cache] [count: 1521, ave time 20.35 msec]
[16293 ms] [prepare:AC] [count: 4, ave time 4073.25 msec]
[11161 ms] [LD] [count: 3, ave time 3720.33 msec]
[ 4256 ms] [TM] [count: 1, ave time 4256.00 msec]
[ 3529 ms] [TS] [count: 1, ave time 3529.00 msec]
[ 1542 ms] [CC] [count: 1, ave time 1542.00 msec]
[  635 ms] [PY] [count: 1, ave time 635.00 msec]
...
```

|

real    0m18,321s
user    0m21,673s
sys     0m3,204s

```
Total time by type:
[10786 ms] [TM] [count: 1, ave time 10786.00 msec]
[ 1568 ms] [prepare:yt-store] [count: 1, ave time 1568.00 msec]
[  151 ms] [prepare:tools] [count: 8, ave time 18.88 msec]
[   61 ms] [prepare:get from local cache] [count: 7, ave time 8.71 msec]
[   48 ms] [prepare:AC] [count: 2, ave time 24.00 msec]
[   14 ms] [prepare:clean] [count: 3, ave time 4.67 msec]
```

||
|#

В новом режиме тест ожидаемо замедляется, так как все питон программы при каждом старте вынуждены читать файлы с фс (ещё через арк), а потом ещё и компилировать байткод, т.е. холодный запуск
commit_hash:24c5a46a8385d3c065abfb6fc5b40f7ad24bb1cb

5 files changed, 78 insertions, 15 deletions

diff --git a/build/plugins/pybuild.py b/build/plugins/pybuild.py
index 744449d109..229d4b1665 100644
--- a/build/plugins/pybuild.py
+++ b/build/plugins/pybuild.py
@@ -8,7 +8,6 @@ import ymake
 from _common import stripext, rootrel_arc_src, listid, pathid, lazy, get_no_lint_value, ugly_conftest_exception
 
 
-YA_IDE_VENV_VAR = 'YA_IDE_VENV'
 PY_NAMESPACE_PREFIX = 'py/namespace'
 BUILTIN_PROTO = 'builtin_proto'
 DEFAULT_FLAKE8_FILE_PROCESSING_TIME = "1.5"  # in seconds
@@ -37,8 +36,8 @@ def is_extended_source_search_enabled(path, unit):
         return False
     if unit.get('NO_EXTENDED_SOURCE_SEARCH') == 'yes':
         return False
-    # contrib is unfriendly to extended source search
-    if unit.resolve_arc_path(path).startswith('$S/contrib/'):
+    # Python itself and contrib/python are unfriendly to extended source search
+    if unit.resolve_arc_path(path).startswith(('$S/contrib/python', '$S/contrib/tools/python3')):
         return False
     return True
 
@@ -252,7 +251,7 @@ def onpy_srcs(unit, *args):
     with_py = not unit.get('PYBUILD_NO_PY')
     with_pyc = not unit.get('PYBUILD_NO_PYC')
     in_proto_library = unit.get('PY_PROTO') or unit.get('PY3_PROTO')
-    venv = unit.get(YA_IDE_VENV_VAR)
+    external_py_files_mode = unit.get('EXTERNAL_PY_FILES') or unit.get('YA_IDE_VENV')
     need_gazetteer_peerdir = False
     trim = 0
 
@@ -542,10 +541,21 @@ def onpy_srcs(unit, *args):
         if py3:
             mod_list_md5 = md5()
             compress = False
+            resfs_mocks = []
+
             for path, mod in pys:
                 mod_list_md5.update(six.ensure_binary(mod))
-                if not (venv and is_extended_source_search_enabled(path, unit)):
-                    dest = 'py/' + mod.replace('.', '/') + '.py'
+                dest = 'py/' + mod.replace('.', '/') + '.py'
+                # In external_py_files mode we want to build python binaries without embedded python files.
+                # The application will still be able to load them from the file system.
+                # However, the import hook still needs meta information about the file locations to work correctly.
+                # That's why we manually register the files in resfs/src/resfs/file, but don't provide any actual files.
+                # For more info see:
+                # - library/python/runtime_py3
+                # - https://docs.yandex-team.ru/ya-make/manual/python/vars
+                if external_py_files_mode and is_extended_source_search_enabled(path, unit):
+                    resfs_mocks += ['-', 'resfs/src/resfs/file/' + dest + '=' + rootrel_arc_src(path, unit)]
+                else:
                     if with_py:
                         res += ['DEST', dest, path]
                     if with_pyc:
@@ -556,6 +566,9 @@ def onpy_srcs(unit, *args):
                     if not compress and ugly_conftest_exception(path):
                         compress = True
 
+            if resfs_mocks:
+                unit.onresource(['DONT_COMPRESS'] + resfs_mocks)
+
             if py_namespaces:
                 # Note: Add md5 to key to prevent key collision if two or more PY_SRCS() used in the same ya.make
                 ns_res = ['DONT_COMPRESS']
diff --git a/build/plugins/ytest.py b/build/plugins/ytest.py
index 4911034877..ef47c32835 100644
--- a/build/plugins/ytest.py
+++ b/build/plugins/ytest.py
@@ -790,13 +790,21 @@ def onadd_check_py_imports(fields, unit, *args):
     if unit.get("TIDY") == "yes":
         # graph changed for clang_tidy tests
         return
+
     if unit.get('NO_CHECK_IMPORTS_FOR_VALUE').strip() == "":
         return
+
     unit.onpeerdir(['library/python/testing/import_test'])
 
     dart_record = create_dart_record(fields, unit, (), {})
     dart_record[df.TestName.KEY] = 'pyimports'
     dart_record[df.ScriptRelPath.KEY] = 'py.imports'
+    # Import tests work correctly in this mode, but can slow down by 2-3 times,
+    # due to the fact that files need to be read from the file system.
+    # Therefore, we disable them, since the external-py-files mode is designed exclusively
+    # to speed up the short cycle of developing regular tests.
+    if unit.get('EXTERNAL_PY_FILES'):
+        dart_record[df.SkipTest.KEY] = 'Import tests disabled in external-py-files mode'
 
     data = dump_test(unit, dart_record)
     if data:
diff --git a/library/python/runtime_py3/enable_external_py_files/ya.make b/library/python/runtime_py3/enable_external_py_files/ya.make
new file mode 100644
index 0000000000..e302dea808
--- /dev/null
+++ b/library/python/runtime_py3/enable_external_py_files/ya.make
@@ -0,0 +1,5 @@
+LIBRARY()
+
+RESOURCE(- py/conf/ENABLE_EXTERNAL_PY_FILES=1)
+
+END()
diff --git a/library/python/runtime_py3/importer.pxi b/library/python/runtime_py3/importer.pxi
index bd65921840..228a00baee 100644
--- a/library/python/runtime_py3/importer.pxi
+++ b/library/python/runtime_py3/importer.pxi
@@ -36,6 +36,8 @@ def _probe(environ_dict, key, default_value=None):
 py_prefix = b'py/'
 py_prefix_len = len(py_prefix)
 
+EXTERNAL_PY_FILES_MODE = __resource.find(b'py/conf/ENABLE_EXTERNAL_PY_FILES') in (b'1', b'yes')
+
 YA_IDE_VENV = __resource.find(res_ya_ide_venv)
 Y_PYTHON_EXTENDED_SOURCE_SEARCH = _probe(_os.environ, env_extended_source_search) or YA_IDE_VENV
 
@@ -78,12 +80,43 @@ def _init_venv():
     raise RuntimeError(f'{cfg_source_root} key not found in {virtual_conf}')
 
 
+def file_bytes(path):
+    # 'open' is not avaiable yet.
+    with FileIO(path, 'r') as f:
+        return f.read()
+
+
+def _guess_source_root():
+    path, tail = _os.getcwd(), 'start'
+
+    while tail:
+        guidence_file = _path_join(path, '.root.path')
+        if _path_isfile(guidence_file):
+            return file_bytes(guidence_file)
+
+        if _path_isfile(_path_join(path, '.arcadia.root')):
+            return _b(path)
+
+        path, tail = _path_split(path)
+
+
 def _get_source_root():
     env_value = _probe(_os.environ, env_source_root)
-    if env_value or not YA_IDE_VENV:
+    if env_value:
         return env_value
 
-    return _init_venv()
+    if EXTERNAL_PY_FILES_MODE:
+        path = _guess_source_root()
+        if path:
+            return path
+
+        raise RuntimeError(
+            "Cannot find source root: binary is built in external-py-files mode, but no env.var. Y_PYTHON_SOURCE_ROOT is specified. Current working directory: " + _os.getcwd()
+        )
+
+    if YA_IDE_VENV:
+        return _init_venv()
+    return None
 
 
 Y_PYTHON_SOURCE_ROOT = _get_source_root()
@@ -103,12 +136,6 @@ def _print(*xs):
     sys.stderr.write(' '.join(parts) + '\n')
 
 
-def file_bytes(path):
-    # 'open' is not avaiable yet.
-    with FileIO(path, 'r') as f:
-        return f.read()
-
-
 def iter_keys(prefix):
     l = len(prefix)
     for idx in range(__resource.count()):
@@ -118,7 +145,7 @@ def iter_keys(prefix):
 
 
 def iter_py_modules(with_keys=False):
-    for key, path in iter_keys(b'resfs/file/' + py_prefix):
+    for key, path in iter_keys(b'resfs/src/resfs/file/' + py_prefix):
         if path.endswith(b'.py'):  # It may also end with '.pyc'.
             mod = _s(path[:-3].replace(b'/', b'.'))
             if with_keys:
diff --git a/library/python/runtime_py3/ya.make b/library/python/runtime_py3/ya.make
index 0ad7d3e13f..dc97c8e2e0 100644
--- a/library/python/runtime_py3/ya.make
+++ b/library/python/runtime_py3/ya.make
@@ -25,6 +25,12 @@ PY_SRCS(
     sitecustomize.pyx
 )
 
+IF (EXTERNAL_PY_FILES)
+    PEERDIR(
+        library/python/runtime_py3/enable_external_py_files
+    )
+ENDIF()
+
 IF (CYTHON_COVERAGE)
     # Let covarage support add all needed files to resources
 ELSE()
@@ -39,6 +45,10 @@ ENDIF()
 
 END()
 
+RECURSE(
+    enable_external_py_files
+)
+
 RECURSE_FOR_TESTS(
     test
 )