[devtools/ya] Dont' write __pycache__ to the arcadia in external-py-files mode

В репозитории больше не будет `__pycache__` при запусках через ya make / test

На примере devtools/ya/handlers/analyze_make/tests
Время конфигурации в горячем сценарии в режиме --ext-py при изменении одного питонфайлика: `Configure time - 6.1 s`
Время конфигурации в горячем сценарии в режиме --ext-py при изменении одного питонфайлика с текущими правками: `Configure time - 4.6 s`
commit_hash:adf5219cd4f347bb572393d4a6debd1436dc0815

1 files changed, 29 insertions, 1 deletions

diff --git a/library/python/runtime_py3/importer.pxi b/library/python/runtime_py3/importer.pxi
index f4f32523067..f2cf1f9f40e 100644
--- a/library/python/runtime_py3/importer.pxi
+++ b/library/python/runtime_py3/importer.pxi
@@ -114,7 +114,7 @@ def _guess_source_root():
 def _get_source_root():
     env_value = _probe(_os.environ, env_source_root)
     if env_value:
-        return env_value
+        return _s(env_value)
 
     if EXTERNAL_PY_FILES_MODE:
         path = _guess_source_root()
@@ -130,6 +130,21 @@ def _get_source_root():
     return None
 
 
+def _guess_pycache_prefix():
+    env_val = _probe(_os.environ, 'PYTHONPYCACHEPREFIX')
+    if env_val:
+        return _s(env_val)
+
+    path, tail = _os.getcwd(), 'start'
+
+    while tail:
+        guidence_file = _path_join(path, '.pycache.path')
+        if _path_isfile(guidence_file):
+            return _s(file_bytes(guidence_file))
+
+        path, tail = _path_split(path)
+
+
 Y_PYTHON_SOURCE_ROOT = _get_source_root()
 
 if EXTERNAL_PY_FILES_MODE:
@@ -143,6 +158,10 @@ if EXTERNAL_PY_FILES_MODE:
 
     setattr(_frozen_importlib_external, 'cache_from_source', patched_cache_from_source)
 
+    pycache_prefix = _guess_pycache_prefix()
+    if pycache_prefix:
+        sys.pycache_prefix = pycache_prefix
+
 
 def _print(*xs):
     """
@@ -359,11 +378,20 @@ class ResourceImporter(SourceFileLoader):
 
     # PEP-302 extension 1 of 3: data loader.
     def get_data(self, path):
+        # XXX
+        # Python machinery operates on absolute paths and uses this method to load bytecode.
+        # That's why we don't try to resolve the path, but try to read it right away.
+        # For more info see get_code() in this file and
+        # data flow in SourceLoader.get_data in contrib/tools/python3/Lib/importlib/_bootstrap_external.py
+        if EXTERNAL_PY_FILES_MODE and path.endswith('.pyc'):
+            return file_bytes(_b(path))
+
         path = _b(path)
         abspath = resfs_resolve(path)
 
         if abspath:
             return file_bytes(abspath)
+
         path = path.replace(_b('\\'), _b('/'))
         data = resfs_read(path, builtin=True)
         if data is None: