Subinterpretor compatible __res module (2nd attempt)

Cython is not yet subinterpreter compatible. There are no ETA when cython is going to support subinterpreters.

This PR removes cython from hermetic python imoprt hooks in order to make them subinterpretr-compatible.
commit_hash:427b6f9db6afa6695659ee147621e1ccb391d3cb

6 files changed, 190 insertions, 2 deletions

diff --git a/library/python/runtime_py3/test/subinterpreter/py3_subinterpreters.cpp b/library/python/runtime_py3/test/subinterpreter/py3_subinterpreters.cpp
new file mode 100644
index 00000000000..0a934d4db50
--- /dev/null
+++ b/library/python/runtime_py3/test/subinterpreter/py3_subinterpreters.cpp
@@ -0,0 +1,82 @@
+#include "stdout_interceptor.h"
+
+#include <util/stream/str.h>
+
+#include <library/cpp/testing/gtest/gtest.h>
+
+#include <Python.h>
+
+#include <thread>
+#include <algorithm>
+
+struct TSubinterpreters: ::testing::Test {
+    static void SetUpTestSuite() {
+        Py_InitializeEx(0);
+        EXPECT_TRUE(TPyStdoutInterceptor::SetupInterceptionSupport());
+    }
+    static void TearDownTestSuite() {
+        Py_Finalize();
+    }
+
+    static void ThreadPyRun(PyInterpreterState* interp, IOutputStream& pyout, const char* pycode) {
+        PyThreadState* state = PyThreadState_New(interp);
+        PyEval_RestoreThread(state);
+
+        {
+            TPyStdoutInterceptor interceptor{pyout};
+            PyRun_SimpleString(pycode);
+        }
+
+        PyThreadState_Clear(state);
+        PyThreadState_DeleteCurrent();
+    }
+};
+
+TEST_F(TSubinterpreters, NonSubinterpreterFlowStillWorks) {
+    TStringStream pyout;
+    TPyStdoutInterceptor interceptor{pyout};
+
+    PyRun_SimpleString("print('Hello World')");
+    EXPECT_EQ(pyout.Str(), "Hello World\n");
+}
+
+TEST_F(TSubinterpreters, ThreadedSubinterpretersFlowWorks) {
+    TStringStream pyout[2];
+
+    PyInterpreterConfig cfg = {
+        .use_main_obmalloc = 0,
+        .allow_fork = 0,
+        .allow_exec = 0,
+        .allow_threads = 1,
+        .allow_daemon_threads = 0,
+        .check_multi_interp_extensions = 1,
+        .gil = PyInterpreterConfig_OWN_GIL,
+    };
+
+    PyThreadState* mainState = PyThreadState_Get();
+    PyThreadState *sub[2] = {nullptr, nullptr};
+    Py_NewInterpreterFromConfig(&sub[0], &cfg);
+    ASSERT_NE(sub[0], nullptr);
+    Py_NewInterpreterFromConfig(&sub[1], &cfg);
+    ASSERT_NE(sub[1], nullptr);
+    PyThreadState_Swap(mainState);
+
+    PyThreadState* savedState = PyEval_SaveThread();
+    std::array<std::thread, 2> threads{
+        std::thread{ThreadPyRun, sub[0]->interp, std::ref(pyout[0]), "print('Hello Thread 0')"},
+        std::thread{ThreadPyRun, sub[1]->interp, std::ref(pyout[1]), "print('Hello Thread 1')"}
+    };
+    std::ranges::for_each(threads, &std::thread::join);
+    PyEval_RestoreThread(savedState);
+
+    PyThreadState_Swap(sub[0]);
+    Py_EndInterpreter(sub[0]);
+
+    PyThreadState_Swap(sub[1]);
+    Py_EndInterpreter(sub[1]);
+
+    PyThreadState_Swap(mainState);
+
+    EXPECT_EQ(pyout[0].Str(), "Hello Thread 0\n");
+    EXPECT_EQ(pyout[1].Str(), "Hello Thread 1\n");
+}
diff --git a/library/python/runtime_py3/test/subinterpreter/stdout_interceptor.cpp b/library/python/runtime_py3/test/subinterpreter/stdout_interceptor.cpp
new file mode 100644
index 00000000000..3cd4b69d012
--- /dev/null
+++ b/library/python/runtime_py3/test/subinterpreter/stdout_interceptor.cpp
@@ -0,0 +1,77 @@
+#include "stdout_interceptor.h"
+
+#include <util/stream/output.h>
+
+namespace {
+
+struct TOStreamWrapper {
+    PyObject_HEAD
+    IOutputStream* Stm = nullptr;
+};
+
+PyObject* Write(TOStreamWrapper *self, PyObject *const *args, Py_ssize_t nargs) noexcept {
+    try {
+        Py_buffer view;
+        for (Py_ssize_t i = 0; i < nargs; ++i) {
+            PyObject* buf = args[i];
+            if (PyUnicode_Check(args[i])) {
+                buf = PyUnicode_AsUTF8String(buf);
+                if (!buf) {
+                    return nullptr;
+                }
+            }
+
+            if (PyObject_GetBuffer(buf, &view, PyBUF_SIMPLE | PyBUF_C_CONTIGUOUS) == -1) {
+                return nullptr;
+            }
+            self->Stm->Write(reinterpret_cast<const char*>(view.buf), view.len);
+            PyBuffer_Release(&view);
+        }
+
+        return Py_None;
+    } catch(const std::exception& err) {
+        PyErr_SetString(PyExc_IOError, err.what());
+    } catch (...) {
+        PyErr_SetString(PyExc_RuntimeError, "Unhandled C++ exception of unknown type");
+    }
+    return nullptr;
+}
+
+PyMethodDef TOStreamWrapperMethods[] = {
+    {"write", reinterpret_cast<PyCFunction>(Write), METH_FASTCALL, PyDoc_STR("write buffer to wrapped C++ stream")},
+    {}
+};
+
+PyTypeObject TOStreamWrapperType {
+    .ob_base = PyVarObject_HEAD_INIT(NULL, 0)
+    .tp_name = "testwrap.OStream",
+    .tp_basicsize = sizeof(TOStreamWrapper),
+    .tp_itemsize = 0,
+    .tp_flags = Py_TPFLAGS_DEFAULT,
+    .tp_doc = PyDoc_STR("C++ IOStream wrapper"),
+    .tp_methods = TOStreamWrapperMethods,
+    .tp_new = PyType_GenericNew,
+};
+
+}
+
+TPyStdoutInterceptor::TPyStdoutInterceptor(IOutputStream& redirectionStream) noexcept
+    : RealStdout_{PySys_GetObject("stdout")}
+{
+    Py_INCREF(RealStdout_);
+
+    PyObject* redirect = TOStreamWrapperType.tp_alloc(&TOStreamWrapperType, 0);
+    reinterpret_cast<TOStreamWrapper*>(redirect)->Stm = &redirectionStream;
+
+    PySys_SetObject("stdout", redirect);
+    Py_DECREF(redirect);
+}
+
+TPyStdoutInterceptor::~TPyStdoutInterceptor() noexcept {
+    PySys_SetObject("stdout", RealStdout_);
+    Py_DECREF(RealStdout_);
+}
+
+bool TPyStdoutInterceptor::SetupInterceptionSupport() noexcept {
+    return PyType_Ready(&TOStreamWrapperType) == 0;
+}
diff --git a/library/python/runtime_py3/test/subinterpreter/stdout_interceptor.h b/library/python/runtime_py3/test/subinterpreter/stdout_interceptor.h
new file mode 100644
index 00000000000..a1e219953f0
--- /dev/null
+++ b/library/python/runtime_py3/test/subinterpreter/stdout_interceptor.h
@@ -0,0 +1,16 @@
+#pragma once
+
+#include <Python.h>
+
+class IOutputStream;
+
+class TPyStdoutInterceptor {
+public:
+    TPyStdoutInterceptor(IOutputStream& redirectionStream) noexcept;
+    ~TPyStdoutInterceptor() noexcept;
+
+    static bool SetupInterceptionSupport() noexcept;
+
+private:
+    PyObject* RealStdout_;
+};
diff --git a/library/python/runtime_py3/test/subinterpreter/ya.make b/library/python/runtime_py3/test/subinterpreter/ya.make
new file mode 100644
index 00000000000..78cc82304c8
--- /dev/null
+++ b/library/python/runtime_py3/test/subinterpreter/ya.make
@@ -0,0 +1,10 @@
+GTEST()
+
+USE_PYTHON3()
+
+SRCS(
+    py3_subinterpreters.cpp
+    stdout_interceptor.cpp
+)
+
+END()
diff --git a/library/python/runtime_py3/test/test_arcadia_source_finder.py b/library/python/runtime_py3/test/test_arcadia_source_finder.py
index 9f794f03591..835e60c6710 100644
--- a/library/python/runtime_py3/test/test_arcadia_source_finder.py
+++ b/library/python/runtime_py3/test/test_arcadia_source_finder.py
@@ -18,7 +18,7 @@ class ImporterMocks:
         self._mock_resources = mock_resources
         self._patchers = [
             patch("__res.iter_keys", wraps=self._iter_keys),
-            patch("__res.__resource.find", wraps=self._resource_find),
+            patch("__res.find", wraps=self._resource_find),
             patch("__res._path_isfile", wraps=self._path_isfile),
             patch("__res._os.listdir", wraps=self._os_listdir),
             patch("__res._os.lstat", wraps=self._os_lstat),
diff --git a/library/python/runtime_py3/test/ya.make b/library/python/runtime_py3/test/ya.make
index e0c4061ad2c..fde64236dca 100644
--- a/library/python/runtime_py3/test/ya.make
+++ b/library/python/runtime_py3/test/ya.make
@@ -34,4 +34,7 @@ RESOURCE_FILES(
 
 END()
 
-RECURSE_FOR_TESTS(traceback)
+RECURSE_FOR_TESTS(
+    subinterpreter
+    traceback
+)