diff options
author | robot-piglet <robot-piglet@yandex-team.com> | 2025-02-28 23:59:20 +0300 |
---|---|---|
committer | robot-piglet <robot-piglet@yandex-team.com> | 2025-03-01 00:13:10 +0300 |
commit | b04e2faf41bf366d5f501c976bda00eb32d55660 (patch) | |
tree | f922ace378c0c471d912c33f2c0231144f898d78 /yql/essentials/udfs/common/python/python_udf | |
parent | 9ba742f4d36b4a3d879b4cf8d9234165413f4a0d (diff) | |
download | ydb-b04e2faf41bf366d5f501c976bda00eb32d55660.tar.gz |
Intermediate changes
commit_hash:e2da3ad430fabaa84a74178b1f2103b09ac69ae7
Diffstat (limited to 'yql/essentials/udfs/common/python/python_udf')
-rw-r--r-- | yql/essentials/udfs/common/python/python_udf/python_udf.cpp | 54 | ||||
-rw-r--r-- | yql/essentials/udfs/common/python/python_udf/python_udf.h | 55 |
2 files changed, 56 insertions, 53 deletions
diff --git a/yql/essentials/udfs/common/python/python_udf/python_udf.cpp b/yql/essentials/udfs/common/python/python_udf/python_udf.cpp index b1739a1775e..1007c75edc3 100644 --- a/yql/essentials/udfs/common/python/python_udf/python_udf.cpp +++ b/yql/essentials/udfs/common/python/python_udf/python_udf.cpp @@ -59,59 +59,7 @@ public: InitYqlModule(pythonFlavor, standalone); - const auto rc = PyRun_SimpleString(R"( -# numpy on import may find installed openblas library and load it, -# which in turn causes it to start CPUCOUNT threads -# with approx. 40Mb memory reserved for each thread; -# -# See more detailed explanation here: https://st.yandex-team.ru/STATLIBS-1715#5bfc68ecbbc039001cec572a -# -# Thus, we reduce negative effects as much as possible -import os -os.environ['OPENBLAS_NUM_THREADS'] = '1' - - -# Following part allows us later to format tracebacks via sys.excepthook -# in thread-safe manner -import sys -import threading -if sys.version_info >= (3, 0): - from io import StringIO, TextIOWrapper as SysStderrType -else: - from cStringIO import StringIO - SysStderrType = file - -class StderrLocal(threading.local): - - def __init__(self): - self.is_real_mode = True - self.buffer = StringIO() - - -class StderrProxy(object): - def __init__(self, stderr): - self._stderr = stderr - self._tls = StderrLocal() - - def _toggle_real_mode(self): - self._tls.is_real_mode = not self._tls.is_real_mode - if not self._tls.is_real_mode: - self._tls.buffer.clear() - - def _get_value(self): - assert not self._tls.is_real_mode - return self._tls.buffer.getvalue() - - def __getattr__(self, attr): - target = self._stderr - if not self._tls.is_real_mode: - target = self._tls.buffer - - return getattr(target, attr) - -if isinstance(sys.stderr, SysStderrType): - sys.stderr = StderrProxy(sys.stderr) -)"); + const auto rc = PyRun_SimpleString(STANDART_STREAM_PROXY_INJECTION_SCRIPT); Y_ABORT_UNLESS(rc >= 0, "Can't setup module"); if (pythonFlavor == EPythonFlavor::Arcadia) { diff --git a/yql/essentials/udfs/common/python/python_udf/python_udf.h b/yql/essentials/udfs/common/python/python_udf/python_udf.h index 16d7da096dd..83b3bb86e6f 100644 --- a/yql/essentials/udfs/common/python/python_udf/python_udf.h +++ b/yql/essentials/udfs/common/python/python_udf/python_udf.h @@ -5,6 +5,61 @@ namespace NYql { namespace NUdf { +inline constexpr char STANDART_STREAM_PROXY_INJECTION_SCRIPT[] = +R"( +# numpy on import may find installed openblas library and load it, +# which in turn causes it to start CPUCOUNT threads +# with approx. 40Mb memory reserved for each thread; +# +# See more detailed explanation here: https://st.yandex-team.ru/STATLIBS-1715#5bfc68ecbbc039001cec572a +# +# Thus, we reduce negative effects as much as possible +import os +os.environ['OPENBLAS_NUM_THREADS'] = '1' + + +# Following part allows us later to format tracebacks via sys.excepthook +# in thread-safe manner +import sys +import threading +if sys.version_info >= (3, 0): + from io import StringIO, TextIOWrapper as SysStderrType +else: + from cStringIO import StringIO + SysStderrType = file + +class StderrLocal(threading.local): + + def __init__(self): + self.is_real_mode = True + self.buffer = StringIO() + + +class StderrProxy(object): + def __init__(self, stderr): + self._stderr = stderr + self._tls = StderrLocal() + + def _toggle_real_mode(self): + self._tls.is_real_mode = not self._tls.is_real_mode + if not self._tls.is_real_mode: + self._tls.buffer.clear() + + def _get_value(self): + assert not self._tls.is_real_mode + return self._tls.buffer.getvalue() + + def __getattr__(self, attr): + target = self._stderr + if not self._tls.is_real_mode: + target = self._tls.buffer + + return getattr(target, attr) + +if isinstance(sys.stderr, SysStderrType): + sys.stderr = StderrProxy(sys.stderr) +)"; + enum class EPythonFlavor { System, Arcadia, |