aboutsummaryrefslogtreecommitdiffstats
path: root/yql/essentials/udfs/common/python/python_udf
diff options
context:
space:
mode:
authorrobot-piglet <robot-piglet@yandex-team.com>2025-02-28 23:59:20 +0300
committerrobot-piglet <robot-piglet@yandex-team.com>2025-03-01 00:13:10 +0300
commitb04e2faf41bf366d5f501c976bda00eb32d55660 (patch)
treef922ace378c0c471d912c33f2c0231144f898d78 /yql/essentials/udfs/common/python/python_udf
parent9ba742f4d36b4a3d879b4cf8d9234165413f4a0d (diff)
downloadydb-b04e2faf41bf366d5f501c976bda00eb32d55660.tar.gz
Intermediate changes
commit_hash:e2da3ad430fabaa84a74178b1f2103b09ac69ae7
Diffstat (limited to 'yql/essentials/udfs/common/python/python_udf')
-rw-r--r--yql/essentials/udfs/common/python/python_udf/python_udf.cpp54
-rw-r--r--yql/essentials/udfs/common/python/python_udf/python_udf.h55
2 files changed, 56 insertions, 53 deletions
diff --git a/yql/essentials/udfs/common/python/python_udf/python_udf.cpp b/yql/essentials/udfs/common/python/python_udf/python_udf.cpp
index b1739a1775e..1007c75edc3 100644
--- a/yql/essentials/udfs/common/python/python_udf/python_udf.cpp
+++ b/yql/essentials/udfs/common/python/python_udf/python_udf.cpp
@@ -59,59 +59,7 @@ public:
InitYqlModule(pythonFlavor, standalone);
- const auto rc = PyRun_SimpleString(R"(
-# numpy on import may find installed openblas library and load it,
-# which in turn causes it to start CPUCOUNT threads
-# with approx. 40Mb memory reserved for each thread;
-#
-# See more detailed explanation here: https://st.yandex-team.ru/STATLIBS-1715#5bfc68ecbbc039001cec572a
-#
-# Thus, we reduce negative effects as much as possible
-import os
-os.environ['OPENBLAS_NUM_THREADS'] = '1'
-
-
-# Following part allows us later to format tracebacks via sys.excepthook
-# in thread-safe manner
-import sys
-import threading
-if sys.version_info >= (3, 0):
- from io import StringIO, TextIOWrapper as SysStderrType
-else:
- from cStringIO import StringIO
- SysStderrType = file
-
-class StderrLocal(threading.local):
-
- def __init__(self):
- self.is_real_mode = True
- self.buffer = StringIO()
-
-
-class StderrProxy(object):
- def __init__(self, stderr):
- self._stderr = stderr
- self._tls = StderrLocal()
-
- def _toggle_real_mode(self):
- self._tls.is_real_mode = not self._tls.is_real_mode
- if not self._tls.is_real_mode:
- self._tls.buffer.clear()
-
- def _get_value(self):
- assert not self._tls.is_real_mode
- return self._tls.buffer.getvalue()
-
- def __getattr__(self, attr):
- target = self._stderr
- if not self._tls.is_real_mode:
- target = self._tls.buffer
-
- return getattr(target, attr)
-
-if isinstance(sys.stderr, SysStderrType):
- sys.stderr = StderrProxy(sys.stderr)
-)");
+ const auto rc = PyRun_SimpleString(STANDART_STREAM_PROXY_INJECTION_SCRIPT);
Y_ABORT_UNLESS(rc >= 0, "Can't setup module");
if (pythonFlavor == EPythonFlavor::Arcadia) {
diff --git a/yql/essentials/udfs/common/python/python_udf/python_udf.h b/yql/essentials/udfs/common/python/python_udf/python_udf.h
index 16d7da096dd..83b3bb86e6f 100644
--- a/yql/essentials/udfs/common/python/python_udf/python_udf.h
+++ b/yql/essentials/udfs/common/python/python_udf/python_udf.h
@@ -5,6 +5,61 @@
namespace NYql {
namespace NUdf {
+inline constexpr char STANDART_STREAM_PROXY_INJECTION_SCRIPT[] =
+R"(
+# numpy on import may find installed openblas library and load it,
+# which in turn causes it to start CPUCOUNT threads
+# with approx. 40Mb memory reserved for each thread;
+#
+# See more detailed explanation here: https://st.yandex-team.ru/STATLIBS-1715#5bfc68ecbbc039001cec572a
+#
+# Thus, we reduce negative effects as much as possible
+import os
+os.environ['OPENBLAS_NUM_THREADS'] = '1'
+
+
+# Following part allows us later to format tracebacks via sys.excepthook
+# in thread-safe manner
+import sys
+import threading
+if sys.version_info >= (3, 0):
+ from io import StringIO, TextIOWrapper as SysStderrType
+else:
+ from cStringIO import StringIO
+ SysStderrType = file
+
+class StderrLocal(threading.local):
+
+ def __init__(self):
+ self.is_real_mode = True
+ self.buffer = StringIO()
+
+
+class StderrProxy(object):
+ def __init__(self, stderr):
+ self._stderr = stderr
+ self._tls = StderrLocal()
+
+ def _toggle_real_mode(self):
+ self._tls.is_real_mode = not self._tls.is_real_mode
+ if not self._tls.is_real_mode:
+ self._tls.buffer.clear()
+
+ def _get_value(self):
+ assert not self._tls.is_real_mode
+ return self._tls.buffer.getvalue()
+
+ def __getattr__(self, attr):
+ target = self._stderr
+ if not self._tls.is_real_mode:
+ target = self._tls.buffer
+
+ return getattr(target, attr)
+
+if isinstance(sys.stderr, SysStderrType):
+ sys.stderr = StderrProxy(sys.stderr)
+)";
+
enum class EPythonFlavor {
System,
Arcadia,