aboutsummaryrefslogtreecommitdiffstats
path: root/library
diff options
context:
space:
mode:
authorsay <say@yandex-team.com>2023-12-08 17:40:48 +0300
committersay <say@yandex-team.com>2023-12-08 19:58:59 +0300
commit914f57e3243f53dd89dd3adb4d8b6d35c47f46ce (patch)
tree98a1f1f1f5e2c38db3a78da10aeb7eb7d4e952e0 /library
parente61293d91ee7c923944f627d8e1138bcb17cacad (diff)
downloadydb-914f57e3243f53dd89dd3adb4d8b6d35c47f46ce.tar.gz
Make support_retries() and get_type() methods of object instead of class ones. Reduce get_type_name() usage.
Diffstat (limited to 'library')
-rw-r--r--library/cpp/pybind/attr.h412
-rw-r--r--library/cpp/pybind/cast.cpp324
-rw-r--r--library/cpp/pybind/cast.h373
-rw-r--r--library/cpp/pybind/embedding.cpp63
-rw-r--r--library/cpp/pybind/embedding.h10
-rw-r--r--library/cpp/pybind/empty.cpp2
-rw-r--r--library/cpp/pybind/exceptions.cpp147
-rw-r--r--library/cpp/pybind/exceptions.h143
-rw-r--r--library/cpp/pybind/init.h25
-rw-r--r--library/cpp/pybind/method.h439
-rw-r--r--library/cpp/pybind/module.cpp72
-rw-r--r--library/cpp/pybind/module.h176
-rw-r--r--library/cpp/pybind/pod.cpp18
-rw-r--r--library/cpp/pybind/pod.h53
-rw-r--r--library/cpp/pybind/ptr.h51
-rw-r--r--library/cpp/pybind/typeattrs.h368
-rw-r--r--library/cpp/pybind/typedesc.cpp79
-rw-r--r--library/cpp/pybind/typedesc.h545
-rw-r--r--library/cpp/pybind/v2.cpp43
-rw-r--r--library/cpp/pybind/v2.h514
-rw-r--r--library/cpp/pybind/ya.make14
-rw-r--r--library/python/archive/__init__.py266
-rw-r--r--library/python/archive/ya.make19
-rw-r--r--library/python/cityhash/cityhash.pyx75
-rw-r--r--library/python/cityhash/hash.cpp32
-rw-r--r--library/python/cityhash/hash.h6
-rw-r--r--library/python/cityhash/ya.make16
-rw-r--r--library/python/codecs/__codecs.pyx61
-rw-r--r--library/python/codecs/__init__.py1
-rw-r--r--library/python/codecs/ya.make16
-rw-r--r--library/python/color/README.md9
-rw-r--r--library/python/color/__init__.py92
-rw-r--r--library/python/color/ya.make13
-rw-r--r--library/python/compress/__init__.py147
-rw-r--r--library/python/compress/ya.make16
-rw-r--r--library/python/coredump_filter/README.md19
-rw-r--r--library/python/coredump_filter/__init__.py1500
-rw-r--r--library/python/coredump_filter/core_proc.js21
-rw-r--r--library/python/coredump_filter/epilog.html2
-rw-r--r--library/python/coredump_filter/prolog.html24
-rw-r--r--library/python/coredump_filter/styles.css116
-rw-r--r--library/python/coredump_filter/ya.make23
-rw-r--r--library/python/json/__init__.py44
-rw-r--r--library/python/json/loads.cpp246
-rw-r--r--library/python/json/loads.h5
-rw-r--r--library/python/json/loads.pyx14
-rw-r--r--library/python/json/ya.make17
-rw-r--r--library/python/par_apply/__init__.py114
-rw-r--r--library/python/par_apply/ya.make11
49 files changed, 0 insertions, 6796 deletions
diff --git a/library/cpp/pybind/attr.h b/library/cpp/pybind/attr.h
deleted file mode 100644
index 5f25a6d73d..0000000000
--- a/library/cpp/pybind/attr.h
+++ /dev/null
@@ -1,412 +0,0 @@
-#pragma once
-
-#define PY_SSIZE_T_CLEAN
-#include <Python.h>
-#include <util/generic/string.h>
-#include <util/generic/map.h>
-#include <util/generic/set.h>
-#include <util/generic/vector.h>
-#include <util/generic/ptr.h>
-
-#include "cast.h"
-#include "exceptions.h"
-
-namespace NPyBind {
- // TBaseAttrGetter
- template <typename TObjType>
- class TBaseAttrGetter {
- public:
- virtual ~TBaseAttrGetter() {
- }
- virtual bool GetAttr(PyObject* owner, const TObjType& self, const TString& attr, PyObject*& res) const = 0;
-
- virtual bool HasAttr(PyObject* owner, const TObjType& self, const TString& attr, const TSet<TString>& hiddenNames) const {
- if (hiddenNames.find(attr) != hiddenNames.end())
- return false;
- PyObject* res = nullptr;
- if (!GetAttr(owner, self, attr, res))
- return false;
- Py_XDECREF(res);
- return true;
- }
- };
-
- template <typename TObjType>
- class TBaseAttrSetter {
- public:
- virtual ~TBaseAttrSetter() {
- }
-
- virtual bool SetAttr(PyObject* owner, TObjType& self, const TString& attr, PyObject* val) = 0;
- };
-
- template <typename TObjType>
- class TAttrGetters {
- public:
- typedef TSimpleSharedPtr<TBaseAttrGetter<TObjType>> TGetterPtr;
-
- private:
- typedef TVector<TGetterPtr> TGetterList;
- typedef TMap<TString, TGetterList> TGetterMap;
-
- const TSet<TString>& HiddenAttrNames;
- TGetterMap Getters;
-
- public:
- TAttrGetters(const TSet<TString>& hiddenNames)
- : HiddenAttrNames(hiddenNames)
- {
- }
-
- void AddGetter(const TString& attr, TGetterPtr getter) {
- Getters[attr].push_back(getter);
- }
-
- PyObject* GetAttr(PyObject* owner, const TObjType& self, const TString& attr) const {
- typename TGetterMap::const_iterator it1 = Getters.find(attr);
- if (it1 == Getters.end())
- it1 = Getters.find("");
- if (it1 == Getters.end())
- return nullptr;
- const TGetterList& lst = it1->second;
- for (typename TGetterList::const_iterator it2 = lst.begin(), end = lst.end(); it2 != end; ++it2) {
- PyObject* res = nullptr;
- if ((*it2)->GetAttr(owner, self, attr, res))
- return res;
- // IMPORTANT!
- // we have to fail GetAttr right there because we've failed because of internal python error/exception and can't continue iterating because
- // it cause subsequent exceptions during call to Py_BuildValue
- // moreover we have to preserve original exception right there
- if (PyErr_Occurred()) {
- break;
- }
- }
- return nullptr;
- }
-
- bool HasAttr(PyObject* owner, const TObjType& self, const TString& attr) const {
- typename TGetterMap::const_iterator it1 = Getters.find(attr);
- if (it1 == Getters.end())
- return false;
- const TGetterList& lst = it1->second;
- for (typename TGetterList::const_iterator it2 = lst.begin(), end = lst.end(); it2 != end; ++it2) {
- if ((*it2)->HasAttr(owner, self, attr, HiddenAttrNames))
- return true;
- }
- return false;
- }
-
- void GetAttrsDictionary(PyObject* owner, const TObjType& self, TMap<TString, PyObject*>& res) const {
- for (typename TGetterMap::const_iterator it = Getters.begin(), end = Getters.end(); it != end; ++it) {
- try {
- if (HasAttr(owner, self, it->first)) {
- auto attrPtr = GetAttr(owner, self, it->first);
- if (attrPtr) {
- res[it->first] = attrPtr;
- }
- if (PyErr_Occurred()) {
- PyErr_Clear(); // Skip python errors as well
- }
- }
- } catch (const std::exception&) {
- // ignore this field
- }
- }
- }
-
- void GetAttrsNames(PyObject* owner, const TObjType& self, TVector<TString>& resultNames) const {
- for (typename TGetterMap::const_iterator it = Getters.begin(), end = Getters.end(); it != end; ++it) {
- if (HasAttr(owner, self, it->first))
- resultNames.push_back(it->first);
- }
- }
- };
-
- template <typename TObjType>
- class TGenericAttrGetter: public TBaseAttrGetter<TObjType> {
- private:
- TString AttrName;
-
- public:
- TGenericAttrGetter(const TString& attrName)
- : AttrName(attrName)
- {
- }
-
- bool GetAttr(PyObject* obj, const TObjType&, const TString&, PyObject*& res) const override {
- auto str = NameFromString(AttrName);
- res = PyObject_GenericGetAttr(obj, str.Get());
- if (!res && !PyErr_Occurred())
- ythrow TPyErr(PyExc_AttributeError) << "Can't get generic attribute '" << AttrName << "'";
- return res;
- }
- };
-
- template <typename TObjType>
- class TAttrSetters {
- private:
- typedef TSimpleSharedPtr<TBaseAttrSetter<TObjType>> TSetterPtr;
- typedef TVector<TSetterPtr> TSetterList;
- typedef TMap<TString, TSetterList> TSetterMap;
-
- TSetterMap Setters;
-
- public:
- void AddSetter(const TString& attr, TSetterPtr setter) {
- Setters[attr].push_back(setter);
- }
-
- bool SetAttr(PyObject* owner, TObjType& self, const TString& attr, PyObject* val) {
- typename TSetterMap::const_iterator it1 = Setters.find(attr);
- if (it1 == Setters.end())
- it1 = Setters.find("");
- if (it1 == Setters.end())
- return false;
- const TSetterList& lst = it1->second;
- for (typename TSetterList::const_iterator it2 = lst.begin(), end = lst.end(); it2 != end; ++it2) {
- if ((*it2)->SetAttr(owner, self, attr, val))
- return true;
- }
- return false;
- }
-
- bool SetAttrDictionary(PyObject* owner, TObjType& self, TMap<TString, PyObject*>& dict) {
- for (TMap<TString, PyObject*>::const_iterator it = dict.begin(), end = dict.end(); it != end; ++it) {
- try {
- SetAttr(owner, self, it->first, it->second);
- } catch (std::exception&) {
- // ignore this field
- }
- }
-
- return true;
- }
- };
-
- /**
- * TMethodAttrGetter - this class maps Python attribute read to C++ method call
- */
- template <typename TObjType, typename TResult, typename TSubObject>
- class TMethodAttrGetter: public TBaseAttrGetter<TObjType> {
- private:
- typedef TResult (TSubObject::*TMethod)() const;
- TMethod Method;
-
- public:
- TMethodAttrGetter(TMethod method)
- : Method(method)
- {
- }
-
- bool GetAttr(PyObject*, const TObjType& self, const TString&, PyObject*& res) const override {
- const TSubObject* sub = dynamic_cast<const TSubObject*>(&self);
- if (sub == nullptr)
- return false;
- res = BuildPyObject((sub->*Method)());
- return (res != nullptr);
- }
- };
-
- template <typename TObjType, typename TFunctor>
- class TFunctorAttrGetter: public TBaseAttrGetter<TObjType> {
- TFunctor Functor;
- public:
- explicit TFunctorAttrGetter(TFunctor functor)
- : Functor(functor)
- {
- }
-
- bool GetAttr(PyObject*, const TObjType& self, const TString&, PyObject*& res) const override {
- res = BuildPyObject(Functor(self));
- return (res != nullptr);
- }
- };
-
-
- /**
- * TMethodAttrGetterWithCheck - this class maps Python attribute read to C++ HasAttr/GetAttr call
- * If HasAttr returns false, None is returned.
- * Otherwise GetAttr is called.
- */
- template <typename TObjType, typename TResult, typename TSubObject>
- class TMethodAttrGetterWithCheck: public TBaseAttrGetter<TObjType> {
- private:
- typedef TResult (TSubObject::*TMethod)() const;
- typedef bool (TSubObject::*TCheckerMethod)() const;
- TMethod Method;
- TCheckerMethod CheckerMethod;
-
- public:
- TMethodAttrGetterWithCheck(TMethod method, TCheckerMethod checkerMethod)
- : Method(method)
- , CheckerMethod(checkerMethod)
- {
- }
-
- bool GetAttr(PyObject*, const TObjType& self, const TString&, PyObject*& res) const override {
- const TSubObject* sub = dynamic_cast<const TSubObject*>(&self);
- if (sub == nullptr)
- return false;
- if ((sub->*CheckerMethod)())
- res = BuildPyObject((sub->*Method)());
- else {
- Py_INCREF(Py_None);
- res = Py_None;
- }
- return (res != nullptr);
- }
- };
-
- template <typename TObjType, typename TResult, typename TSubObject, typename TMapper>
- class TMethodAttrMappingGetter: public TBaseAttrGetter<TObjType> {
- private:
- typedef TResult (TSubObject::*TMethod)() const;
-
- TMethod Method;
- TMapper Mapper;
-
- public:
- TMethodAttrMappingGetter(TMethod method, TMapper mapper)
- : Method(method)
- , Mapper(mapper)
- {
- }
-
- bool GetAttr(PyObject*, const TObjType& self, const TString&, PyObject*& res) const override {
- const TSubObject* sub = dynamic_cast<const TSubObject*>(&self);
- if (sub == nullptr)
- return false;
- res = BuildPyObject(Mapper((sub->*Method)()));
- return (res != nullptr);
- }
- };
-
- template <typename TObjType, typename TResult, typename TSubObject, typename TMapper>
- TSimpleSharedPtr<TBaseAttrGetter<TObjType>>
- CreateMethodAttrMappingGetter(TResult (TSubObject::*method)() const,
- TMapper mapper) {
- return new TMethodAttrMappingGetter<TObjType, TResult, TSubObject, TMapper>(method,
- mapper);
- }
-
- template <typename TObjType, typename TResult, typename TValue, typename TSubObject>
- class TMethodAttrSetter: public TBaseAttrSetter<TObjType> {
- private:
- typedef TResult (TSubObject::*TMethod)(TValue&);
- TMethod Method;
-
- public:
- TMethodAttrSetter(TMethod method)
- : Method(method)
- {
- }
-
- virtual bool SetAttr(PyObject*, TObjType& self, const TString&, PyObject* val) {
- TSubObject* sub = dynamic_cast<TSubObject*>(&self);
- if (sub == nullptr)
- return false;
- TValue value;
- if (!FromPyObject(val, value))
- return false;
- (sub->*Method)(value);
- return true;
- }
- };
-
- template <typename TObjType, typename TValue, typename TFunctor>
- class TFunctorAttrSetter: public TBaseAttrSetter<TObjType> {
- TFunctor Functor;
- public:
- explicit TFunctorAttrSetter(TFunctor functor)
- : Functor(functor)
- {
- }
-
- bool SetAttr(PyObject*, TObjType& self, const TString&, PyObject* val) const override {
- TValue value;
- if (!FromPyObject(val, value))
- return false;
- auto res = BuildPyObject(Functor(self, value));
- return (res != nullptr);
- }
- };
- template <typename TObjType, typename TResult, typename TSubObject>
- TSimpleSharedPtr<TBaseAttrGetter<TObjType>> CreateMethodAttrGetter(TResult (TSubObject::*method)() const) {
- return new TMethodAttrGetter<TObjType, TResult, TSubObject>(method);
- }
-
- template <typename TObjType, typename TFunctor>
- TSimpleSharedPtr<TFunctorAttrGetter<TObjType, TFunctor>> CreateFunctorAttrGetter(TFunctor functor) {
- return MakeSimpleShared<TFunctorAttrGetter<TObjType, TFunctor>>(functor);
- }
-
- template <typename TObjType, typename TResult, typename TSubObject>
- TSimpleSharedPtr<TBaseAttrGetter<TObjType>> CreateMethodAttrGetterWithCheck(
- TResult (TSubObject::*method)() const,
- bool (TSubObject::*checkerMethod)() const) {
- return new TMethodAttrGetterWithCheck<TObjType, TResult, TSubObject>(method, checkerMethod);
- }
-
- template <typename TObjType, typename TResult, typename TValue, typename TSubObject>
- TSimpleSharedPtr<TBaseAttrSetter<TObjType>> CreateMethodAttrSetter(TResult (TSubObject::*method)(TValue&)) {
- return new TMethodAttrSetter<TObjType, TResult, TValue, TSubObject>(method);
- }
-
- template <typename TObjType, typename TFunctor, typename TValue>
- TSimpleSharedPtr<TFunctorAttrSetter<TObjType, TValue, TFunctor>> CreateFunctorAttrSetter(TFunctor functor) {
- return MakeSimpleShared<TFunctorAttrSetter<TObjType, TValue, TFunctor>>(functor);
- }
-
- template <typename TObjType, typename TValue, typename TSubObject>
- class TDirectAttrSetter: public TBaseAttrSetter<TObjType> {
- private:
- typedef TValue TSubObject::*TValueType;
- TValueType Value;
-
- public:
- TDirectAttrSetter(TValueType value)
- : Value(value)
- {
- }
-
- bool SetAttr(PyObject*, TObjType& self, const TString&, PyObject* val) override {
- TSubObject* sub = dynamic_cast<TSubObject*>(&self);
- if (sub == NULL)
- return false;
- if (!FromPyObject(val, sub->*Value))
- return false;
- return true;
- }
- };
-
- template <typename TObjType, typename TValue, typename TSubObject>
- TSimpleSharedPtr<TBaseAttrSetter<TObjType>> CreateAttrSetter(TValue TSubObject::*value) {
- return new TDirectAttrSetter<TObjType, TValue, TSubObject>(value);
- }
-
- template <typename TObjType, typename TValue, typename TSubObject>
- class TDirectAttrGetter: public TBaseAttrGetter<TObjType> {
- private:
- typedef TValue TSubObject::*TValueType;
- TValueType Value;
-
- public:
- TDirectAttrGetter(TValueType value)
- : Value(value)
- {
- }
-
- bool GetAttr(PyObject*, const TObjType& self, const TString&, PyObject*& res) const override {
- const TSubObject* sub = dynamic_cast<const TSubObject*>(&self);
- if (sub == nullptr)
- return false;
- res = BuildPyObject(sub->*Value);
- return (res != nullptr);
- }
- };
-
- template <typename TObjType, typename TValue, typename TSubObject>
- TSimpleSharedPtr<TBaseAttrGetter<TObjType>> CreateAttrGetter(TValue TSubObject::*value) {
- return new TDirectAttrGetter<TObjType, TValue, TSubObject>(value);
- }
-}
diff --git a/library/cpp/pybind/cast.cpp b/library/cpp/pybind/cast.cpp
deleted file mode 100644
index 60a44b0e83..0000000000
--- a/library/cpp/pybind/cast.cpp
+++ /dev/null
@@ -1,324 +0,0 @@
-#include "cast.h"
-#include <util/generic/yexception.h>
-#include <util/generic/buffer.h>
-
-namespace NPyBind {
- PyObject* GetTrueRef(bool incref) {
- if (incref)
- Py_RETURN_TRUE;
- return Py_True;
- }
-
- PyObject* GetFalseRef(bool incref) {
- if (incref)
- Py_RETURN_FALSE;
- return Py_False;
- }
-
- PyObject* BuildPyObject(int val) {
- return Py_BuildValue("i", val);
- }
-
- PyObject* BuildPyObject(unsigned int val) {
- return Py_BuildValue("I", val);
- }
-
- PyObject* BuildPyObject(long int val) {
- return Py_BuildValue("l", val);
- }
-
- PyObject* BuildPyObject(unsigned long int val) {
- return Py_BuildValue("k", val);
- }
-
-#ifdef PY_LONG_LONG
- PyObject* BuildPyObject(PY_LONG_LONG val) {
- return Py_BuildValue("L", val);
- }
-
- PyObject* BuildPyObject(unsigned PY_LONG_LONG val) {
- return Py_BuildValue("K", val);
- }
-#endif
-
- PyObject* BuildPyObject(float val) {
- return Py_BuildValue("f", val);
- }
-
- PyObject* BuildPyObject(double val) {
- return Py_BuildValue("d", val);
- }
-
- PyObject* BuildPyObject(const TStringBuf& val) {
- if (!val.IsInited())
- Py_RETURN_NONE;
-
- PyObject* stringValue = Py_BuildValue("s#", val.data(), static_cast<int>(val.length()));
- if (stringValue != nullptr) {
- return stringValue;
- }
- if (PyErr_ExceptionMatches(PyExc_UnicodeDecodeError)) {
- PyErr_Clear();
- } else {
- return nullptr;
- }
- return Py_BuildValue("y#", val.data(), static_cast<int>(val.length()));
- }
-
- PyObject* BuildPyObject(const char* val) {
- if (val == nullptr)
- Py_RETURN_NONE;
- PyObject* stringValue = Py_BuildValue("s#", val, static_cast<int>(strlen(val)));
- if (stringValue != nullptr) {
- return stringValue;
- }
- if (PyErr_ExceptionMatches(PyExc_UnicodeDecodeError)) {
- PyErr_Clear();
- } else {
- return nullptr;
- }
- return Py_BuildValue("y#", val, static_cast<int>(strlen(val)));
- }
-
- PyObject* BuildPyObject(const TWtringBuf& val) {
- if (!val.IsInited())
- Py_RETURN_NONE;
-#if PY_VERSION_HEX < 0x03030000
- TPyObjectPtr result(PyUnicode_FromUnicode(nullptr, val.size()), true);
- Py_UNICODE* buf = PyUnicode_AS_UNICODE(result.Get());
- if (buf == nullptr)
- Py_RETURN_NONE;
- for (size_t i = 0; i < val.size(); ++i) {
- buf[i] = static_cast<Py_UNICODE>(val[i]);
- }
-#else
- PyObject* unicodeValue = PyUnicode_FromKindAndData(PyUnicode_2BYTE_KIND, val.data(), val.size());
- if (unicodeValue == nullptr)
- Py_RETURN_NONE;
- TPyObjectPtr result(unicodeValue, true);
-#endif
- return result.RefGet();
- }
-
- PyObject* BuildPyObject(const TBuffer& val) {
- TPyObjectPtr res(PyList_New(val.size()), true);
- for (size_t i = 0, size = val.Size(); i < size; ++i)
- PyList_SetItem(res.Get(), i, BuildPyObject(val.Data()[i]));
- return res.RefGet();
- }
-
- PyObject* BuildPyObject(bool val) {
- if (val)
- Py_RETURN_TRUE;
- else
- Py_RETURN_FALSE;
- }
-
- PyObject* BuildPyObject(PyObject* val) {
- Py_XINCREF(val);
- return val;
- }
-
- PyObject* BuildPyObject(TPyObjectPtr ptr) {
- return ptr.RefGet();
- }
-
- /* python represents (http://docs.python.org/c-api/arg.html#Py_BuildValue)
- * char, uchar, short, ushort, int, long as PyInt
- * uint, ulong as PyInt or PyLong (if exceeds sys.maxint)
- * longlong, ulonglong as PyLong
- */
-
- template <>
- bool FromPyObject(PyObject* obj, long& res) {
- if (PyLong_Check(obj)) {
- res = PyLong_AsLong(obj);
- return true;
- }
- if (PyFloat_Check(obj)) {
- res = static_cast<long>(PyFloat_AsDouble(obj));
- return true;
- }
-#if PY_MAJOR_VERSION < 3
- res = PyInt_AsLong(obj);
-#endif
- return -1 != res || !PyErr_Occurred();
- }
-
- template <>
- bool FromPyObject(PyObject* obj, unsigned long& res) {
- long lres;
- if (!FromPyObject(obj, lres))
- return false;
- if (lres < 0)
- return false;
- res = static_cast<unsigned long long>(lres);
- return true;
- }
-
- template <>
- bool FromPyObject(PyObject* obj, int& res) {
- long lres;
- if (!FromPyObject(obj, lres))
- return false;
- res = static_cast<int>(lres);
- return true;
- }
-
- template <>
- bool FromPyObject(PyObject* obj, unsigned char& res) {
- long lres;
- if (!FromPyObject(obj, lres))
- return false;
- res = static_cast<unsigned char>(lres);
- return true;
- }
-
- template <>
- bool FromPyObject(PyObject* obj, char& res) {
- long lres;
- if (!FromPyObject(obj, lres))
- return false;
- res = static_cast<char>(lres);
- return true;
- }
-
- template <>
- bool FromPyObject(PyObject* obj, unsigned int& res) {
- unsigned long lres;
- if (!FromPyObject(obj, lres))
- return false;
- res = static_cast<unsigned int>(lres);
- return true;
- }
-
-#ifdef HAVE_LONG_LONG
- template <>
- bool FromPyObject(PyObject* obj, long long& res) {
- if (PyLong_Check(obj)) {
- res = PyLong_AsLongLong(obj);
- return -1 != res || !PyErr_Occurred();
- }
- long lres;
- if (!FromPyObject(obj, lres))
- return false;
- res = static_cast<long long>(lres);
- return true;
- }
-
- template <>
- bool FromPyObject(PyObject* obj, unsigned long long& res) {
- if (PyLong_Check(obj)) {
- res = PyLong_AsUnsignedLongLong(obj);
- return static_cast<unsigned long long>(-1) != res || !PyErr_Occurred();
- }
- long lres;
- if (!FromPyObject(obj, lres))
- return false;
- res = static_cast<unsigned long long>(lres);
- return true;
- }
-#endif
-
- template <>
- bool FromPyObject(PyObject* obj, double& res) {
- if (PyFloat_Check(obj)) {
- res = PyFloat_AsDouble(obj);
- return true;
- }
- long long lres;
- if (!FromPyObject(obj, lres))
- return false;
- res = static_cast<double>(lres);
- return true;
- }
-
- template <>
- bool FromPyObject(PyObject* obj, float& res) {
- double dres;
- if (!FromPyObject(obj, dres))
- return false;
- res = static_cast<float>(dres);
- return true;
- }
-
- template <>
- bool FromPyObject(PyObject* obj, bool& res) {
- if (!PyBool_Check(obj))
- return false;
- if (obj == Py_True)
- res = true;
- else
- res = false;
- return true;
- }
-
- template <>
- bool FromPyObject(PyObject* obj, PyObject*& res) {
- Py_XINCREF(obj);
- res = obj;
- return true;
- }
-
- template <>
- bool FromPyObject(PyObject* obj, TPyObjectPtr& res) {
- res = TPyObjectPtr(obj);
- return true;
- }
-
- static inline bool _FromPyObject(PyObject* obj, TStringBuf& res) {
- char* str;
- Py_ssize_t len;
-#if PY_MAJOR_VERSION >= 3
- if (PyUnicode_Check(obj)) {
- auto buf = PyUnicode_AsUTF8AndSize(obj, &len);
- res = TStringBuf(buf, len);
- return true;
- }
-#endif
- if (-1 == PyBytes_AsStringAndSize(obj, &str, &len) || 0 > len)
- return false;
- res = TStringBuf(str, len);
- return true;
- }
-
- bool FromPyObject(PyObject* obj, TStringBuf& res) {
- return _FromPyObject(obj, res);
- }
-
- bool FromPyObject(PyObject* obj, TString& res) {
- TStringBuf str;
- if (!_FromPyObject(obj, str))
- return false;
- res = str;
- return true;
- }
-
- bool FromPyObject(PyObject* obj, TUtf16String& res) {
- if (!PyUnicode_Check(obj))
- return false;
- auto str = TPyObjectPtr(PyUnicode_AsUTF16String(obj), true);
- if (!str)
- return false;
- constexpr auto BOM_SIZE = 2;
- size_t len = (static_cast<size_t>(PyBytes_GET_SIZE(str.Get())) - BOM_SIZE) / 2;
- res.resize(len);
- memcpy(res.begin(), PyBytes_AS_STRING(str.Get()) + BOM_SIZE, len * 2);
- return (nullptr == PyErr_Occurred());
- }
-
- bool FromPyObject(PyObject* obj, TBuffer& res) {
- if (!PyList_Check(obj))
- return false;
- size_t cnt = PyList_Size(obj);
- res.Reserve(cnt);
- for (size_t i = 0; i < cnt; ++i) {
- PyObject* item = PyList_GET_ITEM(obj, i);
- char ch = 0;
- if (!FromPyObject(item, ch))
- return false;
- res.Append(ch);
- }
- return true;
- }
-}
diff --git a/library/cpp/pybind/cast.h b/library/cpp/pybind/cast.h
deleted file mode 100644
index 1f3d7d8366..0000000000
--- a/library/cpp/pybind/cast.h
+++ /dev/null
@@ -1,373 +0,0 @@
-#pragma once
-
-#define PY_SSIZE_T_CLEAN
-#include <Python.h>
-#include <util/generic/strbuf.h>
-#include <util/generic/vector.h>
-#include <util/generic/set.h>
-#include <util/generic/yexception.h>
-#include <util/generic/hash.h>
-#include <util/generic/map.h>
-#include <util/generic/maybe.h>
-#include <utility>
-#include <initializer_list>
-#include "ptr.h"
-
-namespace NPyBind {
- PyObject* GetTrueRef(bool incref = true);
- PyObject* GetFalseRef(bool incref = true);
-
- PyObject* BuildPyObject(int val);
- PyObject* BuildPyObject(unsigned int val);
- PyObject* BuildPyObject(long int val);
- PyObject* BuildPyObject(unsigned long int val);
-#ifdef PY_LONG_LONG
- PyObject* BuildPyObject(PY_LONG_LONG val);
- PyObject* BuildPyObject(unsigned PY_LONG_LONG val);
-#endif
- PyObject* BuildPyObject(float val);
- PyObject* BuildPyObject(double val);
- PyObject* BuildPyObject(const TStringBuf& val);
- PyObject* BuildPyObject(const char* val);
- PyObject* BuildPyObject(const TWtringBuf& val);
- PyObject* BuildPyObject(const TBuffer& val);
- PyObject* BuildPyObject(bool val);
- PyObject* BuildPyObject(PyObject*);
- PyObject* BuildPyObject(TPyObjectPtr);
-
- template <typename T>
- PyObject* BuildPyObject(const TVector<T>& val);
-
- template <typename T>
- PyObject* BuildPyObject(const TSet<T>& val);
-
- template <typename TKey, typename TVal>
- PyObject* BuildPyObject(const THashMap<TKey, TVal>& val);
-
- template <typename T1, typename T2>
- PyObject* BuildPyObject(const std::pair<T1, T2>& val) {
- TPyObjectPtr first(BuildPyObject(val.first), true);
- if (!first) {
- return nullptr;
- }
- TPyObjectPtr second(BuildPyObject(val.second), true);
- if (!first || !second) {
- return nullptr;
- }
- TPyObjectPtr res(PyList_New(2), true);
- PyList_SetItem(res.Get(), 0, first.RefGet());
- PyList_SetItem(res.Get(), 1, second.RefGet());
- return res.RefGet();
- }
-
- template <typename T>
- PyObject* BuildPyObject(const TVector<T>& val) {
- TPyObjectPtr res(PyList_New(val.size()), true);
- for (size_t i = 0, size = val.size(); i < size; ++i) {
- auto pythonVal = BuildPyObject(std::move(val[i]));
- if (!pythonVal) {
- return nullptr;
- }
- PyList_SetItem(res.Get(), i, pythonVal);
- }
- return res.RefGet();
- }
-
- template <typename T>
- PyObject* BuildPyObject(TVector<T>&& val) {
- TPyObjectPtr res(PyList_New(val.size()), true);
- for (size_t i = 0, size = val.size(); i < size; ++i) {
- auto pythonVal = BuildPyObject(std::move(val[i]));
- if (!pythonVal) {
- return nullptr;
- }
- PyList_SetItem(res.Get(), i, pythonVal);
- }
- return res.RefGet();
- }
-
- template <typename T>
- PyObject* BuildPyObject(const TSet<T>& val) {
- TPyObjectPtr res(PySet_New(nullptr), true);
- for (const auto& v : val) {
- auto pythonVal = BuildPyObject(std::move(v));
- if (!pythonVal) {
- return nullptr;
- }
- PySet_Add(res.Get(), pythonVal);
- }
- return res.RefGet();
- }
-
- template <typename T>
- PyObject* BuildPyObject(const THashSet<T>& val) {
- TPyObjectPtr res(PySet_New(nullptr), true);
- for (const auto& v : val) {
- auto pythonVal = BuildPyObject(std::move(v));
- if (!pythonVal) {
- return nullptr;
- }
- PySet_Add(res.Get(), pythonVal);
- }
- return res.RefGet();
- }
-
- template <typename TKey, typename TVal>
- PyObject* BuildPyObject(const THashMap<TKey, TVal>& val) {
- TPyObjectPtr res(PyDict_New(), true);
- for (typename THashMap<TKey, TVal>::const_iterator it = val.begin(), end = val.end(); it != end; ++it) {
- auto prevOccurred = PyErr_Occurred();
- Y_UNUSED(prevOccurred);
- TPyObjectPtr k(BuildPyObject(it->first), true);
- if (!k) {
- return nullptr;
- }
- TPyObjectPtr v(BuildPyObject(it->second), true);
- if (!v) {
- return nullptr;
- }
- PyDict_SetItem(res.Get(), k.Get(), v.Get());
- }
- return res.RefGet();
- }
-
- template <typename TKey, typename TVal>
- PyObject* BuildPyObject(const TMap<TKey, TVal>& val) {
- TPyObjectPtr res(PyDict_New(), true);
- for (typename TMap<TKey, TVal>::const_iterator it = val.begin(), end = val.end(); it != end; ++it) {
- TPyObjectPtr k(BuildPyObject(it->first), true);
- if (!k) {
- return nullptr;
- }
- TPyObjectPtr v(BuildPyObject(it->second), true);
- if (!v) {
- return nullptr;
- }
- PyDict_SetItem(res.Get(), k.Get(), v.Get());
- }
- return res.RefGet();
- }
-
-
- template <typename TKey, typename TVal>
- PyObject* BuildPyObject(const TMultiMap<TKey, TVal>& val) {
- TPyObjectPtr res(PyDict_New(), true);
- TMaybe<TKey> prevKey;
- TPyObjectPtr currentEntry(PyList_New(0), true);
- for (const auto& [key, value]: val) {
- if (prevKey.Defined() && prevKey != key) {
- TPyObjectPtr pyPrevKey(BuildPyObject(*prevKey), true);
- if (!pyPrevKey) {
- return nullptr;
- }
- PyDict_SetItem(res.Get(), pyPrevKey.Get(), currentEntry.Get());
- currentEntry = TPyObjectPtr(PyList_New(0), true);
- }
- TPyObjectPtr pyValue(BuildPyObject(value), true);
- if (!pyValue) {
- return nullptr;
- }
- PyList_Append(currentEntry.Get(), pyValue.Get());
- prevKey = key;
- }
-
- if (prevKey.Defined()) {
- TPyObjectPtr pyPrevKey(BuildPyObject(*prevKey), true);
- if (!pyPrevKey) {
- return nullptr;
- }
- PyDict_SetItem(res.Get(), pyPrevKey.Get(), currentEntry.Get());
- }
- return res.RefGet();
- }
-
- template <typename T>
- PyObject* BuildPyObject(const TMaybe<T>& val) {
- if (!val.Defined())
- Py_RETURN_NONE;
- return BuildPyObject(val.GetRef());
- }
-
- template <typename T, typename C, typename D>
- PyObject* BuildPyObject(const TSharedPtr<T, C, D>& val) {
- if (!val.Get())
- Py_RETURN_NONE;
- return BuildPyObject(*val.Get());
- }
-
- template <typename T>
- bool FromPyObject(PyObject* obj, T& res);
-
- bool FromPyObject(PyObject* obj, TString& res);
- bool FromPyObject(PyObject* obj, TStringBuf& res);
- bool FromPyObject(PyObject* obj, TUtf16String& res);
- bool FromPyObject(PyObject* obj, TBuffer& res);
-
- template <typename T>
- bool FromPyObject(PyObject* obj, TMaybe<T>& res) {
- //we need to save current error before trying derserialize the value
- //because it can produce conversion errors in python that we don't need to handle
- struct TError {
- public:
- TError() {
- PyErr_Fetch(&Type, &Value, &Traceback);
- }
- ~TError() {
- PyErr_Restore(Type, Value, Traceback);
-
- }
- private:
- PyObject* Type = nullptr;
- PyObject* Value = nullptr;
- PyObject* Traceback = nullptr;
- } currentPyExcInfo;
- T val;
- if (FromPyObject(obj, val)) {
- res = val;
- return true;
- }
- if (obj == Py_None) {
- res = Nothing();
- return true;
- }
- return false;
- }
-
- template <typename T1, typename T2>
- bool FromPyObject(PyObject* obj, std::pair<T1, T2>& res) {
- PyObject* first;
- PyObject* second;
- if (PyTuple_Check(obj) && 2 == PyTuple_Size(obj)) {
- first = PyTuple_GET_ITEM(obj, 0);
- second = PyTuple_GET_ITEM(obj, 1);
- } else if (PyList_Check(obj) && 2 == PyList_Size(obj)) {
- first = PyList_GET_ITEM(obj, 0);
- second = PyList_GET_ITEM(obj, 1);
- } else {
- return false;
- }
- return FromPyObject(first, res.first) && FromPyObject(second, res.second);
- }
-
- template <typename T>
- bool FromPyObject(PyObject* obj, TVector<T>& res) {
- if (!PyList_Check(obj))
- return false;
- size_t cnt = PyList_Size(obj);
- res.resize(cnt);
- for (size_t i = 0; i < cnt; ++i) {
- PyObject* item = PyList_GET_ITEM(obj, i);
- if (!FromPyObject(item, res[i]))
- return false;
- }
- return true;
- }
-
- template <typename K, typename V>
- bool FromPyObject(PyObject* obj, THashMap<K, V>& res) {
- if (!PyDict_Check(obj))
- return false;
- TPyObjectPtr list(PyDict_Keys(obj), true);
- size_t cnt = PyList_Size(list.Get());
- for (size_t i = 0; i < cnt; ++i) {
- PyObject* key = PyList_GET_ITEM(list.Get(), i);
- PyObject* value = PyDict_GetItem(obj, key);
- K rkey;
- V rvalue;
- if (!FromPyObject(key, rkey))
- return false;
- if (!FromPyObject(value, rvalue))
- return false;
- res[rkey] = rvalue;
- }
- return true;
- }
-
- template <typename K, typename V>
- bool FromPyObject(PyObject* obj, TMap<K, V>& res) {
- if (!PyDict_Check(obj))
- return false;
- TPyObjectPtr list(PyDict_Keys(obj), true);
- size_t cnt = PyList_Size(list.Get());
- for (size_t i = 0; i < cnt; ++i) {
- PyObject* key = PyList_GET_ITEM(list.Get(), i);
- PyObject* value = PyDict_GetItem(obj, key);
- K rkey;
- V rvalue;
- if (!FromPyObject(key, rkey))
- return false;
- if (!FromPyObject(value, rvalue))
- return false;
- res[rkey] = rvalue;
- }
- return true;
- }
-
- class cast_exception: public TBadCastException {
- };
-
- template <typename T>
- T FromPyObject(PyObject* obj) {
- T res;
- if (!FromPyObject(obj, res))
- ythrow cast_exception() << "Cannot cast argument to " << TypeName<T>();
- return res;
- }
-
- template <class... Args, std::size_t... I>
- bool ExtractArgs(std::index_sequence<I...>, PyObject* args, Args&... outArgs) {
- if (!args || !PyTuple_Check(args) || PyTuple_Size(args) != sizeof...(Args))
- return false;
- bool res = true;
- (void)std::initializer_list<bool>{(res = res && NPyBind::FromPyObject(PyTuple_GET_ITEM(args, I), outArgs))...};
- return res;
- }
-
- template <class... Args>
- bool ExtractArgs(PyObject* args, Args&... outArgs) {
- return ExtractArgs(std::index_sequence_for<Args...>(), args, outArgs...);
- }
-
- template <class... Args, std::size_t... I>
- bool ExtractOptionalArgs(std::index_sequence<I...>, PyObject* args, PyObject* kwargs, const char* keywords[], Args&... outArgs) {
- PyObject* pargs[sizeof...(Args)] = {};
- static const char format[sizeof...(Args) + 2] = {'|', ((void)I, 'O')..., 0};
- if (!PyArg_ParseTupleAndKeywords(args, kwargs, format, const_cast<char**>(keywords), &pargs[I]...))
- return false;
- bool res = true;
- (void)std::initializer_list<bool>{(res = res && (!pargs[I] || NPyBind::FromPyObject(pargs[I], outArgs)))...};
- return res;
- }
-
- template <class... Args>
- bool ExtractOptionalArgs(PyObject* args, PyObject* kwargs, const char* keywords[], Args&... outArgs) {
- return ExtractOptionalArgs(std::index_sequence_for<Args...>(), args, kwargs, keywords, outArgs...);
- }
-
- template <typename... Args, std::size_t... I>
- static auto GetArguments(std::index_sequence<I...>, PyObject* args) {
- Y_UNUSED(args); // gcc bug
- return std::make_tuple(FromPyObject<std::remove_cv_t<std::remove_reference_t<Args>>>(PyTuple_GetItem(args, I))...);
- }
-
- template <typename... Args>
- static auto GetArguments(PyObject* args) {
- return GetArguments<Args...>(std::index_sequence_for<Args...>(), args);
- }
-
- inline PyObject* ReturnString(TStringBuf s) {
-#if PY_MAJOR_VERSION >= 3
- return PyUnicode_FromStringAndSize(s.data(), s.size());
-#else
- return PyBytes_FromStringAndSize(s.data(), s.size());
-#endif
- }
-
- inline TPyObjectPtr ReturnBytes(TStringBuf s) {
- return TPyObjectPtr(PyBytes_FromStringAndSize(s.data(), s.size()), true);
- }
-
- inline TPyObjectPtr NameFromString(TStringBuf s) {
- return TPyObjectPtr(ReturnString(s), true);
- }
-}
diff --git a/library/cpp/pybind/embedding.cpp b/library/cpp/pybind/embedding.cpp
deleted file mode 100644
index cf8941a92a..0000000000
--- a/library/cpp/pybind/embedding.cpp
+++ /dev/null
@@ -1,63 +0,0 @@
-#define PY_SSIZE_T_CLEAN
-#include <Python.h>
-
-#include "embedding.h"
-
-#include <util/generic/ptr.h>
-#include <util/generic/yexception.h>
-
-namespace NPyBind {
-#if PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION < 8
- class TDeleteRawMem {
- public:
- template <typename T>
- static inline void Destroy(T* t) noexcept {
- PyMem_RawFree(t);
- }
- };
-
- template <typename T>
- using TRawMemHolder = THolder<T, TDeleteRawMem>;
-
- static void SetProgramName(char* name) {
- TRawMemHolder<wchar_t> wideName(Py_DecodeLocale(name, nullptr));
- Y_ENSURE(wideName);
- Py_SetProgramName(wideName.Get());
- }
-#endif
-
- TEmbedding::TEmbedding(char* argv0) {
-#if PY_MAJOR_VERSION < 3
- Py_SetProgramName(argv0);
- Py_Initialize();
-#elif PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 8
- PyStatus status;
-
- PyConfig config;
- PyConfig_InitPythonConfig(&config);
- // Disable parsing command line arguments
- config.parse_argv = 0;
-
- status = PyConfig_SetBytesString(&config, &config.program_name, argv0);
- if (PyStatus_Exception(status)) {
- PyConfig_Clear(&config);
- Py_ExitStatusException(status);
- }
-
- status = Py_InitializeFromConfig(&config);
- if (PyStatus_Exception(status)) {
- PyConfig_Clear(&config);
- Py_ExitStatusException(status);
- }
-
- PyConfig_Clear(&config);
-#elif PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION < 8
- SetProgramName(argv0);
- Py_Initialize();
-#endif
- }
-
- TEmbedding::~TEmbedding() {
- Py_Finalize();
- }
-}
diff --git a/library/cpp/pybind/embedding.h b/library/cpp/pybind/embedding.h
deleted file mode 100644
index 18553d9f6c..0000000000
--- a/library/cpp/pybind/embedding.h
+++ /dev/null
@@ -1,10 +0,0 @@
-#pragma once
-
-namespace NPyBind {
- class TEmbedding {
- public:
- TEmbedding(char* argv0);
- ~TEmbedding();
- };
-
-}
diff --git a/library/cpp/pybind/empty.cpp b/library/cpp/pybind/empty.cpp
deleted file mode 100644
index 10da997ecc..0000000000
--- a/library/cpp/pybind/empty.cpp
+++ /dev/null
@@ -1,2 +0,0 @@
-#include "init.h"
-#include "v2.h"
diff --git a/library/cpp/pybind/exceptions.cpp b/library/cpp/pybind/exceptions.cpp
deleted file mode 100644
index db1531fc63..0000000000
--- a/library/cpp/pybind/exceptions.cpp
+++ /dev/null
@@ -1,147 +0,0 @@
-#include "exceptions.h"
-#include "cast.h"
-#include "module.h"
-#include <util/generic/algorithm.h>
-
-namespace NPyBind {
-
- namespace NPrivate {
- TPyObjectPtr CreatePyBindModule() {
- return TPyObjectPtr(TExceptionsHolder::DoInitPyBindModule(), true);
- }
- }//NPrivate
-
- TPyObjectPtr TExceptionsHolder::GetException(const TString& name) {
- if (name == "")
- return TPyObjectPtr(nullptr);
- if (!Exceptions[name].Get())
- ythrow yexception() << "Wrong base class '" << name << "'";
- return Exceptions[name];
- }
-
- TPyObjectPtr TExceptionsHolder::GetExceptions(const TVector<TString>& names) {
- TVector<TString> tmp(names.begin(), names.end());
- TVector<TString>::iterator end = std::unique(tmp.begin(), tmp.end());
- TPyObjectPtr tuple(PyTuple_New(std::distance(tmp.begin(), end)), true);
- for (size_t i = 0; i < (size_t)std::distance(tmp.begin(), end); ++i) {
- if (!Exceptions[tmp[i]].Get())
- ythrow yexception() << "Wrong base class '" << tmp[i] << "'";
- PyTuple_SetItem(tuple.Get(), i, Exceptions[tmp[i]].Get());
- }
- return tuple;
- }
-
- // def PyBindObjectReconstructor(cl, props):
- // return cl(__properties__=props)
- static PyObject* PyBindObjectReconstructor(PyObject*, PyObject* args) {
- TPyObjectPtr callable, props;
- if (!ExtractArgs(args, callable, props))
- ythrow yexception() << "Wrong method arguments";
-#if PY_MAJOR_VERSION >= 3
- TPyObjectPtr noArgs(PyTuple_New(0), true);
-#else
- TPyObjectPtr noArgs(PyList_New(0), true);
-#endif
- TPyObjectPtr kw(PyDict_New(), true);
- PyDict_SetItemString(kw.Get(), "__properties__", props.Get());
- TPyObjectPtr res(PyObject_Call(callable.Get(), noArgs.Get(), kw.Get()), true);
- return res.RefGet();
- }
-
- static PyMethodDef PyBindMethods[] = {
- {"PyBindObjectReconstructor", PyBindObjectReconstructor, METH_VARARGS, "Tech method. It's required for unpickling."},
- {nullptr, nullptr, 0, nullptr}};
-
-#if PY_MAJOR_VERSION >= 3
- static struct PyModuleDef moduledef = {
- PyModuleDef_HEAD_INIT,
- "pybind",
- NULL,
- -1,
- PyBindMethods,
- NULL, NULL, NULL, NULL
- };
-
- static PyObject* InitPyBind() {
- return PyModule_Create(&moduledef);
- }
-#else
- static PyObject* InitPyBind() {
- return Py_InitModule("pybind", PyBindMethods);
- }
-#endif
-
- void TExceptionsHolder::DoInitPyBindModule2() {
- DoInitPyBindModule();
- }
-
- PyObject* TExceptionsHolder::DoInitPyBindModule() {
- Instance().Module = NPyBind::TPyObjectPtr(InitPyBind(), true);
- if (!Instance().Module.Get())
- return nullptr;
-
- for (TCheckersVector::const_iterator it = Instance().Checkers.begin(), end = Instance().Checkers.end(); it != end; ++it) {
- TString name = (*it)->GetName();
- if (!!name) {
- //Ref to the object should be incremented before passing to AddObject
- auto res = PyModule_AddObject(Instance().Module.Get(), name.data(), (*it)->GetException().RefGet());
- if (res < 0) {
- ythrow yexception() << "Failed to add object " << name << " to internal module pybind";
- }
- }
- }
- return Instance().Module.RefGet();
- }
-
- void TExceptionsHolder::Clear() {
- //Unfortunately in Python3 we can't retrack this object because of PyError_NewException
- //it's only the safe way to preserve GC gens in valid state during the finalization
- for (auto& ptr: Checkers) {
- if (!dynamic_cast<const TPyErrExceptionsChecker*>(ptr.Get())) { // no need to untrack standard PyExc_* exceptions from TPyErrExceptionsChecker
- if (auto exceptionPtr = ptr->GetException()) {
- PyObject_GC_UnTrack(exceptionPtr.Get());
- }
- }
- }
- Checkers.clear();
- Exceptions.clear();
- Module.Drop();
- }
-
- TExceptionsHolder::TExceptionsHolder() {
- AddException<std::exception>("yexception");
- AddException<TSystemError>("TSystemError", "yexception");
- AddException<TIoException>("TIoException", "yexception");
-
- TVector<TString> names(2);
- names[0] = "TSystemError";
- names[1] = "TIoException";
-
- AddException<TIoSystemError>("TIoSystemError", names);
- AddException<TFileError>("TFileError", "TIoSystemError");
- AddException<TBadCastException>("TBadCastException", "yexception");
-
- Checkers.push_back(new TPyErrExceptionsChecker);
-
- // XXX: In Python 2.6, PyImport_AppendInittab() function takes non-const char*, this causes
- // "ISO C++11 does not allow conversion from string literal to 'char *'" warning.
- static char pybind[] = "pybind";
-#if PY_MAJOR_VERSION >= 3
- PyImport_AppendInittab(pybind, DoInitPyBindModule);
-
- NPrivate::AddFinalizationCallBack([this]() {
- Clear();
- });
-#else
- PyImport_AppendInittab(pybind, DoInitPyBindModule2);
-#endif
- }
-
- NPyBind::TPyObjectPtr TExceptionsHolder::ToPyException(const std::exception& ex) {
- for (TCheckersVector::const_reverse_iterator it = Checkers.rbegin(), end = Checkers.rend(); it != end; ++it) {
- if ((*it)->Check(ex))
- return (*it)->GetException();
- }
- return TPyObjectPtr(nullptr);
- }
-}
diff --git a/library/cpp/pybind/exceptions.h b/library/cpp/pybind/exceptions.h
deleted file mode 100644
index 48e20995e4..0000000000
--- a/library/cpp/pybind/exceptions.h
+++ /dev/null
@@ -1,143 +0,0 @@
-#pragma once
-
-#define PY_SSIZE_T_CLEAN
-#include <Python.h>
-#include <util/generic/yexception.h>
-#include <util/generic/map.h>
-#include <util/generic/vector.h>
-#include "ptr.h"
-
-namespace NPyBind {
- // Usage:
- // ythrow TPyErr(PyExc_TypeError) << "some python type error somewhere in your C++ code";
- //
- class TPyErr: public virtual yexception {
- public:
- TPyErr(PyObject* theException = PyExc_RuntimeError)
- : Exception(theException)
- {
- }
-
- TPyObjectPtr GetException() const {
- return Exception;
- }
-
- private:
- NPyBind::TPyObjectPtr Exception;
- };
-
- //Private api for creating PyBind python module
- //Needs only for overriding pybind python module in library which imports other pybind library
- namespace NPrivate {
- TPyObjectPtr CreatePyBindModule();
- }//NPrivate
- class TExceptionsHolder {
- friend TPyObjectPtr NPrivate::CreatePyBindModule();
- private:
- TExceptionsHolder(const TExceptionsHolder&);
- TExceptionsHolder& operator=(const TExceptionsHolder&);
- TExceptionsHolder();
-
- void Clear();
- TPyObjectPtr GetException(const TString&);
- TPyObjectPtr GetExceptions(const TVector<TString>&);
- private:
- class TExceptionsChecker {
- public:
- virtual ~TExceptionsChecker() {
- }
- virtual bool Check(const std::exception& ex) const = 0;
- virtual TString GetName() const = 0;
- virtual TPyObjectPtr GetException() const = 0;
- };
-
- template <typename TExcType>
- class TConcreteExceptionsChecker: public TExceptionsChecker {
- private:
- TString Name;
- TPyObjectPtr Exception;
-
- public:
- TConcreteExceptionsChecker(const TString& name, TPyObjectPtr exception)
- : Name(name)
- , Exception(exception)
- {
- }
-
- bool Check(const std::exception& ex) const override {
- const std::exception* e = &ex;
- return dynamic_cast<const TExcType*>(e);
- }
-
- TString GetName() const override {
- return Name;
- }
-
- TPyObjectPtr GetException() const override {
- return Exception;
- }
- };
-
- class TPyErrExceptionsChecker: public TExceptionsChecker {
- private:
- mutable TPyObjectPtr Exception;
-
- public:
- TPyErrExceptionsChecker() {
- }
-
- bool Check(const std::exception& ex) const override {
- const TPyErr* err = dynamic_cast<const TPyErr*>(&ex);
- if (err) {
- Exception = err->GetException();
- }
- return err != nullptr;
- }
-
- TString GetName() const override {
- return TString();
- }
-
- TPyObjectPtr GetException() const override {
- return Exception;
- }
- };
-
- typedef TSimpleSharedPtr<TExceptionsChecker> TCheckerPtr;
- typedef TVector<TCheckerPtr> TCheckersVector;
- typedef TMap<TString, TPyObjectPtr> TExceptionsMap;
-
- TPyObjectPtr Module;
- TCheckersVector Checkers;
- TExceptionsMap Exceptions;
-
- static PyObject* DoInitPyBindModule();
- static void DoInitPyBindModule2();
-
- public:
- static TExceptionsHolder& Instance() {
- static TExceptionsHolder Holder;
- return Holder;
- }
-
- template <typename TExcType>
- void AddException(const TString& name, const TString& base = "") {
- TPyObjectPtr baseException(GetException(base));
- TString fullName = TString("pybind.") + name;
- TPyObjectPtr exception(PyErr_NewException(const_cast<char*>(fullName.c_str()), baseException.Get(), nullptr), true);
- Checkers.push_back(new TConcreteExceptionsChecker<TExcType>(name, exception));
- Exceptions[name] = exception;
- }
-
- template <typename TExcType>
- void AddException(const TString& name, const TVector<TString>& bases) {
- TPyObjectPtr baseExceptions(GetExceptions(bases));
- TString fullName = TString("pybind.") + name;
- TPyObjectPtr exception(PyErr_NewException(const_cast<char*>(fullName.c_str()), baseExceptions.Get(), nullptr), true);
- Checkers.push_back(new TConcreteExceptionsChecker<TExcType>(name, exception));
- Exceptions[name] = exception;
- }
-
- NPyBind::TPyObjectPtr ToPyException(const std::exception&);
- };
-}
diff --git a/library/cpp/pybind/init.h b/library/cpp/pybind/init.h
deleted file mode 100644
index 58874574ed..0000000000
--- a/library/cpp/pybind/init.h
+++ /dev/null
@@ -1,25 +0,0 @@
-#pragma once
-
-#define PY_SSIZE_T_CLEAN
-#include <Python.h>
-
-#include "ptr.h"
-
-namespace NPyBind {
-#if PY_MAJOR_VERSION >= 3
-
-#define PYBIND_MODINIT(name) PyMODINIT_FUNC PyInit_##name()
-
- inline PyObject* ModInitReturn(TPyObjectPtr&& modptr) {
- return modptr.Release();
- }
-
-#else
-
-#define PYBIND_MODINIT(name) PyMODINIT_FUNC init##name()
-
- inline void ModInitReturn(TPyObjectPtr&&) {
- }
-
-#endif
-}
diff --git a/library/cpp/pybind/method.h b/library/cpp/pybind/method.h
deleted file mode 100644
index 7c1f6e90e1..0000000000
--- a/library/cpp/pybind/method.h
+++ /dev/null
@@ -1,439 +0,0 @@
-#pragma once
-
-#define PY_SSIZE_T_CLEAN
-#include <Python.h>
-#include <util/generic/string.h>
-#include <util/generic/map.h>
-#include <util/generic/set.h>
-#include <util/generic/vector.h>
-#include <util/generic/ptr.h>
-#include <util/generic/typetraits.h>
-
-#include <util/generic/function.h>
-
-#include "cast.h"
-
-namespace NPyBind {
- template <typename TObjType>
- class TBaseMethodCaller {
- public:
- virtual ~TBaseMethodCaller() {
- }
- virtual bool CallMethod(PyObject* owner, TObjType* self, PyObject* args, PyObject* kwargs, PyObject*& res) const = 0;
- virtual bool HasMethod(PyObject*, TObjType*, const TString&, const TSet<TString>&) {
- return true;
- }
- };
-
- template <typename TObjType>
- class TIsACaller;
-
- template <typename TObjType>
- class TMethodCallers {
- private:
- typedef TSimpleSharedPtr<TBaseMethodCaller<TObjType>> TCallerPtr;
- typedef TVector<TCallerPtr> TCallerList;
- typedef TMap<TString, TCallerList> TCallerMap;
-
- const TSet<TString>& HiddenAttrNames;
- TCallerMap Callers;
-
- public:
- TMethodCallers(const TSet<TString>& hiddenNames)
- : HiddenAttrNames(hiddenNames)
- {
- }
-
- void AddCaller(const TString& name, TCallerPtr caller) {
- Callers[name].push_back(caller);
- }
-
- bool HasCaller(const TString& name) const {
- return Callers.has(name);
- }
-
- PyObject* CallMethod(PyObject* owner, TObjType* self, PyObject* args, PyObject* kwargs, const TString& name) const {
- const TCallerList* lst = Callers.FindPtr(name);
- if (!lst)
- return nullptr;
- for (const auto& caller : *lst) {
- PyObject* res = nullptr;
- PyErr_Clear();
- if (caller->CallMethod(owner, self, args, kwargs, res))
- return res;
- }
- return nullptr;
- }
-
- bool HasMethod(PyObject* owner, TObjType* self, const TString& name) const {
- const TCallerList* lst = Callers.FindPtr(name);
- if (!lst)
- return false;
- for (const auto& caller : *lst) {
- if (caller->HasMethod(owner, self, name, HiddenAttrNames))
- return true;
- }
- return false;
- }
-
- void GetMethodsNames(PyObject* owner, TObjType* self, TVector<TString>& resultNames) const {
- for (const auto& it : Callers) {
- if (HasMethod(owner, self, it.first) && !HiddenAttrNames.contains(it.first))
- resultNames.push_back(it.first);
- }
- }
-
- void GetAllMethodsNames(TVector<TString>& resultNames) const {
- for (const auto& it : Callers) {
- resultNames.push_back(it.first);
- }
- }
-
- void GetPropertiesNames(PyObject*, TObjType* self, TVector<TString>& resultNames) const {
- const TCallerList* lst = Callers.FindPtr("IsA");
- if (!lst)
- return;
- for (const auto& caller : *lst) {
- TIsACaller<TObjType>* isACaller = dynamic_cast<TIsACaller<TObjType>*>(caller.Get());
- if (isACaller) {
- resultNames = isACaller->GetPropertiesNames(self);
- return;
- }
- }
- }
- };
-
- template <typename TObjType>
- class TIsACaller: public TBaseMethodCaller<TObjType> {
- private:
- class TIsAChecker {
- public:
- virtual ~TIsAChecker() {
- }
- virtual bool Check(const TObjType* obj) const = 0;
- };
-
- template <typename TConcrete>
- class TIsAConcreteChecker: public TIsAChecker {
- public:
- bool Check(const TObjType* obj) const override {
- return dynamic_cast<const TConcrete*>(obj) != nullptr;
- }
- };
-
- typedef TSimpleSharedPtr<TIsAChecker> TCheckerPtr;
- typedef TMap<TString, TCheckerPtr> TCheckersMap;
-
- TCheckersMap Checkers;
-
- bool Check(const TString& name, const TObjType* obj) const {
- const TCheckerPtr* checker = Checkers.FindPtr(name);
- if (!checker) {
- PyErr_Format(PyExc_KeyError, "unknown class name: %s", name.data());
- return false;
- }
- return (*checker)->Check(obj);
- }
-
- protected:
- TIsACaller() {
- }
-
- template <typename TConcrete>
- void AddChecker(const TString& name) {
- Checkers[name] = new TIsAConcreteChecker<TConcrete>;
- }
-
- public:
- bool CallMethod(PyObject*, TObjType* self, PyObject* args, PyObject*, PyObject*& res) const override {
- if (args == nullptr || !PyTuple_Check(args))
- return false;
- size_t cnt = PyTuple_Size(args);
- bool result = true;
- for (size_t i = 0; i < cnt; ++i) {
- result = result && Check(
-#if PY_MAJOR_VERSION >= 3
- PyUnicode_AsUTF8(
-#else
- PyString_AsString(
-#endif
- PyTuple_GetItem(args, i)), self);
- }
- if (PyErr_Occurred()) {
- return false;
- }
- res = BuildPyObject(result);
- return true;
- }
-
- TVector<TString> GetPropertiesNames(const TObjType* obj) const {
- TVector<TString> names;
-
- for (const auto& it : Checkers) {
- if (it.second->Check(obj)) {
- names.push_back(it.first);
- }
- }
-
- return names;
- }
- };
-
- template <typename TObjType>
- class TGenericMethodCaller: public TBaseMethodCaller<TObjType> {
- private:
- TString AttrName;
-
- public:
- TGenericMethodCaller(const TString& attrName)
- : AttrName(attrName)
- {
- }
-
- bool CallMethod(PyObject* obj, TObjType*, PyObject* args, PyObject*, PyObject*& res) const override {
- auto str = NameFromString(AttrName);
- PyObject* attr = PyObject_GenericGetAttr(obj, str.Get());
- if (!attr)
- ythrow yexception() << "Can't get generic attribute '" << AttrName << "'";
- res = PyObject_CallObject(attr, args);
- return res != nullptr;
- }
- };
-
-
- template <typename TObjType, typename TSubObject>
- class TSubObjectChecker: public TBaseMethodCaller<TObjType> {
- public:
- ~TSubObjectChecker() override {
- }
-
- bool HasMethod(PyObject*, TObjType* self, const TString&, const TSet<TString>&) override {
- return dynamic_cast<const TSubObject*>(self) != nullptr;
- }
- };
-
- template <typename TFunctor, typename Tuple, typename ResType, typename=std::enable_if_t<!std::is_same_v<ResType, void>>>
- void ApplyFunctor(TFunctor functor, Tuple resultArgs, PyObject*& res) {
- res = BuildPyObject(std::move(Apply(functor, resultArgs)));
- }
-
- template <typename TFunctor, typename Tuple, typename ResType, typename=std::enable_if_t<std::is_same_v<ResType, void>>, typename=void>
- void ApplyFunctor(TFunctor functor, Tuple resultArgs, PyObject*& res) {
- Py_INCREF(Py_None);
- res = Py_None;
- Apply(functor, resultArgs);
- }
-
- template <typename TObjType, typename TResType, typename... Args>
- class TFunctorCaller: public TBaseMethodCaller<TObjType> {
- using TFunctor = std::function<TResType(TObjType&,Args...)>;
- TFunctor Functor;
- public:
- explicit TFunctorCaller(TFunctor functor):
- Functor(functor){}
-
- bool CallMethod(PyObject*, TObjType* self, PyObject* args, PyObject*, PyObject*& res) const {
- auto methodArgsTuple = GetArguments<Args...>(args);
- auto resultArgs = std::tuple_cat(std::tie(*self), methodArgsTuple);
- ApplyFunctor<TFunctor, decltype(resultArgs), TResType>(Functor, resultArgs, res);
- return true;
- }
- };
-
- template <typename TObjType, typename TRealType>
- class TGetStateCaller: public TSubObjectChecker<TObjType, TRealType> {
- protected:
- TPyObjectPtr AddFromCaller(PyObject* obj, const TString& methodName) const {
- PyObject* res = PyObject_CallMethod(obj, const_cast<char*>(methodName.c_str()), const_cast<char*>(""));
- if (!res) {
- PyErr_Clear();
- return TPyObjectPtr(Py_None);
- }
- return TPyObjectPtr(res, true);
- }
-
- void GetStandartAttrsDictionary(PyObject* obj, TRealType*, TMap<TString, TPyObjectPtr>& dict) const {
- TPyObjectPtr attrsDict(PyObject_GetAttrString(obj, "__dict__"), true);
- TMap<TString, TPyObjectPtr> attrs;
- if (!FromPyObject(attrsDict.Get(), attrs))
- ythrow yexception() << "Can't get '__dict__' attribute";
- dict.insert(attrs.begin(), attrs.end());
- }
-
- virtual void GetAttrsDictionary(PyObject* obj, TRealType* self, TMap<TString, TPyObjectPtr>& dict) const = 0;
-
- public:
- bool CallMethod(PyObject* obj, TObjType* self, PyObject* args, PyObject*, PyObject*& res) const override {
- if (!ExtractArgs(args))
- ythrow yexception() << "Can't parse arguments: it should be none";
- TRealType* rself = dynamic_cast<TRealType*>(self);
- if (!rself)
- return false;
- TMap<TString, TPyObjectPtr> dict;
- GetAttrsDictionary(obj, rself, dict);
- res = BuildPyObject(dict);
- return true;
- }
- };
-
- template <typename TObjType, typename TRealType>
- class TSetStateCaller: public TSubObjectChecker<TObjType, TRealType> {
- protected:
- void SetStandartAttrsDictionary(PyObject* obj, TRealType*, TMap<TString, TPyObjectPtr>& dict) const {
- TPyObjectPtr value(BuildPyObject(dict), true);
- PyObject_SetAttrString(obj, "__dict__", value.Get());
- }
-
- virtual void SetAttrsDictionary(PyObject* obj, TRealType* self, TMap<TString, TPyObjectPtr>& dict) const = 0;
-
- public:
- bool CallMethod(PyObject* obj, TObjType* self, PyObject* args, PyObject*, PyObject*& res) const override {
- TMap<TString, TPyObjectPtr> dict;
- if (!ExtractArgs(args, dict))
- ythrow yexception() << "Can't parse arguments: it should be one dictionary";
- TRealType* rself = dynamic_cast<TRealType*>(self);
- if (!rself)
- return false;
- SetAttrsDictionary(obj, rself, dict);
- Py_INCREF(Py_None);
- res = Py_None;
- return true;
- }
- };
-
- template <typename TObjType, typename TResult, typename TSubObject, typename TMethod, typename... Args>
- class TAnyParameterMethodCaller: public TSubObjectChecker<TObjType, TSubObject> {
- private:
- TMethod Method;
-
- public:
- TAnyParameterMethodCaller(TMethod method)
- : Method(method)
- {
- }
-
- public:
- bool CallMethod(PyObject*, TObjType* self, PyObject* args, PyObject*, PyObject*& res) const override {
- TSubObject* sub = dynamic_cast<TSubObject*>(self);
- if (sub == nullptr)
- return false;
- if (args && (!PyTuple_Check(args) || PyTuple_Size(args) != TFunctionArgs<TMethod>::Length)) {
- //ythrow yexception() << "Method takes " << (size_t)(TFunctionArgs<TMethod>::Length) << " arguments, " << PyTuple_Size(args) << " provided";
- return false;
- }
-
- try {
- class Applicant {
- public:
- TResult operator()(Args... theArgs) {
- return (Sub->*Method)(theArgs...);
- }
- TSubObject* Sub;
- TMethod Method;
- };
- res = BuildPyObject(std::move(Apply(Applicant{sub, Method}, GetArguments<Args...>(args))));
- } catch (cast_exception) {
- return false;
- } catch (...) {
- if (PyExc_StopIteration == PyErr_Occurred()) {
- // NB: it's replacement for geo_boost::python::throw_error_already_set();
- return true;
- }
- PyErr_SetString(PyExc_RuntimeError, CurrentExceptionMessage().data());
- return true;
- }
-
- return true;
- }
- };
-
- template <typename TObjType, typename TSubObject, typename TMethod, typename... Args>
- class TAnyParameterMethodCaller<TObjType, void, TSubObject, TMethod, Args...>: public TSubObjectChecker<TObjType, TSubObject> {
- private:
- TMethod Method;
-
- public:
- TAnyParameterMethodCaller(TMethod method)
- : Method(method)
- {
- }
-
- public:
- bool CallMethod(PyObject*, TObjType* self, PyObject* args, PyObject*, PyObject*& res) const override {
- TSubObject* sub = dynamic_cast<TSubObject*>(self);
- if (sub == nullptr) {
- return false;
- }
- if (args && (!PyTuple_Check(args) || PyTuple_Size(args) != TFunctionArgs<TMethod>::Length)) {
- return false;
- }
-
- try {
- class Applicant {
- public:
- void operator()(Args... theArgs) {
- (Sub->*Method)(theArgs...);
- }
- TSubObject* Sub;
- TMethod Method;
- };
-
- Apply(Applicant{sub, Method}, GetArguments<Args...>(args));
-
- Py_INCREF(Py_None);
- res = Py_None;
- } catch (cast_exception) {
- return false;
- } catch (...) {
- PyErr_SetString(PyExc_RuntimeError, CurrentExceptionMessage().data());
- return true;
- }
-
- return true;
- }
- };
-
- template <typename TResult, typename TSubObject, typename... Args>
- struct TConstTraits {
- typedef TResult (TSubObject::*TMethod)(Args... args) const;
- };
-
- template <typename TResult, typename TSubObject, typename... Args>
- struct TNonConstTraits {
- typedef TResult (TSubObject::*TMethod)(Args... args);
- };
-
- template <typename TObjType, typename TResult, typename TSubObject, typename TMethod, typename... Args>
- class TConstMethodCaller: public TAnyParameterMethodCaller<TObjType, TResult, const TSubObject, typename TConstTraits<TResult, TSubObject, Args...>::TMethod, Args...> {
- public:
- TConstMethodCaller(typename TConstTraits<TResult, TSubObject, Args...>::TMethod method)
- : TAnyParameterMethodCaller<TObjType, TResult, const TSubObject, typename TConstTraits<TResult, TSubObject, Args...>::TMethod, Args...>(method)
- {
- }
- };
-
- template <typename TObjType, typename TResult, typename TSubObject, typename... Args>
- TSimpleSharedPtr<TBaseMethodCaller<TObjType>> CreateConstMethodCaller(TResult (TSubObject::*method)(Args...) const) {
- return new TConstMethodCaller<TObjType, TResult, TSubObject, TResult (TSubObject::*)(Args...) const, Args...>(method);
- }
-
- template <typename TObjType, typename TResType, typename... Args>
- TSimpleSharedPtr<TBaseMethodCaller<TObjType>> CreateFunctorCaller(std::function<TResType(TObjType&, Args...)> functor) {
- return new TFunctorCaller<TObjType, TResType, Args...>(functor);
- }
-
- template <typename TObjType, typename TResult, typename TSubObject, typename TMethod, typename... Args>
- class TMethodCaller: public TAnyParameterMethodCaller<TObjType, TResult, TSubObject, typename TNonConstTraits<TResult, TSubObject, Args...>::TMethod, Args...> {
- public:
- TMethodCaller(typename TNonConstTraits<TResult, TSubObject, Args...>::TMethod method)
- : TAnyParameterMethodCaller<TObjType, TResult, TSubObject, typename TNonConstTraits<TResult, TSubObject, Args...>::TMethod, Args...>(method)
- {
- }
- };
-
- template <typename TObjType, typename TResult, typename TSubObject, typename... Args>
- TSimpleSharedPtr<TBaseMethodCaller<TObjType>> CreateMethodCaller(TResult (TSubObject::*method)(Args...)) {
- return new TMethodCaller<TObjType, TResult, TSubObject, TResult (TSubObject::*)(Args...), Args...>(method);
- }
-
-}
diff --git a/library/cpp/pybind/module.cpp b/library/cpp/pybind/module.cpp
deleted file mode 100644
index 63b15de45d..0000000000
--- a/library/cpp/pybind/module.cpp
+++ /dev/null
@@ -1,72 +0,0 @@
-#include "module.h"
-#include "ptr.h"
-
-#include <util/generic/adaptor.h>
-
-namespace NPyBind {
-
-#if PY_MAJOR_VERSION >= 3
- namespace NPrivate {
- struct TFinCallBacksHolder {
- static TVector<TFinalizationCallBack>& GetCallBacks() {
- static TVector<TFinalizationCallBack> res;
- return res;
- }
- };
-
- TAtExitRegistrar::TAtExitRegistrar(TPyObjectPtr module) {
- TPyObjectPtr atExitModuleName(Py_BuildValue("s", "atexit"), true);
- TPyObjectPtr atExitModule(PyImport_Import(atExitModuleName.Get()));
- Y_ABORT_UNLESS(atExitModule);
- TPyObjectPtr finalizerFunc(PyObject_GetAttrString(module.Get(), "finalizer"), true);
- Y_ABORT_UNLESS(finalizerFunc);
- TPyObjectPtr registerName(Py_BuildValue("s", "register"), true);
- PyObject_CallMethodObjArgs(atExitModule.Get(), registerName.Get(), finalizerFunc.Get(), nullptr);
- }
-
- TPyBindModuleRegistrar::TPyBindModuleRegistrar() {
- TPyObjectPtr modules(PySys_GetObject("modules"));
- Y_ENSURE(modules.Get());
- if (Module = NPrivate::CreatePyBindModule()) {
- Y_ABORT_UNLESS(0 == PyDict_SetItemString(modules.Get(), "pybind", Module.RefGet()));
- }
- AddFinalizationCallBack([this]() {
- auto ptr = Module;
- Y_UNUSED(ptr);
- TPyObjectPtr modules(PySys_GetObject("modules"));
- Y_ENSURE(modules.Get());
- TPyObjectPtr pyBindName(Py_BuildValue("s", "pybind"));
- if (PyDict_Contains(modules.Get(), pyBindName.Get()) == 1) {
- Y_ABORT_UNLESS(0==PyDict_DelItemString(modules.Get(), "pybind"));
- }
- if (Module) {
- //We have to untrack the module because some refs from him refers to gc-leaked errors
- //see exceptions.cpp fore more info
- PyObject_GC_UnTrack(Module.Get());
- Module.Drop();
- }
- });
- }
-
- void AddFinalizationCallBack(TFinalizationCallBack callback) {
- TFinCallBacksHolder::GetCallBacks().push_back(callback);
- }
-
- int FinalizeAll() {
- for (auto callback: Reversed(NPrivate::TFinCallBacksHolder::GetCallBacks())) {
- callback();
- }
- return 0;
- }
- }
-#endif
-
-
- TModuleHolder::TModuleHolder()
- : Methods(1, new TVector<TMethodDef>)
- {
-#if PY_MAJOR_VERSION >= 3
- AddModuleMethod<TModuleMethodCaller<decltype(&NPrivate::FinalizeAll), &NPrivate::FinalizeAll>::Call>("finalizer");
-#endif
- }
-}//NPyBind
diff --git a/library/cpp/pybind/module.h b/library/cpp/pybind/module.h
deleted file mode 100644
index 41dcb4dfec..0000000000
--- a/library/cpp/pybind/module.h
+++ /dev/null
@@ -1,176 +0,0 @@
-#pragma once
-
-#define PY_SSIZE_T_CLEAN
-#include <Python.h>
-#include "ptr.h"
-#include "cast.h"
-#include "exceptions.h"
-
-#include <util/generic/function.h>
-
-namespace NPyBind {
-#if PY_MAJOR_VERSION >= 3
- namespace NPrivate {
- using TFinalizationCallBack = std::function<void()>;
- void AddFinalizationCallBack(TFinalizationCallBack);
- class TAtExitRegistrar: private TNonCopyable {
- TAtExitRegistrar(TPyObjectPtr module);
- public:
- static void Instantiate(TPyObjectPtr module) {
- static TAtExitRegistrar registrar(module);
- Y_UNUSED(registrar);
- }
- };
-
- class TPyBindModuleRegistrar: private TNonCopyable {
- TPyBindModuleRegistrar();
- TPyObjectPtr Module;
- public:
- static void Instantiate() {
- static TPyBindModuleRegistrar registrar;
- Y_UNUSED(registrar);
- }
- };
- } //NPrivate
-#endif
-
- class TModuleHolder {
- private:
- TModuleHolder(const TModuleHolder&);
- TModuleHolder& operator=(const TModuleHolder&);
-
- TModuleHolder();
- private:
- typedef PyCFunction TModuleMethod;
-#if PY_MAJOR_VERSION >= 3
- typedef PyObject* (*TModuleInitFunc)();
-#else
- typedef void (*TModuleInitFunc)();
-#endif
-
- struct TMethodDef {
- TString Name;
- TModuleMethod Method;
- TString Description;
- int Flags;
-
- TMethodDef(const TString& name, TModuleMethod method, const TString& descr, int flags)
- : Name(name)
- , Method(method)
- , Description(descr)
- , Flags(flags)
- {
- }
-
- operator PyMethodDef() const {
- PyMethodDef cur = {Name.c_str(), Method, Flags, Description.c_str()};
- return cur;
- }
- };
-
- typedef TSimpleSharedPtr<TVector<TMethodDef>> TMethodDefVecPtr;
- typedef TSimpleSharedPtr<TVector<PyMethodDef>> TPyMethodDefVecPtr;
-
- TVector<TMethodDefVecPtr> Methods;
- TVector<TPyMethodDefVecPtr> Defs;
-#if PY_MAJOR_VERSION >= 3
- //because the md_name will leak otherwise
- class TPyModuleDefWithName {
- PyModuleDef Def;
- TString Name;
- public:
- explicit TPyModuleDefWithName(TString name, TPyMethodDefVecPtr moduleDefs)
- : Name(std::move(name))
- {
- Def = PyModuleDef{
- PyModuleDef_HEAD_INIT,
- Name.c_str(),
- nullptr,
- -1,
- moduleDefs->data(),
- nullptr, nullptr, nullptr, nullptr
- };
- }
- PyModuleDef* GetDefPtr() {
- return &Def;
- }
-
- };
- TVector<TSimpleSharedPtr<TPyModuleDefWithName>> ModuleDefs;
-#endif
-
- template <TModuleMethod method>
- static PyObject* MethodWrapper(PyObject* obj, PyObject* args) {
- try {
- PyObject* res = method(obj, args);
- if (!res && !PyErr_Occurred())
- ythrow yexception() << "\nModule method exited with NULL, but didn't set Error.\n Options:\n -- Return correct value or None;\n -- Set python exception;\n -- Throw c++ exception.";
- return res;
- } catch (const std::exception& ex) {
- PyErr_SetString(TExceptionsHolder::Instance().ToPyException(ex).Get(), ex.what());
- } catch (...) {
- PyErr_SetString(PyExc_RuntimeError, "Unknown error occurred while trying to call module method");
- }
- return nullptr;
- }
-
- public:
- static TModuleHolder& Instance() {
- static TModuleHolder Holder;
- return Holder;
- }
-
- void ImportModule(TPyObjectPtr module, const char* const name, TModuleInitFunc initFunc) {
- PyImport_AppendInittab(const_cast<char*>(name), initFunc);
- TPyObjectPtr importedModule(PyImport_ImportModule(name), true);
- PyModule_AddObject(module.Get(), name, importedModule.Get());
- }
-
- template <TModuleMethod method>
- void AddModuleMethod(const TString& name, const TString& descr = "") {
- Methods.back()->push_back(TMethodDef(name, MethodWrapper<method>, descr, METH_VARARGS));
- }
-
- TPyObjectPtr InitModule(const TString& name) {
- Defs.push_back(new TVector<PyMethodDef>(Methods.back()->begin(), Methods.back()->end()));
- PyMethodDef blank = {nullptr, nullptr, 0, nullptr};
- Defs.back()->push_back(blank);
-#if PY_MAJOR_VERSION >= 3
- ModuleDefs.push_back(MakeSimpleShared<TPyModuleDefWithName>(name, Defs.back()));
- TPyObjectPtr res(PyModule_Create(ModuleDefs.back()->GetDefPtr()));
- NPrivate::TAtExitRegistrar::Instantiate(res);
- NPrivate::TPyBindModuleRegistrar::Instantiate();
-#else
- TPyObjectPtr res(Py_InitModule(name.c_str(), &(Defs.back()->at(0))));
-#endif
- Methods.push_back(new TVector<TMethodDef>);
- return res;
- }
- };
-
- template <typename TMethodSignature, TMethodSignature method>
- class TModuleMethodCaller {
- private:
- template <typename TResult, typename... Args>
- struct TCaller {
- static PyObject* Call(PyObject* args) {
- return BuildPyObject(Apply(method, GetArguments<Args...>(args)));
- }
- };
-
- template <typename TResult, typename... Args>
- static PyObject* InternalCall(TResult (*)(Args...), PyObject* args) {
- return BuildPyObject(Apply(method, GetArguments<Args...>(args)));
- }
-
- public:
- static PyObject* Call(PyObject*, PyObject* args) {
- if (args && (!PyTuple_Check(args) || PyTuple_Size(args) != TFunctionArgs<TMethodSignature>::Length)) {
- ythrow yexception() << "Method takes " << (size_t)(TFunctionArgs<TMethodSignature>::Length) << " arguments, " << PyTuple_Size(args) << " provided";
- }
-
- return InternalCall(method, args);
- }
- };
-
-}
diff --git a/library/cpp/pybind/pod.cpp b/library/cpp/pybind/pod.cpp
deleted file mode 100644
index 3cf030e537..0000000000
--- a/library/cpp/pybind/pod.cpp
+++ /dev/null
@@ -1,18 +0,0 @@
-#include "pod.h"
-
-namespace NPyBind {
- class TPODAttrGetter: public TBaseAttrGetter<TPOD> {
- public:
- bool GetAttr(PyObject*, const TPOD& self, const TString& attr, PyObject*& res) const override {
- res = self.GetAttr(attr.c_str());
- return res != nullptr;
- }
- };
-
- TPODTraits::TPODTraits()
- : MyParent("TPOD", "simple struct")
- {
- AddGetter("", new TPODAttrGetter);
- }
-
-}
diff --git a/library/cpp/pybind/pod.h b/library/cpp/pybind/pod.h
deleted file mode 100644
index 90165fdbec..0000000000
--- a/library/cpp/pybind/pod.h
+++ /dev/null
@@ -1,53 +0,0 @@
-#pragma once
-
-#define PY_SSIZE_T_CLEAN
-#include <Python.h>
-#include "attr.h"
-#include "typedesc.h"
-
-namespace NPyBind {
- struct TPOD {
- TPyObjectPtr Dict;
-
- TPOD()
- : Dict(PyDict_New(), true)
- {
- }
- bool SetAttr(const char* name, PyObject* value) {
- return PyDict_SetItemString(Dict.Get(), name, value) == 0;
- }
- PyObject* GetAttr(const char* name) const {
- PyObject* res = PyDict_GetItemString(Dict.Get(), name);
- Py_XINCREF(res);
- return res;
- }
- };
-
- class TPODTraits: public NPyBind::TPythonType<TPOD, TPOD, TPODTraits> {
- private:
- typedef TPythonType<TPOD, TPOD, TPODTraits> MyParent;
- friend class TPythonType<TPOD, TPOD, TPODTraits>;
- TPODTraits();
-
- public:
- static TPOD* GetObject(TPOD& obj) {
- return &obj;
- }
- };
-
- template <>
- inline bool FromPyObject<TPOD*>(PyObject* obj, TPOD*& res) {
- res = TPODTraits::CastToObject(obj);
- if (res == nullptr)
- return false;
- return true;
- }
- template <>
- inline bool FromPyObject<const TPOD*>(PyObject* obj, const TPOD*& res) {
- res = TPODTraits::CastToObject(obj);
- if (res == nullptr)
- return false;
- return true;
- }
-
-}
diff --git a/library/cpp/pybind/ptr.h b/library/cpp/pybind/ptr.h
deleted file mode 100644
index e136736690..0000000000
--- a/library/cpp/pybind/ptr.h
+++ /dev/null
@@ -1,51 +0,0 @@
-#pragma once
-
-#define PY_SSIZE_T_CLEAN
-#include <Python.h>
-#include <util/generic/ptr.h>
-
-namespace NPyBind {
- template <class T>
- class TPythonIntrusivePtrOps {
- public:
- static inline void Ref(T* t) noexcept {
- Py_XINCREF(t);
- }
-
- static inline void UnRef(T* t) noexcept {
- Py_XDECREF(t);
- }
-
- static inline void DecRef(T* t) noexcept {
- Py_XDECREF(t);
- }
- };
-
- class TPyObjectPtr: public TIntrusivePtr<PyObject, TPythonIntrusivePtrOps<PyObject>> {
- private:
- typedef TIntrusivePtr<PyObject, TPythonIntrusivePtrOps<PyObject>> TParent;
- typedef TPythonIntrusivePtrOps<PyObject> TOps;
-
- public:
- inline TPyObjectPtr() noexcept {
- }
-
- inline explicit TPyObjectPtr(PyObject* obj) noexcept
- : TParent(obj)
- {
- }
-
- inline TPyObjectPtr(PyObject* obj, bool unref) noexcept
- : TParent(obj)
- {
- if (unref)
- TOps::UnRef(TParent::Get());
- }
-
- inline PyObject* RefGet() {
- TOps::Ref(TParent::Get());
- return TParent::Get();
- }
- };
-
-}
diff --git a/library/cpp/pybind/typeattrs.h b/library/cpp/pybind/typeattrs.h
deleted file mode 100644
index a906b9ec2b..0000000000
--- a/library/cpp/pybind/typeattrs.h
+++ /dev/null
@@ -1,368 +0,0 @@
-#pragma once
-
-#include "ptr.h"
-#include "cast.h"
-#include "attr.h"
-#include "method.h"
-
-#include <util/generic/vector.h>
-
-namespace NPyBind {
- template <typename TObject>
- class TPythonTypeAttributes {
- private:
- TAttrGetters<TObject> AttrGetters;
- TAttrSetters<TObject> AttrSetters;
- TMethodCallers<TObject> MethodCallers;
-
- class TGetAttrsNamesCaller;
- class TGetMethodsNamesCaller;
- class TGetAllNamesCaller;
- class TGetPropertiesNamesCaller;
- class TDictAttrGetter;
- class TDictAttrSetter;
- class TGetAttributeMethodCaller;
- class TSetAttrMethodCaller;
- class TGetStrReprMethodCaller;
- class TReduceMethodCaller;
- class TBaseGetStateMethodCaller;
- class TBaseSetStateMethodCaller;
-
- TPythonTypeAttributes(const TPythonTypeAttributes&);
- TPythonTypeAttributes& operator=(const TPythonTypeAttributes&);
-
- static const TSet<TString> HiddenAttrNames;
-
- typedef PyObject* (*GetAttrFunction)(PyObject*, char*);
- typedef int (*SetAttrFunction)(PyObject*, char*, PyObject*);
- GetAttrFunction GetAttr;
- SetAttrFunction SetAttr;
-
- public:
- typedef TSimpleSharedPtr<TBaseAttrGetter<TObject>> TGetterPtr;
- typedef TSimpleSharedPtr<TBaseAttrSetter<TObject>> TSetterPtr;
- typedef TSimpleSharedPtr<TBaseMethodCaller<TObject>> TCallerPtr;
-
- TPythonTypeAttributes(GetAttrFunction getAttr, SetAttrFunction setAttr)
- : AttrGetters(HiddenAttrNames)
- , MethodCallers(HiddenAttrNames)
- , GetAttr(getAttr)
- , SetAttr(setAttr)
- {
- }
-
- void InitCommonAttributes() {
- // attributes
- AddGetter("__dict__", new TDictAttrGetter(AttrGetters));
- AddSetter("__dict__", new TDictAttrSetter(AttrSetters));
-
- // methods
- AddCaller("GetAttrsNames", new TGetAttrsNamesCaller(AttrGetters));
- AddCaller("GetMethodsNames", new TGetMethodsNamesCaller(MethodCallers));
- AddCaller("GetAllNames", new TGetAllNamesCaller(AttrGetters, MethodCallers));
- AddCaller("GetPropertiesNames", new TGetPropertiesNamesCaller(MethodCallers));
- AddCaller("__getattribute__", new TGetAttributeMethodCaller(GetAttr));
- AddCaller("__setattr__", new TSetAttrMethodCaller(SetAttr));
- AddCaller("__str__", new TGetStrReprMethodCaller("__str__"));
- AddCaller("__repr__", new TGetStrReprMethodCaller("__repr__"));
- AddCaller("__reduce_ex__", new TReduceMethodCaller);
- AddCaller("__reduce__", new TReduceMethodCaller);
- AddCaller("__getstate__", new TBaseGetStateMethodCaller);
- AddCaller("__setstate__", new TBaseSetStateMethodCaller);
-
- // generics
- AddGetter("__class__", new TGenericAttrGetter<TObject>("__class__"));
- AddGetter("__doc__", new TGenericAttrGetter<TObject>("__doc__"));
- AddCaller("__sizeof__", new TGenericMethodCaller<TObject>("__sizeof__"));
- AddCaller("__hash__", new TGenericMethodCaller<TObject>("__hash__"));
- }
-
- void AddGetter(const TString& attr, TGetterPtr getter) {
- AttrGetters.AddGetter(attr, getter);
- }
-
- void AddSetter(const TString& attr, TSetterPtr setter) {
- AttrSetters.AddSetter(attr, setter);
- }
-
- void AddCaller(const TString& name, TCallerPtr caller) {
- MethodCallers.AddCaller(name, caller);
- }
-
- const TAttrGetters<TObject>& GetAttrGetters() const {
- return AttrGetters;
- }
-
- TAttrSetters<TObject>& GetAttrSetters() {
- return AttrSetters;
- }
-
- const TMethodCallers<TObject>& GetMethodCallers() const {
- return MethodCallers;
- }
-
- const TSet<TString>& GetHiddenAttrs() const {
- return HiddenAttrNames;
- }
- };
-
- template <typename TObjType>
- class TPythonTypeAttributes<TObjType>::TGetAttrsNamesCaller: public TBaseMethodCaller<TObjType> {
- private:
- const TAttrGetters<TObjType>& AttrGetters;
-
- public:
- TGetAttrsNamesCaller(const TAttrGetters<TObjType>& getters)
- : AttrGetters(getters)
- {
- }
-
- bool CallMethod(PyObject* owner, TObjType* self, PyObject* args, PyObject*, PyObject*& res) const override {
- if (!ExtractArgs(args))
- ythrow yexception() << "Could not parse args for GetAttrsNames() - it should be none";
- TVector<TString> names;
- AttrGetters.GetAttrsNames(owner, *self, names);
- res = BuildPyObject(names);
- return (res != nullptr);
- }
- };
-
- template <typename TObjType>
- class TPythonTypeAttributes<TObjType>::TGetMethodsNamesCaller: public TBaseMethodCaller<TObjType> {
- private:
- const TMethodCallers<TObjType>& MethodCallers;
-
- public:
- TGetMethodsNamesCaller(const TMethodCallers<TObjType>& callers)
- : MethodCallers(callers)
- {
- }
-
- bool CallMethod(PyObject* owner, TObjType* self, PyObject* args, PyObject*, PyObject*& res) const override {
- if (!ExtractArgs(args))
- ythrow yexception() << "Could not parse args for GetMethodsNames() - it should be none";
- TVector<TString> names;
- MethodCallers.GetMethodsNames(owner, self, names);
- res = BuildPyObject(names);
- return (res != nullptr);
- }
- };
-
- template <typename TObjType>
- class TPythonTypeAttributes<TObjType>::TGetAllNamesCaller: public TBaseMethodCaller<TObjType> {
- private:
- const TAttrGetters<TObjType>& AttrGetters;
- const TMethodCallers<TObjType>& MethodCallers;
-
- public:
- TGetAllNamesCaller(const TAttrGetters<TObjType>& getters,
- const TMethodCallers<TObjType>& callers)
- : AttrGetters(getters)
- , MethodCallers(callers)
- {
- }
-
- bool CallMethod(PyObject* owner, TObjType* self, PyObject* args, PyObject*, PyObject*& res) const override {
- if (!ExtractArgs(args))
- ythrow yexception() << "Could not parse args for GetAllNames() - it should be none";
- TVector<TString> names;
- AttrGetters.GetAttrsNames(owner, *self, names);
- MethodCallers.GetMethodsNames(owner, self, names);
- res = BuildPyObject(names);
- return (res != nullptr);
- }
- };
-
- template <typename TObjType>
- class TPythonTypeAttributes<TObjType>::TGetPropertiesNamesCaller: public TBaseMethodCaller<TObjType> {
- private:
- const TMethodCallers<TObjType>& MethodCallers;
-
- public:
- TGetPropertiesNamesCaller(const TMethodCallers<TObjType>& callers)
- : MethodCallers(callers)
- {
- }
-
- public:
- bool CallMethod(PyObject* obj, TObjType* self, PyObject* args, PyObject*, PyObject*& res) const override {
- if (!ExtractArgs(args))
- return false;
-
- TVector<TString> names;
- MethodCallers.GetPropertiesNames(obj, self, names);
- res = BuildPyObject(names);
- return (res != nullptr);
- }
- };
-
- template <typename TObjType>
- class TPythonTypeAttributes<TObjType>::TDictAttrGetter: public TBaseAttrGetter<TObjType> {
- private:
- TAttrGetters<TObjType>& AttrGetters;
-
- public:
- TDictAttrGetter(TAttrGetters<TObjType>& getters)
- : AttrGetters(getters)
- {
- }
-
- bool GetAttr(PyObject* owner, const TObjType& self, const TString&, PyObject*& res) const override {
- TMap<TString, PyObject*> dict;
- AttrGetters.GetAttrsDictionary(owner, self, dict);
- res = BuildPyObject(dict);
- return (res != nullptr);
- }
- };
-
- template <typename TObjType>
- class TPythonTypeAttributes<TObjType>::TDictAttrSetter: public TBaseAttrSetter<TObjType> {
- private:
- TAttrSetters<TObjType>& AttrSetters;
-
- public:
- TDictAttrSetter(TAttrSetters<TObjType>& setters)
- : AttrSetters(setters)
- {
- }
-
- bool SetAttr(PyObject* owner, TObjType& self, const TString&, PyObject* val) override {
- TMap<TString, PyObject*> dict;
- if (!FromPyObject(val, dict))
- ythrow yexception() << "'__dict__' should be set to dictionary";
- if (!AttrSetters.SetAttrDictionary(owner, self, dict))
- return false;
- return true;
- }
- };
-
- template <typename TObjType>
- class TPythonTypeAttributes<TObjType>::TGetAttributeMethodCaller: public TBaseMethodCaller<TObjType> {
- private:
- GetAttrFunction GetAttr;
-
- public:
- TGetAttributeMethodCaller(GetAttrFunction getAttr)
- : GetAttr(getAttr)
- {
- }
-
- bool CallMethod(PyObject* owner, TObjType*, PyObject* args, PyObject*, PyObject*& res) const override {
- TString attrName;
- if (!ExtractArgs(args, attrName))
- ythrow yexception() << "Could not parse args for '__getattribute__' - it should be one string";
- res = GetAttr(owner, const_cast<char*>(attrName.c_str()));
- if (!res)
- // Error already set
- return false;
- return true;
- }
- };
-
- template <typename TObjType>
- class TPythonTypeAttributes<TObjType>::TSetAttrMethodCaller: public TBaseMethodCaller<TObjType> {
- private:
- SetAttrFunction SetAttr;
-
- public:
- TSetAttrMethodCaller(SetAttrFunction setAttr)
- : SetAttr(setAttr)
- {
- }
-
- bool CallMethod(PyObject* owner, TObjType*, PyObject* args, PyObject*, PyObject*& res) const override {
- TString attrName;
- TPyObjectPtr value;
- if (!ExtractArgs(args, attrName, value))
- ythrow yexception() << "Could not parse args for '__setattr__' - it should be one string and value";
- Py_INCREF(Py_None);
- res = Py_None;
- if (-1 == SetAttr(owner, const_cast<char*>(attrName.c_str()), value.Get()))
- // Error already set
- return false;
- return true;
- }
- };
-
- template <typename TObjType>
- class TPythonTypeAttributes<TObjType>::TGetStrReprMethodCaller: public TBaseMethodCaller<TObjType> {
- private:
- TString MethodName;
-
- private:
- const TString GetFullName(PyObject* obj) const {
- TString module, name;
- TPyObjectPtr type(PyObject_Type(obj), true);
- if (!FromPyObject(PyObject_GetAttrString(type.Get(), "__module__"), module) || !FromPyObject(PyObject_GetAttrString(type.Get(), "__name__"), name))
- ythrow yexception() << "Could not get name of object";
- return module + "." + name;
- }
-
- public:
- TGetStrReprMethodCaller(const TString& methodName)
- : MethodName(methodName)
- {
- }
-
- bool CallMethod(PyObject* owner, TObjType*, PyObject* args, PyObject*, PyObject*& res) const override {
- if (args && !ExtractArgs(args))
- ythrow yexception() << "Could not parse args for '" << MethodName << "'";
- TString message = TString("<") + GetFullName(owner) + " object>";
- res = ReturnString(message);
- return (res != nullptr);
- }
- };
-
- template <typename TObjType>
- class TPythonTypeAttributes<TObjType>::TReduceMethodCaller: public TBaseMethodCaller<TObjType> {
- public:
- bool CallMethod(PyObject* owner, TObjType*, PyObject*, PyObject*, PyObject*& res) const override {
- TPyObjectPtr tuple(PyTuple_New(3), true);
- // First component: reconstructor
- TPyObjectPtr pybindName(BuildPyObject("pybind"), true);
- TPyObjectPtr mainModule(PyImport_Import(pybindName.Get()), true);
- TPyObjectPtr recName(BuildPyObject("PyBindObjectReconstructor"), true);
- TPyObjectPtr reconstructor(PyObject_GetAttr(mainModule.Get(), recName.Get()), true);
- // Second component: arguments to rebuild object
- TPyObjectPtr arguments(PyTuple_New(2), true);
- TPyObjectPtr cl(PyObject_GetAttrString(owner, "__class__"), true);
- PyTuple_SET_ITEM(arguments.Get(), 0, cl.RefGet());
- TPyObjectPtr props(PyObject_CallMethod(owner, const_cast<char*>("GetPropertiesNames"), nullptr), true);
- PyTuple_SET_ITEM(arguments.Get(), 1, props.RefGet());
- // Third component: state to fill new object
- TPyObjectPtr state(PyObject_CallMethod(owner, const_cast<char*>("__getstate__"), nullptr), true);
-
- PyTuple_SET_ITEM(tuple.Get(), 0, reconstructor.RefGet());
- PyTuple_SET_ITEM(tuple.Get(), 1, arguments.RefGet());
- PyTuple_SET_ITEM(tuple.Get(), 2, state.RefGet());
- res = tuple.RefGet();
- return (res != nullptr);
- }
- };
-
- template <typename TObjType>
- class TPythonTypeAttributes<TObjType>::TBaseGetStateMethodCaller: public TGetStateCaller<TObjType, TObjType> {
- public:
- void GetAttrsDictionary(PyObject* obj, TObjType* self, TMap<TString, TPyObjectPtr>& dict) const override {
- this->GetStandartAttrsDictionary(obj, self, dict);
- }
- };
-
- template <typename TObjType>
- class TPythonTypeAttributes<TObjType>::TBaseSetStateMethodCaller: public TSetStateCaller<TObjType, TObjType> {
- public:
- void SetAttrsDictionary(PyObject* obj, TObjType* self, TMap<TString, TPyObjectPtr>& dict) const override {
- this->SetStandartAttrsDictionary(obj, self, dict);
- }
- };
-
- static const char* HiddenAttrStrings[] = {
- "__dict__", "__class__", "__dir__", "__delattr__", "__doc__", "__format__", "__getattribute__", "__hash__",
- "__init__", "__new__", "__reduce__", "__reduce_ex__", "__repr__", "__setattr__", "__sizeof__", "__str__",
- "__subclasshook__", "__getstate__", "__setstate__",
- "GetAttrsNames", "GetMethodsNames", "GetAllNames", "GetPropertiesNames"};
-
- template <typename T>
- const TSet<TString> TPythonTypeAttributes<T>::HiddenAttrNames(HiddenAttrStrings, std::end(HiddenAttrStrings));
-
-}
diff --git a/library/cpp/pybind/typedesc.cpp b/library/cpp/pybind/typedesc.cpp
deleted file mode 100644
index 75f39fd126..0000000000
--- a/library/cpp/pybind/typedesc.cpp
+++ /dev/null
@@ -1,79 +0,0 @@
-#include "typedesc.h"
-
-#include <util/generic/singleton.h>
-
-static void RegisterJSONBridgeImpl() {
- PyRun_SimpleString("import json\n"
- "class PyBindEncoder(json.JSONEncoder):\n"
- " def default(self, obj):\n"
- " if isinstance(obj, bytes):\n"
- " try:\n"
- " return obj.decode()\n"
- " except UnicodeDecodeError:\n"
- " return obj.hex()\n"
- " dct = None\n"
- " if hasattr(obj, '__getstate__'):\n"
- " dct = obj.__getstate__()\n"
- " elif hasattr(obj, '__dict__'):\n"
- " dct = obj.__dict__\n"
- " if dct is None:\n"
- " return json.JSONEncoder.default(self, obj)\n"
- " if hasattr(obj, '__class__'):\n"
- " if hasattr(obj.__class__, '__name__'):\n"
- " dct['__name__'] = obj.__class__.__name__\n"
- " if hasattr(obj.__class__, '__module__'):\n"
- " dct['__module__'] = obj.__class__.__module__\n"
- " if hasattr(obj, 'GetPropertiesNames'):\n"
- " dct['__properties__'] = obj.GetPropertiesNames()\n"
- " return dct");
-
- PyRun_SimpleString("def PyBindObjectHook(dct):\n"
- " if '__name__' in dct:\n"
- " name = dct['__name__']\n"
- " module = dct['__module__']\n"
- " del dct['__name__']\n"
- " del dct['__module__']\n"
- " cls = getattr(__import__(module), name)\n"
- " if '__properties__' in dct:\n"
- " props = dct['__properties__']\n"
- " del dct['__properties__']\n"
- " if len(props) == 0:\n"
- " return dct\n"
- " instance = cls(__properties__ = props)\n"
- " else:\n"
- " instance = cls()\n"
- " if hasattr(instance, '__setstate__'):\n"
- " instance.__setstate__(dct)\n"
- " elif hasattr(instance, '__dict__'):\n"
- " instance.__dict__ = dct\n"
- " else:\n"
- " return dct\n"
- " return instance\n"
- " return dct");
-
- PyRun_SimpleString("def json_dump(*args, **kwargs):\n"
- " kwargs['cls'] = PyBindEncoder\n"
- " return json.dump(*args, **kwargs)\n"
- "def json_dumps(*args, **kwargs):\n"
- " kwargs['cls'] = PyBindEncoder\n"
- " return json.dumps(*args, **kwargs)");
-
- PyRun_SimpleString("def json_load(*args, **kwargs):\n"
- " kwargs['object_hook'] = PyBindObjectHook\n"
- " return json.load(*args, **kwargs)\n"
- "def json_loads(*args, **kwargs):\n"
- " kwargs['object_hook'] = PyBindObjectHook\n"
- " return json.loads(*args, **kwargs)");
-}
-
-namespace {
- struct TJSONBridge {
- TJSONBridge() {
- RegisterJSONBridgeImpl();
- }
- };
-}
-
-void NPyBind::RegisterJSONBridge() {
- Singleton<TJSONBridge>();
-}
diff --git a/library/cpp/pybind/typedesc.h b/library/cpp/pybind/typedesc.h
deleted file mode 100644
index 57eacb0f3a..0000000000
--- a/library/cpp/pybind/typedesc.h
+++ /dev/null
@@ -1,545 +0,0 @@
-#pragma once
-
-#define PY_SSIZE_T_CLEAN
-#include <Python.h>
-#include <structmember.h>
-
-#include "typeattrs.h"
-#include "exceptions.h"
-#include "module.h"
-
-namespace NPyBind {
- void RegisterJSONBridge();
-
- namespace NPrivate {
- template <typename>
- class TUnboundClosureHolder;
- template <typename>
- class TUnboundClosure;
- }
-
- // TTraits should be derived from TPythonType
- template <typename TObjectHolder, typename TObject, typename TTraits>
- class TPythonType {
- private:
- TPythonType(const TPythonType&);
- TPythonType& operator=(const TPythonType&);
-
- private:
- typedef typename TPythonTypeAttributes<TObject>::TGetterPtr TGetterPtr;
- typedef typename TPythonTypeAttributes<TObject>::TSetterPtr TSetterPtr;
- typedef typename TPythonTypeAttributes<TObject>::TCallerPtr TCallerPtr;
-
- struct TProxy {
- PyObject_HEAD
- TObjectHolder* Holder;
- };
-
- static PyTypeObject PyType;
- static PyMappingMethods MappingMethods;
- static PyObject* PyTypeObjPtr;
- protected:
- static PyTypeObject* GetPyTypePtr() {
- return &PyType;
- }
- private:
-
- TPythonTypeAttributes<TObject> Attributes;
-
- static int InitObject(PyObject* s, PyObject* args, PyObject* kwargs) {
- try {
- TProxy* self = reinterpret_cast<TProxy*>(s);
- auto str = NameFromString("__properties__");
- if (kwargs && PyDict_Check(kwargs) && PyDict_Contains(kwargs, str.Get())) {
- TPyObjectPtr props(PyDict_GetItem(kwargs, str.Get()));
- TVector<TString> properties;
- FromPyObject(props.Get(), properties);
- self->Holder = TTraits::DoInitPureObject(properties);
- } else {
- self->Holder = (args || kwargs) ? TTraits::DoInitObject(args, kwargs) : nullptr;
- }
- if (PyErr_Occurred())
- return -1;
- return 0;
- } catch (const std::exception& ex) {
- PyErr_SetString(TExceptionsHolder::Instance().ToPyException(ex).Get(), ex.what());
- } catch (...) {
- PyErr_SetString(PyExc_RuntimeError, "Unknown error occurred while trying to init object");
- }
- return -1;
- }
-
- static void DeallocObject(TProxy* self) {
- delete self->Holder;
- Py_TYPE(self)->tp_free(reinterpret_cast<PyObject*>(self));
- }
-
- static PyObject* GetObjectAttr(PyObject* pyObj, char* attr);
- static int SetObjectAttr(PyObject* pyObj, char* attr, PyObject* value);
- static PyObject* GetStr(PyObject*);
- static PyObject* GetRepr(PyObject*);
- static PyObject* GetIter(PyObject*);
- static PyObject* GetNext(PyObject*);
-
- // Fill class __dict__ with functions to make sure methods names will get to dir()
- void FillClassDict() const {
- TVector<TString> names;
- Attributes.GetMethodCallers().GetAllMethodsNames(names);
- for (const auto& name : names) {
- TPyObjectPtr callable = NPrivate::TUnboundClosure<TObject>::Instance().CreatePyObject(new NPrivate::TUnboundClosureHolder<TObject>(&PyType, name));
- PyDict_SetItemString(PyType.tp_dict, name.c_str(), callable.Get());
- }
- }
-
- void InitCommonAttributes() {
- static bool was = false;
- if (was)
- return;
- was = true;
- Attributes.InitCommonAttributes();
- FillClassDict();
- }
-
- protected:
- TPythonType(const char* pyTypeName, const char* typeDescr, PyTypeObject* parentType = nullptr)
- : Attributes(GetObjectAttr, SetObjectAttr)
- {
- PyType.tp_name = pyTypeName;
- PyType.tp_doc = typeDescr;
- Py_INCREF(PyTypeObjPtr);
- if (parentType) {
- Py_INCREF(parentType);
- PyType.tp_base = parentType;
- }
- PyType_Ready(&PyType);
-
- TExceptionsHolder::Instance();
- RegisterJSONBridge();
-
- }
-
- ~TPythonType() {
- }
-
- static TObjectHolder* DoInitObject(PyObject*, PyObject*) {
- return nullptr;
- }
-
- static TObjectHolder* DoInitPureObject(const TVector<TString>&) {
- return nullptr;
- }
-
- static void SetClosure(PyObject* (*call)(PyObject*, PyObject*, PyObject*)) {
- PyType.tp_call = call;
- }
-
- public:
- void AddGetter(const TString& attr, TGetterPtr getter) {
- Attributes.AddGetter(attr, getter);
- }
-
- void AddSetter(const TString& attr, TSetterPtr setter) {
- Attributes.AddSetter(attr, setter);
- }
-
- void AddCaller(const TString& name, TCallerPtr caller) {
- Attributes.AddCaller(name, caller);
- if (name == "__iter__") {
- PyType.tp_iter = GetIter;
- }
- if (name == "next") {
- PyType.tp_iternext = GetNext;
- }
- }
-
- void SetIter(getiterfunc tp_iter) {
- PyType.tp_iter = tp_iter;
- }
-
- void SetIterNext(iternextfunc tp_iternext) {
- PyType.tp_iternext = tp_iternext;
- }
-
- void SetDestructor(destructor tp_dealloc) {
- PyType.tp_dealloc = tp_dealloc;
- }
-
- void SetLengthFunction(lenfunc mp_length) {
- PyType.tp_as_mapping->mp_length = mp_length;
- }
-
- void SetSubscriptFunction(binaryfunc mp_subscript) {
- PyType.tp_as_mapping->mp_subscript = mp_subscript;
- }
-
- void SetAssSubscriptFunction(objobjargproc mp_ass_subscript) {
- PyType.tp_as_mapping->mp_ass_subscript = mp_ass_subscript;
- }
-
- typedef TObject TObjectType;
-
- static TPythonType& Instance() {
- static TTraits Traits;
- Traits.InitCommonAttributes();
- return Traits;
- }
-
- void Register(PyObject* module, const char* typeName) {
- Py_INCREF(PyTypeObjPtr);
- if (0 != PyModule_AddObject(module, typeName, PyTypeObjPtr))
- ythrow yexception() << "can't register type \"" << typeName << "\"";
- }
-
- void Register(PyObject* module, const char* objName, TObjectHolder* hld) {
- if (0 != PyModule_AddObject(module, objName, CreatePyObject(hld).RefGet()))
- ythrow yexception() << "can't register object \"" << objName << "\"";
- }
-
- void Register(TPyObjectPtr module, const TString& typeName) {
- Register(module.Get(), typeName.c_str());
- }
-
- void Register(TPyObjectPtr module, const TString& objName, TObjectHolder* hld) {
- Register(module.Get(), objName.c_str(), hld);
- }
-
- static TObjectHolder* CastToObjectHolder(PyObject* obj) {
- // Call Instance() to make sure PyTypeObjPtr is already created at this point
- Instance();
- if (!PyObject_IsInstance(obj, PyTypeObjPtr))
- return nullptr;
- TProxy* prx = reinterpret_cast<TProxy*>(obj);
- return prx ? prx->Holder : nullptr;
- }
-
- static TObject* CastToObject(PyObject* obj) {
- TObjectHolder* hld = CastToObjectHolder(obj);
- return hld ? TTraits::GetObject(*hld) : nullptr;
- }
-
- static TPyObjectPtr CreatePyObject(TObjectHolder* hld) {
- TPyObjectPtr r(_PyObject_New(&PyType), true);
- TProxy* prx = reinterpret_cast<TProxy*>(r.Get());
- if (prx)
- prx->Holder = hld;
- return r;
- }
- };
-
- template <typename TObjectHolder, typename TObject, typename TTraits>
- PyMappingMethods TPythonType<TObjectHolder, TObject, TTraits>::MappingMethods = {nullptr, nullptr, nullptr};
-
- template <typename TObjectHolder, typename TObject, typename TTraits>
- PyTypeObject TPythonType<TObjectHolder, TObject, TTraits>::PyType = {
- PyVarObject_HEAD_INIT(nullptr, 0) "", sizeof(TProxy), 0, (destructor)&DeallocObject
-#if PY_VERSION_HEX < 0x030800b4
- , nullptr, /*tp_print*/
-#endif
-#if PY_VERSION_HEX >= 0x030800b4
- , 0, /*tp_vectorcall_offset*/
-#endif
- &GetObjectAttr, &SetObjectAttr, nullptr, &GetRepr, nullptr, nullptr, &MappingMethods, nullptr, nullptr, &GetStr, nullptr, nullptr, nullptr,
- Py_TPFLAGS_DEFAULT, "", nullptr, nullptr, nullptr, 0, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, 0, InitObject, PyType_GenericAlloc, PyType_GenericNew, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, 0
-#if PY_MAJOR_VERSION >= 3
- , nullptr
-#endif
-#if PY_VERSION_HEX >= 0x030800b1
- , nullptr /*tp_vectorcall*/
-#endif
-#if PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x03090000
- , nullptr /*tp_print*/
-#endif
-#if PY_VERSION_HEX >= 0x030C0000
- , 0 /*tp_watched*/
-#endif
- };
-
- template <typename TObjectHolder, typename TObject, typename TTraits>
- PyObject* TPythonType<TObjectHolder, TObject, TTraits>::PyTypeObjPtr =
- reinterpret_cast<PyObject*>(&TPythonType<TObjectHolder, TObject, TTraits>::PyType);
-
- namespace NPrivate {
- template <typename TObject>
- class TUnboundClosureHolder {
- private:
- THolder<PyTypeObject> Holder;
- TString Method;
-
- public:
- TUnboundClosureHolder(PyTypeObject* ptr, const TString& meth)
- : Holder(ptr)
- , Method(meth)
- {
- }
-
- PyTypeObject* GetObject() const {
- return Holder.Get();
- }
-
- const TString GetMethod() const {
- return Method;
- }
-
- PyObject* Call(PyObject* obj, PyObject* args, PyObject*) const {
- TPyObjectPtr callable(PyObject_GetAttrString(obj, Method.c_str()), true);
- if (!callable.Get())
- ythrow yexception() << "PyBind can't call method '" << Method << "'";
- TPyObjectPtr res(PyObject_CallObject(callable.Get(), args), true);
- if (!res.Get() && !PyErr_Occurred())
- ythrow yexception() << "PyBind can't call method '" << Method << "'";
- return res.RefGet();
- }
- };
-
- template <typename TObject>
- class TUnboundClosure: public NPyBind::TPythonType<TUnboundClosureHolder<TObject>, PyTypeObject, TUnboundClosure<TObject>> {
- private:
- typedef class NPyBind::TPythonType<TUnboundClosureHolder<TObject>, PyTypeObject, TUnboundClosure<TObject>> TParent;
- friend class NPyBind::TPythonType<TUnboundClosureHolder<TObject>, PyTypeObject, TUnboundClosure<TObject>>;
-
- class TReprMethodCaller: public TBaseMethodCaller<PyTypeObject> {
- public:
- bool CallMethod(PyObject* closure, PyTypeObject*, PyObject*, PyObject*, PyObject*& res) const override {
- TUnboundClosureHolder<TObject>* hld = TParent::CastToObjectHolder(closure);
- TPyObjectPtr type((PyObject*)hld->GetObject());
-
- TString nameStr;
- TPyObjectPtr name(PyObject_GetAttrString(type.Get(), "__name__"), true);
- if (!name.Get() || !FromPyObject(name.Get(), nameStr))
- ythrow yexception() << "Could not get name of object";
-
- TString methodName(hld->GetMethod());
-
- TString message = "<unbound method " + nameStr + "." + methodName + ">";
- res = ReturnString(message);
- return (res != nullptr);
- }
- };
-
- private:
- TUnboundClosure()
- : TParent("", "")
- {
- TParent::AddCaller("__repr__", new TReprMethodCaller());
- TParent::AddCaller("__str__", new TReprMethodCaller());
- TParent::SetClosure(&Call);
- }
-
- static PyObject* Call(PyObject* closure, PyObject* args, PyObject* kwargs) {
- try {
- TUnboundClosureHolder<TObject>* hld = TParent::CastToObjectHolder(closure);
- if (!hld)
- ythrow yexception() << "Can't cast object to TypeHolder";
-
- size_t size = 0;
- if (!PyTuple_Check(args) || (size = PyTuple_Size(args)) < 1)
- ythrow yexception() << "Can't parse first argument: it should be valid object";
- --size;
- TPyObjectPtr obj(PyTuple_GetItem(args, 0));
- TPyObjectPtr newArgs(PyTuple_New(size), true);
-
- for (size_t i = 0; i < size; ++i) {
- TPyObjectPtr item(PyTuple_GetItem(args, i + 1));
- PyTuple_SetItem(newArgs.Get(), i, item.RefGet());
- }
-
- return hld->Call(obj.Get(), newArgs.Get(), kwargs);
- } catch (const std::exception& ex) {
- PyErr_SetString(::NPyBind::TExceptionsHolder::Instance().ToPyException(ex).Get(), ex.what());
- } catch (...) {
- PyErr_SetString(PyExc_RuntimeError, "Unknown error occurred while trying to call method");
- }
- return nullptr;
- }
-
- static PyTypeObject* GetObject(TUnboundClosureHolder<TObject>& obj) {
- return obj.GetObject();
- }
- };
-
- template <typename TObject>
- class TBoundClosureHolder {
- private:
- TPyObjectPtr Ptr;
- TObject* Object;
- TString Method;
- const TMethodCallers<TObject>& MethodCallers;
-
- public:
- TBoundClosureHolder(PyObject* ptr, TObject* obj, const TString& meth, const TMethodCallers<TObject>& callers)
- : Ptr(ptr)
- , Object(obj)
- , Method(meth)
- , MethodCallers(callers)
- {
- }
-
- TPyObjectPtr GetObjectPtr() const {
- return Ptr;
- }
-
- TObject* GetObject() const {
- return Object;
- }
-
- const TString GetMethod() const {
- return Method;
- }
-
- PyObject* Call(PyObject* args, PyObject* kwargs) const {
- PyObject* res = MethodCallers.CallMethod(Ptr.Get(), Object, args, kwargs, Method);
- if (res == nullptr && !PyErr_Occurred())
- ythrow yexception() << "PyBind can't call method '" << Method << "'";
- return res;
- }
- };
-
- template <typename TObject>
- class TBoundClosure: public TPythonType<TBoundClosureHolder<TObject>, TObject, TBoundClosure<TObject>> {
- private:
- typedef TPythonType<TBoundClosureHolder<TObject>, TObject, TBoundClosure<TObject>> TMyParent;
- class TReprMethodCaller: public TBaseMethodCaller<TObject> {
- public:
- bool CallMethod(PyObject* closure, TObject*, PyObject*, PyObject*, PyObject*& res) const override {
- TBoundClosureHolder<TObject>* hld = TMyParent::CastToObjectHolder(closure);
- TPyObjectPtr obj(hld->GetObjectPtr());
- TPyObjectPtr type(PyObject_Type(obj.Get()), true);
-
- TString reprStr;
- TPyObjectPtr repr(PyObject_Repr(obj.Get()), true);
- if (!repr.Get() || !FromPyObject(repr.Get(), reprStr))
- ythrow yexception() << "Could not get repr of object";
-
- TString nameStr;
- TPyObjectPtr name(PyObject_GetAttrString(type.Get(), "__name__"), true);
- if (!name.Get() || !FromPyObject(name.Get(), nameStr))
- ythrow yexception() << "Could not get name of object";
-
- TString methodName(hld->GetMethod());
-
- TString message = "<bound method " + nameStr + "." + methodName + " of " + reprStr + ">";
- res = ReturnString(message);
- return (res != nullptr);
- }
- };
-
- private:
- static PyObject* Call(PyObject* closure, PyObject* args, PyObject* kwargs) {
- try {
- TBoundClosureHolder<TObject>* hld = TMyParent::CastToObjectHolder(closure);
- if (!hld)
- ythrow yexception() << "Can't cast object to ClosureHolder";
-
- return hld->Call(args, kwargs);
- } catch (const std::exception& ex) {
- PyErr_SetString(::NPyBind::TExceptionsHolder::Instance().ToPyException(ex).Get(), ex.what());
- } catch (...) {
- PyErr_SetString(PyExc_RuntimeError, "Unknown error occurred while trying to call method");
- }
- return nullptr;
- }
-
- public:
- TBoundClosure()
- : TMyParent("", "")
- {
- TMyParent::AddCaller("__repr__", new TReprMethodCaller());
- TMyParent::AddCaller("__str__", new TReprMethodCaller());
- TMyParent::SetClosure(&Call);
- }
-
- static TObject* GetObject(const TBoundClosureHolder<TObject>& closure) {
- return closure.GetObject();
- }
- };
-
- }
-
- template <typename TObjectHolder, typename TObject, typename TTraits>
- PyObject* TPythonType<TObjectHolder, TObject, TTraits>::GetObjectAttr(PyObject* pyObj, char* attr) {
- try {
- TObject* obj = CastToObject(pyObj);
- PyObject* res = obj ? Instance().Attributes.GetAttrGetters().GetAttr(pyObj, *obj, attr) : nullptr;
- if (res == nullptr && Instance().Attributes.GetMethodCallers().HasMethod(pyObj, obj, attr)) {
- TPyObjectPtr r = NPrivate::TBoundClosure<TObject>::Instance().CreatePyObject(new NPrivate::TBoundClosureHolder<TObject>(pyObj, obj, attr, Instance().Attributes.GetMethodCallers()));
- res = r.RefGet();
- }
- if (res == nullptr && !PyErr_Occurred())
- ythrow TPyErr(PyExc_AttributeError) << "PyBind can't get attribute '" << attr << "'";
- return res;
- } catch (const std::exception& ex) {
- PyErr_SetString(TExceptionsHolder::Instance().ToPyException(ex).Get(), ex.what());
- } catch (...) {
- PyErr_SetString(PyExc_RuntimeError, (TString("Unknown error occurred while trying to get attribute '") + attr + "'").c_str());
- }
- return nullptr;
- }
-
- template <typename TObjectHolder, typename TObject, typename TTraits>
- int TPythonType<TObjectHolder, TObject, TTraits>::SetObjectAttr(PyObject* pyObj, char* attr, PyObject* value) {
- try {
- TObject* obj = CastToObject(pyObj);
- bool res = obj ? Instance().Attributes.GetAttrSetters().SetAttr(pyObj, *obj, attr, value) : false;
- if (!res && !PyErr_Occurred())
- ythrow yexception() << "PyBind can't set attribute '" << attr << "'";
- return res ? 0 : -1;
- } catch (const std::exception& ex) {
- PyErr_SetString(TExceptionsHolder::Instance().ToPyException(ex).Get(), ex.what());
- } catch (...) {
- PyErr_SetString(PyExc_RuntimeError, (TString("Unknown error occurred while trying to set attribute '") + attr + "'").c_str());
- }
- return -1;
- }
-
- template <typename TObjectHolder, typename TObject, typename TTraits>
- PyObject* TPythonType<TObjectHolder, TObject, TTraits>::GetStr(PyObject* obj) {
- try {
- TObject* self = CastToObject(obj);
- return Instance().Attributes.GetMethodCallers().CallMethod(obj, self, nullptr, nullptr, "__str__");
- } catch (const std::exception& ex) {
- PyErr_SetString(TExceptionsHolder::Instance().ToPyException(ex).Get(), ex.what());
- } catch (...) {
- PyErr_SetString(PyExc_RuntimeError, (TString("Unknown error occurred while trying to call '__str__'").c_str()));
- }
- return nullptr;
- }
-
- template <typename TObjectHolder, typename TObject, typename TTraits>
- PyObject* TPythonType<TObjectHolder, TObject, TTraits>::GetIter(PyObject* obj) {
- try {
- TObject* self = CastToObject(obj);
- return Instance().Attributes.GetMethodCallers().CallMethod(obj, self, nullptr, nullptr, "__iter__");
- } catch (const std::exception& ex) {
- PyErr_SetString(TExceptionsHolder::Instance().ToPyException(ex).Get(), ex.what());
- } catch (...) {
- PyErr_SetString(PyExc_RuntimeError, (TString("Unknown error occurred while trying to call '__iter__'").c_str()));
- }
- return nullptr;
- }
-
- template <typename TObjectHolder, typename TObject, typename TTraits>
- PyObject* TPythonType<TObjectHolder, TObject, TTraits>::GetNext(PyObject* obj) {
- try {
- TObject* self = CastToObject(obj);
- return Instance().Attributes.GetMethodCallers().CallMethod(obj, self, nullptr, nullptr, "next");
- } catch (const std::exception& ex) {
- PyErr_SetString(TExceptionsHolder::Instance().ToPyException(ex).Get(), ex.what());
- } catch (...) {
- PyErr_SetString(PyExc_RuntimeError, (TString("Unknown error occurred while trying to call 'next'").c_str()));
- }
- return nullptr;
- }
-
- template <typename TObjectHolder, typename TObject, typename TTraits>
- PyObject* TPythonType<TObjectHolder, TObject, TTraits>::GetRepr(PyObject* obj) {
- try {
- TObject* self = CastToObject(obj);
- return Instance().Attributes.GetMethodCallers().CallMethod(obj, self, nullptr, nullptr, "__repr__");
- } catch (const std::exception& ex) {
- PyErr_SetString(TExceptionsHolder::Instance().ToPyException(ex).Get(), ex.what());
- } catch (...) {
- PyErr_SetString(PyExc_RuntimeError, (TString("Unknown error occurred while trying to call '__repr__'").c_str()));
- }
- return nullptr;
- }
-}
diff --git a/library/cpp/pybind/v2.cpp b/library/cpp/pybind/v2.cpp
deleted file mode 100644
index edce0be719..0000000000
--- a/library/cpp/pybind/v2.cpp
+++ /dev/null
@@ -1,43 +0,0 @@
-#include "v2.h"
-namespace NPyBind {
- namespace Detail {
- template <>
- PyTypeObject* GetParentType<void>(const TPyModuleDefinition&) {
- return nullptr;
- }
-
-
- template <bool InitEnabled>
- void UpdateClassNamesInModule(TPyModuleDefinition& M, const TString& name, PyTypeObject* pythonType) {
- if (!InitEnabled) {
- return;
- }
- M.ClassName2Type[name] = pythonType;
- }
-
- template <bool InitEnabled>
- void UpdateGetContextInModule(TPyModuleDefinition& M, const TString& name, IGetContextBase* base) {
- if (!InitEnabled) {
- return;
- }
- M.Class2ContextGetter[name] = base;
- }
-
- TPyModuleRegistry::TPyModuleRegistry() {
-#if PY_MAJOR_VERSION >= 3
- NPrivate::AddFinalizationCallBack([this]() {
- if (UnnamedModule) {
- UnnamedModule.Clear();
- }
- Name2Def.clear();
- });
-#endif
- }
- template void UpdateClassNamesInModule<false>(TPyModuleDefinition& M, const TString& name, PyTypeObject* pythonType);
- template void UpdateClassNamesInModule<true>(TPyModuleDefinition& M, const TString& name, PyTypeObject* pythonType);
-
-
- template void UpdateGetContextInModule<false>(TPyModuleDefinition& M, const TString& name, IGetContextBase* pythonType);
- template void UpdateGetContextInModule<true>(TPyModuleDefinition& M, const TString& name, IGetContextBase* pythonType);
- }//Detail
-}//NPyBind
diff --git a/library/cpp/pybind/v2.h b/library/cpp/pybind/v2.h
deleted file mode 100644
index f561d6a380..0000000000
--- a/library/cpp/pybind/v2.h
+++ /dev/null
@@ -1,514 +0,0 @@
-#pragma once
-
-#include <library/cpp/pybind/method.h>
-#include <library/cpp/pybind/typedesc.h>
-#include <library/cpp/pybind/module.h>
-#include <util/generic/hash.h>
-#include <util/generic/hash_set.h>
-#include <util/generic/string.h>
-namespace NPyBind {
-#define DEFINE_CONVERTERS_IMPL(TClass) \
- PyObject* BuildPyObject(typename TClass::TBase&& base) { \
- return TClass::BuildPyObject(std::move(base)); \
- } \
- PyObject* BuildPyObject(const typename TClass::TBase& base) { \
- return TClass::BuildPyObject(base); \
- }
-
-#define DEFINE_CONVERTERS(function) DEFINE_CONVERTERS_IMPL(TFunctionResult<decltype(function)>)
-
-#define DEFINE_TRANSFORMERS_IMPL(TClass) \
- template <> \
- bool ::NPyBind::FromPyObject<typename TClass::TBase*>(PyObject * obj, typename TClass::TBase * &res) { \
- res = TClass::CastToObject(obj); \
- return res != nullptr; \
- } \
- template <> \
- bool ::NPyBind::FromPyObject<typename TClass::TBase const*>(PyObject * obj, typename TClass::TBase const*& res) { \
- res = TClass::CastToObject(obj); \
- return res != nullptr; \
- }
-
-#define DEFINE_TRANSFORMERS(function) DEFINE_TRANSFORMERS_IMPL(TFunctionResult<decltype(function)>)
-
- namespace Detail {
- struct IGetContextBase {
- virtual ~IGetContextBase() = default;
- };
- } //Detail
- struct TPyModuleDefinition {
- static void InitModule(const TString& name);
- static TPyModuleDefinition& GetModule();
-
- TString Name;
- NPyBind::TPyObjectPtr M;
- THashMap<TString, PyTypeObject*> ClassName2Type;
- THashMap<TString, Detail::IGetContextBase*> Class2ContextGetter;
- };
-
- namespace Detail {
- // Manages modules lifecycle
- // IMPORTANT!!! Don't use it in PyBind v1 environment, it will lead to inconsistent state of v1 module
- // UnnamedModule-> new unnamed module stub, this stub become current module. In this case you can add functions to it
- // InitModuleWithName -> convert unnamed module into named one, now you can switch to it in switch, this module remains current
- // SwitchToModule switches to the particular module in registry, this module becomes current.
- class TPyModuleRegistry {
- private:
- TPyModuleRegistry();
- TPyModuleRegistry(const TPyModuleRegistry&) = delete;
- TPyModuleRegistry& operator=(TPyModuleRegistry&) = delete;
- public:
- static TPyModuleRegistry& Get() {
- static TPyModuleRegistry registry;
- return registry;
- }
- TPyModuleDefinition& GetCurrentModule() {
- if (!CurrentModule) {
- GetUnnamedModule();
- }
- return *CurrentModule;
- }
-
- TPyModuleDefinition& GetUnnamedModule() {
- if (!UnnamedModule) {
- UnnamedModule = TPyModuleDefinition();
- CurrentModule = const_cast<TPyModuleDefinition*>(UnnamedModule.Get());
- }
- return *UnnamedModule;
- }
-
- TPyModuleDefinition& InitModuleWithName(const TString& name) {
- if (!UnnamedModule) {
- GetUnnamedModule();
- }
- Name2Def[name] = *UnnamedModule;
- UnnamedModule.Clear();
- CurrentModule = &Name2Def[name];
- return *CurrentModule;
- }
-
- TPyModuleDefinition& SwitchToModuleByName(const TString& name) {
- Y_ENSURE(Name2Def.contains(name));
- Y_ENSURE(UnnamedModule.Empty());
- CurrentModule = &Name2Def[name];
- return *CurrentModule;
- }
- private:
- TPyModuleDefinition* CurrentModule = nullptr;
- TMaybe<TPyModuleDefinition> UnnamedModule;//
- THashMap<TString, TPyModuleDefinition> Name2Def;
- };
- }//Detail
-
- inline void TPyModuleDefinition::InitModule(const TString& name) {
- Detail::TPyModuleRegistry::Get().GetUnnamedModule() = TPyModuleDefinition{name, TModuleHolder::Instance().InitModule(name), {}, {}};
- Detail::TPyModuleRegistry::Get().InitModuleWithName(name);
- }
-
- inline TPyModuleDefinition& TPyModuleDefinition::GetModule() {
- return Detail::TPyModuleRegistry::Get().GetCurrentModule();
- }
-
- namespace Detail {
- template <class TPythonType>
- struct TNameCtx {
- TString ClassShortName;
- static TNameCtx& GetNameCtx() {
- static TNameCtx result;
- return result;
- }
- };
- template <class TBase>
- struct TContextImpl {
- PyTypeObject* ParentType = nullptr;
- TString ClassShortName;
- TString ClassFullName;
- TString ClassDescription;
-
-
- TVector<std::pair<TString, typename TPythonTypeAttributes<TBase>::TCallerPtr>> ListCallers;
- TVector<std::pair<TString, typename TPythonTypeAttributes<TBase>::TGetterPtr>> ListGetters;
- TVector<std::pair<TString, typename TPythonTypeAttributes<TBase>::TSetterPtr>> ListSetters;
- };
-
- template <class TObject>
- struct IGetContext: public IGetContextBase {
- virtual ~IGetContext() = default;
- virtual const TContextImpl<TObject>& GetContext() const = 0;
- };
-
- template <typename THolderClass, typename TBaseClass, bool ShouldEnable, typename=std::enable_if_t<!ShouldEnable || !std::is_default_constructible_v<TBaseClass>>>
- THolderClass* DoInitPureObject(const TVector<TString>&) {
- ythrow yexception() << "Can't create this object in pure mode from python";
- }
-
- template <typename THolderClass, typename TBaseClass, bool ShouldEnable, typename=std::enable_if_t<ShouldEnable && std::is_default_constructible_v<TBaseClass>>, typename=void>
- THolderClass* DoInitPureObject(const TVector<TString>&) {
- return new THolderClass(MakeHolder<TBaseClass>());
- }
-
- template <typename T>
- PyTypeObject* GetParentType(const TPyModuleDefinition& m) {
- auto shortName = Detail::TNameCtx<T>::GetNameCtx().ClassShortName;
- auto it = m.ClassName2Type.find(shortName);
- return (it == m.ClassName2Type.end()) ? nullptr : it->second;
- }
-
- template <>
- PyTypeObject* GetParentType<void>(const TPyModuleDefinition&);
-
- template <bool InitEnabled>
- void UpdateClassNamesInModule(TPyModuleDefinition& M, const TString& name, PyTypeObject* pythonType);
-
- template <bool InitEnabled>
- void UpdateGetContextInModule(TPyModuleDefinition& M, const TString& name, IGetContextBase* base);
- }
-
-
- template <class TParentPyClass_=void>
- struct TPyParentClassTraits {
- using TParentPyClass = TParentPyClass_;
- };
-
- template <bool InitEnabled_, class TParentPyClass_=void>
- struct TPyClassConfigTraits: public TPyParentClassTraits<TParentPyClass_> {
- constexpr static bool InitEnabled = InitEnabled_;
- constexpr static bool RawInit = false;
- };
-
- template <class TParentPyClass_=void>
- struct TPyClassRawInitConfigTraits: public TPyParentClassTraits<TParentPyClass_> {
- constexpr static bool InitEnabled = true;
- constexpr static bool RawInit = true;
- };
-
-
- template <typename TBaseClass, typename TPyClassConfigTraits, typename... ConstructorArgs>
- class TPyClass {
- public:
- using TBase = TBaseClass;
- private:
- using TThisClass = TPyClass<TBaseClass, TPyClassConfigTraits, ConstructorArgs...>;
- using TContext = Detail::TContextImpl<TBase>;
- struct THolder {
- ::THolder<TBase> Holder;
- THolder(::THolder<TBase>&& right)
- : Holder(std::move(right))
- {
- }
- THolder(TBase&& right)
- : Holder(MakeHolder<TBase>(std::move(right)))
- {
- }
- };
-
- class TSelectedTraits: public NPyBind::TPythonType<THolder, TBase, TSelectedTraits> {
- private:
- using TParent = NPyBind::TPythonType<THolder, TBase, TSelectedTraits>;
- friend TParent;
-
- public:
- TSelectedTraits()
- : TParent(TThisClass::GetContext().ClassFullName.data(), TThisClass::GetContext().ClassDescription.data(), TThisClass::GetContext().ParentType)
- {
- for (const auto& caller : TThisClass::GetContext().ListCallers) {
- TParent::AddCaller(caller.first, caller.second);
- }
-
- for (const auto& getter : TThisClass::GetContext().ListGetters) {
- TParent::AddGetter(getter.first, getter.second);
- }
-
- for (const auto& setter : TThisClass::GetContext().ListSetters) {
- TParent::AddSetter(setter.first, setter.second);
- }
- }
-
- static TBase* GetObject(const THolder& holder) {
- return holder.Holder.Get();
- }
-
- static THolder* DoInitObject(PyObject* args, PyObject* kwargs) {
- if constexpr (TPyClassConfigTraits::InitEnabled) {
- if constexpr (TPyClassConfigTraits::RawInit) {
- static_assert(sizeof...(ConstructorArgs) == 0, "Do not pass construction args if use RawInit.");
- return new THolder(::MakeHolder<TBase>(args, kwargs));
- } else {
- if (args && (!PyTuple_Check(args) || PyTuple_Size(args) != sizeof...(ConstructorArgs))) {
- ythrow yexception() << "Method takes " << sizeof...(ConstructorArgs) << " arguments, " << PyTuple_Size(args) << " provided";
- }
- ::THolder<TBaseClass> basePtr{Apply([](auto&&... unpackedArgs) {return new TBase(std::forward<decltype(unpackedArgs)>(unpackedArgs)...); }, GetArguments<ConstructorArgs...>(args))};
- return new THolder(std::move(basePtr));
- }
- } else {
- ythrow yexception() << "Can't create this object from python";
- }
- }
-
- static THolder* DoInitPureObject(const TVector<TString>& properties) {
- return Detail::DoInitPureObject<THolder, TBase, TPyClassConfigTraits::InitEnabled>(properties);
- }
-
- static TBase* CastToObject(PyObject* obj) {
- return TParent::CastToObject(obj);
- }
-
- static PyTypeObject* GetType() {
- return TParent::GetPyTypePtr();
- }
- };
-
- class TContextHolder: public Detail::IGetContext<TBaseClass> {
- public:
- static TContextHolder& GetContextHolder() {
- static TContextHolder holder;
- return holder;
- }
-
- TContext& GetContext() {
- return Context;
- }
- const TContext& GetContext() const override {
- return Context;
- }
- private:
- TContext Context;
- };
-
- template <class TDerivedClass, class TSuperClass>
- class TCallerWrapper: public TBaseMethodCaller<TDerivedClass> {
- public:
- explicit TCallerWrapper(TSimpleSharedPtr<const TBaseMethodCaller<TSuperClass>> baseCaller)
- : BaseCaller(baseCaller) {
- Y_ENSURE(BaseCaller);
- }
-
- bool CallMethod(PyObject* owner, TDerivedClass* self, PyObject* args, PyObject* kwargs, PyObject*& res) const override {
- return BaseCaller->CallMethod(owner, static_cast<TSuperClass*>(self), args, kwargs, res);
- }
-
- private:
- TSimpleSharedPtr<const TBaseMethodCaller<TSuperClass>> BaseCaller;
- };
-
- template <class TDerivedClass, class TSuperClass>
- class TSetterWrapper: public TBaseAttrSetter<TDerivedClass> {
- public:
- explicit TSetterWrapper(TSimpleSharedPtr<TBaseAttrSetter<TSuperClass>> baseSetter)
- : BaseSetter(baseSetter) {
- Y_ENSURE(BaseSetter);
- }
-
- bool SetAttr(PyObject* owner, TDerivedClass& self, const TString& attr, PyObject* val) override {
- return BaseSetter->SetAttr(owner, static_cast<TSuperClass&>(self), attr, val);
- }
-
- private:
- TSimpleSharedPtr<TBaseAttrSetter<TSuperClass>> BaseSetter;
- };
-
- template <class TDerivedClass, class TSuperClass>
- class TGetterWrapper: public TBaseAttrGetter<TDerivedClass> {
- public:
- explicit TGetterWrapper(TSimpleSharedPtr<const TBaseAttrGetter<TSuperClass>> baseGetter)
- : BaseGetter(baseGetter) {
- Y_ENSURE(BaseGetter);
- }
-
- bool GetAttr(PyObject* owner, const TDerivedClass& self, const TString& attr, PyObject*& res) const override {
- return BaseGetter->GetAttr(owner, static_cast<const TSuperClass&>(self), attr, res);
- }
-
- private:
- TSimpleSharedPtr<const TBaseAttrGetter<TSuperClass>> BaseGetter;
- };
-
- template <class TSuperClass, typename=std::enable_if_t<!std::is_same_v<TSuperClass, void>>>
- void ReloadAttrsFromBase() {
- auto shortName = Detail::TNameCtx<TSuperClass>::GetNameCtx().ClassShortName;
- if (!M.Class2ContextGetter.count(shortName)) {
- return;
- }
- auto callerBasePtr = M.Class2ContextGetter[shortName];
- if (auto getContextPtr = dynamic_cast<const Detail::IGetContext<TSuperClass>*>(callerBasePtr)) {
- auto& ctx = getContextPtr->GetContext();
- auto getUniqueNames = [](const auto& collection) {
- THashSet<TString> uniqueNames;
- for (const auto& elem : collection) {
- uniqueNames.insert(elem.first);
- }
- return uniqueNames;
- };
-
- auto uniqueCallerNames = getUniqueNames(GetContext().ListCallers);
- using TConcreteCallerWrapper = TCallerWrapper<TBaseClass, TSuperClass>;
- for (const auto& caller : ctx.ListCallers) {
- if (uniqueCallerNames.contains(caller.first)) {
- continue;
- }
- GetContext().ListCallers.push_back(std::make_pair(caller.first, MakeSimpleShared<TConcreteCallerWrapper>(caller.second)));
- }
-
- auto uniqueGettersNames = getUniqueNames(GetContext().ListGetters);
- using TConcreteGetterWrapper = TGetterWrapper<TBaseClass, TSuperClass>;
- for (const auto& getter : ctx.ListGetters) {
- if (uniqueGettersNames.contains(getter.first)) {
- continue;
- }
- GetContext().ListGetters.push_back(std::make_pair(getter.first, MakeSimpleShared<TConcreteGetterWrapper>(getter.second)));
- }
-
- auto uniqueSetterNames = getUniqueNames(GetContext().ListSetters);
- using TConcreteSetterWrapper = TSetterWrapper<TBaseClass, TSuperClass>;
- for (auto& setter : ctx.ListSetters) {
- if (uniqueSetterNames.contains(setter.first)) {
- continue;
- }
- GetContext().ListSetters.push_back(std::make_pair(setter.first, MakeSimpleShared<TConcreteSetterWrapper>(setter.second)));
- }
- }
- }
-
- template <class TSuperClass, typename=std::enable_if_t<std::is_same_v<TSuperClass, void>>, typename=void>
- void ReloadAttrsFromBase() {
- }
-
- void CompleteImpl() {
- ReloadAttrsFromBase<typename TPyClassConfigTraits::TParentPyClass>();
- TSelectedTraits::Instance().Register(M.M, GetContext().ClassShortName);
- }
-
- static TContext& GetContext() {
- return TContextHolder::GetContextHolder().GetContext();
- }
-
-
- friend struct Detail::TContextImpl<TBase>;//instead of context
- friend struct THolder;
- friend class TSelectedTraits;
-
- using TCallerFunc = std::function<bool(PyObject*, TBaseClass*, PyObject*, PyObject*, PyObject*&)>;
- class TFuncCallerWrapper: public TBaseMethodCaller<TBaseClass> {
- public:
- explicit TFuncCallerWrapper(TCallerFunc func)
- : Func(func) {
- Y_ENSURE(func);
- }
-
- bool CallMethod(PyObject* owner, TBaseClass* self, PyObject* args, PyObject* kwargs, PyObject*& res) const override {
- return Func(owner, self, args, kwargs, res);
- }
- private:
- mutable TCallerFunc Func;
- };
- public:
- TPyClass(const TString& name, const TString& descr = "")
- : M(TPyModuleDefinition::GetModule())
- {
- Detail::UpdateClassNamesInModule<TPyClassConfigTraits::InitEnabled>(M, name, TSelectedTraits::GetType());
- Detail::UpdateGetContextInModule<TPyClassConfigTraits::InitEnabled>(M, name, &TContextHolder::GetContextHolder());
-
- GetContext().ClassFullName = TString::Join(M.Name, ".", name);
- GetContext().ClassShortName = name;
- GetContext().ClassDescription = descr;
- GetContext().ParentType = Detail::GetParentType<typename TPyClassConfigTraits::TParentPyClass>(M);
- Detail::TNameCtx<TBaseClass>::GetNameCtx().ClassShortName = name;
- }
-
- template <typename TMemberFuction, typename = std::enable_if_t<std::is_member_function_pointer_v<TMemberFuction>>, typename=std::enable_if_t<!TIsPointerToConstMemberFunction<TMemberFuction>::value>>
- TThisClass& Def(const TString& name, TMemberFuction t) {
- GetContext().ListCallers.push_back(std::make_pair(name, CreateMethodCaller<TBase>(t)));
- return *this;
- }
-
- template <typename TMemberFuction, typename = std::enable_if_t<std::is_member_function_pointer_v<TMemberFuction>>, typename=std::enable_if_t<TIsPointerToConstMemberFunction<TMemberFuction>::value>, typename=void>
- TThisClass& Def(const TString& name, TMemberFuction t) {
- GetContext().ListCallers.push_back(std::make_pair(name, CreateConstMethodCaller<TBase>(t)));
- return *this;
- }
-
- template <typename TMemberObject, typename = std::enable_if_t<std::is_member_object_pointer_v<TMemberObject>>>
- TThisClass& Def(const TString& name, TMemberObject t) {
- GetContext().ListGetters.push_back(std::make_pair(name, CreateAttrGetter<TBase>(t)));
- GetContext().ListSetters.push_back(std::make_pair(name, CreateAttrSetter<TBase>(t)));
- return *this;
- }
-
- template <typename TResultType, typename... Args>
- TThisClass& DefByFunc(const TString& name, std::function<TResultType(TBaseClass&, Args...)> func) {
- GetContext().ListCallers.push_back(std::make_pair(name, CreateFunctorCaller<TBase, TResultType, Args...>(func)));
- return *this;
- }
-
- TThisClass& DefByFunc(const TString& name, TCallerFunc origFunc) {
- GetContext().ListCallers.push_back(std::make_pair(name, MakeSimpleShared<TFuncCallerWrapper>(origFunc)));
- return *this;
- }
-
- template <typename TMemberObject>
- TThisClass& DefReadonly(const TString& name, TMemberObject t, std::enable_if_t<std::is_member_object_pointer<TMemberObject>::value>* = nullptr) {
- GetContext().ListGetters.push_back(std::make_pair(name, CreateAttrGetter<TBase>(t)));
- return *this;
- }
-
-
- template <typename TMethodGetter, typename TMethodSetter, typename=std::enable_if_t<std::is_member_function_pointer_v<TMethodGetter> && std::is_member_function_pointer_v<TMethodSetter>>>
- TThisClass& AsProperty(const TString& name, TMethodGetter getter, TMethodSetter setter) {
- GetContext().ListGetters.push_back(std::make_pair(name, CreateMethodAttrGetter<TBase>(getter)));
- GetContext().ListSetters.push_back(std::make_pair(name, CreateMethodAttrSetter<TBase>(setter)));
- return *this;
- }
-
- template <typename TMethodGetter, typename TMethodSetter, typename=std::enable_if_t<!std::is_member_function_pointer_v<TMethodGetter> && !std::is_member_function_pointer_v<TMethodSetter>>>
- TThisClass& AsPropertyByFunc(const TString& name, TMethodGetter getter, TMethodSetter setter) {
- GetContext().ListGetters.push_back(std::make_pair(name, CreateFunctorAttrGetter<TBase>(getter)));
- GetContext().ListSetters.push_back(std::make_pair(name, CreateFunctorAttrSetter<TBase>(setter)));
- return *this;
- }
-
- template <typename TMethodGetter, typename=std::enable_if_t<std::is_member_function_pointer_v<TMethodGetter>>>
- TThisClass& AsProperty(const TString& name, TMethodGetter getter) {
- GetContext().ListGetters.push_back(std::make_pair(name, CreateMethodAttrGetter<TBase>(getter)));
- return *this;
- }
-
- template <typename TMethodGetter>
- TThisClass& AsPropertyByFunc(const TString& name, TMethodGetter getter) {
- GetContext().ListGetters.push_back(std::make_pair(name, CreateFunctorAttrGetter<TBase>(getter)));
- return *this;
- }
-
- TThisClass& Complete() {
- if (!Completed) {
- CompleteImpl();
- Completed = true;
- }
- return *this;
- }
-
- public:
- static PyObject* BuildPyObject(TBase&& base) {
- return NPyBind::BuildPyObject(TSelectedTraits::Instance().CreatePyObject(new THolder(std::move(base))));
- }
-
- static PyObject* BuildPyObject(const TBase& base) {
- return NPyBind::BuildPyObject(TSelectedTraits::Instance().CreatePyObject(new THolder(TBase(base)))); // WARN - copy
- }
-
- static TBase* CastToObject(PyObject* obj) {
- return TSelectedTraits::CastToObject(obj);
- }
-
- private:
- TPyModuleDefinition& M;
- bool Completed = false;
- };
-
- template <typename TFunctionSignature, TFunctionSignature function>
- void DefImpl(const TString& name, const TString& descr = "") {
- NPyBind::TModuleHolder::Instance().AddModuleMethod<TModuleMethodCaller<TFunctionSignature, function>::Call>(name, descr);
- }
-
-#define DefFunc(NAME, FUNC) NPyBind::DefImpl<decltype(FUNC), FUNC>(NAME)
-#define DefFuncDescr(NAME, FUNC, DESCR) NPyBind::DefImpl<decltype(FUNC), FUNC>(NAME, DESCR)
-};
diff --git a/library/cpp/pybind/ya.make b/library/cpp/pybind/ya.make
deleted file mode 100644
index 9b7b3413f2..0000000000
--- a/library/cpp/pybind/ya.make
+++ /dev/null
@@ -1,14 +0,0 @@
-PY23_NATIVE_LIBRARY()
-
-SRCS(
- cast.cpp
- pod.cpp
- typedesc.cpp
- module.cpp
- exceptions.cpp
- embedding.cpp
- empty.cpp
- v2.cpp
-)
-
-END()
diff --git a/library/python/archive/__init__.py b/library/python/archive/__init__.py
deleted file mode 100644
index a6e032ff4c..0000000000
--- a/library/python/archive/__init__.py
+++ /dev/null
@@ -1,266 +0,0 @@
-import errno
-import logging
-import os
-import random
-import shutil
-import stat
-import string
-import sys
-
-import six
-
-import libarchive
-import libarchive._libarchive as _libarchive
-
-from pathlib2 import PurePath
-
-logger = logging.getLogger(__name__)
-
-GZIP = "gzip"
-ZSTD = "zstd"
-
-ENCODING = "utf-8"
-
-
-class ConfigureError(Exception):
- pass
-
-
-class Level(object):
- def __init__(self, level):
- self.level = level
-
-
-class Compression(object):
- Fast = Level(1)
- Default = Level(2)
- Best = Level(3)
-
-
-def get_compression_level(filter_name, level):
- if level is None or not filter_name:
- return None
- elif isinstance(level, Level):
- level = {
- GZIP: {
- Compression.Fast: 1,
- Compression.Default: 6,
- Compression.Best: 9,
- },
- ZSTD: {
- Compression.Fast: 1,
- Compression.Default: 3,
- Compression.Best: 22,
- },
- }[filter_name][level]
- return level
-
-
-def encode(value, encoding):
- return value.encode(encoding)
-
-
-def extract_tar(tar_file_path, output_dir, strip_components=None, fail_on_duplicates=True):
- output_dir = encode(output_dir, ENCODING)
- _make_dirs(output_dir)
- with libarchive.Archive(tar_file_path, mode="rb") as tarfile:
- for e in tarfile:
- p = _strip_prefix(e.pathname, strip_components)
- if not p:
- continue
- dest = os.path.join(output_dir, encode(p, ENCODING))
- if e.pathname.endswith("/"):
- _make_dirs(dest)
- continue
-
- if strip_components and fail_on_duplicates:
- if os.path.exists(dest):
- raise Exception(
- "The file {} is duplicated because of strip_components={}".format(dest, strip_components)
- )
-
- _make_dirs(os.path.dirname(dest))
-
- if e.ishardlink():
- src = os.path.join(output_dir, _strip_prefix(e.hardlink, strip_components))
- _hardlink(src, dest)
- continue
- if e.issym():
- src = _strip_prefix(e.linkname, strip_components)
- _symlink(src, dest)
- continue
-
- with open(dest, "wb") as f:
- if hasattr(os, "fchmod"):
- os.fchmod(f.fileno(), e.mode & 0o7777)
- libarchive.call_and_check(
- _libarchive.archive_read_data_into_fd,
- tarfile._a,
- tarfile._a,
- f.fileno(),
- )
-
-
-def _strip_prefix(path, strip_components):
- if not strip_components:
- return path
- p = PurePath(path)
- stripped = str(p.relative_to(*p.parts[:strip_components]))
- return '' if stripped == '.' else stripped
-
-
-def tar(
- paths,
- output,
- compression_filter=None,
- compression_level=None,
- fixed_mtime=None,
- onerror=None,
- postprocess=None,
- dereference=False,
-):
- if isinstance(paths, six.string_types):
- paths = [paths]
-
- if isinstance(output, six.string_types):
- temp_tar_path, stream = (
- output + "." + "".join(random.sample(string.ascii_lowercase, 8)),
- None,
- )
- else:
- temp_tar_path, stream = None, output
-
- compression_level = get_compression_level(compression_filter, compression_level)
-
- try:
- if compression_filter:
- filter_name = compression_filter
- if compression_level is not None:
- filter_opts = {"compression-level": str(compression_level)}
- else:
- filter_opts = {}
- # force gzip don't store mtime of the original file being compressed (http://www.gzip.org/zlib/rfc-gzip.html#file-format)
- if fixed_mtime is not None and compression_filter == GZIP:
- filter_opts["timestamp"] = ""
- else:
- filter_name = filter_opts = None
-
- with libarchive.Archive(
- stream or temp_tar_path,
- mode="wb",
- format="gnu",
- filter=filter_name,
- filter_opts=filter_opts,
- fixed_mtime=fixed_mtime,
- ) as tarfile:
- # determine order if fixed_mtime is specified to produce stable archive
- paths = paths if fixed_mtime is None else sorted(paths)
-
- for p in paths:
- if type(p) == tuple:
- path, arcname = p
- else:
- path, arcname = p, os.path.basename(p)
-
- if os.path.isdir(path):
- for root, dirs, files in os.walk(path, followlinks=dereference):
- if fixed_mtime is None:
- entries = dirs + files
- else:
- entries = sorted(dirs) + sorted(files)
-
- reldir = os.path.relpath(root, path)
- for f in entries:
- _writepath(
- tarfile,
- os.path.join(root, f),
- os.path.normpath(os.path.join(arcname, reldir, f)),
- onerror,
- postprocess,
- dereference,
- )
- else:
- if not os.path.exists(path):
- raise OSError("Specified path doesn't exist: {}".format(path))
- _writepath(tarfile, path, arcname, onerror, postprocess, dereference)
-
- if temp_tar_path:
- os.rename(temp_tar_path, output)
- except Exception:
- if temp_tar_path and os.path.exists(temp_tar_path):
- os.remove(temp_tar_path)
- raise
-
-
-def _writepath(tarfile, src, dst, onerror, postprocess, dereference):
- def tar_writepath(src, dst):
- st = os.lstat(src)
- if stat.S_ISREG(st.st_mode) or stat.S_ISDIR(st.st_mode) or stat.S_ISLNK(st.st_mode):
- if dereference and stat.S_ISLNK(st.st_mode):
- src = os.path.realpath(src)
-
- tarfile.writepath(src, dst)
-
- if postprocess:
- postprocess(src, dst, st.st_mode)
- else:
- logger.debug("Skipping non-regular file '%s' (stat: %s)", src, st)
-
- try:
- return tar_writepath(src, dst)
- except Exception as e:
- if isinstance(e, OSError) and e.errno == errno.ENOENT:
- logger.debug(
- "Skipping missing file '%s' - looks like directory content has changed during archiving",
- src,
- )
- return
-
- if onerror:
- if onerror(src, dst, sys.exc_info()):
- return tar_writepath(src, dst)
- else:
- raise
-
-
-def check_tar(tar_file_path):
- if os.path.isfile(tar_file_path) or os.path.islink(tar_file_path):
- return libarchive.is_archive(tar_file_path)
- return False
-
-
-def _make_dirs(path):
- try:
- os.makedirs(path)
- except OSError as e:
- if e.errno != errno.EEXIST or not os.path.isdir(path):
- raise
-
-
-def _hardlink(src, dst):
- if hasattr(os, "link"):
- os.link(src, dst)
- else:
- shutil.copyfile(src, dst)
-
-
-def _symlink(src, dst):
- if hasattr(os, "symlink"):
- os.symlink(src, dst)
- else:
- # Windows specific case - we cannot copy file right now,
- # because it doesn't exist yet (and would be met later in the archive) or symlink is broken.
- # Act like tar and tarfile - skip such symlinks
- if os.path.exists(src):
- shutil.copytree(src, dst)
-
-
-def get_archive_filter_name(filename):
- filters = libarchive.get_archive_filter_names(filename)
- # https://a.yandex-team.ru/arc/trunk/arcadia/contrib/libs/libarchive/libarchive/archive_read.c?rev=5800047#L522
- assert filters[-1] == "none", filters
- if len(filters) == 1:
- return None
- if len(filters) == 2:
- return filters[0]
- raise Exception("Archive has chain of filter: {}".format(filters))
diff --git a/library/python/archive/ya.make b/library/python/archive/ya.make
deleted file mode 100644
index 5b86a45a42..0000000000
--- a/library/python/archive/ya.make
+++ /dev/null
@@ -1,19 +0,0 @@
-PY23_LIBRARY()
-
-STYLE_PYTHON()
-
-PY_SRCS(
- __init__.py
-)
-
-PEERDIR(
- contrib/python/pathlib2
- contrib/python/python-libarchive
-)
-
-END()
-
-RECURSE_FOR_TESTS(
- benchmark
- test
-)
diff --git a/library/python/cityhash/cityhash.pyx b/library/python/cityhash/cityhash.pyx
deleted file mode 100644
index 6f0046f0d7..0000000000
--- a/library/python/cityhash/cityhash.pyx
+++ /dev/null
@@ -1,75 +0,0 @@
-from libcpp.pair cimport pair
-
-cdef extern from "util/system/types.h":
- ctypedef unsigned long ui64
-
-
-cdef extern from "util/digest/city.h":
- ui64 CityHash64(const char* buf, size_t len) nogil
- pair[ui64, ui64] CityHash128(const char* buf, size_t len) nogil
- ui64 CityHash64WithSeed(const char* buf, size_t len, ui64 seed) nogil
-
-
-cdef extern from "library/python/cityhash/hash.h":
- ui64 FileCityHash128WithSeedHigh64(const char* fpath) nogil except+
- ui64 FileCityHash64(const char* fpath) nogil except+
-
-
-def hash64(content):
- cdef const char* s = content
- cdef size_t size = len(content)
- cdef ui64 res = 0
-
- if size > 128:
- with nogil:
- res = CityHash64(s, size)
- else:
- res = CityHash64(s, size)
-
- return res
-
-def hash128(content):
- cdef const char* s = content
- cdef size_t size = len(content)
- cdef pair[ui64, ui64] res = pair[ui64, ui64](0, 0)
-
- if size > 128:
- with nogil:
- res = CityHash128(s, size)
- else:
- res = CityHash128(s, size)
- return res
-
-
-def hash64seed(content, seed):
- cdef const char* s = content
- cdef size_t size = len(content)
- cdef ui64 _seed = seed;
-
- if size > 128:
- with nogil:
- res = CityHash64WithSeed(s, size, _seed)
- else:
- res = CityHash64WithSeed(s, size, _seed)
-
- return res
-
-
-def filehash64(path):
- cdef const char* p = path
- cdef ui64 res = 0
-
- with nogil:
- res = FileCityHash64(p)
-
- return res
-
-
-def filehash128high64(path):
- cdef const char* p = path
- cdef ui64 res = 0
-
- with nogil:
- res = FileCityHash128WithSeedHigh64(p)
-
- return res
diff --git a/library/python/cityhash/hash.cpp b/library/python/cityhash/hash.cpp
deleted file mode 100644
index 17bd3a75f3..0000000000
--- a/library/python/cityhash/hash.cpp
+++ /dev/null
@@ -1,32 +0,0 @@
-#include "hash.h"
-
-#include <util/digest/city.h>
-#include <util/generic/string.h>
-#include <util/memory/blob.h>
-#include <util/system/file.h>
-#include <util/system/fstat.h>
-
-void ReadFile(const char* fpath, TBlob& blob) {
- TFile f(TString{fpath}, RdOnly | Seq);
- const TFileStat fs(f);
- auto size = fs.Size;
-
- if (size < (64 << 10)) {
- blob = TBlob::FromFileContent(f, 0, size);
- } else {
- blob = TBlob::FromFile(f);
- }
-}
-
-ui64 FileCityHash128WithSeedHigh64(const char* fpath) {
- TBlob blob;
- ReadFile(fpath, blob);
- const uint128 hash = CityHash128WithSeed((const char*)blob.Data(), blob.Size(), uint128(0, blob.Size()));
- return Uint128High64(hash);
-}
-
-ui64 FileCityHash64(const char* fpath) {
- TBlob blob;
- ReadFile(fpath, blob);
- return CityHash64(static_cast<const char*>(blob.Data()), blob.Size());
-}
diff --git a/library/python/cityhash/hash.h b/library/python/cityhash/hash.h
deleted file mode 100644
index 64b22ba74b..0000000000
--- a/library/python/cityhash/hash.h
+++ /dev/null
@@ -1,6 +0,0 @@
-#pragma once
-
-#include <util/system/defaults.h>
-
-ui64 FileCityHash128WithSeedHigh64(const char* fpath);
-ui64 FileCityHash64(const char* fpath);
diff --git a/library/python/cityhash/ya.make b/library/python/cityhash/ya.make
deleted file mode 100644
index 7948e19389..0000000000
--- a/library/python/cityhash/ya.make
+++ /dev/null
@@ -1,16 +0,0 @@
-PY23_LIBRARY()
-
-SRCS(
- hash.cpp
-)
-
-PY_SRCS(
- TOP_LEVEL
- cityhash.pyx
-)
-
-END()
-
-RECURSE_FOR_TESTS(
- test
-)
diff --git a/library/python/codecs/__codecs.pyx b/library/python/codecs/__codecs.pyx
deleted file mode 100644
index 42ec37fe88..0000000000
--- a/library/python/codecs/__codecs.pyx
+++ /dev/null
@@ -1,61 +0,0 @@
-import six
-
-from libcpp cimport bool
-
-from util.generic.string cimport TString, TStringBuf
-
-
-def to_bytes(s):
- try:
- return s.encode('utf-8')
- except AttributeError:
- pass
-
- return s
-
-
-def from_bytes(s):
- if six.PY3:
- return s.decode('utf-8')
-
- return s
-
-
-cdef extern from "library/cpp/blockcodecs/codecs.h" namespace "NBlockCodecs":
- cdef cppclass ICodec:
- void Encode(TStringBuf data, TString& res) nogil
- void Decode(TStringBuf data, TString& res) nogil
-
- cdef const ICodec* Codec(const TStringBuf& name) except +
- cdef TString ListAllCodecsAsString() except +
-
-
-def dumps(name, data):
- name = to_bytes(name)
-
- cdef const ICodec* codec = Codec(TStringBuf(name, len(name)))
- cdef TString res
- cdef TStringBuf cdata = TStringBuf(data, len(data))
-
- with nogil:
- codec.Encode(cdata, res)
-
- return res.c_str()[:res.length()]
-
-
-def loads(name, data):
- name = to_bytes(name)
-
- cdef const ICodec* codec = Codec(TStringBuf(name, len(name)))
- cdef TString res
- cdef TStringBuf cdata = TStringBuf(data, len(data))
-
- with nogil:
- codec.Decode(cdata, res)
-
- return res.c_str()[:res.length()]
-
-def list_all_codecs():
- cdef TString res = ListAllCodecsAsString()
-
- return from_bytes(res.c_str()[:res.length()]).split(',')
diff --git a/library/python/codecs/__init__.py b/library/python/codecs/__init__.py
deleted file mode 100644
index b9fb00deb0..0000000000
--- a/library/python/codecs/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-from __codecs import loads, dumps, list_all_codecs # noqa
diff --git a/library/python/codecs/ya.make b/library/python/codecs/ya.make
deleted file mode 100644
index f42d115d5d..0000000000
--- a/library/python/codecs/ya.make
+++ /dev/null
@@ -1,16 +0,0 @@
-PY23_LIBRARY()
-
-PEERDIR(
- library/cpp/blockcodecs
- contrib/python/six
-)
-
-PY_SRCS(
- __init__.py
-)
-
-BUILDWITH_CYTHON_CPP(__codecs.pyx)
-
-PY_REGISTER(__codecs)
-
-END()
diff --git a/library/python/color/README.md b/library/python/color/README.md
deleted file mode 100644
index 9deae40092..0000000000
--- a/library/python/color/README.md
+++ /dev/null
@@ -1,9 +0,0 @@
-Форк ((termcolor https://github.com/termcolor/termcolor/)) для PY23 с дополнительным функционалом.
-
-Может быть использован для конвертации текстовых спецификаций цвета (например, из markup) в esc-последовательности для корректного отображения в терминале.
-
-Пример использования:
-```python
-from library.python.color import tcolor
-tcolor("some text", "green-bold-on_red") -> '\x1b[32m\x1b[41m\x1b[1msome text\x1b[0m'
-```
diff --git a/library/python/color/__init__.py b/library/python/color/__init__.py
deleted file mode 100644
index a70234945e..0000000000
--- a/library/python/color/__init__.py
+++ /dev/null
@@ -1,92 +0,0 @@
-from __future__ import print_function
-
-import copy
-import os
-
-from termcolor import ATTRIBUTES, COLORS, HIGHLIGHTS, RESET
-
-__all__ = [
- "ATTRIBUTES",
- "COLORS",
- "HIGHLIGHTS",
- "RESET",
- "colored",
- "cprint",
- "tcolor",
- "get_color_by_spec"
-]
-
-ATTRIBUTES = copy.deepcopy(ATTRIBUTES)
-ATTRIBUTES["light"] = ATTRIBUTES['bold']
-
-COLORS = copy.deepcopy(COLORS)
-COLORS['gray'] = COLORS['grey']
-COLORS['purple'] = COLORS['magenta']
-COLORS["reset"] = 0
-
-
-def get_code(code):
- if os.getenv("ANSI_COLORS_DISABLED") is None:
- return "\033[{}m".format(code)
- return ""
-
-
-def get_color_by_spec(color_spec):
- color, on_color, attrs = get_spec(color_spec)
- return get_color(color, on_color, attrs)
-
-
-def get_color(color, on_color, attrs):
- res = ""
-
- if color is not None:
- res += get_code(COLORS[color])
-
- if on_color is not None:
- res += get_code(HIGHLIGHTS[on_color])
-
- if attrs is not None:
- for attr in attrs:
- res += get_code(ATTRIBUTES[attr])
-
- return res
-
-
-def get_spec(color_spec):
- """Parses string text color formatting specification.
-
- Arguments:
- color_spec -- string spec for text color formatting, csv string with
- `color` / `bg_color` / `attr` spec items having "-" as a delimiter.
-
- Returns a tuple: (color, bg-color, attributes list)
-
- Example:
- get_spec("green-bold-on_red") -> (32, 41, [1])
- """
- parts = color_spec.split("-")
- color = None
- on_color = None
- attrs = []
- for part in parts:
- part = part.lower()
- if part in COLORS:
- color = part
- if part in HIGHLIGHTS:
- on_color = part
- if part in ATTRIBUTES:
- attrs.append(part)
- return color, on_color, attrs
-
-
-def tcolor(text, color_spec):
- color, on_color, attrs = get_spec(color_spec)
- return colored(text, color=color, on_color=on_color, attrs=attrs)
-
-
-def colored(text, color=None, on_color=None, attrs=None):
- return get_color(color, on_color, attrs) + text + get_code(COLORS["reset"])
-
-
-def cprint(text, color=None, on_color=None, attrs=None, **kwargs):
- print((colored(text, color, on_color, attrs)), **kwargs)
diff --git a/library/python/color/ya.make b/library/python/color/ya.make
deleted file mode 100644
index ff6740b1d4..0000000000
--- a/library/python/color/ya.make
+++ /dev/null
@@ -1,13 +0,0 @@
-PY23_LIBRARY()
-
-LICENSE(MIT)
-
-PY_SRCS(
- __init__.py
-)
-
-PEERDIR(
- contrib/python/termcolor
-)
-
-END()
diff --git a/library/python/compress/__init__.py b/library/python/compress/__init__.py
deleted file mode 100644
index 380ec47dca..0000000000
--- a/library/python/compress/__init__.py
+++ /dev/null
@@ -1,147 +0,0 @@
-from io import open
-
-import struct
-import json
-import os
-import logging
-
-import library.python.par_apply as lpp
-import library.python.codecs as lpc
-
-
-logger = logging.getLogger('compress')
-
-
-def list_all_codecs():
- return sorted(frozenset(lpc.list_all_codecs()))
-
-
-def find_codec(ext):
- def ext_compress(x):
- return lpc.dumps(ext, x)
-
- def ext_decompress(x):
- return lpc.loads(ext, x)
-
- ext_decompress(ext_compress(b''))
-
- return {'c': ext_compress, 'd': ext_decompress, 'n': ext}
-
-
-def codec_for(path):
- for ext in reversed(path.split('.')):
- try:
- return find_codec(ext)
- except Exception as e:
- logger.debug('in codec_for(): %s', e)
-
- raise Exception('unsupported file %s' % path)
-
-
-def compress(fr, to, codec=None, fopen=open, threads=1):
- if codec:
- codec = find_codec(codec)
- else:
- codec = codec_for(to)
-
- func = codec['c']
-
- def iter_blocks():
- with fopen(fr, 'rb') as f:
- while True:
- chunk = f.read(16 * 1024 * 1024)
-
- if chunk:
- yield chunk
- else:
- yield b''
-
- return
-
- def iter_results():
- info = {
- 'codec': codec['n'],
- }
-
- if fr:
- info['size'] = os.path.getsize(fr)
-
- yield json.dumps(info, sort_keys=True) + '\n'
-
- for c in lpp.par_apply(iter_blocks(), func, threads):
- yield c
-
- with fopen(to, 'wb') as f:
- for c in iter_results():
- logger.debug('complete %s', len(c))
- f.write(struct.pack('<I', len(c)))
-
- try:
- f.write(c)
- except TypeError:
- f.write(c.encode('utf-8'))
-
-
-def decompress(fr, to, codec=None, fopen=open, threads=1):
- def iter_chunks():
- with fopen(fr, 'rb') as f:
- cnt = 0
-
- while True:
- ll = f.read(4)
-
- if ll:
- ll = struct.unpack('<I', ll)[0]
-
- if ll:
- if ll > 100000000:
- raise Exception('broken stream')
-
- yield f.read(ll)
-
- cnt += ll
- else:
- if not cnt:
- raise Exception('empty stream')
-
- return
-
- it = iter_chunks()
- extra = []
-
- for chunk in it:
- hdr = {}
-
- try:
- hdr = json.loads(chunk)
- except Exception as e:
- logger.info('can not parse header, suspect old format: %s', e)
- extra.append(chunk)
-
- break
-
- def resolve_codec():
- if 'codec' in hdr:
- return find_codec(hdr['codec'])
-
- if codec:
- return find_codec(codec)
-
- return codec_for(fr)
-
- dc = resolve_codec()['d']
-
- def iter_all_chunks():
- for x in extra:
- yield x
-
- for x in it:
- yield x
-
- with fopen(to, 'wb') as f:
- for c in lpp.par_apply(iter_all_chunks(), dc, threads):
- if c:
- logger.debug('complete %s', len(c))
- f.write(c)
- else:
- break
diff --git a/library/python/compress/ya.make b/library/python/compress/ya.make
deleted file mode 100644
index bbf2a784e2..0000000000
--- a/library/python/compress/ya.make
+++ /dev/null
@@ -1,16 +0,0 @@
-PY23_LIBRARY()
-
-PEERDIR(
- library/python/codecs
- library/python/par_apply
-)
-
-PY_SRCS(
- __init__.py
-)
-
-END()
-
-RECURSE_FOR_TESTS(
- tests
-)
diff --git a/library/python/coredump_filter/README.md b/library/python/coredump_filter/README.md
deleted file mode 100644
index 87b02e7985..0000000000
--- a/library/python/coredump_filter/README.md
+++ /dev/null
@@ -1,19 +0,0 @@
-# Coredump Filter
-
-- ABC: https://abc.yandex-team.ru/services/cores/
-
-Библиотека для разбора (парсинга) трейсов отладчика gdb/lldb, python traceback-ов и minidump-ов.
-
-На вход принимает текст трейса, на выходе - распаршенный текст + возможность преобразовать
-его в html-формат для удобства чтения.
-
-
-## Основные клиенты
-- [Агрегатор стектрейсов](https://wiki.yandex-team.ru/cores-aggregation)
-
-
-## Правила разработки
-
-Библиотека написана таким образом, чтобы файл `__init__.py` мог работать
-без внешних зависимостей. Это позволяет использовать библиотеку даже в Ter1-окружениях.
-На данный момент этот инвариант не обложен тестами (и это следует исправить).
diff --git a/library/python/coredump_filter/__init__.py b/library/python/coredump_filter/__init__.py
deleted file mode 100644
index de0830cd43..0000000000
--- a/library/python/coredump_filter/__init__.py
+++ /dev/null
@@ -1,1500 +0,0 @@
-#!/usr/bin/env python
-# coding: utf-8
-
-from __future__ import print_function
-
-import six
-import enum
-import datetime
-import os
-import re
-import pkgutil
-import sys
-import hashlib
-import json
-import logging
-
-logger = logging.getLogger(__name__)
-
-
-class CoredumpMode(enum.Enum):
- GDB = "gdb"
- LLDB = "lldb"
- SDC_ASSERT = "sdc_assert"
-
-
-ARCADIA_ROOT_LINK = "https://a.yandex-team.ru/arc/trunk/arcadia/"
-
-ARCADIA_ROOT_DIRS = [
- # hottest paths
- "/util/",
- "/contrib/",
- "/library/",
- "/kernel/",
- "/build/",
- "/search/",
-
- # "/gcc-4.8.2/",
-
- # system paths
- # "/lib/x86_64-linux-gnu/",
-
- # all other stuff
- "/aapi/",
- "/addappter/",
- "/adfox/",
- "/admins/",
- "/ads/",
- "/adv/",
- "/advq/",
- "/afisha/",
- "/afro/",
- "/alet/",
- "/alice/",
- "/analytics/",
- "/antiadblock/",
- "/antirobot/",
- "/apphost/",
- "/april/",
- "/arc/",
- "/arcanum/",
- "/augur/",
- "/aurora/",
- "/autocheck/",
- "/balancer/",
- "/bass/",
- "/billing/",
- "/bindings/",
- "/browser/",
- "/build/",
- "/bunker/",
- "/caas/",
- "/canvas/",
- "/captcha/",
- "/catboost/",
- "/certs/",
- "/ci/",
- "/clickhouse/",
- "/client_analytics/",
- "/cloud/",
- "/cmicot/",
- "/cmnt/",
- "/comdep_analytics/",
- "/commerce/",
- "/contrib/",
- "/crm/",
- "/crowdsourcing/",
- "/crypta/",
- "/cv/",
- "/datacloud/",
- "/datalens/",
- "/data-ui/",
- "/devtools/",
- "/dict/",
- "/direct/",
- "/disk/",
- "/distribution/",
- "/distribution_interface/",
- "/district/",
- "/dj/",
- "/docs/",
- "/douber/",
- "/drive/",
- "/edadeal/",
- "/education/",
- "/entity/",
- "/ether/",
- "/extdata/",
- "/extsearch/",
- "/FactExtract/",
- "/fintech/",
- "/frontend/",
- "/fuzzing/",
- "/games/",
- "/gencfg/",
- "/geobase/",
- "/geoproduct/",
- "/geosuggest/",
- "/geotargeting/",
- "/glycine/",
- "/groups/",
- "/haas/",
- "/health/",
- "/helpdesk/",
- "/hitman/",
- "/home/",
- "/htf/",
- "/hw_watcher/",
- "/hypercube/",
- "/iaas/",
- "/iceberg/",
- "/infra/",
- "/intranet/",
- "/inventori/",
- "/ipreg/",
- "/irt/",
- "/it-office/",
- "/jdk/",
- "/juggler/",
- "/junk/",
- "/jupytercloud/",
- "/kernel/",
- "/keyboard/",
- "/kikimr/",
- "/kinopoisk/",
- "/kinopoisk-ott/",
- "/laas/",
- "/lbs/",
- "/library/",
- "/load/",
- "/locdoc/",
- "/logbroker/",
- "/logfeller/",
- "/mail/",
- "/mapreduce/",
- "/maps/",
- "/maps_adv/",
- "/market/",
- "/mb/",
- "/mds/",
- "/media/",
- "/media-billing/",
- "/media-crm/",
- "/mediapers/",
- "/mediaplanner/",
- "/mediastat/",
- "/media-stories/",
- "/metrika/",
- "/milab/",
- "/ml/",
- "/mlp/",
- "/mlportal/",
- "/mobile/",
- "/modadvert/",
- "/ms/",
- "/mssngr/",
- "/music/",
- "/musickit/",
- "/netsys/",
- "/nginx/",
- "/nirvana/",
- "/noc/",
- "/ofd/",
- "/offline_data/",
- "/opensource/",
- "/orgvisits/",
- "/ott/",
- "/packages/",
- "/partner/",
- "/passport/",
- "/payplatform/",
- "/paysys/",
- "/plus/",
- "/portal/",
- "/portalytics/",
- "/pythia/",
- "/quality/",
- "/quasar/",
- "/razladki/",
- "/regulargeo/",
- "/release_machine/",
- "/rem/",
- "/repo/",
- "/rnd_toolbox/",
- "/robot/",
- "/rtc/",
- "/rtline/",
- "/rtmapreduce/",
- "/rt-research/",
- "/saas/",
- "/samogon/",
- "/samsara/",
- "/sandbox/",
- "/scarab/",
- "/sdc/",
- "/search/",
- "/security/",
- "/semantic-web/",
- "/serp/",
- "/sitesearch/",
- "/skynet/",
- "/smart_devices/",
- "/smarttv/",
- "/smm/",
- "/solomon/",
- "/specsearches/",
- "/speechkit/",
- "/sport/",
- "/sprav/",
- "/statbox/",
- "/strm/",
- "/suburban-trains/",
- "/sup/",
- "/switch/",
- "/talents/",
- "/tasklet/",
- "/taxi/",
- "/taxi_efficiency/",
- "/testenv/",
- "/testpalm/",
- "/testpers/",
- "/toloka/",
- "/toolbox/",
- "/tools/",
- "/tracker/",
- "/traffic/",
- "/transfer_manager/",
- "/travel/",
- "/trust/",
- "/urfu/",
- "/vcs/",
- "/velocity/",
- "/vendor/",
- "/vh/",
- "/voicetech/",
- "/weather/",
- "/web/",
- "/wmconsole/",
- "/xmlsearch/",
- "/yabs/",
- "/yadoc/",
- "/yandex_io/",
- "/yaphone/",
- "/ydf/",
- "/ydo/",
- "/yp/",
- "/yql/",
- "/ysite/",
- "/yt/",
- "/yweb/",
- "/zen/",
- "/zapravki/",
- "/zen/",
- "/zootopia/",
- "/zora/",
-]
-
-MY_PATH = os.path.dirname(os.path.abspath(__file__))
-
-# 0.2.x uses stable hashing
-CORE_PROC_VERSION = "0.2.1"
-
-ARCADIA_ROOT_SIGN = "$S/"
-SIGNAL_NOT_FOUND = "signal not found"
-
-
-class SourceRoot(object):
- def __init__(self):
- self.root = None
-
- def detect(self, source):
- if not source:
- # For example, regexp_4
- return
-
- if source.startswith("/-S/"):
- return source[4:]
-
- if source.startswith("../"):
- return source
-
- """
- if self.root is not None:
- return self.root
- """
-
- min_pos = 100000
- for root_dir in ARCADIA_ROOT_DIRS:
- pos = source.find(root_dir)
- if pos < 0:
- continue
-
- if pos < min_pos:
- min_pos = pos
-
- if min_pos < len(source):
- self.root = source[:min_pos + 1]
-
- def crop(self, source):
- if not source:
- return ""
-
- # detection attempt
- self.detect(source)
-
- if self.root is not None:
- return source.replace(self.root, ARCADIA_ROOT_SIGN, 1)
-
- # when traceback contains only ??, source root cannot be detected
- return source
-
-
-def highlight_func(s):
- return (
- s
- .replace("=", '<span class="symbol">=</span>')
- .replace("(", '<span class="symbol">(</span>')
- .replace(")", '<span class="symbol">)</span>')
- )
-
-
-class FrameBase(object):
- def __init__(
- self,
- frame_no=None,
- addr="",
- func="",
- source="",
- source_no="",
- func_name="",
- ):
- self.frame_no = frame_no
- self.addr = addr
- self.func = func
- self.source = source
- self.source_no = source_no
- self.func_name = func_name
-
- def __str__(self):
- return "{}\t{}\t{}".format(
- self.frame_no,
- self.func,
- self.source,
- )
-
- def to_json(self):
- return {
- "frame_no": self.frame_no,
- "addr": self.addr,
- "func": self.func,
- "func_name": self.func_name,
- "source": self.source,
- "source_no": self.source_no,
- }
-
- def fingerprint(self):
- return self.func_name
-
- def cropped_source(self):
- return self.source
-
- def raw(self):
- return "{frame} {func} {source}".format(
- frame=self.frame_no,
- func=self.func,
- source=self.source,
- )
-
- def html(self):
- source, source_fmt = self.find_source()
- return (
- '<span class="frame">{frame}</span>'
- '<span class="func">{func}</span> '
- '<span class="source">{source}</span>{source_fmt}\n'.format(
- frame=self.frame_no,
- func=highlight_func(self.func.replace("&", "&amp;").replace("<", "&lt;")),
- source=source,
- source_fmt=source_fmt,
- )
- )
-
-
-class LLDBFrame(FrameBase):
- SOURCE_NO_RE = re.compile(r"(.*?[^\d]):(\d+)")
- FUNC_RE = re.compile(r"(\w+\s)?(\w+[\w,:,_,<,>,\s,*]+).*$")
-
- def __init__(
- self,
- frame_no=None,
- addr="",
- func="",
- source="",
- source_no="",
- func_name="",
- ):
- super(LLDBFrame, self).__init__(
- frame_no=frame_no,
- addr=addr,
- func=func,
- source=source,
- source_no=source_no,
- func_name=func_name,
- )
- # .source calculation
-
- func = func.replace("(anonymous namespace)::", "")
- m = self.FUNC_RE.match(func)
- if m:
- self.func_name = m.group(2) # overwrite func_name if name is in func
-
- if source_no:
- self.source_no = source_no
- self.source = source
- else:
- m = self.SOURCE_NO_RE.match(source)
- if m:
- self.source = m.group(1)
- self.source_no = m.group(2)
-
- def find_source(self):
- """
- :return: pair (source, source_fmt)
- """
- source_fmt = ""
-
- if self.source_no:
- source_fmt = ' +<span class="source-no">{}</span>'.format(self.source_no)
-
- return self.source, source_fmt
-
-
-class GDBFrame(FrameBase):
- SOURCE_NO_RE = re.compile(r"(.*):(\d+)")
- # #7 0x00007f105f3a221d in NAppHost::NTransport::TCoroutineExecutor::Poll (this=0x7f08416a5d00,
- # tasks=empty TVector (capacity=32)) at /-S/apphost/lib/executors/executors.cpp:373
- # We match with non-greedy regex a function name that cannot contain equal sign
- FUNC_RE = re.compile(r"(.*?) \(([a-zA-Z0-9_]+=.*|)\)$") # function with kwarg-params or zero params
-
- def __init__(
- self,
- frame_no=None,
- addr="",
- func="",
- source="",
- source_no="",
- func_name="",
- ):
- super(GDBFrame, self).__init__(
- frame_no=frame_no,
- addr=addr,
- func=func,
- source=source,
- source_no=source_no,
- func_name=func_name,
- )
- if not source_no:
- m = self.SOURCE_NO_RE.match(source)
- if m:
- self.source = m.group(1)
- self.source_no = m.group(2)
- if not func_name:
- m = self.FUNC_RE.match(self.func)
- if m:
- self.func_name = m.group(1)
-
- def find_source(self):
- """
- Returns link to arcadia if source is path in arcadia, else just string with path
- :return: pair (source, source_fmt)
- """
- source_fmt = ""
- source = ""
- link = ""
- dirs = self.source.split("/")
- if len(dirs) > 1 and "/{dir}/".format(dir=dirs[1]) in ARCADIA_ROOT_DIRS:
- link = self.source.replace(ARCADIA_ROOT_SIGN, ARCADIA_ROOT_LINK)
- else:
- source = self.source
- if self.source_no:
- source_fmt = ' +<span class="source-no">{}</span>'.format(self.source_no)
- if link:
- link += "?#L{line}".format(line=self.source_no)
-
- if link:
- source = '<a href="{link}">{source}</a>'.format(
- link=link,
- source=self.source,
- )
- return source, source_fmt
-
-
-class SDCAssertFrame(LLDBFrame):
-
- def __init__(
- self,
- frame_no=None,
- addr="",
- func="",
- source="",
- source_no="",
- func_name="",
- ):
- super(SDCAssertFrame, self).__init__(
- frame_no=frame_no,
- addr=addr,
- func=func,
- source=source,
- source_no=source_no,
- func_name=func_name,
- )
- # .source calculation
-
- self.source = source or ""
- if isinstance(source_no, str) and len(source_no) > 0:
- source_no = int(source_no, 16)
- self.source_no = source_no or ""
-
- m = self.FUNC_RE.match(func)
- if m:
- self.func_name = m.group(2)
-
-
-class Stack(object):
- # priority classes
- LOW_IMPORTANT = 25
- DEFAULT_IMPORTANT = 50
- SUSPICIOUS_IMPORTANT = 75
- MAX_IMPORTANT = 100
-
- # default coredump's type
- mode = CoredumpMode.GDB
-
- max_depth = None
-
- fingerprint_blacklist = [
- # bottom frames
- "raise",
- "abort",
- "__gnu_cxx::__verbose_terminate_handler",
- "_cxxabiv1::__terminate",
- "std::terminate",
- "__cxxabiv1::__cxa_throw",
- # top frames
- "start_thread",
- "clone",
- "??",
- "__clone",
- "__libc_start_main",
- "_start",
- "__nanosleep",
- ]
-
- fingerprint_blacklist_prefix = ()
-
- suspicious_functions = [
- "CheckedDelete",
- "NPrivate::Panic",
- "abort",
- "close_all_fds",
- "__cxa_throw",
- ]
-
- low_important_functions_eq = [
- "poll ()",
- "recvfrom ()",
- "pthread_join ()",
- ]
-
- low_important_functions_match = [
- "TCommonSockOps::SendV",
- "WaitD (",
- "SleepT (",
- "Join (",
- "epoll_wait",
- "nanosleep",
- "pthread_cond_wait",
- "pthread_cond_timedwait",
- "gsignal",
- "std::detail::_",
- "std::type_info",
- "ros::NodeHandle",
- ]
-
- def __init__(
- self,
- lines=None,
- source_root=None,
- thread_ptr=0,
- thread_id=None,
- frames=None,
- important=None,
- stack_fp=None,
- fingerprint_hash=None,
- stream=None,
- mode=None, # type: CoredumpMode
- ignore_bad_frames=True,
- ):
- self.lines = lines
- self.source_root = source_root
- self.thread_ptr = thread_ptr
- self.thread_id = thread_id
- if mode is not None:
- self.mode = mode
-
- self.frames = frames or []
- if self.frames and isinstance(frames[0], dict):
- self.frames = [self.frame_factory(f) for f in self.frames]
- self.important = important or self.DEFAULT_IMPORTANT
- if thread_id == 1:
- self.important = self.MAX_IMPORTANT
- self.fingerprint_hash = fingerprint_hash
- self.stack_fp = stack_fp
- self.stream = stream
- self.ignore_bad_frames = ignore_bad_frames
-
- def to_json(self):
- """Should be symmetric with `from_json`."""
- return {
- "mode": self.mode.value,
- "frames": [frame.to_json() for frame in self.frames],
- "important": self.important,
- }
-
- @staticmethod
- def from_json(stack):
- """Should be symmetric with `to_json`."""
- mode = CoredumpMode(stack.get("mode", CoredumpMode.GDB.value))
- # old serialization format support, should be dropped
- lldb_mode = stack.get("lldb_mode", False)
- if lldb_mode:
- mode = CoredumpMode.LLDB
-
- unpacked_stack = {
- "mode": mode,
- "frames": stack["frames"],
- "important": stack.get("important", Stack.DEFAULT_IMPORTANT),
- }
- return mode, unpacked_stack
-
- def frame_factory(self, args):
- frames = {
- CoredumpMode.GDB: GDBFrame,
- CoredumpMode.LLDB: LLDBFrame,
- CoredumpMode.SDC_ASSERT: SDCAssertFrame,
- }
-
- class_object = frames.get(self.mode)
- if not class_object:
- raise Exception("Invalid mode: {}".format(self.mode.value))
-
- return class_object(**args)
-
- def low_important(self):
- return self.important <= self.LOW_IMPORTANT
-
- def check_importance(self, frame):
- # raised priority cannot be lowered
- if self.important > self.DEFAULT_IMPORTANT:
- return
-
- # detect suspicious stacks
- for name in self.suspicious_functions:
- if name in frame.func:
- self.important = self.SUSPICIOUS_IMPORTANT
- return
-
- for name in self.low_important_functions_eq:
- if name == frame.func:
- self.important = self.LOW_IMPORTANT
-
- for name in self.low_important_functions_match:
- if name in frame.func:
- self.important = self.LOW_IMPORTANT
-
- def push_frame(self, frame):
- self.check_importance(frame)
- # ignore duplicated frames
- if len(self.frames) and self.frames[-1].frame_no == frame.frame_no:
- return
- self.frames.append(frame)
-
- def parse(self):
- """
- Parse one stack
- """
- assert self.lines is not None
- assert self.source_root is not None
-
- for line in self.lines:
- match_found = False
- for regexp in self.REGEXPS:
- m = regexp.match(line)
- if m:
- frame_args = m.groupdict()
- if "source" in frame_args:
- frame_args["source"] = self.source_root.crop(frame_args["source"])
-
- self.push_frame(self.frame_factory(frame_args))
- match_found = True
- break
-
- if not match_found:
- self.bad_frame(line)
-
- def bad_frame(self, line):
- if self.ignore_bad_frames:
- logger.warning("Bad frame: %s", line)
- return
-
- raise Exception("Bad frame: `{}`, frame `{}`".format(
- line,
- self.debug(return_result=True),
- ))
-
- def debug(self, return_result=False):
- if self.low_important():
- return ""
-
- res = "\n".join([str(f) for f in self.frames])
- res += "----------------------------- DEBUG END\n"
- if return_result:
- return res
-
- self.stream.write(res)
-
- def raw(self):
- return "\n".join([frame.raw() for frame in self.frames])
-
- def html(self, same_hash=False, same_count=1, return_result=False):
- ans = ""
- pre_class = "important-" + str(self.important)
- if same_hash:
- pre_class += " same-hash"
-
- ans += '<pre class="{0}">'.format(pre_class)
- if not same_hash:
- ans += '<a name="stack{0}"></a>'.format(self.hash())
-
- ans += '<span class="hash"><a href="#stack{0}">#{0}</a>, {1} stack(s) with same hash</span>\n'.format(
- self.hash(), same_count,
- )
-
- for f in self.frames:
- ans += f.html()
- ans += "</pre>\n"
-
- if return_result:
- return ans
-
- self.stream.write(ans)
-
- def fingerprint(self, max_num=None):
- """
- Stack fingerprint: concatenation of non-common stack frames
- FIXME: wipe away `max_num`
- """
- stack_fp = list()
- len_frames = min((max_num or len(self.frames)), len(self.frames))
-
- for f in self.frames[:len_frames]:
- fp = f.fingerprint()
- if not fp:
- continue
-
- if fp in self.fingerprint_blacklist:
- continue
-
- if fp.startswith(self.fingerprint_blacklist_prefix):
- continue
-
- if fp in stack_fp:
- # FIXME: optimize duplicate remover: check only previous frame
- # see also `push_frame`
- continue
-
- stack_fp.append(fp.strip())
-
- if self.max_depth is not None and len(stack_fp) >= self.max_depth:
- break
-
- return "\n".join(stack_fp)
-
- def simple_html(self, num_frames=None):
- if not num_frames:
- num_frames = len(self.frames)
- pre_class = "important-0"
- ans = '<pre class="{0}">'.format(pre_class)
- for i in range(min(len(self.frames), num_frames)):
- ans += self.frames[i].html()
- ans += "</pre>\n"
- return ans
-
- def __str__(self):
- return "\n".join(map(str, self.frames))
-
- def hash(self, max_num=None):
- """
- Entire stack hash for merging same stacks
- """
- if self.fingerprint_hash is None:
- self.fingerprint_hash = int(hashlib.md5(self.fingerprint(max_num).encode("utf-8")).hexdigest()[0:15], 16)
-
- return self.fingerprint_hash
-
-
-class GDBStack(Stack):
-
- mode = CoredumpMode.GDB
-
- REGEXPS = [
- # #6 0x0000000001d9203e in NAsio::TIOService::TImpl::Run (this=0x137b1ec00) at /place/
- # sandbox-data/srcdir/arcadia_cache/library/neh/asio/io_service_impl.cpp:77
-
- re.compile(
- r"#(?P<frame_no>\d+)[ \t]+(?P<addr>0x[0-9a-f]+) in (?P<func>.*) at (?P<source>.*)"
- ),
-
- # #5 TCondVar::WaitD (this=this@entry=0x10196b2b8, mutex=..., deadLine=..., deadLine@entry=...)
- # at /place/sandbox-data/srcdir/arcadia_cache/util/system/condvar.cpp:150
- re.compile(
- r"#(?P<frame_no>\d+)[ \t]+(?P<func>.*) at (?P<source>/.*)"
- ),
-
- # #0 0x00007faf8eb31d84 in pthread_cond_wait@@GLIBC_2.3.2 ()
- # from /lib/x86_64-linux-gnu/libpthread.so.0
- re.compile(
- r"#(?P<frame_no>\d+)[ \t]+(?P<addr>0x[0-9a-f]+) in (?P<func>.*) from (?P<source>.*)"
- ),
-
- # #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
- re.compile(
- r"#(?P<frame_no>\d+)[ \t]+ (?P<func>.*) at (?P<source>.*)"
- ),
-
- # #10 0x0000000000000000 in ?? ()
- re.compile(
- r"#(?P<frame_no>\d+)[ \t]+(?P<addr>0x[0-9a-f]+) in (?P<func>.*)"
- ),
- ]
-
-
-class LLDBStack(Stack):
-
- mode = CoredumpMode.LLDB
-
- REGEXPS = [
- # 0x00007fd7b300a886 libthird_Uparty_Sros_Sros_Ucomm_Sclients_Sroscpp_Sliblibroscpp.so`
- # std::thread::_State_impl<std::thread::_Invoker<std::tuple<ros::PollManager::PollManager()::$_1> > >::_M_run()
- # [inlined] ros::PollManager::threadFunc(this=0x00007fd7b30dab20) at poll_manager.cpp:75:16 # noqa
- re.compile(
- r"[ *]*frame #(?P<frame_no>\d+): (?P<addr>0x[0-9a-f]+).+inlined]\s(?P<func>.+)\sat\s(?P<source>.+)"
- ),
-
- re.compile(
- r"[ *]*frame #(?P<frame_no>\d+): (?P<addr>0x[0-9a-f]+).+?`(?P<func>.+)\sat\s(?P<source>.+)"
- ),
-
- # * frame #0: 0x00007fd7aee51f47 libc.so.6`gsignal + 199
- re.compile(
- r"[ *]*frame #(?P<frame_no>\d+): (?P<addr>0x[0-9a-f]+)\s(?P<source>.+)`(?P<func>.+)\s\+\s(?P<source_no>\d+)"
- ),
- ]
-
- # Take not more than `max_depth` non-filtered frames into fingerprint
- # See CORES-180
- max_depth = 10
-
- fingerprint_blacklist = Stack.fingerprint_blacklist + [
- "ros::ros_wallsleep",
- ]
-
- fingerprint_blacklist_prefix = Stack.fingerprint_blacklist_prefix + (
- "___lldb_unnamed_symbol",
- "__gnu_cxx",
- "__gthread",
- "__pthread",
- "decltype",
- "myriapoda::BuildersRunner",
- "non",
- "std::_Function_handler",
- "std::_Sp_counted_ptr_inplace",
- "std::__invoke_impl",
- "std::__invoke_result",
- "std::__shared_ptr",
- "std::conditional",
- "std::shared_ptr",
- "std::thread::_Invoker",
- "std::thread::_State_impl",
- "yandex::sdc::assert_details_",
- )
-
- suspicious_functions = Stack.suspicious_functions + [
- "Xml",
- "boost",
- "ros",
- "supernode",
- "tensorflow",
- "yandex::sdc",
- ]
-
-
-class PythonStack(Stack):
-
- REGEXPS = [
- re.compile(
- r'File "(?P<source>.*)", line (?P<source_no>\d+), in (?P<func_name>.*)'
- ),
- ]
-
-
-class SDCAssertStack(LLDBStack):
-
- mode = CoredumpMode.SDC_ASSERT
-
- REGEXPS = [
- # 0: ./modules/_shared/libcore_Stools_Slibassert.so(yandex::sdc::assert_details_::PanicV(char const*,
- # long, char const*, char const*, bool, char const*, __va_list_tag*)
- # +0x2aa)[0x7fb83268feaa]
- re.compile(
- r"(?P<frame_no>\d+):\s(?P<source>.+.so)\((?P<func>.+)\+(?P<source_no>.+).+\[(?P<addr>0x[0-9a-f]+)"
- ),
-
- re.compile(
- r"(?P<frame_no>\d+):\s(?P<source>\w+)\((?P<func>.+)\+(?P<source_no>.+).+\[(?P<addr>0x[0-9a-f]+)"
- )
- ]
-
-
-def parse_python_traceback(trace):
- trace = trace.replace("/home/zomb-sandbox/client/", "/")
- trace = trace.replace("/home/zomb-sandbox/tasks/", "/sandbox/")
- trace = trace.split("\n")
- exception = trace[-1] # noqa: F841
- trace = trace[1: -1]
- pairs = zip(trace[::2], trace[1::2])
- stack = Stack(lines=[])
- for frame_no, (path, row) in enumerate(pairs):
- # FIXME: wrap into generic tracer
- m = PythonStack.REGEXPS[0].match(path.strip())
- if m:
- frame_args = m.groupdict()
- if not frame_args["source"].startswith("/"):
- frame_args["source"] = "/" + frame_args["source"]
- frame_args["frame_no"] = str(frame_no)
- frame_args["func"] = row.strip()
- stack.push_frame(GDBFrame(**frame_args))
- return [[stack]], [[stack.raw()]], 6
-
-
-def stack_factory(stack):
- mode, unpacked_stack = Stack.from_json(stack)
-
- if mode == CoredumpMode.GDB:
- return GDBStack(**unpacked_stack)
- elif mode == CoredumpMode.LLDB:
- return LLDBStack(**unpacked_stack)
- elif mode == CoredumpMode.SDC_ASSERT:
- return SDCAssertStack(**unpacked_stack)
-
- raise Exception("Invalid stack mode: {}. ".format(mode))
-
-
-def _read_file(file_name):
- with open(file_name) as f:
- return f.read()
-
-
-def _file_contents(file_name):
- """Return file (or resource) contents as unicode string."""
- if getattr(sys, "is_standalone_binary", False):
- try:
- contents = pkgutil.get_data(__package__, file_name)
- except Exception:
- raise IOError("Failed to find resource: " + file_name)
- else:
- if not os.path.exists(file_name):
- file_name = os.path.join(MY_PATH, file_name)
- contents = _read_file(file_name)
- # py23 compatibility
- if not isinstance(contents, six.text_type):
- contents = contents.decode("utf-8")
- return contents
-
-
-def html_prolog(stream, timestamp):
- prolog = _file_contents("prolog.html")
- assert isinstance(prolog, six.string_types)
- stream.write(prolog.format(
- style=_file_contents("styles.css"),
- coredump_js=_file_contents("core_proc.js"),
- version=CORE_PROC_VERSION,
- timestamp=timestamp,
- ))
-
-
-def html_epilog(stream):
- stream.write(_file_contents("epilog.html"))
-
-
-def detect_coredump_mode(core_text):
- if len(core_text) == 0:
- raise Exception("Text stacktrace is blank")
-
- if "Panic at unixtime" in core_text:
- return CoredumpMode.SDC_ASSERT
-
- if "(lldb)" in core_text:
- return CoredumpMode.LLDB
-
- return CoredumpMode.GDB
-
-
-def filter_stack_dump(
- core_text=None,
- stack_file_name=None,
- use_fingerprint=False,
- sandbox_failed_task_id=None,
- output_stream=None,
- timestamp=datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
- ignore_bad_frames=True,
-):
- """New interface for stacktrace filtering. Preferred to use."""
- if not core_text and not stack_file_name:
- raise ValueError("Either `core_text` or `stack_file_name` should be passed to `filter_stack_dump`. ")
-
- if core_text is not None and stack_file_name:
- raise ValueError("Only one of `core_text` and `stack_file_name` cannot be specified for `filter_stack_dump`. ")
-
- if stack_file_name:
- core_text = _read_file(stack_file_name)
- # further processing uses `core_text` only
-
- mode = detect_coredump_mode(core_text)
- core_lines = core_text.split("\n")
-
- return filter_stackdump(
- file_lines=core_lines,
- ignore_bad_frames=ignore_bad_frames,
- mode=mode,
- sandbox_failed_task_id=sandbox_failed_task_id,
- stream=output_stream,
- timestamp=timestamp,
- use_fingerprint=use_fingerprint,
- use_stream=output_stream is not None,
- )
-
-
-class StackDumperBase(object):
-
- SANDBOX_TASK_RE = re.compile(r".*/[0-9a-f]/[0-9a-f]/([0-9]+)/.*")
- MAX_SAME_STACKS = 30
-
- def __init__(
- self,
- use_fingerprint,
- sandbox_failed_task_id,
- stream,
- use_stream,
- file_lines,
- timestamp,
- mode,
- file_name=None,
- ignore_bad_frames=True,
- ):
- self.source_root = SourceRoot()
- self.use_fingerprint = use_fingerprint
- self.sandbox_task_id = None
- self.sandbox_failed_task_id = sandbox_failed_task_id
- self.stream = stream or sys.stdout
- self.use_stream = use_stream
- self.file_name = file_name
- self.file_lines = file_lines
- self.timestamp = timestamp
- self.ignore_bad_frames = ignore_bad_frames
- self.stack_class = self.get_stack_class(mode)
-
- self.signal = SIGNAL_NOT_FOUND
- self.stacks = []
- self._main_info = []
-
- @staticmethod
- def is_ignored_line(line):
- raise NotImplementedError("Not implemented static method `is_ignored_line`. ")
-
- @staticmethod
- def get_stack_class(mode):
- exist_modes = {}
- for cls in [GDBStack, LLDBStack, SDCAssertStack]:
- current_mode = cls.mode
- if current_mode in exist_modes:
- raise Exception("Duplicate modes are disallowed. Repeated mode: `{}`".format(current_mode.value))
- exist_modes[current_mode] = cls
-
- if mode not in exist_modes:
- raise Exception("Unexpected coredump processing mode: `{}`".format(mode.value))
-
- return exist_modes[mode]
-
- def check_signal(self, line):
- raise NotImplementedError("Not implemented `check_signal`.")
-
- def set_sandbox_task_id(self, task_id):
- self.sandbox_task_id = task_id
-
- def add_main_line(self, line):
- self._main_info.append(line)
-
- def add_stack(self, stack_lines, thread_id):
- if not stack_lines:
- return
-
- stack = self.stack_class(
- lines=stack_lines,
- source_root=self.source_root,
- thread_id=thread_id,
- stream=self.stream,
- ignore_bad_frames=self.ignore_bad_frames,
- )
- self.stacks.append(stack)
-
- def dump(self):
- if self.file_lines is None:
- # FIXME(mvel): LLDB is not handled here
- self.file_lines = get_parsable_gdb_text(_read_file(self.file_name))
-
- self._collect_stacks()
-
- for stack in self.stacks:
- stack.parse()
- # stack.debug()
-
- if self.use_stream:
- if self.use_fingerprint:
- for stack in self.stacks:
- self.stream.write(stack.fingerprint() + "\n")
- self.stream.write("--------------------------------------\n")
- return
- else:
- html_prolog(self.stream, self.timestamp)
-
- if self.sandbox_task_id is not None:
- self.stream.write(
- '<div style="padding-top: 6px; font-size: 18px; font-weight: bold;">'
- 'Coredumped binary build task: '
- '<a href="https://sandbox.yandex-team.ru/task/{0}">{0}</a></div>\n'.format(
- self.sandbox_task_id
- )
- )
-
- if self.sandbox_failed_task_id is not None:
- self.stream.write(
- '<div style="padding-top: 6px; font-size: 18px; font-weight: bold;">'
- 'Sandbox failed task: '
- '<a href="https://sandbox.yandex-team.ru/task/{0}">{0}</a></div>\n'.format(
- self.sandbox_failed_task_id
- )
- )
-
- pre_class = ""
- self.stream.write('<pre class="{0}">\n'.format(pre_class))
- for line in self._main_info:
- self.stream.write(line.replace("&", "&amp;").replace("<", "&lt;") + "\n")
- self.stream.write("</pre>\n")
-
- sorted_stacks = sorted(self.stacks, key=lambda x: (x.important, x.fingerprint()), reverse=True)
-
- prev_hash = None
- all_hash_stacks = []
- cur_hash_stacks = []
- for stack in sorted_stacks:
- if stack.hash() == 0:
- continue
-
- if stack.hash() == prev_hash:
- if len(cur_hash_stacks) < self.MAX_SAME_STACKS:
- # do not collect too much
- cur_hash_stacks.append(stack)
- continue
-
- # hash changed
- if cur_hash_stacks:
- all_hash_stacks.append(cur_hash_stacks)
-
- prev_hash = stack.hash()
- cur_hash_stacks = [stack, ]
-
- # push last
- if cur_hash_stacks:
- all_hash_stacks.append(cur_hash_stacks)
-
- if self.use_stream:
- for cur_hash_stacks in all_hash_stacks:
- same_hash = False
- for stack in cur_hash_stacks:
- stack.html(same_hash=same_hash, same_count=len(cur_hash_stacks))
- same_hash = True
-
- html_epilog(self.stream)
- else:
- raw_hash_stacks = [
- [stack.raw() for stack in common_hash_stacks]
- for common_hash_stacks in all_hash_stacks
- ]
- return all_hash_stacks, raw_hash_stacks, self.signal
-
- def _collect_stacks(self):
- stack_lines = []
- stack_detected = False
- thread_id = None
-
- for line in self.file_lines:
- line = line.strip()
- if self.is_ignored_line(line):
- continue
-
- if "Core was generated" in line:
- m = self.SANDBOX_TASK_RE.match(line)
- if m:
- self.set_sandbox_task_id(int(m.group(1)))
-
- self.check_signal(line)
-
- # [Switching to thread 55 (Thread 0x7f100a94c700 (LWP 21034))]
- # Thread 584 (Thread 0x7ff363c03700 (LWP 2124)):
-
- # see test2 and test3
- tm = self.THREAD_RE.match(line)
- if tm:
- stack_detected = True
- self.add_stack(
- stack_lines=stack_lines,
- thread_id=thread_id,
- )
- stack_lines = []
- thread_id = int(tm.group(1))
- continue
-
- if stack_detected:
- stack_lines.append(line)
- else:
- self.add_main_line(line)
-
- # parse last stack
- self.add_stack(
- stack_lines=stack_lines,
- thread_id=thread_id,
- )
-
-
-class StackDumperGDB(StackDumperBase):
-
- SIGNAL_FLAG = "Program terminated with signal"
- THREAD_RE = re.compile(r".*[Tt]hread (\d+) .*")
- LINE_IN = re.compile(r"\d+\tin ")
-
- def is_ignored_line(self, line):
- if not line:
- return True
-
- if line.startswith("[New "):
- # LWP, Thread, process
- return True
-
- if line.startswith("[Thread "):
- return True
-
- if line.startswith("Using "):
- return True
-
- if line.startswith("warning:"):
- return True
-
- if line.startswith("Python Exception"):
- # TODO: handle this more carefully
- return True
-
- if line[0] != "#" and "No such file or directory" in line:
- return True
-
- if self.LINE_IN.match(line):
- # see test1.txt for example
- # 641 in /place/sandbox-data/srcdir/arcadia/library/coroutine/engine/impl.h
- return True
-
- return False
-
- def check_signal(self, line):
- if self.SIGNAL_FLAG in line:
- self.signal = line[line.find(self.SIGNAL_FLAG) + len(self.SIGNAL_FLAG):].split(",")[0]
-
-
-class StackDumperLLDB(StackDumperBase):
-
- SIGNAL_FLAG = "stop reason = signal"
-
- THREAD_RE = re.compile(r".*thread #(\d+), .*")
-
- SKIP_LINES = {
- "(lldb) bt all",
- "(lldb) script import sys",
- "(lldb) target create",
- "Core file",
-
- # Drop signal interceptor call
- # * frame #0: 0x00007efd49042fb7 libc.so.6`__GI___libc_sigaction at sigaction.c:54
- # TODO(epsilond1): Set MAX_IMPORTANT for some thread
- "__GI___libc_sigaction",
-
- # Drop unnamed symbols at lines like
- # frame #4: 0x00007fd8054156df libstdc++.so.6`___lldb_unnamed_symbol440$$libstdc++.so.6 + 15
- "$$",
- }
-
- @staticmethod
- def is_ignored_line(line):
- if not line:
- return True
-
- for skip_line in StackDumperLLDB.SKIP_LINES:
- if skip_line in line:
- return True
- return False
-
- def check_signal(self, line):
- if self.SIGNAL_FLAG in line and self.signal == SIGNAL_NOT_FOUND:
- self.signal = line.split()[-1]
-
-
-class StackDumperSDCAssert(StackDumperBase):
-
- THREAD_RE = re.compile(
- r"(\d+)(:\s)"
- )
-
- def is_ignored_line(self, line):
- if not line:
- return True
-
- return not re.match(self.THREAD_RE, line)
-
- def check_signal(self, line):
- self.signal = SIGNAL_NOT_FOUND
-
- def _collect_stacks(self):
- stack_lines = []
- for line in self.file_lines:
- line = line.strip()
- if self.is_ignored_line(line):
- continue
- stack_lines.append(line)
- self.check_signal(line)
-
- self.add_stack(
- stack_lines=stack_lines,
- thread_id=0,
- )
-
-
-def filter_stackdump(
- file_name=None,
- use_fingerprint=False,
- sandbox_failed_task_id=None,
- stream=None,
- file_lines=None,
- use_stream=True,
- timestamp=None,
- ignore_bad_frames=True,
- mode=None,
-):
- if mode is None and file_name is not None:
- mode = detect_coredump_mode(_read_file(file_name))
- if mode == CoredumpMode.GDB:
- stack_dumper_cls = StackDumperGDB
- elif mode == CoredumpMode.LLDB:
- stack_dumper_cls = StackDumperLLDB
- elif mode == CoredumpMode.SDC_ASSERT:
- stack_dumper_cls = StackDumperSDCAssert
- else:
- raise Exception("Invalid mode: {}".format(mode.value))
-
- dumper = stack_dumper_cls(
- file_name=file_name,
- use_fingerprint=use_fingerprint,
- sandbox_failed_task_id=sandbox_failed_task_id,
- stream=stream,
- use_stream=use_stream,
- file_lines=file_lines,
- timestamp=timestamp,
- ignore_bad_frames=ignore_bad_frames,
- mode=mode,
- )
-
- return dumper.dump()
-
-
-def get_parsable_gdb_text(core_text):
- # FIXME(mvel): Check encoding?
- # core_text = core_text.encode("ascii", "ignore").decode("ascii")
- core_text = (
- core_text
- # .replace("#", "\n#") # bug here
- .replace("No core", "\nNo core")
- .replace("[New", "\n[New")
- .replace("\n\n", "\n")
- )
-
- return core_text.split("\n")
-
-
-if __name__ == "__main__":
- if len(sys.argv) < 2:
- sys.stderr.write(
- """Traceback filter "Tri Korochki"
-https://wiki.yandex-team.ru/cores-aggregation/
-Usage:
- core_proc.py <traceback.txt> [-f|--fingerprint]
- core_proc.py -v|--version
-"""
- )
- sys.exit(1)
-
- if sys.argv[1] == "--version" or sys.argv[1] == "-v":
- if os.system("svn info 2>/dev/null | grep '^Revision'") != 0:
- print(CORE_PROC_VERSION)
- sys.exit(0)
-
- sandbox_failed_task_id = None
-
- use_fingerprint = False
- if len(sys.argv) >= 3:
- if sys.argv[2] == "-f" or sys.argv[2] == "--fingerprint":
- use_fingerprint = True
- sandbox_failed_task_id = sys.argv[2]
-
- filter_stack_dump(
- core_text=_read_file(sys.argv[1]),
- use_fingerprint=use_fingerprint,
- sandbox_failed_task_id=sandbox_failed_task_id,
- output_stream=sys.stdout,
- )
-
-
-"""
-Stack group is a `Stack` objects list with the same hash (fingerprint).
-"""
-
-
-class StackEncoder(json.JSONEncoder):
- """Stack JSON serializer."""
-
- def default(self, obj):
- if isinstance(obj, Stack):
- return obj.to_json()
-
- return json.JSONEncoder.default(obj)
-
-
-def serialize_stacks(stack_groups):
- """
- Serialize list of stack groups to string (using JSON format).
-
- :param stack_groups: list of stack groups.
- :return: JSON serialized to string
- """
- return json.dumps(stack_groups, cls=StackEncoder)
-
-
-def deserialize_stacks(stack_groups_str):
- """
- Restore JSON-serialized stack data into stack groups.
-
- :param stack_groups_str: JSON-serialized data.
- :return: list of stack groups
- """
- stack_groups_json = json.loads(stack_groups_str)
- # please do not use `map` hell here, it's impossible to debug
- all_stacks = [
- [stack_factory(stack) for stack in stacks]
- for stacks in stack_groups_json
- ]
- return all_stacks
diff --git a/library/python/coredump_filter/core_proc.js b/library/python/coredump_filter/core_proc.js
deleted file mode 100644
index 15413adeae..0000000000
--- a/library/python/coredump_filter/core_proc.js
+++ /dev/null
@@ -1,21 +0,0 @@
-
-
-$(document).ready(function() {
- $('#show-same-stacks').click(function() {
- var stacks = $('.same-hash');
- for (var i = 0; i < stacks.length; ++i)
- $(stacks[i]).show();
- $('#show-same-stacks').hide();
- $('#hide-same-stacks').show();
- return false;
- });
-
- $('#hide-same-stacks').click(function() {
- var stacks = $('.same-hash');
- for (var i = 0; i < stacks.length; ++i)
- $(stacks[i]).hide();
- $('#hide-same-stacks').hide();
- $('#show-same-stacks').show();
- return false;
- });
-});
diff --git a/library/python/coredump_filter/epilog.html b/library/python/coredump_filter/epilog.html
deleted file mode 100644
index b317cc2a91..0000000000
--- a/library/python/coredump_filter/epilog.html
+++ /dev/null
@@ -1,2 +0,0 @@
- </body>
-</html> \ No newline at end of file
diff --git a/library/python/coredump_filter/prolog.html b/library/python/coredump_filter/prolog.html
deleted file mode 100644
index f102a7210d..0000000000
--- a/library/python/coredump_filter/prolog.html
+++ /dev/null
@@ -1,24 +0,0 @@
-<!DOCTYPE html>
-<html>
- <head>
- <style>{style}</style>
- <script src="https://yastatic.net/jquery/1.7.1/jquery.min.js"></script>
- <script>{coredump_js}</script>
- </head>
- <body>
- <h1>Coredump report generated on {timestamp} by Coredump/traceback filter
- <i><a href="http://mvel.at.yandex-team.ru/2373">Tri Korochki</a></i>
- ({version})
- </h1>
- <h3>Author: <a href="https://staff.yandex-team.ru/mvel">mvel@</a> aka Mikhail Veltishchev</h3>
- <h3>&copy; Yandex LLC. All rights reversed</h3>
- <div class="legend">
- <ul style="line-height: 22px">
- <li><span class="important-100">Problem</span> stacks</li>
- <li><span class="important-75">Suspicious</span> stacks</li>
- <li><span class="important-50">Active</span> stacks</li>
- <li><span class="important-25">Non-active</span> stacks</li>
- </ul>
- <a class="show-same-hash" id="show-same-stacks" href="#">Show same stacks</a>
- <a class="show-same-hash" id="hide-same-stacks" href="#" style="display: none">Hide same stacks</a>
- </div>
diff --git a/library/python/coredump_filter/styles.css b/library/python/coredump_filter/styles.css
deleted file mode 100644
index fdd09ce09e..0000000000
--- a/library/python/coredump_filter/styles.css
+++ /dev/null
@@ -1,116 +0,0 @@
-body {
- font-family: sans-serif;
- font-size: 12px;
-}
-
-a {
- text-decoration: none;
- color: #486DEC;
-}
-
-.frame {
- color: #7f7f7f;
- display: inline-block;
- width: 30px;
-}
-
-.addr {
- color: #999999;
- padding-right: 10px;
-}
-
-.func {
- color: #7f0000;
- word-wrap: normal;
-}
-
-.source {
- color: #007f00;
-}
-
-.symbol {
- color: #0000ff;
-}
-
-h1 {
- font-size: 1.5em;
- margin-top: .1em;
- margin-bottom: .2em;
-}
-
-h3 {
- font-size: 1em;
- margin-top: .1em;
- margin-bottom: .2em;
-}
-
-pre {
- overflow-x: auto;
- margin: 6px 0px 6px 0px;
- padding: 0px 12px 6px 12px;
- position: relative;
-}
-
-pre.important-25 {
- background-color: #eeeeee;
-}
-
-span.important-25 {
- background-color: #eeeeee;
- padding: 3px;
-}
-
-pre.important-50 {
- background-color: #e7ffe7;
-}
-
-span.important-50 {
- background-color: #e7ffe7;
- padding: 3px;
-}
-
-pre.important-75 {
- background-color: #ffffcc;
-}
-
-span.important-75 {
- background-color: #ffffcc;
- padding: 3px;
-}
-
-pre.important-100 {
- background-color: #ffdddd;
-}
-
-span.important-100 {
- background-color: #ffdddd;
- padding: 3px;
-}
-
-a.show-same-hash {
- padding: 0px 20px 0px 20px;
-}
-
-/* hidden by default */
-.same-hash {
- display: none;
-}
-
-span.hash {
- position: absolute;
- top: 3px; right: 3px;
-}
-
-div.legend {
- position: absolute;
- z-index: 1;
- top: 5px;
- right: 8px;
- border: 1px solid #7f7f7f;
- border-radius: 3px;
- padding: 0px 20px 3px 0px;
-}
-
-div.legend ul {
- margin: 3px 0px 3px 0px;
-}
diff --git a/library/python/coredump_filter/ya.make b/library/python/coredump_filter/ya.make
deleted file mode 100644
index fc8ec1a45f..0000000000
--- a/library/python/coredump_filter/ya.make
+++ /dev/null
@@ -1,23 +0,0 @@
-PY23_LIBRARY()
-
-PY_SRCS(
- __init__.py
-)
-
-RESOURCE_FILES(
- PREFIX library/python/coredump_filter/
- core_proc.js
- epilog.html
- prolog.html
- styles.css
-)
-
-IF(PYTHON2)
- PEERDIR(contrib/deprecated/python/enum34)
-ENDIF()
-
-END()
-
-RECURSE(
- tests
-)
diff --git a/library/python/json/__init__.py b/library/python/json/__init__.py
deleted file mode 100644
index c6420d5e6d..0000000000
--- a/library/python/json/__init__.py
+++ /dev/null
@@ -1,44 +0,0 @@
-from library.python.json.loads import loads as _loads
-from simplejson import loads as _sj_loads
-
-
-def loads(*args, **kwargs):
- try:
- return _loads(*args, **kwargs)
- except Exception as e:
- if 'invalid syntax at token' in str(e):
- kwargs.pop('intern_keys', None)
- kwargs.pop('intern_vals', None)
- kwargs.pop('may_unicode', None)
- return _sj_loads(*args, **kwargs)
-
- raise
-
-
-from simplejson import load, dump, dumps # noqa
-
-
-def read_file(file_name, **kwargs):
- """
- Read file and return its parsed json contents.
-
- All kwargs will be proxied to `json.load` method as is.
-
- :param file_name: file with json contents
- :return: parsed json contents
- """
- with open(file_name) as f:
- return load(f, **kwargs)
-
-
-def write_file(file_name, contents, **kwargs):
- """
- Dump json data to file.
-
- All kwargs will be proxied to `json.dump` method as is.
-
- :param file_name: file to dump to
- :param contents: JSON-serializable object
- """
- with open(file_name, "w") as f:
- dump(contents, f, **kwargs)
diff --git a/library/python/json/loads.cpp b/library/python/json/loads.cpp
deleted file mode 100644
index 19cdb096ae..0000000000
--- a/library/python/json/loads.cpp
+++ /dev/null
@@ -1,246 +0,0 @@
-#include "loads.h"
-
-#include <Python.h>
-
-#include <library/cpp/json/fast_sax/parser.h>
-
-#include <util/generic/algorithm.h>
-#include <util/generic/stack.h>
-#include <util/generic/vector.h>
-#include <util/generic/ylimits.h>
-#include <util/string/ascii.h>
-
-using namespace NJson;
-
-namespace {
- enum EKind {
- Undefined,
- Array,
- Dict,
- Value,
- Key,
- };
-
- static inline TStringBuf ToStr(EKind kind) noexcept {
- switch (kind) {
- case Undefined:
- return TStringBuf("Undefined");
-
- case Array:
- return TStringBuf("Array");
-
- case Dict:
- return TStringBuf("Dict");
-
- case Value:
- return TStringBuf("Value");
-
- case Key:
- return TStringBuf("Key");
- }
-
- Y_UNREACHABLE();
- }
-
- struct TUnref {
- static inline void Destroy(PyObject* o) noexcept {
- Py_XDECREF(o);
- }
- };
-
- using TObjectPtr = TAutoPtr<PyObject, TUnref>;
-
- static inline TObjectPtr BuildBool(bool val) noexcept {
- if (val) {
- Py_RETURN_TRUE;
- }
-
- Py_RETURN_FALSE;
- }
-
- // Translate python exceptions from object-creating functions into c++ exceptions
- // Such errors are reported by returning nullptr
- // When a python error is set and C++ exception is caught by Cython wrapper,
- // Python exception is propagated, while C++ exception is discarded.
- PyObject* CheckNewObject(PyObject* obj) {
- Y_ENSURE(obj != nullptr, "got python exception");
- return obj;
- }
-
- void CheckRetcode(int retcode) {
- Y_ENSURE(retcode == 0, "got python exception");
- }
-
- static inline TObjectPtr BuildSmall(long val) {
-#if PY_VERSION_HEX >= 0x03000000
- return CheckNewObject(PyLong_FromLong(val));
-#else
- return CheckNewObject(PyInt_FromLong(val));
-#endif
- }
-
- PyObject* CreatePyString(TStringBuf str, bool intern, bool mayUnicode) {
-#if PY_VERSION_HEX >= 0x03000000
- Y_UNUSED(mayUnicode);
- PyObject* pyStr = PyUnicode_FromStringAndSize(str.data(), str.size());
- if (intern) {
- PyUnicode_InternInPlace(&pyStr);
- }
-#else
- const bool needUnicode = mayUnicode && !AllOf(str, IsAscii);
- PyObject* pyStr = needUnicode ? PyUnicode_FromStringAndSize(str.data(), str.size())
- : PyString_FromStringAndSize(str.data(), str.size());
- if (intern && !needUnicode) {
- PyString_InternInPlace(&pyStr);
- }
-#endif
- return pyStr;
- }
-
- struct TVal {
- EKind Kind = Undefined;
- TObjectPtr Val;
-
- inline TVal() noexcept
- : Kind(Undefined)
- {
- }
-
- inline TVal(EKind kind, TObjectPtr val) noexcept
- : Kind(kind)
- , Val(val)
- {
- }
- };
-
- static inline TObjectPtr NoneRef() noexcept {
- Py_RETURN_NONE;
- }
-
- struct TContext: public TJsonCallbacks {
- const bool InternKeys;
- const bool InternVals;
- const bool MayUnicode;
- TStack<TVal, TVector<TVal>> S;
-
- inline TContext(bool internKeys, bool internVals, bool mayUnicode)
- : TJsonCallbacks(true)
- , InternKeys(internKeys)
- , InternVals(internVals)
- , MayUnicode(mayUnicode)
- {
- S.emplace();
- }
-
- inline bool Consume(TObjectPtr o) {
- auto& t = S.top();
-
- if (t.Kind == Array) {
- CheckRetcode(PyList_Append(t.Val.Get(), o.Get()));
- } else if (t.Kind == Key) {
- auto key = S.top().Val;
-
- S.pop();
-
- CheckRetcode(PyDict_SetItem(S.top().Val.Get(), key.Get(), o.Get()));
- } else {
- t = TVal(Value, o);
- }
-
- return true;
- }
-
- inline TObjectPtr Pop(EKind expect) {
- auto res = S.top();
-
- S.pop();
-
- if (res.Kind != expect) {
- ythrow yexception() << "unexpected kind(expect " << ToStr(expect) << ", got " << ToStr(res.Kind) << ")";
- }
-
- return res.Val;
- }
-
- inline void Push(EKind kind, TObjectPtr object) {
- S.push(TVal(kind, object));
- }
-
- virtual bool OnNull() {
- return Consume(NoneRef());
- }
-
- virtual bool OnBoolean(bool v) {
- return Consume(BuildBool(v));
- }
-
- virtual bool OnInteger(long long v) {
- if (v >= (long long)Min<long>()) {
- return Consume(BuildSmall((long)v));
- }
-
- return Consume(CheckNewObject(PyLong_FromLongLong(v)));
- }
-
- virtual bool OnUInteger(unsigned long long v) {
- if (v <= (unsigned long long)Max<long>()) {
- return Consume(BuildSmall((long)v));
- }
-
- return Consume(CheckNewObject(PyLong_FromUnsignedLongLong(v)));
- }
-
- virtual bool OnDouble(double v) {
- return Consume(CheckNewObject(PyFloat_FromDouble(v)));
- }
-
- virtual bool OnString(const TStringBuf& v) {
- return Consume(CheckNewObject(CreatePyString(v, InternVals, MayUnicode)));
- }
-
- virtual bool OnOpenMap() {
- Push(Dict, CheckNewObject(PyDict_New()));
-
- return true;
- }
-
- virtual bool OnCloseMap() {
- return Consume(Pop(Dict));
- }
-
- virtual bool OnMapKey(const TStringBuf& k) {
- Push(Key, CheckNewObject(CreatePyString(k, InternKeys, MayUnicode)));
- return true;
- }
-
- virtual bool OnOpenArray() {
- Push(Array, CheckNewObject(PyList_New(0)));
-
- return true;
- }
-
- virtual bool OnCloseArray() {
- return Consume(Pop(Array));
- }
- };
-}
-
-PyObject* LoadJsonFromString(const char* data, size_t len, bool internKeys, bool internVals, bool mayUnicode) {
- TContext ctx(internKeys, internVals, mayUnicode);
-
- if (!len) {
- ythrow yexception() << "parse error: zero length input string";
- }
-
- if (!NJson::ReadJsonFast(TStringBuf(data, len), &ctx)) {
- ythrow yexception() << "parse error";
- }
-
- auto& s = ctx.S;
-
- if (!s || s.top().Kind != Value) {
- ythrow yexception() << "shit happen";
- }
-
- return s.top().Val.Release();
-}
diff --git a/library/python/json/loads.h b/library/python/json/loads.h
deleted file mode 100644
index 62dcdf6f21..0000000000
--- a/library/python/json/loads.h
+++ /dev/null
@@ -1,5 +0,0 @@
-#pragma once
-
-#include <Python.h>
-
-PyObject* LoadJsonFromString(const char* data, size_t len, bool internKeys = false, bool internVals = false, bool mayUnicode = false);
diff --git a/library/python/json/loads.pyx b/library/python/json/loads.pyx
deleted file mode 100644
index 82e5c6dce7..0000000000
--- a/library/python/json/loads.pyx
+++ /dev/null
@@ -1,14 +0,0 @@
-from libcpp cimport bool
-
-cdef extern from "library/python/json/loads.h":
- object LoadJsonFromString(const char*, size_t, bool internKeys, bool internVals, bool mayUnicode) except +
-
-
-def loads(s, intern_keys = False, intern_vals = False, may_unicode = False):
- if isinstance(s, unicode):
- s = s.encode('utf-8')
-
- try:
- return LoadJsonFromString(s, len(s), intern_keys, intern_vals, may_unicode)
- except Exception as e:
- raise ValueError(str(e))
diff --git a/library/python/json/ya.make b/library/python/json/ya.make
deleted file mode 100644
index 74a82de9d8..0000000000
--- a/library/python/json/ya.make
+++ /dev/null
@@ -1,17 +0,0 @@
-PY23_LIBRARY()
-
-PEERDIR(
- contrib/python/simplejson
- library/cpp/json/fast_sax
-)
-
-PY_SRCS(
- __init__.py
- loads.pyx
-)
-
-SRCS(
- loads.cpp
-)
-
-END()
diff --git a/library/python/par_apply/__init__.py b/library/python/par_apply/__init__.py
deleted file mode 100644
index 19b89ae843..0000000000
--- a/library/python/par_apply/__init__.py
+++ /dev/null
@@ -1,114 +0,0 @@
-import sys
-import threading
-import six
-
-from six.moves import queue
-
-
-def par_apply(seq, func, thr_num, join_polling=None):
- if thr_num < 2:
- for x in seq:
- yield func(x)
-
- return
-
- in_q = queue.Queue()
- out_q = queue.Queue()
-
- def enumerate_blocks():
- n = 0
-
- for b in seq:
- yield n, [b]
- n += 1
-
- yield n, None
-
- def iter_out():
- n = 0
- d = {}
-
- while True:
- if n in d:
- r = d[n]
- del d[n]
- n += 1
-
- yield r
- else:
- res = out_q.get()
-
- d[res[0]] = res
-
- out_iter = iter_out()
-
- def wait_block():
- for x in out_iter:
- return x
-
- def iter_compressed():
- p = 0
-
- for n, b in enumerate_blocks():
- in_q.put((n, b))
-
- while n > p + (thr_num * 2):
- p, b, c = wait_block()
-
- if not b:
- return
-
- yield p, c
-
- while True:
- p, b, c = wait_block()
-
- if not b:
- return
-
- yield p, c
-
- def proc():
- while True:
- data = in_q.get()
-
- if data is None:
- return
-
- n, b = data
-
- if b:
- try:
- res = (func(b[0]), None)
- except Exception:
- res = (None, sys.exc_info())
- else:
- res = (None, None)
-
- out_q.put((n, b, res))
-
- thrs = [threading.Thread(target=proc) for i in range(0, thr_num)]
-
- for t in thrs:
- t.start()
-
- try:
- for p, c in iter_compressed():
- res, err = c
-
- if err:
- six.reraise(*err)
-
- yield res
- finally:
- for t in thrs:
- in_q.put(None)
-
- for t in thrs:
- if join_polling is not None:
- while True:
- t.join(join_polling)
- if not t.is_alive():
- break
- else:
- t.join()
diff --git a/library/python/par_apply/ya.make b/library/python/par_apply/ya.make
deleted file mode 100644
index b14592ab79..0000000000
--- a/library/python/par_apply/ya.make
+++ /dev/null
@@ -1,11 +0,0 @@
-PY23_LIBRARY()
-
-PEERDIR(
- contrib/python/six
-)
-
-PY_SRCS(
- __init__.py
-)
-
-END()