diff options
author | say <say@yandex-team.com> | 2023-12-08 17:40:48 +0300 |
---|---|---|
committer | say <say@yandex-team.com> | 2023-12-08 19:58:59 +0300 |
commit | 914f57e3243f53dd89dd3adb4d8b6d35c47f46ce (patch) | |
tree | 98a1f1f1f5e2c38db3a78da10aeb7eb7d4e952e0 /library | |
parent | e61293d91ee7c923944f627d8e1138bcb17cacad (diff) | |
download | ydb-914f57e3243f53dd89dd3adb4d8b6d35c47f46ce.tar.gz |
Make support_retries() and get_type() methods of object instead of class ones. Reduce get_type_name() usage.
Diffstat (limited to 'library')
49 files changed, 0 insertions, 6796 deletions
diff --git a/library/cpp/pybind/attr.h b/library/cpp/pybind/attr.h deleted file mode 100644 index 5f25a6d73d..0000000000 --- a/library/cpp/pybind/attr.h +++ /dev/null @@ -1,412 +0,0 @@ -#pragma once - -#define PY_SSIZE_T_CLEAN -#include <Python.h> -#include <util/generic/string.h> -#include <util/generic/map.h> -#include <util/generic/set.h> -#include <util/generic/vector.h> -#include <util/generic/ptr.h> - -#include "cast.h" -#include "exceptions.h" - -namespace NPyBind { - // TBaseAttrGetter - template <typename TObjType> - class TBaseAttrGetter { - public: - virtual ~TBaseAttrGetter() { - } - virtual bool GetAttr(PyObject* owner, const TObjType& self, const TString& attr, PyObject*& res) const = 0; - - virtual bool HasAttr(PyObject* owner, const TObjType& self, const TString& attr, const TSet<TString>& hiddenNames) const { - if (hiddenNames.find(attr) != hiddenNames.end()) - return false; - PyObject* res = nullptr; - if (!GetAttr(owner, self, attr, res)) - return false; - Py_XDECREF(res); - return true; - } - }; - - template <typename TObjType> - class TBaseAttrSetter { - public: - virtual ~TBaseAttrSetter() { - } - - virtual bool SetAttr(PyObject* owner, TObjType& self, const TString& attr, PyObject* val) = 0; - }; - - template <typename TObjType> - class TAttrGetters { - public: - typedef TSimpleSharedPtr<TBaseAttrGetter<TObjType>> TGetterPtr; - - private: - typedef TVector<TGetterPtr> TGetterList; - typedef TMap<TString, TGetterList> TGetterMap; - - const TSet<TString>& HiddenAttrNames; - TGetterMap Getters; - - public: - TAttrGetters(const TSet<TString>& hiddenNames) - : HiddenAttrNames(hiddenNames) - { - } - - void AddGetter(const TString& attr, TGetterPtr getter) { - Getters[attr].push_back(getter); - } - - PyObject* GetAttr(PyObject* owner, const TObjType& self, const TString& attr) const { - typename TGetterMap::const_iterator it1 = Getters.find(attr); - if (it1 == Getters.end()) - it1 = Getters.find(""); - if (it1 == Getters.end()) - return nullptr; - const TGetterList& lst = it1->second; - for (typename TGetterList::const_iterator it2 = lst.begin(), end = lst.end(); it2 != end; ++it2) { - PyObject* res = nullptr; - if ((*it2)->GetAttr(owner, self, attr, res)) - return res; - // IMPORTANT! - // we have to fail GetAttr right there because we've failed because of internal python error/exception and can't continue iterating because - // it cause subsequent exceptions during call to Py_BuildValue - // moreover we have to preserve original exception right there - if (PyErr_Occurred()) { - break; - } - } - return nullptr; - } - - bool HasAttr(PyObject* owner, const TObjType& self, const TString& attr) const { - typename TGetterMap::const_iterator it1 = Getters.find(attr); - if (it1 == Getters.end()) - return false; - const TGetterList& lst = it1->second; - for (typename TGetterList::const_iterator it2 = lst.begin(), end = lst.end(); it2 != end; ++it2) { - if ((*it2)->HasAttr(owner, self, attr, HiddenAttrNames)) - return true; - } - return false; - } - - void GetAttrsDictionary(PyObject* owner, const TObjType& self, TMap<TString, PyObject*>& res) const { - for (typename TGetterMap::const_iterator it = Getters.begin(), end = Getters.end(); it != end; ++it) { - try { - if (HasAttr(owner, self, it->first)) { - auto attrPtr = GetAttr(owner, self, it->first); - if (attrPtr) { - res[it->first] = attrPtr; - } - if (PyErr_Occurred()) { - PyErr_Clear(); // Skip python errors as well - } - } - } catch (const std::exception&) { - // ignore this field - } - } - } - - void GetAttrsNames(PyObject* owner, const TObjType& self, TVector<TString>& resultNames) const { - for (typename TGetterMap::const_iterator it = Getters.begin(), end = Getters.end(); it != end; ++it) { - if (HasAttr(owner, self, it->first)) - resultNames.push_back(it->first); - } - } - }; - - template <typename TObjType> - class TGenericAttrGetter: public TBaseAttrGetter<TObjType> { - private: - TString AttrName; - - public: - TGenericAttrGetter(const TString& attrName) - : AttrName(attrName) - { - } - - bool GetAttr(PyObject* obj, const TObjType&, const TString&, PyObject*& res) const override { - auto str = NameFromString(AttrName); - res = PyObject_GenericGetAttr(obj, str.Get()); - if (!res && !PyErr_Occurred()) - ythrow TPyErr(PyExc_AttributeError) << "Can't get generic attribute '" << AttrName << "'"; - return res; - } - }; - - template <typename TObjType> - class TAttrSetters { - private: - typedef TSimpleSharedPtr<TBaseAttrSetter<TObjType>> TSetterPtr; - typedef TVector<TSetterPtr> TSetterList; - typedef TMap<TString, TSetterList> TSetterMap; - - TSetterMap Setters; - - public: - void AddSetter(const TString& attr, TSetterPtr setter) { - Setters[attr].push_back(setter); - } - - bool SetAttr(PyObject* owner, TObjType& self, const TString& attr, PyObject* val) { - typename TSetterMap::const_iterator it1 = Setters.find(attr); - if (it1 == Setters.end()) - it1 = Setters.find(""); - if (it1 == Setters.end()) - return false; - const TSetterList& lst = it1->second; - for (typename TSetterList::const_iterator it2 = lst.begin(), end = lst.end(); it2 != end; ++it2) { - if ((*it2)->SetAttr(owner, self, attr, val)) - return true; - } - return false; - } - - bool SetAttrDictionary(PyObject* owner, TObjType& self, TMap<TString, PyObject*>& dict) { - for (TMap<TString, PyObject*>::const_iterator it = dict.begin(), end = dict.end(); it != end; ++it) { - try { - SetAttr(owner, self, it->first, it->second); - } catch (std::exception&) { - // ignore this field - } - } - - return true; - } - }; - - /** - * TMethodAttrGetter - this class maps Python attribute read to C++ method call - */ - template <typename TObjType, typename TResult, typename TSubObject> - class TMethodAttrGetter: public TBaseAttrGetter<TObjType> { - private: - typedef TResult (TSubObject::*TMethod)() const; - TMethod Method; - - public: - TMethodAttrGetter(TMethod method) - : Method(method) - { - } - - bool GetAttr(PyObject*, const TObjType& self, const TString&, PyObject*& res) const override { - const TSubObject* sub = dynamic_cast<const TSubObject*>(&self); - if (sub == nullptr) - return false; - res = BuildPyObject((sub->*Method)()); - return (res != nullptr); - } - }; - - template <typename TObjType, typename TFunctor> - class TFunctorAttrGetter: public TBaseAttrGetter<TObjType> { - TFunctor Functor; - public: - explicit TFunctorAttrGetter(TFunctor functor) - : Functor(functor) - { - } - - bool GetAttr(PyObject*, const TObjType& self, const TString&, PyObject*& res) const override { - res = BuildPyObject(Functor(self)); - return (res != nullptr); - } - }; - - - /** - * TMethodAttrGetterWithCheck - this class maps Python attribute read to C++ HasAttr/GetAttr call - * If HasAttr returns false, None is returned. - * Otherwise GetAttr is called. - */ - template <typename TObjType, typename TResult, typename TSubObject> - class TMethodAttrGetterWithCheck: public TBaseAttrGetter<TObjType> { - private: - typedef TResult (TSubObject::*TMethod)() const; - typedef bool (TSubObject::*TCheckerMethod)() const; - TMethod Method; - TCheckerMethod CheckerMethod; - - public: - TMethodAttrGetterWithCheck(TMethod method, TCheckerMethod checkerMethod) - : Method(method) - , CheckerMethod(checkerMethod) - { - } - - bool GetAttr(PyObject*, const TObjType& self, const TString&, PyObject*& res) const override { - const TSubObject* sub = dynamic_cast<const TSubObject*>(&self); - if (sub == nullptr) - return false; - if ((sub->*CheckerMethod)()) - res = BuildPyObject((sub->*Method)()); - else { - Py_INCREF(Py_None); - res = Py_None; - } - return (res != nullptr); - } - }; - - template <typename TObjType, typename TResult, typename TSubObject, typename TMapper> - class TMethodAttrMappingGetter: public TBaseAttrGetter<TObjType> { - private: - typedef TResult (TSubObject::*TMethod)() const; - - TMethod Method; - TMapper Mapper; - - public: - TMethodAttrMappingGetter(TMethod method, TMapper mapper) - : Method(method) - , Mapper(mapper) - { - } - - bool GetAttr(PyObject*, const TObjType& self, const TString&, PyObject*& res) const override { - const TSubObject* sub = dynamic_cast<const TSubObject*>(&self); - if (sub == nullptr) - return false; - res = BuildPyObject(Mapper((sub->*Method)())); - return (res != nullptr); - } - }; - - template <typename TObjType, typename TResult, typename TSubObject, typename TMapper> - TSimpleSharedPtr<TBaseAttrGetter<TObjType>> - CreateMethodAttrMappingGetter(TResult (TSubObject::*method)() const, - TMapper mapper) { - return new TMethodAttrMappingGetter<TObjType, TResult, TSubObject, TMapper>(method, - mapper); - } - - template <typename TObjType, typename TResult, typename TValue, typename TSubObject> - class TMethodAttrSetter: public TBaseAttrSetter<TObjType> { - private: - typedef TResult (TSubObject::*TMethod)(TValue&); - TMethod Method; - - public: - TMethodAttrSetter(TMethod method) - : Method(method) - { - } - - virtual bool SetAttr(PyObject*, TObjType& self, const TString&, PyObject* val) { - TSubObject* sub = dynamic_cast<TSubObject*>(&self); - if (sub == nullptr) - return false; - TValue value; - if (!FromPyObject(val, value)) - return false; - (sub->*Method)(value); - return true; - } - }; - - template <typename TObjType, typename TValue, typename TFunctor> - class TFunctorAttrSetter: public TBaseAttrSetter<TObjType> { - TFunctor Functor; - public: - explicit TFunctorAttrSetter(TFunctor functor) - : Functor(functor) - { - } - - bool SetAttr(PyObject*, TObjType& self, const TString&, PyObject* val) const override { - TValue value; - if (!FromPyObject(val, value)) - return false; - auto res = BuildPyObject(Functor(self, value)); - return (res != nullptr); - } - }; - template <typename TObjType, typename TResult, typename TSubObject> - TSimpleSharedPtr<TBaseAttrGetter<TObjType>> CreateMethodAttrGetter(TResult (TSubObject::*method)() const) { - return new TMethodAttrGetter<TObjType, TResult, TSubObject>(method); - } - - template <typename TObjType, typename TFunctor> - TSimpleSharedPtr<TFunctorAttrGetter<TObjType, TFunctor>> CreateFunctorAttrGetter(TFunctor functor) { - return MakeSimpleShared<TFunctorAttrGetter<TObjType, TFunctor>>(functor); - } - - template <typename TObjType, typename TResult, typename TSubObject> - TSimpleSharedPtr<TBaseAttrGetter<TObjType>> CreateMethodAttrGetterWithCheck( - TResult (TSubObject::*method)() const, - bool (TSubObject::*checkerMethod)() const) { - return new TMethodAttrGetterWithCheck<TObjType, TResult, TSubObject>(method, checkerMethod); - } - - template <typename TObjType, typename TResult, typename TValue, typename TSubObject> - TSimpleSharedPtr<TBaseAttrSetter<TObjType>> CreateMethodAttrSetter(TResult (TSubObject::*method)(TValue&)) { - return new TMethodAttrSetter<TObjType, TResult, TValue, TSubObject>(method); - } - - template <typename TObjType, typename TFunctor, typename TValue> - TSimpleSharedPtr<TFunctorAttrSetter<TObjType, TValue, TFunctor>> CreateFunctorAttrSetter(TFunctor functor) { - return MakeSimpleShared<TFunctorAttrSetter<TObjType, TValue, TFunctor>>(functor); - } - - template <typename TObjType, typename TValue, typename TSubObject> - class TDirectAttrSetter: public TBaseAttrSetter<TObjType> { - private: - typedef TValue TSubObject::*TValueType; - TValueType Value; - - public: - TDirectAttrSetter(TValueType value) - : Value(value) - { - } - - bool SetAttr(PyObject*, TObjType& self, const TString&, PyObject* val) override { - TSubObject* sub = dynamic_cast<TSubObject*>(&self); - if (sub == NULL) - return false; - if (!FromPyObject(val, sub->*Value)) - return false; - return true; - } - }; - - template <typename TObjType, typename TValue, typename TSubObject> - TSimpleSharedPtr<TBaseAttrSetter<TObjType>> CreateAttrSetter(TValue TSubObject::*value) { - return new TDirectAttrSetter<TObjType, TValue, TSubObject>(value); - } - - template <typename TObjType, typename TValue, typename TSubObject> - class TDirectAttrGetter: public TBaseAttrGetter<TObjType> { - private: - typedef TValue TSubObject::*TValueType; - TValueType Value; - - public: - TDirectAttrGetter(TValueType value) - : Value(value) - { - } - - bool GetAttr(PyObject*, const TObjType& self, const TString&, PyObject*& res) const override { - const TSubObject* sub = dynamic_cast<const TSubObject*>(&self); - if (sub == nullptr) - return false; - res = BuildPyObject(sub->*Value); - return (res != nullptr); - } - }; - - template <typename TObjType, typename TValue, typename TSubObject> - TSimpleSharedPtr<TBaseAttrGetter<TObjType>> CreateAttrGetter(TValue TSubObject::*value) { - return new TDirectAttrGetter<TObjType, TValue, TSubObject>(value); - } -} diff --git a/library/cpp/pybind/cast.cpp b/library/cpp/pybind/cast.cpp deleted file mode 100644 index 60a44b0e83..0000000000 --- a/library/cpp/pybind/cast.cpp +++ /dev/null @@ -1,324 +0,0 @@ -#include "cast.h" -#include <util/generic/yexception.h> -#include <util/generic/buffer.h> - -namespace NPyBind { - PyObject* GetTrueRef(bool incref) { - if (incref) - Py_RETURN_TRUE; - return Py_True; - } - - PyObject* GetFalseRef(bool incref) { - if (incref) - Py_RETURN_FALSE; - return Py_False; - } - - PyObject* BuildPyObject(int val) { - return Py_BuildValue("i", val); - } - - PyObject* BuildPyObject(unsigned int val) { - return Py_BuildValue("I", val); - } - - PyObject* BuildPyObject(long int val) { - return Py_BuildValue("l", val); - } - - PyObject* BuildPyObject(unsigned long int val) { - return Py_BuildValue("k", val); - } - -#ifdef PY_LONG_LONG - PyObject* BuildPyObject(PY_LONG_LONG val) { - return Py_BuildValue("L", val); - } - - PyObject* BuildPyObject(unsigned PY_LONG_LONG val) { - return Py_BuildValue("K", val); - } -#endif - - PyObject* BuildPyObject(float val) { - return Py_BuildValue("f", val); - } - - PyObject* BuildPyObject(double val) { - return Py_BuildValue("d", val); - } - - PyObject* BuildPyObject(const TStringBuf& val) { - if (!val.IsInited()) - Py_RETURN_NONE; - - PyObject* stringValue = Py_BuildValue("s#", val.data(), static_cast<int>(val.length())); - if (stringValue != nullptr) { - return stringValue; - } - if (PyErr_ExceptionMatches(PyExc_UnicodeDecodeError)) { - PyErr_Clear(); - } else { - return nullptr; - } - return Py_BuildValue("y#", val.data(), static_cast<int>(val.length())); - } - - PyObject* BuildPyObject(const char* val) { - if (val == nullptr) - Py_RETURN_NONE; - PyObject* stringValue = Py_BuildValue("s#", val, static_cast<int>(strlen(val))); - if (stringValue != nullptr) { - return stringValue; - } - if (PyErr_ExceptionMatches(PyExc_UnicodeDecodeError)) { - PyErr_Clear(); - } else { - return nullptr; - } - return Py_BuildValue("y#", val, static_cast<int>(strlen(val))); - } - - PyObject* BuildPyObject(const TWtringBuf& val) { - if (!val.IsInited()) - Py_RETURN_NONE; -#if PY_VERSION_HEX < 0x03030000 - TPyObjectPtr result(PyUnicode_FromUnicode(nullptr, val.size()), true); - Py_UNICODE* buf = PyUnicode_AS_UNICODE(result.Get()); - if (buf == nullptr) - Py_RETURN_NONE; - for (size_t i = 0; i < val.size(); ++i) { - buf[i] = static_cast<Py_UNICODE>(val[i]); - } -#else - PyObject* unicodeValue = PyUnicode_FromKindAndData(PyUnicode_2BYTE_KIND, val.data(), val.size()); - if (unicodeValue == nullptr) - Py_RETURN_NONE; - TPyObjectPtr result(unicodeValue, true); -#endif - return result.RefGet(); - } - - PyObject* BuildPyObject(const TBuffer& val) { - TPyObjectPtr res(PyList_New(val.size()), true); - for (size_t i = 0, size = val.Size(); i < size; ++i) - PyList_SetItem(res.Get(), i, BuildPyObject(val.Data()[i])); - return res.RefGet(); - } - - PyObject* BuildPyObject(bool val) { - if (val) - Py_RETURN_TRUE; - else - Py_RETURN_FALSE; - } - - PyObject* BuildPyObject(PyObject* val) { - Py_XINCREF(val); - return val; - } - - PyObject* BuildPyObject(TPyObjectPtr ptr) { - return ptr.RefGet(); - } - - /* python represents (http://docs.python.org/c-api/arg.html#Py_BuildValue) - * char, uchar, short, ushort, int, long as PyInt - * uint, ulong as PyInt or PyLong (if exceeds sys.maxint) - * longlong, ulonglong as PyLong - */ - - template <> - bool FromPyObject(PyObject* obj, long& res) { - if (PyLong_Check(obj)) { - res = PyLong_AsLong(obj); - return true; - } - if (PyFloat_Check(obj)) { - res = static_cast<long>(PyFloat_AsDouble(obj)); - return true; - } -#if PY_MAJOR_VERSION < 3 - res = PyInt_AsLong(obj); -#endif - return -1 != res || !PyErr_Occurred(); - } - - template <> - bool FromPyObject(PyObject* obj, unsigned long& res) { - long lres; - if (!FromPyObject(obj, lres)) - return false; - if (lres < 0) - return false; - res = static_cast<unsigned long long>(lres); - return true; - } - - template <> - bool FromPyObject(PyObject* obj, int& res) { - long lres; - if (!FromPyObject(obj, lres)) - return false; - res = static_cast<int>(lres); - return true; - } - - template <> - bool FromPyObject(PyObject* obj, unsigned char& res) { - long lres; - if (!FromPyObject(obj, lres)) - return false; - res = static_cast<unsigned char>(lres); - return true; - } - - template <> - bool FromPyObject(PyObject* obj, char& res) { - long lres; - if (!FromPyObject(obj, lres)) - return false; - res = static_cast<char>(lres); - return true; - } - - template <> - bool FromPyObject(PyObject* obj, unsigned int& res) { - unsigned long lres; - if (!FromPyObject(obj, lres)) - return false; - res = static_cast<unsigned int>(lres); - return true; - } - -#ifdef HAVE_LONG_LONG - template <> - bool FromPyObject(PyObject* obj, long long& res) { - if (PyLong_Check(obj)) { - res = PyLong_AsLongLong(obj); - return -1 != res || !PyErr_Occurred(); - } - long lres; - if (!FromPyObject(obj, lres)) - return false; - res = static_cast<long long>(lres); - return true; - } - - template <> - bool FromPyObject(PyObject* obj, unsigned long long& res) { - if (PyLong_Check(obj)) { - res = PyLong_AsUnsignedLongLong(obj); - return static_cast<unsigned long long>(-1) != res || !PyErr_Occurred(); - } - long lres; - if (!FromPyObject(obj, lres)) - return false; - res = static_cast<unsigned long long>(lres); - return true; - } -#endif - - template <> - bool FromPyObject(PyObject* obj, double& res) { - if (PyFloat_Check(obj)) { - res = PyFloat_AsDouble(obj); - return true; - } - long long lres; - if (!FromPyObject(obj, lres)) - return false; - res = static_cast<double>(lres); - return true; - } - - template <> - bool FromPyObject(PyObject* obj, float& res) { - double dres; - if (!FromPyObject(obj, dres)) - return false; - res = static_cast<float>(dres); - return true; - } - - template <> - bool FromPyObject(PyObject* obj, bool& res) { - if (!PyBool_Check(obj)) - return false; - if (obj == Py_True) - res = true; - else - res = false; - return true; - } - - template <> - bool FromPyObject(PyObject* obj, PyObject*& res) { - Py_XINCREF(obj); - res = obj; - return true; - } - - template <> - bool FromPyObject(PyObject* obj, TPyObjectPtr& res) { - res = TPyObjectPtr(obj); - return true; - } - - static inline bool _FromPyObject(PyObject* obj, TStringBuf& res) { - char* str; - Py_ssize_t len; -#if PY_MAJOR_VERSION >= 3 - if (PyUnicode_Check(obj)) { - auto buf = PyUnicode_AsUTF8AndSize(obj, &len); - res = TStringBuf(buf, len); - return true; - } -#endif - if (-1 == PyBytes_AsStringAndSize(obj, &str, &len) || 0 > len) - return false; - res = TStringBuf(str, len); - return true; - } - - bool FromPyObject(PyObject* obj, TStringBuf& res) { - return _FromPyObject(obj, res); - } - - bool FromPyObject(PyObject* obj, TString& res) { - TStringBuf str; - if (!_FromPyObject(obj, str)) - return false; - res = str; - return true; - } - - bool FromPyObject(PyObject* obj, TUtf16String& res) { - if (!PyUnicode_Check(obj)) - return false; - auto str = TPyObjectPtr(PyUnicode_AsUTF16String(obj), true); - if (!str) - return false; - constexpr auto BOM_SIZE = 2; - size_t len = (static_cast<size_t>(PyBytes_GET_SIZE(str.Get())) - BOM_SIZE) / 2; - res.resize(len); - memcpy(res.begin(), PyBytes_AS_STRING(str.Get()) + BOM_SIZE, len * 2); - return (nullptr == PyErr_Occurred()); - } - - bool FromPyObject(PyObject* obj, TBuffer& res) { - if (!PyList_Check(obj)) - return false; - size_t cnt = PyList_Size(obj); - res.Reserve(cnt); - for (size_t i = 0; i < cnt; ++i) { - PyObject* item = PyList_GET_ITEM(obj, i); - char ch = 0; - if (!FromPyObject(item, ch)) - return false; - res.Append(ch); - } - return true; - } -} diff --git a/library/cpp/pybind/cast.h b/library/cpp/pybind/cast.h deleted file mode 100644 index 1f3d7d8366..0000000000 --- a/library/cpp/pybind/cast.h +++ /dev/null @@ -1,373 +0,0 @@ -#pragma once - -#define PY_SSIZE_T_CLEAN -#include <Python.h> -#include <util/generic/strbuf.h> -#include <util/generic/vector.h> -#include <util/generic/set.h> -#include <util/generic/yexception.h> -#include <util/generic/hash.h> -#include <util/generic/map.h> -#include <util/generic/maybe.h> -#include <utility> -#include <initializer_list> -#include "ptr.h" - -namespace NPyBind { - PyObject* GetTrueRef(bool incref = true); - PyObject* GetFalseRef(bool incref = true); - - PyObject* BuildPyObject(int val); - PyObject* BuildPyObject(unsigned int val); - PyObject* BuildPyObject(long int val); - PyObject* BuildPyObject(unsigned long int val); -#ifdef PY_LONG_LONG - PyObject* BuildPyObject(PY_LONG_LONG val); - PyObject* BuildPyObject(unsigned PY_LONG_LONG val); -#endif - PyObject* BuildPyObject(float val); - PyObject* BuildPyObject(double val); - PyObject* BuildPyObject(const TStringBuf& val); - PyObject* BuildPyObject(const char* val); - PyObject* BuildPyObject(const TWtringBuf& val); - PyObject* BuildPyObject(const TBuffer& val); - PyObject* BuildPyObject(bool val); - PyObject* BuildPyObject(PyObject*); - PyObject* BuildPyObject(TPyObjectPtr); - - template <typename T> - PyObject* BuildPyObject(const TVector<T>& val); - - template <typename T> - PyObject* BuildPyObject(const TSet<T>& val); - - template <typename TKey, typename TVal> - PyObject* BuildPyObject(const THashMap<TKey, TVal>& val); - - template <typename T1, typename T2> - PyObject* BuildPyObject(const std::pair<T1, T2>& val) { - TPyObjectPtr first(BuildPyObject(val.first), true); - if (!first) { - return nullptr; - } - TPyObjectPtr second(BuildPyObject(val.second), true); - if (!first || !second) { - return nullptr; - } - TPyObjectPtr res(PyList_New(2), true); - PyList_SetItem(res.Get(), 0, first.RefGet()); - PyList_SetItem(res.Get(), 1, second.RefGet()); - return res.RefGet(); - } - - template <typename T> - PyObject* BuildPyObject(const TVector<T>& val) { - TPyObjectPtr res(PyList_New(val.size()), true); - for (size_t i = 0, size = val.size(); i < size; ++i) { - auto pythonVal = BuildPyObject(std::move(val[i])); - if (!pythonVal) { - return nullptr; - } - PyList_SetItem(res.Get(), i, pythonVal); - } - return res.RefGet(); - } - - template <typename T> - PyObject* BuildPyObject(TVector<T>&& val) { - TPyObjectPtr res(PyList_New(val.size()), true); - for (size_t i = 0, size = val.size(); i < size; ++i) { - auto pythonVal = BuildPyObject(std::move(val[i])); - if (!pythonVal) { - return nullptr; - } - PyList_SetItem(res.Get(), i, pythonVal); - } - return res.RefGet(); - } - - template <typename T> - PyObject* BuildPyObject(const TSet<T>& val) { - TPyObjectPtr res(PySet_New(nullptr), true); - for (const auto& v : val) { - auto pythonVal = BuildPyObject(std::move(v)); - if (!pythonVal) { - return nullptr; - } - PySet_Add(res.Get(), pythonVal); - } - return res.RefGet(); - } - - template <typename T> - PyObject* BuildPyObject(const THashSet<T>& val) { - TPyObjectPtr res(PySet_New(nullptr), true); - for (const auto& v : val) { - auto pythonVal = BuildPyObject(std::move(v)); - if (!pythonVal) { - return nullptr; - } - PySet_Add(res.Get(), pythonVal); - } - return res.RefGet(); - } - - template <typename TKey, typename TVal> - PyObject* BuildPyObject(const THashMap<TKey, TVal>& val) { - TPyObjectPtr res(PyDict_New(), true); - for (typename THashMap<TKey, TVal>::const_iterator it = val.begin(), end = val.end(); it != end; ++it) { - auto prevOccurred = PyErr_Occurred(); - Y_UNUSED(prevOccurred); - TPyObjectPtr k(BuildPyObject(it->first), true); - if (!k) { - return nullptr; - } - TPyObjectPtr v(BuildPyObject(it->second), true); - if (!v) { - return nullptr; - } - PyDict_SetItem(res.Get(), k.Get(), v.Get()); - } - return res.RefGet(); - } - - template <typename TKey, typename TVal> - PyObject* BuildPyObject(const TMap<TKey, TVal>& val) { - TPyObjectPtr res(PyDict_New(), true); - for (typename TMap<TKey, TVal>::const_iterator it = val.begin(), end = val.end(); it != end; ++it) { - TPyObjectPtr k(BuildPyObject(it->first), true); - if (!k) { - return nullptr; - } - TPyObjectPtr v(BuildPyObject(it->second), true); - if (!v) { - return nullptr; - } - PyDict_SetItem(res.Get(), k.Get(), v.Get()); - } - return res.RefGet(); - } - - - template <typename TKey, typename TVal> - PyObject* BuildPyObject(const TMultiMap<TKey, TVal>& val) { - TPyObjectPtr res(PyDict_New(), true); - TMaybe<TKey> prevKey; - TPyObjectPtr currentEntry(PyList_New(0), true); - for (const auto& [key, value]: val) { - if (prevKey.Defined() && prevKey != key) { - TPyObjectPtr pyPrevKey(BuildPyObject(*prevKey), true); - if (!pyPrevKey) { - return nullptr; - } - PyDict_SetItem(res.Get(), pyPrevKey.Get(), currentEntry.Get()); - currentEntry = TPyObjectPtr(PyList_New(0), true); - } - TPyObjectPtr pyValue(BuildPyObject(value), true); - if (!pyValue) { - return nullptr; - } - PyList_Append(currentEntry.Get(), pyValue.Get()); - prevKey = key; - } - - if (prevKey.Defined()) { - TPyObjectPtr pyPrevKey(BuildPyObject(*prevKey), true); - if (!pyPrevKey) { - return nullptr; - } - PyDict_SetItem(res.Get(), pyPrevKey.Get(), currentEntry.Get()); - } - return res.RefGet(); - } - - template <typename T> - PyObject* BuildPyObject(const TMaybe<T>& val) { - if (!val.Defined()) - Py_RETURN_NONE; - return BuildPyObject(val.GetRef()); - } - - template <typename T, typename C, typename D> - PyObject* BuildPyObject(const TSharedPtr<T, C, D>& val) { - if (!val.Get()) - Py_RETURN_NONE; - return BuildPyObject(*val.Get()); - } - - template <typename T> - bool FromPyObject(PyObject* obj, T& res); - - bool FromPyObject(PyObject* obj, TString& res); - bool FromPyObject(PyObject* obj, TStringBuf& res); - bool FromPyObject(PyObject* obj, TUtf16String& res); - bool FromPyObject(PyObject* obj, TBuffer& res); - - template <typename T> - bool FromPyObject(PyObject* obj, TMaybe<T>& res) { - //we need to save current error before trying derserialize the value - //because it can produce conversion errors in python that we don't need to handle - struct TError { - public: - TError() { - PyErr_Fetch(&Type, &Value, &Traceback); - } - ~TError() { - PyErr_Restore(Type, Value, Traceback); - - } - private: - PyObject* Type = nullptr; - PyObject* Value = nullptr; - PyObject* Traceback = nullptr; - } currentPyExcInfo; - T val; - if (FromPyObject(obj, val)) { - res = val; - return true; - } - if (obj == Py_None) { - res = Nothing(); - return true; - } - return false; - } - - template <typename T1, typename T2> - bool FromPyObject(PyObject* obj, std::pair<T1, T2>& res) { - PyObject* first; - PyObject* second; - if (PyTuple_Check(obj) && 2 == PyTuple_Size(obj)) { - first = PyTuple_GET_ITEM(obj, 0); - second = PyTuple_GET_ITEM(obj, 1); - } else if (PyList_Check(obj) && 2 == PyList_Size(obj)) { - first = PyList_GET_ITEM(obj, 0); - second = PyList_GET_ITEM(obj, 1); - } else { - return false; - } - return FromPyObject(first, res.first) && FromPyObject(second, res.second); - } - - template <typename T> - bool FromPyObject(PyObject* obj, TVector<T>& res) { - if (!PyList_Check(obj)) - return false; - size_t cnt = PyList_Size(obj); - res.resize(cnt); - for (size_t i = 0; i < cnt; ++i) { - PyObject* item = PyList_GET_ITEM(obj, i); - if (!FromPyObject(item, res[i])) - return false; - } - return true; - } - - template <typename K, typename V> - bool FromPyObject(PyObject* obj, THashMap<K, V>& res) { - if (!PyDict_Check(obj)) - return false; - TPyObjectPtr list(PyDict_Keys(obj), true); - size_t cnt = PyList_Size(list.Get()); - for (size_t i = 0; i < cnt; ++i) { - PyObject* key = PyList_GET_ITEM(list.Get(), i); - PyObject* value = PyDict_GetItem(obj, key); - K rkey; - V rvalue; - if (!FromPyObject(key, rkey)) - return false; - if (!FromPyObject(value, rvalue)) - return false; - res[rkey] = rvalue; - } - return true; - } - - template <typename K, typename V> - bool FromPyObject(PyObject* obj, TMap<K, V>& res) { - if (!PyDict_Check(obj)) - return false; - TPyObjectPtr list(PyDict_Keys(obj), true); - size_t cnt = PyList_Size(list.Get()); - for (size_t i = 0; i < cnt; ++i) { - PyObject* key = PyList_GET_ITEM(list.Get(), i); - PyObject* value = PyDict_GetItem(obj, key); - K rkey; - V rvalue; - if (!FromPyObject(key, rkey)) - return false; - if (!FromPyObject(value, rvalue)) - return false; - res[rkey] = rvalue; - } - return true; - } - - class cast_exception: public TBadCastException { - }; - - template <typename T> - T FromPyObject(PyObject* obj) { - T res; - if (!FromPyObject(obj, res)) - ythrow cast_exception() << "Cannot cast argument to " << TypeName<T>(); - return res; - } - - template <class... Args, std::size_t... I> - bool ExtractArgs(std::index_sequence<I...>, PyObject* args, Args&... outArgs) { - if (!args || !PyTuple_Check(args) || PyTuple_Size(args) != sizeof...(Args)) - return false; - bool res = true; - (void)std::initializer_list<bool>{(res = res && NPyBind::FromPyObject(PyTuple_GET_ITEM(args, I), outArgs))...}; - return res; - } - - template <class... Args> - bool ExtractArgs(PyObject* args, Args&... outArgs) { - return ExtractArgs(std::index_sequence_for<Args...>(), args, outArgs...); - } - - template <class... Args, std::size_t... I> - bool ExtractOptionalArgs(std::index_sequence<I...>, PyObject* args, PyObject* kwargs, const char* keywords[], Args&... outArgs) { - PyObject* pargs[sizeof...(Args)] = {}; - static const char format[sizeof...(Args) + 2] = {'|', ((void)I, 'O')..., 0}; - if (!PyArg_ParseTupleAndKeywords(args, kwargs, format, const_cast<char**>(keywords), &pargs[I]...)) - return false; - bool res = true; - (void)std::initializer_list<bool>{(res = res && (!pargs[I] || NPyBind::FromPyObject(pargs[I], outArgs)))...}; - return res; - } - - template <class... Args> - bool ExtractOptionalArgs(PyObject* args, PyObject* kwargs, const char* keywords[], Args&... outArgs) { - return ExtractOptionalArgs(std::index_sequence_for<Args...>(), args, kwargs, keywords, outArgs...); - } - - template <typename... Args, std::size_t... I> - static auto GetArguments(std::index_sequence<I...>, PyObject* args) { - Y_UNUSED(args); // gcc bug - return std::make_tuple(FromPyObject<std::remove_cv_t<std::remove_reference_t<Args>>>(PyTuple_GetItem(args, I))...); - } - - template <typename... Args> - static auto GetArguments(PyObject* args) { - return GetArguments<Args...>(std::index_sequence_for<Args...>(), args); - } - - inline PyObject* ReturnString(TStringBuf s) { -#if PY_MAJOR_VERSION >= 3 - return PyUnicode_FromStringAndSize(s.data(), s.size()); -#else - return PyBytes_FromStringAndSize(s.data(), s.size()); -#endif - } - - inline TPyObjectPtr ReturnBytes(TStringBuf s) { - return TPyObjectPtr(PyBytes_FromStringAndSize(s.data(), s.size()), true); - } - - inline TPyObjectPtr NameFromString(TStringBuf s) { - return TPyObjectPtr(ReturnString(s), true); - } -} diff --git a/library/cpp/pybind/embedding.cpp b/library/cpp/pybind/embedding.cpp deleted file mode 100644 index cf8941a92a..0000000000 --- a/library/cpp/pybind/embedding.cpp +++ /dev/null @@ -1,63 +0,0 @@ -#define PY_SSIZE_T_CLEAN -#include <Python.h> - -#include "embedding.h" - -#include <util/generic/ptr.h> -#include <util/generic/yexception.h> - -namespace NPyBind { -#if PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION < 8 - class TDeleteRawMem { - public: - template <typename T> - static inline void Destroy(T* t) noexcept { - PyMem_RawFree(t); - } - }; - - template <typename T> - using TRawMemHolder = THolder<T, TDeleteRawMem>; - - static void SetProgramName(char* name) { - TRawMemHolder<wchar_t> wideName(Py_DecodeLocale(name, nullptr)); - Y_ENSURE(wideName); - Py_SetProgramName(wideName.Get()); - } -#endif - - TEmbedding::TEmbedding(char* argv0) { -#if PY_MAJOR_VERSION < 3 - Py_SetProgramName(argv0); - Py_Initialize(); -#elif PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 8 - PyStatus status; - - PyConfig config; - PyConfig_InitPythonConfig(&config); - // Disable parsing command line arguments - config.parse_argv = 0; - - status = PyConfig_SetBytesString(&config, &config.program_name, argv0); - if (PyStatus_Exception(status)) { - PyConfig_Clear(&config); - Py_ExitStatusException(status); - } - - status = Py_InitializeFromConfig(&config); - if (PyStatus_Exception(status)) { - PyConfig_Clear(&config); - Py_ExitStatusException(status); - } - - PyConfig_Clear(&config); -#elif PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION < 8 - SetProgramName(argv0); - Py_Initialize(); -#endif - } - - TEmbedding::~TEmbedding() { - Py_Finalize(); - } -} diff --git a/library/cpp/pybind/embedding.h b/library/cpp/pybind/embedding.h deleted file mode 100644 index 18553d9f6c..0000000000 --- a/library/cpp/pybind/embedding.h +++ /dev/null @@ -1,10 +0,0 @@ -#pragma once - -namespace NPyBind { - class TEmbedding { - public: - TEmbedding(char* argv0); - ~TEmbedding(); - }; - -} diff --git a/library/cpp/pybind/empty.cpp b/library/cpp/pybind/empty.cpp deleted file mode 100644 index 10da997ecc..0000000000 --- a/library/cpp/pybind/empty.cpp +++ /dev/null @@ -1,2 +0,0 @@ -#include "init.h" -#include "v2.h" diff --git a/library/cpp/pybind/exceptions.cpp b/library/cpp/pybind/exceptions.cpp deleted file mode 100644 index db1531fc63..0000000000 --- a/library/cpp/pybind/exceptions.cpp +++ /dev/null @@ -1,147 +0,0 @@ -#include "exceptions.h" -#include "cast.h" -#include "module.h" -#include <util/generic/algorithm.h> - -namespace NPyBind { - - namespace NPrivate { - TPyObjectPtr CreatePyBindModule() { - return TPyObjectPtr(TExceptionsHolder::DoInitPyBindModule(), true); - } - }//NPrivate - - TPyObjectPtr TExceptionsHolder::GetException(const TString& name) { - if (name == "") - return TPyObjectPtr(nullptr); - if (!Exceptions[name].Get()) - ythrow yexception() << "Wrong base class '" << name << "'"; - return Exceptions[name]; - } - - TPyObjectPtr TExceptionsHolder::GetExceptions(const TVector<TString>& names) { - TVector<TString> tmp(names.begin(), names.end()); - TVector<TString>::iterator end = std::unique(tmp.begin(), tmp.end()); - TPyObjectPtr tuple(PyTuple_New(std::distance(tmp.begin(), end)), true); - for (size_t i = 0; i < (size_t)std::distance(tmp.begin(), end); ++i) { - if (!Exceptions[tmp[i]].Get()) - ythrow yexception() << "Wrong base class '" << tmp[i] << "'"; - PyTuple_SetItem(tuple.Get(), i, Exceptions[tmp[i]].Get()); - } - return tuple; - } - - // def PyBindObjectReconstructor(cl, props): - // return cl(__properties__=props) - static PyObject* PyBindObjectReconstructor(PyObject*, PyObject* args) { - TPyObjectPtr callable, props; - if (!ExtractArgs(args, callable, props)) - ythrow yexception() << "Wrong method arguments"; -#if PY_MAJOR_VERSION >= 3 - TPyObjectPtr noArgs(PyTuple_New(0), true); -#else - TPyObjectPtr noArgs(PyList_New(0), true); -#endif - TPyObjectPtr kw(PyDict_New(), true); - PyDict_SetItemString(kw.Get(), "__properties__", props.Get()); - TPyObjectPtr res(PyObject_Call(callable.Get(), noArgs.Get(), kw.Get()), true); - return res.RefGet(); - } - - static PyMethodDef PyBindMethods[] = { - {"PyBindObjectReconstructor", PyBindObjectReconstructor, METH_VARARGS, "Tech method. It's required for unpickling."}, - {nullptr, nullptr, 0, nullptr}}; - -#if PY_MAJOR_VERSION >= 3 - static struct PyModuleDef moduledef = { - PyModuleDef_HEAD_INIT, - "pybind", - NULL, - -1, - PyBindMethods, - NULL, NULL, NULL, NULL - }; - - static PyObject* InitPyBind() { - return PyModule_Create(&moduledef); - } -#else - static PyObject* InitPyBind() { - return Py_InitModule("pybind", PyBindMethods); - } -#endif - - void TExceptionsHolder::DoInitPyBindModule2() { - DoInitPyBindModule(); - } - - PyObject* TExceptionsHolder::DoInitPyBindModule() { - Instance().Module = NPyBind::TPyObjectPtr(InitPyBind(), true); - if (!Instance().Module.Get()) - return nullptr; - - for (TCheckersVector::const_iterator it = Instance().Checkers.begin(), end = Instance().Checkers.end(); it != end; ++it) { - TString name = (*it)->GetName(); - if (!!name) { - //Ref to the object should be incremented before passing to AddObject - auto res = PyModule_AddObject(Instance().Module.Get(), name.data(), (*it)->GetException().RefGet()); - if (res < 0) { - ythrow yexception() << "Failed to add object " << name << " to internal module pybind"; - } - } - } - return Instance().Module.RefGet(); - } - - void TExceptionsHolder::Clear() { - //Unfortunately in Python3 we can't retrack this object because of PyError_NewException - //it's only the safe way to preserve GC gens in valid state during the finalization - for (auto& ptr: Checkers) { - if (!dynamic_cast<const TPyErrExceptionsChecker*>(ptr.Get())) { // no need to untrack standard PyExc_* exceptions from TPyErrExceptionsChecker - if (auto exceptionPtr = ptr->GetException()) { - PyObject_GC_UnTrack(exceptionPtr.Get()); - } - } - } - Checkers.clear(); - Exceptions.clear(); - Module.Drop(); - } - - TExceptionsHolder::TExceptionsHolder() { - AddException<std::exception>("yexception"); - AddException<TSystemError>("TSystemError", "yexception"); - AddException<TIoException>("TIoException", "yexception"); - - TVector<TString> names(2); - names[0] = "TSystemError"; - names[1] = "TIoException"; - - AddException<TIoSystemError>("TIoSystemError", names); - AddException<TFileError>("TFileError", "TIoSystemError"); - AddException<TBadCastException>("TBadCastException", "yexception"); - - Checkers.push_back(new TPyErrExceptionsChecker); - - // XXX: In Python 2.6, PyImport_AppendInittab() function takes non-const char*, this causes - // "ISO C++11 does not allow conversion from string literal to 'char *'" warning. - static char pybind[] = "pybind"; -#if PY_MAJOR_VERSION >= 3 - PyImport_AppendInittab(pybind, DoInitPyBindModule); - - NPrivate::AddFinalizationCallBack([this]() { - Clear(); - }); -#else - PyImport_AppendInittab(pybind, DoInitPyBindModule2); -#endif - } - - NPyBind::TPyObjectPtr TExceptionsHolder::ToPyException(const std::exception& ex) { - for (TCheckersVector::const_reverse_iterator it = Checkers.rbegin(), end = Checkers.rend(); it != end; ++it) { - if ((*it)->Check(ex)) - return (*it)->GetException(); - } - return TPyObjectPtr(nullptr); - } -} diff --git a/library/cpp/pybind/exceptions.h b/library/cpp/pybind/exceptions.h deleted file mode 100644 index 48e20995e4..0000000000 --- a/library/cpp/pybind/exceptions.h +++ /dev/null @@ -1,143 +0,0 @@ -#pragma once - -#define PY_SSIZE_T_CLEAN -#include <Python.h> -#include <util/generic/yexception.h> -#include <util/generic/map.h> -#include <util/generic/vector.h> -#include "ptr.h" - -namespace NPyBind { - // Usage: - // ythrow TPyErr(PyExc_TypeError) << "some python type error somewhere in your C++ code"; - // - class TPyErr: public virtual yexception { - public: - TPyErr(PyObject* theException = PyExc_RuntimeError) - : Exception(theException) - { - } - - TPyObjectPtr GetException() const { - return Exception; - } - - private: - NPyBind::TPyObjectPtr Exception; - }; - - //Private api for creating PyBind python module - //Needs only for overriding pybind python module in library which imports other pybind library - namespace NPrivate { - TPyObjectPtr CreatePyBindModule(); - }//NPrivate - class TExceptionsHolder { - friend TPyObjectPtr NPrivate::CreatePyBindModule(); - private: - TExceptionsHolder(const TExceptionsHolder&); - TExceptionsHolder& operator=(const TExceptionsHolder&); - TExceptionsHolder(); - - void Clear(); - TPyObjectPtr GetException(const TString&); - TPyObjectPtr GetExceptions(const TVector<TString>&); - private: - class TExceptionsChecker { - public: - virtual ~TExceptionsChecker() { - } - virtual bool Check(const std::exception& ex) const = 0; - virtual TString GetName() const = 0; - virtual TPyObjectPtr GetException() const = 0; - }; - - template <typename TExcType> - class TConcreteExceptionsChecker: public TExceptionsChecker { - private: - TString Name; - TPyObjectPtr Exception; - - public: - TConcreteExceptionsChecker(const TString& name, TPyObjectPtr exception) - : Name(name) - , Exception(exception) - { - } - - bool Check(const std::exception& ex) const override { - const std::exception* e = &ex; - return dynamic_cast<const TExcType*>(e); - } - - TString GetName() const override { - return Name; - } - - TPyObjectPtr GetException() const override { - return Exception; - } - }; - - class TPyErrExceptionsChecker: public TExceptionsChecker { - private: - mutable TPyObjectPtr Exception; - - public: - TPyErrExceptionsChecker() { - } - - bool Check(const std::exception& ex) const override { - const TPyErr* err = dynamic_cast<const TPyErr*>(&ex); - if (err) { - Exception = err->GetException(); - } - return err != nullptr; - } - - TString GetName() const override { - return TString(); - } - - TPyObjectPtr GetException() const override { - return Exception; - } - }; - - typedef TSimpleSharedPtr<TExceptionsChecker> TCheckerPtr; - typedef TVector<TCheckerPtr> TCheckersVector; - typedef TMap<TString, TPyObjectPtr> TExceptionsMap; - - TPyObjectPtr Module; - TCheckersVector Checkers; - TExceptionsMap Exceptions; - - static PyObject* DoInitPyBindModule(); - static void DoInitPyBindModule2(); - - public: - static TExceptionsHolder& Instance() { - static TExceptionsHolder Holder; - return Holder; - } - - template <typename TExcType> - void AddException(const TString& name, const TString& base = "") { - TPyObjectPtr baseException(GetException(base)); - TString fullName = TString("pybind.") + name; - TPyObjectPtr exception(PyErr_NewException(const_cast<char*>(fullName.c_str()), baseException.Get(), nullptr), true); - Checkers.push_back(new TConcreteExceptionsChecker<TExcType>(name, exception)); - Exceptions[name] = exception; - } - - template <typename TExcType> - void AddException(const TString& name, const TVector<TString>& bases) { - TPyObjectPtr baseExceptions(GetExceptions(bases)); - TString fullName = TString("pybind.") + name; - TPyObjectPtr exception(PyErr_NewException(const_cast<char*>(fullName.c_str()), baseExceptions.Get(), nullptr), true); - Checkers.push_back(new TConcreteExceptionsChecker<TExcType>(name, exception)); - Exceptions[name] = exception; - } - - NPyBind::TPyObjectPtr ToPyException(const std::exception&); - }; -} diff --git a/library/cpp/pybind/init.h b/library/cpp/pybind/init.h deleted file mode 100644 index 58874574ed..0000000000 --- a/library/cpp/pybind/init.h +++ /dev/null @@ -1,25 +0,0 @@ -#pragma once - -#define PY_SSIZE_T_CLEAN -#include <Python.h> - -#include "ptr.h" - -namespace NPyBind { -#if PY_MAJOR_VERSION >= 3 - -#define PYBIND_MODINIT(name) PyMODINIT_FUNC PyInit_##name() - - inline PyObject* ModInitReturn(TPyObjectPtr&& modptr) { - return modptr.Release(); - } - -#else - -#define PYBIND_MODINIT(name) PyMODINIT_FUNC init##name() - - inline void ModInitReturn(TPyObjectPtr&&) { - } - -#endif -} diff --git a/library/cpp/pybind/method.h b/library/cpp/pybind/method.h deleted file mode 100644 index 7c1f6e90e1..0000000000 --- a/library/cpp/pybind/method.h +++ /dev/null @@ -1,439 +0,0 @@ -#pragma once - -#define PY_SSIZE_T_CLEAN -#include <Python.h> -#include <util/generic/string.h> -#include <util/generic/map.h> -#include <util/generic/set.h> -#include <util/generic/vector.h> -#include <util/generic/ptr.h> -#include <util/generic/typetraits.h> - -#include <util/generic/function.h> - -#include "cast.h" - -namespace NPyBind { - template <typename TObjType> - class TBaseMethodCaller { - public: - virtual ~TBaseMethodCaller() { - } - virtual bool CallMethod(PyObject* owner, TObjType* self, PyObject* args, PyObject* kwargs, PyObject*& res) const = 0; - virtual bool HasMethod(PyObject*, TObjType*, const TString&, const TSet<TString>&) { - return true; - } - }; - - template <typename TObjType> - class TIsACaller; - - template <typename TObjType> - class TMethodCallers { - private: - typedef TSimpleSharedPtr<TBaseMethodCaller<TObjType>> TCallerPtr; - typedef TVector<TCallerPtr> TCallerList; - typedef TMap<TString, TCallerList> TCallerMap; - - const TSet<TString>& HiddenAttrNames; - TCallerMap Callers; - - public: - TMethodCallers(const TSet<TString>& hiddenNames) - : HiddenAttrNames(hiddenNames) - { - } - - void AddCaller(const TString& name, TCallerPtr caller) { - Callers[name].push_back(caller); - } - - bool HasCaller(const TString& name) const { - return Callers.has(name); - } - - PyObject* CallMethod(PyObject* owner, TObjType* self, PyObject* args, PyObject* kwargs, const TString& name) const { - const TCallerList* lst = Callers.FindPtr(name); - if (!lst) - return nullptr; - for (const auto& caller : *lst) { - PyObject* res = nullptr; - PyErr_Clear(); - if (caller->CallMethod(owner, self, args, kwargs, res)) - return res; - } - return nullptr; - } - - bool HasMethod(PyObject* owner, TObjType* self, const TString& name) const { - const TCallerList* lst = Callers.FindPtr(name); - if (!lst) - return false; - for (const auto& caller : *lst) { - if (caller->HasMethod(owner, self, name, HiddenAttrNames)) - return true; - } - return false; - } - - void GetMethodsNames(PyObject* owner, TObjType* self, TVector<TString>& resultNames) const { - for (const auto& it : Callers) { - if (HasMethod(owner, self, it.first) && !HiddenAttrNames.contains(it.first)) - resultNames.push_back(it.first); - } - } - - void GetAllMethodsNames(TVector<TString>& resultNames) const { - for (const auto& it : Callers) { - resultNames.push_back(it.first); - } - } - - void GetPropertiesNames(PyObject*, TObjType* self, TVector<TString>& resultNames) const { - const TCallerList* lst = Callers.FindPtr("IsA"); - if (!lst) - return; - for (const auto& caller : *lst) { - TIsACaller<TObjType>* isACaller = dynamic_cast<TIsACaller<TObjType>*>(caller.Get()); - if (isACaller) { - resultNames = isACaller->GetPropertiesNames(self); - return; - } - } - } - }; - - template <typename TObjType> - class TIsACaller: public TBaseMethodCaller<TObjType> { - private: - class TIsAChecker { - public: - virtual ~TIsAChecker() { - } - virtual bool Check(const TObjType* obj) const = 0; - }; - - template <typename TConcrete> - class TIsAConcreteChecker: public TIsAChecker { - public: - bool Check(const TObjType* obj) const override { - return dynamic_cast<const TConcrete*>(obj) != nullptr; - } - }; - - typedef TSimpleSharedPtr<TIsAChecker> TCheckerPtr; - typedef TMap<TString, TCheckerPtr> TCheckersMap; - - TCheckersMap Checkers; - - bool Check(const TString& name, const TObjType* obj) const { - const TCheckerPtr* checker = Checkers.FindPtr(name); - if (!checker) { - PyErr_Format(PyExc_KeyError, "unknown class name: %s", name.data()); - return false; - } - return (*checker)->Check(obj); - } - - protected: - TIsACaller() { - } - - template <typename TConcrete> - void AddChecker(const TString& name) { - Checkers[name] = new TIsAConcreteChecker<TConcrete>; - } - - public: - bool CallMethod(PyObject*, TObjType* self, PyObject* args, PyObject*, PyObject*& res) const override { - if (args == nullptr || !PyTuple_Check(args)) - return false; - size_t cnt = PyTuple_Size(args); - bool result = true; - for (size_t i = 0; i < cnt; ++i) { - result = result && Check( -#if PY_MAJOR_VERSION >= 3 - PyUnicode_AsUTF8( -#else - PyString_AsString( -#endif - PyTuple_GetItem(args, i)), self); - } - if (PyErr_Occurred()) { - return false; - } - res = BuildPyObject(result); - return true; - } - - TVector<TString> GetPropertiesNames(const TObjType* obj) const { - TVector<TString> names; - - for (const auto& it : Checkers) { - if (it.second->Check(obj)) { - names.push_back(it.first); - } - } - - return names; - } - }; - - template <typename TObjType> - class TGenericMethodCaller: public TBaseMethodCaller<TObjType> { - private: - TString AttrName; - - public: - TGenericMethodCaller(const TString& attrName) - : AttrName(attrName) - { - } - - bool CallMethod(PyObject* obj, TObjType*, PyObject* args, PyObject*, PyObject*& res) const override { - auto str = NameFromString(AttrName); - PyObject* attr = PyObject_GenericGetAttr(obj, str.Get()); - if (!attr) - ythrow yexception() << "Can't get generic attribute '" << AttrName << "'"; - res = PyObject_CallObject(attr, args); - return res != nullptr; - } - }; - - - template <typename TObjType, typename TSubObject> - class TSubObjectChecker: public TBaseMethodCaller<TObjType> { - public: - ~TSubObjectChecker() override { - } - - bool HasMethod(PyObject*, TObjType* self, const TString&, const TSet<TString>&) override { - return dynamic_cast<const TSubObject*>(self) != nullptr; - } - }; - - template <typename TFunctor, typename Tuple, typename ResType, typename=std::enable_if_t<!std::is_same_v<ResType, void>>> - void ApplyFunctor(TFunctor functor, Tuple resultArgs, PyObject*& res) { - res = BuildPyObject(std::move(Apply(functor, resultArgs))); - } - - template <typename TFunctor, typename Tuple, typename ResType, typename=std::enable_if_t<std::is_same_v<ResType, void>>, typename=void> - void ApplyFunctor(TFunctor functor, Tuple resultArgs, PyObject*& res) { - Py_INCREF(Py_None); - res = Py_None; - Apply(functor, resultArgs); - } - - template <typename TObjType, typename TResType, typename... Args> - class TFunctorCaller: public TBaseMethodCaller<TObjType> { - using TFunctor = std::function<TResType(TObjType&,Args...)>; - TFunctor Functor; - public: - explicit TFunctorCaller(TFunctor functor): - Functor(functor){} - - bool CallMethod(PyObject*, TObjType* self, PyObject* args, PyObject*, PyObject*& res) const { - auto methodArgsTuple = GetArguments<Args...>(args); - auto resultArgs = std::tuple_cat(std::tie(*self), methodArgsTuple); - ApplyFunctor<TFunctor, decltype(resultArgs), TResType>(Functor, resultArgs, res); - return true; - } - }; - - template <typename TObjType, typename TRealType> - class TGetStateCaller: public TSubObjectChecker<TObjType, TRealType> { - protected: - TPyObjectPtr AddFromCaller(PyObject* obj, const TString& methodName) const { - PyObject* res = PyObject_CallMethod(obj, const_cast<char*>(methodName.c_str()), const_cast<char*>("")); - if (!res) { - PyErr_Clear(); - return TPyObjectPtr(Py_None); - } - return TPyObjectPtr(res, true); - } - - void GetStandartAttrsDictionary(PyObject* obj, TRealType*, TMap<TString, TPyObjectPtr>& dict) const { - TPyObjectPtr attrsDict(PyObject_GetAttrString(obj, "__dict__"), true); - TMap<TString, TPyObjectPtr> attrs; - if (!FromPyObject(attrsDict.Get(), attrs)) - ythrow yexception() << "Can't get '__dict__' attribute"; - dict.insert(attrs.begin(), attrs.end()); - } - - virtual void GetAttrsDictionary(PyObject* obj, TRealType* self, TMap<TString, TPyObjectPtr>& dict) const = 0; - - public: - bool CallMethod(PyObject* obj, TObjType* self, PyObject* args, PyObject*, PyObject*& res) const override { - if (!ExtractArgs(args)) - ythrow yexception() << "Can't parse arguments: it should be none"; - TRealType* rself = dynamic_cast<TRealType*>(self); - if (!rself) - return false; - TMap<TString, TPyObjectPtr> dict; - GetAttrsDictionary(obj, rself, dict); - res = BuildPyObject(dict); - return true; - } - }; - - template <typename TObjType, typename TRealType> - class TSetStateCaller: public TSubObjectChecker<TObjType, TRealType> { - protected: - void SetStandartAttrsDictionary(PyObject* obj, TRealType*, TMap<TString, TPyObjectPtr>& dict) const { - TPyObjectPtr value(BuildPyObject(dict), true); - PyObject_SetAttrString(obj, "__dict__", value.Get()); - } - - virtual void SetAttrsDictionary(PyObject* obj, TRealType* self, TMap<TString, TPyObjectPtr>& dict) const = 0; - - public: - bool CallMethod(PyObject* obj, TObjType* self, PyObject* args, PyObject*, PyObject*& res) const override { - TMap<TString, TPyObjectPtr> dict; - if (!ExtractArgs(args, dict)) - ythrow yexception() << "Can't parse arguments: it should be one dictionary"; - TRealType* rself = dynamic_cast<TRealType*>(self); - if (!rself) - return false; - SetAttrsDictionary(obj, rself, dict); - Py_INCREF(Py_None); - res = Py_None; - return true; - } - }; - - template <typename TObjType, typename TResult, typename TSubObject, typename TMethod, typename... Args> - class TAnyParameterMethodCaller: public TSubObjectChecker<TObjType, TSubObject> { - private: - TMethod Method; - - public: - TAnyParameterMethodCaller(TMethod method) - : Method(method) - { - } - - public: - bool CallMethod(PyObject*, TObjType* self, PyObject* args, PyObject*, PyObject*& res) const override { - TSubObject* sub = dynamic_cast<TSubObject*>(self); - if (sub == nullptr) - return false; - if (args && (!PyTuple_Check(args) || PyTuple_Size(args) != TFunctionArgs<TMethod>::Length)) { - //ythrow yexception() << "Method takes " << (size_t)(TFunctionArgs<TMethod>::Length) << " arguments, " << PyTuple_Size(args) << " provided"; - return false; - } - - try { - class Applicant { - public: - TResult operator()(Args... theArgs) { - return (Sub->*Method)(theArgs...); - } - TSubObject* Sub; - TMethod Method; - }; - res = BuildPyObject(std::move(Apply(Applicant{sub, Method}, GetArguments<Args...>(args)))); - } catch (cast_exception) { - return false; - } catch (...) { - if (PyExc_StopIteration == PyErr_Occurred()) { - // NB: it's replacement for geo_boost::python::throw_error_already_set(); - return true; - } - PyErr_SetString(PyExc_RuntimeError, CurrentExceptionMessage().data()); - return true; - } - - return true; - } - }; - - template <typename TObjType, typename TSubObject, typename TMethod, typename... Args> - class TAnyParameterMethodCaller<TObjType, void, TSubObject, TMethod, Args...>: public TSubObjectChecker<TObjType, TSubObject> { - private: - TMethod Method; - - public: - TAnyParameterMethodCaller(TMethod method) - : Method(method) - { - } - - public: - bool CallMethod(PyObject*, TObjType* self, PyObject* args, PyObject*, PyObject*& res) const override { - TSubObject* sub = dynamic_cast<TSubObject*>(self); - if (sub == nullptr) { - return false; - } - if (args && (!PyTuple_Check(args) || PyTuple_Size(args) != TFunctionArgs<TMethod>::Length)) { - return false; - } - - try { - class Applicant { - public: - void operator()(Args... theArgs) { - (Sub->*Method)(theArgs...); - } - TSubObject* Sub; - TMethod Method; - }; - - Apply(Applicant{sub, Method}, GetArguments<Args...>(args)); - - Py_INCREF(Py_None); - res = Py_None; - } catch (cast_exception) { - return false; - } catch (...) { - PyErr_SetString(PyExc_RuntimeError, CurrentExceptionMessage().data()); - return true; - } - - return true; - } - }; - - template <typename TResult, typename TSubObject, typename... Args> - struct TConstTraits { - typedef TResult (TSubObject::*TMethod)(Args... args) const; - }; - - template <typename TResult, typename TSubObject, typename... Args> - struct TNonConstTraits { - typedef TResult (TSubObject::*TMethod)(Args... args); - }; - - template <typename TObjType, typename TResult, typename TSubObject, typename TMethod, typename... Args> - class TConstMethodCaller: public TAnyParameterMethodCaller<TObjType, TResult, const TSubObject, typename TConstTraits<TResult, TSubObject, Args...>::TMethod, Args...> { - public: - TConstMethodCaller(typename TConstTraits<TResult, TSubObject, Args...>::TMethod method) - : TAnyParameterMethodCaller<TObjType, TResult, const TSubObject, typename TConstTraits<TResult, TSubObject, Args...>::TMethod, Args...>(method) - { - } - }; - - template <typename TObjType, typename TResult, typename TSubObject, typename... Args> - TSimpleSharedPtr<TBaseMethodCaller<TObjType>> CreateConstMethodCaller(TResult (TSubObject::*method)(Args...) const) { - return new TConstMethodCaller<TObjType, TResult, TSubObject, TResult (TSubObject::*)(Args...) const, Args...>(method); - } - - template <typename TObjType, typename TResType, typename... Args> - TSimpleSharedPtr<TBaseMethodCaller<TObjType>> CreateFunctorCaller(std::function<TResType(TObjType&, Args...)> functor) { - return new TFunctorCaller<TObjType, TResType, Args...>(functor); - } - - template <typename TObjType, typename TResult, typename TSubObject, typename TMethod, typename... Args> - class TMethodCaller: public TAnyParameterMethodCaller<TObjType, TResult, TSubObject, typename TNonConstTraits<TResult, TSubObject, Args...>::TMethod, Args...> { - public: - TMethodCaller(typename TNonConstTraits<TResult, TSubObject, Args...>::TMethod method) - : TAnyParameterMethodCaller<TObjType, TResult, TSubObject, typename TNonConstTraits<TResult, TSubObject, Args...>::TMethod, Args...>(method) - { - } - }; - - template <typename TObjType, typename TResult, typename TSubObject, typename... Args> - TSimpleSharedPtr<TBaseMethodCaller<TObjType>> CreateMethodCaller(TResult (TSubObject::*method)(Args...)) { - return new TMethodCaller<TObjType, TResult, TSubObject, TResult (TSubObject::*)(Args...), Args...>(method); - } - -} diff --git a/library/cpp/pybind/module.cpp b/library/cpp/pybind/module.cpp deleted file mode 100644 index 63b15de45d..0000000000 --- a/library/cpp/pybind/module.cpp +++ /dev/null @@ -1,72 +0,0 @@ -#include "module.h" -#include "ptr.h" - -#include <util/generic/adaptor.h> - -namespace NPyBind { - -#if PY_MAJOR_VERSION >= 3 - namespace NPrivate { - struct TFinCallBacksHolder { - static TVector<TFinalizationCallBack>& GetCallBacks() { - static TVector<TFinalizationCallBack> res; - return res; - } - }; - - TAtExitRegistrar::TAtExitRegistrar(TPyObjectPtr module) { - TPyObjectPtr atExitModuleName(Py_BuildValue("s", "atexit"), true); - TPyObjectPtr atExitModule(PyImport_Import(atExitModuleName.Get())); - Y_ABORT_UNLESS(atExitModule); - TPyObjectPtr finalizerFunc(PyObject_GetAttrString(module.Get(), "finalizer"), true); - Y_ABORT_UNLESS(finalizerFunc); - TPyObjectPtr registerName(Py_BuildValue("s", "register"), true); - PyObject_CallMethodObjArgs(atExitModule.Get(), registerName.Get(), finalizerFunc.Get(), nullptr); - } - - TPyBindModuleRegistrar::TPyBindModuleRegistrar() { - TPyObjectPtr modules(PySys_GetObject("modules")); - Y_ENSURE(modules.Get()); - if (Module = NPrivate::CreatePyBindModule()) { - Y_ABORT_UNLESS(0 == PyDict_SetItemString(modules.Get(), "pybind", Module.RefGet())); - } - AddFinalizationCallBack([this]() { - auto ptr = Module; - Y_UNUSED(ptr); - TPyObjectPtr modules(PySys_GetObject("modules")); - Y_ENSURE(modules.Get()); - TPyObjectPtr pyBindName(Py_BuildValue("s", "pybind")); - if (PyDict_Contains(modules.Get(), pyBindName.Get()) == 1) { - Y_ABORT_UNLESS(0==PyDict_DelItemString(modules.Get(), "pybind")); - } - if (Module) { - //We have to untrack the module because some refs from him refers to gc-leaked errors - //see exceptions.cpp fore more info - PyObject_GC_UnTrack(Module.Get()); - Module.Drop(); - } - }); - } - - void AddFinalizationCallBack(TFinalizationCallBack callback) { - TFinCallBacksHolder::GetCallBacks().push_back(callback); - } - - int FinalizeAll() { - for (auto callback: Reversed(NPrivate::TFinCallBacksHolder::GetCallBacks())) { - callback(); - } - return 0; - } - } -#endif - - - TModuleHolder::TModuleHolder() - : Methods(1, new TVector<TMethodDef>) - { -#if PY_MAJOR_VERSION >= 3 - AddModuleMethod<TModuleMethodCaller<decltype(&NPrivate::FinalizeAll), &NPrivate::FinalizeAll>::Call>("finalizer"); -#endif - } -}//NPyBind diff --git a/library/cpp/pybind/module.h b/library/cpp/pybind/module.h deleted file mode 100644 index 41dcb4dfec..0000000000 --- a/library/cpp/pybind/module.h +++ /dev/null @@ -1,176 +0,0 @@ -#pragma once - -#define PY_SSIZE_T_CLEAN -#include <Python.h> -#include "ptr.h" -#include "cast.h" -#include "exceptions.h" - -#include <util/generic/function.h> - -namespace NPyBind { -#if PY_MAJOR_VERSION >= 3 - namespace NPrivate { - using TFinalizationCallBack = std::function<void()>; - void AddFinalizationCallBack(TFinalizationCallBack); - class TAtExitRegistrar: private TNonCopyable { - TAtExitRegistrar(TPyObjectPtr module); - public: - static void Instantiate(TPyObjectPtr module) { - static TAtExitRegistrar registrar(module); - Y_UNUSED(registrar); - } - }; - - class TPyBindModuleRegistrar: private TNonCopyable { - TPyBindModuleRegistrar(); - TPyObjectPtr Module; - public: - static void Instantiate() { - static TPyBindModuleRegistrar registrar; - Y_UNUSED(registrar); - } - }; - } //NPrivate -#endif - - class TModuleHolder { - private: - TModuleHolder(const TModuleHolder&); - TModuleHolder& operator=(const TModuleHolder&); - - TModuleHolder(); - private: - typedef PyCFunction TModuleMethod; -#if PY_MAJOR_VERSION >= 3 - typedef PyObject* (*TModuleInitFunc)(); -#else - typedef void (*TModuleInitFunc)(); -#endif - - struct TMethodDef { - TString Name; - TModuleMethod Method; - TString Description; - int Flags; - - TMethodDef(const TString& name, TModuleMethod method, const TString& descr, int flags) - : Name(name) - , Method(method) - , Description(descr) - , Flags(flags) - { - } - - operator PyMethodDef() const { - PyMethodDef cur = {Name.c_str(), Method, Flags, Description.c_str()}; - return cur; - } - }; - - typedef TSimpleSharedPtr<TVector<TMethodDef>> TMethodDefVecPtr; - typedef TSimpleSharedPtr<TVector<PyMethodDef>> TPyMethodDefVecPtr; - - TVector<TMethodDefVecPtr> Methods; - TVector<TPyMethodDefVecPtr> Defs; -#if PY_MAJOR_VERSION >= 3 - //because the md_name will leak otherwise - class TPyModuleDefWithName { - PyModuleDef Def; - TString Name; - public: - explicit TPyModuleDefWithName(TString name, TPyMethodDefVecPtr moduleDefs) - : Name(std::move(name)) - { - Def = PyModuleDef{ - PyModuleDef_HEAD_INIT, - Name.c_str(), - nullptr, - -1, - moduleDefs->data(), - nullptr, nullptr, nullptr, nullptr - }; - } - PyModuleDef* GetDefPtr() { - return &Def; - } - - }; - TVector<TSimpleSharedPtr<TPyModuleDefWithName>> ModuleDefs; -#endif - - template <TModuleMethod method> - static PyObject* MethodWrapper(PyObject* obj, PyObject* args) { - try { - PyObject* res = method(obj, args); - if (!res && !PyErr_Occurred()) - ythrow yexception() << "\nModule method exited with NULL, but didn't set Error.\n Options:\n -- Return correct value or None;\n -- Set python exception;\n -- Throw c++ exception."; - return res; - } catch (const std::exception& ex) { - PyErr_SetString(TExceptionsHolder::Instance().ToPyException(ex).Get(), ex.what()); - } catch (...) { - PyErr_SetString(PyExc_RuntimeError, "Unknown error occurred while trying to call module method"); - } - return nullptr; - } - - public: - static TModuleHolder& Instance() { - static TModuleHolder Holder; - return Holder; - } - - void ImportModule(TPyObjectPtr module, const char* const name, TModuleInitFunc initFunc) { - PyImport_AppendInittab(const_cast<char*>(name), initFunc); - TPyObjectPtr importedModule(PyImport_ImportModule(name), true); - PyModule_AddObject(module.Get(), name, importedModule.Get()); - } - - template <TModuleMethod method> - void AddModuleMethod(const TString& name, const TString& descr = "") { - Methods.back()->push_back(TMethodDef(name, MethodWrapper<method>, descr, METH_VARARGS)); - } - - TPyObjectPtr InitModule(const TString& name) { - Defs.push_back(new TVector<PyMethodDef>(Methods.back()->begin(), Methods.back()->end())); - PyMethodDef blank = {nullptr, nullptr, 0, nullptr}; - Defs.back()->push_back(blank); -#if PY_MAJOR_VERSION >= 3 - ModuleDefs.push_back(MakeSimpleShared<TPyModuleDefWithName>(name, Defs.back())); - TPyObjectPtr res(PyModule_Create(ModuleDefs.back()->GetDefPtr())); - NPrivate::TAtExitRegistrar::Instantiate(res); - NPrivate::TPyBindModuleRegistrar::Instantiate(); -#else - TPyObjectPtr res(Py_InitModule(name.c_str(), &(Defs.back()->at(0)))); -#endif - Methods.push_back(new TVector<TMethodDef>); - return res; - } - }; - - template <typename TMethodSignature, TMethodSignature method> - class TModuleMethodCaller { - private: - template <typename TResult, typename... Args> - struct TCaller { - static PyObject* Call(PyObject* args) { - return BuildPyObject(Apply(method, GetArguments<Args...>(args))); - } - }; - - template <typename TResult, typename... Args> - static PyObject* InternalCall(TResult (*)(Args...), PyObject* args) { - return BuildPyObject(Apply(method, GetArguments<Args...>(args))); - } - - public: - static PyObject* Call(PyObject*, PyObject* args) { - if (args && (!PyTuple_Check(args) || PyTuple_Size(args) != TFunctionArgs<TMethodSignature>::Length)) { - ythrow yexception() << "Method takes " << (size_t)(TFunctionArgs<TMethodSignature>::Length) << " arguments, " << PyTuple_Size(args) << " provided"; - } - - return InternalCall(method, args); - } - }; - -} diff --git a/library/cpp/pybind/pod.cpp b/library/cpp/pybind/pod.cpp deleted file mode 100644 index 3cf030e537..0000000000 --- a/library/cpp/pybind/pod.cpp +++ /dev/null @@ -1,18 +0,0 @@ -#include "pod.h" - -namespace NPyBind { - class TPODAttrGetter: public TBaseAttrGetter<TPOD> { - public: - bool GetAttr(PyObject*, const TPOD& self, const TString& attr, PyObject*& res) const override { - res = self.GetAttr(attr.c_str()); - return res != nullptr; - } - }; - - TPODTraits::TPODTraits() - : MyParent("TPOD", "simple struct") - { - AddGetter("", new TPODAttrGetter); - } - -} diff --git a/library/cpp/pybind/pod.h b/library/cpp/pybind/pod.h deleted file mode 100644 index 90165fdbec..0000000000 --- a/library/cpp/pybind/pod.h +++ /dev/null @@ -1,53 +0,0 @@ -#pragma once - -#define PY_SSIZE_T_CLEAN -#include <Python.h> -#include "attr.h" -#include "typedesc.h" - -namespace NPyBind { - struct TPOD { - TPyObjectPtr Dict; - - TPOD() - : Dict(PyDict_New(), true) - { - } - bool SetAttr(const char* name, PyObject* value) { - return PyDict_SetItemString(Dict.Get(), name, value) == 0; - } - PyObject* GetAttr(const char* name) const { - PyObject* res = PyDict_GetItemString(Dict.Get(), name); - Py_XINCREF(res); - return res; - } - }; - - class TPODTraits: public NPyBind::TPythonType<TPOD, TPOD, TPODTraits> { - private: - typedef TPythonType<TPOD, TPOD, TPODTraits> MyParent; - friend class TPythonType<TPOD, TPOD, TPODTraits>; - TPODTraits(); - - public: - static TPOD* GetObject(TPOD& obj) { - return &obj; - } - }; - - template <> - inline bool FromPyObject<TPOD*>(PyObject* obj, TPOD*& res) { - res = TPODTraits::CastToObject(obj); - if (res == nullptr) - return false; - return true; - } - template <> - inline bool FromPyObject<const TPOD*>(PyObject* obj, const TPOD*& res) { - res = TPODTraits::CastToObject(obj); - if (res == nullptr) - return false; - return true; - } - -} diff --git a/library/cpp/pybind/ptr.h b/library/cpp/pybind/ptr.h deleted file mode 100644 index e136736690..0000000000 --- a/library/cpp/pybind/ptr.h +++ /dev/null @@ -1,51 +0,0 @@ -#pragma once - -#define PY_SSIZE_T_CLEAN -#include <Python.h> -#include <util/generic/ptr.h> - -namespace NPyBind { - template <class T> - class TPythonIntrusivePtrOps { - public: - static inline void Ref(T* t) noexcept { - Py_XINCREF(t); - } - - static inline void UnRef(T* t) noexcept { - Py_XDECREF(t); - } - - static inline void DecRef(T* t) noexcept { - Py_XDECREF(t); - } - }; - - class TPyObjectPtr: public TIntrusivePtr<PyObject, TPythonIntrusivePtrOps<PyObject>> { - private: - typedef TIntrusivePtr<PyObject, TPythonIntrusivePtrOps<PyObject>> TParent; - typedef TPythonIntrusivePtrOps<PyObject> TOps; - - public: - inline TPyObjectPtr() noexcept { - } - - inline explicit TPyObjectPtr(PyObject* obj) noexcept - : TParent(obj) - { - } - - inline TPyObjectPtr(PyObject* obj, bool unref) noexcept - : TParent(obj) - { - if (unref) - TOps::UnRef(TParent::Get()); - } - - inline PyObject* RefGet() { - TOps::Ref(TParent::Get()); - return TParent::Get(); - } - }; - -} diff --git a/library/cpp/pybind/typeattrs.h b/library/cpp/pybind/typeattrs.h deleted file mode 100644 index a906b9ec2b..0000000000 --- a/library/cpp/pybind/typeattrs.h +++ /dev/null @@ -1,368 +0,0 @@ -#pragma once - -#include "ptr.h" -#include "cast.h" -#include "attr.h" -#include "method.h" - -#include <util/generic/vector.h> - -namespace NPyBind { - template <typename TObject> - class TPythonTypeAttributes { - private: - TAttrGetters<TObject> AttrGetters; - TAttrSetters<TObject> AttrSetters; - TMethodCallers<TObject> MethodCallers; - - class TGetAttrsNamesCaller; - class TGetMethodsNamesCaller; - class TGetAllNamesCaller; - class TGetPropertiesNamesCaller; - class TDictAttrGetter; - class TDictAttrSetter; - class TGetAttributeMethodCaller; - class TSetAttrMethodCaller; - class TGetStrReprMethodCaller; - class TReduceMethodCaller; - class TBaseGetStateMethodCaller; - class TBaseSetStateMethodCaller; - - TPythonTypeAttributes(const TPythonTypeAttributes&); - TPythonTypeAttributes& operator=(const TPythonTypeAttributes&); - - static const TSet<TString> HiddenAttrNames; - - typedef PyObject* (*GetAttrFunction)(PyObject*, char*); - typedef int (*SetAttrFunction)(PyObject*, char*, PyObject*); - GetAttrFunction GetAttr; - SetAttrFunction SetAttr; - - public: - typedef TSimpleSharedPtr<TBaseAttrGetter<TObject>> TGetterPtr; - typedef TSimpleSharedPtr<TBaseAttrSetter<TObject>> TSetterPtr; - typedef TSimpleSharedPtr<TBaseMethodCaller<TObject>> TCallerPtr; - - TPythonTypeAttributes(GetAttrFunction getAttr, SetAttrFunction setAttr) - : AttrGetters(HiddenAttrNames) - , MethodCallers(HiddenAttrNames) - , GetAttr(getAttr) - , SetAttr(setAttr) - { - } - - void InitCommonAttributes() { - // attributes - AddGetter("__dict__", new TDictAttrGetter(AttrGetters)); - AddSetter("__dict__", new TDictAttrSetter(AttrSetters)); - - // methods - AddCaller("GetAttrsNames", new TGetAttrsNamesCaller(AttrGetters)); - AddCaller("GetMethodsNames", new TGetMethodsNamesCaller(MethodCallers)); - AddCaller("GetAllNames", new TGetAllNamesCaller(AttrGetters, MethodCallers)); - AddCaller("GetPropertiesNames", new TGetPropertiesNamesCaller(MethodCallers)); - AddCaller("__getattribute__", new TGetAttributeMethodCaller(GetAttr)); - AddCaller("__setattr__", new TSetAttrMethodCaller(SetAttr)); - AddCaller("__str__", new TGetStrReprMethodCaller("__str__")); - AddCaller("__repr__", new TGetStrReprMethodCaller("__repr__")); - AddCaller("__reduce_ex__", new TReduceMethodCaller); - AddCaller("__reduce__", new TReduceMethodCaller); - AddCaller("__getstate__", new TBaseGetStateMethodCaller); - AddCaller("__setstate__", new TBaseSetStateMethodCaller); - - // generics - AddGetter("__class__", new TGenericAttrGetter<TObject>("__class__")); - AddGetter("__doc__", new TGenericAttrGetter<TObject>("__doc__")); - AddCaller("__sizeof__", new TGenericMethodCaller<TObject>("__sizeof__")); - AddCaller("__hash__", new TGenericMethodCaller<TObject>("__hash__")); - } - - void AddGetter(const TString& attr, TGetterPtr getter) { - AttrGetters.AddGetter(attr, getter); - } - - void AddSetter(const TString& attr, TSetterPtr setter) { - AttrSetters.AddSetter(attr, setter); - } - - void AddCaller(const TString& name, TCallerPtr caller) { - MethodCallers.AddCaller(name, caller); - } - - const TAttrGetters<TObject>& GetAttrGetters() const { - return AttrGetters; - } - - TAttrSetters<TObject>& GetAttrSetters() { - return AttrSetters; - } - - const TMethodCallers<TObject>& GetMethodCallers() const { - return MethodCallers; - } - - const TSet<TString>& GetHiddenAttrs() const { - return HiddenAttrNames; - } - }; - - template <typename TObjType> - class TPythonTypeAttributes<TObjType>::TGetAttrsNamesCaller: public TBaseMethodCaller<TObjType> { - private: - const TAttrGetters<TObjType>& AttrGetters; - - public: - TGetAttrsNamesCaller(const TAttrGetters<TObjType>& getters) - : AttrGetters(getters) - { - } - - bool CallMethod(PyObject* owner, TObjType* self, PyObject* args, PyObject*, PyObject*& res) const override { - if (!ExtractArgs(args)) - ythrow yexception() << "Could not parse args for GetAttrsNames() - it should be none"; - TVector<TString> names; - AttrGetters.GetAttrsNames(owner, *self, names); - res = BuildPyObject(names); - return (res != nullptr); - } - }; - - template <typename TObjType> - class TPythonTypeAttributes<TObjType>::TGetMethodsNamesCaller: public TBaseMethodCaller<TObjType> { - private: - const TMethodCallers<TObjType>& MethodCallers; - - public: - TGetMethodsNamesCaller(const TMethodCallers<TObjType>& callers) - : MethodCallers(callers) - { - } - - bool CallMethod(PyObject* owner, TObjType* self, PyObject* args, PyObject*, PyObject*& res) const override { - if (!ExtractArgs(args)) - ythrow yexception() << "Could not parse args for GetMethodsNames() - it should be none"; - TVector<TString> names; - MethodCallers.GetMethodsNames(owner, self, names); - res = BuildPyObject(names); - return (res != nullptr); - } - }; - - template <typename TObjType> - class TPythonTypeAttributes<TObjType>::TGetAllNamesCaller: public TBaseMethodCaller<TObjType> { - private: - const TAttrGetters<TObjType>& AttrGetters; - const TMethodCallers<TObjType>& MethodCallers; - - public: - TGetAllNamesCaller(const TAttrGetters<TObjType>& getters, - const TMethodCallers<TObjType>& callers) - : AttrGetters(getters) - , MethodCallers(callers) - { - } - - bool CallMethod(PyObject* owner, TObjType* self, PyObject* args, PyObject*, PyObject*& res) const override { - if (!ExtractArgs(args)) - ythrow yexception() << "Could not parse args for GetAllNames() - it should be none"; - TVector<TString> names; - AttrGetters.GetAttrsNames(owner, *self, names); - MethodCallers.GetMethodsNames(owner, self, names); - res = BuildPyObject(names); - return (res != nullptr); - } - }; - - template <typename TObjType> - class TPythonTypeAttributes<TObjType>::TGetPropertiesNamesCaller: public TBaseMethodCaller<TObjType> { - private: - const TMethodCallers<TObjType>& MethodCallers; - - public: - TGetPropertiesNamesCaller(const TMethodCallers<TObjType>& callers) - : MethodCallers(callers) - { - } - - public: - bool CallMethod(PyObject* obj, TObjType* self, PyObject* args, PyObject*, PyObject*& res) const override { - if (!ExtractArgs(args)) - return false; - - TVector<TString> names; - MethodCallers.GetPropertiesNames(obj, self, names); - res = BuildPyObject(names); - return (res != nullptr); - } - }; - - template <typename TObjType> - class TPythonTypeAttributes<TObjType>::TDictAttrGetter: public TBaseAttrGetter<TObjType> { - private: - TAttrGetters<TObjType>& AttrGetters; - - public: - TDictAttrGetter(TAttrGetters<TObjType>& getters) - : AttrGetters(getters) - { - } - - bool GetAttr(PyObject* owner, const TObjType& self, const TString&, PyObject*& res) const override { - TMap<TString, PyObject*> dict; - AttrGetters.GetAttrsDictionary(owner, self, dict); - res = BuildPyObject(dict); - return (res != nullptr); - } - }; - - template <typename TObjType> - class TPythonTypeAttributes<TObjType>::TDictAttrSetter: public TBaseAttrSetter<TObjType> { - private: - TAttrSetters<TObjType>& AttrSetters; - - public: - TDictAttrSetter(TAttrSetters<TObjType>& setters) - : AttrSetters(setters) - { - } - - bool SetAttr(PyObject* owner, TObjType& self, const TString&, PyObject* val) override { - TMap<TString, PyObject*> dict; - if (!FromPyObject(val, dict)) - ythrow yexception() << "'__dict__' should be set to dictionary"; - if (!AttrSetters.SetAttrDictionary(owner, self, dict)) - return false; - return true; - } - }; - - template <typename TObjType> - class TPythonTypeAttributes<TObjType>::TGetAttributeMethodCaller: public TBaseMethodCaller<TObjType> { - private: - GetAttrFunction GetAttr; - - public: - TGetAttributeMethodCaller(GetAttrFunction getAttr) - : GetAttr(getAttr) - { - } - - bool CallMethod(PyObject* owner, TObjType*, PyObject* args, PyObject*, PyObject*& res) const override { - TString attrName; - if (!ExtractArgs(args, attrName)) - ythrow yexception() << "Could not parse args for '__getattribute__' - it should be one string"; - res = GetAttr(owner, const_cast<char*>(attrName.c_str())); - if (!res) - // Error already set - return false; - return true; - } - }; - - template <typename TObjType> - class TPythonTypeAttributes<TObjType>::TSetAttrMethodCaller: public TBaseMethodCaller<TObjType> { - private: - SetAttrFunction SetAttr; - - public: - TSetAttrMethodCaller(SetAttrFunction setAttr) - : SetAttr(setAttr) - { - } - - bool CallMethod(PyObject* owner, TObjType*, PyObject* args, PyObject*, PyObject*& res) const override { - TString attrName; - TPyObjectPtr value; - if (!ExtractArgs(args, attrName, value)) - ythrow yexception() << "Could not parse args for '__setattr__' - it should be one string and value"; - Py_INCREF(Py_None); - res = Py_None; - if (-1 == SetAttr(owner, const_cast<char*>(attrName.c_str()), value.Get())) - // Error already set - return false; - return true; - } - }; - - template <typename TObjType> - class TPythonTypeAttributes<TObjType>::TGetStrReprMethodCaller: public TBaseMethodCaller<TObjType> { - private: - TString MethodName; - - private: - const TString GetFullName(PyObject* obj) const { - TString module, name; - TPyObjectPtr type(PyObject_Type(obj), true); - if (!FromPyObject(PyObject_GetAttrString(type.Get(), "__module__"), module) || !FromPyObject(PyObject_GetAttrString(type.Get(), "__name__"), name)) - ythrow yexception() << "Could not get name of object"; - return module + "." + name; - } - - public: - TGetStrReprMethodCaller(const TString& methodName) - : MethodName(methodName) - { - } - - bool CallMethod(PyObject* owner, TObjType*, PyObject* args, PyObject*, PyObject*& res) const override { - if (args && !ExtractArgs(args)) - ythrow yexception() << "Could not parse args for '" << MethodName << "'"; - TString message = TString("<") + GetFullName(owner) + " object>"; - res = ReturnString(message); - return (res != nullptr); - } - }; - - template <typename TObjType> - class TPythonTypeAttributes<TObjType>::TReduceMethodCaller: public TBaseMethodCaller<TObjType> { - public: - bool CallMethod(PyObject* owner, TObjType*, PyObject*, PyObject*, PyObject*& res) const override { - TPyObjectPtr tuple(PyTuple_New(3), true); - // First component: reconstructor - TPyObjectPtr pybindName(BuildPyObject("pybind"), true); - TPyObjectPtr mainModule(PyImport_Import(pybindName.Get()), true); - TPyObjectPtr recName(BuildPyObject("PyBindObjectReconstructor"), true); - TPyObjectPtr reconstructor(PyObject_GetAttr(mainModule.Get(), recName.Get()), true); - // Second component: arguments to rebuild object - TPyObjectPtr arguments(PyTuple_New(2), true); - TPyObjectPtr cl(PyObject_GetAttrString(owner, "__class__"), true); - PyTuple_SET_ITEM(arguments.Get(), 0, cl.RefGet()); - TPyObjectPtr props(PyObject_CallMethod(owner, const_cast<char*>("GetPropertiesNames"), nullptr), true); - PyTuple_SET_ITEM(arguments.Get(), 1, props.RefGet()); - // Third component: state to fill new object - TPyObjectPtr state(PyObject_CallMethod(owner, const_cast<char*>("__getstate__"), nullptr), true); - - PyTuple_SET_ITEM(tuple.Get(), 0, reconstructor.RefGet()); - PyTuple_SET_ITEM(tuple.Get(), 1, arguments.RefGet()); - PyTuple_SET_ITEM(tuple.Get(), 2, state.RefGet()); - res = tuple.RefGet(); - return (res != nullptr); - } - }; - - template <typename TObjType> - class TPythonTypeAttributes<TObjType>::TBaseGetStateMethodCaller: public TGetStateCaller<TObjType, TObjType> { - public: - void GetAttrsDictionary(PyObject* obj, TObjType* self, TMap<TString, TPyObjectPtr>& dict) const override { - this->GetStandartAttrsDictionary(obj, self, dict); - } - }; - - template <typename TObjType> - class TPythonTypeAttributes<TObjType>::TBaseSetStateMethodCaller: public TSetStateCaller<TObjType, TObjType> { - public: - void SetAttrsDictionary(PyObject* obj, TObjType* self, TMap<TString, TPyObjectPtr>& dict) const override { - this->SetStandartAttrsDictionary(obj, self, dict); - } - }; - - static const char* HiddenAttrStrings[] = { - "__dict__", "__class__", "__dir__", "__delattr__", "__doc__", "__format__", "__getattribute__", "__hash__", - "__init__", "__new__", "__reduce__", "__reduce_ex__", "__repr__", "__setattr__", "__sizeof__", "__str__", - "__subclasshook__", "__getstate__", "__setstate__", - "GetAttrsNames", "GetMethodsNames", "GetAllNames", "GetPropertiesNames"}; - - template <typename T> - const TSet<TString> TPythonTypeAttributes<T>::HiddenAttrNames(HiddenAttrStrings, std::end(HiddenAttrStrings)); - -} diff --git a/library/cpp/pybind/typedesc.cpp b/library/cpp/pybind/typedesc.cpp deleted file mode 100644 index 75f39fd126..0000000000 --- a/library/cpp/pybind/typedesc.cpp +++ /dev/null @@ -1,79 +0,0 @@ -#include "typedesc.h" - -#include <util/generic/singleton.h> - -static void RegisterJSONBridgeImpl() { - PyRun_SimpleString("import json\n" - "class PyBindEncoder(json.JSONEncoder):\n" - " def default(self, obj):\n" - " if isinstance(obj, bytes):\n" - " try:\n" - " return obj.decode()\n" - " except UnicodeDecodeError:\n" - " return obj.hex()\n" - " dct = None\n" - " if hasattr(obj, '__getstate__'):\n" - " dct = obj.__getstate__()\n" - " elif hasattr(obj, '__dict__'):\n" - " dct = obj.__dict__\n" - " if dct is None:\n" - " return json.JSONEncoder.default(self, obj)\n" - " if hasattr(obj, '__class__'):\n" - " if hasattr(obj.__class__, '__name__'):\n" - " dct['__name__'] = obj.__class__.__name__\n" - " if hasattr(obj.__class__, '__module__'):\n" - " dct['__module__'] = obj.__class__.__module__\n" - " if hasattr(obj, 'GetPropertiesNames'):\n" - " dct['__properties__'] = obj.GetPropertiesNames()\n" - " return dct"); - - PyRun_SimpleString("def PyBindObjectHook(dct):\n" - " if '__name__' in dct:\n" - " name = dct['__name__']\n" - " module = dct['__module__']\n" - " del dct['__name__']\n" - " del dct['__module__']\n" - " cls = getattr(__import__(module), name)\n" - " if '__properties__' in dct:\n" - " props = dct['__properties__']\n" - " del dct['__properties__']\n" - " if len(props) == 0:\n" - " return dct\n" - " instance = cls(__properties__ = props)\n" - " else:\n" - " instance = cls()\n" - " if hasattr(instance, '__setstate__'):\n" - " instance.__setstate__(dct)\n" - " elif hasattr(instance, '__dict__'):\n" - " instance.__dict__ = dct\n" - " else:\n" - " return dct\n" - " return instance\n" - " return dct"); - - PyRun_SimpleString("def json_dump(*args, **kwargs):\n" - " kwargs['cls'] = PyBindEncoder\n" - " return json.dump(*args, **kwargs)\n" - "def json_dumps(*args, **kwargs):\n" - " kwargs['cls'] = PyBindEncoder\n" - " return json.dumps(*args, **kwargs)"); - - PyRun_SimpleString("def json_load(*args, **kwargs):\n" - " kwargs['object_hook'] = PyBindObjectHook\n" - " return json.load(*args, **kwargs)\n" - "def json_loads(*args, **kwargs):\n" - " kwargs['object_hook'] = PyBindObjectHook\n" - " return json.loads(*args, **kwargs)"); -} - -namespace { - struct TJSONBridge { - TJSONBridge() { - RegisterJSONBridgeImpl(); - } - }; -} - -void NPyBind::RegisterJSONBridge() { - Singleton<TJSONBridge>(); -} diff --git a/library/cpp/pybind/typedesc.h b/library/cpp/pybind/typedesc.h deleted file mode 100644 index 57eacb0f3a..0000000000 --- a/library/cpp/pybind/typedesc.h +++ /dev/null @@ -1,545 +0,0 @@ -#pragma once - -#define PY_SSIZE_T_CLEAN -#include <Python.h> -#include <structmember.h> - -#include "typeattrs.h" -#include "exceptions.h" -#include "module.h" - -namespace NPyBind { - void RegisterJSONBridge(); - - namespace NPrivate { - template <typename> - class TUnboundClosureHolder; - template <typename> - class TUnboundClosure; - } - - // TTraits should be derived from TPythonType - template <typename TObjectHolder, typename TObject, typename TTraits> - class TPythonType { - private: - TPythonType(const TPythonType&); - TPythonType& operator=(const TPythonType&); - - private: - typedef typename TPythonTypeAttributes<TObject>::TGetterPtr TGetterPtr; - typedef typename TPythonTypeAttributes<TObject>::TSetterPtr TSetterPtr; - typedef typename TPythonTypeAttributes<TObject>::TCallerPtr TCallerPtr; - - struct TProxy { - PyObject_HEAD - TObjectHolder* Holder; - }; - - static PyTypeObject PyType; - static PyMappingMethods MappingMethods; - static PyObject* PyTypeObjPtr; - protected: - static PyTypeObject* GetPyTypePtr() { - return &PyType; - } - private: - - TPythonTypeAttributes<TObject> Attributes; - - static int InitObject(PyObject* s, PyObject* args, PyObject* kwargs) { - try { - TProxy* self = reinterpret_cast<TProxy*>(s); - auto str = NameFromString("__properties__"); - if (kwargs && PyDict_Check(kwargs) && PyDict_Contains(kwargs, str.Get())) { - TPyObjectPtr props(PyDict_GetItem(kwargs, str.Get())); - TVector<TString> properties; - FromPyObject(props.Get(), properties); - self->Holder = TTraits::DoInitPureObject(properties); - } else { - self->Holder = (args || kwargs) ? TTraits::DoInitObject(args, kwargs) : nullptr; - } - if (PyErr_Occurred()) - return -1; - return 0; - } catch (const std::exception& ex) { - PyErr_SetString(TExceptionsHolder::Instance().ToPyException(ex).Get(), ex.what()); - } catch (...) { - PyErr_SetString(PyExc_RuntimeError, "Unknown error occurred while trying to init object"); - } - return -1; - } - - static void DeallocObject(TProxy* self) { - delete self->Holder; - Py_TYPE(self)->tp_free(reinterpret_cast<PyObject*>(self)); - } - - static PyObject* GetObjectAttr(PyObject* pyObj, char* attr); - static int SetObjectAttr(PyObject* pyObj, char* attr, PyObject* value); - static PyObject* GetStr(PyObject*); - static PyObject* GetRepr(PyObject*); - static PyObject* GetIter(PyObject*); - static PyObject* GetNext(PyObject*); - - // Fill class __dict__ with functions to make sure methods names will get to dir() - void FillClassDict() const { - TVector<TString> names; - Attributes.GetMethodCallers().GetAllMethodsNames(names); - for (const auto& name : names) { - TPyObjectPtr callable = NPrivate::TUnboundClosure<TObject>::Instance().CreatePyObject(new NPrivate::TUnboundClosureHolder<TObject>(&PyType, name)); - PyDict_SetItemString(PyType.tp_dict, name.c_str(), callable.Get()); - } - } - - void InitCommonAttributes() { - static bool was = false; - if (was) - return; - was = true; - Attributes.InitCommonAttributes(); - FillClassDict(); - } - - protected: - TPythonType(const char* pyTypeName, const char* typeDescr, PyTypeObject* parentType = nullptr) - : Attributes(GetObjectAttr, SetObjectAttr) - { - PyType.tp_name = pyTypeName; - PyType.tp_doc = typeDescr; - Py_INCREF(PyTypeObjPtr); - if (parentType) { - Py_INCREF(parentType); - PyType.tp_base = parentType; - } - PyType_Ready(&PyType); - - TExceptionsHolder::Instance(); - RegisterJSONBridge(); - - } - - ~TPythonType() { - } - - static TObjectHolder* DoInitObject(PyObject*, PyObject*) { - return nullptr; - } - - static TObjectHolder* DoInitPureObject(const TVector<TString>&) { - return nullptr; - } - - static void SetClosure(PyObject* (*call)(PyObject*, PyObject*, PyObject*)) { - PyType.tp_call = call; - } - - public: - void AddGetter(const TString& attr, TGetterPtr getter) { - Attributes.AddGetter(attr, getter); - } - - void AddSetter(const TString& attr, TSetterPtr setter) { - Attributes.AddSetter(attr, setter); - } - - void AddCaller(const TString& name, TCallerPtr caller) { - Attributes.AddCaller(name, caller); - if (name == "__iter__") { - PyType.tp_iter = GetIter; - } - if (name == "next") { - PyType.tp_iternext = GetNext; - } - } - - void SetIter(getiterfunc tp_iter) { - PyType.tp_iter = tp_iter; - } - - void SetIterNext(iternextfunc tp_iternext) { - PyType.tp_iternext = tp_iternext; - } - - void SetDestructor(destructor tp_dealloc) { - PyType.tp_dealloc = tp_dealloc; - } - - void SetLengthFunction(lenfunc mp_length) { - PyType.tp_as_mapping->mp_length = mp_length; - } - - void SetSubscriptFunction(binaryfunc mp_subscript) { - PyType.tp_as_mapping->mp_subscript = mp_subscript; - } - - void SetAssSubscriptFunction(objobjargproc mp_ass_subscript) { - PyType.tp_as_mapping->mp_ass_subscript = mp_ass_subscript; - } - - typedef TObject TObjectType; - - static TPythonType& Instance() { - static TTraits Traits; - Traits.InitCommonAttributes(); - return Traits; - } - - void Register(PyObject* module, const char* typeName) { - Py_INCREF(PyTypeObjPtr); - if (0 != PyModule_AddObject(module, typeName, PyTypeObjPtr)) - ythrow yexception() << "can't register type \"" << typeName << "\""; - } - - void Register(PyObject* module, const char* objName, TObjectHolder* hld) { - if (0 != PyModule_AddObject(module, objName, CreatePyObject(hld).RefGet())) - ythrow yexception() << "can't register object \"" << objName << "\""; - } - - void Register(TPyObjectPtr module, const TString& typeName) { - Register(module.Get(), typeName.c_str()); - } - - void Register(TPyObjectPtr module, const TString& objName, TObjectHolder* hld) { - Register(module.Get(), objName.c_str(), hld); - } - - static TObjectHolder* CastToObjectHolder(PyObject* obj) { - // Call Instance() to make sure PyTypeObjPtr is already created at this point - Instance(); - if (!PyObject_IsInstance(obj, PyTypeObjPtr)) - return nullptr; - TProxy* prx = reinterpret_cast<TProxy*>(obj); - return prx ? prx->Holder : nullptr; - } - - static TObject* CastToObject(PyObject* obj) { - TObjectHolder* hld = CastToObjectHolder(obj); - return hld ? TTraits::GetObject(*hld) : nullptr; - } - - static TPyObjectPtr CreatePyObject(TObjectHolder* hld) { - TPyObjectPtr r(_PyObject_New(&PyType), true); - TProxy* prx = reinterpret_cast<TProxy*>(r.Get()); - if (prx) - prx->Holder = hld; - return r; - } - }; - - template <typename TObjectHolder, typename TObject, typename TTraits> - PyMappingMethods TPythonType<TObjectHolder, TObject, TTraits>::MappingMethods = {nullptr, nullptr, nullptr}; - - template <typename TObjectHolder, typename TObject, typename TTraits> - PyTypeObject TPythonType<TObjectHolder, TObject, TTraits>::PyType = { - PyVarObject_HEAD_INIT(nullptr, 0) "", sizeof(TProxy), 0, (destructor)&DeallocObject -#if PY_VERSION_HEX < 0x030800b4 - , nullptr, /*tp_print*/ -#endif -#if PY_VERSION_HEX >= 0x030800b4 - , 0, /*tp_vectorcall_offset*/ -#endif - &GetObjectAttr, &SetObjectAttr, nullptr, &GetRepr, nullptr, nullptr, &MappingMethods, nullptr, nullptr, &GetStr, nullptr, nullptr, nullptr, - Py_TPFLAGS_DEFAULT, "", nullptr, nullptr, nullptr, 0, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, 0, InitObject, PyType_GenericAlloc, PyType_GenericNew, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, 0 -#if PY_MAJOR_VERSION >= 3 - , nullptr -#endif -#if PY_VERSION_HEX >= 0x030800b1 - , nullptr /*tp_vectorcall*/ -#endif -#if PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x03090000 - , nullptr /*tp_print*/ -#endif -#if PY_VERSION_HEX >= 0x030C0000 - , 0 /*tp_watched*/ -#endif - }; - - template <typename TObjectHolder, typename TObject, typename TTraits> - PyObject* TPythonType<TObjectHolder, TObject, TTraits>::PyTypeObjPtr = - reinterpret_cast<PyObject*>(&TPythonType<TObjectHolder, TObject, TTraits>::PyType); - - namespace NPrivate { - template <typename TObject> - class TUnboundClosureHolder { - private: - THolder<PyTypeObject> Holder; - TString Method; - - public: - TUnboundClosureHolder(PyTypeObject* ptr, const TString& meth) - : Holder(ptr) - , Method(meth) - { - } - - PyTypeObject* GetObject() const { - return Holder.Get(); - } - - const TString GetMethod() const { - return Method; - } - - PyObject* Call(PyObject* obj, PyObject* args, PyObject*) const { - TPyObjectPtr callable(PyObject_GetAttrString(obj, Method.c_str()), true); - if (!callable.Get()) - ythrow yexception() << "PyBind can't call method '" << Method << "'"; - TPyObjectPtr res(PyObject_CallObject(callable.Get(), args), true); - if (!res.Get() && !PyErr_Occurred()) - ythrow yexception() << "PyBind can't call method '" << Method << "'"; - return res.RefGet(); - } - }; - - template <typename TObject> - class TUnboundClosure: public NPyBind::TPythonType<TUnboundClosureHolder<TObject>, PyTypeObject, TUnboundClosure<TObject>> { - private: - typedef class NPyBind::TPythonType<TUnboundClosureHolder<TObject>, PyTypeObject, TUnboundClosure<TObject>> TParent; - friend class NPyBind::TPythonType<TUnboundClosureHolder<TObject>, PyTypeObject, TUnboundClosure<TObject>>; - - class TReprMethodCaller: public TBaseMethodCaller<PyTypeObject> { - public: - bool CallMethod(PyObject* closure, PyTypeObject*, PyObject*, PyObject*, PyObject*& res) const override { - TUnboundClosureHolder<TObject>* hld = TParent::CastToObjectHolder(closure); - TPyObjectPtr type((PyObject*)hld->GetObject()); - - TString nameStr; - TPyObjectPtr name(PyObject_GetAttrString(type.Get(), "__name__"), true); - if (!name.Get() || !FromPyObject(name.Get(), nameStr)) - ythrow yexception() << "Could not get name of object"; - - TString methodName(hld->GetMethod()); - - TString message = "<unbound method " + nameStr + "." + methodName + ">"; - res = ReturnString(message); - return (res != nullptr); - } - }; - - private: - TUnboundClosure() - : TParent("", "") - { - TParent::AddCaller("__repr__", new TReprMethodCaller()); - TParent::AddCaller("__str__", new TReprMethodCaller()); - TParent::SetClosure(&Call); - } - - static PyObject* Call(PyObject* closure, PyObject* args, PyObject* kwargs) { - try { - TUnboundClosureHolder<TObject>* hld = TParent::CastToObjectHolder(closure); - if (!hld) - ythrow yexception() << "Can't cast object to TypeHolder"; - - size_t size = 0; - if (!PyTuple_Check(args) || (size = PyTuple_Size(args)) < 1) - ythrow yexception() << "Can't parse first argument: it should be valid object"; - --size; - TPyObjectPtr obj(PyTuple_GetItem(args, 0)); - TPyObjectPtr newArgs(PyTuple_New(size), true); - - for (size_t i = 0; i < size; ++i) { - TPyObjectPtr item(PyTuple_GetItem(args, i + 1)); - PyTuple_SetItem(newArgs.Get(), i, item.RefGet()); - } - - return hld->Call(obj.Get(), newArgs.Get(), kwargs); - } catch (const std::exception& ex) { - PyErr_SetString(::NPyBind::TExceptionsHolder::Instance().ToPyException(ex).Get(), ex.what()); - } catch (...) { - PyErr_SetString(PyExc_RuntimeError, "Unknown error occurred while trying to call method"); - } - return nullptr; - } - - static PyTypeObject* GetObject(TUnboundClosureHolder<TObject>& obj) { - return obj.GetObject(); - } - }; - - template <typename TObject> - class TBoundClosureHolder { - private: - TPyObjectPtr Ptr; - TObject* Object; - TString Method; - const TMethodCallers<TObject>& MethodCallers; - - public: - TBoundClosureHolder(PyObject* ptr, TObject* obj, const TString& meth, const TMethodCallers<TObject>& callers) - : Ptr(ptr) - , Object(obj) - , Method(meth) - , MethodCallers(callers) - { - } - - TPyObjectPtr GetObjectPtr() const { - return Ptr; - } - - TObject* GetObject() const { - return Object; - } - - const TString GetMethod() const { - return Method; - } - - PyObject* Call(PyObject* args, PyObject* kwargs) const { - PyObject* res = MethodCallers.CallMethod(Ptr.Get(), Object, args, kwargs, Method); - if (res == nullptr && !PyErr_Occurred()) - ythrow yexception() << "PyBind can't call method '" << Method << "'"; - return res; - } - }; - - template <typename TObject> - class TBoundClosure: public TPythonType<TBoundClosureHolder<TObject>, TObject, TBoundClosure<TObject>> { - private: - typedef TPythonType<TBoundClosureHolder<TObject>, TObject, TBoundClosure<TObject>> TMyParent; - class TReprMethodCaller: public TBaseMethodCaller<TObject> { - public: - bool CallMethod(PyObject* closure, TObject*, PyObject*, PyObject*, PyObject*& res) const override { - TBoundClosureHolder<TObject>* hld = TMyParent::CastToObjectHolder(closure); - TPyObjectPtr obj(hld->GetObjectPtr()); - TPyObjectPtr type(PyObject_Type(obj.Get()), true); - - TString reprStr; - TPyObjectPtr repr(PyObject_Repr(obj.Get()), true); - if (!repr.Get() || !FromPyObject(repr.Get(), reprStr)) - ythrow yexception() << "Could not get repr of object"; - - TString nameStr; - TPyObjectPtr name(PyObject_GetAttrString(type.Get(), "__name__"), true); - if (!name.Get() || !FromPyObject(name.Get(), nameStr)) - ythrow yexception() << "Could not get name of object"; - - TString methodName(hld->GetMethod()); - - TString message = "<bound method " + nameStr + "." + methodName + " of " + reprStr + ">"; - res = ReturnString(message); - return (res != nullptr); - } - }; - - private: - static PyObject* Call(PyObject* closure, PyObject* args, PyObject* kwargs) { - try { - TBoundClosureHolder<TObject>* hld = TMyParent::CastToObjectHolder(closure); - if (!hld) - ythrow yexception() << "Can't cast object to ClosureHolder"; - - return hld->Call(args, kwargs); - } catch (const std::exception& ex) { - PyErr_SetString(::NPyBind::TExceptionsHolder::Instance().ToPyException(ex).Get(), ex.what()); - } catch (...) { - PyErr_SetString(PyExc_RuntimeError, "Unknown error occurred while trying to call method"); - } - return nullptr; - } - - public: - TBoundClosure() - : TMyParent("", "") - { - TMyParent::AddCaller("__repr__", new TReprMethodCaller()); - TMyParent::AddCaller("__str__", new TReprMethodCaller()); - TMyParent::SetClosure(&Call); - } - - static TObject* GetObject(const TBoundClosureHolder<TObject>& closure) { - return closure.GetObject(); - } - }; - - } - - template <typename TObjectHolder, typename TObject, typename TTraits> - PyObject* TPythonType<TObjectHolder, TObject, TTraits>::GetObjectAttr(PyObject* pyObj, char* attr) { - try { - TObject* obj = CastToObject(pyObj); - PyObject* res = obj ? Instance().Attributes.GetAttrGetters().GetAttr(pyObj, *obj, attr) : nullptr; - if (res == nullptr && Instance().Attributes.GetMethodCallers().HasMethod(pyObj, obj, attr)) { - TPyObjectPtr r = NPrivate::TBoundClosure<TObject>::Instance().CreatePyObject(new NPrivate::TBoundClosureHolder<TObject>(pyObj, obj, attr, Instance().Attributes.GetMethodCallers())); - res = r.RefGet(); - } - if (res == nullptr && !PyErr_Occurred()) - ythrow TPyErr(PyExc_AttributeError) << "PyBind can't get attribute '" << attr << "'"; - return res; - } catch (const std::exception& ex) { - PyErr_SetString(TExceptionsHolder::Instance().ToPyException(ex).Get(), ex.what()); - } catch (...) { - PyErr_SetString(PyExc_RuntimeError, (TString("Unknown error occurred while trying to get attribute '") + attr + "'").c_str()); - } - return nullptr; - } - - template <typename TObjectHolder, typename TObject, typename TTraits> - int TPythonType<TObjectHolder, TObject, TTraits>::SetObjectAttr(PyObject* pyObj, char* attr, PyObject* value) { - try { - TObject* obj = CastToObject(pyObj); - bool res = obj ? Instance().Attributes.GetAttrSetters().SetAttr(pyObj, *obj, attr, value) : false; - if (!res && !PyErr_Occurred()) - ythrow yexception() << "PyBind can't set attribute '" << attr << "'"; - return res ? 0 : -1; - } catch (const std::exception& ex) { - PyErr_SetString(TExceptionsHolder::Instance().ToPyException(ex).Get(), ex.what()); - } catch (...) { - PyErr_SetString(PyExc_RuntimeError, (TString("Unknown error occurred while trying to set attribute '") + attr + "'").c_str()); - } - return -1; - } - - template <typename TObjectHolder, typename TObject, typename TTraits> - PyObject* TPythonType<TObjectHolder, TObject, TTraits>::GetStr(PyObject* obj) { - try { - TObject* self = CastToObject(obj); - return Instance().Attributes.GetMethodCallers().CallMethod(obj, self, nullptr, nullptr, "__str__"); - } catch (const std::exception& ex) { - PyErr_SetString(TExceptionsHolder::Instance().ToPyException(ex).Get(), ex.what()); - } catch (...) { - PyErr_SetString(PyExc_RuntimeError, (TString("Unknown error occurred while trying to call '__str__'").c_str())); - } - return nullptr; - } - - template <typename TObjectHolder, typename TObject, typename TTraits> - PyObject* TPythonType<TObjectHolder, TObject, TTraits>::GetIter(PyObject* obj) { - try { - TObject* self = CastToObject(obj); - return Instance().Attributes.GetMethodCallers().CallMethod(obj, self, nullptr, nullptr, "__iter__"); - } catch (const std::exception& ex) { - PyErr_SetString(TExceptionsHolder::Instance().ToPyException(ex).Get(), ex.what()); - } catch (...) { - PyErr_SetString(PyExc_RuntimeError, (TString("Unknown error occurred while trying to call '__iter__'").c_str())); - } - return nullptr; - } - - template <typename TObjectHolder, typename TObject, typename TTraits> - PyObject* TPythonType<TObjectHolder, TObject, TTraits>::GetNext(PyObject* obj) { - try { - TObject* self = CastToObject(obj); - return Instance().Attributes.GetMethodCallers().CallMethod(obj, self, nullptr, nullptr, "next"); - } catch (const std::exception& ex) { - PyErr_SetString(TExceptionsHolder::Instance().ToPyException(ex).Get(), ex.what()); - } catch (...) { - PyErr_SetString(PyExc_RuntimeError, (TString("Unknown error occurred while trying to call 'next'").c_str())); - } - return nullptr; - } - - template <typename TObjectHolder, typename TObject, typename TTraits> - PyObject* TPythonType<TObjectHolder, TObject, TTraits>::GetRepr(PyObject* obj) { - try { - TObject* self = CastToObject(obj); - return Instance().Attributes.GetMethodCallers().CallMethod(obj, self, nullptr, nullptr, "__repr__"); - } catch (const std::exception& ex) { - PyErr_SetString(TExceptionsHolder::Instance().ToPyException(ex).Get(), ex.what()); - } catch (...) { - PyErr_SetString(PyExc_RuntimeError, (TString("Unknown error occurred while trying to call '__repr__'").c_str())); - } - return nullptr; - } -} diff --git a/library/cpp/pybind/v2.cpp b/library/cpp/pybind/v2.cpp deleted file mode 100644 index edce0be719..0000000000 --- a/library/cpp/pybind/v2.cpp +++ /dev/null @@ -1,43 +0,0 @@ -#include "v2.h" -namespace NPyBind { - namespace Detail { - template <> - PyTypeObject* GetParentType<void>(const TPyModuleDefinition&) { - return nullptr; - } - - - template <bool InitEnabled> - void UpdateClassNamesInModule(TPyModuleDefinition& M, const TString& name, PyTypeObject* pythonType) { - if (!InitEnabled) { - return; - } - M.ClassName2Type[name] = pythonType; - } - - template <bool InitEnabled> - void UpdateGetContextInModule(TPyModuleDefinition& M, const TString& name, IGetContextBase* base) { - if (!InitEnabled) { - return; - } - M.Class2ContextGetter[name] = base; - } - - TPyModuleRegistry::TPyModuleRegistry() { -#if PY_MAJOR_VERSION >= 3 - NPrivate::AddFinalizationCallBack([this]() { - if (UnnamedModule) { - UnnamedModule.Clear(); - } - Name2Def.clear(); - }); -#endif - } - template void UpdateClassNamesInModule<false>(TPyModuleDefinition& M, const TString& name, PyTypeObject* pythonType); - template void UpdateClassNamesInModule<true>(TPyModuleDefinition& M, const TString& name, PyTypeObject* pythonType); - - - template void UpdateGetContextInModule<false>(TPyModuleDefinition& M, const TString& name, IGetContextBase* pythonType); - template void UpdateGetContextInModule<true>(TPyModuleDefinition& M, const TString& name, IGetContextBase* pythonType); - }//Detail -}//NPyBind diff --git a/library/cpp/pybind/v2.h b/library/cpp/pybind/v2.h deleted file mode 100644 index f561d6a380..0000000000 --- a/library/cpp/pybind/v2.h +++ /dev/null @@ -1,514 +0,0 @@ -#pragma once - -#include <library/cpp/pybind/method.h> -#include <library/cpp/pybind/typedesc.h> -#include <library/cpp/pybind/module.h> -#include <util/generic/hash.h> -#include <util/generic/hash_set.h> -#include <util/generic/string.h> -namespace NPyBind { -#define DEFINE_CONVERTERS_IMPL(TClass) \ - PyObject* BuildPyObject(typename TClass::TBase&& base) { \ - return TClass::BuildPyObject(std::move(base)); \ - } \ - PyObject* BuildPyObject(const typename TClass::TBase& base) { \ - return TClass::BuildPyObject(base); \ - } - -#define DEFINE_CONVERTERS(function) DEFINE_CONVERTERS_IMPL(TFunctionResult<decltype(function)>) - -#define DEFINE_TRANSFORMERS_IMPL(TClass) \ - template <> \ - bool ::NPyBind::FromPyObject<typename TClass::TBase*>(PyObject * obj, typename TClass::TBase * &res) { \ - res = TClass::CastToObject(obj); \ - return res != nullptr; \ - } \ - template <> \ - bool ::NPyBind::FromPyObject<typename TClass::TBase const*>(PyObject * obj, typename TClass::TBase const*& res) { \ - res = TClass::CastToObject(obj); \ - return res != nullptr; \ - } - -#define DEFINE_TRANSFORMERS(function) DEFINE_TRANSFORMERS_IMPL(TFunctionResult<decltype(function)>) - - namespace Detail { - struct IGetContextBase { - virtual ~IGetContextBase() = default; - }; - } //Detail - struct TPyModuleDefinition { - static void InitModule(const TString& name); - static TPyModuleDefinition& GetModule(); - - TString Name; - NPyBind::TPyObjectPtr M; - THashMap<TString, PyTypeObject*> ClassName2Type; - THashMap<TString, Detail::IGetContextBase*> Class2ContextGetter; - }; - - namespace Detail { - // Manages modules lifecycle - // IMPORTANT!!! Don't use it in PyBind v1 environment, it will lead to inconsistent state of v1 module - // UnnamedModule-> new unnamed module stub, this stub become current module. In this case you can add functions to it - // InitModuleWithName -> convert unnamed module into named one, now you can switch to it in switch, this module remains current - // SwitchToModule switches to the particular module in registry, this module becomes current. - class TPyModuleRegistry { - private: - TPyModuleRegistry(); - TPyModuleRegistry(const TPyModuleRegistry&) = delete; - TPyModuleRegistry& operator=(TPyModuleRegistry&) = delete; - public: - static TPyModuleRegistry& Get() { - static TPyModuleRegistry registry; - return registry; - } - TPyModuleDefinition& GetCurrentModule() { - if (!CurrentModule) { - GetUnnamedModule(); - } - return *CurrentModule; - } - - TPyModuleDefinition& GetUnnamedModule() { - if (!UnnamedModule) { - UnnamedModule = TPyModuleDefinition(); - CurrentModule = const_cast<TPyModuleDefinition*>(UnnamedModule.Get()); - } - return *UnnamedModule; - } - - TPyModuleDefinition& InitModuleWithName(const TString& name) { - if (!UnnamedModule) { - GetUnnamedModule(); - } - Name2Def[name] = *UnnamedModule; - UnnamedModule.Clear(); - CurrentModule = &Name2Def[name]; - return *CurrentModule; - } - - TPyModuleDefinition& SwitchToModuleByName(const TString& name) { - Y_ENSURE(Name2Def.contains(name)); - Y_ENSURE(UnnamedModule.Empty()); - CurrentModule = &Name2Def[name]; - return *CurrentModule; - } - private: - TPyModuleDefinition* CurrentModule = nullptr; - TMaybe<TPyModuleDefinition> UnnamedModule;// - THashMap<TString, TPyModuleDefinition> Name2Def; - }; - }//Detail - - inline void TPyModuleDefinition::InitModule(const TString& name) { - Detail::TPyModuleRegistry::Get().GetUnnamedModule() = TPyModuleDefinition{name, TModuleHolder::Instance().InitModule(name), {}, {}}; - Detail::TPyModuleRegistry::Get().InitModuleWithName(name); - } - - inline TPyModuleDefinition& TPyModuleDefinition::GetModule() { - return Detail::TPyModuleRegistry::Get().GetCurrentModule(); - } - - namespace Detail { - template <class TPythonType> - struct TNameCtx { - TString ClassShortName; - static TNameCtx& GetNameCtx() { - static TNameCtx result; - return result; - } - }; - template <class TBase> - struct TContextImpl { - PyTypeObject* ParentType = nullptr; - TString ClassShortName; - TString ClassFullName; - TString ClassDescription; - - - TVector<std::pair<TString, typename TPythonTypeAttributes<TBase>::TCallerPtr>> ListCallers; - TVector<std::pair<TString, typename TPythonTypeAttributes<TBase>::TGetterPtr>> ListGetters; - TVector<std::pair<TString, typename TPythonTypeAttributes<TBase>::TSetterPtr>> ListSetters; - }; - - template <class TObject> - struct IGetContext: public IGetContextBase { - virtual ~IGetContext() = default; - virtual const TContextImpl<TObject>& GetContext() const = 0; - }; - - template <typename THolderClass, typename TBaseClass, bool ShouldEnable, typename=std::enable_if_t<!ShouldEnable || !std::is_default_constructible_v<TBaseClass>>> - THolderClass* DoInitPureObject(const TVector<TString>&) { - ythrow yexception() << "Can't create this object in pure mode from python"; - } - - template <typename THolderClass, typename TBaseClass, bool ShouldEnable, typename=std::enable_if_t<ShouldEnable && std::is_default_constructible_v<TBaseClass>>, typename=void> - THolderClass* DoInitPureObject(const TVector<TString>&) { - return new THolderClass(MakeHolder<TBaseClass>()); - } - - template <typename T> - PyTypeObject* GetParentType(const TPyModuleDefinition& m) { - auto shortName = Detail::TNameCtx<T>::GetNameCtx().ClassShortName; - auto it = m.ClassName2Type.find(shortName); - return (it == m.ClassName2Type.end()) ? nullptr : it->second; - } - - template <> - PyTypeObject* GetParentType<void>(const TPyModuleDefinition&); - - template <bool InitEnabled> - void UpdateClassNamesInModule(TPyModuleDefinition& M, const TString& name, PyTypeObject* pythonType); - - template <bool InitEnabled> - void UpdateGetContextInModule(TPyModuleDefinition& M, const TString& name, IGetContextBase* base); - } - - - template <class TParentPyClass_=void> - struct TPyParentClassTraits { - using TParentPyClass = TParentPyClass_; - }; - - template <bool InitEnabled_, class TParentPyClass_=void> - struct TPyClassConfigTraits: public TPyParentClassTraits<TParentPyClass_> { - constexpr static bool InitEnabled = InitEnabled_; - constexpr static bool RawInit = false; - }; - - template <class TParentPyClass_=void> - struct TPyClassRawInitConfigTraits: public TPyParentClassTraits<TParentPyClass_> { - constexpr static bool InitEnabled = true; - constexpr static bool RawInit = true; - }; - - - template <typename TBaseClass, typename TPyClassConfigTraits, typename... ConstructorArgs> - class TPyClass { - public: - using TBase = TBaseClass; - private: - using TThisClass = TPyClass<TBaseClass, TPyClassConfigTraits, ConstructorArgs...>; - using TContext = Detail::TContextImpl<TBase>; - struct THolder { - ::THolder<TBase> Holder; - THolder(::THolder<TBase>&& right) - : Holder(std::move(right)) - { - } - THolder(TBase&& right) - : Holder(MakeHolder<TBase>(std::move(right))) - { - } - }; - - class TSelectedTraits: public NPyBind::TPythonType<THolder, TBase, TSelectedTraits> { - private: - using TParent = NPyBind::TPythonType<THolder, TBase, TSelectedTraits>; - friend TParent; - - public: - TSelectedTraits() - : TParent(TThisClass::GetContext().ClassFullName.data(), TThisClass::GetContext().ClassDescription.data(), TThisClass::GetContext().ParentType) - { - for (const auto& caller : TThisClass::GetContext().ListCallers) { - TParent::AddCaller(caller.first, caller.second); - } - - for (const auto& getter : TThisClass::GetContext().ListGetters) { - TParent::AddGetter(getter.first, getter.second); - } - - for (const auto& setter : TThisClass::GetContext().ListSetters) { - TParent::AddSetter(setter.first, setter.second); - } - } - - static TBase* GetObject(const THolder& holder) { - return holder.Holder.Get(); - } - - static THolder* DoInitObject(PyObject* args, PyObject* kwargs) { - if constexpr (TPyClassConfigTraits::InitEnabled) { - if constexpr (TPyClassConfigTraits::RawInit) { - static_assert(sizeof...(ConstructorArgs) == 0, "Do not pass construction args if use RawInit."); - return new THolder(::MakeHolder<TBase>(args, kwargs)); - } else { - if (args && (!PyTuple_Check(args) || PyTuple_Size(args) != sizeof...(ConstructorArgs))) { - ythrow yexception() << "Method takes " << sizeof...(ConstructorArgs) << " arguments, " << PyTuple_Size(args) << " provided"; - } - ::THolder<TBaseClass> basePtr{Apply([](auto&&... unpackedArgs) {return new TBase(std::forward<decltype(unpackedArgs)>(unpackedArgs)...); }, GetArguments<ConstructorArgs...>(args))}; - return new THolder(std::move(basePtr)); - } - } else { - ythrow yexception() << "Can't create this object from python"; - } - } - - static THolder* DoInitPureObject(const TVector<TString>& properties) { - return Detail::DoInitPureObject<THolder, TBase, TPyClassConfigTraits::InitEnabled>(properties); - } - - static TBase* CastToObject(PyObject* obj) { - return TParent::CastToObject(obj); - } - - static PyTypeObject* GetType() { - return TParent::GetPyTypePtr(); - } - }; - - class TContextHolder: public Detail::IGetContext<TBaseClass> { - public: - static TContextHolder& GetContextHolder() { - static TContextHolder holder; - return holder; - } - - TContext& GetContext() { - return Context; - } - const TContext& GetContext() const override { - return Context; - } - private: - TContext Context; - }; - - template <class TDerivedClass, class TSuperClass> - class TCallerWrapper: public TBaseMethodCaller<TDerivedClass> { - public: - explicit TCallerWrapper(TSimpleSharedPtr<const TBaseMethodCaller<TSuperClass>> baseCaller) - : BaseCaller(baseCaller) { - Y_ENSURE(BaseCaller); - } - - bool CallMethod(PyObject* owner, TDerivedClass* self, PyObject* args, PyObject* kwargs, PyObject*& res) const override { - return BaseCaller->CallMethod(owner, static_cast<TSuperClass*>(self), args, kwargs, res); - } - - private: - TSimpleSharedPtr<const TBaseMethodCaller<TSuperClass>> BaseCaller; - }; - - template <class TDerivedClass, class TSuperClass> - class TSetterWrapper: public TBaseAttrSetter<TDerivedClass> { - public: - explicit TSetterWrapper(TSimpleSharedPtr<TBaseAttrSetter<TSuperClass>> baseSetter) - : BaseSetter(baseSetter) { - Y_ENSURE(BaseSetter); - } - - bool SetAttr(PyObject* owner, TDerivedClass& self, const TString& attr, PyObject* val) override { - return BaseSetter->SetAttr(owner, static_cast<TSuperClass&>(self), attr, val); - } - - private: - TSimpleSharedPtr<TBaseAttrSetter<TSuperClass>> BaseSetter; - }; - - template <class TDerivedClass, class TSuperClass> - class TGetterWrapper: public TBaseAttrGetter<TDerivedClass> { - public: - explicit TGetterWrapper(TSimpleSharedPtr<const TBaseAttrGetter<TSuperClass>> baseGetter) - : BaseGetter(baseGetter) { - Y_ENSURE(BaseGetter); - } - - bool GetAttr(PyObject* owner, const TDerivedClass& self, const TString& attr, PyObject*& res) const override { - return BaseGetter->GetAttr(owner, static_cast<const TSuperClass&>(self), attr, res); - } - - private: - TSimpleSharedPtr<const TBaseAttrGetter<TSuperClass>> BaseGetter; - }; - - template <class TSuperClass, typename=std::enable_if_t<!std::is_same_v<TSuperClass, void>>> - void ReloadAttrsFromBase() { - auto shortName = Detail::TNameCtx<TSuperClass>::GetNameCtx().ClassShortName; - if (!M.Class2ContextGetter.count(shortName)) { - return; - } - auto callerBasePtr = M.Class2ContextGetter[shortName]; - if (auto getContextPtr = dynamic_cast<const Detail::IGetContext<TSuperClass>*>(callerBasePtr)) { - auto& ctx = getContextPtr->GetContext(); - auto getUniqueNames = [](const auto& collection) { - THashSet<TString> uniqueNames; - for (const auto& elem : collection) { - uniqueNames.insert(elem.first); - } - return uniqueNames; - }; - - auto uniqueCallerNames = getUniqueNames(GetContext().ListCallers); - using TConcreteCallerWrapper = TCallerWrapper<TBaseClass, TSuperClass>; - for (const auto& caller : ctx.ListCallers) { - if (uniqueCallerNames.contains(caller.first)) { - continue; - } - GetContext().ListCallers.push_back(std::make_pair(caller.first, MakeSimpleShared<TConcreteCallerWrapper>(caller.second))); - } - - auto uniqueGettersNames = getUniqueNames(GetContext().ListGetters); - using TConcreteGetterWrapper = TGetterWrapper<TBaseClass, TSuperClass>; - for (const auto& getter : ctx.ListGetters) { - if (uniqueGettersNames.contains(getter.first)) { - continue; - } - GetContext().ListGetters.push_back(std::make_pair(getter.first, MakeSimpleShared<TConcreteGetterWrapper>(getter.second))); - } - - auto uniqueSetterNames = getUniqueNames(GetContext().ListSetters); - using TConcreteSetterWrapper = TSetterWrapper<TBaseClass, TSuperClass>; - for (auto& setter : ctx.ListSetters) { - if (uniqueSetterNames.contains(setter.first)) { - continue; - } - GetContext().ListSetters.push_back(std::make_pair(setter.first, MakeSimpleShared<TConcreteSetterWrapper>(setter.second))); - } - } - } - - template <class TSuperClass, typename=std::enable_if_t<std::is_same_v<TSuperClass, void>>, typename=void> - void ReloadAttrsFromBase() { - } - - void CompleteImpl() { - ReloadAttrsFromBase<typename TPyClassConfigTraits::TParentPyClass>(); - TSelectedTraits::Instance().Register(M.M, GetContext().ClassShortName); - } - - static TContext& GetContext() { - return TContextHolder::GetContextHolder().GetContext(); - } - - - friend struct Detail::TContextImpl<TBase>;//instead of context - friend struct THolder; - friend class TSelectedTraits; - - using TCallerFunc = std::function<bool(PyObject*, TBaseClass*, PyObject*, PyObject*, PyObject*&)>; - class TFuncCallerWrapper: public TBaseMethodCaller<TBaseClass> { - public: - explicit TFuncCallerWrapper(TCallerFunc func) - : Func(func) { - Y_ENSURE(func); - } - - bool CallMethod(PyObject* owner, TBaseClass* self, PyObject* args, PyObject* kwargs, PyObject*& res) const override { - return Func(owner, self, args, kwargs, res); - } - private: - mutable TCallerFunc Func; - }; - public: - TPyClass(const TString& name, const TString& descr = "") - : M(TPyModuleDefinition::GetModule()) - { - Detail::UpdateClassNamesInModule<TPyClassConfigTraits::InitEnabled>(M, name, TSelectedTraits::GetType()); - Detail::UpdateGetContextInModule<TPyClassConfigTraits::InitEnabled>(M, name, &TContextHolder::GetContextHolder()); - - GetContext().ClassFullName = TString::Join(M.Name, ".", name); - GetContext().ClassShortName = name; - GetContext().ClassDescription = descr; - GetContext().ParentType = Detail::GetParentType<typename TPyClassConfigTraits::TParentPyClass>(M); - Detail::TNameCtx<TBaseClass>::GetNameCtx().ClassShortName = name; - } - - template <typename TMemberFuction, typename = std::enable_if_t<std::is_member_function_pointer_v<TMemberFuction>>, typename=std::enable_if_t<!TIsPointerToConstMemberFunction<TMemberFuction>::value>> - TThisClass& Def(const TString& name, TMemberFuction t) { - GetContext().ListCallers.push_back(std::make_pair(name, CreateMethodCaller<TBase>(t))); - return *this; - } - - template <typename TMemberFuction, typename = std::enable_if_t<std::is_member_function_pointer_v<TMemberFuction>>, typename=std::enable_if_t<TIsPointerToConstMemberFunction<TMemberFuction>::value>, typename=void> - TThisClass& Def(const TString& name, TMemberFuction t) { - GetContext().ListCallers.push_back(std::make_pair(name, CreateConstMethodCaller<TBase>(t))); - return *this; - } - - template <typename TMemberObject, typename = std::enable_if_t<std::is_member_object_pointer_v<TMemberObject>>> - TThisClass& Def(const TString& name, TMemberObject t) { - GetContext().ListGetters.push_back(std::make_pair(name, CreateAttrGetter<TBase>(t))); - GetContext().ListSetters.push_back(std::make_pair(name, CreateAttrSetter<TBase>(t))); - return *this; - } - - template <typename TResultType, typename... Args> - TThisClass& DefByFunc(const TString& name, std::function<TResultType(TBaseClass&, Args...)> func) { - GetContext().ListCallers.push_back(std::make_pair(name, CreateFunctorCaller<TBase, TResultType, Args...>(func))); - return *this; - } - - TThisClass& DefByFunc(const TString& name, TCallerFunc origFunc) { - GetContext().ListCallers.push_back(std::make_pair(name, MakeSimpleShared<TFuncCallerWrapper>(origFunc))); - return *this; - } - - template <typename TMemberObject> - TThisClass& DefReadonly(const TString& name, TMemberObject t, std::enable_if_t<std::is_member_object_pointer<TMemberObject>::value>* = nullptr) { - GetContext().ListGetters.push_back(std::make_pair(name, CreateAttrGetter<TBase>(t))); - return *this; - } - - - template <typename TMethodGetter, typename TMethodSetter, typename=std::enable_if_t<std::is_member_function_pointer_v<TMethodGetter> && std::is_member_function_pointer_v<TMethodSetter>>> - TThisClass& AsProperty(const TString& name, TMethodGetter getter, TMethodSetter setter) { - GetContext().ListGetters.push_back(std::make_pair(name, CreateMethodAttrGetter<TBase>(getter))); - GetContext().ListSetters.push_back(std::make_pair(name, CreateMethodAttrSetter<TBase>(setter))); - return *this; - } - - template <typename TMethodGetter, typename TMethodSetter, typename=std::enable_if_t<!std::is_member_function_pointer_v<TMethodGetter> && !std::is_member_function_pointer_v<TMethodSetter>>> - TThisClass& AsPropertyByFunc(const TString& name, TMethodGetter getter, TMethodSetter setter) { - GetContext().ListGetters.push_back(std::make_pair(name, CreateFunctorAttrGetter<TBase>(getter))); - GetContext().ListSetters.push_back(std::make_pair(name, CreateFunctorAttrSetter<TBase>(setter))); - return *this; - } - - template <typename TMethodGetter, typename=std::enable_if_t<std::is_member_function_pointer_v<TMethodGetter>>> - TThisClass& AsProperty(const TString& name, TMethodGetter getter) { - GetContext().ListGetters.push_back(std::make_pair(name, CreateMethodAttrGetter<TBase>(getter))); - return *this; - } - - template <typename TMethodGetter> - TThisClass& AsPropertyByFunc(const TString& name, TMethodGetter getter) { - GetContext().ListGetters.push_back(std::make_pair(name, CreateFunctorAttrGetter<TBase>(getter))); - return *this; - } - - TThisClass& Complete() { - if (!Completed) { - CompleteImpl(); - Completed = true; - } - return *this; - } - - public: - static PyObject* BuildPyObject(TBase&& base) { - return NPyBind::BuildPyObject(TSelectedTraits::Instance().CreatePyObject(new THolder(std::move(base)))); - } - - static PyObject* BuildPyObject(const TBase& base) { - return NPyBind::BuildPyObject(TSelectedTraits::Instance().CreatePyObject(new THolder(TBase(base)))); // WARN - copy - } - - static TBase* CastToObject(PyObject* obj) { - return TSelectedTraits::CastToObject(obj); - } - - private: - TPyModuleDefinition& M; - bool Completed = false; - }; - - template <typename TFunctionSignature, TFunctionSignature function> - void DefImpl(const TString& name, const TString& descr = "") { - NPyBind::TModuleHolder::Instance().AddModuleMethod<TModuleMethodCaller<TFunctionSignature, function>::Call>(name, descr); - } - -#define DefFunc(NAME, FUNC) NPyBind::DefImpl<decltype(FUNC), FUNC>(NAME) -#define DefFuncDescr(NAME, FUNC, DESCR) NPyBind::DefImpl<decltype(FUNC), FUNC>(NAME, DESCR) -}; diff --git a/library/cpp/pybind/ya.make b/library/cpp/pybind/ya.make deleted file mode 100644 index 9b7b3413f2..0000000000 --- a/library/cpp/pybind/ya.make +++ /dev/null @@ -1,14 +0,0 @@ -PY23_NATIVE_LIBRARY() - -SRCS( - cast.cpp - pod.cpp - typedesc.cpp - module.cpp - exceptions.cpp - embedding.cpp - empty.cpp - v2.cpp -) - -END() diff --git a/library/python/archive/__init__.py b/library/python/archive/__init__.py deleted file mode 100644 index a6e032ff4c..0000000000 --- a/library/python/archive/__init__.py +++ /dev/null @@ -1,266 +0,0 @@ -import errno -import logging -import os -import random -import shutil -import stat -import string -import sys - -import six - -import libarchive -import libarchive._libarchive as _libarchive - -from pathlib2 import PurePath - -logger = logging.getLogger(__name__) - -GZIP = "gzip" -ZSTD = "zstd" - -ENCODING = "utf-8" - - -class ConfigureError(Exception): - pass - - -class Level(object): - def __init__(self, level): - self.level = level - - -class Compression(object): - Fast = Level(1) - Default = Level(2) - Best = Level(3) - - -def get_compression_level(filter_name, level): - if level is None or not filter_name: - return None - elif isinstance(level, Level): - level = { - GZIP: { - Compression.Fast: 1, - Compression.Default: 6, - Compression.Best: 9, - }, - ZSTD: { - Compression.Fast: 1, - Compression.Default: 3, - Compression.Best: 22, - }, - }[filter_name][level] - return level - - -def encode(value, encoding): - return value.encode(encoding) - - -def extract_tar(tar_file_path, output_dir, strip_components=None, fail_on_duplicates=True): - output_dir = encode(output_dir, ENCODING) - _make_dirs(output_dir) - with libarchive.Archive(tar_file_path, mode="rb") as tarfile: - for e in tarfile: - p = _strip_prefix(e.pathname, strip_components) - if not p: - continue - dest = os.path.join(output_dir, encode(p, ENCODING)) - if e.pathname.endswith("/"): - _make_dirs(dest) - continue - - if strip_components and fail_on_duplicates: - if os.path.exists(dest): - raise Exception( - "The file {} is duplicated because of strip_components={}".format(dest, strip_components) - ) - - _make_dirs(os.path.dirname(dest)) - - if e.ishardlink(): - src = os.path.join(output_dir, _strip_prefix(e.hardlink, strip_components)) - _hardlink(src, dest) - continue - if e.issym(): - src = _strip_prefix(e.linkname, strip_components) - _symlink(src, dest) - continue - - with open(dest, "wb") as f: - if hasattr(os, "fchmod"): - os.fchmod(f.fileno(), e.mode & 0o7777) - libarchive.call_and_check( - _libarchive.archive_read_data_into_fd, - tarfile._a, - tarfile._a, - f.fileno(), - ) - - -def _strip_prefix(path, strip_components): - if not strip_components: - return path - p = PurePath(path) - stripped = str(p.relative_to(*p.parts[:strip_components])) - return '' if stripped == '.' else stripped - - -def tar( - paths, - output, - compression_filter=None, - compression_level=None, - fixed_mtime=None, - onerror=None, - postprocess=None, - dereference=False, -): - if isinstance(paths, six.string_types): - paths = [paths] - - if isinstance(output, six.string_types): - temp_tar_path, stream = ( - output + "." + "".join(random.sample(string.ascii_lowercase, 8)), - None, - ) - else: - temp_tar_path, stream = None, output - - compression_level = get_compression_level(compression_filter, compression_level) - - try: - if compression_filter: - filter_name = compression_filter - if compression_level is not None: - filter_opts = {"compression-level": str(compression_level)} - else: - filter_opts = {} - # force gzip don't store mtime of the original file being compressed (http://www.gzip.org/zlib/rfc-gzip.html#file-format) - if fixed_mtime is not None and compression_filter == GZIP: - filter_opts["timestamp"] = "" - else: - filter_name = filter_opts = None - - with libarchive.Archive( - stream or temp_tar_path, - mode="wb", - format="gnu", - filter=filter_name, - filter_opts=filter_opts, - fixed_mtime=fixed_mtime, - ) as tarfile: - # determine order if fixed_mtime is specified to produce stable archive - paths = paths if fixed_mtime is None else sorted(paths) - - for p in paths: - if type(p) == tuple: - path, arcname = p - else: - path, arcname = p, os.path.basename(p) - - if os.path.isdir(path): - for root, dirs, files in os.walk(path, followlinks=dereference): - if fixed_mtime is None: - entries = dirs + files - else: - entries = sorted(dirs) + sorted(files) - - reldir = os.path.relpath(root, path) - for f in entries: - _writepath( - tarfile, - os.path.join(root, f), - os.path.normpath(os.path.join(arcname, reldir, f)), - onerror, - postprocess, - dereference, - ) - else: - if not os.path.exists(path): - raise OSError("Specified path doesn't exist: {}".format(path)) - _writepath(tarfile, path, arcname, onerror, postprocess, dereference) - - if temp_tar_path: - os.rename(temp_tar_path, output) - except Exception: - if temp_tar_path and os.path.exists(temp_tar_path): - os.remove(temp_tar_path) - raise - - -def _writepath(tarfile, src, dst, onerror, postprocess, dereference): - def tar_writepath(src, dst): - st = os.lstat(src) - if stat.S_ISREG(st.st_mode) or stat.S_ISDIR(st.st_mode) or stat.S_ISLNK(st.st_mode): - if dereference and stat.S_ISLNK(st.st_mode): - src = os.path.realpath(src) - - tarfile.writepath(src, dst) - - if postprocess: - postprocess(src, dst, st.st_mode) - else: - logger.debug("Skipping non-regular file '%s' (stat: %s)", src, st) - - try: - return tar_writepath(src, dst) - except Exception as e: - if isinstance(e, OSError) and e.errno == errno.ENOENT: - logger.debug( - "Skipping missing file '%s' - looks like directory content has changed during archiving", - src, - ) - return - - if onerror: - if onerror(src, dst, sys.exc_info()): - return tar_writepath(src, dst) - else: - raise - - -def check_tar(tar_file_path): - if os.path.isfile(tar_file_path) or os.path.islink(tar_file_path): - return libarchive.is_archive(tar_file_path) - return False - - -def _make_dirs(path): - try: - os.makedirs(path) - except OSError as e: - if e.errno != errno.EEXIST or not os.path.isdir(path): - raise - - -def _hardlink(src, dst): - if hasattr(os, "link"): - os.link(src, dst) - else: - shutil.copyfile(src, dst) - - -def _symlink(src, dst): - if hasattr(os, "symlink"): - os.symlink(src, dst) - else: - # Windows specific case - we cannot copy file right now, - # because it doesn't exist yet (and would be met later in the archive) or symlink is broken. - # Act like tar and tarfile - skip such symlinks - if os.path.exists(src): - shutil.copytree(src, dst) - - -def get_archive_filter_name(filename): - filters = libarchive.get_archive_filter_names(filename) - # https://a.yandex-team.ru/arc/trunk/arcadia/contrib/libs/libarchive/libarchive/archive_read.c?rev=5800047#L522 - assert filters[-1] == "none", filters - if len(filters) == 1: - return None - if len(filters) == 2: - return filters[0] - raise Exception("Archive has chain of filter: {}".format(filters)) diff --git a/library/python/archive/ya.make b/library/python/archive/ya.make deleted file mode 100644 index 5b86a45a42..0000000000 --- a/library/python/archive/ya.make +++ /dev/null @@ -1,19 +0,0 @@ -PY23_LIBRARY() - -STYLE_PYTHON() - -PY_SRCS( - __init__.py -) - -PEERDIR( - contrib/python/pathlib2 - contrib/python/python-libarchive -) - -END() - -RECURSE_FOR_TESTS( - benchmark - test -) diff --git a/library/python/cityhash/cityhash.pyx b/library/python/cityhash/cityhash.pyx deleted file mode 100644 index 6f0046f0d7..0000000000 --- a/library/python/cityhash/cityhash.pyx +++ /dev/null @@ -1,75 +0,0 @@ -from libcpp.pair cimport pair - -cdef extern from "util/system/types.h": - ctypedef unsigned long ui64 - - -cdef extern from "util/digest/city.h": - ui64 CityHash64(const char* buf, size_t len) nogil - pair[ui64, ui64] CityHash128(const char* buf, size_t len) nogil - ui64 CityHash64WithSeed(const char* buf, size_t len, ui64 seed) nogil - - -cdef extern from "library/python/cityhash/hash.h": - ui64 FileCityHash128WithSeedHigh64(const char* fpath) nogil except+ - ui64 FileCityHash64(const char* fpath) nogil except+ - - -def hash64(content): - cdef const char* s = content - cdef size_t size = len(content) - cdef ui64 res = 0 - - if size > 128: - with nogil: - res = CityHash64(s, size) - else: - res = CityHash64(s, size) - - return res - -def hash128(content): - cdef const char* s = content - cdef size_t size = len(content) - cdef pair[ui64, ui64] res = pair[ui64, ui64](0, 0) - - if size > 128: - with nogil: - res = CityHash128(s, size) - else: - res = CityHash128(s, size) - return res - - -def hash64seed(content, seed): - cdef const char* s = content - cdef size_t size = len(content) - cdef ui64 _seed = seed; - - if size > 128: - with nogil: - res = CityHash64WithSeed(s, size, _seed) - else: - res = CityHash64WithSeed(s, size, _seed) - - return res - - -def filehash64(path): - cdef const char* p = path - cdef ui64 res = 0 - - with nogil: - res = FileCityHash64(p) - - return res - - -def filehash128high64(path): - cdef const char* p = path - cdef ui64 res = 0 - - with nogil: - res = FileCityHash128WithSeedHigh64(p) - - return res diff --git a/library/python/cityhash/hash.cpp b/library/python/cityhash/hash.cpp deleted file mode 100644 index 17bd3a75f3..0000000000 --- a/library/python/cityhash/hash.cpp +++ /dev/null @@ -1,32 +0,0 @@ -#include "hash.h" - -#include <util/digest/city.h> -#include <util/generic/string.h> -#include <util/memory/blob.h> -#include <util/system/file.h> -#include <util/system/fstat.h> - -void ReadFile(const char* fpath, TBlob& blob) { - TFile f(TString{fpath}, RdOnly | Seq); - const TFileStat fs(f); - auto size = fs.Size; - - if (size < (64 << 10)) { - blob = TBlob::FromFileContent(f, 0, size); - } else { - blob = TBlob::FromFile(f); - } -} - -ui64 FileCityHash128WithSeedHigh64(const char* fpath) { - TBlob blob; - ReadFile(fpath, blob); - const uint128 hash = CityHash128WithSeed((const char*)blob.Data(), blob.Size(), uint128(0, blob.Size())); - return Uint128High64(hash); -} - -ui64 FileCityHash64(const char* fpath) { - TBlob blob; - ReadFile(fpath, blob); - return CityHash64(static_cast<const char*>(blob.Data()), blob.Size()); -} diff --git a/library/python/cityhash/hash.h b/library/python/cityhash/hash.h deleted file mode 100644 index 64b22ba74b..0000000000 --- a/library/python/cityhash/hash.h +++ /dev/null @@ -1,6 +0,0 @@ -#pragma once - -#include <util/system/defaults.h> - -ui64 FileCityHash128WithSeedHigh64(const char* fpath); -ui64 FileCityHash64(const char* fpath); diff --git a/library/python/cityhash/ya.make b/library/python/cityhash/ya.make deleted file mode 100644 index 7948e19389..0000000000 --- a/library/python/cityhash/ya.make +++ /dev/null @@ -1,16 +0,0 @@ -PY23_LIBRARY() - -SRCS( - hash.cpp -) - -PY_SRCS( - TOP_LEVEL - cityhash.pyx -) - -END() - -RECURSE_FOR_TESTS( - test -) diff --git a/library/python/codecs/__codecs.pyx b/library/python/codecs/__codecs.pyx deleted file mode 100644 index 42ec37fe88..0000000000 --- a/library/python/codecs/__codecs.pyx +++ /dev/null @@ -1,61 +0,0 @@ -import six - -from libcpp cimport bool - -from util.generic.string cimport TString, TStringBuf - - -def to_bytes(s): - try: - return s.encode('utf-8') - except AttributeError: - pass - - return s - - -def from_bytes(s): - if six.PY3: - return s.decode('utf-8') - - return s - - -cdef extern from "library/cpp/blockcodecs/codecs.h" namespace "NBlockCodecs": - cdef cppclass ICodec: - void Encode(TStringBuf data, TString& res) nogil - void Decode(TStringBuf data, TString& res) nogil - - cdef const ICodec* Codec(const TStringBuf& name) except + - cdef TString ListAllCodecsAsString() except + - - -def dumps(name, data): - name = to_bytes(name) - - cdef const ICodec* codec = Codec(TStringBuf(name, len(name))) - cdef TString res - cdef TStringBuf cdata = TStringBuf(data, len(data)) - - with nogil: - codec.Encode(cdata, res) - - return res.c_str()[:res.length()] - - -def loads(name, data): - name = to_bytes(name) - - cdef const ICodec* codec = Codec(TStringBuf(name, len(name))) - cdef TString res - cdef TStringBuf cdata = TStringBuf(data, len(data)) - - with nogil: - codec.Decode(cdata, res) - - return res.c_str()[:res.length()] - -def list_all_codecs(): - cdef TString res = ListAllCodecsAsString() - - return from_bytes(res.c_str()[:res.length()]).split(',') diff --git a/library/python/codecs/__init__.py b/library/python/codecs/__init__.py deleted file mode 100644 index b9fb00deb0..0000000000 --- a/library/python/codecs/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from __codecs import loads, dumps, list_all_codecs # noqa diff --git a/library/python/codecs/ya.make b/library/python/codecs/ya.make deleted file mode 100644 index f42d115d5d..0000000000 --- a/library/python/codecs/ya.make +++ /dev/null @@ -1,16 +0,0 @@ -PY23_LIBRARY() - -PEERDIR( - library/cpp/blockcodecs - contrib/python/six -) - -PY_SRCS( - __init__.py -) - -BUILDWITH_CYTHON_CPP(__codecs.pyx) - -PY_REGISTER(__codecs) - -END() diff --git a/library/python/color/README.md b/library/python/color/README.md deleted file mode 100644 index 9deae40092..0000000000 --- a/library/python/color/README.md +++ /dev/null @@ -1,9 +0,0 @@ -Форк ((termcolor https://github.com/termcolor/termcolor/)) для PY23 с дополнительным функционалом. - -Может быть использован для конвертации текстовых спецификаций цвета (например, из markup) в esc-последовательности для корректного отображения в терминале. - -Пример использования: -```python -from library.python.color import tcolor -tcolor("some text", "green-bold-on_red") -> '\x1b[32m\x1b[41m\x1b[1msome text\x1b[0m' -``` diff --git a/library/python/color/__init__.py b/library/python/color/__init__.py deleted file mode 100644 index a70234945e..0000000000 --- a/library/python/color/__init__.py +++ /dev/null @@ -1,92 +0,0 @@ -from __future__ import print_function - -import copy -import os - -from termcolor import ATTRIBUTES, COLORS, HIGHLIGHTS, RESET - -__all__ = [ - "ATTRIBUTES", - "COLORS", - "HIGHLIGHTS", - "RESET", - "colored", - "cprint", - "tcolor", - "get_color_by_spec" -] - -ATTRIBUTES = copy.deepcopy(ATTRIBUTES) -ATTRIBUTES["light"] = ATTRIBUTES['bold'] - -COLORS = copy.deepcopy(COLORS) -COLORS['gray'] = COLORS['grey'] -COLORS['purple'] = COLORS['magenta'] -COLORS["reset"] = 0 - - -def get_code(code): - if os.getenv("ANSI_COLORS_DISABLED") is None: - return "\033[{}m".format(code) - return "" - - -def get_color_by_spec(color_spec): - color, on_color, attrs = get_spec(color_spec) - return get_color(color, on_color, attrs) - - -def get_color(color, on_color, attrs): - res = "" - - if color is not None: - res += get_code(COLORS[color]) - - if on_color is not None: - res += get_code(HIGHLIGHTS[on_color]) - - if attrs is not None: - for attr in attrs: - res += get_code(ATTRIBUTES[attr]) - - return res - - -def get_spec(color_spec): - """Parses string text color formatting specification. - - Arguments: - color_spec -- string spec for text color formatting, csv string with - `color` / `bg_color` / `attr` spec items having "-" as a delimiter. - - Returns a tuple: (color, bg-color, attributes list) - - Example: - get_spec("green-bold-on_red") -> (32, 41, [1]) - """ - parts = color_spec.split("-") - color = None - on_color = None - attrs = [] - for part in parts: - part = part.lower() - if part in COLORS: - color = part - if part in HIGHLIGHTS: - on_color = part - if part in ATTRIBUTES: - attrs.append(part) - return color, on_color, attrs - - -def tcolor(text, color_spec): - color, on_color, attrs = get_spec(color_spec) - return colored(text, color=color, on_color=on_color, attrs=attrs) - - -def colored(text, color=None, on_color=None, attrs=None): - return get_color(color, on_color, attrs) + text + get_code(COLORS["reset"]) - - -def cprint(text, color=None, on_color=None, attrs=None, **kwargs): - print((colored(text, color, on_color, attrs)), **kwargs) diff --git a/library/python/color/ya.make b/library/python/color/ya.make deleted file mode 100644 index ff6740b1d4..0000000000 --- a/library/python/color/ya.make +++ /dev/null @@ -1,13 +0,0 @@ -PY23_LIBRARY() - -LICENSE(MIT) - -PY_SRCS( - __init__.py -) - -PEERDIR( - contrib/python/termcolor -) - -END() diff --git a/library/python/compress/__init__.py b/library/python/compress/__init__.py deleted file mode 100644 index 380ec47dca..0000000000 --- a/library/python/compress/__init__.py +++ /dev/null @@ -1,147 +0,0 @@ -from io import open - -import struct -import json -import os -import logging - -import library.python.par_apply as lpp -import library.python.codecs as lpc - - -logger = logging.getLogger('compress') - - -def list_all_codecs(): - return sorted(frozenset(lpc.list_all_codecs())) - - -def find_codec(ext): - def ext_compress(x): - return lpc.dumps(ext, x) - - def ext_decompress(x): - return lpc.loads(ext, x) - - ext_decompress(ext_compress(b'')) - - return {'c': ext_compress, 'd': ext_decompress, 'n': ext} - - -def codec_for(path): - for ext in reversed(path.split('.')): - try: - return find_codec(ext) - except Exception as e: - logger.debug('in codec_for(): %s', e) - - raise Exception('unsupported file %s' % path) - - -def compress(fr, to, codec=None, fopen=open, threads=1): - if codec: - codec = find_codec(codec) - else: - codec = codec_for(to) - - func = codec['c'] - - def iter_blocks(): - with fopen(fr, 'rb') as f: - while True: - chunk = f.read(16 * 1024 * 1024) - - if chunk: - yield chunk - else: - yield b'' - - return - - def iter_results(): - info = { - 'codec': codec['n'], - } - - if fr: - info['size'] = os.path.getsize(fr) - - yield json.dumps(info, sort_keys=True) + '\n' - - for c in lpp.par_apply(iter_blocks(), func, threads): - yield c - - with fopen(to, 'wb') as f: - for c in iter_results(): - logger.debug('complete %s', len(c)) - f.write(struct.pack('<I', len(c))) - - try: - f.write(c) - except TypeError: - f.write(c.encode('utf-8')) - - -def decompress(fr, to, codec=None, fopen=open, threads=1): - def iter_chunks(): - with fopen(fr, 'rb') as f: - cnt = 0 - - while True: - ll = f.read(4) - - if ll: - ll = struct.unpack('<I', ll)[0] - - if ll: - if ll > 100000000: - raise Exception('broken stream') - - yield f.read(ll) - - cnt += ll - else: - if not cnt: - raise Exception('empty stream') - - return - - it = iter_chunks() - extra = [] - - for chunk in it: - hdr = {} - - try: - hdr = json.loads(chunk) - except Exception as e: - logger.info('can not parse header, suspect old format: %s', e) - extra.append(chunk) - - break - - def resolve_codec(): - if 'codec' in hdr: - return find_codec(hdr['codec']) - - if codec: - return find_codec(codec) - - return codec_for(fr) - - dc = resolve_codec()['d'] - - def iter_all_chunks(): - for x in extra: - yield x - - for x in it: - yield x - - with fopen(to, 'wb') as f: - for c in lpp.par_apply(iter_all_chunks(), dc, threads): - if c: - logger.debug('complete %s', len(c)) - f.write(c) - else: - break diff --git a/library/python/compress/ya.make b/library/python/compress/ya.make deleted file mode 100644 index bbf2a784e2..0000000000 --- a/library/python/compress/ya.make +++ /dev/null @@ -1,16 +0,0 @@ -PY23_LIBRARY() - -PEERDIR( - library/python/codecs - library/python/par_apply -) - -PY_SRCS( - __init__.py -) - -END() - -RECURSE_FOR_TESTS( - tests -) diff --git a/library/python/coredump_filter/README.md b/library/python/coredump_filter/README.md deleted file mode 100644 index 87b02e7985..0000000000 --- a/library/python/coredump_filter/README.md +++ /dev/null @@ -1,19 +0,0 @@ -# Coredump Filter - -- ABC: https://abc.yandex-team.ru/services/cores/ - -Библиотека для разбора (парсинга) трейсов отладчика gdb/lldb, python traceback-ов и minidump-ов. - -На вход принимает текст трейса, на выходе - распаршенный текст + возможность преобразовать -его в html-формат для удобства чтения. - - -## Основные клиенты -- [Агрегатор стектрейсов](https://wiki.yandex-team.ru/cores-aggregation) - - -## Правила разработки - -Библиотека написана таким образом, чтобы файл `__init__.py` мог работать -без внешних зависимостей. Это позволяет использовать библиотеку даже в Ter1-окружениях. -На данный момент этот инвариант не обложен тестами (и это следует исправить). diff --git a/library/python/coredump_filter/__init__.py b/library/python/coredump_filter/__init__.py deleted file mode 100644 index de0830cd43..0000000000 --- a/library/python/coredump_filter/__init__.py +++ /dev/null @@ -1,1500 +0,0 @@ -#!/usr/bin/env python -# coding: utf-8 - -from __future__ import print_function - -import six -import enum -import datetime -import os -import re -import pkgutil -import sys -import hashlib -import json -import logging - -logger = logging.getLogger(__name__) - - -class CoredumpMode(enum.Enum): - GDB = "gdb" - LLDB = "lldb" - SDC_ASSERT = "sdc_assert" - - -ARCADIA_ROOT_LINK = "https://a.yandex-team.ru/arc/trunk/arcadia/" - -ARCADIA_ROOT_DIRS = [ - # hottest paths - "/util/", - "/contrib/", - "/library/", - "/kernel/", - "/build/", - "/search/", - - # "/gcc-4.8.2/", - - # system paths - # "/lib/x86_64-linux-gnu/", - - # all other stuff - "/aapi/", - "/addappter/", - "/adfox/", - "/admins/", - "/ads/", - "/adv/", - "/advq/", - "/afisha/", - "/afro/", - "/alet/", - "/alice/", - "/analytics/", - "/antiadblock/", - "/antirobot/", - "/apphost/", - "/april/", - "/arc/", - "/arcanum/", - "/augur/", - "/aurora/", - "/autocheck/", - "/balancer/", - "/bass/", - "/billing/", - "/bindings/", - "/browser/", - "/build/", - "/bunker/", - "/caas/", - "/canvas/", - "/captcha/", - "/catboost/", - "/certs/", - "/ci/", - "/clickhouse/", - "/client_analytics/", - "/cloud/", - "/cmicot/", - "/cmnt/", - "/comdep_analytics/", - "/commerce/", - "/contrib/", - "/crm/", - "/crowdsourcing/", - "/crypta/", - "/cv/", - "/datacloud/", - "/datalens/", - "/data-ui/", - "/devtools/", - "/dict/", - "/direct/", - "/disk/", - "/distribution/", - "/distribution_interface/", - "/district/", - "/dj/", - "/docs/", - "/douber/", - "/drive/", - "/edadeal/", - "/education/", - "/entity/", - "/ether/", - "/extdata/", - "/extsearch/", - "/FactExtract/", - "/fintech/", - "/frontend/", - "/fuzzing/", - "/games/", - "/gencfg/", - "/geobase/", - "/geoproduct/", - "/geosuggest/", - "/geotargeting/", - "/glycine/", - "/groups/", - "/haas/", - "/health/", - "/helpdesk/", - "/hitman/", - "/home/", - "/htf/", - "/hw_watcher/", - "/hypercube/", - "/iaas/", - "/iceberg/", - "/infra/", - "/intranet/", - "/inventori/", - "/ipreg/", - "/irt/", - "/it-office/", - "/jdk/", - "/juggler/", - "/junk/", - "/jupytercloud/", - "/kernel/", - "/keyboard/", - "/kikimr/", - "/kinopoisk/", - "/kinopoisk-ott/", - "/laas/", - "/lbs/", - "/library/", - "/load/", - "/locdoc/", - "/logbroker/", - "/logfeller/", - "/mail/", - "/mapreduce/", - "/maps/", - "/maps_adv/", - "/market/", - "/mb/", - "/mds/", - "/media/", - "/media-billing/", - "/media-crm/", - "/mediapers/", - "/mediaplanner/", - "/mediastat/", - "/media-stories/", - "/metrika/", - "/milab/", - "/ml/", - "/mlp/", - "/mlportal/", - "/mobile/", - "/modadvert/", - "/ms/", - "/mssngr/", - "/music/", - "/musickit/", - "/netsys/", - "/nginx/", - "/nirvana/", - "/noc/", - "/ofd/", - "/offline_data/", - "/opensource/", - "/orgvisits/", - "/ott/", - "/packages/", - "/partner/", - "/passport/", - "/payplatform/", - "/paysys/", - "/plus/", - "/portal/", - "/portalytics/", - "/pythia/", - "/quality/", - "/quasar/", - "/razladki/", - "/regulargeo/", - "/release_machine/", - "/rem/", - "/repo/", - "/rnd_toolbox/", - "/robot/", - "/rtc/", - "/rtline/", - "/rtmapreduce/", - "/rt-research/", - "/saas/", - "/samogon/", - "/samsara/", - "/sandbox/", - "/scarab/", - "/sdc/", - "/search/", - "/security/", - "/semantic-web/", - "/serp/", - "/sitesearch/", - "/skynet/", - "/smart_devices/", - "/smarttv/", - "/smm/", - "/solomon/", - "/specsearches/", - "/speechkit/", - "/sport/", - "/sprav/", - "/statbox/", - "/strm/", - "/suburban-trains/", - "/sup/", - "/switch/", - "/talents/", - "/tasklet/", - "/taxi/", - "/taxi_efficiency/", - "/testenv/", - "/testpalm/", - "/testpers/", - "/toloka/", - "/toolbox/", - "/tools/", - "/tracker/", - "/traffic/", - "/transfer_manager/", - "/travel/", - "/trust/", - "/urfu/", - "/vcs/", - "/velocity/", - "/vendor/", - "/vh/", - "/voicetech/", - "/weather/", - "/web/", - "/wmconsole/", - "/xmlsearch/", - "/yabs/", - "/yadoc/", - "/yandex_io/", - "/yaphone/", - "/ydf/", - "/ydo/", - "/yp/", - "/yql/", - "/ysite/", - "/yt/", - "/yweb/", - "/zen/", - "/zapravki/", - "/zen/", - "/zootopia/", - "/zora/", -] - -MY_PATH = os.path.dirname(os.path.abspath(__file__)) - -# 0.2.x uses stable hashing -CORE_PROC_VERSION = "0.2.1" - -ARCADIA_ROOT_SIGN = "$S/" -SIGNAL_NOT_FOUND = "signal not found" - - -class SourceRoot(object): - def __init__(self): - self.root = None - - def detect(self, source): - if not source: - # For example, regexp_4 - return - - if source.startswith("/-S/"): - return source[4:] - - if source.startswith("../"): - return source - - """ - if self.root is not None: - return self.root - """ - - min_pos = 100000 - for root_dir in ARCADIA_ROOT_DIRS: - pos = source.find(root_dir) - if pos < 0: - continue - - if pos < min_pos: - min_pos = pos - - if min_pos < len(source): - self.root = source[:min_pos + 1] - - def crop(self, source): - if not source: - return "" - - # detection attempt - self.detect(source) - - if self.root is not None: - return source.replace(self.root, ARCADIA_ROOT_SIGN, 1) - - # when traceback contains only ??, source root cannot be detected - return source - - -def highlight_func(s): - return ( - s - .replace("=", '<span class="symbol">=</span>') - .replace("(", '<span class="symbol">(</span>') - .replace(")", '<span class="symbol">)</span>') - ) - - -class FrameBase(object): - def __init__( - self, - frame_no=None, - addr="", - func="", - source="", - source_no="", - func_name="", - ): - self.frame_no = frame_no - self.addr = addr - self.func = func - self.source = source - self.source_no = source_no - self.func_name = func_name - - def __str__(self): - return "{}\t{}\t{}".format( - self.frame_no, - self.func, - self.source, - ) - - def to_json(self): - return { - "frame_no": self.frame_no, - "addr": self.addr, - "func": self.func, - "func_name": self.func_name, - "source": self.source, - "source_no": self.source_no, - } - - def fingerprint(self): - return self.func_name - - def cropped_source(self): - return self.source - - def raw(self): - return "{frame} {func} {source}".format( - frame=self.frame_no, - func=self.func, - source=self.source, - ) - - def html(self): - source, source_fmt = self.find_source() - return ( - '<span class="frame">{frame}</span>' - '<span class="func">{func}</span> ' - '<span class="source">{source}</span>{source_fmt}\n'.format( - frame=self.frame_no, - func=highlight_func(self.func.replace("&", "&").replace("<", "<")), - source=source, - source_fmt=source_fmt, - ) - ) - - -class LLDBFrame(FrameBase): - SOURCE_NO_RE = re.compile(r"(.*?[^\d]):(\d+)") - FUNC_RE = re.compile(r"(\w+\s)?(\w+[\w,:,_,<,>,\s,*]+).*$") - - def __init__( - self, - frame_no=None, - addr="", - func="", - source="", - source_no="", - func_name="", - ): - super(LLDBFrame, self).__init__( - frame_no=frame_no, - addr=addr, - func=func, - source=source, - source_no=source_no, - func_name=func_name, - ) - # .source calculation - - func = func.replace("(anonymous namespace)::", "") - m = self.FUNC_RE.match(func) - if m: - self.func_name = m.group(2) # overwrite func_name if name is in func - - if source_no: - self.source_no = source_no - self.source = source - else: - m = self.SOURCE_NO_RE.match(source) - if m: - self.source = m.group(1) - self.source_no = m.group(2) - - def find_source(self): - """ - :return: pair (source, source_fmt) - """ - source_fmt = "" - - if self.source_no: - source_fmt = ' +<span class="source-no">{}</span>'.format(self.source_no) - - return self.source, source_fmt - - -class GDBFrame(FrameBase): - SOURCE_NO_RE = re.compile(r"(.*):(\d+)") - # #7 0x00007f105f3a221d in NAppHost::NTransport::TCoroutineExecutor::Poll (this=0x7f08416a5d00, - # tasks=empty TVector (capacity=32)) at /-S/apphost/lib/executors/executors.cpp:373 - # We match with non-greedy regex a function name that cannot contain equal sign - FUNC_RE = re.compile(r"(.*?) \(([a-zA-Z0-9_]+=.*|)\)$") # function with kwarg-params or zero params - - def __init__( - self, - frame_no=None, - addr="", - func="", - source="", - source_no="", - func_name="", - ): - super(GDBFrame, self).__init__( - frame_no=frame_no, - addr=addr, - func=func, - source=source, - source_no=source_no, - func_name=func_name, - ) - if not source_no: - m = self.SOURCE_NO_RE.match(source) - if m: - self.source = m.group(1) - self.source_no = m.group(2) - if not func_name: - m = self.FUNC_RE.match(self.func) - if m: - self.func_name = m.group(1) - - def find_source(self): - """ - Returns link to arcadia if source is path in arcadia, else just string with path - :return: pair (source, source_fmt) - """ - source_fmt = "" - source = "" - link = "" - dirs = self.source.split("/") - if len(dirs) > 1 and "/{dir}/".format(dir=dirs[1]) in ARCADIA_ROOT_DIRS: - link = self.source.replace(ARCADIA_ROOT_SIGN, ARCADIA_ROOT_LINK) - else: - source = self.source - if self.source_no: - source_fmt = ' +<span class="source-no">{}</span>'.format(self.source_no) - if link: - link += "?#L{line}".format(line=self.source_no) - - if link: - source = '<a href="{link}">{source}</a>'.format( - link=link, - source=self.source, - ) - return source, source_fmt - - -class SDCAssertFrame(LLDBFrame): - - def __init__( - self, - frame_no=None, - addr="", - func="", - source="", - source_no="", - func_name="", - ): - super(SDCAssertFrame, self).__init__( - frame_no=frame_no, - addr=addr, - func=func, - source=source, - source_no=source_no, - func_name=func_name, - ) - # .source calculation - - self.source = source or "" - if isinstance(source_no, str) and len(source_no) > 0: - source_no = int(source_no, 16) - self.source_no = source_no or "" - - m = self.FUNC_RE.match(func) - if m: - self.func_name = m.group(2) - - -class Stack(object): - # priority classes - LOW_IMPORTANT = 25 - DEFAULT_IMPORTANT = 50 - SUSPICIOUS_IMPORTANT = 75 - MAX_IMPORTANT = 100 - - # default coredump's type - mode = CoredumpMode.GDB - - max_depth = None - - fingerprint_blacklist = [ - # bottom frames - "raise", - "abort", - "__gnu_cxx::__verbose_terminate_handler", - "_cxxabiv1::__terminate", - "std::terminate", - "__cxxabiv1::__cxa_throw", - # top frames - "start_thread", - "clone", - "??", - "__clone", - "__libc_start_main", - "_start", - "__nanosleep", - ] - - fingerprint_blacklist_prefix = () - - suspicious_functions = [ - "CheckedDelete", - "NPrivate::Panic", - "abort", - "close_all_fds", - "__cxa_throw", - ] - - low_important_functions_eq = [ - "poll ()", - "recvfrom ()", - "pthread_join ()", - ] - - low_important_functions_match = [ - "TCommonSockOps::SendV", - "WaitD (", - "SleepT (", - "Join (", - "epoll_wait", - "nanosleep", - "pthread_cond_wait", - "pthread_cond_timedwait", - "gsignal", - "std::detail::_", - "std::type_info", - "ros::NodeHandle", - ] - - def __init__( - self, - lines=None, - source_root=None, - thread_ptr=0, - thread_id=None, - frames=None, - important=None, - stack_fp=None, - fingerprint_hash=None, - stream=None, - mode=None, # type: CoredumpMode - ignore_bad_frames=True, - ): - self.lines = lines - self.source_root = source_root - self.thread_ptr = thread_ptr - self.thread_id = thread_id - if mode is not None: - self.mode = mode - - self.frames = frames or [] - if self.frames and isinstance(frames[0], dict): - self.frames = [self.frame_factory(f) for f in self.frames] - self.important = important or self.DEFAULT_IMPORTANT - if thread_id == 1: - self.important = self.MAX_IMPORTANT - self.fingerprint_hash = fingerprint_hash - self.stack_fp = stack_fp - self.stream = stream - self.ignore_bad_frames = ignore_bad_frames - - def to_json(self): - """Should be symmetric with `from_json`.""" - return { - "mode": self.mode.value, - "frames": [frame.to_json() for frame in self.frames], - "important": self.important, - } - - @staticmethod - def from_json(stack): - """Should be symmetric with `to_json`.""" - mode = CoredumpMode(stack.get("mode", CoredumpMode.GDB.value)) - # old serialization format support, should be dropped - lldb_mode = stack.get("lldb_mode", False) - if lldb_mode: - mode = CoredumpMode.LLDB - - unpacked_stack = { - "mode": mode, - "frames": stack["frames"], - "important": stack.get("important", Stack.DEFAULT_IMPORTANT), - } - return mode, unpacked_stack - - def frame_factory(self, args): - frames = { - CoredumpMode.GDB: GDBFrame, - CoredumpMode.LLDB: LLDBFrame, - CoredumpMode.SDC_ASSERT: SDCAssertFrame, - } - - class_object = frames.get(self.mode) - if not class_object: - raise Exception("Invalid mode: {}".format(self.mode.value)) - - return class_object(**args) - - def low_important(self): - return self.important <= self.LOW_IMPORTANT - - def check_importance(self, frame): - # raised priority cannot be lowered - if self.important > self.DEFAULT_IMPORTANT: - return - - # detect suspicious stacks - for name in self.suspicious_functions: - if name in frame.func: - self.important = self.SUSPICIOUS_IMPORTANT - return - - for name in self.low_important_functions_eq: - if name == frame.func: - self.important = self.LOW_IMPORTANT - - for name in self.low_important_functions_match: - if name in frame.func: - self.important = self.LOW_IMPORTANT - - def push_frame(self, frame): - self.check_importance(frame) - # ignore duplicated frames - if len(self.frames) and self.frames[-1].frame_no == frame.frame_no: - return - self.frames.append(frame) - - def parse(self): - """ - Parse one stack - """ - assert self.lines is not None - assert self.source_root is not None - - for line in self.lines: - match_found = False - for regexp in self.REGEXPS: - m = regexp.match(line) - if m: - frame_args = m.groupdict() - if "source" in frame_args: - frame_args["source"] = self.source_root.crop(frame_args["source"]) - - self.push_frame(self.frame_factory(frame_args)) - match_found = True - break - - if not match_found: - self.bad_frame(line) - - def bad_frame(self, line): - if self.ignore_bad_frames: - logger.warning("Bad frame: %s", line) - return - - raise Exception("Bad frame: `{}`, frame `{}`".format( - line, - self.debug(return_result=True), - )) - - def debug(self, return_result=False): - if self.low_important(): - return "" - - res = "\n".join([str(f) for f in self.frames]) - res += "----------------------------- DEBUG END\n" - if return_result: - return res - - self.stream.write(res) - - def raw(self): - return "\n".join([frame.raw() for frame in self.frames]) - - def html(self, same_hash=False, same_count=1, return_result=False): - ans = "" - pre_class = "important-" + str(self.important) - if same_hash: - pre_class += " same-hash" - - ans += '<pre class="{0}">'.format(pre_class) - if not same_hash: - ans += '<a name="stack{0}"></a>'.format(self.hash()) - - ans += '<span class="hash"><a href="#stack{0}">#{0}</a>, {1} stack(s) with same hash</span>\n'.format( - self.hash(), same_count, - ) - - for f in self.frames: - ans += f.html() - ans += "</pre>\n" - - if return_result: - return ans - - self.stream.write(ans) - - def fingerprint(self, max_num=None): - """ - Stack fingerprint: concatenation of non-common stack frames - FIXME: wipe away `max_num` - """ - stack_fp = list() - len_frames = min((max_num or len(self.frames)), len(self.frames)) - - for f in self.frames[:len_frames]: - fp = f.fingerprint() - if not fp: - continue - - if fp in self.fingerprint_blacklist: - continue - - if fp.startswith(self.fingerprint_blacklist_prefix): - continue - - if fp in stack_fp: - # FIXME: optimize duplicate remover: check only previous frame - # see also `push_frame` - continue - - stack_fp.append(fp.strip()) - - if self.max_depth is not None and len(stack_fp) >= self.max_depth: - break - - return "\n".join(stack_fp) - - def simple_html(self, num_frames=None): - if not num_frames: - num_frames = len(self.frames) - pre_class = "important-0" - ans = '<pre class="{0}">'.format(pre_class) - for i in range(min(len(self.frames), num_frames)): - ans += self.frames[i].html() - ans += "</pre>\n" - return ans - - def __str__(self): - return "\n".join(map(str, self.frames)) - - def hash(self, max_num=None): - """ - Entire stack hash for merging same stacks - """ - if self.fingerprint_hash is None: - self.fingerprint_hash = int(hashlib.md5(self.fingerprint(max_num).encode("utf-8")).hexdigest()[0:15], 16) - - return self.fingerprint_hash - - -class GDBStack(Stack): - - mode = CoredumpMode.GDB - - REGEXPS = [ - # #6 0x0000000001d9203e in NAsio::TIOService::TImpl::Run (this=0x137b1ec00) at /place/ - # sandbox-data/srcdir/arcadia_cache/library/neh/asio/io_service_impl.cpp:77 - - re.compile( - r"#(?P<frame_no>\d+)[ \t]+(?P<addr>0x[0-9a-f]+) in (?P<func>.*) at (?P<source>.*)" - ), - - # #5 TCondVar::WaitD (this=this@entry=0x10196b2b8, mutex=..., deadLine=..., deadLine@entry=...) - # at /place/sandbox-data/srcdir/arcadia_cache/util/system/condvar.cpp:150 - re.compile( - r"#(?P<frame_no>\d+)[ \t]+(?P<func>.*) at (?P<source>/.*)" - ), - - # #0 0x00007faf8eb31d84 in pthread_cond_wait@@GLIBC_2.3.2 () - # from /lib/x86_64-linux-gnu/libpthread.so.0 - re.compile( - r"#(?P<frame_no>\d+)[ \t]+(?P<addr>0x[0-9a-f]+) in (?P<func>.*) from (?P<source>.*)" - ), - - # #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 - re.compile( - r"#(?P<frame_no>\d+)[ \t]+ (?P<func>.*) at (?P<source>.*)" - ), - - # #10 0x0000000000000000 in ?? () - re.compile( - r"#(?P<frame_no>\d+)[ \t]+(?P<addr>0x[0-9a-f]+) in (?P<func>.*)" - ), - ] - - -class LLDBStack(Stack): - - mode = CoredumpMode.LLDB - - REGEXPS = [ - # 0x00007fd7b300a886 libthird_Uparty_Sros_Sros_Ucomm_Sclients_Sroscpp_Sliblibroscpp.so` - # std::thread::_State_impl<std::thread::_Invoker<std::tuple<ros::PollManager::PollManager()::$_1> > >::_M_run() - # [inlined] ros::PollManager::threadFunc(this=0x00007fd7b30dab20) at poll_manager.cpp:75:16 # noqa - re.compile( - r"[ *]*frame #(?P<frame_no>\d+): (?P<addr>0x[0-9a-f]+).+inlined]\s(?P<func>.+)\sat\s(?P<source>.+)" - ), - - re.compile( - r"[ *]*frame #(?P<frame_no>\d+): (?P<addr>0x[0-9a-f]+).+?`(?P<func>.+)\sat\s(?P<source>.+)" - ), - - # * frame #0: 0x00007fd7aee51f47 libc.so.6`gsignal + 199 - re.compile( - r"[ *]*frame #(?P<frame_no>\d+): (?P<addr>0x[0-9a-f]+)\s(?P<source>.+)`(?P<func>.+)\s\+\s(?P<source_no>\d+)" - ), - ] - - # Take not more than `max_depth` non-filtered frames into fingerprint - # See CORES-180 - max_depth = 10 - - fingerprint_blacklist = Stack.fingerprint_blacklist + [ - "ros::ros_wallsleep", - ] - - fingerprint_blacklist_prefix = Stack.fingerprint_blacklist_prefix + ( - "___lldb_unnamed_symbol", - "__gnu_cxx", - "__gthread", - "__pthread", - "decltype", - "myriapoda::BuildersRunner", - "non", - "std::_Function_handler", - "std::_Sp_counted_ptr_inplace", - "std::__invoke_impl", - "std::__invoke_result", - "std::__shared_ptr", - "std::conditional", - "std::shared_ptr", - "std::thread::_Invoker", - "std::thread::_State_impl", - "yandex::sdc::assert_details_", - ) - - suspicious_functions = Stack.suspicious_functions + [ - "Xml", - "boost", - "ros", - "supernode", - "tensorflow", - "yandex::sdc", - ] - - -class PythonStack(Stack): - - REGEXPS = [ - re.compile( - r'File "(?P<source>.*)", line (?P<source_no>\d+), in (?P<func_name>.*)' - ), - ] - - -class SDCAssertStack(LLDBStack): - - mode = CoredumpMode.SDC_ASSERT - - REGEXPS = [ - # 0: ./modules/_shared/libcore_Stools_Slibassert.so(yandex::sdc::assert_details_::PanicV(char const*, - # long, char const*, char const*, bool, char const*, __va_list_tag*) - # +0x2aa)[0x7fb83268feaa] - re.compile( - r"(?P<frame_no>\d+):\s(?P<source>.+.so)\((?P<func>.+)\+(?P<source_no>.+).+\[(?P<addr>0x[0-9a-f]+)" - ), - - re.compile( - r"(?P<frame_no>\d+):\s(?P<source>\w+)\((?P<func>.+)\+(?P<source_no>.+).+\[(?P<addr>0x[0-9a-f]+)" - ) - ] - - -def parse_python_traceback(trace): - trace = trace.replace("/home/zomb-sandbox/client/", "/") - trace = trace.replace("/home/zomb-sandbox/tasks/", "/sandbox/") - trace = trace.split("\n") - exception = trace[-1] # noqa: F841 - trace = trace[1: -1] - pairs = zip(trace[::2], trace[1::2]) - stack = Stack(lines=[]) - for frame_no, (path, row) in enumerate(pairs): - # FIXME: wrap into generic tracer - m = PythonStack.REGEXPS[0].match(path.strip()) - if m: - frame_args = m.groupdict() - if not frame_args["source"].startswith("/"): - frame_args["source"] = "/" + frame_args["source"] - frame_args["frame_no"] = str(frame_no) - frame_args["func"] = row.strip() - stack.push_frame(GDBFrame(**frame_args)) - return [[stack]], [[stack.raw()]], 6 - - -def stack_factory(stack): - mode, unpacked_stack = Stack.from_json(stack) - - if mode == CoredumpMode.GDB: - return GDBStack(**unpacked_stack) - elif mode == CoredumpMode.LLDB: - return LLDBStack(**unpacked_stack) - elif mode == CoredumpMode.SDC_ASSERT: - return SDCAssertStack(**unpacked_stack) - - raise Exception("Invalid stack mode: {}. ".format(mode)) - - -def _read_file(file_name): - with open(file_name) as f: - return f.read() - - -def _file_contents(file_name): - """Return file (or resource) contents as unicode string.""" - if getattr(sys, "is_standalone_binary", False): - try: - contents = pkgutil.get_data(__package__, file_name) - except Exception: - raise IOError("Failed to find resource: " + file_name) - else: - if not os.path.exists(file_name): - file_name = os.path.join(MY_PATH, file_name) - contents = _read_file(file_name) - # py23 compatibility - if not isinstance(contents, six.text_type): - contents = contents.decode("utf-8") - return contents - - -def html_prolog(stream, timestamp): - prolog = _file_contents("prolog.html") - assert isinstance(prolog, six.string_types) - stream.write(prolog.format( - style=_file_contents("styles.css"), - coredump_js=_file_contents("core_proc.js"), - version=CORE_PROC_VERSION, - timestamp=timestamp, - )) - - -def html_epilog(stream): - stream.write(_file_contents("epilog.html")) - - -def detect_coredump_mode(core_text): - if len(core_text) == 0: - raise Exception("Text stacktrace is blank") - - if "Panic at unixtime" in core_text: - return CoredumpMode.SDC_ASSERT - - if "(lldb)" in core_text: - return CoredumpMode.LLDB - - return CoredumpMode.GDB - - -def filter_stack_dump( - core_text=None, - stack_file_name=None, - use_fingerprint=False, - sandbox_failed_task_id=None, - output_stream=None, - timestamp=datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), - ignore_bad_frames=True, -): - """New interface for stacktrace filtering. Preferred to use.""" - if not core_text and not stack_file_name: - raise ValueError("Either `core_text` or `stack_file_name` should be passed to `filter_stack_dump`. ") - - if core_text is not None and stack_file_name: - raise ValueError("Only one of `core_text` and `stack_file_name` cannot be specified for `filter_stack_dump`. ") - - if stack_file_name: - core_text = _read_file(stack_file_name) - # further processing uses `core_text` only - - mode = detect_coredump_mode(core_text) - core_lines = core_text.split("\n") - - return filter_stackdump( - file_lines=core_lines, - ignore_bad_frames=ignore_bad_frames, - mode=mode, - sandbox_failed_task_id=sandbox_failed_task_id, - stream=output_stream, - timestamp=timestamp, - use_fingerprint=use_fingerprint, - use_stream=output_stream is not None, - ) - - -class StackDumperBase(object): - - SANDBOX_TASK_RE = re.compile(r".*/[0-9a-f]/[0-9a-f]/([0-9]+)/.*") - MAX_SAME_STACKS = 30 - - def __init__( - self, - use_fingerprint, - sandbox_failed_task_id, - stream, - use_stream, - file_lines, - timestamp, - mode, - file_name=None, - ignore_bad_frames=True, - ): - self.source_root = SourceRoot() - self.use_fingerprint = use_fingerprint - self.sandbox_task_id = None - self.sandbox_failed_task_id = sandbox_failed_task_id - self.stream = stream or sys.stdout - self.use_stream = use_stream - self.file_name = file_name - self.file_lines = file_lines - self.timestamp = timestamp - self.ignore_bad_frames = ignore_bad_frames - self.stack_class = self.get_stack_class(mode) - - self.signal = SIGNAL_NOT_FOUND - self.stacks = [] - self._main_info = [] - - @staticmethod - def is_ignored_line(line): - raise NotImplementedError("Not implemented static method `is_ignored_line`. ") - - @staticmethod - def get_stack_class(mode): - exist_modes = {} - for cls in [GDBStack, LLDBStack, SDCAssertStack]: - current_mode = cls.mode - if current_mode in exist_modes: - raise Exception("Duplicate modes are disallowed. Repeated mode: `{}`".format(current_mode.value)) - exist_modes[current_mode] = cls - - if mode not in exist_modes: - raise Exception("Unexpected coredump processing mode: `{}`".format(mode.value)) - - return exist_modes[mode] - - def check_signal(self, line): - raise NotImplementedError("Not implemented `check_signal`.") - - def set_sandbox_task_id(self, task_id): - self.sandbox_task_id = task_id - - def add_main_line(self, line): - self._main_info.append(line) - - def add_stack(self, stack_lines, thread_id): - if not stack_lines: - return - - stack = self.stack_class( - lines=stack_lines, - source_root=self.source_root, - thread_id=thread_id, - stream=self.stream, - ignore_bad_frames=self.ignore_bad_frames, - ) - self.stacks.append(stack) - - def dump(self): - if self.file_lines is None: - # FIXME(mvel): LLDB is not handled here - self.file_lines = get_parsable_gdb_text(_read_file(self.file_name)) - - self._collect_stacks() - - for stack in self.stacks: - stack.parse() - # stack.debug() - - if self.use_stream: - if self.use_fingerprint: - for stack in self.stacks: - self.stream.write(stack.fingerprint() + "\n") - self.stream.write("--------------------------------------\n") - return - else: - html_prolog(self.stream, self.timestamp) - - if self.sandbox_task_id is not None: - self.stream.write( - '<div style="padding-top: 6px; font-size: 18px; font-weight: bold;">' - 'Coredumped binary build task: ' - '<a href="https://sandbox.yandex-team.ru/task/{0}">{0}</a></div>\n'.format( - self.sandbox_task_id - ) - ) - - if self.sandbox_failed_task_id is not None: - self.stream.write( - '<div style="padding-top: 6px; font-size: 18px; font-weight: bold;">' - 'Sandbox failed task: ' - '<a href="https://sandbox.yandex-team.ru/task/{0}">{0}</a></div>\n'.format( - self.sandbox_failed_task_id - ) - ) - - pre_class = "" - self.stream.write('<pre class="{0}">\n'.format(pre_class)) - for line in self._main_info: - self.stream.write(line.replace("&", "&").replace("<", "<") + "\n") - self.stream.write("</pre>\n") - - sorted_stacks = sorted(self.stacks, key=lambda x: (x.important, x.fingerprint()), reverse=True) - - prev_hash = None - all_hash_stacks = [] - cur_hash_stacks = [] - for stack in sorted_stacks: - if stack.hash() == 0: - continue - - if stack.hash() == prev_hash: - if len(cur_hash_stacks) < self.MAX_SAME_STACKS: - # do not collect too much - cur_hash_stacks.append(stack) - continue - - # hash changed - if cur_hash_stacks: - all_hash_stacks.append(cur_hash_stacks) - - prev_hash = stack.hash() - cur_hash_stacks = [stack, ] - - # push last - if cur_hash_stacks: - all_hash_stacks.append(cur_hash_stacks) - - if self.use_stream: - for cur_hash_stacks in all_hash_stacks: - same_hash = False - for stack in cur_hash_stacks: - stack.html(same_hash=same_hash, same_count=len(cur_hash_stacks)) - same_hash = True - - html_epilog(self.stream) - else: - raw_hash_stacks = [ - [stack.raw() for stack in common_hash_stacks] - for common_hash_stacks in all_hash_stacks - ] - return all_hash_stacks, raw_hash_stacks, self.signal - - def _collect_stacks(self): - stack_lines = [] - stack_detected = False - thread_id = None - - for line in self.file_lines: - line = line.strip() - if self.is_ignored_line(line): - continue - - if "Core was generated" in line: - m = self.SANDBOX_TASK_RE.match(line) - if m: - self.set_sandbox_task_id(int(m.group(1))) - - self.check_signal(line) - - # [Switching to thread 55 (Thread 0x7f100a94c700 (LWP 21034))] - # Thread 584 (Thread 0x7ff363c03700 (LWP 2124)): - - # see test2 and test3 - tm = self.THREAD_RE.match(line) - if tm: - stack_detected = True - self.add_stack( - stack_lines=stack_lines, - thread_id=thread_id, - ) - stack_lines = [] - thread_id = int(tm.group(1)) - continue - - if stack_detected: - stack_lines.append(line) - else: - self.add_main_line(line) - - # parse last stack - self.add_stack( - stack_lines=stack_lines, - thread_id=thread_id, - ) - - -class StackDumperGDB(StackDumperBase): - - SIGNAL_FLAG = "Program terminated with signal" - THREAD_RE = re.compile(r".*[Tt]hread (\d+) .*") - LINE_IN = re.compile(r"\d+\tin ") - - def is_ignored_line(self, line): - if not line: - return True - - if line.startswith("[New "): - # LWP, Thread, process - return True - - if line.startswith("[Thread "): - return True - - if line.startswith("Using "): - return True - - if line.startswith("warning:"): - return True - - if line.startswith("Python Exception"): - # TODO: handle this more carefully - return True - - if line[0] != "#" and "No such file or directory" in line: - return True - - if self.LINE_IN.match(line): - # see test1.txt for example - # 641 in /place/sandbox-data/srcdir/arcadia/library/coroutine/engine/impl.h - return True - - return False - - def check_signal(self, line): - if self.SIGNAL_FLAG in line: - self.signal = line[line.find(self.SIGNAL_FLAG) + len(self.SIGNAL_FLAG):].split(",")[0] - - -class StackDumperLLDB(StackDumperBase): - - SIGNAL_FLAG = "stop reason = signal" - - THREAD_RE = re.compile(r".*thread #(\d+), .*") - - SKIP_LINES = { - "(lldb) bt all", - "(lldb) script import sys", - "(lldb) target create", - "Core file", - - # Drop signal interceptor call - # * frame #0: 0x00007efd49042fb7 libc.so.6`__GI___libc_sigaction at sigaction.c:54 - # TODO(epsilond1): Set MAX_IMPORTANT for some thread - "__GI___libc_sigaction", - - # Drop unnamed symbols at lines like - # frame #4: 0x00007fd8054156df libstdc++.so.6`___lldb_unnamed_symbol440$$libstdc++.so.6 + 15 - "$$", - } - - @staticmethod - def is_ignored_line(line): - if not line: - return True - - for skip_line in StackDumperLLDB.SKIP_LINES: - if skip_line in line: - return True - return False - - def check_signal(self, line): - if self.SIGNAL_FLAG in line and self.signal == SIGNAL_NOT_FOUND: - self.signal = line.split()[-1] - - -class StackDumperSDCAssert(StackDumperBase): - - THREAD_RE = re.compile( - r"(\d+)(:\s)" - ) - - def is_ignored_line(self, line): - if not line: - return True - - return not re.match(self.THREAD_RE, line) - - def check_signal(self, line): - self.signal = SIGNAL_NOT_FOUND - - def _collect_stacks(self): - stack_lines = [] - for line in self.file_lines: - line = line.strip() - if self.is_ignored_line(line): - continue - stack_lines.append(line) - self.check_signal(line) - - self.add_stack( - stack_lines=stack_lines, - thread_id=0, - ) - - -def filter_stackdump( - file_name=None, - use_fingerprint=False, - sandbox_failed_task_id=None, - stream=None, - file_lines=None, - use_stream=True, - timestamp=None, - ignore_bad_frames=True, - mode=None, -): - if mode is None and file_name is not None: - mode = detect_coredump_mode(_read_file(file_name)) - if mode == CoredumpMode.GDB: - stack_dumper_cls = StackDumperGDB - elif mode == CoredumpMode.LLDB: - stack_dumper_cls = StackDumperLLDB - elif mode == CoredumpMode.SDC_ASSERT: - stack_dumper_cls = StackDumperSDCAssert - else: - raise Exception("Invalid mode: {}".format(mode.value)) - - dumper = stack_dumper_cls( - file_name=file_name, - use_fingerprint=use_fingerprint, - sandbox_failed_task_id=sandbox_failed_task_id, - stream=stream, - use_stream=use_stream, - file_lines=file_lines, - timestamp=timestamp, - ignore_bad_frames=ignore_bad_frames, - mode=mode, - ) - - return dumper.dump() - - -def get_parsable_gdb_text(core_text): - # FIXME(mvel): Check encoding? - # core_text = core_text.encode("ascii", "ignore").decode("ascii") - core_text = ( - core_text - # .replace("#", "\n#") # bug here - .replace("No core", "\nNo core") - .replace("[New", "\n[New") - .replace("\n\n", "\n") - ) - - return core_text.split("\n") - - -if __name__ == "__main__": - if len(sys.argv) < 2: - sys.stderr.write( - """Traceback filter "Tri Korochki" -https://wiki.yandex-team.ru/cores-aggregation/ -Usage: - core_proc.py <traceback.txt> [-f|--fingerprint] - core_proc.py -v|--version -""" - ) - sys.exit(1) - - if sys.argv[1] == "--version" or sys.argv[1] == "-v": - if os.system("svn info 2>/dev/null | grep '^Revision'") != 0: - print(CORE_PROC_VERSION) - sys.exit(0) - - sandbox_failed_task_id = None - - use_fingerprint = False - if len(sys.argv) >= 3: - if sys.argv[2] == "-f" or sys.argv[2] == "--fingerprint": - use_fingerprint = True - sandbox_failed_task_id = sys.argv[2] - - filter_stack_dump( - core_text=_read_file(sys.argv[1]), - use_fingerprint=use_fingerprint, - sandbox_failed_task_id=sandbox_failed_task_id, - output_stream=sys.stdout, - ) - - -""" -Stack group is a `Stack` objects list with the same hash (fingerprint). -""" - - -class StackEncoder(json.JSONEncoder): - """Stack JSON serializer.""" - - def default(self, obj): - if isinstance(obj, Stack): - return obj.to_json() - - return json.JSONEncoder.default(obj) - - -def serialize_stacks(stack_groups): - """ - Serialize list of stack groups to string (using JSON format). - - :param stack_groups: list of stack groups. - :return: JSON serialized to string - """ - return json.dumps(stack_groups, cls=StackEncoder) - - -def deserialize_stacks(stack_groups_str): - """ - Restore JSON-serialized stack data into stack groups. - - :param stack_groups_str: JSON-serialized data. - :return: list of stack groups - """ - stack_groups_json = json.loads(stack_groups_str) - # please do not use `map` hell here, it's impossible to debug - all_stacks = [ - [stack_factory(stack) for stack in stacks] - for stacks in stack_groups_json - ] - return all_stacks diff --git a/library/python/coredump_filter/core_proc.js b/library/python/coredump_filter/core_proc.js deleted file mode 100644 index 15413adeae..0000000000 --- a/library/python/coredump_filter/core_proc.js +++ /dev/null @@ -1,21 +0,0 @@ - - -$(document).ready(function() { - $('#show-same-stacks').click(function() { - var stacks = $('.same-hash'); - for (var i = 0; i < stacks.length; ++i) - $(stacks[i]).show(); - $('#show-same-stacks').hide(); - $('#hide-same-stacks').show(); - return false; - }); - - $('#hide-same-stacks').click(function() { - var stacks = $('.same-hash'); - for (var i = 0; i < stacks.length; ++i) - $(stacks[i]).hide(); - $('#hide-same-stacks').hide(); - $('#show-same-stacks').show(); - return false; - }); -}); diff --git a/library/python/coredump_filter/epilog.html b/library/python/coredump_filter/epilog.html deleted file mode 100644 index b317cc2a91..0000000000 --- a/library/python/coredump_filter/epilog.html +++ /dev/null @@ -1,2 +0,0 @@ - </body> -</html>
\ No newline at end of file diff --git a/library/python/coredump_filter/prolog.html b/library/python/coredump_filter/prolog.html deleted file mode 100644 index f102a7210d..0000000000 --- a/library/python/coredump_filter/prolog.html +++ /dev/null @@ -1,24 +0,0 @@ -<!DOCTYPE html> -<html> - <head> - <style>{style}</style> - <script src="https://yastatic.net/jquery/1.7.1/jquery.min.js"></script> - <script>{coredump_js}</script> - </head> - <body> - <h1>Coredump report generated on {timestamp} by Coredump/traceback filter - <i><a href="http://mvel.at.yandex-team.ru/2373">Tri Korochki</a></i> - ({version}) - </h1> - <h3>Author: <a href="https://staff.yandex-team.ru/mvel">mvel@</a> aka Mikhail Veltishchev</h3> - <h3>© Yandex LLC. All rights reversed</h3> - <div class="legend"> - <ul style="line-height: 22px"> - <li><span class="important-100">Problem</span> stacks</li> - <li><span class="important-75">Suspicious</span> stacks</li> - <li><span class="important-50">Active</span> stacks</li> - <li><span class="important-25">Non-active</span> stacks</li> - </ul> - <a class="show-same-hash" id="show-same-stacks" href="#">Show same stacks</a> - <a class="show-same-hash" id="hide-same-stacks" href="#" style="display: none">Hide same stacks</a> - </div> diff --git a/library/python/coredump_filter/styles.css b/library/python/coredump_filter/styles.css deleted file mode 100644 index fdd09ce09e..0000000000 --- a/library/python/coredump_filter/styles.css +++ /dev/null @@ -1,116 +0,0 @@ -body { - font-family: sans-serif; - font-size: 12px; -} - -a { - text-decoration: none; - color: #486DEC; -} - -.frame { - color: #7f7f7f; - display: inline-block; - width: 30px; -} - -.addr { - color: #999999; - padding-right: 10px; -} - -.func { - color: #7f0000; - word-wrap: normal; -} - -.source { - color: #007f00; -} - -.symbol { - color: #0000ff; -} - -h1 { - font-size: 1.5em; - margin-top: .1em; - margin-bottom: .2em; -} - -h3 { - font-size: 1em; - margin-top: .1em; - margin-bottom: .2em; -} - -pre { - overflow-x: auto; - margin: 6px 0px 6px 0px; - padding: 0px 12px 6px 12px; - position: relative; -} - -pre.important-25 { - background-color: #eeeeee; -} - -span.important-25 { - background-color: #eeeeee; - padding: 3px; -} - -pre.important-50 { - background-color: #e7ffe7; -} - -span.important-50 { - background-color: #e7ffe7; - padding: 3px; -} - -pre.important-75 { - background-color: #ffffcc; -} - -span.important-75 { - background-color: #ffffcc; - padding: 3px; -} - -pre.important-100 { - background-color: #ffdddd; -} - -span.important-100 { - background-color: #ffdddd; - padding: 3px; -} - -a.show-same-hash { - padding: 0px 20px 0px 20px; -} - -/* hidden by default */ -.same-hash { - display: none; -} - -span.hash { - position: absolute; - top: 3px; right: 3px; -} - -div.legend { - position: absolute; - z-index: 1; - top: 5px; - right: 8px; - border: 1px solid #7f7f7f; - border-radius: 3px; - padding: 0px 20px 3px 0px; -} - -div.legend ul { - margin: 3px 0px 3px 0px; -} diff --git a/library/python/coredump_filter/ya.make b/library/python/coredump_filter/ya.make deleted file mode 100644 index fc8ec1a45f..0000000000 --- a/library/python/coredump_filter/ya.make +++ /dev/null @@ -1,23 +0,0 @@ -PY23_LIBRARY() - -PY_SRCS( - __init__.py -) - -RESOURCE_FILES( - PREFIX library/python/coredump_filter/ - core_proc.js - epilog.html - prolog.html - styles.css -) - -IF(PYTHON2) - PEERDIR(contrib/deprecated/python/enum34) -ENDIF() - -END() - -RECURSE( - tests -) diff --git a/library/python/json/__init__.py b/library/python/json/__init__.py deleted file mode 100644 index c6420d5e6d..0000000000 --- a/library/python/json/__init__.py +++ /dev/null @@ -1,44 +0,0 @@ -from library.python.json.loads import loads as _loads -from simplejson import loads as _sj_loads - - -def loads(*args, **kwargs): - try: - return _loads(*args, **kwargs) - except Exception as e: - if 'invalid syntax at token' in str(e): - kwargs.pop('intern_keys', None) - kwargs.pop('intern_vals', None) - kwargs.pop('may_unicode', None) - return _sj_loads(*args, **kwargs) - - raise - - -from simplejson import load, dump, dumps # noqa - - -def read_file(file_name, **kwargs): - """ - Read file and return its parsed json contents. - - All kwargs will be proxied to `json.load` method as is. - - :param file_name: file with json contents - :return: parsed json contents - """ - with open(file_name) as f: - return load(f, **kwargs) - - -def write_file(file_name, contents, **kwargs): - """ - Dump json data to file. - - All kwargs will be proxied to `json.dump` method as is. - - :param file_name: file to dump to - :param contents: JSON-serializable object - """ - with open(file_name, "w") as f: - dump(contents, f, **kwargs) diff --git a/library/python/json/loads.cpp b/library/python/json/loads.cpp deleted file mode 100644 index 19cdb096ae..0000000000 --- a/library/python/json/loads.cpp +++ /dev/null @@ -1,246 +0,0 @@ -#include "loads.h" - -#include <Python.h> - -#include <library/cpp/json/fast_sax/parser.h> - -#include <util/generic/algorithm.h> -#include <util/generic/stack.h> -#include <util/generic/vector.h> -#include <util/generic/ylimits.h> -#include <util/string/ascii.h> - -using namespace NJson; - -namespace { - enum EKind { - Undefined, - Array, - Dict, - Value, - Key, - }; - - static inline TStringBuf ToStr(EKind kind) noexcept { - switch (kind) { - case Undefined: - return TStringBuf("Undefined"); - - case Array: - return TStringBuf("Array"); - - case Dict: - return TStringBuf("Dict"); - - case Value: - return TStringBuf("Value"); - - case Key: - return TStringBuf("Key"); - } - - Y_UNREACHABLE(); - } - - struct TUnref { - static inline void Destroy(PyObject* o) noexcept { - Py_XDECREF(o); - } - }; - - using TObjectPtr = TAutoPtr<PyObject, TUnref>; - - static inline TObjectPtr BuildBool(bool val) noexcept { - if (val) { - Py_RETURN_TRUE; - } - - Py_RETURN_FALSE; - } - - // Translate python exceptions from object-creating functions into c++ exceptions - // Such errors are reported by returning nullptr - // When a python error is set and C++ exception is caught by Cython wrapper, - // Python exception is propagated, while C++ exception is discarded. - PyObject* CheckNewObject(PyObject* obj) { - Y_ENSURE(obj != nullptr, "got python exception"); - return obj; - } - - void CheckRetcode(int retcode) { - Y_ENSURE(retcode == 0, "got python exception"); - } - - static inline TObjectPtr BuildSmall(long val) { -#if PY_VERSION_HEX >= 0x03000000 - return CheckNewObject(PyLong_FromLong(val)); -#else - return CheckNewObject(PyInt_FromLong(val)); -#endif - } - - PyObject* CreatePyString(TStringBuf str, bool intern, bool mayUnicode) { -#if PY_VERSION_HEX >= 0x03000000 - Y_UNUSED(mayUnicode); - PyObject* pyStr = PyUnicode_FromStringAndSize(str.data(), str.size()); - if (intern) { - PyUnicode_InternInPlace(&pyStr); - } -#else - const bool needUnicode = mayUnicode && !AllOf(str, IsAscii); - PyObject* pyStr = needUnicode ? PyUnicode_FromStringAndSize(str.data(), str.size()) - : PyString_FromStringAndSize(str.data(), str.size()); - if (intern && !needUnicode) { - PyString_InternInPlace(&pyStr); - } -#endif - return pyStr; - } - - struct TVal { - EKind Kind = Undefined; - TObjectPtr Val; - - inline TVal() noexcept - : Kind(Undefined) - { - } - - inline TVal(EKind kind, TObjectPtr val) noexcept - : Kind(kind) - , Val(val) - { - } - }; - - static inline TObjectPtr NoneRef() noexcept { - Py_RETURN_NONE; - } - - struct TContext: public TJsonCallbacks { - const bool InternKeys; - const bool InternVals; - const bool MayUnicode; - TStack<TVal, TVector<TVal>> S; - - inline TContext(bool internKeys, bool internVals, bool mayUnicode) - : TJsonCallbacks(true) - , InternKeys(internKeys) - , InternVals(internVals) - , MayUnicode(mayUnicode) - { - S.emplace(); - } - - inline bool Consume(TObjectPtr o) { - auto& t = S.top(); - - if (t.Kind == Array) { - CheckRetcode(PyList_Append(t.Val.Get(), o.Get())); - } else if (t.Kind == Key) { - auto key = S.top().Val; - - S.pop(); - - CheckRetcode(PyDict_SetItem(S.top().Val.Get(), key.Get(), o.Get())); - } else { - t = TVal(Value, o); - } - - return true; - } - - inline TObjectPtr Pop(EKind expect) { - auto res = S.top(); - - S.pop(); - - if (res.Kind != expect) { - ythrow yexception() << "unexpected kind(expect " << ToStr(expect) << ", got " << ToStr(res.Kind) << ")"; - } - - return res.Val; - } - - inline void Push(EKind kind, TObjectPtr object) { - S.push(TVal(kind, object)); - } - - virtual bool OnNull() { - return Consume(NoneRef()); - } - - virtual bool OnBoolean(bool v) { - return Consume(BuildBool(v)); - } - - virtual bool OnInteger(long long v) { - if (v >= (long long)Min<long>()) { - return Consume(BuildSmall((long)v)); - } - - return Consume(CheckNewObject(PyLong_FromLongLong(v))); - } - - virtual bool OnUInteger(unsigned long long v) { - if (v <= (unsigned long long)Max<long>()) { - return Consume(BuildSmall((long)v)); - } - - return Consume(CheckNewObject(PyLong_FromUnsignedLongLong(v))); - } - - virtual bool OnDouble(double v) { - return Consume(CheckNewObject(PyFloat_FromDouble(v))); - } - - virtual bool OnString(const TStringBuf& v) { - return Consume(CheckNewObject(CreatePyString(v, InternVals, MayUnicode))); - } - - virtual bool OnOpenMap() { - Push(Dict, CheckNewObject(PyDict_New())); - - return true; - } - - virtual bool OnCloseMap() { - return Consume(Pop(Dict)); - } - - virtual bool OnMapKey(const TStringBuf& k) { - Push(Key, CheckNewObject(CreatePyString(k, InternKeys, MayUnicode))); - return true; - } - - virtual bool OnOpenArray() { - Push(Array, CheckNewObject(PyList_New(0))); - - return true; - } - - virtual bool OnCloseArray() { - return Consume(Pop(Array)); - } - }; -} - -PyObject* LoadJsonFromString(const char* data, size_t len, bool internKeys, bool internVals, bool mayUnicode) { - TContext ctx(internKeys, internVals, mayUnicode); - - if (!len) { - ythrow yexception() << "parse error: zero length input string"; - } - - if (!NJson::ReadJsonFast(TStringBuf(data, len), &ctx)) { - ythrow yexception() << "parse error"; - } - - auto& s = ctx.S; - - if (!s || s.top().Kind != Value) { - ythrow yexception() << "shit happen"; - } - - return s.top().Val.Release(); -} diff --git a/library/python/json/loads.h b/library/python/json/loads.h deleted file mode 100644 index 62dcdf6f21..0000000000 --- a/library/python/json/loads.h +++ /dev/null @@ -1,5 +0,0 @@ -#pragma once - -#include <Python.h> - -PyObject* LoadJsonFromString(const char* data, size_t len, bool internKeys = false, bool internVals = false, bool mayUnicode = false); diff --git a/library/python/json/loads.pyx b/library/python/json/loads.pyx deleted file mode 100644 index 82e5c6dce7..0000000000 --- a/library/python/json/loads.pyx +++ /dev/null @@ -1,14 +0,0 @@ -from libcpp cimport bool - -cdef extern from "library/python/json/loads.h": - object LoadJsonFromString(const char*, size_t, bool internKeys, bool internVals, bool mayUnicode) except + - - -def loads(s, intern_keys = False, intern_vals = False, may_unicode = False): - if isinstance(s, unicode): - s = s.encode('utf-8') - - try: - return LoadJsonFromString(s, len(s), intern_keys, intern_vals, may_unicode) - except Exception as e: - raise ValueError(str(e)) diff --git a/library/python/json/ya.make b/library/python/json/ya.make deleted file mode 100644 index 74a82de9d8..0000000000 --- a/library/python/json/ya.make +++ /dev/null @@ -1,17 +0,0 @@ -PY23_LIBRARY() - -PEERDIR( - contrib/python/simplejson - library/cpp/json/fast_sax -) - -PY_SRCS( - __init__.py - loads.pyx -) - -SRCS( - loads.cpp -) - -END() diff --git a/library/python/par_apply/__init__.py b/library/python/par_apply/__init__.py deleted file mode 100644 index 19b89ae843..0000000000 --- a/library/python/par_apply/__init__.py +++ /dev/null @@ -1,114 +0,0 @@ -import sys -import threading -import six - -from six.moves import queue - - -def par_apply(seq, func, thr_num, join_polling=None): - if thr_num < 2: - for x in seq: - yield func(x) - - return - - in_q = queue.Queue() - out_q = queue.Queue() - - def enumerate_blocks(): - n = 0 - - for b in seq: - yield n, [b] - n += 1 - - yield n, None - - def iter_out(): - n = 0 - d = {} - - while True: - if n in d: - r = d[n] - del d[n] - n += 1 - - yield r - else: - res = out_q.get() - - d[res[0]] = res - - out_iter = iter_out() - - def wait_block(): - for x in out_iter: - return x - - def iter_compressed(): - p = 0 - - for n, b in enumerate_blocks(): - in_q.put((n, b)) - - while n > p + (thr_num * 2): - p, b, c = wait_block() - - if not b: - return - - yield p, c - - while True: - p, b, c = wait_block() - - if not b: - return - - yield p, c - - def proc(): - while True: - data = in_q.get() - - if data is None: - return - - n, b = data - - if b: - try: - res = (func(b[0]), None) - except Exception: - res = (None, sys.exc_info()) - else: - res = (None, None) - - out_q.put((n, b, res)) - - thrs = [threading.Thread(target=proc) for i in range(0, thr_num)] - - for t in thrs: - t.start() - - try: - for p, c in iter_compressed(): - res, err = c - - if err: - six.reraise(*err) - - yield res - finally: - for t in thrs: - in_q.put(None) - - for t in thrs: - if join_polling is not None: - while True: - t.join(join_polling) - if not t.is_alive(): - break - else: - t.join() diff --git a/library/python/par_apply/ya.make b/library/python/par_apply/ya.make deleted file mode 100644 index b14592ab79..0000000000 --- a/library/python/par_apply/ya.make +++ /dev/null @@ -1,11 +0,0 @@ -PY23_LIBRARY() - -PEERDIR( - contrib/python/six -) - -PY_SRCS( - __init__.py -) - -END() |