diff options
author | thegeorg <thegeorg@yandex-team.com> | 2023-10-03 11:19:48 +0300 |
---|---|---|
committer | thegeorg <thegeorg@yandex-team.com> | 2023-10-03 11:43:28 +0300 |
commit | cda0c13f23f6b169fb0a49dc504b40a0aaecea09 (patch) | |
tree | 26476e92e5af2c856e017afb1df8f8dff42495bf /library/cpp/pybind | |
parent | 4854116da9c5e3c95bb8440f2ea997c54b6e1a61 (diff) | |
download | ydb-cda0c13f23f6b169fb0a49dc504b40a0aaecea09.tar.gz |
Move contrib/tools/jdk to build/platform/java/jdk/testing
Diffstat (limited to 'library/cpp/pybind')
-rw-r--r-- | library/cpp/pybind/attr.h | 412 | ||||
-rw-r--r-- | library/cpp/pybind/cast.cpp | 324 | ||||
-rw-r--r-- | library/cpp/pybind/cast.h | 373 | ||||
-rw-r--r-- | library/cpp/pybind/embedding.cpp | 63 | ||||
-rw-r--r-- | library/cpp/pybind/embedding.h | 10 | ||||
-rw-r--r-- | library/cpp/pybind/empty.cpp | 2 | ||||
-rw-r--r-- | library/cpp/pybind/exceptions.cpp | 147 | ||||
-rw-r--r-- | library/cpp/pybind/exceptions.h | 143 | ||||
-rw-r--r-- | library/cpp/pybind/init.h | 25 | ||||
-rw-r--r-- | library/cpp/pybind/method.h | 439 | ||||
-rw-r--r-- | library/cpp/pybind/module.cpp | 72 | ||||
-rw-r--r-- | library/cpp/pybind/module.h | 176 | ||||
-rw-r--r-- | library/cpp/pybind/pod.cpp | 18 | ||||
-rw-r--r-- | library/cpp/pybind/pod.h | 53 | ||||
-rw-r--r-- | library/cpp/pybind/ptr.h | 51 | ||||
-rw-r--r-- | library/cpp/pybind/typeattrs.h | 368 | ||||
-rw-r--r-- | library/cpp/pybind/typedesc.cpp | 79 | ||||
-rw-r--r-- | library/cpp/pybind/typedesc.h | 545 | ||||
-rw-r--r-- | library/cpp/pybind/v2.cpp | 43 | ||||
-rw-r--r-- | library/cpp/pybind/v2.h | 514 | ||||
-rw-r--r-- | library/cpp/pybind/ya.make | 14 |
21 files changed, 3871 insertions, 0 deletions
diff --git a/library/cpp/pybind/attr.h b/library/cpp/pybind/attr.h new file mode 100644 index 0000000000..5f25a6d73d --- /dev/null +++ b/library/cpp/pybind/attr.h @@ -0,0 +1,412 @@ +#pragma once + +#define PY_SSIZE_T_CLEAN +#include <Python.h> +#include <util/generic/string.h> +#include <util/generic/map.h> +#include <util/generic/set.h> +#include <util/generic/vector.h> +#include <util/generic/ptr.h> + +#include "cast.h" +#include "exceptions.h" + +namespace NPyBind { + // TBaseAttrGetter + template <typename TObjType> + class TBaseAttrGetter { + public: + virtual ~TBaseAttrGetter() { + } + virtual bool GetAttr(PyObject* owner, const TObjType& self, const TString& attr, PyObject*& res) const = 0; + + virtual bool HasAttr(PyObject* owner, const TObjType& self, const TString& attr, const TSet<TString>& hiddenNames) const { + if (hiddenNames.find(attr) != hiddenNames.end()) + return false; + PyObject* res = nullptr; + if (!GetAttr(owner, self, attr, res)) + return false; + Py_XDECREF(res); + return true; + } + }; + + template <typename TObjType> + class TBaseAttrSetter { + public: + virtual ~TBaseAttrSetter() { + } + + virtual bool SetAttr(PyObject* owner, TObjType& self, const TString& attr, PyObject* val) = 0; + }; + + template <typename TObjType> + class TAttrGetters { + public: + typedef TSimpleSharedPtr<TBaseAttrGetter<TObjType>> TGetterPtr; + + private: + typedef TVector<TGetterPtr> TGetterList; + typedef TMap<TString, TGetterList> TGetterMap; + + const TSet<TString>& HiddenAttrNames; + TGetterMap Getters; + + public: + TAttrGetters(const TSet<TString>& hiddenNames) + : HiddenAttrNames(hiddenNames) + { + } + + void AddGetter(const TString& attr, TGetterPtr getter) { + Getters[attr].push_back(getter); + } + + PyObject* GetAttr(PyObject* owner, const TObjType& self, const TString& attr) const { + typename TGetterMap::const_iterator it1 = Getters.find(attr); + if (it1 == Getters.end()) + it1 = Getters.find(""); + if (it1 == Getters.end()) + return nullptr; + const TGetterList& lst = it1->second; + for (typename TGetterList::const_iterator it2 = lst.begin(), end = lst.end(); it2 != end; ++it2) { + PyObject* res = nullptr; + if ((*it2)->GetAttr(owner, self, attr, res)) + return res; + // IMPORTANT! + // we have to fail GetAttr right there because we've failed because of internal python error/exception and can't continue iterating because + // it cause subsequent exceptions during call to Py_BuildValue + // moreover we have to preserve original exception right there + if (PyErr_Occurred()) { + break; + } + } + return nullptr; + } + + bool HasAttr(PyObject* owner, const TObjType& self, const TString& attr) const { + typename TGetterMap::const_iterator it1 = Getters.find(attr); + if (it1 == Getters.end()) + return false; + const TGetterList& lst = it1->second; + for (typename TGetterList::const_iterator it2 = lst.begin(), end = lst.end(); it2 != end; ++it2) { + if ((*it2)->HasAttr(owner, self, attr, HiddenAttrNames)) + return true; + } + return false; + } + + void GetAttrsDictionary(PyObject* owner, const TObjType& self, TMap<TString, PyObject*>& res) const { + for (typename TGetterMap::const_iterator it = Getters.begin(), end = Getters.end(); it != end; ++it) { + try { + if (HasAttr(owner, self, it->first)) { + auto attrPtr = GetAttr(owner, self, it->first); + if (attrPtr) { + res[it->first] = attrPtr; + } + if (PyErr_Occurred()) { + PyErr_Clear(); // Skip python errors as well + } + } + } catch (const std::exception&) { + // ignore this field + } + } + } + + void GetAttrsNames(PyObject* owner, const TObjType& self, TVector<TString>& resultNames) const { + for (typename TGetterMap::const_iterator it = Getters.begin(), end = Getters.end(); it != end; ++it) { + if (HasAttr(owner, self, it->first)) + resultNames.push_back(it->first); + } + } + }; + + template <typename TObjType> + class TGenericAttrGetter: public TBaseAttrGetter<TObjType> { + private: + TString AttrName; + + public: + TGenericAttrGetter(const TString& attrName) + : AttrName(attrName) + { + } + + bool GetAttr(PyObject* obj, const TObjType&, const TString&, PyObject*& res) const override { + auto str = NameFromString(AttrName); + res = PyObject_GenericGetAttr(obj, str.Get()); + if (!res && !PyErr_Occurred()) + ythrow TPyErr(PyExc_AttributeError) << "Can't get generic attribute '" << AttrName << "'"; + return res; + } + }; + + template <typename TObjType> + class TAttrSetters { + private: + typedef TSimpleSharedPtr<TBaseAttrSetter<TObjType>> TSetterPtr; + typedef TVector<TSetterPtr> TSetterList; + typedef TMap<TString, TSetterList> TSetterMap; + + TSetterMap Setters; + + public: + void AddSetter(const TString& attr, TSetterPtr setter) { + Setters[attr].push_back(setter); + } + + bool SetAttr(PyObject* owner, TObjType& self, const TString& attr, PyObject* val) { + typename TSetterMap::const_iterator it1 = Setters.find(attr); + if (it1 == Setters.end()) + it1 = Setters.find(""); + if (it1 == Setters.end()) + return false; + const TSetterList& lst = it1->second; + for (typename TSetterList::const_iterator it2 = lst.begin(), end = lst.end(); it2 != end; ++it2) { + if ((*it2)->SetAttr(owner, self, attr, val)) + return true; + } + return false; + } + + bool SetAttrDictionary(PyObject* owner, TObjType& self, TMap<TString, PyObject*>& dict) { + for (TMap<TString, PyObject*>::const_iterator it = dict.begin(), end = dict.end(); it != end; ++it) { + try { + SetAttr(owner, self, it->first, it->second); + } catch (std::exception&) { + // ignore this field + } + } + + return true; + } + }; + + /** + * TMethodAttrGetter - this class maps Python attribute read to C++ method call + */ + template <typename TObjType, typename TResult, typename TSubObject> + class TMethodAttrGetter: public TBaseAttrGetter<TObjType> { + private: + typedef TResult (TSubObject::*TMethod)() const; + TMethod Method; + + public: + TMethodAttrGetter(TMethod method) + : Method(method) + { + } + + bool GetAttr(PyObject*, const TObjType& self, const TString&, PyObject*& res) const override { + const TSubObject* sub = dynamic_cast<const TSubObject*>(&self); + if (sub == nullptr) + return false; + res = BuildPyObject((sub->*Method)()); + return (res != nullptr); + } + }; + + template <typename TObjType, typename TFunctor> + class TFunctorAttrGetter: public TBaseAttrGetter<TObjType> { + TFunctor Functor; + public: + explicit TFunctorAttrGetter(TFunctor functor) + : Functor(functor) + { + } + + bool GetAttr(PyObject*, const TObjType& self, const TString&, PyObject*& res) const override { + res = BuildPyObject(Functor(self)); + return (res != nullptr); + } + }; + + + /** + * TMethodAttrGetterWithCheck - this class maps Python attribute read to C++ HasAttr/GetAttr call + * If HasAttr returns false, None is returned. + * Otherwise GetAttr is called. + */ + template <typename TObjType, typename TResult, typename TSubObject> + class TMethodAttrGetterWithCheck: public TBaseAttrGetter<TObjType> { + private: + typedef TResult (TSubObject::*TMethod)() const; + typedef bool (TSubObject::*TCheckerMethod)() const; + TMethod Method; + TCheckerMethod CheckerMethod; + + public: + TMethodAttrGetterWithCheck(TMethod method, TCheckerMethod checkerMethod) + : Method(method) + , CheckerMethod(checkerMethod) + { + } + + bool GetAttr(PyObject*, const TObjType& self, const TString&, PyObject*& res) const override { + const TSubObject* sub = dynamic_cast<const TSubObject*>(&self); + if (sub == nullptr) + return false; + if ((sub->*CheckerMethod)()) + res = BuildPyObject((sub->*Method)()); + else { + Py_INCREF(Py_None); + res = Py_None; + } + return (res != nullptr); + } + }; + + template <typename TObjType, typename TResult, typename TSubObject, typename TMapper> + class TMethodAttrMappingGetter: public TBaseAttrGetter<TObjType> { + private: + typedef TResult (TSubObject::*TMethod)() const; + + TMethod Method; + TMapper Mapper; + + public: + TMethodAttrMappingGetter(TMethod method, TMapper mapper) + : Method(method) + , Mapper(mapper) + { + } + + bool GetAttr(PyObject*, const TObjType& self, const TString&, PyObject*& res) const override { + const TSubObject* sub = dynamic_cast<const TSubObject*>(&self); + if (sub == nullptr) + return false; + res = BuildPyObject(Mapper((sub->*Method)())); + return (res != nullptr); + } + }; + + template <typename TObjType, typename TResult, typename TSubObject, typename TMapper> + TSimpleSharedPtr<TBaseAttrGetter<TObjType>> + CreateMethodAttrMappingGetter(TResult (TSubObject::*method)() const, + TMapper mapper) { + return new TMethodAttrMappingGetter<TObjType, TResult, TSubObject, TMapper>(method, + mapper); + } + + template <typename TObjType, typename TResult, typename TValue, typename TSubObject> + class TMethodAttrSetter: public TBaseAttrSetter<TObjType> { + private: + typedef TResult (TSubObject::*TMethod)(TValue&); + TMethod Method; + + public: + TMethodAttrSetter(TMethod method) + : Method(method) + { + } + + virtual bool SetAttr(PyObject*, TObjType& self, const TString&, PyObject* val) { + TSubObject* sub = dynamic_cast<TSubObject*>(&self); + if (sub == nullptr) + return false; + TValue value; + if (!FromPyObject(val, value)) + return false; + (sub->*Method)(value); + return true; + } + }; + + template <typename TObjType, typename TValue, typename TFunctor> + class TFunctorAttrSetter: public TBaseAttrSetter<TObjType> { + TFunctor Functor; + public: + explicit TFunctorAttrSetter(TFunctor functor) + : Functor(functor) + { + } + + bool SetAttr(PyObject*, TObjType& self, const TString&, PyObject* val) const override { + TValue value; + if (!FromPyObject(val, value)) + return false; + auto res = BuildPyObject(Functor(self, value)); + return (res != nullptr); + } + }; + template <typename TObjType, typename TResult, typename TSubObject> + TSimpleSharedPtr<TBaseAttrGetter<TObjType>> CreateMethodAttrGetter(TResult (TSubObject::*method)() const) { + return new TMethodAttrGetter<TObjType, TResult, TSubObject>(method); + } + + template <typename TObjType, typename TFunctor> + TSimpleSharedPtr<TFunctorAttrGetter<TObjType, TFunctor>> CreateFunctorAttrGetter(TFunctor functor) { + return MakeSimpleShared<TFunctorAttrGetter<TObjType, TFunctor>>(functor); + } + + template <typename TObjType, typename TResult, typename TSubObject> + TSimpleSharedPtr<TBaseAttrGetter<TObjType>> CreateMethodAttrGetterWithCheck( + TResult (TSubObject::*method)() const, + bool (TSubObject::*checkerMethod)() const) { + return new TMethodAttrGetterWithCheck<TObjType, TResult, TSubObject>(method, checkerMethod); + } + + template <typename TObjType, typename TResult, typename TValue, typename TSubObject> + TSimpleSharedPtr<TBaseAttrSetter<TObjType>> CreateMethodAttrSetter(TResult (TSubObject::*method)(TValue&)) { + return new TMethodAttrSetter<TObjType, TResult, TValue, TSubObject>(method); + } + + template <typename TObjType, typename TFunctor, typename TValue> + TSimpleSharedPtr<TFunctorAttrSetter<TObjType, TValue, TFunctor>> CreateFunctorAttrSetter(TFunctor functor) { + return MakeSimpleShared<TFunctorAttrSetter<TObjType, TValue, TFunctor>>(functor); + } + + template <typename TObjType, typename TValue, typename TSubObject> + class TDirectAttrSetter: public TBaseAttrSetter<TObjType> { + private: + typedef TValue TSubObject::*TValueType; + TValueType Value; + + public: + TDirectAttrSetter(TValueType value) + : Value(value) + { + } + + bool SetAttr(PyObject*, TObjType& self, const TString&, PyObject* val) override { + TSubObject* sub = dynamic_cast<TSubObject*>(&self); + if (sub == NULL) + return false; + if (!FromPyObject(val, sub->*Value)) + return false; + return true; + } + }; + + template <typename TObjType, typename TValue, typename TSubObject> + TSimpleSharedPtr<TBaseAttrSetter<TObjType>> CreateAttrSetter(TValue TSubObject::*value) { + return new TDirectAttrSetter<TObjType, TValue, TSubObject>(value); + } + + template <typename TObjType, typename TValue, typename TSubObject> + class TDirectAttrGetter: public TBaseAttrGetter<TObjType> { + private: + typedef TValue TSubObject::*TValueType; + TValueType Value; + + public: + TDirectAttrGetter(TValueType value) + : Value(value) + { + } + + bool GetAttr(PyObject*, const TObjType& self, const TString&, PyObject*& res) const override { + const TSubObject* sub = dynamic_cast<const TSubObject*>(&self); + if (sub == nullptr) + return false; + res = BuildPyObject(sub->*Value); + return (res != nullptr); + } + }; + + template <typename TObjType, typename TValue, typename TSubObject> + TSimpleSharedPtr<TBaseAttrGetter<TObjType>> CreateAttrGetter(TValue TSubObject::*value) { + return new TDirectAttrGetter<TObjType, TValue, TSubObject>(value); + } +} diff --git a/library/cpp/pybind/cast.cpp b/library/cpp/pybind/cast.cpp new file mode 100644 index 0000000000..60a44b0e83 --- /dev/null +++ b/library/cpp/pybind/cast.cpp @@ -0,0 +1,324 @@ +#include "cast.h" +#include <util/generic/yexception.h> +#include <util/generic/buffer.h> + +namespace NPyBind { + PyObject* GetTrueRef(bool incref) { + if (incref) + Py_RETURN_TRUE; + return Py_True; + } + + PyObject* GetFalseRef(bool incref) { + if (incref) + Py_RETURN_FALSE; + return Py_False; + } + + PyObject* BuildPyObject(int val) { + return Py_BuildValue("i", val); + } + + PyObject* BuildPyObject(unsigned int val) { + return Py_BuildValue("I", val); + } + + PyObject* BuildPyObject(long int val) { + return Py_BuildValue("l", val); + } + + PyObject* BuildPyObject(unsigned long int val) { + return Py_BuildValue("k", val); + } + +#ifdef PY_LONG_LONG + PyObject* BuildPyObject(PY_LONG_LONG val) { + return Py_BuildValue("L", val); + } + + PyObject* BuildPyObject(unsigned PY_LONG_LONG val) { + return Py_BuildValue("K", val); + } +#endif + + PyObject* BuildPyObject(float val) { + return Py_BuildValue("f", val); + } + + PyObject* BuildPyObject(double val) { + return Py_BuildValue("d", val); + } + + PyObject* BuildPyObject(const TStringBuf& val) { + if (!val.IsInited()) + Py_RETURN_NONE; + + PyObject* stringValue = Py_BuildValue("s#", val.data(), static_cast<int>(val.length())); + if (stringValue != nullptr) { + return stringValue; + } + if (PyErr_ExceptionMatches(PyExc_UnicodeDecodeError)) { + PyErr_Clear(); + } else { + return nullptr; + } + return Py_BuildValue("y#", val.data(), static_cast<int>(val.length())); + } + + PyObject* BuildPyObject(const char* val) { + if (val == nullptr) + Py_RETURN_NONE; + PyObject* stringValue = Py_BuildValue("s#", val, static_cast<int>(strlen(val))); + if (stringValue != nullptr) { + return stringValue; + } + if (PyErr_ExceptionMatches(PyExc_UnicodeDecodeError)) { + PyErr_Clear(); + } else { + return nullptr; + } + return Py_BuildValue("y#", val, static_cast<int>(strlen(val))); + } + + PyObject* BuildPyObject(const TWtringBuf& val) { + if (!val.IsInited()) + Py_RETURN_NONE; +#if PY_VERSION_HEX < 0x03030000 + TPyObjectPtr result(PyUnicode_FromUnicode(nullptr, val.size()), true); + Py_UNICODE* buf = PyUnicode_AS_UNICODE(result.Get()); + if (buf == nullptr) + Py_RETURN_NONE; + for (size_t i = 0; i < val.size(); ++i) { + buf[i] = static_cast<Py_UNICODE>(val[i]); + } +#else + PyObject* unicodeValue = PyUnicode_FromKindAndData(PyUnicode_2BYTE_KIND, val.data(), val.size()); + if (unicodeValue == nullptr) + Py_RETURN_NONE; + TPyObjectPtr result(unicodeValue, true); +#endif + return result.RefGet(); + } + + PyObject* BuildPyObject(const TBuffer& val) { + TPyObjectPtr res(PyList_New(val.size()), true); + for (size_t i = 0, size = val.Size(); i < size; ++i) + PyList_SetItem(res.Get(), i, BuildPyObject(val.Data()[i])); + return res.RefGet(); + } + + PyObject* BuildPyObject(bool val) { + if (val) + Py_RETURN_TRUE; + else + Py_RETURN_FALSE; + } + + PyObject* BuildPyObject(PyObject* val) { + Py_XINCREF(val); + return val; + } + + PyObject* BuildPyObject(TPyObjectPtr ptr) { + return ptr.RefGet(); + } + + /* python represents (http://docs.python.org/c-api/arg.html#Py_BuildValue) + * char, uchar, short, ushort, int, long as PyInt + * uint, ulong as PyInt or PyLong (if exceeds sys.maxint) + * longlong, ulonglong as PyLong + */ + + template <> + bool FromPyObject(PyObject* obj, long& res) { + if (PyLong_Check(obj)) { + res = PyLong_AsLong(obj); + return true; + } + if (PyFloat_Check(obj)) { + res = static_cast<long>(PyFloat_AsDouble(obj)); + return true; + } +#if PY_MAJOR_VERSION < 3 + res = PyInt_AsLong(obj); +#endif + return -1 != res || !PyErr_Occurred(); + } + + template <> + bool FromPyObject(PyObject* obj, unsigned long& res) { + long lres; + if (!FromPyObject(obj, lres)) + return false; + if (lres < 0) + return false; + res = static_cast<unsigned long long>(lres); + return true; + } + + template <> + bool FromPyObject(PyObject* obj, int& res) { + long lres; + if (!FromPyObject(obj, lres)) + return false; + res = static_cast<int>(lres); + return true; + } + + template <> + bool FromPyObject(PyObject* obj, unsigned char& res) { + long lres; + if (!FromPyObject(obj, lres)) + return false; + res = static_cast<unsigned char>(lres); + return true; + } + + template <> + bool FromPyObject(PyObject* obj, char& res) { + long lres; + if (!FromPyObject(obj, lres)) + return false; + res = static_cast<char>(lres); + return true; + } + + template <> + bool FromPyObject(PyObject* obj, unsigned int& res) { + unsigned long lres; + if (!FromPyObject(obj, lres)) + return false; + res = static_cast<unsigned int>(lres); + return true; + } + +#ifdef HAVE_LONG_LONG + template <> + bool FromPyObject(PyObject* obj, long long& res) { + if (PyLong_Check(obj)) { + res = PyLong_AsLongLong(obj); + return -1 != res || !PyErr_Occurred(); + } + long lres; + if (!FromPyObject(obj, lres)) + return false; + res = static_cast<long long>(lres); + return true; + } + + template <> + bool FromPyObject(PyObject* obj, unsigned long long& res) { + if (PyLong_Check(obj)) { + res = PyLong_AsUnsignedLongLong(obj); + return static_cast<unsigned long long>(-1) != res || !PyErr_Occurred(); + } + long lres; + if (!FromPyObject(obj, lres)) + return false; + res = static_cast<unsigned long long>(lres); + return true; + } +#endif + + template <> + bool FromPyObject(PyObject* obj, double& res) { + if (PyFloat_Check(obj)) { + res = PyFloat_AsDouble(obj); + return true; + } + long long lres; + if (!FromPyObject(obj, lres)) + return false; + res = static_cast<double>(lres); + return true; + } + + template <> + bool FromPyObject(PyObject* obj, float& res) { + double dres; + if (!FromPyObject(obj, dres)) + return false; + res = static_cast<float>(dres); + return true; + } + + template <> + bool FromPyObject(PyObject* obj, bool& res) { + if (!PyBool_Check(obj)) + return false; + if (obj == Py_True) + res = true; + else + res = false; + return true; + } + + template <> + bool FromPyObject(PyObject* obj, PyObject*& res) { + Py_XINCREF(obj); + res = obj; + return true; + } + + template <> + bool FromPyObject(PyObject* obj, TPyObjectPtr& res) { + res = TPyObjectPtr(obj); + return true; + } + + static inline bool _FromPyObject(PyObject* obj, TStringBuf& res) { + char* str; + Py_ssize_t len; +#if PY_MAJOR_VERSION >= 3 + if (PyUnicode_Check(obj)) { + auto buf = PyUnicode_AsUTF8AndSize(obj, &len); + res = TStringBuf(buf, len); + return true; + } +#endif + if (-1 == PyBytes_AsStringAndSize(obj, &str, &len) || 0 > len) + return false; + res = TStringBuf(str, len); + return true; + } + + bool FromPyObject(PyObject* obj, TStringBuf& res) { + return _FromPyObject(obj, res); + } + + bool FromPyObject(PyObject* obj, TString& res) { + TStringBuf str; + if (!_FromPyObject(obj, str)) + return false; + res = str; + return true; + } + + bool FromPyObject(PyObject* obj, TUtf16String& res) { + if (!PyUnicode_Check(obj)) + return false; + auto str = TPyObjectPtr(PyUnicode_AsUTF16String(obj), true); + if (!str) + return false; + constexpr auto BOM_SIZE = 2; + size_t len = (static_cast<size_t>(PyBytes_GET_SIZE(str.Get())) - BOM_SIZE) / 2; + res.resize(len); + memcpy(res.begin(), PyBytes_AS_STRING(str.Get()) + BOM_SIZE, len * 2); + return (nullptr == PyErr_Occurred()); + } + + bool FromPyObject(PyObject* obj, TBuffer& res) { + if (!PyList_Check(obj)) + return false; + size_t cnt = PyList_Size(obj); + res.Reserve(cnt); + for (size_t i = 0; i < cnt; ++i) { + PyObject* item = PyList_GET_ITEM(obj, i); + char ch = 0; + if (!FromPyObject(item, ch)) + return false; + res.Append(ch); + } + return true; + } +} diff --git a/library/cpp/pybind/cast.h b/library/cpp/pybind/cast.h new file mode 100644 index 0000000000..1f3d7d8366 --- /dev/null +++ b/library/cpp/pybind/cast.h @@ -0,0 +1,373 @@ +#pragma once + +#define PY_SSIZE_T_CLEAN +#include <Python.h> +#include <util/generic/strbuf.h> +#include <util/generic/vector.h> +#include <util/generic/set.h> +#include <util/generic/yexception.h> +#include <util/generic/hash.h> +#include <util/generic/map.h> +#include <util/generic/maybe.h> +#include <utility> +#include <initializer_list> +#include "ptr.h" + +namespace NPyBind { + PyObject* GetTrueRef(bool incref = true); + PyObject* GetFalseRef(bool incref = true); + + PyObject* BuildPyObject(int val); + PyObject* BuildPyObject(unsigned int val); + PyObject* BuildPyObject(long int val); + PyObject* BuildPyObject(unsigned long int val); +#ifdef PY_LONG_LONG + PyObject* BuildPyObject(PY_LONG_LONG val); + PyObject* BuildPyObject(unsigned PY_LONG_LONG val); +#endif + PyObject* BuildPyObject(float val); + PyObject* BuildPyObject(double val); + PyObject* BuildPyObject(const TStringBuf& val); + PyObject* BuildPyObject(const char* val); + PyObject* BuildPyObject(const TWtringBuf& val); + PyObject* BuildPyObject(const TBuffer& val); + PyObject* BuildPyObject(bool val); + PyObject* BuildPyObject(PyObject*); + PyObject* BuildPyObject(TPyObjectPtr); + + template <typename T> + PyObject* BuildPyObject(const TVector<T>& val); + + template <typename T> + PyObject* BuildPyObject(const TSet<T>& val); + + template <typename TKey, typename TVal> + PyObject* BuildPyObject(const THashMap<TKey, TVal>& val); + + template <typename T1, typename T2> + PyObject* BuildPyObject(const std::pair<T1, T2>& val) { + TPyObjectPtr first(BuildPyObject(val.first), true); + if (!first) { + return nullptr; + } + TPyObjectPtr second(BuildPyObject(val.second), true); + if (!first || !second) { + return nullptr; + } + TPyObjectPtr res(PyList_New(2), true); + PyList_SetItem(res.Get(), 0, first.RefGet()); + PyList_SetItem(res.Get(), 1, second.RefGet()); + return res.RefGet(); + } + + template <typename T> + PyObject* BuildPyObject(const TVector<T>& val) { + TPyObjectPtr res(PyList_New(val.size()), true); + for (size_t i = 0, size = val.size(); i < size; ++i) { + auto pythonVal = BuildPyObject(std::move(val[i])); + if (!pythonVal) { + return nullptr; + } + PyList_SetItem(res.Get(), i, pythonVal); + } + return res.RefGet(); + } + + template <typename T> + PyObject* BuildPyObject(TVector<T>&& val) { + TPyObjectPtr res(PyList_New(val.size()), true); + for (size_t i = 0, size = val.size(); i < size; ++i) { + auto pythonVal = BuildPyObject(std::move(val[i])); + if (!pythonVal) { + return nullptr; + } + PyList_SetItem(res.Get(), i, pythonVal); + } + return res.RefGet(); + } + + template <typename T> + PyObject* BuildPyObject(const TSet<T>& val) { + TPyObjectPtr res(PySet_New(nullptr), true); + for (const auto& v : val) { + auto pythonVal = BuildPyObject(std::move(v)); + if (!pythonVal) { + return nullptr; + } + PySet_Add(res.Get(), pythonVal); + } + return res.RefGet(); + } + + template <typename T> + PyObject* BuildPyObject(const THashSet<T>& val) { + TPyObjectPtr res(PySet_New(nullptr), true); + for (const auto& v : val) { + auto pythonVal = BuildPyObject(std::move(v)); + if (!pythonVal) { + return nullptr; + } + PySet_Add(res.Get(), pythonVal); + } + return res.RefGet(); + } + + template <typename TKey, typename TVal> + PyObject* BuildPyObject(const THashMap<TKey, TVal>& val) { + TPyObjectPtr res(PyDict_New(), true); + for (typename THashMap<TKey, TVal>::const_iterator it = val.begin(), end = val.end(); it != end; ++it) { + auto prevOccurred = PyErr_Occurred(); + Y_UNUSED(prevOccurred); + TPyObjectPtr k(BuildPyObject(it->first), true); + if (!k) { + return nullptr; + } + TPyObjectPtr v(BuildPyObject(it->second), true); + if (!v) { + return nullptr; + } + PyDict_SetItem(res.Get(), k.Get(), v.Get()); + } + return res.RefGet(); + } + + template <typename TKey, typename TVal> + PyObject* BuildPyObject(const TMap<TKey, TVal>& val) { + TPyObjectPtr res(PyDict_New(), true); + for (typename TMap<TKey, TVal>::const_iterator it = val.begin(), end = val.end(); it != end; ++it) { + TPyObjectPtr k(BuildPyObject(it->first), true); + if (!k) { + return nullptr; + } + TPyObjectPtr v(BuildPyObject(it->second), true); + if (!v) { + return nullptr; + } + PyDict_SetItem(res.Get(), k.Get(), v.Get()); + } + return res.RefGet(); + } + + + template <typename TKey, typename TVal> + PyObject* BuildPyObject(const TMultiMap<TKey, TVal>& val) { + TPyObjectPtr res(PyDict_New(), true); + TMaybe<TKey> prevKey; + TPyObjectPtr currentEntry(PyList_New(0), true); + for (const auto& [key, value]: val) { + if (prevKey.Defined() && prevKey != key) { + TPyObjectPtr pyPrevKey(BuildPyObject(*prevKey), true); + if (!pyPrevKey) { + return nullptr; + } + PyDict_SetItem(res.Get(), pyPrevKey.Get(), currentEntry.Get()); + currentEntry = TPyObjectPtr(PyList_New(0), true); + } + TPyObjectPtr pyValue(BuildPyObject(value), true); + if (!pyValue) { + return nullptr; + } + PyList_Append(currentEntry.Get(), pyValue.Get()); + prevKey = key; + } + + if (prevKey.Defined()) { + TPyObjectPtr pyPrevKey(BuildPyObject(*prevKey), true); + if (!pyPrevKey) { + return nullptr; + } + PyDict_SetItem(res.Get(), pyPrevKey.Get(), currentEntry.Get()); + } + return res.RefGet(); + } + + template <typename T> + PyObject* BuildPyObject(const TMaybe<T>& val) { + if (!val.Defined()) + Py_RETURN_NONE; + return BuildPyObject(val.GetRef()); + } + + template <typename T, typename C, typename D> + PyObject* BuildPyObject(const TSharedPtr<T, C, D>& val) { + if (!val.Get()) + Py_RETURN_NONE; + return BuildPyObject(*val.Get()); + } + + template <typename T> + bool FromPyObject(PyObject* obj, T& res); + + bool FromPyObject(PyObject* obj, TString& res); + bool FromPyObject(PyObject* obj, TStringBuf& res); + bool FromPyObject(PyObject* obj, TUtf16String& res); + bool FromPyObject(PyObject* obj, TBuffer& res); + + template <typename T> + bool FromPyObject(PyObject* obj, TMaybe<T>& res) { + //we need to save current error before trying derserialize the value + //because it can produce conversion errors in python that we don't need to handle + struct TError { + public: + TError() { + PyErr_Fetch(&Type, &Value, &Traceback); + } + ~TError() { + PyErr_Restore(Type, Value, Traceback); + + } + private: + PyObject* Type = nullptr; + PyObject* Value = nullptr; + PyObject* Traceback = nullptr; + } currentPyExcInfo; + T val; + if (FromPyObject(obj, val)) { + res = val; + return true; + } + if (obj == Py_None) { + res = Nothing(); + return true; + } + return false; + } + + template <typename T1, typename T2> + bool FromPyObject(PyObject* obj, std::pair<T1, T2>& res) { + PyObject* first; + PyObject* second; + if (PyTuple_Check(obj) && 2 == PyTuple_Size(obj)) { + first = PyTuple_GET_ITEM(obj, 0); + second = PyTuple_GET_ITEM(obj, 1); + } else if (PyList_Check(obj) && 2 == PyList_Size(obj)) { + first = PyList_GET_ITEM(obj, 0); + second = PyList_GET_ITEM(obj, 1); + } else { + return false; + } + return FromPyObject(first, res.first) && FromPyObject(second, res.second); + } + + template <typename T> + bool FromPyObject(PyObject* obj, TVector<T>& res) { + if (!PyList_Check(obj)) + return false; + size_t cnt = PyList_Size(obj); + res.resize(cnt); + for (size_t i = 0; i < cnt; ++i) { + PyObject* item = PyList_GET_ITEM(obj, i); + if (!FromPyObject(item, res[i])) + return false; + } + return true; + } + + template <typename K, typename V> + bool FromPyObject(PyObject* obj, THashMap<K, V>& res) { + if (!PyDict_Check(obj)) + return false; + TPyObjectPtr list(PyDict_Keys(obj), true); + size_t cnt = PyList_Size(list.Get()); + for (size_t i = 0; i < cnt; ++i) { + PyObject* key = PyList_GET_ITEM(list.Get(), i); + PyObject* value = PyDict_GetItem(obj, key); + K rkey; + V rvalue; + if (!FromPyObject(key, rkey)) + return false; + if (!FromPyObject(value, rvalue)) + return false; + res[rkey] = rvalue; + } + return true; + } + + template <typename K, typename V> + bool FromPyObject(PyObject* obj, TMap<K, V>& res) { + if (!PyDict_Check(obj)) + return false; + TPyObjectPtr list(PyDict_Keys(obj), true); + size_t cnt = PyList_Size(list.Get()); + for (size_t i = 0; i < cnt; ++i) { + PyObject* key = PyList_GET_ITEM(list.Get(), i); + PyObject* value = PyDict_GetItem(obj, key); + K rkey; + V rvalue; + if (!FromPyObject(key, rkey)) + return false; + if (!FromPyObject(value, rvalue)) + return false; + res[rkey] = rvalue; + } + return true; + } + + class cast_exception: public TBadCastException { + }; + + template <typename T> + T FromPyObject(PyObject* obj) { + T res; + if (!FromPyObject(obj, res)) + ythrow cast_exception() << "Cannot cast argument to " << TypeName<T>(); + return res; + } + + template <class... Args, std::size_t... I> + bool ExtractArgs(std::index_sequence<I...>, PyObject* args, Args&... outArgs) { + if (!args || !PyTuple_Check(args) || PyTuple_Size(args) != sizeof...(Args)) + return false; + bool res = true; + (void)std::initializer_list<bool>{(res = res && NPyBind::FromPyObject(PyTuple_GET_ITEM(args, I), outArgs))...}; + return res; + } + + template <class... Args> + bool ExtractArgs(PyObject* args, Args&... outArgs) { + return ExtractArgs(std::index_sequence_for<Args...>(), args, outArgs...); + } + + template <class... Args, std::size_t... I> + bool ExtractOptionalArgs(std::index_sequence<I...>, PyObject* args, PyObject* kwargs, const char* keywords[], Args&... outArgs) { + PyObject* pargs[sizeof...(Args)] = {}; + static const char format[sizeof...(Args) + 2] = {'|', ((void)I, 'O')..., 0}; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, format, const_cast<char**>(keywords), &pargs[I]...)) + return false; + bool res = true; + (void)std::initializer_list<bool>{(res = res && (!pargs[I] || NPyBind::FromPyObject(pargs[I], outArgs)))...}; + return res; + } + + template <class... Args> + bool ExtractOptionalArgs(PyObject* args, PyObject* kwargs, const char* keywords[], Args&... outArgs) { + return ExtractOptionalArgs(std::index_sequence_for<Args...>(), args, kwargs, keywords, outArgs...); + } + + template <typename... Args, std::size_t... I> + static auto GetArguments(std::index_sequence<I...>, PyObject* args) { + Y_UNUSED(args); // gcc bug + return std::make_tuple(FromPyObject<std::remove_cv_t<std::remove_reference_t<Args>>>(PyTuple_GetItem(args, I))...); + } + + template <typename... Args> + static auto GetArguments(PyObject* args) { + return GetArguments<Args...>(std::index_sequence_for<Args...>(), args); + } + + inline PyObject* ReturnString(TStringBuf s) { +#if PY_MAJOR_VERSION >= 3 + return PyUnicode_FromStringAndSize(s.data(), s.size()); +#else + return PyBytes_FromStringAndSize(s.data(), s.size()); +#endif + } + + inline TPyObjectPtr ReturnBytes(TStringBuf s) { + return TPyObjectPtr(PyBytes_FromStringAndSize(s.data(), s.size()), true); + } + + inline TPyObjectPtr NameFromString(TStringBuf s) { + return TPyObjectPtr(ReturnString(s), true); + } +} diff --git a/library/cpp/pybind/embedding.cpp b/library/cpp/pybind/embedding.cpp new file mode 100644 index 0000000000..cf8941a92a --- /dev/null +++ b/library/cpp/pybind/embedding.cpp @@ -0,0 +1,63 @@ +#define PY_SSIZE_T_CLEAN +#include <Python.h> + +#include "embedding.h" + +#include <util/generic/ptr.h> +#include <util/generic/yexception.h> + +namespace NPyBind { +#if PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION < 8 + class TDeleteRawMem { + public: + template <typename T> + static inline void Destroy(T* t) noexcept { + PyMem_RawFree(t); + } + }; + + template <typename T> + using TRawMemHolder = THolder<T, TDeleteRawMem>; + + static void SetProgramName(char* name) { + TRawMemHolder<wchar_t> wideName(Py_DecodeLocale(name, nullptr)); + Y_ENSURE(wideName); + Py_SetProgramName(wideName.Get()); + } +#endif + + TEmbedding::TEmbedding(char* argv0) { +#if PY_MAJOR_VERSION < 3 + Py_SetProgramName(argv0); + Py_Initialize(); +#elif PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 8 + PyStatus status; + + PyConfig config; + PyConfig_InitPythonConfig(&config); + // Disable parsing command line arguments + config.parse_argv = 0; + + status = PyConfig_SetBytesString(&config, &config.program_name, argv0); + if (PyStatus_Exception(status)) { + PyConfig_Clear(&config); + Py_ExitStatusException(status); + } + + status = Py_InitializeFromConfig(&config); + if (PyStatus_Exception(status)) { + PyConfig_Clear(&config); + Py_ExitStatusException(status); + } + + PyConfig_Clear(&config); +#elif PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION < 8 + SetProgramName(argv0); + Py_Initialize(); +#endif + } + + TEmbedding::~TEmbedding() { + Py_Finalize(); + } +} diff --git a/library/cpp/pybind/embedding.h b/library/cpp/pybind/embedding.h new file mode 100644 index 0000000000..18553d9f6c --- /dev/null +++ b/library/cpp/pybind/embedding.h @@ -0,0 +1,10 @@ +#pragma once + +namespace NPyBind { + class TEmbedding { + public: + TEmbedding(char* argv0); + ~TEmbedding(); + }; + +} diff --git a/library/cpp/pybind/empty.cpp b/library/cpp/pybind/empty.cpp new file mode 100644 index 0000000000..10da997ecc --- /dev/null +++ b/library/cpp/pybind/empty.cpp @@ -0,0 +1,2 @@ +#include "init.h" +#include "v2.h" diff --git a/library/cpp/pybind/exceptions.cpp b/library/cpp/pybind/exceptions.cpp new file mode 100644 index 0000000000..db1531fc63 --- /dev/null +++ b/library/cpp/pybind/exceptions.cpp @@ -0,0 +1,147 @@ +#include "exceptions.h" +#include "cast.h" +#include "module.h" +#include <util/generic/algorithm.h> + +namespace NPyBind { + + namespace NPrivate { + TPyObjectPtr CreatePyBindModule() { + return TPyObjectPtr(TExceptionsHolder::DoInitPyBindModule(), true); + } + }//NPrivate + + TPyObjectPtr TExceptionsHolder::GetException(const TString& name) { + if (name == "") + return TPyObjectPtr(nullptr); + if (!Exceptions[name].Get()) + ythrow yexception() << "Wrong base class '" << name << "'"; + return Exceptions[name]; + } + + TPyObjectPtr TExceptionsHolder::GetExceptions(const TVector<TString>& names) { + TVector<TString> tmp(names.begin(), names.end()); + TVector<TString>::iterator end = std::unique(tmp.begin(), tmp.end()); + TPyObjectPtr tuple(PyTuple_New(std::distance(tmp.begin(), end)), true); + for (size_t i = 0; i < (size_t)std::distance(tmp.begin(), end); ++i) { + if (!Exceptions[tmp[i]].Get()) + ythrow yexception() << "Wrong base class '" << tmp[i] << "'"; + PyTuple_SetItem(tuple.Get(), i, Exceptions[tmp[i]].Get()); + } + return tuple; + } + + // def PyBindObjectReconstructor(cl, props): + // return cl(__properties__=props) + static PyObject* PyBindObjectReconstructor(PyObject*, PyObject* args) { + TPyObjectPtr callable, props; + if (!ExtractArgs(args, callable, props)) + ythrow yexception() << "Wrong method arguments"; +#if PY_MAJOR_VERSION >= 3 + TPyObjectPtr noArgs(PyTuple_New(0), true); +#else + TPyObjectPtr noArgs(PyList_New(0), true); +#endif + TPyObjectPtr kw(PyDict_New(), true); + PyDict_SetItemString(kw.Get(), "__properties__", props.Get()); + TPyObjectPtr res(PyObject_Call(callable.Get(), noArgs.Get(), kw.Get()), true); + return res.RefGet(); + } + + static PyMethodDef PyBindMethods[] = { + {"PyBindObjectReconstructor", PyBindObjectReconstructor, METH_VARARGS, "Tech method. It's required for unpickling."}, + {nullptr, nullptr, 0, nullptr}}; + +#if PY_MAJOR_VERSION >= 3 + static struct PyModuleDef moduledef = { + PyModuleDef_HEAD_INIT, + "pybind", + NULL, + -1, + PyBindMethods, + NULL, NULL, NULL, NULL + }; + + static PyObject* InitPyBind() { + return PyModule_Create(&moduledef); + } +#else + static PyObject* InitPyBind() { + return Py_InitModule("pybind", PyBindMethods); + } +#endif + + void TExceptionsHolder::DoInitPyBindModule2() { + DoInitPyBindModule(); + } + + PyObject* TExceptionsHolder::DoInitPyBindModule() { + Instance().Module = NPyBind::TPyObjectPtr(InitPyBind(), true); + if (!Instance().Module.Get()) + return nullptr; + + for (TCheckersVector::const_iterator it = Instance().Checkers.begin(), end = Instance().Checkers.end(); it != end; ++it) { + TString name = (*it)->GetName(); + if (!!name) { + //Ref to the object should be incremented before passing to AddObject + auto res = PyModule_AddObject(Instance().Module.Get(), name.data(), (*it)->GetException().RefGet()); + if (res < 0) { + ythrow yexception() << "Failed to add object " << name << " to internal module pybind"; + } + } + } + return Instance().Module.RefGet(); + } + + void TExceptionsHolder::Clear() { + //Unfortunately in Python3 we can't retrack this object because of PyError_NewException + //it's only the safe way to preserve GC gens in valid state during the finalization + for (auto& ptr: Checkers) { + if (!dynamic_cast<const TPyErrExceptionsChecker*>(ptr.Get())) { // no need to untrack standard PyExc_* exceptions from TPyErrExceptionsChecker + if (auto exceptionPtr = ptr->GetException()) { + PyObject_GC_UnTrack(exceptionPtr.Get()); + } + } + } + Checkers.clear(); + Exceptions.clear(); + Module.Drop(); + } + + TExceptionsHolder::TExceptionsHolder() { + AddException<std::exception>("yexception"); + AddException<TSystemError>("TSystemError", "yexception"); + AddException<TIoException>("TIoException", "yexception"); + + TVector<TString> names(2); + names[0] = "TSystemError"; + names[1] = "TIoException"; + + AddException<TIoSystemError>("TIoSystemError", names); + AddException<TFileError>("TFileError", "TIoSystemError"); + AddException<TBadCastException>("TBadCastException", "yexception"); + + Checkers.push_back(new TPyErrExceptionsChecker); + + // XXX: In Python 2.6, PyImport_AppendInittab() function takes non-const char*, this causes + // "ISO C++11 does not allow conversion from string literal to 'char *'" warning. + static char pybind[] = "pybind"; +#if PY_MAJOR_VERSION >= 3 + PyImport_AppendInittab(pybind, DoInitPyBindModule); + + NPrivate::AddFinalizationCallBack([this]() { + Clear(); + }); +#else + PyImport_AppendInittab(pybind, DoInitPyBindModule2); +#endif + } + + NPyBind::TPyObjectPtr TExceptionsHolder::ToPyException(const std::exception& ex) { + for (TCheckersVector::const_reverse_iterator it = Checkers.rbegin(), end = Checkers.rend(); it != end; ++it) { + if ((*it)->Check(ex)) + return (*it)->GetException(); + } + return TPyObjectPtr(nullptr); + } +} diff --git a/library/cpp/pybind/exceptions.h b/library/cpp/pybind/exceptions.h new file mode 100644 index 0000000000..48e20995e4 --- /dev/null +++ b/library/cpp/pybind/exceptions.h @@ -0,0 +1,143 @@ +#pragma once + +#define PY_SSIZE_T_CLEAN +#include <Python.h> +#include <util/generic/yexception.h> +#include <util/generic/map.h> +#include <util/generic/vector.h> +#include "ptr.h" + +namespace NPyBind { + // Usage: + // ythrow TPyErr(PyExc_TypeError) << "some python type error somewhere in your C++ code"; + // + class TPyErr: public virtual yexception { + public: + TPyErr(PyObject* theException = PyExc_RuntimeError) + : Exception(theException) + { + } + + TPyObjectPtr GetException() const { + return Exception; + } + + private: + NPyBind::TPyObjectPtr Exception; + }; + + //Private api for creating PyBind python module + //Needs only for overriding pybind python module in library which imports other pybind library + namespace NPrivate { + TPyObjectPtr CreatePyBindModule(); + }//NPrivate + class TExceptionsHolder { + friend TPyObjectPtr NPrivate::CreatePyBindModule(); + private: + TExceptionsHolder(const TExceptionsHolder&); + TExceptionsHolder& operator=(const TExceptionsHolder&); + TExceptionsHolder(); + + void Clear(); + TPyObjectPtr GetException(const TString&); + TPyObjectPtr GetExceptions(const TVector<TString>&); + private: + class TExceptionsChecker { + public: + virtual ~TExceptionsChecker() { + } + virtual bool Check(const std::exception& ex) const = 0; + virtual TString GetName() const = 0; + virtual TPyObjectPtr GetException() const = 0; + }; + + template <typename TExcType> + class TConcreteExceptionsChecker: public TExceptionsChecker { + private: + TString Name; + TPyObjectPtr Exception; + + public: + TConcreteExceptionsChecker(const TString& name, TPyObjectPtr exception) + : Name(name) + , Exception(exception) + { + } + + bool Check(const std::exception& ex) const override { + const std::exception* e = &ex; + return dynamic_cast<const TExcType*>(e); + } + + TString GetName() const override { + return Name; + } + + TPyObjectPtr GetException() const override { + return Exception; + } + }; + + class TPyErrExceptionsChecker: public TExceptionsChecker { + private: + mutable TPyObjectPtr Exception; + + public: + TPyErrExceptionsChecker() { + } + + bool Check(const std::exception& ex) const override { + const TPyErr* err = dynamic_cast<const TPyErr*>(&ex); + if (err) { + Exception = err->GetException(); + } + return err != nullptr; + } + + TString GetName() const override { + return TString(); + } + + TPyObjectPtr GetException() const override { + return Exception; + } + }; + + typedef TSimpleSharedPtr<TExceptionsChecker> TCheckerPtr; + typedef TVector<TCheckerPtr> TCheckersVector; + typedef TMap<TString, TPyObjectPtr> TExceptionsMap; + + TPyObjectPtr Module; + TCheckersVector Checkers; + TExceptionsMap Exceptions; + + static PyObject* DoInitPyBindModule(); + static void DoInitPyBindModule2(); + + public: + static TExceptionsHolder& Instance() { + static TExceptionsHolder Holder; + return Holder; + } + + template <typename TExcType> + void AddException(const TString& name, const TString& base = "") { + TPyObjectPtr baseException(GetException(base)); + TString fullName = TString("pybind.") + name; + TPyObjectPtr exception(PyErr_NewException(const_cast<char*>(fullName.c_str()), baseException.Get(), nullptr), true); + Checkers.push_back(new TConcreteExceptionsChecker<TExcType>(name, exception)); + Exceptions[name] = exception; + } + + template <typename TExcType> + void AddException(const TString& name, const TVector<TString>& bases) { + TPyObjectPtr baseExceptions(GetExceptions(bases)); + TString fullName = TString("pybind.") + name; + TPyObjectPtr exception(PyErr_NewException(const_cast<char*>(fullName.c_str()), baseExceptions.Get(), nullptr), true); + Checkers.push_back(new TConcreteExceptionsChecker<TExcType>(name, exception)); + Exceptions[name] = exception; + } + + NPyBind::TPyObjectPtr ToPyException(const std::exception&); + }; +} diff --git a/library/cpp/pybind/init.h b/library/cpp/pybind/init.h new file mode 100644 index 0000000000..58874574ed --- /dev/null +++ b/library/cpp/pybind/init.h @@ -0,0 +1,25 @@ +#pragma once + +#define PY_SSIZE_T_CLEAN +#include <Python.h> + +#include "ptr.h" + +namespace NPyBind { +#if PY_MAJOR_VERSION >= 3 + +#define PYBIND_MODINIT(name) PyMODINIT_FUNC PyInit_##name() + + inline PyObject* ModInitReturn(TPyObjectPtr&& modptr) { + return modptr.Release(); + } + +#else + +#define PYBIND_MODINIT(name) PyMODINIT_FUNC init##name() + + inline void ModInitReturn(TPyObjectPtr&&) { + } + +#endif +} diff --git a/library/cpp/pybind/method.h b/library/cpp/pybind/method.h new file mode 100644 index 0000000000..7c1f6e90e1 --- /dev/null +++ b/library/cpp/pybind/method.h @@ -0,0 +1,439 @@ +#pragma once + +#define PY_SSIZE_T_CLEAN +#include <Python.h> +#include <util/generic/string.h> +#include <util/generic/map.h> +#include <util/generic/set.h> +#include <util/generic/vector.h> +#include <util/generic/ptr.h> +#include <util/generic/typetraits.h> + +#include <util/generic/function.h> + +#include "cast.h" + +namespace NPyBind { + template <typename TObjType> + class TBaseMethodCaller { + public: + virtual ~TBaseMethodCaller() { + } + virtual bool CallMethod(PyObject* owner, TObjType* self, PyObject* args, PyObject* kwargs, PyObject*& res) const = 0; + virtual bool HasMethod(PyObject*, TObjType*, const TString&, const TSet<TString>&) { + return true; + } + }; + + template <typename TObjType> + class TIsACaller; + + template <typename TObjType> + class TMethodCallers { + private: + typedef TSimpleSharedPtr<TBaseMethodCaller<TObjType>> TCallerPtr; + typedef TVector<TCallerPtr> TCallerList; + typedef TMap<TString, TCallerList> TCallerMap; + + const TSet<TString>& HiddenAttrNames; + TCallerMap Callers; + + public: + TMethodCallers(const TSet<TString>& hiddenNames) + : HiddenAttrNames(hiddenNames) + { + } + + void AddCaller(const TString& name, TCallerPtr caller) { + Callers[name].push_back(caller); + } + + bool HasCaller(const TString& name) const { + return Callers.has(name); + } + + PyObject* CallMethod(PyObject* owner, TObjType* self, PyObject* args, PyObject* kwargs, const TString& name) const { + const TCallerList* lst = Callers.FindPtr(name); + if (!lst) + return nullptr; + for (const auto& caller : *lst) { + PyObject* res = nullptr; + PyErr_Clear(); + if (caller->CallMethod(owner, self, args, kwargs, res)) + return res; + } + return nullptr; + } + + bool HasMethod(PyObject* owner, TObjType* self, const TString& name) const { + const TCallerList* lst = Callers.FindPtr(name); + if (!lst) + return false; + for (const auto& caller : *lst) { + if (caller->HasMethod(owner, self, name, HiddenAttrNames)) + return true; + } + return false; + } + + void GetMethodsNames(PyObject* owner, TObjType* self, TVector<TString>& resultNames) const { + for (const auto& it : Callers) { + if (HasMethod(owner, self, it.first) && !HiddenAttrNames.contains(it.first)) + resultNames.push_back(it.first); + } + } + + void GetAllMethodsNames(TVector<TString>& resultNames) const { + for (const auto& it : Callers) { + resultNames.push_back(it.first); + } + } + + void GetPropertiesNames(PyObject*, TObjType* self, TVector<TString>& resultNames) const { + const TCallerList* lst = Callers.FindPtr("IsA"); + if (!lst) + return; + for (const auto& caller : *lst) { + TIsACaller<TObjType>* isACaller = dynamic_cast<TIsACaller<TObjType>*>(caller.Get()); + if (isACaller) { + resultNames = isACaller->GetPropertiesNames(self); + return; + } + } + } + }; + + template <typename TObjType> + class TIsACaller: public TBaseMethodCaller<TObjType> { + private: + class TIsAChecker { + public: + virtual ~TIsAChecker() { + } + virtual bool Check(const TObjType* obj) const = 0; + }; + + template <typename TConcrete> + class TIsAConcreteChecker: public TIsAChecker { + public: + bool Check(const TObjType* obj) const override { + return dynamic_cast<const TConcrete*>(obj) != nullptr; + } + }; + + typedef TSimpleSharedPtr<TIsAChecker> TCheckerPtr; + typedef TMap<TString, TCheckerPtr> TCheckersMap; + + TCheckersMap Checkers; + + bool Check(const TString& name, const TObjType* obj) const { + const TCheckerPtr* checker = Checkers.FindPtr(name); + if (!checker) { + PyErr_Format(PyExc_KeyError, "unknown class name: %s", name.data()); + return false; + } + return (*checker)->Check(obj); + } + + protected: + TIsACaller() { + } + + template <typename TConcrete> + void AddChecker(const TString& name) { + Checkers[name] = new TIsAConcreteChecker<TConcrete>; + } + + public: + bool CallMethod(PyObject*, TObjType* self, PyObject* args, PyObject*, PyObject*& res) const override { + if (args == nullptr || !PyTuple_Check(args)) + return false; + size_t cnt = PyTuple_Size(args); + bool result = true; + for (size_t i = 0; i < cnt; ++i) { + result = result && Check( +#if PY_MAJOR_VERSION >= 3 + PyUnicode_AsUTF8( +#else + PyString_AsString( +#endif + PyTuple_GetItem(args, i)), self); + } + if (PyErr_Occurred()) { + return false; + } + res = BuildPyObject(result); + return true; + } + + TVector<TString> GetPropertiesNames(const TObjType* obj) const { + TVector<TString> names; + + for (const auto& it : Checkers) { + if (it.second->Check(obj)) { + names.push_back(it.first); + } + } + + return names; + } + }; + + template <typename TObjType> + class TGenericMethodCaller: public TBaseMethodCaller<TObjType> { + private: + TString AttrName; + + public: + TGenericMethodCaller(const TString& attrName) + : AttrName(attrName) + { + } + + bool CallMethod(PyObject* obj, TObjType*, PyObject* args, PyObject*, PyObject*& res) const override { + auto str = NameFromString(AttrName); + PyObject* attr = PyObject_GenericGetAttr(obj, str.Get()); + if (!attr) + ythrow yexception() << "Can't get generic attribute '" << AttrName << "'"; + res = PyObject_CallObject(attr, args); + return res != nullptr; + } + }; + + + template <typename TObjType, typename TSubObject> + class TSubObjectChecker: public TBaseMethodCaller<TObjType> { + public: + ~TSubObjectChecker() override { + } + + bool HasMethod(PyObject*, TObjType* self, const TString&, const TSet<TString>&) override { + return dynamic_cast<const TSubObject*>(self) != nullptr; + } + }; + + template <typename TFunctor, typename Tuple, typename ResType, typename=std::enable_if_t<!std::is_same_v<ResType, void>>> + void ApplyFunctor(TFunctor functor, Tuple resultArgs, PyObject*& res) { + res = BuildPyObject(std::move(Apply(functor, resultArgs))); + } + + template <typename TFunctor, typename Tuple, typename ResType, typename=std::enable_if_t<std::is_same_v<ResType, void>>, typename=void> + void ApplyFunctor(TFunctor functor, Tuple resultArgs, PyObject*& res) { + Py_INCREF(Py_None); + res = Py_None; + Apply(functor, resultArgs); + } + + template <typename TObjType, typename TResType, typename... Args> + class TFunctorCaller: public TBaseMethodCaller<TObjType> { + using TFunctor = std::function<TResType(TObjType&,Args...)>; + TFunctor Functor; + public: + explicit TFunctorCaller(TFunctor functor): + Functor(functor){} + + bool CallMethod(PyObject*, TObjType* self, PyObject* args, PyObject*, PyObject*& res) const { + auto methodArgsTuple = GetArguments<Args...>(args); + auto resultArgs = std::tuple_cat(std::tie(*self), methodArgsTuple); + ApplyFunctor<TFunctor, decltype(resultArgs), TResType>(Functor, resultArgs, res); + return true; + } + }; + + template <typename TObjType, typename TRealType> + class TGetStateCaller: public TSubObjectChecker<TObjType, TRealType> { + protected: + TPyObjectPtr AddFromCaller(PyObject* obj, const TString& methodName) const { + PyObject* res = PyObject_CallMethod(obj, const_cast<char*>(methodName.c_str()), const_cast<char*>("")); + if (!res) { + PyErr_Clear(); + return TPyObjectPtr(Py_None); + } + return TPyObjectPtr(res, true); + } + + void GetStandartAttrsDictionary(PyObject* obj, TRealType*, TMap<TString, TPyObjectPtr>& dict) const { + TPyObjectPtr attrsDict(PyObject_GetAttrString(obj, "__dict__"), true); + TMap<TString, TPyObjectPtr> attrs; + if (!FromPyObject(attrsDict.Get(), attrs)) + ythrow yexception() << "Can't get '__dict__' attribute"; + dict.insert(attrs.begin(), attrs.end()); + } + + virtual void GetAttrsDictionary(PyObject* obj, TRealType* self, TMap<TString, TPyObjectPtr>& dict) const = 0; + + public: + bool CallMethod(PyObject* obj, TObjType* self, PyObject* args, PyObject*, PyObject*& res) const override { + if (!ExtractArgs(args)) + ythrow yexception() << "Can't parse arguments: it should be none"; + TRealType* rself = dynamic_cast<TRealType*>(self); + if (!rself) + return false; + TMap<TString, TPyObjectPtr> dict; + GetAttrsDictionary(obj, rself, dict); + res = BuildPyObject(dict); + return true; + } + }; + + template <typename TObjType, typename TRealType> + class TSetStateCaller: public TSubObjectChecker<TObjType, TRealType> { + protected: + void SetStandartAttrsDictionary(PyObject* obj, TRealType*, TMap<TString, TPyObjectPtr>& dict) const { + TPyObjectPtr value(BuildPyObject(dict), true); + PyObject_SetAttrString(obj, "__dict__", value.Get()); + } + + virtual void SetAttrsDictionary(PyObject* obj, TRealType* self, TMap<TString, TPyObjectPtr>& dict) const = 0; + + public: + bool CallMethod(PyObject* obj, TObjType* self, PyObject* args, PyObject*, PyObject*& res) const override { + TMap<TString, TPyObjectPtr> dict; + if (!ExtractArgs(args, dict)) + ythrow yexception() << "Can't parse arguments: it should be one dictionary"; + TRealType* rself = dynamic_cast<TRealType*>(self); + if (!rself) + return false; + SetAttrsDictionary(obj, rself, dict); + Py_INCREF(Py_None); + res = Py_None; + return true; + } + }; + + template <typename TObjType, typename TResult, typename TSubObject, typename TMethod, typename... Args> + class TAnyParameterMethodCaller: public TSubObjectChecker<TObjType, TSubObject> { + private: + TMethod Method; + + public: + TAnyParameterMethodCaller(TMethod method) + : Method(method) + { + } + + public: + bool CallMethod(PyObject*, TObjType* self, PyObject* args, PyObject*, PyObject*& res) const override { + TSubObject* sub = dynamic_cast<TSubObject*>(self); + if (sub == nullptr) + return false; + if (args && (!PyTuple_Check(args) || PyTuple_Size(args) != TFunctionArgs<TMethod>::Length)) { + //ythrow yexception() << "Method takes " << (size_t)(TFunctionArgs<TMethod>::Length) << " arguments, " << PyTuple_Size(args) << " provided"; + return false; + } + + try { + class Applicant { + public: + TResult operator()(Args... theArgs) { + return (Sub->*Method)(theArgs...); + } + TSubObject* Sub; + TMethod Method; + }; + res = BuildPyObject(std::move(Apply(Applicant{sub, Method}, GetArguments<Args...>(args)))); + } catch (cast_exception) { + return false; + } catch (...) { + if (PyExc_StopIteration == PyErr_Occurred()) { + // NB: it's replacement for geo_boost::python::throw_error_already_set(); + return true; + } + PyErr_SetString(PyExc_RuntimeError, CurrentExceptionMessage().data()); + return true; + } + + return true; + } + }; + + template <typename TObjType, typename TSubObject, typename TMethod, typename... Args> + class TAnyParameterMethodCaller<TObjType, void, TSubObject, TMethod, Args...>: public TSubObjectChecker<TObjType, TSubObject> { + private: + TMethod Method; + + public: + TAnyParameterMethodCaller(TMethod method) + : Method(method) + { + } + + public: + bool CallMethod(PyObject*, TObjType* self, PyObject* args, PyObject*, PyObject*& res) const override { + TSubObject* sub = dynamic_cast<TSubObject*>(self); + if (sub == nullptr) { + return false; + } + if (args && (!PyTuple_Check(args) || PyTuple_Size(args) != TFunctionArgs<TMethod>::Length)) { + return false; + } + + try { + class Applicant { + public: + void operator()(Args... theArgs) { + (Sub->*Method)(theArgs...); + } + TSubObject* Sub; + TMethod Method; + }; + + Apply(Applicant{sub, Method}, GetArguments<Args...>(args)); + + Py_INCREF(Py_None); + res = Py_None; + } catch (cast_exception) { + return false; + } catch (...) { + PyErr_SetString(PyExc_RuntimeError, CurrentExceptionMessage().data()); + return true; + } + + return true; + } + }; + + template <typename TResult, typename TSubObject, typename... Args> + struct TConstTraits { + typedef TResult (TSubObject::*TMethod)(Args... args) const; + }; + + template <typename TResult, typename TSubObject, typename... Args> + struct TNonConstTraits { + typedef TResult (TSubObject::*TMethod)(Args... args); + }; + + template <typename TObjType, typename TResult, typename TSubObject, typename TMethod, typename... Args> + class TConstMethodCaller: public TAnyParameterMethodCaller<TObjType, TResult, const TSubObject, typename TConstTraits<TResult, TSubObject, Args...>::TMethod, Args...> { + public: + TConstMethodCaller(typename TConstTraits<TResult, TSubObject, Args...>::TMethod method) + : TAnyParameterMethodCaller<TObjType, TResult, const TSubObject, typename TConstTraits<TResult, TSubObject, Args...>::TMethod, Args...>(method) + { + } + }; + + template <typename TObjType, typename TResult, typename TSubObject, typename... Args> + TSimpleSharedPtr<TBaseMethodCaller<TObjType>> CreateConstMethodCaller(TResult (TSubObject::*method)(Args...) const) { + return new TConstMethodCaller<TObjType, TResult, TSubObject, TResult (TSubObject::*)(Args...) const, Args...>(method); + } + + template <typename TObjType, typename TResType, typename... Args> + TSimpleSharedPtr<TBaseMethodCaller<TObjType>> CreateFunctorCaller(std::function<TResType(TObjType&, Args...)> functor) { + return new TFunctorCaller<TObjType, TResType, Args...>(functor); + } + + template <typename TObjType, typename TResult, typename TSubObject, typename TMethod, typename... Args> + class TMethodCaller: public TAnyParameterMethodCaller<TObjType, TResult, TSubObject, typename TNonConstTraits<TResult, TSubObject, Args...>::TMethod, Args...> { + public: + TMethodCaller(typename TNonConstTraits<TResult, TSubObject, Args...>::TMethod method) + : TAnyParameterMethodCaller<TObjType, TResult, TSubObject, typename TNonConstTraits<TResult, TSubObject, Args...>::TMethod, Args...>(method) + { + } + }; + + template <typename TObjType, typename TResult, typename TSubObject, typename... Args> + TSimpleSharedPtr<TBaseMethodCaller<TObjType>> CreateMethodCaller(TResult (TSubObject::*method)(Args...)) { + return new TMethodCaller<TObjType, TResult, TSubObject, TResult (TSubObject::*)(Args...), Args...>(method); + } + +} diff --git a/library/cpp/pybind/module.cpp b/library/cpp/pybind/module.cpp new file mode 100644 index 0000000000..b7b003d3b0 --- /dev/null +++ b/library/cpp/pybind/module.cpp @@ -0,0 +1,72 @@ +#include "module.h" +#include "ptr.h" + +#include <util/generic/adaptor.h> + +namespace NPyBind { + +#if PY_MAJOR_VERSION >= 3 + namespace NPrivate { + struct TFinCallBacksHolder { + static TVector<TFinalizationCallBack>& GetCallBacks() { + static TVector<TFinalizationCallBack> res; + return res; + } + }; + + TAtExitRegistrar::TAtExitRegistrar(TPyObjectPtr module) { + TPyObjectPtr atExitModuleName(Py_BuildValue("s", "atexit"), true); + TPyObjectPtr atExitModule(PyImport_Import(atExitModuleName.Get())); + Y_VERIFY(atExitModule); + TPyObjectPtr finalizerFunc(PyObject_GetAttrString(module.Get(), "finalizer"), true); + Y_VERIFY(finalizerFunc); + TPyObjectPtr registerName(Py_BuildValue("s", "register"), true); + PyObject_CallMethodObjArgs(atExitModule.Get(), registerName.Get(), finalizerFunc.Get(), nullptr); + } + + TPyBindModuleRegistrar::TPyBindModuleRegistrar() { + TPyObjectPtr modules(PySys_GetObject("modules")); + Y_ENSURE(modules.Get()); + if (Module = NPrivate::CreatePyBindModule()) { + Y_VERIFY(0 == PyDict_SetItemString(modules.Get(), "pybind", Module.RefGet())); + } + AddFinalizationCallBack([this]() { + auto ptr = Module; + Y_UNUSED(ptr); + TPyObjectPtr modules(PySys_GetObject("modules")); + Y_ENSURE(modules.Get()); + TPyObjectPtr pyBindName(Py_BuildValue("s", "pybind")); + if (PyDict_Contains(modules.Get(), pyBindName.Get()) == 1) { + Y_VERIFY(0==PyDict_DelItemString(modules.Get(), "pybind")); + } + if (Module) { + //We have to untrack the module because some refs from him refers to gc-leaked errors + //see exceptions.cpp fore more info + PyObject_GC_UnTrack(Module.Get()); + Module.Drop(); + } + }); + } + + void AddFinalizationCallBack(TFinalizationCallBack callback) { + TFinCallBacksHolder::GetCallBacks().push_back(callback); + } + + int FinalizeAll() { + for (auto callback: Reversed(NPrivate::TFinCallBacksHolder::GetCallBacks())) { + callback(); + } + return 0; + } + } +#endif + + + TModuleHolder::TModuleHolder() + : Methods(1, new TVector<TMethodDef>) + { +#if PY_MAJOR_VERSION >= 3 + AddModuleMethod<TModuleMethodCaller<decltype(&NPrivate::FinalizeAll), &NPrivate::FinalizeAll>::Call>("finalizer"); +#endif + } +}//NPyBind diff --git a/library/cpp/pybind/module.h b/library/cpp/pybind/module.h new file mode 100644 index 0000000000..41dcb4dfec --- /dev/null +++ b/library/cpp/pybind/module.h @@ -0,0 +1,176 @@ +#pragma once + +#define PY_SSIZE_T_CLEAN +#include <Python.h> +#include "ptr.h" +#include "cast.h" +#include "exceptions.h" + +#include <util/generic/function.h> + +namespace NPyBind { +#if PY_MAJOR_VERSION >= 3 + namespace NPrivate { + using TFinalizationCallBack = std::function<void()>; + void AddFinalizationCallBack(TFinalizationCallBack); + class TAtExitRegistrar: private TNonCopyable { + TAtExitRegistrar(TPyObjectPtr module); + public: + static void Instantiate(TPyObjectPtr module) { + static TAtExitRegistrar registrar(module); + Y_UNUSED(registrar); + } + }; + + class TPyBindModuleRegistrar: private TNonCopyable { + TPyBindModuleRegistrar(); + TPyObjectPtr Module; + public: + static void Instantiate() { + static TPyBindModuleRegistrar registrar; + Y_UNUSED(registrar); + } + }; + } //NPrivate +#endif + + class TModuleHolder { + private: + TModuleHolder(const TModuleHolder&); + TModuleHolder& operator=(const TModuleHolder&); + + TModuleHolder(); + private: + typedef PyCFunction TModuleMethod; +#if PY_MAJOR_VERSION >= 3 + typedef PyObject* (*TModuleInitFunc)(); +#else + typedef void (*TModuleInitFunc)(); +#endif + + struct TMethodDef { + TString Name; + TModuleMethod Method; + TString Description; + int Flags; + + TMethodDef(const TString& name, TModuleMethod method, const TString& descr, int flags) + : Name(name) + , Method(method) + , Description(descr) + , Flags(flags) + { + } + + operator PyMethodDef() const { + PyMethodDef cur = {Name.c_str(), Method, Flags, Description.c_str()}; + return cur; + } + }; + + typedef TSimpleSharedPtr<TVector<TMethodDef>> TMethodDefVecPtr; + typedef TSimpleSharedPtr<TVector<PyMethodDef>> TPyMethodDefVecPtr; + + TVector<TMethodDefVecPtr> Methods; + TVector<TPyMethodDefVecPtr> Defs; +#if PY_MAJOR_VERSION >= 3 + //because the md_name will leak otherwise + class TPyModuleDefWithName { + PyModuleDef Def; + TString Name; + public: + explicit TPyModuleDefWithName(TString name, TPyMethodDefVecPtr moduleDefs) + : Name(std::move(name)) + { + Def = PyModuleDef{ + PyModuleDef_HEAD_INIT, + Name.c_str(), + nullptr, + -1, + moduleDefs->data(), + nullptr, nullptr, nullptr, nullptr + }; + } + PyModuleDef* GetDefPtr() { + return &Def; + } + + }; + TVector<TSimpleSharedPtr<TPyModuleDefWithName>> ModuleDefs; +#endif + + template <TModuleMethod method> + static PyObject* MethodWrapper(PyObject* obj, PyObject* args) { + try { + PyObject* res = method(obj, args); + if (!res && !PyErr_Occurred()) + ythrow yexception() << "\nModule method exited with NULL, but didn't set Error.\n Options:\n -- Return correct value or None;\n -- Set python exception;\n -- Throw c++ exception."; + return res; + } catch (const std::exception& ex) { + PyErr_SetString(TExceptionsHolder::Instance().ToPyException(ex).Get(), ex.what()); + } catch (...) { + PyErr_SetString(PyExc_RuntimeError, "Unknown error occurred while trying to call module method"); + } + return nullptr; + } + + public: + static TModuleHolder& Instance() { + static TModuleHolder Holder; + return Holder; + } + + void ImportModule(TPyObjectPtr module, const char* const name, TModuleInitFunc initFunc) { + PyImport_AppendInittab(const_cast<char*>(name), initFunc); + TPyObjectPtr importedModule(PyImport_ImportModule(name), true); + PyModule_AddObject(module.Get(), name, importedModule.Get()); + } + + template <TModuleMethod method> + void AddModuleMethod(const TString& name, const TString& descr = "") { + Methods.back()->push_back(TMethodDef(name, MethodWrapper<method>, descr, METH_VARARGS)); + } + + TPyObjectPtr InitModule(const TString& name) { + Defs.push_back(new TVector<PyMethodDef>(Methods.back()->begin(), Methods.back()->end())); + PyMethodDef blank = {nullptr, nullptr, 0, nullptr}; + Defs.back()->push_back(blank); +#if PY_MAJOR_VERSION >= 3 + ModuleDefs.push_back(MakeSimpleShared<TPyModuleDefWithName>(name, Defs.back())); + TPyObjectPtr res(PyModule_Create(ModuleDefs.back()->GetDefPtr())); + NPrivate::TAtExitRegistrar::Instantiate(res); + NPrivate::TPyBindModuleRegistrar::Instantiate(); +#else + TPyObjectPtr res(Py_InitModule(name.c_str(), &(Defs.back()->at(0)))); +#endif + Methods.push_back(new TVector<TMethodDef>); + return res; + } + }; + + template <typename TMethodSignature, TMethodSignature method> + class TModuleMethodCaller { + private: + template <typename TResult, typename... Args> + struct TCaller { + static PyObject* Call(PyObject* args) { + return BuildPyObject(Apply(method, GetArguments<Args...>(args))); + } + }; + + template <typename TResult, typename... Args> + static PyObject* InternalCall(TResult (*)(Args...), PyObject* args) { + return BuildPyObject(Apply(method, GetArguments<Args...>(args))); + } + + public: + static PyObject* Call(PyObject*, PyObject* args) { + if (args && (!PyTuple_Check(args) || PyTuple_Size(args) != TFunctionArgs<TMethodSignature>::Length)) { + ythrow yexception() << "Method takes " << (size_t)(TFunctionArgs<TMethodSignature>::Length) << " arguments, " << PyTuple_Size(args) << " provided"; + } + + return InternalCall(method, args); + } + }; + +} diff --git a/library/cpp/pybind/pod.cpp b/library/cpp/pybind/pod.cpp new file mode 100644 index 0000000000..3cf030e537 --- /dev/null +++ b/library/cpp/pybind/pod.cpp @@ -0,0 +1,18 @@ +#include "pod.h" + +namespace NPyBind { + class TPODAttrGetter: public TBaseAttrGetter<TPOD> { + public: + bool GetAttr(PyObject*, const TPOD& self, const TString& attr, PyObject*& res) const override { + res = self.GetAttr(attr.c_str()); + return res != nullptr; + } + }; + + TPODTraits::TPODTraits() + : MyParent("TPOD", "simple struct") + { + AddGetter("", new TPODAttrGetter); + } + +} diff --git a/library/cpp/pybind/pod.h b/library/cpp/pybind/pod.h new file mode 100644 index 0000000000..90165fdbec --- /dev/null +++ b/library/cpp/pybind/pod.h @@ -0,0 +1,53 @@ +#pragma once + +#define PY_SSIZE_T_CLEAN +#include <Python.h> +#include "attr.h" +#include "typedesc.h" + +namespace NPyBind { + struct TPOD { + TPyObjectPtr Dict; + + TPOD() + : Dict(PyDict_New(), true) + { + } + bool SetAttr(const char* name, PyObject* value) { + return PyDict_SetItemString(Dict.Get(), name, value) == 0; + } + PyObject* GetAttr(const char* name) const { + PyObject* res = PyDict_GetItemString(Dict.Get(), name); + Py_XINCREF(res); + return res; + } + }; + + class TPODTraits: public NPyBind::TPythonType<TPOD, TPOD, TPODTraits> { + private: + typedef TPythonType<TPOD, TPOD, TPODTraits> MyParent; + friend class TPythonType<TPOD, TPOD, TPODTraits>; + TPODTraits(); + + public: + static TPOD* GetObject(TPOD& obj) { + return &obj; + } + }; + + template <> + inline bool FromPyObject<TPOD*>(PyObject* obj, TPOD*& res) { + res = TPODTraits::CastToObject(obj); + if (res == nullptr) + return false; + return true; + } + template <> + inline bool FromPyObject<const TPOD*>(PyObject* obj, const TPOD*& res) { + res = TPODTraits::CastToObject(obj); + if (res == nullptr) + return false; + return true; + } + +} diff --git a/library/cpp/pybind/ptr.h b/library/cpp/pybind/ptr.h new file mode 100644 index 0000000000..e136736690 --- /dev/null +++ b/library/cpp/pybind/ptr.h @@ -0,0 +1,51 @@ +#pragma once + +#define PY_SSIZE_T_CLEAN +#include <Python.h> +#include <util/generic/ptr.h> + +namespace NPyBind { + template <class T> + class TPythonIntrusivePtrOps { + public: + static inline void Ref(T* t) noexcept { + Py_XINCREF(t); + } + + static inline void UnRef(T* t) noexcept { + Py_XDECREF(t); + } + + static inline void DecRef(T* t) noexcept { + Py_XDECREF(t); + } + }; + + class TPyObjectPtr: public TIntrusivePtr<PyObject, TPythonIntrusivePtrOps<PyObject>> { + private: + typedef TIntrusivePtr<PyObject, TPythonIntrusivePtrOps<PyObject>> TParent; + typedef TPythonIntrusivePtrOps<PyObject> TOps; + + public: + inline TPyObjectPtr() noexcept { + } + + inline explicit TPyObjectPtr(PyObject* obj) noexcept + : TParent(obj) + { + } + + inline TPyObjectPtr(PyObject* obj, bool unref) noexcept + : TParent(obj) + { + if (unref) + TOps::UnRef(TParent::Get()); + } + + inline PyObject* RefGet() { + TOps::Ref(TParent::Get()); + return TParent::Get(); + } + }; + +} diff --git a/library/cpp/pybind/typeattrs.h b/library/cpp/pybind/typeattrs.h new file mode 100644 index 0000000000..a906b9ec2b --- /dev/null +++ b/library/cpp/pybind/typeattrs.h @@ -0,0 +1,368 @@ +#pragma once + +#include "ptr.h" +#include "cast.h" +#include "attr.h" +#include "method.h" + +#include <util/generic/vector.h> + +namespace NPyBind { + template <typename TObject> + class TPythonTypeAttributes { + private: + TAttrGetters<TObject> AttrGetters; + TAttrSetters<TObject> AttrSetters; + TMethodCallers<TObject> MethodCallers; + + class TGetAttrsNamesCaller; + class TGetMethodsNamesCaller; + class TGetAllNamesCaller; + class TGetPropertiesNamesCaller; + class TDictAttrGetter; + class TDictAttrSetter; + class TGetAttributeMethodCaller; + class TSetAttrMethodCaller; + class TGetStrReprMethodCaller; + class TReduceMethodCaller; + class TBaseGetStateMethodCaller; + class TBaseSetStateMethodCaller; + + TPythonTypeAttributes(const TPythonTypeAttributes&); + TPythonTypeAttributes& operator=(const TPythonTypeAttributes&); + + static const TSet<TString> HiddenAttrNames; + + typedef PyObject* (*GetAttrFunction)(PyObject*, char*); + typedef int (*SetAttrFunction)(PyObject*, char*, PyObject*); + GetAttrFunction GetAttr; + SetAttrFunction SetAttr; + + public: + typedef TSimpleSharedPtr<TBaseAttrGetter<TObject>> TGetterPtr; + typedef TSimpleSharedPtr<TBaseAttrSetter<TObject>> TSetterPtr; + typedef TSimpleSharedPtr<TBaseMethodCaller<TObject>> TCallerPtr; + + TPythonTypeAttributes(GetAttrFunction getAttr, SetAttrFunction setAttr) + : AttrGetters(HiddenAttrNames) + , MethodCallers(HiddenAttrNames) + , GetAttr(getAttr) + , SetAttr(setAttr) + { + } + + void InitCommonAttributes() { + // attributes + AddGetter("__dict__", new TDictAttrGetter(AttrGetters)); + AddSetter("__dict__", new TDictAttrSetter(AttrSetters)); + + // methods + AddCaller("GetAttrsNames", new TGetAttrsNamesCaller(AttrGetters)); + AddCaller("GetMethodsNames", new TGetMethodsNamesCaller(MethodCallers)); + AddCaller("GetAllNames", new TGetAllNamesCaller(AttrGetters, MethodCallers)); + AddCaller("GetPropertiesNames", new TGetPropertiesNamesCaller(MethodCallers)); + AddCaller("__getattribute__", new TGetAttributeMethodCaller(GetAttr)); + AddCaller("__setattr__", new TSetAttrMethodCaller(SetAttr)); + AddCaller("__str__", new TGetStrReprMethodCaller("__str__")); + AddCaller("__repr__", new TGetStrReprMethodCaller("__repr__")); + AddCaller("__reduce_ex__", new TReduceMethodCaller); + AddCaller("__reduce__", new TReduceMethodCaller); + AddCaller("__getstate__", new TBaseGetStateMethodCaller); + AddCaller("__setstate__", new TBaseSetStateMethodCaller); + + // generics + AddGetter("__class__", new TGenericAttrGetter<TObject>("__class__")); + AddGetter("__doc__", new TGenericAttrGetter<TObject>("__doc__")); + AddCaller("__sizeof__", new TGenericMethodCaller<TObject>("__sizeof__")); + AddCaller("__hash__", new TGenericMethodCaller<TObject>("__hash__")); + } + + void AddGetter(const TString& attr, TGetterPtr getter) { + AttrGetters.AddGetter(attr, getter); + } + + void AddSetter(const TString& attr, TSetterPtr setter) { + AttrSetters.AddSetter(attr, setter); + } + + void AddCaller(const TString& name, TCallerPtr caller) { + MethodCallers.AddCaller(name, caller); + } + + const TAttrGetters<TObject>& GetAttrGetters() const { + return AttrGetters; + } + + TAttrSetters<TObject>& GetAttrSetters() { + return AttrSetters; + } + + const TMethodCallers<TObject>& GetMethodCallers() const { + return MethodCallers; + } + + const TSet<TString>& GetHiddenAttrs() const { + return HiddenAttrNames; + } + }; + + template <typename TObjType> + class TPythonTypeAttributes<TObjType>::TGetAttrsNamesCaller: public TBaseMethodCaller<TObjType> { + private: + const TAttrGetters<TObjType>& AttrGetters; + + public: + TGetAttrsNamesCaller(const TAttrGetters<TObjType>& getters) + : AttrGetters(getters) + { + } + + bool CallMethod(PyObject* owner, TObjType* self, PyObject* args, PyObject*, PyObject*& res) const override { + if (!ExtractArgs(args)) + ythrow yexception() << "Could not parse args for GetAttrsNames() - it should be none"; + TVector<TString> names; + AttrGetters.GetAttrsNames(owner, *self, names); + res = BuildPyObject(names); + return (res != nullptr); + } + }; + + template <typename TObjType> + class TPythonTypeAttributes<TObjType>::TGetMethodsNamesCaller: public TBaseMethodCaller<TObjType> { + private: + const TMethodCallers<TObjType>& MethodCallers; + + public: + TGetMethodsNamesCaller(const TMethodCallers<TObjType>& callers) + : MethodCallers(callers) + { + } + + bool CallMethod(PyObject* owner, TObjType* self, PyObject* args, PyObject*, PyObject*& res) const override { + if (!ExtractArgs(args)) + ythrow yexception() << "Could not parse args for GetMethodsNames() - it should be none"; + TVector<TString> names; + MethodCallers.GetMethodsNames(owner, self, names); + res = BuildPyObject(names); + return (res != nullptr); + } + }; + + template <typename TObjType> + class TPythonTypeAttributes<TObjType>::TGetAllNamesCaller: public TBaseMethodCaller<TObjType> { + private: + const TAttrGetters<TObjType>& AttrGetters; + const TMethodCallers<TObjType>& MethodCallers; + + public: + TGetAllNamesCaller(const TAttrGetters<TObjType>& getters, + const TMethodCallers<TObjType>& callers) + : AttrGetters(getters) + , MethodCallers(callers) + { + } + + bool CallMethod(PyObject* owner, TObjType* self, PyObject* args, PyObject*, PyObject*& res) const override { + if (!ExtractArgs(args)) + ythrow yexception() << "Could not parse args for GetAllNames() - it should be none"; + TVector<TString> names; + AttrGetters.GetAttrsNames(owner, *self, names); + MethodCallers.GetMethodsNames(owner, self, names); + res = BuildPyObject(names); + return (res != nullptr); + } + }; + + template <typename TObjType> + class TPythonTypeAttributes<TObjType>::TGetPropertiesNamesCaller: public TBaseMethodCaller<TObjType> { + private: + const TMethodCallers<TObjType>& MethodCallers; + + public: + TGetPropertiesNamesCaller(const TMethodCallers<TObjType>& callers) + : MethodCallers(callers) + { + } + + public: + bool CallMethod(PyObject* obj, TObjType* self, PyObject* args, PyObject*, PyObject*& res) const override { + if (!ExtractArgs(args)) + return false; + + TVector<TString> names; + MethodCallers.GetPropertiesNames(obj, self, names); + res = BuildPyObject(names); + return (res != nullptr); + } + }; + + template <typename TObjType> + class TPythonTypeAttributes<TObjType>::TDictAttrGetter: public TBaseAttrGetter<TObjType> { + private: + TAttrGetters<TObjType>& AttrGetters; + + public: + TDictAttrGetter(TAttrGetters<TObjType>& getters) + : AttrGetters(getters) + { + } + + bool GetAttr(PyObject* owner, const TObjType& self, const TString&, PyObject*& res) const override { + TMap<TString, PyObject*> dict; + AttrGetters.GetAttrsDictionary(owner, self, dict); + res = BuildPyObject(dict); + return (res != nullptr); + } + }; + + template <typename TObjType> + class TPythonTypeAttributes<TObjType>::TDictAttrSetter: public TBaseAttrSetter<TObjType> { + private: + TAttrSetters<TObjType>& AttrSetters; + + public: + TDictAttrSetter(TAttrSetters<TObjType>& setters) + : AttrSetters(setters) + { + } + + bool SetAttr(PyObject* owner, TObjType& self, const TString&, PyObject* val) override { + TMap<TString, PyObject*> dict; + if (!FromPyObject(val, dict)) + ythrow yexception() << "'__dict__' should be set to dictionary"; + if (!AttrSetters.SetAttrDictionary(owner, self, dict)) + return false; + return true; + } + }; + + template <typename TObjType> + class TPythonTypeAttributes<TObjType>::TGetAttributeMethodCaller: public TBaseMethodCaller<TObjType> { + private: + GetAttrFunction GetAttr; + + public: + TGetAttributeMethodCaller(GetAttrFunction getAttr) + : GetAttr(getAttr) + { + } + + bool CallMethod(PyObject* owner, TObjType*, PyObject* args, PyObject*, PyObject*& res) const override { + TString attrName; + if (!ExtractArgs(args, attrName)) + ythrow yexception() << "Could not parse args for '__getattribute__' - it should be one string"; + res = GetAttr(owner, const_cast<char*>(attrName.c_str())); + if (!res) + // Error already set + return false; + return true; + } + }; + + template <typename TObjType> + class TPythonTypeAttributes<TObjType>::TSetAttrMethodCaller: public TBaseMethodCaller<TObjType> { + private: + SetAttrFunction SetAttr; + + public: + TSetAttrMethodCaller(SetAttrFunction setAttr) + : SetAttr(setAttr) + { + } + + bool CallMethod(PyObject* owner, TObjType*, PyObject* args, PyObject*, PyObject*& res) const override { + TString attrName; + TPyObjectPtr value; + if (!ExtractArgs(args, attrName, value)) + ythrow yexception() << "Could not parse args for '__setattr__' - it should be one string and value"; + Py_INCREF(Py_None); + res = Py_None; + if (-1 == SetAttr(owner, const_cast<char*>(attrName.c_str()), value.Get())) + // Error already set + return false; + return true; + } + }; + + template <typename TObjType> + class TPythonTypeAttributes<TObjType>::TGetStrReprMethodCaller: public TBaseMethodCaller<TObjType> { + private: + TString MethodName; + + private: + const TString GetFullName(PyObject* obj) const { + TString module, name; + TPyObjectPtr type(PyObject_Type(obj), true); + if (!FromPyObject(PyObject_GetAttrString(type.Get(), "__module__"), module) || !FromPyObject(PyObject_GetAttrString(type.Get(), "__name__"), name)) + ythrow yexception() << "Could not get name of object"; + return module + "." + name; + } + + public: + TGetStrReprMethodCaller(const TString& methodName) + : MethodName(methodName) + { + } + + bool CallMethod(PyObject* owner, TObjType*, PyObject* args, PyObject*, PyObject*& res) const override { + if (args && !ExtractArgs(args)) + ythrow yexception() << "Could not parse args for '" << MethodName << "'"; + TString message = TString("<") + GetFullName(owner) + " object>"; + res = ReturnString(message); + return (res != nullptr); + } + }; + + template <typename TObjType> + class TPythonTypeAttributes<TObjType>::TReduceMethodCaller: public TBaseMethodCaller<TObjType> { + public: + bool CallMethod(PyObject* owner, TObjType*, PyObject*, PyObject*, PyObject*& res) const override { + TPyObjectPtr tuple(PyTuple_New(3), true); + // First component: reconstructor + TPyObjectPtr pybindName(BuildPyObject("pybind"), true); + TPyObjectPtr mainModule(PyImport_Import(pybindName.Get()), true); + TPyObjectPtr recName(BuildPyObject("PyBindObjectReconstructor"), true); + TPyObjectPtr reconstructor(PyObject_GetAttr(mainModule.Get(), recName.Get()), true); + // Second component: arguments to rebuild object + TPyObjectPtr arguments(PyTuple_New(2), true); + TPyObjectPtr cl(PyObject_GetAttrString(owner, "__class__"), true); + PyTuple_SET_ITEM(arguments.Get(), 0, cl.RefGet()); + TPyObjectPtr props(PyObject_CallMethod(owner, const_cast<char*>("GetPropertiesNames"), nullptr), true); + PyTuple_SET_ITEM(arguments.Get(), 1, props.RefGet()); + // Third component: state to fill new object + TPyObjectPtr state(PyObject_CallMethod(owner, const_cast<char*>("__getstate__"), nullptr), true); + + PyTuple_SET_ITEM(tuple.Get(), 0, reconstructor.RefGet()); + PyTuple_SET_ITEM(tuple.Get(), 1, arguments.RefGet()); + PyTuple_SET_ITEM(tuple.Get(), 2, state.RefGet()); + res = tuple.RefGet(); + return (res != nullptr); + } + }; + + template <typename TObjType> + class TPythonTypeAttributes<TObjType>::TBaseGetStateMethodCaller: public TGetStateCaller<TObjType, TObjType> { + public: + void GetAttrsDictionary(PyObject* obj, TObjType* self, TMap<TString, TPyObjectPtr>& dict) const override { + this->GetStandartAttrsDictionary(obj, self, dict); + } + }; + + template <typename TObjType> + class TPythonTypeAttributes<TObjType>::TBaseSetStateMethodCaller: public TSetStateCaller<TObjType, TObjType> { + public: + void SetAttrsDictionary(PyObject* obj, TObjType* self, TMap<TString, TPyObjectPtr>& dict) const override { + this->SetStandartAttrsDictionary(obj, self, dict); + } + }; + + static const char* HiddenAttrStrings[] = { + "__dict__", "__class__", "__dir__", "__delattr__", "__doc__", "__format__", "__getattribute__", "__hash__", + "__init__", "__new__", "__reduce__", "__reduce_ex__", "__repr__", "__setattr__", "__sizeof__", "__str__", + "__subclasshook__", "__getstate__", "__setstate__", + "GetAttrsNames", "GetMethodsNames", "GetAllNames", "GetPropertiesNames"}; + + template <typename T> + const TSet<TString> TPythonTypeAttributes<T>::HiddenAttrNames(HiddenAttrStrings, std::end(HiddenAttrStrings)); + +} diff --git a/library/cpp/pybind/typedesc.cpp b/library/cpp/pybind/typedesc.cpp new file mode 100644 index 0000000000..75f39fd126 --- /dev/null +++ b/library/cpp/pybind/typedesc.cpp @@ -0,0 +1,79 @@ +#include "typedesc.h" + +#include <util/generic/singleton.h> + +static void RegisterJSONBridgeImpl() { + PyRun_SimpleString("import json\n" + "class PyBindEncoder(json.JSONEncoder):\n" + " def default(self, obj):\n" + " if isinstance(obj, bytes):\n" + " try:\n" + " return obj.decode()\n" + " except UnicodeDecodeError:\n" + " return obj.hex()\n" + " dct = None\n" + " if hasattr(obj, '__getstate__'):\n" + " dct = obj.__getstate__()\n" + " elif hasattr(obj, '__dict__'):\n" + " dct = obj.__dict__\n" + " if dct is None:\n" + " return json.JSONEncoder.default(self, obj)\n" + " if hasattr(obj, '__class__'):\n" + " if hasattr(obj.__class__, '__name__'):\n" + " dct['__name__'] = obj.__class__.__name__\n" + " if hasattr(obj.__class__, '__module__'):\n" + " dct['__module__'] = obj.__class__.__module__\n" + " if hasattr(obj, 'GetPropertiesNames'):\n" + " dct['__properties__'] = obj.GetPropertiesNames()\n" + " return dct"); + + PyRun_SimpleString("def PyBindObjectHook(dct):\n" + " if '__name__' in dct:\n" + " name = dct['__name__']\n" + " module = dct['__module__']\n" + " del dct['__name__']\n" + " del dct['__module__']\n" + " cls = getattr(__import__(module), name)\n" + " if '__properties__' in dct:\n" + " props = dct['__properties__']\n" + " del dct['__properties__']\n" + " if len(props) == 0:\n" + " return dct\n" + " instance = cls(__properties__ = props)\n" + " else:\n" + " instance = cls()\n" + " if hasattr(instance, '__setstate__'):\n" + " instance.__setstate__(dct)\n" + " elif hasattr(instance, '__dict__'):\n" + " instance.__dict__ = dct\n" + " else:\n" + " return dct\n" + " return instance\n" + " return dct"); + + PyRun_SimpleString("def json_dump(*args, **kwargs):\n" + " kwargs['cls'] = PyBindEncoder\n" + " return json.dump(*args, **kwargs)\n" + "def json_dumps(*args, **kwargs):\n" + " kwargs['cls'] = PyBindEncoder\n" + " return json.dumps(*args, **kwargs)"); + + PyRun_SimpleString("def json_load(*args, **kwargs):\n" + " kwargs['object_hook'] = PyBindObjectHook\n" + " return json.load(*args, **kwargs)\n" + "def json_loads(*args, **kwargs):\n" + " kwargs['object_hook'] = PyBindObjectHook\n" + " return json.loads(*args, **kwargs)"); +} + +namespace { + struct TJSONBridge { + TJSONBridge() { + RegisterJSONBridgeImpl(); + } + }; +} + +void NPyBind::RegisterJSONBridge() { + Singleton<TJSONBridge>(); +} diff --git a/library/cpp/pybind/typedesc.h b/library/cpp/pybind/typedesc.h new file mode 100644 index 0000000000..57eacb0f3a --- /dev/null +++ b/library/cpp/pybind/typedesc.h @@ -0,0 +1,545 @@ +#pragma once + +#define PY_SSIZE_T_CLEAN +#include <Python.h> +#include <structmember.h> + +#include "typeattrs.h" +#include "exceptions.h" +#include "module.h" + +namespace NPyBind { + void RegisterJSONBridge(); + + namespace NPrivate { + template <typename> + class TUnboundClosureHolder; + template <typename> + class TUnboundClosure; + } + + // TTraits should be derived from TPythonType + template <typename TObjectHolder, typename TObject, typename TTraits> + class TPythonType { + private: + TPythonType(const TPythonType&); + TPythonType& operator=(const TPythonType&); + + private: + typedef typename TPythonTypeAttributes<TObject>::TGetterPtr TGetterPtr; + typedef typename TPythonTypeAttributes<TObject>::TSetterPtr TSetterPtr; + typedef typename TPythonTypeAttributes<TObject>::TCallerPtr TCallerPtr; + + struct TProxy { + PyObject_HEAD + TObjectHolder* Holder; + }; + + static PyTypeObject PyType; + static PyMappingMethods MappingMethods; + static PyObject* PyTypeObjPtr; + protected: + static PyTypeObject* GetPyTypePtr() { + return &PyType; + } + private: + + TPythonTypeAttributes<TObject> Attributes; + + static int InitObject(PyObject* s, PyObject* args, PyObject* kwargs) { + try { + TProxy* self = reinterpret_cast<TProxy*>(s); + auto str = NameFromString("__properties__"); + if (kwargs && PyDict_Check(kwargs) && PyDict_Contains(kwargs, str.Get())) { + TPyObjectPtr props(PyDict_GetItem(kwargs, str.Get())); + TVector<TString> properties; + FromPyObject(props.Get(), properties); + self->Holder = TTraits::DoInitPureObject(properties); + } else { + self->Holder = (args || kwargs) ? TTraits::DoInitObject(args, kwargs) : nullptr; + } + if (PyErr_Occurred()) + return -1; + return 0; + } catch (const std::exception& ex) { + PyErr_SetString(TExceptionsHolder::Instance().ToPyException(ex).Get(), ex.what()); + } catch (...) { + PyErr_SetString(PyExc_RuntimeError, "Unknown error occurred while trying to init object"); + } + return -1; + } + + static void DeallocObject(TProxy* self) { + delete self->Holder; + Py_TYPE(self)->tp_free(reinterpret_cast<PyObject*>(self)); + } + + static PyObject* GetObjectAttr(PyObject* pyObj, char* attr); + static int SetObjectAttr(PyObject* pyObj, char* attr, PyObject* value); + static PyObject* GetStr(PyObject*); + static PyObject* GetRepr(PyObject*); + static PyObject* GetIter(PyObject*); + static PyObject* GetNext(PyObject*); + + // Fill class __dict__ with functions to make sure methods names will get to dir() + void FillClassDict() const { + TVector<TString> names; + Attributes.GetMethodCallers().GetAllMethodsNames(names); + for (const auto& name : names) { + TPyObjectPtr callable = NPrivate::TUnboundClosure<TObject>::Instance().CreatePyObject(new NPrivate::TUnboundClosureHolder<TObject>(&PyType, name)); + PyDict_SetItemString(PyType.tp_dict, name.c_str(), callable.Get()); + } + } + + void InitCommonAttributes() { + static bool was = false; + if (was) + return; + was = true; + Attributes.InitCommonAttributes(); + FillClassDict(); + } + + protected: + TPythonType(const char* pyTypeName, const char* typeDescr, PyTypeObject* parentType = nullptr) + : Attributes(GetObjectAttr, SetObjectAttr) + { + PyType.tp_name = pyTypeName; + PyType.tp_doc = typeDescr; + Py_INCREF(PyTypeObjPtr); + if (parentType) { + Py_INCREF(parentType); + PyType.tp_base = parentType; + } + PyType_Ready(&PyType); + + TExceptionsHolder::Instance(); + RegisterJSONBridge(); + + } + + ~TPythonType() { + } + + static TObjectHolder* DoInitObject(PyObject*, PyObject*) { + return nullptr; + } + + static TObjectHolder* DoInitPureObject(const TVector<TString>&) { + return nullptr; + } + + static void SetClosure(PyObject* (*call)(PyObject*, PyObject*, PyObject*)) { + PyType.tp_call = call; + } + + public: + void AddGetter(const TString& attr, TGetterPtr getter) { + Attributes.AddGetter(attr, getter); + } + + void AddSetter(const TString& attr, TSetterPtr setter) { + Attributes.AddSetter(attr, setter); + } + + void AddCaller(const TString& name, TCallerPtr caller) { + Attributes.AddCaller(name, caller); + if (name == "__iter__") { + PyType.tp_iter = GetIter; + } + if (name == "next") { + PyType.tp_iternext = GetNext; + } + } + + void SetIter(getiterfunc tp_iter) { + PyType.tp_iter = tp_iter; + } + + void SetIterNext(iternextfunc tp_iternext) { + PyType.tp_iternext = tp_iternext; + } + + void SetDestructor(destructor tp_dealloc) { + PyType.tp_dealloc = tp_dealloc; + } + + void SetLengthFunction(lenfunc mp_length) { + PyType.tp_as_mapping->mp_length = mp_length; + } + + void SetSubscriptFunction(binaryfunc mp_subscript) { + PyType.tp_as_mapping->mp_subscript = mp_subscript; + } + + void SetAssSubscriptFunction(objobjargproc mp_ass_subscript) { + PyType.tp_as_mapping->mp_ass_subscript = mp_ass_subscript; + } + + typedef TObject TObjectType; + + static TPythonType& Instance() { + static TTraits Traits; + Traits.InitCommonAttributes(); + return Traits; + } + + void Register(PyObject* module, const char* typeName) { + Py_INCREF(PyTypeObjPtr); + if (0 != PyModule_AddObject(module, typeName, PyTypeObjPtr)) + ythrow yexception() << "can't register type \"" << typeName << "\""; + } + + void Register(PyObject* module, const char* objName, TObjectHolder* hld) { + if (0 != PyModule_AddObject(module, objName, CreatePyObject(hld).RefGet())) + ythrow yexception() << "can't register object \"" << objName << "\""; + } + + void Register(TPyObjectPtr module, const TString& typeName) { + Register(module.Get(), typeName.c_str()); + } + + void Register(TPyObjectPtr module, const TString& objName, TObjectHolder* hld) { + Register(module.Get(), objName.c_str(), hld); + } + + static TObjectHolder* CastToObjectHolder(PyObject* obj) { + // Call Instance() to make sure PyTypeObjPtr is already created at this point + Instance(); + if (!PyObject_IsInstance(obj, PyTypeObjPtr)) + return nullptr; + TProxy* prx = reinterpret_cast<TProxy*>(obj); + return prx ? prx->Holder : nullptr; + } + + static TObject* CastToObject(PyObject* obj) { + TObjectHolder* hld = CastToObjectHolder(obj); + return hld ? TTraits::GetObject(*hld) : nullptr; + } + + static TPyObjectPtr CreatePyObject(TObjectHolder* hld) { + TPyObjectPtr r(_PyObject_New(&PyType), true); + TProxy* prx = reinterpret_cast<TProxy*>(r.Get()); + if (prx) + prx->Holder = hld; + return r; + } + }; + + template <typename TObjectHolder, typename TObject, typename TTraits> + PyMappingMethods TPythonType<TObjectHolder, TObject, TTraits>::MappingMethods = {nullptr, nullptr, nullptr}; + + template <typename TObjectHolder, typename TObject, typename TTraits> + PyTypeObject TPythonType<TObjectHolder, TObject, TTraits>::PyType = { + PyVarObject_HEAD_INIT(nullptr, 0) "", sizeof(TProxy), 0, (destructor)&DeallocObject +#if PY_VERSION_HEX < 0x030800b4 + , nullptr, /*tp_print*/ +#endif +#if PY_VERSION_HEX >= 0x030800b4 + , 0, /*tp_vectorcall_offset*/ +#endif + &GetObjectAttr, &SetObjectAttr, nullptr, &GetRepr, nullptr, nullptr, &MappingMethods, nullptr, nullptr, &GetStr, nullptr, nullptr, nullptr, + Py_TPFLAGS_DEFAULT, "", nullptr, nullptr, nullptr, 0, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, 0, InitObject, PyType_GenericAlloc, PyType_GenericNew, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, 0 +#if PY_MAJOR_VERSION >= 3 + , nullptr +#endif +#if PY_VERSION_HEX >= 0x030800b1 + , nullptr /*tp_vectorcall*/ +#endif +#if PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x03090000 + , nullptr /*tp_print*/ +#endif +#if PY_VERSION_HEX >= 0x030C0000 + , 0 /*tp_watched*/ +#endif + }; + + template <typename TObjectHolder, typename TObject, typename TTraits> + PyObject* TPythonType<TObjectHolder, TObject, TTraits>::PyTypeObjPtr = + reinterpret_cast<PyObject*>(&TPythonType<TObjectHolder, TObject, TTraits>::PyType); + + namespace NPrivate { + template <typename TObject> + class TUnboundClosureHolder { + private: + THolder<PyTypeObject> Holder; + TString Method; + + public: + TUnboundClosureHolder(PyTypeObject* ptr, const TString& meth) + : Holder(ptr) + , Method(meth) + { + } + + PyTypeObject* GetObject() const { + return Holder.Get(); + } + + const TString GetMethod() const { + return Method; + } + + PyObject* Call(PyObject* obj, PyObject* args, PyObject*) const { + TPyObjectPtr callable(PyObject_GetAttrString(obj, Method.c_str()), true); + if (!callable.Get()) + ythrow yexception() << "PyBind can't call method '" << Method << "'"; + TPyObjectPtr res(PyObject_CallObject(callable.Get(), args), true); + if (!res.Get() && !PyErr_Occurred()) + ythrow yexception() << "PyBind can't call method '" << Method << "'"; + return res.RefGet(); + } + }; + + template <typename TObject> + class TUnboundClosure: public NPyBind::TPythonType<TUnboundClosureHolder<TObject>, PyTypeObject, TUnboundClosure<TObject>> { + private: + typedef class NPyBind::TPythonType<TUnboundClosureHolder<TObject>, PyTypeObject, TUnboundClosure<TObject>> TParent; + friend class NPyBind::TPythonType<TUnboundClosureHolder<TObject>, PyTypeObject, TUnboundClosure<TObject>>; + + class TReprMethodCaller: public TBaseMethodCaller<PyTypeObject> { + public: + bool CallMethod(PyObject* closure, PyTypeObject*, PyObject*, PyObject*, PyObject*& res) const override { + TUnboundClosureHolder<TObject>* hld = TParent::CastToObjectHolder(closure); + TPyObjectPtr type((PyObject*)hld->GetObject()); + + TString nameStr; + TPyObjectPtr name(PyObject_GetAttrString(type.Get(), "__name__"), true); + if (!name.Get() || !FromPyObject(name.Get(), nameStr)) + ythrow yexception() << "Could not get name of object"; + + TString methodName(hld->GetMethod()); + + TString message = "<unbound method " + nameStr + "." + methodName + ">"; + res = ReturnString(message); + return (res != nullptr); + } + }; + + private: + TUnboundClosure() + : TParent("", "") + { + TParent::AddCaller("__repr__", new TReprMethodCaller()); + TParent::AddCaller("__str__", new TReprMethodCaller()); + TParent::SetClosure(&Call); + } + + static PyObject* Call(PyObject* closure, PyObject* args, PyObject* kwargs) { + try { + TUnboundClosureHolder<TObject>* hld = TParent::CastToObjectHolder(closure); + if (!hld) + ythrow yexception() << "Can't cast object to TypeHolder"; + + size_t size = 0; + if (!PyTuple_Check(args) || (size = PyTuple_Size(args)) < 1) + ythrow yexception() << "Can't parse first argument: it should be valid object"; + --size; + TPyObjectPtr obj(PyTuple_GetItem(args, 0)); + TPyObjectPtr newArgs(PyTuple_New(size), true); + + for (size_t i = 0; i < size; ++i) { + TPyObjectPtr item(PyTuple_GetItem(args, i + 1)); + PyTuple_SetItem(newArgs.Get(), i, item.RefGet()); + } + + return hld->Call(obj.Get(), newArgs.Get(), kwargs); + } catch (const std::exception& ex) { + PyErr_SetString(::NPyBind::TExceptionsHolder::Instance().ToPyException(ex).Get(), ex.what()); + } catch (...) { + PyErr_SetString(PyExc_RuntimeError, "Unknown error occurred while trying to call method"); + } + return nullptr; + } + + static PyTypeObject* GetObject(TUnboundClosureHolder<TObject>& obj) { + return obj.GetObject(); + } + }; + + template <typename TObject> + class TBoundClosureHolder { + private: + TPyObjectPtr Ptr; + TObject* Object; + TString Method; + const TMethodCallers<TObject>& MethodCallers; + + public: + TBoundClosureHolder(PyObject* ptr, TObject* obj, const TString& meth, const TMethodCallers<TObject>& callers) + : Ptr(ptr) + , Object(obj) + , Method(meth) + , MethodCallers(callers) + { + } + + TPyObjectPtr GetObjectPtr() const { + return Ptr; + } + + TObject* GetObject() const { + return Object; + } + + const TString GetMethod() const { + return Method; + } + + PyObject* Call(PyObject* args, PyObject* kwargs) const { + PyObject* res = MethodCallers.CallMethod(Ptr.Get(), Object, args, kwargs, Method); + if (res == nullptr && !PyErr_Occurred()) + ythrow yexception() << "PyBind can't call method '" << Method << "'"; + return res; + } + }; + + template <typename TObject> + class TBoundClosure: public TPythonType<TBoundClosureHolder<TObject>, TObject, TBoundClosure<TObject>> { + private: + typedef TPythonType<TBoundClosureHolder<TObject>, TObject, TBoundClosure<TObject>> TMyParent; + class TReprMethodCaller: public TBaseMethodCaller<TObject> { + public: + bool CallMethod(PyObject* closure, TObject*, PyObject*, PyObject*, PyObject*& res) const override { + TBoundClosureHolder<TObject>* hld = TMyParent::CastToObjectHolder(closure); + TPyObjectPtr obj(hld->GetObjectPtr()); + TPyObjectPtr type(PyObject_Type(obj.Get()), true); + + TString reprStr; + TPyObjectPtr repr(PyObject_Repr(obj.Get()), true); + if (!repr.Get() || !FromPyObject(repr.Get(), reprStr)) + ythrow yexception() << "Could not get repr of object"; + + TString nameStr; + TPyObjectPtr name(PyObject_GetAttrString(type.Get(), "__name__"), true); + if (!name.Get() || !FromPyObject(name.Get(), nameStr)) + ythrow yexception() << "Could not get name of object"; + + TString methodName(hld->GetMethod()); + + TString message = "<bound method " + nameStr + "." + methodName + " of " + reprStr + ">"; + res = ReturnString(message); + return (res != nullptr); + } + }; + + private: + static PyObject* Call(PyObject* closure, PyObject* args, PyObject* kwargs) { + try { + TBoundClosureHolder<TObject>* hld = TMyParent::CastToObjectHolder(closure); + if (!hld) + ythrow yexception() << "Can't cast object to ClosureHolder"; + + return hld->Call(args, kwargs); + } catch (const std::exception& ex) { + PyErr_SetString(::NPyBind::TExceptionsHolder::Instance().ToPyException(ex).Get(), ex.what()); + } catch (...) { + PyErr_SetString(PyExc_RuntimeError, "Unknown error occurred while trying to call method"); + } + return nullptr; + } + + public: + TBoundClosure() + : TMyParent("", "") + { + TMyParent::AddCaller("__repr__", new TReprMethodCaller()); + TMyParent::AddCaller("__str__", new TReprMethodCaller()); + TMyParent::SetClosure(&Call); + } + + static TObject* GetObject(const TBoundClosureHolder<TObject>& closure) { + return closure.GetObject(); + } + }; + + } + + template <typename TObjectHolder, typename TObject, typename TTraits> + PyObject* TPythonType<TObjectHolder, TObject, TTraits>::GetObjectAttr(PyObject* pyObj, char* attr) { + try { + TObject* obj = CastToObject(pyObj); + PyObject* res = obj ? Instance().Attributes.GetAttrGetters().GetAttr(pyObj, *obj, attr) : nullptr; + if (res == nullptr && Instance().Attributes.GetMethodCallers().HasMethod(pyObj, obj, attr)) { + TPyObjectPtr r = NPrivate::TBoundClosure<TObject>::Instance().CreatePyObject(new NPrivate::TBoundClosureHolder<TObject>(pyObj, obj, attr, Instance().Attributes.GetMethodCallers())); + res = r.RefGet(); + } + if (res == nullptr && !PyErr_Occurred()) + ythrow TPyErr(PyExc_AttributeError) << "PyBind can't get attribute '" << attr << "'"; + return res; + } catch (const std::exception& ex) { + PyErr_SetString(TExceptionsHolder::Instance().ToPyException(ex).Get(), ex.what()); + } catch (...) { + PyErr_SetString(PyExc_RuntimeError, (TString("Unknown error occurred while trying to get attribute '") + attr + "'").c_str()); + } + return nullptr; + } + + template <typename TObjectHolder, typename TObject, typename TTraits> + int TPythonType<TObjectHolder, TObject, TTraits>::SetObjectAttr(PyObject* pyObj, char* attr, PyObject* value) { + try { + TObject* obj = CastToObject(pyObj); + bool res = obj ? Instance().Attributes.GetAttrSetters().SetAttr(pyObj, *obj, attr, value) : false; + if (!res && !PyErr_Occurred()) + ythrow yexception() << "PyBind can't set attribute '" << attr << "'"; + return res ? 0 : -1; + } catch (const std::exception& ex) { + PyErr_SetString(TExceptionsHolder::Instance().ToPyException(ex).Get(), ex.what()); + } catch (...) { + PyErr_SetString(PyExc_RuntimeError, (TString("Unknown error occurred while trying to set attribute '") + attr + "'").c_str()); + } + return -1; + } + + template <typename TObjectHolder, typename TObject, typename TTraits> + PyObject* TPythonType<TObjectHolder, TObject, TTraits>::GetStr(PyObject* obj) { + try { + TObject* self = CastToObject(obj); + return Instance().Attributes.GetMethodCallers().CallMethod(obj, self, nullptr, nullptr, "__str__"); + } catch (const std::exception& ex) { + PyErr_SetString(TExceptionsHolder::Instance().ToPyException(ex).Get(), ex.what()); + } catch (...) { + PyErr_SetString(PyExc_RuntimeError, (TString("Unknown error occurred while trying to call '__str__'").c_str())); + } + return nullptr; + } + + template <typename TObjectHolder, typename TObject, typename TTraits> + PyObject* TPythonType<TObjectHolder, TObject, TTraits>::GetIter(PyObject* obj) { + try { + TObject* self = CastToObject(obj); + return Instance().Attributes.GetMethodCallers().CallMethod(obj, self, nullptr, nullptr, "__iter__"); + } catch (const std::exception& ex) { + PyErr_SetString(TExceptionsHolder::Instance().ToPyException(ex).Get(), ex.what()); + } catch (...) { + PyErr_SetString(PyExc_RuntimeError, (TString("Unknown error occurred while trying to call '__iter__'").c_str())); + } + return nullptr; + } + + template <typename TObjectHolder, typename TObject, typename TTraits> + PyObject* TPythonType<TObjectHolder, TObject, TTraits>::GetNext(PyObject* obj) { + try { + TObject* self = CastToObject(obj); + return Instance().Attributes.GetMethodCallers().CallMethod(obj, self, nullptr, nullptr, "next"); + } catch (const std::exception& ex) { + PyErr_SetString(TExceptionsHolder::Instance().ToPyException(ex).Get(), ex.what()); + } catch (...) { + PyErr_SetString(PyExc_RuntimeError, (TString("Unknown error occurred while trying to call 'next'").c_str())); + } + return nullptr; + } + + template <typename TObjectHolder, typename TObject, typename TTraits> + PyObject* TPythonType<TObjectHolder, TObject, TTraits>::GetRepr(PyObject* obj) { + try { + TObject* self = CastToObject(obj); + return Instance().Attributes.GetMethodCallers().CallMethod(obj, self, nullptr, nullptr, "__repr__"); + } catch (const std::exception& ex) { + PyErr_SetString(TExceptionsHolder::Instance().ToPyException(ex).Get(), ex.what()); + } catch (...) { + PyErr_SetString(PyExc_RuntimeError, (TString("Unknown error occurred while trying to call '__repr__'").c_str())); + } + return nullptr; + } +} diff --git a/library/cpp/pybind/v2.cpp b/library/cpp/pybind/v2.cpp new file mode 100644 index 0000000000..edce0be719 --- /dev/null +++ b/library/cpp/pybind/v2.cpp @@ -0,0 +1,43 @@ +#include "v2.h" +namespace NPyBind { + namespace Detail { + template <> + PyTypeObject* GetParentType<void>(const TPyModuleDefinition&) { + return nullptr; + } + + + template <bool InitEnabled> + void UpdateClassNamesInModule(TPyModuleDefinition& M, const TString& name, PyTypeObject* pythonType) { + if (!InitEnabled) { + return; + } + M.ClassName2Type[name] = pythonType; + } + + template <bool InitEnabled> + void UpdateGetContextInModule(TPyModuleDefinition& M, const TString& name, IGetContextBase* base) { + if (!InitEnabled) { + return; + } + M.Class2ContextGetter[name] = base; + } + + TPyModuleRegistry::TPyModuleRegistry() { +#if PY_MAJOR_VERSION >= 3 + NPrivate::AddFinalizationCallBack([this]() { + if (UnnamedModule) { + UnnamedModule.Clear(); + } + Name2Def.clear(); + }); +#endif + } + template void UpdateClassNamesInModule<false>(TPyModuleDefinition& M, const TString& name, PyTypeObject* pythonType); + template void UpdateClassNamesInModule<true>(TPyModuleDefinition& M, const TString& name, PyTypeObject* pythonType); + + + template void UpdateGetContextInModule<false>(TPyModuleDefinition& M, const TString& name, IGetContextBase* pythonType); + template void UpdateGetContextInModule<true>(TPyModuleDefinition& M, const TString& name, IGetContextBase* pythonType); + }//Detail +}//NPyBind diff --git a/library/cpp/pybind/v2.h b/library/cpp/pybind/v2.h new file mode 100644 index 0000000000..f561d6a380 --- /dev/null +++ b/library/cpp/pybind/v2.h @@ -0,0 +1,514 @@ +#pragma once + +#include <library/cpp/pybind/method.h> +#include <library/cpp/pybind/typedesc.h> +#include <library/cpp/pybind/module.h> +#include <util/generic/hash.h> +#include <util/generic/hash_set.h> +#include <util/generic/string.h> +namespace NPyBind { +#define DEFINE_CONVERTERS_IMPL(TClass) \ + PyObject* BuildPyObject(typename TClass::TBase&& base) { \ + return TClass::BuildPyObject(std::move(base)); \ + } \ + PyObject* BuildPyObject(const typename TClass::TBase& base) { \ + return TClass::BuildPyObject(base); \ + } + +#define DEFINE_CONVERTERS(function) DEFINE_CONVERTERS_IMPL(TFunctionResult<decltype(function)>) + +#define DEFINE_TRANSFORMERS_IMPL(TClass) \ + template <> \ + bool ::NPyBind::FromPyObject<typename TClass::TBase*>(PyObject * obj, typename TClass::TBase * &res) { \ + res = TClass::CastToObject(obj); \ + return res != nullptr; \ + } \ + template <> \ + bool ::NPyBind::FromPyObject<typename TClass::TBase const*>(PyObject * obj, typename TClass::TBase const*& res) { \ + res = TClass::CastToObject(obj); \ + return res != nullptr; \ + } + +#define DEFINE_TRANSFORMERS(function) DEFINE_TRANSFORMERS_IMPL(TFunctionResult<decltype(function)>) + + namespace Detail { + struct IGetContextBase { + virtual ~IGetContextBase() = default; + }; + } //Detail + struct TPyModuleDefinition { + static void InitModule(const TString& name); + static TPyModuleDefinition& GetModule(); + + TString Name; + NPyBind::TPyObjectPtr M; + THashMap<TString, PyTypeObject*> ClassName2Type; + THashMap<TString, Detail::IGetContextBase*> Class2ContextGetter; + }; + + namespace Detail { + // Manages modules lifecycle + // IMPORTANT!!! Don't use it in PyBind v1 environment, it will lead to inconsistent state of v1 module + // UnnamedModule-> new unnamed module stub, this stub become current module. In this case you can add functions to it + // InitModuleWithName -> convert unnamed module into named one, now you can switch to it in switch, this module remains current + // SwitchToModule switches to the particular module in registry, this module becomes current. + class TPyModuleRegistry { + private: + TPyModuleRegistry(); + TPyModuleRegistry(const TPyModuleRegistry&) = delete; + TPyModuleRegistry& operator=(TPyModuleRegistry&) = delete; + public: + static TPyModuleRegistry& Get() { + static TPyModuleRegistry registry; + return registry; + } + TPyModuleDefinition& GetCurrentModule() { + if (!CurrentModule) { + GetUnnamedModule(); + } + return *CurrentModule; + } + + TPyModuleDefinition& GetUnnamedModule() { + if (!UnnamedModule) { + UnnamedModule = TPyModuleDefinition(); + CurrentModule = const_cast<TPyModuleDefinition*>(UnnamedModule.Get()); + } + return *UnnamedModule; + } + + TPyModuleDefinition& InitModuleWithName(const TString& name) { + if (!UnnamedModule) { + GetUnnamedModule(); + } + Name2Def[name] = *UnnamedModule; + UnnamedModule.Clear(); + CurrentModule = &Name2Def[name]; + return *CurrentModule; + } + + TPyModuleDefinition& SwitchToModuleByName(const TString& name) { + Y_ENSURE(Name2Def.contains(name)); + Y_ENSURE(UnnamedModule.Empty()); + CurrentModule = &Name2Def[name]; + return *CurrentModule; + } + private: + TPyModuleDefinition* CurrentModule = nullptr; + TMaybe<TPyModuleDefinition> UnnamedModule;// + THashMap<TString, TPyModuleDefinition> Name2Def; + }; + }//Detail + + inline void TPyModuleDefinition::InitModule(const TString& name) { + Detail::TPyModuleRegistry::Get().GetUnnamedModule() = TPyModuleDefinition{name, TModuleHolder::Instance().InitModule(name), {}, {}}; + Detail::TPyModuleRegistry::Get().InitModuleWithName(name); + } + + inline TPyModuleDefinition& TPyModuleDefinition::GetModule() { + return Detail::TPyModuleRegistry::Get().GetCurrentModule(); + } + + namespace Detail { + template <class TPythonType> + struct TNameCtx { + TString ClassShortName; + static TNameCtx& GetNameCtx() { + static TNameCtx result; + return result; + } + }; + template <class TBase> + struct TContextImpl { + PyTypeObject* ParentType = nullptr; + TString ClassShortName; + TString ClassFullName; + TString ClassDescription; + + + TVector<std::pair<TString, typename TPythonTypeAttributes<TBase>::TCallerPtr>> ListCallers; + TVector<std::pair<TString, typename TPythonTypeAttributes<TBase>::TGetterPtr>> ListGetters; + TVector<std::pair<TString, typename TPythonTypeAttributes<TBase>::TSetterPtr>> ListSetters; + }; + + template <class TObject> + struct IGetContext: public IGetContextBase { + virtual ~IGetContext() = default; + virtual const TContextImpl<TObject>& GetContext() const = 0; + }; + + template <typename THolderClass, typename TBaseClass, bool ShouldEnable, typename=std::enable_if_t<!ShouldEnable || !std::is_default_constructible_v<TBaseClass>>> + THolderClass* DoInitPureObject(const TVector<TString>&) { + ythrow yexception() << "Can't create this object in pure mode from python"; + } + + template <typename THolderClass, typename TBaseClass, bool ShouldEnable, typename=std::enable_if_t<ShouldEnable && std::is_default_constructible_v<TBaseClass>>, typename=void> + THolderClass* DoInitPureObject(const TVector<TString>&) { + return new THolderClass(MakeHolder<TBaseClass>()); + } + + template <typename T> + PyTypeObject* GetParentType(const TPyModuleDefinition& m) { + auto shortName = Detail::TNameCtx<T>::GetNameCtx().ClassShortName; + auto it = m.ClassName2Type.find(shortName); + return (it == m.ClassName2Type.end()) ? nullptr : it->second; + } + + template <> + PyTypeObject* GetParentType<void>(const TPyModuleDefinition&); + + template <bool InitEnabled> + void UpdateClassNamesInModule(TPyModuleDefinition& M, const TString& name, PyTypeObject* pythonType); + + template <bool InitEnabled> + void UpdateGetContextInModule(TPyModuleDefinition& M, const TString& name, IGetContextBase* base); + } + + + template <class TParentPyClass_=void> + struct TPyParentClassTraits { + using TParentPyClass = TParentPyClass_; + }; + + template <bool InitEnabled_, class TParentPyClass_=void> + struct TPyClassConfigTraits: public TPyParentClassTraits<TParentPyClass_> { + constexpr static bool InitEnabled = InitEnabled_; + constexpr static bool RawInit = false; + }; + + template <class TParentPyClass_=void> + struct TPyClassRawInitConfigTraits: public TPyParentClassTraits<TParentPyClass_> { + constexpr static bool InitEnabled = true; + constexpr static bool RawInit = true; + }; + + + template <typename TBaseClass, typename TPyClassConfigTraits, typename... ConstructorArgs> + class TPyClass { + public: + using TBase = TBaseClass; + private: + using TThisClass = TPyClass<TBaseClass, TPyClassConfigTraits, ConstructorArgs...>; + using TContext = Detail::TContextImpl<TBase>; + struct THolder { + ::THolder<TBase> Holder; + THolder(::THolder<TBase>&& right) + : Holder(std::move(right)) + { + } + THolder(TBase&& right) + : Holder(MakeHolder<TBase>(std::move(right))) + { + } + }; + + class TSelectedTraits: public NPyBind::TPythonType<THolder, TBase, TSelectedTraits> { + private: + using TParent = NPyBind::TPythonType<THolder, TBase, TSelectedTraits>; + friend TParent; + + public: + TSelectedTraits() + : TParent(TThisClass::GetContext().ClassFullName.data(), TThisClass::GetContext().ClassDescription.data(), TThisClass::GetContext().ParentType) + { + for (const auto& caller : TThisClass::GetContext().ListCallers) { + TParent::AddCaller(caller.first, caller.second); + } + + for (const auto& getter : TThisClass::GetContext().ListGetters) { + TParent::AddGetter(getter.first, getter.second); + } + + for (const auto& setter : TThisClass::GetContext().ListSetters) { + TParent::AddSetter(setter.first, setter.second); + } + } + + static TBase* GetObject(const THolder& holder) { + return holder.Holder.Get(); + } + + static THolder* DoInitObject(PyObject* args, PyObject* kwargs) { + if constexpr (TPyClassConfigTraits::InitEnabled) { + if constexpr (TPyClassConfigTraits::RawInit) { + static_assert(sizeof...(ConstructorArgs) == 0, "Do not pass construction args if use RawInit."); + return new THolder(::MakeHolder<TBase>(args, kwargs)); + } else { + if (args && (!PyTuple_Check(args) || PyTuple_Size(args) != sizeof...(ConstructorArgs))) { + ythrow yexception() << "Method takes " << sizeof...(ConstructorArgs) << " arguments, " << PyTuple_Size(args) << " provided"; + } + ::THolder<TBaseClass> basePtr{Apply([](auto&&... unpackedArgs) {return new TBase(std::forward<decltype(unpackedArgs)>(unpackedArgs)...); }, GetArguments<ConstructorArgs...>(args))}; + return new THolder(std::move(basePtr)); + } + } else { + ythrow yexception() << "Can't create this object from python"; + } + } + + static THolder* DoInitPureObject(const TVector<TString>& properties) { + return Detail::DoInitPureObject<THolder, TBase, TPyClassConfigTraits::InitEnabled>(properties); + } + + static TBase* CastToObject(PyObject* obj) { + return TParent::CastToObject(obj); + } + + static PyTypeObject* GetType() { + return TParent::GetPyTypePtr(); + } + }; + + class TContextHolder: public Detail::IGetContext<TBaseClass> { + public: + static TContextHolder& GetContextHolder() { + static TContextHolder holder; + return holder; + } + + TContext& GetContext() { + return Context; + } + const TContext& GetContext() const override { + return Context; + } + private: + TContext Context; + }; + + template <class TDerivedClass, class TSuperClass> + class TCallerWrapper: public TBaseMethodCaller<TDerivedClass> { + public: + explicit TCallerWrapper(TSimpleSharedPtr<const TBaseMethodCaller<TSuperClass>> baseCaller) + : BaseCaller(baseCaller) { + Y_ENSURE(BaseCaller); + } + + bool CallMethod(PyObject* owner, TDerivedClass* self, PyObject* args, PyObject* kwargs, PyObject*& res) const override { + return BaseCaller->CallMethod(owner, static_cast<TSuperClass*>(self), args, kwargs, res); + } + + private: + TSimpleSharedPtr<const TBaseMethodCaller<TSuperClass>> BaseCaller; + }; + + template <class TDerivedClass, class TSuperClass> + class TSetterWrapper: public TBaseAttrSetter<TDerivedClass> { + public: + explicit TSetterWrapper(TSimpleSharedPtr<TBaseAttrSetter<TSuperClass>> baseSetter) + : BaseSetter(baseSetter) { + Y_ENSURE(BaseSetter); + } + + bool SetAttr(PyObject* owner, TDerivedClass& self, const TString& attr, PyObject* val) override { + return BaseSetter->SetAttr(owner, static_cast<TSuperClass&>(self), attr, val); + } + + private: + TSimpleSharedPtr<TBaseAttrSetter<TSuperClass>> BaseSetter; + }; + + template <class TDerivedClass, class TSuperClass> + class TGetterWrapper: public TBaseAttrGetter<TDerivedClass> { + public: + explicit TGetterWrapper(TSimpleSharedPtr<const TBaseAttrGetter<TSuperClass>> baseGetter) + : BaseGetter(baseGetter) { + Y_ENSURE(BaseGetter); + } + + bool GetAttr(PyObject* owner, const TDerivedClass& self, const TString& attr, PyObject*& res) const override { + return BaseGetter->GetAttr(owner, static_cast<const TSuperClass&>(self), attr, res); + } + + private: + TSimpleSharedPtr<const TBaseAttrGetter<TSuperClass>> BaseGetter; + }; + + template <class TSuperClass, typename=std::enable_if_t<!std::is_same_v<TSuperClass, void>>> + void ReloadAttrsFromBase() { + auto shortName = Detail::TNameCtx<TSuperClass>::GetNameCtx().ClassShortName; + if (!M.Class2ContextGetter.count(shortName)) { + return; + } + auto callerBasePtr = M.Class2ContextGetter[shortName]; + if (auto getContextPtr = dynamic_cast<const Detail::IGetContext<TSuperClass>*>(callerBasePtr)) { + auto& ctx = getContextPtr->GetContext(); + auto getUniqueNames = [](const auto& collection) { + THashSet<TString> uniqueNames; + for (const auto& elem : collection) { + uniqueNames.insert(elem.first); + } + return uniqueNames; + }; + + auto uniqueCallerNames = getUniqueNames(GetContext().ListCallers); + using TConcreteCallerWrapper = TCallerWrapper<TBaseClass, TSuperClass>; + for (const auto& caller : ctx.ListCallers) { + if (uniqueCallerNames.contains(caller.first)) { + continue; + } + GetContext().ListCallers.push_back(std::make_pair(caller.first, MakeSimpleShared<TConcreteCallerWrapper>(caller.second))); + } + + auto uniqueGettersNames = getUniqueNames(GetContext().ListGetters); + using TConcreteGetterWrapper = TGetterWrapper<TBaseClass, TSuperClass>; + for (const auto& getter : ctx.ListGetters) { + if (uniqueGettersNames.contains(getter.first)) { + continue; + } + GetContext().ListGetters.push_back(std::make_pair(getter.first, MakeSimpleShared<TConcreteGetterWrapper>(getter.second))); + } + + auto uniqueSetterNames = getUniqueNames(GetContext().ListSetters); + using TConcreteSetterWrapper = TSetterWrapper<TBaseClass, TSuperClass>; + for (auto& setter : ctx.ListSetters) { + if (uniqueSetterNames.contains(setter.first)) { + continue; + } + GetContext().ListSetters.push_back(std::make_pair(setter.first, MakeSimpleShared<TConcreteSetterWrapper>(setter.second))); + } + } + } + + template <class TSuperClass, typename=std::enable_if_t<std::is_same_v<TSuperClass, void>>, typename=void> + void ReloadAttrsFromBase() { + } + + void CompleteImpl() { + ReloadAttrsFromBase<typename TPyClassConfigTraits::TParentPyClass>(); + TSelectedTraits::Instance().Register(M.M, GetContext().ClassShortName); + } + + static TContext& GetContext() { + return TContextHolder::GetContextHolder().GetContext(); + } + + + friend struct Detail::TContextImpl<TBase>;//instead of context + friend struct THolder; + friend class TSelectedTraits; + + using TCallerFunc = std::function<bool(PyObject*, TBaseClass*, PyObject*, PyObject*, PyObject*&)>; + class TFuncCallerWrapper: public TBaseMethodCaller<TBaseClass> { + public: + explicit TFuncCallerWrapper(TCallerFunc func) + : Func(func) { + Y_ENSURE(func); + } + + bool CallMethod(PyObject* owner, TBaseClass* self, PyObject* args, PyObject* kwargs, PyObject*& res) const override { + return Func(owner, self, args, kwargs, res); + } + private: + mutable TCallerFunc Func; + }; + public: + TPyClass(const TString& name, const TString& descr = "") + : M(TPyModuleDefinition::GetModule()) + { + Detail::UpdateClassNamesInModule<TPyClassConfigTraits::InitEnabled>(M, name, TSelectedTraits::GetType()); + Detail::UpdateGetContextInModule<TPyClassConfigTraits::InitEnabled>(M, name, &TContextHolder::GetContextHolder()); + + GetContext().ClassFullName = TString::Join(M.Name, ".", name); + GetContext().ClassShortName = name; + GetContext().ClassDescription = descr; + GetContext().ParentType = Detail::GetParentType<typename TPyClassConfigTraits::TParentPyClass>(M); + Detail::TNameCtx<TBaseClass>::GetNameCtx().ClassShortName = name; + } + + template <typename TMemberFuction, typename = std::enable_if_t<std::is_member_function_pointer_v<TMemberFuction>>, typename=std::enable_if_t<!TIsPointerToConstMemberFunction<TMemberFuction>::value>> + TThisClass& Def(const TString& name, TMemberFuction t) { + GetContext().ListCallers.push_back(std::make_pair(name, CreateMethodCaller<TBase>(t))); + return *this; + } + + template <typename TMemberFuction, typename = std::enable_if_t<std::is_member_function_pointer_v<TMemberFuction>>, typename=std::enable_if_t<TIsPointerToConstMemberFunction<TMemberFuction>::value>, typename=void> + TThisClass& Def(const TString& name, TMemberFuction t) { + GetContext().ListCallers.push_back(std::make_pair(name, CreateConstMethodCaller<TBase>(t))); + return *this; + } + + template <typename TMemberObject, typename = std::enable_if_t<std::is_member_object_pointer_v<TMemberObject>>> + TThisClass& Def(const TString& name, TMemberObject t) { + GetContext().ListGetters.push_back(std::make_pair(name, CreateAttrGetter<TBase>(t))); + GetContext().ListSetters.push_back(std::make_pair(name, CreateAttrSetter<TBase>(t))); + return *this; + } + + template <typename TResultType, typename... Args> + TThisClass& DefByFunc(const TString& name, std::function<TResultType(TBaseClass&, Args...)> func) { + GetContext().ListCallers.push_back(std::make_pair(name, CreateFunctorCaller<TBase, TResultType, Args...>(func))); + return *this; + } + + TThisClass& DefByFunc(const TString& name, TCallerFunc origFunc) { + GetContext().ListCallers.push_back(std::make_pair(name, MakeSimpleShared<TFuncCallerWrapper>(origFunc))); + return *this; + } + + template <typename TMemberObject> + TThisClass& DefReadonly(const TString& name, TMemberObject t, std::enable_if_t<std::is_member_object_pointer<TMemberObject>::value>* = nullptr) { + GetContext().ListGetters.push_back(std::make_pair(name, CreateAttrGetter<TBase>(t))); + return *this; + } + + + template <typename TMethodGetter, typename TMethodSetter, typename=std::enable_if_t<std::is_member_function_pointer_v<TMethodGetter> && std::is_member_function_pointer_v<TMethodSetter>>> + TThisClass& AsProperty(const TString& name, TMethodGetter getter, TMethodSetter setter) { + GetContext().ListGetters.push_back(std::make_pair(name, CreateMethodAttrGetter<TBase>(getter))); + GetContext().ListSetters.push_back(std::make_pair(name, CreateMethodAttrSetter<TBase>(setter))); + return *this; + } + + template <typename TMethodGetter, typename TMethodSetter, typename=std::enable_if_t<!std::is_member_function_pointer_v<TMethodGetter> && !std::is_member_function_pointer_v<TMethodSetter>>> + TThisClass& AsPropertyByFunc(const TString& name, TMethodGetter getter, TMethodSetter setter) { + GetContext().ListGetters.push_back(std::make_pair(name, CreateFunctorAttrGetter<TBase>(getter))); + GetContext().ListSetters.push_back(std::make_pair(name, CreateFunctorAttrSetter<TBase>(setter))); + return *this; + } + + template <typename TMethodGetter, typename=std::enable_if_t<std::is_member_function_pointer_v<TMethodGetter>>> + TThisClass& AsProperty(const TString& name, TMethodGetter getter) { + GetContext().ListGetters.push_back(std::make_pair(name, CreateMethodAttrGetter<TBase>(getter))); + return *this; + } + + template <typename TMethodGetter> + TThisClass& AsPropertyByFunc(const TString& name, TMethodGetter getter) { + GetContext().ListGetters.push_back(std::make_pair(name, CreateFunctorAttrGetter<TBase>(getter))); + return *this; + } + + TThisClass& Complete() { + if (!Completed) { + CompleteImpl(); + Completed = true; + } + return *this; + } + + public: + static PyObject* BuildPyObject(TBase&& base) { + return NPyBind::BuildPyObject(TSelectedTraits::Instance().CreatePyObject(new THolder(std::move(base)))); + } + + static PyObject* BuildPyObject(const TBase& base) { + return NPyBind::BuildPyObject(TSelectedTraits::Instance().CreatePyObject(new THolder(TBase(base)))); // WARN - copy + } + + static TBase* CastToObject(PyObject* obj) { + return TSelectedTraits::CastToObject(obj); + } + + private: + TPyModuleDefinition& M; + bool Completed = false; + }; + + template <typename TFunctionSignature, TFunctionSignature function> + void DefImpl(const TString& name, const TString& descr = "") { + NPyBind::TModuleHolder::Instance().AddModuleMethod<TModuleMethodCaller<TFunctionSignature, function>::Call>(name, descr); + } + +#define DefFunc(NAME, FUNC) NPyBind::DefImpl<decltype(FUNC), FUNC>(NAME) +#define DefFuncDescr(NAME, FUNC, DESCR) NPyBind::DefImpl<decltype(FUNC), FUNC>(NAME, DESCR) +}; diff --git a/library/cpp/pybind/ya.make b/library/cpp/pybind/ya.make new file mode 100644 index 0000000000..9b7b3413f2 --- /dev/null +++ b/library/cpp/pybind/ya.make @@ -0,0 +1,14 @@ +PY23_NATIVE_LIBRARY() + +SRCS( + cast.cpp + pod.cpp + typedesc.cpp + module.cpp + exceptions.cpp + embedding.cpp + empty.cpp + v2.cpp +) + +END() |