diff options
| author | prettyboy <[email protected]> | 2023-09-08 00:22:12 +0300 | 
|---|---|---|
| committer | prettyboy <[email protected]> | 2023-09-08 00:46:04 +0300 | 
| commit | 3a6cd865171eed9b89bf536cd242285f8b583a91 (patch) | |
| tree | 25e2756c125f7484fb118e0d5724212199662389 /library/cpp | |
| parent | 67f3f216950849664a29035458cfaa5d12a62846 (diff) | |
[build/plugins/ytest] Allow prebuilt linters for opensource
Без этого, ydb или не сможет запускать flake8 с помощью ya make.
Или к ним поедет сборка flake8.
Возможно последнее и не так плохо, но сейчас предлагается пока так
Diffstat (limited to 'library/cpp')
| -rw-r--r-- | library/cpp/pybind/attr.h | 412 | ||||
| -rw-r--r-- | library/cpp/pybind/cast.cpp | 324 | ||||
| -rw-r--r-- | library/cpp/pybind/cast.h | 373 | ||||
| -rw-r--r-- | library/cpp/pybind/embedding.cpp | 63 | ||||
| -rw-r--r-- | library/cpp/pybind/embedding.h | 10 | ||||
| -rw-r--r-- | library/cpp/pybind/empty.cpp | 2 | ||||
| -rw-r--r-- | library/cpp/pybind/exceptions.cpp | 147 | ||||
| -rw-r--r-- | library/cpp/pybind/exceptions.h | 143 | ||||
| -rw-r--r-- | library/cpp/pybind/init.h | 25 | ||||
| -rw-r--r-- | library/cpp/pybind/method.h | 439 | ||||
| -rw-r--r-- | library/cpp/pybind/module.cpp | 72 | ||||
| -rw-r--r-- | library/cpp/pybind/module.h | 176 | ||||
| -rw-r--r-- | library/cpp/pybind/pod.cpp | 18 | ||||
| -rw-r--r-- | library/cpp/pybind/pod.h | 53 | ||||
| -rw-r--r-- | library/cpp/pybind/ptr.h | 51 | ||||
| -rw-r--r-- | library/cpp/pybind/typeattrs.h | 368 | ||||
| -rw-r--r-- | library/cpp/pybind/typedesc.cpp | 79 | ||||
| -rw-r--r-- | library/cpp/pybind/typedesc.h | 545 | ||||
| -rw-r--r-- | library/cpp/pybind/v2.cpp | 43 | ||||
| -rw-r--r-- | library/cpp/pybind/v2.h | 514 | ||||
| -rw-r--r-- | library/cpp/pybind/ya.make | 14 | 
21 files changed, 3871 insertions, 0 deletions
| diff --git a/library/cpp/pybind/attr.h b/library/cpp/pybind/attr.h new file mode 100644 index 00000000000..5f25a6d73d1 --- /dev/null +++ b/library/cpp/pybind/attr.h @@ -0,0 +1,412 @@ +#pragma once + +#define PY_SSIZE_T_CLEAN +#include <Python.h> +#include <util/generic/string.h> +#include <util/generic/map.h> +#include <util/generic/set.h> +#include <util/generic/vector.h> +#include <util/generic/ptr.h> + +#include "cast.h" +#include "exceptions.h" + +namespace NPyBind { +    // TBaseAttrGetter +    template <typename TObjType> +    class TBaseAttrGetter { +    public: +        virtual ~TBaseAttrGetter() { +        } +        virtual bool GetAttr(PyObject* owner, const TObjType& self, const TString& attr, PyObject*& res) const = 0; + +        virtual bool HasAttr(PyObject* owner, const TObjType& self, const TString& attr, const TSet<TString>& hiddenNames) const { +            if (hiddenNames.find(attr) != hiddenNames.end()) +                return false; +            PyObject* res = nullptr; +            if (!GetAttr(owner, self, attr, res)) +                return false; +            Py_XDECREF(res); +            return true; +        } +    }; + +    template <typename TObjType> +    class TBaseAttrSetter { +    public: +        virtual ~TBaseAttrSetter() { +        } + +        virtual bool SetAttr(PyObject* owner, TObjType& self, const TString& attr, PyObject* val) = 0; +    }; + +    template <typename TObjType> +    class TAttrGetters { +    public: +        typedef TSimpleSharedPtr<TBaseAttrGetter<TObjType>> TGetterPtr; + +    private: +        typedef TVector<TGetterPtr> TGetterList; +        typedef TMap<TString, TGetterList> TGetterMap; + +        const TSet<TString>& HiddenAttrNames; +        TGetterMap Getters; + +    public: +        TAttrGetters(const TSet<TString>& hiddenNames) +            : HiddenAttrNames(hiddenNames) +        { +        } + +        void AddGetter(const TString& attr, TGetterPtr getter) { +            Getters[attr].push_back(getter); +        } + +        PyObject* GetAttr(PyObject* owner, const TObjType& self, const TString& attr) const { +            typename TGetterMap::const_iterator it1 = Getters.find(attr); +            if (it1 == Getters.end()) +                it1 = Getters.find(""); +            if (it1 == Getters.end()) +                return nullptr; +            const TGetterList& lst = it1->second; +            for (typename TGetterList::const_iterator it2 = lst.begin(), end = lst.end(); it2 != end; ++it2) { +                PyObject* res = nullptr; +                if ((*it2)->GetAttr(owner, self, attr, res)) +                    return res; +                // IMPORTANT! +                // we have to fail GetAttr right there  because we've failed because of internal python error/exception and can't continue iterating because +                // it cause subsequent exceptions during call to Py_BuildValue +                // moreover we have to preserve original exception right there +                if (PyErr_Occurred()) { +                    break; +                } +            } +            return nullptr; +        } + +        bool HasAttr(PyObject* owner, const TObjType& self, const TString& attr) const { +            typename TGetterMap::const_iterator it1 = Getters.find(attr); +            if (it1 == Getters.end()) +                return false; +            const TGetterList& lst = it1->second; +            for (typename TGetterList::const_iterator it2 = lst.begin(), end = lst.end(); it2 != end; ++it2) { +                if ((*it2)->HasAttr(owner, self, attr, HiddenAttrNames)) +                    return true; +            } +            return false; +        } + +        void GetAttrsDictionary(PyObject* owner, const TObjType& self, TMap<TString, PyObject*>& res) const { +            for (typename TGetterMap::const_iterator it = Getters.begin(), end = Getters.end(); it != end; ++it) { +                try { +                    if (HasAttr(owner, self, it->first)) { +                        auto attrPtr = GetAttr(owner, self, it->first); +                        if (attrPtr) { +                            res[it->first] = attrPtr; +                        } +                        if (PyErr_Occurred()) { +                            PyErr_Clear(); // Skip python errors as well +                        } +                    } +                } catch (const std::exception&) { +                    // ignore this field +                } +            } +        } + +        void GetAttrsNames(PyObject* owner, const TObjType& self, TVector<TString>& resultNames) const { +            for (typename TGetterMap::const_iterator it = Getters.begin(), end = Getters.end(); it != end; ++it) { +                if (HasAttr(owner, self, it->first)) +                    resultNames.push_back(it->first); +            } +        } +    }; + +    template <typename TObjType> +    class TGenericAttrGetter: public TBaseAttrGetter<TObjType> { +    private: +        TString AttrName; + +    public: +        TGenericAttrGetter(const TString& attrName) +            : AttrName(attrName) +        { +        } + +        bool GetAttr(PyObject* obj, const TObjType&, const TString&, PyObject*& res) const override { +            auto str = NameFromString(AttrName); +            res = PyObject_GenericGetAttr(obj, str.Get()); +            if (!res && !PyErr_Occurred()) +                ythrow TPyErr(PyExc_AttributeError) << "Can't get generic attribute '" << AttrName << "'"; +            return res; +        } +    }; + +    template <typename TObjType> +    class TAttrSetters { +    private: +        typedef TSimpleSharedPtr<TBaseAttrSetter<TObjType>> TSetterPtr; +        typedef TVector<TSetterPtr> TSetterList; +        typedef TMap<TString, TSetterList> TSetterMap; + +        TSetterMap Setters; + +    public: +        void AddSetter(const TString& attr, TSetterPtr setter) { +            Setters[attr].push_back(setter); +        } + +        bool SetAttr(PyObject* owner, TObjType& self, const TString& attr, PyObject* val) { +            typename TSetterMap::const_iterator it1 = Setters.find(attr); +            if (it1 == Setters.end()) +                it1 = Setters.find(""); +            if (it1 == Setters.end()) +                return false; +            const TSetterList& lst = it1->second; +            for (typename TSetterList::const_iterator it2 = lst.begin(), end = lst.end(); it2 != end; ++it2) { +                if ((*it2)->SetAttr(owner, self, attr, val)) +                    return true; +            } +            return false; +        } + +        bool SetAttrDictionary(PyObject* owner, TObjType& self, TMap<TString, PyObject*>& dict) { +            for (TMap<TString, PyObject*>::const_iterator it = dict.begin(), end = dict.end(); it != end; ++it) { +                try { +                    SetAttr(owner, self, it->first, it->second); +                } catch (std::exception&) { +                    // ignore this field +                } +            } + +            return true; +        } +    }; + +    /** +      * TMethodAttrGetter - this class maps Python attribute read to C++ method call +      */ +    template <typename TObjType, typename TResult, typename TSubObject> +    class TMethodAttrGetter: public TBaseAttrGetter<TObjType> { +    private: +        typedef TResult (TSubObject::*TMethod)() const; +        TMethod Method; + +    public: +        TMethodAttrGetter(TMethod method) +            : Method(method) +        { +        } + +        bool GetAttr(PyObject*, const TObjType& self, const TString&, PyObject*& res) const override { +            const TSubObject* sub = dynamic_cast<const TSubObject*>(&self); +            if (sub == nullptr) +                return false; +            res = BuildPyObject((sub->*Method)()); +            return (res != nullptr); +        } +    }; + +    template <typename TObjType, typename TFunctor> +    class TFunctorAttrGetter: public TBaseAttrGetter<TObjType> { +        TFunctor Functor; +    public: +        explicit TFunctorAttrGetter(TFunctor functor) +            : Functor(functor) +        { +        } + +        bool GetAttr(PyObject*, const TObjType& self, const TString&, PyObject*& res) const override { +            res = BuildPyObject(Functor(self)); +            return (res != nullptr); +        } +    }; + + +    /** +      * TMethodAttrGetterWithCheck - this class maps Python attribute read to C++ HasAttr/GetAttr call +      *     If HasAttr returns false, None is returned. +      *     Otherwise GetAttr is called. +      */ +    template <typename TObjType, typename TResult, typename TSubObject> +    class TMethodAttrGetterWithCheck: public TBaseAttrGetter<TObjType> { +    private: +        typedef TResult (TSubObject::*TMethod)() const; +        typedef bool (TSubObject::*TCheckerMethod)() const; +        TMethod Method; +        TCheckerMethod CheckerMethod; + +    public: +        TMethodAttrGetterWithCheck(TMethod method, TCheckerMethod checkerMethod) +            : Method(method) +            , CheckerMethod(checkerMethod) +        { +        } + +        bool GetAttr(PyObject*, const TObjType& self, const TString&, PyObject*& res) const override { +            const TSubObject* sub = dynamic_cast<const TSubObject*>(&self); +            if (sub == nullptr) +                return false; +            if ((sub->*CheckerMethod)()) +                res = BuildPyObject((sub->*Method)()); +            else { +                Py_INCREF(Py_None); +                res = Py_None; +            } +            return (res != nullptr); +        } +    }; + +    template <typename TObjType, typename TResult, typename TSubObject, typename TMapper> +    class TMethodAttrMappingGetter: public TBaseAttrGetter<TObjType> { +    private: +        typedef TResult (TSubObject::*TMethod)() const; + +        TMethod Method; +        TMapper Mapper; + +    public: +        TMethodAttrMappingGetter(TMethod method, TMapper mapper) +            : Method(method) +            , Mapper(mapper) +        { +        } + +        bool GetAttr(PyObject*, const TObjType& self, const TString&, PyObject*& res) const override { +            const TSubObject* sub = dynamic_cast<const TSubObject*>(&self); +            if (sub == nullptr) +                return false; +            res = BuildPyObject(Mapper((sub->*Method)())); +            return (res != nullptr); +        } +    }; + +    template <typename TObjType, typename TResult, typename TSubObject, typename TMapper> +    TSimpleSharedPtr<TBaseAttrGetter<TObjType>> +    CreateMethodAttrMappingGetter(TResult (TSubObject::*method)() const, +                                  TMapper mapper) { +        return new TMethodAttrMappingGetter<TObjType, TResult, TSubObject, TMapper>(method, +                                                                                    mapper); +    } + +    template <typename TObjType, typename TResult, typename TValue, typename TSubObject> +    class TMethodAttrSetter: public TBaseAttrSetter<TObjType> { +    private: +        typedef TResult (TSubObject::*TMethod)(TValue&); +        TMethod Method; + +    public: +        TMethodAttrSetter(TMethod method) +            : Method(method) +        { +        } + +        virtual bool SetAttr(PyObject*, TObjType& self, const TString&, PyObject* val) { +            TSubObject* sub = dynamic_cast<TSubObject*>(&self); +            if (sub == nullptr) +                return false; +            TValue value; +            if (!FromPyObject(val, value)) +                return false; +            (sub->*Method)(value); +            return true; +        } +    }; + +    template <typename TObjType, typename TValue, typename TFunctor> +    class TFunctorAttrSetter: public TBaseAttrSetter<TObjType> { +        TFunctor Functor; +    public: +        explicit TFunctorAttrSetter(TFunctor functor) +            : Functor(functor) +        { +        } + +        bool SetAttr(PyObject*, TObjType& self, const TString&, PyObject* val) const override { +            TValue value; +            if (!FromPyObject(val, value)) +                return false; +            auto res = BuildPyObject(Functor(self, value)); +            return (res != nullptr); +        } +    }; +    template <typename TObjType, typename TResult, typename TSubObject> +    TSimpleSharedPtr<TBaseAttrGetter<TObjType>> CreateMethodAttrGetter(TResult (TSubObject::*method)() const) { +        return new TMethodAttrGetter<TObjType, TResult, TSubObject>(method); +    } + +    template <typename TObjType, typename TFunctor> +    TSimpleSharedPtr<TFunctorAttrGetter<TObjType, TFunctor>> CreateFunctorAttrGetter(TFunctor functor) { +        return MakeSimpleShared<TFunctorAttrGetter<TObjType, TFunctor>>(functor); +    } + +    template <typename TObjType, typename TResult, typename TSubObject> +    TSimpleSharedPtr<TBaseAttrGetter<TObjType>> CreateMethodAttrGetterWithCheck( +        TResult (TSubObject::*method)() const, +        bool (TSubObject::*checkerMethod)() const) { +        return new TMethodAttrGetterWithCheck<TObjType, TResult, TSubObject>(method, checkerMethod); +    } + +    template <typename TObjType, typename TResult, typename TValue, typename TSubObject> +    TSimpleSharedPtr<TBaseAttrSetter<TObjType>> CreateMethodAttrSetter(TResult (TSubObject::*method)(TValue&)) { +        return new TMethodAttrSetter<TObjType, TResult, TValue, TSubObject>(method); +    } + +    template <typename TObjType, typename TFunctor, typename TValue> +    TSimpleSharedPtr<TFunctorAttrSetter<TObjType, TValue, TFunctor>> CreateFunctorAttrSetter(TFunctor functor) { +        return MakeSimpleShared<TFunctorAttrSetter<TObjType, TValue, TFunctor>>(functor); +    } + +    template <typename TObjType, typename TValue, typename TSubObject> +    class TDirectAttrSetter: public TBaseAttrSetter<TObjType> { +    private: +        typedef TValue TSubObject::*TValueType; +        TValueType Value; + +    public: +        TDirectAttrSetter(TValueType value) +            : Value(value) +        { +        } + +        bool SetAttr(PyObject*, TObjType& self, const TString&, PyObject* val) override { +            TSubObject* sub = dynamic_cast<TSubObject*>(&self); +            if (sub == NULL) +                return false; +            if (!FromPyObject(val, sub->*Value)) +                return false; +            return true; +        } +    }; + +    template <typename TObjType, typename TValue, typename TSubObject> +    TSimpleSharedPtr<TBaseAttrSetter<TObjType>> CreateAttrSetter(TValue TSubObject::*value) { +        return new TDirectAttrSetter<TObjType, TValue, TSubObject>(value); +    } + +    template <typename TObjType, typename TValue, typename TSubObject> +    class TDirectAttrGetter: public TBaseAttrGetter<TObjType> { +    private: +        typedef TValue TSubObject::*TValueType; +        TValueType Value; + +    public: +        TDirectAttrGetter(TValueType value) +            : Value(value) +        { +        } + +        bool GetAttr(PyObject*, const TObjType& self, const TString&, PyObject*& res) const override { +            const TSubObject* sub = dynamic_cast<const TSubObject*>(&self); +            if (sub == nullptr) +                return false; +            res = BuildPyObject(sub->*Value); +            return (res != nullptr); +        } +    }; + +    template <typename TObjType, typename TValue, typename TSubObject> +    TSimpleSharedPtr<TBaseAttrGetter<TObjType>> CreateAttrGetter(TValue TSubObject::*value) { +        return new TDirectAttrGetter<TObjType, TValue, TSubObject>(value); +    } +} diff --git a/library/cpp/pybind/cast.cpp b/library/cpp/pybind/cast.cpp new file mode 100644 index 00000000000..60a44b0e833 --- /dev/null +++ b/library/cpp/pybind/cast.cpp @@ -0,0 +1,324 @@ +#include "cast.h" +#include <util/generic/yexception.h> +#include <util/generic/buffer.h> + +namespace NPyBind { +    PyObject* GetTrueRef(bool incref) { +        if (incref) +            Py_RETURN_TRUE; +        return Py_True; +    } + +    PyObject* GetFalseRef(bool incref) { +        if (incref) +            Py_RETURN_FALSE; +        return Py_False; +    } + +    PyObject* BuildPyObject(int val) { +        return Py_BuildValue("i", val); +    } + +    PyObject* BuildPyObject(unsigned int val) { +        return Py_BuildValue("I", val); +    } + +    PyObject* BuildPyObject(long int val) { +        return Py_BuildValue("l", val); +    } + +    PyObject* BuildPyObject(unsigned long int val) { +        return Py_BuildValue("k", val); +    } + +#ifdef PY_LONG_LONG +    PyObject* BuildPyObject(PY_LONG_LONG val) { +        return Py_BuildValue("L", val); +    } + +    PyObject* BuildPyObject(unsigned PY_LONG_LONG val) { +        return Py_BuildValue("K", val); +    } +#endif + +    PyObject* BuildPyObject(float val) { +        return Py_BuildValue("f", val); +    } + +    PyObject* BuildPyObject(double val) { +        return Py_BuildValue("d", val); +    } + +    PyObject* BuildPyObject(const TStringBuf& val) { +        if (!val.IsInited()) +            Py_RETURN_NONE; + +        PyObject* stringValue = Py_BuildValue("s#", val.data(), static_cast<int>(val.length())); +        if (stringValue != nullptr) { +            return stringValue; +        } +        if (PyErr_ExceptionMatches(PyExc_UnicodeDecodeError)) { +            PyErr_Clear(); +        } else { +            return nullptr; +        } +        return Py_BuildValue("y#", val.data(), static_cast<int>(val.length())); +    } + +    PyObject* BuildPyObject(const char* val) { +        if (val == nullptr) +            Py_RETURN_NONE; +        PyObject* stringValue = Py_BuildValue("s#", val, static_cast<int>(strlen(val))); +        if (stringValue != nullptr) { +            return stringValue; +        } +        if (PyErr_ExceptionMatches(PyExc_UnicodeDecodeError)) { +            PyErr_Clear(); +        } else { +            return nullptr; +        } +        return Py_BuildValue("y#", val, static_cast<int>(strlen(val))); +    } + +    PyObject* BuildPyObject(const TWtringBuf& val) { +        if (!val.IsInited()) +            Py_RETURN_NONE; +#if PY_VERSION_HEX < 0x03030000 +        TPyObjectPtr result(PyUnicode_FromUnicode(nullptr, val.size()), true); +        Py_UNICODE* buf = PyUnicode_AS_UNICODE(result.Get()); +        if (buf == nullptr) +            Py_RETURN_NONE; +        for (size_t i = 0; i < val.size(); ++i) { +            buf[i] = static_cast<Py_UNICODE>(val[i]); +        } +#else +        PyObject* unicodeValue = PyUnicode_FromKindAndData(PyUnicode_2BYTE_KIND, val.data(), val.size()); +        if (unicodeValue == nullptr) +            Py_RETURN_NONE; +        TPyObjectPtr result(unicodeValue, true); +#endif +        return result.RefGet(); +    } + +    PyObject* BuildPyObject(const TBuffer& val) { +        TPyObjectPtr res(PyList_New(val.size()), true); +        for (size_t i = 0, size = val.Size(); i < size; ++i) +            PyList_SetItem(res.Get(), i, BuildPyObject(val.Data()[i])); +        return res.RefGet(); +    } + +    PyObject* BuildPyObject(bool val) { +        if (val) +            Py_RETURN_TRUE; +        else +            Py_RETURN_FALSE; +    } + +    PyObject* BuildPyObject(PyObject* val) { +        Py_XINCREF(val); +        return val; +    } + +    PyObject* BuildPyObject(TPyObjectPtr ptr) { +        return ptr.RefGet(); +    } + +    /* python represents (http://docs.python.org/c-api/arg.html#Py_BuildValue) +     * char, uchar, short, ushort, int, long as PyInt +     * uint, ulong as PyInt or PyLong (if exceeds sys.maxint) +     * longlong, ulonglong as PyLong +     */ + +    template <> +    bool FromPyObject(PyObject* obj, long& res) { +        if (PyLong_Check(obj)) { +            res = PyLong_AsLong(obj); +            return true; +        } +        if (PyFloat_Check(obj)) { +            res = static_cast<long>(PyFloat_AsDouble(obj)); +            return true; +        } +#if PY_MAJOR_VERSION < 3 +        res = PyInt_AsLong(obj); +#endif +        return -1 != res || !PyErr_Occurred(); +    } + +    template <> +    bool FromPyObject(PyObject* obj, unsigned long& res) { +        long lres; +        if (!FromPyObject(obj, lres)) +            return false; +        if (lres < 0) +            return false; +        res = static_cast<unsigned long long>(lres); +        return true; +    } + +    template <> +    bool FromPyObject(PyObject* obj, int& res) { +        long lres; +        if (!FromPyObject(obj, lres)) +            return false; +        res = static_cast<int>(lres); +        return true; +    } + +    template <> +    bool FromPyObject(PyObject* obj, unsigned char& res) { +        long lres; +        if (!FromPyObject(obj, lres)) +            return false; +        res = static_cast<unsigned char>(lres); +        return true; +    } + +    template <> +    bool FromPyObject(PyObject* obj, char& res) { +        long lres; +        if (!FromPyObject(obj, lres)) +            return false; +        res = static_cast<char>(lres); +        return true; +    } + +    template <> +    bool FromPyObject(PyObject* obj, unsigned int& res) { +        unsigned long lres; +        if (!FromPyObject(obj, lres)) +            return false; +        res = static_cast<unsigned int>(lres); +        return true; +    } + +#ifdef HAVE_LONG_LONG +    template <> +    bool FromPyObject(PyObject* obj, long long& res) { +        if (PyLong_Check(obj)) { +            res = PyLong_AsLongLong(obj); +            return -1 != res || !PyErr_Occurred(); +        } +        long lres; +        if (!FromPyObject(obj, lres)) +            return false; +        res = static_cast<long long>(lres); +        return true; +    } + +    template <> +    bool FromPyObject(PyObject* obj, unsigned long long& res) { +        if (PyLong_Check(obj)) { +            res = PyLong_AsUnsignedLongLong(obj); +            return static_cast<unsigned long long>(-1) != res || !PyErr_Occurred(); +        } +        long lres; +        if (!FromPyObject(obj, lres)) +            return false; +        res = static_cast<unsigned long long>(lres); +        return true; +    } +#endif + +    template <> +    bool FromPyObject(PyObject* obj, double& res) { +        if (PyFloat_Check(obj)) { +            res = PyFloat_AsDouble(obj); +            return true; +        } +        long long lres; +        if (!FromPyObject(obj, lres)) +            return false; +        res = static_cast<double>(lres); +        return true; +    } + +    template <> +    bool FromPyObject(PyObject* obj, float& res) { +        double dres; +        if (!FromPyObject(obj, dres)) +            return false; +        res = static_cast<float>(dres); +        return true; +    } + +    template <> +    bool FromPyObject(PyObject* obj, bool& res) { +        if (!PyBool_Check(obj)) +            return false; +        if (obj == Py_True) +            res = true; +        else +            res = false; +        return true; +    } + +    template <> +    bool FromPyObject(PyObject* obj, PyObject*& res) { +        Py_XINCREF(obj); +        res = obj; +        return true; +    } + +    template <> +    bool FromPyObject(PyObject* obj, TPyObjectPtr& res) { +        res = TPyObjectPtr(obj); +        return true; +    } + +    static inline bool _FromPyObject(PyObject* obj, TStringBuf& res) { +        char* str; +        Py_ssize_t len; +#if PY_MAJOR_VERSION >= 3 +        if (PyUnicode_Check(obj)) { +            auto buf = PyUnicode_AsUTF8AndSize(obj, &len); +            res = TStringBuf(buf, len); +            return true; +        } +#endif +        if (-1 == PyBytes_AsStringAndSize(obj, &str, &len) || 0 > len) +            return false; +        res = TStringBuf(str, len); +        return true; +    } + +    bool FromPyObject(PyObject* obj, TStringBuf& res) { +        return _FromPyObject(obj, res); +    } + +    bool FromPyObject(PyObject* obj, TString& res) { +        TStringBuf str; +        if (!_FromPyObject(obj, str)) +            return false; +        res = str; +        return true; +    } + +    bool FromPyObject(PyObject* obj, TUtf16String& res) { +        if (!PyUnicode_Check(obj)) +            return false; +        auto str = TPyObjectPtr(PyUnicode_AsUTF16String(obj), true); +        if (!str) +            return false; +        constexpr auto BOM_SIZE = 2; +        size_t len = (static_cast<size_t>(PyBytes_GET_SIZE(str.Get())) - BOM_SIZE) / 2; +        res.resize(len); +        memcpy(res.begin(), PyBytes_AS_STRING(str.Get()) + BOM_SIZE, len * 2); +        return (nullptr == PyErr_Occurred()); +    } + +    bool FromPyObject(PyObject* obj, TBuffer& res) { +        if (!PyList_Check(obj)) +            return false; +        size_t cnt = PyList_Size(obj); +        res.Reserve(cnt); +        for (size_t i = 0; i < cnt; ++i) { +            PyObject* item = PyList_GET_ITEM(obj, i); +            char ch = 0; +            if (!FromPyObject(item, ch)) +                return false; +            res.Append(ch); +        } +        return true; +    } +} diff --git a/library/cpp/pybind/cast.h b/library/cpp/pybind/cast.h new file mode 100644 index 00000000000..1f3d7d8366f --- /dev/null +++ b/library/cpp/pybind/cast.h @@ -0,0 +1,373 @@ +#pragma once + +#define PY_SSIZE_T_CLEAN +#include <Python.h> +#include <util/generic/strbuf.h> +#include <util/generic/vector.h> +#include <util/generic/set.h> +#include <util/generic/yexception.h> +#include <util/generic/hash.h> +#include <util/generic/map.h> +#include <util/generic/maybe.h> +#include <utility> +#include <initializer_list> +#include "ptr.h" + +namespace NPyBind { +    PyObject* GetTrueRef(bool incref = true); +    PyObject* GetFalseRef(bool incref = true); + +    PyObject* BuildPyObject(int val); +    PyObject* BuildPyObject(unsigned int val); +    PyObject* BuildPyObject(long int val); +    PyObject* BuildPyObject(unsigned long int val); +#ifdef PY_LONG_LONG +    PyObject* BuildPyObject(PY_LONG_LONG val); +    PyObject* BuildPyObject(unsigned PY_LONG_LONG val); +#endif +    PyObject* BuildPyObject(float val); +    PyObject* BuildPyObject(double val); +    PyObject* BuildPyObject(const TStringBuf& val); +    PyObject* BuildPyObject(const char* val); +    PyObject* BuildPyObject(const TWtringBuf& val); +    PyObject* BuildPyObject(const TBuffer& val); +    PyObject* BuildPyObject(bool val); +    PyObject* BuildPyObject(PyObject*); +    PyObject* BuildPyObject(TPyObjectPtr); + +    template <typename T> +    PyObject* BuildPyObject(const TVector<T>& val); + +    template <typename T> +    PyObject* BuildPyObject(const TSet<T>& val); + +    template <typename TKey, typename TVal> +    PyObject* BuildPyObject(const THashMap<TKey, TVal>& val); + +    template <typename T1, typename T2> +    PyObject* BuildPyObject(const std::pair<T1, T2>& val) { +        TPyObjectPtr first(BuildPyObject(val.first), true); +        if (!first) { +            return nullptr; +        } +        TPyObjectPtr second(BuildPyObject(val.second), true); +        if (!first || !second) { +            return nullptr; +        } +        TPyObjectPtr res(PyList_New(2), true); +        PyList_SetItem(res.Get(), 0, first.RefGet()); +        PyList_SetItem(res.Get(), 1, second.RefGet()); +        return res.RefGet(); +    } + +    template <typename T> +    PyObject* BuildPyObject(const TVector<T>& val) { +        TPyObjectPtr res(PyList_New(val.size()), true); +        for (size_t i = 0, size = val.size(); i < size; ++i) { +            auto pythonVal = BuildPyObject(std::move(val[i])); +            if (!pythonVal) { +                return nullptr; +            } +            PyList_SetItem(res.Get(), i, pythonVal); +        } +        return res.RefGet(); +    } + +    template <typename T> +    PyObject* BuildPyObject(TVector<T>&& val) { +        TPyObjectPtr res(PyList_New(val.size()), true); +        for (size_t i = 0, size = val.size(); i < size; ++i) { +            auto pythonVal = BuildPyObject(std::move(val[i])); +            if (!pythonVal) { +                return nullptr; +            } +            PyList_SetItem(res.Get(), i, pythonVal); +        } +        return res.RefGet(); +    } + +    template <typename T> +    PyObject* BuildPyObject(const TSet<T>& val) { +        TPyObjectPtr res(PySet_New(nullptr), true); +        for (const auto& v : val) { +            auto pythonVal = BuildPyObject(std::move(v)); +            if (!pythonVal) { +                return nullptr; +            } +            PySet_Add(res.Get(), pythonVal); +        } +        return res.RefGet(); +    } + +    template <typename T> +    PyObject* BuildPyObject(const THashSet<T>& val) { +        TPyObjectPtr res(PySet_New(nullptr), true); +        for (const auto& v : val) { +            auto pythonVal = BuildPyObject(std::move(v)); +            if (!pythonVal) { +                return nullptr; +            } +            PySet_Add(res.Get(), pythonVal); +        } +        return res.RefGet(); +    } + +    template <typename TKey, typename TVal> +    PyObject* BuildPyObject(const THashMap<TKey, TVal>& val) { +        TPyObjectPtr res(PyDict_New(), true); +        for (typename THashMap<TKey, TVal>::const_iterator it = val.begin(), end = val.end(); it != end; ++it) { +            auto prevOccurred = PyErr_Occurred(); +            Y_UNUSED(prevOccurred); +            TPyObjectPtr k(BuildPyObject(it->first), true); +            if (!k) { +                return nullptr; +            } +            TPyObjectPtr v(BuildPyObject(it->second), true); +            if (!v) { +                return nullptr; +            } +            PyDict_SetItem(res.Get(), k.Get(), v.Get()); +        } +        return res.RefGet(); +    } + +    template <typename TKey, typename TVal> +    PyObject* BuildPyObject(const TMap<TKey, TVal>& val) { +        TPyObjectPtr res(PyDict_New(), true); +        for (typename TMap<TKey, TVal>::const_iterator it = val.begin(), end = val.end(); it != end; ++it) { +            TPyObjectPtr k(BuildPyObject(it->first), true); +            if (!k) { +                return nullptr; +            } +            TPyObjectPtr v(BuildPyObject(it->second), true); +            if (!v) { +                return nullptr; +            } +            PyDict_SetItem(res.Get(), k.Get(), v.Get()); +        } +        return res.RefGet(); +    } + + +    template <typename TKey, typename TVal> +    PyObject* BuildPyObject(const TMultiMap<TKey, TVal>& val) { +        TPyObjectPtr res(PyDict_New(), true); +        TMaybe<TKey> prevKey; +        TPyObjectPtr currentEntry(PyList_New(0), true); +        for (const auto& [key, value]: val) { +            if (prevKey.Defined() && prevKey != key) { +                TPyObjectPtr pyPrevKey(BuildPyObject(*prevKey), true); +                if (!pyPrevKey) { +                    return nullptr; +                } +                PyDict_SetItem(res.Get(), pyPrevKey.Get(), currentEntry.Get()); +                currentEntry = TPyObjectPtr(PyList_New(0), true); +            } +            TPyObjectPtr pyValue(BuildPyObject(value), true); +            if (!pyValue) { +                return nullptr; +            } +            PyList_Append(currentEntry.Get(), pyValue.Get()); +            prevKey = key; +        } + +        if (prevKey.Defined()) { +            TPyObjectPtr pyPrevKey(BuildPyObject(*prevKey), true); +            if (!pyPrevKey) { +                return nullptr; +            } +            PyDict_SetItem(res.Get(), pyPrevKey.Get(), currentEntry.Get()); +        } +        return res.RefGet(); +    } + +    template <typename T> +    PyObject* BuildPyObject(const TMaybe<T>& val) { +        if (!val.Defined()) +            Py_RETURN_NONE; +        return BuildPyObject(val.GetRef()); +    } + +    template <typename T, typename C, typename D> +    PyObject* BuildPyObject(const TSharedPtr<T, C, D>& val) { +        if (!val.Get()) +            Py_RETURN_NONE; +        return BuildPyObject(*val.Get()); +    } + +    template <typename T> +    bool FromPyObject(PyObject* obj, T& res); + +    bool FromPyObject(PyObject* obj, TString& res); +    bool FromPyObject(PyObject* obj, TStringBuf& res); +    bool FromPyObject(PyObject* obj, TUtf16String& res); +    bool FromPyObject(PyObject* obj, TBuffer& res); + +    template <typename T> +    bool FromPyObject(PyObject* obj, TMaybe<T>& res) { +        //we need to save current error before trying derserialize the value +        //because it can produce conversion errors in python that we don't need to handle +        struct TError { +        public: +            TError() { +                PyErr_Fetch(&Type, &Value, &Traceback); +            } +            ~TError() { +                PyErr_Restore(Type, Value, Traceback); + +            } +        private: +            PyObject* Type = nullptr; +            PyObject* Value = nullptr; +            PyObject* Traceback = nullptr; +        } currentPyExcInfo; +        T val; +        if (FromPyObject(obj, val)) { +            res = val; +            return true; +        } +        if (obj == Py_None) { +            res = Nothing(); +            return true; +        } +        return false; +    } + +    template <typename T1, typename T2> +    bool FromPyObject(PyObject* obj, std::pair<T1, T2>& res) { +        PyObject* first; +        PyObject* second; +        if (PyTuple_Check(obj) && 2 == PyTuple_Size(obj)) { +            first = PyTuple_GET_ITEM(obj, 0); +            second = PyTuple_GET_ITEM(obj, 1); +        } else if (PyList_Check(obj) && 2 == PyList_Size(obj)) { +            first = PyList_GET_ITEM(obj, 0); +            second = PyList_GET_ITEM(obj, 1); +        } else { +            return false; +        } +        return FromPyObject(first, res.first) && FromPyObject(second, res.second); +    } + +    template <typename T> +    bool FromPyObject(PyObject* obj, TVector<T>& res) { +        if (!PyList_Check(obj)) +            return false; +        size_t cnt = PyList_Size(obj); +        res.resize(cnt); +        for (size_t i = 0; i < cnt; ++i) { +            PyObject* item = PyList_GET_ITEM(obj, i); +            if (!FromPyObject(item, res[i])) +                return false; +        } +        return true; +    } + +    template <typename K, typename V> +    bool FromPyObject(PyObject* obj, THashMap<K, V>& res) { +        if (!PyDict_Check(obj)) +            return false; +        TPyObjectPtr list(PyDict_Keys(obj), true); +        size_t cnt = PyList_Size(list.Get()); +        for (size_t i = 0; i < cnt; ++i) { +            PyObject* key = PyList_GET_ITEM(list.Get(), i); +            PyObject* value = PyDict_GetItem(obj, key); +            K rkey; +            V rvalue; +            if (!FromPyObject(key, rkey)) +                return false; +            if (!FromPyObject(value, rvalue)) +                return false; +            res[rkey] = rvalue; +        } +        return true; +    } + +    template <typename K, typename V> +    bool FromPyObject(PyObject* obj, TMap<K, V>& res) { +        if (!PyDict_Check(obj)) +            return false; +        TPyObjectPtr list(PyDict_Keys(obj), true); +        size_t cnt = PyList_Size(list.Get()); +        for (size_t i = 0; i < cnt; ++i) { +            PyObject* key = PyList_GET_ITEM(list.Get(), i); +            PyObject* value = PyDict_GetItem(obj, key); +            K rkey; +            V rvalue; +            if (!FromPyObject(key, rkey)) +                return false; +            if (!FromPyObject(value, rvalue)) +                return false; +            res[rkey] = rvalue; +        } +        return true; +    } + +    class cast_exception: public TBadCastException { +    }; + +    template <typename T> +    T FromPyObject(PyObject* obj) { +        T res; +        if (!FromPyObject(obj, res)) +            ythrow cast_exception() << "Cannot cast argument to " << TypeName<T>(); +        return res; +    } + +    template <class... Args, std::size_t... I> +    bool ExtractArgs(std::index_sequence<I...>, PyObject* args, Args&... outArgs) { +        if (!args || !PyTuple_Check(args) || PyTuple_Size(args) != sizeof...(Args)) +            return false; +        bool res = true; +        (void)std::initializer_list<bool>{(res = res && NPyBind::FromPyObject(PyTuple_GET_ITEM(args, I), outArgs))...}; +        return res; +    } + +    template <class... Args> +    bool ExtractArgs(PyObject* args, Args&... outArgs) { +        return ExtractArgs(std::index_sequence_for<Args...>(), args, outArgs...); +    } + +    template <class... Args, std::size_t... I> +    bool ExtractOptionalArgs(std::index_sequence<I...>, PyObject* args, PyObject* kwargs, const char* keywords[], Args&... outArgs) { +        PyObject* pargs[sizeof...(Args)] = {}; +        static const char format[sizeof...(Args) + 2] = {'|', ((void)I, 'O')..., 0}; +        if (!PyArg_ParseTupleAndKeywords(args, kwargs, format, const_cast<char**>(keywords), &pargs[I]...)) +            return false; +        bool res = true; +        (void)std::initializer_list<bool>{(res = res && (!pargs[I] || NPyBind::FromPyObject(pargs[I], outArgs)))...}; +        return res; +    } + +    template <class... Args> +    bool ExtractOptionalArgs(PyObject* args, PyObject* kwargs, const char* keywords[], Args&... outArgs) { +        return ExtractOptionalArgs(std::index_sequence_for<Args...>(), args, kwargs, keywords, outArgs...); +    } + +    template <typename... Args, std::size_t... I> +    static auto GetArguments(std::index_sequence<I...>, PyObject* args) { +        Y_UNUSED(args); // gcc bug +        return std::make_tuple(FromPyObject<std::remove_cv_t<std::remove_reference_t<Args>>>(PyTuple_GetItem(args, I))...); +    } + +    template <typename... Args> +    static auto GetArguments(PyObject* args) { +        return GetArguments<Args...>(std::index_sequence_for<Args...>(), args); +    } + +    inline PyObject* ReturnString(TStringBuf s) { +#if PY_MAJOR_VERSION >= 3 +        return PyUnicode_FromStringAndSize(s.data(), s.size()); +#else +        return PyBytes_FromStringAndSize(s.data(), s.size()); +#endif +    } + +    inline TPyObjectPtr ReturnBytes(TStringBuf s) { +        return TPyObjectPtr(PyBytes_FromStringAndSize(s.data(), s.size()), true); +    } + +    inline TPyObjectPtr NameFromString(TStringBuf s) { +        return TPyObjectPtr(ReturnString(s), true); +    } +} diff --git a/library/cpp/pybind/embedding.cpp b/library/cpp/pybind/embedding.cpp new file mode 100644 index 00000000000..cf8941a92af --- /dev/null +++ b/library/cpp/pybind/embedding.cpp @@ -0,0 +1,63 @@ +#define PY_SSIZE_T_CLEAN +#include <Python.h> + +#include "embedding.h" + +#include <util/generic/ptr.h> +#include <util/generic/yexception.h> + +namespace NPyBind { +#if PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION < 8 +    class TDeleteRawMem { +    public: +        template <typename T> +        static inline void Destroy(T* t) noexcept { +            PyMem_RawFree(t); +        } +    }; + +    template <typename T> +    using TRawMemHolder = THolder<T, TDeleteRawMem>; + +    static void SetProgramName(char* name) { +        TRawMemHolder<wchar_t> wideName(Py_DecodeLocale(name, nullptr)); +        Y_ENSURE(wideName); +        Py_SetProgramName(wideName.Get()); +    } +#endif + +    TEmbedding::TEmbedding(char* argv0) { +#if PY_MAJOR_VERSION < 3 +        Py_SetProgramName(argv0); +        Py_Initialize(); +#elif PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 8 +        PyStatus status; + +        PyConfig config; +        PyConfig_InitPythonConfig(&config); +        // Disable parsing command line arguments +        config.parse_argv = 0; + +        status = PyConfig_SetBytesString(&config, &config.program_name, argv0); +        if (PyStatus_Exception(status)) { +            PyConfig_Clear(&config); +            Py_ExitStatusException(status); +        } + +        status = Py_InitializeFromConfig(&config); +        if (PyStatus_Exception(status)) { +            PyConfig_Clear(&config); +            Py_ExitStatusException(status); +        } + +        PyConfig_Clear(&config); +#elif PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION < 8 +        SetProgramName(argv0); +        Py_Initialize(); +#endif +    } + +    TEmbedding::~TEmbedding() { +        Py_Finalize(); +    } +} diff --git a/library/cpp/pybind/embedding.h b/library/cpp/pybind/embedding.h new file mode 100644 index 00000000000..18553d9f6c0 --- /dev/null +++ b/library/cpp/pybind/embedding.h @@ -0,0 +1,10 @@ +#pragma once + +namespace NPyBind { +    class TEmbedding { +    public: +        TEmbedding(char* argv0); +        ~TEmbedding(); +    }; + +} diff --git a/library/cpp/pybind/empty.cpp b/library/cpp/pybind/empty.cpp new file mode 100644 index 00000000000..10da997ecce --- /dev/null +++ b/library/cpp/pybind/empty.cpp @@ -0,0 +1,2 @@ +#include "init.h" +#include "v2.h" diff --git a/library/cpp/pybind/exceptions.cpp b/library/cpp/pybind/exceptions.cpp new file mode 100644 index 00000000000..db1531fc632 --- /dev/null +++ b/library/cpp/pybind/exceptions.cpp @@ -0,0 +1,147 @@ +#include "exceptions.h" +#include "cast.h" +#include "module.h" +#include <util/generic/algorithm.h> + +namespace NPyBind { + +    namespace NPrivate { +        TPyObjectPtr CreatePyBindModule() { +            return TPyObjectPtr(TExceptionsHolder::DoInitPyBindModule(), true); +        } +    }//NPrivate + +    TPyObjectPtr TExceptionsHolder::GetException(const TString& name) { +        if (name == "") +            return TPyObjectPtr(nullptr); +        if (!Exceptions[name].Get()) +            ythrow yexception() << "Wrong base class '" << name << "'"; +        return Exceptions[name]; +    } + +    TPyObjectPtr TExceptionsHolder::GetExceptions(const TVector<TString>& names) { +        TVector<TString> tmp(names.begin(), names.end()); +        TVector<TString>::iterator end = std::unique(tmp.begin(), tmp.end()); +        TPyObjectPtr tuple(PyTuple_New(std::distance(tmp.begin(), end)), true); +        for (size_t i = 0; i < (size_t)std::distance(tmp.begin(), end); ++i) { +            if (!Exceptions[tmp[i]].Get()) +                ythrow yexception() << "Wrong base class '" << tmp[i] << "'"; +            PyTuple_SetItem(tuple.Get(), i, Exceptions[tmp[i]].Get()); +        } +        return tuple; +    } + +    // def PyBindObjectReconstructor(cl, props): +    //    return cl(__properties__=props) +    static PyObject* PyBindObjectReconstructor(PyObject*, PyObject* args) { +        TPyObjectPtr callable, props; +        if (!ExtractArgs(args, callable, props)) +            ythrow yexception() << "Wrong method arguments"; +#if PY_MAJOR_VERSION >= 3 +        TPyObjectPtr noArgs(PyTuple_New(0), true); +#else +        TPyObjectPtr noArgs(PyList_New(0), true); +#endif +        TPyObjectPtr kw(PyDict_New(), true); +        PyDict_SetItemString(kw.Get(), "__properties__", props.Get()); +        TPyObjectPtr res(PyObject_Call(callable.Get(), noArgs.Get(), kw.Get()), true); +        return res.RefGet(); +    } + +    static PyMethodDef PyBindMethods[] = { +        {"PyBindObjectReconstructor", PyBindObjectReconstructor, METH_VARARGS, "Tech method. It's required for unpickling."}, +        {nullptr, nullptr, 0, nullptr}}; + +#if PY_MAJOR_VERSION >= 3 +    static struct PyModuleDef moduledef = { +                PyModuleDef_HEAD_INIT, +                "pybind", +                NULL, +                -1, +                PyBindMethods, +                NULL, NULL, NULL, NULL +    }; + +    static PyObject* InitPyBind() { +        return PyModule_Create(&moduledef); +    } +#else +    static PyObject* InitPyBind() { +        return Py_InitModule("pybind", PyBindMethods); +    } +#endif + +    void TExceptionsHolder::DoInitPyBindModule2() { +        DoInitPyBindModule(); +    } + +    PyObject* TExceptionsHolder::DoInitPyBindModule() { +        Instance().Module = NPyBind::TPyObjectPtr(InitPyBind(), true); +        if (!Instance().Module.Get()) +            return nullptr; + +        for (TCheckersVector::const_iterator it = Instance().Checkers.begin(), end = Instance().Checkers.end(); it != end; ++it) { +            TString name = (*it)->GetName(); +            if (!!name) { +                //Ref to the object should be incremented before passing to AddObject +                auto res = PyModule_AddObject(Instance().Module.Get(), name.data(), (*it)->GetException().RefGet()); +                if (res < 0) { +                    ythrow yexception() << "Failed to add object " << name << " to internal module pybind"; +                } +            } +        } +        return Instance().Module.RefGet(); +    } + +    void TExceptionsHolder::Clear() { +        //Unfortunately in Python3 we can't retrack this object because of PyError_NewException +        //it's only the safe way to preserve GC gens in valid state during the finalization +        for (auto& ptr: Checkers) { +            if (!dynamic_cast<const TPyErrExceptionsChecker*>(ptr.Get())) {  // no need to untrack standard PyExc_* exceptions from TPyErrExceptionsChecker +                if (auto exceptionPtr = ptr->GetException()) { +                    PyObject_GC_UnTrack(exceptionPtr.Get()); +                } +            } +        } +        Checkers.clear(); +        Exceptions.clear(); +        Module.Drop(); +    } + +    TExceptionsHolder::TExceptionsHolder() { +        AddException<std::exception>("yexception"); +        AddException<TSystemError>("TSystemError", "yexception"); +        AddException<TIoException>("TIoException", "yexception"); + +        TVector<TString> names(2); +        names[0] = "TSystemError"; +        names[1] = "TIoException"; + +        AddException<TIoSystemError>("TIoSystemError", names); +        AddException<TFileError>("TFileError", "TIoSystemError"); +        AddException<TBadCastException>("TBadCastException", "yexception"); + +        Checkers.push_back(new TPyErrExceptionsChecker); + +        // XXX: In Python 2.6, PyImport_AppendInittab() function takes non-const char*, this causes +        // "ISO C++11 does not allow conversion from string literal to 'char *'" warning. +        static char pybind[] = "pybind"; +#if PY_MAJOR_VERSION >= 3 +        PyImport_AppendInittab(pybind, DoInitPyBindModule); + +        NPrivate::AddFinalizationCallBack([this]() { +            Clear(); +        }); +#else +        PyImport_AppendInittab(pybind, DoInitPyBindModule2); +#endif +    } + +    NPyBind::TPyObjectPtr TExceptionsHolder::ToPyException(const std::exception& ex) { +        for (TCheckersVector::const_reverse_iterator it = Checkers.rbegin(), end = Checkers.rend(); it != end; ++it) { +            if ((*it)->Check(ex)) +                return (*it)->GetException(); +        } +        return TPyObjectPtr(nullptr); +    } +} diff --git a/library/cpp/pybind/exceptions.h b/library/cpp/pybind/exceptions.h new file mode 100644 index 00000000000..48e20995e46 --- /dev/null +++ b/library/cpp/pybind/exceptions.h @@ -0,0 +1,143 @@ +#pragma once + +#define PY_SSIZE_T_CLEAN +#include <Python.h> +#include <util/generic/yexception.h> +#include <util/generic/map.h> +#include <util/generic/vector.h> +#include "ptr.h" + +namespace NPyBind { +    // Usage: +    //   ythrow TPyErr(PyExc_TypeError) << "some python type error somewhere in your C++ code"; +    // +    class TPyErr: public virtual yexception { +    public: +        TPyErr(PyObject* theException = PyExc_RuntimeError) +            : Exception(theException) +        { +        } + +        TPyObjectPtr GetException() const { +            return Exception; +        } + +    private: +        NPyBind::TPyObjectPtr Exception; +    }; + +    //Private api for creating PyBind python module +    //Needs only for overriding pybind python module in library which imports other pybind library +    namespace NPrivate { +        TPyObjectPtr CreatePyBindModule(); +    }//NPrivate +    class TExceptionsHolder { +        friend TPyObjectPtr NPrivate::CreatePyBindModule(); +    private: +        TExceptionsHolder(const TExceptionsHolder&); +        TExceptionsHolder& operator=(const TExceptionsHolder&); +        TExceptionsHolder(); + +        void Clear(); +        TPyObjectPtr GetException(const TString&); +        TPyObjectPtr GetExceptions(const TVector<TString>&); +    private: +        class TExceptionsChecker { +        public: +            virtual ~TExceptionsChecker() { +            } +            virtual bool Check(const std::exception& ex) const = 0; +            virtual TString GetName() const = 0; +            virtual TPyObjectPtr GetException() const = 0; +        }; + +        template <typename TExcType> +        class TConcreteExceptionsChecker: public TExceptionsChecker { +        private: +            TString Name; +            TPyObjectPtr Exception; + +        public: +            TConcreteExceptionsChecker(const TString& name, TPyObjectPtr exception) +                : Name(name) +                , Exception(exception) +            { +            } + +            bool Check(const std::exception& ex) const override { +                const std::exception* e = &ex; +                return dynamic_cast<const TExcType*>(e); +            } + +            TString GetName() const override { +                return Name; +            } + +            TPyObjectPtr GetException() const override { +                return Exception; +            } +        }; + +        class TPyErrExceptionsChecker: public TExceptionsChecker { +        private: +            mutable TPyObjectPtr Exception; + +        public: +            TPyErrExceptionsChecker() { +            } + +            bool Check(const std::exception& ex) const override { +                const TPyErr* err = dynamic_cast<const TPyErr*>(&ex); +                if (err) { +                    Exception = err->GetException(); +                } +                return err != nullptr; +            } + +            TString GetName() const override { +                return TString(); +            } + +            TPyObjectPtr GetException() const override { +                return Exception; +            } +        }; + +        typedef TSimpleSharedPtr<TExceptionsChecker> TCheckerPtr; +        typedef TVector<TCheckerPtr> TCheckersVector; +        typedef TMap<TString, TPyObjectPtr> TExceptionsMap; + +        TPyObjectPtr Module; +        TCheckersVector Checkers; +        TExceptionsMap Exceptions; + +        static PyObject* DoInitPyBindModule(); +        static void DoInitPyBindModule2(); + +    public: +        static TExceptionsHolder& Instance() { +            static TExceptionsHolder Holder; +            return Holder; +        } + +        template <typename TExcType> +        void AddException(const TString& name, const TString& base = "") { +            TPyObjectPtr baseException(GetException(base)); +            TString fullName = TString("pybind.") + name; +            TPyObjectPtr exception(PyErr_NewException(const_cast<char*>(fullName.c_str()), baseException.Get(), nullptr), true); +            Checkers.push_back(new TConcreteExceptionsChecker<TExcType>(name, exception)); +            Exceptions[name] = exception; +        } + +        template <typename TExcType> +        void AddException(const TString& name, const TVector<TString>& bases) { +            TPyObjectPtr baseExceptions(GetExceptions(bases)); +            TString fullName = TString("pybind.") + name; +            TPyObjectPtr exception(PyErr_NewException(const_cast<char*>(fullName.c_str()), baseExceptions.Get(), nullptr), true); +            Checkers.push_back(new TConcreteExceptionsChecker<TExcType>(name, exception)); +            Exceptions[name] = exception; +        } + +        NPyBind::TPyObjectPtr ToPyException(const std::exception&); +    }; +} diff --git a/library/cpp/pybind/init.h b/library/cpp/pybind/init.h new file mode 100644 index 00000000000..58874574ed6 --- /dev/null +++ b/library/cpp/pybind/init.h @@ -0,0 +1,25 @@ +#pragma once + +#define PY_SSIZE_T_CLEAN +#include <Python.h> + +#include "ptr.h" + +namespace NPyBind { +#if PY_MAJOR_VERSION >= 3 + +#define PYBIND_MODINIT(name) PyMODINIT_FUNC PyInit_##name() + +    inline PyObject* ModInitReturn(TPyObjectPtr&& modptr) { +        return modptr.Release(); +    } + +#else + +#define PYBIND_MODINIT(name) PyMODINIT_FUNC init##name() + +    inline void ModInitReturn(TPyObjectPtr&&) { +    } + +#endif +} diff --git a/library/cpp/pybind/method.h b/library/cpp/pybind/method.h new file mode 100644 index 00000000000..7c1f6e90e1b --- /dev/null +++ b/library/cpp/pybind/method.h @@ -0,0 +1,439 @@ +#pragma once + +#define PY_SSIZE_T_CLEAN +#include <Python.h> +#include <util/generic/string.h> +#include <util/generic/map.h> +#include <util/generic/set.h> +#include <util/generic/vector.h> +#include <util/generic/ptr.h> +#include <util/generic/typetraits.h> + +#include <util/generic/function.h> + +#include "cast.h" + +namespace NPyBind { +    template <typename TObjType> +    class TBaseMethodCaller { +    public: +        virtual ~TBaseMethodCaller() { +        } +        virtual bool CallMethod(PyObject* owner, TObjType* self, PyObject* args, PyObject* kwargs, PyObject*& res) const = 0; +        virtual bool HasMethod(PyObject*, TObjType*, const TString&, const TSet<TString>&) { +            return true; +        } +    }; + +    template <typename TObjType> +    class TIsACaller; + +    template <typename TObjType> +    class TMethodCallers { +    private: +        typedef TSimpleSharedPtr<TBaseMethodCaller<TObjType>> TCallerPtr; +        typedef TVector<TCallerPtr> TCallerList; +        typedef TMap<TString, TCallerList> TCallerMap; + +        const TSet<TString>& HiddenAttrNames; +        TCallerMap Callers; + +    public: +        TMethodCallers(const TSet<TString>& hiddenNames) +            : HiddenAttrNames(hiddenNames) +        { +        } + +        void AddCaller(const TString& name, TCallerPtr caller) { +            Callers[name].push_back(caller); +        } + +        bool HasCaller(const TString& name) const { +            return Callers.has(name); +        } + +        PyObject* CallMethod(PyObject* owner, TObjType* self, PyObject* args, PyObject* kwargs, const TString& name) const { +            const TCallerList* lst = Callers.FindPtr(name); +            if (!lst) +                return nullptr; +            for (const auto& caller : *lst) { +                PyObject* res = nullptr; +                PyErr_Clear(); +                if (caller->CallMethod(owner, self, args, kwargs, res)) +                    return res; +            } +            return nullptr; +        } + +        bool HasMethod(PyObject* owner, TObjType* self, const TString& name) const { +            const TCallerList* lst = Callers.FindPtr(name); +            if (!lst) +                return false; +            for (const auto& caller : *lst) { +                if (caller->HasMethod(owner, self, name, HiddenAttrNames)) +                    return true; +            } +            return false; +        } + +        void GetMethodsNames(PyObject* owner, TObjType* self, TVector<TString>& resultNames) const { +            for (const auto& it : Callers) { +                if (HasMethod(owner, self, it.first) && !HiddenAttrNames.contains(it.first)) +                    resultNames.push_back(it.first); +            } +        } + +        void GetAllMethodsNames(TVector<TString>& resultNames) const { +            for (const auto& it : Callers) { +                resultNames.push_back(it.first); +            } +        } + +        void GetPropertiesNames(PyObject*, TObjType* self, TVector<TString>& resultNames) const { +            const TCallerList* lst = Callers.FindPtr("IsA"); +            if (!lst) +                return; +            for (const auto& caller : *lst) { +                TIsACaller<TObjType>* isACaller = dynamic_cast<TIsACaller<TObjType>*>(caller.Get()); +                if (isACaller) { +                    resultNames = isACaller->GetPropertiesNames(self); +                    return; +                } +            } +        } +    }; + +    template <typename TObjType> +    class TIsACaller: public TBaseMethodCaller<TObjType> { +    private: +        class TIsAChecker { +        public: +            virtual ~TIsAChecker() { +            } +            virtual bool Check(const TObjType* obj) const = 0; +        }; + +        template <typename TConcrete> +        class TIsAConcreteChecker: public TIsAChecker { +        public: +            bool Check(const TObjType* obj) const override { +                return dynamic_cast<const TConcrete*>(obj) != nullptr; +            } +        }; + +        typedef TSimpleSharedPtr<TIsAChecker> TCheckerPtr; +        typedef TMap<TString, TCheckerPtr> TCheckersMap; + +        TCheckersMap Checkers; + +        bool Check(const TString& name, const TObjType* obj) const { +            const TCheckerPtr* checker = Checkers.FindPtr(name); +            if (!checker) { +                PyErr_Format(PyExc_KeyError, "unknown class name: %s", name.data()); +                return false; +            } +            return (*checker)->Check(obj); +        } + +    protected: +        TIsACaller() { +        } + +        template <typename TConcrete> +        void AddChecker(const TString& name) { +            Checkers[name] = new TIsAConcreteChecker<TConcrete>; +        } + +    public: +        bool CallMethod(PyObject*, TObjType* self, PyObject* args, PyObject*, PyObject*& res) const override { +            if (args == nullptr || !PyTuple_Check(args)) +                return false; +            size_t cnt = PyTuple_Size(args); +            bool result = true; +            for (size_t i = 0; i < cnt; ++i) { +                result = result && Check( +#if PY_MAJOR_VERSION >= 3 +                        PyUnicode_AsUTF8( +#else +                        PyString_AsString( +#endif +                            PyTuple_GetItem(args, i)), self); +            } +            if (PyErr_Occurred()) { +                return false; +            } +            res = BuildPyObject(result); +            return true; +        } + +        TVector<TString> GetPropertiesNames(const TObjType* obj) const { +            TVector<TString> names; + +            for (const auto& it : Checkers) { +                if (it.second->Check(obj)) { +                    names.push_back(it.first); +                } +            } + +            return names; +        } +    }; + +    template <typename TObjType> +    class TGenericMethodCaller: public TBaseMethodCaller<TObjType> { +    private: +        TString AttrName; + +    public: +        TGenericMethodCaller(const TString& attrName) +            : AttrName(attrName) +        { +        } + +        bool CallMethod(PyObject* obj, TObjType*, PyObject* args, PyObject*, PyObject*& res) const override { +            auto str = NameFromString(AttrName); +            PyObject* attr = PyObject_GenericGetAttr(obj, str.Get()); +            if (!attr) +                ythrow yexception() << "Can't get generic attribute '" << AttrName << "'"; +            res = PyObject_CallObject(attr, args); +            return res != nullptr; +        } +    }; + + +    template <typename TObjType, typename TSubObject> +    class TSubObjectChecker: public TBaseMethodCaller<TObjType> { +    public: +        ~TSubObjectChecker() override { +        } + +        bool HasMethod(PyObject*, TObjType* self, const TString&, const TSet<TString>&) override { +            return dynamic_cast<const TSubObject*>(self) != nullptr; +        } +    }; + +    template <typename TFunctor, typename Tuple, typename ResType, typename=std::enable_if_t<!std::is_same_v<ResType, void>>> +    void ApplyFunctor(TFunctor functor, Tuple resultArgs, PyObject*& res) { +        res = BuildPyObject(std::move(Apply(functor, resultArgs))); +    } + +    template <typename TFunctor, typename Tuple, typename ResType, typename=std::enable_if_t<std::is_same_v<ResType, void>>, typename=void> +    void ApplyFunctor(TFunctor functor, Tuple resultArgs, PyObject*& res) { +        Py_INCREF(Py_None); +        res = Py_None; +        Apply(functor, resultArgs); +    } + +    template <typename TObjType, typename TResType, typename... Args> +    class TFunctorCaller: public TBaseMethodCaller<TObjType> { +        using TFunctor = std::function<TResType(TObjType&,Args...)>; +        TFunctor Functor; +    public: +        explicit TFunctorCaller(TFunctor functor): +            Functor(functor){} + +        bool CallMethod(PyObject*, TObjType* self, PyObject* args, PyObject*, PyObject*& res) const { +            auto methodArgsTuple = GetArguments<Args...>(args); +            auto resultArgs = std::tuple_cat(std::tie(*self), methodArgsTuple); +            ApplyFunctor<TFunctor, decltype(resultArgs), TResType>(Functor, resultArgs, res); +            return true; +        } +    }; + +    template <typename TObjType, typename TRealType> +    class TGetStateCaller: public TSubObjectChecker<TObjType, TRealType> { +    protected: +        TPyObjectPtr AddFromCaller(PyObject* obj, const TString& methodName) const { +            PyObject* res = PyObject_CallMethod(obj, const_cast<char*>(methodName.c_str()), const_cast<char*>("")); +            if (!res) { +                PyErr_Clear(); +                return TPyObjectPtr(Py_None); +            } +            return TPyObjectPtr(res, true); +        } + +        void GetStandartAttrsDictionary(PyObject* obj, TRealType*, TMap<TString, TPyObjectPtr>& dict) const { +            TPyObjectPtr attrsDict(PyObject_GetAttrString(obj, "__dict__"), true); +            TMap<TString, TPyObjectPtr> attrs; +            if (!FromPyObject(attrsDict.Get(), attrs)) +                ythrow yexception() << "Can't get '__dict__' attribute"; +            dict.insert(attrs.begin(), attrs.end()); +        } + +        virtual void GetAttrsDictionary(PyObject* obj, TRealType* self, TMap<TString, TPyObjectPtr>& dict) const = 0; + +    public: +        bool CallMethod(PyObject* obj, TObjType* self, PyObject* args, PyObject*, PyObject*& res) const override { +            if (!ExtractArgs(args)) +                ythrow yexception() << "Can't parse arguments: it should be none"; +            TRealType* rself = dynamic_cast<TRealType*>(self); +            if (!rself) +                return false; +            TMap<TString, TPyObjectPtr> dict; +            GetAttrsDictionary(obj, rself, dict); +            res = BuildPyObject(dict); +            return true; +        } +    }; + +    template <typename TObjType, typename TRealType> +    class TSetStateCaller: public TSubObjectChecker<TObjType, TRealType> { +    protected: +        void SetStandartAttrsDictionary(PyObject* obj, TRealType*, TMap<TString, TPyObjectPtr>& dict) const { +            TPyObjectPtr value(BuildPyObject(dict), true); +            PyObject_SetAttrString(obj, "__dict__", value.Get()); +        } + +        virtual void SetAttrsDictionary(PyObject* obj, TRealType* self, TMap<TString, TPyObjectPtr>& dict) const = 0; + +    public: +        bool CallMethod(PyObject* obj, TObjType* self, PyObject* args, PyObject*, PyObject*& res) const override { +            TMap<TString, TPyObjectPtr> dict; +            if (!ExtractArgs(args, dict)) +                ythrow yexception() << "Can't parse arguments: it should be one dictionary"; +            TRealType* rself = dynamic_cast<TRealType*>(self); +            if (!rself) +                return false; +            SetAttrsDictionary(obj, rself, dict); +            Py_INCREF(Py_None); +            res = Py_None; +            return true; +        } +    }; + +    template <typename TObjType, typename TResult, typename TSubObject, typename TMethod, typename... Args> +    class TAnyParameterMethodCaller: public TSubObjectChecker<TObjType, TSubObject> { +    private: +        TMethod Method; + +    public: +        TAnyParameterMethodCaller(TMethod method) +            : Method(method) +        { +        } + +    public: +        bool CallMethod(PyObject*, TObjType* self, PyObject* args, PyObject*, PyObject*& res) const override { +            TSubObject* sub = dynamic_cast<TSubObject*>(self); +            if (sub == nullptr) +                return false; +            if (args && (!PyTuple_Check(args) || PyTuple_Size(args) != TFunctionArgs<TMethod>::Length)) { +                //ythrow yexception() << "Method takes " << (size_t)(TFunctionArgs<TMethod>::Length) << " arguments, " << PyTuple_Size(args) << " provided"; +                return false; +            } + +            try { +                class Applicant { +                public: +                    TResult operator()(Args... theArgs) { +                        return (Sub->*Method)(theArgs...); +                    } +                    TSubObject* Sub; +                    TMethod Method; +                }; +                res = BuildPyObject(std::move(Apply(Applicant{sub, Method}, GetArguments<Args...>(args)))); +            } catch (cast_exception) { +                return false; +            } catch (...) { +                if (PyExc_StopIteration == PyErr_Occurred()) { +                    // NB: it's replacement for geo_boost::python::throw_error_already_set(); +                    return true; +                } +                PyErr_SetString(PyExc_RuntimeError, CurrentExceptionMessage().data()); +                return true; +            } + +            return true; +        } +    }; + +    template <typename TObjType, typename TSubObject, typename TMethod, typename... Args> +    class TAnyParameterMethodCaller<TObjType, void, TSubObject, TMethod, Args...>: public TSubObjectChecker<TObjType, TSubObject> { +    private: +        TMethod Method; + +    public: +        TAnyParameterMethodCaller(TMethod method) +            : Method(method) +        { +        } + +    public: +        bool CallMethod(PyObject*, TObjType* self, PyObject* args, PyObject*, PyObject*& res) const override { +            TSubObject* sub = dynamic_cast<TSubObject*>(self); +            if (sub == nullptr) { +                return false; +            } +            if (args && (!PyTuple_Check(args) || PyTuple_Size(args) != TFunctionArgs<TMethod>::Length)) { +                return false; +            } + +            try { +                class Applicant { +                public: +                    void operator()(Args... theArgs) { +                        (Sub->*Method)(theArgs...); +                    } +                    TSubObject* Sub; +                    TMethod Method; +                }; + +                Apply(Applicant{sub, Method}, GetArguments<Args...>(args)); + +                Py_INCREF(Py_None); +                res = Py_None; +            } catch (cast_exception) { +                return false; +            } catch (...) { +                PyErr_SetString(PyExc_RuntimeError, CurrentExceptionMessage().data()); +                return true; +            } + +            return true; +        } +    }; + +    template <typename TResult, typename TSubObject, typename... Args> +    struct TConstTraits { +        typedef TResult (TSubObject::*TMethod)(Args... args) const; +    }; + +    template <typename TResult, typename TSubObject, typename... Args> +    struct TNonConstTraits { +        typedef TResult (TSubObject::*TMethod)(Args... args); +    }; + +    template <typename TObjType, typename TResult, typename TSubObject, typename TMethod, typename... Args> +    class TConstMethodCaller: public TAnyParameterMethodCaller<TObjType, TResult, const TSubObject, typename TConstTraits<TResult, TSubObject, Args...>::TMethod, Args...> { +    public: +        TConstMethodCaller(typename TConstTraits<TResult, TSubObject, Args...>::TMethod method) +            : TAnyParameterMethodCaller<TObjType, TResult, const TSubObject, typename TConstTraits<TResult, TSubObject, Args...>::TMethod, Args...>(method) +        { +        } +    }; + +    template <typename TObjType, typename TResult, typename TSubObject, typename... Args> +    TSimpleSharedPtr<TBaseMethodCaller<TObjType>> CreateConstMethodCaller(TResult (TSubObject::*method)(Args...) const) { +        return new TConstMethodCaller<TObjType, TResult, TSubObject, TResult (TSubObject::*)(Args...) const, Args...>(method); +    } + +    template <typename TObjType, typename TResType, typename... Args> +    TSimpleSharedPtr<TBaseMethodCaller<TObjType>> CreateFunctorCaller(std::function<TResType(TObjType&, Args...)> functor) { +        return new TFunctorCaller<TObjType, TResType, Args...>(functor); +    } + +    template <typename TObjType, typename TResult, typename TSubObject, typename TMethod, typename... Args> +    class TMethodCaller: public TAnyParameterMethodCaller<TObjType, TResult, TSubObject, typename TNonConstTraits<TResult, TSubObject, Args...>::TMethod, Args...> { +    public: +        TMethodCaller(typename TNonConstTraits<TResult, TSubObject, Args...>::TMethod method) +            : TAnyParameterMethodCaller<TObjType, TResult, TSubObject, typename TNonConstTraits<TResult, TSubObject, Args...>::TMethod, Args...>(method) +        { +        } +    }; + +    template <typename TObjType, typename TResult, typename TSubObject, typename... Args> +    TSimpleSharedPtr<TBaseMethodCaller<TObjType>> CreateMethodCaller(TResult (TSubObject::*method)(Args...)) { +        return new TMethodCaller<TObjType, TResult, TSubObject, TResult (TSubObject::*)(Args...), Args...>(method); +    } + +} diff --git a/library/cpp/pybind/module.cpp b/library/cpp/pybind/module.cpp new file mode 100644 index 00000000000..b7b003d3b08 --- /dev/null +++ b/library/cpp/pybind/module.cpp @@ -0,0 +1,72 @@ +#include "module.h" +#include "ptr.h" + +#include <util/generic/adaptor.h> + +namespace NPyBind { + +#if PY_MAJOR_VERSION >= 3 +    namespace NPrivate { +        struct TFinCallBacksHolder { +            static TVector<TFinalizationCallBack>& GetCallBacks() { +                static TVector<TFinalizationCallBack> res; +                return res; +            } +        }; + +        TAtExitRegistrar::TAtExitRegistrar(TPyObjectPtr module) { +            TPyObjectPtr atExitModuleName(Py_BuildValue("s", "atexit"), true); +            TPyObjectPtr atExitModule(PyImport_Import(atExitModuleName.Get())); +            Y_VERIFY(atExitModule); +            TPyObjectPtr finalizerFunc(PyObject_GetAttrString(module.Get(), "finalizer"), true); +            Y_VERIFY(finalizerFunc); +            TPyObjectPtr registerName(Py_BuildValue("s", "register"), true); +            PyObject_CallMethodObjArgs(atExitModule.Get(), registerName.Get(), finalizerFunc.Get(), nullptr); +        } + +        TPyBindModuleRegistrar::TPyBindModuleRegistrar() { +            TPyObjectPtr modules(PySys_GetObject("modules")); +            Y_ENSURE(modules.Get()); +            if (Module = NPrivate::CreatePyBindModule()) { +                Y_VERIFY(0 == PyDict_SetItemString(modules.Get(), "pybind", Module.RefGet())); +            } +            AddFinalizationCallBack([this]() { +                auto ptr = Module; +                Y_UNUSED(ptr); +                TPyObjectPtr modules(PySys_GetObject("modules")); +                Y_ENSURE(modules.Get()); +                TPyObjectPtr pyBindName(Py_BuildValue("s", "pybind")); +                if (PyDict_Contains(modules.Get(), pyBindName.Get()) == 1) { +                    Y_VERIFY(0==PyDict_DelItemString(modules.Get(), "pybind")); +                } +                if (Module) { +                    //We have to untrack the module because some refs from him refers to gc-leaked errors +                    //see exceptions.cpp fore more info +                    PyObject_GC_UnTrack(Module.Get()); +                    Module.Drop(); +                } +            }); +        } + +        void AddFinalizationCallBack(TFinalizationCallBack callback) { +            TFinCallBacksHolder::GetCallBacks().push_back(callback); +        } + +        int FinalizeAll() { +            for (auto callback: Reversed(NPrivate::TFinCallBacksHolder::GetCallBacks())) { +                callback(); +            } +            return 0; +        } +    } +#endif + + +    TModuleHolder::TModuleHolder() +        : Methods(1, new TVector<TMethodDef>) +    { +#if PY_MAJOR_VERSION >= 3 +        AddModuleMethod<TModuleMethodCaller<decltype(&NPrivate::FinalizeAll), &NPrivate::FinalizeAll>::Call>("finalizer"); +#endif +    } +}//NPyBind diff --git a/library/cpp/pybind/module.h b/library/cpp/pybind/module.h new file mode 100644 index 00000000000..41dcb4dfec8 --- /dev/null +++ b/library/cpp/pybind/module.h @@ -0,0 +1,176 @@ +#pragma once + +#define PY_SSIZE_T_CLEAN +#include <Python.h> +#include "ptr.h" +#include "cast.h" +#include "exceptions.h" + +#include <util/generic/function.h> + +namespace NPyBind { +#if PY_MAJOR_VERSION >= 3 +    namespace NPrivate { +        using TFinalizationCallBack = std::function<void()>; +        void AddFinalizationCallBack(TFinalizationCallBack); +        class TAtExitRegistrar: private TNonCopyable { +            TAtExitRegistrar(TPyObjectPtr module); +        public: +            static void Instantiate(TPyObjectPtr module) { +                static TAtExitRegistrar registrar(module); +                Y_UNUSED(registrar); +            } +        }; + +        class TPyBindModuleRegistrar: private TNonCopyable { +            TPyBindModuleRegistrar(); +            TPyObjectPtr Module; +        public: +            static void Instantiate() { +                static TPyBindModuleRegistrar registrar; +                Y_UNUSED(registrar); +            } +        }; +    } //NPrivate +#endif + +    class TModuleHolder { +    private: +        TModuleHolder(const TModuleHolder&); +        TModuleHolder& operator=(const TModuleHolder&); + +        TModuleHolder(); +    private: +        typedef PyCFunction TModuleMethod; +#if PY_MAJOR_VERSION >= 3 +        typedef PyObject* (*TModuleInitFunc)(); +#else +        typedef void (*TModuleInitFunc)(); +#endif + +        struct TMethodDef { +            TString Name; +            TModuleMethod Method; +            TString Description; +            int Flags; + +            TMethodDef(const TString& name, TModuleMethod method, const TString& descr, int flags) +                : Name(name) +                , Method(method) +                , Description(descr) +                , Flags(flags) +            { +            } + +            operator PyMethodDef() const { +                PyMethodDef cur = {Name.c_str(), Method, Flags, Description.c_str()}; +                return cur; +            } +        }; + +        typedef TSimpleSharedPtr<TVector<TMethodDef>> TMethodDefVecPtr; +        typedef TSimpleSharedPtr<TVector<PyMethodDef>> TPyMethodDefVecPtr; + +        TVector<TMethodDefVecPtr> Methods; +        TVector<TPyMethodDefVecPtr> Defs; +#if PY_MAJOR_VERSION >= 3 +        //because the md_name will leak otherwise +        class TPyModuleDefWithName { +            PyModuleDef Def; +            TString Name; +        public: +            explicit TPyModuleDefWithName(TString name, TPyMethodDefVecPtr moduleDefs) +                : Name(std::move(name)) +            { +                Def = PyModuleDef{ +                    PyModuleDef_HEAD_INIT, +                    Name.c_str(), +                    nullptr, +                    -1, +                    moduleDefs->data(), +                    nullptr, nullptr, nullptr, nullptr +                }; +            } +            PyModuleDef* GetDefPtr() { +                return &Def; +            } + +        }; +        TVector<TSimpleSharedPtr<TPyModuleDefWithName>> ModuleDefs; +#endif + +        template <TModuleMethod method> +        static PyObject* MethodWrapper(PyObject* obj, PyObject* args) { +            try { +                PyObject* res = method(obj, args); +                if (!res && !PyErr_Occurred()) +                    ythrow yexception() << "\nModule method exited with NULL, but didn't set Error.\n Options:\n -- Return correct value or None;\n -- Set python exception;\n -- Throw c++ exception."; +                return res; +            } catch (const std::exception& ex) { +                PyErr_SetString(TExceptionsHolder::Instance().ToPyException(ex).Get(), ex.what()); +            } catch (...) { +                PyErr_SetString(PyExc_RuntimeError, "Unknown error occurred while trying to call module method"); +            } +            return nullptr; +        } + +    public: +        static TModuleHolder& Instance() { +            static TModuleHolder Holder; +            return Holder; +        } + +        void ImportModule(TPyObjectPtr module, const char* const name, TModuleInitFunc initFunc) { +            PyImport_AppendInittab(const_cast<char*>(name), initFunc); +            TPyObjectPtr importedModule(PyImport_ImportModule(name), true); +            PyModule_AddObject(module.Get(), name, importedModule.Get()); +        } + +        template <TModuleMethod method> +        void AddModuleMethod(const TString& name, const TString& descr = "") { +            Methods.back()->push_back(TMethodDef(name, MethodWrapper<method>, descr, METH_VARARGS)); +        } + +        TPyObjectPtr InitModule(const TString& name) { +            Defs.push_back(new TVector<PyMethodDef>(Methods.back()->begin(), Methods.back()->end())); +            PyMethodDef blank = {nullptr, nullptr, 0, nullptr}; +            Defs.back()->push_back(blank); +#if PY_MAJOR_VERSION >= 3 +            ModuleDefs.push_back(MakeSimpleShared<TPyModuleDefWithName>(name, Defs.back())); +            TPyObjectPtr res(PyModule_Create(ModuleDefs.back()->GetDefPtr())); +            NPrivate::TAtExitRegistrar::Instantiate(res); +            NPrivate::TPyBindModuleRegistrar::Instantiate(); +#else +            TPyObjectPtr res(Py_InitModule(name.c_str(), &(Defs.back()->at(0)))); +#endif +            Methods.push_back(new TVector<TMethodDef>); +            return res; +        } +    }; + +    template <typename TMethodSignature, TMethodSignature method> +    class TModuleMethodCaller { +    private: +        template <typename TResult, typename... Args> +        struct TCaller { +            static PyObject* Call(PyObject* args) { +                return BuildPyObject(Apply(method, GetArguments<Args...>(args))); +            } +        }; + +        template <typename TResult, typename... Args> +        static PyObject* InternalCall(TResult (*)(Args...), PyObject* args) { +            return BuildPyObject(Apply(method, GetArguments<Args...>(args))); +        } + +    public: +        static PyObject* Call(PyObject*, PyObject* args) { +            if (args && (!PyTuple_Check(args) || PyTuple_Size(args) != TFunctionArgs<TMethodSignature>::Length)) { +                ythrow yexception() << "Method takes " << (size_t)(TFunctionArgs<TMethodSignature>::Length) << " arguments, " << PyTuple_Size(args) << " provided"; +            } + +            return InternalCall(method, args); +        } +    }; + +} diff --git a/library/cpp/pybind/pod.cpp b/library/cpp/pybind/pod.cpp new file mode 100644 index 00000000000..3cf030e537b --- /dev/null +++ b/library/cpp/pybind/pod.cpp @@ -0,0 +1,18 @@ +#include "pod.h" + +namespace NPyBind { +    class TPODAttrGetter: public TBaseAttrGetter<TPOD> { +    public: +        bool GetAttr(PyObject*, const TPOD& self, const TString& attr, PyObject*& res) const override { +            res = self.GetAttr(attr.c_str()); +            return res != nullptr; +        } +    }; + +    TPODTraits::TPODTraits() +        : MyParent("TPOD", "simple struct") +    { +        AddGetter("", new TPODAttrGetter); +    } + +} diff --git a/library/cpp/pybind/pod.h b/library/cpp/pybind/pod.h new file mode 100644 index 00000000000..90165fdbec6 --- /dev/null +++ b/library/cpp/pybind/pod.h @@ -0,0 +1,53 @@ +#pragma once + +#define PY_SSIZE_T_CLEAN +#include <Python.h> +#include "attr.h" +#include "typedesc.h" + +namespace NPyBind { +    struct TPOD { +        TPyObjectPtr Dict; + +        TPOD() +            : Dict(PyDict_New(), true) +        { +        } +        bool SetAttr(const char* name, PyObject* value) { +            return PyDict_SetItemString(Dict.Get(), name, value) == 0; +        } +        PyObject* GetAttr(const char* name) const { +            PyObject* res = PyDict_GetItemString(Dict.Get(), name); +            Py_XINCREF(res); +            return res; +        } +    }; + +    class TPODTraits: public NPyBind::TPythonType<TPOD, TPOD, TPODTraits> { +    private: +        typedef TPythonType<TPOD, TPOD, TPODTraits> MyParent; +        friend class TPythonType<TPOD, TPOD, TPODTraits>; +        TPODTraits(); + +    public: +        static TPOD* GetObject(TPOD& obj) { +            return &obj; +        } +    }; + +    template <> +    inline bool FromPyObject<TPOD*>(PyObject* obj, TPOD*& res) { +        res = TPODTraits::CastToObject(obj); +        if (res == nullptr) +            return false; +        return true; +    } +    template <> +    inline bool FromPyObject<const TPOD*>(PyObject* obj, const TPOD*& res) { +        res = TPODTraits::CastToObject(obj); +        if (res == nullptr) +            return false; +        return true; +    } + +} diff --git a/library/cpp/pybind/ptr.h b/library/cpp/pybind/ptr.h new file mode 100644 index 00000000000..e1367366904 --- /dev/null +++ b/library/cpp/pybind/ptr.h @@ -0,0 +1,51 @@ +#pragma once + +#define PY_SSIZE_T_CLEAN +#include <Python.h> +#include <util/generic/ptr.h> + +namespace NPyBind { +    template <class T> +    class TPythonIntrusivePtrOps { +    public: +        static inline void Ref(T* t) noexcept { +            Py_XINCREF(t); +        } + +        static inline void UnRef(T* t) noexcept { +            Py_XDECREF(t); +        } + +        static inline void DecRef(T* t) noexcept { +            Py_XDECREF(t); +        } +    }; + +    class TPyObjectPtr: public TIntrusivePtr<PyObject, TPythonIntrusivePtrOps<PyObject>> { +    private: +        typedef TIntrusivePtr<PyObject, TPythonIntrusivePtrOps<PyObject>> TParent; +        typedef TPythonIntrusivePtrOps<PyObject> TOps; + +    public: +        inline TPyObjectPtr() noexcept { +        } + +        inline explicit TPyObjectPtr(PyObject* obj) noexcept +            : TParent(obj) +        { +        } + +        inline TPyObjectPtr(PyObject* obj, bool unref) noexcept +            : TParent(obj) +        { +            if (unref) +                TOps::UnRef(TParent::Get()); +        } + +        inline PyObject* RefGet() { +            TOps::Ref(TParent::Get()); +            return TParent::Get(); +        } +    }; + +} diff --git a/library/cpp/pybind/typeattrs.h b/library/cpp/pybind/typeattrs.h new file mode 100644 index 00000000000..a906b9ec2b0 --- /dev/null +++ b/library/cpp/pybind/typeattrs.h @@ -0,0 +1,368 @@ +#pragma once + +#include "ptr.h" +#include "cast.h" +#include "attr.h" +#include "method.h" + +#include <util/generic/vector.h> + +namespace NPyBind { +    template <typename TObject> +    class TPythonTypeAttributes { +    private: +        TAttrGetters<TObject> AttrGetters; +        TAttrSetters<TObject> AttrSetters; +        TMethodCallers<TObject> MethodCallers; + +        class TGetAttrsNamesCaller; +        class TGetMethodsNamesCaller; +        class TGetAllNamesCaller; +        class TGetPropertiesNamesCaller; +        class TDictAttrGetter; +        class TDictAttrSetter; +        class TGetAttributeMethodCaller; +        class TSetAttrMethodCaller; +        class TGetStrReprMethodCaller; +        class TReduceMethodCaller; +        class TBaseGetStateMethodCaller; +        class TBaseSetStateMethodCaller; + +        TPythonTypeAttributes(const TPythonTypeAttributes&); +        TPythonTypeAttributes& operator=(const TPythonTypeAttributes&); + +        static const TSet<TString> HiddenAttrNames; + +        typedef PyObject* (*GetAttrFunction)(PyObject*, char*); +        typedef int (*SetAttrFunction)(PyObject*, char*, PyObject*); +        GetAttrFunction GetAttr; +        SetAttrFunction SetAttr; + +    public: +        typedef TSimpleSharedPtr<TBaseAttrGetter<TObject>> TGetterPtr; +        typedef TSimpleSharedPtr<TBaseAttrSetter<TObject>> TSetterPtr; +        typedef TSimpleSharedPtr<TBaseMethodCaller<TObject>> TCallerPtr; + +        TPythonTypeAttributes(GetAttrFunction getAttr, SetAttrFunction setAttr) +            : AttrGetters(HiddenAttrNames) +            , MethodCallers(HiddenAttrNames) +            , GetAttr(getAttr) +            , SetAttr(setAttr) +        { +        } + +        void InitCommonAttributes() { +            // attributes +            AddGetter("__dict__", new TDictAttrGetter(AttrGetters)); +            AddSetter("__dict__", new TDictAttrSetter(AttrSetters)); + +            // methods +            AddCaller("GetAttrsNames", new TGetAttrsNamesCaller(AttrGetters)); +            AddCaller("GetMethodsNames", new TGetMethodsNamesCaller(MethodCallers)); +            AddCaller("GetAllNames", new TGetAllNamesCaller(AttrGetters, MethodCallers)); +            AddCaller("GetPropertiesNames", new TGetPropertiesNamesCaller(MethodCallers)); +            AddCaller("__getattribute__", new TGetAttributeMethodCaller(GetAttr)); +            AddCaller("__setattr__", new TSetAttrMethodCaller(SetAttr)); +            AddCaller("__str__", new TGetStrReprMethodCaller("__str__")); +            AddCaller("__repr__", new TGetStrReprMethodCaller("__repr__")); +            AddCaller("__reduce_ex__", new TReduceMethodCaller); +            AddCaller("__reduce__", new TReduceMethodCaller); +            AddCaller("__getstate__", new TBaseGetStateMethodCaller); +            AddCaller("__setstate__", new TBaseSetStateMethodCaller); + +            // generics +            AddGetter("__class__", new TGenericAttrGetter<TObject>("__class__")); +            AddGetter("__doc__", new TGenericAttrGetter<TObject>("__doc__")); +            AddCaller("__sizeof__", new TGenericMethodCaller<TObject>("__sizeof__")); +            AddCaller("__hash__", new TGenericMethodCaller<TObject>("__hash__")); +        } + +        void AddGetter(const TString& attr, TGetterPtr getter) { +            AttrGetters.AddGetter(attr, getter); +        } + +        void AddSetter(const TString& attr, TSetterPtr setter) { +            AttrSetters.AddSetter(attr, setter); +        } + +        void AddCaller(const TString& name, TCallerPtr caller) { +            MethodCallers.AddCaller(name, caller); +        } + +        const TAttrGetters<TObject>& GetAttrGetters() const { +            return AttrGetters; +        } + +        TAttrSetters<TObject>& GetAttrSetters() { +            return AttrSetters; +        } + +        const TMethodCallers<TObject>& GetMethodCallers() const { +            return MethodCallers; +        } + +        const TSet<TString>& GetHiddenAttrs() const { +            return HiddenAttrNames; +        } +    }; + +    template <typename TObjType> +    class TPythonTypeAttributes<TObjType>::TGetAttrsNamesCaller: public TBaseMethodCaller<TObjType> { +    private: +        const TAttrGetters<TObjType>& AttrGetters; + +    public: +        TGetAttrsNamesCaller(const TAttrGetters<TObjType>& getters) +            : AttrGetters(getters) +        { +        } + +        bool CallMethod(PyObject* owner, TObjType* self, PyObject* args, PyObject*, PyObject*& res) const override { +            if (!ExtractArgs(args)) +                ythrow yexception() << "Could not parse args for GetAttrsNames() - it should be none"; +            TVector<TString> names; +            AttrGetters.GetAttrsNames(owner, *self, names); +            res = BuildPyObject(names); +            return (res != nullptr); +        } +    }; + +    template <typename TObjType> +    class TPythonTypeAttributes<TObjType>::TGetMethodsNamesCaller: public TBaseMethodCaller<TObjType> { +    private: +        const TMethodCallers<TObjType>& MethodCallers; + +    public: +        TGetMethodsNamesCaller(const TMethodCallers<TObjType>& callers) +            : MethodCallers(callers) +        { +        } + +        bool CallMethod(PyObject* owner, TObjType* self, PyObject* args, PyObject*, PyObject*& res) const override { +            if (!ExtractArgs(args)) +                ythrow yexception() << "Could not parse args for GetMethodsNames() - it should be none"; +            TVector<TString> names; +            MethodCallers.GetMethodsNames(owner, self, names); +            res = BuildPyObject(names); +            return (res != nullptr); +        } +    }; + +    template <typename TObjType> +    class TPythonTypeAttributes<TObjType>::TGetAllNamesCaller: public TBaseMethodCaller<TObjType> { +    private: +        const TAttrGetters<TObjType>& AttrGetters; +        const TMethodCallers<TObjType>& MethodCallers; + +    public: +        TGetAllNamesCaller(const TAttrGetters<TObjType>& getters, +                           const TMethodCallers<TObjType>& callers) +            : AttrGetters(getters) +            , MethodCallers(callers) +        { +        } + +        bool CallMethod(PyObject* owner, TObjType* self, PyObject* args, PyObject*, PyObject*& res) const override { +            if (!ExtractArgs(args)) +                ythrow yexception() << "Could not parse args for GetAllNames() - it should be none"; +            TVector<TString> names; +            AttrGetters.GetAttrsNames(owner, *self, names); +            MethodCallers.GetMethodsNames(owner, self, names); +            res = BuildPyObject(names); +            return (res != nullptr); +        } +    }; + +    template <typename TObjType> +    class TPythonTypeAttributes<TObjType>::TGetPropertiesNamesCaller: public TBaseMethodCaller<TObjType> { +    private: +        const TMethodCallers<TObjType>& MethodCallers; + +    public: +        TGetPropertiesNamesCaller(const TMethodCallers<TObjType>& callers) +            : MethodCallers(callers) +        { +        } + +    public: +        bool CallMethod(PyObject* obj, TObjType* self, PyObject* args, PyObject*, PyObject*& res) const override { +            if (!ExtractArgs(args)) +                return false; + +            TVector<TString> names; +            MethodCallers.GetPropertiesNames(obj, self, names); +            res = BuildPyObject(names); +            return (res != nullptr); +        } +    }; + +    template <typename TObjType> +    class TPythonTypeAttributes<TObjType>::TDictAttrGetter: public TBaseAttrGetter<TObjType> { +    private: +        TAttrGetters<TObjType>& AttrGetters; + +    public: +        TDictAttrGetter(TAttrGetters<TObjType>& getters) +            : AttrGetters(getters) +        { +        } + +        bool GetAttr(PyObject* owner, const TObjType& self, const TString&, PyObject*& res) const override { +            TMap<TString, PyObject*> dict; +            AttrGetters.GetAttrsDictionary(owner, self, dict); +            res = BuildPyObject(dict); +            return (res != nullptr); +        } +    }; + +    template <typename TObjType> +    class TPythonTypeAttributes<TObjType>::TDictAttrSetter: public TBaseAttrSetter<TObjType> { +    private: +        TAttrSetters<TObjType>& AttrSetters; + +    public: +        TDictAttrSetter(TAttrSetters<TObjType>& setters) +            : AttrSetters(setters) +        { +        } + +        bool SetAttr(PyObject* owner, TObjType& self, const TString&, PyObject* val) override { +            TMap<TString, PyObject*> dict; +            if (!FromPyObject(val, dict)) +                ythrow yexception() << "'__dict__' should be set to dictionary"; +            if (!AttrSetters.SetAttrDictionary(owner, self, dict)) +                return false; +            return true; +        } +    }; + +    template <typename TObjType> +    class TPythonTypeAttributes<TObjType>::TGetAttributeMethodCaller: public TBaseMethodCaller<TObjType> { +    private: +        GetAttrFunction GetAttr; + +    public: +        TGetAttributeMethodCaller(GetAttrFunction getAttr) +            : GetAttr(getAttr) +        { +        } + +        bool CallMethod(PyObject* owner, TObjType*, PyObject* args, PyObject*, PyObject*& res) const override { +            TString attrName; +            if (!ExtractArgs(args, attrName)) +                ythrow yexception() << "Could not parse args for '__getattribute__' - it should be one string"; +            res = GetAttr(owner, const_cast<char*>(attrName.c_str())); +            if (!res) +                // Error already set +                return false; +            return true; +        } +    }; + +    template <typename TObjType> +    class TPythonTypeAttributes<TObjType>::TSetAttrMethodCaller: public TBaseMethodCaller<TObjType> { +    private: +        SetAttrFunction SetAttr; + +    public: +        TSetAttrMethodCaller(SetAttrFunction setAttr) +            : SetAttr(setAttr) +        { +        } + +        bool CallMethod(PyObject* owner, TObjType*, PyObject* args, PyObject*, PyObject*& res) const override { +            TString attrName; +            TPyObjectPtr value; +            if (!ExtractArgs(args, attrName, value)) +                ythrow yexception() << "Could not parse args for '__setattr__' - it should be one string and value"; +            Py_INCREF(Py_None); +            res = Py_None; +            if (-1 == SetAttr(owner, const_cast<char*>(attrName.c_str()), value.Get())) +                // Error already set +                return false; +            return true; +        } +    }; + +    template <typename TObjType> +    class TPythonTypeAttributes<TObjType>::TGetStrReprMethodCaller: public TBaseMethodCaller<TObjType> { +    private: +        TString MethodName; + +    private: +        const TString GetFullName(PyObject* obj) const { +            TString module, name; +            TPyObjectPtr type(PyObject_Type(obj), true); +            if (!FromPyObject(PyObject_GetAttrString(type.Get(), "__module__"), module) || !FromPyObject(PyObject_GetAttrString(type.Get(), "__name__"), name)) +                ythrow yexception() << "Could not get name of object"; +            return module + "." + name; +        } + +    public: +        TGetStrReprMethodCaller(const TString& methodName) +            : MethodName(methodName) +        { +        } + +        bool CallMethod(PyObject* owner, TObjType*, PyObject* args, PyObject*, PyObject*& res) const override { +            if (args && !ExtractArgs(args)) +                ythrow yexception() << "Could not parse args for '" << MethodName << "'"; +            TString message = TString("<") + GetFullName(owner) + " object>"; +            res = ReturnString(message); +            return (res != nullptr); +        } +    }; + +    template <typename TObjType> +    class TPythonTypeAttributes<TObjType>::TReduceMethodCaller: public TBaseMethodCaller<TObjType> { +    public: +        bool CallMethod(PyObject* owner, TObjType*, PyObject*, PyObject*, PyObject*& res) const override { +            TPyObjectPtr tuple(PyTuple_New(3), true); +            // First component: reconstructor +            TPyObjectPtr pybindName(BuildPyObject("pybind"), true); +            TPyObjectPtr mainModule(PyImport_Import(pybindName.Get()), true); +            TPyObjectPtr recName(BuildPyObject("PyBindObjectReconstructor"), true); +            TPyObjectPtr reconstructor(PyObject_GetAttr(mainModule.Get(), recName.Get()), true); +            // Second component: arguments to rebuild object +            TPyObjectPtr arguments(PyTuple_New(2), true); +            TPyObjectPtr cl(PyObject_GetAttrString(owner, "__class__"), true); +            PyTuple_SET_ITEM(arguments.Get(), 0, cl.RefGet()); +            TPyObjectPtr props(PyObject_CallMethod(owner, const_cast<char*>("GetPropertiesNames"), nullptr), true); +            PyTuple_SET_ITEM(arguments.Get(), 1, props.RefGet()); +            // Third component: state to fill new object +            TPyObjectPtr state(PyObject_CallMethod(owner, const_cast<char*>("__getstate__"), nullptr), true); + +            PyTuple_SET_ITEM(tuple.Get(), 0, reconstructor.RefGet()); +            PyTuple_SET_ITEM(tuple.Get(), 1, arguments.RefGet()); +            PyTuple_SET_ITEM(tuple.Get(), 2, state.RefGet()); +            res = tuple.RefGet(); +            return (res != nullptr); +        } +    }; + +    template <typename TObjType> +    class TPythonTypeAttributes<TObjType>::TBaseGetStateMethodCaller: public TGetStateCaller<TObjType, TObjType> { +    public: +        void GetAttrsDictionary(PyObject* obj, TObjType* self, TMap<TString, TPyObjectPtr>& dict) const override { +            this->GetStandartAttrsDictionary(obj, self, dict); +        } +    }; + +    template <typename TObjType> +    class TPythonTypeAttributes<TObjType>::TBaseSetStateMethodCaller: public TSetStateCaller<TObjType, TObjType> { +    public: +        void SetAttrsDictionary(PyObject* obj, TObjType* self, TMap<TString, TPyObjectPtr>& dict) const override { +            this->SetStandartAttrsDictionary(obj, self, dict); +        } +    }; + +    static const char* HiddenAttrStrings[] = { +        "__dict__", "__class__", "__dir__", "__delattr__", "__doc__", "__format__", "__getattribute__", "__hash__", +        "__init__", "__new__", "__reduce__", "__reduce_ex__", "__repr__", "__setattr__", "__sizeof__", "__str__", +        "__subclasshook__", "__getstate__", "__setstate__", +        "GetAttrsNames", "GetMethodsNames", "GetAllNames", "GetPropertiesNames"}; + +    template <typename T> +    const TSet<TString> TPythonTypeAttributes<T>::HiddenAttrNames(HiddenAttrStrings, std::end(HiddenAttrStrings)); + +} diff --git a/library/cpp/pybind/typedesc.cpp b/library/cpp/pybind/typedesc.cpp new file mode 100644 index 00000000000..75f39fd1260 --- /dev/null +++ b/library/cpp/pybind/typedesc.cpp @@ -0,0 +1,79 @@ +#include "typedesc.h" + +#include <util/generic/singleton.h> + +static void RegisterJSONBridgeImpl() { +    PyRun_SimpleString("import json\n" +                       "class PyBindEncoder(json.JSONEncoder):\n" +                       "    def default(self, obj):\n" +                       "        if isinstance(obj, bytes):\n" +                       "            try:\n" +                       "                return obj.decode()\n" +                       "            except UnicodeDecodeError:\n" +                       "                return obj.hex()\n" +                       "        dct = None\n" +                       "        if hasattr(obj, '__getstate__'):\n" +                       "            dct = obj.__getstate__()\n" +                       "        elif hasattr(obj, '__dict__'):\n" +                       "            dct = obj.__dict__\n" +                       "        if dct is None:\n" +                       "            return json.JSONEncoder.default(self, obj)\n" +                       "        if hasattr(obj, '__class__'):\n" +                       "            if hasattr(obj.__class__, '__name__'):\n" +                       "                dct['__name__'] = obj.__class__.__name__\n" +                       "            if hasattr(obj.__class__, '__module__'):\n" +                       "                dct['__module__'] = obj.__class__.__module__\n" +                       "        if hasattr(obj, 'GetPropertiesNames'):\n" +                       "            dct['__properties__'] = obj.GetPropertiesNames()\n" +                       "        return dct"); + +    PyRun_SimpleString("def PyBindObjectHook(dct):\n" +                       "    if '__name__' in dct:\n" +                       "        name = dct['__name__']\n" +                       "        module = dct['__module__']\n" +                       "        del dct['__name__']\n" +                       "        del dct['__module__']\n" +                       "        cls = getattr(__import__(module), name)\n" +                       "        if '__properties__' in dct:\n" +                       "            props = dct['__properties__']\n" +                       "            del dct['__properties__']\n" +                       "            if len(props) == 0:\n" +                       "                return dct\n" +                       "            instance = cls(__properties__ = props)\n" +                       "        else:\n" +                       "            instance = cls()\n" +                       "        if hasattr(instance, '__setstate__'):\n" +                       "            instance.__setstate__(dct)\n" +                       "        elif hasattr(instance, '__dict__'):\n" +                       "            instance.__dict__ = dct\n" +                       "        else:\n" +                       "            return dct\n" +                       "        return instance\n" +                       "    return dct"); + +    PyRun_SimpleString("def json_dump(*args, **kwargs):\n" +                       "    kwargs['cls'] = PyBindEncoder\n" +                       "    return json.dump(*args, **kwargs)\n" +                       "def json_dumps(*args, **kwargs):\n" +                       "    kwargs['cls'] = PyBindEncoder\n" +                       "    return json.dumps(*args, **kwargs)"); + +    PyRun_SimpleString("def json_load(*args, **kwargs):\n" +                       "    kwargs['object_hook'] = PyBindObjectHook\n" +                       "    return json.load(*args, **kwargs)\n" +                       "def json_loads(*args, **kwargs):\n" +                       "    kwargs['object_hook'] = PyBindObjectHook\n" +                       "    return json.loads(*args, **kwargs)"); +} + +namespace { +    struct TJSONBridge { +        TJSONBridge() { +            RegisterJSONBridgeImpl(); +        } +    }; +} + +void NPyBind::RegisterJSONBridge() { +    Singleton<TJSONBridge>(); +} diff --git a/library/cpp/pybind/typedesc.h b/library/cpp/pybind/typedesc.h new file mode 100644 index 00000000000..57eacb0f3ae --- /dev/null +++ b/library/cpp/pybind/typedesc.h @@ -0,0 +1,545 @@ +#pragma once + +#define PY_SSIZE_T_CLEAN +#include <Python.h> +#include <structmember.h> + +#include "typeattrs.h" +#include "exceptions.h" +#include "module.h" + +namespace NPyBind { +    void RegisterJSONBridge(); + +    namespace NPrivate { +        template <typename> +        class TUnboundClosureHolder; +        template <typename> +        class TUnboundClosure; +    } + +    // TTraits should be derived from TPythonType +    template <typename TObjectHolder, typename TObject, typename TTraits> +    class TPythonType { +    private: +        TPythonType(const TPythonType&); +        TPythonType& operator=(const TPythonType&); + +    private: +        typedef typename TPythonTypeAttributes<TObject>::TGetterPtr TGetterPtr; +        typedef typename TPythonTypeAttributes<TObject>::TSetterPtr TSetterPtr; +        typedef typename TPythonTypeAttributes<TObject>::TCallerPtr TCallerPtr; + +        struct TProxy { +            PyObject_HEAD +                TObjectHolder* Holder; +        }; + +        static PyTypeObject PyType; +        static PyMappingMethods MappingMethods; +        static PyObject* PyTypeObjPtr; +    protected: +        static PyTypeObject* GetPyTypePtr() { +            return &PyType; +        } +    private: + +        TPythonTypeAttributes<TObject> Attributes; + +        static int InitObject(PyObject* s, PyObject* args, PyObject* kwargs) { +            try { +                TProxy* self = reinterpret_cast<TProxy*>(s); +                auto str = NameFromString("__properties__"); +                if (kwargs && PyDict_Check(kwargs) && PyDict_Contains(kwargs, str.Get())) { +                    TPyObjectPtr props(PyDict_GetItem(kwargs, str.Get())); +                    TVector<TString> properties; +                    FromPyObject(props.Get(), properties); +                    self->Holder = TTraits::DoInitPureObject(properties); +                } else { +                    self->Holder = (args || kwargs) ? TTraits::DoInitObject(args, kwargs) : nullptr; +                } +                if (PyErr_Occurred()) +                    return -1; +                return 0; +            } catch (const std::exception& ex) { +                PyErr_SetString(TExceptionsHolder::Instance().ToPyException(ex).Get(), ex.what()); +            } catch (...) { +                PyErr_SetString(PyExc_RuntimeError, "Unknown error occurred while trying to init object"); +            } +            return -1; +        } + +        static void DeallocObject(TProxy* self) { +            delete self->Holder; +            Py_TYPE(self)->tp_free(reinterpret_cast<PyObject*>(self)); +        } + +        static PyObject* GetObjectAttr(PyObject* pyObj, char* attr); +        static int SetObjectAttr(PyObject* pyObj, char* attr, PyObject* value); +        static PyObject* GetStr(PyObject*); +        static PyObject* GetRepr(PyObject*); +        static PyObject* GetIter(PyObject*); +        static PyObject* GetNext(PyObject*); + +        // Fill class __dict__ with functions to make sure methods names will get to dir() +        void FillClassDict() const { +            TVector<TString> names; +            Attributes.GetMethodCallers().GetAllMethodsNames(names); +            for (const auto& name : names) { +                TPyObjectPtr callable = NPrivate::TUnboundClosure<TObject>::Instance().CreatePyObject(new NPrivate::TUnboundClosureHolder<TObject>(&PyType, name)); +                PyDict_SetItemString(PyType.tp_dict, name.c_str(), callable.Get()); +            } +        } + +        void InitCommonAttributes() { +            static bool was = false; +            if (was) +                return; +            was = true; +            Attributes.InitCommonAttributes(); +            FillClassDict(); +        } + +    protected: +        TPythonType(const char* pyTypeName, const char* typeDescr, PyTypeObject* parentType = nullptr) +            : Attributes(GetObjectAttr, SetObjectAttr) +        { +            PyType.tp_name = pyTypeName; +            PyType.tp_doc = typeDescr; +            Py_INCREF(PyTypeObjPtr); +            if (parentType) { +                Py_INCREF(parentType); +                PyType.tp_base = parentType; +            } +            PyType_Ready(&PyType); + +            TExceptionsHolder::Instance(); +            RegisterJSONBridge(); + +        } + +        ~TPythonType() { +        } + +        static TObjectHolder* DoInitObject(PyObject*, PyObject*) { +            return nullptr; +        } + +        static TObjectHolder* DoInitPureObject(const TVector<TString>&) { +            return nullptr; +        } + +        static void SetClosure(PyObject* (*call)(PyObject*, PyObject*, PyObject*)) { +            PyType.tp_call = call; +        } + +    public: +        void AddGetter(const TString& attr, TGetterPtr getter) { +            Attributes.AddGetter(attr, getter); +        } + +        void AddSetter(const TString& attr, TSetterPtr setter) { +            Attributes.AddSetter(attr, setter); +        } + +        void AddCaller(const TString& name, TCallerPtr caller) { +            Attributes.AddCaller(name, caller); +            if (name == "__iter__") { +                PyType.tp_iter = GetIter; +            } +            if (name == "next") { +                PyType.tp_iternext = GetNext; +            } +        } + +        void SetIter(getiterfunc tp_iter) { +            PyType.tp_iter = tp_iter; +        } + +        void SetIterNext(iternextfunc tp_iternext) { +            PyType.tp_iternext = tp_iternext; +        } + +        void SetDestructor(destructor tp_dealloc) { +            PyType.tp_dealloc = tp_dealloc; +        } + +        void SetLengthFunction(lenfunc mp_length) { +            PyType.tp_as_mapping->mp_length = mp_length; +        } + +        void SetSubscriptFunction(binaryfunc mp_subscript) { +            PyType.tp_as_mapping->mp_subscript = mp_subscript; +        } + +        void SetAssSubscriptFunction(objobjargproc mp_ass_subscript) { +            PyType.tp_as_mapping->mp_ass_subscript = mp_ass_subscript; +        } + +        typedef TObject TObjectType; + +        static TPythonType& Instance() { +            static TTraits Traits; +            Traits.InitCommonAttributes(); +            return Traits; +        } + +        void Register(PyObject* module, const char* typeName) { +            Py_INCREF(PyTypeObjPtr); +            if (0 != PyModule_AddObject(module, typeName, PyTypeObjPtr)) +                ythrow yexception() << "can't register type \"" << typeName << "\""; +        } + +        void Register(PyObject* module, const char* objName, TObjectHolder* hld) { +            if (0 != PyModule_AddObject(module, objName, CreatePyObject(hld).RefGet())) +                ythrow yexception() << "can't register object \"" << objName << "\""; +        } + +        void Register(TPyObjectPtr module, const TString& typeName) { +            Register(module.Get(), typeName.c_str()); +        } + +        void Register(TPyObjectPtr module, const TString& objName, TObjectHolder* hld) { +            Register(module.Get(), objName.c_str(), hld); +        } + +        static TObjectHolder* CastToObjectHolder(PyObject* obj) { +            // Call Instance() to make sure PyTypeObjPtr is already created at this point +            Instance(); +            if (!PyObject_IsInstance(obj, PyTypeObjPtr)) +                return nullptr; +            TProxy* prx = reinterpret_cast<TProxy*>(obj); +            return prx ? prx->Holder : nullptr; +        } + +        static TObject* CastToObject(PyObject* obj) { +            TObjectHolder* hld = CastToObjectHolder(obj); +            return hld ? TTraits::GetObject(*hld) : nullptr; +        } + +        static TPyObjectPtr CreatePyObject(TObjectHolder* hld) { +            TPyObjectPtr r(_PyObject_New(&PyType), true); +            TProxy* prx = reinterpret_cast<TProxy*>(r.Get()); +            if (prx) +                prx->Holder = hld; +            return r; +        } +    }; + +    template <typename TObjectHolder, typename TObject, typename TTraits> +    PyMappingMethods TPythonType<TObjectHolder, TObject, TTraits>::MappingMethods = {nullptr, nullptr, nullptr}; + +    template <typename TObjectHolder, typename TObject, typename TTraits> +    PyTypeObject TPythonType<TObjectHolder, TObject, TTraits>::PyType = { +        PyVarObject_HEAD_INIT(nullptr, 0) "", sizeof(TProxy), 0, (destructor)&DeallocObject +#if PY_VERSION_HEX < 0x030800b4 +        , nullptr, /*tp_print*/ +#endif +#if PY_VERSION_HEX >= 0x030800b4 +        , 0, /*tp_vectorcall_offset*/ +#endif +        &GetObjectAttr, &SetObjectAttr, nullptr, &GetRepr, nullptr, nullptr, &MappingMethods, nullptr, nullptr, &GetStr, nullptr, nullptr, nullptr, +        Py_TPFLAGS_DEFAULT, "", nullptr, nullptr, nullptr, 0, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, 0, InitObject, PyType_GenericAlloc, PyType_GenericNew, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, 0 +#if PY_MAJOR_VERSION >= 3 +        , nullptr +#endif +#if PY_VERSION_HEX >= 0x030800b1 +        , nullptr /*tp_vectorcall*/ +#endif +#if PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x03090000 +        , nullptr /*tp_print*/ +#endif +#if PY_VERSION_HEX >= 0x030C0000 +        , 0 /*tp_watched*/ +#endif +    }; + +    template <typename TObjectHolder, typename TObject, typename TTraits> +    PyObject* TPythonType<TObjectHolder, TObject, TTraits>::PyTypeObjPtr = +        reinterpret_cast<PyObject*>(&TPythonType<TObjectHolder, TObject, TTraits>::PyType); + +    namespace NPrivate { +        template <typename TObject> +        class TUnboundClosureHolder { +        private: +            THolder<PyTypeObject> Holder; +            TString Method; + +        public: +            TUnboundClosureHolder(PyTypeObject* ptr, const TString& meth) +                : Holder(ptr) +                , Method(meth) +            { +            } + +            PyTypeObject* GetObject() const { +                return Holder.Get(); +            } + +            const TString GetMethod() const { +                return Method; +            } + +            PyObject* Call(PyObject* obj, PyObject* args, PyObject*) const { +                TPyObjectPtr callable(PyObject_GetAttrString(obj, Method.c_str()), true); +                if (!callable.Get()) +                    ythrow yexception() << "PyBind can't call method '" << Method << "'"; +                TPyObjectPtr res(PyObject_CallObject(callable.Get(), args), true); +                if (!res.Get() && !PyErr_Occurred()) +                    ythrow yexception() << "PyBind can't call method '" << Method << "'"; +                return res.RefGet(); +            } +        }; + +        template <typename TObject> +        class TUnboundClosure: public NPyBind::TPythonType<TUnboundClosureHolder<TObject>, PyTypeObject, TUnboundClosure<TObject>> { +        private: +            typedef class NPyBind::TPythonType<TUnboundClosureHolder<TObject>, PyTypeObject, TUnboundClosure<TObject>> TParent; +            friend class NPyBind::TPythonType<TUnboundClosureHolder<TObject>, PyTypeObject, TUnboundClosure<TObject>>; + +            class TReprMethodCaller: public TBaseMethodCaller<PyTypeObject> { +            public: +                bool CallMethod(PyObject* closure, PyTypeObject*, PyObject*, PyObject*, PyObject*& res) const override { +                    TUnboundClosureHolder<TObject>* hld = TParent::CastToObjectHolder(closure); +                    TPyObjectPtr type((PyObject*)hld->GetObject()); + +                    TString nameStr; +                    TPyObjectPtr name(PyObject_GetAttrString(type.Get(), "__name__"), true); +                    if (!name.Get() || !FromPyObject(name.Get(), nameStr)) +                        ythrow yexception() << "Could not get name of object"; + +                    TString methodName(hld->GetMethod()); + +                    TString message = "<unbound method " + nameStr + "." + methodName + ">"; +                    res = ReturnString(message); +                    return (res != nullptr); +                } +            }; + +        private: +            TUnboundClosure() +                : TParent("", "") +            { +                TParent::AddCaller("__repr__", new TReprMethodCaller()); +                TParent::AddCaller("__str__", new TReprMethodCaller()); +                TParent::SetClosure(&Call); +            } + +            static PyObject* Call(PyObject* closure, PyObject* args, PyObject* kwargs) { +                try { +                    TUnboundClosureHolder<TObject>* hld = TParent::CastToObjectHolder(closure); +                    if (!hld) +                        ythrow yexception() << "Can't cast object to TypeHolder"; + +                    size_t size = 0; +                    if (!PyTuple_Check(args) || (size = PyTuple_Size(args)) < 1) +                        ythrow yexception() << "Can't parse first argument: it should be valid object"; +                    --size; +                    TPyObjectPtr obj(PyTuple_GetItem(args, 0)); +                    TPyObjectPtr newArgs(PyTuple_New(size), true); + +                    for (size_t i = 0; i < size; ++i) { +                        TPyObjectPtr item(PyTuple_GetItem(args, i + 1)); +                        PyTuple_SetItem(newArgs.Get(), i, item.RefGet()); +                    } + +                    return hld->Call(obj.Get(), newArgs.Get(), kwargs); +                } catch (const std::exception& ex) { +                    PyErr_SetString(::NPyBind::TExceptionsHolder::Instance().ToPyException(ex).Get(), ex.what()); +                } catch (...) { +                    PyErr_SetString(PyExc_RuntimeError, "Unknown error occurred while trying to call method"); +                } +                return nullptr; +            } + +            static PyTypeObject* GetObject(TUnboundClosureHolder<TObject>& obj) { +                return obj.GetObject(); +            } +        }; + +        template <typename TObject> +        class TBoundClosureHolder { +        private: +            TPyObjectPtr Ptr; +            TObject* Object; +            TString Method; +            const TMethodCallers<TObject>& MethodCallers; + +        public: +            TBoundClosureHolder(PyObject* ptr, TObject* obj, const TString& meth, const TMethodCallers<TObject>& callers) +                : Ptr(ptr) +                , Object(obj) +                , Method(meth) +                , MethodCallers(callers) +            { +            } + +            TPyObjectPtr GetObjectPtr() const { +                return Ptr; +            } + +            TObject* GetObject() const { +                return Object; +            } + +            const TString GetMethod() const { +                return Method; +            } + +            PyObject* Call(PyObject* args, PyObject* kwargs) const { +                PyObject* res = MethodCallers.CallMethod(Ptr.Get(), Object, args, kwargs, Method); +                if (res == nullptr && !PyErr_Occurred()) +                    ythrow yexception() << "PyBind can't call method '" << Method << "'"; +                return res; +            } +        }; + +        template <typename TObject> +        class TBoundClosure: public TPythonType<TBoundClosureHolder<TObject>, TObject, TBoundClosure<TObject>> { +        private: +            typedef TPythonType<TBoundClosureHolder<TObject>, TObject, TBoundClosure<TObject>> TMyParent; +            class TReprMethodCaller: public TBaseMethodCaller<TObject> { +            public: +                bool CallMethod(PyObject* closure, TObject*, PyObject*, PyObject*, PyObject*& res) const override { +                    TBoundClosureHolder<TObject>* hld = TMyParent::CastToObjectHolder(closure); +                    TPyObjectPtr obj(hld->GetObjectPtr()); +                    TPyObjectPtr type(PyObject_Type(obj.Get()), true); + +                    TString reprStr; +                    TPyObjectPtr repr(PyObject_Repr(obj.Get()), true); +                    if (!repr.Get() || !FromPyObject(repr.Get(), reprStr)) +                        ythrow yexception() << "Could not get repr of object"; + +                    TString nameStr; +                    TPyObjectPtr name(PyObject_GetAttrString(type.Get(), "__name__"), true); +                    if (!name.Get() || !FromPyObject(name.Get(), nameStr)) +                        ythrow yexception() << "Could not get name of object"; + +                    TString methodName(hld->GetMethod()); + +                    TString message = "<bound method " + nameStr + "." + methodName + " of " + reprStr + ">"; +                    res = ReturnString(message); +                    return (res != nullptr); +                } +            }; + +        private: +            static PyObject* Call(PyObject* closure, PyObject* args, PyObject* kwargs) { +                try { +                    TBoundClosureHolder<TObject>* hld = TMyParent::CastToObjectHolder(closure); +                    if (!hld) +                        ythrow yexception() << "Can't cast object to ClosureHolder"; + +                    return hld->Call(args, kwargs); +                } catch (const std::exception& ex) { +                    PyErr_SetString(::NPyBind::TExceptionsHolder::Instance().ToPyException(ex).Get(), ex.what()); +                } catch (...) { +                    PyErr_SetString(PyExc_RuntimeError, "Unknown error occurred while trying to call method"); +                } +                return nullptr; +            } + +        public: +            TBoundClosure() +                : TMyParent("", "") +            { +                TMyParent::AddCaller("__repr__", new TReprMethodCaller()); +                TMyParent::AddCaller("__str__", new TReprMethodCaller()); +                TMyParent::SetClosure(&Call); +            } + +            static TObject* GetObject(const TBoundClosureHolder<TObject>& closure) { +                return closure.GetObject(); +            } +        }; + +    } + +    template <typename TObjectHolder, typename TObject, typename TTraits> +    PyObject* TPythonType<TObjectHolder, TObject, TTraits>::GetObjectAttr(PyObject* pyObj, char* attr) { +        try { +            TObject* obj = CastToObject(pyObj); +            PyObject* res = obj ? Instance().Attributes.GetAttrGetters().GetAttr(pyObj, *obj, attr) : nullptr; +            if (res == nullptr && Instance().Attributes.GetMethodCallers().HasMethod(pyObj, obj, attr)) { +                TPyObjectPtr r = NPrivate::TBoundClosure<TObject>::Instance().CreatePyObject(new NPrivate::TBoundClosureHolder<TObject>(pyObj, obj, attr, Instance().Attributes.GetMethodCallers())); +                res = r.RefGet(); +            } +            if (res == nullptr && !PyErr_Occurred()) +                ythrow TPyErr(PyExc_AttributeError) << "PyBind can't get attribute '" << attr << "'"; +            return res; +        } catch (const std::exception& ex) { +            PyErr_SetString(TExceptionsHolder::Instance().ToPyException(ex).Get(), ex.what()); +        } catch (...) { +            PyErr_SetString(PyExc_RuntimeError, (TString("Unknown error occurred while trying to get attribute '") + attr + "'").c_str()); +        } +        return nullptr; +    } + +    template <typename TObjectHolder, typename TObject, typename TTraits> +    int TPythonType<TObjectHolder, TObject, TTraits>::SetObjectAttr(PyObject* pyObj, char* attr, PyObject* value) { +        try { +            TObject* obj = CastToObject(pyObj); +            bool res = obj ? Instance().Attributes.GetAttrSetters().SetAttr(pyObj, *obj, attr, value) : false; +            if (!res && !PyErr_Occurred()) +                ythrow yexception() << "PyBind can't set attribute '" << attr << "'"; +            return res ? 0 : -1; +        } catch (const std::exception& ex) { +            PyErr_SetString(TExceptionsHolder::Instance().ToPyException(ex).Get(), ex.what()); +        } catch (...) { +            PyErr_SetString(PyExc_RuntimeError, (TString("Unknown error occurred while trying to set attribute '") + attr + "'").c_str()); +        } +        return -1; +    } + +    template <typename TObjectHolder, typename TObject, typename TTraits> +    PyObject* TPythonType<TObjectHolder, TObject, TTraits>::GetStr(PyObject* obj) { +        try { +            TObject* self = CastToObject(obj); +            return Instance().Attributes.GetMethodCallers().CallMethod(obj, self, nullptr, nullptr, "__str__"); +        } catch (const std::exception& ex) { +            PyErr_SetString(TExceptionsHolder::Instance().ToPyException(ex).Get(), ex.what()); +        } catch (...) { +            PyErr_SetString(PyExc_RuntimeError, (TString("Unknown error occurred while trying to call '__str__'").c_str())); +        } +        return nullptr; +    } + +    template <typename TObjectHolder, typename TObject, typename TTraits> +    PyObject* TPythonType<TObjectHolder, TObject, TTraits>::GetIter(PyObject* obj) { +        try { +            TObject* self = CastToObject(obj); +            return Instance().Attributes.GetMethodCallers().CallMethod(obj, self, nullptr, nullptr, "__iter__"); +        } catch (const std::exception& ex) { +            PyErr_SetString(TExceptionsHolder::Instance().ToPyException(ex).Get(), ex.what()); +        } catch (...) { +            PyErr_SetString(PyExc_RuntimeError, (TString("Unknown error occurred while trying to call '__iter__'").c_str())); +        } +        return nullptr; +    } + +    template <typename TObjectHolder, typename TObject, typename TTraits> +    PyObject* TPythonType<TObjectHolder, TObject, TTraits>::GetNext(PyObject* obj) { +        try { +            TObject* self = CastToObject(obj); +            return Instance().Attributes.GetMethodCallers().CallMethod(obj, self, nullptr, nullptr, "next"); +        } catch (const std::exception& ex) { +            PyErr_SetString(TExceptionsHolder::Instance().ToPyException(ex).Get(), ex.what()); +        } catch (...) { +            PyErr_SetString(PyExc_RuntimeError, (TString("Unknown error occurred while trying to call 'next'").c_str())); +        } +        return nullptr; +    } + +    template <typename TObjectHolder, typename TObject, typename TTraits> +    PyObject* TPythonType<TObjectHolder, TObject, TTraits>::GetRepr(PyObject* obj) { +        try { +            TObject* self = CastToObject(obj); +            return Instance().Attributes.GetMethodCallers().CallMethod(obj, self, nullptr, nullptr, "__repr__"); +        } catch (const std::exception& ex) { +            PyErr_SetString(TExceptionsHolder::Instance().ToPyException(ex).Get(), ex.what()); +        } catch (...) { +            PyErr_SetString(PyExc_RuntimeError, (TString("Unknown error occurred while trying to call '__repr__'").c_str())); +        } +        return nullptr; +    } +} diff --git a/library/cpp/pybind/v2.cpp b/library/cpp/pybind/v2.cpp new file mode 100644 index 00000000000..edce0be7195 --- /dev/null +++ b/library/cpp/pybind/v2.cpp @@ -0,0 +1,43 @@ +#include "v2.h" +namespace NPyBind { +    namespace Detail { +        template <> +        PyTypeObject* GetParentType<void>(const TPyModuleDefinition&) { +            return nullptr; +        } + + +        template <bool InitEnabled> +        void UpdateClassNamesInModule(TPyModuleDefinition& M, const TString& name, PyTypeObject* pythonType) { +            if (!InitEnabled) { +                return; +            } +            M.ClassName2Type[name] = pythonType; +        } + +        template <bool InitEnabled> +        void UpdateGetContextInModule(TPyModuleDefinition& M, const TString& name, IGetContextBase* base) { +            if (!InitEnabled) { +                return; +            } +            M.Class2ContextGetter[name] = base; +        } + +        TPyModuleRegistry::TPyModuleRegistry() { +#if PY_MAJOR_VERSION >= 3 +            NPrivate::AddFinalizationCallBack([this]() { +                if (UnnamedModule) { +                    UnnamedModule.Clear(); +                } +                Name2Def.clear(); +            }); +#endif +        } +        template void UpdateClassNamesInModule<false>(TPyModuleDefinition& M, const TString& name, PyTypeObject* pythonType); +        template void UpdateClassNamesInModule<true>(TPyModuleDefinition& M, const TString& name, PyTypeObject* pythonType); + + +        template void UpdateGetContextInModule<false>(TPyModuleDefinition& M, const TString& name, IGetContextBase* pythonType); +        template void UpdateGetContextInModule<true>(TPyModuleDefinition& M, const TString& name, IGetContextBase* pythonType); +    }//Detail +}//NPyBind diff --git a/library/cpp/pybind/v2.h b/library/cpp/pybind/v2.h new file mode 100644 index 00000000000..f561d6a380f --- /dev/null +++ b/library/cpp/pybind/v2.h @@ -0,0 +1,514 @@ +#pragma once + +#include <library/cpp/pybind/method.h> +#include <library/cpp/pybind/typedesc.h> +#include <library/cpp/pybind/module.h> +#include <util/generic/hash.h> +#include <util/generic/hash_set.h> +#include <util/generic/string.h> +namespace NPyBind { +#define DEFINE_CONVERTERS_IMPL(TClass)                            \ +    PyObject* BuildPyObject(typename TClass::TBase&& base) {      \ +        return TClass::BuildPyObject(std::move(base));            \ +    }                                                             \ +    PyObject* BuildPyObject(const typename TClass::TBase& base) { \ +        return TClass::BuildPyObject(base);                       \ +    } + +#define DEFINE_CONVERTERS(function) DEFINE_CONVERTERS_IMPL(TFunctionResult<decltype(function)>) + +#define DEFINE_TRANSFORMERS_IMPL(TClass)                                                                              \ +    template <>                                                                                                       \ +    bool ::NPyBind::FromPyObject<typename TClass::TBase*>(PyObject * obj, typename TClass::TBase * &res) {            \ +        res = TClass::CastToObject(obj);                                                                              \ +        return res != nullptr;                                                                                        \ +    }                                                                                                                 \ +    template <>                                                                                                       \ +    bool ::NPyBind::FromPyObject<typename TClass::TBase const*>(PyObject * obj, typename TClass::TBase const*& res) { \ +        res = TClass::CastToObject(obj);                                                                              \ +        return res != nullptr;                                                                                        \ +    } + +#define DEFINE_TRANSFORMERS(function) DEFINE_TRANSFORMERS_IMPL(TFunctionResult<decltype(function)>) + +    namespace Detail { +        struct IGetContextBase { +            virtual ~IGetContextBase() = default; +        }; +    } //Detail +    struct TPyModuleDefinition { +        static void InitModule(const TString& name); +        static TPyModuleDefinition& GetModule(); + +        TString Name; +        NPyBind::TPyObjectPtr M; +        THashMap<TString, PyTypeObject*> ClassName2Type; +        THashMap<TString, Detail::IGetContextBase*> Class2ContextGetter; +    }; + +    namespace Detail { +        // Manages modules lifecycle +        // IMPORTANT!!! Don't use it in PyBind v1 environment, it will lead to inconsistent state of v1 module +        // UnnamedModule-> new unnamed module stub, this stub become current module. In this case you can add functions to it +        // InitModuleWithName -> convert unnamed module into named one, now you can switch to it in switch, this module remains current +        // SwitchToModule switches to the particular module in registry, this module becomes current. +        class TPyModuleRegistry { +        private: +            TPyModuleRegistry(); +            TPyModuleRegistry(const TPyModuleRegistry&) = delete; +            TPyModuleRegistry& operator=(TPyModuleRegistry&) = delete; +        public: +            static TPyModuleRegistry& Get() { +                static TPyModuleRegistry registry; +                return registry; +            } +            TPyModuleDefinition& GetCurrentModule() { +                if (!CurrentModule) { +                    GetUnnamedModule(); +                } +                return *CurrentModule; +            } + +            TPyModuleDefinition& GetUnnamedModule() { +                if (!UnnamedModule) { +                    UnnamedModule = TPyModuleDefinition(); +                    CurrentModule = const_cast<TPyModuleDefinition*>(UnnamedModule.Get()); +                } +                return *UnnamedModule; +            } + +            TPyModuleDefinition& InitModuleWithName(const TString& name) { +                if (!UnnamedModule) { +                    GetUnnamedModule(); +                } +                Name2Def[name] = *UnnamedModule; +                UnnamedModule.Clear(); +                CurrentModule = &Name2Def[name]; +                return *CurrentModule; +            } + +            TPyModuleDefinition& SwitchToModuleByName(const TString& name) { +                Y_ENSURE(Name2Def.contains(name)); +                Y_ENSURE(UnnamedModule.Empty()); +                CurrentModule = &Name2Def[name]; +                return *CurrentModule; +            } +        private: +            TPyModuleDefinition* CurrentModule = nullptr; +            TMaybe<TPyModuleDefinition> UnnamedModule;// +            THashMap<TString, TPyModuleDefinition> Name2Def; +        }; +    }//Detail + +    inline void TPyModuleDefinition::InitModule(const TString& name) { +        Detail::TPyModuleRegistry::Get().GetUnnamedModule() = TPyModuleDefinition{name, TModuleHolder::Instance().InitModule(name), {}, {}}; +        Detail::TPyModuleRegistry::Get().InitModuleWithName(name); +    } + +    inline TPyModuleDefinition& TPyModuleDefinition::GetModule() { +        return Detail::TPyModuleRegistry::Get().GetCurrentModule(); +    } + +    namespace Detail { +        template <class TPythonType> +        struct TNameCtx { +            TString ClassShortName; +            static TNameCtx& GetNameCtx() { +                static TNameCtx result; +                return result; +            } +        }; +        template <class TBase> +        struct TContextImpl { +            PyTypeObject* ParentType = nullptr; +            TString ClassShortName; +            TString ClassFullName; +            TString ClassDescription; + + +            TVector<std::pair<TString, typename TPythonTypeAttributes<TBase>::TCallerPtr>> ListCallers; +            TVector<std::pair<TString, typename TPythonTypeAttributes<TBase>::TGetterPtr>> ListGetters; +            TVector<std::pair<TString, typename TPythonTypeAttributes<TBase>::TSetterPtr>> ListSetters; +        }; + +        template <class TObject> +        struct IGetContext: public IGetContextBase { +            virtual ~IGetContext() = default; +            virtual const TContextImpl<TObject>& GetContext() const = 0; +        }; + +        template <typename THolderClass, typename TBaseClass, bool ShouldEnable, typename=std::enable_if_t<!ShouldEnable || !std::is_default_constructible_v<TBaseClass>>> +        THolderClass* DoInitPureObject(const TVector<TString>&) { +            ythrow yexception() << "Can't create this object in pure mode from python"; +        } + +        template <typename THolderClass, typename TBaseClass, bool ShouldEnable, typename=std::enable_if_t<ShouldEnable && std::is_default_constructible_v<TBaseClass>>, typename=void> +        THolderClass* DoInitPureObject(const TVector<TString>&) { +            return new THolderClass(MakeHolder<TBaseClass>()); +        } + +        template <typename T> +        PyTypeObject* GetParentType(const TPyModuleDefinition& m) { +            auto shortName = Detail::TNameCtx<T>::GetNameCtx().ClassShortName; +            auto it = m.ClassName2Type.find(shortName); +            return (it == m.ClassName2Type.end()) ? nullptr : it->second; +        } + +        template <> +        PyTypeObject* GetParentType<void>(const TPyModuleDefinition&); + +        template <bool InitEnabled> +        void UpdateClassNamesInModule(TPyModuleDefinition& M, const TString& name, PyTypeObject* pythonType); + +        template <bool InitEnabled> +        void UpdateGetContextInModule(TPyModuleDefinition& M, const TString& name, IGetContextBase* base); +    } + + +    template <class TParentPyClass_=void> +    struct TPyParentClassTraits { +        using TParentPyClass = TParentPyClass_; +    }; + +    template <bool InitEnabled_, class TParentPyClass_=void> +    struct TPyClassConfigTraits: public TPyParentClassTraits<TParentPyClass_> { +        constexpr static bool InitEnabled = InitEnabled_; +        constexpr static bool RawInit = false; +    }; + +    template <class TParentPyClass_=void> +    struct TPyClassRawInitConfigTraits: public TPyParentClassTraits<TParentPyClass_> { +        constexpr static bool InitEnabled = true; +        constexpr static bool RawInit = true; +    }; + + +    template <typename TBaseClass, typename TPyClassConfigTraits, typename... ConstructorArgs> +    class TPyClass  { +    public: +        using TBase = TBaseClass; +    private: +        using TThisClass = TPyClass<TBaseClass, TPyClassConfigTraits, ConstructorArgs...>; +        using TContext = Detail::TContextImpl<TBase>; +        struct THolder { +            ::THolder<TBase> Holder; +            THolder(::THolder<TBase>&& right) +                : Holder(std::move(right)) +            { +            } +            THolder(TBase&& right) +                : Holder(MakeHolder<TBase>(std::move(right))) +            { +            } +        }; + +        class TSelectedTraits: public NPyBind::TPythonType<THolder, TBase, TSelectedTraits> { +        private: +            using TParent = NPyBind::TPythonType<THolder, TBase, TSelectedTraits>; +            friend TParent; + +        public: +            TSelectedTraits() +                : TParent(TThisClass::GetContext().ClassFullName.data(), TThisClass::GetContext().ClassDescription.data(), TThisClass::GetContext().ParentType) +            { +                for (const auto& caller : TThisClass::GetContext().ListCallers) { +                    TParent::AddCaller(caller.first, caller.second); +                } + +                for (const auto& getter : TThisClass::GetContext().ListGetters) { +                    TParent::AddGetter(getter.first, getter.second); +                } + +                for (const auto& setter : TThisClass::GetContext().ListSetters) { +                    TParent::AddSetter(setter.first, setter.second); +                } +            } + +            static TBase* GetObject(const THolder& holder) { +                return holder.Holder.Get(); +            } + +            static THolder* DoInitObject(PyObject* args, PyObject* kwargs) { +                if constexpr (TPyClassConfigTraits::InitEnabled) { +                    if constexpr (TPyClassConfigTraits::RawInit) { +                        static_assert(sizeof...(ConstructorArgs) == 0, "Do not pass construction args if use RawInit."); +                        return new THolder(::MakeHolder<TBase>(args, kwargs)); +                    } else { +                        if (args && (!PyTuple_Check(args) || PyTuple_Size(args) != sizeof...(ConstructorArgs))) { +                            ythrow yexception() << "Method takes " << sizeof...(ConstructorArgs) << " arguments, " << PyTuple_Size(args) << " provided"; +                        } +                        ::THolder<TBaseClass> basePtr{Apply([](auto&&... unpackedArgs) {return new TBase(std::forward<decltype(unpackedArgs)>(unpackedArgs)...); }, GetArguments<ConstructorArgs...>(args))}; +                        return new THolder(std::move(basePtr)); +                    } +                } else { +                    ythrow yexception() << "Can't create this object from python"; +                } +            } + +            static THolder* DoInitPureObject(const TVector<TString>& properties) { +                return Detail::DoInitPureObject<THolder, TBase, TPyClassConfigTraits::InitEnabled>(properties); +            } + +            static TBase* CastToObject(PyObject* obj) { +                return TParent::CastToObject(obj); +            } + +            static PyTypeObject* GetType() { +                return TParent::GetPyTypePtr(); +            } +        }; + +        class TContextHolder: public Detail::IGetContext<TBaseClass> { +        public: +            static TContextHolder& GetContextHolder() { +                static TContextHolder holder; +                return holder; +            } + +            TContext& GetContext() { +                return Context; +            } +            const TContext& GetContext() const override { +                return Context; +            } +        private: +            TContext Context; +        }; + +        template <class TDerivedClass, class TSuperClass> +        class TCallerWrapper: public TBaseMethodCaller<TDerivedClass> { +        public: +            explicit TCallerWrapper(TSimpleSharedPtr<const TBaseMethodCaller<TSuperClass>> baseCaller) +                : BaseCaller(baseCaller) { +                Y_ENSURE(BaseCaller); +            } + +            bool CallMethod(PyObject* owner, TDerivedClass* self, PyObject* args, PyObject* kwargs, PyObject*& res) const override { +                return BaseCaller->CallMethod(owner, static_cast<TSuperClass*>(self), args, kwargs, res); +            } + +        private: +            TSimpleSharedPtr<const TBaseMethodCaller<TSuperClass>> BaseCaller; +        }; + +        template <class TDerivedClass, class TSuperClass> +        class TSetterWrapper: public TBaseAttrSetter<TDerivedClass> { +        public: +            explicit TSetterWrapper(TSimpleSharedPtr<TBaseAttrSetter<TSuperClass>> baseSetter) +                : BaseSetter(baseSetter) { +                Y_ENSURE(BaseSetter); +            } + +            bool SetAttr(PyObject* owner, TDerivedClass& self, const TString& attr, PyObject* val) override { +                return BaseSetter->SetAttr(owner, static_cast<TSuperClass&>(self), attr, val); +            } + +        private: +            TSimpleSharedPtr<TBaseAttrSetter<TSuperClass>> BaseSetter; +        }; + +        template <class TDerivedClass, class TSuperClass> +        class TGetterWrapper: public TBaseAttrGetter<TDerivedClass> { +        public: +            explicit TGetterWrapper(TSimpleSharedPtr<const TBaseAttrGetter<TSuperClass>> baseGetter) +                : BaseGetter(baseGetter) { +                Y_ENSURE(BaseGetter); +            } + +            bool GetAttr(PyObject* owner, const TDerivedClass& self, const TString& attr, PyObject*& res) const override { +                return BaseGetter->GetAttr(owner, static_cast<const TSuperClass&>(self), attr, res); +            } + +        private: +            TSimpleSharedPtr<const TBaseAttrGetter<TSuperClass>> BaseGetter; +        }; + +        template <class TSuperClass, typename=std::enable_if_t<!std::is_same_v<TSuperClass, void>>> +        void ReloadAttrsFromBase() { +            auto shortName = Detail::TNameCtx<TSuperClass>::GetNameCtx().ClassShortName; +            if (!M.Class2ContextGetter.count(shortName)) { +                return; +            } +            auto callerBasePtr = M.Class2ContextGetter[shortName]; +            if (auto getContextPtr = dynamic_cast<const Detail::IGetContext<TSuperClass>*>(callerBasePtr)) { +                auto& ctx = getContextPtr->GetContext(); +                auto getUniqueNames = [](const auto& collection) { +                    THashSet<TString> uniqueNames; +                    for (const auto& elem : collection) { +                        uniqueNames.insert(elem.first); +                    } +                    return uniqueNames; +                }; + +                auto uniqueCallerNames = getUniqueNames(GetContext().ListCallers); +                using TConcreteCallerWrapper = TCallerWrapper<TBaseClass, TSuperClass>; +                for (const auto& caller : ctx.ListCallers) { +                    if (uniqueCallerNames.contains(caller.first)) { +                        continue; +                    } +                    GetContext().ListCallers.push_back(std::make_pair(caller.first, MakeSimpleShared<TConcreteCallerWrapper>(caller.second))); +                } + +                auto uniqueGettersNames = getUniqueNames(GetContext().ListGetters); +                using TConcreteGetterWrapper = TGetterWrapper<TBaseClass, TSuperClass>; +                for (const auto& getter : ctx.ListGetters) { +                    if (uniqueGettersNames.contains(getter.first)) { +                        continue; +                    } +                    GetContext().ListGetters.push_back(std::make_pair(getter.first, MakeSimpleShared<TConcreteGetterWrapper>(getter.second))); +                } + +                auto uniqueSetterNames = getUniqueNames(GetContext().ListSetters); +                using TConcreteSetterWrapper = TSetterWrapper<TBaseClass, TSuperClass>; +                for (auto& setter : ctx.ListSetters) { +                    if (uniqueSetterNames.contains(setter.first)) { +                        continue; +                    } +                    GetContext().ListSetters.push_back(std::make_pair(setter.first, MakeSimpleShared<TConcreteSetterWrapper>(setter.second))); +                } +            } +        } + +        template <class TSuperClass, typename=std::enable_if_t<std::is_same_v<TSuperClass, void>>, typename=void> +        void ReloadAttrsFromBase() { +        } + +        void CompleteImpl() { +            ReloadAttrsFromBase<typename TPyClassConfigTraits::TParentPyClass>(); +            TSelectedTraits::Instance().Register(M.M, GetContext().ClassShortName); +        } + +        static TContext& GetContext() { +            return TContextHolder::GetContextHolder().GetContext(); +        } + + +        friend struct Detail::TContextImpl<TBase>;//instead of context +        friend struct THolder; +        friend class TSelectedTraits; + +        using TCallerFunc = std::function<bool(PyObject*, TBaseClass*, PyObject*, PyObject*, PyObject*&)>; +        class TFuncCallerWrapper: public TBaseMethodCaller<TBaseClass> { +        public: +            explicit TFuncCallerWrapper(TCallerFunc func) +                : Func(func) { +                Y_ENSURE(func); +            } + +            bool CallMethod(PyObject* owner, TBaseClass* self, PyObject* args, PyObject* kwargs, PyObject*& res) const override { +                return Func(owner, self, args, kwargs, res); +            } +        private: +            mutable TCallerFunc Func; +        }; +    public: +        TPyClass(const TString& name, const TString& descr = "") +            : M(TPyModuleDefinition::GetModule()) +        { +            Detail::UpdateClassNamesInModule<TPyClassConfigTraits::InitEnabled>(M, name, TSelectedTraits::GetType()); +            Detail::UpdateGetContextInModule<TPyClassConfigTraits::InitEnabled>(M, name, &TContextHolder::GetContextHolder()); + +            GetContext().ClassFullName = TString::Join(M.Name, ".", name); +            GetContext().ClassShortName = name; +            GetContext().ClassDescription = descr; +            GetContext().ParentType = Detail::GetParentType<typename TPyClassConfigTraits::TParentPyClass>(M); +            Detail::TNameCtx<TBaseClass>::GetNameCtx().ClassShortName = name; +        } + +        template <typename TMemberFuction, typename = std::enable_if_t<std::is_member_function_pointer_v<TMemberFuction>>, typename=std::enable_if_t<!TIsPointerToConstMemberFunction<TMemberFuction>::value>> +        TThisClass& Def(const TString& name, TMemberFuction t) { +            GetContext().ListCallers.push_back(std::make_pair(name, CreateMethodCaller<TBase>(t))); +            return *this; +        } + +        template <typename TMemberFuction, typename = std::enable_if_t<std::is_member_function_pointer_v<TMemberFuction>>, typename=std::enable_if_t<TIsPointerToConstMemberFunction<TMemberFuction>::value>, typename=void> +        TThisClass& Def(const TString& name, TMemberFuction t) { +            GetContext().ListCallers.push_back(std::make_pair(name, CreateConstMethodCaller<TBase>(t))); +            return *this; +        } + +        template <typename TMemberObject, typename = std::enable_if_t<std::is_member_object_pointer_v<TMemberObject>>> +        TThisClass& Def(const TString& name, TMemberObject t)  { +            GetContext().ListGetters.push_back(std::make_pair(name, CreateAttrGetter<TBase>(t))); +            GetContext().ListSetters.push_back(std::make_pair(name, CreateAttrSetter<TBase>(t))); +            return *this; +        } + +        template <typename TResultType, typename... Args> +        TThisClass& DefByFunc(const TString& name, std::function<TResultType(TBaseClass&, Args...)> func) { +            GetContext().ListCallers.push_back(std::make_pair(name, CreateFunctorCaller<TBase, TResultType, Args...>(func))); +            return *this; +        } + +        TThisClass& DefByFunc(const TString& name, TCallerFunc origFunc) { +            GetContext().ListCallers.push_back(std::make_pair(name, MakeSimpleShared<TFuncCallerWrapper>(origFunc))); +            return *this; +        } + +        template <typename TMemberObject> +        TThisClass& DefReadonly(const TString& name, TMemberObject t, std::enable_if_t<std::is_member_object_pointer<TMemberObject>::value>* = nullptr) { +            GetContext().ListGetters.push_back(std::make_pair(name, CreateAttrGetter<TBase>(t))); +            return *this; +        } + + +        template <typename TMethodGetter, typename TMethodSetter, typename=std::enable_if_t<std::is_member_function_pointer_v<TMethodGetter> && std::is_member_function_pointer_v<TMethodSetter>>> +        TThisClass& AsProperty(const TString& name, TMethodGetter getter, TMethodSetter setter) { +            GetContext().ListGetters.push_back(std::make_pair(name, CreateMethodAttrGetter<TBase>(getter))); +            GetContext().ListSetters.push_back(std::make_pair(name, CreateMethodAttrSetter<TBase>(setter))); +            return *this; +        } + +        template <typename TMethodGetter, typename TMethodSetter, typename=std::enable_if_t<!std::is_member_function_pointer_v<TMethodGetter> && !std::is_member_function_pointer_v<TMethodSetter>>> +        TThisClass& AsPropertyByFunc(const TString& name, TMethodGetter getter, TMethodSetter setter) { +            GetContext().ListGetters.push_back(std::make_pair(name, CreateFunctorAttrGetter<TBase>(getter))); +            GetContext().ListSetters.push_back(std::make_pair(name, CreateFunctorAttrSetter<TBase>(setter))); +            return *this; +        } + +        template <typename TMethodGetter, typename=std::enable_if_t<std::is_member_function_pointer_v<TMethodGetter>>> +        TThisClass& AsProperty(const TString& name, TMethodGetter getter) { +            GetContext().ListGetters.push_back(std::make_pair(name, CreateMethodAttrGetter<TBase>(getter))); +            return *this; +        } + +        template <typename TMethodGetter> +        TThisClass& AsPropertyByFunc(const TString& name, TMethodGetter getter) { +            GetContext().ListGetters.push_back(std::make_pair(name, CreateFunctorAttrGetter<TBase>(getter))); +            return *this; +        } + +        TThisClass& Complete() { +            if (!Completed) { +                CompleteImpl(); +                Completed = true; +            } +            return *this; +        } + +    public: +        static PyObject* BuildPyObject(TBase&& base) { +            return NPyBind::BuildPyObject(TSelectedTraits::Instance().CreatePyObject(new THolder(std::move(base)))); +        } + +        static PyObject* BuildPyObject(const TBase& base) { +            return NPyBind::BuildPyObject(TSelectedTraits::Instance().CreatePyObject(new THolder(TBase(base)))); // WARN - copy +        } + +        static TBase* CastToObject(PyObject* obj) { +            return TSelectedTraits::CastToObject(obj); +        } + +    private: +        TPyModuleDefinition& M; +        bool Completed = false; +    }; + +    template <typename TFunctionSignature, TFunctionSignature function> +    void DefImpl(const TString& name, const TString& descr = "") { +        NPyBind::TModuleHolder::Instance().AddModuleMethod<TModuleMethodCaller<TFunctionSignature, function>::Call>(name, descr); +    } + +#define DefFunc(NAME, FUNC) NPyBind::DefImpl<decltype(FUNC), FUNC>(NAME) +#define DefFuncDescr(NAME, FUNC, DESCR) NPyBind::DefImpl<decltype(FUNC), FUNC>(NAME, DESCR) +}; diff --git a/library/cpp/pybind/ya.make b/library/cpp/pybind/ya.make new file mode 100644 index 00000000000..9b7b3413f2d --- /dev/null +++ b/library/cpp/pybind/ya.make @@ -0,0 +1,14 @@ +PY23_NATIVE_LIBRARY() + +SRCS( +    cast.cpp +    pod.cpp +    typedesc.cpp +    module.cpp +    exceptions.cpp +    embedding.cpp +    empty.cpp +    v2.cpp +) + +END() | 
