aboutsummaryrefslogtreecommitdiffstats
path: root/library
diff options
context:
space:
mode:
authorthegeorg <thegeorg@yandex-team.com>2023-10-03 11:19:48 +0300
committerthegeorg <thegeorg@yandex-team.com>2023-10-03 11:43:28 +0300
commitcda0c13f23f6b169fb0a49dc504b40a0aaecea09 (patch)
tree26476e92e5af2c856e017afb1df8f8dff42495bf /library
parent4854116da9c5e3c95bb8440f2ea997c54b6e1a61 (diff)
downloadydb-cda0c13f23f6b169fb0a49dc504b40a0aaecea09.tar.gz
Move contrib/tools/jdk to build/platform/java/jdk/testing
Diffstat (limited to 'library')
-rw-r--r--library/cpp/pybind/attr.h412
-rw-r--r--library/cpp/pybind/cast.cpp324
-rw-r--r--library/cpp/pybind/cast.h373
-rw-r--r--library/cpp/pybind/embedding.cpp63
-rw-r--r--library/cpp/pybind/embedding.h10
-rw-r--r--library/cpp/pybind/empty.cpp2
-rw-r--r--library/cpp/pybind/exceptions.cpp147
-rw-r--r--library/cpp/pybind/exceptions.h143
-rw-r--r--library/cpp/pybind/init.h25
-rw-r--r--library/cpp/pybind/method.h439
-rw-r--r--library/cpp/pybind/module.cpp72
-rw-r--r--library/cpp/pybind/module.h176
-rw-r--r--library/cpp/pybind/pod.cpp18
-rw-r--r--library/cpp/pybind/pod.h53
-rw-r--r--library/cpp/pybind/ptr.h51
-rw-r--r--library/cpp/pybind/typeattrs.h368
-rw-r--r--library/cpp/pybind/typedesc.cpp79
-rw-r--r--library/cpp/pybind/typedesc.h545
-rw-r--r--library/cpp/pybind/v2.cpp43
-rw-r--r--library/cpp/pybind/v2.h514
-rw-r--r--library/cpp/pybind/ya.make14
-rw-r--r--library/python/archive/__init__.py266
-rw-r--r--library/python/archive/ya.make19
-rw-r--r--library/python/cityhash/cityhash.pyx75
-rw-r--r--library/python/cityhash/hash.cpp32
-rw-r--r--library/python/cityhash/hash.h6
-rw-r--r--library/python/cityhash/ya.make16
-rw-r--r--library/python/codecs/__codecs.pyx61
-rw-r--r--library/python/codecs/__init__.py1
-rw-r--r--library/python/codecs/ya.make16
-rw-r--r--library/python/compress/__init__.py147
-rw-r--r--library/python/compress/ya.make16
-rw-r--r--library/python/json/__init__.py44
-rw-r--r--library/python/json/loads.cpp246
-rw-r--r--library/python/json/loads.h5
-rw-r--r--library/python/json/loads.pyx14
-rw-r--r--library/python/json/ya.make17
-rw-r--r--library/python/par_apply/__init__.py114
-rw-r--r--library/python/par_apply/ya.make11
-rw-r--r--library/python/retry/__init__.py250
-rw-r--r--library/python/retry/ya.make11
-rw-r--r--library/python/testing/yatest_common/ut/test.py17
-rw-r--r--library/python/testing/yatest_common/ut/ya.make17
-rw-r--r--library/python/testing/yatest_common/ya.make4
-rw-r--r--library/python/testing/yatest_common/yatest/common/runtime.py2
45 files changed, 5277 insertions, 1 deletions
diff --git a/library/cpp/pybind/attr.h b/library/cpp/pybind/attr.h
new file mode 100644
index 00000000000..5f25a6d73d1
--- /dev/null
+++ b/library/cpp/pybind/attr.h
@@ -0,0 +1,412 @@
+#pragma once
+
+#define PY_SSIZE_T_CLEAN
+#include <Python.h>
+#include <util/generic/string.h>
+#include <util/generic/map.h>
+#include <util/generic/set.h>
+#include <util/generic/vector.h>
+#include <util/generic/ptr.h>
+
+#include "cast.h"
+#include "exceptions.h"
+
+namespace NPyBind {
+ // TBaseAttrGetter
+ template <typename TObjType>
+ class TBaseAttrGetter {
+ public:
+ virtual ~TBaseAttrGetter() {
+ }
+ virtual bool GetAttr(PyObject* owner, const TObjType& self, const TString& attr, PyObject*& res) const = 0;
+
+ virtual bool HasAttr(PyObject* owner, const TObjType& self, const TString& attr, const TSet<TString>& hiddenNames) const {
+ if (hiddenNames.find(attr) != hiddenNames.end())
+ return false;
+ PyObject* res = nullptr;
+ if (!GetAttr(owner, self, attr, res))
+ return false;
+ Py_XDECREF(res);
+ return true;
+ }
+ };
+
+ template <typename TObjType>
+ class TBaseAttrSetter {
+ public:
+ virtual ~TBaseAttrSetter() {
+ }
+
+ virtual bool SetAttr(PyObject* owner, TObjType& self, const TString& attr, PyObject* val) = 0;
+ };
+
+ template <typename TObjType>
+ class TAttrGetters {
+ public:
+ typedef TSimpleSharedPtr<TBaseAttrGetter<TObjType>> TGetterPtr;
+
+ private:
+ typedef TVector<TGetterPtr> TGetterList;
+ typedef TMap<TString, TGetterList> TGetterMap;
+
+ const TSet<TString>& HiddenAttrNames;
+ TGetterMap Getters;
+
+ public:
+ TAttrGetters(const TSet<TString>& hiddenNames)
+ : HiddenAttrNames(hiddenNames)
+ {
+ }
+
+ void AddGetter(const TString& attr, TGetterPtr getter) {
+ Getters[attr].push_back(getter);
+ }
+
+ PyObject* GetAttr(PyObject* owner, const TObjType& self, const TString& attr) const {
+ typename TGetterMap::const_iterator it1 = Getters.find(attr);
+ if (it1 == Getters.end())
+ it1 = Getters.find("");
+ if (it1 == Getters.end())
+ return nullptr;
+ const TGetterList& lst = it1->second;
+ for (typename TGetterList::const_iterator it2 = lst.begin(), end = lst.end(); it2 != end; ++it2) {
+ PyObject* res = nullptr;
+ if ((*it2)->GetAttr(owner, self, attr, res))
+ return res;
+ // IMPORTANT!
+ // we have to fail GetAttr right there because we've failed because of internal python error/exception and can't continue iterating because
+ // it cause subsequent exceptions during call to Py_BuildValue
+ // moreover we have to preserve original exception right there
+ if (PyErr_Occurred()) {
+ break;
+ }
+ }
+ return nullptr;
+ }
+
+ bool HasAttr(PyObject* owner, const TObjType& self, const TString& attr) const {
+ typename TGetterMap::const_iterator it1 = Getters.find(attr);
+ if (it1 == Getters.end())
+ return false;
+ const TGetterList& lst = it1->second;
+ for (typename TGetterList::const_iterator it2 = lst.begin(), end = lst.end(); it2 != end; ++it2) {
+ if ((*it2)->HasAttr(owner, self, attr, HiddenAttrNames))
+ return true;
+ }
+ return false;
+ }
+
+ void GetAttrsDictionary(PyObject* owner, const TObjType& self, TMap<TString, PyObject*>& res) const {
+ for (typename TGetterMap::const_iterator it = Getters.begin(), end = Getters.end(); it != end; ++it) {
+ try {
+ if (HasAttr(owner, self, it->first)) {
+ auto attrPtr = GetAttr(owner, self, it->first);
+ if (attrPtr) {
+ res[it->first] = attrPtr;
+ }
+ if (PyErr_Occurred()) {
+ PyErr_Clear(); // Skip python errors as well
+ }
+ }
+ } catch (const std::exception&) {
+ // ignore this field
+ }
+ }
+ }
+
+ void GetAttrsNames(PyObject* owner, const TObjType& self, TVector<TString>& resultNames) const {
+ for (typename TGetterMap::const_iterator it = Getters.begin(), end = Getters.end(); it != end; ++it) {
+ if (HasAttr(owner, self, it->first))
+ resultNames.push_back(it->first);
+ }
+ }
+ };
+
+ template <typename TObjType>
+ class TGenericAttrGetter: public TBaseAttrGetter<TObjType> {
+ private:
+ TString AttrName;
+
+ public:
+ TGenericAttrGetter(const TString& attrName)
+ : AttrName(attrName)
+ {
+ }
+
+ bool GetAttr(PyObject* obj, const TObjType&, const TString&, PyObject*& res) const override {
+ auto str = NameFromString(AttrName);
+ res = PyObject_GenericGetAttr(obj, str.Get());
+ if (!res && !PyErr_Occurred())
+ ythrow TPyErr(PyExc_AttributeError) << "Can't get generic attribute '" << AttrName << "'";
+ return res;
+ }
+ };
+
+ template <typename TObjType>
+ class TAttrSetters {
+ private:
+ typedef TSimpleSharedPtr<TBaseAttrSetter<TObjType>> TSetterPtr;
+ typedef TVector<TSetterPtr> TSetterList;
+ typedef TMap<TString, TSetterList> TSetterMap;
+
+ TSetterMap Setters;
+
+ public:
+ void AddSetter(const TString& attr, TSetterPtr setter) {
+ Setters[attr].push_back(setter);
+ }
+
+ bool SetAttr(PyObject* owner, TObjType& self, const TString& attr, PyObject* val) {
+ typename TSetterMap::const_iterator it1 = Setters.find(attr);
+ if (it1 == Setters.end())
+ it1 = Setters.find("");
+ if (it1 == Setters.end())
+ return false;
+ const TSetterList& lst = it1->second;
+ for (typename TSetterList::const_iterator it2 = lst.begin(), end = lst.end(); it2 != end; ++it2) {
+ if ((*it2)->SetAttr(owner, self, attr, val))
+ return true;
+ }
+ return false;
+ }
+
+ bool SetAttrDictionary(PyObject* owner, TObjType& self, TMap<TString, PyObject*>& dict) {
+ for (TMap<TString, PyObject*>::const_iterator it = dict.begin(), end = dict.end(); it != end; ++it) {
+ try {
+ SetAttr(owner, self, it->first, it->second);
+ } catch (std::exception&) {
+ // ignore this field
+ }
+ }
+
+ return true;
+ }
+ };
+
+ /**
+ * TMethodAttrGetter - this class maps Python attribute read to C++ method call
+ */
+ template <typename TObjType, typename TResult, typename TSubObject>
+ class TMethodAttrGetter: public TBaseAttrGetter<TObjType> {
+ private:
+ typedef TResult (TSubObject::*TMethod)() const;
+ TMethod Method;
+
+ public:
+ TMethodAttrGetter(TMethod method)
+ : Method(method)
+ {
+ }
+
+ bool GetAttr(PyObject*, const TObjType& self, const TString&, PyObject*& res) const override {
+ const TSubObject* sub = dynamic_cast<const TSubObject*>(&self);
+ if (sub == nullptr)
+ return false;
+ res = BuildPyObject((sub->*Method)());
+ return (res != nullptr);
+ }
+ };
+
+ template <typename TObjType, typename TFunctor>
+ class TFunctorAttrGetter: public TBaseAttrGetter<TObjType> {
+ TFunctor Functor;
+ public:
+ explicit TFunctorAttrGetter(TFunctor functor)
+ : Functor(functor)
+ {
+ }
+
+ bool GetAttr(PyObject*, const TObjType& self, const TString&, PyObject*& res) const override {
+ res = BuildPyObject(Functor(self));
+ return (res != nullptr);
+ }
+ };
+
+
+ /**
+ * TMethodAttrGetterWithCheck - this class maps Python attribute read to C++ HasAttr/GetAttr call
+ * If HasAttr returns false, None is returned.
+ * Otherwise GetAttr is called.
+ */
+ template <typename TObjType, typename TResult, typename TSubObject>
+ class TMethodAttrGetterWithCheck: public TBaseAttrGetter<TObjType> {
+ private:
+ typedef TResult (TSubObject::*TMethod)() const;
+ typedef bool (TSubObject::*TCheckerMethod)() const;
+ TMethod Method;
+ TCheckerMethod CheckerMethod;
+
+ public:
+ TMethodAttrGetterWithCheck(TMethod method, TCheckerMethod checkerMethod)
+ : Method(method)
+ , CheckerMethod(checkerMethod)
+ {
+ }
+
+ bool GetAttr(PyObject*, const TObjType& self, const TString&, PyObject*& res) const override {
+ const TSubObject* sub = dynamic_cast<const TSubObject*>(&self);
+ if (sub == nullptr)
+ return false;
+ if ((sub->*CheckerMethod)())
+ res = BuildPyObject((sub->*Method)());
+ else {
+ Py_INCREF(Py_None);
+ res = Py_None;
+ }
+ return (res != nullptr);
+ }
+ };
+
+ template <typename TObjType, typename TResult, typename TSubObject, typename TMapper>
+ class TMethodAttrMappingGetter: public TBaseAttrGetter<TObjType> {
+ private:
+ typedef TResult (TSubObject::*TMethod)() const;
+
+ TMethod Method;
+ TMapper Mapper;
+
+ public:
+ TMethodAttrMappingGetter(TMethod method, TMapper mapper)
+ : Method(method)
+ , Mapper(mapper)
+ {
+ }
+
+ bool GetAttr(PyObject*, const TObjType& self, const TString&, PyObject*& res) const override {
+ const TSubObject* sub = dynamic_cast<const TSubObject*>(&self);
+ if (sub == nullptr)
+ return false;
+ res = BuildPyObject(Mapper((sub->*Method)()));
+ return (res != nullptr);
+ }
+ };
+
+ template <typename TObjType, typename TResult, typename TSubObject, typename TMapper>
+ TSimpleSharedPtr<TBaseAttrGetter<TObjType>>
+ CreateMethodAttrMappingGetter(TResult (TSubObject::*method)() const,
+ TMapper mapper) {
+ return new TMethodAttrMappingGetter<TObjType, TResult, TSubObject, TMapper>(method,
+ mapper);
+ }
+
+ template <typename TObjType, typename TResult, typename TValue, typename TSubObject>
+ class TMethodAttrSetter: public TBaseAttrSetter<TObjType> {
+ private:
+ typedef TResult (TSubObject::*TMethod)(TValue&);
+ TMethod Method;
+
+ public:
+ TMethodAttrSetter(TMethod method)
+ : Method(method)
+ {
+ }
+
+ virtual bool SetAttr(PyObject*, TObjType& self, const TString&, PyObject* val) {
+ TSubObject* sub = dynamic_cast<TSubObject*>(&self);
+ if (sub == nullptr)
+ return false;
+ TValue value;
+ if (!FromPyObject(val, value))
+ return false;
+ (sub->*Method)(value);
+ return true;
+ }
+ };
+
+ template <typename TObjType, typename TValue, typename TFunctor>
+ class TFunctorAttrSetter: public TBaseAttrSetter<TObjType> {
+ TFunctor Functor;
+ public:
+ explicit TFunctorAttrSetter(TFunctor functor)
+ : Functor(functor)
+ {
+ }
+
+ bool SetAttr(PyObject*, TObjType& self, const TString&, PyObject* val) const override {
+ TValue value;
+ if (!FromPyObject(val, value))
+ return false;
+ auto res = BuildPyObject(Functor(self, value));
+ return (res != nullptr);
+ }
+ };
+ template <typename TObjType, typename TResult, typename TSubObject>
+ TSimpleSharedPtr<TBaseAttrGetter<TObjType>> CreateMethodAttrGetter(TResult (TSubObject::*method)() const) {
+ return new TMethodAttrGetter<TObjType, TResult, TSubObject>(method);
+ }
+
+ template <typename TObjType, typename TFunctor>
+ TSimpleSharedPtr<TFunctorAttrGetter<TObjType, TFunctor>> CreateFunctorAttrGetter(TFunctor functor) {
+ return MakeSimpleShared<TFunctorAttrGetter<TObjType, TFunctor>>(functor);
+ }
+
+ template <typename TObjType, typename TResult, typename TSubObject>
+ TSimpleSharedPtr<TBaseAttrGetter<TObjType>> CreateMethodAttrGetterWithCheck(
+ TResult (TSubObject::*method)() const,
+ bool (TSubObject::*checkerMethod)() const) {
+ return new TMethodAttrGetterWithCheck<TObjType, TResult, TSubObject>(method, checkerMethod);
+ }
+
+ template <typename TObjType, typename TResult, typename TValue, typename TSubObject>
+ TSimpleSharedPtr<TBaseAttrSetter<TObjType>> CreateMethodAttrSetter(TResult (TSubObject::*method)(TValue&)) {
+ return new TMethodAttrSetter<TObjType, TResult, TValue, TSubObject>(method);
+ }
+
+ template <typename TObjType, typename TFunctor, typename TValue>
+ TSimpleSharedPtr<TFunctorAttrSetter<TObjType, TValue, TFunctor>> CreateFunctorAttrSetter(TFunctor functor) {
+ return MakeSimpleShared<TFunctorAttrSetter<TObjType, TValue, TFunctor>>(functor);
+ }
+
+ template <typename TObjType, typename TValue, typename TSubObject>
+ class TDirectAttrSetter: public TBaseAttrSetter<TObjType> {
+ private:
+ typedef TValue TSubObject::*TValueType;
+ TValueType Value;
+
+ public:
+ TDirectAttrSetter(TValueType value)
+ : Value(value)
+ {
+ }
+
+ bool SetAttr(PyObject*, TObjType& self, const TString&, PyObject* val) override {
+ TSubObject* sub = dynamic_cast<TSubObject*>(&self);
+ if (sub == NULL)
+ return false;
+ if (!FromPyObject(val, sub->*Value))
+ return false;
+ return true;
+ }
+ };
+
+ template <typename TObjType, typename TValue, typename TSubObject>
+ TSimpleSharedPtr<TBaseAttrSetter<TObjType>> CreateAttrSetter(TValue TSubObject::*value) {
+ return new TDirectAttrSetter<TObjType, TValue, TSubObject>(value);
+ }
+
+ template <typename TObjType, typename TValue, typename TSubObject>
+ class TDirectAttrGetter: public TBaseAttrGetter<TObjType> {
+ private:
+ typedef TValue TSubObject::*TValueType;
+ TValueType Value;
+
+ public:
+ TDirectAttrGetter(TValueType value)
+ : Value(value)
+ {
+ }
+
+ bool GetAttr(PyObject*, const TObjType& self, const TString&, PyObject*& res) const override {
+ const TSubObject* sub = dynamic_cast<const TSubObject*>(&self);
+ if (sub == nullptr)
+ return false;
+ res = BuildPyObject(sub->*Value);
+ return (res != nullptr);
+ }
+ };
+
+ template <typename TObjType, typename TValue, typename TSubObject>
+ TSimpleSharedPtr<TBaseAttrGetter<TObjType>> CreateAttrGetter(TValue TSubObject::*value) {
+ return new TDirectAttrGetter<TObjType, TValue, TSubObject>(value);
+ }
+}
diff --git a/library/cpp/pybind/cast.cpp b/library/cpp/pybind/cast.cpp
new file mode 100644
index 00000000000..60a44b0e833
--- /dev/null
+++ b/library/cpp/pybind/cast.cpp
@@ -0,0 +1,324 @@
+#include "cast.h"
+#include <util/generic/yexception.h>
+#include <util/generic/buffer.h>
+
+namespace NPyBind {
+ PyObject* GetTrueRef(bool incref) {
+ if (incref)
+ Py_RETURN_TRUE;
+ return Py_True;
+ }
+
+ PyObject* GetFalseRef(bool incref) {
+ if (incref)
+ Py_RETURN_FALSE;
+ return Py_False;
+ }
+
+ PyObject* BuildPyObject(int val) {
+ return Py_BuildValue("i", val);
+ }
+
+ PyObject* BuildPyObject(unsigned int val) {
+ return Py_BuildValue("I", val);
+ }
+
+ PyObject* BuildPyObject(long int val) {
+ return Py_BuildValue("l", val);
+ }
+
+ PyObject* BuildPyObject(unsigned long int val) {
+ return Py_BuildValue("k", val);
+ }
+
+#ifdef PY_LONG_LONG
+ PyObject* BuildPyObject(PY_LONG_LONG val) {
+ return Py_BuildValue("L", val);
+ }
+
+ PyObject* BuildPyObject(unsigned PY_LONG_LONG val) {
+ return Py_BuildValue("K", val);
+ }
+#endif
+
+ PyObject* BuildPyObject(float val) {
+ return Py_BuildValue("f", val);
+ }
+
+ PyObject* BuildPyObject(double val) {
+ return Py_BuildValue("d", val);
+ }
+
+ PyObject* BuildPyObject(const TStringBuf& val) {
+ if (!val.IsInited())
+ Py_RETURN_NONE;
+
+ PyObject* stringValue = Py_BuildValue("s#", val.data(), static_cast<int>(val.length()));
+ if (stringValue != nullptr) {
+ return stringValue;
+ }
+ if (PyErr_ExceptionMatches(PyExc_UnicodeDecodeError)) {
+ PyErr_Clear();
+ } else {
+ return nullptr;
+ }
+ return Py_BuildValue("y#", val.data(), static_cast<int>(val.length()));
+ }
+
+ PyObject* BuildPyObject(const char* val) {
+ if (val == nullptr)
+ Py_RETURN_NONE;
+ PyObject* stringValue = Py_BuildValue("s#", val, static_cast<int>(strlen(val)));
+ if (stringValue != nullptr) {
+ return stringValue;
+ }
+ if (PyErr_ExceptionMatches(PyExc_UnicodeDecodeError)) {
+ PyErr_Clear();
+ } else {
+ return nullptr;
+ }
+ return Py_BuildValue("y#", val, static_cast<int>(strlen(val)));
+ }
+
+ PyObject* BuildPyObject(const TWtringBuf& val) {
+ if (!val.IsInited())
+ Py_RETURN_NONE;
+#if PY_VERSION_HEX < 0x03030000
+ TPyObjectPtr result(PyUnicode_FromUnicode(nullptr, val.size()), true);
+ Py_UNICODE* buf = PyUnicode_AS_UNICODE(result.Get());
+ if (buf == nullptr)
+ Py_RETURN_NONE;
+ for (size_t i = 0; i < val.size(); ++i) {
+ buf[i] = static_cast<Py_UNICODE>(val[i]);
+ }
+#else
+ PyObject* unicodeValue = PyUnicode_FromKindAndData(PyUnicode_2BYTE_KIND, val.data(), val.size());
+ if (unicodeValue == nullptr)
+ Py_RETURN_NONE;
+ TPyObjectPtr result(unicodeValue, true);
+#endif
+ return result.RefGet();
+ }
+
+ PyObject* BuildPyObject(const TBuffer& val) {
+ TPyObjectPtr res(PyList_New(val.size()), true);
+ for (size_t i = 0, size = val.Size(); i < size; ++i)
+ PyList_SetItem(res.Get(), i, BuildPyObject(val.Data()[i]));
+ return res.RefGet();
+ }
+
+ PyObject* BuildPyObject(bool val) {
+ if (val)
+ Py_RETURN_TRUE;
+ else
+ Py_RETURN_FALSE;
+ }
+
+ PyObject* BuildPyObject(PyObject* val) {
+ Py_XINCREF(val);
+ return val;
+ }
+
+ PyObject* BuildPyObject(TPyObjectPtr ptr) {
+ return ptr.RefGet();
+ }
+
+ /* python represents (http://docs.python.org/c-api/arg.html#Py_BuildValue)
+ * char, uchar, short, ushort, int, long as PyInt
+ * uint, ulong as PyInt or PyLong (if exceeds sys.maxint)
+ * longlong, ulonglong as PyLong
+ */
+
+ template <>
+ bool FromPyObject(PyObject* obj, long& res) {
+ if (PyLong_Check(obj)) {
+ res = PyLong_AsLong(obj);
+ return true;
+ }
+ if (PyFloat_Check(obj)) {
+ res = static_cast<long>(PyFloat_AsDouble(obj));
+ return true;
+ }
+#if PY_MAJOR_VERSION < 3
+ res = PyInt_AsLong(obj);
+#endif
+ return -1 != res || !PyErr_Occurred();
+ }
+
+ template <>
+ bool FromPyObject(PyObject* obj, unsigned long& res) {
+ long lres;
+ if (!FromPyObject(obj, lres))
+ return false;
+ if (lres < 0)
+ return false;
+ res = static_cast<unsigned long long>(lres);
+ return true;
+ }
+
+ template <>
+ bool FromPyObject(PyObject* obj, int& res) {
+ long lres;
+ if (!FromPyObject(obj, lres))
+ return false;
+ res = static_cast<int>(lres);
+ return true;
+ }
+
+ template <>
+ bool FromPyObject(PyObject* obj, unsigned char& res) {
+ long lres;
+ if (!FromPyObject(obj, lres))
+ return false;
+ res = static_cast<unsigned char>(lres);
+ return true;
+ }
+
+ template <>
+ bool FromPyObject(PyObject* obj, char& res) {
+ long lres;
+ if (!FromPyObject(obj, lres))
+ return false;
+ res = static_cast<char>(lres);
+ return true;
+ }
+
+ template <>
+ bool FromPyObject(PyObject* obj, unsigned int& res) {
+ unsigned long lres;
+ if (!FromPyObject(obj, lres))
+ return false;
+ res = static_cast<unsigned int>(lres);
+ return true;
+ }
+
+#ifdef HAVE_LONG_LONG
+ template <>
+ bool FromPyObject(PyObject* obj, long long& res) {
+ if (PyLong_Check(obj)) {
+ res = PyLong_AsLongLong(obj);
+ return -1 != res || !PyErr_Occurred();
+ }
+ long lres;
+ if (!FromPyObject(obj, lres))
+ return false;
+ res = static_cast<long long>(lres);
+ return true;
+ }
+
+ template <>
+ bool FromPyObject(PyObject* obj, unsigned long long& res) {
+ if (PyLong_Check(obj)) {
+ res = PyLong_AsUnsignedLongLong(obj);
+ return static_cast<unsigned long long>(-1) != res || !PyErr_Occurred();
+ }
+ long lres;
+ if (!FromPyObject(obj, lres))
+ return false;
+ res = static_cast<unsigned long long>(lres);
+ return true;
+ }
+#endif
+
+ template <>
+ bool FromPyObject(PyObject* obj, double& res) {
+ if (PyFloat_Check(obj)) {
+ res = PyFloat_AsDouble(obj);
+ return true;
+ }
+ long long lres;
+ if (!FromPyObject(obj, lres))
+ return false;
+ res = static_cast<double>(lres);
+ return true;
+ }
+
+ template <>
+ bool FromPyObject(PyObject* obj, float& res) {
+ double dres;
+ if (!FromPyObject(obj, dres))
+ return false;
+ res = static_cast<float>(dres);
+ return true;
+ }
+
+ template <>
+ bool FromPyObject(PyObject* obj, bool& res) {
+ if (!PyBool_Check(obj))
+ return false;
+ if (obj == Py_True)
+ res = true;
+ else
+ res = false;
+ return true;
+ }
+
+ template <>
+ bool FromPyObject(PyObject* obj, PyObject*& res) {
+ Py_XINCREF(obj);
+ res = obj;
+ return true;
+ }
+
+ template <>
+ bool FromPyObject(PyObject* obj, TPyObjectPtr& res) {
+ res = TPyObjectPtr(obj);
+ return true;
+ }
+
+ static inline bool _FromPyObject(PyObject* obj, TStringBuf& res) {
+ char* str;
+ Py_ssize_t len;
+#if PY_MAJOR_VERSION >= 3
+ if (PyUnicode_Check(obj)) {
+ auto buf = PyUnicode_AsUTF8AndSize(obj, &len);
+ res = TStringBuf(buf, len);
+ return true;
+ }
+#endif
+ if (-1 == PyBytes_AsStringAndSize(obj, &str, &len) || 0 > len)
+ return false;
+ res = TStringBuf(str, len);
+ return true;
+ }
+
+ bool FromPyObject(PyObject* obj, TStringBuf& res) {
+ return _FromPyObject(obj, res);
+ }
+
+ bool FromPyObject(PyObject* obj, TString& res) {
+ TStringBuf str;
+ if (!_FromPyObject(obj, str))
+ return false;
+ res = str;
+ return true;
+ }
+
+ bool FromPyObject(PyObject* obj, TUtf16String& res) {
+ if (!PyUnicode_Check(obj))
+ return false;
+ auto str = TPyObjectPtr(PyUnicode_AsUTF16String(obj), true);
+ if (!str)
+ return false;
+ constexpr auto BOM_SIZE = 2;
+ size_t len = (static_cast<size_t>(PyBytes_GET_SIZE(str.Get())) - BOM_SIZE) / 2;
+ res.resize(len);
+ memcpy(res.begin(), PyBytes_AS_STRING(str.Get()) + BOM_SIZE, len * 2);
+ return (nullptr == PyErr_Occurred());
+ }
+
+ bool FromPyObject(PyObject* obj, TBuffer& res) {
+ if (!PyList_Check(obj))
+ return false;
+ size_t cnt = PyList_Size(obj);
+ res.Reserve(cnt);
+ for (size_t i = 0; i < cnt; ++i) {
+ PyObject* item = PyList_GET_ITEM(obj, i);
+ char ch = 0;
+ if (!FromPyObject(item, ch))
+ return false;
+ res.Append(ch);
+ }
+ return true;
+ }
+}
diff --git a/library/cpp/pybind/cast.h b/library/cpp/pybind/cast.h
new file mode 100644
index 00000000000..1f3d7d8366f
--- /dev/null
+++ b/library/cpp/pybind/cast.h
@@ -0,0 +1,373 @@
+#pragma once
+
+#define PY_SSIZE_T_CLEAN
+#include <Python.h>
+#include <util/generic/strbuf.h>
+#include <util/generic/vector.h>
+#include <util/generic/set.h>
+#include <util/generic/yexception.h>
+#include <util/generic/hash.h>
+#include <util/generic/map.h>
+#include <util/generic/maybe.h>
+#include <utility>
+#include <initializer_list>
+#include "ptr.h"
+
+namespace NPyBind {
+ PyObject* GetTrueRef(bool incref = true);
+ PyObject* GetFalseRef(bool incref = true);
+
+ PyObject* BuildPyObject(int val);
+ PyObject* BuildPyObject(unsigned int val);
+ PyObject* BuildPyObject(long int val);
+ PyObject* BuildPyObject(unsigned long int val);
+#ifdef PY_LONG_LONG
+ PyObject* BuildPyObject(PY_LONG_LONG val);
+ PyObject* BuildPyObject(unsigned PY_LONG_LONG val);
+#endif
+ PyObject* BuildPyObject(float val);
+ PyObject* BuildPyObject(double val);
+ PyObject* BuildPyObject(const TStringBuf& val);
+ PyObject* BuildPyObject(const char* val);
+ PyObject* BuildPyObject(const TWtringBuf& val);
+ PyObject* BuildPyObject(const TBuffer& val);
+ PyObject* BuildPyObject(bool val);
+ PyObject* BuildPyObject(PyObject*);
+ PyObject* BuildPyObject(TPyObjectPtr);
+
+ template <typename T>
+ PyObject* BuildPyObject(const TVector<T>& val);
+
+ template <typename T>
+ PyObject* BuildPyObject(const TSet<T>& val);
+
+ template <typename TKey, typename TVal>
+ PyObject* BuildPyObject(const THashMap<TKey, TVal>& val);
+
+ template <typename T1, typename T2>
+ PyObject* BuildPyObject(const std::pair<T1, T2>& val) {
+ TPyObjectPtr first(BuildPyObject(val.first), true);
+ if (!first) {
+ return nullptr;
+ }
+ TPyObjectPtr second(BuildPyObject(val.second), true);
+ if (!first || !second) {
+ return nullptr;
+ }
+ TPyObjectPtr res(PyList_New(2), true);
+ PyList_SetItem(res.Get(), 0, first.RefGet());
+ PyList_SetItem(res.Get(), 1, second.RefGet());
+ return res.RefGet();
+ }
+
+ template <typename T>
+ PyObject* BuildPyObject(const TVector<T>& val) {
+ TPyObjectPtr res(PyList_New(val.size()), true);
+ for (size_t i = 0, size = val.size(); i < size; ++i) {
+ auto pythonVal = BuildPyObject(std::move(val[i]));
+ if (!pythonVal) {
+ return nullptr;
+ }
+ PyList_SetItem(res.Get(), i, pythonVal);
+ }
+ return res.RefGet();
+ }
+
+ template <typename T>
+ PyObject* BuildPyObject(TVector<T>&& val) {
+ TPyObjectPtr res(PyList_New(val.size()), true);
+ for (size_t i = 0, size = val.size(); i < size; ++i) {
+ auto pythonVal = BuildPyObject(std::move(val[i]));
+ if (!pythonVal) {
+ return nullptr;
+ }
+ PyList_SetItem(res.Get(), i, pythonVal);
+ }
+ return res.RefGet();
+ }
+
+ template <typename T>
+ PyObject* BuildPyObject(const TSet<T>& val) {
+ TPyObjectPtr res(PySet_New(nullptr), true);
+ for (const auto& v : val) {
+ auto pythonVal = BuildPyObject(std::move(v));
+ if (!pythonVal) {
+ return nullptr;
+ }
+ PySet_Add(res.Get(), pythonVal);
+ }
+ return res.RefGet();
+ }
+
+ template <typename T>
+ PyObject* BuildPyObject(const THashSet<T>& val) {
+ TPyObjectPtr res(PySet_New(nullptr), true);
+ for (const auto& v : val) {
+ auto pythonVal = BuildPyObject(std::move(v));
+ if (!pythonVal) {
+ return nullptr;
+ }
+ PySet_Add(res.Get(), pythonVal);
+ }
+ return res.RefGet();
+ }
+
+ template <typename TKey, typename TVal>
+ PyObject* BuildPyObject(const THashMap<TKey, TVal>& val) {
+ TPyObjectPtr res(PyDict_New(), true);
+ for (typename THashMap<TKey, TVal>::const_iterator it = val.begin(), end = val.end(); it != end; ++it) {
+ auto prevOccurred = PyErr_Occurred();
+ Y_UNUSED(prevOccurred);
+ TPyObjectPtr k(BuildPyObject(it->first), true);
+ if (!k) {
+ return nullptr;
+ }
+ TPyObjectPtr v(BuildPyObject(it->second), true);
+ if (!v) {
+ return nullptr;
+ }
+ PyDict_SetItem(res.Get(), k.Get(), v.Get());
+ }
+ return res.RefGet();
+ }
+
+ template <typename TKey, typename TVal>
+ PyObject* BuildPyObject(const TMap<TKey, TVal>& val) {
+ TPyObjectPtr res(PyDict_New(), true);
+ for (typename TMap<TKey, TVal>::const_iterator it = val.begin(), end = val.end(); it != end; ++it) {
+ TPyObjectPtr k(BuildPyObject(it->first), true);
+ if (!k) {
+ return nullptr;
+ }
+ TPyObjectPtr v(BuildPyObject(it->second), true);
+ if (!v) {
+ return nullptr;
+ }
+ PyDict_SetItem(res.Get(), k.Get(), v.Get());
+ }
+ return res.RefGet();
+ }
+
+
+ template <typename TKey, typename TVal>
+ PyObject* BuildPyObject(const TMultiMap<TKey, TVal>& val) {
+ TPyObjectPtr res(PyDict_New(), true);
+ TMaybe<TKey> prevKey;
+ TPyObjectPtr currentEntry(PyList_New(0), true);
+ for (const auto& [key, value]: val) {
+ if (prevKey.Defined() && prevKey != key) {
+ TPyObjectPtr pyPrevKey(BuildPyObject(*prevKey), true);
+ if (!pyPrevKey) {
+ return nullptr;
+ }
+ PyDict_SetItem(res.Get(), pyPrevKey.Get(), currentEntry.Get());
+ currentEntry = TPyObjectPtr(PyList_New(0), true);
+ }
+ TPyObjectPtr pyValue(BuildPyObject(value), true);
+ if (!pyValue) {
+ return nullptr;
+ }
+ PyList_Append(currentEntry.Get(), pyValue.Get());
+ prevKey = key;
+ }
+
+ if (prevKey.Defined()) {
+ TPyObjectPtr pyPrevKey(BuildPyObject(*prevKey), true);
+ if (!pyPrevKey) {
+ return nullptr;
+ }
+ PyDict_SetItem(res.Get(), pyPrevKey.Get(), currentEntry.Get());
+ }
+ return res.RefGet();
+ }
+
+ template <typename T>
+ PyObject* BuildPyObject(const TMaybe<T>& val) {
+ if (!val.Defined())
+ Py_RETURN_NONE;
+ return BuildPyObject(val.GetRef());
+ }
+
+ template <typename T, typename C, typename D>
+ PyObject* BuildPyObject(const TSharedPtr<T, C, D>& val) {
+ if (!val.Get())
+ Py_RETURN_NONE;
+ return BuildPyObject(*val.Get());
+ }
+
+ template <typename T>
+ bool FromPyObject(PyObject* obj, T& res);
+
+ bool FromPyObject(PyObject* obj, TString& res);
+ bool FromPyObject(PyObject* obj, TStringBuf& res);
+ bool FromPyObject(PyObject* obj, TUtf16String& res);
+ bool FromPyObject(PyObject* obj, TBuffer& res);
+
+ template <typename T>
+ bool FromPyObject(PyObject* obj, TMaybe<T>& res) {
+ //we need to save current error before trying derserialize the value
+ //because it can produce conversion errors in python that we don't need to handle
+ struct TError {
+ public:
+ TError() {
+ PyErr_Fetch(&Type, &Value, &Traceback);
+ }
+ ~TError() {
+ PyErr_Restore(Type, Value, Traceback);
+
+ }
+ private:
+ PyObject* Type = nullptr;
+ PyObject* Value = nullptr;
+ PyObject* Traceback = nullptr;
+ } currentPyExcInfo;
+ T val;
+ if (FromPyObject(obj, val)) {
+ res = val;
+ return true;
+ }
+ if (obj == Py_None) {
+ res = Nothing();
+ return true;
+ }
+ return false;
+ }
+
+ template <typename T1, typename T2>
+ bool FromPyObject(PyObject* obj, std::pair<T1, T2>& res) {
+ PyObject* first;
+ PyObject* second;
+ if (PyTuple_Check(obj) && 2 == PyTuple_Size(obj)) {
+ first = PyTuple_GET_ITEM(obj, 0);
+ second = PyTuple_GET_ITEM(obj, 1);
+ } else if (PyList_Check(obj) && 2 == PyList_Size(obj)) {
+ first = PyList_GET_ITEM(obj, 0);
+ second = PyList_GET_ITEM(obj, 1);
+ } else {
+ return false;
+ }
+ return FromPyObject(first, res.first) && FromPyObject(second, res.second);
+ }
+
+ template <typename T>
+ bool FromPyObject(PyObject* obj, TVector<T>& res) {
+ if (!PyList_Check(obj))
+ return false;
+ size_t cnt = PyList_Size(obj);
+ res.resize(cnt);
+ for (size_t i = 0; i < cnt; ++i) {
+ PyObject* item = PyList_GET_ITEM(obj, i);
+ if (!FromPyObject(item, res[i]))
+ return false;
+ }
+ return true;
+ }
+
+ template <typename K, typename V>
+ bool FromPyObject(PyObject* obj, THashMap<K, V>& res) {
+ if (!PyDict_Check(obj))
+ return false;
+ TPyObjectPtr list(PyDict_Keys(obj), true);
+ size_t cnt = PyList_Size(list.Get());
+ for (size_t i = 0; i < cnt; ++i) {
+ PyObject* key = PyList_GET_ITEM(list.Get(), i);
+ PyObject* value = PyDict_GetItem(obj, key);
+ K rkey;
+ V rvalue;
+ if (!FromPyObject(key, rkey))
+ return false;
+ if (!FromPyObject(value, rvalue))
+ return false;
+ res[rkey] = rvalue;
+ }
+ return true;
+ }
+
+ template <typename K, typename V>
+ bool FromPyObject(PyObject* obj, TMap<K, V>& res) {
+ if (!PyDict_Check(obj))
+ return false;
+ TPyObjectPtr list(PyDict_Keys(obj), true);
+ size_t cnt = PyList_Size(list.Get());
+ for (size_t i = 0; i < cnt; ++i) {
+ PyObject* key = PyList_GET_ITEM(list.Get(), i);
+ PyObject* value = PyDict_GetItem(obj, key);
+ K rkey;
+ V rvalue;
+ if (!FromPyObject(key, rkey))
+ return false;
+ if (!FromPyObject(value, rvalue))
+ return false;
+ res[rkey] = rvalue;
+ }
+ return true;
+ }
+
+ class cast_exception: public TBadCastException {
+ };
+
+ template <typename T>
+ T FromPyObject(PyObject* obj) {
+ T res;
+ if (!FromPyObject(obj, res))
+ ythrow cast_exception() << "Cannot cast argument to " << TypeName<T>();
+ return res;
+ }
+
+ template <class... Args, std::size_t... I>
+ bool ExtractArgs(std::index_sequence<I...>, PyObject* args, Args&... outArgs) {
+ if (!args || !PyTuple_Check(args) || PyTuple_Size(args) != sizeof...(Args))
+ return false;
+ bool res = true;
+ (void)std::initializer_list<bool>{(res = res && NPyBind::FromPyObject(PyTuple_GET_ITEM(args, I), outArgs))...};
+ return res;
+ }
+
+ template <class... Args>
+ bool ExtractArgs(PyObject* args, Args&... outArgs) {
+ return ExtractArgs(std::index_sequence_for<Args...>(), args, outArgs...);
+ }
+
+ template <class... Args, std::size_t... I>
+ bool ExtractOptionalArgs(std::index_sequence<I...>, PyObject* args, PyObject* kwargs, const char* keywords[], Args&... outArgs) {
+ PyObject* pargs[sizeof...(Args)] = {};
+ static const char format[sizeof...(Args) + 2] = {'|', ((void)I, 'O')..., 0};
+ if (!PyArg_ParseTupleAndKeywords(args, kwargs, format, const_cast<char**>(keywords), &pargs[I]...))
+ return false;
+ bool res = true;
+ (void)std::initializer_list<bool>{(res = res && (!pargs[I] || NPyBind::FromPyObject(pargs[I], outArgs)))...};
+ return res;
+ }
+
+ template <class... Args>
+ bool ExtractOptionalArgs(PyObject* args, PyObject* kwargs, const char* keywords[], Args&... outArgs) {
+ return ExtractOptionalArgs(std::index_sequence_for<Args...>(), args, kwargs, keywords, outArgs...);
+ }
+
+ template <typename... Args, std::size_t... I>
+ static auto GetArguments(std::index_sequence<I...>, PyObject* args) {
+ Y_UNUSED(args); // gcc bug
+ return std::make_tuple(FromPyObject<std::remove_cv_t<std::remove_reference_t<Args>>>(PyTuple_GetItem(args, I))...);
+ }
+
+ template <typename... Args>
+ static auto GetArguments(PyObject* args) {
+ return GetArguments<Args...>(std::index_sequence_for<Args...>(), args);
+ }
+
+ inline PyObject* ReturnString(TStringBuf s) {
+#if PY_MAJOR_VERSION >= 3
+ return PyUnicode_FromStringAndSize(s.data(), s.size());
+#else
+ return PyBytes_FromStringAndSize(s.data(), s.size());
+#endif
+ }
+
+ inline TPyObjectPtr ReturnBytes(TStringBuf s) {
+ return TPyObjectPtr(PyBytes_FromStringAndSize(s.data(), s.size()), true);
+ }
+
+ inline TPyObjectPtr NameFromString(TStringBuf s) {
+ return TPyObjectPtr(ReturnString(s), true);
+ }
+}
diff --git a/library/cpp/pybind/embedding.cpp b/library/cpp/pybind/embedding.cpp
new file mode 100644
index 00000000000..cf8941a92af
--- /dev/null
+++ b/library/cpp/pybind/embedding.cpp
@@ -0,0 +1,63 @@
+#define PY_SSIZE_T_CLEAN
+#include <Python.h>
+
+#include "embedding.h"
+
+#include <util/generic/ptr.h>
+#include <util/generic/yexception.h>
+
+namespace NPyBind {
+#if PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION < 8
+ class TDeleteRawMem {
+ public:
+ template <typename T>
+ static inline void Destroy(T* t) noexcept {
+ PyMem_RawFree(t);
+ }
+ };
+
+ template <typename T>
+ using TRawMemHolder = THolder<T, TDeleteRawMem>;
+
+ static void SetProgramName(char* name) {
+ TRawMemHolder<wchar_t> wideName(Py_DecodeLocale(name, nullptr));
+ Y_ENSURE(wideName);
+ Py_SetProgramName(wideName.Get());
+ }
+#endif
+
+ TEmbedding::TEmbedding(char* argv0) {
+#if PY_MAJOR_VERSION < 3
+ Py_SetProgramName(argv0);
+ Py_Initialize();
+#elif PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 8
+ PyStatus status;
+
+ PyConfig config;
+ PyConfig_InitPythonConfig(&config);
+ // Disable parsing command line arguments
+ config.parse_argv = 0;
+
+ status = PyConfig_SetBytesString(&config, &config.program_name, argv0);
+ if (PyStatus_Exception(status)) {
+ PyConfig_Clear(&config);
+ Py_ExitStatusException(status);
+ }
+
+ status = Py_InitializeFromConfig(&config);
+ if (PyStatus_Exception(status)) {
+ PyConfig_Clear(&config);
+ Py_ExitStatusException(status);
+ }
+
+ PyConfig_Clear(&config);
+#elif PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION < 8
+ SetProgramName(argv0);
+ Py_Initialize();
+#endif
+ }
+
+ TEmbedding::~TEmbedding() {
+ Py_Finalize();
+ }
+}
diff --git a/library/cpp/pybind/embedding.h b/library/cpp/pybind/embedding.h
new file mode 100644
index 00000000000..18553d9f6c0
--- /dev/null
+++ b/library/cpp/pybind/embedding.h
@@ -0,0 +1,10 @@
+#pragma once
+
+namespace NPyBind {
+ class TEmbedding {
+ public:
+ TEmbedding(char* argv0);
+ ~TEmbedding();
+ };
+
+}
diff --git a/library/cpp/pybind/empty.cpp b/library/cpp/pybind/empty.cpp
new file mode 100644
index 00000000000..10da997ecce
--- /dev/null
+++ b/library/cpp/pybind/empty.cpp
@@ -0,0 +1,2 @@
+#include "init.h"
+#include "v2.h"
diff --git a/library/cpp/pybind/exceptions.cpp b/library/cpp/pybind/exceptions.cpp
new file mode 100644
index 00000000000..db1531fc632
--- /dev/null
+++ b/library/cpp/pybind/exceptions.cpp
@@ -0,0 +1,147 @@
+#include "exceptions.h"
+#include "cast.h"
+#include "module.h"
+#include <util/generic/algorithm.h>
+
+namespace NPyBind {
+
+ namespace NPrivate {
+ TPyObjectPtr CreatePyBindModule() {
+ return TPyObjectPtr(TExceptionsHolder::DoInitPyBindModule(), true);
+ }
+ }//NPrivate
+
+ TPyObjectPtr TExceptionsHolder::GetException(const TString& name) {
+ if (name == "")
+ return TPyObjectPtr(nullptr);
+ if (!Exceptions[name].Get())
+ ythrow yexception() << "Wrong base class '" << name << "'";
+ return Exceptions[name];
+ }
+
+ TPyObjectPtr TExceptionsHolder::GetExceptions(const TVector<TString>& names) {
+ TVector<TString> tmp(names.begin(), names.end());
+ TVector<TString>::iterator end = std::unique(tmp.begin(), tmp.end());
+ TPyObjectPtr tuple(PyTuple_New(std::distance(tmp.begin(), end)), true);
+ for (size_t i = 0; i < (size_t)std::distance(tmp.begin(), end); ++i) {
+ if (!Exceptions[tmp[i]].Get())
+ ythrow yexception() << "Wrong base class '" << tmp[i] << "'";
+ PyTuple_SetItem(tuple.Get(), i, Exceptions[tmp[i]].Get());
+ }
+ return tuple;
+ }
+
+ // def PyBindObjectReconstructor(cl, props):
+ // return cl(__properties__=props)
+ static PyObject* PyBindObjectReconstructor(PyObject*, PyObject* args) {
+ TPyObjectPtr callable, props;
+ if (!ExtractArgs(args, callable, props))
+ ythrow yexception() << "Wrong method arguments";
+#if PY_MAJOR_VERSION >= 3
+ TPyObjectPtr noArgs(PyTuple_New(0), true);
+#else
+ TPyObjectPtr noArgs(PyList_New(0), true);
+#endif
+ TPyObjectPtr kw(PyDict_New(), true);
+ PyDict_SetItemString(kw.Get(), "__properties__", props.Get());
+ TPyObjectPtr res(PyObject_Call(callable.Get(), noArgs.Get(), kw.Get()), true);
+ return res.RefGet();
+ }
+
+ static PyMethodDef PyBindMethods[] = {
+ {"PyBindObjectReconstructor", PyBindObjectReconstructor, METH_VARARGS, "Tech method. It's required for unpickling."},
+ {nullptr, nullptr, 0, nullptr}};
+
+#if PY_MAJOR_VERSION >= 3
+ static struct PyModuleDef moduledef = {
+ PyModuleDef_HEAD_INIT,
+ "pybind",
+ NULL,
+ -1,
+ PyBindMethods,
+ NULL, NULL, NULL, NULL
+ };
+
+ static PyObject* InitPyBind() {
+ return PyModule_Create(&moduledef);
+ }
+#else
+ static PyObject* InitPyBind() {
+ return Py_InitModule("pybind", PyBindMethods);
+ }
+#endif
+
+ void TExceptionsHolder::DoInitPyBindModule2() {
+ DoInitPyBindModule();
+ }
+
+ PyObject* TExceptionsHolder::DoInitPyBindModule() {
+ Instance().Module = NPyBind::TPyObjectPtr(InitPyBind(), true);
+ if (!Instance().Module.Get())
+ return nullptr;
+
+ for (TCheckersVector::const_iterator it = Instance().Checkers.begin(), end = Instance().Checkers.end(); it != end; ++it) {
+ TString name = (*it)->GetName();
+ if (!!name) {
+ //Ref to the object should be incremented before passing to AddObject
+ auto res = PyModule_AddObject(Instance().Module.Get(), name.data(), (*it)->GetException().RefGet());
+ if (res < 0) {
+ ythrow yexception() << "Failed to add object " << name << " to internal module pybind";
+ }
+ }
+ }
+ return Instance().Module.RefGet();
+ }
+
+ void TExceptionsHolder::Clear() {
+ //Unfortunately in Python3 we can't retrack this object because of PyError_NewException
+ //it's only the safe way to preserve GC gens in valid state during the finalization
+ for (auto& ptr: Checkers) {
+ if (!dynamic_cast<const TPyErrExceptionsChecker*>(ptr.Get())) { // no need to untrack standard PyExc_* exceptions from TPyErrExceptionsChecker
+ if (auto exceptionPtr = ptr->GetException()) {
+ PyObject_GC_UnTrack(exceptionPtr.Get());
+ }
+ }
+ }
+ Checkers.clear();
+ Exceptions.clear();
+ Module.Drop();
+ }
+
+ TExceptionsHolder::TExceptionsHolder() {
+ AddException<std::exception>("yexception");
+ AddException<TSystemError>("TSystemError", "yexception");
+ AddException<TIoException>("TIoException", "yexception");
+
+ TVector<TString> names(2);
+ names[0] = "TSystemError";
+ names[1] = "TIoException";
+
+ AddException<TIoSystemError>("TIoSystemError", names);
+ AddException<TFileError>("TFileError", "TIoSystemError");
+ AddException<TBadCastException>("TBadCastException", "yexception");
+
+ Checkers.push_back(new TPyErrExceptionsChecker);
+
+ // XXX: In Python 2.6, PyImport_AppendInittab() function takes non-const char*, this causes
+ // "ISO C++11 does not allow conversion from string literal to 'char *'" warning.
+ static char pybind[] = "pybind";
+#if PY_MAJOR_VERSION >= 3
+ PyImport_AppendInittab(pybind, DoInitPyBindModule);
+
+ NPrivate::AddFinalizationCallBack([this]() {
+ Clear();
+ });
+#else
+ PyImport_AppendInittab(pybind, DoInitPyBindModule2);
+#endif
+ }
+
+ NPyBind::TPyObjectPtr TExceptionsHolder::ToPyException(const std::exception& ex) {
+ for (TCheckersVector::const_reverse_iterator it = Checkers.rbegin(), end = Checkers.rend(); it != end; ++it) {
+ if ((*it)->Check(ex))
+ return (*it)->GetException();
+ }
+ return TPyObjectPtr(nullptr);
+ }
+}
diff --git a/library/cpp/pybind/exceptions.h b/library/cpp/pybind/exceptions.h
new file mode 100644
index 00000000000..48e20995e46
--- /dev/null
+++ b/library/cpp/pybind/exceptions.h
@@ -0,0 +1,143 @@
+#pragma once
+
+#define PY_SSIZE_T_CLEAN
+#include <Python.h>
+#include <util/generic/yexception.h>
+#include <util/generic/map.h>
+#include <util/generic/vector.h>
+#include "ptr.h"
+
+namespace NPyBind {
+ // Usage:
+ // ythrow TPyErr(PyExc_TypeError) << "some python type error somewhere in your C++ code";
+ //
+ class TPyErr: public virtual yexception {
+ public:
+ TPyErr(PyObject* theException = PyExc_RuntimeError)
+ : Exception(theException)
+ {
+ }
+
+ TPyObjectPtr GetException() const {
+ return Exception;
+ }
+
+ private:
+ NPyBind::TPyObjectPtr Exception;
+ };
+
+ //Private api for creating PyBind python module
+ //Needs only for overriding pybind python module in library which imports other pybind library
+ namespace NPrivate {
+ TPyObjectPtr CreatePyBindModule();
+ }//NPrivate
+ class TExceptionsHolder {
+ friend TPyObjectPtr NPrivate::CreatePyBindModule();
+ private:
+ TExceptionsHolder(const TExceptionsHolder&);
+ TExceptionsHolder& operator=(const TExceptionsHolder&);
+ TExceptionsHolder();
+
+ void Clear();
+ TPyObjectPtr GetException(const TString&);
+ TPyObjectPtr GetExceptions(const TVector<TString>&);
+ private:
+ class TExceptionsChecker {
+ public:
+ virtual ~TExceptionsChecker() {
+ }
+ virtual bool Check(const std::exception& ex) const = 0;
+ virtual TString GetName() const = 0;
+ virtual TPyObjectPtr GetException() const = 0;
+ };
+
+ template <typename TExcType>
+ class TConcreteExceptionsChecker: public TExceptionsChecker {
+ private:
+ TString Name;
+ TPyObjectPtr Exception;
+
+ public:
+ TConcreteExceptionsChecker(const TString& name, TPyObjectPtr exception)
+ : Name(name)
+ , Exception(exception)
+ {
+ }
+
+ bool Check(const std::exception& ex) const override {
+ const std::exception* e = &ex;
+ return dynamic_cast<const TExcType*>(e);
+ }
+
+ TString GetName() const override {
+ return Name;
+ }
+
+ TPyObjectPtr GetException() const override {
+ return Exception;
+ }
+ };
+
+ class TPyErrExceptionsChecker: public TExceptionsChecker {
+ private:
+ mutable TPyObjectPtr Exception;
+
+ public:
+ TPyErrExceptionsChecker() {
+ }
+
+ bool Check(const std::exception& ex) const override {
+ const TPyErr* err = dynamic_cast<const TPyErr*>(&ex);
+ if (err) {
+ Exception = err->GetException();
+ }
+ return err != nullptr;
+ }
+
+ TString GetName() const override {
+ return TString();
+ }
+
+ TPyObjectPtr GetException() const override {
+ return Exception;
+ }
+ };
+
+ typedef TSimpleSharedPtr<TExceptionsChecker> TCheckerPtr;
+ typedef TVector<TCheckerPtr> TCheckersVector;
+ typedef TMap<TString, TPyObjectPtr> TExceptionsMap;
+
+ TPyObjectPtr Module;
+ TCheckersVector Checkers;
+ TExceptionsMap Exceptions;
+
+ static PyObject* DoInitPyBindModule();
+ static void DoInitPyBindModule2();
+
+ public:
+ static TExceptionsHolder& Instance() {
+ static TExceptionsHolder Holder;
+ return Holder;
+ }
+
+ template <typename TExcType>
+ void AddException(const TString& name, const TString& base = "") {
+ TPyObjectPtr baseException(GetException(base));
+ TString fullName = TString("pybind.") + name;
+ TPyObjectPtr exception(PyErr_NewException(const_cast<char*>(fullName.c_str()), baseException.Get(), nullptr), true);
+ Checkers.push_back(new TConcreteExceptionsChecker<TExcType>(name, exception));
+ Exceptions[name] = exception;
+ }
+
+ template <typename TExcType>
+ void AddException(const TString& name, const TVector<TString>& bases) {
+ TPyObjectPtr baseExceptions(GetExceptions(bases));
+ TString fullName = TString("pybind.") + name;
+ TPyObjectPtr exception(PyErr_NewException(const_cast<char*>(fullName.c_str()), baseExceptions.Get(), nullptr), true);
+ Checkers.push_back(new TConcreteExceptionsChecker<TExcType>(name, exception));
+ Exceptions[name] = exception;
+ }
+
+ NPyBind::TPyObjectPtr ToPyException(const std::exception&);
+ };
+}
diff --git a/library/cpp/pybind/init.h b/library/cpp/pybind/init.h
new file mode 100644
index 00000000000..58874574ed6
--- /dev/null
+++ b/library/cpp/pybind/init.h
@@ -0,0 +1,25 @@
+#pragma once
+
+#define PY_SSIZE_T_CLEAN
+#include <Python.h>
+
+#include "ptr.h"
+
+namespace NPyBind {
+#if PY_MAJOR_VERSION >= 3
+
+#define PYBIND_MODINIT(name) PyMODINIT_FUNC PyInit_##name()
+
+ inline PyObject* ModInitReturn(TPyObjectPtr&& modptr) {
+ return modptr.Release();
+ }
+
+#else
+
+#define PYBIND_MODINIT(name) PyMODINIT_FUNC init##name()
+
+ inline void ModInitReturn(TPyObjectPtr&&) {
+ }
+
+#endif
+}
diff --git a/library/cpp/pybind/method.h b/library/cpp/pybind/method.h
new file mode 100644
index 00000000000..7c1f6e90e1b
--- /dev/null
+++ b/library/cpp/pybind/method.h
@@ -0,0 +1,439 @@
+#pragma once
+
+#define PY_SSIZE_T_CLEAN
+#include <Python.h>
+#include <util/generic/string.h>
+#include <util/generic/map.h>
+#include <util/generic/set.h>
+#include <util/generic/vector.h>
+#include <util/generic/ptr.h>
+#include <util/generic/typetraits.h>
+
+#include <util/generic/function.h>
+
+#include "cast.h"
+
+namespace NPyBind {
+ template <typename TObjType>
+ class TBaseMethodCaller {
+ public:
+ virtual ~TBaseMethodCaller() {
+ }
+ virtual bool CallMethod(PyObject* owner, TObjType* self, PyObject* args, PyObject* kwargs, PyObject*& res) const = 0;
+ virtual bool HasMethod(PyObject*, TObjType*, const TString&, const TSet<TString>&) {
+ return true;
+ }
+ };
+
+ template <typename TObjType>
+ class TIsACaller;
+
+ template <typename TObjType>
+ class TMethodCallers {
+ private:
+ typedef TSimpleSharedPtr<TBaseMethodCaller<TObjType>> TCallerPtr;
+ typedef TVector<TCallerPtr> TCallerList;
+ typedef TMap<TString, TCallerList> TCallerMap;
+
+ const TSet<TString>& HiddenAttrNames;
+ TCallerMap Callers;
+
+ public:
+ TMethodCallers(const TSet<TString>& hiddenNames)
+ : HiddenAttrNames(hiddenNames)
+ {
+ }
+
+ void AddCaller(const TString& name, TCallerPtr caller) {
+ Callers[name].push_back(caller);
+ }
+
+ bool HasCaller(const TString& name) const {
+ return Callers.has(name);
+ }
+
+ PyObject* CallMethod(PyObject* owner, TObjType* self, PyObject* args, PyObject* kwargs, const TString& name) const {
+ const TCallerList* lst = Callers.FindPtr(name);
+ if (!lst)
+ return nullptr;
+ for (const auto& caller : *lst) {
+ PyObject* res = nullptr;
+ PyErr_Clear();
+ if (caller->CallMethod(owner, self, args, kwargs, res))
+ return res;
+ }
+ return nullptr;
+ }
+
+ bool HasMethod(PyObject* owner, TObjType* self, const TString& name) const {
+ const TCallerList* lst = Callers.FindPtr(name);
+ if (!lst)
+ return false;
+ for (const auto& caller : *lst) {
+ if (caller->HasMethod(owner, self, name, HiddenAttrNames))
+ return true;
+ }
+ return false;
+ }
+
+ void GetMethodsNames(PyObject* owner, TObjType* self, TVector<TString>& resultNames) const {
+ for (const auto& it : Callers) {
+ if (HasMethod(owner, self, it.first) && !HiddenAttrNames.contains(it.first))
+ resultNames.push_back(it.first);
+ }
+ }
+
+ void GetAllMethodsNames(TVector<TString>& resultNames) const {
+ for (const auto& it : Callers) {
+ resultNames.push_back(it.first);
+ }
+ }
+
+ void GetPropertiesNames(PyObject*, TObjType* self, TVector<TString>& resultNames) const {
+ const TCallerList* lst = Callers.FindPtr("IsA");
+ if (!lst)
+ return;
+ for (const auto& caller : *lst) {
+ TIsACaller<TObjType>* isACaller = dynamic_cast<TIsACaller<TObjType>*>(caller.Get());
+ if (isACaller) {
+ resultNames = isACaller->GetPropertiesNames(self);
+ return;
+ }
+ }
+ }
+ };
+
+ template <typename TObjType>
+ class TIsACaller: public TBaseMethodCaller<TObjType> {
+ private:
+ class TIsAChecker {
+ public:
+ virtual ~TIsAChecker() {
+ }
+ virtual bool Check(const TObjType* obj) const = 0;
+ };
+
+ template <typename TConcrete>
+ class TIsAConcreteChecker: public TIsAChecker {
+ public:
+ bool Check(const TObjType* obj) const override {
+ return dynamic_cast<const TConcrete*>(obj) != nullptr;
+ }
+ };
+
+ typedef TSimpleSharedPtr<TIsAChecker> TCheckerPtr;
+ typedef TMap<TString, TCheckerPtr> TCheckersMap;
+
+ TCheckersMap Checkers;
+
+ bool Check(const TString& name, const TObjType* obj) const {
+ const TCheckerPtr* checker = Checkers.FindPtr(name);
+ if (!checker) {
+ PyErr_Format(PyExc_KeyError, "unknown class name: %s", name.data());
+ return false;
+ }
+ return (*checker)->Check(obj);
+ }
+
+ protected:
+ TIsACaller() {
+ }
+
+ template <typename TConcrete>
+ void AddChecker(const TString& name) {
+ Checkers[name] = new TIsAConcreteChecker<TConcrete>;
+ }
+
+ public:
+ bool CallMethod(PyObject*, TObjType* self, PyObject* args, PyObject*, PyObject*& res) const override {
+ if (args == nullptr || !PyTuple_Check(args))
+ return false;
+ size_t cnt = PyTuple_Size(args);
+ bool result = true;
+ for (size_t i = 0; i < cnt; ++i) {
+ result = result && Check(
+#if PY_MAJOR_VERSION >= 3
+ PyUnicode_AsUTF8(
+#else
+ PyString_AsString(
+#endif
+ PyTuple_GetItem(args, i)), self);
+ }
+ if (PyErr_Occurred()) {
+ return false;
+ }
+ res = BuildPyObject(result);
+ return true;
+ }
+
+ TVector<TString> GetPropertiesNames(const TObjType* obj) const {
+ TVector<TString> names;
+
+ for (const auto& it : Checkers) {
+ if (it.second->Check(obj)) {
+ names.push_back(it.first);
+ }
+ }
+
+ return names;
+ }
+ };
+
+ template <typename TObjType>
+ class TGenericMethodCaller: public TBaseMethodCaller<TObjType> {
+ private:
+ TString AttrName;
+
+ public:
+ TGenericMethodCaller(const TString& attrName)
+ : AttrName(attrName)
+ {
+ }
+
+ bool CallMethod(PyObject* obj, TObjType*, PyObject* args, PyObject*, PyObject*& res) const override {
+ auto str = NameFromString(AttrName);
+ PyObject* attr = PyObject_GenericGetAttr(obj, str.Get());
+ if (!attr)
+ ythrow yexception() << "Can't get generic attribute '" << AttrName << "'";
+ res = PyObject_CallObject(attr, args);
+ return res != nullptr;
+ }
+ };
+
+
+ template <typename TObjType, typename TSubObject>
+ class TSubObjectChecker: public TBaseMethodCaller<TObjType> {
+ public:
+ ~TSubObjectChecker() override {
+ }
+
+ bool HasMethod(PyObject*, TObjType* self, const TString&, const TSet<TString>&) override {
+ return dynamic_cast<const TSubObject*>(self) != nullptr;
+ }
+ };
+
+ template <typename TFunctor, typename Tuple, typename ResType, typename=std::enable_if_t<!std::is_same_v<ResType, void>>>
+ void ApplyFunctor(TFunctor functor, Tuple resultArgs, PyObject*& res) {
+ res = BuildPyObject(std::move(Apply(functor, resultArgs)));
+ }
+
+ template <typename TFunctor, typename Tuple, typename ResType, typename=std::enable_if_t<std::is_same_v<ResType, void>>, typename=void>
+ void ApplyFunctor(TFunctor functor, Tuple resultArgs, PyObject*& res) {
+ Py_INCREF(Py_None);
+ res = Py_None;
+ Apply(functor, resultArgs);
+ }
+
+ template <typename TObjType, typename TResType, typename... Args>
+ class TFunctorCaller: public TBaseMethodCaller<TObjType> {
+ using TFunctor = std::function<TResType(TObjType&,Args...)>;
+ TFunctor Functor;
+ public:
+ explicit TFunctorCaller(TFunctor functor):
+ Functor(functor){}
+
+ bool CallMethod(PyObject*, TObjType* self, PyObject* args, PyObject*, PyObject*& res) const {
+ auto methodArgsTuple = GetArguments<Args...>(args);
+ auto resultArgs = std::tuple_cat(std::tie(*self), methodArgsTuple);
+ ApplyFunctor<TFunctor, decltype(resultArgs), TResType>(Functor, resultArgs, res);
+ return true;
+ }
+ };
+
+ template <typename TObjType, typename TRealType>
+ class TGetStateCaller: public TSubObjectChecker<TObjType, TRealType> {
+ protected:
+ TPyObjectPtr AddFromCaller(PyObject* obj, const TString& methodName) const {
+ PyObject* res = PyObject_CallMethod(obj, const_cast<char*>(methodName.c_str()), const_cast<char*>(""));
+ if (!res) {
+ PyErr_Clear();
+ return TPyObjectPtr(Py_None);
+ }
+ return TPyObjectPtr(res, true);
+ }
+
+ void GetStandartAttrsDictionary(PyObject* obj, TRealType*, TMap<TString, TPyObjectPtr>& dict) const {
+ TPyObjectPtr attrsDict(PyObject_GetAttrString(obj, "__dict__"), true);
+ TMap<TString, TPyObjectPtr> attrs;
+ if (!FromPyObject(attrsDict.Get(), attrs))
+ ythrow yexception() << "Can't get '__dict__' attribute";
+ dict.insert(attrs.begin(), attrs.end());
+ }
+
+ virtual void GetAttrsDictionary(PyObject* obj, TRealType* self, TMap<TString, TPyObjectPtr>& dict) const = 0;
+
+ public:
+ bool CallMethod(PyObject* obj, TObjType* self, PyObject* args, PyObject*, PyObject*& res) const override {
+ if (!ExtractArgs(args))
+ ythrow yexception() << "Can't parse arguments: it should be none";
+ TRealType* rself = dynamic_cast<TRealType*>(self);
+ if (!rself)
+ return false;
+ TMap<TString, TPyObjectPtr> dict;
+ GetAttrsDictionary(obj, rself, dict);
+ res = BuildPyObject(dict);
+ return true;
+ }
+ };
+
+ template <typename TObjType, typename TRealType>
+ class TSetStateCaller: public TSubObjectChecker<TObjType, TRealType> {
+ protected:
+ void SetStandartAttrsDictionary(PyObject* obj, TRealType*, TMap<TString, TPyObjectPtr>& dict) const {
+ TPyObjectPtr value(BuildPyObject(dict), true);
+ PyObject_SetAttrString(obj, "__dict__", value.Get());
+ }
+
+ virtual void SetAttrsDictionary(PyObject* obj, TRealType* self, TMap<TString, TPyObjectPtr>& dict) const = 0;
+
+ public:
+ bool CallMethod(PyObject* obj, TObjType* self, PyObject* args, PyObject*, PyObject*& res) const override {
+ TMap<TString, TPyObjectPtr> dict;
+ if (!ExtractArgs(args, dict))
+ ythrow yexception() << "Can't parse arguments: it should be one dictionary";
+ TRealType* rself = dynamic_cast<TRealType*>(self);
+ if (!rself)
+ return false;
+ SetAttrsDictionary(obj, rself, dict);
+ Py_INCREF(Py_None);
+ res = Py_None;
+ return true;
+ }
+ };
+
+ template <typename TObjType, typename TResult, typename TSubObject, typename TMethod, typename... Args>
+ class TAnyParameterMethodCaller: public TSubObjectChecker<TObjType, TSubObject> {
+ private:
+ TMethod Method;
+
+ public:
+ TAnyParameterMethodCaller(TMethod method)
+ : Method(method)
+ {
+ }
+
+ public:
+ bool CallMethod(PyObject*, TObjType* self, PyObject* args, PyObject*, PyObject*& res) const override {
+ TSubObject* sub = dynamic_cast<TSubObject*>(self);
+ if (sub == nullptr)
+ return false;
+ if (args && (!PyTuple_Check(args) || PyTuple_Size(args) != TFunctionArgs<TMethod>::Length)) {
+ //ythrow yexception() << "Method takes " << (size_t)(TFunctionArgs<TMethod>::Length) << " arguments, " << PyTuple_Size(args) << " provided";
+ return false;
+ }
+
+ try {
+ class Applicant {
+ public:
+ TResult operator()(Args... theArgs) {
+ return (Sub->*Method)(theArgs...);
+ }
+ TSubObject* Sub;
+ TMethod Method;
+ };
+ res = BuildPyObject(std::move(Apply(Applicant{sub, Method}, GetArguments<Args...>(args))));
+ } catch (cast_exception) {
+ return false;
+ } catch (...) {
+ if (PyExc_StopIteration == PyErr_Occurred()) {
+ // NB: it's replacement for geo_boost::python::throw_error_already_set();
+ return true;
+ }
+ PyErr_SetString(PyExc_RuntimeError, CurrentExceptionMessage().data());
+ return true;
+ }
+
+ return true;
+ }
+ };
+
+ template <typename TObjType, typename TSubObject, typename TMethod, typename... Args>
+ class TAnyParameterMethodCaller<TObjType, void, TSubObject, TMethod, Args...>: public TSubObjectChecker<TObjType, TSubObject> {
+ private:
+ TMethod Method;
+
+ public:
+ TAnyParameterMethodCaller(TMethod method)
+ : Method(method)
+ {
+ }
+
+ public:
+ bool CallMethod(PyObject*, TObjType* self, PyObject* args, PyObject*, PyObject*& res) const override {
+ TSubObject* sub = dynamic_cast<TSubObject*>(self);
+ if (sub == nullptr) {
+ return false;
+ }
+ if (args && (!PyTuple_Check(args) || PyTuple_Size(args) != TFunctionArgs<TMethod>::Length)) {
+ return false;
+ }
+
+ try {
+ class Applicant {
+ public:
+ void operator()(Args... theArgs) {
+ (Sub->*Method)(theArgs...);
+ }
+ TSubObject* Sub;
+ TMethod Method;
+ };
+
+ Apply(Applicant{sub, Method}, GetArguments<Args...>(args));
+
+ Py_INCREF(Py_None);
+ res = Py_None;
+ } catch (cast_exception) {
+ return false;
+ } catch (...) {
+ PyErr_SetString(PyExc_RuntimeError, CurrentExceptionMessage().data());
+ return true;
+ }
+
+ return true;
+ }
+ };
+
+ template <typename TResult, typename TSubObject, typename... Args>
+ struct TConstTraits {
+ typedef TResult (TSubObject::*TMethod)(Args... args) const;
+ };
+
+ template <typename TResult, typename TSubObject, typename... Args>
+ struct TNonConstTraits {
+ typedef TResult (TSubObject::*TMethod)(Args... args);
+ };
+
+ template <typename TObjType, typename TResult, typename TSubObject, typename TMethod, typename... Args>
+ class TConstMethodCaller: public TAnyParameterMethodCaller<TObjType, TResult, const TSubObject, typename TConstTraits<TResult, TSubObject, Args...>::TMethod, Args...> {
+ public:
+ TConstMethodCaller(typename TConstTraits<TResult, TSubObject, Args...>::TMethod method)
+ : TAnyParameterMethodCaller<TObjType, TResult, const TSubObject, typename TConstTraits<TResult, TSubObject, Args...>::TMethod, Args...>(method)
+ {
+ }
+ };
+
+ template <typename TObjType, typename TResult, typename TSubObject, typename... Args>
+ TSimpleSharedPtr<TBaseMethodCaller<TObjType>> CreateConstMethodCaller(TResult (TSubObject::*method)(Args...) const) {
+ return new TConstMethodCaller<TObjType, TResult, TSubObject, TResult (TSubObject::*)(Args...) const, Args...>(method);
+ }
+
+ template <typename TObjType, typename TResType, typename... Args>
+ TSimpleSharedPtr<TBaseMethodCaller<TObjType>> CreateFunctorCaller(std::function<TResType(TObjType&, Args...)> functor) {
+ return new TFunctorCaller<TObjType, TResType, Args...>(functor);
+ }
+
+ template <typename TObjType, typename TResult, typename TSubObject, typename TMethod, typename... Args>
+ class TMethodCaller: public TAnyParameterMethodCaller<TObjType, TResult, TSubObject, typename TNonConstTraits<TResult, TSubObject, Args...>::TMethod, Args...> {
+ public:
+ TMethodCaller(typename TNonConstTraits<TResult, TSubObject, Args...>::TMethod method)
+ : TAnyParameterMethodCaller<TObjType, TResult, TSubObject, typename TNonConstTraits<TResult, TSubObject, Args...>::TMethod, Args...>(method)
+ {
+ }
+ };
+
+ template <typename TObjType, typename TResult, typename TSubObject, typename... Args>
+ TSimpleSharedPtr<TBaseMethodCaller<TObjType>> CreateMethodCaller(TResult (TSubObject::*method)(Args...)) {
+ return new TMethodCaller<TObjType, TResult, TSubObject, TResult (TSubObject::*)(Args...), Args...>(method);
+ }
+
+}
diff --git a/library/cpp/pybind/module.cpp b/library/cpp/pybind/module.cpp
new file mode 100644
index 00000000000..b7b003d3b08
--- /dev/null
+++ b/library/cpp/pybind/module.cpp
@@ -0,0 +1,72 @@
+#include "module.h"
+#include "ptr.h"
+
+#include <util/generic/adaptor.h>
+
+namespace NPyBind {
+
+#if PY_MAJOR_VERSION >= 3
+ namespace NPrivate {
+ struct TFinCallBacksHolder {
+ static TVector<TFinalizationCallBack>& GetCallBacks() {
+ static TVector<TFinalizationCallBack> res;
+ return res;
+ }
+ };
+
+ TAtExitRegistrar::TAtExitRegistrar(TPyObjectPtr module) {
+ TPyObjectPtr atExitModuleName(Py_BuildValue("s", "atexit"), true);
+ TPyObjectPtr atExitModule(PyImport_Import(atExitModuleName.Get()));
+ Y_VERIFY(atExitModule);
+ TPyObjectPtr finalizerFunc(PyObject_GetAttrString(module.Get(), "finalizer"), true);
+ Y_VERIFY(finalizerFunc);
+ TPyObjectPtr registerName(Py_BuildValue("s", "register"), true);
+ PyObject_CallMethodObjArgs(atExitModule.Get(), registerName.Get(), finalizerFunc.Get(), nullptr);
+ }
+
+ TPyBindModuleRegistrar::TPyBindModuleRegistrar() {
+ TPyObjectPtr modules(PySys_GetObject("modules"));
+ Y_ENSURE(modules.Get());
+ if (Module = NPrivate::CreatePyBindModule()) {
+ Y_VERIFY(0 == PyDict_SetItemString(modules.Get(), "pybind", Module.RefGet()));
+ }
+ AddFinalizationCallBack([this]() {
+ auto ptr = Module;
+ Y_UNUSED(ptr);
+ TPyObjectPtr modules(PySys_GetObject("modules"));
+ Y_ENSURE(modules.Get());
+ TPyObjectPtr pyBindName(Py_BuildValue("s", "pybind"));
+ if (PyDict_Contains(modules.Get(), pyBindName.Get()) == 1) {
+ Y_VERIFY(0==PyDict_DelItemString(modules.Get(), "pybind"));
+ }
+ if (Module) {
+ //We have to untrack the module because some refs from him refers to gc-leaked errors
+ //see exceptions.cpp fore more info
+ PyObject_GC_UnTrack(Module.Get());
+ Module.Drop();
+ }
+ });
+ }
+
+ void AddFinalizationCallBack(TFinalizationCallBack callback) {
+ TFinCallBacksHolder::GetCallBacks().push_back(callback);
+ }
+
+ int FinalizeAll() {
+ for (auto callback: Reversed(NPrivate::TFinCallBacksHolder::GetCallBacks())) {
+ callback();
+ }
+ return 0;
+ }
+ }
+#endif
+
+
+ TModuleHolder::TModuleHolder()
+ : Methods(1, new TVector<TMethodDef>)
+ {
+#if PY_MAJOR_VERSION >= 3
+ AddModuleMethod<TModuleMethodCaller<decltype(&NPrivate::FinalizeAll), &NPrivate::FinalizeAll>::Call>("finalizer");
+#endif
+ }
+}//NPyBind
diff --git a/library/cpp/pybind/module.h b/library/cpp/pybind/module.h
new file mode 100644
index 00000000000..41dcb4dfec8
--- /dev/null
+++ b/library/cpp/pybind/module.h
@@ -0,0 +1,176 @@
+#pragma once
+
+#define PY_SSIZE_T_CLEAN
+#include <Python.h>
+#include "ptr.h"
+#include "cast.h"
+#include "exceptions.h"
+
+#include <util/generic/function.h>
+
+namespace NPyBind {
+#if PY_MAJOR_VERSION >= 3
+ namespace NPrivate {
+ using TFinalizationCallBack = std::function<void()>;
+ void AddFinalizationCallBack(TFinalizationCallBack);
+ class TAtExitRegistrar: private TNonCopyable {
+ TAtExitRegistrar(TPyObjectPtr module);
+ public:
+ static void Instantiate(TPyObjectPtr module) {
+ static TAtExitRegistrar registrar(module);
+ Y_UNUSED(registrar);
+ }
+ };
+
+ class TPyBindModuleRegistrar: private TNonCopyable {
+ TPyBindModuleRegistrar();
+ TPyObjectPtr Module;
+ public:
+ static void Instantiate() {
+ static TPyBindModuleRegistrar registrar;
+ Y_UNUSED(registrar);
+ }
+ };
+ } //NPrivate
+#endif
+
+ class TModuleHolder {
+ private:
+ TModuleHolder(const TModuleHolder&);
+ TModuleHolder& operator=(const TModuleHolder&);
+
+ TModuleHolder();
+ private:
+ typedef PyCFunction TModuleMethod;
+#if PY_MAJOR_VERSION >= 3
+ typedef PyObject* (*TModuleInitFunc)();
+#else
+ typedef void (*TModuleInitFunc)();
+#endif
+
+ struct TMethodDef {
+ TString Name;
+ TModuleMethod Method;
+ TString Description;
+ int Flags;
+
+ TMethodDef(const TString& name, TModuleMethod method, const TString& descr, int flags)
+ : Name(name)
+ , Method(method)
+ , Description(descr)
+ , Flags(flags)
+ {
+ }
+
+ operator PyMethodDef() const {
+ PyMethodDef cur = {Name.c_str(), Method, Flags, Description.c_str()};
+ return cur;
+ }
+ };
+
+ typedef TSimpleSharedPtr<TVector<TMethodDef>> TMethodDefVecPtr;
+ typedef TSimpleSharedPtr<TVector<PyMethodDef>> TPyMethodDefVecPtr;
+
+ TVector<TMethodDefVecPtr> Methods;
+ TVector<TPyMethodDefVecPtr> Defs;
+#if PY_MAJOR_VERSION >= 3
+ //because the md_name will leak otherwise
+ class TPyModuleDefWithName {
+ PyModuleDef Def;
+ TString Name;
+ public:
+ explicit TPyModuleDefWithName(TString name, TPyMethodDefVecPtr moduleDefs)
+ : Name(std::move(name))
+ {
+ Def = PyModuleDef{
+ PyModuleDef_HEAD_INIT,
+ Name.c_str(),
+ nullptr,
+ -1,
+ moduleDefs->data(),
+ nullptr, nullptr, nullptr, nullptr
+ };
+ }
+ PyModuleDef* GetDefPtr() {
+ return &Def;
+ }
+
+ };
+ TVector<TSimpleSharedPtr<TPyModuleDefWithName>> ModuleDefs;
+#endif
+
+ template <TModuleMethod method>
+ static PyObject* MethodWrapper(PyObject* obj, PyObject* args) {
+ try {
+ PyObject* res = method(obj, args);
+ if (!res && !PyErr_Occurred())
+ ythrow yexception() << "\nModule method exited with NULL, but didn't set Error.\n Options:\n -- Return correct value or None;\n -- Set python exception;\n -- Throw c++ exception.";
+ return res;
+ } catch (const std::exception& ex) {
+ PyErr_SetString(TExceptionsHolder::Instance().ToPyException(ex).Get(), ex.what());
+ } catch (...) {
+ PyErr_SetString(PyExc_RuntimeError, "Unknown error occurred while trying to call module method");
+ }
+ return nullptr;
+ }
+
+ public:
+ static TModuleHolder& Instance() {
+ static TModuleHolder Holder;
+ return Holder;
+ }
+
+ void ImportModule(TPyObjectPtr module, const char* const name, TModuleInitFunc initFunc) {
+ PyImport_AppendInittab(const_cast<char*>(name), initFunc);
+ TPyObjectPtr importedModule(PyImport_ImportModule(name), true);
+ PyModule_AddObject(module.Get(), name, importedModule.Get());
+ }
+
+ template <TModuleMethod method>
+ void AddModuleMethod(const TString& name, const TString& descr = "") {
+ Methods.back()->push_back(TMethodDef(name, MethodWrapper<method>, descr, METH_VARARGS));
+ }
+
+ TPyObjectPtr InitModule(const TString& name) {
+ Defs.push_back(new TVector<PyMethodDef>(Methods.back()->begin(), Methods.back()->end()));
+ PyMethodDef blank = {nullptr, nullptr, 0, nullptr};
+ Defs.back()->push_back(blank);
+#if PY_MAJOR_VERSION >= 3
+ ModuleDefs.push_back(MakeSimpleShared<TPyModuleDefWithName>(name, Defs.back()));
+ TPyObjectPtr res(PyModule_Create(ModuleDefs.back()->GetDefPtr()));
+ NPrivate::TAtExitRegistrar::Instantiate(res);
+ NPrivate::TPyBindModuleRegistrar::Instantiate();
+#else
+ TPyObjectPtr res(Py_InitModule(name.c_str(), &(Defs.back()->at(0))));
+#endif
+ Methods.push_back(new TVector<TMethodDef>);
+ return res;
+ }
+ };
+
+ template <typename TMethodSignature, TMethodSignature method>
+ class TModuleMethodCaller {
+ private:
+ template <typename TResult, typename... Args>
+ struct TCaller {
+ static PyObject* Call(PyObject* args) {
+ return BuildPyObject(Apply(method, GetArguments<Args...>(args)));
+ }
+ };
+
+ template <typename TResult, typename... Args>
+ static PyObject* InternalCall(TResult (*)(Args...), PyObject* args) {
+ return BuildPyObject(Apply(method, GetArguments<Args...>(args)));
+ }
+
+ public:
+ static PyObject* Call(PyObject*, PyObject* args) {
+ if (args && (!PyTuple_Check(args) || PyTuple_Size(args) != TFunctionArgs<TMethodSignature>::Length)) {
+ ythrow yexception() << "Method takes " << (size_t)(TFunctionArgs<TMethodSignature>::Length) << " arguments, " << PyTuple_Size(args) << " provided";
+ }
+
+ return InternalCall(method, args);
+ }
+ };
+
+}
diff --git a/library/cpp/pybind/pod.cpp b/library/cpp/pybind/pod.cpp
new file mode 100644
index 00000000000..3cf030e537b
--- /dev/null
+++ b/library/cpp/pybind/pod.cpp
@@ -0,0 +1,18 @@
+#include "pod.h"
+
+namespace NPyBind {
+ class TPODAttrGetter: public TBaseAttrGetter<TPOD> {
+ public:
+ bool GetAttr(PyObject*, const TPOD& self, const TString& attr, PyObject*& res) const override {
+ res = self.GetAttr(attr.c_str());
+ return res != nullptr;
+ }
+ };
+
+ TPODTraits::TPODTraits()
+ : MyParent("TPOD", "simple struct")
+ {
+ AddGetter("", new TPODAttrGetter);
+ }
+
+}
diff --git a/library/cpp/pybind/pod.h b/library/cpp/pybind/pod.h
new file mode 100644
index 00000000000..90165fdbec6
--- /dev/null
+++ b/library/cpp/pybind/pod.h
@@ -0,0 +1,53 @@
+#pragma once
+
+#define PY_SSIZE_T_CLEAN
+#include <Python.h>
+#include "attr.h"
+#include "typedesc.h"
+
+namespace NPyBind {
+ struct TPOD {
+ TPyObjectPtr Dict;
+
+ TPOD()
+ : Dict(PyDict_New(), true)
+ {
+ }
+ bool SetAttr(const char* name, PyObject* value) {
+ return PyDict_SetItemString(Dict.Get(), name, value) == 0;
+ }
+ PyObject* GetAttr(const char* name) const {
+ PyObject* res = PyDict_GetItemString(Dict.Get(), name);
+ Py_XINCREF(res);
+ return res;
+ }
+ };
+
+ class TPODTraits: public NPyBind::TPythonType<TPOD, TPOD, TPODTraits> {
+ private:
+ typedef TPythonType<TPOD, TPOD, TPODTraits> MyParent;
+ friend class TPythonType<TPOD, TPOD, TPODTraits>;
+ TPODTraits();
+
+ public:
+ static TPOD* GetObject(TPOD& obj) {
+ return &obj;
+ }
+ };
+
+ template <>
+ inline bool FromPyObject<TPOD*>(PyObject* obj, TPOD*& res) {
+ res = TPODTraits::CastToObject(obj);
+ if (res == nullptr)
+ return false;
+ return true;
+ }
+ template <>
+ inline bool FromPyObject<const TPOD*>(PyObject* obj, const TPOD*& res) {
+ res = TPODTraits::CastToObject(obj);
+ if (res == nullptr)
+ return false;
+ return true;
+ }
+
+}
diff --git a/library/cpp/pybind/ptr.h b/library/cpp/pybind/ptr.h
new file mode 100644
index 00000000000..e1367366904
--- /dev/null
+++ b/library/cpp/pybind/ptr.h
@@ -0,0 +1,51 @@
+#pragma once
+
+#define PY_SSIZE_T_CLEAN
+#include <Python.h>
+#include <util/generic/ptr.h>
+
+namespace NPyBind {
+ template <class T>
+ class TPythonIntrusivePtrOps {
+ public:
+ static inline void Ref(T* t) noexcept {
+ Py_XINCREF(t);
+ }
+
+ static inline void UnRef(T* t) noexcept {
+ Py_XDECREF(t);
+ }
+
+ static inline void DecRef(T* t) noexcept {
+ Py_XDECREF(t);
+ }
+ };
+
+ class TPyObjectPtr: public TIntrusivePtr<PyObject, TPythonIntrusivePtrOps<PyObject>> {
+ private:
+ typedef TIntrusivePtr<PyObject, TPythonIntrusivePtrOps<PyObject>> TParent;
+ typedef TPythonIntrusivePtrOps<PyObject> TOps;
+
+ public:
+ inline TPyObjectPtr() noexcept {
+ }
+
+ inline explicit TPyObjectPtr(PyObject* obj) noexcept
+ : TParent(obj)
+ {
+ }
+
+ inline TPyObjectPtr(PyObject* obj, bool unref) noexcept
+ : TParent(obj)
+ {
+ if (unref)
+ TOps::UnRef(TParent::Get());
+ }
+
+ inline PyObject* RefGet() {
+ TOps::Ref(TParent::Get());
+ return TParent::Get();
+ }
+ };
+
+}
diff --git a/library/cpp/pybind/typeattrs.h b/library/cpp/pybind/typeattrs.h
new file mode 100644
index 00000000000..a906b9ec2b0
--- /dev/null
+++ b/library/cpp/pybind/typeattrs.h
@@ -0,0 +1,368 @@
+#pragma once
+
+#include "ptr.h"
+#include "cast.h"
+#include "attr.h"
+#include "method.h"
+
+#include <util/generic/vector.h>
+
+namespace NPyBind {
+ template <typename TObject>
+ class TPythonTypeAttributes {
+ private:
+ TAttrGetters<TObject> AttrGetters;
+ TAttrSetters<TObject> AttrSetters;
+ TMethodCallers<TObject> MethodCallers;
+
+ class TGetAttrsNamesCaller;
+ class TGetMethodsNamesCaller;
+ class TGetAllNamesCaller;
+ class TGetPropertiesNamesCaller;
+ class TDictAttrGetter;
+ class TDictAttrSetter;
+ class TGetAttributeMethodCaller;
+ class TSetAttrMethodCaller;
+ class TGetStrReprMethodCaller;
+ class TReduceMethodCaller;
+ class TBaseGetStateMethodCaller;
+ class TBaseSetStateMethodCaller;
+
+ TPythonTypeAttributes(const TPythonTypeAttributes&);
+ TPythonTypeAttributes& operator=(const TPythonTypeAttributes&);
+
+ static const TSet<TString> HiddenAttrNames;
+
+ typedef PyObject* (*GetAttrFunction)(PyObject*, char*);
+ typedef int (*SetAttrFunction)(PyObject*, char*, PyObject*);
+ GetAttrFunction GetAttr;
+ SetAttrFunction SetAttr;
+
+ public:
+ typedef TSimpleSharedPtr<TBaseAttrGetter<TObject>> TGetterPtr;
+ typedef TSimpleSharedPtr<TBaseAttrSetter<TObject>> TSetterPtr;
+ typedef TSimpleSharedPtr<TBaseMethodCaller<TObject>> TCallerPtr;
+
+ TPythonTypeAttributes(GetAttrFunction getAttr, SetAttrFunction setAttr)
+ : AttrGetters(HiddenAttrNames)
+ , MethodCallers(HiddenAttrNames)
+ , GetAttr(getAttr)
+ , SetAttr(setAttr)
+ {
+ }
+
+ void InitCommonAttributes() {
+ // attributes
+ AddGetter("__dict__", new TDictAttrGetter(AttrGetters));
+ AddSetter("__dict__", new TDictAttrSetter(AttrSetters));
+
+ // methods
+ AddCaller("GetAttrsNames", new TGetAttrsNamesCaller(AttrGetters));
+ AddCaller("GetMethodsNames", new TGetMethodsNamesCaller(MethodCallers));
+ AddCaller("GetAllNames", new TGetAllNamesCaller(AttrGetters, MethodCallers));
+ AddCaller("GetPropertiesNames", new TGetPropertiesNamesCaller(MethodCallers));
+ AddCaller("__getattribute__", new TGetAttributeMethodCaller(GetAttr));
+ AddCaller("__setattr__", new TSetAttrMethodCaller(SetAttr));
+ AddCaller("__str__", new TGetStrReprMethodCaller("__str__"));
+ AddCaller("__repr__", new TGetStrReprMethodCaller("__repr__"));
+ AddCaller("__reduce_ex__", new TReduceMethodCaller);
+ AddCaller("__reduce__", new TReduceMethodCaller);
+ AddCaller("__getstate__", new TBaseGetStateMethodCaller);
+ AddCaller("__setstate__", new TBaseSetStateMethodCaller);
+
+ // generics
+ AddGetter("__class__", new TGenericAttrGetter<TObject>("__class__"));
+ AddGetter("__doc__", new TGenericAttrGetter<TObject>("__doc__"));
+ AddCaller("__sizeof__", new TGenericMethodCaller<TObject>("__sizeof__"));
+ AddCaller("__hash__", new TGenericMethodCaller<TObject>("__hash__"));
+ }
+
+ void AddGetter(const TString& attr, TGetterPtr getter) {
+ AttrGetters.AddGetter(attr, getter);
+ }
+
+ void AddSetter(const TString& attr, TSetterPtr setter) {
+ AttrSetters.AddSetter(attr, setter);
+ }
+
+ void AddCaller(const TString& name, TCallerPtr caller) {
+ MethodCallers.AddCaller(name, caller);
+ }
+
+ const TAttrGetters<TObject>& GetAttrGetters() const {
+ return AttrGetters;
+ }
+
+ TAttrSetters<TObject>& GetAttrSetters() {
+ return AttrSetters;
+ }
+
+ const TMethodCallers<TObject>& GetMethodCallers() const {
+ return MethodCallers;
+ }
+
+ const TSet<TString>& GetHiddenAttrs() const {
+ return HiddenAttrNames;
+ }
+ };
+
+ template <typename TObjType>
+ class TPythonTypeAttributes<TObjType>::TGetAttrsNamesCaller: public TBaseMethodCaller<TObjType> {
+ private:
+ const TAttrGetters<TObjType>& AttrGetters;
+
+ public:
+ TGetAttrsNamesCaller(const TAttrGetters<TObjType>& getters)
+ : AttrGetters(getters)
+ {
+ }
+
+ bool CallMethod(PyObject* owner, TObjType* self, PyObject* args, PyObject*, PyObject*& res) const override {
+ if (!ExtractArgs(args))
+ ythrow yexception() << "Could not parse args for GetAttrsNames() - it should be none";
+ TVector<TString> names;
+ AttrGetters.GetAttrsNames(owner, *self, names);
+ res = BuildPyObject(names);
+ return (res != nullptr);
+ }
+ };
+
+ template <typename TObjType>
+ class TPythonTypeAttributes<TObjType>::TGetMethodsNamesCaller: public TBaseMethodCaller<TObjType> {
+ private:
+ const TMethodCallers<TObjType>& MethodCallers;
+
+ public:
+ TGetMethodsNamesCaller(const TMethodCallers<TObjType>& callers)
+ : MethodCallers(callers)
+ {
+ }
+
+ bool CallMethod(PyObject* owner, TObjType* self, PyObject* args, PyObject*, PyObject*& res) const override {
+ if (!ExtractArgs(args))
+ ythrow yexception() << "Could not parse args for GetMethodsNames() - it should be none";
+ TVector<TString> names;
+ MethodCallers.GetMethodsNames(owner, self, names);
+ res = BuildPyObject(names);
+ return (res != nullptr);
+ }
+ };
+
+ template <typename TObjType>
+ class TPythonTypeAttributes<TObjType>::TGetAllNamesCaller: public TBaseMethodCaller<TObjType> {
+ private:
+ const TAttrGetters<TObjType>& AttrGetters;
+ const TMethodCallers<TObjType>& MethodCallers;
+
+ public:
+ TGetAllNamesCaller(const TAttrGetters<TObjType>& getters,
+ const TMethodCallers<TObjType>& callers)
+ : AttrGetters(getters)
+ , MethodCallers(callers)
+ {
+ }
+
+ bool CallMethod(PyObject* owner, TObjType* self, PyObject* args, PyObject*, PyObject*& res) const override {
+ if (!ExtractArgs(args))
+ ythrow yexception() << "Could not parse args for GetAllNames() - it should be none";
+ TVector<TString> names;
+ AttrGetters.GetAttrsNames(owner, *self, names);
+ MethodCallers.GetMethodsNames(owner, self, names);
+ res = BuildPyObject(names);
+ return (res != nullptr);
+ }
+ };
+
+ template <typename TObjType>
+ class TPythonTypeAttributes<TObjType>::TGetPropertiesNamesCaller: public TBaseMethodCaller<TObjType> {
+ private:
+ const TMethodCallers<TObjType>& MethodCallers;
+
+ public:
+ TGetPropertiesNamesCaller(const TMethodCallers<TObjType>& callers)
+ : MethodCallers(callers)
+ {
+ }
+
+ public:
+ bool CallMethod(PyObject* obj, TObjType* self, PyObject* args, PyObject*, PyObject*& res) const override {
+ if (!ExtractArgs(args))
+ return false;
+
+ TVector<TString> names;
+ MethodCallers.GetPropertiesNames(obj, self, names);
+ res = BuildPyObject(names);
+ return (res != nullptr);
+ }
+ };
+
+ template <typename TObjType>
+ class TPythonTypeAttributes<TObjType>::TDictAttrGetter: public TBaseAttrGetter<TObjType> {
+ private:
+ TAttrGetters<TObjType>& AttrGetters;
+
+ public:
+ TDictAttrGetter(TAttrGetters<TObjType>& getters)
+ : AttrGetters(getters)
+ {
+ }
+
+ bool GetAttr(PyObject* owner, const TObjType& self, const TString&, PyObject*& res) const override {
+ TMap<TString, PyObject*> dict;
+ AttrGetters.GetAttrsDictionary(owner, self, dict);
+ res = BuildPyObject(dict);
+ return (res != nullptr);
+ }
+ };
+
+ template <typename TObjType>
+ class TPythonTypeAttributes<TObjType>::TDictAttrSetter: public TBaseAttrSetter<TObjType> {
+ private:
+ TAttrSetters<TObjType>& AttrSetters;
+
+ public:
+ TDictAttrSetter(TAttrSetters<TObjType>& setters)
+ : AttrSetters(setters)
+ {
+ }
+
+ bool SetAttr(PyObject* owner, TObjType& self, const TString&, PyObject* val) override {
+ TMap<TString, PyObject*> dict;
+ if (!FromPyObject(val, dict))
+ ythrow yexception() << "'__dict__' should be set to dictionary";
+ if (!AttrSetters.SetAttrDictionary(owner, self, dict))
+ return false;
+ return true;
+ }
+ };
+
+ template <typename TObjType>
+ class TPythonTypeAttributes<TObjType>::TGetAttributeMethodCaller: public TBaseMethodCaller<TObjType> {
+ private:
+ GetAttrFunction GetAttr;
+
+ public:
+ TGetAttributeMethodCaller(GetAttrFunction getAttr)
+ : GetAttr(getAttr)
+ {
+ }
+
+ bool CallMethod(PyObject* owner, TObjType*, PyObject* args, PyObject*, PyObject*& res) const override {
+ TString attrName;
+ if (!ExtractArgs(args, attrName))
+ ythrow yexception() << "Could not parse args for '__getattribute__' - it should be one string";
+ res = GetAttr(owner, const_cast<char*>(attrName.c_str()));
+ if (!res)
+ // Error already set
+ return false;
+ return true;
+ }
+ };
+
+ template <typename TObjType>
+ class TPythonTypeAttributes<TObjType>::TSetAttrMethodCaller: public TBaseMethodCaller<TObjType> {
+ private:
+ SetAttrFunction SetAttr;
+
+ public:
+ TSetAttrMethodCaller(SetAttrFunction setAttr)
+ : SetAttr(setAttr)
+ {
+ }
+
+ bool CallMethod(PyObject* owner, TObjType*, PyObject* args, PyObject*, PyObject*& res) const override {
+ TString attrName;
+ TPyObjectPtr value;
+ if (!ExtractArgs(args, attrName, value))
+ ythrow yexception() << "Could not parse args for '__setattr__' - it should be one string and value";
+ Py_INCREF(Py_None);
+ res = Py_None;
+ if (-1 == SetAttr(owner, const_cast<char*>(attrName.c_str()), value.Get()))
+ // Error already set
+ return false;
+ return true;
+ }
+ };
+
+ template <typename TObjType>
+ class TPythonTypeAttributes<TObjType>::TGetStrReprMethodCaller: public TBaseMethodCaller<TObjType> {
+ private:
+ TString MethodName;
+
+ private:
+ const TString GetFullName(PyObject* obj) const {
+ TString module, name;
+ TPyObjectPtr type(PyObject_Type(obj), true);
+ if (!FromPyObject(PyObject_GetAttrString(type.Get(), "__module__"), module) || !FromPyObject(PyObject_GetAttrString(type.Get(), "__name__"), name))
+ ythrow yexception() << "Could not get name of object";
+ return module + "." + name;
+ }
+
+ public:
+ TGetStrReprMethodCaller(const TString& methodName)
+ : MethodName(methodName)
+ {
+ }
+
+ bool CallMethod(PyObject* owner, TObjType*, PyObject* args, PyObject*, PyObject*& res) const override {
+ if (args && !ExtractArgs(args))
+ ythrow yexception() << "Could not parse args for '" << MethodName << "'";
+ TString message = TString("<") + GetFullName(owner) + " object>";
+ res = ReturnString(message);
+ return (res != nullptr);
+ }
+ };
+
+ template <typename TObjType>
+ class TPythonTypeAttributes<TObjType>::TReduceMethodCaller: public TBaseMethodCaller<TObjType> {
+ public:
+ bool CallMethod(PyObject* owner, TObjType*, PyObject*, PyObject*, PyObject*& res) const override {
+ TPyObjectPtr tuple(PyTuple_New(3), true);
+ // First component: reconstructor
+ TPyObjectPtr pybindName(BuildPyObject("pybind"), true);
+ TPyObjectPtr mainModule(PyImport_Import(pybindName.Get()), true);
+ TPyObjectPtr recName(BuildPyObject("PyBindObjectReconstructor"), true);
+ TPyObjectPtr reconstructor(PyObject_GetAttr(mainModule.Get(), recName.Get()), true);
+ // Second component: arguments to rebuild object
+ TPyObjectPtr arguments(PyTuple_New(2), true);
+ TPyObjectPtr cl(PyObject_GetAttrString(owner, "__class__"), true);
+ PyTuple_SET_ITEM(arguments.Get(), 0, cl.RefGet());
+ TPyObjectPtr props(PyObject_CallMethod(owner, const_cast<char*>("GetPropertiesNames"), nullptr), true);
+ PyTuple_SET_ITEM(arguments.Get(), 1, props.RefGet());
+ // Third component: state to fill new object
+ TPyObjectPtr state(PyObject_CallMethod(owner, const_cast<char*>("__getstate__"), nullptr), true);
+
+ PyTuple_SET_ITEM(tuple.Get(), 0, reconstructor.RefGet());
+ PyTuple_SET_ITEM(tuple.Get(), 1, arguments.RefGet());
+ PyTuple_SET_ITEM(tuple.Get(), 2, state.RefGet());
+ res = tuple.RefGet();
+ return (res != nullptr);
+ }
+ };
+
+ template <typename TObjType>
+ class TPythonTypeAttributes<TObjType>::TBaseGetStateMethodCaller: public TGetStateCaller<TObjType, TObjType> {
+ public:
+ void GetAttrsDictionary(PyObject* obj, TObjType* self, TMap<TString, TPyObjectPtr>& dict) const override {
+ this->GetStandartAttrsDictionary(obj, self, dict);
+ }
+ };
+
+ template <typename TObjType>
+ class TPythonTypeAttributes<TObjType>::TBaseSetStateMethodCaller: public TSetStateCaller<TObjType, TObjType> {
+ public:
+ void SetAttrsDictionary(PyObject* obj, TObjType* self, TMap<TString, TPyObjectPtr>& dict) const override {
+ this->SetStandartAttrsDictionary(obj, self, dict);
+ }
+ };
+
+ static const char* HiddenAttrStrings[] = {
+ "__dict__", "__class__", "__dir__", "__delattr__", "__doc__", "__format__", "__getattribute__", "__hash__",
+ "__init__", "__new__", "__reduce__", "__reduce_ex__", "__repr__", "__setattr__", "__sizeof__", "__str__",
+ "__subclasshook__", "__getstate__", "__setstate__",
+ "GetAttrsNames", "GetMethodsNames", "GetAllNames", "GetPropertiesNames"};
+
+ template <typename T>
+ const TSet<TString> TPythonTypeAttributes<T>::HiddenAttrNames(HiddenAttrStrings, std::end(HiddenAttrStrings));
+
+}
diff --git a/library/cpp/pybind/typedesc.cpp b/library/cpp/pybind/typedesc.cpp
new file mode 100644
index 00000000000..75f39fd1260
--- /dev/null
+++ b/library/cpp/pybind/typedesc.cpp
@@ -0,0 +1,79 @@
+#include "typedesc.h"
+
+#include <util/generic/singleton.h>
+
+static void RegisterJSONBridgeImpl() {
+ PyRun_SimpleString("import json\n"
+ "class PyBindEncoder(json.JSONEncoder):\n"
+ " def default(self, obj):\n"
+ " if isinstance(obj, bytes):\n"
+ " try:\n"
+ " return obj.decode()\n"
+ " except UnicodeDecodeError:\n"
+ " return obj.hex()\n"
+ " dct = None\n"
+ " if hasattr(obj, '__getstate__'):\n"
+ " dct = obj.__getstate__()\n"
+ " elif hasattr(obj, '__dict__'):\n"
+ " dct = obj.__dict__\n"
+ " if dct is None:\n"
+ " return json.JSONEncoder.default(self, obj)\n"
+ " if hasattr(obj, '__class__'):\n"
+ " if hasattr(obj.__class__, '__name__'):\n"
+ " dct['__name__'] = obj.__class__.__name__\n"
+ " if hasattr(obj.__class__, '__module__'):\n"
+ " dct['__module__'] = obj.__class__.__module__\n"
+ " if hasattr(obj, 'GetPropertiesNames'):\n"
+ " dct['__properties__'] = obj.GetPropertiesNames()\n"
+ " return dct");
+
+ PyRun_SimpleString("def PyBindObjectHook(dct):\n"
+ " if '__name__' in dct:\n"
+ " name = dct['__name__']\n"
+ " module = dct['__module__']\n"
+ " del dct['__name__']\n"
+ " del dct['__module__']\n"
+ " cls = getattr(__import__(module), name)\n"
+ " if '__properties__' in dct:\n"
+ " props = dct['__properties__']\n"
+ " del dct['__properties__']\n"
+ " if len(props) == 0:\n"
+ " return dct\n"
+ " instance = cls(__properties__ = props)\n"
+ " else:\n"
+ " instance = cls()\n"
+ " if hasattr(instance, '__setstate__'):\n"
+ " instance.__setstate__(dct)\n"
+ " elif hasattr(instance, '__dict__'):\n"
+ " instance.__dict__ = dct\n"
+ " else:\n"
+ " return dct\n"
+ " return instance\n"
+ " return dct");
+
+ PyRun_SimpleString("def json_dump(*args, **kwargs):\n"
+ " kwargs['cls'] = PyBindEncoder\n"
+ " return json.dump(*args, **kwargs)\n"
+ "def json_dumps(*args, **kwargs):\n"
+ " kwargs['cls'] = PyBindEncoder\n"
+ " return json.dumps(*args, **kwargs)");
+
+ PyRun_SimpleString("def json_load(*args, **kwargs):\n"
+ " kwargs['object_hook'] = PyBindObjectHook\n"
+ " return json.load(*args, **kwargs)\n"
+ "def json_loads(*args, **kwargs):\n"
+ " kwargs['object_hook'] = PyBindObjectHook\n"
+ " return json.loads(*args, **kwargs)");
+}
+
+namespace {
+ struct TJSONBridge {
+ TJSONBridge() {
+ RegisterJSONBridgeImpl();
+ }
+ };
+}
+
+void NPyBind::RegisterJSONBridge() {
+ Singleton<TJSONBridge>();
+}
diff --git a/library/cpp/pybind/typedesc.h b/library/cpp/pybind/typedesc.h
new file mode 100644
index 00000000000..57eacb0f3ae
--- /dev/null
+++ b/library/cpp/pybind/typedesc.h
@@ -0,0 +1,545 @@
+#pragma once
+
+#define PY_SSIZE_T_CLEAN
+#include <Python.h>
+#include <structmember.h>
+
+#include "typeattrs.h"
+#include "exceptions.h"
+#include "module.h"
+
+namespace NPyBind {
+ void RegisterJSONBridge();
+
+ namespace NPrivate {
+ template <typename>
+ class TUnboundClosureHolder;
+ template <typename>
+ class TUnboundClosure;
+ }
+
+ // TTraits should be derived from TPythonType
+ template <typename TObjectHolder, typename TObject, typename TTraits>
+ class TPythonType {
+ private:
+ TPythonType(const TPythonType&);
+ TPythonType& operator=(const TPythonType&);
+
+ private:
+ typedef typename TPythonTypeAttributes<TObject>::TGetterPtr TGetterPtr;
+ typedef typename TPythonTypeAttributes<TObject>::TSetterPtr TSetterPtr;
+ typedef typename TPythonTypeAttributes<TObject>::TCallerPtr TCallerPtr;
+
+ struct TProxy {
+ PyObject_HEAD
+ TObjectHolder* Holder;
+ };
+
+ static PyTypeObject PyType;
+ static PyMappingMethods MappingMethods;
+ static PyObject* PyTypeObjPtr;
+ protected:
+ static PyTypeObject* GetPyTypePtr() {
+ return &PyType;
+ }
+ private:
+
+ TPythonTypeAttributes<TObject> Attributes;
+
+ static int InitObject(PyObject* s, PyObject* args, PyObject* kwargs) {
+ try {
+ TProxy* self = reinterpret_cast<TProxy*>(s);
+ auto str = NameFromString("__properties__");
+ if (kwargs && PyDict_Check(kwargs) && PyDict_Contains(kwargs, str.Get())) {
+ TPyObjectPtr props(PyDict_GetItem(kwargs, str.Get()));
+ TVector<TString> properties;
+ FromPyObject(props.Get(), properties);
+ self->Holder = TTraits::DoInitPureObject(properties);
+ } else {
+ self->Holder = (args || kwargs) ? TTraits::DoInitObject(args, kwargs) : nullptr;
+ }
+ if (PyErr_Occurred())
+ return -1;
+ return 0;
+ } catch (const std::exception& ex) {
+ PyErr_SetString(TExceptionsHolder::Instance().ToPyException(ex).Get(), ex.what());
+ } catch (...) {
+ PyErr_SetString(PyExc_RuntimeError, "Unknown error occurred while trying to init object");
+ }
+ return -1;
+ }
+
+ static void DeallocObject(TProxy* self) {
+ delete self->Holder;
+ Py_TYPE(self)->tp_free(reinterpret_cast<PyObject*>(self));
+ }
+
+ static PyObject* GetObjectAttr(PyObject* pyObj, char* attr);
+ static int SetObjectAttr(PyObject* pyObj, char* attr, PyObject* value);
+ static PyObject* GetStr(PyObject*);
+ static PyObject* GetRepr(PyObject*);
+ static PyObject* GetIter(PyObject*);
+ static PyObject* GetNext(PyObject*);
+
+ // Fill class __dict__ with functions to make sure methods names will get to dir()
+ void FillClassDict() const {
+ TVector<TString> names;
+ Attributes.GetMethodCallers().GetAllMethodsNames(names);
+ for (const auto& name : names) {
+ TPyObjectPtr callable = NPrivate::TUnboundClosure<TObject>::Instance().CreatePyObject(new NPrivate::TUnboundClosureHolder<TObject>(&PyType, name));
+ PyDict_SetItemString(PyType.tp_dict, name.c_str(), callable.Get());
+ }
+ }
+
+ void InitCommonAttributes() {
+ static bool was = false;
+ if (was)
+ return;
+ was = true;
+ Attributes.InitCommonAttributes();
+ FillClassDict();
+ }
+
+ protected:
+ TPythonType(const char* pyTypeName, const char* typeDescr, PyTypeObject* parentType = nullptr)
+ : Attributes(GetObjectAttr, SetObjectAttr)
+ {
+ PyType.tp_name = pyTypeName;
+ PyType.tp_doc = typeDescr;
+ Py_INCREF(PyTypeObjPtr);
+ if (parentType) {
+ Py_INCREF(parentType);
+ PyType.tp_base = parentType;
+ }
+ PyType_Ready(&PyType);
+
+ TExceptionsHolder::Instance();
+ RegisterJSONBridge();
+
+ }
+
+ ~TPythonType() {
+ }
+
+ static TObjectHolder* DoInitObject(PyObject*, PyObject*) {
+ return nullptr;
+ }
+
+ static TObjectHolder* DoInitPureObject(const TVector<TString>&) {
+ return nullptr;
+ }
+
+ static void SetClosure(PyObject* (*call)(PyObject*, PyObject*, PyObject*)) {
+ PyType.tp_call = call;
+ }
+
+ public:
+ void AddGetter(const TString& attr, TGetterPtr getter) {
+ Attributes.AddGetter(attr, getter);
+ }
+
+ void AddSetter(const TString& attr, TSetterPtr setter) {
+ Attributes.AddSetter(attr, setter);
+ }
+
+ void AddCaller(const TString& name, TCallerPtr caller) {
+ Attributes.AddCaller(name, caller);
+ if (name == "__iter__") {
+ PyType.tp_iter = GetIter;
+ }
+ if (name == "next") {
+ PyType.tp_iternext = GetNext;
+ }
+ }
+
+ void SetIter(getiterfunc tp_iter) {
+ PyType.tp_iter = tp_iter;
+ }
+
+ void SetIterNext(iternextfunc tp_iternext) {
+ PyType.tp_iternext = tp_iternext;
+ }
+
+ void SetDestructor(destructor tp_dealloc) {
+ PyType.tp_dealloc = tp_dealloc;
+ }
+
+ void SetLengthFunction(lenfunc mp_length) {
+ PyType.tp_as_mapping->mp_length = mp_length;
+ }
+
+ void SetSubscriptFunction(binaryfunc mp_subscript) {
+ PyType.tp_as_mapping->mp_subscript = mp_subscript;
+ }
+
+ void SetAssSubscriptFunction(objobjargproc mp_ass_subscript) {
+ PyType.tp_as_mapping->mp_ass_subscript = mp_ass_subscript;
+ }
+
+ typedef TObject TObjectType;
+
+ static TPythonType& Instance() {
+ static TTraits Traits;
+ Traits.InitCommonAttributes();
+ return Traits;
+ }
+
+ void Register(PyObject* module, const char* typeName) {
+ Py_INCREF(PyTypeObjPtr);
+ if (0 != PyModule_AddObject(module, typeName, PyTypeObjPtr))
+ ythrow yexception() << "can't register type \"" << typeName << "\"";
+ }
+
+ void Register(PyObject* module, const char* objName, TObjectHolder* hld) {
+ if (0 != PyModule_AddObject(module, objName, CreatePyObject(hld).RefGet()))
+ ythrow yexception() << "can't register object \"" << objName << "\"";
+ }
+
+ void Register(TPyObjectPtr module, const TString& typeName) {
+ Register(module.Get(), typeName.c_str());
+ }
+
+ void Register(TPyObjectPtr module, const TString& objName, TObjectHolder* hld) {
+ Register(module.Get(), objName.c_str(), hld);
+ }
+
+ static TObjectHolder* CastToObjectHolder(PyObject* obj) {
+ // Call Instance() to make sure PyTypeObjPtr is already created at this point
+ Instance();
+ if (!PyObject_IsInstance(obj, PyTypeObjPtr))
+ return nullptr;
+ TProxy* prx = reinterpret_cast<TProxy*>(obj);
+ return prx ? prx->Holder : nullptr;
+ }
+
+ static TObject* CastToObject(PyObject* obj) {
+ TObjectHolder* hld = CastToObjectHolder(obj);
+ return hld ? TTraits::GetObject(*hld) : nullptr;
+ }
+
+ static TPyObjectPtr CreatePyObject(TObjectHolder* hld) {
+ TPyObjectPtr r(_PyObject_New(&PyType), true);
+ TProxy* prx = reinterpret_cast<TProxy*>(r.Get());
+ if (prx)
+ prx->Holder = hld;
+ return r;
+ }
+ };
+
+ template <typename TObjectHolder, typename TObject, typename TTraits>
+ PyMappingMethods TPythonType<TObjectHolder, TObject, TTraits>::MappingMethods = {nullptr, nullptr, nullptr};
+
+ template <typename TObjectHolder, typename TObject, typename TTraits>
+ PyTypeObject TPythonType<TObjectHolder, TObject, TTraits>::PyType = {
+ PyVarObject_HEAD_INIT(nullptr, 0) "", sizeof(TProxy), 0, (destructor)&DeallocObject
+#if PY_VERSION_HEX < 0x030800b4
+ , nullptr, /*tp_print*/
+#endif
+#if PY_VERSION_HEX >= 0x030800b4
+ , 0, /*tp_vectorcall_offset*/
+#endif
+ &GetObjectAttr, &SetObjectAttr, nullptr, &GetRepr, nullptr, nullptr, &MappingMethods, nullptr, nullptr, &GetStr, nullptr, nullptr, nullptr,
+ Py_TPFLAGS_DEFAULT, "", nullptr, nullptr, nullptr, 0, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, 0, InitObject, PyType_GenericAlloc, PyType_GenericNew, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, 0
+#if PY_MAJOR_VERSION >= 3
+ , nullptr
+#endif
+#if PY_VERSION_HEX >= 0x030800b1
+ , nullptr /*tp_vectorcall*/
+#endif
+#if PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x03090000
+ , nullptr /*tp_print*/
+#endif
+#if PY_VERSION_HEX >= 0x030C0000
+ , 0 /*tp_watched*/
+#endif
+ };
+
+ template <typename TObjectHolder, typename TObject, typename TTraits>
+ PyObject* TPythonType<TObjectHolder, TObject, TTraits>::PyTypeObjPtr =
+ reinterpret_cast<PyObject*>(&TPythonType<TObjectHolder, TObject, TTraits>::PyType);
+
+ namespace NPrivate {
+ template <typename TObject>
+ class TUnboundClosureHolder {
+ private:
+ THolder<PyTypeObject> Holder;
+ TString Method;
+
+ public:
+ TUnboundClosureHolder(PyTypeObject* ptr, const TString& meth)
+ : Holder(ptr)
+ , Method(meth)
+ {
+ }
+
+ PyTypeObject* GetObject() const {
+ return Holder.Get();
+ }
+
+ const TString GetMethod() const {
+ return Method;
+ }
+
+ PyObject* Call(PyObject* obj, PyObject* args, PyObject*) const {
+ TPyObjectPtr callable(PyObject_GetAttrString(obj, Method.c_str()), true);
+ if (!callable.Get())
+ ythrow yexception() << "PyBind can't call method '" << Method << "'";
+ TPyObjectPtr res(PyObject_CallObject(callable.Get(), args), true);
+ if (!res.Get() && !PyErr_Occurred())
+ ythrow yexception() << "PyBind can't call method '" << Method << "'";
+ return res.RefGet();
+ }
+ };
+
+ template <typename TObject>
+ class TUnboundClosure: public NPyBind::TPythonType<TUnboundClosureHolder<TObject>, PyTypeObject, TUnboundClosure<TObject>> {
+ private:
+ typedef class NPyBind::TPythonType<TUnboundClosureHolder<TObject>, PyTypeObject, TUnboundClosure<TObject>> TParent;
+ friend class NPyBind::TPythonType<TUnboundClosureHolder<TObject>, PyTypeObject, TUnboundClosure<TObject>>;
+
+ class TReprMethodCaller: public TBaseMethodCaller<PyTypeObject> {
+ public:
+ bool CallMethod(PyObject* closure, PyTypeObject*, PyObject*, PyObject*, PyObject*& res) const override {
+ TUnboundClosureHolder<TObject>* hld = TParent::CastToObjectHolder(closure);
+ TPyObjectPtr type((PyObject*)hld->GetObject());
+
+ TString nameStr;
+ TPyObjectPtr name(PyObject_GetAttrString(type.Get(), "__name__"), true);
+ if (!name.Get() || !FromPyObject(name.Get(), nameStr))
+ ythrow yexception() << "Could not get name of object";
+
+ TString methodName(hld->GetMethod());
+
+ TString message = "<unbound method " + nameStr + "." + methodName + ">";
+ res = ReturnString(message);
+ return (res != nullptr);
+ }
+ };
+
+ private:
+ TUnboundClosure()
+ : TParent("", "")
+ {
+ TParent::AddCaller("__repr__", new TReprMethodCaller());
+ TParent::AddCaller("__str__", new TReprMethodCaller());
+ TParent::SetClosure(&Call);
+ }
+
+ static PyObject* Call(PyObject* closure, PyObject* args, PyObject* kwargs) {
+ try {
+ TUnboundClosureHolder<TObject>* hld = TParent::CastToObjectHolder(closure);
+ if (!hld)
+ ythrow yexception() << "Can't cast object to TypeHolder";
+
+ size_t size = 0;
+ if (!PyTuple_Check(args) || (size = PyTuple_Size(args)) < 1)
+ ythrow yexception() << "Can't parse first argument: it should be valid object";
+ --size;
+ TPyObjectPtr obj(PyTuple_GetItem(args, 0));
+ TPyObjectPtr newArgs(PyTuple_New(size), true);
+
+ for (size_t i = 0; i < size; ++i) {
+ TPyObjectPtr item(PyTuple_GetItem(args, i + 1));
+ PyTuple_SetItem(newArgs.Get(), i, item.RefGet());
+ }
+
+ return hld->Call(obj.Get(), newArgs.Get(), kwargs);
+ } catch (const std::exception& ex) {
+ PyErr_SetString(::NPyBind::TExceptionsHolder::Instance().ToPyException(ex).Get(), ex.what());
+ } catch (...) {
+ PyErr_SetString(PyExc_RuntimeError, "Unknown error occurred while trying to call method");
+ }
+ return nullptr;
+ }
+
+ static PyTypeObject* GetObject(TUnboundClosureHolder<TObject>& obj) {
+ return obj.GetObject();
+ }
+ };
+
+ template <typename TObject>
+ class TBoundClosureHolder {
+ private:
+ TPyObjectPtr Ptr;
+ TObject* Object;
+ TString Method;
+ const TMethodCallers<TObject>& MethodCallers;
+
+ public:
+ TBoundClosureHolder(PyObject* ptr, TObject* obj, const TString& meth, const TMethodCallers<TObject>& callers)
+ : Ptr(ptr)
+ , Object(obj)
+ , Method(meth)
+ , MethodCallers(callers)
+ {
+ }
+
+ TPyObjectPtr GetObjectPtr() const {
+ return Ptr;
+ }
+
+ TObject* GetObject() const {
+ return Object;
+ }
+
+ const TString GetMethod() const {
+ return Method;
+ }
+
+ PyObject* Call(PyObject* args, PyObject* kwargs) const {
+ PyObject* res = MethodCallers.CallMethod(Ptr.Get(), Object, args, kwargs, Method);
+ if (res == nullptr && !PyErr_Occurred())
+ ythrow yexception() << "PyBind can't call method '" << Method << "'";
+ return res;
+ }
+ };
+
+ template <typename TObject>
+ class TBoundClosure: public TPythonType<TBoundClosureHolder<TObject>, TObject, TBoundClosure<TObject>> {
+ private:
+ typedef TPythonType<TBoundClosureHolder<TObject>, TObject, TBoundClosure<TObject>> TMyParent;
+ class TReprMethodCaller: public TBaseMethodCaller<TObject> {
+ public:
+ bool CallMethod(PyObject* closure, TObject*, PyObject*, PyObject*, PyObject*& res) const override {
+ TBoundClosureHolder<TObject>* hld = TMyParent::CastToObjectHolder(closure);
+ TPyObjectPtr obj(hld->GetObjectPtr());
+ TPyObjectPtr type(PyObject_Type(obj.Get()), true);
+
+ TString reprStr;
+ TPyObjectPtr repr(PyObject_Repr(obj.Get()), true);
+ if (!repr.Get() || !FromPyObject(repr.Get(), reprStr))
+ ythrow yexception() << "Could not get repr of object";
+
+ TString nameStr;
+ TPyObjectPtr name(PyObject_GetAttrString(type.Get(), "__name__"), true);
+ if (!name.Get() || !FromPyObject(name.Get(), nameStr))
+ ythrow yexception() << "Could not get name of object";
+
+ TString methodName(hld->GetMethod());
+
+ TString message = "<bound method " + nameStr + "." + methodName + " of " + reprStr + ">";
+ res = ReturnString(message);
+ return (res != nullptr);
+ }
+ };
+
+ private:
+ static PyObject* Call(PyObject* closure, PyObject* args, PyObject* kwargs) {
+ try {
+ TBoundClosureHolder<TObject>* hld = TMyParent::CastToObjectHolder(closure);
+ if (!hld)
+ ythrow yexception() << "Can't cast object to ClosureHolder";
+
+ return hld->Call(args, kwargs);
+ } catch (const std::exception& ex) {
+ PyErr_SetString(::NPyBind::TExceptionsHolder::Instance().ToPyException(ex).Get(), ex.what());
+ } catch (...) {
+ PyErr_SetString(PyExc_RuntimeError, "Unknown error occurred while trying to call method");
+ }
+ return nullptr;
+ }
+
+ public:
+ TBoundClosure()
+ : TMyParent("", "")
+ {
+ TMyParent::AddCaller("__repr__", new TReprMethodCaller());
+ TMyParent::AddCaller("__str__", new TReprMethodCaller());
+ TMyParent::SetClosure(&Call);
+ }
+
+ static TObject* GetObject(const TBoundClosureHolder<TObject>& closure) {
+ return closure.GetObject();
+ }
+ };
+
+ }
+
+ template <typename TObjectHolder, typename TObject, typename TTraits>
+ PyObject* TPythonType<TObjectHolder, TObject, TTraits>::GetObjectAttr(PyObject* pyObj, char* attr) {
+ try {
+ TObject* obj = CastToObject(pyObj);
+ PyObject* res = obj ? Instance().Attributes.GetAttrGetters().GetAttr(pyObj, *obj, attr) : nullptr;
+ if (res == nullptr && Instance().Attributes.GetMethodCallers().HasMethod(pyObj, obj, attr)) {
+ TPyObjectPtr r = NPrivate::TBoundClosure<TObject>::Instance().CreatePyObject(new NPrivate::TBoundClosureHolder<TObject>(pyObj, obj, attr, Instance().Attributes.GetMethodCallers()));
+ res = r.RefGet();
+ }
+ if (res == nullptr && !PyErr_Occurred())
+ ythrow TPyErr(PyExc_AttributeError) << "PyBind can't get attribute '" << attr << "'";
+ return res;
+ } catch (const std::exception& ex) {
+ PyErr_SetString(TExceptionsHolder::Instance().ToPyException(ex).Get(), ex.what());
+ } catch (...) {
+ PyErr_SetString(PyExc_RuntimeError, (TString("Unknown error occurred while trying to get attribute '") + attr + "'").c_str());
+ }
+ return nullptr;
+ }
+
+ template <typename TObjectHolder, typename TObject, typename TTraits>
+ int TPythonType<TObjectHolder, TObject, TTraits>::SetObjectAttr(PyObject* pyObj, char* attr, PyObject* value) {
+ try {
+ TObject* obj = CastToObject(pyObj);
+ bool res = obj ? Instance().Attributes.GetAttrSetters().SetAttr(pyObj, *obj, attr, value) : false;
+ if (!res && !PyErr_Occurred())
+ ythrow yexception() << "PyBind can't set attribute '" << attr << "'";
+ return res ? 0 : -1;
+ } catch (const std::exception& ex) {
+ PyErr_SetString(TExceptionsHolder::Instance().ToPyException(ex).Get(), ex.what());
+ } catch (...) {
+ PyErr_SetString(PyExc_RuntimeError, (TString("Unknown error occurred while trying to set attribute '") + attr + "'").c_str());
+ }
+ return -1;
+ }
+
+ template <typename TObjectHolder, typename TObject, typename TTraits>
+ PyObject* TPythonType<TObjectHolder, TObject, TTraits>::GetStr(PyObject* obj) {
+ try {
+ TObject* self = CastToObject(obj);
+ return Instance().Attributes.GetMethodCallers().CallMethod(obj, self, nullptr, nullptr, "__str__");
+ } catch (const std::exception& ex) {
+ PyErr_SetString(TExceptionsHolder::Instance().ToPyException(ex).Get(), ex.what());
+ } catch (...) {
+ PyErr_SetString(PyExc_RuntimeError, (TString("Unknown error occurred while trying to call '__str__'").c_str()));
+ }
+ return nullptr;
+ }
+
+ template <typename TObjectHolder, typename TObject, typename TTraits>
+ PyObject* TPythonType<TObjectHolder, TObject, TTraits>::GetIter(PyObject* obj) {
+ try {
+ TObject* self = CastToObject(obj);
+ return Instance().Attributes.GetMethodCallers().CallMethod(obj, self, nullptr, nullptr, "__iter__");
+ } catch (const std::exception& ex) {
+ PyErr_SetString(TExceptionsHolder::Instance().ToPyException(ex).Get(), ex.what());
+ } catch (...) {
+ PyErr_SetString(PyExc_RuntimeError, (TString("Unknown error occurred while trying to call '__iter__'").c_str()));
+ }
+ return nullptr;
+ }
+
+ template <typename TObjectHolder, typename TObject, typename TTraits>
+ PyObject* TPythonType<TObjectHolder, TObject, TTraits>::GetNext(PyObject* obj) {
+ try {
+ TObject* self = CastToObject(obj);
+ return Instance().Attributes.GetMethodCallers().CallMethod(obj, self, nullptr, nullptr, "next");
+ } catch (const std::exception& ex) {
+ PyErr_SetString(TExceptionsHolder::Instance().ToPyException(ex).Get(), ex.what());
+ } catch (...) {
+ PyErr_SetString(PyExc_RuntimeError, (TString("Unknown error occurred while trying to call 'next'").c_str()));
+ }
+ return nullptr;
+ }
+
+ template <typename TObjectHolder, typename TObject, typename TTraits>
+ PyObject* TPythonType<TObjectHolder, TObject, TTraits>::GetRepr(PyObject* obj) {
+ try {
+ TObject* self = CastToObject(obj);
+ return Instance().Attributes.GetMethodCallers().CallMethod(obj, self, nullptr, nullptr, "__repr__");
+ } catch (const std::exception& ex) {
+ PyErr_SetString(TExceptionsHolder::Instance().ToPyException(ex).Get(), ex.what());
+ } catch (...) {
+ PyErr_SetString(PyExc_RuntimeError, (TString("Unknown error occurred while trying to call '__repr__'").c_str()));
+ }
+ return nullptr;
+ }
+}
diff --git a/library/cpp/pybind/v2.cpp b/library/cpp/pybind/v2.cpp
new file mode 100644
index 00000000000..edce0be7195
--- /dev/null
+++ b/library/cpp/pybind/v2.cpp
@@ -0,0 +1,43 @@
+#include "v2.h"
+namespace NPyBind {
+ namespace Detail {
+ template <>
+ PyTypeObject* GetParentType<void>(const TPyModuleDefinition&) {
+ return nullptr;
+ }
+
+
+ template <bool InitEnabled>
+ void UpdateClassNamesInModule(TPyModuleDefinition& M, const TString& name, PyTypeObject* pythonType) {
+ if (!InitEnabled) {
+ return;
+ }
+ M.ClassName2Type[name] = pythonType;
+ }
+
+ template <bool InitEnabled>
+ void UpdateGetContextInModule(TPyModuleDefinition& M, const TString& name, IGetContextBase* base) {
+ if (!InitEnabled) {
+ return;
+ }
+ M.Class2ContextGetter[name] = base;
+ }
+
+ TPyModuleRegistry::TPyModuleRegistry() {
+#if PY_MAJOR_VERSION >= 3
+ NPrivate::AddFinalizationCallBack([this]() {
+ if (UnnamedModule) {
+ UnnamedModule.Clear();
+ }
+ Name2Def.clear();
+ });
+#endif
+ }
+ template void UpdateClassNamesInModule<false>(TPyModuleDefinition& M, const TString& name, PyTypeObject* pythonType);
+ template void UpdateClassNamesInModule<true>(TPyModuleDefinition& M, const TString& name, PyTypeObject* pythonType);
+
+
+ template void UpdateGetContextInModule<false>(TPyModuleDefinition& M, const TString& name, IGetContextBase* pythonType);
+ template void UpdateGetContextInModule<true>(TPyModuleDefinition& M, const TString& name, IGetContextBase* pythonType);
+ }//Detail
+}//NPyBind
diff --git a/library/cpp/pybind/v2.h b/library/cpp/pybind/v2.h
new file mode 100644
index 00000000000..f561d6a380f
--- /dev/null
+++ b/library/cpp/pybind/v2.h
@@ -0,0 +1,514 @@
+#pragma once
+
+#include <library/cpp/pybind/method.h>
+#include <library/cpp/pybind/typedesc.h>
+#include <library/cpp/pybind/module.h>
+#include <util/generic/hash.h>
+#include <util/generic/hash_set.h>
+#include <util/generic/string.h>
+namespace NPyBind {
+#define DEFINE_CONVERTERS_IMPL(TClass) \
+ PyObject* BuildPyObject(typename TClass::TBase&& base) { \
+ return TClass::BuildPyObject(std::move(base)); \
+ } \
+ PyObject* BuildPyObject(const typename TClass::TBase& base) { \
+ return TClass::BuildPyObject(base); \
+ }
+
+#define DEFINE_CONVERTERS(function) DEFINE_CONVERTERS_IMPL(TFunctionResult<decltype(function)>)
+
+#define DEFINE_TRANSFORMERS_IMPL(TClass) \
+ template <> \
+ bool ::NPyBind::FromPyObject<typename TClass::TBase*>(PyObject * obj, typename TClass::TBase * &res) { \
+ res = TClass::CastToObject(obj); \
+ return res != nullptr; \
+ } \
+ template <> \
+ bool ::NPyBind::FromPyObject<typename TClass::TBase const*>(PyObject * obj, typename TClass::TBase const*& res) { \
+ res = TClass::CastToObject(obj); \
+ return res != nullptr; \
+ }
+
+#define DEFINE_TRANSFORMERS(function) DEFINE_TRANSFORMERS_IMPL(TFunctionResult<decltype(function)>)
+
+ namespace Detail {
+ struct IGetContextBase {
+ virtual ~IGetContextBase() = default;
+ };
+ } //Detail
+ struct TPyModuleDefinition {
+ static void InitModule(const TString& name);
+ static TPyModuleDefinition& GetModule();
+
+ TString Name;
+ NPyBind::TPyObjectPtr M;
+ THashMap<TString, PyTypeObject*> ClassName2Type;
+ THashMap<TString, Detail::IGetContextBase*> Class2ContextGetter;
+ };
+
+ namespace Detail {
+ // Manages modules lifecycle
+ // IMPORTANT!!! Don't use it in PyBind v1 environment, it will lead to inconsistent state of v1 module
+ // UnnamedModule-> new unnamed module stub, this stub become current module. In this case you can add functions to it
+ // InitModuleWithName -> convert unnamed module into named one, now you can switch to it in switch, this module remains current
+ // SwitchToModule switches to the particular module in registry, this module becomes current.
+ class TPyModuleRegistry {
+ private:
+ TPyModuleRegistry();
+ TPyModuleRegistry(const TPyModuleRegistry&) = delete;
+ TPyModuleRegistry& operator=(TPyModuleRegistry&) = delete;
+ public:
+ static TPyModuleRegistry& Get() {
+ static TPyModuleRegistry registry;
+ return registry;
+ }
+ TPyModuleDefinition& GetCurrentModule() {
+ if (!CurrentModule) {
+ GetUnnamedModule();
+ }
+ return *CurrentModule;
+ }
+
+ TPyModuleDefinition& GetUnnamedModule() {
+ if (!UnnamedModule) {
+ UnnamedModule = TPyModuleDefinition();
+ CurrentModule = const_cast<TPyModuleDefinition*>(UnnamedModule.Get());
+ }
+ return *UnnamedModule;
+ }
+
+ TPyModuleDefinition& InitModuleWithName(const TString& name) {
+ if (!UnnamedModule) {
+ GetUnnamedModule();
+ }
+ Name2Def[name] = *UnnamedModule;
+ UnnamedModule.Clear();
+ CurrentModule = &Name2Def[name];
+ return *CurrentModule;
+ }
+
+ TPyModuleDefinition& SwitchToModuleByName(const TString& name) {
+ Y_ENSURE(Name2Def.contains(name));
+ Y_ENSURE(UnnamedModule.Empty());
+ CurrentModule = &Name2Def[name];
+ return *CurrentModule;
+ }
+ private:
+ TPyModuleDefinition* CurrentModule = nullptr;
+ TMaybe<TPyModuleDefinition> UnnamedModule;//
+ THashMap<TString, TPyModuleDefinition> Name2Def;
+ };
+ }//Detail
+
+ inline void TPyModuleDefinition::InitModule(const TString& name) {
+ Detail::TPyModuleRegistry::Get().GetUnnamedModule() = TPyModuleDefinition{name, TModuleHolder::Instance().InitModule(name), {}, {}};
+ Detail::TPyModuleRegistry::Get().InitModuleWithName(name);
+ }
+
+ inline TPyModuleDefinition& TPyModuleDefinition::GetModule() {
+ return Detail::TPyModuleRegistry::Get().GetCurrentModule();
+ }
+
+ namespace Detail {
+ template <class TPythonType>
+ struct TNameCtx {
+ TString ClassShortName;
+ static TNameCtx& GetNameCtx() {
+ static TNameCtx result;
+ return result;
+ }
+ };
+ template <class TBase>
+ struct TContextImpl {
+ PyTypeObject* ParentType = nullptr;
+ TString ClassShortName;
+ TString ClassFullName;
+ TString ClassDescription;
+
+
+ TVector<std::pair<TString, typename TPythonTypeAttributes<TBase>::TCallerPtr>> ListCallers;
+ TVector<std::pair<TString, typename TPythonTypeAttributes<TBase>::TGetterPtr>> ListGetters;
+ TVector<std::pair<TString, typename TPythonTypeAttributes<TBase>::TSetterPtr>> ListSetters;
+ };
+
+ template <class TObject>
+ struct IGetContext: public IGetContextBase {
+ virtual ~IGetContext() = default;
+ virtual const TContextImpl<TObject>& GetContext() const = 0;
+ };
+
+ template <typename THolderClass, typename TBaseClass, bool ShouldEnable, typename=std::enable_if_t<!ShouldEnable || !std::is_default_constructible_v<TBaseClass>>>
+ THolderClass* DoInitPureObject(const TVector<TString>&) {
+ ythrow yexception() << "Can't create this object in pure mode from python";
+ }
+
+ template <typename THolderClass, typename TBaseClass, bool ShouldEnable, typename=std::enable_if_t<ShouldEnable && std::is_default_constructible_v<TBaseClass>>, typename=void>
+ THolderClass* DoInitPureObject(const TVector<TString>&) {
+ return new THolderClass(MakeHolder<TBaseClass>());
+ }
+
+ template <typename T>
+ PyTypeObject* GetParentType(const TPyModuleDefinition& m) {
+ auto shortName = Detail::TNameCtx<T>::GetNameCtx().ClassShortName;
+ auto it = m.ClassName2Type.find(shortName);
+ return (it == m.ClassName2Type.end()) ? nullptr : it->second;
+ }
+
+ template <>
+ PyTypeObject* GetParentType<void>(const TPyModuleDefinition&);
+
+ template <bool InitEnabled>
+ void UpdateClassNamesInModule(TPyModuleDefinition& M, const TString& name, PyTypeObject* pythonType);
+
+ template <bool InitEnabled>
+ void UpdateGetContextInModule(TPyModuleDefinition& M, const TString& name, IGetContextBase* base);
+ }
+
+
+ template <class TParentPyClass_=void>
+ struct TPyParentClassTraits {
+ using TParentPyClass = TParentPyClass_;
+ };
+
+ template <bool InitEnabled_, class TParentPyClass_=void>
+ struct TPyClassConfigTraits: public TPyParentClassTraits<TParentPyClass_> {
+ constexpr static bool InitEnabled = InitEnabled_;
+ constexpr static bool RawInit = false;
+ };
+
+ template <class TParentPyClass_=void>
+ struct TPyClassRawInitConfigTraits: public TPyParentClassTraits<TParentPyClass_> {
+ constexpr static bool InitEnabled = true;
+ constexpr static bool RawInit = true;
+ };
+
+
+ template <typename TBaseClass, typename TPyClassConfigTraits, typename... ConstructorArgs>
+ class TPyClass {
+ public:
+ using TBase = TBaseClass;
+ private:
+ using TThisClass = TPyClass<TBaseClass, TPyClassConfigTraits, ConstructorArgs...>;
+ using TContext = Detail::TContextImpl<TBase>;
+ struct THolder {
+ ::THolder<TBase> Holder;
+ THolder(::THolder<TBase>&& right)
+ : Holder(std::move(right))
+ {
+ }
+ THolder(TBase&& right)
+ : Holder(MakeHolder<TBase>(std::move(right)))
+ {
+ }
+ };
+
+ class TSelectedTraits: public NPyBind::TPythonType<THolder, TBase, TSelectedTraits> {
+ private:
+ using TParent = NPyBind::TPythonType<THolder, TBase, TSelectedTraits>;
+ friend TParent;
+
+ public:
+ TSelectedTraits()
+ : TParent(TThisClass::GetContext().ClassFullName.data(), TThisClass::GetContext().ClassDescription.data(), TThisClass::GetContext().ParentType)
+ {
+ for (const auto& caller : TThisClass::GetContext().ListCallers) {
+ TParent::AddCaller(caller.first, caller.second);
+ }
+
+ for (const auto& getter : TThisClass::GetContext().ListGetters) {
+ TParent::AddGetter(getter.first, getter.second);
+ }
+
+ for (const auto& setter : TThisClass::GetContext().ListSetters) {
+ TParent::AddSetter(setter.first, setter.second);
+ }
+ }
+
+ static TBase* GetObject(const THolder& holder) {
+ return holder.Holder.Get();
+ }
+
+ static THolder* DoInitObject(PyObject* args, PyObject* kwargs) {
+ if constexpr (TPyClassConfigTraits::InitEnabled) {
+ if constexpr (TPyClassConfigTraits::RawInit) {
+ static_assert(sizeof...(ConstructorArgs) == 0, "Do not pass construction args if use RawInit.");
+ return new THolder(::MakeHolder<TBase>(args, kwargs));
+ } else {
+ if (args && (!PyTuple_Check(args) || PyTuple_Size(args) != sizeof...(ConstructorArgs))) {
+ ythrow yexception() << "Method takes " << sizeof...(ConstructorArgs) << " arguments, " << PyTuple_Size(args) << " provided";
+ }
+ ::THolder<TBaseClass> basePtr{Apply([](auto&&... unpackedArgs) {return new TBase(std::forward<decltype(unpackedArgs)>(unpackedArgs)...); }, GetArguments<ConstructorArgs...>(args))};
+ return new THolder(std::move(basePtr));
+ }
+ } else {
+ ythrow yexception() << "Can't create this object from python";
+ }
+ }
+
+ static THolder* DoInitPureObject(const TVector<TString>& properties) {
+ return Detail::DoInitPureObject<THolder, TBase, TPyClassConfigTraits::InitEnabled>(properties);
+ }
+
+ static TBase* CastToObject(PyObject* obj) {
+ return TParent::CastToObject(obj);
+ }
+
+ static PyTypeObject* GetType() {
+ return TParent::GetPyTypePtr();
+ }
+ };
+
+ class TContextHolder: public Detail::IGetContext<TBaseClass> {
+ public:
+ static TContextHolder& GetContextHolder() {
+ static TContextHolder holder;
+ return holder;
+ }
+
+ TContext& GetContext() {
+ return Context;
+ }
+ const TContext& GetContext() const override {
+ return Context;
+ }
+ private:
+ TContext Context;
+ };
+
+ template <class TDerivedClass, class TSuperClass>
+ class TCallerWrapper: public TBaseMethodCaller<TDerivedClass> {
+ public:
+ explicit TCallerWrapper(TSimpleSharedPtr<const TBaseMethodCaller<TSuperClass>> baseCaller)
+ : BaseCaller(baseCaller) {
+ Y_ENSURE(BaseCaller);
+ }
+
+ bool CallMethod(PyObject* owner, TDerivedClass* self, PyObject* args, PyObject* kwargs, PyObject*& res) const override {
+ return BaseCaller->CallMethod(owner, static_cast<TSuperClass*>(self), args, kwargs, res);
+ }
+
+ private:
+ TSimpleSharedPtr<const TBaseMethodCaller<TSuperClass>> BaseCaller;
+ };
+
+ template <class TDerivedClass, class TSuperClass>
+ class TSetterWrapper: public TBaseAttrSetter<TDerivedClass> {
+ public:
+ explicit TSetterWrapper(TSimpleSharedPtr<TBaseAttrSetter<TSuperClass>> baseSetter)
+ : BaseSetter(baseSetter) {
+ Y_ENSURE(BaseSetter);
+ }
+
+ bool SetAttr(PyObject* owner, TDerivedClass& self, const TString& attr, PyObject* val) override {
+ return BaseSetter->SetAttr(owner, static_cast<TSuperClass&>(self), attr, val);
+ }
+
+ private:
+ TSimpleSharedPtr<TBaseAttrSetter<TSuperClass>> BaseSetter;
+ };
+
+ template <class TDerivedClass, class TSuperClass>
+ class TGetterWrapper: public TBaseAttrGetter<TDerivedClass> {
+ public:
+ explicit TGetterWrapper(TSimpleSharedPtr<const TBaseAttrGetter<TSuperClass>> baseGetter)
+ : BaseGetter(baseGetter) {
+ Y_ENSURE(BaseGetter);
+ }
+
+ bool GetAttr(PyObject* owner, const TDerivedClass& self, const TString& attr, PyObject*& res) const override {
+ return BaseGetter->GetAttr(owner, static_cast<const TSuperClass&>(self), attr, res);
+ }
+
+ private:
+ TSimpleSharedPtr<const TBaseAttrGetter<TSuperClass>> BaseGetter;
+ };
+
+ template <class TSuperClass, typename=std::enable_if_t<!std::is_same_v<TSuperClass, void>>>
+ void ReloadAttrsFromBase() {
+ auto shortName = Detail::TNameCtx<TSuperClass>::GetNameCtx().ClassShortName;
+ if (!M.Class2ContextGetter.count(shortName)) {
+ return;
+ }
+ auto callerBasePtr = M.Class2ContextGetter[shortName];
+ if (auto getContextPtr = dynamic_cast<const Detail::IGetContext<TSuperClass>*>(callerBasePtr)) {
+ auto& ctx = getContextPtr->GetContext();
+ auto getUniqueNames = [](const auto& collection) {
+ THashSet<TString> uniqueNames;
+ for (const auto& elem : collection) {
+ uniqueNames.insert(elem.first);
+ }
+ return uniqueNames;
+ };
+
+ auto uniqueCallerNames = getUniqueNames(GetContext().ListCallers);
+ using TConcreteCallerWrapper = TCallerWrapper<TBaseClass, TSuperClass>;
+ for (const auto& caller : ctx.ListCallers) {
+ if (uniqueCallerNames.contains(caller.first)) {
+ continue;
+ }
+ GetContext().ListCallers.push_back(std::make_pair(caller.first, MakeSimpleShared<TConcreteCallerWrapper>(caller.second)));
+ }
+
+ auto uniqueGettersNames = getUniqueNames(GetContext().ListGetters);
+ using TConcreteGetterWrapper = TGetterWrapper<TBaseClass, TSuperClass>;
+ for (const auto& getter : ctx.ListGetters) {
+ if (uniqueGettersNames.contains(getter.first)) {
+ continue;
+ }
+ GetContext().ListGetters.push_back(std::make_pair(getter.first, MakeSimpleShared<TConcreteGetterWrapper>(getter.second)));
+ }
+
+ auto uniqueSetterNames = getUniqueNames(GetContext().ListSetters);
+ using TConcreteSetterWrapper = TSetterWrapper<TBaseClass, TSuperClass>;
+ for (auto& setter : ctx.ListSetters) {
+ if (uniqueSetterNames.contains(setter.first)) {
+ continue;
+ }
+ GetContext().ListSetters.push_back(std::make_pair(setter.first, MakeSimpleShared<TConcreteSetterWrapper>(setter.second)));
+ }
+ }
+ }
+
+ template <class TSuperClass, typename=std::enable_if_t<std::is_same_v<TSuperClass, void>>, typename=void>
+ void ReloadAttrsFromBase() {
+ }
+
+ void CompleteImpl() {
+ ReloadAttrsFromBase<typename TPyClassConfigTraits::TParentPyClass>();
+ TSelectedTraits::Instance().Register(M.M, GetContext().ClassShortName);
+ }
+
+ static TContext& GetContext() {
+ return TContextHolder::GetContextHolder().GetContext();
+ }
+
+
+ friend struct Detail::TContextImpl<TBase>;//instead of context
+ friend struct THolder;
+ friend class TSelectedTraits;
+
+ using TCallerFunc = std::function<bool(PyObject*, TBaseClass*, PyObject*, PyObject*, PyObject*&)>;
+ class TFuncCallerWrapper: public TBaseMethodCaller<TBaseClass> {
+ public:
+ explicit TFuncCallerWrapper(TCallerFunc func)
+ : Func(func) {
+ Y_ENSURE(func);
+ }
+
+ bool CallMethod(PyObject* owner, TBaseClass* self, PyObject* args, PyObject* kwargs, PyObject*& res) const override {
+ return Func(owner, self, args, kwargs, res);
+ }
+ private:
+ mutable TCallerFunc Func;
+ };
+ public:
+ TPyClass(const TString& name, const TString& descr = "")
+ : M(TPyModuleDefinition::GetModule())
+ {
+ Detail::UpdateClassNamesInModule<TPyClassConfigTraits::InitEnabled>(M, name, TSelectedTraits::GetType());
+ Detail::UpdateGetContextInModule<TPyClassConfigTraits::InitEnabled>(M, name, &TContextHolder::GetContextHolder());
+
+ GetContext().ClassFullName = TString::Join(M.Name, ".", name);
+ GetContext().ClassShortName = name;
+ GetContext().ClassDescription = descr;
+ GetContext().ParentType = Detail::GetParentType<typename TPyClassConfigTraits::TParentPyClass>(M);
+ Detail::TNameCtx<TBaseClass>::GetNameCtx().ClassShortName = name;
+ }
+
+ template <typename TMemberFuction, typename = std::enable_if_t<std::is_member_function_pointer_v<TMemberFuction>>, typename=std::enable_if_t<!TIsPointerToConstMemberFunction<TMemberFuction>::value>>
+ TThisClass& Def(const TString& name, TMemberFuction t) {
+ GetContext().ListCallers.push_back(std::make_pair(name, CreateMethodCaller<TBase>(t)));
+ return *this;
+ }
+
+ template <typename TMemberFuction, typename = std::enable_if_t<std::is_member_function_pointer_v<TMemberFuction>>, typename=std::enable_if_t<TIsPointerToConstMemberFunction<TMemberFuction>::value>, typename=void>
+ TThisClass& Def(const TString& name, TMemberFuction t) {
+ GetContext().ListCallers.push_back(std::make_pair(name, CreateConstMethodCaller<TBase>(t)));
+ return *this;
+ }
+
+ template <typename TMemberObject, typename = std::enable_if_t<std::is_member_object_pointer_v<TMemberObject>>>
+ TThisClass& Def(const TString& name, TMemberObject t) {
+ GetContext().ListGetters.push_back(std::make_pair(name, CreateAttrGetter<TBase>(t)));
+ GetContext().ListSetters.push_back(std::make_pair(name, CreateAttrSetter<TBase>(t)));
+ return *this;
+ }
+
+ template <typename TResultType, typename... Args>
+ TThisClass& DefByFunc(const TString& name, std::function<TResultType(TBaseClass&, Args...)> func) {
+ GetContext().ListCallers.push_back(std::make_pair(name, CreateFunctorCaller<TBase, TResultType, Args...>(func)));
+ return *this;
+ }
+
+ TThisClass& DefByFunc(const TString& name, TCallerFunc origFunc) {
+ GetContext().ListCallers.push_back(std::make_pair(name, MakeSimpleShared<TFuncCallerWrapper>(origFunc)));
+ return *this;
+ }
+
+ template <typename TMemberObject>
+ TThisClass& DefReadonly(const TString& name, TMemberObject t, std::enable_if_t<std::is_member_object_pointer<TMemberObject>::value>* = nullptr) {
+ GetContext().ListGetters.push_back(std::make_pair(name, CreateAttrGetter<TBase>(t)));
+ return *this;
+ }
+
+
+ template <typename TMethodGetter, typename TMethodSetter, typename=std::enable_if_t<std::is_member_function_pointer_v<TMethodGetter> && std::is_member_function_pointer_v<TMethodSetter>>>
+ TThisClass& AsProperty(const TString& name, TMethodGetter getter, TMethodSetter setter) {
+ GetContext().ListGetters.push_back(std::make_pair(name, CreateMethodAttrGetter<TBase>(getter)));
+ GetContext().ListSetters.push_back(std::make_pair(name, CreateMethodAttrSetter<TBase>(setter)));
+ return *this;
+ }
+
+ template <typename TMethodGetter, typename TMethodSetter, typename=std::enable_if_t<!std::is_member_function_pointer_v<TMethodGetter> && !std::is_member_function_pointer_v<TMethodSetter>>>
+ TThisClass& AsPropertyByFunc(const TString& name, TMethodGetter getter, TMethodSetter setter) {
+ GetContext().ListGetters.push_back(std::make_pair(name, CreateFunctorAttrGetter<TBase>(getter)));
+ GetContext().ListSetters.push_back(std::make_pair(name, CreateFunctorAttrSetter<TBase>(setter)));
+ return *this;
+ }
+
+ template <typename TMethodGetter, typename=std::enable_if_t<std::is_member_function_pointer_v<TMethodGetter>>>
+ TThisClass& AsProperty(const TString& name, TMethodGetter getter) {
+ GetContext().ListGetters.push_back(std::make_pair(name, CreateMethodAttrGetter<TBase>(getter)));
+ return *this;
+ }
+
+ template <typename TMethodGetter>
+ TThisClass& AsPropertyByFunc(const TString& name, TMethodGetter getter) {
+ GetContext().ListGetters.push_back(std::make_pair(name, CreateFunctorAttrGetter<TBase>(getter)));
+ return *this;
+ }
+
+ TThisClass& Complete() {
+ if (!Completed) {
+ CompleteImpl();
+ Completed = true;
+ }
+ return *this;
+ }
+
+ public:
+ static PyObject* BuildPyObject(TBase&& base) {
+ return NPyBind::BuildPyObject(TSelectedTraits::Instance().CreatePyObject(new THolder(std::move(base))));
+ }
+
+ static PyObject* BuildPyObject(const TBase& base) {
+ return NPyBind::BuildPyObject(TSelectedTraits::Instance().CreatePyObject(new THolder(TBase(base)))); // WARN - copy
+ }
+
+ static TBase* CastToObject(PyObject* obj) {
+ return TSelectedTraits::CastToObject(obj);
+ }
+
+ private:
+ TPyModuleDefinition& M;
+ bool Completed = false;
+ };
+
+ template <typename TFunctionSignature, TFunctionSignature function>
+ void DefImpl(const TString& name, const TString& descr = "") {
+ NPyBind::TModuleHolder::Instance().AddModuleMethod<TModuleMethodCaller<TFunctionSignature, function>::Call>(name, descr);
+ }
+
+#define DefFunc(NAME, FUNC) NPyBind::DefImpl<decltype(FUNC), FUNC>(NAME)
+#define DefFuncDescr(NAME, FUNC, DESCR) NPyBind::DefImpl<decltype(FUNC), FUNC>(NAME, DESCR)
+};
diff --git a/library/cpp/pybind/ya.make b/library/cpp/pybind/ya.make
new file mode 100644
index 00000000000..9b7b3413f2d
--- /dev/null
+++ b/library/cpp/pybind/ya.make
@@ -0,0 +1,14 @@
+PY23_NATIVE_LIBRARY()
+
+SRCS(
+ cast.cpp
+ pod.cpp
+ typedesc.cpp
+ module.cpp
+ exceptions.cpp
+ embedding.cpp
+ empty.cpp
+ v2.cpp
+)
+
+END()
diff --git a/library/python/archive/__init__.py b/library/python/archive/__init__.py
new file mode 100644
index 00000000000..a6e032ff4c5
--- /dev/null
+++ b/library/python/archive/__init__.py
@@ -0,0 +1,266 @@
+import errno
+import logging
+import os
+import random
+import shutil
+import stat
+import string
+import sys
+
+import six
+
+import libarchive
+import libarchive._libarchive as _libarchive
+
+from pathlib2 import PurePath
+
+logger = logging.getLogger(__name__)
+
+GZIP = "gzip"
+ZSTD = "zstd"
+
+ENCODING = "utf-8"
+
+
+class ConfigureError(Exception):
+ pass
+
+
+class Level(object):
+ def __init__(self, level):
+ self.level = level
+
+
+class Compression(object):
+ Fast = Level(1)
+ Default = Level(2)
+ Best = Level(3)
+
+
+def get_compression_level(filter_name, level):
+ if level is None or not filter_name:
+ return None
+ elif isinstance(level, Level):
+ level = {
+ GZIP: {
+ Compression.Fast: 1,
+ Compression.Default: 6,
+ Compression.Best: 9,
+ },
+ ZSTD: {
+ Compression.Fast: 1,
+ Compression.Default: 3,
+ Compression.Best: 22,
+ },
+ }[filter_name][level]
+ return level
+
+
+def encode(value, encoding):
+ return value.encode(encoding)
+
+
+def extract_tar(tar_file_path, output_dir, strip_components=None, fail_on_duplicates=True):
+ output_dir = encode(output_dir, ENCODING)
+ _make_dirs(output_dir)
+ with libarchive.Archive(tar_file_path, mode="rb") as tarfile:
+ for e in tarfile:
+ p = _strip_prefix(e.pathname, strip_components)
+ if not p:
+ continue
+ dest = os.path.join(output_dir, encode(p, ENCODING))
+ if e.pathname.endswith("/"):
+ _make_dirs(dest)
+ continue
+
+ if strip_components and fail_on_duplicates:
+ if os.path.exists(dest):
+ raise Exception(
+ "The file {} is duplicated because of strip_components={}".format(dest, strip_components)
+ )
+
+ _make_dirs(os.path.dirname(dest))
+
+ if e.ishardlink():
+ src = os.path.join(output_dir, _strip_prefix(e.hardlink, strip_components))
+ _hardlink(src, dest)
+ continue
+ if e.issym():
+ src = _strip_prefix(e.linkname, strip_components)
+ _symlink(src, dest)
+ continue
+
+ with open(dest, "wb") as f:
+ if hasattr(os, "fchmod"):
+ os.fchmod(f.fileno(), e.mode & 0o7777)
+ libarchive.call_and_check(
+ _libarchive.archive_read_data_into_fd,
+ tarfile._a,
+ tarfile._a,
+ f.fileno(),
+ )
+
+
+def _strip_prefix(path, strip_components):
+ if not strip_components:
+ return path
+ p = PurePath(path)
+ stripped = str(p.relative_to(*p.parts[:strip_components]))
+ return '' if stripped == '.' else stripped
+
+
+def tar(
+ paths,
+ output,
+ compression_filter=None,
+ compression_level=None,
+ fixed_mtime=None,
+ onerror=None,
+ postprocess=None,
+ dereference=False,
+):
+ if isinstance(paths, six.string_types):
+ paths = [paths]
+
+ if isinstance(output, six.string_types):
+ temp_tar_path, stream = (
+ output + "." + "".join(random.sample(string.ascii_lowercase, 8)),
+ None,
+ )
+ else:
+ temp_tar_path, stream = None, output
+
+ compression_level = get_compression_level(compression_filter, compression_level)
+
+ try:
+ if compression_filter:
+ filter_name = compression_filter
+ if compression_level is not None:
+ filter_opts = {"compression-level": str(compression_level)}
+ else:
+ filter_opts = {}
+ # force gzip don't store mtime of the original file being compressed (http://www.gzip.org/zlib/rfc-gzip.html#file-format)
+ if fixed_mtime is not None and compression_filter == GZIP:
+ filter_opts["timestamp"] = ""
+ else:
+ filter_name = filter_opts = None
+
+ with libarchive.Archive(
+ stream or temp_tar_path,
+ mode="wb",
+ format="gnu",
+ filter=filter_name,
+ filter_opts=filter_opts,
+ fixed_mtime=fixed_mtime,
+ ) as tarfile:
+ # determine order if fixed_mtime is specified to produce stable archive
+ paths = paths if fixed_mtime is None else sorted(paths)
+
+ for p in paths:
+ if type(p) == tuple:
+ path, arcname = p
+ else:
+ path, arcname = p, os.path.basename(p)
+
+ if os.path.isdir(path):
+ for root, dirs, files in os.walk(path, followlinks=dereference):
+ if fixed_mtime is None:
+ entries = dirs + files
+ else:
+ entries = sorted(dirs) + sorted(files)
+
+ reldir = os.path.relpath(root, path)
+ for f in entries:
+ _writepath(
+ tarfile,
+ os.path.join(root, f),
+ os.path.normpath(os.path.join(arcname, reldir, f)),
+ onerror,
+ postprocess,
+ dereference,
+ )
+ else:
+ if not os.path.exists(path):
+ raise OSError("Specified path doesn't exist: {}".format(path))
+ _writepath(tarfile, path, arcname, onerror, postprocess, dereference)
+
+ if temp_tar_path:
+ os.rename(temp_tar_path, output)
+ except Exception:
+ if temp_tar_path and os.path.exists(temp_tar_path):
+ os.remove(temp_tar_path)
+ raise
+
+
+def _writepath(tarfile, src, dst, onerror, postprocess, dereference):
+ def tar_writepath(src, dst):
+ st = os.lstat(src)
+ if stat.S_ISREG(st.st_mode) or stat.S_ISDIR(st.st_mode) or stat.S_ISLNK(st.st_mode):
+ if dereference and stat.S_ISLNK(st.st_mode):
+ src = os.path.realpath(src)
+
+ tarfile.writepath(src, dst)
+
+ if postprocess:
+ postprocess(src, dst, st.st_mode)
+ else:
+ logger.debug("Skipping non-regular file '%s' (stat: %s)", src, st)
+
+ try:
+ return tar_writepath(src, dst)
+ except Exception as e:
+ if isinstance(e, OSError) and e.errno == errno.ENOENT:
+ logger.debug(
+ "Skipping missing file '%s' - looks like directory content has changed during archiving",
+ src,
+ )
+ return
+
+ if onerror:
+ if onerror(src, dst, sys.exc_info()):
+ return tar_writepath(src, dst)
+ else:
+ raise
+
+
+def check_tar(tar_file_path):
+ if os.path.isfile(tar_file_path) or os.path.islink(tar_file_path):
+ return libarchive.is_archive(tar_file_path)
+ return False
+
+
+def _make_dirs(path):
+ try:
+ os.makedirs(path)
+ except OSError as e:
+ if e.errno != errno.EEXIST or not os.path.isdir(path):
+ raise
+
+
+def _hardlink(src, dst):
+ if hasattr(os, "link"):
+ os.link(src, dst)
+ else:
+ shutil.copyfile(src, dst)
+
+
+def _symlink(src, dst):
+ if hasattr(os, "symlink"):
+ os.symlink(src, dst)
+ else:
+ # Windows specific case - we cannot copy file right now,
+ # because it doesn't exist yet (and would be met later in the archive) or symlink is broken.
+ # Act like tar and tarfile - skip such symlinks
+ if os.path.exists(src):
+ shutil.copytree(src, dst)
+
+
+def get_archive_filter_name(filename):
+ filters = libarchive.get_archive_filter_names(filename)
+ # https://a.yandex-team.ru/arc/trunk/arcadia/contrib/libs/libarchive/libarchive/archive_read.c?rev=5800047#L522
+ assert filters[-1] == "none", filters
+ if len(filters) == 1:
+ return None
+ if len(filters) == 2:
+ return filters[0]
+ raise Exception("Archive has chain of filter: {}".format(filters))
diff --git a/library/python/archive/ya.make b/library/python/archive/ya.make
new file mode 100644
index 00000000000..5b86a45a422
--- /dev/null
+++ b/library/python/archive/ya.make
@@ -0,0 +1,19 @@
+PY23_LIBRARY()
+
+STYLE_PYTHON()
+
+PY_SRCS(
+ __init__.py
+)
+
+PEERDIR(
+ contrib/python/pathlib2
+ contrib/python/python-libarchive
+)
+
+END()
+
+RECURSE_FOR_TESTS(
+ benchmark
+ test
+)
diff --git a/library/python/cityhash/cityhash.pyx b/library/python/cityhash/cityhash.pyx
new file mode 100644
index 00000000000..6f0046f0d77
--- /dev/null
+++ b/library/python/cityhash/cityhash.pyx
@@ -0,0 +1,75 @@
+from libcpp.pair cimport pair
+
+cdef extern from "util/system/types.h":
+ ctypedef unsigned long ui64
+
+
+cdef extern from "util/digest/city.h":
+ ui64 CityHash64(const char* buf, size_t len) nogil
+ pair[ui64, ui64] CityHash128(const char* buf, size_t len) nogil
+ ui64 CityHash64WithSeed(const char* buf, size_t len, ui64 seed) nogil
+
+
+cdef extern from "library/python/cityhash/hash.h":
+ ui64 FileCityHash128WithSeedHigh64(const char* fpath) nogil except+
+ ui64 FileCityHash64(const char* fpath) nogil except+
+
+
+def hash64(content):
+ cdef const char* s = content
+ cdef size_t size = len(content)
+ cdef ui64 res = 0
+
+ if size > 128:
+ with nogil:
+ res = CityHash64(s, size)
+ else:
+ res = CityHash64(s, size)
+
+ return res
+
+def hash128(content):
+ cdef const char* s = content
+ cdef size_t size = len(content)
+ cdef pair[ui64, ui64] res = pair[ui64, ui64](0, 0)
+
+ if size > 128:
+ with nogil:
+ res = CityHash128(s, size)
+ else:
+ res = CityHash128(s, size)
+ return res
+
+
+def hash64seed(content, seed):
+ cdef const char* s = content
+ cdef size_t size = len(content)
+ cdef ui64 _seed = seed;
+
+ if size > 128:
+ with nogil:
+ res = CityHash64WithSeed(s, size, _seed)
+ else:
+ res = CityHash64WithSeed(s, size, _seed)
+
+ return res
+
+
+def filehash64(path):
+ cdef const char* p = path
+ cdef ui64 res = 0
+
+ with nogil:
+ res = FileCityHash64(p)
+
+ return res
+
+
+def filehash128high64(path):
+ cdef const char* p = path
+ cdef ui64 res = 0
+
+ with nogil:
+ res = FileCityHash128WithSeedHigh64(p)
+
+ return res
diff --git a/library/python/cityhash/hash.cpp b/library/python/cityhash/hash.cpp
new file mode 100644
index 00000000000..17bd3a75f35
--- /dev/null
+++ b/library/python/cityhash/hash.cpp
@@ -0,0 +1,32 @@
+#include "hash.h"
+
+#include <util/digest/city.h>
+#include <util/generic/string.h>
+#include <util/memory/blob.h>
+#include <util/system/file.h>
+#include <util/system/fstat.h>
+
+void ReadFile(const char* fpath, TBlob& blob) {
+ TFile f(TString{fpath}, RdOnly | Seq);
+ const TFileStat fs(f);
+ auto size = fs.Size;
+
+ if (size < (64 << 10)) {
+ blob = TBlob::FromFileContent(f, 0, size);
+ } else {
+ blob = TBlob::FromFile(f);
+ }
+}
+
+ui64 FileCityHash128WithSeedHigh64(const char* fpath) {
+ TBlob blob;
+ ReadFile(fpath, blob);
+ const uint128 hash = CityHash128WithSeed((const char*)blob.Data(), blob.Size(), uint128(0, blob.Size()));
+ return Uint128High64(hash);
+}
+
+ui64 FileCityHash64(const char* fpath) {
+ TBlob blob;
+ ReadFile(fpath, blob);
+ return CityHash64(static_cast<const char*>(blob.Data()), blob.Size());
+}
diff --git a/library/python/cityhash/hash.h b/library/python/cityhash/hash.h
new file mode 100644
index 00000000000..64b22ba74b0
--- /dev/null
+++ b/library/python/cityhash/hash.h
@@ -0,0 +1,6 @@
+#pragma once
+
+#include <util/system/defaults.h>
+
+ui64 FileCityHash128WithSeedHigh64(const char* fpath);
+ui64 FileCityHash64(const char* fpath);
diff --git a/library/python/cityhash/ya.make b/library/python/cityhash/ya.make
new file mode 100644
index 00000000000..7948e19389b
--- /dev/null
+++ b/library/python/cityhash/ya.make
@@ -0,0 +1,16 @@
+PY23_LIBRARY()
+
+SRCS(
+ hash.cpp
+)
+
+PY_SRCS(
+ TOP_LEVEL
+ cityhash.pyx
+)
+
+END()
+
+RECURSE_FOR_TESTS(
+ test
+)
diff --git a/library/python/codecs/__codecs.pyx b/library/python/codecs/__codecs.pyx
new file mode 100644
index 00000000000..42ec37fe886
--- /dev/null
+++ b/library/python/codecs/__codecs.pyx
@@ -0,0 +1,61 @@
+import six
+
+from libcpp cimport bool
+
+from util.generic.string cimport TString, TStringBuf
+
+
+def to_bytes(s):
+ try:
+ return s.encode('utf-8')
+ except AttributeError:
+ pass
+
+ return s
+
+
+def from_bytes(s):
+ if six.PY3:
+ return s.decode('utf-8')
+
+ return s
+
+
+cdef extern from "library/cpp/blockcodecs/codecs.h" namespace "NBlockCodecs":
+ cdef cppclass ICodec:
+ void Encode(TStringBuf data, TString& res) nogil
+ void Decode(TStringBuf data, TString& res) nogil
+
+ cdef const ICodec* Codec(const TStringBuf& name) except +
+ cdef TString ListAllCodecsAsString() except +
+
+
+def dumps(name, data):
+ name = to_bytes(name)
+
+ cdef const ICodec* codec = Codec(TStringBuf(name, len(name)))
+ cdef TString res
+ cdef TStringBuf cdata = TStringBuf(data, len(data))
+
+ with nogil:
+ codec.Encode(cdata, res)
+
+ return res.c_str()[:res.length()]
+
+
+def loads(name, data):
+ name = to_bytes(name)
+
+ cdef const ICodec* codec = Codec(TStringBuf(name, len(name)))
+ cdef TString res
+ cdef TStringBuf cdata = TStringBuf(data, len(data))
+
+ with nogil:
+ codec.Decode(cdata, res)
+
+ return res.c_str()[:res.length()]
+
+def list_all_codecs():
+ cdef TString res = ListAllCodecsAsString()
+
+ return from_bytes(res.c_str()[:res.length()]).split(',')
diff --git a/library/python/codecs/__init__.py b/library/python/codecs/__init__.py
new file mode 100644
index 00000000000..b9fb00deb0a
--- /dev/null
+++ b/library/python/codecs/__init__.py
@@ -0,0 +1 @@
+from __codecs import loads, dumps, list_all_codecs # noqa
diff --git a/library/python/codecs/ya.make b/library/python/codecs/ya.make
new file mode 100644
index 00000000000..f42d115d5d1
--- /dev/null
+++ b/library/python/codecs/ya.make
@@ -0,0 +1,16 @@
+PY23_LIBRARY()
+
+PEERDIR(
+ library/cpp/blockcodecs
+ contrib/python/six
+)
+
+PY_SRCS(
+ __init__.py
+)
+
+BUILDWITH_CYTHON_CPP(__codecs.pyx)
+
+PY_REGISTER(__codecs)
+
+END()
diff --git a/library/python/compress/__init__.py b/library/python/compress/__init__.py
new file mode 100644
index 00000000000..380ec47dca8
--- /dev/null
+++ b/library/python/compress/__init__.py
@@ -0,0 +1,147 @@
+from io import open
+
+import struct
+import json
+import os
+import logging
+
+import library.python.par_apply as lpp
+import library.python.codecs as lpc
+
+
+logger = logging.getLogger('compress')
+
+
+def list_all_codecs():
+ return sorted(frozenset(lpc.list_all_codecs()))
+
+
+def find_codec(ext):
+ def ext_compress(x):
+ return lpc.dumps(ext, x)
+
+ def ext_decompress(x):
+ return lpc.loads(ext, x)
+
+ ext_decompress(ext_compress(b''))
+
+ return {'c': ext_compress, 'd': ext_decompress, 'n': ext}
+
+
+def codec_for(path):
+ for ext in reversed(path.split('.')):
+ try:
+ return find_codec(ext)
+ except Exception as e:
+ logger.debug('in codec_for(): %s', e)
+
+ raise Exception('unsupported file %s' % path)
+
+
+def compress(fr, to, codec=None, fopen=open, threads=1):
+ if codec:
+ codec = find_codec(codec)
+ else:
+ codec = codec_for(to)
+
+ func = codec['c']
+
+ def iter_blocks():
+ with fopen(fr, 'rb') as f:
+ while True:
+ chunk = f.read(16 * 1024 * 1024)
+
+ if chunk:
+ yield chunk
+ else:
+ yield b''
+
+ return
+
+ def iter_results():
+ info = {
+ 'codec': codec['n'],
+ }
+
+ if fr:
+ info['size'] = os.path.getsize(fr)
+
+ yield json.dumps(info, sort_keys=True) + '\n'
+
+ for c in lpp.par_apply(iter_blocks(), func, threads):
+ yield c
+
+ with fopen(to, 'wb') as f:
+ for c in iter_results():
+ logger.debug('complete %s', len(c))
+ f.write(struct.pack('<I', len(c)))
+
+ try:
+ f.write(c)
+ except TypeError:
+ f.write(c.encode('utf-8'))
+
+
+def decompress(fr, to, codec=None, fopen=open, threads=1):
+ def iter_chunks():
+ with fopen(fr, 'rb') as f:
+ cnt = 0
+
+ while True:
+ ll = f.read(4)
+
+ if ll:
+ ll = struct.unpack('<I', ll)[0]
+
+ if ll:
+ if ll > 100000000:
+ raise Exception('broken stream')
+
+ yield f.read(ll)
+
+ cnt += ll
+ else:
+ if not cnt:
+ raise Exception('empty stream')
+
+ return
+
+ it = iter_chunks()
+ extra = []
+
+ for chunk in it:
+ hdr = {}
+
+ try:
+ hdr = json.loads(chunk)
+ except Exception as e:
+ logger.info('can not parse header, suspect old format: %s', e)
+ extra.append(chunk)
+
+ break
+
+ def resolve_codec():
+ if 'codec' in hdr:
+ return find_codec(hdr['codec'])
+
+ if codec:
+ return find_codec(codec)
+
+ return codec_for(fr)
+
+ dc = resolve_codec()['d']
+
+ def iter_all_chunks():
+ for x in extra:
+ yield x
+
+ for x in it:
+ yield x
+
+ with fopen(to, 'wb') as f:
+ for c in lpp.par_apply(iter_all_chunks(), dc, threads):
+ if c:
+ logger.debug('complete %s', len(c))
+ f.write(c)
+ else:
+ break
diff --git a/library/python/compress/ya.make b/library/python/compress/ya.make
new file mode 100644
index 00000000000..bbf2a784e29
--- /dev/null
+++ b/library/python/compress/ya.make
@@ -0,0 +1,16 @@
+PY23_LIBRARY()
+
+PEERDIR(
+ library/python/codecs
+ library/python/par_apply
+)
+
+PY_SRCS(
+ __init__.py
+)
+
+END()
+
+RECURSE_FOR_TESTS(
+ tests
+)
diff --git a/library/python/json/__init__.py b/library/python/json/__init__.py
new file mode 100644
index 00000000000..c6420d5e6d2
--- /dev/null
+++ b/library/python/json/__init__.py
@@ -0,0 +1,44 @@
+from library.python.json.loads import loads as _loads
+from simplejson import loads as _sj_loads
+
+
+def loads(*args, **kwargs):
+ try:
+ return _loads(*args, **kwargs)
+ except Exception as e:
+ if 'invalid syntax at token' in str(e):
+ kwargs.pop('intern_keys', None)
+ kwargs.pop('intern_vals', None)
+ kwargs.pop('may_unicode', None)
+ return _sj_loads(*args, **kwargs)
+
+ raise
+
+
+from simplejson import load, dump, dumps # noqa
+
+
+def read_file(file_name, **kwargs):
+ """
+ Read file and return its parsed json contents.
+
+ All kwargs will be proxied to `json.load` method as is.
+
+ :param file_name: file with json contents
+ :return: parsed json contents
+ """
+ with open(file_name) as f:
+ return load(f, **kwargs)
+
+
+def write_file(file_name, contents, **kwargs):
+ """
+ Dump json data to file.
+
+ All kwargs will be proxied to `json.dump` method as is.
+
+ :param file_name: file to dump to
+ :param contents: JSON-serializable object
+ """
+ with open(file_name, "w") as f:
+ dump(contents, f, **kwargs)
diff --git a/library/python/json/loads.cpp b/library/python/json/loads.cpp
new file mode 100644
index 00000000000..19cdb096aef
--- /dev/null
+++ b/library/python/json/loads.cpp
@@ -0,0 +1,246 @@
+#include "loads.h"
+
+#include <Python.h>
+
+#include <library/cpp/json/fast_sax/parser.h>
+
+#include <util/generic/algorithm.h>
+#include <util/generic/stack.h>
+#include <util/generic/vector.h>
+#include <util/generic/ylimits.h>
+#include <util/string/ascii.h>
+
+using namespace NJson;
+
+namespace {
+ enum EKind {
+ Undefined,
+ Array,
+ Dict,
+ Value,
+ Key,
+ };
+
+ static inline TStringBuf ToStr(EKind kind) noexcept {
+ switch (kind) {
+ case Undefined:
+ return TStringBuf("Undefined");
+
+ case Array:
+ return TStringBuf("Array");
+
+ case Dict:
+ return TStringBuf("Dict");
+
+ case Value:
+ return TStringBuf("Value");
+
+ case Key:
+ return TStringBuf("Key");
+ }
+
+ Y_UNREACHABLE();
+ }
+
+ struct TUnref {
+ static inline void Destroy(PyObject* o) noexcept {
+ Py_XDECREF(o);
+ }
+ };
+
+ using TObjectPtr = TAutoPtr<PyObject, TUnref>;
+
+ static inline TObjectPtr BuildBool(bool val) noexcept {
+ if (val) {
+ Py_RETURN_TRUE;
+ }
+
+ Py_RETURN_FALSE;
+ }
+
+ // Translate python exceptions from object-creating functions into c++ exceptions
+ // Such errors are reported by returning nullptr
+ // When a python error is set and C++ exception is caught by Cython wrapper,
+ // Python exception is propagated, while C++ exception is discarded.
+ PyObject* CheckNewObject(PyObject* obj) {
+ Y_ENSURE(obj != nullptr, "got python exception");
+ return obj;
+ }
+
+ void CheckRetcode(int retcode) {
+ Y_ENSURE(retcode == 0, "got python exception");
+ }
+
+ static inline TObjectPtr BuildSmall(long val) {
+#if PY_VERSION_HEX >= 0x03000000
+ return CheckNewObject(PyLong_FromLong(val));
+#else
+ return CheckNewObject(PyInt_FromLong(val));
+#endif
+ }
+
+ PyObject* CreatePyString(TStringBuf str, bool intern, bool mayUnicode) {
+#if PY_VERSION_HEX >= 0x03000000
+ Y_UNUSED(mayUnicode);
+ PyObject* pyStr = PyUnicode_FromStringAndSize(str.data(), str.size());
+ if (intern) {
+ PyUnicode_InternInPlace(&pyStr);
+ }
+#else
+ const bool needUnicode = mayUnicode && !AllOf(str, IsAscii);
+ PyObject* pyStr = needUnicode ? PyUnicode_FromStringAndSize(str.data(), str.size())
+ : PyString_FromStringAndSize(str.data(), str.size());
+ if (intern && !needUnicode) {
+ PyString_InternInPlace(&pyStr);
+ }
+#endif
+ return pyStr;
+ }
+
+ struct TVal {
+ EKind Kind = Undefined;
+ TObjectPtr Val;
+
+ inline TVal() noexcept
+ : Kind(Undefined)
+ {
+ }
+
+ inline TVal(EKind kind, TObjectPtr val) noexcept
+ : Kind(kind)
+ , Val(val)
+ {
+ }
+ };
+
+ static inline TObjectPtr NoneRef() noexcept {
+ Py_RETURN_NONE;
+ }
+
+ struct TContext: public TJsonCallbacks {
+ const bool InternKeys;
+ const bool InternVals;
+ const bool MayUnicode;
+ TStack<TVal, TVector<TVal>> S;
+
+ inline TContext(bool internKeys, bool internVals, bool mayUnicode)
+ : TJsonCallbacks(true)
+ , InternKeys(internKeys)
+ , InternVals(internVals)
+ , MayUnicode(mayUnicode)
+ {
+ S.emplace();
+ }
+
+ inline bool Consume(TObjectPtr o) {
+ auto& t = S.top();
+
+ if (t.Kind == Array) {
+ CheckRetcode(PyList_Append(t.Val.Get(), o.Get()));
+ } else if (t.Kind == Key) {
+ auto key = S.top().Val;
+
+ S.pop();
+
+ CheckRetcode(PyDict_SetItem(S.top().Val.Get(), key.Get(), o.Get()));
+ } else {
+ t = TVal(Value, o);
+ }
+
+ return true;
+ }
+
+ inline TObjectPtr Pop(EKind expect) {
+ auto res = S.top();
+
+ S.pop();
+
+ if (res.Kind != expect) {
+ ythrow yexception() << "unexpected kind(expect " << ToStr(expect) << ", got " << ToStr(res.Kind) << ")";
+ }
+
+ return res.Val;
+ }
+
+ inline void Push(EKind kind, TObjectPtr object) {
+ S.push(TVal(kind, object));
+ }
+
+ virtual bool OnNull() {
+ return Consume(NoneRef());
+ }
+
+ virtual bool OnBoolean(bool v) {
+ return Consume(BuildBool(v));
+ }
+
+ virtual bool OnInteger(long long v) {
+ if (v >= (long long)Min<long>()) {
+ return Consume(BuildSmall((long)v));
+ }
+
+ return Consume(CheckNewObject(PyLong_FromLongLong(v)));
+ }
+
+ virtual bool OnUInteger(unsigned long long v) {
+ if (v <= (unsigned long long)Max<long>()) {
+ return Consume(BuildSmall((long)v));
+ }
+
+ return Consume(CheckNewObject(PyLong_FromUnsignedLongLong(v)));
+ }
+
+ virtual bool OnDouble(double v) {
+ return Consume(CheckNewObject(PyFloat_FromDouble(v)));
+ }
+
+ virtual bool OnString(const TStringBuf& v) {
+ return Consume(CheckNewObject(CreatePyString(v, InternVals, MayUnicode)));
+ }
+
+ virtual bool OnOpenMap() {
+ Push(Dict, CheckNewObject(PyDict_New()));
+
+ return true;
+ }
+
+ virtual bool OnCloseMap() {
+ return Consume(Pop(Dict));
+ }
+
+ virtual bool OnMapKey(const TStringBuf& k) {
+ Push(Key, CheckNewObject(CreatePyString(k, InternKeys, MayUnicode)));
+ return true;
+ }
+
+ virtual bool OnOpenArray() {
+ Push(Array, CheckNewObject(PyList_New(0)));
+
+ return true;
+ }
+
+ virtual bool OnCloseArray() {
+ return Consume(Pop(Array));
+ }
+ };
+}
+
+PyObject* LoadJsonFromString(const char* data, size_t len, bool internKeys, bool internVals, bool mayUnicode) {
+ TContext ctx(internKeys, internVals, mayUnicode);
+
+ if (!len) {
+ ythrow yexception() << "parse error: zero length input string";
+ }
+
+ if (!NJson::ReadJsonFast(TStringBuf(data, len), &ctx)) {
+ ythrow yexception() << "parse error";
+ }
+
+ auto& s = ctx.S;
+
+ if (!s || s.top().Kind != Value) {
+ ythrow yexception() << "shit happen";
+ }
+
+ return s.top().Val.Release();
+}
diff --git a/library/python/json/loads.h b/library/python/json/loads.h
new file mode 100644
index 00000000000..62dcdf6f213
--- /dev/null
+++ b/library/python/json/loads.h
@@ -0,0 +1,5 @@
+#pragma once
+
+#include <Python.h>
+
+PyObject* LoadJsonFromString(const char* data, size_t len, bool internKeys = false, bool internVals = false, bool mayUnicode = false);
diff --git a/library/python/json/loads.pyx b/library/python/json/loads.pyx
new file mode 100644
index 00000000000..82e5c6dce7b
--- /dev/null
+++ b/library/python/json/loads.pyx
@@ -0,0 +1,14 @@
+from libcpp cimport bool
+
+cdef extern from "library/python/json/loads.h":
+ object LoadJsonFromString(const char*, size_t, bool internKeys, bool internVals, bool mayUnicode) except +
+
+
+def loads(s, intern_keys = False, intern_vals = False, may_unicode = False):
+ if isinstance(s, unicode):
+ s = s.encode('utf-8')
+
+ try:
+ return LoadJsonFromString(s, len(s), intern_keys, intern_vals, may_unicode)
+ except Exception as e:
+ raise ValueError(str(e))
diff --git a/library/python/json/ya.make b/library/python/json/ya.make
new file mode 100644
index 00000000000..74a82de9d8f
--- /dev/null
+++ b/library/python/json/ya.make
@@ -0,0 +1,17 @@
+PY23_LIBRARY()
+
+PEERDIR(
+ contrib/python/simplejson
+ library/cpp/json/fast_sax
+)
+
+PY_SRCS(
+ __init__.py
+ loads.pyx
+)
+
+SRCS(
+ loads.cpp
+)
+
+END()
diff --git a/library/python/par_apply/__init__.py b/library/python/par_apply/__init__.py
new file mode 100644
index 00000000000..19b89ae8431
--- /dev/null
+++ b/library/python/par_apply/__init__.py
@@ -0,0 +1,114 @@
+import sys
+import threading
+import six
+
+from six.moves import queue
+
+
+def par_apply(seq, func, thr_num, join_polling=None):
+ if thr_num < 2:
+ for x in seq:
+ yield func(x)
+
+ return
+
+ in_q = queue.Queue()
+ out_q = queue.Queue()
+
+ def enumerate_blocks():
+ n = 0
+
+ for b in seq:
+ yield n, [b]
+ n += 1
+
+ yield n, None
+
+ def iter_out():
+ n = 0
+ d = {}
+
+ while True:
+ if n in d:
+ r = d[n]
+ del d[n]
+ n += 1
+
+ yield r
+ else:
+ res = out_q.get()
+
+ d[res[0]] = res
+
+ out_iter = iter_out()
+
+ def wait_block():
+ for x in out_iter:
+ return x
+
+ def iter_compressed():
+ p = 0
+
+ for n, b in enumerate_blocks():
+ in_q.put((n, b))
+
+ while n > p + (thr_num * 2):
+ p, b, c = wait_block()
+
+ if not b:
+ return
+
+ yield p, c
+
+ while True:
+ p, b, c = wait_block()
+
+ if not b:
+ return
+
+ yield p, c
+
+ def proc():
+ while True:
+ data = in_q.get()
+
+ if data is None:
+ return
+
+ n, b = data
+
+ if b:
+ try:
+ res = (func(b[0]), None)
+ except Exception:
+ res = (None, sys.exc_info())
+ else:
+ res = (None, None)
+
+ out_q.put((n, b, res))
+
+ thrs = [threading.Thread(target=proc) for i in range(0, thr_num)]
+
+ for t in thrs:
+ t.start()
+
+ try:
+ for p, c in iter_compressed():
+ res, err = c
+
+ if err:
+ six.reraise(*err)
+
+ yield res
+ finally:
+ for t in thrs:
+ in_q.put(None)
+
+ for t in thrs:
+ if join_polling is not None:
+ while True:
+ t.join(join_polling)
+ if not t.is_alive():
+ break
+ else:
+ t.join()
diff --git a/library/python/par_apply/ya.make b/library/python/par_apply/ya.make
new file mode 100644
index 00000000000..b14592ab799
--- /dev/null
+++ b/library/python/par_apply/ya.make
@@ -0,0 +1,11 @@
+PY23_LIBRARY()
+
+PEERDIR(
+ contrib/python/six
+)
+
+PY_SRCS(
+ __init__.py
+)
+
+END()
diff --git a/library/python/retry/__init__.py b/library/python/retry/__init__.py
new file mode 100644
index 00000000000..5520a70332a
--- /dev/null
+++ b/library/python/retry/__init__.py
@@ -0,0 +1,250 @@
+import copy
+import datetime
+import functools
+import itertools
+import logging
+import random
+import time
+
+
+"""
+Retry library provides an ability to retry function calls in a configurable way.
+
+To retry a certain function call use `retry_call` function. To make function auto-retriable use `retry`
+or `retry_intrusive` decorators. Both `retry_call` and `retry` optionally accept retry configuration object
+or its fields as kwargs. The `retry_intrusive` is designed for methods and uses intrusive configuration object.
+
+>>> retry_call(foo)
+>>> retry_call(foo, foo_args, foo_kwargs)
+>>> retry_call(foo, foo_args, foo_kwargs, conf=conf)
+
+>>> @retry()
+>>> def foo(...):
+>>> ...
+
+>>> @retry(conf)
+>>> def foo(...):
+>>> ...
+
+>>> class Obj(object):
+>>> def __init__(self):
+>>> self.retry_conf = conf
+>>>
+>>> @retry_intrusive
+>>> def foo(self, ...):
+>>> ...
+
+This library differs from its alternatives:
+ * `retry` contrib library lacks time-based limits, reusable configuration objects and is generally less flexible
+ * `retrying` contrib library is somewhat more complex, but also lacks reusable configuration objects
+"""
+
+
+DEFAULT_SLEEP_FUNC = time.sleep
+LOGGER = logging.getLogger(__name__)
+
+
+class RetryConf(object):
+ """
+ Configuration object defines retry behaviour and is composed of these fields:
+ * `retriable` - function that decides if an exception should trigger retrying
+ * `get_delay` - function that returns a number of seconds retrier must wait before doing the next attempt
+ * `max_time` - maximum `datetime.timedelta` that can pass after the first call for any retry attempt to be done
+ * `max_times` - maximum number of retry attempts (note retries, not tries/calls)
+ * `handle_error` - function that is called for each failed call attempt
+ * `logger` - logger object to record retry warnings with
+ * `sleep` - custom sleep function to use for waiting
+
+ >>> RetryConf(max_time=datetime.timedelta(seconds=30), max_times=10)
+
+ Empty configuration retries indefinitely on any exceptions raised.
+
+ By default `DEFAULT_CONF` if used, which retries indefinitely, waiting 1 sec with 1.2 backoff between attempts, and
+ also logging with built-in logger object.
+
+ Configuration must be cloned before modification to create separate configuration:
+
+ >>> DEFAULT_CONF.clone()
+
+ There are various methods that provide convenient clone-and-modify shortcuts and "retry recipes".
+ """
+
+ _PROPS = {
+ "retriable": lambda e: True,
+ "get_delay": lambda n, raised_after, last: 0,
+ "max_time": None,
+ "max_times": None,
+ "handle_error": None,
+ "logger": None,
+ "sleep": DEFAULT_SLEEP_FUNC,
+ }
+
+ def __init__(self, **kwargs):
+ for prop, default_value in self._PROPS.items():
+ setattr(self, prop, default_value)
+ self._set(**kwargs)
+
+ def __repr__(self):
+ return repr(self.__dict__)
+
+ def clone(self, **kwargs):
+ """
+ Clone configuration.
+ """
+
+ obj = copy.copy(self)
+ obj._set(**kwargs)
+ return obj
+
+ def on(self, *errors):
+ """
+ Clone and retry on specific exception types (retriable shortcut):
+
+ >>> conf = conf.on(MyException, MyOtherException)
+ """
+
+ obj = self.clone()
+ obj.retriable = lambda e: isinstance(e, errors)
+ return obj
+
+ def waiting(self, delay=0, backoff=1.0, jitter=0, limit=None):
+ """
+ Clone and wait between attempts with backoff, jitter and limit (get_delay shortcut):
+
+ >>> conf = conf.waiting(delay)
+ >>> conf = conf.waiting(delay, backoff=2.0) # initial delay with backoff x2 on each attempt
+ >>> conf = conf.waiting(delay, jitter=3) # jitter from 0 to 3 seconds
+ >>> conf = conf.waiting(delay, backoff=2.0, limit=60) # delay with backoff, but not greater than a minute
+
+ All these options can be combined together, of course.
+ """
+
+ def get_delay(n, raised_after, last):
+ if n == 1:
+ return delay
+
+ s = last * backoff
+ s += random.uniform(0, jitter)
+ if limit is not None:
+ s = min(s, limit)
+ return s
+
+ obj = self.clone()
+ obj.get_delay = get_delay
+ return obj
+
+ def upto(self, seconds=0, **other_timedelta_kwargs):
+ """
+ Clone and do retry attempts only for some time (max_time shortcut):
+
+ >>> conf = conf.upto(30) # retrying for 30 seconds
+ >>> conf = conf.upto(hours=1, minutes=20) # retrying for 1:20
+
+ Any `datetime.timedelta` kwargs can be used here.
+ """
+
+ obj = self.clone()
+ obj.max_time = datetime.timedelta(seconds=seconds, **other_timedelta_kwargs)
+ return obj
+
+ def upto_retries(self, retries=0):
+ """
+ Set limit for retry attempts number (max_times shortcut):
+
+ >>> conf = conf.upto_retries(10)
+ """
+
+ obj = self.clone()
+ obj.max_times = retries
+ return obj
+
+ def _set(self, **kwargs):
+ for prop, value in kwargs.items():
+ if prop not in self._PROPS:
+ continue
+ setattr(self, prop, value)
+
+
+DEFAULT_CONF = RetryConf(logger=LOGGER).waiting(1, backoff=1.2)
+
+
+def retry_call(f, f_args=(), f_kwargs={}, conf=DEFAULT_CONF, **kwargs):
+ """
+ Retry function call.
+
+ :param f: function to be retried
+ :param f_args: target function args
+ :param f_kwargs: target function kwargs
+ :param conf: configuration
+ """
+
+ if kwargs:
+ conf = conf.clone(**kwargs)
+ return _retry(conf, functools.partial(f, *f_args, **f_kwargs))
+
+
+def retry(conf=DEFAULT_CONF, **kwargs):
+ """
+ Retrying decorator.
+
+ :param conf: configuration
+ """
+
+ if kwargs:
+ conf = conf.clone(**kwargs)
+
+ def decorator(f):
+ @functools.wraps(f)
+ def wrapped(*f_args, **f_kwargs):
+ return _retry(conf, functools.partial(f, *f_args, **f_kwargs))
+
+ return wrapped
+
+ return decorator
+
+
+def retry_intrusive(f):
+ """
+ Retrying method decorator that uses an intrusive conf (obj.retry_conf).
+ """
+
+ @functools.wraps(f)
+ def wrapped(obj, *f_args, **f_kwargs):
+ assert hasattr(obj, "retry_conf"), "Object must have retry_conf attribute for decorator to run"
+ return _retry(obj.retry_conf, functools.partial(f, obj, *f_args, **f_kwargs))
+
+ return wrapped
+
+
+def _retry(conf, f):
+ start = datetime.datetime.now()
+ delay = 0
+ for n in itertools.count(1):
+ try:
+ return f()
+ except Exception as error:
+ raised_after = datetime.datetime.now() - start
+ if conf.handle_error:
+ conf.handle_error(error, n, raised_after)
+ delay = conf.get_delay(n, raised_after, delay)
+ retry_after = raised_after + datetime.timedelta(seconds=delay)
+ retrying = (
+ conf.retriable(error)
+ and (conf.max_times is None or n <= conf.max_times)
+ and (conf.max_time is None or retry_after <= conf.max_time)
+ )
+ if not retrying:
+ raise
+ if delay:
+ conf.sleep(delay)
+ if conf.logger:
+ conf.logger.warning(
+ "Retrying (try %d) after %s (%s + %s sec) on %s: %s",
+ n,
+ retry_after,
+ raised_after,
+ delay,
+ error.__class__.__name__,
+ error,
+ exc_info=True,
+ )
diff --git a/library/python/retry/ya.make b/library/python/retry/ya.make
new file mode 100644
index 00000000000..dcbf5567cd9
--- /dev/null
+++ b/library/python/retry/ya.make
@@ -0,0 +1,11 @@
+PY23_LIBRARY()
+
+STYLE_PYTHON()
+
+PY_SRCS(__init__.py)
+
+END()
+
+RECURSE_FOR_TESTS(
+ tests
+)
diff --git a/library/python/testing/yatest_common/ut/test.py b/library/python/testing/yatest_common/ut/test.py
new file mode 100644
index 00000000000..bffdf1e3535
--- /dev/null
+++ b/library/python/testing/yatest_common/ut/test.py
@@ -0,0 +1,17 @@
+import os
+import tarfile
+
+import yatest.common
+
+import yalibrary.tools
+
+
+def test_jdk_from_package_equals_jdk_tool_from_yaconf_json():
+ jdk_path = yatest.common.binary_path(os.path.join('build', 'platform', 'java', 'jdk', 'testing'))
+ os.makedirs("extracted")
+ with tarfile.open(os.path.join(jdk_path, "jdk.tar")) as tf:
+ tf.extractall("extracted")
+ jdk_tool_path = yalibrary.tools.toolchain_root('java', None, None)
+ with open(os.path.join("extracted", "release")) as jdk_path_release:
+ with open(os.path.join(jdk_tool_path, "release")) as jdk_tool_path_release:
+ assert jdk_path_release.read() == jdk_tool_path_release.read()
diff --git a/library/python/testing/yatest_common/ut/ya.make b/library/python/testing/yatest_common/ut/ya.make
new file mode 100644
index 00000000000..cc56fa1b6ce
--- /dev/null
+++ b/library/python/testing/yatest_common/ut/ya.make
@@ -0,0 +1,17 @@
+PY2TEST()
+
+TEST_SRCS(test.py)
+
+PEERDIR(
+ devtools/ya/yalibrary/tools
+)
+
+DEPENDS(
+ build/platform/java/jdk/testing
+)
+
+REQUIREMENTS(
+ network:full
+)
+
+END()
diff --git a/library/python/testing/yatest_common/ya.make b/library/python/testing/yatest_common/ya.make
index 2f5aa3bf390..9798bb40bbc 100644
--- a/library/python/testing/yatest_common/ya.make
+++ b/library/python/testing/yatest_common/ya.make
@@ -36,3 +36,7 @@ IF (NOT CATBOOST_OPENSOURCE)
ENDIF()
END()
+
+RECURSE(
+ ut
+)
diff --git a/library/python/testing/yatest_common/yatest/common/runtime.py b/library/python/testing/yatest_common/yatest/common/runtime.py
index b43666b596f..7f6f9df4405 100644
--- a/library/python/testing/yatest_common/yatest/common/runtime.py
+++ b/library/python/testing/yatest_common/yatest/common/runtime.py
@@ -158,7 +158,7 @@ def java_path():
"""
from . import runtime_java
- return runtime_java.get_java_path(binary_path(os.path.join('contrib', 'tools', 'jdk')))
+ return runtime_java.get_java_path(binary_path(os.path.join('build', 'platform', 'java', 'jdk', 'testing')))
def java_home():