publishFullContrib: true for ydb

<HIDDEN_URL>
commit_hash:c82a80ac4594723cebf2c7387dec9c60217f603e

10 files changed, 458 insertions, 0 deletions

diff --git a/contrib/python/protobuf/py3/patches/disable-deprecated-warning.patch b/contrib/python/protobuf/py3/patches/disable-deprecated-warning.patch
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/contrib/python/protobuf/py3/patches/disable-deprecated-warning.patch
diff --git a/contrib/python/protobuf/py3/patches/fix-deprecated-import.patch b/contrib/python/protobuf/py3/patches/fix-deprecated-import.patch
new file mode 100644
index 00000000000..d6743ec1eba
--- /dev/null
+++ b/contrib/python/protobuf/py3/patches/fix-deprecated-import.patch
@@ -0,0 +1,10 @@
+diff --git a/python/google/__init__.py b/python/google/__init__.py
+index 5585614..69e3be5 100644
+--- a/python/google/__init__.py
++++ b/python/google/__init__.py
+@@ -1,4 +1 @@
+-try:
+-  __import__('pkg_resources').declare_namespace(__name__)
+-except ImportError:
+-  __path__ = __import__('pkgutil').extend_path(__path__, __name__)
++__path__ = __import__('pkgutil').extend_path(__path__, __name__)
diff --git a/contrib/python/protobuf/py3/patches/fix-deprecated-warning.patch b/contrib/python/protobuf/py3/patches/fix-deprecated-warning.patch
new file mode 100644
index 00000000000..2480373701a
--- /dev/null
+++ b/contrib/python/protobuf/py3/patches/fix-deprecated-warning.patch
@@ -0,0 +1,13 @@
+diff --git a/python/google/protobuf/internal/well_known_types.py b/python/google/protobuf/internal/well_known_types.py
+index e340f90..ff6929e 100644
+--- a/python/google/protobuf/internal/well_known_types.py
++++ b/python/google/protobuf/internal/well_known_types.py
+@@ -90,7 +90,7 @@ class Any(object):
+     return '/' in self.type_url and self.TypeName() == descriptor.full_name
+ 
+ 
+-_EPOCH_DATETIME_NAIVE = datetime.datetime.utcfromtimestamp(0)
++_EPOCH_DATETIME_NAIVE = datetime.datetime(1970, 1, 1, tzinfo=None)
+ _EPOCH_DATETIME_AWARE = datetime.datetime.fromtimestamp(
+     0, tz=datetime.timezone.utc)
+ 
diff --git a/contrib/python/protobuf/py3/patches/lower-upper.patch b/contrib/python/protobuf/py3/patches/lower-upper.patch
new file mode 100644
index 00000000000..ad2fe0c0c5c
--- /dev/null
+++ b/contrib/python/protobuf/py3/patches/lower-upper.patch
@@ -0,0 +1,17 @@
+diff --git a/python/google/protobuf/pyext/message.cc b/python/google/protobuf/pyext/message.cc
+index 3ff31e2..31d4003 100644
+--- a/python/google/protobuf/pyext/message.cc
++++ b/python/google/protobuf/pyext/message.cc
+@@ -121,11 +121,7 @@ namespace message_meta {
+ namespace {
+ // Copied over from internal 'google/protobuf/stubs/strutil.h'.
+ inline void LowerString(std::string* s) {
+-  std::string::iterator end = s->end();
+-  for (std::string::iterator i = s->begin(); i != end; ++i) {
+-    // tolower() changes based on locale.  We don't want this!
+-    if ('A' <= *i && *i <= 'Z') *i += 'a' - 'A';
+-  }
++  s->to_lower();
+ }
+ }  // namespace
+ 
diff --git a/contrib/python/protobuf/py3/patches/pr10403-support-pyhon-3.11.patch_ignored b/contrib/python/protobuf/py3/patches/pr10403-support-pyhon-3.11.patch_ignored
new file mode 100644
index 00000000000..e06256b2858
--- /dev/null
+++ b/contrib/python/protobuf/py3/patches/pr10403-support-pyhon-3.11.patch_ignored
@@ -0,0 +1,132 @@
+From da973aff2adab60a9e516d3202c111dbdde1a50f Mon Sep 17 00:00:00 2001
+From: Alexander Shadchin <[email protected]>
+Date: Sun, 14 Aug 2022 21:13:49 +0300
+Subject: [PATCH] Fix build with Python 3.11
+
+The PyFrameObject structure members have been removed from the public C API.
+---
+ python/google/protobuf/pyext/descriptor.cc | 75 ++++++++++++++++++----
+ 1 file changed, 62 insertions(+), 13 deletions(-)
+
+diff --git a/python/google/protobuf/pyext/descriptor.cc b/python/google/protobuf/pyext/descriptor.cc
+index fc83acf01a7..fc97b0fa6c1 100644
+--- a/google/protobuf/pyext/descriptor.cc
++++ b/google/protobuf/pyext/descriptor.cc
+@@ -56,6 +56,37 @@
+               : 0)                                               \
+        : PyBytes_AsStringAndSize(ob, (charpp), (sizep)))
+ 
++#if PY_VERSION_HEX < 0x030900B1 && !defined(PYPY_VERSION)
++static PyCodeObject* PyFrame_GetCode(PyFrameObject *frame)
++{
++    Py_INCREF(frame->f_code);
++    return frame->f_code;
++}
++
++static PyFrameObject* PyFrame_GetBack(PyFrameObject *frame)
++{
++    Py_XINCREF(frame->f_back);
++    return frame->f_back;
++}
++#endif
++
++#if PY_VERSION_HEX < 0x030B00A7 && !defined(PYPY_VERSION)
++static PyObject* PyFrame_GetLocals(PyFrameObject *frame)
++{
++    if (PyFrame_FastToLocalsWithError(frame) < 0) {
++        return NULL;
++    }
++    Py_INCREF(frame->f_locals);
++    return frame->f_locals;
++}
++
++static PyObject* PyFrame_GetGlobals(PyFrameObject *frame)
++{
++    Py_INCREF(frame->f_globals);
++    return frame->f_globals;
++}
++#endif
++
+ namespace google {
+ namespace protobuf {
+ namespace python {
+@@ -127,48 +127,66 @@ bool _CalledFromGeneratedFile(int stacklevel) {
+   // This check is not critical and is somewhat difficult to implement correctly
+   // in PyPy.
+   PyFrameObject* frame = PyEval_GetFrame();
++  PyCodeObject* frame_code = nullptr;
++  PyObject* frame_globals = nullptr;
++  PyObject* frame_locals = nullptr;
++  bool result = false;
++
+   if (frame == nullptr) {
+-    return false;
++    goto exit;
+   }
++  Py_INCREF(frame);
+   while (stacklevel-- > 0) {
+-    frame = frame->f_back;
++    PyFrameObject* next_frame = PyFrame_GetBack(frame);
++    Py_DECREF(frame);
++    frame = next_frame;
+     if (frame == nullptr) {
+-      return false;
++      goto exit;
+     }
+   }
+ 
+-  if (frame->f_code->co_filename == nullptr) {
+-    return false;
++  frame_code = PyFrame_GetCode(frame);
++  if (frame_code->co_filename == nullptr) {
++    goto exit;
+   }
+   char* filename;
+   Py_ssize_t filename_size;
+-  if (PyString_AsStringAndSize(frame->f_code->co_filename,
++  if (PyString_AsStringAndSize(frame_code->co_filename,
+                                &filename, &filename_size) < 0) {
+     // filename is not a string.
+     PyErr_Clear();
+-    return false;
++    goto exit;
+   }
+   if ((filename_size < 3) ||
+       (strcmp(&filename[filename_size - 3], ".py") != 0)) {
+     // Cython's stack does not have .py file name and is not at global module
+     // scope.
+-    return true;
++    result = true;
++    goto exit;
+   }
+   if (filename_size < 7) {
+     // filename is too short.
+-    return false;
++    goto exit;
+   }
+   if (strcmp(&filename[filename_size - 7], "_pb2.py") != 0) {
+     // Filename is not ending with _pb2.
+-    return false;
++    goto exit;
+   }
+ 
+-  if (frame->f_globals != frame->f_locals) {
++  frame_globals = PyFrame_GetGlobals(frame);
++  frame_locals = PyFrame_GetLocals(frame);
++  if (frame_globals != frame_locals) {
+     // Not at global module scope
+-    return false;
++    goto exit;
+   }
+ #endif
+-  return true;
++  result = true;
++exit:
++  Py_XDECREF(frame_globals);
++  Py_XDECREF(frame_locals);
++  Py_XDECREF(frame_code);
++  Py_XDECREF(frame);
++  return result;
+ }
+ 
+ // If the calling code is not a _pb2.py file, raise AttributeError.
diff --git a/contrib/python/protobuf/py3/patches/pr15999-register-as-virtual-subclasses.patch b/contrib/python/protobuf/py3/patches/pr15999-register-as-virtual-subclasses.patch
new file mode 100644
index 00000000000..05d00c02fe7
--- /dev/null
+++ b/contrib/python/protobuf/py3/patches/pr15999-register-as-virtual-subclasses.patch
@@ -0,0 +1,167 @@
+diff --git a/python/google/protobuf/pyext/map_container.cc b/python/google/protobuf/pyext/map_container.cc
+index 7d690c1..2d75c09 100644
+--- a/python/google/protobuf/pyext/map_container.cc
++++ b/python/google/protobuf/pyext/map_container.cc
+@@ -53,7 +53,7 @@ namespace python {
+ class MapReflectionFriend {
+  public:
+   // Methods that are in common between the map types.
+-  static PyObject* Contains(PyObject* _self, PyObject* key);
++  static int Contains(PyObject* _self, PyObject* key);
+   static Py_ssize_t Length(PyObject* _self);
+   static PyObject* GetIterator(PyObject *_self);
+   static PyObject* IterNext(PyObject* _self);
+@@ -352,7 +352,7 @@ PyObject* MapReflectionFriend::MergeFrom(PyObject* _self, PyObject* arg) {
+   Py_RETURN_NONE;
+ }
+ 
+-PyObject* MapReflectionFriend::Contains(PyObject* _self, PyObject* key) {
++int MapReflectionFriend::Contains(PyObject* _self, PyObject* key) {
+   MapContainer* self = GetMap(_self);
+ 
+   const Message* message = self->parent->message;
+@@ -360,14 +360,14 @@ PyObject* MapReflectionFriend::Contains(PyObject* _self, PyObject* key) {
+   MapKey map_key;
+ 
+   if (!PythonToMapKey(self, key, &map_key)) {
+-    return nullptr;
++    return -1;
+   }
+ 
+   if (reflection->ContainsMapKey(*message, self->parent_field_descriptor,
+                                  map_key)) {
+-    Py_RETURN_TRUE;
++    return 1;
+   } else {
+-    Py_RETURN_FALSE;
++    return 0;
+   }
+ }
+ 
+@@ -466,12 +466,12 @@ static PyObject* ScalarMapGet(PyObject* self, PyObject* args,
+     return nullptr;
+   }
+ 
+-  ScopedPyObjectPtr is_present(MapReflectionFriend::Contains(self, key));
+-  if (is_present.get() == nullptr) {
++  auto is_present = MapReflectionFriend::Contains(self, key);
++  if (is_present < 0) {
+     return nullptr;
+   }
+ 
+-  if (PyObject_IsTrue(is_present.get())) {
++  if (is_present == 1) {
+     return MapReflectionFriend::ScalarMapGetItem(self, key);
+   } else {
+     if (default_value != nullptr) {
+@@ -525,8 +525,6 @@ static void ScalarMapDealloc(PyObject* _self) {
+ }
+ 
+ static PyMethodDef ScalarMapMethods[] = {
+-    {"__contains__", MapReflectionFriend::Contains, METH_O,
+-     "Tests whether a key is a member of the map."},
+     {"clear", (PyCFunction)Clear, METH_NOARGS,
+      "Removes all elements from the map."},
+     {"get", (PyCFunction)ScalarMapGet, METH_VARARGS | METH_KEYWORDS,
+@@ -551,6 +549,7 @@ static PyType_Slot ScalarMapContainer_Type_slots[] = {
+     {Py_mp_subscript, (void*)MapReflectionFriend::ScalarMapGetItem},
+     {Py_mp_ass_subscript, (void*)MapReflectionFriend::ScalarMapSetItem},
+     {Py_tp_methods, (void*)ScalarMapMethods},
++    {Py_sq_contains, (void*)MapReflectionFriend::Contains},
+     {Py_tp_iter, (void*)MapReflectionFriend::GetIterator},
+     {Py_tp_repr, (void*)MapReflectionFriend::ScalarMapToStr},
+     {0, nullptr},
+@@ -710,12 +709,12 @@ PyObject* MessageMapGet(PyObject* self, PyObject* args, PyObject* kwargs) {
+     return nullptr;
+   }
+ 
+-  ScopedPyObjectPtr is_present(MapReflectionFriend::Contains(self, key));
+-  if (is_present.get() == nullptr) {
++  auto is_present = MapReflectionFriend::Contains(self, key);
++  if (is_present < 0) {
+     return nullptr;
+   }
+ 
+-  if (PyObject_IsTrue(is_present.get())) {
++  if (is_present == 1) {
+     return MapReflectionFriend::MessageMapGetItem(self, key);
+   } else {
+     if (default_value != nullptr) {
+@@ -740,8 +739,6 @@ static void MessageMapDealloc(PyObject* _self) {
+ }
+ 
+ static PyMethodDef MessageMapMethods[] = {
+-    {"__contains__", (PyCFunction)MapReflectionFriend::Contains, METH_O,
+-     "Tests whether the map contains this element."},
+     {"clear", (PyCFunction)Clear, METH_NOARGS,
+      "Removes all elements from the map."},
+     {"get", (PyCFunction)MessageMapGet, METH_VARARGS | METH_KEYWORDS,
+@@ -768,6 +765,7 @@ static PyType_Slot MessageMapContainer_Type_slots[] = {
+     {Py_mp_subscript, (void*)MapReflectionFriend::MessageMapGetItem},
+     {Py_mp_ass_subscript, (void*)MapReflectionFriend::MessageMapSetItem},
+     {Py_tp_methods, (void*)MessageMapMethods},
++    {Py_sq_contains, (void*)MapReflectionFriend::Contains},
+     {Py_tp_iter, (void*)MapReflectionFriend::GetIterator},
+     {Py_tp_repr, (void*)MapReflectionFriend::MessageMapToStr},
+     {0, nullptr}};
+@@ -893,6 +891,33 @@ PyTypeObject MapIterator_Type = {
+     nullptr,                        //  tp_init
+ };
+ 
++
++PyTypeObject* PyUpb_AddClassWithRegister(PyType_Spec* spec,
++                                         PyObject* virtual_base,
++                                         const char** methods) {
++  PyObject* type = PyType_FromSpec(spec);
++  PyObject* ret1 = PyObject_CallMethod(virtual_base, "register", "O", type);
++  if (!ret1) {
++    Py_XDECREF(type);
++    return NULL;
++  }
++  for (size_t i = 0; methods[i] != NULL; i++) {
++    PyObject* method = PyObject_GetAttrString(virtual_base, methods[i]);
++    if (!method) {
++      Py_XDECREF(type);
++      return NULL;
++    }
++    int ret2 = PyObject_SetAttrString(type, methods[i], method);
++    if (ret2 < 0) {
++      Py_XDECREF(type);
++      return NULL;
++    }
++  }
++
++  return (PyTypeObject*)type;
++}
++
++
+ bool InitMapContainers() {
+   // ScalarMapContainer_Type derives from our MutableMapping type.
+   ScopedPyObjectPtr abc(PyImport_ImportModule("collections.abc"));
+@@ -907,20 +932,23 @@ bool InitMapContainers() {
+   }
+ 
+   Py_INCREF(mutable_mapping.get());
+-  ScopedPyObjectPtr bases(PyTuple_Pack(1, mutable_mapping.get()));
++  ScopedPyObjectPtr bases(Py_BuildValue("O", mutable_mapping.get()));
+   if (bases == nullptr) {
+     return false;
+   }
+ 
++  const char* methods[] = {"keys", "items",   "values", "__eq__",     "__ne__",
++                           "pop",  "popitem", "update", "setdefault", NULL};
++
+   ScalarMapContainer_Type = reinterpret_cast<PyTypeObject*>(
+-      PyType_FromSpecWithBases(&ScalarMapContainer_Type_spec, bases.get()));
++      PyUpb_AddClassWithRegister(&ScalarMapContainer_Type_spec, bases.get(), methods));
+ 
+   if (PyType_Ready(&MapIterator_Type) < 0) {
+     return false;
+   }
+ 
+   MessageMapContainer_Type = reinterpret_cast<PyTypeObject*>(
+-      PyType_FromSpecWithBases(&MessageMapContainer_Type_spec, bases.get()));
++      PyUpb_AddClassWithRegister(&MessageMapContainer_Type_spec, bases.get(), methods));
+   return true;
+ }
+ 
diff --git a/contrib/python/protobuf/py3/patches/pr8765-raise-decode-error.patch b/contrib/python/protobuf/py3/patches/pr8765-raise-decode-error.patch
new file mode 100644
index 00000000000..8027b6b7bc5
--- /dev/null
+++ b/contrib/python/protobuf/py3/patches/pr8765-raise-decode-error.patch
@@ -0,0 +1,17 @@
+diff --git a/python/google/protobuf/pyext/message.cc b/python/google/protobuf/pyext/message.cc
+index 31d4003..7d69688 100644
+--- a/python/google/protobuf/pyext/message.cc
++++ b/python/google/protobuf/pyext/message.cc
+@@ -1920,10 +1920,8 @@ static PyObject* MergeFromString(CMessage* self, PyObject* arg) {
+   // ctx has an explicit limit set (length of string_view), so we have to
+   // check we ended at that limit.
+   if (!ctx.EndedAtLimit()) {
+-    // TODO(jieluo): Raise error and return NULL instead.
+-    // b/27494216
+-    PyErr_Warn(nullptr, "Unexpected end-group tag: Not all data was converted");
+-    return PyLong_FromLong(data.len - ctx.BytesUntilLimit(ptr));
++    PyErr_Format(DecodeError_class, "Unexpected end-group tag: Not all data was converted");
++    return nullptr;
+   }
+   return PyLong_FromLong(data.len);
+ }
diff --git a/contrib/python/protobuf/py3/patches/roll-back-fix-compilation.patch_ignored b/contrib/python/protobuf/py3/patches/roll-back-fix-compilation.patch_ignored
new file mode 100644
index 00000000000..38db10d49ff
--- /dev/null
+++ b/contrib/python/protobuf/py3/patches/roll-back-fix-compilation.patch_ignored
@@ -0,0 +1,13 @@
+diff --git a/google/protobuf/pyext/message.cc b/google/protobuf/pyext/message.cc
+index c55f5687..19470c76 100644
+--- src/python/google/protobuf/pyext/message.cc
++++ src/python/google/protobuf/pyext/message.cc
+@@ -1065,7 +1065,7 @@ int DeleteRepeatedField(
+     }
+   }
+ 
+-  Arena* arena = Arena::InternalHelper<Message>::GetArenaForAllocation(message);
++  Arena* arena = Arena::InternalGetArenaForAllocation(message);
+   GOOGLE_DCHECK_EQ(arena, nullptr)
+       << "python protobuf is expected to be allocated from heap";
+   // Remove items, starting from the end.
diff --git a/contrib/python/protobuf/py3/patches/wait-send-in-upstream-fix-pydebug.patch b/contrib/python/protobuf/py3/patches/wait-send-in-upstream-fix-pydebug.patch
new file mode 100644
index 00000000000..d880bbb2759
--- /dev/null
+++ b/contrib/python/protobuf/py3/patches/wait-send-in-upstream-fix-pydebug.patch
@@ -0,0 +1,66 @@
+diff --git a/python/google/protobuf/pyext/descriptor.cc b/python/google/protobuf/pyext/descriptor.cc
+index de9a248..8efe348 100644
+--- a/python/google/protobuf/pyext/descriptor.cc
++++ b/python/google/protobuf/pyext/descriptor.cc
+@@ -443,6 +443,9 @@ PyObject* NewInternedDescriptor(PyTypeObject* type,
+ 
+ static void Dealloc(PyObject* pself) {
+   PyBaseDescriptor* self = reinterpret_cast<PyBaseDescriptor*>(pself);
++  if (PyObject_GC_IsTracked(pself)) {
++    PyObject_GC_UnTrack(pself);
++  }
+   // Remove from interned dictionary
+   interned_descriptors->erase(self->descriptor);
+   Py_CLEAR(self->pool);
+diff --git a/python/google/protobuf/pyext/descriptor_pool.cc b/python/google/protobuf/pyext/descriptor_pool.cc
+index 0dd57b2..bfbbe4a 100644
+--- a/python/google/protobuf/pyext/descriptor_pool.cc
++++ b/python/google/protobuf/pyext/descriptor_pool.cc
+@@ -212,6 +212,9 @@ static PyObject* New(PyTypeObject* type,
+ 
+ static void Dealloc(PyObject* pself) {
+   PyDescriptorPool* self = reinterpret_cast<PyDescriptorPool*>(pself);
++  if (PyObject_GC_IsTracked(pself)) {
++    PyObject_GC_UnTrack(pself);
++  }
+   descriptor_pool_map->erase(self->pool);
+   Py_CLEAR(self->py_message_factory);
+   for (std::unordered_map<const void*, PyObject*>::iterator it =
+diff --git a/python/google/protobuf/pyext/map_container.cc b/python/google/protobuf/pyext/map_container.cc
+index 2d75c09..fe48b21 100644
+--- a/python/google/protobuf/pyext/map_container.cc
++++ b/python/google/protobuf/pyext/map_container.cc
+@@ -515,6 +515,9 @@ PyObject* MapReflectionFriend::ScalarMapToStr(PyObject* _self) {
+ 
+ static void ScalarMapDealloc(PyObject* _self) {
+   MapContainer* self = GetMap(_self);
++  if (PyObject_GC_IsTracked(_self)) {
++    PyObject_GC_UnTrack(_self);
++  }
+   self->RemoveFromParentCache();
+   PyTypeObject *type = Py_TYPE(_self);
+   type->tp_free(_self);
+@@ -728,6 +731,9 @@ PyObject* MessageMapGet(PyObject* self, PyObject* args, PyObject* kwargs) {
+ 
+ static void MessageMapDealloc(PyObject* _self) {
+   MessageMapContainer* self = GetMessageMap(_self);
++  if (PyObject_GC_IsTracked(_self)) {
++    PyObject_GC_UnTrack(_self);
++  }
+   self->RemoveFromParentCache();
+   Py_DECREF(self->message_class);
+   PyTypeObject *type = Py_TYPE(_self);
+diff --git a/python/google/protobuf/pyext/message_factory.cc b/python/google/protobuf/pyext/message_factory.cc
+index 27aa5e4..060cc76 100644
+--- a/python/google/protobuf/pyext/message_factory.cc
++++ b/python/google/protobuf/pyext/message_factory.cc
+@@ -104,6 +104,9 @@ PyObject* New(PyTypeObject* type, PyObject* args, PyObject* kwargs) {
+ static void Dealloc(PyObject* pself) {
+   PyMessageFactory* self = reinterpret_cast<PyMessageFactory*>(pself);
+ 
++  if (PyObject_GC_IsTracked(pself)) {
++    PyObject_GC_UnTrack(pself);
++  }
+   typedef PyMessageFactory::ClassesByMessageMap::iterator iterator;
+   for (iterator it = self->classes_by_descriptor->begin();
+        it != self->classes_by_descriptor->end(); ++it) {
diff --git a/contrib/python/protobuf/py3/patches/z_00_yandex_src_migration.sh b/contrib/python/protobuf/py3/patches/z_00_yandex_src_migration.sh
new file mode 100644
index 00000000000..3c743d7732a
--- /dev/null
+++ b/contrib/python/protobuf/py3/patches/z_00_yandex_src_migration.sh
@@ -0,0 +1,23 @@
+cat << EOF > fix.py
+import sys
+
+def fix_line(l):
+    l = l.replace('absl/', 'y_absl/')
+    l = l.replace('absl::', 'y_absl::')
+    l = l.replace('ABSL_', 'Y_ABSL_')
+    l = l.replace('std::string', 'TProtoStringType')
+
+    return l
+
+print('\n'.join(fix_line(x) for x in sys.stdin.read().split('\n')).strip())
+EOF
+
+(
+    find . -type f -name '*.cc'
+    find . -type f -name '*.h'
+) | while read l; do
+    cat ${l} | python3 ./fix.py > _
+    mv _ ${l}
+done
+
+rm fix.py