diff options
author | nechda <nechda@yandex-team.com> | 2024-07-11 01:51:42 +0300 |
---|---|---|
committer | nechda <nechda@yandex-team.com> | 2024-07-11 02:00:59 +0300 |
commit | d006a88e7b23ee01f2d921ddd472095315091d2f (patch) | |
tree | 697bc2fdcce619aeee64af62f33bd7babe799114 | |
parent | 1c80bc50980e66b306543ebd1ed4cce471cf6671 (diff) | |
download | ydb-d006a88e7b23ee01f2d921ddd472095315091d2f.tar.gz |
Update py-protobuf to 3.21.7
Привет! Этот PR обновляет python-protobuf библиотеку до версии 3.21.7
Если у вас возникают какие-то проблемы, пишите в [DEVTOOLSSUPPORT](https://st.yandex-team.ru/DEVTOOLSSUPPORT)
fa476305c5613976658b9ae7392f36a086aa0a9f
21 files changed, 763 insertions, 52 deletions
diff --git a/contrib/python/protobuf/py3/.dist-info/METADATA b/contrib/python/protobuf/py3/.dist-info/METADATA index 13ac7d644c..11099f6d54 100644 --- a/contrib/python/protobuf/py3/.dist-info/METADATA +++ b/contrib/python/protobuf/py3/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: protobuf -Version: 3.20.3 +Version: 4.21.7 Summary: Protocol Buffers Home-page: https://developers.google.com/protocol-buffers/ Download-URL: https://github.com/protocolbuffers/protobuf/releases diff --git a/contrib/python/protobuf/py3/google/protobuf/__init__.py b/contrib/python/protobuf/py3/google/protobuf/__init__.py index 3087605b2e..70d564a90a 100644 --- a/contrib/python/protobuf/py3/google/protobuf/__init__.py +++ b/contrib/python/protobuf/py3/google/protobuf/__init__.py @@ -30,4 +30,4 @@ # Copyright 2007 Google Inc. All Rights Reserved. -__version__ = '3.20.3' +__version__ = '4.21.7' diff --git a/contrib/python/protobuf/py3/google/protobuf/descriptor.py b/contrib/python/protobuf/py3/google/protobuf/descriptor.py index ad70be9a11..f5a0caa6bd 100644 --- a/contrib/python/protobuf/py3/google/protobuf/descriptor.py +++ b/contrib/python/protobuf/py3/google/protobuf/descriptor.py @@ -40,11 +40,15 @@ import warnings from google.protobuf.internal import api_implementation _USE_C_DESCRIPTORS = False -if api_implementation.Type() == 'cpp': +if api_implementation.Type() != 'python': # Used by MakeDescriptor in cpp mode import binascii import os - from google.protobuf.pyext import _message + # pylint: disable=protected-access + _message = api_implementation._c_module + # TODO(jieluo): Remove this import after fix api_implementation + if _message is None: + from google.protobuf.pyext import _message _USE_C_DESCRIPTORS = True @@ -598,13 +602,13 @@ class FieldDescriptor(DescriptorBase): self.is_extension = is_extension self.extension_scope = extension_scope self.containing_oneof = containing_oneof - if api_implementation.Type() == 'cpp': + if api_implementation.Type() == 'python': + self._cdescriptor = None + else: if is_extension: self._cdescriptor = _message.default_pool.FindExtensionByName(full_name) else: self._cdescriptor = _message.default_pool.FindFieldByName(full_name) - else: - self._cdescriptor = None @property def camelcase_name(self): @@ -1135,7 +1139,7 @@ def MakeDescriptor(desc_proto, package='', build_file_if_cpp=True, Returns: A Descriptor for protobuf messages. """ - if api_implementation.Type() == 'cpp' and build_file_if_cpp: + if api_implementation.Type() != 'python' and build_file_if_cpp: # The C++ implementation requires all descriptors to be backed by the same # definition in the C++ descriptor pool. To do this, we build a # FileDescriptorProto with the same definition as this descriptor and build diff --git a/contrib/python/protobuf/py3/google/protobuf/internal/api_implementation.py b/contrib/python/protobuf/py3/google/protobuf/internal/api_implementation.py index 7fef237670..74586487a8 100644 --- a/contrib/python/protobuf/py3/google/protobuf/internal/api_implementation.py +++ b/contrib/python/protobuf/py3/google/protobuf/internal/api_implementation.py @@ -31,41 +31,96 @@ """Determine which implementation of the protobuf API is used in this process. """ +import importlib import os import sys import warnings + +def _ApiVersionToImplementationType(api_version): + if api_version == 2: + return 'cpp' + if api_version == 1: + raise ValueError('api_version=1 is no longer supported.') + if api_version == 0: + return 'python' + return None + + +_implementation_type = None try: # pylint: disable=g-import-not-at-top from google.protobuf.internal import _api_implementation # The compile-time constants in the _api_implementation module can be used to # switch to a certain implementation of the Python API at build time. - _api_version = _api_implementation.api_version + _implementation_type = _ApiVersionToImplementationType( + _api_implementation.api_version) except ImportError: - _api_version = -1 # Unspecified by compiler flags. + pass # Unspecified by compiler flags. + -if _api_version == 1: - raise ValueError('api_version=1 is no longer supported.') +def _CanImport(mod_name): + try: + mod = importlib.import_module(mod_name) + # Work around a known issue in the classic bootstrap .par import hook. + if not mod: + raise ImportError(mod_name + ' import succeeded but was None') + return True + except ImportError: + return False -_default_implementation_type = ('cpp' if _api_version > 0 else 'python') +if _implementation_type is None: + if _CanImport('google._upb._message'): + _implementation_type = 'upb' + elif _CanImport('google.protobuf.pyext._message'): + _implementation_type = 'cpp' + else: + _implementation_type = 'python' # This environment variable can be used to switch to a certain implementation # of the Python API, overriding the compile-time constants in the -# _api_implementation module. Right now only 'python' and 'cpp' are valid -# values. Any other value will be ignored. +# _api_implementation module. Right now only 'python', 'cpp' and 'upb' are +# valid values. Any other value will raise error. _implementation_type = os.getenv('PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION', - _default_implementation_type) + _implementation_type) -if _implementation_type != 'python': - _implementation_type = 'cpp' +if _implementation_type not in ('python', 'cpp', 'upb'): + raise ValueError('PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION {0} is not ' + 'supported. Please set to \'python\', \'cpp\' or ' + '\'upb\'.'.format(_implementation_type)) if 'PyPy' in sys.version and _implementation_type == 'cpp': warnings.warn('PyPy does not work yet with cpp protocol buffers. ' 'Falling back to the python implementation.') _implementation_type = 'python' +_c_module = None + +if _implementation_type == 'cpp': + try: + # pylint: disable=g-import-not-at-top + from google.protobuf.pyext import _message + _c_module = _message + del _message + except ImportError: + # TODO(jieluo): fail back to python + warnings.warn( + 'Selected implementation cpp is not available.') + pass + +if _implementation_type == 'upb': + try: + # pylint: disable=g-import-not-at-top + from google._upb import _message + _c_module = _message + del _message + except ImportError: + warnings.warn('Selected implementation upb is not available. ' + 'Falling back to the python implementation.') + _implementation_type = 'python' + pass # Detect if serialization should be deterministic by default try: @@ -103,6 +158,7 @@ def _SetType(implementation_type): # See comment on 'Type' above. +# TODO(jieluo): Remove the API, it returns a constant. b/228102101 def Version(): return 2 diff --git a/contrib/python/protobuf/py3/google/protobuf/internal/decoder.py b/contrib/python/protobuf/py3/google/protobuf/internal/decoder.py index bc1b7b785c..a916276319 100644 --- a/contrib/python/protobuf/py3/google/protobuf/internal/decoder.py +++ b/contrib/python/protobuf/py3/google/protobuf/internal/decoder.py @@ -831,6 +831,45 @@ def MessageSetItemDecoder(descriptor): return DecodeItem + +def UnknownMessageSetItemDecoder(): + """Returns a decoder for a Unknown MessageSet item.""" + + type_id_tag_bytes = encoder.TagBytes(2, wire_format.WIRETYPE_VARINT) + message_tag_bytes = encoder.TagBytes(3, wire_format.WIRETYPE_LENGTH_DELIMITED) + item_end_tag_bytes = encoder.TagBytes(1, wire_format.WIRETYPE_END_GROUP) + + def DecodeUnknownItem(buffer): + pos = 0 + end = len(buffer) + message_start = -1 + message_end = -1 + while 1: + (tag_bytes, pos) = ReadTag(buffer, pos) + if tag_bytes == type_id_tag_bytes: + (type_id, pos) = _DecodeVarint(buffer, pos) + elif tag_bytes == message_tag_bytes: + (size, message_start) = _DecodeVarint(buffer, pos) + pos = message_end = message_start + size + elif tag_bytes == item_end_tag_bytes: + break + else: + pos = SkipField(buffer, pos, end, tag_bytes) + if pos == -1: + raise _DecodeError('Missing group end tag.') + + if pos > end: + raise _DecodeError('Truncated message.') + + if type_id == -1: + raise _DecodeError('MessageSet item missing type_id.') + if message_start == -1: + raise _DecodeError('MessageSet item missing message.') + + return (type_id, buffer[message_start:message_end].tobytes()) + + return DecodeUnknownItem + # -------------------------------------------------------------------- def MapDecoder(field_descriptor, new_default, is_message_map): diff --git a/contrib/python/protobuf/py3/google/protobuf/internal/python_message.py b/contrib/python/protobuf/py3/google/protobuf/internal/python_message.py index 2921d5cb6e..5550b425c4 100644 --- a/contrib/python/protobuf/py3/google/protobuf/internal/python_message.py +++ b/contrib/python/protobuf/py3/google/protobuf/internal/python_message.py @@ -1479,7 +1479,7 @@ class _Listener(object): In order to support semantics like: - foo.bar.baz.qux = 23 + foo.bar.baz.moo = 23 assert foo.HasField('bar') ...child objects must have back references to their parents. diff --git a/contrib/python/protobuf/py3/google/protobuf/internal/well_known_types.py b/contrib/python/protobuf/py3/google/protobuf/internal/well_known_types.py index 3f1aa9f846..3cc97f6a55 100644 --- a/contrib/python/protobuf/py3/google/protobuf/internal/well_known_types.py +++ b/contrib/python/protobuf/py3/google/protobuf/internal/well_known_types.py @@ -868,6 +868,7 @@ class ListValue(object): collections.abc.MutableSequence.register(ListValue) +# LINT.IfChange(wktbases) WKTBASES = { 'google.protobuf.Any': Any, 'google.protobuf.Duration': Duration, @@ -876,3 +877,4 @@ WKTBASES = { 'google.protobuf.Struct': Struct, 'google.protobuf.Timestamp': Timestamp, } +# LINT.ThenChange(//depot/google.protobuf/compiler/python/pyi_generator.cc:wktbases) diff --git a/contrib/python/protobuf/py3/google/protobuf/message_factory.py b/contrib/python/protobuf/py3/google/protobuf/message_factory.py index 3656fa6874..8d65204581 100644 --- a/contrib/python/protobuf/py3/google/protobuf/message_factory.py +++ b/contrib/python/protobuf/py3/google/protobuf/message_factory.py @@ -43,10 +43,10 @@ from google.protobuf.internal import api_implementation from google.protobuf import descriptor_pool from google.protobuf import message -if api_implementation.Type() == 'cpp': - from google.protobuf.pyext import cpp_message as message_impl -else: +if api_implementation.Type() == 'python': from google.protobuf.internal import python_message as message_impl +else: + from google.protobuf.pyext import cpp_message as message_impl # pylint: disable=g-import-not-at-top # The type of all Message classes. @@ -118,6 +118,8 @@ class MessageFactory(object): self.GetPrototype(extension.containing_type) extended_class = self._classes[extension.containing_type] extended_class.RegisterExtension(extension) + if extension.message_type: + self.GetPrototype(extension.message_type) return result_class def GetMessages(self, files): @@ -154,6 +156,8 @@ class MessageFactory(object): self.GetPrototype(extension.containing_type) extended_class = self._classes[extension.containing_type] extended_class.RegisterExtension(extension) + if extension.message_type: + self.GetPrototype(extension.message_type) return result diff --git a/contrib/python/protobuf/py3/google/protobuf/pyext/cpp_message.py b/contrib/python/protobuf/py3/google/protobuf/pyext/cpp_message.py index fc8eb32d79..ca290299f1 100644 --- a/contrib/python/protobuf/py3/google/protobuf/pyext/cpp_message.py +++ b/contrib/python/protobuf/py3/google/protobuf/pyext/cpp_message.py @@ -36,7 +36,14 @@ Descriptor objects at runtime backed by the protocol buffer C++ API. __author__ = 'tibell@google.com (Johan Tibell)' -from google.protobuf.pyext import _message +from google.protobuf.internal import api_implementation + + +# pylint: disable=protected-access +_message = api_implementation._c_module +# TODO(jieluo): Remove this import after fix api_implementation +if _message is None: + from google.protobuf.pyext import _message class GeneratedProtocolMessageType(_message.MessageMeta): diff --git a/contrib/python/protobuf/py3/google/protobuf/pyext/extension_dict.cc b/contrib/python/protobuf/py3/google/protobuf/pyext/extension_dict.cc index 692029f682..66703da898 100644 --- a/contrib/python/protobuf/py3/google/protobuf/pyext/extension_dict.cc +++ b/contrib/python/protobuf/py3/google/protobuf/pyext/extension_dict.cc @@ -331,7 +331,7 @@ static PyObject* RichCompare(ExtensionDict* self, PyObject* other, int opid) { } bool equals = false; if (PyObject_TypeCheck(other, &ExtensionDict_Type)) { - equals = self->parent == reinterpret_cast<ExtensionDict*>(other)->parent;; + equals = self->parent == reinterpret_cast<ExtensionDict*>(other)->parent; } if (equals ^ (opid == Py_EQ)) { Py_RETURN_FALSE; diff --git a/contrib/python/protobuf/py3/google/protobuf/pyext/extension_dict.h b/contrib/python/protobuf/py3/google/protobuf/pyext/extension_dict.h index a0581941bd..86d2451a00 100644 --- a/contrib/python/protobuf/py3/google/protobuf/pyext/extension_dict.h +++ b/contrib/python/protobuf/py3/google/protobuf/pyext/extension_dict.h @@ -37,8 +37,6 @@ #define PY_SSIZE_T_CLEAN #include <Python.h> -#include <memory> - #include <google/protobuf/pyext/message.h> namespace google { diff --git a/contrib/python/protobuf/py3/google/protobuf/pyext/map_container.cc b/contrib/python/protobuf/py3/google/protobuf/pyext/map_container.cc index 2c22e022d0..17bf0988ed 100644 --- a/contrib/python/protobuf/py3/google/protobuf/pyext/map_container.cc +++ b/contrib/python/protobuf/py3/google/protobuf/pyext/map_container.cc @@ -264,7 +264,7 @@ static bool PythonToMapValueRef(MapContainer* self, PyObject* obj, case FieldDescriptor::CPPTYPE_BOOL: { GOOGLE_CHECK_GET_BOOL(obj, value, false); value_ref->SetBoolValue(value); - return true;; + return true; } case FieldDescriptor::CPPTYPE_STRING: { TProtoStringType str; diff --git a/contrib/python/protobuf/py3/google/protobuf/pyext/map_container.h b/contrib/python/protobuf/py3/google/protobuf/pyext/map_container.h index e14136efab..af334d2e77 100644 --- a/contrib/python/protobuf/py3/google/protobuf/pyext/map_container.h +++ b/contrib/python/protobuf/py3/google/protobuf/pyext/map_container.h @@ -35,7 +35,6 @@ #include <Python.h> #include <cstdint> -#include <memory> #include <google/protobuf/descriptor.h> #include <google/protobuf/message.h> diff --git a/contrib/python/protobuf/py3/google/protobuf/pyext/message.cc b/contrib/python/protobuf/py3/google/protobuf/pyext/message.cc index d416925f53..0a1474a233 100644 --- a/contrib/python/protobuf/py3/google/protobuf/pyext/message.cc +++ b/contrib/python/protobuf/py3/google/protobuf/pyext/message.cc @@ -68,6 +68,7 @@ #include <google/protobuf/pyext/repeated_scalar_container.h> #include <google/protobuf/pyext/safe_numerics.h> #include <google/protobuf/pyext/scoped_pyobject_ptr.h> +#include <google/protobuf/pyext/unknown_field_set.h> #include <google/protobuf/pyext/unknown_fields.h> #include <google/protobuf/util/message_differencer.h> #include <google/protobuf/io/strtod.h> @@ -2418,7 +2419,7 @@ static PyObject* GetExtensionDict(CMessage* self, void *closure) { return reinterpret_cast<PyObject*>(extension_dict); } -static PyObject* UnknownFieldSet(CMessage* self) { +static PyObject* GetUnknownFields(CMessage* self) { if (self->unknown_field_set == nullptr) { self->unknown_field_set = unknown_fields::NewPyUnknownFields(self); } else { @@ -2487,7 +2488,7 @@ static PyMethodDef Methods[] = { "Serializes the message to a string, only for initialized messages."}, {"SetInParent", (PyCFunction)SetInParent, METH_NOARGS, "Sets the has bit of the given field in its parent message."}, - {"UnknownFields", (PyCFunction)UnknownFieldSet, METH_NOARGS, + {"UnknownFields", (PyCFunction)GetUnknownFields, METH_NOARGS, "Parse unknown field set"}, {"WhichOneof", (PyCFunction)WhichOneof, METH_O, "Returns the name of the field set inside a oneof, " @@ -2964,15 +2965,20 @@ bool InitProto2MessageModule(PyObject *m) { return false; } + if (PyType_Ready(&PyUnknownFieldSet_Type) < 0) { + return false; + } + PyModule_AddObject(m, "UnknownFieldSet", - reinterpret_cast<PyObject*>(&PyUnknownFields_Type)); + reinterpret_cast<PyObject*>(&PyUnknownFieldSet_Type)); if (PyType_Ready(&PyUnknownFieldRef_Type) < 0) { return false; } - PyModule_AddObject(m, "UnknownField", - reinterpret_cast<PyObject*>(&PyUnknownFieldRef_Type)); + if (PyType_Ready(&PyUnknownField_Type) < 0) { + return false; + } // Initialize Map container types. if (!InitMapContainers()) { diff --git a/contrib/python/protobuf/py3/google/protobuf/pyext/repeated_composite_container.h b/contrib/python/protobuf/py3/google/protobuf/pyext/repeated_composite_container.h index 30536eabd4..6fa6e176f1 100644 --- a/contrib/python/protobuf/py3/google/protobuf/pyext/repeated_composite_container.h +++ b/contrib/python/protobuf/py3/google/protobuf/pyext/repeated_composite_container.h @@ -37,10 +37,6 @@ #define PY_SSIZE_T_CLEAN #include <Python.h> -#include <memory> -#include <string> -#include <vector> - #include <google/protobuf/pyext/message.h> namespace google { diff --git a/contrib/python/protobuf/py3/google/protobuf/pyext/repeated_scalar_container.h b/contrib/python/protobuf/py3/google/protobuf/pyext/repeated_scalar_container.h index 372b6f0a35..67423ab4ce 100644 --- a/contrib/python/protobuf/py3/google/protobuf/pyext/repeated_scalar_container.h +++ b/contrib/python/protobuf/py3/google/protobuf/pyext/repeated_scalar_container.h @@ -37,8 +37,6 @@ #define PY_SSIZE_T_CLEAN #include <Python.h> -#include <memory> - #include <google/protobuf/descriptor.h> #include <google/protobuf/pyext/message.h> diff --git a/contrib/python/protobuf/py3/google/protobuf/pyext/unknown_field_set.cc b/contrib/python/protobuf/py3/google/protobuf/pyext/unknown_field_set.cc new file mode 100644 index 0000000000..42f9bbcb04 --- /dev/null +++ b/contrib/python/protobuf/py3/google/protobuf/pyext/unknown_field_set.cc @@ -0,0 +1,355 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <google/protobuf/pyext/unknown_field_set.h> + +#define PY_SSIZE_T_CLEAN +#include <Python.h> + +#include <memory> +#include <set> + +#include <google/protobuf/message.h> +#include <google/protobuf/unknown_field_set.h> +#include <google/protobuf/wire_format_lite.h> +#include <google/protobuf/pyext/message.h> +#include <google/protobuf/pyext/scoped_pyobject_ptr.h> + +namespace google { +namespace protobuf { +namespace python { + +namespace unknown_field_set { + +static Py_ssize_t Len(PyObject* pself) { + PyUnknownFieldSet* self = reinterpret_cast<PyUnknownFieldSet*>(pself); + if (self->fields == nullptr) { + PyErr_Format(PyExc_ValueError, "UnknownFieldSet does not exist. "); + return -1; + } + return self->fields->field_count(); +} + +PyObject* NewPyUnknownField(PyUnknownFieldSet* parent, Py_ssize_t index); + +static PyObject* Item(PyObject* pself, Py_ssize_t index) { + PyUnknownFieldSet* self = reinterpret_cast<PyUnknownFieldSet*>(pself); + if (self->fields == nullptr) { + PyErr_Format(PyExc_ValueError, "UnknownFieldSet does not exist. "); + return nullptr; + } + Py_ssize_t total_size = self->fields->field_count(); + if (index < 0) { + index = total_size + index; + } + if (index < 0 || index >= total_size) { + PyErr_Format(PyExc_IndexError, "index (%zd) out of range", index); + return nullptr; + } + return unknown_field_set::NewPyUnknownField(self, index); +} + +PyObject* New(PyTypeObject* type, PyObject* args, PyObject* kwargs) { + if (args == nullptr || PyTuple_Size(args) != 1) { + PyErr_SetString(PyExc_TypeError, + "Must provide a message to create UnknownFieldSet"); + return nullptr; + } + + PyObject* c_message; + if (!PyArg_ParseTuple(args, "O", &c_message)) { + PyErr_SetString(PyExc_TypeError, + "Must provide a message to create UnknownFieldSet"); + return nullptr; + } + + if (!PyObject_TypeCheck(c_message, CMessage_Type)) { + PyErr_Format(PyExc_TypeError, + "Parameter to UnknownFieldSet() must be a message " + "got %s.", + Py_TYPE(c_message)->tp_name); + return nullptr; + } + + PyUnknownFieldSet* self = reinterpret_cast<PyUnknownFieldSet*>( + PyType_GenericAlloc(&PyUnknownFieldSet_Type, 0)); + if (self == nullptr) { + return nullptr; + } + + // Top UnknownFieldSet should set parent nullptr. + self->parent = nullptr; + + // Copy c_message's UnknownFieldSet. + Message* message = reinterpret_cast<CMessage*>(c_message)->message; + const Reflection* reflection = message->GetReflection(); + self->fields = new google::protobuf::UnknownFieldSet; + self->fields->MergeFrom(reflection->GetUnknownFields(*message)); + return reinterpret_cast<PyObject*>(self); +} + +PyObject* NewPyUnknownField(PyUnknownFieldSet* parent, Py_ssize_t index) { + PyUnknownField* self = reinterpret_cast<PyUnknownField*>( + PyType_GenericAlloc(&PyUnknownField_Type, 0)); + if (self == nullptr) { + return nullptr; + } + + Py_INCREF(parent); + self->parent = parent; + self->index = index; + + return reinterpret_cast<PyObject*>(self); +} + +static void Dealloc(PyObject* pself) { + PyUnknownFieldSet* self = reinterpret_cast<PyUnknownFieldSet*>(pself); + if (self->parent == nullptr) { + delete self->fields; + } else { + Py_CLEAR(self->parent); + } + auto* py_type = Py_TYPE(pself); + self->~PyUnknownFieldSet(); + py_type->tp_free(pself); +} + +static PySequenceMethods SqMethods = { + Len, /* sq_length */ + nullptr, /* sq_concat */ + nullptr, /* sq_repeat */ + Item, /* sq_item */ + nullptr, /* sq_slice */ + nullptr, /* sq_ass_item */ +}; + +} // namespace unknown_field_set + +PyTypeObject PyUnknownFieldSet_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) FULL_MODULE_NAME + ".PyUnknownFieldSet", // tp_name + sizeof(PyUnknownFieldSet), // tp_basicsize + 0, // tp_itemsize + unknown_field_set::Dealloc, // tp_dealloc +#if PY_VERSION_HEX < 0x03080000 + nullptr, // tp_print +#else + 0, // tp_vectorcall_offset +#endif + nullptr, // tp_getattr + nullptr, // tp_setattr + nullptr, // tp_compare + nullptr, // tp_repr + nullptr, // tp_as_number + &unknown_field_set::SqMethods, // tp_as_sequence + nullptr, // tp_as_mapping + PyObject_HashNotImplemented, // tp_hash + nullptr, // tp_call + nullptr, // tp_str + nullptr, // tp_getattro + nullptr, // tp_setattro + nullptr, // tp_as_buffer + Py_TPFLAGS_DEFAULT, // tp_flags + "unknown field set", // tp_doc + nullptr, // tp_traverse + nullptr, // tp_clear + nullptr, // tp_richcompare + 0, // tp_weaklistoffset + nullptr, // tp_iter + nullptr, // tp_iternext + nullptr, // tp_methods + nullptr, // tp_members + nullptr, // tp_getset + nullptr, // tp_base + nullptr, // tp_dict + nullptr, // tp_descr_get + nullptr, // tp_descr_set + 0, // tp_dictoffset + nullptr, // tp_init + nullptr, // tp_alloc + unknown_field_set::New, // tp_new +}; + +namespace unknown_field { +static PyObject* PyUnknownFieldSet_FromUnknownFieldSet( + PyUnknownFieldSet* parent, const UnknownFieldSet& fields) { + PyUnknownFieldSet* self = reinterpret_cast<PyUnknownFieldSet*>( + PyType_GenericAlloc(&PyUnknownFieldSet_Type, 0)); + if (self == nullptr) { + return nullptr; + } + + Py_INCREF(parent); + self->parent = parent; + self->fields = const_cast<UnknownFieldSet*>(&fields); + + return reinterpret_cast<PyObject*>(self); +} + +const UnknownField* GetUnknownField(PyUnknownField* self) { + const UnknownFieldSet* fields = self->parent->fields; + if (fields == nullptr) { + PyErr_Format(PyExc_ValueError, "UnknownField does not exist. "); + return nullptr; + } + Py_ssize_t total_size = fields->field_count(); + if (self->index >= total_size) { + PyErr_Format(PyExc_ValueError, "UnknownField does not exist. "); + return nullptr; + } + return &fields->field(self->index); +} + +static PyObject* GetFieldNumber(PyUnknownField* self, void* closure) { + const UnknownField* unknown_field = GetUnknownField(self); + if (unknown_field == nullptr) { + return nullptr; + } + return PyLong_FromLong(unknown_field->number()); +} + +using internal::WireFormatLite; +static PyObject* GetWireType(PyUnknownField* self, void* closure) { + const UnknownField* unknown_field = GetUnknownField(self); + if (unknown_field == nullptr) { + return nullptr; + } + + // Assign a default value to suppress may-uninitialized warnings (errors + // when built in some places). + WireFormatLite::WireType wire_type = WireFormatLite::WIRETYPE_VARINT; + switch (unknown_field->type()) { + case UnknownField::TYPE_VARINT: + wire_type = WireFormatLite::WIRETYPE_VARINT; + break; + case UnknownField::TYPE_FIXED32: + wire_type = WireFormatLite::WIRETYPE_FIXED32; + break; + case UnknownField::TYPE_FIXED64: + wire_type = WireFormatLite::WIRETYPE_FIXED64; + break; + case UnknownField::TYPE_LENGTH_DELIMITED: + wire_type = WireFormatLite::WIRETYPE_LENGTH_DELIMITED; + break; + case UnknownField::TYPE_GROUP: + wire_type = WireFormatLite::WIRETYPE_START_GROUP; + break; + } + return PyLong_FromLong(wire_type); +} + +static PyObject* GetData(PyUnknownField* self, void* closure) { + const UnknownField* field = GetUnknownField(self); + if (field == nullptr) { + return nullptr; + } + PyObject* data = nullptr; + switch (field->type()) { + case UnknownField::TYPE_VARINT: + data = PyLong_FromUnsignedLongLong(field->varint()); + break; + case UnknownField::TYPE_FIXED32: + data = PyLong_FromUnsignedLong(field->fixed32()); + break; + case UnknownField::TYPE_FIXED64: + data = PyLong_FromUnsignedLongLong(field->fixed64()); + break; + case UnknownField::TYPE_LENGTH_DELIMITED: + data = PyBytes_FromStringAndSize(field->length_delimited().data(), + field->GetLengthDelimitedSize()); + break; + case UnknownField::TYPE_GROUP: + data = + PyUnknownFieldSet_FromUnknownFieldSet(self->parent, field->group()); + break; + } + return data; +} + +static void Dealloc(PyObject* pself) { + PyUnknownField* self = reinterpret_cast<PyUnknownField*>(pself); + Py_CLEAR(self->parent); +} + +static PyGetSetDef Getters[] = { + {"field_number", (getter)GetFieldNumber, nullptr}, + {"wire_type", (getter)GetWireType, nullptr}, + {"data", (getter)GetData, nullptr}, + {nullptr}, +}; + +} // namespace unknown_field + +PyTypeObject PyUnknownField_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) FULL_MODULE_NAME + ".PyUnknownField", // tp_name + sizeof(PyUnknownField), // tp_basicsize + 0, // tp_itemsize + unknown_field::Dealloc, // tp_dealloc +#if PY_VERSION_HEX < 0x03080000 + nullptr, // tp_print +#else + 0, // tp_vectorcall_offset +#endif + nullptr, // tp_getattr + nullptr, // tp_setattr + nullptr, // tp_compare + nullptr, // tp_repr + nullptr, // tp_as_number + nullptr, // tp_as_sequence + nullptr, // tp_as_mapping + PyObject_HashNotImplemented, // tp_hash + nullptr, // tp_call + nullptr, // tp_str + nullptr, // tp_getattro + nullptr, // tp_setattro + nullptr, // tp_as_buffer + Py_TPFLAGS_DEFAULT, // tp_flags + "unknown field", // tp_doc + nullptr, // tp_traverse + nullptr, // tp_clear + nullptr, // tp_richcompare + 0, // tp_weaklistoffset + nullptr, // tp_iter + nullptr, // tp_iternext + nullptr, // tp_methods + nullptr, // tp_members + unknown_field::Getters, // tp_getset + nullptr, // tp_base + nullptr, // tp_dict + nullptr, // tp_descr_get + nullptr, // tp_descr_set + 0, // tp_dictoffset + nullptr, // tp_init +}; + +} // namespace python +} // namespace protobuf +} // namespace google diff --git a/contrib/python/protobuf/py3/google/protobuf/pyext/unknown_field_set.h b/contrib/python/protobuf/py3/google/protobuf/pyext/unknown_field_set.h new file mode 100644 index 0000000000..3fa764d01e --- /dev/null +++ b/contrib/python/protobuf/py3/google/protobuf/pyext/unknown_field_set.h @@ -0,0 +1,78 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifndef GOOGLE_PROTOBUF_PYTHON_CPP_UNKNOWN_FIELD_SET_H__ +#define GOOGLE_PROTOBUF_PYTHON_CPP_UNKNOWN_FIELD_SET_H__ + +#define PY_SSIZE_T_CLEAN +#include <Python.h> + +#include <memory> +#include <set> + +#include <google/protobuf/pyext/message.h> + +namespace google { +namespace protobuf { + +class UnknownField; +class UnknownFieldSet; + +namespace python { +struct CMessage; + +struct PyUnknownFieldSet { + PyObject_HEAD; + // If parent is nullptr, it is a top UnknownFieldSet. + PyUnknownFieldSet* parent; + + // Top UnknownFieldSet owns fields pointer. Sub UnknownFieldSet + // does not own fields pointer. + UnknownFieldSet* fields; +}; + +struct PyUnknownField { + PyObject_HEAD; + // Every Python PyUnknownField holds a reference to its parent + // PyUnknownFieldSet in order to keep it alive. + PyUnknownFieldSet* parent; + + // The UnknownField index in UnknownFieldSet. + Py_ssize_t index; +}; + +extern PyTypeObject PyUnknownFieldSet_Type; +extern PyTypeObject PyUnknownField_Type; + +} // namespace python +} // namespace protobuf +} // namespace google + +#endif // GOOGLE_PROTOBUF_PYTHON_CPP_UNKNOWN_FIELD_SET_H__ diff --git a/contrib/python/protobuf/py3/google/protobuf/text_format.py b/contrib/python/protobuf/py3/google/protobuf/text_format.py index 412385c26f..a6d8bcf648 100644 --- a/contrib/python/protobuf/py3/google/protobuf/text_format.py +++ b/contrib/python/protobuf/py3/google/protobuf/text_format.py @@ -53,6 +53,7 @@ from google.protobuf.internal import decoder from google.protobuf.internal import type_checkers from google.protobuf import descriptor from google.protobuf import text_encoding +from google.protobuf import unknown_fields # pylint: disable=g-import-not-at-top __all__ = ['MessageToString', 'Parse', 'PrintMessage', 'PrintField', @@ -136,8 +137,6 @@ def MessageToString( Args: message: The protocol buffers message. as_utf8: Return unescaped Unicode for non-ASCII characters. - In Python 3 actual Unicode characters may appear as is in strings. - In Python 2 the return value will be valid UTF-8 rather than only ASCII. as_one_line: Don't introduce newlines between fields. use_short_repeated_primitives: Use short repeated format for primitives. pointy_brackets: If True, use angle brackets instead of curly braces for @@ -223,6 +222,36 @@ def PrintMessage(message, message_formatter=None, print_unknown_fields=False, force_colon=False): + """Convert the message to text format and write it to the out stream. + + Args: + message: The Message object to convert to text format. + out: A file handle to write the message to. + indent: The initial indent level for pretty print. + as_utf8: Return unescaped Unicode for non-ASCII characters. + as_one_line: Don't introduce newlines between fields. + use_short_repeated_primitives: Use short repeated format for primitives. + pointy_brackets: If True, use angle brackets instead of curly braces for + nesting. + use_index_order: If True, print fields of a proto message using the order + defined in source code instead of the field number. By default, use the + field number order. + float_format: If set, use this to specify float field formatting + (per the "Format Specification Mini-Language"); otherwise, shortest + float that has same value in wire will be printed. Also affect double + field if double_format is not set but float_format is set. + double_format: If set, use this to specify double field formatting + (per the "Format Specification Mini-Language"); if it is not set but + float_format is set, use float_format. Otherwise, str() is used. + use_field_number: If True, print field numbers instead of names. + descriptor_pool: A DescriptorPool used to resolve Any types. + message_formatter: A function(message, indent, as_one_line): unicode|None + to custom format selected sub-messages (usually based on message type). + Use to pretty print parts of the protobuf for easier diffing. + print_unknown_fields: If True, unknown fields will be printed. + force_colon: If set, a colon will be added after the field name even if + the field is a proto message. + """ printer = _Printer( out=out, indent=indent, as_utf8=as_utf8, as_one_line=as_one_line, @@ -347,8 +376,6 @@ class _Printer(object): out: To record the text format result. indent: The initial indent level for pretty print. as_utf8: Return unescaped Unicode for non-ASCII characters. - In Python 3 actual Unicode characters may appear as is in strings. - In Python 2 the return value will be valid UTF-8 rather than ASCII. as_one_line: Don't introduce newlines between fields. use_short_repeated_primitives: Use short repeated format for primitives. pointy_brackets: If True, use angle brackets instead of curly braces for @@ -454,12 +481,12 @@ class _Printer(object): self.PrintField(field, value) if self.print_unknown_fields: - self._PrintUnknownFields(message.UnknownFields()) + self._PrintUnknownFields(unknown_fields.UnknownFieldSet(message)) - def _PrintUnknownFields(self, unknown_fields): + def _PrintUnknownFields(self, unknown_field_set): """Print unknown fields.""" out = self.out - for field in unknown_fields: + for field in unknown_field_set: out.write(' ' * self.indent) out.write(str(field.field_number)) if field.wire_type == WIRETYPE_START_GROUP: @@ -859,7 +886,10 @@ class _Parser(object): expanded_any_end_token = '}' expanded_any_sub_message = _BuildMessageFromTypeName(packed_type_name, self.descriptor_pool) - if not expanded_any_sub_message: + # Direct comparison with None is used instead of implicit bool conversion + # to avoid false positives with falsy initial values, e.g. for + # google.protobuf.ListValue. + if expanded_any_sub_message is None: raise ParseError('Type %s not found in descriptor pool' % packed_type_name) while not tokenizer.TryConsume(expanded_any_end_token): @@ -1143,9 +1173,12 @@ def _SkipFieldContents(tokenizer): # start with "{" or "<" which indicates the beginning of a message body. # If there is no ":" or there is a "{" or "<" after ":", this field has # to be a message or the input is ill-formed. - if tokenizer.TryConsume(':') and not tokenizer.LookingAt( - '{') and not tokenizer.LookingAt('<'): - _SkipFieldValue(tokenizer) + if tokenizer.TryConsume( + ':') and not tokenizer.LookingAt('{') and not tokenizer.LookingAt('<'): + if tokenizer.LookingAt('['): + _SkipRepeatedFieldValue(tokenizer) + else: + _SkipFieldValue(tokenizer) else: _SkipFieldMessage(tokenizer) @@ -1214,6 +1247,20 @@ def _SkipFieldValue(tokenizer): raise ParseError('Invalid field value: ' + tokenizer.token) +def _SkipRepeatedFieldValue(tokenizer): + """Skips over a repeated field value. + + Args: + tokenizer: A tokenizer to parse the field value. + """ + tokenizer.Consume('[') + if not tokenizer.LookingAt(']'): + _SkipFieldValue(tokenizer) + while tokenizer.TryConsume(','): + _SkipFieldValue(tokenizer) + tokenizer.Consume(']') + + class Tokenizer(object): """Protocol buffer text representation tokenizer. diff --git a/contrib/python/protobuf/py3/google/protobuf/unknown_fields.py b/contrib/python/protobuf/py3/google/protobuf/unknown_fields.py new file mode 100644 index 0000000000..3bd828619f --- /dev/null +++ b/contrib/python/protobuf/py3/google/protobuf/unknown_fields.py @@ -0,0 +1,120 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Contains Unknown Fields APIs. + +Simple usage example: + unknown_field_set = UnknownFieldSet(message) + for unknown_field in unknown_field_set: + wire_type = unknown_field.wire_type + field_number = unknown_field.field_number + data = unknown_field.data +""" + + +from google.protobuf.internal import api_implementation + +if api_implementation._c_module is not None: # pylint: disable=protected-access + UnknownFieldSet = api_implementation._c_module.UnknownFieldSet # pylint: disable=protected-access +else: + from google.protobuf.internal import decoder # pylint: disable=g-import-not-at-top + from google.protobuf.internal import wire_format # pylint: disable=g-import-not-at-top + + class UnknownField: + """A parsed unknown field.""" + + # Disallows assignment to other attributes. + __slots__ = ['_field_number', '_wire_type', '_data'] + + def __init__(self, field_number, wire_type, data): + self._field_number = field_number + self._wire_type = wire_type + self._data = data + return + + @property + def field_number(self): + return self._field_number + + @property + def wire_type(self): + return self._wire_type + + @property + def data(self): + return self._data + + class UnknownFieldSet: + """UnknownField container.""" + + # Disallows assignment to other attributes. + __slots__ = ['_values'] + + def __init__(self, msg): + + def InternalAdd(field_number, wire_type, data): + unknown_field = UnknownField(field_number, wire_type, data) + self._values.append(unknown_field) + + self._values = [] + msg_des = msg.DESCRIPTOR + # pylint: disable=protected-access + unknown_fields = msg._unknown_fields + if (msg_des.has_options and + msg_des.GetOptions().message_set_wire_format): + local_decoder = decoder.UnknownMessageSetItemDecoder() + for _, buffer in unknown_fields: + (field_number, data) = local_decoder(memoryview(buffer)) + InternalAdd(field_number, wire_format.WIRETYPE_LENGTH_DELIMITED, data) + else: + for tag_bytes, buffer in unknown_fields: + # pylint: disable=protected-access + (tag, _) = decoder._DecodeVarint(tag_bytes, 0) + field_number, wire_type = wire_format.UnpackTag(tag) + if field_number == 0: + raise RuntimeError('Field number 0 is illegal.') + (data, _) = decoder._DecodeUnknownField( + memoryview(buffer), 0, wire_type) + InternalAdd(field_number, wire_type, data) + + def __getitem__(self, index): + size = len(self._values) + if index < 0: + index += size + if index < 0 or index >= size: + raise IndexError('index %d out of range'.index) + + return self._values[index] + + def __len__(self): + return len(self._values) + + def __iter__(self): + return iter(self._values) diff --git a/contrib/python/protobuf/py3/ya.make b/contrib/python/protobuf/py3/ya.make index 93b23a6f9b..c00e01b812 100644 --- a/contrib/python/protobuf/py3/ya.make +++ b/contrib/python/protobuf/py3/ya.make @@ -9,9 +9,9 @@ LICENSE( LICENSE_TEXTS(.yandex_meta/licenses.list.txt) -VERSION(3.20.3) +VERSION(4.21.7) -ORIGINAL_SOURCE(mirror://pypi/p/protobuf/protobuf-3.20.3.tar.gz) +ORIGINAL_SOURCE(mirror://pypi/p/protobuf/protobuf-4.21.7.tar.gz) PEERDIR( contrib/libs/protobuf @@ -46,6 +46,7 @@ SRCS( google/protobuf/pyext/message_module.cc google/protobuf/pyext/repeated_composite_container.cc google/protobuf/pyext/repeated_scalar_container.cc + google/protobuf/pyext/unknown_field_set.cc google/protobuf/pyext/unknown_fields.cc ) @@ -87,6 +88,7 @@ PY_SRCS( google/protobuf/symbol_database.py google/protobuf/text_encoding.py google/protobuf/text_format.py + google/protobuf/unknown_fields.py google/protobuf/util/__init__.py ) |