Update protobuf to 3.20.2 and pyprotobuf to 3.20.3

Если это pull-request что-то сломал, то:
- если это тест с канонизацией и еще нет pr с переканонизацией, то переканонизируйте пожалуйста сами
- проверьте, что тест не флапает
- приходите в [DEVTOOLSSUPPORT](https://st.yandex-team.ru/createTicket?queue=DEVTOOLSSUPPORT) - там вам обязательно помогут
987be5ed151f827f7f292f32420470b04b71a91d

1 files changed, 99 insertions, 179 deletions

diff --git a/contrib/libs/protoc/src/google/protobuf/compiler/cpp/cpp_file.cc b/contrib/libs/protoc/src/google/protobuf/compiler/cpp/cpp_file.cc
index 2a98ad0b02c..f27d876e2ac 100644
--- a/contrib/libs/protoc/src/google/protobuf/compiler/cpp/cpp_file.cc
+++ b/contrib/libs/protoc/src/google/protobuf/compiler/cpp/cpp_file.cc
@@ -43,16 +43,16 @@
 #include <unordered_set>
 #include <vector>
 
+#include <google/protobuf/compiler/scc.h>
+#include <google/protobuf/io/printer.h>
+#include <google/protobuf/stubs/strutil.h>
 #include <google/protobuf/compiler/cpp/cpp_enum.h>
 #include <google/protobuf/compiler/cpp/cpp_extension.h>
 #include <google/protobuf/compiler/cpp/cpp_field.h>
 #include <google/protobuf/compiler/cpp/cpp_helpers.h>
 #include <google/protobuf/compiler/cpp/cpp_message.h>
 #include <google/protobuf/compiler/cpp/cpp_service.h>
-#include <google/protobuf/compiler/scc.h>
 #include <google/protobuf/descriptor.pb.h>
-#include <google/protobuf/io/printer.h>
-#include <google/protobuf/stubs/strutil.h>
 
 // Must be last.
 #include <google/protobuf/port_def.inc>
@@ -88,6 +88,23 @@ std::vector<const T*> Sorted(const std::unordered_set<const T*>& vals) {
   return sorted;
 }
 
+// TODO(b/203101078): remove pragmas that suppresses uninitialized warnings when
+// clang bug is fixed.
+inline void MuteWuninitialized(Formatter& format) {
+  format(
+      "#if defined(__llvm__)\n"
+      "  #pragma clang diagnostic push\n"
+      "  #pragma clang diagnostic ignored \"-Wuninitialized\"\n"
+      "#endif  // __llvm__\n");
+}
+
+inline void UnmuteWuninitialized(Formatter& format) {
+  format(
+      "#if defined(__llvm__)\n"
+      "  #pragma clang diagnostic pop\n"
+      "#endif  // __llvm__\n");
+}
+
 }  // namespace
 
 FileGenerator::FileGenerator(const FileDescriptor* file, const Options& options)
@@ -347,7 +364,14 @@ void FileGenerator::DoIncludeFile(const TProtoStringType& google3_name,
              options_.runtime_include_base, path);
     }
   } else {
-    format("#include \"$1$\"", google3_name);
+    TProtoStringType path = google3_name;
+    // The bootstrapped proto generated code needs to use the
+    // third_party/protobuf header paths to avoid circular dependencies.
+    if (options_.bootstrap) {
+      path = StringReplace(google3_name, "net/proto2/public",
+                           "third_party/protobuf", false);
+    }
+    format("#include \"$1$\"", path);
   }
 
   if (do_export) {
@@ -444,12 +468,28 @@ void FileGenerator::GenerateSourceIncludes(io::Printer* printer) {
 
   format("// @@protoc_insertion_point(includes)\n");
   IncludeFile("net/proto2/public/port_def.inc", printer);
+}
+
+void FileGenerator::GenerateSourcePrelude(io::Printer* printer) {
+  Formatter format(printer, variables_);
 
   // For MSVC builds, we use #pragma init_seg to move the initialization of our
   // libraries to happen before the user code.
   // This worksaround the fact that MSVC does not do constant initializers when
   // required by the standard.
   format("\nPROTOBUF_PRAGMA_INIT_SEG\n");
+
+  // Generate convenience aliases.
+  format(
+      "\n"
+      "namespace _pb = ::$1$;\n"
+      "namespace _pbi = _pb::internal;\n",
+      ProtobufNamespace(options_));
+  if (HasGeneratedMethods(file_, options_) &&
+      options_.tctable_mode != Options::kTCTableNever) {
+    format("namespace _fl = _pbi::field_layout;\n");
+  }
+  format("\n");
 }
 
 void FileGenerator::GenerateSourceDefaultInstance(int idx,
@@ -462,8 +502,8 @@ void FileGenerator::GenerateSourceDefaultInstance(int idx,
   // destructor that we need to elide.
   format(
       "struct $1$ {\n"
-      "  constexpr $1$()\n"
-      "    : _instance(::$proto_ns$::internal::ConstantInitialized{}) {}\n"
+      "  PROTOBUF_CONSTEXPR $1$()\n"
+      "      : _instance(::_pbi::ConstantInitialized{}) {}\n"
       "  ~$1$() {}\n"
       "  union {\n"
       "    $2$ _instance;\n"
@@ -475,7 +515,8 @@ void FileGenerator::GenerateSourceDefaultInstance(int idx,
   // enough. However, the empty destructor fails to be elided in some
   // configurations (like non-opt or with certain sanitizers). NO_DESTROY is
   // there just to improve performance and binary size in these builds.
-  format("PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT $1$ $2$;\n",
+  format("PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT "
+         "PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 $1$ $2$;\n",
          DefaultInstanceType(generator->descriptor_, options_),
          DefaultInstanceName(generator->descriptor_, options_));
 
@@ -484,19 +525,21 @@ void FileGenerator::GenerateSourceDefaultInstance(int idx,
     if (IsStringInlined(field, options_)) {
       // Force the initialization of the inlined string in the default instance.
       format(
-          "PROTOBUF_ATTRIBUTE_INIT_PRIORITY std::true_type "
+          "PROTOBUF_ATTRIBUTE_INIT_PRIORITY2 std::true_type "
           "$1$::_init_inline_$2$_ = "
-          "($3$._instance.$2$_.Init(), std::true_type{});\n",
+          "($3$._instance.$4$.Init(), std::true_type{});\n",
           ClassName(generator->descriptor_), FieldName(field),
-          DefaultInstanceName(generator->descriptor_, options_));
+          DefaultInstanceName(generator->descriptor_, options_),
+          FieldMemberName(field));
     }
   }
 
   if (options_.lite_implicit_weak_fields) {
-    format("$1$* $2$ = &$3$;\n",
-           DefaultInstanceType(generator->descriptor_, options_),
-           DefaultInstancePtr(generator->descriptor_, options_),
-           DefaultInstanceName(generator->descriptor_, options_));
+    format(
+        "PROTOBUF_CONSTINIT const void* $1$ =\n"
+        "    &$2$;\n",
+        DefaultInstancePtr(generator->descriptor_, options_),
+        DefaultInstanceName(generator->descriptor_, options_));
   }
 }
 
@@ -550,11 +593,10 @@ void FileGenerator::GenerateInternalForwardDeclarations(
     for (auto instance : Sorted(refs.weak_default_instances)) {
       ns.ChangeTo(Namespace(instance, options_));
       if (options_.lite_implicit_weak_fields) {
-        format("extern $1$ $2$;\n", DefaultInstanceType(instance, options_),
-               DefaultInstanceName(instance, options_));
-        format("__attribute__((weak)) $1$* $2$ = nullptr;\n",
-               DefaultInstanceType(instance, options_),
-               DefaultInstancePtr(instance, options_));
+        format(
+            "PROTOBUF_CONSTINIT __attribute__((weak)) const void* $1$ =\n"
+            "    &::_pbi::implicit_weak_message_default_instance;\n",
+            DefaultInstancePtr(instance, options_));
       } else {
         format("extern __attribute__((weak)) $1$ $2$;\n",
                DefaultInstanceType(instance, options_),
@@ -565,8 +607,7 @@ void FileGenerator::GenerateInternalForwardDeclarations(
 
   for (auto file : Sorted(refs.weak_reflection_files)) {
     format(
-        "extern __attribute__((weak)) const "
-        "::$proto_ns$::internal::DescriptorTable $1$;\n",
+        "extern __attribute__((weak)) const ::_pbi::DescriptorTable $1$;\n",
         DescriptorTableName(file, options_));
   }
 }
@@ -574,6 +615,9 @@ void FileGenerator::GenerateInternalForwardDeclarations(
 void FileGenerator::GenerateSourceForMessage(int idx, io::Printer* printer) {
   Formatter format(printer, variables_);
   GenerateSourceIncludes(printer);
+  GenerateSourcePrelude(printer);
+
+  if (IsAnyMessage(file_, options_)) MuteWuninitialized(format);
 
   CrossFileReferences refs;
   ForEachField(message_generators_[idx]->descriptor_,
@@ -602,6 +646,8 @@ void FileGenerator::GenerateSourceForMessage(int idx, io::Printer* printer) {
     message_generators_[idx]->GenerateSourceInProto2Namespace(printer);
   }
 
+  if (IsAnyMessage(file_, options_)) UnmuteWuninitialized(format);
+
   format(
       "\n"
       "// @@protoc_insertion_point(global_scope)\n");
@@ -610,6 +656,7 @@ void FileGenerator::GenerateSourceForMessage(int idx, io::Printer* printer) {
 void FileGenerator::GenerateSourceForExtension(int idx, io::Printer* printer) {
   Formatter format(printer, variables_);
   GenerateSourceIncludes(printer);
+  GenerateSourcePrelude(printer);
   NamespaceOpener ns(Namespace(file_, options_), format);
   extension_generators_[idx]->GenerateDefinition(printer);
 }
@@ -617,10 +664,9 @@ void FileGenerator::GenerateSourceForExtension(int idx, io::Printer* printer) {
 void FileGenerator::GenerateGlobalSource(io::Printer* printer) {
   Formatter format(printer, variables_);
   GenerateSourceIncludes(printer);
+  GenerateSourcePrelude(printer);
 
   {
-    GenerateTables(printer);
-
     // Define the code to initialize reflection. This code uses a global
     // constructor to register reflection data with the runtime pre-main.
     if (HasDescriptorMethods(file_, options_)) {
@@ -639,10 +685,13 @@ void FileGenerator::GenerateGlobalSource(io::Printer* printer) {
 void FileGenerator::GenerateSource(io::Printer* printer) {
   Formatter format(printer, variables_);
   GenerateSourceIncludes(printer);
+  GenerateSourcePrelude(printer);
   CrossFileReferences refs;
   GetCrossFileReferencesForFile(file_, &refs);
   GenerateInternalForwardDeclarations(refs, printer);
 
+  if (IsAnyMessage(file_, options_)) MuteWuninitialized(format);
+
   {
     NamespaceOpener ns(Namespace(file_, options_), format);
 
@@ -653,8 +702,6 @@ void FileGenerator::GenerateSource(io::Printer* printer) {
   }
 
   {
-    GenerateTables(printer);
-
     if (HasDescriptorMethods(file_, options_)) {
       // Define the code to initialize reflection. This code uses a global
       // constructor to register reflection data with the runtime pre-main.
@@ -711,6 +758,8 @@ void FileGenerator::GenerateSource(io::Printer* printer) {
       "\n"
       "// @@protoc_insertion_point(global_scope)\n");
 
+  if (IsAnyMessage(file_, options_)) UnmuteWuninitialized(format);
+
   IncludeFile("net/proto2/public/port_undef.inc", printer);
 }
 
@@ -718,31 +767,30 @@ void FileGenerator::GenerateReflectionInitializationCode(io::Printer* printer) {
   Formatter format(printer, variables_);
 
   if (!message_generators_.empty()) {
-    format("static ::$proto_ns$::Metadata $file_level_metadata$[$1$];\n",
+    format("static ::_pb::Metadata $file_level_metadata$[$1$];\n",
            message_generators_.size());
   }
   if (!enum_generators_.empty()) {
     format(
-        "static "
-        "const ::$proto_ns$::EnumDescriptor* "
+        "static const ::_pb::EnumDescriptor* "
         "$file_level_enum_descriptors$[$1$];\n",
         enum_generators_.size());
   } else {
     format(
         "static "
-        "constexpr ::$proto_ns$::EnumDescriptor const** "
+        "constexpr ::_pb::EnumDescriptor const** "
         "$file_level_enum_descriptors$ = nullptr;\n");
   }
   if (HasGenericServices(file_, options_) && file_->service_count() > 0) {
     format(
         "static "
-        "const ::$proto_ns$::ServiceDescriptor* "
+        "const ::_pb::ServiceDescriptor* "
         "$file_level_service_descriptors$[$1$];\n",
         file_->service_count());
   } else {
     format(
         "static "
-        "constexpr ::$proto_ns$::ServiceDescriptor const** "
+        "constexpr ::_pb::ServiceDescriptor const** "
         "$file_level_service_descriptors$ = nullptr;\n");
   }
 
@@ -760,7 +808,7 @@ void FileGenerator::GenerateReflectionInitializationCode(io::Printer* printer) {
     format.Outdent();
     format(
         "};\n"
-        "static const ::$proto_ns$::internal::MigrationSchema schemas[] "
+        "static const ::_pbi::MigrationSchema schemas[] "
         "PROTOBUF_SECTION_VARIABLE(protodesc_cold) = {\n");
     format.Indent();
     {
@@ -774,16 +822,13 @@ void FileGenerator::GenerateReflectionInitializationCode(io::Printer* printer) {
     format.Outdent();
     format(
         "};\n"
-        "\nstatic "
-        "::$proto_ns$::Message const * const file_default_instances[] = {\n");
+        "\nstatic const ::_pb::Message* const file_default_instances[] = {\n");
     format.Indent();
     for (int i = 0; i < message_generators_.size(); i++) {
       const Descriptor* descriptor = message_generators_[i]->descriptor_;
-      format(
-          "reinterpret_cast<const "
-          "::$proto_ns$::Message*>(&$1$::_$2$_default_instance_),\n",
-          Namespace(descriptor, options_),  // 1
-          ClassName(descriptor));           // 2
+      format("&$1$::_$2$_default_instance_._instance,\n",
+             Namespace(descriptor, options_),  // 1
+             ClassName(descriptor));           // 2
     }
     format.Outdent();
     format(
@@ -794,10 +839,8 @@ void FileGenerator::GenerateReflectionInitializationCode(io::Printer* printer) {
     format(
         // MSVC doesn't like empty arrays, so we add a dummy.
         "const $uint32$ $tablename$::offsets[1] = {};\n"
-        "static constexpr ::$proto_ns$::internal::MigrationSchema* schemas = "
-        "nullptr;"
-        "\n"
-        "static constexpr ::$proto_ns$::Message* const* "
+        "static constexpr ::_pbi::MigrationSchema* schemas = nullptr;\n"
+        "static constexpr ::_pb::Message* const* "
         "file_default_instances = nullptr;\n"
         "\n");
   }
@@ -852,7 +895,7 @@ void FileGenerator::GenerateReflectionInitializationCode(io::Printer* printer) {
   // Build array of DescriptorTable deps.
   if (num_deps > 0) {
     format(
-        "static const ::$proto_ns$::internal::DescriptorTable*const "
+        "static const ::_pbi::DescriptorTable* const "
         "$desc_table$_deps[$1$] = {\n",
         num_deps);
 
@@ -872,13 +915,14 @@ void FileGenerator::GenerateReflectionInitializationCode(io::Printer* printer) {
   // so disable for now.
   bool eager = false;
   format(
-      "static ::$proto_ns$::internal::once_flag $desc_table$_once;\n"
-      "const ::$proto_ns$::internal::DescriptorTable $desc_table$ = {\n"
-      "  false, $1$, $2$, $3$, \"$filename$\", \n"
-      "  &$desc_table$_once, $4$, $5$, $6$,\n"
-      "  schemas, file_default_instances, $tablename$::offsets,\n"
-      "  $7$, $file_level_enum_descriptors$, "
-      "$file_level_service_descriptors$,\n"
+      "static ::_pbi::once_flag $desc_table$_once;\n"
+      "const ::_pbi::DescriptorTable $desc_table$ = {\n"
+      "    false, $1$, $2$, $3$,\n"
+      "    \"$filename$\",\n"
+      "    &$desc_table$_once, $4$, $5$, $6$,\n"
+      "    schemas, file_default_instances, $tablename$::offsets,\n"
+      "    $7$, $file_level_enum_descriptors$,\n"
+      "    $file_level_service_descriptors$,\n"
       "};\n"
       // This function exists to be marked as weak.
       // It can significantly speed up compilation by breaking up LLVM's SCC in
@@ -891,7 +935,7 @@ void FileGenerator::GenerateReflectionInitializationCode(io::Printer* printer) {
       //   vtables -> GetMetadata
       // By adding a weak function here we break the connection from the
       // individual vtables back into the descriptor table.
-      "PROTOBUF_ATTRIBUTE_WEAK const ::$proto_ns$::internal::DescriptorTable* "
+      "PROTOBUF_ATTRIBUTE_WEAK const ::_pbi::DescriptorTable* "
       "$desc_table$_getter() {\n"
       "  return &$desc_table$;\n"
       "}\n"
@@ -909,123 +953,12 @@ void FileGenerator::GenerateReflectionInitializationCode(io::Printer* printer) {
   if (file_->name() != "net/proto2/proto/descriptor.proto") {
     format(
         "// Force running AddDescriptors() at dynamic initialization time.\n"
-        "PROTOBUF_ATTRIBUTE_INIT_PRIORITY "
-        "static ::$proto_ns$::internal::AddDescriptorsRunner "
-        "$1$(&$desc_table$);\n",
+        "PROTOBUF_ATTRIBUTE_INIT_PRIORITY2 "
+        "static ::_pbi::AddDescriptorsRunner $1$(&$desc_table$);\n",
         UniqueName("dynamic_init_dummy", file_, options_));
   }
 }
 
-void FileGenerator::GenerateTables(io::Printer* printer) {
-  Formatter format(printer, variables_);
-  if (options_.table_driven_parsing) {
-    // TODO(ckennelly): Gate this with the same options flag to enable
-    // table-driven parsing.
-    format(
-        "PROTOBUF_CONSTEXPR_VAR ::$proto_ns$::internal::ParseTableField\n"
-        "    const $tablename$::entries[] "
-        "PROTOBUF_SECTION_VARIABLE(protodesc_cold) = {\n");
-    format.Indent();
-
-    std::vector<size_t> entries;
-    size_t count = 0;
-    for (int i = 0; i < message_generators_.size(); i++) {
-      size_t value = message_generators_[i]->GenerateParseOffsets(printer);
-      entries.push_back(value);
-      count += value;
-    }
-
-    // We need these arrays to exist, and MSVC does not like empty arrays.
-    if (count == 0) {
-      format("{0, 0, 0, ::$proto_ns$::internal::kInvalidMask, 0, 0},\n");
-    }
-
-    format.Outdent();
-    format(
-        "};\n"
-        "\n"
-        "PROTOBUF_CONSTEXPR_VAR "
-        "::$proto_ns$::internal::AuxiliaryParseTableField\n"
-        "    const $tablename$::aux[] "
-        "PROTOBUF_SECTION_VARIABLE(protodesc_cold) = {\n");
-    format.Indent();
-
-    std::vector<size_t> aux_entries;
-    count = 0;
-    for (int i = 0; i < message_generators_.size(); i++) {
-      size_t value = message_generators_[i]->GenerateParseAuxTable(printer);
-      aux_entries.push_back(value);
-      count += value;
-    }
-
-    if (count == 0) {
-      format("::$proto_ns$::internal::AuxiliaryParseTableField(),\n");
-    }
-
-    format.Outdent();
-    format(
-        "};\n"
-        "PROTOBUF_CONSTEXPR_VAR ::$proto_ns$::internal::ParseTable const\n"
-        "    $tablename$::schema[] "
-        "PROTOBUF_SECTION_VARIABLE(protodesc_cold) = {\n");
-    format.Indent();
-
-    size_t offset = 0;
-    size_t aux_offset = 0;
-    for (int i = 0; i < message_generators_.size(); i++) {
-      message_generators_[i]->GenerateParseTable(printer, offset, aux_offset);
-      offset += entries[i];
-      aux_offset += aux_entries[i];
-    }
-
-    if (message_generators_.empty()) {
-      format("{ nullptr, nullptr, 0, -1, -1, false },\n");
-    }
-
-    format.Outdent();
-    format(
-        "};\n"
-        "\n");
-  }
-
-  if (!message_generators_.empty() && options_.table_driven_serialization) {
-    format(
-        "const ::$proto_ns$::internal::FieldMetadata "
-        "$tablename$::field_metadata[] "
-        "= {\n");
-    format.Indent();
-    std::vector<int> field_metadata_offsets;
-    int idx = 0;
-    for (int i = 0; i < message_generators_.size(); i++) {
-      field_metadata_offsets.push_back(idx);
-      idx += message_generators_[i]->GenerateFieldMetadata(printer);
-    }
-    field_metadata_offsets.push_back(idx);
-    format.Outdent();
-    format(
-        "};\n"
-        "const ::$proto_ns$::internal::SerializationTable "
-        "$tablename$::serialization_table[] = {\n");
-    format.Indent();
-    // We rely on the order we layout the tables to match the order we
-    // calculate them with FlattenMessagesInFile, so we check here that
-    // these match exactly.
-    std::vector<const Descriptor*> calculated_order =
-        FlattenMessagesInFile(file_);
-    GOOGLE_CHECK_EQ(calculated_order.size(), message_generators_.size());
-    for (int i = 0; i < message_generators_.size(); i++) {
-      GOOGLE_CHECK_EQ(calculated_order[i], message_generators_[i]->descriptor_);
-      format("{$1$, $tablename$::field_metadata + $2$},\n",
-             field_metadata_offsets[i + 1] - field_metadata_offsets[i],  // 1
-             field_metadata_offsets[i]);                                 // 2
-    }
-    format.Outdent();
-    format(
-        "};\n"
-        "\n");
-  }
-}
-
 class FileGenerator::ForwardDeclarations {
  public:
   void AddMessage(const Descriptor* d) { classes_[ClassName(d)] = d; }
@@ -1229,7 +1162,6 @@ void FileGenerator::GenerateLibraryIncludes(io::Printer* printer) {
   if (HasSimpleBaseClasses(file_, options_)) {
     IncludeFile("net/proto2/public/generated_message_bases.h", printer);
   }
-  IncludeFile("net/proto2/public/generated_message_table_driven.h", printer);
   if (HasGeneratedMethods(file_, options_) &&
       options_.tctable_mode != Options::kTCTableNever) {
     IncludeFile("net/proto2/public/generated_message_tctable_decl.h", printer);
@@ -1362,20 +1294,8 @@ void FileGenerator::GenerateGlobalStateFunctionDeclarations(
       "\n"
       "// Internal implementation detail -- do not use these members.\n"
       "struct $dllexport_decl $$tablename$ {\n"
-      // These tables describe how to serialize and parse messages. Used
-      // for table driven code.
-      "  static const ::$proto_ns$::internal::ParseTableField entries[]\n"
-      "    PROTOBUF_SECTION_VARIABLE(protodesc_cold);\n"
-      "  static const ::$proto_ns$::internal::AuxiliaryParseTableField aux[]\n"
-      "    PROTOBUF_SECTION_VARIABLE(protodesc_cold);\n"
-      "  static const ::$proto_ns$::internal::ParseTable schema[$1$]\n"
-      "    PROTOBUF_SECTION_VARIABLE(protodesc_cold);\n"
-      "  static const ::$proto_ns$::internal::FieldMetadata field_metadata[];\n"
-      "  static const ::$proto_ns$::internal::SerializationTable "
-      "serialization_table[];\n"
       "  static const $uint32$ offsets[];\n"
-      "};\n",
-      std::max(size_t(1), message_generators_.size()));
+      "};\n");
   if (HasDescriptorMethods(file_, options_)) {
     format(
         "$dllexport_decl $extern const ::$proto_ns$::internal::DescriptorTable "