aboutsummaryrefslogtreecommitdiffstats
path: root/library/cpp/protobuf
diff options
context:
space:
mode:
authorDaniil Cherednik <dcherednik@ydb.tech>2023-10-23 20:34:16 +0000
committerDaniil Cherednik <dcherednik@ydb.tech>2023-10-23 20:34:16 +0000
commite84c813452e9ed62415b2d17a117008fce909a3d (patch)
tree998632d6f08419de5b644940799f3be9408ab46b /library/cpp/protobuf
parentdf6e99640a1489cde9b7cc5b58a3747c6ec28921 (diff)
downloadydb-stable-23-3.tar.gz
Intermediate changesstable-23-3
x-stable-origin-commit: 8b96eef194d7fb3b315816b97322e8dd90bf3d94
Diffstat (limited to 'library/cpp/protobuf')
-rw-r--r--library/cpp/protobuf/json/json2proto.cpp46
-rw-r--r--library/cpp/protobuf/json/json2proto.h10
-rw-r--r--library/cpp/protobuf/json/unknown_fields_collector.h29
-rw-r--r--library/cpp/protobuf/json/ut/unknown_fields_collector_ut.cpp165
-rw-r--r--library/cpp/protobuf/json/ut/ya.make1
5 files changed, 249 insertions, 2 deletions
diff --git a/library/cpp/protobuf/json/json2proto.cpp b/library/cpp/protobuf/json/json2proto.cpp
index be8682e239..bb6cc66de3 100644
--- a/library/cpp/protobuf/json/json2proto.cpp
+++ b/library/cpp/protobuf/json/json2proto.cpp
@@ -258,6 +258,10 @@ Json2SingleField(const NJson::TJsonValue& json,
const NJson::TJsonValue& fieldJson = name ? json[name] : json;
+ if (name && config.UnknownFieldsCollector) {
+ config.UnknownFieldsCollector->OnEnterMapItem(name);
+ }
+
switch (field.cpp_type()) {
JSON_TO_FIELD(CPPTYPE_INT32, field.name(), fieldJson, IsInteger, SetInt32, GetInteger);
JSON_TO_FIELD(CPPTYPE_INT64, field.name(), fieldJson, IsInteger, SetInt64, GetInteger);
@@ -295,6 +299,10 @@ Json2SingleField(const NJson::TJsonValue& json,
ythrow yexception() << "Unknown protobuf field type: "
<< static_cast<int>(field.cpp_type()) << ".";
}
+
+ if (name && config.UnknownFieldsCollector) {
+ config.UnknownFieldsCollector->OnLeaveMapItem();
+ }
}
static void
@@ -397,6 +405,10 @@ Json2RepeatedField(const NJson::TJsonValue& json,
if (fieldJson.GetType() == NJson::JSON_UNDEFINED || fieldJson.GetType() == NJson::JSON_NULL)
return;
+ if (config.UnknownFieldsCollector) {
+ config.UnknownFieldsCollector->OnEnterMapItem(name);
+ }
+
bool isMap = fieldJson.GetType() == NJson::JSON_MAP;
if (isMap) {
if (!config.MapAsObject) {
@@ -421,7 +433,13 @@ Json2RepeatedField(const NJson::TJsonValue& json,
for (const auto& x : jsonMap) {
const TString& key = x.first;
const NJson::TJsonValue& jsonValue = x.second;
+ if (config.UnknownFieldsCollector) {
+ config.UnknownFieldsCollector->OnEnterMapItem(key);
+ }
Json2RepeatedFieldValue(jsonValue, proto, field, config, reflection, key);
+ if (config.UnknownFieldsCollector) {
+ config.UnknownFieldsCollector->OnLeaveMapItem();
+ }
}
} else {
if (config.ReplaceRepeatedFields) {
@@ -429,17 +447,37 @@ Json2RepeatedField(const NJson::TJsonValue& json,
}
if (fieldJson.GetType() == NJson::JSON_ARRAY) {
const NJson::TJsonValue::TArray& jsonArray = fieldJson.GetArray();
+ ui64 id = 0;
for (const NJson::TJsonValue& jsonValue : jsonArray) {
+ if (config.UnknownFieldsCollector) {
+ config.UnknownFieldsCollector->OnEnterArrayItem(id);
+ }
Json2RepeatedFieldValue(jsonValue, proto, field, config, reflection);
+ if (config.UnknownFieldsCollector) {
+ config.UnknownFieldsCollector->OnLeaveArrayItem();
+ }
+ ++id;
}
} else if (config.ValueVectorizer) {
+ ui64 id = 0;
for (const NJson::TJsonValue& jsonValue : config.ValueVectorizer(fieldJson)) {
+ if (config.UnknownFieldsCollector) {
+ config.UnknownFieldsCollector->OnEnterArrayItem(id);
+ }
Json2RepeatedFieldValue(jsonValue, proto, field, config, reflection);
+ if (config.UnknownFieldsCollector) {
+ config.UnknownFieldsCollector->OnLeaveArrayItem();
+ }
+ ++id;
}
} else if (config.VectorizeScalars) {
Json2RepeatedFieldValue(fieldJson, proto, field, config, reflection);
}
}
+
+ if (config.UnknownFieldsCollector) {
+ config.UnknownFieldsCollector->OnLeaveMapItem();
+ }
}
namespace NProtobufJson {
@@ -463,14 +501,18 @@ namespace NProtobufJson {
}
}
- if (!config.AllowUnknownFields) {
+ if (!config.AllowUnknownFields || config.UnknownFieldsCollector) {
THashMap<TString, bool> knownFields;
for (int f = 0, endF = descriptor->field_count(); f < endF; ++f) {
const google::protobuf::FieldDescriptor* field = descriptor->field(f);
knownFields[GetFieldName(*field, config)] = 1;
}
for (const auto& f : json.GetMap()) {
- Y_ENSURE(knownFields.contains(f.first), "unknown field \"" << f.first << "\" for \"" << descriptor->full_name() << "\"");
+ const bool isFieldKnown = knownFields.contains(f.first);
+ Y_ENSURE(config.AllowUnknownFields || isFieldKnown, "unknown field \"" << f.first << "\" for \"" << descriptor->full_name() << "\"");
+ if (!isFieldKnown) {
+ config.UnknownFieldsCollector->OnUnknownField(f.first, *descriptor);
+ }
}
}
}
diff --git a/library/cpp/protobuf/json/json2proto.h b/library/cpp/protobuf/json/json2proto.h
index d576fabce7..5952e28c3d 100644
--- a/library/cpp/protobuf/json/json2proto.h
+++ b/library/cpp/protobuf/json/json2proto.h
@@ -2,10 +2,12 @@
#include "string_transform.h"
#include "name_generator.h"
+#include "unknown_fields_collector.h"
#include <library/cpp/json/json_reader.h>
#include <library/cpp/json/json_value.h>
+#include <util/generic/ptr.h>
#include <util/stream/input.h>
#include <util/stream/str.h>
#include <util/stream/mem.h>
@@ -108,6 +110,11 @@ namespace NProtobufJson {
return *this;
}
+ TSelf& SetUnknownFieldsCollector(TSimpleSharedPtr<IUnknownFieldsCollector> value) {
+ UnknownFieldsCollector = std::move(value);
+ return *this;
+ }
+
FldNameMode FieldNameMode = FieldNameOriginalCase;
bool AllowUnknownFields = true;
@@ -152,6 +159,9 @@ namespace NProtobufJson {
/// Allow nonstandard conversions, e.g. google.protobuf.Duration from String
bool AllowString2TimeConversion = false;
+
+ /// Stores information about unknown fields
+ TSimpleSharedPtr<IUnknownFieldsCollector> UnknownFieldsCollector = nullptr;
};
/// @throw yexception
diff --git a/library/cpp/protobuf/json/unknown_fields_collector.h b/library/cpp/protobuf/json/unknown_fields_collector.h
new file mode 100644
index 0000000000..1e71f2164c
--- /dev/null
+++ b/library/cpp/protobuf/json/unknown_fields_collector.h
@@ -0,0 +1,29 @@
+#pragma once
+
+#include <util/generic/string.h>
+
+namespace google {
+ namespace protobuf {
+ class FieldDescriptor;
+ class Descriptor;
+ }
+}
+
+namespace NProtobufJson {
+ /* Methods OnEnter.../OnLeave... are called on every field of structure
+ * during traverse and should be used to build context
+ * Method OnUnknownField are called every time when field which can't
+ * be mapped
+ */
+ struct IUnknownFieldsCollector {
+ virtual ~IUnknownFieldsCollector() = default;
+
+ virtual void OnEnterMapItem(const TString& key) = 0;
+ virtual void OnLeaveMapItem() = 0;
+
+ virtual void OnEnterArrayItem(ui64 id) = 0;
+ virtual void OnLeaveArrayItem() = 0;
+
+ virtual void OnUnknownField(const TString& key, const google::protobuf::Descriptor& value) = 0;
+ };
+}
diff --git a/library/cpp/protobuf/json/ut/unknown_fields_collector_ut.cpp b/library/cpp/protobuf/json/ut/unknown_fields_collector_ut.cpp
new file mode 100644
index 0000000000..00a2152a68
--- /dev/null
+++ b/library/cpp/protobuf/json/ut/unknown_fields_collector_ut.cpp
@@ -0,0 +1,165 @@
+#include "json.h"
+#include "proto.h"
+#include "proto2json.h"
+
+#include <library/cpp/protobuf/json/json2proto.h>
+
+#include <library/cpp/testing/unittest/registar.h>
+
+#include <util/generic/set.h>
+#include <util/generic/string.h>
+
+using namespace NProtobufJson;
+using namespace NProtobufJsonTest;
+
+struct TTestUnknownFieldsCollector : public IUnknownFieldsCollector {
+ void OnEnterMapItem(const TString& key) override {
+ CurrentPath.push_back(key);
+ }
+
+ void OnEnterArrayItem(ui64 id) override {
+ CurrentPath.push_back(ToString(id));
+ }
+
+ void OnLeaveMapItem() override {
+ CurrentPath.pop_back();
+ }
+
+ void OnLeaveArrayItem() override {
+ CurrentPath.pop_back();
+ }
+
+ void OnUnknownField(const TString& key, const google::protobuf::Descriptor& value) override {
+ TString path;
+ for (auto& piece : CurrentPath) {
+ path.append("/");
+ path.append(piece);
+ }
+ path.append("/");
+ path.append(key);
+ UnknownKeys.insert(std::move(path));
+ Y_UNUSED(value);
+ }
+
+ TVector<TString> CurrentPath;
+ TSet<TString> UnknownKeys;
+};
+
+Y_UNIT_TEST_SUITE(TUnknownFieldsCollectorTest) {
+ Y_UNIT_TEST(TestFlatOptional) {
+ TFlatOptional proto;
+ TSimpleSharedPtr<TTestUnknownFieldsCollector> collector = new TTestUnknownFieldsCollector;
+ TJson2ProtoConfig cfg;
+ cfg.SetUnknownFieldsCollector(collector).SetAllowUnknownFields(true);
+
+ Json2Proto(TStringBuf(R"({"42":42,"I32":11,"test":2,"string":"str","String":"string","obj":{"inner":{}},"arr":[1,2,3]})"), proto, cfg);
+ TSet<TString> expectedKeys = {
+ {"/42"},
+ {"/arr"},
+ {"/obj"},
+ {"/string"},
+ {"/test"},
+ };
+ UNIT_ASSERT(collector->CurrentPath.empty());
+ UNIT_ASSERT_VALUES_EQUAL(collector->UnknownKeys, expectedKeys);
+ }
+
+ Y_UNIT_TEST(TestFlatRepeated) {
+ TFlatRepeated proto;
+ TSimpleSharedPtr<TTestUnknownFieldsCollector> collector = new TTestUnknownFieldsCollector;
+ TJson2ProtoConfig cfg;
+ cfg.SetUnknownFieldsCollector(collector).SetAllowUnknownFields(true);
+
+ Json2Proto(TStringBuf(R"({"42":42,"I32":[11,12],"test":12,"string":"str","String":["string1","string2"],"obj":{"inner":{}},"arr":[1,2,3]})"), proto, cfg);
+ TSet<TString> expectedKeys = {
+ {"/42"},
+ {"/arr"},
+ {"/obj"},
+ {"/string"},
+ {"/test"},
+ };
+ UNIT_ASSERT(collector->CurrentPath.empty());
+ UNIT_ASSERT_VALUES_EQUAL(collector->UnknownKeys, expectedKeys);
+ }
+
+ Y_UNIT_TEST(TestCompositeOptional) {
+ TCompositeOptional proto;
+ TSimpleSharedPtr<TTestUnknownFieldsCollector> collector = new TTestUnknownFieldsCollector;
+ TJson2ProtoConfig cfg;
+ cfg.SetUnknownFieldsCollector(collector).SetAllowUnknownFields(true);
+
+ Json2Proto(TStringBuf(R"({"Part":{"42":42,"I32":11,"test":12,"string":"str","String":"string"},"string2":"str"})"), proto, cfg);
+ TSet<TString> expectedKeys = {
+ {"/Part/42"},
+ {"/Part/string"},
+ {"/Part/test"},
+ {"/string2"},
+ };
+ UNIT_ASSERT(collector->CurrentPath.empty());
+ UNIT_ASSERT_VALUES_EQUAL(collector->UnknownKeys, expectedKeys);
+ }
+
+ Y_UNIT_TEST(TestCompositeRepeated) {
+ TCompositeRepeated proto;
+ TSimpleSharedPtr<TTestUnknownFieldsCollector> collector = new TTestUnknownFieldsCollector;
+ TJson2ProtoConfig cfg;
+ cfg.SetUnknownFieldsCollector(collector).SetAllowUnknownFields(true);
+
+ Json2Proto(TStringBuf(R"({"Part":[)"
+ R"( {"42":42,"I32":11,"test":12,"string":"str","String":"string"},)"
+ R"( {"abc":"d"})"
+ R"(],)"
+ R"("string2":"str"})"), proto, cfg);
+ TSet<TString> expectedKeys = {
+ {"/Part/0/42"},
+ {"/Part/0/string"},
+ {"/Part/0/test"},
+ {"/Part/1/abc"},
+ {"/string2"},
+ };
+ UNIT_ASSERT(collector->CurrentPath.empty());
+ UNIT_ASSERT_VALUES_EQUAL(collector->UnknownKeys, expectedKeys);
+ }
+
+ Y_UNIT_TEST(TestCompleMapType) {
+ TComplexMapType proto;
+ TSimpleSharedPtr<TTestUnknownFieldsCollector> collector = new TTestUnknownFieldsCollector;
+ TJson2ProtoConfig cfg;
+ cfg.SetUnknownFieldsCollector(collector).SetAllowUnknownFields(true);
+
+ Json2Proto(TStringBuf(R"({"42":42,)"
+ R"("Nested":[)"
+ R"( {"key":"abc","value":{"string":"string","Nested":[{"key":"def","value":{"string2":"string2"}}]}},)"
+ R"( {"key":"car","value":{"string3":"string3"}})"
+ R"(]})"), proto, cfg);
+ TSet<TString> expectedKeys = {
+ {"/42"},
+ {"/Nested/0/value/Nested/0/value/string2"},
+ {"/Nested/0/value/string"},
+ {"/Nested/1/value/string3"},
+ };
+ UNIT_ASSERT(collector->CurrentPath.empty());
+ UNIT_ASSERT_VALUES_EQUAL(collector->UnknownKeys, expectedKeys);
+ }
+
+ Y_UNIT_TEST(TestCompleMapTypeMapAsObject) {
+ TComplexMapType proto;
+ TSimpleSharedPtr<TTestUnknownFieldsCollector> collector = new TTestUnknownFieldsCollector;
+ TJson2ProtoConfig cfg;
+ cfg.SetUnknownFieldsCollector(collector).SetAllowUnknownFields(true).SetMapAsObject(true);
+
+ Json2Proto(TStringBuf(R"({"42":42,)"
+ R"("Nested":{)"
+ R"( "abc":{"string":"string","Nested":{"def":{"string2":"string2"}}},)"
+ R"( "car":{"string3":"string3"})"
+ R"(}})"), proto, cfg);
+ TSet<TString> expectedKeys = {
+ {"/42"},
+ {"/Nested/abc/Nested/def/string2"},
+ {"/Nested/abc/string"},
+ {"/Nested/car/string3"},
+ };
+ UNIT_ASSERT(collector->CurrentPath.empty());
+ UNIT_ASSERT_VALUES_EQUAL(collector->UnknownKeys, expectedKeys);
+ }
+} // TJson2ProtoTest
diff --git a/library/cpp/protobuf/json/ut/ya.make b/library/cpp/protobuf/json/ut/ya.make
index 2a5391e375..11690ceb46 100644
--- a/library/cpp/protobuf/json/ut/ya.make
+++ b/library/cpp/protobuf/json/ut/ya.make
@@ -10,6 +10,7 @@ SRCS(
filter_ut.proto
test.proto
util_ut.cpp
+ unknown_fields_collector_ut.cpp
)
GENERATE_ENUM_SERIALIZATION(test.pb.h)