diff options
author | Daniil Cherednik <dcherednik@ydb.tech> | 2023-10-23 20:34:16 +0000 |
---|---|---|
committer | Daniil Cherednik <dcherednik@ydb.tech> | 2023-10-23 20:34:16 +0000 |
commit | e84c813452e9ed62415b2d17a117008fce909a3d (patch) | |
tree | 998632d6f08419de5b644940799f3be9408ab46b /library/cpp/protobuf | |
parent | df6e99640a1489cde9b7cc5b58a3747c6ec28921 (diff) | |
download | ydb-stable-23-3.tar.gz |
Intermediate changesstable-23-3
x-stable-origin-commit: 8b96eef194d7fb3b315816b97322e8dd90bf3d94
Diffstat (limited to 'library/cpp/protobuf')
-rw-r--r-- | library/cpp/protobuf/json/json2proto.cpp | 46 | ||||
-rw-r--r-- | library/cpp/protobuf/json/json2proto.h | 10 | ||||
-rw-r--r-- | library/cpp/protobuf/json/unknown_fields_collector.h | 29 | ||||
-rw-r--r-- | library/cpp/protobuf/json/ut/unknown_fields_collector_ut.cpp | 165 | ||||
-rw-r--r-- | library/cpp/protobuf/json/ut/ya.make | 1 |
5 files changed, 249 insertions, 2 deletions
diff --git a/library/cpp/protobuf/json/json2proto.cpp b/library/cpp/protobuf/json/json2proto.cpp index be8682e239..bb6cc66de3 100644 --- a/library/cpp/protobuf/json/json2proto.cpp +++ b/library/cpp/protobuf/json/json2proto.cpp @@ -258,6 +258,10 @@ Json2SingleField(const NJson::TJsonValue& json, const NJson::TJsonValue& fieldJson = name ? json[name] : json; + if (name && config.UnknownFieldsCollector) { + config.UnknownFieldsCollector->OnEnterMapItem(name); + } + switch (field.cpp_type()) { JSON_TO_FIELD(CPPTYPE_INT32, field.name(), fieldJson, IsInteger, SetInt32, GetInteger); JSON_TO_FIELD(CPPTYPE_INT64, field.name(), fieldJson, IsInteger, SetInt64, GetInteger); @@ -295,6 +299,10 @@ Json2SingleField(const NJson::TJsonValue& json, ythrow yexception() << "Unknown protobuf field type: " << static_cast<int>(field.cpp_type()) << "."; } + + if (name && config.UnknownFieldsCollector) { + config.UnknownFieldsCollector->OnLeaveMapItem(); + } } static void @@ -397,6 +405,10 @@ Json2RepeatedField(const NJson::TJsonValue& json, if (fieldJson.GetType() == NJson::JSON_UNDEFINED || fieldJson.GetType() == NJson::JSON_NULL) return; + if (config.UnknownFieldsCollector) { + config.UnknownFieldsCollector->OnEnterMapItem(name); + } + bool isMap = fieldJson.GetType() == NJson::JSON_MAP; if (isMap) { if (!config.MapAsObject) { @@ -421,7 +433,13 @@ Json2RepeatedField(const NJson::TJsonValue& json, for (const auto& x : jsonMap) { const TString& key = x.first; const NJson::TJsonValue& jsonValue = x.second; + if (config.UnknownFieldsCollector) { + config.UnknownFieldsCollector->OnEnterMapItem(key); + } Json2RepeatedFieldValue(jsonValue, proto, field, config, reflection, key); + if (config.UnknownFieldsCollector) { + config.UnknownFieldsCollector->OnLeaveMapItem(); + } } } else { if (config.ReplaceRepeatedFields) { @@ -429,17 +447,37 @@ Json2RepeatedField(const NJson::TJsonValue& json, } if (fieldJson.GetType() == NJson::JSON_ARRAY) { const NJson::TJsonValue::TArray& jsonArray = fieldJson.GetArray(); + ui64 id = 0; for (const NJson::TJsonValue& jsonValue : jsonArray) { + if (config.UnknownFieldsCollector) { + config.UnknownFieldsCollector->OnEnterArrayItem(id); + } Json2RepeatedFieldValue(jsonValue, proto, field, config, reflection); + if (config.UnknownFieldsCollector) { + config.UnknownFieldsCollector->OnLeaveArrayItem(); + } + ++id; } } else if (config.ValueVectorizer) { + ui64 id = 0; for (const NJson::TJsonValue& jsonValue : config.ValueVectorizer(fieldJson)) { + if (config.UnknownFieldsCollector) { + config.UnknownFieldsCollector->OnEnterArrayItem(id); + } Json2RepeatedFieldValue(jsonValue, proto, field, config, reflection); + if (config.UnknownFieldsCollector) { + config.UnknownFieldsCollector->OnLeaveArrayItem(); + } + ++id; } } else if (config.VectorizeScalars) { Json2RepeatedFieldValue(fieldJson, proto, field, config, reflection); } } + + if (config.UnknownFieldsCollector) { + config.UnknownFieldsCollector->OnLeaveMapItem(); + } } namespace NProtobufJson { @@ -463,14 +501,18 @@ namespace NProtobufJson { } } - if (!config.AllowUnknownFields) { + if (!config.AllowUnknownFields || config.UnknownFieldsCollector) { THashMap<TString, bool> knownFields; for (int f = 0, endF = descriptor->field_count(); f < endF; ++f) { const google::protobuf::FieldDescriptor* field = descriptor->field(f); knownFields[GetFieldName(*field, config)] = 1; } for (const auto& f : json.GetMap()) { - Y_ENSURE(knownFields.contains(f.first), "unknown field \"" << f.first << "\" for \"" << descriptor->full_name() << "\""); + const bool isFieldKnown = knownFields.contains(f.first); + Y_ENSURE(config.AllowUnknownFields || isFieldKnown, "unknown field \"" << f.first << "\" for \"" << descriptor->full_name() << "\""); + if (!isFieldKnown) { + config.UnknownFieldsCollector->OnUnknownField(f.first, *descriptor); + } } } } diff --git a/library/cpp/protobuf/json/json2proto.h b/library/cpp/protobuf/json/json2proto.h index d576fabce7..5952e28c3d 100644 --- a/library/cpp/protobuf/json/json2proto.h +++ b/library/cpp/protobuf/json/json2proto.h @@ -2,10 +2,12 @@ #include "string_transform.h" #include "name_generator.h" +#include "unknown_fields_collector.h" #include <library/cpp/json/json_reader.h> #include <library/cpp/json/json_value.h> +#include <util/generic/ptr.h> #include <util/stream/input.h> #include <util/stream/str.h> #include <util/stream/mem.h> @@ -108,6 +110,11 @@ namespace NProtobufJson { return *this; } + TSelf& SetUnknownFieldsCollector(TSimpleSharedPtr<IUnknownFieldsCollector> value) { + UnknownFieldsCollector = std::move(value); + return *this; + } + FldNameMode FieldNameMode = FieldNameOriginalCase; bool AllowUnknownFields = true; @@ -152,6 +159,9 @@ namespace NProtobufJson { /// Allow nonstandard conversions, e.g. google.protobuf.Duration from String bool AllowString2TimeConversion = false; + + /// Stores information about unknown fields + TSimpleSharedPtr<IUnknownFieldsCollector> UnknownFieldsCollector = nullptr; }; /// @throw yexception diff --git a/library/cpp/protobuf/json/unknown_fields_collector.h b/library/cpp/protobuf/json/unknown_fields_collector.h new file mode 100644 index 0000000000..1e71f2164c --- /dev/null +++ b/library/cpp/protobuf/json/unknown_fields_collector.h @@ -0,0 +1,29 @@ +#pragma once + +#include <util/generic/string.h> + +namespace google { + namespace protobuf { + class FieldDescriptor; + class Descriptor; + } +} + +namespace NProtobufJson { + /* Methods OnEnter.../OnLeave... are called on every field of structure + * during traverse and should be used to build context + * Method OnUnknownField are called every time when field which can't + * be mapped + */ + struct IUnknownFieldsCollector { + virtual ~IUnknownFieldsCollector() = default; + + virtual void OnEnterMapItem(const TString& key) = 0; + virtual void OnLeaveMapItem() = 0; + + virtual void OnEnterArrayItem(ui64 id) = 0; + virtual void OnLeaveArrayItem() = 0; + + virtual void OnUnknownField(const TString& key, const google::protobuf::Descriptor& value) = 0; + }; +} diff --git a/library/cpp/protobuf/json/ut/unknown_fields_collector_ut.cpp b/library/cpp/protobuf/json/ut/unknown_fields_collector_ut.cpp new file mode 100644 index 0000000000..00a2152a68 --- /dev/null +++ b/library/cpp/protobuf/json/ut/unknown_fields_collector_ut.cpp @@ -0,0 +1,165 @@ +#include "json.h" +#include "proto.h" +#include "proto2json.h" + +#include <library/cpp/protobuf/json/json2proto.h> + +#include <library/cpp/testing/unittest/registar.h> + +#include <util/generic/set.h> +#include <util/generic/string.h> + +using namespace NProtobufJson; +using namespace NProtobufJsonTest; + +struct TTestUnknownFieldsCollector : public IUnknownFieldsCollector { + void OnEnterMapItem(const TString& key) override { + CurrentPath.push_back(key); + } + + void OnEnterArrayItem(ui64 id) override { + CurrentPath.push_back(ToString(id)); + } + + void OnLeaveMapItem() override { + CurrentPath.pop_back(); + } + + void OnLeaveArrayItem() override { + CurrentPath.pop_back(); + } + + void OnUnknownField(const TString& key, const google::protobuf::Descriptor& value) override { + TString path; + for (auto& piece : CurrentPath) { + path.append("/"); + path.append(piece); + } + path.append("/"); + path.append(key); + UnknownKeys.insert(std::move(path)); + Y_UNUSED(value); + } + + TVector<TString> CurrentPath; + TSet<TString> UnknownKeys; +}; + +Y_UNIT_TEST_SUITE(TUnknownFieldsCollectorTest) { + Y_UNIT_TEST(TestFlatOptional) { + TFlatOptional proto; + TSimpleSharedPtr<TTestUnknownFieldsCollector> collector = new TTestUnknownFieldsCollector; + TJson2ProtoConfig cfg; + cfg.SetUnknownFieldsCollector(collector).SetAllowUnknownFields(true); + + Json2Proto(TStringBuf(R"({"42":42,"I32":11,"test":2,"string":"str","String":"string","obj":{"inner":{}},"arr":[1,2,3]})"), proto, cfg); + TSet<TString> expectedKeys = { + {"/42"}, + {"/arr"}, + {"/obj"}, + {"/string"}, + {"/test"}, + }; + UNIT_ASSERT(collector->CurrentPath.empty()); + UNIT_ASSERT_VALUES_EQUAL(collector->UnknownKeys, expectedKeys); + } + + Y_UNIT_TEST(TestFlatRepeated) { + TFlatRepeated proto; + TSimpleSharedPtr<TTestUnknownFieldsCollector> collector = new TTestUnknownFieldsCollector; + TJson2ProtoConfig cfg; + cfg.SetUnknownFieldsCollector(collector).SetAllowUnknownFields(true); + + Json2Proto(TStringBuf(R"({"42":42,"I32":[11,12],"test":12,"string":"str","String":["string1","string2"],"obj":{"inner":{}},"arr":[1,2,3]})"), proto, cfg); + TSet<TString> expectedKeys = { + {"/42"}, + {"/arr"}, + {"/obj"}, + {"/string"}, + {"/test"}, + }; + UNIT_ASSERT(collector->CurrentPath.empty()); + UNIT_ASSERT_VALUES_EQUAL(collector->UnknownKeys, expectedKeys); + } + + Y_UNIT_TEST(TestCompositeOptional) { + TCompositeOptional proto; + TSimpleSharedPtr<TTestUnknownFieldsCollector> collector = new TTestUnknownFieldsCollector; + TJson2ProtoConfig cfg; + cfg.SetUnknownFieldsCollector(collector).SetAllowUnknownFields(true); + + Json2Proto(TStringBuf(R"({"Part":{"42":42,"I32":11,"test":12,"string":"str","String":"string"},"string2":"str"})"), proto, cfg); + TSet<TString> expectedKeys = { + {"/Part/42"}, + {"/Part/string"}, + {"/Part/test"}, + {"/string2"}, + }; + UNIT_ASSERT(collector->CurrentPath.empty()); + UNIT_ASSERT_VALUES_EQUAL(collector->UnknownKeys, expectedKeys); + } + + Y_UNIT_TEST(TestCompositeRepeated) { + TCompositeRepeated proto; + TSimpleSharedPtr<TTestUnknownFieldsCollector> collector = new TTestUnknownFieldsCollector; + TJson2ProtoConfig cfg; + cfg.SetUnknownFieldsCollector(collector).SetAllowUnknownFields(true); + + Json2Proto(TStringBuf(R"({"Part":[)" + R"( {"42":42,"I32":11,"test":12,"string":"str","String":"string"},)" + R"( {"abc":"d"})" + R"(],)" + R"("string2":"str"})"), proto, cfg); + TSet<TString> expectedKeys = { + {"/Part/0/42"}, + {"/Part/0/string"}, + {"/Part/0/test"}, + {"/Part/1/abc"}, + {"/string2"}, + }; + UNIT_ASSERT(collector->CurrentPath.empty()); + UNIT_ASSERT_VALUES_EQUAL(collector->UnknownKeys, expectedKeys); + } + + Y_UNIT_TEST(TestCompleMapType) { + TComplexMapType proto; + TSimpleSharedPtr<TTestUnknownFieldsCollector> collector = new TTestUnknownFieldsCollector; + TJson2ProtoConfig cfg; + cfg.SetUnknownFieldsCollector(collector).SetAllowUnknownFields(true); + + Json2Proto(TStringBuf(R"({"42":42,)" + R"("Nested":[)" + R"( {"key":"abc","value":{"string":"string","Nested":[{"key":"def","value":{"string2":"string2"}}]}},)" + R"( {"key":"car","value":{"string3":"string3"}})" + R"(]})"), proto, cfg); + TSet<TString> expectedKeys = { + {"/42"}, + {"/Nested/0/value/Nested/0/value/string2"}, + {"/Nested/0/value/string"}, + {"/Nested/1/value/string3"}, + }; + UNIT_ASSERT(collector->CurrentPath.empty()); + UNIT_ASSERT_VALUES_EQUAL(collector->UnknownKeys, expectedKeys); + } + + Y_UNIT_TEST(TestCompleMapTypeMapAsObject) { + TComplexMapType proto; + TSimpleSharedPtr<TTestUnknownFieldsCollector> collector = new TTestUnknownFieldsCollector; + TJson2ProtoConfig cfg; + cfg.SetUnknownFieldsCollector(collector).SetAllowUnknownFields(true).SetMapAsObject(true); + + Json2Proto(TStringBuf(R"({"42":42,)" + R"("Nested":{)" + R"( "abc":{"string":"string","Nested":{"def":{"string2":"string2"}}},)" + R"( "car":{"string3":"string3"})" + R"(}})"), proto, cfg); + TSet<TString> expectedKeys = { + {"/42"}, + {"/Nested/abc/Nested/def/string2"}, + {"/Nested/abc/string"}, + {"/Nested/car/string3"}, + }; + UNIT_ASSERT(collector->CurrentPath.empty()); + UNIT_ASSERT_VALUES_EQUAL(collector->UnknownKeys, expectedKeys); + } +} // TJson2ProtoTest diff --git a/library/cpp/protobuf/json/ut/ya.make b/library/cpp/protobuf/json/ut/ya.make index 2a5391e375..11690ceb46 100644 --- a/library/cpp/protobuf/json/ut/ya.make +++ b/library/cpp/protobuf/json/ut/ya.make @@ -10,6 +10,7 @@ SRCS( filter_ut.proto test.proto util_ut.cpp + unknown_fields_collector_ut.cpp ) GENERATE_ENUM_SERIALIZATION(test.pb.h) |