diff options
author | robot-piglet <robot-piglet@yandex-team.com> | 2024-02-24 16:55:29 +0300 |
---|---|---|
committer | robot-piglet <robot-piglet@yandex-team.com> | 2024-02-24 17:04:44 +0300 |
commit | 3cd9fd5e8335820231d39e6d9f7722ef3eb47c0c (patch) | |
tree | 064caf2f1b18b36761acc38112e298e8823b0b6e | |
parent | 594a0b528d492d6574df185c932b62ec63b50fea (diff) | |
download | ydb-3cd9fd5e8335820231d39e6d9f7722ef3eb47c0c.tar.gz |
Intermediate changes
-rw-r--r-- | yt/yt/core/logging/config.cpp | 3 | ||||
-rw-r--r-- | yt/yt/core/logging/config.h | 4 | ||||
-rw-r--r-- | yt/yt/core/logging/formatter.cpp | 13 | ||||
-rw-r--r-- | yt/yt/core/logging/formatter.h | 4 | ||||
-rw-r--r-- | yt/yt/core/logging/log_manager.cpp | 2 | ||||
-rw-r--r-- | yt/yt/core/logging/unittests/logging_ut.cpp | 32 | ||||
-rw-r--r-- | yt/yt/core/ya.make | 1 | ||||
-rw-r--r-- | yt/yt/core/ytree/public.h | 7 | ||||
-rw-r--r-- | yt/yt/core/ytree/unittests/ya.make | 1 | ||||
-rw-r--r-- | yt/yt/core/ytree/unittests/yson_serializable_ut.cpp | 1171 | ||||
-rw-r--r-- | yt/yt/core/ytree/unittests/yson_struct_ut.cpp | 168 | ||||
-rw-r--r-- | yt/yt/core/ytree/yson_schema-inl.h | 7 | ||||
-rw-r--r-- | yt/yt/core/ytree/yson_serializable-inl.h | 1037 | ||||
-rw-r--r-- | yt/yt/core/ytree/yson_serializable.cpp | 385 | ||||
-rw-r--r-- | yt/yt/core/ytree/yson_serializable.h | 268 | ||||
-rw-r--r-- | yt/yt/core/ytree/yson_struct.h | 4 | ||||
-rw-r--r-- | yt/yt/core/ytree/yson_struct_detail-inl.h | 27 | ||||
-rw-r--r-- | yt/yt/core/ytree/yson_struct_detail.h | 2 | ||||
-rw-r--r-- | yt/yt/core/ytree/yson_struct_enum.h (renamed from yt/yt/core/ytree/yson_serialize_common.h) | 0 |
19 files changed, 65 insertions, 3071 deletions
diff --git a/yt/yt/core/logging/config.cpp b/yt/yt/core/logging/config.cpp index 81694f3d6f..213f9baf79 100644 --- a/yt/yt/core/logging/config.cpp +++ b/yt/yt/core/logging/config.cpp @@ -92,7 +92,8 @@ void TLogWriterConfig::Register(TRegistrar registrar) .Default(); registrar.Parameter("enable_source_location", &TThis::EnableSourceLocation) .Default(false); - registrar.Parameter("enable_instant", &TThis::EnableInstant) + registrar.Parameter("enable_system_fields", &TThis::EnableSystemFields) + .Alias("enable_instant") .Default(true); registrar.Parameter("json_format", &TThis::JsonFormat) .Default(); diff --git a/yt/yt/core/logging/config.h b/yt/yt/core/logging/config.h index 5dbebb21cc..8053d36dbe 100644 --- a/yt/yt/core/logging/config.h +++ b/yt/yt/core/logging/config.h @@ -89,10 +89,8 @@ public: //! Plain text formatter options. bool EnableSourceLocation; - //! Enable writing instant field. - bool EnableInstant; - //! Structured formatter options. + bool EnableSystemFields; THashMap<TString, NYTree::INodePtr> CommonFields; NJson::TJsonFormatConfigPtr JsonFormat; diff --git a/yt/yt/core/logging/formatter.cpp b/yt/yt/core/logging/formatter.cpp index 75a3c4f5be..7c468c16b7 100644 --- a/yt/yt/core/logging/formatter.cpp +++ b/yt/yt/core/logging/formatter.cpp @@ -153,12 +153,12 @@ TStructuredLogFormatter::TStructuredLogFormatter( THashMap<TString, NYTree::INodePtr> commonFields, bool enableSystemMessages, bool enableSourceLocation, - bool enableInstant, + bool enableSystemFields, NJson::TJsonFormatConfigPtr jsonFormat) : TLogFormatterBase(enableSystemMessages, enableSourceLocation) , Format_(format) , CommonFields_(std::move(commonFields)) - , EnableInstant_(enableInstant) + , EnableSystemFields_(enableSystemFields) , JsonFormat_(!jsonFormat && (Format_ == ELogFormat::Json) ? New<NJson::TJsonFormatConfig>() : std::move(jsonFormat)) @@ -199,11 +199,12 @@ i64 TStructuredLogFormatter::WriteFormatted(IOutputStream* stream, const TLogEve .DoIf(event.MessageKind == ELogMessageKind::Unstructured, [&] (auto fluent) { fluent.Item("message").Value(event.MessageRef.ToStringBuf()); }) - .DoIf(EnableInstant_, [&] (auto fluent) { - fluent.Item("instant").Value(dateTimeBuffer.GetBuffer()); + .DoIf(EnableSystemFields_, [&] (auto fluent) { + fluent + .Item("instant").Value(dateTimeBuffer.GetBuffer()) + .Item("level").Value(FormatEnum(event.Level)) + .Item("category").Value(event.Category->Name); }) - .Item("level").Value(FormatEnum(event.Level)) - .Item("category").Value(event.Category->Name) .DoIf(event.Family == ELogFamily::PlainText, [&] (auto fluent) { if (event.FiberId != TFiberId()) { fluent.Item("fiber_id").Value(Format("%x", event.FiberId)); diff --git a/yt/yt/core/logging/formatter.h b/yt/yt/core/logging/formatter.h index e4d41cf719..70913773f8 100644 --- a/yt/yt/core/logging/formatter.h +++ b/yt/yt/core/logging/formatter.h @@ -69,7 +69,7 @@ public: THashMap<TString, NYTree::INodePtr> commonFields, bool enableSystemMessages = true, bool enableSourceLocation = false, - bool enableInstant = true, + bool enableSystemFields = true, NJson::TJsonFormatConfigPtr jsonFormat = nullptr); i64 WriteFormatted(IOutputStream* outputStream, const TLogEvent& event) override; @@ -80,7 +80,7 @@ public: private: const ELogFormat Format_; const THashMap<TString, NYTree::INodePtr> CommonFields_; - const bool EnableInstant_; + const bool EnableSystemFields_; const NJson::TJsonFormatConfigPtr JsonFormat_; TCachingDateFormatter CachingDateFormatter_; diff --git a/yt/yt/core/logging/log_manager.cpp b/yt/yt/core/logging/log_manager.cpp index b31e28df5a..1481e4053d 100644 --- a/yt/yt/core/logging/log_manager.cpp +++ b/yt/yt/core/logging/log_manager.cpp @@ -852,7 +852,7 @@ private: writerConfig->CommonFields, writerConfig->AreSystemMessagesEnabled(), writerConfig->EnableSourceLocation, - writerConfig->EnableInstant, + writerConfig->EnableSystemFields, writerConfig->JsonFormat); default: diff --git a/yt/yt/core/logging/unittests/logging_ut.cpp b/yt/yt/core/logging/unittests/logging_ut.cpp index fd0bf73423..9448c56b9b 100644 --- a/yt/yt/core/logging/unittests/logging_ut.cpp +++ b/yt/yt/core/logging/unittests/logging_ut.cpp @@ -659,7 +659,7 @@ TEST_F(TLoggingTest, StructuredLoggingJsonFormat) /*commonFields*/ THashMap<TString, INodePtr>{}, /*enableControlMessages*/ true, /*enableSourceLocation*/ false, - /*enableInstant*/ true, + /*enableSystemFields*/ true, jsonFormat); auto writer = CreateFileLogWriter( @@ -763,6 +763,36 @@ TEST_F(TLoggingTest, StructuredValidationWithSamplingRate) EXPECT_GT(counter, 0); } +TEST_F(TLoggingTest, StructuredLoggingDisableSystemFields) +{ + TLogEvent event; + event.Family = ELogFamily::Structured; + event.Category = Logger.GetCategory(); + event.Level = ELogLevel::Debug; + event.MessageRef = BuildYsonStringFluently<EYsonType::MapFragment>() + .Item("message").Value("test_message") + .Finish() + .ToSharedRef(); + event.MessageKind = ELogMessageKind::Structured; + + auto formatter = std::make_unique<TStructuredLogFormatter>( + ELogFormat::Yson, + /*commonFields*/ THashMap<TString, INodePtr>{}, + /*enableControlMessages*/ true, + /*enableSourceLocation*/ false, + /*enableSystemFields*/ false); + + TStringStream stringStream; + formatter->WriteFormatted(&stringStream, event); + + auto message = DeserializeStructuredEvent(stringStream.Str(), ELogFormat::Yson); + EXPECT_EQ(message->GetChildOrThrow("message")->AsString()->GetValue(), "test_message"); + + EXPECT_EQ(message->FindChild("instant"), nullptr); + EXPECT_EQ(message->FindChild("level"), nullptr); + EXPECT_EQ(message->FindChild("category"), nullptr); +} + //////////////////////////////////////////////////////////////////////////////// class TBuiltinRotationTest diff --git a/yt/yt/core/ya.make b/yt/yt/core/ya.make index 552f269b6a..cac3d60635 100644 --- a/yt/yt/core/ya.make +++ b/yt/yt/core/ya.make @@ -280,7 +280,6 @@ SRCS( ytree/ypath_detail.cpp ytree/ypath_resolver.cpp ytree/ypath_service.cpp - ytree/yson_serializable.cpp ytree/yson_struct.cpp ytree/yson_struct_detail.cpp diff --git a/yt/yt/core/ytree/public.h b/yt/yt/core/ytree/public.h index f3265bb085..2c51214527 100644 --- a/yt/yt/core/ytree/public.h +++ b/yt/yt/core/ytree/public.h @@ -19,13 +19,13 @@ class TAttributeFilter; struct TAttributeFilter; -class TYsonSerializableLite; -class TYsonSerializable; - struct IYsonStructMeta; class TYsonStructBase; class TYsonStructLite; +template <class T> +concept CYsonStructDerived = std::derived_from<T, TYsonStructBase>; + DECLARE_REFCOUNTED_STRUCT(INode) using IConstNodePtr = TIntrusivePtr<const INode>; DECLARE_REFCOUNTED_STRUCT(ICompositeNode) @@ -76,7 +76,6 @@ constexpr i64 DefaultVirtualChildLimit = 1000; //! NB: Changing this value will invalidate all changelogs! constexpr int MaxYPathResolveIterations = 256; -DECLARE_REFCOUNTED_CLASS(TYsonSerializable) DECLARE_REFCOUNTED_CLASS(TYsonStruct) DECLARE_REFCOUNTED_CLASS(TYPathServiceContextWrapper) diff --git a/yt/yt/core/ytree/unittests/ya.make b/yt/yt/core/ytree/unittests/ya.make index 843bb07605..034ba0a1d5 100644 --- a/yt/yt/core/ytree/unittests/ya.make +++ b/yt/yt/core/ytree/unittests/ya.make @@ -17,7 +17,6 @@ SRCS( tree_builder_ut.cpp lazy_ypath_service_ut.cpp yson_schema_ut.cpp - yson_serializable_ut.cpp yson_struct_ut.cpp ytree_fluent_ut.cpp ytree_ut.cpp diff --git a/yt/yt/core/ytree/unittests/yson_serializable_ut.cpp b/yt/yt/core/ytree/unittests/yson_serializable_ut.cpp deleted file mode 100644 index 12835350a3..0000000000 --- a/yt/yt/core/ytree/unittests/yson_serializable_ut.cpp +++ /dev/null @@ -1,1171 +0,0 @@ -#include <yt/yt/core/test_framework/framework.h> - -#include <yt/yt/core/yson/writer.h> - -#include <yt/yt/core/ytree/ephemeral_node_factory.h> -#include <yt/yt/core/ytree/fluent.h> -#include <yt/yt/core/ytree/tree_builder.h> -#include <yt/yt/core/ytree/tree_visitor.h> -#include <yt/yt/core/ytree/ypath_client.h> -#include <yt/yt/core/ytree/yson_serializable.h> -#include <yt/yt/core/ytree/yson_struct.h> - -#include <array> - -namespace NYT::NYTree { -namespace { - -using namespace NYson; - -//////////////////////////////////////////////////////////////////////////////// - -DEFINE_ENUM(ETestEnum, - (Value0) - (Value1) - (Value2) -); - -//////////////////////////////////////////////////////////////////////////////// - -struct TTestSubconfig - : public TYsonSerializable -{ - int MyInt; - unsigned int MyUint; - bool MyBool; - std::vector<TString> MyStringList; - ETestEnum MyEnum; - - TTestSubconfig() - { - RegisterParameter("my_int", MyInt).Default(100).InRange(95, 205); - RegisterParameter("my_uint", MyUint).Default(50).InRange(31, 117); - RegisterParameter("my_bool", MyBool).Default(false); - RegisterParameter("my_string_list", MyStringList).Default(); - RegisterParameter("my_enum", MyEnum).Default(ETestEnum::Value1); - } -}; - -using TTestSubconfigPtr = TIntrusivePtr<TTestSubconfig>; - -//////////////////////////////////////////////////////////////////////////////// - -class TTestConfig - : public TYsonSerializable -{ -public: - TString MyString; - TTestSubconfigPtr Subconfig; - std::vector<TTestSubconfigPtr> SubconfigList; - std::unordered_map<TString, TTestSubconfigPtr> SubconfigMap; - std::optional<i64> NullableInt; - - TTestConfig() - { - SetUnrecognizedStrategy(EUnrecognizedStrategy::KeepRecursive); - - RegisterParameter("my_string", MyString).NonEmpty(); - RegisterParameter("sub", Subconfig).DefaultNew(); - RegisterParameter("sub_list", SubconfigList).Default(); - RegisterParameter("sub_map", SubconfigMap).Default(); - RegisterParameter("nullable_int", NullableInt).Default(); - - RegisterPreprocessor([&] () { - MyString = "x"; - Subconfig->MyInt = 200; - }); - } -}; - -using TTestConfigPtr = TIntrusivePtr<TTestConfig>; - -//////////////////////////////////////////////////////////////////////////////// - -class TSimpleYsonSerializable - : public TYsonSerializable -{ -public: - int IntValue; - - TSimpleYsonSerializable() - { - RegisterParameter("int_value", IntValue) - .Default(1); - } -}; - -//////////////////////////////////////////////////////////////////////////////// - -class TSimpleYsonStruct - : public TYsonStruct -{ -public: - int IntValue; - - REGISTER_YSON_STRUCT(TSimpleYsonStruct); - - static void Register(TRegistrar registrar) - { - registrar.Parameter("int_value", &TSimpleYsonStruct::IntValue) - .Default(1); - } -}; - -//////////////////////////////////////////////////////////////////////////////// - -class TYsonSerializableWithSimpleYsonStruct - : public TYsonSerializable -{ -public: - TIntrusivePtr<TSimpleYsonStruct> YsonStruct; - - TYsonSerializableWithSimpleYsonStruct() - { - SetUnrecognizedStrategy(EUnrecognizedStrategy::KeepRecursive); - - RegisterParameter("yson_struct", YsonStruct) - .DefaultNew(); - } -}; - -//////////////////////////////////////////////////////////////////////////////// - -// {LoadFromNode} -using TYsonSerializableParseTestParameter = std::tuple<bool>; - -class TYsonSerializableParseTest - : public ::testing::TestWithParam<TYsonSerializableParseTestParameter> -{ -public: - template <typename T> - TIntrusivePtr<T> Load( - const INodePtr& node, - bool postprocess = true, - bool setDefaults = true, - const NYPath::TYPath& path = {}) - { - auto [loadFromNode] = GetParam(); - auto config = New<T>(); - if (loadFromNode) { - config->Load(node, postprocess, setDefaults, path); - } else { - auto ysonString = ConvertToYsonString(node); - auto string = ysonString.ToString(); - TStringInput input(string); - TYsonPullParser parser(&input, EYsonType::Node); - auto cursor = TYsonPullParserCursor(&parser); - config->Load(&cursor, postprocess, setDefaults, path); - } - return config; - } - - template <typename T> - TIntrusivePtr<T> Load(const TYsonStringBuf& yson) - { - return Load<T>(ConvertTo<INodePtr>(yson)); - } -}; - -INSTANTIATE_TEST_SUITE_P( - LoadFromNode, - TYsonSerializableParseTest, - ::testing::Values(TYsonSerializableParseTestParameter{ - true - }) -); - -INSTANTIATE_TEST_SUITE_P( - LoadFromCursor, - TYsonSerializableParseTest, - ::testing::Values(TYsonSerializableParseTestParameter{ - false - }) -); - -//////////////////////////////////////////////////////////////////////////////// - -void TestCompleteSubconfig(TTestSubconfig* subconfig) -{ - EXPECT_EQ(99, subconfig->MyInt); - EXPECT_EQ(101u, subconfig->MyUint); - EXPECT_TRUE(subconfig->MyBool); - EXPECT_EQ(3u, subconfig->MyStringList.size()); - EXPECT_EQ("ListItem0", subconfig->MyStringList[0]); - EXPECT_EQ("ListItem1", subconfig->MyStringList[1]); - EXPECT_EQ("ListItem2", subconfig->MyStringList[2]); - EXPECT_EQ(ETestEnum::Value2, subconfig->MyEnum); -} - -TEST_P(TYsonSerializableParseTest, Complete) -{ - auto configNode = BuildYsonNodeFluently() - .BeginMap() - .Item("my_string").Value("TestString") - .Item("sub").BeginMap() - .Item("my_int").Value(99) - .Item("my_uint").Value(101) - .Item("my_bool").Value(true) - .Item("my_enum").Value("value2") - .Item("my_string_list").BeginList() - .Item().Value("ListItem0") - .Item().Value("ListItem1") - .Item().Value("ListItem2") - .EndList() - .EndMap() - .Item("sub_list").BeginList() - .Item().BeginMap() - .Item("my_int").Value(99) - .Item("my_uint").Value(101) - .Item("my_bool").Value(true) - .Item("my_enum").Value("value2") - .Item("my_string_list").BeginList() - .Item().Value("ListItem0") - .Item().Value("ListItem1") - .Item().Value("ListItem2") - .EndList() - .EndMap() - .Item().BeginMap() - .Item("my_int").Value(99) - .Item("my_uint").Value(101) - .Item("my_bool").Value(true) - .Item("my_enum").Value("value2") - .Item("my_string_list").BeginList() - .Item().Value("ListItem0") - .Item().Value("ListItem1") - .Item().Value("ListItem2") - .EndList() - .EndMap() - .EndList() - .Item("sub_map").BeginMap() - .Item("sub1").BeginMap() - .Item("my_int").Value(99) - .Item("my_uint").Value(101) - .Item("my_bool").Value(true) - .Item("my_enum").Value("value2") - .Item("my_string_list").BeginList() - .Item().Value("ListItem0") - .Item().Value("ListItem1") - .Item().Value("ListItem2") - .EndList() - .EndMap() - .Item("sub2").BeginMap() - .Item("my_int").Value(99) - .Item("my_uint").Value(101) - .Item("my_bool").Value(true) - .Item("my_enum").Value("value2") - .Item("my_string_list").BeginList() - .Item().Value("ListItem0") - .Item().Value("ListItem1") - .Item().Value("ListItem2") - .EndList() - .EndMap() - .EndMap() - .EndMap(); - - auto config = Load<TTestConfig>(configNode->AsMap()); - - EXPECT_EQ("TestString", config->MyString); - TestCompleteSubconfig(config->Subconfig.Get()); - EXPECT_EQ(2u, config->SubconfigList.size()); - TestCompleteSubconfig(config->SubconfigList[0].Get()); - TestCompleteSubconfig(config->SubconfigList[1].Get()); - EXPECT_EQ(2u, config->SubconfigMap.size()); - auto it1 = config->SubconfigMap.find("sub1"); - EXPECT_FALSE(it1 == config->SubconfigMap.end()); - TestCompleteSubconfig(it1->second.Get()); - auto it2 = config->SubconfigMap.find("sub2"); - EXPECT_FALSE(it2 == config->SubconfigMap.end()); - TestCompleteSubconfig(it2->second.Get()); -} - -TEST_P(TYsonSerializableParseTest, MissingParameter) -{ - auto configNode = BuildYsonNodeFluently() - .BeginMap() - .Item("my_string").Value("TestString") - .Item("sub").BeginMap() - .Item("my_bool").Value(true) - .EndMap() - .EndMap(); - - auto config = Load<TTestConfig>(configNode->AsMap()); - - EXPECT_EQ("TestString", config->MyString); - EXPECT_EQ(200, config->Subconfig->MyInt); - EXPECT_TRUE(config->Subconfig->MyBool); - EXPECT_EQ(0u, config->Subconfig->MyStringList.size()); - EXPECT_EQ(ETestEnum::Value1, config->Subconfig->MyEnum); - EXPECT_EQ(0u, config->SubconfigList.size()); - EXPECT_EQ(0u, config->SubconfigMap.size()); -} - -TEST_P(TYsonSerializableParseTest, MissingSubconfig) -{ - auto configNode = BuildYsonNodeFluently() - .BeginMap() - .Item("my_string").Value("TestString") - .EndMap(); - - auto config = Load<TTestConfig>(configNode->AsMap()); - - EXPECT_EQ("TestString", config->MyString); - EXPECT_EQ(200, config->Subconfig->MyInt); - EXPECT_FALSE(config->Subconfig->MyBool); - EXPECT_EQ(0u, config->Subconfig->MyStringList.size()); - EXPECT_EQ(ETestEnum::Value1, config->Subconfig->MyEnum); - EXPECT_EQ(0u, config->SubconfigList.size()); - EXPECT_EQ(0u, config->SubconfigMap.size()); -} - -TEST_P(TYsonSerializableParseTest, UnrecognizedSimple) -{ - auto configNode = BuildYsonNodeFluently() - .BeginMap() - .Item("my_string").Value("TestString") - .Item("option").Value(1) - .EndMap(); - - auto config = Load<TTestConfig>(configNode->AsMap()); - - auto unrecognizedNode = config->GetUnrecognized(); - auto unrecognizedRecursivelyNode = config->GetUnrecognizedRecursively(); - EXPECT_TRUE(AreNodesEqual(unrecognizedNode, unrecognizedRecursivelyNode)); - EXPECT_EQ(1, unrecognizedNode->GetChildCount()); - for (const auto& [key, child] : unrecognizedNode->GetChildren()) { - EXPECT_EQ("option", key); - EXPECT_EQ(1, child->AsInt64()->GetValue()); - } - - auto output = ConvertToYsonString(config, NYson::EYsonFormat::Text); - auto deserializedConfig = ConvertTo<TTestConfigPtr>(output); - EXPECT_TRUE(AreNodesEqual(ConvertToNode(config), ConvertToNode(deserializedConfig))); -} - -TEST_P(TYsonSerializableParseTest, UnrecognizedRecursive) -{ - auto configNode = BuildYsonNodeFluently() - .BeginMap() - .Item("my_string").Value("TestString") - .Item("option").Value(1) - .Item("sub").BeginMap() - .Item("sub_option").Value(42) - .EndMap() - .EndMap(); - - auto config = Load<TTestConfig>(configNode->AsMap()); - - auto unrecognizedRecursivelyNode = config->GetUnrecognizedRecursively(); - EXPECT_EQ(2, unrecognizedRecursivelyNode->GetChildCount()); - for (const auto& [key, child] : unrecognizedRecursivelyNode->GetChildren()) { - if (key == "option") { - EXPECT_EQ(1, child->AsInt64()->GetValue()); - } else { - EXPECT_EQ("sub", key); - EXPECT_EQ(42, child->AsMap()->GetChildOrThrow("sub_option")->AsInt64()->GetValue()); - } - } - - auto output = ConvertToYsonString(config, NYson::EYsonFormat::Text); - auto deserializedConfig = ConvertTo<TTestConfigPtr>(output); - EXPECT_TRUE(AreNodesEqual(ConvertToNode(config), ConvertToNode(deserializedConfig))); -} - -TEST_P(TYsonSerializableParseTest, UnrecognizedWithNestedYsonStruct) -{ - auto configNode = BuildYsonNodeFluently() - .BeginMap() - .Item("yson_struct").BeginMap() - .Item("unrecognized").Value(1) - .EndMap() - .EndMap(); - - auto config = Load<TYsonSerializableWithSimpleYsonStruct>(configNode->AsMap()); - - auto unrecognized = config->GetRecursiveUnrecognized(); - EXPECT_EQ( - ConvertToYsonString(configNode, EYsonFormat::Text).AsStringBuf(), - ConvertToYsonString(unrecognized, EYsonFormat::Text).AsStringBuf()); -} - -TEST_P(TYsonSerializableParseTest, MissingRequiredParameter) -{ - auto configNode = BuildYsonNodeFluently() - .BeginMap() - .Item("sub").BeginMap() - .Item("my_int").Value(99) - .Item("my_bool").Value(true) - .EndMap() - .EndMap(); - - EXPECT_THROW(Load<TTestConfig>(configNode->AsMap()), std::exception); -} - -TEST_P(TYsonSerializableParseTest, IncorrectNodeType) -{ - auto builder = CreateBuilderFromFactory(GetEphemeralNodeFactory()); - builder->BeginTree(); - BuildYsonFluently(builder.get()) - .BeginMap() - .Item("my_string").Value(1) // incorrect type - .EndMap(); - auto configNode = builder->EndTree(); - - EXPECT_THROW(Load<TTestConfig>(configNode->AsMap()), std::exception); -} - -TEST_P(TYsonSerializableParseTest, ArithmeticOverflow) -{ - auto builder = CreateBuilderFromFactory(GetEphemeralNodeFactory()); - builder->BeginTree(); - BuildYsonFluently(builder.get()) - .BeginMap() - .Item("my_string").Value("TestString") - .Item("sub").BeginMap() - .Item("my_int").Value(Max<i64>()) - .Item("my_bool").Value(true) - .Item("my_enum").Value("Value2") - .Item("my_string_list").BeginList() - .Item().Value("ListItem0") - .Item().Value("ListItem1") - .Item().Value("ListItem2") - .EndList() - .EndMap() - .EndMap(); - auto configNode = builder->EndTree(); - - EXPECT_THROW(Load<TTestConfig>(configNode->AsMap()), std::exception); -} - -TEST_P(TYsonSerializableParseTest, Postprocess) -{ - auto builder = CreateBuilderFromFactory(GetEphemeralNodeFactory()); - builder->BeginTree(); - BuildYsonFluently(builder.get()) - .BeginMap() - .Item("my_string").Value("") // empty! - .EndMap(); - auto configNode = builder->EndTree(); - - auto config = Load<TTestConfig>(configNode, false); - EXPECT_THROW(config->Postprocess(), std::exception); -} - -TEST_P(TYsonSerializableParseTest, PostprocessSubconfig) -{ - auto builder = CreateBuilderFromFactory(GetEphemeralNodeFactory()); - builder->BeginTree(); - BuildYsonFluently(builder.get()) - .BeginMap() - .Item("my_string").Value("TestString") - .Item("sub").BeginMap() - .Item("my_int").Value(210) // out of range - .EndMap() - .EndMap(); - auto configNode = builder->EndTree(); - - auto config = Load<TTestConfig>(configNode, false); - EXPECT_THROW(config->Postprocess(), std::exception); -} - -TEST_P(TYsonSerializableParseTest, PostprocessSubconfigList) -{ - auto builder = CreateBuilderFromFactory(GetEphemeralNodeFactory()); - builder->BeginTree(); - BuildYsonFluently(builder.get()) - .BeginMap() - .Item("my_string").Value("TestString") - .Item("sub_list").BeginList() - .Item().BeginMap() - .Item("my_int").Value(210) // out of range - .EndMap() - .EndList() - .EndMap(); - auto configNode = builder->EndTree(); - - auto config = Load<TTestConfig>(configNode, false); - EXPECT_THROW(config->Postprocess(), std::exception); -} - -TEST_P(TYsonSerializableParseTest, PostprocessSubconfigMap) -{ - auto builder = CreateBuilderFromFactory(GetEphemeralNodeFactory()); - builder->BeginTree(); - BuildYsonFluently(builder.get()) - .BeginMap() - .Item("my_string").Value("TestString") - .Item("sub_map").BeginMap() - .Item("sub").BeginMap() - .Item("my_int").Value(210) // out of range - .EndMap() - .EndMap() - .EndMap(); - auto configNode = builder->EndTree(); - - auto config = Load<TTestConfig>(configNode, false); - EXPECT_THROW(config->Postprocess(), std::exception); -} - -TEST(TYsonSerializableTest, SaveSingleParameter) -{ - auto config = New<TTestConfig>(); - config->MyString = "test"; - config->NullableInt = 10; - - auto builder = CreateBuilderFromFactory(GetEphemeralNodeFactory()); - builder->BeginTree(); - config->SaveParameter("my_string", builder.get()); - auto actual = ConvertTo<TString>(builder->EndTree()); - EXPECT_EQ("test", actual); -} - -TEST(TYsonSerializableTest, LoadSingleParameter) -{ - auto config = New<TTestConfig>(); - config->NullableInt = 10; - - config->LoadParameter("my_string", ConvertToNode("test"), EMergeStrategy::Default); - EXPECT_EQ("test", config->MyString); - EXPECT_EQ(10, config->NullableInt); -} - -TEST(TYsonSerializableTest, LoadSingleParameterWithMergeStrategy) -{ - auto builder = CreateBuilderFromFactory(GetEphemeralNodeFactory()); - builder->BeginTree(); - BuildYsonFluently(builder.get()) - .BeginMap() - .Item("my_int").Value(100) - .EndMap(); - auto subConfig = builder->EndTree(); - - auto config1 = New<TTestConfig>(); - config1->Subconfig->MyBool = true; - config1->LoadParameter("sub", subConfig, EMergeStrategy::Default); - EXPECT_EQ(100, config1->Subconfig->MyInt); - EXPECT_TRUE(config1->Subconfig->MyBool); // Subconfig merged by default. - - auto config2 = New<TTestConfig>(); - config2->Subconfig->MyBool = true; - config2->LoadParameter("sub", subConfig, EMergeStrategy::Overwrite); - EXPECT_EQ(100, config2->Subconfig->MyInt); - EXPECT_FALSE(config2->Subconfig->MyBool); // Overwrite destroyed previous values. -} - -TEST(TYsonSerializableTest, ResetSingleParameter) -{ - auto config = New<TTestSubconfig>(); - config->MyInt = 10; - config->MyUint = 10; - - config->ResetParameter("my_int"); - EXPECT_EQ(100, config->MyInt); // Default value. - EXPECT_EQ(10u, config->MyUint); -} - -TEST(TYsonSerializableTest, Save) -{ - auto config = New<TTestConfig>(); - - // add non-default fields; - config->MyString = "hello!"; - config->SubconfigList.push_back(New<TTestSubconfig>()); - config->SubconfigMap["item"] = New<TTestSubconfig>(); - config->NullableInt = 42; - - auto output = ConvertToYsonString(config, NYson::EYsonFormat::Text); - - TString subconfigYson = - "{\"my_bool\"=%false;" - "\"my_enum\"=\"value1\";" - "\"my_int\"=200;" - "\"my_uint\"=50u;" - "\"my_string_list\"=[]}"; - - TString subconfigYsonOrigin = - "{\"my_bool\"=%false;" - "\"my_enum\"=\"value1\";" - "\"my_int\"=100;" - "\"my_uint\"=50u;" - "\"my_string_list\"=[]}"; - - TString expectedYson; - expectedYson += "{\"my_string\"=\"hello!\";"; - expectedYson += "\"sub\"=" + subconfigYson + ";"; - expectedYson += "\"sub_list\"=[" + subconfigYsonOrigin + "];"; - expectedYson += "\"sub_map\"={\"item\"=" + subconfigYsonOrigin + "};"; - expectedYson += "\"nullable_int\"=42}"; - - EXPECT_TRUE(AreNodesEqual( - ConvertToNode(TYsonString(expectedYson)), - ConvertToNode(output))); -} - -TEST(TYsonSerializableTest, TestConfigUpdate) -{ - auto config = New<TTestConfig>(); - { - auto newConfig = UpdateYsonSerializable(config, nullptr); - EXPECT_EQ(newConfig->Subconfig->MyInt, 200); - } - - { - auto newConfig = UpdateYsonSerializable(config, ConvertToNode(TYsonString(TStringBuf("{\"sub\"={\"my_int\"=150}}")))); - EXPECT_EQ(newConfig->Subconfig->MyInt, 150); - } - - { - auto newConfig = UpdateYsonSerializable(config, ConvertToNode(TYsonString(TStringBuf("{\"sub\"={\"my_int_\"=150}}")))); - EXPECT_EQ(newConfig->Subconfig->MyInt, 200); - } -} - -TEST(TYsonSerializableTest, NoDefaultNewAliasing) -{ - auto config1 = New<TTestConfig>(); - auto config2 = New<TTestConfig>(); - EXPECT_NE(config1->Subconfig, config2->Subconfig); -} - -TEST(TYsonSerializableTest, Reconfigure) -{ - auto config = New<TTestConfig>(); - auto subconfig = config->Subconfig; - - EXPECT_EQ("x", config->MyString); - EXPECT_EQ(200, subconfig->MyInt); - - auto configNode1 = BuildYsonNodeFluently() - .BeginMap() - .Item("my_string").Value("y") - .EndMap(); - ReconfigureYsonSerializable(config, configNode1); - - EXPECT_EQ("y", config->MyString); - EXPECT_EQ(subconfig, config->Subconfig); - EXPECT_EQ(200, subconfig->MyInt); - - auto configNode2 = BuildYsonNodeFluently() - .BeginMap() - .Item("my_string").Value("z") - .Item("sub").BeginMap() - .Item("my_int").Value(95) - .EndMap() - .EndMap(); - ReconfigureYsonSerializable(config, configNode2); - - EXPECT_EQ("z", config->MyString); - EXPECT_EQ(subconfig, config->Subconfig); - EXPECT_EQ(95, subconfig->MyInt); -} - -//////////////////////////////////////////////////////////////////////////////// - -class TTestConfigLite - : public TYsonSerializableLite -{ -public: - TString MyString; - std::optional<i64> NullableInt; - - TTestConfigLite() - { - RegisterParameter("my_string", MyString).NonEmpty(); - RegisterParameter("nullable_int", NullableInt).Default(); - } -}; - -TEST(TYsonSerializableTest, SaveLite) -{ - TTestConfigLite config; - - config.MyString = "hello!"; - config.NullableInt = 42; - - auto output = ConvertToYsonString(config, NYson::EYsonFormat::Text); - - TString expectedYson; - expectedYson += "{\"my_string\"=\"hello!\";"; - expectedYson += "\"nullable_int\"=42}"; - - EXPECT_TRUE(AreNodesEqual( - ConvertToNode(TYsonString(expectedYson)), - ConvertToNode(output))); -} - -//////////////////////////////////////////////////////////////////////////////// - -class TTestConfigWithAliases - : public TYsonSerializable -{ -public: - TString Value; - - TTestConfigWithAliases() - { - RegisterParameter("key", Value) - .Alias("alias1") - .Alias("alias2"); - } -}; - -TEST_P(TYsonSerializableParseTest, Aliases1) -{ - auto builder = CreateBuilderFromFactory(GetEphemeralNodeFactory()); - builder->BeginTree(); - BuildYsonFluently(builder.get()) - .BeginMap() - .Item("key").Value("value") - .EndMap(); - auto configNode = builder->EndTree(); - - auto config = Load<TTestConfigWithAliases>(configNode->AsMap(), false); - - EXPECT_EQ("value", config->Value); -} - -TEST_P(TYsonSerializableParseTest, Aliases2) -{ - auto builder = CreateBuilderFromFactory(GetEphemeralNodeFactory()); - builder->BeginTree(); - BuildYsonFluently(builder.get()) - .BeginMap() - .Item("alias1").Value("value") - .EndMap(); - auto configNode = builder->EndTree(); - - auto config = Load<TTestConfigWithAliases>(configNode->AsMap(), false); - - EXPECT_EQ("value", config->Value); -} - -TEST_P(TYsonSerializableParseTest, Aliases3) -{ - auto builder = CreateBuilderFromFactory(GetEphemeralNodeFactory()); - builder->BeginTree(); - BuildYsonFluently(builder.get()) - .BeginMap() - .Item("alias1").Value("value") - .Item("alias2").Value("value") - .EndMap(); - auto configNode = builder->EndTree(); - - auto config = Load<TTestConfigWithAliases>(configNode->AsMap(), false); - - EXPECT_EQ("value", config->Value); -} - -TEST_P(TYsonSerializableParseTest, Aliases4) -{ - auto builder = CreateBuilderFromFactory(GetEphemeralNodeFactory()); - builder->BeginTree(); - BuildYsonFluently(builder.get()) - .BeginMap() - .Item("alias1").Value("value1") - .Item("alias2").Value("value2") - .EndMap(); - auto configNode = builder->EndTree(); - - EXPECT_THROW(Load<TTestConfigWithAliases>(configNode->AsMap()), std::exception); -} - -TEST_P(TYsonSerializableParseTest, Aliases5) -{ - auto builder = CreateBuilderFromFactory(GetEphemeralNodeFactory()); - builder->BeginTree(); - BuildYsonFluently(builder.get()) - .BeginMap() - .EndMap(); - auto configNode = builder->EndTree(); - - EXPECT_THROW(Load<TTestConfigWithAliases>(configNode->AsMap()), std::exception); -} - -TEST_P(TYsonSerializableParseTest, ParameterTuplesAndContainers) -{ - class TTestClass - : public NYTree::TYsonSerializable - { - public: - std::vector<TString> Vector; - std::array<TString, 3> Array; - std::pair<size_t, TString> Pair; - std::set<TString> Set; - std::map<TString, int> Map; - std::multiset<int> MultiSet; - std::unordered_set<TString> UnorderedSet; - std::unordered_map<TString, int> UnorderedMap; - std::unordered_multiset<size_t> UnorderedMultiSet; - - TTestClass() - { - RegisterParameter("vector", Vector) - .Default(); - RegisterParameter("array", Array) - .Default(); - RegisterParameter("pair", Pair) - .Default(); - RegisterParameter("set", Set) - .Default(); - RegisterParameter("map", Map) - .Default(); - RegisterParameter("multiset", MultiSet) - .Default(); - RegisterParameter("unordered_set", UnorderedSet) - .Default(); - RegisterParameter("unordered_map", UnorderedMap) - .Default(); - RegisterParameter("unordered_multiset", UnorderedMultiSet) - .Default(); - } - }; - - auto original = New<TTestClass>(); - original->Vector = { "fceswf", "sadfcesa" }; - original->Array = {{ "UYTUY", ":LL:a", "78678678" }}; - original->Pair = { 7U, "UYTUY" }; - original->Set = { " q!", "12343e", "svvr", "0001" }; - original->Map = { {"!", 4398}, {"zzz", 0} }; - original->MultiSet = { 33, 33, 22, 22, 11 }; - original->UnorderedSet = { "41", "52", "001", "set" }; - original->UnorderedMap = { {"12345", 8}, {"XXX", 9}, {"XYZ", 42} }; - original->UnorderedMultiSet = { 1U, 2U, 1U, 0U, 0U }; - - auto deserialized = Load<TTestClass>(ConvertToYsonString(*original)); - - EXPECT_EQ(original->Vector, deserialized->Vector); - EXPECT_EQ(original->Array, deserialized->Array); - EXPECT_EQ(original->Pair, deserialized->Pair); - EXPECT_EQ(original->Set, deserialized->Set); - EXPECT_EQ(original->Map, deserialized->Map); - EXPECT_EQ(original->MultiSet, deserialized->MultiSet); - EXPECT_EQ(original->UnorderedSet, deserialized->UnorderedSet); - EXPECT_EQ(original->UnorderedMap, deserialized->UnorderedMap); - EXPECT_EQ(original->UnorderedMultiSet, deserialized->UnorderedMultiSet); -} - -//////////////////////////////////////////////////////////////////////////////// - -TEST(TYsonSerializableTest, EnumAsKeyToYHash) -{ - THashMap<ETestEnum, TString> deserialized, original = { - {ETestEnum::Value0, "abc"} - }; - - TString serialized = "{\"value0\"=\"abc\";}"; - ASSERT_EQ(serialized, ConvertToYsonString(original, EYsonFormat::Text).AsStringBuf()); - - Deserialize(deserialized, ConvertToNode(TYsonString(serialized, EYsonType::Node))); - - ASSERT_EQ(original, deserialized); -} - -//////////////////////////////////////////////////////////////////////////////// - -TEST_P(TYsonSerializableParseTest, NullableWithNonNullDefault) -{ - class TConfig - : public TYsonSerializable - { - public: - std::optional<int> Value; - - TConfig() - { - RegisterParameter("value", Value) - .Default(123); - } - }; - - { - auto config = Load<TConfig>(TYsonStringBuf("{}")); - EXPECT_EQ(123, *config->Value); - EXPECT_EQ(123, ConvertToNode(config)->AsMap()->GetChildValueOrThrow<i64>("value")); - } - - { - auto config = Load<TConfig>(TYsonStringBuf("{value=#}")); - EXPECT_FALSE(config->Value); - EXPECT_EQ(ENodeType::Entity, ConvertToNode(config)->AsMap()->GetChildOrThrow("value")->GetType()); - } -} - -//////////////////////////////////////////////////////////////////////////////// - -TEST(TYsonSerializableTest, DontSerializeDefault) -{ - class TConfig - : public TYsonSerializable - { - public: - int Value; - int OtherValue; - - TConfig() - { - RegisterParameter("value", Value) - .Default(123); - RegisterParameter("other_value", OtherValue) - .Default(456) - .DontSerializeDefault(); - } - }; - - { - auto config = New<TConfig>(); - auto output = ConvertToYsonString(config, NYson::EYsonFormat::Text); - - TString expectedYson = "{\"value\"=123;}"; - EXPECT_TRUE(AreNodesEqual( - ConvertToNode(TYsonString(expectedYson)), - ConvertToNode(output))); - } - - { - auto config = New<TConfig>(); - config->OtherValue = 789; - auto output = ConvertToYsonString(config, NYson::EYsonFormat::Text); - - TString expectedYson = "{\"value\"=123;\"other_value\"=789;}"; - EXPECT_TRUE(AreNodesEqual( - ConvertToNode(TYsonString(expectedYson)), - ConvertToNode(output))); - } -} - -class TYsonStructClass - : public TYsonStruct -{ -public: - int IntValue; - - REGISTER_YSON_STRUCT(TYsonStructClass); - - static void Register(TRegistrar registrar) - { - registrar.Parameter("int_value", &TThis::IntValue) - .Default(1); - } -}; - -class TYsonSerializableClass - : public TYsonSerializable -{ -public: - THashMap<TString, TIntrusivePtr<TYsonStructClass>> YsonStructHashMap; - - TIntrusivePtr<TYsonStructClass> YsonStructValue; - - TYsonSerializableClass() - { - RegisterParameter("yson_struct_hash_map", YsonStructHashMap) - .Default(); - - RegisterParameter("yson_struct_value", YsonStructValue) - .DefaultNew(); - - RegisterPreprocessor([&] () { - YsonStructValue->IntValue = 5; - }); - } -}; - -TEST_P(TYsonSerializableParseTest, YsonStructNestedToYsonSerializableSimple) -{ - { - auto config = New<TYsonSerializableClass>(); - EXPECT_EQ(config->YsonStructValue->IntValue, 5); - - config->YsonStructHashMap["x"] = New<TYsonStructClass>(); - config->YsonStructHashMap["x"]->IntValue = 10; - config->YsonStructValue->IntValue = 2; - - auto output = ConvertToYsonString(config, NYson::EYsonFormat::Text); - TString expectedYson = "{yson_struct_hash_map={x={int_value=10}};yson_struct_value={int_value=2}}"; - EXPECT_TRUE(AreNodesEqual( - ConvertToNode(TYsonString(expectedYson)), - ConvertToNode(TYsonString(output.AsStringBuf())))); - - auto deserialized = Load<TYsonSerializableClass>(output); - EXPECT_EQ(deserialized->YsonStructHashMap["x"]->IntValue, 10); - EXPECT_EQ(deserialized->YsonStructValue->IntValue, 2); - - } -} - -TEST_P(TYsonSerializableParseTest, YsonStructNestedToYsonSerializableDeserializesFromEmpty) -{ - { - auto testInput = TYsonString(TStringBuf("{yson_struct_value={}}")); - auto deserialized = Load<TYsonSerializableClass>(testInput); - EXPECT_EQ(deserialized->YsonStructValue->IntValue, 5); - } -} - -//////////////////////////////////////////////////////////////////////////////// - -class TNestedYsonStructClass - : public TYsonStruct -{ -public: - int IntValue; - - REGISTER_YSON_STRUCT(TNestedYsonStructClass); - - static void Register(TRegistrar registrar) - { - registrar.Parameter("int_value", &TThis::IntValue) - .Default(1); - registrar.Postprocessor([&] (TNestedYsonStructClass* klass) { - klass->IntValue = 10; - }); - } -}; - -class TYsonSerializableClass2 - : public TYsonSerializable -{ -public: - THashMap<TString, TIntrusivePtr<TNestedYsonStructClass>> YsonStructHashMap; - - TYsonSerializableClass2() - { - RegisterParameter("yson_struct_hash_map", YsonStructHashMap) - .Default(); - } -}; - -TEST_P(TYsonSerializableParseTest, PostprocessIsPropagatedFromYsonSerializableToYsonStruct) -{ - auto testInput = TYsonString(TStringBuf("{yson_struct_hash_map={x={int_value=2}}}")); - auto deserialized = Load<TYsonSerializableClass2>(testInput); - EXPECT_EQ(deserialized->YsonStructHashMap["x"]->IntValue, 10); -} - -//////////////////////////////////////////////////////////////////////////////// - -template <class T> -TIntrusivePtr<T> CreateCustomDefault() -{ - auto result = New<T>(); - result->IntValue = 10; - return result; -} - -class TYsonSerializableWithNestedStructsAndCustomDefaults - : public TYsonSerializable -{ -public: - TIntrusivePtr<TSimpleYsonSerializable> YsonSerializable; - TIntrusivePtr<TSimpleYsonStruct> YsonStruct; - - TYsonSerializableWithNestedStructsAndCustomDefaults() - { - RegisterParameter("yson_serializable", YsonSerializable) - .Default(CreateCustomDefault<TSimpleYsonSerializable>()); - RegisterParameter("yson_struct", YsonStruct) - .Default(CreateCustomDefault<TSimpleYsonStruct>()); - } -}; - -TEST(TYsonSerializableTest, TestCustomDefaultsOfNestedStructsAreDiscardedOnDeserialize) -{ - auto testInput = TYsonString(TStringBuf("{}")); - auto deserialized = ConvertTo<TIntrusivePtr<TYsonSerializableWithNestedStructsAndCustomDefaults>>(testInput); - EXPECT_EQ(deserialized->YsonSerializable->IntValue, 1); - EXPECT_EQ(deserialized->YsonStruct->IntValue, 1); -} - -//////////////////////////////////////////////////////////////////////////////// - -class TYsonSerializableWithNestedStructsAndPreprocessors - : public TYsonSerializable -{ -public: - TIntrusivePtr<TSimpleYsonSerializable> YsonSerializable; - TIntrusivePtr<TSimpleYsonStruct> YsonStruct; - - TYsonSerializableWithNestedStructsAndPreprocessors() - { - RegisterParameter("yson_serializable", YsonSerializable) - .Default(); - RegisterParameter("yson_struct", YsonStruct) - .Default(); - RegisterPreprocessor([&] () { - YsonSerializable = CreateCustomDefault<TSimpleYsonSerializable>(); - YsonStruct = CreateCustomDefault<TSimpleYsonStruct>(); - }); - } -}; - -TEST(TYsonSerializableTest, TestPreprocessorsEffectsOnNestedStructsArePreservedOnDeserialize) -{ - auto testInput = TYsonString(TStringBuf("{}")); - auto deserialized = ConvertTo<TIntrusivePtr<TYsonSerializableWithNestedStructsAndPreprocessors>>(testInput); - EXPECT_EQ(deserialized->YsonSerializable->IntValue, 10); - EXPECT_EQ(deserialized->YsonStruct->IntValue, 10); -} - -//////////////////////////////////////////////////////////////////////////////// - -TEST(TYsonSerializableTest, TestStable) -{ - class TInner - : public NYTree::TYsonSerializable - { - public: - int A; - int B; - int C; - int D; - int Q; - - TInner() - { - RegisterParameter("b", B) - .Default(2); - RegisterParameter("a", A) - .Default(1); - RegisterParameter("c", C) - .Default(3); - RegisterParameter("q", Q) - .Default(9); - RegisterParameter("d", D) - .Default(4); - } - }; - - class TOuter - : public TYsonSerializable - { - public: - TIntrusivePtr<TInner> Inner; - TOuter() - { - RegisterParameter("inner", Inner) - .DefaultNew(); - } - }; - - { - auto outer = New<TOuter>(); - auto output = ConvertToYsonString(*outer, NYson::EYsonFormat::Text); - - auto result = BuildYsonStringFluently(NYson::EYsonFormat::Text) - .BeginMap() - .Item("inner").BeginMap() - .Item("a").Value(1) - .Item("b").Value(2) - .Item("c").Value(3) - .Item("d").Value(4) - .Item("q").Value(9) - .EndMap() - .EndMap(); - - EXPECT_EQ(result, output); - } -} - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace -} // namespace NYT::NYTree diff --git a/yt/yt/core/ytree/unittests/yson_struct_ut.cpp b/yt/yt/core/ytree/unittests/yson_struct_ut.cpp index cc507e3178..71457da7aa 100644 --- a/yt/yt/core/ytree/unittests/yson_struct_ut.cpp +++ b/yt/yt/core/ytree/unittests/yson_struct_ut.cpp @@ -7,7 +7,6 @@ #include <yt/yt/core/ytree/tree_builder.h> #include <yt/yt/core/ytree/tree_visitor.h> #include <yt/yt/core/ytree/ypath_client.h> -#include <yt/yt/core/ytree/yson_serializable.h> #include <yt/yt/core/ytree/yson_struct.h> #include <util/stream/buffer.h> @@ -104,21 +103,6 @@ using TTestConfigPtr = TIntrusivePtr<TTestConfig>; //////////////////////////////////////////////////////////////////////////////// -class TSimpleYsonSerializable - : public TYsonSerializable -{ -public: - int IntValue; - - TSimpleYsonSerializable() - { - RegisterParameter("int_value", IntValue) - .Default(1); - } -}; - -//////////////////////////////////////////////////////////////////////////////// - class TSimpleYsonStruct : public TYsonStruct { @@ -136,25 +120,6 @@ public: //////////////////////////////////////////////////////////////////////////////// -class TYsonStructWithSimpleYsonSerializable - : public TYsonStruct -{ -public: - TIntrusivePtr<TSimpleYsonSerializable> YsonSerializable; - - REGISTER_YSON_STRUCT(TYsonStructWithSimpleYsonSerializable); - - static void Register(TRegistrar registrar) - { - registrar.UnrecognizedStrategy(EUnrecognizedStrategy::KeepRecursive); - - registrar.Parameter("yson_serializable", &TYsonStructWithSimpleYsonSerializable::YsonSerializable) - .DefaultNew(); - } -}; - -//////////////////////////////////////////////////////////////////////////////// - auto GetCompleteConfigNode(int offset = 0) { return BuildYsonNodeFluently() @@ -501,23 +466,6 @@ TEST_P(TYsonStructParseTest, UnrecognizedRecursiveTwoLevelNesting) ConvertToYsonString(unrecognized, EYsonFormat::Text).AsStringBuf()); } -TEST_P(TYsonStructParseTest, UnrecognizedWithNestedYsonSerializable) -{ - auto configNode = BuildYsonNodeFluently() - .BeginMap() - .Item("yson_serializable").BeginMap() - .Item("unrecognized").Value(1) - .EndMap() - .EndMap(); - - auto config = Load<TYsonStructWithSimpleYsonSerializable>(configNode->AsMap()); - - auto unrecognized = config->GetRecursiveUnrecognized(); - EXPECT_EQ( - ConvertToYsonString(configNode, EYsonFormat::Text).AsStringBuf(), - ConvertToYsonString(unrecognized, EYsonFormat::Text).AsStringBuf()); -} - TEST_P(TYsonStructParseTest, MissingRequiredParameter) { auto configNode = BuildYsonNodeFluently() @@ -1245,76 +1193,6 @@ TEST(TYsonStructTest, RegisterBaseFieldInDerived) //////////////////////////////////////////////////////////////////////////////// -class TYsonSerializableClass - : public TYsonSerializable -{ -public: - TYsonSerializableClass() - { - RegisterParameter("int_value", IntValue) - .Default(1); - } - - int IntValue; -}; - -class TYsonStructClass - : public TYsonStruct -{ -public: - THashMap<TString, TIntrusivePtr<TYsonSerializableClass>> YsonSerializableHashMap; - - TIntrusivePtr<TYsonSerializableClass> YsonSerializableValue; - - REGISTER_YSON_STRUCT(TYsonStructClass); - - static void Register(TRegistrar registrar) - { - registrar.Parameter("yson_serializable_hash_map", &TThis::YsonSerializableHashMap) - .Default(); - - registrar.Parameter("yson_serializable_value", &TThis::YsonSerializableValue) - .DefaultNew(); - - registrar.Preprocessor([] (TYsonStructClass* klass) { - klass->YsonSerializableValue->IntValue = 5; - }); - } -}; - -TEST(TYsonStructTest, YsonSerializableNestedToYsonStructSimple) -{ - { - auto config = New<TYsonStructClass>(); - EXPECT_EQ(config->YsonSerializableValue->IntValue, 5); - - config->YsonSerializableHashMap["x"] = New<TYsonSerializableClass>(); - config->YsonSerializableHashMap["x"]->IntValue = 10; - config->YsonSerializableValue->IntValue = 2; - - auto output = ConvertToYsonString(config, NYson::EYsonFormat::Text); - TString expectedYson = "{yson_serializable_hash_map={x={int_value=10}};yson_serializable_value={int_value=2}}"; - EXPECT_TRUE(AreNodesEqual( - ConvertToNode(TYsonString(expectedYson)), - ConvertToNode(TYsonString(output.AsStringBuf())))); - - auto deserialized = ConvertTo<TIntrusivePtr<TYsonStructClass>>(output); - EXPECT_EQ(deserialized->YsonSerializableHashMap["x"]->IntValue, 10); - EXPECT_EQ(deserialized->YsonSerializableValue->IntValue, 2); - - } -} - -TEST(TYsonStructTest, YsonSerializableNestedToYsonStructDeserializesFromEmpty) -{ - { - auto testInput = TYsonString(TStringBuf("{yson_serializable_value={}}")); - auto deserialized = ConvertTo<TIntrusivePtr<TYsonStructClass>>(testInput); - EXPECT_EQ(deserialized->YsonSerializableValue->IntValue, 5); - } -} -//////////////////////////////////////////////////////////////////////////////// - class TClassLevelPostprocessConfig : public TYsonStruct { @@ -1394,43 +1272,6 @@ TEST(TYsonStructTest, RecursiveConfig) //////////////////////////////////////////////////////////////////////////////// -class TNestedYsonSerializableClass - : public TYsonSerializable -{ -public: - TNestedYsonSerializableClass() - { - RegisterParameter("int_value", IntValue) - .Default(1); - RegisterPostprocessor([&] { - IntValue = 10; - }); - } - - int IntValue; -}; - -class TYsonStructClass2 - : public TYsonStruct -{ -public: - THashMap<TString, TIntrusivePtr<TNestedYsonSerializableClass>> YsonSerializableHashMap; - - REGISTER_YSON_STRUCT(TYsonStructClass2); - - static void Register(TRegistrar registrar) - { - registrar.Parameter("yson_serializable_hash_map", &TYsonStructClass2::YsonSerializableHashMap) - .Default(); - } -}; - -TEST(TYsonStructTest, PostprocessIsPropagatedFromYsonStructToYsonSerializable) -{ - auto testInput = TYsonString(TStringBuf("{yson_serializable_hash_map={x={int_value=2}}}")); - auto deserialized = ConvertTo<TIntrusivePtr<TYsonStructClass2>>(testInput); - EXPECT_EQ(deserialized->YsonSerializableHashMap["x"]->IntValue, 10); -} template <class T> TIntrusivePtr<T> CreateCustomDefault() @@ -1444,15 +1285,12 @@ class TYsonStructWithNestedStructsAndCustomDefaults : public TYsonStruct { public: - TIntrusivePtr<TSimpleYsonSerializable> YsonSerializable; TIntrusivePtr<TSimpleYsonStruct> YsonStruct; REGISTER_YSON_STRUCT(TYsonStructWithNestedStructsAndCustomDefaults); static void Register(TRegistrar registrar) { - registrar.Parameter("yson_serializable", &TThis::YsonSerializable) - .DefaultCtor([] () { return CreateCustomDefault<TSimpleYsonSerializable>(); }); registrar.Parameter("yson_struct", &TThis::YsonStruct) .DefaultCtor([] () { return CreateCustomDefault<TSimpleYsonStruct>(); }); } @@ -1461,7 +1299,6 @@ public: TEST(TYsonStructTest, TestCustomDefaultsOfNestedStructsAreNotDiscardedOnDeserialize) { auto deserialized = ConvertTo<TIntrusivePtr<TYsonStructWithNestedStructsAndCustomDefaults>>(TYsonString(TStringBuf("{}"))); - EXPECT_EQ(deserialized->YsonSerializable->IntValue, 10); EXPECT_EQ(deserialized->YsonStruct->IntValue, 10); } @@ -1471,7 +1308,6 @@ class TYsonStructWithNestedStructsAndPreprocessors : public TYsonStruct { public: - TIntrusivePtr<TSimpleYsonSerializable> YsonSerializable; TIntrusivePtr<TSimpleYsonStruct> YsonStruct; REGISTER_YSON_STRUCT(TYsonStructWithNestedStructsAndPreprocessors); @@ -1480,10 +1316,7 @@ public: { registrar.Parameter("yson_struct", &TThis::YsonStruct) .Default(); - registrar.Parameter("yson_serializable", &TThis::YsonSerializable) - .Default(); registrar.Preprocessor([] (TThis* s) { - s->YsonSerializable = CreateCustomDefault<TSimpleYsonSerializable>(); s->YsonStruct = CreateCustomDefault<TSimpleYsonStruct>(); }); } @@ -1492,7 +1325,6 @@ public: TEST(TYsonStructTest, TestPreprocessorsEffectsOnNestedStructsArePreservedOnDeserialize) { auto deserialized = ConvertTo<TIntrusivePtr<TYsonStructWithNestedStructsAndPreprocessors>>(TYsonString(TStringBuf("{}"))); - EXPECT_EQ(deserialized->YsonSerializable->IntValue, 10); EXPECT_EQ(deserialized->YsonStruct->IntValue, 10); } diff --git a/yt/yt/core/ytree/yson_schema-inl.h b/yt/yt/core/ytree/yson_schema-inl.h index ae8e3d5966..821048f19f 100644 --- a/yt/yt/core/ytree/yson_schema-inl.h +++ b/yt/yt/core/ytree/yson_schema-inl.h @@ -21,9 +21,6 @@ template <class T> concept CIsEnum = TEnumTraits<T>::IsEnum; template <class T> -concept CIsYsonStruct = std::derived_from<T, TYsonStructBase>; - -template <class T> concept CIsProtobufMessage = std::derived_from<std::decay_t<T>, google::protobuf::Message>; template <class T> @@ -92,7 +89,7 @@ void WriteSchema(const T&, NYson::IYsonConsumer* consumer) .EndMap(); } -template <CIsYsonStruct T> +template <CYsonStructDerived T> void WriteSchema(const NYT::TIntrusivePtr<T>& value, NYson::IYsonConsumer* consumer) { BuildYsonFluently(consumer) @@ -104,7 +101,7 @@ void WriteSchema(const NYT::TIntrusivePtr<T>& value, NYson::IYsonConsumer* consu .EndMap(); } -template <CIsYsonStruct T> +template <CYsonStructDerived T> void WriteSchema(const T& value, NYson::IYsonConsumer* consumer) { return value.WriteSchema(consumer); diff --git a/yt/yt/core/ytree/yson_serializable-inl.h b/yt/yt/core/ytree/yson_serializable-inl.h deleted file mode 100644 index a53c6aa26a..0000000000 --- a/yt/yt/core/ytree/yson_serializable-inl.h +++ /dev/null @@ -1,1037 +0,0 @@ -#ifndef YSON_SERIALIZABLE_INL_H_ -#error "Direct inclusion of this file is not allowed, include yson_serializable.h" -// For the sake of sane code completion. -#include "yson_serializable.h" -#endif - -#include "convert.h" -#include "serialize.h" -#include "tree_visitor.h" - -#include <yt/yt/core/yson/consumer.h> - -#include <yt/yt/core/misc/guid.h> -#include <yt/yt/core/misc/serialize.h> - -#include <yt/yt/core/ypath/token.h> - -#include <yt/yt/core/ytree/ypath_client.h> -#include <yt/yt/core/ytree/convert.h> - -#include <yt/yt/core/actions/bind.h> - -#include <library/cpp/yt/misc/enum.h> -#include <library/cpp/yt/misc/wrapper_traits.h> - -#include <util/datetime/base.h> - -#include <optional> - -namespace NYT::NYTree { - -//////////////////////////////////////////////////////////////////////////////// - -namespace NDetail { - -template <class T> -concept IsYsonStructOrYsonSerializable = std::derived_from<T, TYsonStructBase> || std::derived_from<T, TYsonSerializableLite>; - -template <class T> -void LoadFromNode( - T& parameter, - NYTree::INodePtr node, - const NYPath::TYPath& path, - EMergeStrategy /*mergeStrategy*/, - bool /*keepUnrecognizedRecursively*/) -{ - try { - Deserialize(parameter, node); - } catch (const std::exception& ex) { - THROW_ERROR_EXCEPTION("Error reading parameter %v", path) - << ex; - } -} - -// INodePtr -template <> -inline void LoadFromNode( - NYTree::INodePtr& parameter, - NYTree::INodePtr node, - const NYPath::TYPath& /*path*/, - EMergeStrategy mergeStrategy, - bool /*keepUnrecognizedRecursively*/) -{ - switch (mergeStrategy) { - case EMergeStrategy::Default: - case EMergeStrategy::Overwrite: { - parameter = node; - break; - } - - case EMergeStrategy::Combine: { - if (!parameter) { - parameter = node; - } else { - parameter = PatchNode(parameter, node); - } - break; - } - - default: - YT_UNIMPLEMENTED(); - } -} - -// TYsonSerializable -template <IsYsonStructOrYsonSerializable T> -void LoadFromNode( - TIntrusivePtr<T>& parameter, - NYTree::INodePtr node, - const NYPath::TYPath& path, - EMergeStrategy mergeStrategy, - bool keepUnrecognizedRecursively) -{ - if (!parameter || mergeStrategy == EMergeStrategy::Overwrite) { - parameter = New<T>(); - } - - if (keepUnrecognizedRecursively) { - parameter->SetUnrecognizedStrategy(EUnrecognizedStrategy::KeepRecursive); - } - - switch (mergeStrategy) { - case EMergeStrategy::Default: - case EMergeStrategy::Overwrite: - case EMergeStrategy::Combine: { - parameter->Load(node, false, false, path); - break; - } - - default: - YT_UNIMPLEMENTED(); - } -} - -// std::optional -template <class T> -void LoadFromNode( - std::optional<T>& parameter, - NYTree::INodePtr node, - const NYPath::TYPath& path, - EMergeStrategy mergeStrategy, - bool keepUnrecognizedRecursively) -{ - switch (mergeStrategy) { - case EMergeStrategy::Default: - case EMergeStrategy::Overwrite: { - if (node->GetType() == NYTree::ENodeType::Entity) { - parameter = std::nullopt; - } else { - T value; - LoadFromNode(value, node, path, EMergeStrategy::Overwrite, keepUnrecognizedRecursively); - parameter = std::move(value); - } - break; - } - - default: - YT_UNIMPLEMENTED(); - } -} - -// std::vector -template <class... T> -void LoadFromNode( - std::vector<T...>& parameter, - NYTree::INodePtr node, - const NYPath::TYPath& path, - EMergeStrategy mergeStrategy, - bool keepUnrecognizedRecursively) -{ - switch (mergeStrategy) { - case EMergeStrategy::Default: - case EMergeStrategy::Overwrite: { - auto listNode = node->AsList(); - auto size = listNode->GetChildCount(); - parameter.resize(size); - for (int i = 0; i < size; ++i) { - LoadFromNode( - parameter[i], - listNode->GetChildOrThrow(i), - path + "/" + NYPath::ToYPathLiteral(i), - EMergeStrategy::Overwrite, - keepUnrecognizedRecursively); - } - break; - } - - default: - YT_UNIMPLEMENTED(); - } -} - -template <class T> -T DeserializeMapKey(TStringBuf value) -{ - if constexpr (TEnumTraits<T>::IsEnum) { - return ParseEnum<T>(value); - } else if constexpr (std::is_same_v<T, TGuid>) { - return TGuid::FromString(value); - } else { - return FromString<T>(value); - } -} - -// For any map. -template <template <typename...> class Map, class... T, class M = typename Map<T...>::mapped_type> -void LoadFromNode( - Map<T...>& parameter, - NYTree::INodePtr node, - const NYPath::TYPath& path, - EMergeStrategy mergeStrategy, - bool keepUnrecognizedRecursively) -{ - switch (mergeStrategy) { - case EMergeStrategy::Default: - case EMergeStrategy::Overwrite: { - auto mapNode = node->AsMap(); - parameter.clear(); - for (const auto& [key, child] : mapNode->GetChildren()) { - M value; - LoadFromNode( - value, - child, - path + "/" + NYPath::ToYPathLiteral(key), - EMergeStrategy::Overwrite, - keepUnrecognizedRecursively); - parameter.emplace(DeserializeMapKey<typename Map<T...>::key_type>(key), std::move(value)); - } - break; - } - case EMergeStrategy::Combine: { - auto mapNode = node->AsMap(); - for (const auto& [key, child] : mapNode->GetChildren()) { - M value; - LoadFromNode( - value, - child, - path + "/" + NYPath::ToYPathLiteral(key), - EMergeStrategy::Combine, - keepUnrecognizedRecursively); - parameter[DeserializeMapKey<typename Map<T...>::key_type>(key)] = std::move(value); - } - break; - } - - default: - YT_UNIMPLEMENTED(); - } -} -//////////////////////////////////////////////////////////////////////////////// - -template <class T> -void LoadFromCursor( - T& parameter, - NYson::TYsonPullParserCursor* cursor, - const NYPath::TYPath& path, - EMergeStrategy /*mergeStrategy*/, - bool /*keepUnrecognizedRecursively*/) -{ - try { - Deserialize(parameter, cursor); - } catch (const std::exception& ex) { - THROW_ERROR_EXCEPTION("Error reading parameter %v", path) - << ex; - } -} - -//////////////////////////////////////////////////////////////////////////////// - -template <IsYsonStructOrYsonSerializable T> -void LoadFromCursor( - TIntrusivePtr<T>& parameterValue, - NYson::TYsonPullParserCursor* cursor, - const NYPath::TYPath& path, - EMergeStrategy mergeStrategy, - bool keepUnrecognizedRecursively); - -template <class... T> -void LoadFromCursor( - std::vector<T...>& parameter, - NYson::TYsonPullParserCursor* cursor, - const NYPath::TYPath& path, - EMergeStrategy mergeStrategy, - bool keepUnrecognizedRecursively); - -// std::optional -template <class T> -void LoadFromCursor( - std::optional<T>& parameter, - NYson::TYsonPullParserCursor* cursor, - const NYPath::TYPath& path, - EMergeStrategy mergeStrategy, - bool keepUnrecognizedRecursively); - -template <template <typename...> class Map, class... T, class M = typename Map<T...>::mapped_type> -void LoadFromCursor( - Map<T...>& parameter, - NYson::TYsonPullParserCursor* cursor, - const NYPath::TYPath& path, - EMergeStrategy mergeStrategy, - bool keepUnrecognizedRecursively); - -//////////////////////////////////////////////////////////////////////////////// - -// INodePtr -template <> -inline void LoadFromCursor( - NYTree::INodePtr& parameter, - NYson::TYsonPullParserCursor* cursor, - const NYPath::TYPath& path, - EMergeStrategy mergeStrategy, - bool keepUnrecognizedRecursively) -{ - try { - auto node = NYson::ExtractTo<INodePtr>(cursor); - LoadFromNode(parameter, std::move(node), path, mergeStrategy, keepUnrecognizedRecursively); - } catch (const std::exception& ex) { - THROW_ERROR_EXCEPTION("Error loading parameter %v", path) - << ex; - } -} - -// TYsonStruct or TYsonSerializable -template <IsYsonStructOrYsonSerializable T> -void LoadFromCursor( - TIntrusivePtr<T>& parameterValue, - NYson::TYsonPullParserCursor* cursor, - const NYPath::TYPath& path, - EMergeStrategy mergeStrategy, - bool keepUnrecognizedRecursively) -{ - if (!parameterValue || mergeStrategy == EMergeStrategy::Overwrite) { - parameterValue = New<T>(); - } - - if (keepUnrecognizedRecursively) { - parameterValue->SetUnrecognizedStrategy(EUnrecognizedStrategy::KeepRecursive); - } - - switch (mergeStrategy) { - case EMergeStrategy::Default: - case EMergeStrategy::Overwrite: - case EMergeStrategy::Combine: { - parameterValue->Load(cursor, false, false, path); - break; - } - - default: - YT_UNIMPLEMENTED(); - } -} - -// std::optional -template <class T> -void LoadFromCursor( - std::optional<T>& parameter, - NYson::TYsonPullParserCursor* cursor, - const NYPath::TYPath& path, - EMergeStrategy mergeStrategy, - bool keepUnrecognizedRecursively) -{ - try { - switch (mergeStrategy) { - case EMergeStrategy::Default: - case EMergeStrategy::Overwrite: { - if ((*cursor)->GetType() == NYson::EYsonItemType::EntityValue) { - parameter = std::nullopt; - cursor->Next(); - } else { - T value; - LoadFromCursor(value, cursor, path, EMergeStrategy::Overwrite, keepUnrecognizedRecursively); - parameter = std::move(value); - } - break; - } - - default: - YT_UNIMPLEMENTED(); - } - } catch (const std::exception& ex) { - THROW_ERROR_EXCEPTION("Error loading parameter %v", path) - << ex; - } -} - -// std::vector -template <class... T> -void LoadFromCursor( - std::vector<T...>& parameter, - NYson::TYsonPullParserCursor* cursor, - const NYPath::TYPath& path, - EMergeStrategy mergeStrategy, - bool keepUnrecognizedRecursively) -{ - try { - switch (mergeStrategy) { - case EMergeStrategy::Default: - case EMergeStrategy::Overwrite: { - parameter.clear(); - int index = 0; - cursor->ParseList([&](NYson::TYsonPullParserCursor* cursor) { - LoadFromCursor( - parameter.emplace_back(), - cursor, - path + "/" + NYPath::ToYPathLiteral(index), - EMergeStrategy::Overwrite, - keepUnrecognizedRecursively); - ++index; - }); - break; - } - - default: - YT_UNIMPLEMENTED(); - } - } catch (const std::exception& ex) { - THROW_ERROR_EXCEPTION("Error loading parameter %v", path) - << ex; - } -} - -template <class TMapping, class TValue, class TEmplacer, class TSetter> -void LoadMappingFromCursor( - TMapping& mapping, - TEmplacer emplacer, - TSetter setter, - NYson::TYsonPullParserCursor* cursor, - const NYPath::TYPath& path, - EMergeStrategy mergeStrategy, - bool keepUnrecognizedRecursively) -{ - try { - auto doParse = [&] (auto setterOrEmplacer, EMergeStrategy mergeStrategy) { - cursor->ParseMap([&] (NYson::TYsonPullParserCursor* cursor) { - auto key = ExtractTo<TString>(cursor); - TValue value; - LoadFromCursor( - value, - cursor, - path + "/" + NYPath::ToYPathLiteral(key), - mergeStrategy, - keepUnrecognizedRecursively); - setterOrEmplacer(mapping, key, std::move(value)); - }); - }; - - switch (mergeStrategy) { - case EMergeStrategy::Default: - case EMergeStrategy::Overwrite: { - mapping = {}; - doParse(emplacer, EMergeStrategy::Overwrite); - break; - } - case EMergeStrategy::Combine: { - doParse(setter, EMergeStrategy::Combine); - break; - } - default: - YT_UNIMPLEMENTED(); - } - } catch (const std::exception& ex) { - THROW_ERROR_EXCEPTION("Error loading parameter %v", path) - << ex; - } -} - -// For any map. -template <template <typename...> class Map, class... T, class M> -void LoadFromCursor( - Map<T...>& parameter, - NYson::TYsonPullParserCursor* cursor, - const NYPath::TYPath& path, - EMergeStrategy mergeStrategy, - bool keepUnrecognizedRecursively) -{ - try { - auto doParse = [&] (const auto& setterOrEmplacer, EMergeStrategy mergeStrategy) { - cursor->ParseMap([&] (NYson::TYsonPullParserCursor* cursor) { - auto key = ExtractTo<TString>(cursor); - M value; - LoadFromCursor( - value, - cursor, - path + "/" + NYPath::ToYPathLiteral(key), - mergeStrategy, - keepUnrecognizedRecursively); - setterOrEmplacer(key, std::move(value)); - }); - }; - - switch (mergeStrategy) { - case EMergeStrategy::Default: - case EMergeStrategy::Overwrite: { - parameter.clear(); - auto emplacer = [&] (auto key, M&& value) { - parameter.emplace(DeserializeMapKey<typename Map<T...>::key_type>(key), std::move(value)); - }; - doParse(emplacer, EMergeStrategy::Overwrite); - break; - } - case EMergeStrategy::Combine: { - auto setter = [&] (auto key, M&& value) { - parameter[DeserializeMapKey<typename Map<T...>::key_type>(key)] = std::move(value); - }; - doParse(setter, EMergeStrategy::Combine); - break; - } - default: - YT_UNIMPLEMENTED(); - } - } catch (const std::exception& ex) { - THROW_ERROR_EXCEPTION("Error loading parameter %v", path) - << ex; - } -} - -//////////////////////////////////////////////////////////////////////////////// - -// For all classes except descendants of TYsonSerializableLite and their intrusive pointers -// we do not attempt to extract unrecognzied members. C++ prohibits function template specialization -// so we have to deal with static struct members. -template <class T> -struct TGetUnrecognizedRecursively -{ - static IMapNodePtr Do(const T& /*parameter*/) - { - return nullptr; - } -}; - -template <IsYsonStructOrYsonSerializable T> -struct TGetUnrecognizedRecursively<T> -{ - static IMapNodePtr Do(const T& parameter) - { - return parameter.GetRecursiveUnrecognized(); - } -}; - -template <IsYsonStructOrYsonSerializable T> -struct TGetUnrecognizedRecursively<TIntrusivePtr<T>> -{ - static IMapNodePtr Do(const TIntrusivePtr<T>& parameter) - { - return parameter ? parameter->GetRecursiveUnrecognized() : nullptr; - } -}; - -//////////////////////////////////////////////////////////////////////////////// - -// all -template <class F> -void InvokeForComposites( - const void* /*parameter*/, - const NYPath::TYPath& /*path*/, - const F& /*func*/) -{ } - -// TYsonSerializable or TYsonStruct -template <IsYsonStructOrYsonSerializable T, class F> -inline void InvokeForComposites( - const TIntrusivePtr<T>* parameter, - const NYPath::TYPath& path, - const F& func) -{ - func(*parameter, path); -} - -// std::vector -template <class... T, class F> -inline void InvokeForComposites( - const std::vector<T...>* parameter, - const NYPath::TYPath& path, - const F& func) -{ - for (size_t i = 0; i < parameter->size(); ++i) { - InvokeForComposites( - &(*parameter)[i], - path + "/" + NYPath::ToYPathLiteral(i), - func); - } -} - -// For any map. -template <template<typename...> class Map, class... T, class F, class M = typename Map<T...>::mapped_type> -inline void InvokeForComposites( - const Map<T...>* parameter, - const NYPath::TYPath& path, - const F& func) -{ - for (const auto& [key, value] : *parameter) { - InvokeForComposites( - &value, - path + "/" + NYPath::ToYPathLiteral(key), - func); - } -} - -//////////////////////////////////////////////////////////////////////////////// - -// all -template <class F> -void InvokeForComposites( - const void* /*parameter*/, - const F& /*func*/) -{ } - -// TYsonStruct or TYsonSerializable -template <IsYsonStructOrYsonSerializable T, class F> -inline void InvokeForComposites(const TIntrusivePtr<T>* parameter, const F& func) -{ - func(*parameter); -} - -// std::vector -template <class... T, class F> -inline void InvokeForComposites(const std::vector<T...>* parameter, const F& func) -{ - for (const auto& item : *parameter) { - InvokeForComposites(&item, func); - } -} - -// For any map. -template <template<typename...> class Map, class... T, class F, class M = typename Map<T...>::mapped_type> -inline void InvokeForComposites(const Map<T...>* parameter, const F& func) -{ - for (const auto& [key, value] : *parameter) { - InvokeForComposites(&value, func); - } -} - -// TODO(shakurov): get rid of this once concept support makes it into the standard -// library implementation. Use equality-comparability instead. -template <class T> -concept SupportsDontSerializeDefaultImpl = - std::is_arithmetic_v<T> || - std::is_same_v<T, TString> || - std::is_same_v<T, TDuration> || - std::is_same_v<T, TGuid> || - std::is_same_v<T, std::optional<std::vector<TString>>> || - std::is_same_v<T, THashSet<TString>>; - -template <class T> -concept SupportsDontSerializeDefault = - SupportsDontSerializeDefaultImpl<typename TWrapperTraits<T>::TRecursiveUnwrapped>; - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NDetail - -template <class T> -TYsonSerializableLite::TParameter<T>::TParameter(TString key, T& parameter) - : Key(std::move(key)) - , Parameter(parameter) - , MergeStrategy(EMergeStrategy::Default) -{ } - -template <class T> -void TYsonSerializableLite::TParameter<T>::Load( - NYTree::INodePtr node, - const NYPath::TYPath& path, - std::optional<EMergeStrategy> mergeStrategy) -{ - if (node) { - NDetail::LoadFromNode( - Parameter, - node, - path, - mergeStrategy.value_or(MergeStrategy), - KeepUnrecognizedRecursively); - } else if (!DefaultValue) { - THROW_ERROR_EXCEPTION("Missing required parameter %v", - path); - } -} - -template <class T> -void TYsonSerializableLite::TParameter<T>::SafeLoad( - NYTree::INodePtr node, - const NYPath::TYPath& path, - const std::function<void()>& validate, - std::optional<EMergeStrategy> mergeStrategy) -{ - if (node) { - T oldValue = Parameter; - try { - NDetail::LoadFromNode( - Parameter, - node, - path, - mergeStrategy.value_or(MergeStrategy), - KeepUnrecognizedRecursively); - validate(); - } catch (const std::exception&) { - Parameter = std::move(oldValue); - throw; - } - } -} - - -template <class T> -void TYsonSerializableLite::TParameter<T>::Load( - NYson::TYsonPullParserCursor* cursor, - const NYPath::TYPath& path, - std::optional<EMergeStrategy> mergeStrategy) -{ - if (cursor) { - NDetail::LoadFromCursor( - Parameter, - cursor, - path, - mergeStrategy.value_or(MergeStrategy), - KeepUnrecognizedRecursively); - } else if (!DefaultValue) { - THROW_ERROR_EXCEPTION("Missing required parameter %v", - path); - } -} - -template <class T> -void TYsonSerializableLite::TParameter<T>::SafeLoad( - NYson::TYsonPullParserCursor* cursor, - const NYPath::TYPath& path, - const std::function<void()>& validate, - std::optional<EMergeStrategy> mergeStrategy) -{ - if (cursor) { - T oldValue = Parameter; - try { - NDetail::LoadFromCursor( - Parameter, - cursor, - path, - mergeStrategy.value_or(MergeStrategy), - KeepUnrecognizedRecursively); - validate(); - } catch (const std::exception&) { - Parameter = std::move(oldValue); - throw; - } - } -} - -template <class T> -void TYsonSerializableLite::TParameter<T>::Postprocess(const NYPath::TYPath& path) const -{ - for (const auto& postprocessor : Postprocessors) { - try { - postprocessor(Parameter); - } catch (const std::exception& ex) { - THROW_ERROR_EXCEPTION("Postprocess failed at %v", - path.empty() ? "root" : path) - << ex; - } - } - - NYT::NYTree::NDetail::InvokeForComposites( - &Parameter, - path, - [] <NDetail::IsYsonStructOrYsonSerializable TStruct> (TIntrusivePtr<TStruct> obj, const NYPath::TYPath& subpath) { - if (obj) { - obj->Postprocess(subpath); - } - }); -} - -template <class T> -void TYsonSerializableLite::TParameter<T>::SetDefaults() -{ - if (DefaultValue) { - Parameter = *DefaultValue; - } - - NYT::NYTree::NDetail::InvokeForComposites( - &Parameter, - [] <NDetail::IsYsonStructOrYsonSerializable TStruct> (TIntrusivePtr<TStruct> obj) { - if (obj) { - obj->SetDefaults(); - } - }); -} - -template <class T> -void TYsonSerializableLite::TParameter<T>::Save(NYson::IYsonConsumer* consumer) const -{ - using NYTree::Serialize; - Serialize(Parameter, consumer); -} - -template <class T> -bool TYsonSerializableLite::TParameter<T>::CanOmitValue() const -{ - if constexpr (NDetail::SupportsDontSerializeDefault<T>) { - if (!SerializeDefault && Parameter == DefaultValue) { - return true; - } - } - - return NYT::NYTree::NDetail::CanOmitValue(&Parameter, DefaultValue ? &*DefaultValue : nullptr); -} - -template <class T> -TYsonSerializableLite::TParameter<T>& TYsonSerializableLite::TParameter<T>::Alias(const TString& name) -{ - Aliases.push_back(name); - return *this; -} - -template <class T> -const std::vector<TString>& TYsonSerializableLite::TParameter<T>::GetAliases() const -{ - return Aliases; -} - -template <class T> -const TString& TYsonSerializableLite::TParameter<T>::GetKey() const -{ - return Key; -} - -template <class T> -TYsonSerializableLite::TParameter<T>& TYsonSerializableLite::TParameter<T>::Optional() -{ - DefaultValue = Parameter; - return *this; -} - -template <class T> -TYsonSerializableLite::TParameter<T>& TYsonSerializableLite::TParameter<T>::Default(const T& defaultValue) -{ - DefaultValue = defaultValue; - Parameter = defaultValue; - return *this; -} - -template <class T> -TYsonSerializableLite::TParameter<T>& TYsonSerializableLite::TParameter<T>::DontSerializeDefault() -{ - // We should check for equality-comparability here but it is rather hard - // to do the deep validation. - static_assert( - NDetail::SupportsDontSerializeDefault<T>, - "DontSerializeDefault requires |Parameter| to be TString, TDuration, an arithmetic type or an optional of those"); - - SerializeDefault = false; - return *this; -} - -template <class T> -template <class... TArgs> -TYsonSerializableLite::TParameter<T>& TYsonSerializableLite::TParameter<T>::DefaultNew(TArgs&&... args) -{ - return Default(New<typename T::TUnderlying>(std::forward<TArgs>(args)...)); -} - -template <class T> -TYsonSerializableLite::TParameter<T>& TYsonSerializableLite::TParameter<T>::CheckThat(TPostprocessor postprocessor) -{ - Postprocessors.push_back(std::move(postprocessor)); - return *this; -} - -template <class T> -TYsonSerializableLite::TParameter<T>& TYsonSerializableLite::TParameter<T>::MergeBy(EMergeStrategy strategy) -{ - MergeStrategy = strategy; - return *this; -} - -template <class T> -IMapNodePtr TYsonSerializableLite::TParameter<T>::GetUnrecognizedRecursively() const -{ - return NDetail::TGetUnrecognizedRecursively<T>::Do(Parameter); -} - -template <class T> -void TYsonSerializableLite::TParameter<T>::SetKeepUnrecognizedRecursively() -{ - KeepUnrecognizedRecursively = true; -} - -//////////////////////////////////////////////////////////////////////////////// -// Standard postprocessors - -#define DEFINE_POSTPROCESSOR(method, condition, error) \ - template <class T> \ - TYsonSerializableLite::TParameter<T>& TYsonSerializableLite::TParameter<T>::method \ - { \ - return CheckThat([=] (const T& parameter) { \ - using ::ToString; \ - std::optional<TValueType> nullableParameter(parameter); \ - if (nullableParameter) { \ - const auto& actual = *nullableParameter; \ - if (!(condition)) { \ - THROW_ERROR error; \ - } \ - } \ - }); \ - } - -DEFINE_POSTPROCESSOR( - GreaterThan(TValueType expected), - actual > expected, - TError("Expected > %v, found %v", expected, actual) -) - -DEFINE_POSTPROCESSOR( - GreaterThanOrEqual(TValueType expected), - actual >= expected, - TError("Expected >= %v, found %v", expected, actual) -) - -DEFINE_POSTPROCESSOR( - LessThan(TValueType expected), - actual < expected, - TError("Expected < %v, found %v", expected, actual) -) - -DEFINE_POSTPROCESSOR( - LessThanOrEqual(TValueType expected), - actual <= expected, - TError("Expected <= %v, found %v", expected, actual) -) - -DEFINE_POSTPROCESSOR( - InRange(TValueType lowerBound, TValueType upperBound), - lowerBound <= actual && actual <= upperBound, - TError("Expected in range [%v,%v], found %v", lowerBound, upperBound, actual) -) - -DEFINE_POSTPROCESSOR( - NonEmpty(), - actual.size() > 0, - TError("Value must not be empty") -) - -#undef DEFINE_POSTPROCESSOR - -//////////////////////////////////////////////////////////////////////////////// - -template <class T> -TYsonSerializableLite::TParameter<T>& TYsonSerializableLite::RegisterParameter( - TString parameterName, - T& value) -{ - auto parameter = New<TParameter<T>>(parameterName, value); - if (UnrecognizedStrategy == EUnrecognizedStrategy::KeepRecursive) { - parameter->SetKeepUnrecognizedRecursively(); - } - YT_VERIFY(Parameters.emplace(std::move(parameterName), parameter).second); - return *parameter; -} - -//////////////////////////////////////////////////////////////////////////////// - -template <class T> -TIntrusivePtr<T> CloneYsonSerializable(const TIntrusivePtr<T>& obj) -{ - static_assert( - std::is_convertible_v<T*, TYsonSerializable*>, - "'obj' must be convertible to TYsonSerializable"); - - return NYTree::ConvertTo<TIntrusivePtr<T>>(NYson::ConvertToYsonString(*obj)); -} - -template <class T> -std::vector<TIntrusivePtr<T>> CloneYsonSerializables(const std::vector<TIntrusivePtr<T>>& objs) -{ - std::vector<TIntrusivePtr<T>> clonedObjs; - clonedObjs.reserve(objs.size()); - for (const auto& obj : objs) { - clonedObjs.push_back(CloneYsonSerializable(obj)); - } - return clonedObjs; -} - -template <class T> -THashMap<TString, TIntrusivePtr<T>> CloneYsonSerializables(const THashMap<TString, TIntrusivePtr<T>>& objs) -{ - THashMap<TString, TIntrusivePtr<T>> clonedObjs; - clonedObjs.reserve(objs.size()); - for (const auto& [key, obj] : objs) { - clonedObjs.emplace(key, CloneYsonSerializable(obj)); - } - return clonedObjs; -} - -template <class T> -TIntrusivePtr<T> UpdateYsonSerializable( - const TIntrusivePtr<T>& obj, - const NYTree::INodePtr& patch) -{ - static_assert( - std::is_convertible_v<T*, TYsonSerializable*>, - "'obj' must be convertible to TYsonSerializable"); - - using NYTree::INodePtr; - using NYTree::ConvertTo; - - if (patch) { - return ConvertTo<TIntrusivePtr<T>>(PatchNode(ConvertTo<INodePtr>(obj), patch)); - } else { - return CloneYsonSerializable(obj); - } -} - -template <class T> -TIntrusivePtr<T> UpdateYsonSerializable( - const TIntrusivePtr<T>& obj, - const NYson::TYsonString& patch) -{ - if (!patch) { - return obj; - } - - return UpdateYsonSerializable(obj, ConvertToNode(patch)); -} - -template <class T> -bool ReconfigureYsonSerializable( - const TIntrusivePtr<T>& config, - const NYson::TYsonString& newConfigYson) -{ - return ReconfigureYsonSerializable(config, ConvertToNode(newConfigYson)); -} - -template <class T> -bool ReconfigureYsonSerializable( - const TIntrusivePtr<T>& config, - const TIntrusivePtr<T>& newConfig) -{ - return ReconfigureYsonSerializable(config, ConvertToNode(newConfig)); -} - -template <class T> -bool ReconfigureYsonSerializable( - const TIntrusivePtr<T>& config, - const NYTree::INodePtr& newConfigNode) -{ - auto configNode = NYTree::ConvertToNode(config); - - auto newConfig = NYTree::ConvertTo<TIntrusivePtr<T>>(newConfigNode); - auto newCanonicalConfigNode = NYTree::ConvertToNode(newConfig); - - if (NYTree::AreNodesEqual(configNode, newCanonicalConfigNode)) { - return false; - } - - config->Load(newConfigNode); - return true; -} - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NYTree diff --git a/yt/yt/core/ytree/yson_serializable.cpp b/yt/yt/core/ytree/yson_serializable.cpp deleted file mode 100644 index f9557e1988..0000000000 --- a/yt/yt/core/ytree/yson_serializable.cpp +++ /dev/null @@ -1,385 +0,0 @@ -#include "yson_serializable.h" - -#include <yt/yt/core/yson/consumer.h> - -#include <yt/yt/core/yson/consumer.h> -#include <yt/yt/core/yson/token_writer.h> - -#include <yt/yt/core/ytree/ephemeral_node_factory.h> -#include <yt/yt/core/ytree/node.h> -#include <yt/yt/core/ytree/ypath_detail.h> - -#include <util/generic/algorithm.h> - -namespace NYT::NYTree { - -using namespace NYPath; -using namespace NYson; - -//////////////////////////////////////////////////////////////////////////////// - -TYsonSerializableLite::TYsonSerializableLite() -{ } - -IMapNodePtr TYsonSerializableLite::GetUnrecognized() const -{ - return Unrecognized; -} - -IMapNodePtr TYsonSerializableLite::GetRecursiveUnrecognized() const -{ - return GetUnrecognizedRecursively(); -} - -IMapNodePtr TYsonSerializableLite::GetUnrecognizedRecursively() const -{ - // Take a copy of `Unrecognized` and add parameter->GetUnrecognizedRecursively() - // for all parameters that are TYsonSerializable's themselves. - auto result = Unrecognized ? ConvertTo<IMapNodePtr>(Unrecognized) : GetEphemeralNodeFactory()->CreateMap(); - for (const auto& [name, parameter] : Parameters) { - auto unrecognized = parameter->GetUnrecognizedRecursively(); - if (unrecognized && unrecognized->AsMap()->GetChildCount() > 0) { - result->AddChild(name, unrecognized); - } - } - return result; -} - -void TYsonSerializableLite::SetUnrecognizedStrategy(EUnrecognizedStrategy strategy) -{ - UnrecognizedStrategy = strategy; - if (strategy == EUnrecognizedStrategy::KeepRecursive) { - for (const auto& [name, parameter] : Parameters) { - parameter->SetKeepUnrecognizedRecursively(); - } - } -} - -THashSet<TString> TYsonSerializableLite::GetRegisteredKeys() const -{ - THashSet<TString> result(Parameters.size()); - for (const auto& [name, parameter] : Parameters) { - result.insert(name); - for (const auto& alias : parameter->GetAliases()) { - result.insert(alias); - } - } - return result; -} - -void TYsonSerializableLite::Load( - INodePtr node, - bool postprocess, - bool setDefaults, - const TYPath& path) -{ - YT_VERIFY(node); - - if (setDefaults) { - SetDefaults(); - } - - auto mapNode = node->AsMap(); - for (const auto& [name, parameter] : Parameters) { - TString key = name; - auto child = mapNode->FindChild(name); // can be NULL - for (const auto& alias : parameter->GetAliases()) { - auto otherChild = mapNode->FindChild(alias); - if (child && otherChild && !AreNodesEqual(child, otherChild)) { - THROW_ERROR_EXCEPTION("Different values for aliased parameters %Qv and %Qv", key, alias) - << TErrorAttribute("main_value", child) - << TErrorAttribute("aliased_value", otherChild); - } - if (!child && otherChild) { - child = otherChild; - key = alias; - } - } - auto childPath = path + "/" + key; - parameter->Load(child, childPath); - } - - if (UnrecognizedStrategy != EUnrecognizedStrategy::Drop) { - auto registeredKeys = GetRegisteredKeys(); - if (!Unrecognized) { - Unrecognized = GetEphemeralNodeFactory()->CreateMap(); - } - for (const auto& [key, child] : mapNode->GetChildren()) { - if (registeredKeys.find(key) == registeredKeys.end()) { - Unrecognized->RemoveChild(key); - YT_VERIFY(Unrecognized->AddChild(key, ConvertToNode(child))); - } - } - } - - if (postprocess) { - Postprocess(path); - } -} - -void TYsonSerializableLite::Load( - TYsonPullParserCursor* cursor, - bool postprocess, - bool setDefaults, - const TYPath& path) -{ - YT_VERIFY(cursor); - - if (setDefaults) { - SetDefaults(); - } - - THashMap<TStringBuf, IParameter*> keyToParameter; - THashSet<IParameter*> pendingParameters; - for (const auto& [key, parameter] : Parameters) { - EmplaceOrCrash(keyToParameter, key, parameter.Get()); - for (const auto& alias : parameter->GetAliases()) { - EmplaceOrCrash(keyToParameter, alias, parameter.Get()); - } - InsertOrCrash(pendingParameters, parameter.Get()); - } - - THashMap<TString, TString> aliasedData; - - auto processPossibleAlias = [&] ( - IParameter* parameter, - TStringBuf key, - TYsonPullParserCursor* cursor) - { - TStringStream ss; - { - TCheckedInDebugYsonTokenWriter writer(&ss); - cursor->TransferComplexValue(&writer); - } - auto data = std::move(ss.Str()); - const auto& canonicalKey = parameter->GetKey(); - auto aliasedDataIt = aliasedData.find(canonicalKey); - if (aliasedDataIt != aliasedData.end()) { - auto firstNode = ConvertTo<INodePtr>(TYsonStringBuf(aliasedDataIt->second)); - auto secondNode = ConvertTo<INodePtr>(TYsonStringBuf(data)); - if (!AreNodesEqual(firstNode, secondNode)) { - THROW_ERROR_EXCEPTION("Different values for aliased parameters %Qv and %Qv", canonicalKey, key) - << TErrorAttribute("main_value", firstNode) - << TErrorAttribute("aliased_value", secondNode); - } - return; - } - { - TStringInput input(data); - TYsonPullParser parser(&input, NYson::EYsonType::Node); - TYsonPullParserCursor newCursor(&parser); - auto childPath = path + "/" + key; - parameter->Load(&newCursor, childPath); - } - EmplaceOrCrash(aliasedData, canonicalKey, std::move(data)); - }; - - auto processUnrecognized = [&, this] (const TString& key, TYsonPullParserCursor* cursor) { - if (UnrecognizedStrategy == EUnrecognizedStrategy::Drop) { - cursor->SkipComplexValue(); - return; - } - if (!Unrecognized) { - Unrecognized = GetEphemeralNodeFactory()->CreateMap(); - } - Unrecognized->RemoveChild(key); - auto added = Unrecognized->AddChild(key, ExtractTo<INodePtr>(cursor)); - YT_VERIFY(added); - }; - - cursor->ParseMap([&] (TYsonPullParserCursor* cursor) { - auto key = ExtractTo<TString>(cursor); - auto it = keyToParameter.find(key); - if (it == keyToParameter.end()) { - processUnrecognized(key, cursor); - return; - } - - auto parameter = it->second; - if (parameter->GetAliases().empty()) { - auto childPath = path + "/" + key; - parameter->Load(cursor, childPath); - } else { - processPossibleAlias(parameter, key, cursor); - } - // NB: Key may be missing in case of aliasing. - pendingParameters.erase(parameter); - }); - - for (auto parameter : pendingParameters) { - auto childPath = path + "/" + parameter->GetKey(); - parameter->Load(/*cursor*/ nullptr, childPath); - } - - if (postprocess) { - Postprocess(path); - } -} - -void TYsonSerializableLite::Save(IYsonConsumer* consumer) const -{ - consumer->OnBeginMap(); - - for (const auto& [name, parameter] : SortHashMapByKeys(Parameters)) { - if (!parameter->CanOmitValue()) { - consumer->OnKeyedItem(name); - parameter->Save(consumer); - } - } - - if (Unrecognized) { - auto children = Unrecognized->GetChildren(); - SortByFirst(children); - for (const auto& [key, child] : children) { - consumer->OnKeyedItem(key); - Serialize(child, consumer); - } - } - - consumer->OnEndMap(); -} - -void TYsonSerializableLite::Postprocess(const TYPath& path) const -{ - for (const auto& [name, parameter] : Parameters) { - parameter->Postprocess(path + "/" + name); - } - - try { - for (const auto& postprocessor : Postprocessors) { - postprocessor(); - } - } catch (const std::exception& ex) { - THROW_ERROR_EXCEPTION("Postprocess failed at %v", - path.empty() ? "root" : path) - << ex; - } -} - -void TYsonSerializableLite::SetDefaults() -{ - for (const auto& [name, parameter] : Parameters) { - parameter->SetDefaults(); - } - for (const auto& initializer : Preprocessors) { - initializer(); - } -} - -void TYsonSerializableLite::RegisterPreprocessor(const TPreprocessor& func) -{ - func(); - Preprocessors.push_back(func); -} - -void TYsonSerializableLite::RegisterPostprocessor(const TPostprocessor& func) -{ - Postprocessors.push_back(func); -} - -void TYsonSerializableLite::SaveParameter(const TString& key, IYsonConsumer* consumer) const -{ - GetParameter(key)->Save(consumer); -} - -void TYsonSerializableLite::LoadParameter(const TString& key, const NYTree::INodePtr& node, EMergeStrategy mergeStrategy) const -{ - const auto& parameter = GetParameter(key); - auto validate = [&] () { - parameter->Postprocess("/" + key); - try { - for (const auto& postprocessor : Postprocessors) { - postprocessor(); - } - } catch (const std::exception& ex) { - THROW_ERROR_EXCEPTION( - "Postprocess failed while loading parameter %Qv from value %Qv", - key, - ConvertToYsonString(node, EYsonFormat::Text)) - << ex; - } - }; - parameter->SafeLoad(node, /*path*/ "", validate, mergeStrategy); -} - -void TYsonSerializableLite::ResetParameter(const TString& key) const -{ - GetParameter(key)->SetDefaults(); -} - -TYsonSerializableLite::IParameterPtr TYsonSerializableLite::GetParameter(const TString& keyOrAlias) const -{ - auto it = Parameters.find(keyOrAlias); - if (it != Parameters.end()) { - return it->second; - } - - for (const auto& [_, parameter] : Parameters) { - if (Count(parameter->GetAliases(), keyOrAlias) > 0) { - return parameter; - } - } - THROW_ERROR_EXCEPTION("Key or alias %Qv not found in yson serializable", keyOrAlias); -} - -int TYsonSerializableLite::GetParameterCount() const -{ - return Parameters.size(); -} - -std::vector<TString> TYsonSerializableLite::GetAllParameterAliases(const TString& key) const -{ - auto parameter = GetParameter(key); - auto result = parameter->GetAliases(); - result.push_back(parameter->GetKey()); - return result; -} - -//////////////////////////////////////////////////////////////////////////////// - -void Serialize(const TYsonSerializableLite& value, IYsonConsumer* consumer) -{ - value.Save(consumer); -} - -void Deserialize(TYsonSerializableLite& value, INodePtr node) -{ - value.Load(node); -} - -void Deserialize(TYsonSerializableLite& value, NYson::TYsonPullParserCursor* cursor) -{ - value.Load(cursor); -} - -//////////////////////////////////////////////////////////////////////////////// - -DEFINE_REFCOUNTED_TYPE(TYsonSerializable) - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NYTree - - -namespace NYT { - -using namespace NYTree; - -//////////////////////////////////////////////////////////////////////////////// - -void TBinaryYsonSerializer::Save(TStreamSaveContext& context, const TYsonSerializableLite& obj) -{ - auto str = ConvertToYsonString(obj); - NYT::Save(context, str); -} - -void TBinaryYsonSerializer::Load(TStreamLoadContext& context, TYsonSerializableLite& obj) -{ - auto str = NYT::Load<TYsonString>(context); - auto node = ConvertTo<INodePtr>(str); - obj.Load(node); -} - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT diff --git a/yt/yt/core/ytree/yson_serializable.h b/yt/yt/core/ytree/yson_serializable.h deleted file mode 100644 index 50f99e47b1..0000000000 --- a/yt/yt/core/ytree/yson_serializable.h +++ /dev/null @@ -1,268 +0,0 @@ -#pragma once - -#include "public.h" -#include "node.h" -#include "yson_serialize_common.h" - -#include <yt/yt/core/misc/error.h> -#include <yt/yt/core/misc/mpl.h> -#include <yt/yt/core/misc/property.h> - -#include <yt/yt/core/yson/public.h> - -#include <functional> -#include <optional> - -namespace NYT::NYTree { - -//////////////////////////////////////////////////////////////////////////////// - -class TYsonSerializableLite - : private TNonCopyable -{ -public: - using TPostprocessor = std::function<void()>; - using TPreprocessor = std::function<void()>; - - struct IParameter - : public TRefCounted - { - virtual void Load( - NYTree::INodePtr node, - const NYPath::TYPath& path, - std::optional<EMergeStrategy> mergeStrategy = std::nullopt) = 0; - - virtual void SafeLoad( - NYTree::INodePtr node, - const NYPath::TYPath& path, - const std::function<void()>& validate, - std::optional<EMergeStrategy> mergeStrategy = std::nullopt) = 0; - - virtual void Load( - NYson::TYsonPullParserCursor* cursor, - const NYPath::TYPath& path, - std::optional<EMergeStrategy> mergeStrategy = std::nullopt) = 0; - - virtual void SafeLoad( - NYson::TYsonPullParserCursor* cursor, - const NYPath::TYPath& path, - const std::function<void()>& validate, - std::optional<EMergeStrategy> mergeStrategy = std::nullopt) = 0; - - virtual void Postprocess(const NYPath::TYPath& path) const = 0; - virtual void SetDefaults() = 0; - virtual void Save(NYson::IYsonConsumer* consumer) const = 0; - virtual bool CanOmitValue() const = 0; - virtual const TString& GetKey() const = 0; - virtual const std::vector<TString>& GetAliases() const = 0; - virtual IMapNodePtr GetUnrecognizedRecursively() const = 0; - virtual void SetKeepUnrecognizedRecursively() = 0; - }; - - using IParameterPtr = TIntrusivePtr<IParameter>; - - template <class T> - class TParameter - : public IParameter - { - public: - using TPostprocessor = std::function<void(const T&)>; - using TValueType = typename TOptionalTraits<T>::TValue; - - TParameter(TString key, T& parameter); - - void Load( - NYTree::INodePtr node, - const NYPath::TYPath& path, - std::optional<EMergeStrategy> mergeStrategy = std::nullopt) override; - - void SafeLoad( - NYTree::INodePtr node, - const NYPath::TYPath& path, - const std::function<void()>& validate, - std::optional<EMergeStrategy> mergeStrategy = std::nullopt) override; - - void Load( - NYson::TYsonPullParserCursor* cursor, - const NYPath::TYPath& path, - std::optional<EMergeStrategy> mergeStrategy = std::nullopt) override; - - void SafeLoad( - NYson::TYsonPullParserCursor* cursor, - const NYPath::TYPath& path, - const std::function<void()>& validate, - std::optional<EMergeStrategy> mergeStrategy = std::nullopt) override; - - void Postprocess(const NYPath::TYPath& path) const override; - void SetDefaults() override; - void Save(NYson::IYsonConsumer* consumer) const override; - bool CanOmitValue() const override; - const TString& GetKey() const override; - const std::vector<TString>& GetAliases() const override; - IMapNodePtr GetUnrecognizedRecursively() const override; - void SetKeepUnrecognizedRecursively() override; - - public: - TParameter& Optional(); - TParameter& Default(const T& defaultValue = T()); - TParameter& DontSerializeDefault(); - TParameter& CheckThat(TPostprocessor validator); - TParameter& GreaterThan(TValueType value); - TParameter& GreaterThanOrEqual(TValueType value); - TParameter& LessThan(TValueType value); - TParameter& LessThanOrEqual(TValueType value); - TParameter& InRange(TValueType lowerBound, TValueType upperBound); - TParameter& NonEmpty(); - TParameter& Alias(const TString& name); - TParameter& MergeBy(EMergeStrategy strategy); - - template <class... TArgs> - TParameter& DefaultNew(TArgs&&... args); - - private: - TString Key; - T& Parameter; - std::optional<T> DefaultValue; - bool SerializeDefault = true; - std::vector<TPostprocessor> Postprocessors; - std::vector<TString> Aliases; - EMergeStrategy MergeStrategy; - bool KeepUnrecognizedRecursively = false; - }; - -public: - TYsonSerializableLite(); - - void Load( - NYTree::INodePtr node, - bool postprocess = true, - bool setDefaults = true, - const NYPath::TYPath& path = ""); - - void Load( - NYson::TYsonPullParserCursor* cursor, - bool postprocess = true, - bool setDefaults = true, - const NYPath::TYPath& path = ""); - - void Postprocess(const NYPath::TYPath& path = "") const; - - void SetDefaults(); - - void Save(NYson::IYsonConsumer* consumer) const; - - IMapNodePtr GetUnrecognized() const; - IMapNodePtr GetUnrecognizedRecursively() const; - IMapNodePtr GetRecursiveUnrecognized() const; - - void SetUnrecognizedStrategy(EUnrecognizedStrategy strategy); - - THashSet<TString> GetRegisteredKeys() const; - int GetParameterCount() const; - - void SaveParameter(const TString& key, NYson::IYsonConsumer* consumer) const; - void LoadParameter(const TString& key, const NYTree::INodePtr& node, EMergeStrategy mergeStrategy) const; - void ResetParameter(const TString& key) const; - - std::vector<TString> GetAllParameterAliases(const TString& key) const; - -protected: - template <class T> - TParameter<T>& RegisterParameter( - TString parameterName, - T& value); - - void RegisterPreprocessor(const TPreprocessor& func); - void RegisterPostprocessor(const TPostprocessor& func); - -private: - template <class T> - friend class TParameter; - - THashMap<TString, IParameterPtr> Parameters; - - NYTree::IMapNodePtr Unrecognized; - EUnrecognizedStrategy UnrecognizedStrategy = EUnrecognizedStrategy::Drop; - - std::vector<TPreprocessor> Preprocessors; - std::vector<TPostprocessor> Postprocessors; - - IParameterPtr GetParameter(const TString& keyOrAlias) const; -}; - -//////////////////////////////////////////////////////////////////////////////// - -class TYsonSerializable - : public TRefCounted - , public TYsonSerializableLite -{ }; - -//////////////////////////////////////////////////////////////////////////////// - -template <class T> -TIntrusivePtr<T> CloneYsonSerializable(const TIntrusivePtr<T>& obj); -template <class T> -std::vector<TIntrusivePtr<T>> CloneYsonSerializables(const std::vector<TIntrusivePtr<T>>& objs); -template <class T> -THashMap<TString, TIntrusivePtr<T>> CloneYsonSerializables(const THashMap<TString, TIntrusivePtr<T>>& objs); - -void Serialize(const TYsonSerializableLite& value, NYson::IYsonConsumer* consumer); -void Deserialize(TYsonSerializableLite& value, NYTree::INodePtr node); -void Deserialize(TYsonSerializableLite& value, NYson::TYsonPullParserCursor* cursor); - -template <class T> -TIntrusivePtr<T> UpdateYsonSerializable( - const TIntrusivePtr<T>& obj, - const NYTree::INodePtr& patch); - -template <class T> -TIntrusivePtr<T> UpdateYsonSerializable( - const TIntrusivePtr<T>& obj, - const NYson::TYsonString& patch); - -template <class T> -bool ReconfigureYsonSerializable( - const TIntrusivePtr<T>& config, - const NYson::TYsonString& newConfigYson); - -template <class T> -bool ReconfigureYsonSerializable( - const TIntrusivePtr<T>& config, - const TIntrusivePtr<T>& newConfig); - -template <class T> -bool ReconfigureYsonSerializable( - const TIntrusivePtr<T>& config, - const NYTree::INodePtr& newConfigNode); - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NYTree - - -namespace NYT { - -//////////////////////////////////////////////////////////////////////////////// - -struct TBinaryYsonSerializer -{ - static void Save(TStreamSaveContext& context, const NYTree::TYsonSerializableLite& obj); - static void Load(TStreamLoadContext& context, NYTree::TYsonSerializableLite& obj); -}; - -template <class T, class C> -struct TSerializerTraits< - T, - C, - typename std::enable_if_t<std::is_convertible_v<T&, NYTree::TYsonSerializableLite&>>> -{ - using TSerializer = TBinaryYsonSerializer; -}; - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT - -#define YSON_SERIALIZABLE_INL_H_ -#include "yson_serializable-inl.h" -#undef YSON_SERIALIZABLE_INL_H_ diff --git a/yt/yt/core/ytree/yson_struct.h b/yt/yt/core/ytree/yson_struct.h index f28f5808f7..ea4a488387 100644 --- a/yt/yt/core/ytree/yson_struct.h +++ b/yt/yt/core/ytree/yson_struct.h @@ -1,7 +1,7 @@ #pragma once #include "node.h" -#include "yson_serialize_common.h" +#include "yson_struct_enum.h" #include <yt/yt/core/misc/error.h> #include <yt/yt/core/misc/mpl.h> @@ -10,6 +10,8 @@ #include <yt/yt/core/yson/public.h> #include <yt/yt/library/syncmap/map.h> +#include <library/cpp/yt/misc/enum.h> + #include <util/generic/algorithm.h> #include <functional> diff --git a/yt/yt/core/ytree/yson_struct_detail-inl.h b/yt/yt/core/ytree/yson_struct_detail-inl.h index 06a54ff6ae..6785af863d 100644 --- a/yt/yt/core/ytree/yson_struct_detail-inl.h +++ b/yt/yt/core/ytree/yson_struct_detail-inl.h @@ -18,9 +18,6 @@ namespace NYT::NYTree { namespace NPrivate { -template <class T> -concept IsYsonStructOrYsonSerializable = std::derived_from<T, TYsonStructBase> || std::derived_from<T, TYsonSerializableLite>; - // TODO(shakurov): get rid of this once concept support makes it into the standard // library implementation. Use equality-comparability instead. template <class T> @@ -89,8 +86,8 @@ inline void LoadFromNode( } } -// TYsonStruct or TYsonSerializable -template <IsYsonStructOrYsonSerializable T> +// TYsonStruct +template <CYsonStructDerived T> void LoadFromNode( TIntrusivePtr<T>& parameterValue, NYTree::INodePtr node, @@ -313,7 +310,7 @@ void LoadFromCursor( //////////////////////////////////////////////////////////////////////////////// -template <IsYsonStructOrYsonSerializable T> +template <CYsonStructDerived T> void LoadFromCursor( TIntrusivePtr<T>& parameterValue, NYson::TYsonPullParserCursor* cursor, @@ -366,8 +363,8 @@ inline void LoadFromCursor( } } -// TYsonStruct or TYsonSerializable -template <IsYsonStructOrYsonSerializable T> +// TYsonStruct +template <CYsonStructDerived T> void LoadFromCursor( TIntrusivePtr<T>& parameterValue, NYson::TYsonPullParserCursor* cursor, @@ -544,7 +541,7 @@ struct TGetRecursiveUnrecognized } }; -template <IsYsonStructOrYsonSerializable T> +template <CYsonStructDerived T> struct TGetRecursiveUnrecognized<T> { static IMapNodePtr Do(const T& parameter) @@ -553,7 +550,7 @@ struct TGetRecursiveUnrecognized<T> } }; -template <IsYsonStructOrYsonSerializable T> +template <CYsonStructDerived T> struct TGetRecursiveUnrecognized<TIntrusivePtr<T>> { static IMapNodePtr Do(const TIntrusivePtr<T>& parameter) @@ -572,8 +569,8 @@ void InvokeForComposites( const F& /*func*/) { } -// TYsonStruct or TYsonSerializable -template <IsYsonStructOrYsonSerializable T, class F> +// TYsonStruct +template <CYsonStructDerived T, class F> inline void InvokeForComposites( const TIntrusivePtr<T>* parameterValue, const NYPath::TYPath& path, @@ -621,8 +618,8 @@ void InvokeForComposites( const F& /*func*/) { } -// TYsonStruct or TYsonSerializable -template <IsYsonStructOrYsonSerializable T, class F> +// TYsonStruct +template <CYsonStructDerived T, class F> inline void InvokeForComposites(const TIntrusivePtr<T>* parameter, const F& func) { func(*parameter); @@ -788,7 +785,7 @@ void TYsonStructParameter<TValue>::Postprocess(const TYsonStructBase* self, cons NPrivate::InvokeForComposites( &value, path, - [] <NPrivate::IsYsonStructOrYsonSerializable T> (TIntrusivePtr<T> obj, const NYPath::TYPath& subpath) { + [] <CYsonStructDerived T> (TIntrusivePtr<T> obj, const NYPath::TYPath& subpath) { if (obj) { obj->Postprocess(subpath); } diff --git a/yt/yt/core/ytree/yson_struct_detail.h b/yt/yt/core/ytree/yson_struct_detail.h index d4e9fea19a..e6d5a1b595 100644 --- a/yt/yt/core/ytree/yson_struct_detail.h +++ b/yt/yt/core/ytree/yson_struct_detail.h @@ -1,6 +1,6 @@ #pragma once -#include "yson_serialize_common.h" +#include "yson_struct_enum.h" #include <yt/yt/core/yson/public.h> #include <yt/yt/core/ypath/public.h> diff --git a/yt/yt/core/ytree/yson_serialize_common.h b/yt/yt/core/ytree/yson_struct_enum.h index 47824666fc..47824666fc 100644 --- a/yt/yt/core/ytree/yson_serialize_common.h +++ b/yt/yt/core/ytree/yson_struct_enum.h |