diff options
author | jerzy <jerzy@yandex-team.com> | 2024-01-22 19:07:04 +0300 |
---|---|---|
committer | Alexander Smirnov <alex@ydb.tech> | 2024-01-24 15:01:59 +0300 |
commit | 128a824326772bedbabd7c38f67992d09eed0864 (patch) | |
tree | ab93a17dcf8d7d4a1941ffb35e3bb1797441dd43 /yt | |
parent | 6c47bd7b377ea2052b0701797f3b9bce06d30f6d (diff) | |
download | ydb-128a824326772bedbabd7c38f67992d09eed0864.tar.gz |
- Release new version of ytyndexer
Diffstat (limited to 'yt')
-rw-r--r-- | yt/yt/client/scheduler/public.h | 1 | ||||
-rw-r--r-- | yt/yt/core/yson/config.cpp | 19 | ||||
-rw-r--r-- | yt/yt/core/yson/config.h | 27 | ||||
-rw-r--r-- | yt/yt/core/yson/protobuf_interop.cpp | 44 | ||||
-rw-r--r-- | yt/yt/core/yson/protobuf_interop.h | 2 | ||||
-rw-r--r-- | yt/yt/core/yson/public.h | 10 | ||||
-rw-r--r-- | yt/yt/core/yson/unittests/proto/protobuf_yson_ut.proto | 14 | ||||
-rw-r--r-- | yt/yt/core/yson/unittests/protobuf_yson_ut.cpp | 169 | ||||
-rw-r--r-- | yt/yt/library/program/config.cpp | 2 | ||||
-rw-r--r-- | yt/yt/library/program/config.h | 1 | ||||
-rw-r--r-- | yt/yt/library/program/helpers.cpp | 6 |
11 files changed, 202 insertions, 93 deletions
diff --git a/yt/yt/client/scheduler/public.h b/yt/yt/client/scheduler/public.h index 655d03f7f4..97e408f835 100644 --- a/yt/yt/client/scheduler/public.h +++ b/yt/yt/client/scheduler/public.h @@ -132,6 +132,7 @@ DEFINE_ENUM(EAbortReason, ((JobSettlementTimedOut) ( 50)) ((NonexistentPoolTree) ( 51)) ((WrongSchedulingSegmentModule) ( 52)) + ((NodeUnresolved) ( 53)) ((SchedulingFirst) (100)) ((SchedulingTimeout) (101)) ((SchedulingResourceOvercommit) (102)) diff --git a/yt/yt/core/yson/config.cpp b/yt/yt/core/yson/config.cpp index 3f6ccf94de..62632a5a39 100644 --- a/yt/yt/core/yson/config.cpp +++ b/yt/yt/core/yson/config.cpp @@ -4,12 +4,29 @@ namespace NYT::NYson { //////////////////////////////////////////////////////////////////////////////// -void TProtobufInteropDynamicConfig::Register(TRegistrar registrar) +void TProtobufInteropConfig::Register(TRegistrar registrar) { + registrar.Parameter("default_enum_yson_storage_type", &TThis::DefaultEnumYsonStorageType) + .Default(EEnumYsonStorageType::String); registrar.Parameter("utf8_check", &TThis::Utf8Check) .Default(EUtf8Check::Disable); } +TProtobufInteropConfigPtr TProtobufInteropConfig::ApplyDynamic( + const TProtobufInteropDynamicConfigPtr& dynamicConfig) const +{ + auto mergedConfig = CloneYsonStruct(MakeStrong(this)); + NYTree::UpdateYsonStructField(mergedConfig->Utf8Check, dynamicConfig->Utf8Check); + mergedConfig->Postprocess(); + return mergedConfig; +} + +void TProtobufInteropDynamicConfig::Register(TRegistrar registrar) +{ + registrar.Parameter("utf8_check", &TThis::Utf8Check) + .Optional(); +} + //////////////////////////////////////////////////////////////////////////////// } // namespace NYT::NYson diff --git a/yt/yt/core/yson/config.h b/yt/yt/core/yson/config.h index f4d987b2e9..8e34bc233f 100644 --- a/yt/yt/core/yson/config.h +++ b/yt/yt/core/yson/config.h @@ -12,14 +12,35 @@ namespace NYT::NYson { //////////////////////////////////////////////////////////////////////////////// -// TODO(kmokrov): Drop after YTORM-843 -class TProtobufInteropDynamicConfig +// TODO(kmokrov): Drop Utf8Check after YTORM-843 +class TProtobufInteropConfig : public NYTree::TYsonStruct { public: - // Check if string field contain actual UTF-8 string. + // Default enum storage type for protobuf to yson conversion. + EEnumYsonStorageType DefaultEnumYsonStorageType; + // Check if string field contains actual UTF-8 string. EUtf8Check Utf8Check; + TProtobufInteropConfigPtr ApplyDynamic(const TProtobufInteropDynamicConfigPtr& dynamicConfig) const; + + REGISTER_YSON_STRUCT(TProtobufInteropConfig); + + static void Register(TRegistrar registrar); +}; + +DEFINE_REFCOUNTED_TYPE(TProtobufInteropConfig) + +//////////////////////////////////////////////////////////////////////////////// + +// TODO(kmokrov): Drop Utf8Check after YTORM-843 +class TProtobufInteropDynamicConfig + : public NYTree::TYsonStruct +{ +public: + // Check if string field contains actual UTF-8 string. + std::optional<EUtf8Check> Utf8Check; + REGISTER_YSON_STRUCT(TProtobufInteropDynamicConfig); static void Register(TRegistrar registrar); diff --git a/yt/yt/core/yson/protobuf_interop.cpp b/yt/yt/core/yson/protobuf_interop.cpp index f9d32b34d5..ee4493cb98 100644 --- a/yt/yt/core/yson/protobuf_interop.cpp +++ b/yt/yt/core/yson/protobuf_interop.cpp @@ -158,7 +158,7 @@ TString DeriveYsonName(const TString& protobufName, const google::protobuf::File struct TProtobufInteropConfigSingleton { - TAtomicIntrusivePtr<TProtobufInteropDynamicConfig> Config{New<TProtobufInteropDynamicConfig>()}; + TAtomicIntrusivePtr<TProtobufInteropConfig> Config{New<TProtobufInteropConfig>()}; }; TProtobufInteropConfigSingleton* GlobalProtobufInteropConfig() @@ -166,7 +166,7 @@ TProtobufInteropConfigSingleton* GlobalProtobufInteropConfig() return LeakySingleton<TProtobufInteropConfigSingleton>(); } -TProtobufInteropDynamicConfigPtr GetProtobufInteropConfig() +TProtobufInteropConfigPtr GetProtobufInteropConfig() { return GlobalProtobufInteropConfig()->Config.Acquire(); } @@ -177,7 +177,7 @@ TProtobufInteropDynamicConfigPtr GetProtobufInteropConfig() //////////////////////////////////////////////////////////////////////////////// -void SetProtobufInteropConfig(TProtobufInteropDynamicConfigPtr config) +void SetProtobufInteropConfig(TProtobufInteropConfigPtr config) { GlobalProtobufInteropConfig()->Config.Store(std::move(config)); } @@ -385,7 +385,9 @@ public: , YsonMap_(descriptor->options().GetExtension(NYT::NYson::NProto::yson_map)) , Required_(descriptor->options().GetExtension(NYT::NYson::NProto::required)) , Converter_(registry->FindMessageBytesFieldConverter(descriptor->containing_type(), descriptor->index())) - , EnumYsonStorageType_(descriptor->options().GetExtension(NYT::NYson::NProto::enum_yson_storage_type)) + , EnumYsonStorageType_(descriptor->options().HasExtension(NYT::NYson::NProto::enum_yson_storage_type) ? + std::optional(descriptor->options().GetExtension(NYT::NYson::NProto::enum_yson_storage_type)) : + std::nullopt) { if (YsonMap_ && !descriptor->is_map()) { THROW_ERROR_EXCEPTION("Field %v is not a map and cannot be annotated with \"yson_map\" option", @@ -496,9 +498,19 @@ public: return Converter_; } - const NYT::NYson::NProto::EEnumYsonStorageType& GetEnumYsonStorageType() const + EEnumYsonStorageType GetEnumYsonStorageType() const { - return EnumYsonStorageType_; + if (EnumYsonStorageType_) { + switch (*EnumYsonStorageType_) { + case NYT::NYson::NProto::EEnumYsonStorageType::EYST_STRING: + return EEnumYsonStorageType::String; + case NYT::NYson::NProto::EEnumYsonStorageType::EYST_INT: + return EEnumYsonStorageType::Int; + } + } + + auto config = GetProtobufInteropConfig(); + return config->DefaultEnumYsonStorageType; } void WriteSchema(IYsonConsumer* consumer) const @@ -582,7 +594,7 @@ private: const bool YsonMap_; const bool Required_; const std::optional<TProtobufMessageBytesFieldConverter> Converter_; - const NYT::NYson::NProto::EEnumYsonStorageType EnumYsonStorageType_; + const std::optional<NYT::NYson::NProto::EEnumYsonStorageType> EnumYsonStorageType_; }; //////////////////////////////////////////////////////////////////////////////// @@ -2499,14 +2511,13 @@ private: case FieldDescriptor::TYPE_ENUM: { ParseScalar([&] { - using NYT::NYson::NProto::EEnumYsonStorageType; switch(field->GetEnumYsonStorageType()) { - case EEnumYsonStorageType::EYST_INT: - storeEnumAsInt(unsignedValue); - break; - case EEnumYsonStorageType::EYST_STRING: + case EEnumYsonStorageType::String: storeEnumAsString(unsignedValue); break; + case EEnumYsonStorageType::Int: + storeEnumAsInt(unsignedValue); + break; } }); break; @@ -2742,14 +2753,13 @@ private: case FieldDescriptor::TYPE_ENUM: { ParseVarintPacked<ui32>(length, field, [&] (auto value) { - using NYT::NYson::NProto::EEnumYsonStorageType; switch(field->GetEnumYsonStorageType()) { - case EEnumYsonStorageType::EYST_INT: - storeEnumAsInt(value); - break; - case EEnumYsonStorageType::EYST_STRING: + case EEnumYsonStorageType::String: storeEnumAsString(value); break; + case EEnumYsonStorageType::Int: + storeEnumAsInt(value); + break; } }); break; diff --git a/yt/yt/core/yson/protobuf_interop.h b/yt/yt/core/yson/protobuf_interop.h index 5bfe3fd93b..8ee1ed2afc 100644 --- a/yt/yt/core/yson/protobuf_interop.h +++ b/yt/yt/core/yson/protobuf_interop.h @@ -276,7 +276,7 @@ TString YsonStringToProto( //////////////////////////////////////////////////////////////////////////////// -void SetProtobufInteropConfig(TProtobufInteropDynamicConfigPtr config); +void SetProtobufInteropConfig(TProtobufInteropConfigPtr config); //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/core/yson/public.h b/yt/yt/core/yson/public.h index d5fb112202..62d309fd7d 100644 --- a/yt/yt/core/yson/public.h +++ b/yt/yt/core/yson/public.h @@ -73,6 +73,16 @@ DEFINE_ENUM(EUtf8Check, (ThrowOnFail) ); +//////////////////////////////////////////////////////////////////////////////// + +DEFINE_ENUM(EEnumYsonStorageType, + (String) + (Int) +); + +//////////////////////////////////////////////////////////////////////////////// + +DECLARE_REFCOUNTED_CLASS(TProtobufInteropConfig); DECLARE_REFCOUNTED_CLASS(TProtobufInteropDynamicConfig); //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/core/yson/unittests/proto/protobuf_yson_ut.proto b/yt/yt/core/yson/unittests/proto/protobuf_yson_ut.proto index 3ca0813ce5..0dc275f429 100644 --- a/yt/yt/core/yson/unittests/proto/protobuf_yson_ut.proto +++ b/yt/yt/core/yson/unittests/proto/protobuf_yson_ut.proto @@ -173,11 +173,15 @@ message TMessageWithEnums VALUE1 = 1; }; - optional EEnum enum_int = 1 [(NYT.NYson.NProto.enum_yson_storage_type) = EYST_INT]; - repeated EEnum enum_rep_not_packed_int = 2 [(NYT.NYson.NProto.enum_yson_storage_type) = EYST_INT, packed = false]; - repeated EEnum enum_rep_packed_int = 3 [(NYT.NYson.NProto.enum_yson_storage_type) = EYST_INT, packed = true]; + optional EEnum enum_int = 1 [(NYT.NYson.NProto.enum_yson_storage_type) = EYST_INT]; + repeated EEnum enum_rep_not_packed_int = 2 [(NYT.NYson.NProto.enum_yson_storage_type) = EYST_INT, packed = false]; + repeated EEnum enum_rep_packed_int = 3 [(NYT.NYson.NProto.enum_yson_storage_type) = EYST_INT, packed = true]; - optional EEnum enum_string = 4 [(NYT.NYson.NProto.enum_yson_storage_type) = EYST_STRING]; + optional EEnum enum_string = 4 [(NYT.NYson.NProto.enum_yson_storage_type) = EYST_STRING]; repeated EEnum enum_rep_not_packed_string = 5 [(NYT.NYson.NProto.enum_yson_storage_type) = EYST_STRING, packed = false]; - repeated EEnum enum_rep_packed_string = 6 [(NYT.NYson.NProto.enum_yson_storage_type) = EYST_STRING, packed = true]; + repeated EEnum enum_rep_packed_string = 6 [(NYT.NYson.NProto.enum_yson_storage_type) = EYST_STRING, packed = true]; + + optional EEnum enum_without_option = 7; + repeated EEnum enum_rep_not_packed_without_option = 8 [packed = false]; + repeated EEnum enum_rep_packed_without_option = 9 [packed = true]; } diff --git a/yt/yt/core/yson/unittests/protobuf_yson_ut.cpp b/yt/yt/core/yson/unittests/protobuf_yson_ut.cpp index 87c15c9600..35c422a389 100644 --- a/yt/yt/core/yson/unittests/protobuf_yson_ut.cpp +++ b/yt/yt/core/yson/unittests/protobuf_yson_ut.cpp @@ -1020,7 +1020,7 @@ TEST(TYsonToProtobufTest, Entities) TEST(TYsonToProtobufTest, ValidUtf8StringCheck) { for (auto option: {EUtf8Check::Disable, EUtf8Check::LogOnFail, EUtf8Check::ThrowOnFail}) { - auto config = New<TProtobufInteropDynamicConfig>(); + auto config = New<TProtobufInteropConfig>(); config->Utf8Check = option; SetProtobufInteropConfig(config); @@ -2635,74 +2635,117 @@ TEST(TPackedRepeatedProtobufTest, TestSerializeDeserialize) TEST(TEnumYsonStorageTypeTest, TestDeserializeSerialize) { - NProto::TMessageWithEnums message; - { - auto zero = NProto::TMessageWithEnums_EEnum::TMessageWithEnums_EEnum_VALUE0; - auto one = NProto::TMessageWithEnums_EEnum::TMessageWithEnums_EEnum_VALUE1; - - message.set_enum_int(zero); - message.add_enum_rep_not_packed_int(zero); - message.add_enum_rep_not_packed_int(one); - message.add_enum_rep_packed_int(zero); - message.add_enum_rep_packed_int(one); - - message.set_enum_string(one); - message.add_enum_rep_not_packed_string(one); - message.add_enum_rep_not_packed_string(zero); - message.add_enum_rep_packed_string(one); - message.add_enum_rep_packed_string(zero); - } + for (auto storageType: {EEnumYsonStorageType::String, EEnumYsonStorageType::Int}) { + auto config = New<TProtobufInteropConfig>(); + config->DefaultEnumYsonStorageType = storageType; + SetProtobufInteropConfig(config); - // Proto message to yson. - TString stringWithYson; - { - TString protobufString; - Y_UNUSED(message.SerializeToString(&protobufString)); - TStringOutput ysonOutputStream(stringWithYson); - TYsonWriter ysonWriter(&ysonOutputStream, EYsonFormat::Pretty); - ArrayInputStream protobufInput(protobufString.data(), protobufString.length()); + NProto::TMessageWithEnums message; + { + auto zero = NProto::TMessageWithEnums_EEnum::TMessageWithEnums_EEnum_VALUE0; + auto one = NProto::TMessageWithEnums_EEnum::TMessageWithEnums_EEnum_VALUE1; + + message.set_enum_int(zero); + message.add_enum_rep_not_packed_int(zero); + message.add_enum_rep_not_packed_int(one); + message.add_enum_rep_packed_int(zero); + message.add_enum_rep_packed_int(one); + + message.set_enum_string(one); + message.add_enum_rep_not_packed_string(one); + message.add_enum_rep_not_packed_string(zero); + message.add_enum_rep_packed_string(one); + message.add_enum_rep_packed_string(zero); + + message.set_enum_without_option(zero); + message.add_enum_rep_not_packed_without_option(zero); + message.add_enum_rep_not_packed_without_option(one); + message.add_enum_rep_packed_without_option(zero); + message.add_enum_rep_packed_without_option(one); + } - EXPECT_NO_THROW(ParseProtobuf(&ysonWriter, &protobufInput, ReflectProtobufMessageType<NProto::TMessageWithEnums>())); - } + // Proto message to yson. + TString stringWithYson; + { + TString protobufString; + Y_UNUSED(message.SerializeToString(&protobufString)); + TStringOutput ysonOutputStream(stringWithYson); + TYsonWriter ysonWriter(&ysonOutputStream, EYsonFormat::Pretty); + ArrayInputStream protobufInput(protobufString.data(), protobufString.length()); - // Check enum representation in yson. - auto resultedNode = ConvertToNode(TYsonString(stringWithYson)); - { - auto expectedNode = BuildYsonNodeFluently() - .BeginMap() - .Item("enum_int").Value(0) - .Item("enum_rep_not_packed_int").BeginList() - .Item().Value(0) - .Item().Value(1) - .EndList() - .Item("enum_rep_packed_int").BeginList() - .Item().Value(0) - .Item().Value(1) - .EndList() - .Item("enum_string").Value("VALUE1") - .Item("enum_rep_not_packed_string").BeginList() - .Item().Value("VALUE1") - .Item().Value("VALUE0") - .EndList() - .Item("enum_rep_packed_string").BeginList() - .Item().Value("VALUE1") - .Item().Value("VALUE0") - .EndList() - .EndMap(); + EXPECT_NO_THROW(ParseProtobuf(&ysonWriter, &protobufInput, ReflectProtobufMessageType<NProto::TMessageWithEnums>())); + } - EXPECT_TRUE(AreNodesEqual(resultedNode, expectedNode)); - } + // Check enum representation in yson. + auto resultedNode = ConvertToNode(TYsonString(stringWithYson)); + { + auto expectedNode = BuildYsonNodeFluently() + .BeginMap() + .Item("enum_int").Value(0) + .Item("enum_rep_not_packed_int").BeginList() + .Item().Value(0) + .Item().Value(1) + .EndList() + .Item("enum_rep_packed_int").BeginList() + .Item().Value(0) + .Item().Value(1) + .EndList() - // Yson to proto message. - NProto::TMessageWithEnums resultedMessage; - DeserializeProtobufMessage( - resultedMessage, - ReflectProtobufMessageType<NProto::TMessageWithEnums>(), - resultedNode, - TProtobufWriterOptions{}); + .Item("enum_string").Value("VALUE1") + .Item("enum_rep_not_packed_string").BeginList() + .Item().Value("VALUE1") + .Item().Value("VALUE0") + .EndList() + .Item("enum_rep_packed_string").BeginList() + .Item().Value("VALUE1") + .Item().Value("VALUE0") + .EndList() + .EndMap(); + + auto map = expectedNode->AsMap(); + switch (storageType) { + case EEnumYsonStorageType::String: + map->AsMap()->AddChild("enum_without_option", BuildYsonNodeFluently().Value("VALUE0")); + map->AsMap()->AddChild("enum_rep_not_packed_without_option", BuildYsonNodeFluently() + .BeginList() + .Item().Value("VALUE0") + .Item().Value("VALUE1") + .EndList()); + map->AsMap()->AddChild("enum_rep_packed_without_option", BuildYsonNodeFluently() + .BeginList() + .Item().Value("VALUE0") + .Item().Value("VALUE1") + .EndList()); + break; + case EEnumYsonStorageType::Int: + map->AsMap()->AddChild("enum_without_option", BuildYsonNodeFluently().Value(0)); + map->AsMap()->AddChild("enum_rep_not_packed_without_option", BuildYsonNodeFluently() + .BeginList() + .Item().Value(0) + .Item().Value(1) + .EndList()); + map->AsMap()->AddChild("enum_rep_packed_without_option", BuildYsonNodeFluently() + .BeginList() + .Item().Value(0) + .Item().Value(1) + .EndList()); + break; + } + + EXPECT_TRUE(AreNodesEqual(resultedNode, expectedNode)); + } - // Check that original message is equal to its deserialized + serialized version - EXPECT_TRUE(google::protobuf::util::MessageDifferencer::Equals(message, resultedMessage)); + // Yson to proto message. + NProto::TMessageWithEnums resultedMessage; + DeserializeProtobufMessage( + resultedMessage, + ReflectProtobufMessageType<NProto::TMessageWithEnums>(), + resultedNode, + TProtobufWriterOptions{}); + + // Check that original message is equal to its deserialized + serialized version + EXPECT_TRUE(google::protobuf::util::MessageDifferencer::Equals(message, resultedMessage)); + } } TEST(TYsonToProtobufTest, Casing) diff --git a/yt/yt/library/program/config.cpp b/yt/yt/library/program/config.cpp index 304f9c761c..4b8a8c4abe 100644 --- a/yt/yt/library/program/config.cpp +++ b/yt/yt/library/program/config.cpp @@ -107,6 +107,8 @@ void TSingletonsConfig::Register(TRegistrar registrar) .Optional(); registrar.Parameter("heap_profiler", &TThis::HeapProfiler) .DefaultNew(); + registrar.Parameter("protobuf_interop", &TThis::ProtobufInterop) + .DefaultNew(); registrar.Postprocessor([] (TThis* config) { if (config->ResourceTrackerVCpuFactor && !config->EnableResourceTracker) { diff --git a/yt/yt/library/program/config.h b/yt/yt/library/program/config.h index d4420b7278..e96e5d191b 100644 --- a/yt/yt/library/program/config.h +++ b/yt/yt/library/program/config.h @@ -150,6 +150,7 @@ public: bool EnableResourceTracker; std::optional<double> ResourceTrackerVCpuFactor; THeapProfilerConfigPtr HeapProfiler; + NYson::TProtobufInteropConfigPtr ProtobufInterop; REGISTER_YSON_STRUCT(TSingletonsConfig); diff --git a/yt/yt/library/program/helpers.cpp b/yt/yt/library/program/helpers.cpp index dc06c9b044..2fdf033791 100644 --- a/yt/yt/library/program/helpers.cpp +++ b/yt/yt/library/program/helpers.cpp @@ -231,6 +231,8 @@ void ConfigureSingletonsImpl(const TConfig& config) NProfiling::SetVCpuFactor(config->ResourceTrackerVCpuFactor.value()); } } + + NYson::SetProtobufInteropConfig(config->ProtobufInterop); } void ConfigureSingletons(const TSingletonsConfigPtr& config) @@ -274,9 +276,7 @@ void ReconfigureSingletonsImpl(const TStaticConfig& config, const TDynamicConfig ConfigureTCMalloc(config->TCMalloc); } - if (dynamicConfig->ProtobufInterop) { - NYson::SetProtobufInteropConfig(dynamicConfig->ProtobufInterop); - } + NYson::SetProtobufInteropConfig(config->ProtobufInterop->ApplyDynamic(dynamicConfig->ProtobufInterop)); } void ReconfigureSingletons(const TSingletonsConfigPtr& config, const TSingletonsDynamicConfigPtr& dynamicConfig) |