- Release new version of ytyndexer

11 files changed, 202 insertions, 93 deletions

diff --git a/yt/yt/client/scheduler/public.h b/yt/yt/client/scheduler/public.h
index 655d03f7f4..97e408f835 100644
--- a/yt/yt/client/scheduler/public.h
+++ b/yt/yt/client/scheduler/public.h
@@ -132,6 +132,7 @@ DEFINE_ENUM(EAbortReason,
     ((JobSettlementTimedOut)           ( 50))
     ((NonexistentPoolTree)             ( 51))
     ((WrongSchedulingSegmentModule)    ( 52))
+    ((NodeUnresolved)                  ( 53))
     ((SchedulingFirst)                 (100))
     ((SchedulingTimeout)               (101))
     ((SchedulingResourceOvercommit)    (102))
diff --git a/yt/yt/core/yson/config.cpp b/yt/yt/core/yson/config.cpp
index 3f6ccf94de..62632a5a39 100644
--- a/yt/yt/core/yson/config.cpp
+++ b/yt/yt/core/yson/config.cpp
@@ -4,12 +4,29 @@ namespace NYT::NYson {
 
 ////////////////////////////////////////////////////////////////////////////////
 
-void TProtobufInteropDynamicConfig::Register(TRegistrar registrar)
+void TProtobufInteropConfig::Register(TRegistrar registrar)
 {
+    registrar.Parameter("default_enum_yson_storage_type", &TThis::DefaultEnumYsonStorageType)
+        .Default(EEnumYsonStorageType::String);
     registrar.Parameter("utf8_check", &TThis::Utf8Check)
         .Default(EUtf8Check::Disable);
 }
 
+TProtobufInteropConfigPtr TProtobufInteropConfig::ApplyDynamic(
+    const TProtobufInteropDynamicConfigPtr& dynamicConfig) const
+{
+    auto mergedConfig = CloneYsonStruct(MakeStrong(this));
+    NYTree::UpdateYsonStructField(mergedConfig->Utf8Check, dynamicConfig->Utf8Check);
+    mergedConfig->Postprocess();
+    return mergedConfig;
+}
+
+void TProtobufInteropDynamicConfig::Register(TRegistrar registrar)
+{
+    registrar.Parameter("utf8_check", &TThis::Utf8Check)
+        .Optional();
+}
+
 ////////////////////////////////////////////////////////////////////////////////
 
 } // namespace NYT::NYson
diff --git a/yt/yt/core/yson/config.h b/yt/yt/core/yson/config.h
index f4d987b2e9..8e34bc233f 100644
--- a/yt/yt/core/yson/config.h
+++ b/yt/yt/core/yson/config.h
@@ -12,14 +12,35 @@ namespace NYT::NYson {
 
 ////////////////////////////////////////////////////////////////////////////////
 
-// TODO(kmokrov): Drop after YTORM-843
-class TProtobufInteropDynamicConfig
+// TODO(kmokrov): Drop Utf8Check after YTORM-843
+class TProtobufInteropConfig
     : public NYTree::TYsonStruct
 {
 public:
-    // Check if string field contain actual UTF-8 string.
+    // Default enum storage type for protobuf to yson conversion.
+    EEnumYsonStorageType DefaultEnumYsonStorageType;
+    // Check if string field contains actual UTF-8 string.
     EUtf8Check Utf8Check;
 
+    TProtobufInteropConfigPtr ApplyDynamic(const TProtobufInteropDynamicConfigPtr& dynamicConfig) const;
+
+    REGISTER_YSON_STRUCT(TProtobufInteropConfig);
+
+    static void Register(TRegistrar registrar);
+};
+
+DEFINE_REFCOUNTED_TYPE(TProtobufInteropConfig)
+
+////////////////////////////////////////////////////////////////////////////////
+
+// TODO(kmokrov): Drop Utf8Check after YTORM-843
+class TProtobufInteropDynamicConfig
+    : public NYTree::TYsonStruct
+{
+public:
+    // Check if string field contains actual UTF-8 string.
+    std::optional<EUtf8Check> Utf8Check;
+
     REGISTER_YSON_STRUCT(TProtobufInteropDynamicConfig);
 
     static void Register(TRegistrar registrar);
diff --git a/yt/yt/core/yson/protobuf_interop.cpp b/yt/yt/core/yson/protobuf_interop.cpp
index f9d32b34d5..ee4493cb98 100644
--- a/yt/yt/core/yson/protobuf_interop.cpp
+++ b/yt/yt/core/yson/protobuf_interop.cpp
@@ -158,7 +158,7 @@ TString DeriveYsonName(const TString& protobufName, const google::protobuf::File
 
 struct TProtobufInteropConfigSingleton
 {
-    TAtomicIntrusivePtr<TProtobufInteropDynamicConfig> Config{New<TProtobufInteropDynamicConfig>()};
+    TAtomicIntrusivePtr<TProtobufInteropConfig> Config{New<TProtobufInteropConfig>()};
 };
 
 TProtobufInteropConfigSingleton* GlobalProtobufInteropConfig()
@@ -166,7 +166,7 @@ TProtobufInteropConfigSingleton* GlobalProtobufInteropConfig()
     return LeakySingleton<TProtobufInteropConfigSingleton>();
 }
 
-TProtobufInteropDynamicConfigPtr GetProtobufInteropConfig()
+TProtobufInteropConfigPtr GetProtobufInteropConfig()
 {
     return GlobalProtobufInteropConfig()->Config.Acquire();
 }
@@ -177,7 +177,7 @@ TProtobufInteropDynamicConfigPtr GetProtobufInteropConfig()
 
 ////////////////////////////////////////////////////////////////////////////////
 
-void SetProtobufInteropConfig(TProtobufInteropDynamicConfigPtr config)
+void SetProtobufInteropConfig(TProtobufInteropConfigPtr config)
 {
     GlobalProtobufInteropConfig()->Config.Store(std::move(config));
 }
@@ -385,7 +385,9 @@ public:
         , YsonMap_(descriptor->options().GetExtension(NYT::NYson::NProto::yson_map))
         , Required_(descriptor->options().GetExtension(NYT::NYson::NProto::required))
         , Converter_(registry->FindMessageBytesFieldConverter(descriptor->containing_type(), descriptor->index()))
-        , EnumYsonStorageType_(descriptor->options().GetExtension(NYT::NYson::NProto::enum_yson_storage_type))
+        , EnumYsonStorageType_(descriptor->options().HasExtension(NYT::NYson::NProto::enum_yson_storage_type) ?
+            std::optional(descriptor->options().GetExtension(NYT::NYson::NProto::enum_yson_storage_type)) :
+            std::nullopt)
     {
         if (YsonMap_ && !descriptor->is_map()) {
             THROW_ERROR_EXCEPTION("Field %v is not a map and cannot be annotated with \"yson_map\" option",
@@ -496,9 +498,19 @@ public:
         return Converter_;
     }
 
-    const NYT::NYson::NProto::EEnumYsonStorageType& GetEnumYsonStorageType() const
+    EEnumYsonStorageType GetEnumYsonStorageType() const
     {
-        return EnumYsonStorageType_;
+        if (EnumYsonStorageType_) {
+            switch (*EnumYsonStorageType_) {
+                case NYT::NYson::NProto::EEnumYsonStorageType::EYST_STRING:
+                    return EEnumYsonStorageType::String;
+                case NYT::NYson::NProto::EEnumYsonStorageType::EYST_INT:
+                    return EEnumYsonStorageType::Int;
+            }
+        }
+
+        auto config = GetProtobufInteropConfig();
+        return config->DefaultEnumYsonStorageType;
     }
 
     void WriteSchema(IYsonConsumer* consumer) const
@@ -582,7 +594,7 @@ private:
     const bool YsonMap_;
     const bool Required_;
     const std::optional<TProtobufMessageBytesFieldConverter> Converter_;
-    const NYT::NYson::NProto::EEnumYsonStorageType EnumYsonStorageType_;
+    const std::optional<NYT::NYson::NProto::EEnumYsonStorageType> EnumYsonStorageType_;
 };
 
 ////////////////////////////////////////////////////////////////////////////////
@@ -2499,14 +2511,13 @@ private:
 
                     case FieldDescriptor::TYPE_ENUM: {
                         ParseScalar([&] {
-                            using NYT::NYson::NProto::EEnumYsonStorageType;
                             switch(field->GetEnumYsonStorageType()) {
-                                case EEnumYsonStorageType::EYST_INT:
-                                    storeEnumAsInt(unsignedValue);
-                                    break;
-                                case EEnumYsonStorageType::EYST_STRING:
+                                case EEnumYsonStorageType::String:
                                     storeEnumAsString(unsignedValue);
                                     break;
+                                case EEnumYsonStorageType::Int:
+                                    storeEnumAsInt(unsignedValue);
+                                    break;
                             }
                         });
                         break;
@@ -2742,14 +2753,13 @@ private:
 
                     case FieldDescriptor::TYPE_ENUM: {
                         ParseVarintPacked<ui32>(length, field, [&] (auto value) {
-                            using NYT::NYson::NProto::EEnumYsonStorageType;
                             switch(field->GetEnumYsonStorageType()) {
-                                case EEnumYsonStorageType::EYST_INT:
-                                    storeEnumAsInt(value);
-                                    break;
-                                case EEnumYsonStorageType::EYST_STRING:
+                                case EEnumYsonStorageType::String:
                                     storeEnumAsString(value);
                                     break;
+                                case EEnumYsonStorageType::Int:
+                                    storeEnumAsInt(value);
+                                    break;
                             }
                         });
                         break;
diff --git a/yt/yt/core/yson/protobuf_interop.h b/yt/yt/core/yson/protobuf_interop.h
index 5bfe3fd93b..8ee1ed2afc 100644
--- a/yt/yt/core/yson/protobuf_interop.h
+++ b/yt/yt/core/yson/protobuf_interop.h
@@ -276,7 +276,7 @@ TString YsonStringToProto(
 
 ////////////////////////////////////////////////////////////////////////////////
 
-void SetProtobufInteropConfig(TProtobufInteropDynamicConfigPtr config);
+void SetProtobufInteropConfig(TProtobufInteropConfigPtr config);
 
 ////////////////////////////////////////////////////////////////////////////////
 
diff --git a/yt/yt/core/yson/public.h b/yt/yt/core/yson/public.h
index d5fb112202..62d309fd7d 100644
--- a/yt/yt/core/yson/public.h
+++ b/yt/yt/core/yson/public.h
@@ -73,6 +73,16 @@ DEFINE_ENUM(EUtf8Check,
     (ThrowOnFail)
 );
 
+////////////////////////////////////////////////////////////////////////////////
+
+DEFINE_ENUM(EEnumYsonStorageType,
+    (String)
+    (Int)
+);
+
+////////////////////////////////////////////////////////////////////////////////
+
+DECLARE_REFCOUNTED_CLASS(TProtobufInteropConfig);
 DECLARE_REFCOUNTED_CLASS(TProtobufInteropDynamicConfig);
 
 ////////////////////////////////////////////////////////////////////////////////
diff --git a/yt/yt/core/yson/unittests/proto/protobuf_yson_ut.proto b/yt/yt/core/yson/unittests/proto/protobuf_yson_ut.proto
index 3ca0813ce5..0dc275f429 100644
--- a/yt/yt/core/yson/unittests/proto/protobuf_yson_ut.proto
+++ b/yt/yt/core/yson/unittests/proto/protobuf_yson_ut.proto
@@ -173,11 +173,15 @@ message TMessageWithEnums
         VALUE1 = 1;
     };
 
-    optional EEnum enum_int = 1                   [(NYT.NYson.NProto.enum_yson_storage_type) = EYST_INT];
-    repeated EEnum enum_rep_not_packed_int = 2    [(NYT.NYson.NProto.enum_yson_storage_type) = EYST_INT, packed = false];
-    repeated EEnum enum_rep_packed_int = 3        [(NYT.NYson.NProto.enum_yson_storage_type) = EYST_INT, packed = true];
+    optional EEnum enum_int = 1 [(NYT.NYson.NProto.enum_yson_storage_type) = EYST_INT];
+    repeated EEnum enum_rep_not_packed_int = 2 [(NYT.NYson.NProto.enum_yson_storage_type) = EYST_INT, packed = false];
+    repeated EEnum enum_rep_packed_int = 3 [(NYT.NYson.NProto.enum_yson_storage_type) = EYST_INT, packed = true];
 
-    optional EEnum enum_string = 4                [(NYT.NYson.NProto.enum_yson_storage_type) = EYST_STRING];
+    optional EEnum enum_string = 4 [(NYT.NYson.NProto.enum_yson_storage_type) = EYST_STRING];
     repeated EEnum enum_rep_not_packed_string = 5 [(NYT.NYson.NProto.enum_yson_storage_type) = EYST_STRING, packed = false];
-    repeated EEnum enum_rep_packed_string = 6     [(NYT.NYson.NProto.enum_yson_storage_type) = EYST_STRING, packed = true];
+    repeated EEnum enum_rep_packed_string = 6 [(NYT.NYson.NProto.enum_yson_storage_type) = EYST_STRING, packed = true];
+
+    optional EEnum enum_without_option = 7;
+    repeated EEnum enum_rep_not_packed_without_option = 8 [packed = false];
+    repeated EEnum enum_rep_packed_without_option = 9 [packed = true];
 }
diff --git a/yt/yt/core/yson/unittests/protobuf_yson_ut.cpp b/yt/yt/core/yson/unittests/protobuf_yson_ut.cpp
index 87c15c9600..35c422a389 100644
--- a/yt/yt/core/yson/unittests/protobuf_yson_ut.cpp
+++ b/yt/yt/core/yson/unittests/protobuf_yson_ut.cpp
@@ -1020,7 +1020,7 @@ TEST(TYsonToProtobufTest, Entities)
 TEST(TYsonToProtobufTest, ValidUtf8StringCheck)
 {
     for (auto option: {EUtf8Check::Disable, EUtf8Check::LogOnFail, EUtf8Check::ThrowOnFail}) {
-        auto config = New<TProtobufInteropDynamicConfig>();
+        auto config = New<TProtobufInteropConfig>();
         config->Utf8Check = option;
         SetProtobufInteropConfig(config);
 
@@ -2635,74 +2635,117 @@ TEST(TPackedRepeatedProtobufTest, TestSerializeDeserialize)
 
 TEST(TEnumYsonStorageTypeTest, TestDeserializeSerialize)
 {
-    NProto::TMessageWithEnums message;
-    {
-        auto zero = NProto::TMessageWithEnums_EEnum::TMessageWithEnums_EEnum_VALUE0;
-        auto one = NProto::TMessageWithEnums_EEnum::TMessageWithEnums_EEnum_VALUE1;
-
-        message.set_enum_int(zero);
-        message.add_enum_rep_not_packed_int(zero);
-        message.add_enum_rep_not_packed_int(one);
-        message.add_enum_rep_packed_int(zero);
-        message.add_enum_rep_packed_int(one);
-
-        message.set_enum_string(one);
-        message.add_enum_rep_not_packed_string(one);
-        message.add_enum_rep_not_packed_string(zero);
-        message.add_enum_rep_packed_string(one);
-        message.add_enum_rep_packed_string(zero);
-    }
+    for (auto storageType: {EEnumYsonStorageType::String, EEnumYsonStorageType::Int}) {
+        auto config = New<TProtobufInteropConfig>();
+        config->DefaultEnumYsonStorageType = storageType;
+        SetProtobufInteropConfig(config);
 
-    // Proto message to yson.
-    TString stringWithYson;
-    {
-        TString protobufString;
-        Y_UNUSED(message.SerializeToString(&protobufString));
-        TStringOutput ysonOutputStream(stringWithYson);
-        TYsonWriter ysonWriter(&ysonOutputStream, EYsonFormat::Pretty);
-        ArrayInputStream protobufInput(protobufString.data(), protobufString.length());
+        NProto::TMessageWithEnums message;
+        {
+            auto zero = NProto::TMessageWithEnums_EEnum::TMessageWithEnums_EEnum_VALUE0;
+            auto one = NProto::TMessageWithEnums_EEnum::TMessageWithEnums_EEnum_VALUE1;
+
+            message.set_enum_int(zero);
+            message.add_enum_rep_not_packed_int(zero);
+            message.add_enum_rep_not_packed_int(one);
+            message.add_enum_rep_packed_int(zero);
+            message.add_enum_rep_packed_int(one);
+
+            message.set_enum_string(one);
+            message.add_enum_rep_not_packed_string(one);
+            message.add_enum_rep_not_packed_string(zero);
+            message.add_enum_rep_packed_string(one);
+            message.add_enum_rep_packed_string(zero);
+
+            message.set_enum_without_option(zero);
+            message.add_enum_rep_not_packed_without_option(zero);
+            message.add_enum_rep_not_packed_without_option(one);
+            message.add_enum_rep_packed_without_option(zero);
+            message.add_enum_rep_packed_without_option(one);
+        }
 
-        EXPECT_NO_THROW(ParseProtobuf(&ysonWriter, &protobufInput, ReflectProtobufMessageType<NProto::TMessageWithEnums>()));
-    }
+        // Proto message to yson.
+        TString stringWithYson;
+        {
+            TString protobufString;
+            Y_UNUSED(message.SerializeToString(&protobufString));
+            TStringOutput ysonOutputStream(stringWithYson);
+            TYsonWriter ysonWriter(&ysonOutputStream, EYsonFormat::Pretty);
+            ArrayInputStream protobufInput(protobufString.data(), protobufString.length());
 
-    // Check enum representation in yson.
-    auto resultedNode = ConvertToNode(TYsonString(stringWithYson));
-    {
-        auto expectedNode = BuildYsonNodeFluently()
-            .BeginMap()
-                .Item("enum_int").Value(0)
-                .Item("enum_rep_not_packed_int").BeginList()
-                    .Item().Value(0)
-                    .Item().Value(1)
-                .EndList()
-                .Item("enum_rep_packed_int").BeginList()
-                    .Item().Value(0)
-                    .Item().Value(1)
-                .EndList()
-                .Item("enum_string").Value("VALUE1")
-                .Item("enum_rep_not_packed_string").BeginList()
-                    .Item().Value("VALUE1")
-                    .Item().Value("VALUE0")
-                .EndList()
-                .Item("enum_rep_packed_string").BeginList()
-                    .Item().Value("VALUE1")
-                    .Item().Value("VALUE0")
-                .EndList()
-            .EndMap();
+            EXPECT_NO_THROW(ParseProtobuf(&ysonWriter, &protobufInput, ReflectProtobufMessageType<NProto::TMessageWithEnums>()));
+        }
 
-        EXPECT_TRUE(AreNodesEqual(resultedNode, expectedNode));
-    }
+        // Check enum representation in yson.
+        auto resultedNode = ConvertToNode(TYsonString(stringWithYson));
+        {
+            auto expectedNode = BuildYsonNodeFluently()
+                .BeginMap()
+                    .Item("enum_int").Value(0)
+                    .Item("enum_rep_not_packed_int").BeginList()
+                        .Item().Value(0)
+                        .Item().Value(1)
+                    .EndList()
+                    .Item("enum_rep_packed_int").BeginList()
+                        .Item().Value(0)
+                        .Item().Value(1)
+                    .EndList()
 
-    // Yson to proto message.
-    NProto::TMessageWithEnums resultedMessage;
-    DeserializeProtobufMessage(
-        resultedMessage,
-        ReflectProtobufMessageType<NProto::TMessageWithEnums>(),
-        resultedNode,
-        TProtobufWriterOptions{});
+                    .Item("enum_string").Value("VALUE1")
+                    .Item("enum_rep_not_packed_string").BeginList()
+                        .Item().Value("VALUE1")
+                        .Item().Value("VALUE0")
+                    .EndList()
+                    .Item("enum_rep_packed_string").BeginList()
+                        .Item().Value("VALUE1")
+                        .Item().Value("VALUE0")
+                    .EndList()
+                .EndMap();
+
+                auto map = expectedNode->AsMap();
+                switch (storageType) {
+                    case EEnumYsonStorageType::String:
+                        map->AsMap()->AddChild("enum_without_option", BuildYsonNodeFluently().Value("VALUE0"));
+                        map->AsMap()->AddChild("enum_rep_not_packed_without_option", BuildYsonNodeFluently()
+                            .BeginList()
+                                .Item().Value("VALUE0")
+                                .Item().Value("VALUE1")
+                            .EndList());
+                        map->AsMap()->AddChild("enum_rep_packed_without_option", BuildYsonNodeFluently()
+                            .BeginList()
+                                .Item().Value("VALUE0")
+                                .Item().Value("VALUE1")
+                            .EndList());
+                        break;
+                    case EEnumYsonStorageType::Int:
+                        map->AsMap()->AddChild("enum_without_option", BuildYsonNodeFluently().Value(0));
+                        map->AsMap()->AddChild("enum_rep_not_packed_without_option", BuildYsonNodeFluently()
+                            .BeginList()
+                                .Item().Value(0)
+                                .Item().Value(1)
+                            .EndList());
+                        map->AsMap()->AddChild("enum_rep_packed_without_option", BuildYsonNodeFluently()
+                            .BeginList()
+                                .Item().Value(0)
+                                .Item().Value(1)
+                            .EndList());
+                        break;
+                }
+
+            EXPECT_TRUE(AreNodesEqual(resultedNode, expectedNode));
+        }
 
-    // Check that original message is equal to its deserialized + serialized version
-    EXPECT_TRUE(google::protobuf::util::MessageDifferencer::Equals(message, resultedMessage));
+        // Yson to proto message.
+        NProto::TMessageWithEnums resultedMessage;
+        DeserializeProtobufMessage(
+            resultedMessage,
+            ReflectProtobufMessageType<NProto::TMessageWithEnums>(),
+            resultedNode,
+            TProtobufWriterOptions{});
+
+        // Check that original message is equal to its deserialized + serialized version
+        EXPECT_TRUE(google::protobuf::util::MessageDifferencer::Equals(message, resultedMessage));
+    }
 }
 
 TEST(TYsonToProtobufTest, Casing)
diff --git a/yt/yt/library/program/config.cpp b/yt/yt/library/program/config.cpp
index 304f9c761c..4b8a8c4abe 100644
--- a/yt/yt/library/program/config.cpp
+++ b/yt/yt/library/program/config.cpp
@@ -107,6 +107,8 @@ void TSingletonsConfig::Register(TRegistrar registrar)
         .Optional();
     registrar.Parameter("heap_profiler", &TThis::HeapProfiler)
         .DefaultNew();
+    registrar.Parameter("protobuf_interop", &TThis::ProtobufInterop)
+        .DefaultNew();
 
     registrar.Postprocessor([] (TThis* config) {
         if (config->ResourceTrackerVCpuFactor && !config->EnableResourceTracker) {
diff --git a/yt/yt/library/program/config.h b/yt/yt/library/program/config.h
index d4420b7278..e96e5d191b 100644
--- a/yt/yt/library/program/config.h
+++ b/yt/yt/library/program/config.h
@@ -150,6 +150,7 @@ public:
     bool EnableResourceTracker;
     std::optional<double> ResourceTrackerVCpuFactor;
     THeapProfilerConfigPtr HeapProfiler;
+    NYson::TProtobufInteropConfigPtr ProtobufInterop;
 
     REGISTER_YSON_STRUCT(TSingletonsConfig);
 
diff --git a/yt/yt/library/program/helpers.cpp b/yt/yt/library/program/helpers.cpp
index dc06c9b044..2fdf033791 100644
--- a/yt/yt/library/program/helpers.cpp
+++ b/yt/yt/library/program/helpers.cpp
@@ -231,6 +231,8 @@ void ConfigureSingletonsImpl(const TConfig& config)
             NProfiling::SetVCpuFactor(config->ResourceTrackerVCpuFactor.value());
         }
     }
+
+    NYson::SetProtobufInteropConfig(config->ProtobufInterop);
 }
 
 void ConfigureSingletons(const TSingletonsConfigPtr& config)
@@ -274,9 +276,7 @@ void ReconfigureSingletonsImpl(const TStaticConfig& config, const TDynamicConfig
         ConfigureTCMalloc(config->TCMalloc);
     }
 
-    if (dynamicConfig->ProtobufInterop) {
-        NYson::SetProtobufInteropConfig(dynamicConfig->ProtobufInterop);
-    }
+    NYson::SetProtobufInteropConfig(config->ProtobufInterop->ApplyDynamic(dynamicConfig->ProtobufInterop));
 }
 
 void ReconfigureSingletons(const TSingletonsConfigPtr& config, const TSingletonsDynamicConfigPtr& dynamicConfig)