YQL-17295 write YT schema into lineage too

29 files changed, 164 insertions, 87 deletions

diff --git a/ydb/library/yql/core/services/yql_lineage.cpp b/ydb/library/yql/core/services/yql_lineage.cpp
index 6e20baf3ec..3c874762c4 100644
--- a/ydb/library/yql/core/services/yql_lineage.cpp
+++ b/ydb/library/yql/core/services/yql_lineage.cpp
@@ -41,7 +41,8 @@ public:
         writer.OnBeginList();
         for (const auto& r : Reads_) {
             TVector<TPinInfo> inputs;
-            r.second->GetPlanFormatter().GetInputs(*r.first, inputs);
+            auto& formatter = r.second->GetPlanFormatter();
+            formatter.GetInputs(*r.first, inputs);
             for (const auto& i : inputs) {
                 auto id = ++NextReadId_;
                 ReadIds_[r.first].push_back(id);
@@ -52,7 +53,12 @@ public:
                 writer.OnKeyedItem("Name");
                 writer.OnStringScalar(i.DisplayName);
                 writer.OnKeyedItem("Schema");
-                WriteSchema(writer, *r.first->GetTypeAnn()->Cast<TTupleExprType>()->GetItems()[1]->Cast<TListExprType>()->GetItemType()->Cast<TStructExprType>());
+                const auto& itemType = *r.first->GetTypeAnn()->Cast<TTupleExprType>()->GetItems()[1]->Cast<TListExprType>()->GetItemType()->Cast<TStructExprType>();
+                WriteSchema(writer, itemType, nullptr);
+                if (formatter.WriteSchemaHeader(writer)) {
+                    WriteSchema(writer, itemType, &formatter);
+                }
+
                 writer.OnEndMap();
             }
         }
@@ -63,7 +69,8 @@ public:
         for (const auto& w : Writes_) {
             auto data = w.first->Child(3);
             TVector<TPinInfo> outputs;
-            w.second->GetPlanFormatter().GetOutputs(*w.first, outputs);
+            auto& formatter = w.second->GetPlanFormatter();
+            formatter.GetOutputs(*w.first, outputs);
             YQL_ENSURE(outputs.size() == 1);
             auto id = ++NextWriteId_;
             WriteIds_[w.first] = id;
@@ -74,7 +81,12 @@ public:
             writer.OnKeyedItem("Name");
             writer.OnStringScalar(outputs.front().DisplayName);
             writer.OnKeyedItem("Schema");
-            WriteSchema(writer, *data->GetTypeAnn()->Cast<TListExprType>()->GetItemType()->Cast<TStructExprType>());
+            const auto& itemType = *data->GetTypeAnn()->Cast<TListExprType>()->GetItemType()->Cast<TStructExprType>();
+            WriteSchema(writer, itemType, nullptr);
+            if (formatter.WriteSchemaHeader(writer)) {
+                WriteSchema(writer, itemType, &formatter);
+            }
+
             writer.OnKeyedItem("Lineage");
             auto lineage = CollectLineage(*data);
             WriteLineage(writer, *lineage);
@@ -87,7 +99,7 @@ public:
     }
 
 private:
-    void WriteSchema(NYson::TYsonWriter& writer, const TStructExprType& structType) {
+    void WriteSchema(NYson::TYsonWriter& writer, const TStructExprType& structType, IPlanFormatter* formatter) {
         writer.OnBeginMap();
         for (const auto& i : structType.GetItems()) {
             if (i->GetName().StartsWith("_yql_sys_")) {
@@ -95,7 +107,11 @@ private:
             }
 
             writer.OnKeyedItem(i->GetName());
-            writer.OnStringScalar(FormatType(i->GetItemType()));
+            if (formatter) {
+                formatter->WriteTypeDetails(writer, *i->GetItemType());
+            } else {
+                writer.OnStringScalar(FormatType(i->GetItemType()));
+            }
         }
 
         writer.OnEndMap();
diff --git a/ydb/library/yql/core/yql_data_provider.h b/ydb/library/yql/core/yql_data_provider.h
index 259d1817bb..fd8a69594b 100644
--- a/ydb/library/yql/core/yql_data_provider.h
+++ b/ydb/library/yql/core/yql_data_provider.h
@@ -57,6 +57,9 @@ public:
     virtual void WritePullDetails(const TExprNode& node, NYson::TYsonWriter& writer) = 0;
     virtual void WritePinDetails(const TExprNode& node, NYson::TYsonWriter& writer) = 0;
     virtual TString GetOperationDisplayName(const TExprNode& node) = 0;
+    // returns false if provider schemas aren't supported
+    virtual bool WriteSchemaHeader(NYson::TYsonWriter& writer) = 0;
+    virtual void WriteTypeDetails(NYson::TYsonWriter& writer, const TTypeAnnotationNode& type) = 0;
 };
 
 class ITrackableNodeProcessor {
diff --git a/ydb/library/yql/providers/common/provider/yql_data_provider_impl.cpp b/ydb/library/yql/providers/common/provider/yql_data_provider_impl.cpp
index 8d855a77b3..7471f81063 100644
--- a/ydb/library/yql/providers/common/provider/yql_data_provider_impl.cpp
+++ b/ydb/library/yql/providers/common/provider/yql_data_provider_impl.cpp
@@ -62,6 +62,16 @@ TString TPlanFormatterBase::GetOperationDisplayName(const TExprNode& node) {
     return TString(node.Content());
 }
 
+bool TPlanFormatterBase::WriteSchemaHeader(NYson::TYsonWriter& writer) {
+    Y_UNUSED(writer);
+    return false;
+}
+
+void TPlanFormatterBase::WriteTypeDetails(NYson::TYsonWriter& writer, const TTypeAnnotationNode& type) {
+    Y_UNUSED(writer);
+    Y_UNUSED(type);
+}
+
 void TTrackableNodeProcessorBase::GetUsedNodes(const TExprNode& node, TVector<TString>& usedNodeIds) {
     Y_UNUSED(node);
     usedNodeIds.clear();
diff --git a/ydb/library/yql/providers/common/provider/yql_data_provider_impl.h b/ydb/library/yql/providers/common/provider/yql_data_provider_impl.h
index 4c4e88013d..fa98e096d3 100644
--- a/ydb/library/yql/providers/common/provider/yql_data_provider_impl.h
+++ b/ydb/library/yql/providers/common/provider/yql_data_provider_impl.h
@@ -22,6 +22,8 @@ public:
     void WritePullDetails(const TExprNode& node, NYson::TYsonWriter& writer) override;
     void WritePinDetails(const TExprNode& node, NYson::TYsonWriter& writer) override;
     TString GetOperationDisplayName(const TExprNode& node) override;
+    bool WriteSchemaHeader(NYson::TYsonWriter& writer) override;
+    void WriteTypeDetails(NYson::TYsonWriter& writer, const TTypeAnnotationNode& type) override;
 };
 
 class TTrackableNodeProcessorBase : public ITrackableNodeProcessor {
diff --git a/ydb/library/yql/providers/yt/lib/schema/CMakeLists.darwin-arm64.txt b/ydb/library/yql/providers/yt/lib/schema/CMakeLists.darwin-arm64.txt
index 0a26927e4a..c91c2f57ef 100644
--- a/ydb/library/yql/providers/yt/lib/schema/CMakeLists.darwin-arm64.txt
+++ b/ydb/library/yql/providers/yt/lib/schema/CMakeLists.darwin-arm64.txt
@@ -16,6 +16,7 @@ target_link_libraries(yt-lib-schema PUBLIC
   library-yql-utils
   yql-utils-log
   providers-common-codec
+  common-schema-expr
   providers-yt-common
 )
 target_sources(yt-lib-schema PRIVATE
diff --git a/ydb/library/yql/providers/yt/lib/schema/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/providers/yt/lib/schema/CMakeLists.darwin-x86_64.txt
index 0a26927e4a..c91c2f57ef 100644
--- a/ydb/library/yql/providers/yt/lib/schema/CMakeLists.darwin-x86_64.txt
+++ b/ydb/library/yql/providers/yt/lib/schema/CMakeLists.darwin-x86_64.txt
@@ -16,6 +16,7 @@ target_link_libraries(yt-lib-schema PUBLIC
   library-yql-utils
   yql-utils-log
   providers-common-codec
+  common-schema-expr
   providers-yt-common
 )
 target_sources(yt-lib-schema PRIVATE
diff --git a/ydb/library/yql/providers/yt/lib/schema/CMakeLists.linux-aarch64.txt b/ydb/library/yql/providers/yt/lib/schema/CMakeLists.linux-aarch64.txt
index 13ad80d741..c06d025f4e 100644
--- a/ydb/library/yql/providers/yt/lib/schema/CMakeLists.linux-aarch64.txt
+++ b/ydb/library/yql/providers/yt/lib/schema/CMakeLists.linux-aarch64.txt
@@ -17,6 +17,7 @@ target_link_libraries(yt-lib-schema PUBLIC
   library-yql-utils
   yql-utils-log
   providers-common-codec
+  common-schema-expr
   providers-yt-common
 )
 target_sources(yt-lib-schema PRIVATE
diff --git a/ydb/library/yql/providers/yt/lib/schema/CMakeLists.linux-x86_64.txt b/ydb/library/yql/providers/yt/lib/schema/CMakeLists.linux-x86_64.txt
index 13ad80d741..c06d025f4e 100644
--- a/ydb/library/yql/providers/yt/lib/schema/CMakeLists.linux-x86_64.txt
+++ b/ydb/library/yql/providers/yt/lib/schema/CMakeLists.linux-x86_64.txt
@@ -17,6 +17,7 @@ target_link_libraries(yt-lib-schema PUBLIC
   library-yql-utils
   yql-utils-log
   providers-common-codec
+  common-schema-expr
   providers-yt-common
 )
 target_sources(yt-lib-schema PRIVATE
diff --git a/ydb/library/yql/providers/yt/lib/schema/CMakeLists.windows-x86_64.txt b/ydb/library/yql/providers/yt/lib/schema/CMakeLists.windows-x86_64.txt
index 0a26927e4a..c91c2f57ef 100644
--- a/ydb/library/yql/providers/yt/lib/schema/CMakeLists.windows-x86_64.txt
+++ b/ydb/library/yql/providers/yt/lib/schema/CMakeLists.windows-x86_64.txt
@@ -16,6 +16,7 @@ target_link_libraries(yt-lib-schema PUBLIC
   library-yql-utils
   yql-utils-log
   providers-common-codec
+  common-schema-expr
   providers-yt-common
 )
 target_sources(yt-lib-schema PRIVATE
diff --git a/ydb/library/yql/providers/yt/lib/schema/schema.cpp b/ydb/library/yql/providers/yt/lib/schema/schema.cpp
index fb9035121d..7ac3ac1ef8 100644
--- a/ydb/library/yql/providers/yt/lib/schema/schema.cpp
+++ b/ydb/library/yql/providers/yt/lib/schema/schema.cpp
@@ -2,11 +2,13 @@
 
 #include <ydb/library/yql/providers/yt/common/yql_names.h>
 #include <ydb/library/yql/providers/common/codec/yql_codec_type_flags.h>
+#include <ydb/library/yql/providers/common/schema/expr/yql_expr_schema.h>
 
 #include <ydb/library/yql/utils/log/log.h>
 #include <ydb/library/yql/utils/yql_panic.h>
 
 #include <library/cpp/yson/node/node_io.h>
+#include <library/cpp/yson/node/node_builder.h>
 
 #include <util/string/cast.h>
 #include <util/generic/yexception.h>
@@ -1017,4 +1019,11 @@ NYT::TSortColumns ToYTSortColumns(const TVector<std::pair<TString, bool>>& sortC
     return res;
 }
 
+TString GetTypeV3String(const TTypeAnnotationNode& type, ui64 nativeTypeCompatibility) {
+    NYT::TNode typeNode;
+    NYT::TNodeBuilder nodeBuilder(&typeNode);
+    NCommon::WriteTypeToYson(nodeBuilder, &type);
+    return NYT::NodeToCanonicalYsonString(RowSpecYqlTypeToYtNativeType(typeNode, nativeTypeCompatibility));
+}
+
 } // NYql
diff --git a/ydb/library/yql/providers/yt/lib/schema/schema.h b/ydb/library/yql/providers/yt/lib/schema/schema.h
index 694533ea69..d532e4b718 100644
--- a/ydb/library/yql/providers/yt/lib/schema/schema.h
+++ b/ydb/library/yql/providers/yt/lib/schema/schema.h
@@ -1,5 +1,7 @@
 #pragma once
 
+#include <ydb/library/yql/providers/common/codec/yql_codec_type_flags.h>
+
 #include <yt/cpp/mapreduce/interface/common.h>
 
 #include <library/cpp/yson/node/node.h>
@@ -12,6 +14,8 @@
 
 namespace NYql {
 
+class TTypeAnnotationNode;
+
 struct TYTSortInfo {
     TVector<std::pair<TString, int>> Keys;
     bool Unique = false;
@@ -26,5 +30,6 @@ bool ValidateTableSchema(const TString& tableName, const NYT::TNode& attributes,
 void MergeInferredSchemeWithSort(NYT::TNode& schema, TYTSortInfo& sortInfo);
 NYT::TTableSchema RowSpecToYTSchema(const NYT::TNode& rowSpec, ui64 nativeTypeCompatibility);
 NYT::TSortColumns ToYTSortColumns(const TVector<std::pair<TString, bool>>& sortColumns);
+TString GetTypeV3String(const TTypeAnnotationNode& type, ui64 nativeTypeCompatibility = NTCF_ALL);
 
 } // NYql
diff --git a/ydb/library/yql/providers/yt/lib/schema/ya.make b/ydb/library/yql/providers/yt/lib/schema/ya.make
index f7fd644fe8..97e8b28b2d 100644
--- a/ydb/library/yql/providers/yt/lib/schema/ya.make
+++ b/ydb/library/yql/providers/yt/lib/schema/ya.make
@@ -10,6 +10,7 @@ PEERDIR(
     ydb/library/yql/utils
     ydb/library/yql/utils/log
     ydb/library/yql/providers/common/codec
+    ydb/library/yql/providers/common/schema/expr
     ydb/library/yql/providers/yt/common
 )
 
diff --git a/ydb/library/yql/providers/yt/provider/CMakeLists.darwin-arm64.txt b/ydb/library/yql/providers/yt/provider/CMakeLists.darwin-arm64.txt
index c3499508b5..a55b537886 100644
--- a/ydb/library/yql/providers/yt/provider/CMakeLists.darwin-arm64.txt
+++ b/ydb/library/yql/providers/yt/provider/CMakeLists.darwin-arm64.txt
@@ -66,6 +66,7 @@ target_link_libraries(providers-yt-provider PUBLIC
   yt-lib-mkql_helpers
   yt-lib-res_pull
   yt-lib-row_spec
+  yt-lib-schema
   yt-lib-skiff
   yt-lib-yson_helpers
   providers-yt-opt
diff --git a/ydb/library/yql/providers/yt/provider/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/providers/yt/provider/CMakeLists.darwin-x86_64.txt
index c3499508b5..a55b537886 100644
--- a/ydb/library/yql/providers/yt/provider/CMakeLists.darwin-x86_64.txt
+++ b/ydb/library/yql/providers/yt/provider/CMakeLists.darwin-x86_64.txt
@@ -66,6 +66,7 @@ target_link_libraries(providers-yt-provider PUBLIC
   yt-lib-mkql_helpers
   yt-lib-res_pull
   yt-lib-row_spec
+  yt-lib-schema
   yt-lib-skiff
   yt-lib-yson_helpers
   providers-yt-opt
diff --git a/ydb/library/yql/providers/yt/provider/CMakeLists.linux-aarch64.txt b/ydb/library/yql/providers/yt/provider/CMakeLists.linux-aarch64.txt
index 01abb47076..885a4f2be9 100644
--- a/ydb/library/yql/providers/yt/provider/CMakeLists.linux-aarch64.txt
+++ b/ydb/library/yql/providers/yt/provider/CMakeLists.linux-aarch64.txt
@@ -67,6 +67,7 @@ target_link_libraries(providers-yt-provider PUBLIC
   yt-lib-mkql_helpers
   yt-lib-res_pull
   yt-lib-row_spec
+  yt-lib-schema
   yt-lib-skiff
   yt-lib-yson_helpers
   providers-yt-opt
diff --git a/ydb/library/yql/providers/yt/provider/CMakeLists.linux-x86_64.txt b/ydb/library/yql/providers/yt/provider/CMakeLists.linux-x86_64.txt
index 01abb47076..885a4f2be9 100644
--- a/ydb/library/yql/providers/yt/provider/CMakeLists.linux-x86_64.txt
+++ b/ydb/library/yql/providers/yt/provider/CMakeLists.linux-x86_64.txt
@@ -67,6 +67,7 @@ target_link_libraries(providers-yt-provider PUBLIC
   yt-lib-mkql_helpers
   yt-lib-res_pull
   yt-lib-row_spec
+  yt-lib-schema
   yt-lib-skiff
   yt-lib-yson_helpers
   providers-yt-opt
diff --git a/ydb/library/yql/providers/yt/provider/CMakeLists.windows-x86_64.txt b/ydb/library/yql/providers/yt/provider/CMakeLists.windows-x86_64.txt
index c3499508b5..a55b537886 100644
--- a/ydb/library/yql/providers/yt/provider/CMakeLists.windows-x86_64.txt
+++ b/ydb/library/yql/providers/yt/provider/CMakeLists.windows-x86_64.txt
@@ -66,6 +66,7 @@ target_link_libraries(providers-yt-provider PUBLIC
   yt-lib-mkql_helpers
   yt-lib-res_pull
   yt-lib-row_spec
+  yt-lib-schema
   yt-lib-skiff
   yt-lib-yson_helpers
   providers-yt-opt
diff --git a/ydb/library/yql/providers/yt/provider/ya.make b/ydb/library/yql/providers/yt/provider/ya.make
index b07485f878..8c79642442 100644
--- a/ydb/library/yql/providers/yt/provider/ya.make
+++ b/ydb/library/yql/providers/yt/provider/ya.make
@@ -88,6 +88,7 @@ PEERDIR(
     ydb/library/yql/providers/yt/lib/mkql_helpers
     ydb/library/yql/providers/yt/lib/res_pull
     ydb/library/yql/providers/yt/lib/row_spec
+    ydb/library/yql/providers/yt/lib/schema
     ydb/library/yql/providers/yt/lib/skiff
     ydb/library/yql/providers/yt/lib/yson_helpers
     ydb/library/yql/providers/yt/opt
diff --git a/ydb/library/yql/providers/yt/provider/yql_yt_datasink.cpp b/ydb/library/yql/providers/yt/provider/yql_yt_datasink.cpp
index 075fa08709..e5be775a2c 100644
--- a/ydb/library/yql/providers/yt/provider/yql_yt_datasink.cpp
+++ b/ydb/library/yql/providers/yt/provider/yql_yt_datasink.cpp
@@ -7,6 +7,7 @@
 #include <ydb/library/yql/providers/yt/expr_nodes/yql_yt_expr_nodes.h>
 #include <ydb/library/yql/providers/yt/common/yql_names.h>
 #include <ydb/library/yql/providers/yt/common/yql_configuration.h>
+#include <ydb/library/yql/providers/yt/lib/schema/schema.h>
 #include <ydb/library/yql/providers/common/codec/yql_codec_type_flags.h>
 #include <ydb/library/yql/providers/common/provider/yql_provider_names.h>
 #include <ydb/library/yql/providers/common/provider/yql_provider.h>
@@ -499,6 +500,15 @@ public:
         return TString{node.Content()};
     }
 
+    bool WriteSchemaHeader(NYson::TYsonWriter& writer) override {
+        writer.OnKeyedItem("YtSchema");
+        return true;
+    }
+
+    void WriteTypeDetails(NYson::TYsonWriter& writer, const TTypeAnnotationNode& type) override {
+        writer.OnStringScalar(GetTypeV3String(type));
+    }
+
     ITrackableNodeProcessor& GetTrackableNodeProcessor() override {
         return *TrackableNodeProcessor_;
     }
diff --git a/ydb/library/yql/providers/yt/provider/yql_yt_datasource.cpp b/ydb/library/yql/providers/yt/provider/yql_yt_datasource.cpp
index 4d96812bab..6c0713cee2 100644
--- a/ydb/library/yql/providers/yt/provider/yql_yt_datasource.cpp
+++ b/ydb/library/yql/providers/yt/provider/yql_yt_datasource.cpp
@@ -10,6 +10,7 @@
 #include <ydb/library/yql/providers/result/expr_nodes/yql_res_expr_nodes.h>
 #include <ydb/library/yql/providers/yt/common/yql_names.h>
 #include <ydb/library/yql/providers/yt/common/yql_configuration.h>
+#include <ydb/library/yql/providers/yt/lib/schema/schema.h>
 #include <ydb/library/yql/providers/common/provider/yql_provider_names.h>
 #include <ydb/library/yql/providers/common/provider/yql_provider.h>
 #include <ydb/library/yql/providers/common/provider/yql_data_provider_impl.h>
@@ -433,6 +434,15 @@ public:
         }
     }
 
+    bool WriteSchemaHeader(NYson::TYsonWriter& writer) override {
+        writer.OnKeyedItem("YtSchema");
+        return true;
+    }
+
+    void WriteTypeDetails(NYson::TYsonWriter& writer, const TTypeAnnotationNode& type) override {
+        writer.OnStringScalar(GetTypeV3String(type));
+    }
+
     ITrackableNodeProcessor& GetTrackableNodeProcessor() override {
         return *TrackableNodeProcessor_;
     }
diff --git a/ydb/library/yql/tests/sql/yt_native_file/part0/canondata/result.json b/ydb/library/yql/tests/sql/yt_native_file/part0/canondata/result.json
index 43ff070ecf..892086ee05 100644
--- a/ydb/library/yql/tests/sql/yt_native_file/part0/canondata/result.json
+++ b/ydb/library/yql/tests/sql/yt_native_file/part0/canondata/result.json
@@ -2979,9 +2979,9 @@
     ],
     "test.test[lineage-select_field-default.txt-Lineage]": [
         {
-            "checksum": "77d3c06dffeab4578e2fa991c8a154bb",
-            "size": 683,
-            "uri": "https://{canondata_backend}/1936947/df9b81afb4a3789c183adb602db0c26fc8dd98f9/resource.tar.gz#test.test_lineage-select_field-default.txt-Lineage_/results.txt"
+            "checksum": "76522ff9c85d6989407abd6a1aff0859",
+            "size": 922,
+            "uri": "https://{canondata_backend}/1777230/e59af0e7f96e8fc7e1bb411d89f5a5bf7f957b73/resource.tar.gz#test.test_lineage-select_field-default.txt-Lineage_/results.txt"
         }
     ],
     "test.test[lineage-select_field-default.txt-Plan]": [
@@ -3007,9 +3007,9 @@
     ],
     "test.test[lineage-select_nested_table_row-default.txt-Lineage]": [
         {
-            "checksum": "562f2a1c1be7e8b71620e55329ec3f07",
-            "size": 1019,
-            "uri": "https://{canondata_backend}/1936947/df9b81afb4a3789c183adb602db0c26fc8dd98f9/resource.tar.gz#test.test_lineage-select_nested_table_row-default.txt-Lineage_/results.txt"
+            "checksum": "5b6775b942673edd82a288aa800287de",
+            "size": 1262,
+            "uri": "https://{canondata_backend}/1777230/e59af0e7f96e8fc7e1bb411d89f5a5bf7f957b73/resource.tar.gz#test.test_lineage-select_nested_table_row-default.txt-Lineage_/results.txt"
         }
     ],
     "test.test[lineage-select_nested_table_row-default.txt-Plan]": [
@@ -3035,9 +3035,9 @@
     ],
     "test.test[lineage-window_member_struct-default.txt-Lineage]": [
         {
-            "checksum": "c6b31864c2ac75af3ac10084e4f7101d",
-            "size": 1207,
-            "uri": "https://{canondata_backend}/1031349/5d84bc5db6dd385c4575ea21e6461bbae876c0ad/resource.tar.gz#test.test_lineage-window_member_struct-default.txt-Lineage_/results.txt"
+            "checksum": "8c0c9c8cf3dab1578139b7ea161a7953",
+            "size": 1641,
+            "uri": "https://{canondata_backend}/1777230/e59af0e7f96e8fc7e1bb411d89f5a5bf7f957b73/resource.tar.gz#test.test_lineage-window_member_struct-default.txt-Lineage_/results.txt"
         }
     ],
     "test.test[lineage-window_member_struct-default.txt-Plan]": [
diff --git a/ydb/library/yql/tests/sql/yt_native_file/part1/canondata/result.json b/ydb/library/yql/tests/sql/yt_native_file/part1/canondata/result.json
index 26d3550ebf..e95ff31d82 100644
--- a/ydb/library/yql/tests/sql/yt_native_file/part1/canondata/result.json
+++ b/ydb/library/yql/tests/sql/yt_native_file/part1/canondata/result.json
@@ -2652,9 +2652,9 @@
     ],
     "test.test[lineage-group_by_asstruct_key-default.txt-Lineage]": [
         {
-            "checksum": "2c1b422f80c7a7f9aab747eb66ef844e",
-            "size": 914,
-            "uri": "https://{canondata_backend}/1871002/96f16894ac580cd8c299379a7d7cd71c8d8834c9/resource.tar.gz#test.test_lineage-group_by_asstruct_key-default.txt-Lineage_/results.txt"
+            "checksum": "bc3917a40f5ac45438aba58cf7027a81",
+            "size": 1187,
+            "uri": "https://{canondata_backend}/1784826/8a2f0d0a55779bdbe0866a62e74103aa1da6a4f8/resource.tar.gz#test.test_lineage-group_by_asstruct_key-default.txt-Lineage_/results.txt"
         }
     ],
     "test.test[lineage-group_by_asstruct_key-default.txt-Plan]": [
@@ -2680,9 +2680,9 @@
     ],
     "test.test[lineage-if_struct-default.txt-Lineage]": [
         {
-            "checksum": "a645e4eb7366e3752a49146c03fffcbf",
-            "size": 1197,
-            "uri": "https://{canondata_backend}/1924537/2a4ead87e83a857ebf5401c24109391652de8b31/resource.tar.gz#test.test_lineage-if_struct-default.txt-Lineage_/results.txt"
+            "checksum": "33d7b1a3dfc294b0302c7c8336db82d1",
+            "size": 1593,
+            "uri": "https://{canondata_backend}/1784826/8a2f0d0a55779bdbe0866a62e74103aa1da6a4f8/resource.tar.gz#test.test_lineage-if_struct-default.txt-Lineage_/results.txt"
         }
     ],
     "test.test[lineage-if_struct-default.txt-Plan]": [
@@ -2708,9 +2708,9 @@
     ],
     "test.test[lineage-member_over_if_struct-default.txt-Lineage]": [
         {
-            "checksum": "c6b31864c2ac75af3ac10084e4f7101d",
-            "size": 1207,
-            "uri": "https://{canondata_backend}/1924537/2a4ead87e83a857ebf5401c24109391652de8b31/resource.tar.gz#test.test_lineage-member_over_if_struct-default.txt-Lineage_/results.txt"
+            "checksum": "8c0c9c8cf3dab1578139b7ea161a7953",
+            "size": 1641,
+            "uri": "https://{canondata_backend}/1784826/8a2f0d0a55779bdbe0866a62e74103aa1da6a4f8/resource.tar.gz#test.test_lineage-member_over_if_struct-default.txt-Lineage_/results.txt"
         }
     ],
     "test.test[lineage-member_over_if_struct-default.txt-Plan]": [
@@ -2736,9 +2736,9 @@
     ],
     "test.test[lineage-scalar_context--Lineage]": [
         {
-            "checksum": "d1b96f432d15d0bb3dcb9ea58354ea90",
-            "size": 949,
-            "uri": "https://{canondata_backend}/1942525/e0184724944958b962ec71cc92bb7a92ce861a41/resource.tar.gz#test.test_lineage-scalar_context--Lineage_/results.txt"
+            "checksum": "ee93db99707395bb2dca328e0282a31f",
+            "size": 1332,
+            "uri": "https://{canondata_backend}/1784826/8a2f0d0a55779bdbe0866a62e74103aa1da6a4f8/resource.tar.gz#test.test_lineage-scalar_context--Lineage_/results.txt"
         }
     ],
     "test.test[lineage-scalar_context--Plan]": [
@@ -2764,9 +2764,9 @@
     ],
     "test.test[lineage-select_all_filter-default.txt-Lineage]": [
         {
-            "checksum": "f63afb055a4d342bd9978e3f23efad57",
-            "size": 1192,
-            "uri": "https://{canondata_backend}/1936947/91260df91a0d59c509dd590c89de7ca402ab4ed7/resource.tar.gz#test.test_lineage-select_all_filter-default.txt-Lineage_/results.txt"
+            "checksum": "b0b905172b61d319b738ba2351f072b5",
+            "size": 1512,
+            "uri": "https://{canondata_backend}/1784826/8a2f0d0a55779bdbe0866a62e74103aa1da6a4f8/resource.tar.gz#test.test_lineage-select_all_filter-default.txt-Lineage_/results.txt"
         }
     ],
     "test.test[lineage-select_all_filter-default.txt-Plan]": [
diff --git a/ydb/library/yql/tests/sql/yt_native_file/part2/canondata/result.json b/ydb/library/yql/tests/sql/yt_native_file/part2/canondata/result.json
index 7f1a7a5da0..ee901a960f 100644
--- a/ydb/library/yql/tests/sql/yt_native_file/part2/canondata/result.json
+++ b/ydb/library/yql/tests/sql/yt_native_file/part2/canondata/result.json
@@ -3131,9 +3131,9 @@
     ],
     "test.test[lineage-select_field_filter-default.txt-Lineage]": [
         {
-            "checksum": "77d3c06dffeab4578e2fa991c8a154bb",
-            "size": 683,
-            "uri": "https://{canondata_backend}/995452/08f31ba6aa44c68618d79e138715de1877b14c19/resource.tar.gz#test.test_lineage-select_field_filter-default.txt-Lineage_/results.txt"
+            "checksum": "76522ff9c85d6989407abd6a1aff0859",
+            "size": 922,
+            "uri": "https://{canondata_backend}/1031349/7e26f4b640da113d1db3fa3a8ff8b28e97a45c0a/resource.tar.gz#test.test_lineage-select_field_filter-default.txt-Lineage_/results.txt"
         }
     ],
     "test.test[lineage-select_field_filter-default.txt-Plan]": [
@@ -3159,9 +3159,9 @@
     ],
     "test.test[lineage-window_asstruct-default.txt-Lineage]": [
         {
-            "checksum": "92795fe4af0a2d3f05dd0ebf2c7afa8f",
-            "size": 934,
-            "uri": "https://{canondata_backend}/995452/e205aff3d9b723852c079c68f0802c3a86c5a41d/resource.tar.gz#test.test_lineage-window_asstruct-default.txt-Lineage_/results.txt"
+            "checksum": "fed71b80911f72157fa57758b39491c5",
+            "size": 1283,
+            "uri": "https://{canondata_backend}/1031349/7e26f4b640da113d1db3fa3a8ff8b28e97a45c0a/resource.tar.gz#test.test_lineage-window_asstruct-default.txt-Lineage_/results.txt"
         }
     ],
     "test.test[lineage-window_asstruct-default.txt-Plan]": [
@@ -3187,9 +3187,9 @@
     ],
     "test.test[lineage-window_many-default.txt-Lineage]": [
         {
-            "checksum": "7dbeee3beab2930e414a816fc9b00517",
-            "size": 1781,
-            "uri": "https://{canondata_backend}/1942278/8c7405cdcc21ea5164cd9e5fb378c7a09dad6583/resource.tar.gz#test.test_lineage-window_many-default.txt-Lineage_/results.txt"
+            "checksum": "5939638fb05e89a09ca32307ebc5adb7",
+            "size": 2306,
+            "uri": "https://{canondata_backend}/1031349/7e26f4b640da113d1db3fa3a8ff8b28e97a45c0a/resource.tar.gz#test.test_lineage-window_many-default.txt-Lineage_/results.txt"
         }
     ],
     "test.test[lineage-window_many-default.txt-Plan]": [
diff --git a/ydb/library/yql/tests/sql/yt_native_file/part3/canondata/result.json b/ydb/library/yql/tests/sql/yt_native_file/part3/canondata/result.json
index d4b030346a..1899e84dee 100644
--- a/ydb/library/yql/tests/sql/yt_native_file/part3/canondata/result.json
+++ b/ydb/library/yql/tests/sql/yt_native_file/part3/canondata/result.json
@@ -2611,9 +2611,9 @@
     ],
     "test.test[lineage-select_union_all-default.txt-Lineage]": [
         {
-            "checksum": "dc367289a81fc4bb5fe6f3888b1d2285",
-            "size": 850,
-            "uri": "https://{canondata_backend}/1936947/deffcc9d962b794e742d3b8ff51d1e256afd0238/resource.tar.gz#test.test_lineage-select_union_all-default.txt-Lineage_/results.txt"
+            "checksum": "2d02d3f2c6a825079782e3160a6b4d3f",
+            "size": 1087,
+            "uri": "https://{canondata_backend}/995452/4f19c8fe7e77938f61e37ae99c34d717614b55a8/resource.tar.gz#test.test_lineage-select_union_all-default.txt-Lineage_/results.txt"
         }
     ],
     "test.test[lineage-select_union_all-default.txt-Plan]": [
@@ -2639,9 +2639,9 @@
     ],
     "test.test[lineage-some_tablerow-default.txt-Lineage]": [
         {
-            "checksum": "c6b31864c2ac75af3ac10084e4f7101d",
-            "size": 1207,
-            "uri": "https://{canondata_backend}/995452/ea151ade22503e898db09f69b7b2e95103712788/resource.tar.gz#test.test_lineage-some_tablerow-default.txt-Lineage_/results.txt"
+            "checksum": "8c0c9c8cf3dab1578139b7ea161a7953",
+            "size": 1641,
+            "uri": "https://{canondata_backend}/995452/4f19c8fe7e77938f61e37ae99c34d717614b55a8/resource.tar.gz#test.test_lineage-some_tablerow-default.txt-Lineage_/results.txt"
         }
     ],
     "test.test[lineage-some_tablerow-default.txt-Plan]": [
diff --git a/ydb/library/yql/tests/sql/yt_native_file/part4/canondata/result.json b/ydb/library/yql/tests/sql/yt_native_file/part4/canondata/result.json
index 685d21a1f9..8105dee3c0 100644
--- a/ydb/library/yql/tests/sql/yt_native_file/part4/canondata/result.json
+++ b/ydb/library/yql/tests/sql/yt_native_file/part4/canondata/result.json
@@ -2974,9 +2974,9 @@
     ],
     "test.test[lineage-flatten_list_nested_lambda--Lineage]": [
         {
-            "checksum": "660a1143bfed1911a820ba30deb475c9",
-            "size": 1105,
-            "uri": "https://{canondata_backend}/1899731/b9ffa3e1e9b029d040e77c2fd6806f9a389fa390/resource.tar.gz#test.test_lineage-flatten_list_nested_lambda--Lineage_/results.txt"
+            "checksum": "6c1ec378e3be49249b9329ec32ccc44f",
+            "size": 1450,
+            "uri": "https://{canondata_backend}/1784117/8b15eeb28fa4a434e40085469b2cbe2db0a4cb0e/resource.tar.gz#test.test_lineage-flatten_list_nested_lambda--Lineage_/results.txt"
         }
     ],
     "test.test[lineage-flatten_list_nested_lambda--Plan]": [
@@ -3002,9 +3002,9 @@
     ],
     "test.test[lineage-join_as_struct-default.txt-Lineage]": [
         {
-            "checksum": "5b0fbd4a1beb69af1d4ec414234cd968",
-            "size": 1412,
-            "uri": "https://{canondata_backend}/1923547/5a49f4ae9fdb921701bf2a8fb8c78ed10df9652f/resource.tar.gz#test.test_lineage-join_as_struct-default.txt-Lineage_/results.txt"
+            "checksum": "eb6cafcb508dc1023a6b9098e696183e",
+            "size": 1757,
+            "uri": "https://{canondata_backend}/1784117/8b15eeb28fa4a434e40085469b2cbe2db0a4cb0e/resource.tar.gz#test.test_lineage-join_as_struct-default.txt-Lineage_/results.txt"
         }
     ],
     "test.test[lineage-join_as_struct-default.txt-Plan]": [
@@ -3030,9 +3030,9 @@
     ],
     "test.test[lineage-select_field_order_by-default.txt-Lineage]": [
         {
-            "checksum": "dabd7822c5e9f2643c4bbbacab4a2043",
-            "size": 936,
-            "uri": "https://{canondata_backend}/1936997/0f0f7b9b3aa810a6eac66c336119ee61e55c3c97/resource.tar.gz#test.test_lineage-select_field_order_by-default.txt-Lineage_/results.txt"
+            "checksum": "908f3aa7fb5bcabe8e318758d8fb952f",
+            "size": 1215,
+            "uri": "https://{canondata_backend}/1784117/8b15eeb28fa4a434e40085469b2cbe2db0a4cb0e/resource.tar.gz#test.test_lineage-select_field_order_by-default.txt-Lineage_/results.txt"
         }
     ],
     "test.test[lineage-select_field_order_by-default.txt-Plan]": [
@@ -3058,9 +3058,9 @@
     ],
     "test.test[lineage-select_field_rename-default.txt-Lineage]": [
         {
-            "checksum": "c841eecc4ffa196bd0a916c0ed5c8684",
-            "size": 685,
-            "uri": "https://{canondata_backend}/1936997/0f0f7b9b3aa810a6eac66c336119ee61e55c3c97/resource.tar.gz#test.test_lineage-select_field_rename-default.txt-Lineage_/results.txt"
+            "checksum": "6427d52727651b5d80680e2c3e8bbd96",
+            "size": 925,
+            "uri": "https://{canondata_backend}/1784117/8b15eeb28fa4a434e40085469b2cbe2db0a4cb0e/resource.tar.gz#test.test_lineage-select_field_rename-default.txt-Lineage_/results.txt"
         }
     ],
     "test.test[lineage-select_field_rename-default.txt-Plan]": [
@@ -3086,9 +3086,9 @@
     ],
     "test.test[lineage-select_group_by_all-default.txt-Lineage]": [
         {
-            "checksum": "fe3fc33b5e6efcb1bd30aeb5cb2ce7dd",
-            "size": 768,
-            "uri": "https://{canondata_backend}/1942278/f6fda42eb5790b5111f1f319e147830ac91c125b/resource.tar.gz#test.test_lineage-select_group_by_all-default.txt-Lineage_/results.txt"
+            "checksum": "8689914e36f87d9df32be40118d108e5",
+            "size": 1091,
+            "uri": "https://{canondata_backend}/1784117/8b15eeb28fa4a434e40085469b2cbe2db0a4cb0e/resource.tar.gz#test.test_lineage-select_group_by_all-default.txt-Lineage_/results.txt"
         }
     ],
     "test.test[lineage-select_group_by_all-default.txt-Plan]": [
@@ -3114,9 +3114,9 @@
     ],
     "test.test[lineage-window_one-default.txt-Lineage]": [
         {
-            "checksum": "a34c8dadfa7c462191f78fc069fe9eca",
-            "size": 753,
-            "uri": "https://{canondata_backend}/1936997/0f0f7b9b3aa810a6eac66c336119ee61e55c3c97/resource.tar.gz#test.test_lineage-window_one-default.txt-Lineage_/results.txt"
+            "checksum": "05b6780a194be49fa533aace52926984",
+            "size": 1034,
+            "uri": "https://{canondata_backend}/1784117/8b15eeb28fa4a434e40085469b2cbe2db0a4cb0e/resource.tar.gz#test.test_lineage-window_one-default.txt-Lineage_/results.txt"
         }
     ],
     "test.test[lineage-window_one-default.txt-Plan]": [
diff --git a/ydb/library/yql/tests/sql/yt_native_file/part5/canondata/result.json b/ydb/library/yql/tests/sql/yt_native_file/part5/canondata/result.json
index 0be9637f38..34252810b6 100644
--- a/ydb/library/yql/tests/sql/yt_native_file/part5/canondata/result.json
+++ b/ydb/library/yql/tests/sql/yt_native_file/part5/canondata/result.json
@@ -3079,9 +3079,9 @@
     ],
     "test.test[lineage-union_all_tablerow-default.txt-Lineage]": [
         {
-            "checksum": "4cc4604f44f6d70d1365158bec1f1a1d",
-            "size": 1274,
-            "uri": "https://{canondata_backend}/1871002/673102593fb2549964b5dfea94f627e25e5dd8cc/resource.tar.gz#test.test_lineage-union_all_tablerow-default.txt-Lineage_/results.txt"
+            "checksum": "b0de4513ba7d24f66c3315a23fc82da3",
+            "size": 1781,
+            "uri": "https://{canondata_backend}/1903885/a993e318364b61f58765de1b942062d91ca8d490/resource.tar.gz#test.test_lineage-union_all_tablerow-default.txt-Lineage_/results.txt"
         }
     ],
     "test.test[lineage-union_all_tablerow-default.txt-Plan]": [
@@ -3107,9 +3107,9 @@
     ],
     "test.test[lineage-window_tablerow-default.txt-Lineage]": [
         {
-            "checksum": "c6b31864c2ac75af3ac10084e4f7101d",
-            "size": 1207,
-            "uri": "https://{canondata_backend}/1871002/673102593fb2549964b5dfea94f627e25e5dd8cc/resource.tar.gz#test.test_lineage-window_tablerow-default.txt-Lineage_/results.txt"
+            "checksum": "8c0c9c8cf3dab1578139b7ea161a7953",
+            "size": 1641,
+            "uri": "https://{canondata_backend}/1903885/a993e318364b61f58765de1b942062d91ca8d490/resource.tar.gz#test.test_lineage-window_tablerow-default.txt-Lineage_/results.txt"
         }
     ],
     "test.test[lineage-window_tablerow-default.txt-Plan]": [
diff --git a/ydb/library/yql/tests/sql/yt_native_file/part7/canondata/result.json b/ydb/library/yql/tests/sql/yt_native_file/part7/canondata/result.json
index 5428231c69..fe6531777b 100644
--- a/ydb/library/yql/tests/sql/yt_native_file/part7/canondata/result.json
+++ b/ydb/library/yql/tests/sql/yt_native_file/part7/canondata/result.json
@@ -2593,9 +2593,9 @@
     ],
     "test.test[lineage-select_all-default.txt-Lineage]": [
         {
-            "checksum": "f63afb055a4d342bd9978e3f23efad57",
-            "size": 1192,
-            "uri": "https://{canondata_backend}/1936947/b529f4021f8b62f7004a07e3b51df9d20ff5b9ac/resource.tar.gz#test.test_lineage-select_all-default.txt-Lineage_/results.txt"
+            "checksum": "b0b905172b61d319b738ba2351f072b5",
+            "size": 1512,
+            "uri": "https://{canondata_backend}/1903885/cdcb91a056db131076ec83588bd6c6466601235d/resource.tar.gz#test.test_lineage-select_all-default.txt-Lineage_/results.txt"
         }
     ],
     "test.test[lineage-select_all-default.txt-Plan]": [
diff --git a/ydb/library/yql/tests/sql/yt_native_file/part8/canondata/result.json b/ydb/library/yql/tests/sql/yt_native_file/part8/canondata/result.json
index ef63ce113e..12f312f208 100644
--- a/ydb/library/yql/tests/sql/yt_native_file/part8/canondata/result.json
+++ b/ydb/library/yql/tests/sql/yt_native_file/part8/canondata/result.json
@@ -3114,9 +3114,9 @@
     ],
     "test.test[lineage-select_group_by_key-default.txt-Lineage]": [
         {
-            "checksum": "d206604c4213f59d58459213458663f0",
-            "size": 1005,
-            "uri": "https://{canondata_backend}/1937367/20cf7c53120f5dc778f37887b8cf90e1ff28bc0f/resource.tar.gz#test.test_lineage-select_group_by_key-default.txt-Lineage_/results.txt"
+            "checksum": "2f4388c32e2079752296709aa51ffa2e",
+            "size": 1328,
+            "uri": "https://{canondata_backend}/1937492/cd988abba8ecb60e2569d3dc25c564d6962909a9/resource.tar.gz#test.test_lineage-select_group_by_key-default.txt-Lineage_/results.txt"
         }
     ],
     "test.test[lineage-select_group_by_key-default.txt-Plan]": [
diff --git a/ydb/library/yql/tests/sql/yt_native_file/part9/canondata/result.json b/ydb/library/yql/tests/sql/yt_native_file/part9/canondata/result.json
index 8a3f32bdaa..914b5fa6bd 100644
--- a/ydb/library/yql/tests/sql/yt_native_file/part9/canondata/result.json
+++ b/ydb/library/yql/tests/sql/yt_native_file/part9/canondata/result.json
@@ -2839,9 +2839,9 @@
     ],
     "test.test[lineage-select_field_limit_offset-default.txt-Lineage]": [
         {
-            "checksum": "77d3c06dffeab4578e2fa991c8a154bb",
-            "size": 683,
-            "uri": "https://{canondata_backend}/1936997/79d0dce5d03826aeda88ddafc2403624f6ddbf98/resource.tar.gz#test.test_lineage-select_field_limit_offset-default.txt-Lineage_/results.txt"
+            "checksum": "76522ff9c85d6989407abd6a1aff0859",
+            "size": 922,
+            "uri": "https://{canondata_backend}/1784826/1318eebc45f03235dbb15cb2d32e5836157be88c/resource.tar.gz#test.test_lineage-select_field_limit_offset-default.txt-Lineage_/results.txt"
         }
     ],
     "test.test[lineage-select_field_limit_offset-default.txt-Plan]": [
@@ -2867,9 +2867,9 @@
     ],
     "test.test[lineage-select_join-default.txt-Lineage]": [
         {
-            "checksum": "ba48df70b0e8331e052a6bb733da1e56",
-            "size": 924,
-            "uri": "https://{canondata_backend}/1936997/79d0dce5d03826aeda88ddafc2403624f6ddbf98/resource.tar.gz#test.test_lineage-select_join-default.txt-Lineage_/results.txt"
+            "checksum": "adacc5271c364c7f0ff88afd862457b1",
+            "size": 1197,
+            "uri": "https://{canondata_backend}/1784826/1318eebc45f03235dbb15cb2d32e5836157be88c/resource.tar.gz#test.test_lineage-select_join-default.txt-Lineage_/results.txt"
         }
     ],
     "test.test[lineage-select_join-default.txt-Plan]": [
@@ -2895,9 +2895,9 @@
     ],
     "test.test[lineage-select_mix_fields-default.txt-Lineage]": [
         {
-            "checksum": "6aad010def814c20df535d6cf3baac92",
-            "size": 852,
-            "uri": "https://{canondata_backend}/1936997/79d0dce5d03826aeda88ddafc2403624f6ddbf98/resource.tar.gz#test.test_lineage-select_mix_fields-default.txt-Lineage_/results.txt"
+            "checksum": "6483e3c6c076c19a74da3b9321f83802",
+            "size": 1095,
+            "uri": "https://{canondata_backend}/1784826/1318eebc45f03235dbb15cb2d32e5836157be88c/resource.tar.gz#test.test_lineage-select_mix_fields-default.txt-Lineage_/results.txt"
         }
     ],
     "test.test[lineage-select_mix_fields-default.txt-Plan]": [
@@ -2923,9 +2923,9 @@
     ],
     "test.test[lineage-select_table_row-default.txt-Lineage]": [
         {
-            "checksum": "aec8e7c91159ac156f94fee6b22fa9ea",
-            "size": 1064,
-            "uri": "https://{canondata_backend}/1936997/79d0dce5d03826aeda88ddafc2403624f6ddbf98/resource.tar.gz#test.test_lineage-select_table_row-default.txt-Lineage_/results.txt"
+            "checksum": "d6b9ee98d364556d381f9657e48c7b14",
+            "size": 1459,
+            "uri": "https://{canondata_backend}/1784826/1318eebc45f03235dbb15cb2d32e5836157be88c/resource.tar.gz#test.test_lineage-select_table_row-default.txt-Lineage_/results.txt"
         }
     ],
     "test.test[lineage-select_table_row-default.txt-Plan]": [