YQL-19409: Column duplicates in yt codec

YQL-19409: Column duplicates in yt codec
commit_hash:65386b6c2642fc2d19b202be9fb379b22c3cbf46

10 files changed, 93 insertions, 10 deletions

diff --git a/yql/essentials/core/dq_integration/yql_dq_integration.h b/yql/essentials/core/dq_integration/yql_dq_integration.h
index 1ac9427f65..bc61105a6b 100644
--- a/yql/essentials/core/dq_integration/yql_dq_integration.h
+++ b/yql/essentials/core/dq_integration/yql_dq_integration.h
@@ -4,6 +4,7 @@
 #include <yql/essentials/core/yql_data_provider.h>
 #include <yql/essentials/core/yql_statistics.h>
 #include <yql/essentials/core/expr_nodes/yql_expr_nodes.h>
+#include <yql/essentials/core/yql_type_annotation.h>
 #include <yql/essentials/public/issue/yql_issue.h>
 
 #include <library/cpp/yson/writer.h>
@@ -81,7 +82,7 @@ public:
     virtual void FillSinkSettings(const TExprNode& node, ::google::protobuf::Any& settings, TString& sinkType) = 0;
     virtual void FillTransformSettings(const TExprNode& node, ::google::protobuf::Any& settings) = 0;
     virtual void Annotate(const TExprNode& node, THashMap<TString, TString>& params) = 0;
-    virtual bool PrepareFullResultTableParams(const TExprNode& root, TExprContext& ctx, THashMap<TString, TString>& params, THashMap<TString, TString>& secureParams) = 0;
+    virtual bool PrepareFullResultTableParams(const TExprNode& root, TExprContext& ctx, THashMap<TString, TString>& params, THashMap<TString, TString>& secureParams, const TMaybe<TColumnOrder>& = {}) = 0;
     virtual void WriteFullResultTableRef(NYson::TYsonWriter& writer, const TVector<TString>& columns, const THashMap<TString, TString>& graphParams) = 0;
 
     // Fill plan operator properties for sources/sinks
diff --git a/yql/essentials/core/yql_type_annotation.h b/yql/essentials/core/yql_type_annotation.h
index b508dc6778..ac0f0f36bd 100644
--- a/yql/essentials/core/yql_type_annotation.h
+++ b/yql/essentials/core/yql_type_annotation.h
@@ -212,6 +212,33 @@ public:
     const TOrderedItem& back() const {
         return Order_.back();
     }
+
+    TVector<TString> GetLogicalNames() const {
+        TVector<TString> res;
+        res.reserve(Order_.size());
+        for (const auto &[name, _]: Order_) {
+            res.emplace_back(name);
+        }
+        return res;
+    }
+
+    TVector<TString> GetPhysicalNames() const {
+        TVector<TString> res;
+        res.reserve(Order_.size());
+        for (const auto &[_, name]: Order_) {
+            res.emplace_back(name);
+        }
+        return res;
+    }
+
+    bool HasDuplicates() const {
+        for (const auto& e: Order_) {
+            if (e.PhysicalName != e.LogicalName) {
+                return true;
+            }
+        }
+        return false;
+    }
 private:
     THashMap<TString, TString> GeneratedToOriginal_;
     THashMap<TString, uint64_t> UseCount_;
diff --git a/yql/essentials/providers/common/dq/yql_dq_integration_impl.cpp b/yql/essentials/providers/common/dq/yql_dq_integration_impl.cpp
index 4782a79d57..098cc7a266 100644
--- a/yql/essentials/providers/common/dq/yql_dq_integration_impl.cpp
+++ b/yql/essentials/providers/common/dq/yql_dq_integration_impl.cpp
@@ -74,7 +74,7 @@ void TDqIntegrationBase::FillTransformSettings(const TExprNode&, ::google::proto
 void TDqIntegrationBase::Annotate(const TExprNode&, THashMap<TString, TString>&) {
 }
 
-bool TDqIntegrationBase::PrepareFullResultTableParams(const TExprNode&, TExprContext&, THashMap<TString, TString>&, THashMap<TString, TString>&) {
+bool TDqIntegrationBase::PrepareFullResultTableParams(const TExprNode&, TExprContext&, THashMap<TString, TString>&, THashMap<TString, TString>&, const TMaybe<TColumnOrder>&) {
     return false;
 }
 
diff --git a/yql/essentials/providers/common/dq/yql_dq_integration_impl.h b/yql/essentials/providers/common/dq/yql_dq_integration_impl.h
index 1fbcb33d77..d4e1597b85 100644
--- a/yql/essentials/providers/common/dq/yql_dq_integration_impl.h
+++ b/yql/essentials/providers/common/dq/yql_dq_integration_impl.h
@@ -23,7 +23,7 @@ public:
     void FillSinkSettings(const TExprNode& node, ::google::protobuf::Any& settings, TString& sinkType) override;
     void FillTransformSettings(const TExprNode& node, ::google::protobuf::Any& settings) override;
     void Annotate(const TExprNode& node, THashMap<TString, TString>& params) override;
-    bool PrepareFullResultTableParams(const TExprNode& root, TExprContext& ctx, THashMap<TString, TString>& params, THashMap<TString, TString>& secureParams) override;
+    bool PrepareFullResultTableParams(const TExprNode& root, TExprContext& ctx, THashMap<TString, TString>& params, THashMap<TString, TString>& secureParams, const TMaybe<TColumnOrder>& order = {}) override;
     void WriteFullResultTableRef(NYson::TYsonWriter& writer, const TVector<TString>& columns, const THashMap<TString, TString>& graphParams) override;
     bool FillSourcePlanProperties(const NNodes::TExprBase& node, TMap<TString, NJson::TJsonValue>& properties) override;
     bool FillSinkPlanProperties(const NNodes::TExprBase& node, TMap<TString, NJson::TJsonValue>& properties) override;
diff --git a/yql/essentials/providers/common/schema/expr/yql_expr_schema.cpp b/yql/essentials/providers/common/schema/expr/yql_expr_schema.cpp
index 58a138b764..1ac4c89927 100644
--- a/yql/essentials/providers/common/schema/expr/yql_expr_schema.cpp
+++ b/yql/essentials/providers/common/schema/expr/yql_expr_schema.cpp
@@ -46,7 +46,7 @@ class TExprTypeSaver: public TSaver<TExprTypeSaver<TSaver>> {
     struct TMappingOrderedStructAdaptor {
         TVector<std::pair<TStringBuf, const TTypeAnnotationNode*>> Members;
 
-        TMappingOrderedStructAdaptor(const TStructMemberMapper& mapper, const TMaybe<TColumnOrder>& columns, const TStructExprType* type)
+        TMappingOrderedStructAdaptor(const TStructMemberMapper& mapper, const TMaybe<TColumnOrder>& columns, const TStructExprType* type, bool writePhysical = true)
         {
             TMap<TStringBuf, const TTypeAnnotationNode*> members;
             for (auto& item: type->GetItems()) {
@@ -61,7 +61,7 @@ class TExprTypeSaver: public TSaver<TExprTypeSaver<TSaver>> {
                 for (auto& [column, gen_column] : *columns) {
                     auto it = members.find(gen_column);
                     if (it != members.end()) {
-                        Members.emplace_back(column, it->second);
+                        Members.emplace_back(writePhysical ? gen_column : column, it->second);
                     }
                 }
             } else {
@@ -229,9 +229,9 @@ public:
         }
     }
 
-    void SaveStructType(const TStructExprType* type, const TMaybe<TColumnOrder>& columns, const TStructMemberMapper& mapper) {
+    void SaveStructType(const TStructExprType* type, const TMaybe<TColumnOrder>& columns, const TStructMemberMapper& mapper, bool physical = true) {
         if (mapper || columns) {
-            TBase::SaveStructType(TMappingOrderedStructAdaptor(mapper, columns, type));
+            TBase::SaveStructType(TMappingOrderedStructAdaptor(mapper, columns, type, physical));
         } else {
             Save(type);
         }
@@ -240,7 +240,12 @@ public:
 
 void SaveStructTypeToYson(NYson::TYsonConsumerBase& writer, const TStructExprType* type, const TMaybe<TColumnOrder>& columns, const TStructMemberMapper& mapper, bool extendedForm) {
     TExprTypeSaver<TYqlTypeYsonSaverImpl> saver(writer, extendedForm);
-    saver.SaveStructType(type, columns, mapper);
+    saver.SaveStructType(type, columns, mapper, true);
+}
+
+void SaveStructTypeToYsonWithLogicalNames(NYson::TYsonConsumerBase& writer, const TStructExprType* type, const TMaybe<TColumnOrder>& columns, const TStructMemberMapper& mapper, bool extendedForm) {
+    TExprTypeSaver<TYqlTypeYsonSaverImpl> saver(writer, extendedForm);
+    saver.SaveStructType(type, columns, mapper, false);
 }
 
 void WriteTypeToYson(NYson::TYsonConsumerBase& writer, const TTypeAnnotationNode* type, bool extendedForm) {
@@ -315,8 +320,9 @@ struct TExprTypeLoader {
     }
     TMaybe<TType> LoadStructType(const TVector<std::pair<TString, TType>>& members, ui32 /*level*/) {
         TVector<const TItemExprType*> items;
+        TColumnOrder order;
         for (auto& member: members) {
-            items.push_back(Ctx.MakeType<TItemExprType>(member.first, member.second));
+            items.push_back(Ctx.MakeType<TItemExprType>(order.AddColumn(member.first), member.second));
         }
         auto ret = Ctx.MakeType<TStructExprType>(items);
         YQL_ENSURE(ret->Validate(TPosition(), Ctx));
@@ -430,7 +436,7 @@ void WriteResOrPullType(NYson::TYsonConsumerBase& writer, const TTypeAnnotationN
         writer.OnStringScalar("ListType");
         writer.OnListItem();
 
-        SaveStructTypeToYson(writer, type->Cast<TListExprType>()->GetItemType()->Cast<TStructExprType>(), columns, {}, true);
+        SaveStructTypeToYsonWithLogicalNames(writer, type->Cast<TListExprType>()->GetItemType()->Cast<TStructExprType>(), columns, {}, true);
 
         writer.OnEndList();
     }
diff --git a/yql/essentials/tests/sql/sql2yql/canondata/result.json b/yql/essentials/tests/sql/sql2yql/canondata/result.json
index da8e4a69ce..715524f3f9 100644
--- a/yql/essentials/tests/sql/sql2yql/canondata/result.json
+++ b/yql/essentials/tests/sql/sql2yql/canondata/result.json
@@ -15672,6 +15672,13 @@
             "uri": "https://{canondata_backend}/1937429/434276f26b2857be3c5ad3fdbbf877d2bf775ac5/resource.tar.gz#test_sql2yql.test_pg_duplicated-ambigous_ref_/sql.yql"
         }
     ],
+    "test_sql2yql.test[pg_duplicated-duplicated_rowspec]": [
+        {
+            "checksum": "70c2665c84dec52c9318ad1b43450c08",
+            "size": 623,
+            "uri": "https://{canondata_backend}/1871002/c5bebc3ea8669574f080ea93959c29fb53e4bc67/resource.tar.gz#test_sql2yql.test_pg_duplicated-duplicated_rowspec_/sql.yql"
+        }
+    ],
     "test_sql2yql.test[pg_duplicated-order_by_duplicated_proj_on_column_ref]": [
         {
             "checksum": "e0bd0cd714e708b53afbfb0770cf3e30",
diff --git a/yql/essentials/tests/sql/suites/pg_duplicated/duplicated_rowspec.cfg b/yql/essentials/tests/sql/suites/pg_duplicated/duplicated_rowspec.cfg
new file mode 100644
index 0000000000..ab354ca5a0
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/pg_duplicated/duplicated_rowspec.cfg
@@ -0,0 +1 @@
+in input duplicated_rowspec.txt
\ No newline at end of file
diff --git a/yql/essentials/tests/sql/suites/pg_duplicated/duplicated_rowspec.sql b/yql/essentials/tests/sql/suites/pg_duplicated/duplicated_rowspec.sql
new file mode 100644
index 0000000000..4d7bd0853f
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/pg_duplicated/duplicated_rowspec.sql
@@ -0,0 +1,3 @@
+--!syntax_pg
+
+select * from plato.Input;
diff --git a/yql/essentials/tests/sql/suites/pg_duplicated/duplicated_rowspec.txt b/yql/essentials/tests/sql/suites/pg_duplicated/duplicated_rowspec.txt
new file mode 100644
index 0000000000..60840629ed
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/pg_duplicated/duplicated_rowspec.txt
@@ -0,0 +1,2 @@
+{x=1;x_generated_2=333333333333;};
+{x=3;x_generated_2=444444444444;};
\ No newline at end of file
diff --git a/yql/essentials/tests/sql/suites/pg_duplicated/duplicated_rowspec.txt.attr b/yql/essentials/tests/sql/suites/pg_duplicated/duplicated_rowspec.txt.attr
new file mode 100644
index 0000000000..66ac98585a
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/pg_duplicated/duplicated_rowspec.txt.attr
@@ -0,0 +1,36 @@
+{
+    "_yql_row_spec"={
+        "ColumnOrder"=["x";"x"];
+        "NativeYtTypeFlags"=0u;
+        "StrictSchema"= %true;
+        "TypePatch"= [
+            "StructType"; [
+                ["x"; ["PgType";"int4"]];
+                ["x_generated_2"; ["PgType";"int8"]];
+            ]
+        ]
+    };
+    "schema"=<
+        "strict"=%true;
+        "unique_keys"=%false;
+    >[
+        {
+            "name"= "x";
+            "required"= %false;
+            "type"= "int32";
+            "type_v3"= {
+                "item"= "int32";
+                "type_name"= "optional";
+            }
+        };
+        {
+            "name"= "x_generated_2";
+            "required"= %false;
+            "type"= "int64";
+            "type_v3"= {
+                "item"= "int64";
+                "type_name"= "optional";
+            }
+        };
+    ]
+}
\ No newline at end of file