diff options
author | mrlolthe1st <mrlolthe1st@yandex-team.com> | 2024-12-24 22:01:20 +0300 |
---|---|---|
committer | mrlolthe1st <mrlolthe1st@yandex-team.com> | 2024-12-24 22:18:36 +0300 |
commit | 845baeff50212ec002c6a67e16bdf7c03c997fbf (patch) | |
tree | bb2553525d6dcaf5ed72fd3f7114e44a6ccfc5e2 /yql | |
parent | d8df7f1c1f12dbb8cc66503852ee49362c824832 (diff) | |
download | ydb-845baeff50212ec002c6a67e16bdf7c03c997fbf.tar.gz |
YQL-19409: Column duplicates in yt codec
YQL-19409: Column duplicates in yt codec
commit_hash:65386b6c2642fc2d19b202be9fb379b22c3cbf46
Diffstat (limited to 'yql')
10 files changed, 93 insertions, 10 deletions
diff --git a/yql/essentials/core/dq_integration/yql_dq_integration.h b/yql/essentials/core/dq_integration/yql_dq_integration.h index 1ac9427f65..bc61105a6b 100644 --- a/yql/essentials/core/dq_integration/yql_dq_integration.h +++ b/yql/essentials/core/dq_integration/yql_dq_integration.h @@ -4,6 +4,7 @@ #include <yql/essentials/core/yql_data_provider.h> #include <yql/essentials/core/yql_statistics.h> #include <yql/essentials/core/expr_nodes/yql_expr_nodes.h> +#include <yql/essentials/core/yql_type_annotation.h> #include <yql/essentials/public/issue/yql_issue.h> #include <library/cpp/yson/writer.h> @@ -81,7 +82,7 @@ public: virtual void FillSinkSettings(const TExprNode& node, ::google::protobuf::Any& settings, TString& sinkType) = 0; virtual void FillTransformSettings(const TExprNode& node, ::google::protobuf::Any& settings) = 0; virtual void Annotate(const TExprNode& node, THashMap<TString, TString>& params) = 0; - virtual bool PrepareFullResultTableParams(const TExprNode& root, TExprContext& ctx, THashMap<TString, TString>& params, THashMap<TString, TString>& secureParams) = 0; + virtual bool PrepareFullResultTableParams(const TExprNode& root, TExprContext& ctx, THashMap<TString, TString>& params, THashMap<TString, TString>& secureParams, const TMaybe<TColumnOrder>& = {}) = 0; virtual void WriteFullResultTableRef(NYson::TYsonWriter& writer, const TVector<TString>& columns, const THashMap<TString, TString>& graphParams) = 0; // Fill plan operator properties for sources/sinks diff --git a/yql/essentials/core/yql_type_annotation.h b/yql/essentials/core/yql_type_annotation.h index b508dc6778..ac0f0f36bd 100644 --- a/yql/essentials/core/yql_type_annotation.h +++ b/yql/essentials/core/yql_type_annotation.h @@ -212,6 +212,33 @@ public: const TOrderedItem& back() const { return Order_.back(); } + + TVector<TString> GetLogicalNames() const { + TVector<TString> res; + res.reserve(Order_.size()); + for (const auto &[name, _]: Order_) { + res.emplace_back(name); + } + return res; + } + + TVector<TString> GetPhysicalNames() const { + TVector<TString> res; + res.reserve(Order_.size()); + for (const auto &[_, name]: Order_) { + res.emplace_back(name); + } + return res; + } + + bool HasDuplicates() const { + for (const auto& e: Order_) { + if (e.PhysicalName != e.LogicalName) { + return true; + } + } + return false; + } private: THashMap<TString, TString> GeneratedToOriginal_; THashMap<TString, uint64_t> UseCount_; diff --git a/yql/essentials/providers/common/dq/yql_dq_integration_impl.cpp b/yql/essentials/providers/common/dq/yql_dq_integration_impl.cpp index 4782a79d57..098cc7a266 100644 --- a/yql/essentials/providers/common/dq/yql_dq_integration_impl.cpp +++ b/yql/essentials/providers/common/dq/yql_dq_integration_impl.cpp @@ -74,7 +74,7 @@ void TDqIntegrationBase::FillTransformSettings(const TExprNode&, ::google::proto void TDqIntegrationBase::Annotate(const TExprNode&, THashMap<TString, TString>&) { } -bool TDqIntegrationBase::PrepareFullResultTableParams(const TExprNode&, TExprContext&, THashMap<TString, TString>&, THashMap<TString, TString>&) { +bool TDqIntegrationBase::PrepareFullResultTableParams(const TExprNode&, TExprContext&, THashMap<TString, TString>&, THashMap<TString, TString>&, const TMaybe<TColumnOrder>&) { return false; } diff --git a/yql/essentials/providers/common/dq/yql_dq_integration_impl.h b/yql/essentials/providers/common/dq/yql_dq_integration_impl.h index 1fbcb33d77..d4e1597b85 100644 --- a/yql/essentials/providers/common/dq/yql_dq_integration_impl.h +++ b/yql/essentials/providers/common/dq/yql_dq_integration_impl.h @@ -23,7 +23,7 @@ public: void FillSinkSettings(const TExprNode& node, ::google::protobuf::Any& settings, TString& sinkType) override; void FillTransformSettings(const TExprNode& node, ::google::protobuf::Any& settings) override; void Annotate(const TExprNode& node, THashMap<TString, TString>& params) override; - bool PrepareFullResultTableParams(const TExprNode& root, TExprContext& ctx, THashMap<TString, TString>& params, THashMap<TString, TString>& secureParams) override; + bool PrepareFullResultTableParams(const TExprNode& root, TExprContext& ctx, THashMap<TString, TString>& params, THashMap<TString, TString>& secureParams, const TMaybe<TColumnOrder>& order = {}) override; void WriteFullResultTableRef(NYson::TYsonWriter& writer, const TVector<TString>& columns, const THashMap<TString, TString>& graphParams) override; bool FillSourcePlanProperties(const NNodes::TExprBase& node, TMap<TString, NJson::TJsonValue>& properties) override; bool FillSinkPlanProperties(const NNodes::TExprBase& node, TMap<TString, NJson::TJsonValue>& properties) override; diff --git a/yql/essentials/providers/common/schema/expr/yql_expr_schema.cpp b/yql/essentials/providers/common/schema/expr/yql_expr_schema.cpp index 58a138b764..1ac4c89927 100644 --- a/yql/essentials/providers/common/schema/expr/yql_expr_schema.cpp +++ b/yql/essentials/providers/common/schema/expr/yql_expr_schema.cpp @@ -46,7 +46,7 @@ class TExprTypeSaver: public TSaver<TExprTypeSaver<TSaver>> { struct TMappingOrderedStructAdaptor { TVector<std::pair<TStringBuf, const TTypeAnnotationNode*>> Members; - TMappingOrderedStructAdaptor(const TStructMemberMapper& mapper, const TMaybe<TColumnOrder>& columns, const TStructExprType* type) + TMappingOrderedStructAdaptor(const TStructMemberMapper& mapper, const TMaybe<TColumnOrder>& columns, const TStructExprType* type, bool writePhysical = true) { TMap<TStringBuf, const TTypeAnnotationNode*> members; for (auto& item: type->GetItems()) { @@ -61,7 +61,7 @@ class TExprTypeSaver: public TSaver<TExprTypeSaver<TSaver>> { for (auto& [column, gen_column] : *columns) { auto it = members.find(gen_column); if (it != members.end()) { - Members.emplace_back(column, it->second); + Members.emplace_back(writePhysical ? gen_column : column, it->second); } } } else { @@ -229,9 +229,9 @@ public: } } - void SaveStructType(const TStructExprType* type, const TMaybe<TColumnOrder>& columns, const TStructMemberMapper& mapper) { + void SaveStructType(const TStructExprType* type, const TMaybe<TColumnOrder>& columns, const TStructMemberMapper& mapper, bool physical = true) { if (mapper || columns) { - TBase::SaveStructType(TMappingOrderedStructAdaptor(mapper, columns, type)); + TBase::SaveStructType(TMappingOrderedStructAdaptor(mapper, columns, type, physical)); } else { Save(type); } @@ -240,7 +240,12 @@ public: void SaveStructTypeToYson(NYson::TYsonConsumerBase& writer, const TStructExprType* type, const TMaybe<TColumnOrder>& columns, const TStructMemberMapper& mapper, bool extendedForm) { TExprTypeSaver<TYqlTypeYsonSaverImpl> saver(writer, extendedForm); - saver.SaveStructType(type, columns, mapper); + saver.SaveStructType(type, columns, mapper, true); +} + +void SaveStructTypeToYsonWithLogicalNames(NYson::TYsonConsumerBase& writer, const TStructExprType* type, const TMaybe<TColumnOrder>& columns, const TStructMemberMapper& mapper, bool extendedForm) { + TExprTypeSaver<TYqlTypeYsonSaverImpl> saver(writer, extendedForm); + saver.SaveStructType(type, columns, mapper, false); } void WriteTypeToYson(NYson::TYsonConsumerBase& writer, const TTypeAnnotationNode* type, bool extendedForm) { @@ -315,8 +320,9 @@ struct TExprTypeLoader { } TMaybe<TType> LoadStructType(const TVector<std::pair<TString, TType>>& members, ui32 /*level*/) { TVector<const TItemExprType*> items; + TColumnOrder order; for (auto& member: members) { - items.push_back(Ctx.MakeType<TItemExprType>(member.first, member.second)); + items.push_back(Ctx.MakeType<TItemExprType>(order.AddColumn(member.first), member.second)); } auto ret = Ctx.MakeType<TStructExprType>(items); YQL_ENSURE(ret->Validate(TPosition(), Ctx)); @@ -430,7 +436,7 @@ void WriteResOrPullType(NYson::TYsonConsumerBase& writer, const TTypeAnnotationN writer.OnStringScalar("ListType"); writer.OnListItem(); - SaveStructTypeToYson(writer, type->Cast<TListExprType>()->GetItemType()->Cast<TStructExprType>(), columns, {}, true); + SaveStructTypeToYsonWithLogicalNames(writer, type->Cast<TListExprType>()->GetItemType()->Cast<TStructExprType>(), columns, {}, true); writer.OnEndList(); } diff --git a/yql/essentials/tests/sql/sql2yql/canondata/result.json b/yql/essentials/tests/sql/sql2yql/canondata/result.json index da8e4a69ce..715524f3f9 100644 --- a/yql/essentials/tests/sql/sql2yql/canondata/result.json +++ b/yql/essentials/tests/sql/sql2yql/canondata/result.json @@ -15672,6 +15672,13 @@ "uri": "https://{canondata_backend}/1937429/434276f26b2857be3c5ad3fdbbf877d2bf775ac5/resource.tar.gz#test_sql2yql.test_pg_duplicated-ambigous_ref_/sql.yql" } ], + "test_sql2yql.test[pg_duplicated-duplicated_rowspec]": [ + { + "checksum": "70c2665c84dec52c9318ad1b43450c08", + "size": 623, + "uri": "https://{canondata_backend}/1871002/c5bebc3ea8669574f080ea93959c29fb53e4bc67/resource.tar.gz#test_sql2yql.test_pg_duplicated-duplicated_rowspec_/sql.yql" + } + ], "test_sql2yql.test[pg_duplicated-order_by_duplicated_proj_on_column_ref]": [ { "checksum": "e0bd0cd714e708b53afbfb0770cf3e30", diff --git a/yql/essentials/tests/sql/suites/pg_duplicated/duplicated_rowspec.cfg b/yql/essentials/tests/sql/suites/pg_duplicated/duplicated_rowspec.cfg new file mode 100644 index 0000000000..ab354ca5a0 --- /dev/null +++ b/yql/essentials/tests/sql/suites/pg_duplicated/duplicated_rowspec.cfg @@ -0,0 +1 @@ +in input duplicated_rowspec.txt
\ No newline at end of file diff --git a/yql/essentials/tests/sql/suites/pg_duplicated/duplicated_rowspec.sql b/yql/essentials/tests/sql/suites/pg_duplicated/duplicated_rowspec.sql new file mode 100644 index 0000000000..4d7bd0853f --- /dev/null +++ b/yql/essentials/tests/sql/suites/pg_duplicated/duplicated_rowspec.sql @@ -0,0 +1,3 @@ +--!syntax_pg + +select * from plato.Input; diff --git a/yql/essentials/tests/sql/suites/pg_duplicated/duplicated_rowspec.txt b/yql/essentials/tests/sql/suites/pg_duplicated/duplicated_rowspec.txt new file mode 100644 index 0000000000..60840629ed --- /dev/null +++ b/yql/essentials/tests/sql/suites/pg_duplicated/duplicated_rowspec.txt @@ -0,0 +1,2 @@ +{x=1;x_generated_2=333333333333;}; +{x=3;x_generated_2=444444444444;};
\ No newline at end of file diff --git a/yql/essentials/tests/sql/suites/pg_duplicated/duplicated_rowspec.txt.attr b/yql/essentials/tests/sql/suites/pg_duplicated/duplicated_rowspec.txt.attr new file mode 100644 index 0000000000..66ac98585a --- /dev/null +++ b/yql/essentials/tests/sql/suites/pg_duplicated/duplicated_rowspec.txt.attr @@ -0,0 +1,36 @@ +{ + "_yql_row_spec"={ + "ColumnOrder"=["x";"x"]; + "NativeYtTypeFlags"=0u; + "StrictSchema"= %true; + "TypePatch"= [ + "StructType"; [ + ["x"; ["PgType";"int4"]]; + ["x_generated_2"; ["PgType";"int8"]]; + ] + ] + }; + "schema"=< + "strict"=%true; + "unique_keys"=%false; + >[ + { + "name"= "x"; + "required"= %false; + "type"= "int32"; + "type_v3"= { + "item"= "int32"; + "type_name"= "optional"; + } + }; + { + "name"= "x_generated_2"; + "required"= %false; + "type"= "int64"; + "type_v3"= { + "item"= "int64"; + "type_name"= "optional"; + } + }; + ] +}
\ No newline at end of file |