aboutsummaryrefslogtreecommitdiffstats
path: root/yql
diff options
context:
space:
mode:
authormrlolthe1st <mrlolthe1st@yandex-team.com>2024-12-24 22:01:20 +0300
committermrlolthe1st <mrlolthe1st@yandex-team.com>2024-12-24 22:18:36 +0300
commit845baeff50212ec002c6a67e16bdf7c03c997fbf (patch)
treebb2553525d6dcaf5ed72fd3f7114e44a6ccfc5e2 /yql
parentd8df7f1c1f12dbb8cc66503852ee49362c824832 (diff)
downloadydb-845baeff50212ec002c6a67e16bdf7c03c997fbf.tar.gz
YQL-19409: Column duplicates in yt codec
YQL-19409: Column duplicates in yt codec commit_hash:65386b6c2642fc2d19b202be9fb379b22c3cbf46
Diffstat (limited to 'yql')
-rw-r--r--yql/essentials/core/dq_integration/yql_dq_integration.h3
-rw-r--r--yql/essentials/core/yql_type_annotation.h27
-rw-r--r--yql/essentials/providers/common/dq/yql_dq_integration_impl.cpp2
-rw-r--r--yql/essentials/providers/common/dq/yql_dq_integration_impl.h2
-rw-r--r--yql/essentials/providers/common/schema/expr/yql_expr_schema.cpp20
-rw-r--r--yql/essentials/tests/sql/sql2yql/canondata/result.json7
-rw-r--r--yql/essentials/tests/sql/suites/pg_duplicated/duplicated_rowspec.cfg1
-rw-r--r--yql/essentials/tests/sql/suites/pg_duplicated/duplicated_rowspec.sql3
-rw-r--r--yql/essentials/tests/sql/suites/pg_duplicated/duplicated_rowspec.txt2
-rw-r--r--yql/essentials/tests/sql/suites/pg_duplicated/duplicated_rowspec.txt.attr36
10 files changed, 93 insertions, 10 deletions
diff --git a/yql/essentials/core/dq_integration/yql_dq_integration.h b/yql/essentials/core/dq_integration/yql_dq_integration.h
index 1ac9427f65..bc61105a6b 100644
--- a/yql/essentials/core/dq_integration/yql_dq_integration.h
+++ b/yql/essentials/core/dq_integration/yql_dq_integration.h
@@ -4,6 +4,7 @@
#include <yql/essentials/core/yql_data_provider.h>
#include <yql/essentials/core/yql_statistics.h>
#include <yql/essentials/core/expr_nodes/yql_expr_nodes.h>
+#include <yql/essentials/core/yql_type_annotation.h>
#include <yql/essentials/public/issue/yql_issue.h>
#include <library/cpp/yson/writer.h>
@@ -81,7 +82,7 @@ public:
virtual void FillSinkSettings(const TExprNode& node, ::google::protobuf::Any& settings, TString& sinkType) = 0;
virtual void FillTransformSettings(const TExprNode& node, ::google::protobuf::Any& settings) = 0;
virtual void Annotate(const TExprNode& node, THashMap<TString, TString>& params) = 0;
- virtual bool PrepareFullResultTableParams(const TExprNode& root, TExprContext& ctx, THashMap<TString, TString>& params, THashMap<TString, TString>& secureParams) = 0;
+ virtual bool PrepareFullResultTableParams(const TExprNode& root, TExprContext& ctx, THashMap<TString, TString>& params, THashMap<TString, TString>& secureParams, const TMaybe<TColumnOrder>& = {}) = 0;
virtual void WriteFullResultTableRef(NYson::TYsonWriter& writer, const TVector<TString>& columns, const THashMap<TString, TString>& graphParams) = 0;
// Fill plan operator properties for sources/sinks
diff --git a/yql/essentials/core/yql_type_annotation.h b/yql/essentials/core/yql_type_annotation.h
index b508dc6778..ac0f0f36bd 100644
--- a/yql/essentials/core/yql_type_annotation.h
+++ b/yql/essentials/core/yql_type_annotation.h
@@ -212,6 +212,33 @@ public:
const TOrderedItem& back() const {
return Order_.back();
}
+
+ TVector<TString> GetLogicalNames() const {
+ TVector<TString> res;
+ res.reserve(Order_.size());
+ for (const auto &[name, _]: Order_) {
+ res.emplace_back(name);
+ }
+ return res;
+ }
+
+ TVector<TString> GetPhysicalNames() const {
+ TVector<TString> res;
+ res.reserve(Order_.size());
+ for (const auto &[_, name]: Order_) {
+ res.emplace_back(name);
+ }
+ return res;
+ }
+
+ bool HasDuplicates() const {
+ for (const auto& e: Order_) {
+ if (e.PhysicalName != e.LogicalName) {
+ return true;
+ }
+ }
+ return false;
+ }
private:
THashMap<TString, TString> GeneratedToOriginal_;
THashMap<TString, uint64_t> UseCount_;
diff --git a/yql/essentials/providers/common/dq/yql_dq_integration_impl.cpp b/yql/essentials/providers/common/dq/yql_dq_integration_impl.cpp
index 4782a79d57..098cc7a266 100644
--- a/yql/essentials/providers/common/dq/yql_dq_integration_impl.cpp
+++ b/yql/essentials/providers/common/dq/yql_dq_integration_impl.cpp
@@ -74,7 +74,7 @@ void TDqIntegrationBase::FillTransformSettings(const TExprNode&, ::google::proto
void TDqIntegrationBase::Annotate(const TExprNode&, THashMap<TString, TString>&) {
}
-bool TDqIntegrationBase::PrepareFullResultTableParams(const TExprNode&, TExprContext&, THashMap<TString, TString>&, THashMap<TString, TString>&) {
+bool TDqIntegrationBase::PrepareFullResultTableParams(const TExprNode&, TExprContext&, THashMap<TString, TString>&, THashMap<TString, TString>&, const TMaybe<TColumnOrder>&) {
return false;
}
diff --git a/yql/essentials/providers/common/dq/yql_dq_integration_impl.h b/yql/essentials/providers/common/dq/yql_dq_integration_impl.h
index 1fbcb33d77..d4e1597b85 100644
--- a/yql/essentials/providers/common/dq/yql_dq_integration_impl.h
+++ b/yql/essentials/providers/common/dq/yql_dq_integration_impl.h
@@ -23,7 +23,7 @@ public:
void FillSinkSettings(const TExprNode& node, ::google::protobuf::Any& settings, TString& sinkType) override;
void FillTransformSettings(const TExprNode& node, ::google::protobuf::Any& settings) override;
void Annotate(const TExprNode& node, THashMap<TString, TString>& params) override;
- bool PrepareFullResultTableParams(const TExprNode& root, TExprContext& ctx, THashMap<TString, TString>& params, THashMap<TString, TString>& secureParams) override;
+ bool PrepareFullResultTableParams(const TExprNode& root, TExprContext& ctx, THashMap<TString, TString>& params, THashMap<TString, TString>& secureParams, const TMaybe<TColumnOrder>& order = {}) override;
void WriteFullResultTableRef(NYson::TYsonWriter& writer, const TVector<TString>& columns, const THashMap<TString, TString>& graphParams) override;
bool FillSourcePlanProperties(const NNodes::TExprBase& node, TMap<TString, NJson::TJsonValue>& properties) override;
bool FillSinkPlanProperties(const NNodes::TExprBase& node, TMap<TString, NJson::TJsonValue>& properties) override;
diff --git a/yql/essentials/providers/common/schema/expr/yql_expr_schema.cpp b/yql/essentials/providers/common/schema/expr/yql_expr_schema.cpp
index 58a138b764..1ac4c89927 100644
--- a/yql/essentials/providers/common/schema/expr/yql_expr_schema.cpp
+++ b/yql/essentials/providers/common/schema/expr/yql_expr_schema.cpp
@@ -46,7 +46,7 @@ class TExprTypeSaver: public TSaver<TExprTypeSaver<TSaver>> {
struct TMappingOrderedStructAdaptor {
TVector<std::pair<TStringBuf, const TTypeAnnotationNode*>> Members;
- TMappingOrderedStructAdaptor(const TStructMemberMapper& mapper, const TMaybe<TColumnOrder>& columns, const TStructExprType* type)
+ TMappingOrderedStructAdaptor(const TStructMemberMapper& mapper, const TMaybe<TColumnOrder>& columns, const TStructExprType* type, bool writePhysical = true)
{
TMap<TStringBuf, const TTypeAnnotationNode*> members;
for (auto& item: type->GetItems()) {
@@ -61,7 +61,7 @@ class TExprTypeSaver: public TSaver<TExprTypeSaver<TSaver>> {
for (auto& [column, gen_column] : *columns) {
auto it = members.find(gen_column);
if (it != members.end()) {
- Members.emplace_back(column, it->second);
+ Members.emplace_back(writePhysical ? gen_column : column, it->second);
}
}
} else {
@@ -229,9 +229,9 @@ public:
}
}
- void SaveStructType(const TStructExprType* type, const TMaybe<TColumnOrder>& columns, const TStructMemberMapper& mapper) {
+ void SaveStructType(const TStructExprType* type, const TMaybe<TColumnOrder>& columns, const TStructMemberMapper& mapper, bool physical = true) {
if (mapper || columns) {
- TBase::SaveStructType(TMappingOrderedStructAdaptor(mapper, columns, type));
+ TBase::SaveStructType(TMappingOrderedStructAdaptor(mapper, columns, type, physical));
} else {
Save(type);
}
@@ -240,7 +240,12 @@ public:
void SaveStructTypeToYson(NYson::TYsonConsumerBase& writer, const TStructExprType* type, const TMaybe<TColumnOrder>& columns, const TStructMemberMapper& mapper, bool extendedForm) {
TExprTypeSaver<TYqlTypeYsonSaverImpl> saver(writer, extendedForm);
- saver.SaveStructType(type, columns, mapper);
+ saver.SaveStructType(type, columns, mapper, true);
+}
+
+void SaveStructTypeToYsonWithLogicalNames(NYson::TYsonConsumerBase& writer, const TStructExprType* type, const TMaybe<TColumnOrder>& columns, const TStructMemberMapper& mapper, bool extendedForm) {
+ TExprTypeSaver<TYqlTypeYsonSaverImpl> saver(writer, extendedForm);
+ saver.SaveStructType(type, columns, mapper, false);
}
void WriteTypeToYson(NYson::TYsonConsumerBase& writer, const TTypeAnnotationNode* type, bool extendedForm) {
@@ -315,8 +320,9 @@ struct TExprTypeLoader {
}
TMaybe<TType> LoadStructType(const TVector<std::pair<TString, TType>>& members, ui32 /*level*/) {
TVector<const TItemExprType*> items;
+ TColumnOrder order;
for (auto& member: members) {
- items.push_back(Ctx.MakeType<TItemExprType>(member.first, member.second));
+ items.push_back(Ctx.MakeType<TItemExprType>(order.AddColumn(member.first), member.second));
}
auto ret = Ctx.MakeType<TStructExprType>(items);
YQL_ENSURE(ret->Validate(TPosition(), Ctx));
@@ -430,7 +436,7 @@ void WriteResOrPullType(NYson::TYsonConsumerBase& writer, const TTypeAnnotationN
writer.OnStringScalar("ListType");
writer.OnListItem();
- SaveStructTypeToYson(writer, type->Cast<TListExprType>()->GetItemType()->Cast<TStructExprType>(), columns, {}, true);
+ SaveStructTypeToYsonWithLogicalNames(writer, type->Cast<TListExprType>()->GetItemType()->Cast<TStructExprType>(), columns, {}, true);
writer.OnEndList();
}
diff --git a/yql/essentials/tests/sql/sql2yql/canondata/result.json b/yql/essentials/tests/sql/sql2yql/canondata/result.json
index da8e4a69ce..715524f3f9 100644
--- a/yql/essentials/tests/sql/sql2yql/canondata/result.json
+++ b/yql/essentials/tests/sql/sql2yql/canondata/result.json
@@ -15672,6 +15672,13 @@
"uri": "https://{canondata_backend}/1937429/434276f26b2857be3c5ad3fdbbf877d2bf775ac5/resource.tar.gz#test_sql2yql.test_pg_duplicated-ambigous_ref_/sql.yql"
}
],
+ "test_sql2yql.test[pg_duplicated-duplicated_rowspec]": [
+ {
+ "checksum": "70c2665c84dec52c9318ad1b43450c08",
+ "size": 623,
+ "uri": "https://{canondata_backend}/1871002/c5bebc3ea8669574f080ea93959c29fb53e4bc67/resource.tar.gz#test_sql2yql.test_pg_duplicated-duplicated_rowspec_/sql.yql"
+ }
+ ],
"test_sql2yql.test[pg_duplicated-order_by_duplicated_proj_on_column_ref]": [
{
"checksum": "e0bd0cd714e708b53afbfb0770cf3e30",
diff --git a/yql/essentials/tests/sql/suites/pg_duplicated/duplicated_rowspec.cfg b/yql/essentials/tests/sql/suites/pg_duplicated/duplicated_rowspec.cfg
new file mode 100644
index 0000000000..ab354ca5a0
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/pg_duplicated/duplicated_rowspec.cfg
@@ -0,0 +1 @@
+in input duplicated_rowspec.txt \ No newline at end of file
diff --git a/yql/essentials/tests/sql/suites/pg_duplicated/duplicated_rowspec.sql b/yql/essentials/tests/sql/suites/pg_duplicated/duplicated_rowspec.sql
new file mode 100644
index 0000000000..4d7bd0853f
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/pg_duplicated/duplicated_rowspec.sql
@@ -0,0 +1,3 @@
+--!syntax_pg
+
+select * from plato.Input;
diff --git a/yql/essentials/tests/sql/suites/pg_duplicated/duplicated_rowspec.txt b/yql/essentials/tests/sql/suites/pg_duplicated/duplicated_rowspec.txt
new file mode 100644
index 0000000000..60840629ed
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/pg_duplicated/duplicated_rowspec.txt
@@ -0,0 +1,2 @@
+{x=1;x_generated_2=333333333333;};
+{x=3;x_generated_2=444444444444;}; \ No newline at end of file
diff --git a/yql/essentials/tests/sql/suites/pg_duplicated/duplicated_rowspec.txt.attr b/yql/essentials/tests/sql/suites/pg_duplicated/duplicated_rowspec.txt.attr
new file mode 100644
index 0000000000..66ac98585a
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/pg_duplicated/duplicated_rowspec.txt.attr
@@ -0,0 +1,36 @@
+{
+ "_yql_row_spec"={
+ "ColumnOrder"=["x";"x"];
+ "NativeYtTypeFlags"=0u;
+ "StrictSchema"= %true;
+ "TypePatch"= [
+ "StructType"; [
+ ["x"; ["PgType";"int4"]];
+ ["x_generated_2"; ["PgType";"int8"]];
+ ]
+ ]
+ };
+ "schema"=<
+ "strict"=%true;
+ "unique_keys"=%false;
+ >[
+ {
+ "name"= "x";
+ "required"= %false;
+ "type"= "int32";
+ "type_v3"= {
+ "item"= "int32";
+ "type_name"= "optional";
+ }
+ };
+ {
+ "name"= "x_generated_2";
+ "required"= %false;
+ "type"= "int64";
+ "type_v3"= {
+ "item"= "int64";
+ "type_name"= "optional";
+ }
+ };
+ ]
+} \ No newline at end of file