Fix aux column handling in finalyzing optimizers

YQL-19332

\* Changelog entry

Type: fix

Component: yql
commit_hash:bd508d1a1c23e797eff08bbc7da11d5cbde05fb3

6 files changed, 50 insertions, 3 deletions

diff --git a/yql/essentials/tests/sql/sql2yql/canondata/result.json b/yql/essentials/tests/sql/sql2yql/canondata/result.json
index dac11c7bd37..ad1c6773945 100644
--- a/yql/essentials/tests/sql/sql2yql/canondata/result.json
+++ b/yql/essentials/tests/sql/sql2yql/canondata/result.json
@@ -7559,6 +7559,13 @@
             "uri": "https://{canondata_backend}/1936273/4a1b39013e1bae40e722cff8ccef8829784964e2/resource.tar.gz#test_sql2yql.test_hor_join-yql-6477_table_path_/sql.yql"
         }
     ],
+    "test_sql2yql.test[hor_join-yql19332_aux_cols]": [
+        {
+            "checksum": "195e1068ca8926582b211c2b2eb5bfbf",
+            "size": 3144,
+            "uri": "https://{canondata_backend}/1784117/39173fc32d12059e969bfcf094e71dd92d4acaeb/resource.tar.gz#test_sql2yql.test_hor_join-yql19332_aux_cols_/sql.yql"
+        }
+    ],
     "test_sql2yql.test[in-basic_in]": [
         {
             "checksum": "d65b144596eb506f4cc5f1f191a89832",
@@ -25272,6 +25279,11 @@
             "uri": "file://test_sql_format.test_hor_join-yql-6477_table_path_/formatted.sql"
         }
     ],
+    "test_sql_format.test[hor_join-yql19332_aux_cols]": [
+        {
+            "uri": "file://test_sql_format.test_hor_join-yql19332_aux_cols_/formatted.sql"
+        }
+    ],
     "test_sql_format.test[in-basic_in]": [
         {
             "uri": "file://test_sql_format.test_in-basic_in_/formatted.sql"
diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_hor_join-yql19332_aux_cols_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_hor_join-yql19332_aux_cols_/formatted.sql
new file mode 100644
index 00000000000..e5628f6d894
--- /dev/null
+++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_hor_join-yql19332_aux_cols_/formatted.sql
@@ -0,0 +1,20 @@
+-- ignore runonopt plan diff
+USE plato;
+PRAGMA DisableSimpleColumns;
+PRAGMA yt.JoinMergeTablesLimit = "10";
+
+$i =
+    SELECT
+        *
+    FROM concat(Input2, Input3)
+    WHERE value != "1";
+
+SELECT
+    b.key,
+    b.subkey,
+    a.value
+FROM Input1
+    AS a
+JOIN $i
+    AS b
+ON (a.key, a.subkey) == (b.key, b.subkey);
diff --git a/yql/essentials/tests/sql/suites/hor_join/yql19332_aux_cols.cfg b/yql/essentials/tests/sql/suites/hor_join/yql19332_aux_cols.cfg
new file mode 100644
index 00000000000..3eb7660ec93
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/hor_join/yql19332_aux_cols.cfg
@@ -0,0 +1,3 @@
+in Input1 sorted.txt
+in Input2 sorted.txt
+in Input3 sorted.txt
diff --git a/yql/essentials/tests/sql/suites/hor_join/yql19332_aux_cols.sql b/yql/essentials/tests/sql/suites/hor_join/yql19332_aux_cols.sql
new file mode 100644
index 00000000000..c84769777c1
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/hor_join/yql19332_aux_cols.sql
@@ -0,0 +1,9 @@
+-- ignore runonopt plan diff
+USE plato;
+pragma DisableSimpleColumns;
+pragma yt.JoinMergeTablesLimit="10";
+
+$i = select * from concat(Input2, Input3) where value != "1";
+
+select b.key, b.subkey, a.value from Input1 as a
+join $i as b on (a.key, a.subkey) == (b.key, b.subkey)
diff --git a/yt/yql/providers/yt/provider/yql_yt_horizontal_join.cpp b/yt/yql/providers/yt/provider/yql_yt_horizontal_join.cpp
index b35f82aca04..f680d995686 100644
--- a/yt/yql/providers/yt/provider/yql_yt_horizontal_join.cpp
+++ b/yt/yql/providers/yt/provider/yql_yt_horizontal_join.cpp
@@ -999,7 +999,7 @@ bool THorizontalJoinOptimizer::MakeJoinedMap(TPositionHandle pos, TExprContext&
             else if (nextNewOutIndex < outRemap.size()) {
                 TVector<TExprBase> tupleTypes;
                 for (auto out: joinedMapOuts) {
-                    auto itemType = out.Ref().GetTypeAnn()->Cast<TListExprType>()->GetItemType();
+                    auto itemType = TYqlRowSpecInfo(out.RowSpec()).GetExtendedType(ctx);
                     tupleTypes.push_back(TExprBase(ExpandType(pos, *itemType, ctx)));
                 }
                 TExprBase varType = Build<TCoVariantType>(ctx, pos)
diff --git a/yt/yql/providers/yt/provider/yql_yt_physical_finalizing.cpp b/yt/yql/providers/yt/provider/yql_yt_physical_finalizing.cpp
index 01cf4708e7f..1ebcfcc1e12 100644
--- a/yt/yql/providers/yt/provider/yql_yt_physical_finalizing.cpp
+++ b/yt/yql/providers/yt/provider/yql_yt_physical_finalizing.cpp
@@ -1026,10 +1026,12 @@ private:
         bool changedOutSort = false;
         TVector<TYtOutTable> outTables;
         TExprNode::TListType filterColumns(op.Output().Size());
+        TTypeAnnotationNode::TListType varItemTypes(op.Output().Size());
         for (size_t i = 0; i < op.Output().Size(); ++i) {
             auto out = op.Output().Item(i);
+            auto rowSpec = TYtTableBaseInfo::GetRowSpec(out);
+            varItemTypes[i] = rowSpec->GetExtendedType(ctx);
             if (unorderedOuts.Test(i)) {
-                auto rowSpec = TYtTableBaseInfo::GetRowSpec(out);
                 YQL_ENSURE(rowSpec);
                 if (rowSpec->IsSorted()) {
                     if (rowSpec->HasAuxColumns()) {
@@ -1038,6 +1040,7 @@ private:
                             columns.emplace_back(item->GetName());
                         }
                         filterColumns[i] = ToAtomList(columns, node.Pos(), ctx);
+                        varItemTypes[i] = rowSpec->GetType();
                     }
                     rowSpec->ClearSortness(ctx);
                     outTables.push_back(TYtOutTable(ctx.ChangeChild(out.Ref(), TYtOutTable::idx_RowSpec, rowSpec->ToExprNode(ctx, out.Pos()).Ptr())));
@@ -1156,7 +1159,7 @@ private:
                         .Build()
                         .Done().Ptr();
                 } else {
-                    auto varType = ExpandType(lambda->Pos(), *GetSeqItemType(node.Child(lambdaIdx)->GetTypeAnn()), ctx);
+                    auto varType = ExpandType(lambda->Pos(), *ctx.MakeType<TVariantExprType>(ctx.MakeType<TTupleExprType>(varItemTypes)), ctx);
                     TVector<TExprBase> visitArgs;
                     for (size_t i = 0; i < op.Output().Size(); ++i) {
                         visitArgs.push_back(Build<TCoAtom>(ctx, lambda->Pos()).Value(ToString(i)).Done());