Merge branch 'rightlib' into mergelibs-241120-1113

30 files changed, 301 insertions, 0 deletions

diff --git a/yql/essentials/tests/sql/suites/csee/closure_in_l1_and_l2.sql b/yql/essentials/tests/sql/suites/csee/closure_in_l1_and_l2.sql
new file mode 100644
index 0000000000..4feaabc625
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/csee/closure_in_l1_and_l2.sql
@@ -0,0 +1,12 @@
+SELECT
+    AsList(
+        ListMap(AsList(3,4),($x)->{
+            return $x
+            +Yql::Fold(AsList(9,10), 1, ($c,$d)->{return $c + $d + Yql::Fold(ListMap(AsList(5,6),($y)->{
+              return $x+$y
+            }), 0, ($a,$b)->{return $a+$b})})
+            +Yql::Fold(ListMap(AsList(7,8),($y)->{
+              return $x+$y
+            }), 0, ($a,$b)->{return $a+$b})
+        })
+    );
diff --git a/yql/essentials/tests/sql/suites/csee/closure_in_l2_and_l1.sql b/yql/essentials/tests/sql/suites/csee/closure_in_l2_and_l1.sql
new file mode 100644
index 0000000000..0456e07b1f
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/csee/closure_in_l2_and_l1.sql
@@ -0,0 +1,12 @@
+SELECT
+    AsList(
+        ListMap(AsList(3,4),($x)->{
+            return $x
+            +Yql::Fold(ListMap(AsList(7,8),($y)->{
+              return $x+$y
+            }), 0, ($a,$b)->{return $a+$b})
+            +Yql::Fold(AsList(9,10), 1, ($c,$d)->{return $c + $d + Yql::Fold(ListMap(AsList(5,6),($y)->{
+              return $x+$y
+            }), 0, ($a,$b)->{return $a+$b})})
+        })
+    );
diff --git a/yql/essentials/tests/sql/suites/csee/closure_l2.sql b/yql/essentials/tests/sql/suites/csee/closure_l2.sql
new file mode 100644
index 0000000000..16153c80f7
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/csee/closure_l2.sql
@@ -0,0 +1,4 @@
+SELECT
+    ListMap(AsList(1,2),($x)->{
+        return ListMap(AsList(100,101),($y)->{return $x+$y})
+    });
diff --git a/yql/essentials/tests/sql/suites/csee/complete_l2.sql b/yql/essentials/tests/sql/suites/csee/complete_l2.sql
new file mode 100644
index 0000000000..59ccdb43b3
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/csee/complete_l2.sql
@@ -0,0 +1,6 @@
+SELECT
+    AsList(
+        ListMap(AsList(1,2),($x)->{
+            return $x+Yql::Fold(AsList(1),0,($a,$b)->{return $a+$b})
+        })
+    );
\ No newline at end of file
diff --git a/yql/essentials/tests/sql/suites/csee/const_body_diff_lambda.sql b/yql/essentials/tests/sql/suites/csee/const_body_diff_lambda.sql
new file mode 100644
index 0000000000..7fb449778c
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/csee/const_body_diff_lambda.sql
@@ -0,0 +1,5 @@
+SELECT
+    AsTuple(
+        ListMap(AsList(1,2),($_x)->{return 3+4}),
+        ListMap(AsList("foo","bar"),($_x)->{return 3+4})
+    );
diff --git a/yql/essentials/tests/sql/suites/csee/const_body_same_lambda.sql b/yql/essentials/tests/sql/suites/csee/const_body_same_lambda.sql
new file mode 100644
index 0000000000..042e743c55
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/csee/const_body_same_lambda.sql
@@ -0,0 +1,5 @@
+SELECT
+    AsTuple(
+        ListMap(AsList(1,2),($_x)->{return 3+4}),
+        ListMap(AsList(5,6),($_x)->{return 3+4})
+    );
diff --git a/yql/essentials/tests/sql/suites/csee/default.cfg b/yql/essentials/tests/sql/suites/csee/default.cfg
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/csee/default.cfg
diff --git a/yql/essentials/tests/sql/suites/csee/expr.sql b/yql/essentials/tests/sql/suites/csee/expr.sql
new file mode 100644
index 0000000000..d859286e89
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/csee/expr.sql
@@ -0,0 +1 @@
+SELECT (1 + 2) * (1 + 2);
diff --git a/yql/essentials/tests/sql/suites/csee/expr_in_l0_and_l1.sql b/yql/essentials/tests/sql/suites/csee/expr_in_l0_and_l1.sql
new file mode 100644
index 0000000000..1c7a7298d8
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/csee/expr_in_l0_and_l1.sql
@@ -0,0 +1,7 @@
+SELECT
+    AsList(
+        AsList(1+2),
+        ListMap(AsList(1,2),($x)->{
+            return $x+(1+2)
+        }),
+    );
diff --git a/yql/essentials/tests/sql/suites/csee/expr_in_l1.sql b/yql/essentials/tests/sql/suites/csee/expr_in_l1.sql
new file mode 100644
index 0000000000..0d69aae125
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/csee/expr_in_l1.sql
@@ -0,0 +1,6 @@
+SELECT
+    AsList(
+        ListMap(AsList(1,2),($x)->{
+            return ($x+1)*($x+1)
+        }),
+    );
diff --git a/yql/essentials/tests/sql/suites/csee/expr_in_l1_and_l0.sql b/yql/essentials/tests/sql/suites/csee/expr_in_l1_and_l0.sql
new file mode 100644
index 0000000000..5379245254
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/csee/expr_in_l1_and_l0.sql
@@ -0,0 +1,7 @@
+SELECT
+    AsList(
+        ListMap(AsList(1,2),($x)->{
+            return $x+(1+2)
+        }),
+        AsList(1+2)
+    );
diff --git a/yql/essentials/tests/sql/suites/csee/l2_dup_l1.sql b/yql/essentials/tests/sql/suites/csee/l2_dup_l1.sql
new file mode 100644
index 0000000000..75ac138279
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/csee/l2_dup_l1.sql
@@ -0,0 +1,9 @@
+SELECT
+    AsList(
+        ListMap(AsList(1,2),($x)->{
+            return ($x+1)*($x+1)+Yql::Fold(AsList(1),0,($a,$b)->{return $a+$b})
+        }),
+        ListMap(AsList(10,11),($x)->{
+            return ($x+1)*($x+1)+Yql::Fold(AsList(1),0,($a,$b)->{return $a+$b})
+        })
+    );
\ No newline at end of file
diff --git a/yql/essentials/tests/sql/suites/csee/l2_dup_l1_many.sql b/yql/essentials/tests/sql/suites/csee/l2_dup_l1_many.sql
new file mode 100644
index 0000000000..e1a0c69772
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/csee/l2_dup_l1_many.sql
@@ -0,0 +1,9 @@
+SELECT
+    AsList(
+        ListMap(AsList(1,2),($_x)->{
+            return Yql::Fold(AsList(1),0,($a,$b)->{return $a+$b}) + Yql::Fold(AsList(1),0,($a,$b)->{return $a+$b})
+        }),
+        ListMap(AsList(10,11),($_x)->{
+            return Yql::Fold(AsList(1),0,($a,$b)->{return $a+$b}) * Yql::Fold(AsList(1),0,($a,$b)->{return $a+$b})
+        })
+    );
diff --git a/yql/essentials/tests/sql/suites/csee/lambda_in_l0_and_l1.sql b/yql/essentials/tests/sql/suites/csee/lambda_in_l0_and_l1.sql
new file mode 100644
index 0000000000..a5f244b9e6
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/csee/lambda_in_l0_and_l1.sql
@@ -0,0 +1,11 @@
+SELECT
+    AsList(
+        ListMap(AsList(1,2),($x)->{
+            return $x+1
+        }),
+        ListMap(AsList(3,4),($x)->{
+            return $x+Yql::Fold(ListMap(AsList(5,6),($x)->{
+              return $x+1
+            }), 0, ($a,$b)->{return $a+$b})
+        })
+    );
diff --git a/yql/essentials/tests/sql/suites/csee/lambda_in_l1_and_l0.sql b/yql/essentials/tests/sql/suites/csee/lambda_in_l1_and_l0.sql
new file mode 100644
index 0000000000..7a6a51cec0
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/csee/lambda_in_l1_and_l0.sql
@@ -0,0 +1,11 @@
+SELECT
+    AsList(
+        ListMap(AsList(3,4),($x)->{
+            return $x+Yql::Fold(ListMap(AsList(5,6),($x)->{
+              return $x+1
+            }), 0, ($a,$b)->{return $a+$b})
+        }),
+        ListMap(AsList(1,2),($x)->{
+            return $x+1
+        })
+    );
diff --git a/yql/essentials/tests/sql/suites/csee/nested_closure_in_l1_and_l2.sql b/yql/essentials/tests/sql/suites/csee/nested_closure_in_l1_and_l2.sql
new file mode 100644
index 0000000000..72f48c9697
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/csee/nested_closure_in_l1_and_l2.sql
@@ -0,0 +1,12 @@
+SELECT
+    AsList(
+        ListMap(AsList(3,4),($x)->{
+            return $x
+            +Yql::Fold(AsList(9,10), 1, ($c,$d)->{return $c + $d + Yql::Fold(ListMap(AsList(5,6),($y)->{
+              return $x+Yql::Fold(AsList(11,12), 2, ($e,$f)->{return $e+$f+$y})
+            }), 0, ($a,$b)->{return $a+$b+Yql::Fold(AsList(13,14), 3, ($i,$j)->{return $i+$j+$a})})})
+            +Yql::Fold(ListMap(AsList(7,8),($y)->{
+              return $x+Yql::Fold(AsList(11,12), 2, ($e,$f)->{return $e+$f+$y})
+            }), 0, ($a,$b)->{return $a+$b+Yql::Fold(AsList(13,14), 3, ($i,$j)->{return $i+$j+$a})})
+        })
+    );
diff --git a/yql/essentials/tests/sql/suites/csee/nested_closure_in_l1_and_l2_unordered.sql b/yql/essentials/tests/sql/suites/csee/nested_closure_in_l1_and_l2_unordered.sql
new file mode 100644
index 0000000000..ffdf9c639e
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/csee/nested_closure_in_l1_and_l2_unordered.sql
@@ -0,0 +1,12 @@
+SELECT
+    AsList(
+        ListMap(AsList(3,4),($x)->{
+            return MIN_OF($x,
+            Yql::Fold(AsList(9,10), 100, ($c,$d)->{return MIN_OF($c,$d,Yql::Fold(ListMap(AsList(5,6),($y)->{
+              return MIN_OF($x,Yql::Fold(AsList(11,12), 10, ($e,$f)->{return MIN_OF($f,$y,$e)}))
+            }), 100, ($a,$b)->{return MIN_OF($a,$b,Yql::Fold(AsList(13,14), 100, ($i,$j)->{return MIN_OF($i,$j,$a)}))}))}),
+            Yql::Fold(ListMap(AsList(7,8),($y)->{
+              return MIN_OF($x,Yql::Fold(AsList(11,12), 10, ($e,$f)->{return MIN_OF($e,$f,$y)}))
+            }), 100, ($a,$b)->{return MIN_OF($b,Yql::Fold(AsList(13,14), 100, ($i,$j)->{return MIN_OF($a,$j,$i)}),$a)}))
+        })
+    );
diff --git a/yql/essentials/tests/sql/suites/csee/nested_closure_in_l2_and_l1.sql b/yql/essentials/tests/sql/suites/csee/nested_closure_in_l2_and_l1.sql
new file mode 100644
index 0000000000..1adc7e0bdb
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/csee/nested_closure_in_l2_and_l1.sql
@@ -0,0 +1,12 @@
+SELECT
+    AsList(
+        ListMap(AsList(3,4),($x)->{
+            return $x
+            +Yql::Fold(ListMap(AsList(7,8),($y)->{
+              return $x+Yql::Fold(AsList(11,12), 2, ($e,$f)->{return $e+$f+$y})
+            }), 0, ($a,$b)->{return $a+$b+Yql::Fold(AsList(13,14), 3, ($i,$j)->{return $i+$j+$a})})
+            +Yql::Fold(AsList(9,10), 1, ($c,$d)->{return $c + $d + Yql::Fold(ListMap(AsList(5,6),($y)->{
+              return $x+Yql::Fold(AsList(11,12), 2, ($e,$f)->{return $e+$f+$y})
+            }), 0, ($a,$b)->{return $a+$b+Yql::Fold(AsList(13,14), 3, ($i,$j)->{return $i+$j+$a})})})
+        })
+    );
diff --git a/yql/essentials/tests/sql/suites/csee/nested_closure_in_l2_and_l1_unordered.sql b/yql/essentials/tests/sql/suites/csee/nested_closure_in_l2_and_l1_unordered.sql
new file mode 100644
index 0000000000..05475188d3
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/csee/nested_closure_in_l2_and_l1_unordered.sql
@@ -0,0 +1,13 @@
+SELECT
+    AsList(
+        ListMap(AsList(33,42),($x)->{
+            return MAX_OF($x,
+            Yql::Fold(ListMap(AsList(7,8),($y)->{
+              return MAX_OF($x,Yql::Fold(AsList(11,12), 2, ($e,$f)->{return MAX_OF($e,$f,$y)}))
+            }), 0, ($a,$b)->{return MAX_OF($a,$b,Yql::Fold(AsList(13,14), 3, ($i,$j)->{return MAX_OF($i,$j,$a)}))}),
+            Yql::Fold(AsList(9,10), 1, ($c,$d)->{return MAX_OF($c,$d,Yql::Fold(ListMap(AsList(5,6),($y)->{
+              return MAX_OF($x,Yql::Fold(AsList(11,12), 2, ($e,$f)->{return MAX_OF($f,$y,$e)}))
+            }), 0, ($a,$b)->{return MAX_OF(Yql::Fold(AsList(13,14), 3, ($i,$j)->{return MAX_OF($j,$a,$i)}),$a,$b)}))}))
+        })
+    );
+
diff --git a/yql/essentials/tests/sql/suites/csee/same_closure_l2.sql b/yql/essentials/tests/sql/suites/csee/same_closure_l2.sql
new file mode 100644
index 0000000000..b39ecaf0bb
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/csee/same_closure_l2.sql
@@ -0,0 +1,6 @@
+SELECT
+    ListMap(AsList(1,2),($x)->{
+        return ListExtend(
+            ListMap(AsList(100,101),($y)->{return $x+$y}),
+            ListMap(AsList(100,101),($y)->{return $x+$y}))
+    })
diff --git a/yql/essentials/tests/sql/suites/csee/same_complete_l2.sql b/yql/essentials/tests/sql/suites/csee/same_complete_l2.sql
new file mode 100644
index 0000000000..80c7af98cf
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/csee/same_complete_l2.sql
@@ -0,0 +1,9 @@
+SELECT
+    AsList(
+        ListMap(AsList(1,2),($x)->{
+            return $x+Yql::Fold(AsList(1),0,($a,$b)->{return $a+$b})
+        }),
+        ListMap(AsList(10,11),($x)->{
+            return $x+Yql::Fold(AsList(1),0,($a,$b)->{return $a+$b})
+        })
+    );
diff --git a/yql/essentials/tests/sql/suites/csee/same_complete_l2_expr.sql b/yql/essentials/tests/sql/suites/csee/same_complete_l2_expr.sql
new file mode 100644
index 0000000000..d6ed82b87f
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/csee/same_complete_l2_expr.sql
@@ -0,0 +1,9 @@
+SELECT
+    AsList(
+        ListMap(AsList(1,2),($x)->{
+            return $x+Yql::Fold(AsList(1),0,($a,$b)->{return $a+$b})
+        }),
+        ListMap(AsList(1,2),($x)->{
+            return $x+Yql::Fold(AsList(1),0,($a,$b)->{return $a+$b})
+        })
+    );
diff --git a/yql/essentials/tests/sql/suites/csee/same_free_closure_size2.sql b/yql/essentials/tests/sql/suites/csee/same_free_closure_size2.sql
new file mode 100644
index 0000000000..c145deeab4
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/csee/same_free_closure_size2.sql
@@ -0,0 +1,11 @@
+SELECT
+    AsTuple(
+            YQL::FlatMap(3/1, ($x)->{
+                return YQL::FlatMap(1/1,($y)->{
+            return Just($x + $y)
+            })}),
+            YQL::FlatMap(4/1, ($x)->{
+                return YQL::FlatMap(1/1,($y)->{
+            return Just($x + $y)
+            })})
+        )
diff --git a/yql/essentials/tests/sql/suites/csee/same_free_closure_size2_arg_both_level.sql b/yql/essentials/tests/sql/suites/csee/same_free_closure_size2_arg_both_level.sql
new file mode 100644
index 0000000000..a6b9a568fd
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/csee/same_free_closure_size2_arg_both_level.sql
@@ -0,0 +1,11 @@
+SELECT
+    AsTuple(
+            YQL::FlatMap(3/1, ($x)->{
+                return $x + YQL::FlatMap(1/1,($y)->{
+            return Just($x + $y)
+            })}),
+            YQL::FlatMap(4/1, ($x)->{
+                return $x + YQL::FlatMap(1/1,($y)->{
+            return Just($x + $y)
+            })})
+        )
diff --git a/yql/essentials/tests/sql/suites/csee/same_free_closure_size2_arg_both_level_expr.sql b/yql/essentials/tests/sql/suites/csee/same_free_closure_size2_arg_both_level_expr.sql
new file mode 100644
index 0000000000..ae43452088
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/csee/same_free_closure_size2_arg_both_level_expr.sql
@@ -0,0 +1,11 @@
+SELECT
+    AsTuple(
+            YQL::FlatMap(3/1, ($x)->{
+                return $x + YQL::FlatMap(1/1,($y)->{
+            return Just($x + $y)
+            })}),
+            YQL::FlatMap(3/1, ($x)->{
+                return $x + YQL::FlatMap(1/1,($y)->{
+            return Just($x + $y)
+            })})
+        )
diff --git a/yql/essentials/tests/sql/suites/csee/same_free_closure_size2_expr.sql b/yql/essentials/tests/sql/suites/csee/same_free_closure_size2_expr.sql
new file mode 100644
index 0000000000..b348cbbe8a
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/csee/same_free_closure_size2_expr.sql
@@ -0,0 +1,11 @@
+SELECT
+    AsTuple(
+            YQL::FlatMap(3/1, ($x)->{
+                return YQL::FlatMap(1/1,($y)->{
+            return Just($x + $y)
+            })}),
+            YQL::FlatMap(3/1, ($x)->{
+                return YQL::FlatMap(1/1,($y)->{
+            return Just($x + $y)
+            })})
+        )
diff --git a/yql/essentials/tests/sql/suites/csee/same_l1.sql b/yql/essentials/tests/sql/suites/csee/same_l1.sql
new file mode 100644
index 0000000000..d1483c600f
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/csee/same_l1.sql
@@ -0,0 +1,5 @@
+SELECT
+    AsList(
+        ListMap(AsList(1,2),($x)->{return $x+1}),
+        ListMap(AsList(10,11),($x)->{return $x+1})
+    );
\ No newline at end of file
diff --git a/yql/essentials/tests/sql/suites/csee/same_l1_expr.sql b/yql/essentials/tests/sql/suites/csee/same_l1_expr.sql
new file mode 100644
index 0000000000..167e0dca0d
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/csee/same_l1_expr.sql
@@ -0,0 +1,5 @@
+SELECT
+    AsList(
+        ListMap(AsList(1,2),($x)->{return $x+1}),
+        ListMap(AsList(1,2),($x)->{return $x+1})
+    );
\ No newline at end of file
diff --git a/yql/essentials/tests/sql/suites/csee/yql-7237.cfg b/yql/essentials/tests/sql/suites/csee/yql-7237.cfg
new file mode 100644
index 0000000000..c7f999ccb9
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/csee/yql-7237.cfg
@@ -0,0 +1 @@
+udf strings_udf
diff --git a/yql/essentials/tests/sql/suites/csee/yql-7237.sql b/yql/essentials/tests/sql/suites/csee/yql-7237.sql
new file mode 100644
index 0000000000..52a0a68ea7
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/csee/yql-7237.sql
@@ -0,0 +1,68 @@
+/* syntax version 1 */
+USE plato;
+
+$push_final_data = AsList(
+    AsStruct("manufacturer" AS manufacturer, "state" AS state));
+
+INSERT INTO @push_final
+SELECT * FROM AS_TABLE($push_final_data);
+COMMIT;
+
+$manufacturer_name_fix = ($manufacturer) -> {
+    $lowered_manufacturer = CAST(Unicode::ToLower(CAST(String::Strip($manufacturer) AS Utf8)) AS String);
+    $in = AsList(
+        "oysters", -- bullshit in naming
+        "qumo", -- bullshit in naming
+        "texet", -- bullshit in naming
+        "alcatel", -- bullshit in naming
+        "dexp", -- bullshit in naming
+        "haier", -- bullshit in naming
+        "dexp", -- bullshit in naming
+        "asus", -- ASUSTek Computer Inc & ASUS both usable
+        "yota", -- Yota Devices & Yota Devices Limited ...
+        "ark" -- "ark" & "ark electronic technology" & "ark_electronic_technology"
+    );
+    $lambda = ($substring) -> {
+        RETURN FIND($lowered_manufacturer, $substring) IS NULL;
+    };
+    $list = ListSkipWhile($in, $lambda);
+    RETURN IF(ListHasItems($list), $list[0], $lowered_manufacturer);
+};
+
+$manufacturers_whitelist = (
+    SELECT
+        man as manufacturer
+    FROM
+    (
+        SELECT
+            man, COUNT(*) as cnt
+        FROM
+            @push_final
+        GROUP BY
+            $manufacturer_name_fix(manufacturer) as man
+    )
+    WHERE
+        cnt > 1000
+);
+
+$push_final_preprocessing = (
+    SELECT
+        $manufacturer_name_fix(manufacturer) as manufacturer,
+        state
+    FROM
+        @push_final
+);
+
+SELECT
+    COALESCE(fixed_manufacturer, "other") as manufacturer,
+    L.* WITHOUT L.manufacturer
+FROM
+    $push_final_preprocessing as L
+    LEFT JOIN
+    (
+        SELECT
+            manufacturer as fixed_manufacturer
+        FROM
+            $manufacturers_whitelist
+    ) as R
+    ON(L.manufacturer = R.fixed_manufacturer);