Merge pull request #12554 from vitalyisaev2/YQ-3839.with_rightlib.3

Import from Arcadia + YDB FQ: turning gateways_config.proto into a file without external dependencies

17 files changed, 207 insertions, 5 deletions

diff --git a/yql/essentials/tests/sql/sql2yql/canondata/result.json b/yql/essentials/tests/sql/sql2yql/canondata/result.json
index ebec884c2a..47f93caece 100644
--- a/yql/essentials/tests/sql/sql2yql/canondata/result.json
+++ b/yql/essentials/tests/sql/sql2yql/canondata/result.json
@@ -1833,6 +1833,13 @@
             "uri": "https://{canondata_backend}/1937429/434276f26b2857be3c5ad3fdbbf877d2bf775ac5/resource.tar.gz#test_sql2yql.test_aggregate-avg_with_having_/sql.yql"
         }
     ],
+    "test_sql2yql.test[aggregate-compact_distinct]": [
+        {
+            "checksum": "c6aa750c244ef573293be11d07397a54",
+            "size": 3863,
+            "uri": "https://{canondata_backend}/1773845/f26b5f394704b0b9c6108eac91165c8420756fb1/resource.tar.gz#test_sql2yql.test_aggregate-compact_distinct_/sql.yql"
+        }
+    ],
     "test_sql2yql.test[aggregate-compare_by]": [
         {
             "checksum": "a9eacab25486aef4e5000b928097e2f7",
@@ -2610,6 +2617,13 @@
             "uri": "https://{canondata_backend}/1937429/434276f26b2857be3c5ad3fdbbf877d2bf775ac5/resource.tar.gz#test_sql2yql.test_aggregate-native_desc_group_compact_by_/sql.yql"
         }
     ],
+    "test_sql2yql.test[aggregate-no_compact_distinct]": [
+        {
+            "checksum": "7a533d8080711b15f39de0c29270692f",
+            "size": 3771,
+            "uri": "https://{canondata_backend}/1899731/184b8df51a2ed58cdacc5419a09c9170dbd7ef88/resource.tar.gz#test_sql2yql.test_aggregate-no_compact_distinct_/sql.yql"
+        }
+    ],
     "test_sql2yql.test[aggregate-null_type]": [
         {
             "checksum": "33f4556d059ae6520e3e360b3a780337",
@@ -16253,6 +16267,13 @@
             "uri": "https://{canondata_backend}/1936273/4a1b39013e1bae40e722cff8ccef8829784964e2/resource.tar.gz#test_sql2yql.test_produce-reduce_with_python_row_repack_/sql.yql"
         }
     ],
+    "test_sql2yql.test[produce-reduce_with_trivial_remaps]": [
+        {
+            "checksum": "b094793320668f635c5e837a71e84b50",
+            "size": 2276,
+            "uri": "https://{canondata_backend}/1942100/fc877ab79aa3673db82f29099d27c26f000fde66/resource.tar.gz#test_sql2yql.test_produce-reduce_with_trivial_remaps_/sql.yql"
+        }
+    ],
     "test_sql2yql.test[produce-yql-10297]": [
         {
             "checksum": "0efb35a2c6333d72f2edddd180c70a00",
@@ -21252,6 +21273,11 @@
             "uri": "file://test_sql_format.test_aggregate-avg_with_having_/formatted.sql"
         }
     ],
+    "test_sql_format.test[aggregate-compact_distinct]": [
+        {
+            "uri": "file://test_sql_format.test_aggregate-compact_distinct_/formatted.sql"
+        }
+    ],
     "test_sql_format.test[aggregate-compare_by]": [
         {
             "uri": "file://test_sql_format.test_aggregate-compare_by_/formatted.sql"
@@ -21807,6 +21833,11 @@
             "uri": "file://test_sql_format.test_aggregate-native_desc_group_compact_by_/formatted.sql"
         }
     ],
+    "test_sql_format.test[aggregate-no_compact_distinct]": [
+        {
+            "uri": "file://test_sql_format.test_aggregate-no_compact_distinct_/formatted.sql"
+        }
+    ],
     "test_sql_format.test[aggregate-null_type]": [
         {
             "uri": "file://test_sql_format.test_aggregate-null_type_/formatted.sql"
@@ -29302,6 +29333,11 @@
             "uri": "file://test_sql_format.test_produce-reduce_with_python_row_repack_/formatted.sql"
         }
     ],
+    "test_sql_format.test[produce-reduce_with_trivial_remaps]": [
+        {
+            "uri": "file://test_sql_format.test_produce-reduce_with_trivial_remaps_/formatted.sql"
+        }
+    ],
     "test_sql_format.test[produce-yql-10297]": [
         {
             "uri": "file://test_sql_format.test_produce-yql-10297_/formatted.sql"
diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_aggregate-compact_distinct_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_aggregate-compact_distinct_/formatted.sql
new file mode 100644
index 0000000000..c59d082159
--- /dev/null
+++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_aggregate-compact_distinct_/formatted.sql
@@ -0,0 +1,36 @@
+USE plato;
+
+PRAGMA AnsiOptionalAs;
+PRAGMA yt.CompactForDistinct;
+
+$x = (
+    SELECT
+        key,
+        AVG(DISTINCT CAST(subkey AS float)) s
+    FROM
+        InputB
+    GROUP BY
+        key
+);
+
+$y = (
+    SELECT
+        key,
+        SUM(CAST(subkey AS float)) s
+    FROM
+        InputC
+    GROUP BY
+        key
+);
+
+SELECT
+    x.key,
+    x.s AS s1,
+    y.s AS s2
+FROM
+    $x x
+FULL OUTER JOIN
+    $y y
+ON
+    x.key == y.key
+;
diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_aggregate-no_compact_distinct_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_aggregate-no_compact_distinct_/formatted.sql
new file mode 100644
index 0000000000..96db5d308f
--- /dev/null
+++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_aggregate-no_compact_distinct_/formatted.sql
@@ -0,0 +1,35 @@
+USE plato;
+
+PRAGMA AnsiOptionalAs;
+
+$x = (
+    SELECT
+        key,
+        AVG(DISTINCT CAST(subkey AS float)) s
+    FROM
+        InputB
+    GROUP BY
+        key
+);
+
+$y = (
+    SELECT
+        key,
+        SUM(CAST(subkey AS float)) s
+    FROM
+        InputC
+    GROUP BY
+        key
+);
+
+SELECT
+    x.key,
+    x.s AS s1,
+    y.s AS s2
+FROM
+    $x x
+FULL OUTER JOIN
+    $y y
+ON
+    x.key == y.key
+;
diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_expr-variant_builtins_opt_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_expr-variant_builtins_opt_/formatted.sql
index 3023dcf5c2..e4aa4de0a1 100644
--- a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_expr-variant_builtins_opt_/formatted.sql
+++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_expr-variant_builtins_opt_/formatted.sql
@@ -25,6 +25,6 @@ $var_2 = VARIANT ("8", "1", $vartype_t);
 SELECT
     Visit(Just($var_1), $handle_a, $handle_b),
     Visit(Just($var_2), $handle_a, $handle_b),
-    VisitOrDefault(Just($var_2), $handle_a, Just(777u)),
+    VisitOrDefault(Just($var_2), Just(777u), $handle_a),
     VariantItem(Just($var_b))
 ;
diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_expr-variant_tuple_builtins_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_expr-variant_tuple_builtins_/formatted.sql
index b1e21cda10..bdca3d25be 100644
--- a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_expr-variant_tuple_builtins_/formatted.sql
+++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_expr-variant_tuple_builtins_/formatted.sql
@@ -25,7 +25,7 @@ SELECT
 ;
 
 $visitor_def = ($var) -> {
-    RETURN VisitOrDefault($var, $handle_num, $handle_flag, 999);
+    RETURN VisitOrDefault($var, 999, $handle_num, $handle_flag);
 };
 
 SELECT
diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_produce-reduce_with_trivial_remaps_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_produce-reduce_with_trivial_remaps_/formatted.sql
new file mode 100644
index 0000000000..a29b85cedb
--- /dev/null
+++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_produce-reduce_with_trivial_remaps_/formatted.sql
@@ -0,0 +1,23 @@
+USE plato;
+
+PRAGMA warning("disable", "4510");
+
+$udf = ($_key, $stream) -> {
+    $init = ($item) -> (AsStruct(1u AS cnt, $item AS row));
+    $switch = ($_item, $_state) -> (FALSE);
+    $update = ($item, $state) -> (
+        AsStruct(
+            $state.cnt + 1u AS cnt,
+            if(($item.value > $state.row.value) ?? FALSE, $item, $state.row) AS row
+        )
+    );
+    $state = YQL::Collect(YQL::Condense1($stream, $init, $switch, $update));
+    RETURN $state;
+};
+
+REDUCE CONCAT(Input1, Input2)
+PRESORT
+    subkey
+ON
+    key
+USING $udf(TableRow());
diff --git a/yql/essentials/tests/sql/suites/aggregate/compact_distinct.cfg b/yql/essentials/tests/sql/suites/aggregate/compact_distinct.cfg
new file mode 100644
index 0000000000..711689a5cb
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/aggregate/compact_distinct.cfg
@@ -0,0 +1,3 @@
+providers yt
+in InputB input2.txt
+in InputC input3.txt
diff --git a/yql/essentials/tests/sql/suites/aggregate/compact_distinct.sql b/yql/essentials/tests/sql/suites/aggregate/compact_distinct.sql
new file mode 100644
index 0000000000..4176ad60aa
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/aggregate/compact_distinct.sql
@@ -0,0 +1,14 @@
+USE plato;
+
+pragma AnsiOptionalAs;
+pragma yt.CompactForDistinct;
+
+$x = (
+    SELECT key, AVG(DISTINCT Cast(subkey as float)) s FROM InputB GROUP BY key
+);
+
+$y = (
+    SELECT key, SUM(Cast(subkey as float)) s FROM InputC GROUP BY key
+);
+
+SELECT x.key, x.s AS s1, y.s AS s2 FROM $x x FULL OUTER JOIN $y y ON  x.key = y.key;
diff --git a/yql/essentials/tests/sql/suites/aggregate/no_compact_distinct.cfg b/yql/essentials/tests/sql/suites/aggregate/no_compact_distinct.cfg
new file mode 100644
index 0000000000..711689a5cb
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/aggregate/no_compact_distinct.cfg
@@ -0,0 +1,3 @@
+providers yt
+in InputB input2.txt
+in InputC input3.txt
diff --git a/yql/essentials/tests/sql/suites/aggregate/no_compact_distinct.sql b/yql/essentials/tests/sql/suites/aggregate/no_compact_distinct.sql
new file mode 100644
index 0000000000..757c9d55cc
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/aggregate/no_compact_distinct.sql
@@ -0,0 +1,13 @@
+USE plato;
+
+pragma AnsiOptionalAs;
+
+$x = (
+    SELECT key, AVG(DISTINCT Cast(subkey as float)) s FROM InputB GROUP BY key
+);
+
+$y = (
+    SELECT key, SUM(Cast(subkey as float)) s FROM InputC GROUP BY key
+);
+
+SELECT x.key, x.s AS s1, y.s AS s2 FROM $x x FULL OUTER JOIN $y y ON x.key = y.key;
diff --git a/yql/essentials/tests/sql/suites/expr/variant_builtins_opt.sql b/yql/essentials/tests/sql/suites/expr/variant_builtins_opt.sql
index 443802fe56..3d28d3f09a 100644
--- a/yql/essentials/tests/sql/suites/expr/variant_builtins_opt.sql
+++ b/yql/essentials/tests/sql/suites/expr/variant_builtins_opt.sql
@@ -21,6 +21,6 @@ $var_2 = Variant("8", "1", $vartype_t);
 SELECT
     Visit(Just($var_1), $handle_a, $handle_b),
     Visit(Just($var_2), $handle_a, $handle_b),
-    VisitOrDefault(Just($var_2), $handle_a, Just(777u)),
+    VisitOrDefault(Just($var_2), Just(777u), $handle_a),
     VariantItem(Just($var_b))
 ;
diff --git a/yql/essentials/tests/sql/suites/expr/variant_tuple_builtins.sql b/yql/essentials/tests/sql/suites/expr/variant_tuple_builtins.sql
index 35d1775708..075f061974 100644
--- a/yql/essentials/tests/sql/suites/expr/variant_tuple_builtins.sql
+++ b/yql/essentials/tests/sql/suites/expr/variant_tuple_builtins.sql
@@ -12,7 +12,7 @@ SELECT
     $visitor(NULL)
 ;
 
-$visitor_def = ($var) -> { return VisitOrDefault($var, $handle_num, $handle_flag, 999); };
+$visitor_def = ($var) -> { return VisitOrDefault($var, 999, $handle_num, $handle_flag); };
 SELECT
     $visitor_def(Variant(5, "0", $vartype)),
     $visitor_def(Just(Variant(True, "1", $vartype))),
diff --git a/yql/essentials/tests/sql/suites/join/cbo_4tables.cfg b/yql/essentials/tests/sql/suites/join/cbo_4tables.cfg
index a3f4d2aec3..a5101acc8c 100644
--- a/yql/essentials/tests/sql/suites/join/cbo_4tables.cfg
+++ b/yql/essentials/tests/sql/suites/join/cbo_4tables.cfg
@@ -1,4 +1,4 @@
-providers dummy
+providers yt
 in InputA cbo_4tables_a.txt
 in InputB cbo_4tables_b.txt
 in InputC cbo_4tables_c.txt
diff --git a/yql/essentials/tests/sql/suites/produce/reduce_with_trivial_remaps.cfg b/yql/essentials/tests/sql/suites/produce/reduce_with_trivial_remaps.cfg
new file mode 100644
index 0000000000..1516335c81
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/produce/reduce_with_trivial_remaps.cfg
@@ -0,0 +1,3 @@
+in Input1 sorted1.txt
+in Input2 sorted2.txt
+providers yt
diff --git a/yql/essentials/tests/sql/suites/produce/reduce_with_trivial_remaps.sql b/yql/essentials/tests/sql/suites/produce/reduce_with_trivial_remaps.sql
new file mode 100644
index 0000000000..65e70e8715
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/produce/reduce_with_trivial_remaps.sql
@@ -0,0 +1,15 @@
+use plato;
+pragma warning("disable", "4510");
+
+$udf = ($_key, $stream) -> {
+   $init = ($item) -> (AsStruct(1u as cnt, $item as row));
+   $switch = ($_item, $_state) -> (false);
+   $update = ($item, $state) -> (AsStruct($state.cnt + 1u as cnt,
+      if(($item.value > $state.row.value) ?? false, $item, $state.row) as row));
+   $state = YQL::Collect(YQL::Condense1($stream, $init, $switch, $update));
+   return $state;
+};
+
+REDUCE CONCAT(Input1,Input2)
+presort subkey
+ON key USING $udf(TableRow());
diff --git a/yql/essentials/tests/sql/suites/produce/sorted2.txt b/yql/essentials/tests/sql/suites/produce/sorted2.txt
new file mode 100644
index 0000000000..b214aab0d9
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/produce/sorted2.txt
@@ -0,0 +1,10 @@
+{"key"="023";"subkey"="3";"value"="aaa"};
+{"key"="037";"subkey"="5";"value"="ddd"};
+{"key"="075";"subkey"="1";"value"="abc"};
+{"key"="150";"subkey"="1";"value"="aaa"};
+{"key"="150";"subkey"="3";"value"="iii"};
+{"key"="150";"subkey"="8";"value"="zzz"};
+{"key"="200";"subkey"="7";"value"="qqq"};
+{"key"="527";"subkey"="4";"value"="bbb"};
+{"key"="761";"subkey"="6";"value"="ccc"};
+{"key"="911";"subkey"="2";"value"="kkk"};
diff --git a/yql/essentials/tests/sql/suites/produce/sorted2.txt.attr b/yql/essentials/tests/sql/suites/produce/sorted2.txt.attr
new file mode 100644
index 0000000000..5d3821a576
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/produce/sorted2.txt.attr
@@ -0,0 +1,11 @@
+{"_yql_row_spec"={
+    "Type"=["StructType";[
+        ["key";["DataType";"String"]];
+        ["subkey";["DataType";"String"]];
+        ["value";["DataType";"Utf8"]]
+    ]];
+    "SortDirections"=[1;1;1;];
+    "SortedBy"=["key";"subkey";"value";];
+    "SortedByTypes"=[["DataType";"String";];["DataType";"String";];["DataType";"Utf8";];];
+    "SortMembers"=["key";"subkey";"value";];
+}}