Merge pull request #11788 from ydb-platform/mergelibs-241120-1113

Library import 241120-1113

32 files changed, 200 insertions, 0 deletions

diff --git a/yql/essentials/tests/sql/suites/dq/blacklisted_pragmas.cfg b/yql/essentials/tests/sql/suites/dq/blacklisted_pragmas.cfg
new file mode 100644
index 0000000000..628ba758c4
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/dq/blacklisted_pragmas.cfg
@@ -0,0 +1,3 @@
+providers dq
+in Input input.txt
+out Output output.txt
diff --git a/yql/essentials/tests/sql/suites/dq/blacklisted_pragmas.sql b/yql/essentials/tests/sql/suites/dq/blacklisted_pragmas.sql
new file mode 100644
index 0000000000..48ed7e56a7
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/dq/blacklisted_pragmas.sql
@@ -0,0 +1,10 @@
+/* postgres can not */
+/* dq can not */
+/* syntax version 1 */
+use plato;
+pragma OrderedColumns;
+pragma yt.MaxRowWeight='32M';
+
+insert into Input
+select key, subkey, value from Input;
+
diff --git a/yql/essentials/tests/sql/suites/dq/blacklisted_pragmas1.cfg b/yql/essentials/tests/sql/suites/dq/blacklisted_pragmas1.cfg
new file mode 100644
index 0000000000..628ba758c4
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/dq/blacklisted_pragmas1.cfg
@@ -0,0 +1,3 @@
+providers dq
+in Input input.txt
+out Output output.txt
diff --git a/yql/essentials/tests/sql/suites/dq/blacklisted_pragmas1.sql b/yql/essentials/tests/sql/suites/dq/blacklisted_pragmas1.sql
new file mode 100644
index 0000000000..a5ff262828
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/dq/blacklisted_pragmas1.sql
@@ -0,0 +1,10 @@
+/* postgres can not */
+/* dq can not */
+/* syntax version 1 */
+use plato;
+pragma OrderedColumns;
+pragma yt.PoolTrees='test';
+
+insert into Input
+select key, subkey, value from Input;
+
diff --git a/yql/essentials/tests/sql/suites/dq/default.cfg b/yql/essentials/tests/sql/suites/dq/default.cfg
new file mode 100644
index 0000000000..5a04bd7d71
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/dq/default.cfg
@@ -0,0 +1,2 @@
+providers dq
+in Input input.txt
diff --git a/yql/essentials/tests/sql/suites/dq/dq_replicate_ok.sql b/yql/essentials/tests/sql/suites/dq/dq_replicate_ok.sql
new file mode 100644
index 0000000000..7fe22bdd86
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/dq/dq_replicate_ok.sql
@@ -0,0 +1,6 @@
+use plato;
+SELECT key
+    , COUNT(*) AS cnt
+    , COUNT(DISTINCT value) AS uniq
+FROM Input
+GROUP BY key
diff --git a/yql/essentials/tests/sql/suites/dq/input.txt b/yql/essentials/tests/sql/suites/dq/input.txt
new file mode 100644
index 0000000000..65949ea745
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/dq/input.txt
@@ -0,0 +1,4 @@
+{"key"="075";"subkey"="1";"value"="abc"};
+{"key"="800";"subkey"="2";"value"="ddd"};
+{"key"="020";"subkey"="3";"value"="q"};
+{"key"="150";"subkey"="4";"value"="qzz"};
diff --git a/yql/essentials/tests/sql/suites/dq/join_cbo_native_3_tables.cfg b/yql/essentials/tests/sql/suites/dq/join_cbo_native_3_tables.cfg
new file mode 100644
index 0000000000..041e8638ac
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/dq/join_cbo_native_3_tables.cfg
@@ -0,0 +1,5 @@
+providers dq
+in Input1 input.txt
+in Input2 input.txt
+in Input3 input.txt
+in Input4 input.txt
diff --git a/yql/essentials/tests/sql/suites/dq/join_cbo_native_3_tables.sql b/yql/essentials/tests/sql/suites/dq/join_cbo_native_3_tables.sql
new file mode 100644
index 0000000000..163762e45a
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/dq/join_cbo_native_3_tables.sql
@@ -0,0 +1,10 @@
+use plato;
+
+pragma CostBasedOptimizer="native";
+
+select i1.value, i2.value, i3.value, i4.value 
+from Input1 as i1
+join Input2 as i2 on i1.key=i2.key
+join Input3 as i3 on i1.key=i3.key
+join Input4 as i4 on i1.key=i4.key
+order by i1.value, i2.value, i3.value, i4.value;
diff --git a/yql/essentials/tests/sql/suites/dq/list.txt b/yql/essentials/tests/sql/suites/dq/list.txt
new file mode 100644
index 0000000000..085426e7dc
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/dq/list.txt
@@ -0,0 +1,4 @@
+075
+800
+020
+150
diff --git a/yql/essentials/tests/sql/suites/dq/mem_limit.cfg b/yql/essentials/tests/sql/suites/dq/mem_limit.cfg
new file mode 100644
index 0000000000..33c765a8f2
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/dq/mem_limit.cfg
@@ -0,0 +1,2 @@
+xfail
+providers dq
diff --git a/yql/essentials/tests/sql/suites/dq/mem_limit.sql b/yql/essentials/tests/sql/suites/dq/mem_limit.sql
new file mode 100644
index 0000000000..b3bac08441
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/dq/mem_limit.sql
@@ -0,0 +1,7 @@
+/* syntax version 1 */
+/* dqfile can not */
+USE plato;
+
+PRAGMA DqEngine="force";
+PRAGMA dq.MemoryLimit="1M";
+SELECT String::JoinFromList(ListMap(ListFromRange(0, 1000000), ($_) -> { return "0"; }), "");
diff --git a/yql/essentials/tests/sql/suites/dq/pool_trees_whitelist.cfg b/yql/essentials/tests/sql/suites/dq/pool_trees_whitelist.cfg
new file mode 100644
index 0000000000..628ba758c4
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/dq/pool_trees_whitelist.cfg
@@ -0,0 +1,3 @@
+providers dq
+in Input input.txt
+out Output output.txt
diff --git a/yql/essentials/tests/sql/suites/dq/pool_trees_whitelist.sql b/yql/essentials/tests/sql/suites/dq/pool_trees_whitelist.sql
new file mode 100644
index 0000000000..cb4b754ff4
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/dq/pool_trees_whitelist.sql
@@ -0,0 +1,8 @@
+/* postgres can not */
+/* syntax version 1 */
+use plato;
+pragma OrderedColumns;
+pragma yt.PoolTrees='physical,cloud';
+
+insert into Input
+select key, subkey, value from Input;
diff --git a/yql/essentials/tests/sql/suites/dq/precompute_asyncfile.cfg b/yql/essentials/tests/sql/suites/dq/precompute_asyncfile.cfg
new file mode 100644
index 0000000000..144206fd2c
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/dq/precompute_asyncfile.cfg
@@ -0,0 +1,4 @@
+providers dq
+in Input input.txt
+file list.txt list.txt
+udf file_udf
\ No newline at end of file
diff --git a/yql/essentials/tests/sql/suites/dq/precompute_asyncfile.sql b/yql/essentials/tests/sql/suites/dq/precompute_asyncfile.sql
new file mode 100644
index 0000000000..a75d060c8e
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/dq/precompute_asyncfile.sql
@@ -0,0 +1,5 @@
+use plato;
+
+$a = select max(key) from Input where key in ParseFile("String", "list.txt");
+
+select * from Input where key >= $a;
diff --git a/yql/essentials/tests/sql/suites/dq/precompute_parallel.cfg b/yql/essentials/tests/sql/suites/dq/precompute_parallel.cfg
new file mode 100644
index 0000000000..cf64efcf8e
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/dq/precompute_parallel.cfg
@@ -0,0 +1,4 @@
+providers dq
+in Input input.txt
+out Output1 output1.txt
+out Output2 output2.txt
diff --git a/yql/essentials/tests/sql/suites/dq/precompute_parallel.sql b/yql/essentials/tests/sql/suites/dq/precompute_parallel.sql
new file mode 100644
index 0000000000..4f3924af4c
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/dq/precompute_parallel.sql
@@ -0,0 +1,7 @@
+use plato;
+
+$a = select key from Input order by key limit 1;
+
+insert into Output1 select * from Input where key <= $a;
+insert into Output2 select * from Input where key >= $a;
+
diff --git a/yql/essentials/tests/sql/suites/dq/precompute_parallel_indep.cfg b/yql/essentials/tests/sql/suites/dq/precompute_parallel_indep.cfg
new file mode 100644
index 0000000000..1ff2e85554
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/dq/precompute_parallel_indep.cfg
@@ -0,0 +1,5 @@
+providers dq
+in Input1 input.txt
+in Input2 input.txt
+in Input3 input.txt
+out Output output.txt
diff --git a/yql/essentials/tests/sql/suites/dq/precompute_parallel_indep.sql b/yql/essentials/tests/sql/suites/dq/precompute_parallel_indep.sql
new file mode 100644
index 0000000000..c9a5dbc8f4
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/dq/precompute_parallel_indep.sql
@@ -0,0 +1,5 @@
+use plato;
+
+insert into Output select sum(cast(key as int32)) from Input1;
+insert into Output select sum(cast(key as int32)) from Input2;
+insert into Output select sum(cast(key as int32)) from Input3;
diff --git a/yql/essentials/tests/sql/suites/dq/precompute_parallel_mix.cfg b/yql/essentials/tests/sql/suites/dq/precompute_parallel_mix.cfg
new file mode 100644
index 0000000000..cf64efcf8e
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/dq/precompute_parallel_mix.cfg
@@ -0,0 +1,4 @@
+providers dq
+in Input input.txt
+out Output1 output1.txt
+out Output2 output2.txt
diff --git a/yql/essentials/tests/sql/suites/dq/precompute_parallel_mix.sql b/yql/essentials/tests/sql/suites/dq/precompute_parallel_mix.sql
new file mode 100644
index 0000000000..f68eaabcc9
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/dq/precompute_parallel_mix.sql
@@ -0,0 +1,7 @@
+use plato;
+
+$a = select key from Input order by key limit 1;
+$b = select key from Input order by key limit 1 offset 1;
+
+insert into Output1 select * from Input where key <= $a;
+insert into Output2 select * from Input where key >= $a and key != $b;
diff --git a/yql/essentials/tests/sql/suites/dq/precompute_result.sql b/yql/essentials/tests/sql/suites/dq/precompute_result.sql
new file mode 100644
index 0000000000..c00a9af0de
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/dq/precompute_result.sql
@@ -0,0 +1,6 @@
+use plato;
+
+$l = select min(cast(key as uint64)) as key from Input;
+$l = EvaluateExpr($l);
+
+select * from Input limit $l ?? 0;
diff --git a/yql/essentials/tests/sql/suites/dq/precompute_tree.sql b/yql/essentials/tests/sql/suites/dq/precompute_tree.sql
new file mode 100644
index 0000000000..2073f261e3
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/dq/precompute_tree.sql
@@ -0,0 +1,9 @@
+use plato;
+
+$a = select max(key) from Input;
+
+$b = select min(subkey) from Input where key < $a;
+
+$c = select min(key) from Input where key < "080";
+
+select * from Input where key > $c and subkey > $b;
diff --git a/yql/essentials/tests/sql/suites/dq/read_cost.sql b/yql/essentials/tests/sql/suites/dq/read_cost.sql
new file mode 100644
index 0000000000..173c2f1593
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/dq/read_cost.sql
@@ -0,0 +1,8 @@
+pragma warning("disable", "4510");
+pragma CostBasedOptimizer="PG";
+use plato;
+
+$foo = select subkey, key, value as v from Input order by subkey asc, key desc limit 10;
+$x = process $foo;
+
+select YQL::CostsOf($x) as costs;
diff --git a/yql/essentials/tests/sql/suites/dq/read_cost_native.sql b/yql/essentials/tests/sql/suites/dq/read_cost_native.sql
new file mode 100644
index 0000000000..65e744bb77
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/dq/read_cost_native.sql
@@ -0,0 +1,8 @@
+pragma warning("disable", "4510");
+pragma CostBasedOptimizer="Native";
+use plato;
+
+$foo = select subkey, key, value as v from Input order by subkey asc, key desc limit 10;
+$x = process $foo;
+
+select YQL::CostsOf($x) as costs;
diff --git a/yql/essentials/tests/sql/suites/dq/truncate_local.sql b/yql/essentials/tests/sql/suites/dq/truncate_local.sql
new file mode 100644
index 0000000000..71872dcf16
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/dq/truncate_local.sql
@@ -0,0 +1,3 @@
+pragma DqEngine="auto";
+select * from AS_TABLE(ListMap(ListFromRange(1,10000),($x)->(<|a:$x|>)));
+
diff --git a/yql/essentials/tests/sql/suites/dq/wrong_script.cfg b/yql/essentials/tests/sql/suites/dq/wrong_script.cfg
new file mode 100644
index 0000000000..33c765a8f2
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/dq/wrong_script.cfg
@@ -0,0 +1,2 @@
+xfail
+providers dq
diff --git a/yql/essentials/tests/sql/suites/dq/wrong_script.sql b/yql/essentials/tests/sql/suites/dq/wrong_script.sql
new file mode 100644
index 0000000000..7774e498a2
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/dq/wrong_script.sql
@@ -0,0 +1,13 @@
+/* custom error:Failed to cast arguments*/
+/* dqfile can not */
+USE plato;
+
+$f=Python3::f(@@
+def f(x):
+    """
+    Callable<(Int32)->Int32>
+    """
+    return ""
+@@);
+
+select $f(0);
diff --git a/yql/essentials/tests/sql/suites/dq/wrong_script_segf.cfg b/yql/essentials/tests/sql/suites/dq/wrong_script_segf.cfg
new file mode 100644
index 0000000000..33c765a8f2
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/dq/wrong_script_segf.cfg
@@ -0,0 +1,2 @@
+xfail
+providers dq
diff --git a/yql/essentials/tests/sql/suites/dq/wrong_script_segf.sql b/yql/essentials/tests/sql/suites/dq/wrong_script_segf.sql
new file mode 100644
index 0000000000..a679ed4c2c
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/dq/wrong_script_segf.sql
@@ -0,0 +1,17 @@
+/* custom error:PrintBacktraceToStderr*/
+/* dqfile can not */
+USE plato;
+
+$f=Python3::f(@@
+def f(x):
+    """
+    Callable<(Int32)->Int32>
+    """
+    import ctypes
+    def deref(addr, typ):
+        return ctypes.cast(addr, ctypes.POINTER(typ)).contents
+    print(deref(1, ctypes.c_int))
+    return 0
+@@);
+
+select $f(0);
diff --git a/yql/essentials/tests/sql/suites/dq/wrong_script_timeout.sql b/yql/essentials/tests/sql/suites/dq/wrong_script_timeout.sql
new file mode 100644
index 0000000000..8493ab87e8
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/dq/wrong_script_timeout.sql
@@ -0,0 +1,14 @@
+/* dqfile can not */
+USE plato;
+
+$f=Python3::f(@@
+def f(x):
+    """
+    Callable<(Int32)->Int32>
+    """
+    import time
+    time.sleep(60)
+    return 0
+@@);
+
+select $f(0);