Merge pull request #11788 from ydb-platform/mergelibs-241120-1113

Library import 241120-1113

54 files changed, 440 insertions, 0 deletions

diff --git a/yql/essentials/tests/sql/suites/udf/automap_null.cfg b/yql/essentials/tests/sql/suites/udf/automap_null.cfg
new file mode 100644
index 0000000000..9354002fbc
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/udf/automap_null.cfg
@@ -0,0 +1 @@
+udf strings_udf
\ No newline at end of file
diff --git a/yql/essentials/tests/sql/suites/udf/automap_null.sql b/yql/essentials/tests/sql/suites/udf/automap_null.sql
new file mode 100644
index 0000000000..6ca93e6f09
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/udf/automap_null.sql
@@ -0,0 +1,5 @@
+/* syntax version 1 */
+/* postgres can not */
+select String::CollapseText("abc",1);
+select String::CollapseText(Nothing(String?),1);
+select String::CollapseText(null,1);
diff --git a/yql/essentials/tests/sql/suites/udf/complex_return_type.cfg b/yql/essentials/tests/sql/suites/udf/complex_return_type.cfg
new file mode 100644
index 0000000000..8df9e9bd77
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/udf/complex_return_type.cfg
@@ -0,0 +1 @@
+udf simple_udf
diff --git a/yql/essentials/tests/sql/suites/udf/complex_return_type.sql b/yql/essentials/tests/sql/suites/udf/complex_return_type.sql
new file mode 100644
index 0000000000..aaa361e1b5
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/udf/complex_return_type.sql
@@ -0,0 +1 @@
+select ListSort(DictItems(SimpleUdf::ComplexReturnType("banana")));
diff --git a/yql/essentials/tests/sql/suites/udf/default.cfg b/yql/essentials/tests/sql/suites/udf/default.cfg
new file mode 100644
index 0000000000..58878f8945
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/udf/default.cfg
@@ -0,0 +1 @@
+in Input input.txt
diff --git a/yql/essentials/tests/sql/suites/udf/generic_udf.cfg b/yql/essentials/tests/sql/suites/udf/generic_udf.cfg
new file mode 100644
index 0000000000..7daac60a2e
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/udf/generic_udf.cfg
@@ -0,0 +1 @@
+udf type_inspection_udf
diff --git a/yql/essentials/tests/sql/suites/udf/generic_udf.sql b/yql/essentials/tests/sql/suites/udf/generic_udf.sql
new file mode 100644
index 0000000000..e5d2a1e364
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/udf/generic_udf.sql
@@ -0,0 +1,2 @@
+/* postgres can not */
+select TypeInspection::Zip(AsList("A","B","C"),AsList(1,2,3));
diff --git a/yql/essentials/tests/sql/suites/udf/input.txt b/yql/essentials/tests/sql/suites/udf/input.txt
new file mode 100644
index 0000000000..65949ea745
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/udf/input.txt
@@ -0,0 +1,4 @@
+{"key"="075";"subkey"="1";"value"="abc"};
+{"key"="800";"subkey"="2";"value"="ddd"};
+{"key"="020";"subkey"="3";"value"="q"};
+{"key"="150";"subkey"="4";"value"="qzz"};
diff --git a/yql/essentials/tests/sql/suites/udf/input_tutorial_users.txt b/yql/essentials/tests/sql/suites/udf/input_tutorial_users.txt
new file mode 100644
index 0000000000..4a18a0dd29
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/udf/input_tutorial_users.txt
@@ -0,0 +1,12 @@
+{"key"="15";"subkey"="213";"value"="Anya"};
+{"key"="25";"subkey"="225";"value"="Petr"};
+{"key"="17";"subkey"="1";"value"="Masha"};
+{"key"="5";"subkey"="225";"value"="Alena"};
+{"key"="23";"subkey"="2";"value"="Irina"};
+{"key"="13";"subkey"="21";"value"="Inna"};
+{"key"="33";"subkey"="125";"value"="Ivan"};
+{"key"="45";"subkey"="225";"value"="Asya"};
+{"key"="27";"subkey"="125";"value"="German"};
+{"key"="41";"subkey"="225";"value"="Olya"};
+{"key"="35";"subkey"="2";"value"="Slava"};
+{"key"="56";"subkey"="2";"value"="Elena"};
diff --git a/yql/essentials/tests/sql/suites/udf/named_args.cfg b/yql/essentials/tests/sql/suites/udf/named_args.cfg
new file mode 100644
index 0000000000..8df9e9bd77
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/udf/named_args.cfg
@@ -0,0 +1 @@
+udf simple_udf
diff --git a/yql/essentials/tests/sql/suites/udf/named_args.sql b/yql/essentials/tests/sql/suites/udf/named_args.sql
new file mode 100644
index 0000000000..2aa5ea265d
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/udf/named_args.sql
@@ -0,0 +1,12 @@
+/* postgres can not */
+select
+  SimpleUdf::NamedArgs(0) as named_args0,
+  SimpleUdf::NamedArgs(1, 101 as D, 13 as C) as named_args1,
+  SimpleUdf::NamedArgs(1, 21 as C, 113 as D) as named_args2,
+  SimpleUdf::NamedArgs(1, 42 as C) as named_args3,
+  SimpleUdf::NamedArgs(1, 128 as D) as named_args4,
+  SimpleUdf::NamedArgs(100, 500, 404 as D, 44 as C) as named_args5,
+  SimpleUdf::NamedArgs(100, 500, 55 as C, 505 as D) as named_args6,
+  SimpleUdf::NamedArgs(100, 500, 25 as C) as named_args7,
+  SimpleUdf::NamedArgs(100, 500, 606 as D) as named_args8,
+  SimpleUdf::NamedArgs(100, 500, 64, 512 as D) as named_args9
diff --git a/yql/essentials/tests/sql/suites/udf/named_args_for_script.cfg b/yql/essentials/tests/sql/suites/udf/named_args_for_script.cfg
new file mode 100644
index 0000000000..f773dab653
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/udf/named_args_for_script.cfg
@@ -0,0 +1,3 @@
+in Input input_tutorial_users.txt
+udf python3_udf
+
diff --git a/yql/essentials/tests/sql/suites/udf/named_args_for_script.sql b/yql/essentials/tests/sql/suites/udf/named_args_for_script.sql
new file mode 100644
index 0000000000..523b97853a
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/udf/named_args_for_script.sql
@@ -0,0 +1,18 @@
+/* postgres can not */
+/* syntax version 1 */
+use plato;
+$udfScript = @@
+def AppendInfo(a_name, a_age = None, a_region = None):
+    res = a_name.decode('utf-8')
+    if a_age:
+        res += ", age: " + repr(a_age)
+    if a_region:
+        res += ", region: " + repr(a_region)
+    return res.encode('utf-8')
+@@;
+
+$udf = Python3::AppendInfo(Callable<(name:String, [age:Int32?, region:Int32?])->String>, $udfScript);
+
+$data = (select cast(key as int32) as age, cast(subkey as int32) as region, value as name from Input);
+
+select $udf(region as region, name as name) as val from $data;
diff --git a/yql/essentials/tests/sql/suites/udf/named_args_for_script_with_posargs.cfg b/yql/essentials/tests/sql/suites/udf/named_args_for_script_with_posargs.cfg
new file mode 100644
index 0000000000..289c4251bb
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/udf/named_args_for_script_with_posargs.cfg
@@ -0,0 +1,2 @@
+in Input input.txt
+udf python3_udf
diff --git a/yql/essentials/tests/sql/suites/udf/named_args_for_script_with_posargs.sql b/yql/essentials/tests/sql/suites/udf/named_args_for_script_with_posargs.sql
new file mode 100644
index 0000000000..e393ea92d0
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/udf/named_args_for_script_with_posargs.sql
@@ -0,0 +1,18 @@
+/* postgres can not */
+/* syntax version 1 */
+use plato;
+$udfScript = @@
+def AppendInfo(a_name, a_age = None, a_region = None):
+    res = a_name.decode('utf-8')
+    if a_age:
+        res += ", age: " + repr(a_age)
+    if a_region:
+        res += ", region: " + repr(a_region)
+    return res.encode('utf-8')
+@@;
+
+$udf = Python3::AppendInfo(Callable<(name:String, [age:Int32?, region:Int32?])->String>, $udfScript);
+
+$data = (select cast(key as int32) as age, cast(subkey as int32) as region, value as name from Input);
+
+select $udf(name, region as region, age as age) as val from $data;
diff --git a/yql/essentials/tests/sql/suites/udf/named_args_for_script_with_posargs2.cfg b/yql/essentials/tests/sql/suites/udf/named_args_for_script_with_posargs2.cfg
new file mode 100644
index 0000000000..289c4251bb
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/udf/named_args_for_script_with_posargs2.cfg
@@ -0,0 +1,2 @@
+in Input input.txt
+udf python3_udf
diff --git a/yql/essentials/tests/sql/suites/udf/named_args_for_script_with_posargs2.sql b/yql/essentials/tests/sql/suites/udf/named_args_for_script_with_posargs2.sql
new file mode 100644
index 0000000000..bcc8bfebaf
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/udf/named_args_for_script_with_posargs2.sql
@@ -0,0 +1,18 @@
+/* postgres can not */
+/* syntax version 1 */
+use plato;
+$udfScript = @@
+def AppendInfo(a_name, a_age = None, a_region = None):
+    res = a_name.decode('utf-8')
+    if a_age:
+        res += ", age: " + repr(a_age)
+    if a_region:
+        res += ", region: " + repr(a_region)
+    return res.encode('utf-8')
+@@;
+
+$udf = Python3::AppendInfo(Callable<(name:String, [age:Int32?, region:Int32?])->String>, $udfScript);
+
+$data = (select cast(key as int32) as age, cast(subkey as int32) as region, value as name from Input);
+
+select $udf(name, age, region as region) as val from $data;
diff --git a/yql/essentials/tests/sql/suites/udf/named_args_for_script_with_posargs_reuse_args_fail.cfg b/yql/essentials/tests/sql/suites/udf/named_args_for_script_with_posargs_reuse_args_fail.cfg
new file mode 100644
index 0000000000..0311eba913
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/udf/named_args_for_script_with_posargs_reuse_args_fail.cfg
@@ -0,0 +1,3 @@
+xfail
+in Input input_tutorial_users.txt
+udf python3_udf
diff --git a/yql/essentials/tests/sql/suites/udf/named_args_for_script_with_posargs_reuse_args_fail.sql b/yql/essentials/tests/sql/suites/udf/named_args_for_script_with_posargs_reuse_args_fail.sql
new file mode 100644
index 0000000000..21f66fa806
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/udf/named_args_for_script_with_posargs_reuse_args_fail.sql
@@ -0,0 +1,18 @@
+/* postgres can not */
+/* syntax version 1 */
+use plato;
+$udfScript = @@
+def AppendInfo(a_name, a_age = None, a_region = None):
+    res = a_name.decode('utf-8')
+    if a_age:
+        res += ", age: " + repr(a_age)
+    if a_region:
+        res += ", region: " + repr(a_region)
+    return res.encode('utf-8')
+@@;
+
+$udf = Python3::AppendInfo(Callable<(name:String, [age:Int32?, region:Int32?])->String>, $udfScript);
+
+$data = (select cast(key as int32) as age, cast(subkey as int32) as region, value as name from Input);
+
+select $udf(name, age, age as age) as val from $data; -- age is reused as named after positional
diff --git a/yql/essentials/tests/sql/suites/udf/named_args_for_script_with_wrong_order.sqlx b/yql/essentials/tests/sql/suites/udf/named_args_for_script_with_wrong_order.sqlx
new file mode 100644
index 0000000000..0f7acf017b
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/udf/named_args_for_script_with_wrong_order.sqlx
@@ -0,0 +1,18 @@
+/* postgres can not */
+/* syntax version 1 */
+use plato;
+$udfScript = @@
+def AppendInfo(a_name, a_age = None, a_region = None):
+    res = a_name.decode('utf-8')
+    if a_age:
+        res += ", age: " + repr(a_age)
+    if a_region:
+        res += ", region: " + repr(a_region)
+    return res.encode('utf-8')
+@@;
+
+$udf = Python3::AppendInfo(Callable<(String, [age:Int32?, region:Int32?])->String>, $udfScript);
+
+$data = (select cast(key as uint32) as age, cast(subkey as uint32) as region, value as name from Input);
+
+select $udf(name, region as region, age) as val from $data;
diff --git a/yql/essentials/tests/sql/suites/udf/python_script.cfg b/yql/essentials/tests/sql/suites/udf/python_script.cfg
new file mode 100644
index 0000000000..289c4251bb
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/udf/python_script.cfg
@@ -0,0 +1,2 @@
+in Input input.txt
+udf python3_udf
diff --git a/yql/essentials/tests/sql/suites/udf/python_script.py b/yql/essentials/tests/sql/suites/udf/python_script.py
new file mode 100644
index 0000000000..29f52fab0c
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/udf/python_script.py
@@ -0,0 +1,2 @@
+def AppendNum(s, n):
+    return s + str(n).encode('utf-8')
diff --git a/yql/essentials/tests/sql/suites/udf/python_script.sql b/yql/essentials/tests/sql/suites/udf/python_script.sql
new file mode 100644
index 0000000000..cb3c5075c9
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/udf/python_script.sql
@@ -0,0 +1,11 @@
+/* postgres can not */
+/* syntax version 1 */
+use plato;
+$udfScript = @@
+def AppendNum(name, age):
+    return name + str(age).encode('utf-8')
+@@;
+
+$udf = Python3::AppendNum(Callable<(String, Int32?)->String>, $udfScript);
+
+select $udf(value, cast(subkey as Int32)) as val from Input;
diff --git a/yql/essentials/tests/sql/suites/udf/python_script_from_file.cfg b/yql/essentials/tests/sql/suites/udf/python_script_from_file.cfg
new file mode 100644
index 0000000000..79b7571fec
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/udf/python_script_from_file.cfg
@@ -0,0 +1,4 @@
+in Input input.txt
+file python_script.py python_script.py
+udf python2_udf
+providers yt
diff --git a/yql/essentials/tests/sql/suites/udf/python_script_from_file.sql b/yql/essentials/tests/sql/suites/udf/python_script_from_file.sql
new file mode 100644
index 0000000000..259ec838aa
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/udf/python_script_from_file.sql
@@ -0,0 +1,7 @@
+/* postgres can not */
+/* syntax version 1 */
+use plato;
+$udfScript = FileContent("python_script.py");
+$udf = Python::AppendNum(Callable<(String, Int32?)->String>, $udfScript);
+
+select $udf(value, cast(subkey as Int32)) as value from Input;
diff --git a/yql/essentials/tests/sql/suites/udf/python_struct.cfg b/yql/essentials/tests/sql/suites/udf/python_struct.cfg
new file mode 100644
index 0000000000..d86459a8db
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/udf/python_struct.cfg
@@ -0,0 +1,3 @@
+in Input input.txt
+udf python2_udf
+providers yt
diff --git a/yql/essentials/tests/sql/suites/udf/python_struct.sql b/yql/essentials/tests/sql/suites/udf/python_struct.sql
new file mode 100644
index 0000000000..ed7fc8eeb3
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/udf/python_struct.sql
@@ -0,0 +1,23 @@
+/* postgres can not */
+use plato;
+
+$udf = YQL::@@(block '(
+    (let $udfScript (String '@@@@
+class Person:
+    def __init__(self, name, age):
+        self.name = name
+        self.age = age
+
+def NewPerson(name, age):
+    return Person(name, age)
+@@@@))
+    (let ui32 (DataType 'Uint32))
+    (let str (DataType 'String))
+    (let personType (StructType '('name str) '('age ui32)))
+    (let udfType (CallableType '() '(personType) '(str) '(ui32)))
+    (let udf (ScriptUdf 'Python 'NewPerson udfType $udfScript))
+    (return udf)
+))@@;
+
+$persons = (select $udf(value, 100) as val from Input);
+select val from $persons;
diff --git a/yql/essentials/tests/sql/suites/udf/regexp_udf.cfg b/yql/essentials/tests/sql/suites/udf/regexp_udf.cfg
new file mode 100644
index 0000000000..ce7c51a32a
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/udf/regexp_udf.cfg
@@ -0,0 +1,2 @@
+in Input input.txt
+udf pire_udf
diff --git a/yql/essentials/tests/sql/suites/udf/regexp_udf.sql b/yql/essentials/tests/sql/suites/udf/regexp_udf.sql
new file mode 100644
index 0000000000..f3fba96738
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/udf/regexp_udf.sql
@@ -0,0 +1,3 @@
+/* postgres can not */
+$regexp = Pire::Match("q.*");
+select * from plato.Input where $regexp(value);
diff --git a/yql/essentials/tests/sql/suites/udf/same_udf_modules.cfg b/yql/essentials/tests/sql/suites/udf/same_udf_modules.cfg
new file mode 100644
index 0000000000..e035127424
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/udf/same_udf_modules.cfg
@@ -0,0 +1 @@
+udf re2_udf
diff --git a/yql/essentials/tests/sql/suites/udf/same_udf_modules.sql b/yql/essentials/tests/sql/suites/udf/same_udf_modules.sql
new file mode 100644
index 0000000000..9d5103074f
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/udf/same_udf_modules.sql
@@ -0,0 +1,15 @@
+/* postgres can not */
+$value = "xaaxaaxaa";
+$match = Re2::Match("[ax]+\\d");
+$grep = Re2Posix::Grep("a.*");
+$capture = Re2::Capture(".*(?P<foo>xa?)(a{2,}).*");
+$replace = Re2::Replace("x(a+)x");
+$count = Re2::Count("a");
+ 
+SELECT
+    $match($value) AS match,
+    $grep($value) AS grep,
+    $capture($value) AS capture,
+    $capture($value)._1 AS capture_member,
+    $replace($value, "b\\1z") AS replace,
+    $count($value) AS count;
diff --git a/yql/essentials/tests/sql/suites/udf/sqlproject_grounds.sql b/yql/essentials/tests/sql/suites/udf/sqlproject_grounds.sql
new file mode 100644
index 0000000000..2c903ee41a
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/udf/sqlproject_grounds.sql
@@ -0,0 +1,8 @@
+/* syntax version 1 */
+/* postgres can not */
+
+$ts_bytes = String::HexDecode('0a0b0c0d');
+$ts_int = FromBytes($ts_bytes, Uint32);
+SELECT 
+    $ts_int,
+    DateTime::FromSeconds($ts_int)
diff --git a/yql/essentials/tests/sql/suites/udf/trivial_udf.cfg b/yql/essentials/tests/sql/suites/udf/trivial_udf.cfg
new file mode 100644
index 0000000000..b16a0e0534
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/udf/trivial_udf.cfg
@@ -0,0 +1 @@
+udf datetime_udf
diff --git a/yql/essentials/tests/sql/suites/udf/trivial_udf.sql b/yql/essentials/tests/sql/suites/udf/trivial_udf.sql
new file mode 100644
index 0000000000..ecc31fc71b
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/udf/trivial_udf.sql
@@ -0,0 +1,3 @@
+/* syntax version 1 */
+/* postgres can not */
+SELECT Unicode::ToUpper("foo"u);
diff --git a/yql/essentials/tests/sql/suites/udf/two_regexps.cfg b/yql/essentials/tests/sql/suites/udf/two_regexps.cfg
new file mode 100644
index 0000000000..ce7c51a32a
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/udf/two_regexps.cfg
@@ -0,0 +1,2 @@
+in Input input.txt
+udf pire_udf
diff --git a/yql/essentials/tests/sql/suites/udf/two_regexps.sql b/yql/essentials/tests/sql/suites/udf/two_regexps.sql
new file mode 100644
index 0000000000..23c16a81cd
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/udf/two_regexps.sql
@@ -0,0 +1,5 @@
+/* postgres can not */
+$is_abc = Pire::Match(".*abc");
+$_is_wtf = Pire::Match(".*wtf");
+
+SELECT * FROM plato.Input WHERE $is_abc(value) LIMIT 10;
diff --git a/yql/essentials/tests/sql/suites/udf/udaf.cfg b/yql/essentials/tests/sql/suites/udf/udaf.cfg
new file mode 100644
index 0000000000..289c4251bb
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/udf/udaf.cfg
@@ -0,0 +1,2 @@
+in Input input.txt
+udf python3_udf
diff --git a/yql/essentials/tests/sql/suites/udf/udaf.sql b/yql/essentials/tests/sql/suites/udf/udaf.sql
new file mode 100644
index 0000000000..ea2d7fa7ba
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/udf/udaf.sql
@@ -0,0 +1,49 @@
+/* postgres can not */
+/* syntax version 1 */
+$script = @@
+import heapq
+import json
+
+N_SMALLEST = 3
+
+def create(item):
+    return [item]
+
+def add(state, item):
+    heapq.heappush(state, item)
+    return heapq.nsmallest(N_SMALLEST, state)
+
+def merge(state_a, state_b):
+    merged = heapq.merge(state_a, state_b)
+    return heapq.nsmallest(N_SMALLEST, merged)
+
+def get_result(state):
+    result = heapq.nsmallest(N_SMALLEST, state)
+    return '%d smallest items: %s' % (
+        N_SMALLEST,
+        ', '.join(map(str, result))
+    )
+
+def serialize(state):
+    return json.dumps(state)
+
+def deserialize(serialized):
+    return json.loads(serialized)
+@@;
+
+$create = Python3::create(Callable<(Double)->Resource<Python3>>, $script);
+$add = Python3::add(Callable<(Resource<Python3>,Double)->Resource<Python3>>, $script);
+$merge = Python3::merge(Callable<(Resource<Python3>,Resource<Python3>)->Resource<Python3>>, $script);
+$get_result = Python3::get_result(Callable<(Resource<Python3>)->String>, $script);
+$serialize = Python3::serialize(Callable<(Resource<Python3>)->String>, $script);
+$deserialize = Python3::deserialize(Callable<(String)->Resource<Python3>>, $script);
+
+SELECT UDAF(
+    CAST(key AS Double),
+    $create,
+    $add,
+    $merge,
+    $get_result,
+    $serialize,
+    $deserialize
+) FROM plato.Input; 
diff --git a/yql/essentials/tests/sql/suites/udf/udaf_default.sql b/yql/essentials/tests/sql/suites/udf/udaf_default.sql
new file mode 100644
index 0000000000..88182c32b6
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/udf/udaf_default.sql
@@ -0,0 +1,25 @@
+/* syntax version 1 */
+/* postgres can not */
+/* syntax version 1 */
+
+$create = ($item, $_parent) -> { return AsList($item) };
+$add = ($state, $item, $_parent) -> { return Yql::Append($state, $item) };
+$merge = ($state1, $state2) -> { return ListExtend($state1,$state2) };
+$get_result = ($state) -> { return $state };
+$serialize = ($state) -> { return $state };
+$deserialize = ($state) -> { return $state };
+$default = ($result_type)->{ return Yql::List($result_type) };
+
+$udaf_factory = AGGREGATION_FACTORY(
+    "UDAF",
+    $create,
+    $add,
+    $merge,
+    $get_result,
+    $serialize,
+    $deserialize,
+    $default
+);
+
+select ListAggregate(AsList(1,2),$udaf_factory);
+select ListAggregate(ListCreate(Int32),$udaf_factory);
diff --git a/yql/essentials/tests/sql/suites/udf/udaf_distinct.cfg b/yql/essentials/tests/sql/suites/udf/udaf_distinct.cfg
new file mode 100644
index 0000000000..289c4251bb
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/udf/udaf_distinct.cfg
@@ -0,0 +1,2 @@
+in Input input.txt
+udf python3_udf
diff --git a/yql/essentials/tests/sql/suites/udf/udaf_distinct.sql b/yql/essentials/tests/sql/suites/udf/udaf_distinct.sql
new file mode 100644
index 0000000000..2bde48ca55
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/udf/udaf_distinct.sql
@@ -0,0 +1,53 @@
+/* postgres can not */
+/* syntax version 1 */
+$script = @@
+import heapq
+import json
+
+N_SMALLEST = 3
+
+def create(item):
+    return [item]
+
+def add(state, item):
+    heapq.heappush(state, item)
+    return heapq.nsmallest(N_SMALLEST, state)
+
+def merge(state_a, state_b):
+    merged = heapq.merge(state_a, state_b)
+    return heapq.nsmallest(N_SMALLEST, merged)
+
+def get_result(state):
+    result = heapq.nsmallest(N_SMALLEST, state)
+    return '%d smallest items: %s' % (
+        N_SMALLEST,
+        ', '.join(map(str, result))
+    )
+
+def serialize(state):
+    return json.dumps(state)
+
+def deserialize(serialized):
+    return json.loads(serialized)
+@@;
+
+$create = Python3::create(Callable<(Int64)->Resource<Python3>>, $script);
+$add = Python3::add(Callable<(Resource<Python3>,Int64)->Resource<Python3>>, $script);
+$merge = Python3::merge(Callable<(Resource<Python3>,Resource<Python3>)->Resource<Python3>>, $script);
+$get_result = Python3::get_result(Callable<(Resource<Python3>)->String>, $script);
+$serialize = Python3::serialize(Callable<(Resource<Python3>)->String>, $script);
+$deserialize = Python3::deserialize(Callable<(String)->Resource<Python3>>, $script);
+
+SELECT UDAF(
+    DISTINCT item,
+    $create,
+    $add,
+    $merge,
+    $get_result,
+    $serialize,
+    $deserialize
+) FROM (
+    SELECT
+        CAST(LENGTH(value) AS Int64) AS item
+    FROM plato.Input
+); 
diff --git a/yql/essentials/tests/sql/suites/udf/udaf_lambda.sql b/yql/essentials/tests/sql/suites/udf/udaf_lambda.sql
new file mode 100644
index 0000000000..9e2a4d11b5
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/udf/udaf_lambda.sql
@@ -0,0 +1,18 @@
+/* postgres can not */
+$create = ($item, $_parent) -> { return $item };
+$add = ($state, $item, $_parent) -> { return $item + $state };
+$merge = ($state1, $state2) -> { return $state1 + $state2 };
+$get_result = ($state) -> { return $state };
+$serialize = ($state) -> { return $state };
+$deserialize = ($state) -> { return $state };
+
+SELECT UDAF(
+    length(key),
+    $create,
+    $add,
+    $merge,
+    $get_result,
+    $serialize,
+    $deserialize,
+    0u
+) FROM plato.Input;
diff --git a/yql/essentials/tests/sql/suites/udf/udaf_short.cfg b/yql/essentials/tests/sql/suites/udf/udaf_short.cfg
new file mode 100644
index 0000000000..289c4251bb
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/udf/udaf_short.cfg
@@ -0,0 +1,2 @@
+in Input input.txt
+udf python3_udf
diff --git a/yql/essentials/tests/sql/suites/udf/udaf_short.sql b/yql/essentials/tests/sql/suites/udf/udaf_short.sql
new file mode 100644
index 0000000000..9072ffb8cc
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/udf/udaf_short.sql
@@ -0,0 +1,27 @@
+/* postgres can not */
+/* syntax version 1 */
+$script = @@
+def create(item):
+    return item
+
+def add(state, item):
+    return state + item 
+
+def merge(state_a, state_b):
+    return state_a + state_b 
+@@;
+
+$create = Python3::create(Callable<(Int64)->Int64>, $script);
+$add = Python3::add(Callable<(Int64,Int64)->Int64>, $script);
+$merge = Python3::merge(Callable<(Int64,Int64)->Int64>, $script);
+
+SELECT UDAF(
+    item,
+    $create,
+    $add,
+    $merge
+) FROM (
+    SELECT
+        CAST(LENGTH(value) AS Int64) AS item
+    FROM plato.Input
+); 
diff --git a/yql/essentials/tests/sql/suites/udf/udf.cfg b/yql/essentials/tests/sql/suites/udf/udf.cfg
new file mode 100644
index 0000000000..abdcfc8271
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/udf/udf.cfg
@@ -0,0 +1,2 @@
+in Input input.txt
+udf simple_udf
diff --git a/yql/essentials/tests/sql/suites/udf/udf.sql b/yql/essentials/tests/sql/suites/udf/udf.sql
new file mode 100644
index 0000000000..8b6d115640
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/udf/udf.sql
@@ -0,0 +1,2 @@
+/* postgres can not */
+select key, subkey, SimpleUdf::Concat(value, "test") as value from plato.Input;
\ No newline at end of file
diff --git a/yql/essentials/tests/sql/suites/udf/udf_call_with_group_and_limit.cfg b/yql/essentials/tests/sql/suites/udf/udf_call_with_group_and_limit.cfg
new file mode 100644
index 0000000000..abdcfc8271
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/udf/udf_call_with_group_and_limit.cfg
@@ -0,0 +1,2 @@
+in Input input.txt
+udf simple_udf
diff --git a/yql/essentials/tests/sql/suites/udf/udf_call_with_group_and_limit.sql b/yql/essentials/tests/sql/suites/udf/udf_call_with_group_and_limit.sql
new file mode 100644
index 0000000000..c2285575ba
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/udf/udf_call_with_group_and_limit.sql
@@ -0,0 +1,6 @@
+/* postgres can not */
+select SimpleUdf::Echo(key) as key, count(*) as count 
+    from plato.Input 
+    group by key
+    order by key /* sort for stable results only */
+    limit 2;
\ No newline at end of file
diff --git a/yql/essentials/tests/sql/suites/udf/udf_empty.cfg b/yql/essentials/tests/sql/suites/udf/udf_empty.cfg
new file mode 100644
index 0000000000..8df9e9bd77
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/udf/udf_empty.cfg
@@ -0,0 +1 @@
+udf simple_udf
diff --git a/yql/essentials/tests/sql/suites/udf/udf_empty.sql b/yql/essentials/tests/sql/suites/udf/udf_empty.sql
new file mode 100644
index 0000000000..645668e0c6
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/udf/udf_empty.sql
@@ -0,0 +1,2 @@
+/* postgres can not */
+select SimpleUdf::ReturnNull(""), SimpleUdf::ReturnVoid(""), SimpleUdf::ReturnEmpty(""), SimpleUdf::ReturnEmpty("") is null;
diff --git a/yql/essentials/tests/sql/suites/udf/udf_result_member.cfg b/yql/essentials/tests/sql/suites/udf/udf_result_member.cfg
new file mode 100644
index 0000000000..1f757514f3
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/udf/udf_result_member.cfg
@@ -0,0 +1 @@
+udf structs_udf
diff --git a/yql/essentials/tests/sql/suites/udf/udf_result_member.sql b/yql/essentials/tests/sql/suites/udf/udf_result_member.sql
new file mode 100644
index 0000000000..58295ef434
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/udf/udf_result_member.sql
@@ -0,0 +1,3 @@
+/* postgres can not */
+$person = Person::New("Vasya", "Pupkin", 33);
+select $person.FirstName as name, $person.Age as age;
diff --git a/yql/essentials/tests/sql/suites/udf/wrong_args_fail.cfg b/yql/essentials/tests/sql/suites/udf/wrong_args_fail.cfg
new file mode 100644
index 0000000000..f4ea431abc
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/udf/wrong_args_fail.cfg
@@ -0,0 +1,2 @@
+xfail
+udf strings_udf
\ No newline at end of file
diff --git a/yql/essentials/tests/sql/suites/udf/wrong_args_fail.sql b/yql/essentials/tests/sql/suites/udf/wrong_args_fail.sql
new file mode 100644
index 0000000000..759bfd3816
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/udf/wrong_args_fail.sql
@@ -0,0 +1,8 @@
+/* postgres can not */
+
+-- Find has optional args
+select String::ReplaceAll(); -- too few
+select String::ReplaceAll("abc"); -- too few
+
+select String::ReplaceAll("abc", "b", 2, 4); -- too many
+select String::ReplaceAll("abc" , "b", 2, 4, 44); -- too many