Move yql/udfs/common/ to /yql/essentials YQL-19206

Except the following directories:

* clickhouse/client
* datetime
* knn
* roaring
commit_hash:c7da95636144d28db109d6b17ddc762e9bacb59f

23 files changed, 431 insertions, 0 deletions

diff --git a/yql/essentials/udfs/common/python/python3_small/test/canondata/result.json b/yql/essentials/udfs/common/python/python3_small/test/canondata/result.json
new file mode 100644
index 00000000000..dd55da78b53
--- /dev/null
+++ b/yql/essentials/udfs/common/python/python3_small/test/canondata/result.json
@@ -0,0 +1,61 @@
+{
+    "test.test[Annotations]": [
+        {
+            "checksum": "19c6d906cb8617cf9d2b5d484e09caf8",
+            "size": 7570,
+            "uri": "https://{canondata_backend}/212715/49b4751c22bd43fa7057cc92ae5cbedb40404f40/resource.tar.gz#test.test_Annotations_/results.txt"
+        }
+    ],
+    "test.test[BytesDecodeModeStrict]": [
+        {
+            "checksum": "f8534cff0843faaf876c41e0875dcf05",
+            "size": 3120,
+            "uri": "https://{canondata_backend}/1775319/4c4fed0942b33bcc70d44f7dd2972a8e05c6db97/resource.tar.gz#test.test_BytesDecodeModeStrict_/results.txt"
+        }
+    ],
+    "test.test[Cleanup]": [
+        {
+            "checksum": "036e77892757e48fa3fb319ed324b019",
+            "size": 954,
+            "uri": "https://{canondata_backend}/1871182/9909e0b25b15bb1f21d5def23fb072d64c82f07e/resource.tar.gz#test.test_Cleanup_/results.txt"
+        }
+    ],
+    "test.test[CustomYsonConverter]": [
+        {
+            "checksum": "7716204e544d2fcb9313412c3919e66d",
+            "size": 1625,
+            "uri": "https://{canondata_backend}/1130705/576535b56a4e74992911431865e5edd0f7d55520/resource.tar.gz#test.test_CustomYsonConverter_/results.txt"
+        }
+    ],
+    "test.test[Data]": [
+        {
+            "checksum": "f40e83806b294be420681fdfbf2133e8",
+            "size": 25268,
+            "uri": "https://{canondata_backend}/1031349/7065a0985fe0cd26a754a5bee7a4c808836a4692/resource.tar.gz#test.test_Data_/results.txt"
+        }
+    ],
+    "test.test[Excepthook]": [
+        {
+            "uri": "file://test.test_Excepthook_/extracted"
+        }
+    ],
+    "test.test[GreedyInputContainers]": [
+        {
+            "checksum": "02a619c86f180e8a4c536087d64bab6d",
+            "size": 1328,
+            "uri": "https://{canondata_backend}/995452/085d43bbd16f44afc51d6cafed42465a3d20215c/resource.tar.gz#test.test_GreedyInputContainers_/results.txt"
+        }
+    ],
+    "test.test[OptionalNested]": [
+        {
+            "uri": "file://test.test_OptionalNested_/extracted"
+        }
+    ],
+    "test.test[Switch]": [
+        {
+            "checksum": "e60320702512bdcecd5c663f387ee939",
+            "size": 9172,
+            "uri": "https://{canondata_backend}/1130705/493ee46b1e8f2e848ab928f97913d332cb4fffc7/resource.tar.gz#test.test_Switch_/results.txt"
+        }
+    ]
+}
diff --git a/yql/essentials/udfs/common/python/python3_small/test/canondata/test.test_Excepthook_/extracted b/yql/essentials/udfs/common/python/python3_small/test/canondata/test.test_Excepthook_/extracted
new file mode 100644
index 00000000000..b260fe7616b
--- /dev/null
+++ b/yql/essentials/udfs/common/python/python3_small/test/canondata/test.test_Excepthook_/extracted
@@ -0,0 +1,15 @@
+<tmp_path>/program.sql:<main>: Fatal: Execution
+
+    <tmp_path>/program.sql:<main>:44:1: Fatal: Execution of node: Result
+    	SELECT $udf(@@{"abc":1}@@);
+	^
+        <tmp_path>/program.sql:<main>:40:17: Fatal: Failed to execute:
+CUSTOM_EXCEPTHOOK
+True
+Traceback (most recent call last):
+  File "embedded:f", line 31, in f
+Exception
+
+
+        	$udf = Python3::f(Callable<(String)->String>, $script);
+	                ^
\ No newline at end of file
diff --git a/yql/essentials/udfs/common/python/python3_small/test/canondata/test.test_OptionalNested_/extracted b/yql/essentials/udfs/common/python/python3_small/test/canondata/test.test_OptionalNested_/extracted
new file mode 100644
index 00000000000..413eb2f4ec0
--- /dev/null
+++ b/yql/essentials/udfs/common/python/python3_small/test/canondata/test.test_OptionalNested_/extracted
@@ -0,0 +1,14 @@
+<tmp_path>/program.sql:<main>: Error: Type annotation
+
+    <tmp_path>/program.sql:<main>:12:1: Error: At function: RemovePrefixMembers, At function: Unordered, At function: PersistableRepr, At function: OrderedSqlProject, At function: SqlProjectItem
+    	SELECT $optOptList("42");
+	^
+        <tmp_path>/program.sql:<main>:12:8: Error: At function: Apply
+        	SELECT $optOptList("42");
+	       ^
+            <tmp_path>/program.sql:<main>:2:24: Error: At function: ScriptUdf
+            	$optOptList = Python3::opt_opt_list(Callable<(String)->List<String>??>, @@
+	                       ^
+                <tmp_path>/program.sql:<main>:2:24: Error: Nested optionals are unsupported in script UDF
+                	$optOptList = Python3::opt_opt_list(Callable<(String)->List<String>??>, @@
+	                       ^
\ No newline at end of file
diff --git a/yql/essentials/udfs/common/python/python3_small/test/cases/Annotations.in b/yql/essentials/udfs/common/python/python3_small/test/cases/Annotations.in
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/yql/essentials/udfs/common/python/python3_small/test/cases/Annotations.in
diff --git a/yql/essentials/udfs/common/python/python3_small/test/cases/Annotations.sql b/yql/essentials/udfs/common/python/python3_small/test/cases/Annotations.sql
new file mode 100644
index 00000000000..3f845322e20
--- /dev/null
+++ b/yql/essentials/udfs/common/python/python3_small/test/cases/Annotations.sql
@@ -0,0 +1,67 @@
+--sanitizer ignore memory
+$script = @@ 
+from yql.typing import *
+
+def primitive(a0:Bool,a1:Int8,a2:Uint8,a3:Int16,a4:Uint16,a5:Int32,a6:Uint32,
+   a7:Int64,a8:Uint64,a9:Float,a10:Double,a11:String,a12:Utf8,a13:Yson,a14:Json,
+   a15:Uuid,a16:Date,a17:Datetime,a18:Timestamp,a19:Interval,a20:TzDate,
+   a21:TzDatetime,a22:TzTimestamp)->Decimal(10,3):
+   pass
+
+def singletons(a0:Void,a1:Null,a2:EmptyStruct,a3:EmptyTuple)->Void:
+   pass
+
+def containers(a0:Optional[Int32],a1:List[List[Bool]],a2:Stream[String],a3:Dict[Int32,String],
+   a4:Tuple[Int32,String],a5:Tuple[Int32],a6:Struct["a":Int32,"b":String],a7:Struct["a":Int32],
+   a8:Variant[Int32,String],a9:Variant[Int32],a10:Variant["a":Int32,"b":String],a11:Variant["a":Int32])->List[String]:
+   pass
+
+def special(a0:Resource["Python3"],a1:Tagged[Int32,"foo"])->Void:
+   pass
+
+def c0()->Callable[0,Int32]: pass
+def c1()->Callable[1,Int32,Optional[List[Int32]]]: pass
+def c2()->Callable[1,Int32,Int32,Optional[List[Int32]]]: pass
+def c3()->Callable[0,Int32,"a":Int32:{AutoMap}]: pass
+def c4()->Callable[0,Int32,"":Int32:{AutoMap}]: pass
+def c5()->Callable[0,Int32,"":Int32:{}]: pass
+def c6()->Callable[0,Int32,"foo":Int32]: pass
+
+def f0(x:Optional[Int32]=None,y:Optional[Int32]=None)->Void: pass
+def f1(x:Optional[Int32],y:Optional[Int32]=None)->Void: pass
+def f2(x:Optional[Int32],y:Optional[Int32])->Void: pass
+def f3(x:slice("",Int32,{AutoMap}), y:slice("name",String))->Void: pass
+
+@@;
+
+$t = ($name)->{
+    return FormatType(EvaluateType(
+       ParseTypeHandle(Core::PythonFuncSignature(AsAtom("Python3"), $script, $name))));
+};
+
+-- Singletons
+
+select $t("primitive");
+select $t("singletons");
+
+-- Containers & Special
+
+select $t("containers");
+select $t("special");
+
+-- Callable   
+select 
+  $t("c0") as c0,
+  $t("c1") as c1,
+  $t("c2") as c2,
+  $t("c3") as c3,
+  $t("c4") as c4,
+  $t("c5") as c5,
+  $t("c6") as c6;  
+
+-- Top level
+select
+  $t("f0") as f0,
+  $t("f1") as f1,
+  $t("f2") as f2,
+  $t("f3") as f3;
diff --git a/yql/essentials/udfs/common/python/python3_small/test/cases/BytesDecodeModeStrict.in b/yql/essentials/udfs/common/python/python3_small/test/cases/BytesDecodeModeStrict.in
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/yql/essentials/udfs/common/python/python3_small/test/cases/BytesDecodeModeStrict.in
diff --git a/yql/essentials/udfs/common/python/python3_small/test/cases/BytesDecodeModeStrict.sql b/yql/essentials/udfs/common/python/python3_small/test/cases/BytesDecodeModeStrict.sql
new file mode 100644
index 00000000000..e540dbf38ab
--- /dev/null
+++ b/yql/essentials/udfs/common/python/python3_small/test/cases/BytesDecodeModeStrict.sql
@@ -0,0 +1,11 @@
+--sanitizer ignore memory
+$script = @@
+def f(string, uuid, yson):
+    return (string, str(type(string)), uuid, str(type(uuid)), yson, str(type(yson)))
+
+f._yql_bytes_decode_mode = 'strict'
+@@;
+
+$udf = Python3::f(Callable<(String?, UUid?, Yson?)->Tuple<String?, String, UUid?, String, Yson?, String>>, $script);
+
+SELECT $udf("string", UUid('1812bc18-5838-4cde-98aa-287302697b90'), cast(@@{"abc"=1}@@ as yson));
diff --git a/yql/essentials/udfs/common/python/python3_small/test/cases/Cleanup.in b/yql/essentials/udfs/common/python/python3_small/test/cases/Cleanup.in
new file mode 100644
index 00000000000..d5ddcb40830
--- /dev/null
+++ b/yql/essentials/udfs/common/python/python3_small/test/cases/Cleanup.in
@@ -0,0 +1 @@
+{"key"="1";"subkey"="2";"value"="3"};
diff --git a/yql/essentials/udfs/common/python/python3_small/test/cases/Cleanup.sql b/yql/essentials/udfs/common/python/python3_small/test/cases/Cleanup.sql
new file mode 100644
index 00000000000..9db98402923
--- /dev/null
+++ b/yql/essentials/udfs/common/python/python3_small/test/cases/Cleanup.sql
@@ -0,0 +1,12 @@
+--sanitizer ignore memory
+$udfScript = @@
+import yql
+def mapper(records):
+    yql.g = records
+    for record in records:
+        yield dict(yid=b"bla", rnd=0.)
+@@;
+
+$udf = Python3::mapper(Callable<(Stream<Struct<key:String, subkey:String, value:String>>)->Stream<Struct<yid:String, rnd:Double>>>, $udfScript);
+
+PROCESS Input using $udf(TableRows());
diff --git a/yql/essentials/udfs/common/python/python3_small/test/cases/CustomYsonConverter.in b/yql/essentials/udfs/common/python/python3_small/test/cases/CustomYsonConverter.in
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/yql/essentials/udfs/common/python/python3_small/test/cases/CustomYsonConverter.in
diff --git a/yql/essentials/udfs/common/python/python3_small/test/cases/CustomYsonConverter.sql b/yql/essentials/udfs/common/python/python3_small/test/cases/CustomYsonConverter.sql
new file mode 100644
index 00000000000..43dd00cb3df
--- /dev/null
+++ b/yql/essentials/udfs/common/python/python3_small/test/cases/CustomYsonConverter.sql
@@ -0,0 +1,20 @@
+--sanitizer ignore memory
+/* syntax version 1 */
+$script = @@
+import json
+
+def yloads(z):
+  return json.loads(str(z, 'latin-1').replace("=",":"))
+
+def ydumps(z):
+  return bytes(json.dumps(z).replace(":","="), 'latin-1')
+
+def f(s):
+    return (s.get("abc",0),s)
+
+f._yql_convert_yson = (yloads,ydumps)
+@@;
+
+$udf = Python3::f(Callable<(Yson?)->Tuple<Int64, Yson?>>, $script);
+
+SELECT $udf(cast(@@{"abc"=1}@@ as yson));
diff --git a/yql/essentials/udfs/common/python/python3_small/test/cases/Data.in b/yql/essentials/udfs/common/python/python3_small/test/cases/Data.in
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/yql/essentials/udfs/common/python/python3_small/test/cases/Data.in
diff --git a/yql/essentials/udfs/common/python/python3_small/test/cases/Data.sql b/yql/essentials/udfs/common/python/python3_small/test/cases/Data.sql
new file mode 100644
index 00000000000..3f7de07d5c2
--- /dev/null
+++ b/yql/essentials/udfs/common/python/python3_small/test/cases/Data.sql
@@ -0,0 +1,61 @@
+--sanitizer ignore memory 
+$data = AsTuple(
+    Bool("true"),
+    Bool("FalsE"),
+    Int8("-128"),
+    Int8("127"),
+    Uint8("0"),
+    Uint8("255"),
+    Int16("-32768"),
+    Int16("32767"),
+    Uint16("0"),
+    Uint16("65535"),
+    Int32("-2147483648"),
+    Int32("2147483647"),
+    Uint32("0"),
+    Uint32("4294967295"),
+    Int64("-9223372036854775808"),
+    Int64("9223372036854775807"),
+    Uint64("0"),
+    Uint64("18446744073709551615"),
+    Float("0"),
+    Float("1"),
+    Float("-1e30"),
+    Float("-inf"),
+    Float("+inf"),
+    Float("nan"),
+    Double("0"),
+    Double("1"),
+    Double("-1e300"),
+    Double("-inf"),
+    Double("+inf"),
+    Double("nan"),
+    String("foo\xffbar"),
+    Utf8("привет"),
+    Yson("<a=1>[3;%false]"),
+    Json(@@{"a":1,"b":null}@@),
+    Date("2000-01-01"),
+    Datetime("2000-01-01T01:02:03Z"),
+    Timestamp("2000-01-01T01:02:03.4Z"),
+    Interval("P1DT12H"),
+    TzDate("2000-01-01,Europe/Moscow"),
+    TzDatetime("2000-01-01T01:02:03,Europe/Moscow"),
+    TzTimestamp("2000-01-01T01:02:03.4,Europe/Moscow"),
+    Uuid('31323334-3536-3738-393a-3b3c3d3e3f40'),
+    Decimal('3.1415926535897932384626433832795029', 35, 34),
+    Decimal('-.00000000000000000000000000000000001', 35, 35),
+    Decimal('NAN', 10, 5),
+    Decimal('-iNf', 1, 0)
+);
+
+$type = CallableType(0,
+   TypeOf($data),
+   TypeOf($data)
+);
+
+$f = Python3::f($type, @@
+def f(x):
+   return x
+@@);
+
+select $data, $f($data);
diff --git a/yql/essentials/udfs/common/python/python3_small/test/cases/Excepthook.cfg b/yql/essentials/udfs/common/python/python3_small/test/cases/Excepthook.cfg
new file mode 100644
index 00000000000..5dae597903c
--- /dev/null
+++ b/yql/essentials/udfs/common/python/python3_small/test/cases/Excepthook.cfg
@@ -0,0 +1 @@
+xfail
diff --git a/yql/essentials/udfs/common/python/python3_small/test/cases/Excepthook.sql b/yql/essentials/udfs/common/python/python3_small/test/cases/Excepthook.sql
new file mode 100644
index 00000000000..100086c9e4e
--- /dev/null
+++ b/yql/essentials/udfs/common/python/python3_small/test/cases/Excepthook.sql
@@ -0,0 +1,23 @@
+--sanitizer ignore memory
+/* syntax version 1 */
+$script = @@
+import sys
+import traceback
+
+
+def excepthook(*args):
+    print('CUSTOM_EXCEPTHOOK', file=sys.stderr)
+    print(all(_ for _ in args), file=sys.stderr)
+    print("".join(traceback.format_exception(*args)), file=sys.stderr)
+
+
+sys.excepthook = excepthook
+
+
+def f(string):
+    raise Exception()
+@@;
+
+$udf = Python3::f(Callable<(String)->String>, $script);
+
+SELECT $udf(@@{"abc":1}@@);
diff --git a/yql/essentials/udfs/common/python/python3_small/test/cases/GreedyInputContainers.in b/yql/essentials/udfs/common/python/python3_small/test/cases/GreedyInputContainers.in
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/yql/essentials/udfs/common/python/python3_small/test/cases/GreedyInputContainers.in
diff --git a/yql/essentials/udfs/common/python/python3_small/test/cases/GreedyInputContainers.sql b/yql/essentials/udfs/common/python/python3_small/test/cases/GreedyInputContainers.sql
new file mode 100644
index 00000000000..a43af8791d6
--- /dev/null
+++ b/yql/essentials/udfs/common/python/python3_small/test/cases/GreedyInputContainers.sql
@@ -0,0 +1,19 @@
+--sanitizer ignore memory
+/* syntax version 1 */
+$s = @@
+def list_func(lst):
+  return lst.count(1)
+list_func._yql_lazy_input = False
+@@;
+
+$u = Python3::list_func(Callable<(List<Int32>)->Int32>, $s);
+select $u(AsList(1,2,3));
+
+$s = @@
+def dict_func(dict):
+  return list(dict.values()).count(b"b")
+dict_func._yql_lazy_input = False
+@@;
+
+$v = Python3::dict_func(Callable<(Dict<Int32, String>)->Int32>, $s);
+select $v(AsDict(AsTuple(1,"a"),AsTuple(2,"b")));
diff --git a/yql/essentials/udfs/common/python/python3_small/test/cases/OptionalNested.cfg b/yql/essentials/udfs/common/python/python3_small/test/cases/OptionalNested.cfg
new file mode 100644
index 00000000000..5dae597903c
--- /dev/null
+++ b/yql/essentials/udfs/common/python/python3_small/test/cases/OptionalNested.cfg
@@ -0,0 +1 @@
+xfail
diff --git a/yql/essentials/udfs/common/python/python3_small/test/cases/OptionalNested.sql b/yql/essentials/udfs/common/python/python3_small/test/cases/OptionalNested.sql
new file mode 100644
index 00000000000..33396f036a7
--- /dev/null
+++ b/yql/essentials/udfs/common/python/python3_small/test/cases/OptionalNested.sql
@@ -0,0 +1,7 @@
+--sanitizer ignore memory
+$optOptList = Python3::opt_opt_list(Callable<(String)->List<String>??>, @@
+def opt_opt_list(in_str):
+    return [in_str] if len(in_str) % 2 == 0 else None
+@@);
+
+SELECT $optOptList("42");
diff --git a/yql/essentials/udfs/common/python/python3_small/test/cases/Switch.in b/yql/essentials/udfs/common/python/python3_small/test/cases/Switch.in
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/yql/essentials/udfs/common/python/python3_small/test/cases/Switch.in
diff --git a/yql/essentials/udfs/common/python/python3_small/test/cases/Switch.sql b/yql/essentials/udfs/common/python/python3_small/test/cases/Switch.sql
new file mode 100644
index 00000000000..c2576a72e45
--- /dev/null
+++ b/yql/essentials/udfs/common/python/python3_small/test/cases/Switch.sql
@@ -0,0 +1,92 @@
+--sanitizer ignore memory
+/* syntax version 1 */
+$x = AsList(1,2,3);
+
+$s1 = @@
+def f(input):
+    for x in input:
+        yield x
+@@;
+
+$s2 = @@
+class Iter:
+    def __init__(self, input):
+        self.input = input
+
+    def __next__(self):
+        return next(self.input)
+@@;
+
+$s3 = @@
+class CallableIter:
+    def __init__(self, input):
+        self.input = input
+
+    def __call__(self):
+        def f(input):
+            for x in input:
+                yield x
+
+        return f(self.input)
+@@;
+
+$s4 = @@
+class Iterable:
+    def __init__(self, input):
+        self.input = input
+
+    def __iter__(self):
+        return iter(self.input)
+@@;
+
+$f1 = Python3::f(Callable<(Stream<Int32>)->Stream<Int32>>, $s1);
+
+$f2 = Python3::Iter(Callable<(Stream<Int32>)->Stream<Int32>>, $s2);
+
+$f3 = Python3::CallableIter(Callable<(Stream<Int32>)->Stream<Int32>>, $s3);
+
+$f4 = Python3::Iterable(Callable<(Stream<Int32>)->Stream<Int32>>, $s4);
+
+$g = ($stream)->{
+    return $stream;
+};
+
+select Yql::Collect($g(Yql::Iterator($x, Yql::DependsOn("A1"))));
+
+select Yql::Collect($f1(Yql::Iterator($x, Yql::DependsOn("A2"))));
+
+select Yql::Collect($f2(Yql::Iterator($x, Yql::DependsOn("A3"))));
+
+select Yql::Collect($f3(Yql::Iterator($x, Yql::DependsOn("A4"))));
+
+select Yql::Collect($f4(Yql::Iterator($x, Yql::DependsOn("A5"))));
+
+select Yql::Collect(Yql::Switch(
+    Yql::Iterator($x, Yql::DependsOn("B1")),
+    AsAtom('0'),
+    AsTuple(AsAtom('0')),
+    $g));
+
+select Yql::Collect(Yql::Switch(
+    Yql::Iterator($x, Yql::DependsOn("B2")),
+    AsAtom('0'),
+    AsTuple(AsAtom('0')),
+    $f1));
+
+select Yql::Collect(Yql::Switch(
+    Yql::Iterator($x, Yql::DependsOn("B3")),
+    AsAtom('0'),
+    AsTuple(AsAtom('0')),
+    $f2));
+
+select Yql::Collect(Yql::Switch(
+    Yql::Iterator($x, Yql::DependsOn("B4")),
+    AsAtom('0'),
+    AsTuple(AsAtom('0')),
+    $f3));
+
+select Yql::Collect(Yql::Switch(
+    Yql::Iterator($x, Yql::DependsOn("B5")),
+    AsAtom('0'),
+    AsTuple(AsAtom('0')),
+    $f4));
diff --git a/yql/essentials/udfs/common/python/python3_small/test/ya.make b/yql/essentials/udfs/common/python/python3_small/test/ya.make
new file mode 100644
index 00000000000..ac03d946685
--- /dev/null
+++ b/yql/essentials/udfs/common/python/python3_small/test/ya.make
@@ -0,0 +1,10 @@
+YQL_UDF_TEST_CONTRIB()
+
+TIMEOUT(300)
+SIZE(MEDIUM)
+
+DEPENDS(
+    yql/essentials/udfs/common/python/python3_small
+)
+
+END()
diff --git a/yql/essentials/udfs/common/python/python3_small/ya.make b/yql/essentials/udfs/common/python/python3_small/ya.make
new file mode 100644
index 00000000000..f815fa8d757
--- /dev/null
+++ b/yql/essentials/udfs/common/python/python3_small/ya.make
@@ -0,0 +1,16 @@
+YQL_PYTHON3_UDF(python3_udf)
+
+REGISTER_YQL_PYTHON_UDF(
+    NAME Python3
+    RESOURCE_NAME Python3
+)
+
+PEERDIR(
+    yql/essentials/public/udf
+)
+
+END()
+
+RECURSE_FOR_TESTS(
+    test
+)