Merge pull request #11788 from ydb-platform/mergelibs-241120-1113

Library import 241120-1113

21 files changed, 311 insertions, 0 deletions

diff --git a/yql/essentials/tests/s-expressions/suites/TableContent/FilterTable.yql b/yql/essentials/tests/s-expressions/suites/TableContent/FilterTable.yql
new file mode 100644
index 0000000000..47e6d9a517
--- /dev/null
+++ b/yql/essentials/tests/s-expressions/suites/TableContent/FilterTable.yql
@@ -0,0 +1,15 @@
+(
+    #comment
+    (let mr_source (DataSource 'yt 'plato))
+    (let table1 (Right! (Read! world mr_source (Key '('table (String 'Input))) (Void) '('('inline)))))
+    (let tresh (String '300))
+    (let table1low (FlatMap table1 (lambda '(item) (block '(
+           (let intValueOpt (Member item 'key))
+           (return (ListIf (< intValueOpt tresh) item))
+    )))))
+    (let mr_sink (DataSink 'yt 'plato))
+    (let world (Write! world mr_sink (Key '('table (String 'Output))) table1low '('('mode 'append))))
+    (let world (Commit! world mr_sink))
+    (return world)
+)
+
diff --git a/yql/essentials/tests/s-expressions/suites/TableContent/FilterWithComplexSkip.cfg b/yql/essentials/tests/s-expressions/suites/TableContent/FilterWithComplexSkip.cfg
new file mode 100644
index 0000000000..386ab9ac7f
--- /dev/null
+++ b/yql/essentials/tests/s-expressions/suites/TableContent/FilterWithComplexSkip.cfg
@@ -0,0 +1,3 @@
+in Input input.txt
+out Output output.txt
+udf string_udf
diff --git a/yql/essentials/tests/s-expressions/suites/TableContent/FilterWithComplexSkip.yql b/yql/essentials/tests/s-expressions/suites/TableContent/FilterWithComplexSkip.yql
new file mode 100644
index 0000000000..3d7eadd8df
--- /dev/null
+++ b/yql/essentials/tests/s-expressions/suites/TableContent/FilterWithComplexSkip.yql
@@ -0,0 +1,17 @@
+(
+    #comment
+    (let mr_source (DataSource 'yt 'plato))
+    (let strip (Udf 'String.Strip))
+    (let skip (Unwrap (Cast (Apply strip (String '" 4")) 'Uint64)))
+    (let table1 (Right! (Read! world mr_source (Key '('table (String 'Input))) (Void) '('('inline) '('skip skip) '('take (Uint64 '3))))))
+    (let tresh (String '300))
+    (let table1low (FlatMap table1 (lambda '(item) (block '(
+           (let intValueOpt (Member item 'key))
+           (return (ListIf (< intValueOpt tresh) item))
+    )))))
+    (let mr_sink (DataSink 'yt 'plato))
+    (let world (Write! world mr_sink (Key '('table (String 'Output))) table1low '('('mode 'append))))
+    (let world (Commit! world mr_sink))
+    (return world)
+)
+
diff --git a/yql/essentials/tests/s-expressions/suites/TableContent/FilterWithFields.yql b/yql/essentials/tests/s-expressions/suites/TableContent/FilterWithFields.yql
new file mode 100644
index 0000000000..e5293e0a57
--- /dev/null
+++ b/yql/essentials/tests/s-expressions/suites/TableContent/FilterWithFields.yql
@@ -0,0 +1,15 @@
+(
+    #comment
+    (let mr_source (DataSource 'yt 'plato))
+    (let table1 (Right! (Read! world mr_source (Key '('table (String 'Input))) '('key 'value) '('('inline)))))
+    (let tresh (String '300))
+    (let table1low (FlatMap table1 (lambda '(item) (block '(
+           (let intValueOpt (Member item 'key))
+           (return (ListIf (< intValueOpt tresh) item))
+    )))))
+    (let mr_sink (DataSink 'yt 'plato))
+    (let world (Write! world mr_sink (Key '('table (String 'Output))) table1low '('('mode 'append))))
+    (let world (Commit! world mr_sink))
+    (return world)
+)
+
diff --git a/yql/essentials/tests/s-expressions/suites/TableContent/FilterWithTake.yql b/yql/essentials/tests/s-expressions/suites/TableContent/FilterWithTake.yql
new file mode 100644
index 0000000000..e0d516738d
--- /dev/null
+++ b/yql/essentials/tests/s-expressions/suites/TableContent/FilterWithTake.yql
@@ -0,0 +1,15 @@
+(
+    #comment
+    (let mr_source (DataSource 'yt 'plato))
+    (let table1 (Right! (Read! world mr_source (Key '('table (String 'Input))) (Void) '('('inline) '('take (Uint64 '3))))))
+    (let tresh (String '300))
+    (let table1low (FlatMap table1 (lambda '(item) (block '(
+           (let intValueOpt (Member item 'key))
+           (return (ListIf (< intValueOpt tresh) item))
+    )))))
+    (let mr_sink (DataSink 'yt 'plato))
+    (let world (Write! world mr_sink (Key '('table (String 'Output))) table1low '('('mode 'append))))
+    (let world (Commit! world mr_sink))
+    (return world)
+)
+
diff --git a/yql/essentials/tests/s-expressions/suites/TableContent/InsideCombine.yql b/yql/essentials/tests/s-expressions/suites/TableContent/InsideCombine.yql
new file mode 100644
index 0000000000..529cd9be35
--- /dev/null
+++ b/yql/essentials/tests/s-expressions/suites/TableContent/InsideCombine.yql
@@ -0,0 +1,32 @@
+(
+    #comment
+    (let mr_source (DataSource 'yt 'plato))
+    (let x (Read! world mr_source (Key '('table (String 'Input))) '('key 'subkey 'value) '()))
+    (let world (Left! x))
+    (let table1 (Right! x))
+
+    (let mrtc (Right! (Read! world mr_source (Key '('table (String 'Input))) (Void) '('('inline)))))
+    (let filter (FlatMap mrtc (lambda '(item) (block '(
+           (let value (Member item 'value))
+           (return (ListIf (< value (String 'qqq)) item))
+    )))))
+    (let filter (ToDict mrtc (lambda '(x) (Member x 'value)) (lambda '(x) (Member x 'subkey)) '('Hashed 'One 'Compact)))
+
+    (let preMap (lambda '(item) (Just item)))
+    (let keyExtractor (lambda '(item) (Member item 'key)))
+    (let init (lambda '(key item) (Contains filter (Member item 'value))))
+    (let update (lambda '(key item state) (And state (Contains filter (Member item 'value)))))
+    (let finish (lambda '(key state) (block '(
+        (let s (Struct))
+        (let s (AddMember s 'key key))
+        (let s (AddMember s 'subkey (String '.)))
+        (let s (AddMember s 'value (ToString state)))
+        (return (Just s))
+    ))))
+    (let out (CombineByKey table1 preMap keyExtractor init update finish))
+
+    (let mr_sink (DataSink 'yt 'plato))
+    (let world (Write! world mr_sink (Key '('table (String 'Output))) out '('('mode 'append))))
+    (let world (Commit! world mr_sink))
+    (return world)
+)
diff --git a/yql/essentials/tests/s-expressions/suites/TableContent/InsideMap.yql b/yql/essentials/tests/s-expressions/suites/TableContent/InsideMap.yql
new file mode 100644
index 0000000000..1c9999ee3a
--- /dev/null
+++ b/yql/essentials/tests/s-expressions/suites/TableContent/InsideMap.yql
@@ -0,0 +1,24 @@
+(
+    #comment
+    (let mr_source (DataSource 'yt 'plato))
+    (let x (Read! world mr_source (Key '('table (String 'Input))) '('key 'subkey 'value) '()))
+    (let world (Left! x))
+    (let table1 (Right! x))
+
+    (let mrtc (Right! (Read! world mr_source (Key '('table (String 'Input))) (Void) '('('inline)))))
+    (let mrtc (FlatMap mrtc (lambda '(item) (block '(
+           (let key (Member item 'key))
+           (return (ListIf (> key (String '100)) item))
+    )))))
+    (let filter (ToDict mrtc (lambda '(x) (Member x 'key)) (lambda '(x) (Member x 'value)) '('Hashed 'One 'Compact)))
+
+    (let tresh (String '500))
+    (let out (FlatMap table1 (lambda '(item) (block '(
+           (let key (Member item 'key))
+           (return (ListIf (And (< key tresh) (Contains filter key)) item))
+    )))))
+    (let mr_sink (DataSink 'yt 'plato))
+    (let world (Write! world mr_sink (Key '('table (String 'Output))) out '('('mode 'append))))
+    (let world (Commit! world mr_sink))
+    (return world)
+)
diff --git a/yql/essentials/tests/s-expressions/suites/TableContent/InsideReduce.yql b/yql/essentials/tests/s-expressions/suites/TableContent/InsideReduce.yql
new file mode 100644
index 0000000000..71d61c2a0c
--- /dev/null
+++ b/yql/essentials/tests/s-expressions/suites/TableContent/InsideReduce.yql
@@ -0,0 +1,31 @@
+(
+    #comment
+    (let mr_source (DataSource 'yt 'plato))
+    (let x (Read! world mr_source (Key '('table (String 'Input))) '('key 'subkey 'value) '()))
+    (let world (Left! x))
+    (let table1 (Right! x))
+
+    (let mrtc (Right! (Read! world mr_source (Key '('table (String 'Input))) (Void) '('('inline)))))
+    (let mrtc (FlatMap mrtc (lambda '(item) (block '(
+           (let value (Member item 'value))
+           (return (ListIf (< value (String 'qqq)) item))
+    )))))
+    (let filter (ToDict mrtc (lambda '(x) (Member x 'value)) (lambda '(x) (Member x 'subkey)) '('Hashed 'One 'Compact)))
+
+    (let keySelector (lambda '(x) (Member x 'key)))
+    (let listHandler (lambda '(stream) (FlatMap stream (lambda '(pair) (block '(
+        (let key (Nth pair '0))
+        (let list (ForwardList (Nth pair '1)))
+        (let r (FlatMap list (lambda '(item) (block '(
+            (return (ListIf (Contains filter (Member item 'value)) item))
+        )))))
+        (return (FlatListIf (< key (String '500)) r))
+    ))))))
+    (let out (PartitionByKey table1 keySelector (Void) (Void) listHandler))
+    (let out (Sort out (Bool 'true) (lambda '(x) (Member x 'key))))
+
+    (let mr_sink (DataSink 'yt 'plato))
+    (let world (Write! world mr_sink (Key '('table (String 'Output))) out '('('mode 'append))))
+    (let world (Commit! world mr_sink))
+    (return world)
+)
diff --git a/yql/essentials/tests/s-expressions/suites/TableContent/MixedOther.cfg b/yql/essentials/tests/s-expressions/suites/TableContent/MixedOther.cfg
new file mode 100644
index 0000000000..d14bf78557
--- /dev/null
+++ b/yql/essentials/tests/s-expressions/suites/TableContent/MixedOther.cfg
@@ -0,0 +1,4 @@
+in Input input.txt
+in Input1 input_other.txt
+in Input2 input_other_strict.txt
+out Output output.txt
diff --git a/yql/essentials/tests/s-expressions/suites/TableContent/MixedOther.yql b/yql/essentials/tests/s-expressions/suites/TableContent/MixedOther.yql
new file mode 100644
index 0000000000..adde05c66d
--- /dev/null
+++ b/yql/essentials/tests/s-expressions/suites/TableContent/MixedOther.yql
@@ -0,0 +1,21 @@
+(
+    #comment
+    (let mr_source (DataSource 'yt 'plato))
+    (let x (Read! world mr_source (Key '('table (String 'Input))) '('key 'subkey 'value) '()))
+    (let world (Left! x))
+    (let table1 (Right! x))
+
+    (let r (Right! (Read! world mr_source '((Key '('table (String 'Input1))) (Key '('table (String 'Input2)))) '('_other) '('('inline)))))
+
+    (let out (FlatMap table1 (lambda '(item) (block '(
+        (let key (Member item 'key))
+        (return (Map r (lambda '(x) (block '(
+            (let value (Lookup (Member x '_other) (String 'value)))
+            (return (AsStruct '('key key) '('value value))) 
+        )))))
+    )))))
+    (let mr_sink (DataSink 'yt 'plato))
+    (let world (Write! world mr_sink (Key '('table (String 'Output))) out '('('mode 'append))))
+    (let world (Commit! world mr_sink))
+    (return world)
+)
diff --git a/yql/essentials/tests/s-expressions/suites/TableContent/NonStrict.cfg b/yql/essentials/tests/s-expressions/suites/TableContent/NonStrict.cfg
new file mode 100644
index 0000000000..2372a67784
--- /dev/null
+++ b/yql/essentials/tests/s-expressions/suites/TableContent/NonStrict.cfg
@@ -0,0 +1,3 @@
+in Input input.txt
+in Input1 input_other.txt
+out Output output.txt
diff --git a/yql/essentials/tests/s-expressions/suites/TableContent/NonStrict.yql b/yql/essentials/tests/s-expressions/suites/TableContent/NonStrict.yql
new file mode 100644
index 0000000000..5ef7216f81
--- /dev/null
+++ b/yql/essentials/tests/s-expressions/suites/TableContent/NonStrict.yql
@@ -0,0 +1,21 @@
+(
+    #comment
+    (let mr_source (DataSource 'yt 'plato))
+    (let x (Read! world mr_source (Key '('table (String 'Input))) '('key 'subkey 'value) '()))
+    (let world (Left! x))
+    (let table1 (Right! x))
+
+    (let r (Right! (Read! world mr_source (Key '('table (String 'Input1))) '('_other) '('('inline)))))
+
+    (let out (FlatMap table1 (lambda '(item) (block '(
+        (let key (Member item 'key))
+        (return (Map r (lambda '(x) (block '(
+            (let value (Lookup (Member x '_other) (String 'value)))
+            (return (AsStruct '('key key) '('value value))) 
+        )))))
+    )))))
+    (let mr_sink (DataSink 'yt 'plato))
+    (let world (Write! world mr_sink (Key '('table (String 'Output))) out '('('mode 'append))))
+    (let world (Commit! world mr_sink))
+    (return world)
+)
diff --git a/yql/essentials/tests/s-expressions/suites/TableContent/StrictOther.cfg b/yql/essentials/tests/s-expressions/suites/TableContent/StrictOther.cfg
new file mode 100644
index 0000000000..938bc7ecfc
--- /dev/null
+++ b/yql/essentials/tests/s-expressions/suites/TableContent/StrictOther.cfg
@@ -0,0 +1,3 @@
+in Input input.txt
+in Input1 input_other_strict.txt
+out Output output.txt
diff --git a/yql/essentials/tests/s-expressions/suites/TableContent/StrictOther.yql b/yql/essentials/tests/s-expressions/suites/TableContent/StrictOther.yql
new file mode 100644
index 0000000000..5ef7216f81
--- /dev/null
+++ b/yql/essentials/tests/s-expressions/suites/TableContent/StrictOther.yql
@@ -0,0 +1,21 @@
+(
+    #comment
+    (let mr_source (DataSource 'yt 'plato))
+    (let x (Read! world mr_source (Key '('table (String 'Input))) '('key 'subkey 'value) '()))
+    (let world (Left! x))
+    (let table1 (Right! x))
+
+    (let r (Right! (Read! world mr_source (Key '('table (String 'Input1))) '('_other) '('('inline)))))
+
+    (let out (FlatMap table1 (lambda '(item) (block '(
+        (let key (Member item 'key))
+        (return (Map r (lambda '(x) (block '(
+            (let value (Lookup (Member x '_other) (String 'value)))
+            (return (AsStruct '('key key) '('value value))) 
+        )))))
+    )))))
+    (let mr_sink (DataSink 'yt 'plato))
+    (let world (Write! world mr_sink (Key '('table (String 'Output))) out '('('mode 'append))))
+    (let world (Commit! world mr_sink))
+    (return world)
+)
diff --git a/yql/essentials/tests/s-expressions/suites/TableContent/default.cfg b/yql/essentials/tests/s-expressions/suites/TableContent/default.cfg
new file mode 100644
index 0000000000..ad52c79527
--- /dev/null
+++ b/yql/essentials/tests/s-expressions/suites/TableContent/default.cfg
@@ -0,0 +1,2 @@
+in Input input.txt
+out Output output.txt
diff --git a/yql/essentials/tests/s-expressions/suites/TableContent/input.txt b/yql/essentials/tests/s-expressions/suites/TableContent/input.txt
new file mode 100644
index 0000000000..d1fa0973d9
--- /dev/null
+++ b/yql/essentials/tests/s-expressions/suites/TableContent/input.txt
@@ -0,0 +1,8 @@
+{"key"="075";"subkey"=".";"value"="abc"};
+{"key"="911";"subkey"=".";"value"="kkk"};
+{"key"="023";"subkey"=".";"value"="aaa"};
+{"key"="527";"subkey"=".";"value"="bbb"};
+{"key"="037";"subkey"=".";"value"="ddd"};
+{"key"="761";"subkey"=".";"value"="ccc"};
+{"key"="200";"subkey"=".";"value"="qqq"};
+{"key"="150";"subkey"=".";"value"="zzz"};
diff --git a/yql/essentials/tests/s-expressions/suites/TableContent/input.txt.attr b/yql/essentials/tests/s-expressions/suites/TableContent/input.txt.attr
new file mode 100644
index 0000000000..b6100e5fd0
--- /dev/null
+++ b/yql/essentials/tests/s-expressions/suites/TableContent/input.txt.attr
@@ -0,0 +1,30 @@
+{
+    "_yql_row_spec" = {
+        "Type" = [
+            "StructType";
+            [
+                [
+                    "key";
+                    [
+                        "DataType";
+                        "String"
+                    ]
+                ];
+                [
+                    "subkey";
+                    [
+                        "DataType";
+                        "String"
+                    ]
+                ];
+                [
+                    "value";
+                    [
+                        "DataType";
+                        "String"
+                    ]
+                ]
+            ]
+        ]
+    }
+}
\ No newline at end of file
diff --git a/yql/essentials/tests/s-expressions/suites/TableContent/input_other.txt b/yql/essentials/tests/s-expressions/suites/TableContent/input_other.txt
new file mode 100644
index 0000000000..65949ea745
--- /dev/null
+++ b/yql/essentials/tests/s-expressions/suites/TableContent/input_other.txt
@@ -0,0 +1,4 @@
+{"key"="075";"subkey"="1";"value"="abc"};
+{"key"="800";"subkey"="2";"value"="ddd"};
+{"key"="020";"subkey"="3";"value"="q"};
+{"key"="150";"subkey"="4";"value"="qzz"};
diff --git a/yql/essentials/tests/s-expressions/suites/TableContent/input_other.txt.attr b/yql/essentials/tests/s-expressions/suites/TableContent/input_other.txt.attr
new file mode 100644
index 0000000000..371a0d6ec2
--- /dev/null
+++ b/yql/essentials/tests/s-expressions/suites/TableContent/input_other.txt.attr
@@ -0,0 +1,10 @@
+{
+    "schema" = <
+        "strict" = %false;
+        "unique_keys" = %false
+    >
+    [{
+        "name" = "key";
+        "type" = "string"
+    }];
+}
diff --git a/yql/essentials/tests/s-expressions/suites/TableContent/input_other_strict.txt b/yql/essentials/tests/s-expressions/suites/TableContent/input_other_strict.txt
new file mode 100644
index 0000000000..b043b210b0
--- /dev/null
+++ b/yql/essentials/tests/s-expressions/suites/TableContent/input_other_strict.txt
@@ -0,0 +1,4 @@
+{"key"="075";"_other"=[["subkey";"1"];["value";"abc"]]};
+{"key"="800";"_other"=[["subkey";"2"];["value";"ddd"]]};
+{"key"="020";"_other"=[["subkey";"3"];["value";"q"]]};
+{"key"="150";"_other"=[["subkey";"4"];["value";"qzz"]]};
diff --git a/yql/essentials/tests/s-expressions/suites/TableContent/input_other_strict.txt.attr b/yql/essentials/tests/s-expressions/suites/TableContent/input_other_strict.txt.attr
new file mode 100644
index 0000000000..e7d68f4eea
--- /dev/null
+++ b/yql/essentials/tests/s-expressions/suites/TableContent/input_other_strict.txt.attr
@@ -0,0 +1,28 @@
+{
+    "_yql_row_spec" = {"Type" = [
+        "StructType";
+        [
+            [
+                "_other";
+                [
+                    "DictType";
+                    [
+                        "DataType";
+                        "String"
+                    ];
+                    [
+                        "DataType";
+                        "String"
+                    ]
+                ]
+            ];
+            [
+                "key";
+                [
+                    "DataType";
+                    "Yson"
+                ]
+            ]
+        ]
+    ]};
+}