Merge pull request #11788 from ydb-platform/mergelibs-241120-1113

Library import 241120-1113

25 files changed, 238 insertions, 0 deletions

diff --git a/yql/essentials/tests/s-expressions/suites/File/ConfigureFileFromUrl.yql b/yql/essentials/tests/s-expressions/suites/File/ConfigureFileFromUrl.yql
new file mode 100644
index 0000000000..b6d23e3c3e
--- /dev/null
+++ b/yql/essentials/tests/s-expressions/suites/File/ConfigureFileFromUrl.yql
@@ -0,0 +1,11 @@
+# canonize yson here
+(
+(let config (DataSource 'config))
+(let world (Configure! world config 'PureDataSource 'yt))
+(let world (Configure! world config 'AddFileByUrl 'Tony 'https://yadi.sk/a/pk0iwMkNuNYgM/57b9c14c6fcc2a565515fab9))
+
+(let res_sink (DataSink 'result))
+(let world (Write! world res_sink (Key) (Files '/home) '('('type))))
+(let world (Commit! world res_sink))
+(return world)
+)
diff --git a/yql/essentials/tests/s-expressions/suites/File/DifferentAliasesToOneFile.cfg b/yql/essentials/tests/s-expressions/suites/File/DifferentAliasesToOneFile.cfg
new file mode 100644
index 0000000000..03037b8d6f
--- /dev/null
+++ b/yql/essentials/tests/s-expressions/suites/File/DifferentAliasesToOneFile.cfg
@@ -0,0 +1,3 @@
+res result.txt
+file OneFile one_file.txt
+file AnotherFile one_file.txt
diff --git a/yql/essentials/tests/s-expressions/suites/File/DifferentAliasesToOneFile.yql b/yql/essentials/tests/s-expressions/suites/File/DifferentAliasesToOneFile.yql
new file mode 100644
index 0000000000..6e65cc1e65
--- /dev/null
+++ b/yql/essentials/tests/s-expressions/suites/File/DifferentAliasesToOneFile.yql
@@ -0,0 +1,13 @@
+(
+#comment
+(let config (DataSource 'config))
+(let world (Configure! world config 'PureDataSource 'yt))
+
+(let res_sink (DataSink 'result))
+(let results (AsDict
+    '((String 'OneFile) (FileContent 'OneFile))
+    '((String 'AnotherFile) (FileContent 'AnotherFile))
+))
+(let world (Write! world res_sink (Key) results '('('type))))
+(return (Commit! world res_sink))
+)
diff --git a/yql/essentials/tests/s-expressions/suites/File/DifferentAliasesToSameFiles.cfg b/yql/essentials/tests/s-expressions/suites/File/DifferentAliasesToSameFiles.cfg
new file mode 100644
index 0000000000..a566dd4aa0
--- /dev/null
+++ b/yql/essentials/tests/s-expressions/suites/File/DifferentAliasesToSameFiles.cfg
@@ -0,0 +1,3 @@
+res result.txt
+file OneFile one_file.txt
+file AnotherFile another_file.txt
diff --git a/yql/essentials/tests/s-expressions/suites/File/DifferentAliasesToSameFiles.yql b/yql/essentials/tests/s-expressions/suites/File/DifferentAliasesToSameFiles.yql
new file mode 100644
index 0000000000..6e65cc1e65
--- /dev/null
+++ b/yql/essentials/tests/s-expressions/suites/File/DifferentAliasesToSameFiles.yql
@@ -0,0 +1,13 @@
+(
+#comment
+(let config (DataSource 'config))
+(let world (Configure! world config 'PureDataSource 'yt))
+
+(let res_sink (DataSink 'result))
+(let results (AsDict
+    '((String 'OneFile) (FileContent 'OneFile))
+    '((String 'AnotherFile) (FileContent 'AnotherFile))
+))
+(let world (Write! world res_sink (Key) results '('('type))))
+(return (Commit! world res_sink))
+)
diff --git a/yql/essentials/tests/s-expressions/suites/File/FileContent.yql b/yql/essentials/tests/s-expressions/suites/File/FileContent.yql
new file mode 100644
index 0000000000..3e34cb8a58
--- /dev/null
+++ b/yql/essentials/tests/s-expressions/suites/File/FileContent.yql
@@ -0,0 +1,10 @@
+(
+#comment
+(let config (DataSource 'config))
+(let world (Configure! world config 'PureDataSource 'yt))
+
+(let res_sink (DataSink 'result))
+(let world (Write! world res_sink (Key) (FileContent 'MyFile) '()))
+(let world (Commit! world res_sink))
+(return world)
+)
diff --git a/yql/essentials/tests/s-expressions/suites/File/FilePath.yql b/yql/essentials/tests/s-expressions/suites/File/FilePath.yql
new file mode 100644
index 0000000000..1b036e6da4
--- /dev/null
+++ b/yql/essentials/tests/s-expressions/suites/File/FilePath.yql
@@ -0,0 +1,11 @@
+# do not check yson here
+(
+#comment
+(let config (DataSource 'config))
+(let world (Configure! world config 'PureDataSource 'yt))
+
+(let res_sink (DataSink 'result))
+(let world (Write! world res_sink (Key) (FilePath 'MyFile) '()))
+(let world (Commit! world res_sink))
+(return world)
+)
diff --git a/yql/essentials/tests/s-expressions/suites/File/FilePathFromRoot.yql b/yql/essentials/tests/s-expressions/suites/File/FilePathFromRoot.yql
new file mode 100644
index 0000000000..fa4d5835ce
--- /dev/null
+++ b/yql/essentials/tests/s-expressions/suites/File/FilePathFromRoot.yql
@@ -0,0 +1,11 @@
+# do not check yson here
+(
+#comment
+(let config (DataSource 'config))
+(let world (Configure! world config 'PureDataSource 'yt))
+
+(let res_sink (DataSink 'result))
+(let world (Write! world res_sink (Key) (FilePath '/home/MyFile) '()))
+(let world (Commit! world res_sink))
+(return world)
+)
diff --git a/yql/essentials/tests/s-expressions/suites/File/FillWithFileContent.cfg b/yql/essentials/tests/s-expressions/suites/File/FillWithFileContent.cfg
new file mode 100644
index 0000000000..5c9bcce3f8
--- /dev/null
+++ b/yql/essentials/tests/s-expressions/suites/File/FillWithFileContent.cfg
@@ -0,0 +1,3 @@
+file MyFile file1.txt
+in Input input.txt
+out Output output.txt
diff --git a/yql/essentials/tests/s-expressions/suites/File/FillWithFileContent.yql b/yql/essentials/tests/s-expressions/suites/File/FillWithFileContent.yql
new file mode 100644
index 0000000000..203ac40254
--- /dev/null
+++ b/yql/essentials/tests/s-expressions/suites/File/FillWithFileContent.yql
@@ -0,0 +1,11 @@
+(
+#comment
+(let data (AsList 
+(AsStruct '('key (String 'key1)) '('subkey (String '"")) '('value (FileContent 'MyFile)))
+))
+
+(let mr_sink (DataSink 'yt (quote plato)))
+(let world (Write! world mr_sink (Key '('table (String 'Output))) data '('('mode 'append))))
+(let world (Commit! world mr_sink))
+(return world)
+)
diff --git a/yql/essentials/tests/s-expressions/suites/File/FolderPath.yql b/yql/essentials/tests/s-expressions/suites/File/FolderPath.yql
new file mode 100644
index 0000000000..e676a8a14d
--- /dev/null
+++ b/yql/essentials/tests/s-expressions/suites/File/FolderPath.yql
@@ -0,0 +1,11 @@
+# canonize yson here
+(
+#comment
+(let config (DataSource 'config))
+(let world (Configure! world config 'PureDataSource 'yt))
+
+(let res_sink (DataSink 'result))
+(let world (Write! world res_sink (Key) (FolderPath '"/lib") '()))
+(let world (Commit! world res_sink))
+(return world)
+)
diff --git a/yql/essentials/tests/s-expressions/suites/File/HomeFiles.yql b/yql/essentials/tests/s-expressions/suites/File/HomeFiles.yql
new file mode 100644
index 0000000000..78799b4126
--- /dev/null
+++ b/yql/essentials/tests/s-expressions/suites/File/HomeFiles.yql
@@ -0,0 +1,11 @@
+# canonize yson here
+(
+#comment
+(let config (DataSource 'config))
+(let world (Configure! world config 'PureDataSource 'yt))
+
+(let res_sink (DataSink 'result))
+(let world (Write! world res_sink (Key) (Files '/home) '('('type))))
+(let world (Commit! world res_sink))
+(return world)
+)
diff --git a/yql/essentials/tests/s-expressions/suites/File/MapWithFileContent.cfg b/yql/essentials/tests/s-expressions/suites/File/MapWithFileContent.cfg
new file mode 100644
index 0000000000..5c9bcce3f8
--- /dev/null
+++ b/yql/essentials/tests/s-expressions/suites/File/MapWithFileContent.cfg
@@ -0,0 +1,3 @@
+file MyFile file1.txt
+in Input input.txt
+out Output output.txt
diff --git a/yql/essentials/tests/s-expressions/suites/File/MapWithFileContent.yql b/yql/essentials/tests/s-expressions/suites/File/MapWithFileContent.yql
new file mode 100644
index 0000000000..d481de6809
--- /dev/null
+++ b/yql/essentials/tests/s-expressions/suites/File/MapWithFileContent.yql
@@ -0,0 +1,19 @@
+(
+#comment
+(let mr_source (DataSource 'yt 'plato))
+(let x (Read! world mr_source (Key '('table (String 'Input))) '('key 'subkey 'value) '()))
+(let world (Left! x))
+(let table1 (Right! x))
+(let table1map (Map table1 (lambda '(item) (block '(
+   (let newValue (FileContent 'MyFile))
+   (let s (Struct))
+   (let s (AddMember s 'key (Member item 'key)))
+   (let s (AddMember s 'subkey (Member item 'subkey)))
+   (let s (AddMember s 'value newValue))
+   (return s)
+)))))
+(let mr_sink (DataSink 'yt (quote plato)))
+(let world (Write! world mr_sink (Key '('table (String 'Output))) table1map '('('mode 'append))))
+(let world (Commit! world mr_sink))
+(return world)
+)
diff --git a/yql/essentials/tests/s-expressions/suites/File/MapWithFilePath.cfg b/yql/essentials/tests/s-expressions/suites/File/MapWithFilePath.cfg
new file mode 100644
index 0000000000..b99236a7de
--- /dev/null
+++ b/yql/essentials/tests/s-expressions/suites/File/MapWithFilePath.cfg
@@ -0,0 +1,4 @@
+file MyFile file1.txt
+in Input input.txt
+out Output output.txt
+udf re2_udf
diff --git a/yql/essentials/tests/s-expressions/suites/File/MapWithFilePath.yql b/yql/essentials/tests/s-expressions/suites/File/MapWithFilePath.yql
new file mode 100644
index 0000000000..fcd006e0aa
--- /dev/null
+++ b/yql/essentials/tests/s-expressions/suites/File/MapWithFilePath.yql
@@ -0,0 +1,22 @@
+# canonize yson here
+(
+#comment
+(let udf (Udf '"Re2.Replace" '((String '"(^|.*/)([^/]+)$") (Null))))
+(let mr_source (DataSource 'yt 'plato))
+(let x (Read! world mr_source (Key '('table (String 'Input))) '('key 'subkey 'value) '()))
+(let world (Left! x))
+(let table1 (Right! x))
+(let table1map (Map table1 (lambda '(item) (block '(
+   (let newValue (FilePath 'MyFile))
+   (let newValue (Apply udf newValue (String '"./\\2")))
+   (let s (Struct))
+   (let s (AddMember s 'key (Member item 'key)))
+   (let s (AddMember s 'subkey (Member item 'subkey)))
+   (let s (AddMember s 'value newValue))
+   (return s)
+)))))
+(let mr_sink (DataSink 'yt (quote plato)))
+(let world (Write! world mr_sink (Key '('table (String 'Output))) table1map '('('mode 'append))))
+(let world (Commit! world mr_sink))
+(return world)
+)
diff --git a/yql/essentials/tests/s-expressions/suites/File/MapWithFolderPath.cfg b/yql/essentials/tests/s-expressions/suites/File/MapWithFolderPath.cfg
new file mode 100644
index 0000000000..f1a778e02d
--- /dev/null
+++ b/yql/essentials/tests/s-expressions/suites/File/MapWithFolderPath.cfg
@@ -0,0 +1,4 @@
+in Input input.txt
+out Output output.txt
+mount ../mounts.txt
+udf re2_udf
diff --git a/yql/essentials/tests/s-expressions/suites/File/MapWithFolderPath.yql b/yql/essentials/tests/s-expressions/suites/File/MapWithFolderPath.yql
new file mode 100644
index 0000000000..a9bd5a3ada
--- /dev/null
+++ b/yql/essentials/tests/s-expressions/suites/File/MapWithFolderPath.yql
@@ -0,0 +1,22 @@
+# canonize yson here
+(
+#comment
+(let udf (Udf '"Re2.Replace" '((String '"(^|.*/)([^/]+/?)$") (Null))))
+(let mr_source (DataSource 'yt 'plato))
+(let x (Read! world mr_source (Key '('table (String 'Input))) '('key 'subkey 'value) '()))
+(let world (Left! x))
+(let table1 (Right! x))
+(let table1map (Map table1 (lambda '(item) (block '(
+   (let newValue (FolderPath '"/lib"))
+   (let newValue (Apply udf newValue (String '"./\\2")))
+   (let s (Struct))
+   (let s (AddMember s 'key (Member item 'key)))
+   (let s (AddMember s 'subkey (Member item 'subkey)))
+   (let s (AddMember s 'value newValue))
+   (return s)
+)))))
+(let mr_sink (DataSink 'yt (quote plato)))
+(let world (Write! world mr_sink (Key '('table (String 'Output))) table1map '('('mode 'append))))
+(let world (Commit! world mr_sink))
+(return world)
+)
diff --git a/yql/essentials/tests/s-expressions/suites/File/RootFiles.yql b/yql/essentials/tests/s-expressions/suites/File/RootFiles.yql
new file mode 100644
index 0000000000..cf275fb841
--- /dev/null
+++ b/yql/essentials/tests/s-expressions/suites/File/RootFiles.yql
@@ -0,0 +1,11 @@
+# canonize yson here
+(
+#comment
+(let config (DataSource 'config))
+(let world (Configure! world config 'PureDataSource 'yt))
+
+(let res_sink (DataSink 'result))
+(let world (Write! world res_sink (Key) (Files '/) '('('type))))
+(let world (Commit! world res_sink))
+(return world)
+)
diff --git a/yql/essentials/tests/s-expressions/suites/File/another_file.txt b/yql/essentials/tests/s-expressions/suites/File/another_file.txt
new file mode 100644
index 0000000000..b07dc8bb88
--- /dev/null
+++ b/yql/essentials/tests/s-expressions/suites/File/another_file.txt
@@ -0,0 +1 @@
+Lorem ipsum dolor sit amet, consectetur adipiscing elit...
diff --git a/yql/essentials/tests/s-expressions/suites/File/default.cfg b/yql/essentials/tests/s-expressions/suites/File/default.cfg
new file mode 100644
index 0000000000..c870e59de5
--- /dev/null
+++ b/yql/essentials/tests/s-expressions/suites/File/default.cfg
@@ -0,0 +1,4 @@
+file MyFile file1.txt
+res result.txt
+mount ../mounts.txt
+udf string_udf
diff --git a/yql/essentials/tests/s-expressions/suites/File/file1.txt b/yql/essentials/tests/s-expressions/suites/File/file1.txt
new file mode 100644
index 0000000000..882c86d1f1
--- /dev/null
+++ b/yql/essentials/tests/s-expressions/suites/File/file1.txt
@@ -0,0 +1,2 @@
+some data 1
+some data 2
diff --git a/yql/essentials/tests/s-expressions/suites/File/input.txt b/yql/essentials/tests/s-expressions/suites/File/input.txt
new file mode 100644
index 0000000000..65949ea745
--- /dev/null
+++ b/yql/essentials/tests/s-expressions/suites/File/input.txt
@@ -0,0 +1,4 @@
+{"key"="075";"subkey"="1";"value"="abc"};
+{"key"="800";"subkey"="2";"value"="ddd"};
+{"key"="020";"subkey"="3";"value"="q"};
+{"key"="150";"subkey"="4";"value"="qzz"};
diff --git a/yql/essentials/tests/s-expressions/suites/File/input.txt.attr b/yql/essentials/tests/s-expressions/suites/File/input.txt.attr
new file mode 100644
index 0000000000..b6100e5fd0
--- /dev/null
+++ b/yql/essentials/tests/s-expressions/suites/File/input.txt.attr
@@ -0,0 +1,30 @@
+{
+    "_yql_row_spec" = {
+        "Type" = [
+            "StructType";
+            [
+                [
+                    "key";
+                    [
+                        "DataType";
+                        "String"
+                    ]
+                ];
+                [
+                    "subkey";
+                    [
+                        "DataType";
+                        "String"
+                    ]
+                ];
+                [
+                    "value";
+                    [
+                        "DataType";
+                        "String"
+                    ]
+                ]
+            ]
+        ]
+    }
+}
\ No newline at end of file
diff --git a/yql/essentials/tests/s-expressions/suites/File/one_file.txt b/yql/essentials/tests/s-expressions/suites/File/one_file.txt
new file mode 100644
index 0000000000..b07dc8bb88
--- /dev/null
+++ b/yql/essentials/tests/s-expressions/suites/File/one_file.txt
@@ -0,0 +1 @@
+Lorem ipsum dolor sit amet, consectetur adipiscing elit...