Intermediate changes

commit_hash:b05d89eb8dfeb6b3f2fe069a57d8f2297b8e3638

5 files changed, 107 insertions, 1 deletions

diff --git a/yql/essentials/cfg/tests/gateways-experimental.conf b/yql/essentials/cfg/tests/gateways-experimental.conf
index a68414581b7..0c3dfaea61c 100644
--- a/yql/essentials/cfg/tests/gateways-experimental.conf
+++ b/yql/essentials/cfg/tests/gateways-experimental.conf
@@ -40,8 +40,18 @@ Yt {
     }
 
     DefaultSettings {
+        Name: "JobBlockInput"
+        Value: "true"
+    }
+
+    DefaultSettings {
         Name: "JobBlockInputSupportedTypes"
-        Value: "tuple"
+        Value: "tuple,pg"
+    }
+
+    DefaultSettings {
+        Name: "BlockReaderSupportedTypes"
+        Value: "tuple,pg"
     }
 
     DefaultSettings {
@@ -50,12 +60,31 @@ Yt {
     }
 
     DefaultSettings {
+        Name: "BlockReaderSupportedDataTypes"
+        Value: "Int8,Uint8,Int16,Uint16,Int32,Uint32,Int64,Uint64,Bool,Double,String,Utf8,Yson,Float"
+    }
+
+    DefaultSettings {
         Name: "ReportEquiJoinStats"
         Value: "true"
     }
+
+    DefaultSettings {
+        Name: "UseRPCReaderInDq"
+        Value: "true"
+    }
+
+    DefaultSettings {
+        Name: "DQRPCReaderTimeout"
+        Value: "600s"
+    }
 }
 
 Dq {
+    DefaultSettings {
+        Name: "UseBlockReader"
+        Value: "true"
+    }
 }
 
 YqlCore {
diff --git a/yql/essentials/tests/s-expressions/minirun/part0/canondata/result.json b/yql/essentials/tests/s-expressions/minirun/part0/canondata/result.json
index 0a80f2cd10c..5902893a5ac 100644
--- a/yql/essentials/tests/s-expressions/minirun/part0/canondata/result.json
+++ b/yql/essentials/tests/s-expressions/minirun/part0/canondata/result.json
@@ -13,6 +13,20 @@
             "uri": "https://{canondata_backend}/1942671/fe442fb9178873beea37ece387e67a5d541a25b2/resource.tar.gz#test.test_Aggregation-AggrCount-default.txt-Results_/results.txt"
         }
     ],
+    "test.test[Blocks-WideToBlocks-WideFromBlocks-default.txt-Debug]": [
+        {
+            "checksum": "7838538438ce14638ded18f1defb2380",
+            "size": 627,
+            "uri": "https://{canondata_backend}/1925842/3d4a337c6b21036cefb828b5de69295a7ec717f6/resource.tar.gz#test.test_Blocks-WideToBlocks-WideFromBlocks-default.txt-Debug_/opt.yql"
+        }
+    ],
+    "test.test[Blocks-WideToBlocks-WideFromBlocks-default.txt-Results]": [
+        {
+            "checksum": "f98666abfa816dc9b28ac6c28c9b5e7c",
+            "size": 1257,
+            "uri": "https://{canondata_backend}/1925842/3d4a337c6b21036cefb828b5de69295a7ec717f6/resource.tar.gz#test.test_Blocks-WideToBlocks-WideFromBlocks-default.txt-Results_/results.txt"
+        }
+    ],
     "test.test[Builtins-ByteString-default.txt-Debug]": [
         {
             "checksum": "056b689b9c8cf44642288e79f8cd0185",
diff --git a/yql/essentials/tests/s-expressions/minirun/part8/canondata/result.json b/yql/essentials/tests/s-expressions/minirun/part8/canondata/result.json
index 2e739109485..3d8ae83f3b5 100644
--- a/yql/essentials/tests/s-expressions/minirun/part8/canondata/result.json
+++ b/yql/essentials/tests/s-expressions/minirun/part8/canondata/result.json
@@ -55,6 +55,20 @@
             "uri": "https://{canondata_backend}/1937429/6dc717bd36879ce84e2fa1eb85b97eefce0733e9/resource.tar.gz#test.test_Blocks-ListFromBlocks-default.txt-Results_/results.txt"
         }
     ],
+    "test.test[Blocks-WideFromBlocks-WideToBlocks-default.txt-Debug]": [
+        {
+            "checksum": "d6c0024db8898aef8f8b31b1025667cc",
+            "size": 483,
+            "uri": "https://{canondata_backend}/1925842/70c39ce7d6aa898d26d58abf8d512eebac34de7f/resource.tar.gz#test.test_Blocks-WideFromBlocks-WideToBlocks-default.txt-Debug_/opt.yql"
+        }
+    ],
+    "test.test[Blocks-WideFromBlocks-WideToBlocks-default.txt-Results]": [
+        {
+            "checksum": "3326d454244059e96a0b61b3fdf53480",
+            "size": 878,
+            "uri": "https://{canondata_backend}/1925842/70c39ce7d6aa898d26d58abf8d512eebac34de7f/resource.tar.gz#test.test_Blocks-WideFromBlocks-WideToBlocks-default.txt-Results_/results.txt"
+        }
+    ],
     "test.test[Builtins-ToIntegral-default.txt-Debug]": [
         {
             "checksum": "33f569baf5940bbd79fbf635f47cc363",
diff --git a/yql/essentials/tests/s-expressions/suites/Blocks/WideFromBlocks-WideToBlocks.yqls b/yql/essentials/tests/s-expressions/suites/Blocks/WideFromBlocks-WideToBlocks.yqls
new file mode 100644
index 00000000000..cd1a5d60d23
--- /dev/null
+++ b/yql/essentials/tests/s-expressions/suites/Blocks/WideFromBlocks-WideToBlocks.yqls
@@ -0,0 +1,24 @@
+(
+# XXX: Explicitly enable PeepHoleBlock and PeepHoleFinalBlock steps.
+(let world (Configure! world (DataSource 'config) 'BlockEngine 'force))
+(let wconf (DataSink 'result))
+
+(let x1 (AsStruct '('"x" (Int32 '"1"))))
+(let x2 (AsStruct '('"x" (Int32 '"2"))))
+(let x3 (AsStruct '('"x" (Int32 '"3"))))
+(let x4 (AsStruct '('"x" (Int32 '"4"))))
+
+(let list (AsList x1 x2 x3 x4))
+
+(let expandLambda (lambda '(item) (Member item '"x")))
+(let wideStream (FromFlow (ExpandMap (ToFlow list) expandLambda)))
+
+(let nopFromBlocksToBlocks (WideFromBlocks (WideToBlocks wideStream)))
+
+(let narrowLambda (lambda '(x) (AsStruct '('"x" x))))
+(let scalarList (ForwardList (NarrowMap (ToFlow nopFromBlocksToBlocks) narrowLambda)))
+
+(let world (Write! world wconf (Key) scalarList '('('type))))
+(let world (Commit! world wconf))
+(return world)
+)
diff --git a/yql/essentials/tests/s-expressions/suites/Blocks/WideToBlocks-WideFromBlocks.yqls b/yql/essentials/tests/s-expressions/suites/Blocks/WideToBlocks-WideFromBlocks.yqls
new file mode 100644
index 00000000000..cafd0f45a81
--- /dev/null
+++ b/yql/essentials/tests/s-expressions/suites/Blocks/WideToBlocks-WideFromBlocks.yqls
@@ -0,0 +1,25 @@
+(
+# XXX: Explicitly enable PeepHoleBlock and PeepHoleFinalBlock steps.
+(let world (Configure! world (DataSource 'config) 'BlockEngine 'force))
+(let wconf (DataSink 'result))
+
+(let x1 (AsStruct '('"x" (Int32 '"1"))))
+(let x2 (AsStruct '('"x" (Int32 '"2"))))
+(let x3 (AsStruct '('"x" (Int32 '"3"))))
+(let x4 (AsStruct '('"x" (Int32 '"4"))))
+(let scalar0 (AsScalar (Int32 '"0")))
+
+(let blockList (ListToBlocks (AsList x1 x2 x3 x4)))
+
+(let expandLambda (lambda '(item) (Member item '"x") scalar0 (Member item '"_yql_block_length")))
+(let wideBlocksStream (FromFlow (ExpandMap (ToFlow blockList) expandLambda)))
+
+(let nopToBlocksFromBlocks (WideToBlocks (WideFromBlocks wideBlocksStream)))
+
+(let narrowLambda (lambda '(x scalar blockLength) (AsStruct '('"x" x) '('"scalar" scalar) '('"_yql_block_length" blockLength))))
+(let scalarList (ListFromBlocks (ForwardList (NarrowMap (ToFlow nopToBlocksFromBlocks) narrowLambda))))
+
+(let world (Write! world wconf (Key) scalarList '('('type))))
+(let world (Commit! world wconf))
+(return world)
+)