diff options
author | robot-piglet <robot-piglet@yandex-team.com> | 2025-01-29 16:02:20 +0300 |
---|---|---|
committer | robot-piglet <robot-piglet@yandex-team.com> | 2025-01-29 16:15:08 +0300 |
commit | be36d7b60684c12f59cbbb6ef49017e5d7f4440d (patch) | |
tree | 0b917cf5c1fcf02e86a3d91caaff09321a824b97 | |
parent | 4c83154820595e855ddc50b8d0e6c96098b0adb8 (diff) | |
download | ydb-be36d7b60684c12f59cbbb6ef49017e5d7f4440d.tar.gz |
Intermediate changes
commit_hash:22545c4aa243324b513d4c90832d07cc85d8a6cf
32 files changed, 636 insertions, 67 deletions
diff --git a/yql/essentials/tests/sql/minirun/part0/canondata/result.json b/yql/essentials/tests/sql/minirun/part0/canondata/result.json index 209766edb6..cecc1d216b 100644 --- a/yql/essentials/tests/sql/minirun/part0/canondata/result.json +++ b/yql/essentials/tests/sql/minirun/part0/canondata/result.json @@ -261,6 +261,27 @@ "uri": "https://{canondata_backend}/1920236/e2416b57cd9baa140c086892d357d5945ed0fdb1/resource.tar.gz#test.test_blocks-and-default.txt-Results_/results.txt" } ], + "test.test[blocks-pg_call-default.txt-Debug]": [ + { + "checksum": "34def2ef08f7d1306e6bfe6afe7bde8f", + "size": 701, + "uri": "https://{canondata_backend}/1814674/d1d41a17170e55fd1f93d85f4708eb5282bce7d2/resource.tar.gz#test.test_blocks-pg_call-default.txt-Debug_/opt.yql" + } + ], + "test.test[blocks-pg_call-default.txt-Peephole]": [ + { + "checksum": "4b035ae018652e54247b56a686325bb4", + "size": 887, + "uri": "https://{canondata_backend}/1814674/d1d41a17170e55fd1f93d85f4708eb5282bce7d2/resource.tar.gz#test.test_blocks-pg_call-default.txt-Peephole_/opt.yql" + } + ], + "test.test[blocks-pg_call-default.txt-Results]": [ + { + "checksum": "0c1271771d3ec413f22fdd8c115f2dd6", + "size": 1370, + "uri": "https://{canondata_backend}/1814674/d1d41a17170e55fd1f93d85f4708eb5282bce7d2/resource.tar.gz#test.test_blocks-pg_call-default.txt-Results_/results.txt" + } + ], "test.test[blocks-xor-default.txt-Debug]": [ { "checksum": "f0b56bd3576e6d9f549d692dff6522c8", diff --git a/yql/essentials/tests/sql/minirun/part3/canondata/result.json b/yql/essentials/tests/sql/minirun/part3/canondata/result.json index 4db0b28687..518dcf729d 100644 --- a/yql/essentials/tests/sql/minirun/part3/canondata/result.json +++ b/yql/essentials/tests/sql/minirun/part3/canondata/result.json @@ -285,6 +285,27 @@ "uri": "file://test.test_bitcast_implicit-div_bitcast-default.txt-Results_/extracted" } ], + "test.test[blocks-frompg-default.txt-Debug]": [ + { + "checksum": "52d5daf2447e1de0d4cf3150468c9de5", + "size": 361, + "uri": "https://{canondata_backend}/1775319/25a92a1cd773ccebf59751427286ef1fe45aeeb2/resource.tar.gz#test.test_blocks-frompg-default.txt-Debug_/opt.yql" + } + ], + "test.test[blocks-frompg-default.txt-Peephole]": [ + { + "checksum": "e245ccef92be3a037238558f8908427b", + "size": 531, + "uri": "https://{canondata_backend}/1775319/25a92a1cd773ccebf59751427286ef1fe45aeeb2/resource.tar.gz#test.test_blocks-frompg-default.txt-Peephole_/opt.yql" + } + ], + "test.test[blocks-frompg-default.txt-Results]": [ + { + "checksum": "19876b1dd470207f15ec98224449a5a3", + "size": 958, + "uri": "https://{canondata_backend}/1775319/25a92a1cd773ccebf59751427286ef1fe45aeeb2/resource.tar.gz#test.test_blocks-frompg-default.txt-Results_/results.txt" + } + ], "test.test[blocks-or_opt-default.txt-Debug]": [ { "checksum": "85fb6a988595cab34e899acfd0ef8dd2", diff --git a/yql/essentials/tests/sql/minirun/part5/canondata/result.json b/yql/essentials/tests/sql/minirun/part5/canondata/result.json index 406b42e25d..0fc1d285af 100644 --- a/yql/essentials/tests/sql/minirun/part5/canondata/result.json +++ b/yql/essentials/tests/sql/minirun/part5/canondata/result.json @@ -345,6 +345,27 @@ "uri": "https://{canondata_backend}/1942671/ce51ce34754ddc48cce99d0eb0d68c58043ce298/resource.tar.gz#test.test_blocks-and_opt-default.txt-Results_/results.txt" } ], + "test.test[blocks-decimal_binop-default.txt-Debug]": [ + { + "checksum": "36ae0e625c8e7bc9ff869827c5b36a33", + "size": 389, + "uri": "https://{canondata_backend}/1775319/488597481a390ab007602d4f2c7cb21f0d5833de/resource.tar.gz#test.test_blocks-decimal_binop-default.txt-Debug_/opt.yql" + } + ], + "test.test[blocks-decimal_binop-default.txt-Peephole]": [ + { + "checksum": "e165bc99d0eeb90c4e09d377dfa0609b", + "size": 732, + "uri": "https://{canondata_backend}/1775319/488597481a390ab007602d4f2c7cb21f0d5833de/resource.tar.gz#test.test_blocks-decimal_binop-default.txt-Peephole_/opt.yql" + } + ], + "test.test[blocks-decimal_binop-default.txt-Results]": [ + { + "checksum": "0a482cb584916b15cf01e8bfddbd92fb", + "size": 1535, + "uri": "https://{canondata_backend}/1775319/488597481a390ab007602d4f2c7cb21f0d5833de/resource.tar.gz#test.test_blocks-decimal_binop-default.txt-Results_/results.txt" + } + ], "test.test[blocks-filter-default.txt-Debug]": [ { "checksum": "3dea830ec45f78ed78603576b054b4ca", @@ -366,6 +387,27 @@ "uri": "https://{canondata_backend}/1937150/d87c401e99a94fad9fdc66f7c4a0d9c670b84949/resource.tar.gz#test.test_blocks-filter-default.txt-Results_/results.txt" } ], + "test.test[blocks-just-default.txt-Debug]": [ + { + "checksum": "248412653c63c6892f76b6ed3c91a2ae", + "size": 230, + "uri": "https://{canondata_backend}/1775319/488597481a390ab007602d4f2c7cb21f0d5833de/resource.tar.gz#test.test_blocks-just-default.txt-Debug_/opt.yql" + } + ], + "test.test[blocks-just-default.txt-Peephole]": [ + { + "checksum": "d51f71d182470ef3d43b3082d7d48fb8", + "size": 468, + "uri": "https://{canondata_backend}/1775319/488597481a390ab007602d4f2c7cb21f0d5833de/resource.tar.gz#test.test_blocks-just-default.txt-Peephole_/opt.yql" + } + ], + "test.test[blocks-just-default.txt-Results]": [ + { + "checksum": "f9a40ce704e450668912cbb5f4baeef9", + "size": 887, + "uri": "https://{canondata_backend}/1775319/488597481a390ab007602d4f2c7cb21f0d5833de/resource.tar.gz#test.test_blocks-just-default.txt-Results_/results.txt" + } + ], "test.test[blocks-not-default.txt-Debug]": [ { "checksum": "11ca4c4fbc3e6a5463a95ce9496effdf", diff --git a/yql/essentials/tests/sql/minirun/part6/canondata/result.json b/yql/essentials/tests/sql/minirun/part6/canondata/result.json index 8c8ddbc9f5..088e4d2a31 100644 --- a/yql/essentials/tests/sql/minirun/part6/canondata/result.json +++ b/yql/essentials/tests/sql/minirun/part6/canondata/result.json @@ -317,6 +317,27 @@ "uri": "https://{canondata_backend}/1920236/8a88b220fe1ff8b571b6813b812d7623629101fd/resource.tar.gz#test.test_blocks-coalesce_scalar-default.txt-Results_/results.txt" } ], + "test.test[blocks-member-default.txt-Debug]": [ + { + "checksum": "b73a845c0abd3dddfca9888bbf83560a", + "size": 416, + "uri": "https://{canondata_backend}/1900335/81a360a61ba1555c418f20e4c99272eb22766040/resource.tar.gz#test.test_blocks-member-default.txt-Debug_/opt.yql" + } + ], + "test.test[blocks-member-default.txt-Peephole]": [ + { + "checksum": "b40c6c197f750c6a604329621bd37300", + "size": 586, + "uri": "https://{canondata_backend}/1900335/81a360a61ba1555c418f20e4c99272eb22766040/resource.tar.gz#test.test_blocks-member-default.txt-Peephole_/opt.yql" + } + ], + "test.test[blocks-member-default.txt-Results]": [ + { + "checksum": "8b1bee266dfa36a94d77f728d69f95ad", + "size": 961, + "uri": "https://{canondata_backend}/1900335/81a360a61ba1555c418f20e4c99272eb22766040/resource.tar.gz#test.test_blocks-member-default.txt-Results_/results.txt" + } + ], "test.test[blocks-projection_add_ints_filter-default.txt-Debug]": [ { "checksum": "0dc7c149ab903b62717b7afe506ab893", diff --git a/yql/essentials/tests/sql/minirun/part7/canondata/result.json b/yql/essentials/tests/sql/minirun/part7/canondata/result.json index e4d2c99469..3adddc5fb4 100644 --- a/yql/essentials/tests/sql/minirun/part7/canondata/result.json +++ b/yql/essentials/tests/sql/minirun/part7/canondata/result.json @@ -209,6 +209,27 @@ "uri": "https://{canondata_backend}/1920236/d08588ad74710b9926df45a5b372c37153cb8b1e/resource.tar.gz#test.test_bigdate-misc-default.txt-Results_/results.txt" } ], + "test.test[blocks-as_tuple-default.txt-Debug]": [ + { + "checksum": "0e965c2d9b0db25f4d2fab145332abbb", + "size": 288, + "uri": "https://{canondata_backend}/1775319/ebad1e34262c6a71e8c6dbf23efe7ba30e371e75/resource.tar.gz#test.test_blocks-as_tuple-default.txt-Debug_/opt.yql" + } + ], + "test.test[blocks-as_tuple-default.txt-Peephole]": [ + { + "checksum": "dc0005ecb6750c691d66fe6c50e00d0c", + "size": 603, + "uri": "https://{canondata_backend}/1775319/ebad1e34262c6a71e8c6dbf23efe7ba30e371e75/resource.tar.gz#test.test_blocks-as_tuple-default.txt-Peephole_/opt.yql" + } + ], + "test.test[blocks-as_tuple-default.txt-Results]": [ + { + "checksum": "41c77ef11f22f1696fcda9bb0c1a4515", + "size": 2005, + "uri": "https://{canondata_backend}/1775319/ebad1e34262c6a71e8c6dbf23efe7ba30e371e75/resource.tar.gz#test.test_blocks-as_tuple-default.txt-Results_/results.txt" + } + ], "test.test[blocks-or-default.txt-Debug]": [ { "checksum": "4c3c28b4c505737765e9379d3386b4ff", @@ -230,6 +251,27 @@ "uri": "https://{canondata_backend}/1942671/a9486852a02ad9ecba5005ac92cde1261882e326/resource.tar.gz#test.test_blocks-or-default.txt-Results_/results.txt" } ], + "test.test[blocks-topg-default.txt-Debug]": [ + { + "checksum": "689f15316094944d5027603befe3f31a", + "size": 367, + "uri": "https://{canondata_backend}/1775319/ebad1e34262c6a71e8c6dbf23efe7ba30e371e75/resource.tar.gz#test.test_blocks-topg-default.txt-Debug_/opt.yql" + } + ], + "test.test[blocks-topg-default.txt-Peephole]": [ + { + "checksum": "adfdcd16ed6e227522ee922fd1e818c8", + "size": 537, + "uri": "https://{canondata_backend}/1775319/ebad1e34262c6a71e8c6dbf23efe7ba30e371e75/resource.tar.gz#test.test_blocks-topg-default.txt-Peephole_/opt.yql" + } + ], + "test.test[blocks-topg-default.txt-Results]": [ + { + "checksum": "200c7ace6be24a899424965b6aba5fb3", + "size": 804, + "uri": "https://{canondata_backend}/1775319/ebad1e34262c6a71e8c6dbf23efe7ba30e371e75/resource.tar.gz#test.test_blocks-topg-default.txt-Results_/results.txt" + } + ], "test.test[case-case_opt_then-default.txt-Debug]": [ { "checksum": "456497d68c16bc496296525c7d2fadd5", diff --git a/yql/essentials/tests/sql/minirun/part8/canondata/result.json b/yql/essentials/tests/sql/minirun/part8/canondata/result.json index a041fe1469..de9da9c5b2 100644 --- a/yql/essentials/tests/sql/minirun/part8/canondata/result.json +++ b/yql/essentials/tests/sql/minirun/part8/canondata/result.json @@ -394,6 +394,27 @@ "uri": "https://{canondata_backend}/1942671/1fe07dc7779a4d1b05048ad4f78bfbeb3a86d998/resource.tar.gz#test.test_blocks-if-default.txt-Results_/results.txt" } ], + "test.test[blocks-pg_op-default.txt-Debug]": [ + { + "checksum": "726a0ff9b5ab151301d52192bfc274d2", + "size": 677, + "uri": "https://{canondata_backend}/1900335/9f3e189e481af8d65c06be72d7112a9ca1209c31/resource.tar.gz#test.test_blocks-pg_op-default.txt-Debug_/opt.yql" + } + ], + "test.test[blocks-pg_op-default.txt-Peephole]": [ + { + "checksum": "7b89533ec36887d33beaa65ae0662d3f", + "size": 863, + "uri": "https://{canondata_backend}/1900335/9f3e189e481af8d65c06be72d7112a9ca1209c31/resource.tar.gz#test.test_blocks-pg_op-default.txt-Peephole_/opt.yql" + } + ], + "test.test[blocks-pg_op-default.txt-Results]": [ + { + "checksum": "88cf292f03f7446cbd4a93928a3e0e38", + "size": 1398, + "uri": "https://{canondata_backend}/1900335/9f3e189e481af8d65c06be72d7112a9ca1209c31/resource.tar.gz#test.test_blocks-pg_op-default.txt-Results_/results.txt" + } + ], "test.test[blocks-xor_opt-default.txt-Debug]": [ { "checksum": "91e8a0bee949446ecd2a7ed7703756fe", diff --git a/yql/essentials/tests/sql/minirun/part9/canondata/result.json b/yql/essentials/tests/sql/minirun/part9/canondata/result.json index 84ff18d585..4d4e7caa3b 100644 --- a/yql/essentials/tests/sql/minirun/part9/canondata/result.json +++ b/yql/essentials/tests/sql/minirun/part9/canondata/result.json @@ -209,6 +209,27 @@ "uri": "https://{canondata_backend}/1920236/19a9934ebc9455bf1caab37058331acdf3c4ee26/resource.tar.gz#test.test_blocks-if_scalar-default.txt-Results_/results.txt" } ], + "test.test[blocks-nth-default.txt-Debug]": [ + { + "checksum": "33ec59138b81baf86a6a09b1a69cb923", + "size": 421, + "uri": "https://{canondata_backend}/1900335/892d26e90090b841f50bacde742fd87e2ef88a38/resource.tar.gz#test.test_blocks-nth-default.txt-Debug_/opt.yql" + } + ], + "test.test[blocks-nth-default.txt-Peephole]": [ + { + "checksum": "e06d2c11c8bad42802c715436faa44c0", + "size": 591, + "uri": "https://{canondata_backend}/1900335/892d26e90090b841f50bacde742fd87e2ef88a38/resource.tar.gz#test.test_blocks-nth-default.txt-Peephole_/opt.yql" + } + ], + "test.test[blocks-nth-default.txt-Results]": [ + { + "checksum": "8b1bee266dfa36a94d77f728d69f95ad", + "size": 961, + "uri": "https://{canondata_backend}/1900335/892d26e90090b841f50bacde742fd87e2ef88a38/resource.tar.gz#test.test_blocks-nth-default.txt-Results_/results.txt" + } + ], "test.test[blocks-projection_add_ints-default.txt-Debug]": [ { "checksum": "0dc7c149ab903b62717b7afe506ab893", diff --git a/yql/essentials/tests/sql/sql2yql/canondata/result.json b/yql/essentials/tests/sql/sql2yql/canondata/result.json index 4d5c62cb0b..9fec09ba5f 100644 --- a/yql/essentials/tests/sql/sql2yql/canondata/result.json +++ b/yql/essentials/tests/sql/sql2yql/canondata/result.json @@ -1343,6 +1343,13 @@ "uri": "https://{canondata_backend}/1916746/15a617669fb2b654b6be1fc493d61b99fb1f6f81/resource.tar.gz#test_sql2yql.test_blocks-and_scalar_/sql.yql" } ], + "test_sql2yql.test[blocks-as_tuple]": [ + { + "checksum": "601f02d489707b615a9ff16a4fe1d3f5", + "size": 1304, + "uri": "https://{canondata_backend}/1900335/c447765ddbde200b8fe3ee8091f4d625b36b6bc6/resource.tar.gz#test_sql2yql.test_blocks-as_tuple_/sql.yql" + } + ], "test_sql2yql.test[blocks-coalesce]": [ { "checksum": "d407fef7972254e7c63c8b55750b8d4a", @@ -1357,6 +1364,13 @@ "uri": "https://{canondata_backend}/1916746/15a617669fb2b654b6be1fc493d61b99fb1f6f81/resource.tar.gz#test_sql2yql.test_blocks-coalesce_scalar_/sql.yql" } ], + "test_sql2yql.test[blocks-decimal_binop]": [ + { + "checksum": "ba0d166a10075020edf9630c95f688e0", + "size": 1541, + "uri": "https://{canondata_backend}/1900335/c447765ddbde200b8fe3ee8091f4d625b36b6bc6/resource.tar.gz#test_sql2yql.test_blocks-decimal_binop_/sql.yql" + } + ], "test_sql2yql.test[blocks-empty_as_table]": [ { "checksum": "bbda02d622e3f3c4f698ea1962d9f902", @@ -1378,6 +1392,13 @@ "uri": "https://{canondata_backend}/1130705/7bfa4c7bef8ecf6ffba8b79d210708dc39bab04e/resource.tar.gz#test_sql2yql.test_blocks-filter_/sql.yql" } ], + "test_sql2yql.test[blocks-frompg]": [ + { + "checksum": "0049311ab78f33b07832a7abf8792156", + "size": 1126, + "uri": "https://{canondata_backend}/1900335/c447765ddbde200b8fe3ee8091f4d625b36b6bc6/resource.tar.gz#test_sql2yql.test_blocks-frompg_/sql.yql" + } + ], "test_sql2yql.test[blocks-if]": [ { "checksum": "2b573fa3534410c28b1273b496c93231", @@ -1392,6 +1413,20 @@ "uri": "https://{canondata_backend}/1916746/15a617669fb2b654b6be1fc493d61b99fb1f6f81/resource.tar.gz#test_sql2yql.test_blocks-if_scalar_/sql.yql" } ], + "test_sql2yql.test[blocks-just]": [ + { + "checksum": "6e4505f61db05c3ede047edf22c68997", + "size": 1081, + "uri": "https://{canondata_backend}/1900335/c447765ddbde200b8fe3ee8091f4d625b36b6bc6/resource.tar.gz#test_sql2yql.test_blocks-just_/sql.yql" + } + ], + "test_sql2yql.test[blocks-member]": [ + { + "checksum": "cf5cf545239509b0ec384396da6b77c0", + "size": 1197, + "uri": "https://{canondata_backend}/1900335/c447765ddbde200b8fe3ee8091f4d625b36b6bc6/resource.tar.gz#test_sql2yql.test_blocks-member_/sql.yql" + } + ], "test_sql2yql.test[blocks-not]": [ { "checksum": "60ffa8348b787bc96b78945aa79545d2", @@ -1406,6 +1441,13 @@ "uri": "https://{canondata_backend}/1916746/15a617669fb2b654b6be1fc493d61b99fb1f6f81/resource.tar.gz#test_sql2yql.test_blocks-not_opt_/sql.yql" } ], + "test_sql2yql.test[blocks-nth]": [ + { + "checksum": "4bce64c6fd9a1faed86401b27bc05ecb", + "size": 1175, + "uri": "https://{canondata_backend}/1900335/c447765ddbde200b8fe3ee8091f4d625b36b6bc6/resource.tar.gz#test_sql2yql.test_blocks-nth_/sql.yql" + } + ], "test_sql2yql.test[blocks-or]": [ { "checksum": "1e7e91aab2807a411a6a8b24ff2db384", @@ -1434,6 +1476,20 @@ "uri": "https://{canondata_backend}/1916746/15a617669fb2b654b6be1fc493d61b99fb1f6f81/resource.tar.gz#test_sql2yql.test_blocks-or_scalar_/sql.yql" } ], + "test_sql2yql.test[blocks-pg_call]": [ + { + "checksum": "32062809d5a13c1cbf4288a824da4a7f", + "size": 1680, + "uri": "https://{canondata_backend}/1900335/c447765ddbde200b8fe3ee8091f4d625b36b6bc6/resource.tar.gz#test_sql2yql.test_blocks-pg_call_/sql.yql" + } + ], + "test_sql2yql.test[blocks-pg_op]": [ + { + "checksum": "6fde398b5c3fed0e3fa982b753fe6018", + "size": 1658, + "uri": "https://{canondata_backend}/1900335/c447765ddbde200b8fe3ee8091f4d625b36b6bc6/resource.tar.gz#test_sql2yql.test_blocks-pg_op_/sql.yql" + } + ], "test_sql2yql.test[blocks-projection_add_ints]": [ { "checksum": "f84fc85138d4f1d4b72cf4d37a2b8ff2", @@ -1448,6 +1504,13 @@ "uri": "https://{canondata_backend}/1130705/7bfa4c7bef8ecf6ffba8b79d210708dc39bab04e/resource.tar.gz#test_sql2yql.test_blocks-projection_add_ints_filter_/sql.yql" } ], + "test_sql2yql.test[blocks-topg]": [ + { + "checksum": "0f669eec4c7c11d663db474f09902bd4", + "size": 1107, + "uri": "https://{canondata_backend}/1900335/c447765ddbde200b8fe3ee8091f4d625b36b6bc6/resource.tar.gz#test_sql2yql.test_blocks-topg_/sql.yql" + } + ], "test_sql2yql.test[blocks-xor]": [ { "checksum": "879bea5e15aa53dc4f3444f0fa17de49", @@ -8029,6 +8092,11 @@ "uri": "file://test_sql_format.test_blocks-and_scalar_/formatted.sql" } ], + "test_sql_format.test[blocks-as_tuple]": [ + { + "uri": "file://test_sql_format.test_blocks-as_tuple_/formatted.sql" + } + ], "test_sql_format.test[blocks-coalesce]": [ { "uri": "file://test_sql_format.test_blocks-coalesce_/formatted.sql" @@ -8039,6 +8107,11 @@ "uri": "file://test_sql_format.test_blocks-coalesce_scalar_/formatted.sql" } ], + "test_sql_format.test[blocks-decimal_binop]": [ + { + "uri": "file://test_sql_format.test_blocks-decimal_binop_/formatted.sql" + } + ], "test_sql_format.test[blocks-empty_as_table]": [ { "uri": "file://test_sql_format.test_blocks-empty_as_table_/formatted.sql" @@ -8054,6 +8127,11 @@ "uri": "file://test_sql_format.test_blocks-filter_/formatted.sql" } ], + "test_sql_format.test[blocks-frompg]": [ + { + "uri": "file://test_sql_format.test_blocks-frompg_/formatted.sql" + } + ], "test_sql_format.test[blocks-if]": [ { "uri": "file://test_sql_format.test_blocks-if_/formatted.sql" @@ -8064,6 +8142,16 @@ "uri": "file://test_sql_format.test_blocks-if_scalar_/formatted.sql" } ], + "test_sql_format.test[blocks-just]": [ + { + "uri": "file://test_sql_format.test_blocks-just_/formatted.sql" + } + ], + "test_sql_format.test[blocks-member]": [ + { + "uri": "file://test_sql_format.test_blocks-member_/formatted.sql" + } + ], "test_sql_format.test[blocks-not]": [ { "uri": "file://test_sql_format.test_blocks-not_/formatted.sql" @@ -8074,6 +8162,11 @@ "uri": "file://test_sql_format.test_blocks-not_opt_/formatted.sql" } ], + "test_sql_format.test[blocks-nth]": [ + { + "uri": "file://test_sql_format.test_blocks-nth_/formatted.sql" + } + ], "test_sql_format.test[blocks-or]": [ { "uri": "file://test_sql_format.test_blocks-or_/formatted.sql" @@ -8094,6 +8187,16 @@ "uri": "file://test_sql_format.test_blocks-or_scalar_/formatted.sql" } ], + "test_sql_format.test[blocks-pg_call]": [ + { + "uri": "file://test_sql_format.test_blocks-pg_call_/formatted.sql" + } + ], + "test_sql_format.test[blocks-pg_op]": [ + { + "uri": "file://test_sql_format.test_blocks-pg_op_/formatted.sql" + } + ], "test_sql_format.test[blocks-projection_add_ints]": [ { "uri": "file://test_sql_format.test_blocks-projection_add_ints_/formatted.sql" @@ -8104,6 +8207,11 @@ "uri": "file://test_sql_format.test_blocks-projection_add_ints_filter_/formatted.sql" } ], + "test_sql_format.test[blocks-topg]": [ + { + "uri": "file://test_sql_format.test_blocks-topg_/formatted.sql" + } + ], "test_sql_format.test[blocks-xor]": [ { "uri": "file://test_sql_format.test_blocks-xor_/formatted.sql" diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_blocks-as_tuple_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_blocks-as_tuple_/formatted.sql new file mode 100644 index 0000000000..9ece992b4f --- /dev/null +++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_blocks-as_tuple_/formatted.sql @@ -0,0 +1,8 @@ +$data = [<|x: 1, y: 'foo'|>]; + +SELECT + (x, 1), + (x, y) +FROM + as_table($data) +; diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_blocks-decimal_binop_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_blocks-decimal_binop_/formatted.sql new file mode 100644 index 0000000000..6cb72a57b8 --- /dev/null +++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_blocks-decimal_binop_/formatted.sql @@ -0,0 +1,11 @@ +$data = [ + <|x: Decimal('3', 5, 1), y: Decimal('2', 5, 1)|>, +]; + +SELECT + x * y, + x / y, + x % y +FROM + as_table($data) +; diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_blocks-frompg_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_blocks-frompg_/formatted.sql new file mode 100644 index 0000000000..41d7bad3b4 --- /dev/null +++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_blocks-frompg_/formatted.sql @@ -0,0 +1,7 @@ +$data = [<|x: 1p|>, <|x: NULL|>]; + +SELECT + FromPg(x) +FROM + as_table($data) +; diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_blocks-just_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_blocks-just_/formatted.sql new file mode 100644 index 0000000000..223604cae2 --- /dev/null +++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_blocks-just_/formatted.sql @@ -0,0 +1,7 @@ +$data = [<|x: 1|>]; + +SELECT + just(x) +FROM + as_table($data) +; diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_blocks-member_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_blocks-member_/formatted.sql new file mode 100644 index 0000000000..ff82e55e50 --- /dev/null +++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_blocks-member_/formatted.sql @@ -0,0 +1,7 @@ +$data = [<|x: <|a: 'foo'|>|>, <|x: <|a: NULL|>|>]; + +SELECT + x.a +FROM + as_table($data) +; diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_blocks-nth_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_blocks-nth_/formatted.sql new file mode 100644 index 0000000000..33104f38cd --- /dev/null +++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_blocks-nth_/formatted.sql @@ -0,0 +1,7 @@ +$data = [<|x: (1, 'foo')|>, <|x: NULL|>]; + +SELECT + x.1 +FROM + as_table($data) +; diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_blocks-pg_call_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_blocks-pg_call_/formatted.sql new file mode 100644 index 0000000000..4cb1104020 --- /dev/null +++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_blocks-pg_call_/formatted.sql @@ -0,0 +1,13 @@ +$data = [ + <|x: 0p, y: 0p|>, + <|x: 0p, y: 1p|>, + <|x: 1p, y: 0p|>, + <|x: 1p, y: 1p|>, +]; + +SELECT + PgCall('int4pl', x, y), + PgCall('int4pl', x, 1p) +FROM + as_table($data) +; diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_blocks-pg_op_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_blocks-pg_op_/formatted.sql new file mode 100644 index 0000000000..6bca912f57 --- /dev/null +++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_blocks-pg_op_/formatted.sql @@ -0,0 +1,13 @@ +$data = [ + <|x: 0p, y: 0p|>, + <|x: 0p, y: 1p|>, + <|x: 1p, y: 0p|>, + <|x: 1p, y: 1p|>, +]; + +SELECT + PgOp('=', x, y), + PgOp('=', x, 1p) +FROM + as_table($data) +; diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_blocks-topg_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_blocks-topg_/formatted.sql new file mode 100644 index 0000000000..d8819bc415 --- /dev/null +++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_blocks-topg_/formatted.sql @@ -0,0 +1,7 @@ +$data = [<|x: 1|>, <|x: NULL|>]; + +SELECT + ToPg(x) +FROM + as_table($data) +; diff --git a/yql/essentials/tests/sql/suites/blocks/as_tuple.sql b/yql/essentials/tests/sql/suites/blocks/as_tuple.sql new file mode 100644 index 0000000000..d585acd643 --- /dev/null +++ b/yql/essentials/tests/sql/suites/blocks/as_tuple.sql @@ -0,0 +1,3 @@ +$data = [<|x:1,y:'foo'|>]; + +select (x,1),(x,y) from as_table($data); diff --git a/yql/essentials/tests/sql/suites/blocks/decimal_binop.sql b/yql/essentials/tests/sql/suites/blocks/decimal_binop.sql new file mode 100644 index 0000000000..3a4d1e587c --- /dev/null +++ b/yql/essentials/tests/sql/suites/blocks/decimal_binop.sql @@ -0,0 +1,5 @@ +$data = [ + <|x:Decimal('3',5,1),y:Decimal('2',5,1)|>, +]; + +select x * y, x / y, x % y from as_table($data); diff --git a/yql/essentials/tests/sql/suites/blocks/frompg.sql b/yql/essentials/tests/sql/suites/blocks/frompg.sql new file mode 100644 index 0000000000..24073f8214 --- /dev/null +++ b/yql/essentials/tests/sql/suites/blocks/frompg.sql @@ -0,0 +1,3 @@ +$data = [<|x:1p|>,<|x:null|>]; + +select FromPg(x) from as_table($data); diff --git a/yql/essentials/tests/sql/suites/blocks/just.sql b/yql/essentials/tests/sql/suites/blocks/just.sql new file mode 100644 index 0000000000..6ebc57f7d8 --- /dev/null +++ b/yql/essentials/tests/sql/suites/blocks/just.sql @@ -0,0 +1,3 @@ +$data = [<|x:1|>]; + +select just(x) from as_table($data); diff --git a/yql/essentials/tests/sql/suites/blocks/member.sql b/yql/essentials/tests/sql/suites/blocks/member.sql new file mode 100644 index 0000000000..d9599c9776 --- /dev/null +++ b/yql/essentials/tests/sql/suites/blocks/member.sql @@ -0,0 +1,3 @@ +$data = [<|x:<|a:'foo'|>|>,<|x:<|a:null|>|>]; + +select x.a from as_table($data); diff --git a/yql/essentials/tests/sql/suites/blocks/nth.sql b/yql/essentials/tests/sql/suites/blocks/nth.sql new file mode 100644 index 0000000000..46e7354750 --- /dev/null +++ b/yql/essentials/tests/sql/suites/blocks/nth.sql @@ -0,0 +1,3 @@ +$data = [<|x:(1,'foo')|>,<|x:null|>]; + +select x.1 from as_table($data); diff --git a/yql/essentials/tests/sql/suites/blocks/pg_call.sql b/yql/essentials/tests/sql/suites/blocks/pg_call.sql new file mode 100644 index 0000000000..1964e6cd7c --- /dev/null +++ b/yql/essentials/tests/sql/suites/blocks/pg_call.sql @@ -0,0 +1,8 @@ +$data = [ + <|x:0p,y:0p|>, + <|x:0p,y:1p|>, + <|x:1p,y:0p|>, + <|x:1p,y:1p|>, +]; + +select PgCall("int4pl",x,y),PgCall("int4pl",x,1p) from as_table($data); diff --git a/yql/essentials/tests/sql/suites/blocks/pg_op.sql b/yql/essentials/tests/sql/suites/blocks/pg_op.sql new file mode 100644 index 0000000000..a7a7851302 --- /dev/null +++ b/yql/essentials/tests/sql/suites/blocks/pg_op.sql @@ -0,0 +1,8 @@ +$data = [ + <|x:0p,y:0p|>, + <|x:0p,y:1p|>, + <|x:1p,y:0p|>, + <|x:1p,y:1p|>, +]; + +select PgOp("=",x,y),PgOp("=",x,1p) from as_table($data); diff --git a/yql/essentials/tests/sql/suites/blocks/topg.sql b/yql/essentials/tests/sql/suites/blocks/topg.sql new file mode 100644 index 0000000000..2907990604 --- /dev/null +++ b/yql/essentials/tests/sql/suites/blocks/topg.sql @@ -0,0 +1,3 @@ +$data = [<|x:1|>,<|x:null|>]; + +select ToPg(x) from as_table($data); diff --git a/yql/essentials/tools/arrow_kernels_dump/ya.make b/yql/essentials/tools/arrow_kernels_dump/ya.make index d33cfbd5b9..95b00f95b2 100644 --- a/yql/essentials/tools/arrow_kernels_dump/ya.make +++ b/yql/essentials/tools/arrow_kernels_dump/ya.make @@ -12,7 +12,7 @@ ENDIF() PEERDIR( yql/essentials/minikql/invoke_builtins - yql/essentials/minikql/invoke_builtins/llvm14 + yql/essentials/minikql/invoke_builtins/llvm16 yql/essentials/public/udf/service/terminate_policy yql/essentials/public/udf yql/essentials/parser/pg_wrapper diff --git a/yql/essentials/tools/minirun/ya.make b/yql/essentials/tools/minirun/ya.make index 951ef4b414..8687e908a5 100644 --- a/yql/essentials/tools/minirun/ya.make +++ b/yql/essentials/tools/minirun/ya.make @@ -18,8 +18,8 @@ PEERDIR( yql/essentials/tools/yql_facade_run yql/essentials/providers/pure yql/essentials/providers/common/provider - yql/essentials/minikql/invoke_builtins/llvm14 - yql/essentials/minikql/comp_nodes/llvm14 + yql/essentials/minikql/invoke_builtins/llvm16 + yql/essentials/minikql/comp_nodes/llvm16 yql/essentials/parser/pg_wrapper yql/essentials/public/udf/service/terminate_policy yql/essentials/sql/pg diff --git a/yql/essentials/tools/pgrun/ya.make b/yql/essentials/tools/pgrun/ya.make index 75b63c79e0..420679420d 100644 --- a/yql/essentials/tools/pgrun/ya.make +++ b/yql/essentials/tools/pgrun/ya.make @@ -22,7 +22,7 @@ PEERDIR( yql/essentials/core/file_storage/proto yql/essentials/core/file_storage/http_download yql/essentials/core/services/mounts - yql/essentials/minikql/comp_nodes/llvm14 + yql/essentials/minikql/comp_nodes/llvm16 yql/essentials/protos yql/essentials/public/udf/service/exception_policy yql/essentials/utils/backtrace @@ -37,7 +37,7 @@ PEERDIR( yt/yql/providers/yt/common yt/yql/providers/yt/lib/schema yt/yql/providers/yt/gateway/file - yt/yql/providers/yt/comp_nodes/llvm14 + yt/yql/providers/yt/comp_nodes/llvm16 yt/yql/providers/yt/codec/codegen yql/essentials/providers/pg/provider yql/essentials/core/url_preprocessing diff --git a/yt/yt/core/misc/async_slru_cache-inl.h b/yt/yt/core/misc/async_slru_cache-inl.h index 3d10650bf6..5520cbfb2b 100644 --- a/yt/yt/core/misc/async_slru_cache-inl.h +++ b/yt/yt/core/misc/async_slru_cache-inl.h @@ -203,10 +203,28 @@ const TKey& TAsyncCacheValueBase<TKey, TValue, THash>::GetKey() const } template <class TKey, class TValue, class THash> +TIntrusivePtr<typename TAsyncCacheValueBase<TKey, TValue, THash>::TCache> TAsyncCacheValueBase<TKey, TValue, THash>::TryGetCache() const +{ + return Cache_.Load().Lock(); +} + +template <class TKey, class TValue, class THash> +void TAsyncCacheValueBase<TKey, TValue, THash>::SetCache(TWeakPtr<TCache> cache) +{ + Cache_.Store(cache); +} + +template <class TKey, class TValue, class THash> +void TAsyncCacheValueBase<TKey, TValue, THash>::ResetCache() +{ + Cache_.Store(nullptr); +} + +template <class TKey, class TValue, class THash> void TAsyncCacheValueBase<TKey, TValue, THash>::UpdateWeight() const { - if (auto cache = Cache_.Lock()) { - cache->UpdateWeight(GetKey()); + if (auto cache = TryGetCache()) { + cache->UpdateWeight(Key_); } } @@ -218,7 +236,7 @@ TAsyncCacheValueBase<TKey, TValue, THash>::TAsyncCacheValueBase(const TKey& key) template <class TKey, class TValue, class THash> NYT::TAsyncCacheValueBase<TKey, TValue, THash>::~TAsyncCacheValueBase() { - if (auto cache = Cache_.Lock()) { + if (auto cache = TryGetCache()) { cache->Unregister(Key_); } } @@ -448,7 +466,7 @@ void TAsyncSlruCacheBase<TKey, TValue, THash>::Touch(const TValuePtr& value) auto readerGuard = ReaderGuard(shard->SpinLock); - if (value->Cache_.Lock() != this || !value->Item_) { + if (value->TryGetCache() != this || !value->Item_) { return; } @@ -755,7 +773,7 @@ void TAsyncSlruCacheBase<TKey, TValue, THash>::EndInsert(const TInsertCookie& in shard->DrainTouchBuffer(); - value->Cache_ = MakeWeak(this); + value->SetCache(MakeWeak(this)); auto* item = GetOrCrash(shard->ItemMap, key); item->Value = value; @@ -841,11 +859,12 @@ void TAsyncSlruCacheBase<TKey, TValue, THash>::Unregister(const TKey& key) auto* shard = GetShardByKey(key); auto guard = WriterGuard(shard->SpinLock); - - shard->DrainTouchBuffer(); - - YT_VERIFY(shard->ItemMap.find(key) == shard->ItemMap.end()); - YT_VERIFY(shard->ValueMap.erase(key) == 1); + if (auto it = shard->ValueMap.find(key); + it != shard->ValueMap.end()) + { + shard->DrainTouchBuffer(); + shard->ValueMap.erase(it); + } } template <class TKey, class TValue, class THash> @@ -890,7 +909,7 @@ void TAsyncSlruCacheBase<TKey, TValue, THash>::DoTryRemove( } if (forbidResurrection || !IsResurrectionSupported()) { - valueIt->second->Cache_.Reset(); + valueIt->second->ResetCache(); valueMap.erase(valueIt); } @@ -915,9 +934,11 @@ void TAsyncSlruCacheBase<TKey, TValue, THash>::DoTryRemove( delete item; - guard.Release(); - OnRemoved(actualValue); + + // It is necessary to remove the guard before the actual value is destroyed. + // Otherwise, it will lead to a deadlock in unregister. + guard.Release(); } template <class TKey, class TValue, class THash> @@ -1323,11 +1344,11 @@ TAsyncSlruCacheBase<TKey, TValue, THash>::TShard::Trim(const TIntrusiveListWithA for (const auto& item : evictedItems) { auto value = item.Value; - YT_VERIFY(ItemMap.erase(value->GetKey()) == 1); + EraseOrCrash(ItemMap, value->GetKey()); if (!Parent->IsResurrectionSupported()) { - YT_VERIFY(ValueMap.erase(value->GetKey()) == 1); - value->Cache_.Reset(); + EraseOrCrash(ValueMap, value->GetKey()); + value->ResetCache(); } YT_VERIFY(value->Item_ == &item); diff --git a/yt/yt/core/misc/async_slru_cache.h b/yt/yt/core/misc/async_slru_cache.h index e4395185bb..b3f6a53b0a 100644 --- a/yt/yt/core/misc/async_slru_cache.h +++ b/yt/yt/core/misc/async_slru_cache.h @@ -10,6 +10,7 @@ #include <yt/yt/library/profiling/sensor.h> +#include <library/cpp/yt/threading/atomic_object.h> #include <library/cpp/yt/threading/rw_spin_lock.h> #include <atomic> @@ -26,20 +27,26 @@ class TAsyncCacheValueBase : public virtual TRefCounted { public: + using TCache = TAsyncSlruCacheBase<TKey, TValue, THash>; + virtual ~TAsyncCacheValueBase(); const TKey& GetKey() const; void UpdateWeight() const; + TIntrusivePtr<TCache> TryGetCache() const; + void SetCache(TWeakPtr<TCache> cache); + void ResetCache(); + protected: explicit TAsyncCacheValueBase(const TKey& key); private: - using TCache = TAsyncSlruCacheBase<TKey, TValue, THash>; - friend class TAsyncSlruCacheBase<TKey, TValue, THash>; + friend TCache; + + NThreading::TAtomicObject<TWeakPtr<TCache>> Cache_; - TWeakPtr<TCache> Cache_; TKey Key_; typename TCache::TItem* Item_ = nullptr; }; @@ -225,6 +232,8 @@ protected: // If item weight ever changes, UpdateWeight() should be called to apply the changes. virtual i64 GetWeight(const TValuePtr& value) const; + // These methods are executed under the cache write lock. + // Therefore, these methods should not perform heavy operations. virtual void OnAdded(const TValuePtr& value); virtual void OnRemoved(const TValuePtr& value); diff --git a/yt/yt/core/misc/unittests/async_slru_cache_ut.cpp b/yt/yt/core/misc/unittests/async_slru_cache_ut.cpp index b2d72822db..15a7924030 100644 --- a/yt/yt/core/misc/unittests/async_slru_cache_ut.cpp +++ b/yt/yt/core/misc/unittests/async_slru_cache_ut.cpp @@ -4,6 +4,8 @@ #include <yt/yt/core/concurrency/fair_share_action_queue.h> #include <yt/yt/core/concurrency/thread_pool.h> +#include <yt/yt/core/actions/new_with_offloaded_dtor.h> + #include <yt/yt/core/misc/async_slru_cache.h> #include <yt/yt/core/misc/property.h> @@ -1055,17 +1057,31 @@ DEFINE_ENUM(EStressOperation, ((UpdateCookieWeight) (10)) ); +struct TAsyncSlruCacheStressTestParams +{ + bool EnableResurrection = false; + bool Sync = true; + int ThreadCount = 1; + int CacheSize = 100; +}; + class TAsyncSlruCacheStressTest - : public ::testing::TestWithParam<bool> + : public ::testing::TestWithParam<TAsyncSlruCacheStressTestParams> { }; TEST_P(TAsyncSlruCacheStressTest, Stress) { - constexpr int cacheSize = 100; - constexpr int stepCount = 1'000'000; + constexpr int stepCount = 100'000; + constexpr int batchCount = 10'000; constexpr double forbidResurrectionProbability = 0.25; - const bool enableResurrection = GetParam(); + const auto params = GetParam(); + const int cacheSize = params.CacheSize; + const bool enableResurrection = params.EnableResurrection; + const bool sync = params.Sync; + const int threadCount = params.ThreadCount; + + auto threadPool = NConcurrency::CreateThreadPool(threadCount, "StressTest"); auto config = CreateCacheConfig(cacheSize); auto cache = New<TCountingSlruCache>(std::move(config), enableResurrection); @@ -1082,10 +1098,12 @@ TEST_P(TAsyncSlruCacheStressTest, Stress) std::uniform_int_distribution<int> weightDistribution(1, 10); std::uniform_int_distribution<int> cookieWeightDistribution(0, 10); - std::uniform_int_distribution<int> keyDistribution(1, 20); - std::uniform_int_distribution<int> capacityDistribution(50, 150); + std::uniform_int_distribution<int> keyDistribution(1, cacheSize); + std::uniform_int_distribution<int> capacityDistribution(static_cast<int>(cacheSize * 0.5), static_cast<int>(cacheSize * 1.5)); std::uniform_real_distribution<double> youngerSizeFractionDistribution(0.0, 1.0); + NThreading::TSpinLock lock; + std::vector<TCountingSlruCache::TInsertCookie> activeInsertCookies; // Pointers to all the values that are either present in cache now or were in cache @@ -1102,6 +1120,7 @@ TEST_P(TAsyncSlruCacheStressTest, Stress) THashMap<int, TWeakPtr<TSimpleCachedValue>> lastInsertedValues; auto pickCacheValue = [&] () -> TSimpleCachedValuePtr { + auto guard = Guard(lock); while (!cacheValues.empty()) { size_t cacheValueIndex = randomGenerator() % cacheValues.size(); std::swap(cacheValues[cacheValueIndex], cacheValues.back()); @@ -1114,13 +1133,12 @@ TEST_P(TAsyncSlruCacheStressTest, Stress) return nullptr; }; - for (int step = 0; step < stepCount; ++step) { - auto operation = operations[randomGenerator() % operations.size()]; - + auto runAction = [&] (const EStressOperation operation, const int step) -> void{ switch (operation) { case EStressOperation::Find: { - auto value = cache->Find(keyDistribution(randomGenerator)); - if (value) { + auto key = keyDistribution(randomGenerator); + auto value = cache->Find(key); + if (value && sync) { ASSERT_EQ(lastInsertedValues[value->GetKey()].Lock(), value); } break; @@ -1128,9 +1146,10 @@ TEST_P(TAsyncSlruCacheStressTest, Stress) case EStressOperation::Lookup: { auto key = keyDistribution(randomGenerator); auto valueFuture = cache->Lookup(key); - if (!valueFuture) { + if (!valueFuture || !sync) { break; } + if (valueFuture.IsSet()) { ASSERT_TRUE(valueFuture.Get().IsOK()); const auto& value = valueFuture.Get().Value(); @@ -1154,10 +1173,12 @@ TEST_P(TAsyncSlruCacheStressTest, Stress) int key = keyDistribution(randomGenerator); auto cookieWeight = cookieWeightDistribution(randomGenerator); auto cookie = cache->BeginInsert(key, cookieWeight); + + auto guard = Guard(lock); if (cookie.IsActive()) { activeInsertCookies.emplace_back(std::move(cookie)); lastInsertedValues[key] = nullptr; - } else { + } else if (sync) { auto valueFuture = cookie.GetValue(); ASSERT_TRUE(static_cast<bool>(valueFuture)); if (valueFuture.IsSet()) { @@ -1173,34 +1194,48 @@ TEST_P(TAsyncSlruCacheStressTest, Stress) break; } case EStressOperation::EndInsert: { - if (activeInsertCookies.empty()) { - break; - } - size_t cookieIndex = randomGenerator() % activeInsertCookies.size(); - std::swap(activeInsertCookies[cookieIndex], activeInsertCookies.back()); - auto value = New<TSimpleCachedValue>( - /*key*/ activeInsertCookies.back().GetKey(), - /*value*/ step, - /*weight*/ weightDistribution(randomGenerator)); - cacheValues.emplace_back(value); - if (enableResurrection) { - heldValues.push_back(value); + TCountingSlruCache::TInsertCookie cookie; + TSimpleCachedValuePtr value; + + { + auto guard = Guard(lock); + if (activeInsertCookies.empty()) { + break; + } + + size_t cookieIndex = randomGenerator() % activeInsertCookies.size(); + std::swap(activeInsertCookies[cookieIndex], activeInsertCookies.back()); + value = New<TSimpleCachedValue>( + /*key*/ activeInsertCookies.back().GetKey(), + /*value*/ step, + /*weight*/ weightDistribution(randomGenerator)); + cacheValues.emplace_back(value); + if (enableResurrection) { + heldValues.push_back(value); + } + lastInsertedValues[value->GetKey()] = value; + ASSERT_TRUE(!sync || activeInsertCookies.back().IsActive()); + cookie = std::move(activeInsertCookies[activeInsertCookies.size() - 1]); + activeInsertCookies.pop_back(); } - lastInsertedValues[value->GetKey()] = value; - ASSERT_TRUE(activeInsertCookies.back().IsActive()); - activeInsertCookies.back().EndInsert(std::move(value)); - activeInsertCookies.pop_back(); + cookie.EndInsert(std::move(value)); break; } case EStressOperation::CancelInsert: { - if (activeInsertCookies.empty()) { - break; + TCountingSlruCache::TInsertCookie cookie; + + { + auto guard = Guard(lock); + if (activeInsertCookies.empty()) { + break; + } + size_t cookieIndex = randomGenerator() % activeInsertCookies.size(); + std::swap(activeInsertCookies[cookieIndex], activeInsertCookies.back()); + ASSERT_TRUE(!sync || activeInsertCookies.back().IsActive()); + cookie = std::move(activeInsertCookies[activeInsertCookies.size() - 1]); + activeInsertCookies.pop_back(); } - size_t cookieIndex = randomGenerator() % activeInsertCookies.size(); - std::swap(activeInsertCookies[cookieIndex], activeInsertCookies.back()); - ASSERT_TRUE(activeInsertCookies.back().IsActive()); - activeInsertCookies.back().Cancel(TError("Cancelled")); - activeInsertCookies.pop_back(); + cookie.Cancel(TError("Cancelled")); break; } case EStressOperation::TryRemove: { @@ -1208,6 +1243,11 @@ TEST_P(TAsyncSlruCacheStressTest, Stress) bool forbidResurrection = distribution(randomGenerator); auto key = keyDistribution(randomGenerator); cache->TryRemove(key, forbidResurrection); + + if (!sync) { + break; + } + if (!enableResurrection || forbidResurrection) { lastInsertedValues[key] = nullptr; } @@ -1223,35 +1263,118 @@ TEST_P(TAsyncSlruCacheStressTest, Stress) break; } case EStressOperation::ReleaseValue: { + auto guard = Guard(lock); if (heldValues.empty()) { break; } size_t valueIndex = randomGenerator() % heldValues.size(); std::swap(heldValues[valueIndex], heldValues.back()); + heldValues.pop_back(); break; } case EStressOperation::Reconfigure: { auto config = New<TSlruCacheDynamicConfig>(); - config->Capacity = capacityDistribution(randomGenerator); + config->Capacity = std::max(0, capacityDistribution(randomGenerator)); config->YoungerSizeFraction = youngerSizeFractionDistribution(randomGenerator); cache->Reconfigure(std::move(config)); break; } case EStressOperation::UpdateCookieWeight: { - if (activeInsertCookies.empty()) { - break; + TCountingSlruCache::TInsertCookie cookie; + int cookieWeight; + { + auto guard = Guard(lock); + if (activeInsertCookies.empty()) { + break; + } + size_t cookieIndex = randomGenerator() % activeInsertCookies.size(); + std::swap(activeInsertCookies[cookieIndex], activeInsertCookies.back()); + cookieWeight = cookieWeightDistribution(randomGenerator); + activeInsertCookies.back().UpdateWeight(cookieWeight); } - size_t cookieIndex = randomGenerator() % activeInsertCookies.size(); - auto cookieWeight = cookieWeightDistribution(randomGenerator); - activeInsertCookies[cookieIndex].UpdateWeight(cookieWeight); break; } } + }; + + auto invoker = threadPool->GetInvoker(); + + std::vector<TFuture<void>> actions; + + auto syncActions = [&] { + NConcurrency::WaitFor(AllSucceeded(actions)) + .ThrowOnError(); + actions.clear(); + }; + + for (int step = 0; step < stepCount; ++step) { + auto operation = operations[randomGenerator() % operations.size()]; + if (sync) { + runAction(operation, step); + } else { + auto future = BIND(runAction) + .AsyncVia(invoker) + .Run(operation, step); + + actions.push_back(future); + + if (actions.size() > batchCount) { + syncActions(); + } + } } + + syncActions(); + threadPool->Shutdown(); } -INSTANTIATE_TEST_SUITE_P(Stress, TAsyncSlruCacheStressTest, ::testing::Values(false, true)); +INSTANTIATE_TEST_SUITE_P(Stress, TAsyncSlruCacheStressTest, ::testing::Values( + TAsyncSlruCacheStressTestParams{ + .EnableResurrection = false, + .Sync = true, + .ThreadCount = 1, + }, + TAsyncSlruCacheStressTestParams{ + .EnableResurrection = true, + .Sync = true, + .ThreadCount = 1, + }, + TAsyncSlruCacheStressTestParams{ + .EnableResurrection = false, + .Sync = false, + .ThreadCount = 16, + .CacheSize = 1, + }, + TAsyncSlruCacheStressTestParams{ + .EnableResurrection = true, + .Sync = false, + .ThreadCount = 16, + .CacheSize = 1, + }, + TAsyncSlruCacheStressTestParams{ + .EnableResurrection = false, + .Sync = false, + .ThreadCount = 16, + .CacheSize = 10, + }, + TAsyncSlruCacheStressTestParams{ + .EnableResurrection = true, + .Sync = false, + .ThreadCount = 16, + .CacheSize = 10, + }, + TAsyncSlruCacheStressTestParams{ + .EnableResurrection = false, + .Sync = false, + .ThreadCount = 16, + }, + TAsyncSlruCacheStressTestParams{ + .EnableResurrection = true, + .Sync = false, + .ThreadCount = 16, + } +)); //////////////////////////////////////////////////////////////////////////////// |