diff options
author | vvvv <vvvv@yandex-team.com> | 2025-01-20 16:38:10 +0300 |
---|---|---|
committer | vvvv <vvvv@yandex-team.com> | 2025-01-20 17:00:30 +0300 |
commit | 6b3e7777077b0112e088056abc7093db18d7bec0 (patch) | |
tree | dd79c706aadcb821936dfcf375a58b6adb2d00fd /yql/essentials/tests/sql/minirun | |
parent | b6a2451a8ed4ad6c210751fcf6ae3d28998235f1 (diff) | |
download | ydb-6b3e7777077b0112e088056abc7093db18d7bec0.tar.gz |
Introduced TableSource wrapper and Blocks/Peephole mode for minirun tests
init
commit_hash:22d9a4470f726b8efcd86aaf043bfa5552c2b35e
Diffstat (limited to 'yql/essentials/tests/sql/minirun')
15 files changed, 148 insertions, 56 deletions
diff --git a/yql/essentials/tests/sql/minirun/part0/test.py b/yql/essentials/tests/sql/minirun/part0/test.py index 9c41b3dbc7..ab149d4b47 100644 --- a/yql/essentials/tests/sql/minirun/part0/test.py +++ b/yql/essentials/tests/sql/minirun/part0/test.py @@ -1,16 +1,14 @@ -import pytest import yatest -from pure import run_test, DATA_PATH +from pure import run_test, DATA_PATH, mode_expander from test_utils import pytest_generate_tests_for_part from yql_utils import pytest_get_current_part def pytest_generate_tests(metafunc): current_part, part_count = pytest_get_current_part(yatest.common.source_path(__file__)) - return pytest_generate_tests_for_part(metafunc, current_part, part_count, data_path=DATA_PATH) + return pytest_generate_tests_for_part(metafunc, current_part, part_count, data_path=DATA_PATH, mode_expander=mode_expander) -@pytest.mark.parametrize('what', ['Results', 'Debug', 'RunOnOpt', 'LLVM']) -def test(suite, case, cfg, tmpdir, what, yql_http_file_server): +def test(suite, case, cfg, what, tmpdir, yql_http_file_server): return run_test(suite, case, cfg, tmpdir, what, yql_http_file_server) diff --git a/yql/essentials/tests/sql/minirun/part1/canondata/result.json b/yql/essentials/tests/sql/minirun/part1/canondata/result.json index 88857905bb..4a1ab738f6 100644 --- a/yql/essentials/tests/sql/minirun/part1/canondata/result.json +++ b/yql/essentials/tests/sql/minirun/part1/canondata/result.json @@ -209,6 +209,27 @@ "uri": "https://{canondata_backend}/1920236/fd3dfd41a8e63d7fa9cb2e8c3d8e6d8d5f51f2cb/resource.tar.gz#test.test_bigdate-tzstrliterals-default.txt-Results_/results.txt" } ], + "test.test[blocks-empty_as_table-default.txt-Debug]": [ + { + "checksum": "02505b6facd77cffd49cb50c6594bfb4", + "size": 193, + "uri": "https://{canondata_backend}/1881367/d69a84f4c03a928385a2fa1e80f00c900ecc2329/resource.tar.gz#test.test_blocks-empty_as_table-default.txt-Debug_/opt.yql" + } + ], + "test.test[blocks-empty_as_table-default.txt-Peephole]": [ + { + "checksum": "a25bc5529ca43bbc2156117a5ab492f0", + "size": 207, + "uri": "https://{canondata_backend}/1599023/13c5bc706a21d8feb94abd7c7eacc9cc0eadb23f/resource.tar.gz#test.test_blocks-empty_as_table-default.txt-Peephole_/opt.yql" + } + ], + "test.test[blocks-empty_as_table-default.txt-Results]": [ + { + "checksum": "d3e6d1b7aac397559c3118e30efbef9d", + "size": 251, + "uri": "https://{canondata_backend}/1881367/d69a84f4c03a928385a2fa1e80f00c900ecc2329/resource.tar.gz#test.test_blocks-empty_as_table-default.txt-Results_/results.txt" + } + ], "test.test[compute_range-tuples_compare-default.txt-Debug]": [ { "checksum": "501d11676cf33c61787adfa1afed5db6", diff --git a/yql/essentials/tests/sql/minirun/part1/test.py b/yql/essentials/tests/sql/minirun/part1/test.py index 9c41b3dbc7..ab149d4b47 100644 --- a/yql/essentials/tests/sql/minirun/part1/test.py +++ b/yql/essentials/tests/sql/minirun/part1/test.py @@ -1,16 +1,14 @@ -import pytest import yatest -from pure import run_test, DATA_PATH +from pure import run_test, DATA_PATH, mode_expander from test_utils import pytest_generate_tests_for_part from yql_utils import pytest_get_current_part def pytest_generate_tests(metafunc): current_part, part_count = pytest_get_current_part(yatest.common.source_path(__file__)) - return pytest_generate_tests_for_part(metafunc, current_part, part_count, data_path=DATA_PATH) + return pytest_generate_tests_for_part(metafunc, current_part, part_count, data_path=DATA_PATH, mode_expander=mode_expander) -@pytest.mark.parametrize('what', ['Results', 'Debug', 'RunOnOpt', 'LLVM']) -def test(suite, case, cfg, tmpdir, what, yql_http_file_server): +def test(suite, case, cfg, what, tmpdir, yql_http_file_server): return run_test(suite, case, cfg, tmpdir, what, yql_http_file_server) diff --git a/yql/essentials/tests/sql/minirun/part2/test.py b/yql/essentials/tests/sql/minirun/part2/test.py index 9c41b3dbc7..ab149d4b47 100644 --- a/yql/essentials/tests/sql/minirun/part2/test.py +++ b/yql/essentials/tests/sql/minirun/part2/test.py @@ -1,16 +1,14 @@ -import pytest import yatest -from pure import run_test, DATA_PATH +from pure import run_test, DATA_PATH, mode_expander from test_utils import pytest_generate_tests_for_part from yql_utils import pytest_get_current_part def pytest_generate_tests(metafunc): current_part, part_count = pytest_get_current_part(yatest.common.source_path(__file__)) - return pytest_generate_tests_for_part(metafunc, current_part, part_count, data_path=DATA_PATH) + return pytest_generate_tests_for_part(metafunc, current_part, part_count, data_path=DATA_PATH, mode_expander=mode_expander) -@pytest.mark.parametrize('what', ['Results', 'Debug', 'RunOnOpt', 'LLVM']) -def test(suite, case, cfg, tmpdir, what, yql_http_file_server): +def test(suite, case, cfg, what, tmpdir, yql_http_file_server): return run_test(suite, case, cfg, tmpdir, what, yql_http_file_server) diff --git a/yql/essentials/tests/sql/minirun/part3/test.py b/yql/essentials/tests/sql/minirun/part3/test.py index 9c41b3dbc7..ab149d4b47 100644 --- a/yql/essentials/tests/sql/minirun/part3/test.py +++ b/yql/essentials/tests/sql/minirun/part3/test.py @@ -1,16 +1,14 @@ -import pytest import yatest -from pure import run_test, DATA_PATH +from pure import run_test, DATA_PATH, mode_expander from test_utils import pytest_generate_tests_for_part from yql_utils import pytest_get_current_part def pytest_generate_tests(metafunc): current_part, part_count = pytest_get_current_part(yatest.common.source_path(__file__)) - return pytest_generate_tests_for_part(metafunc, current_part, part_count, data_path=DATA_PATH) + return pytest_generate_tests_for_part(metafunc, current_part, part_count, data_path=DATA_PATH, mode_expander=mode_expander) -@pytest.mark.parametrize('what', ['Results', 'Debug', 'RunOnOpt', 'LLVM']) -def test(suite, case, cfg, tmpdir, what, yql_http_file_server): +def test(suite, case, cfg, what, tmpdir, yql_http_file_server): return run_test(suite, case, cfg, tmpdir, what, yql_http_file_server) diff --git a/yql/essentials/tests/sql/minirun/part4/test.py b/yql/essentials/tests/sql/minirun/part4/test.py index 9c41b3dbc7..ab149d4b47 100644 --- a/yql/essentials/tests/sql/minirun/part4/test.py +++ b/yql/essentials/tests/sql/minirun/part4/test.py @@ -1,16 +1,14 @@ -import pytest import yatest -from pure import run_test, DATA_PATH +from pure import run_test, DATA_PATH, mode_expander from test_utils import pytest_generate_tests_for_part from yql_utils import pytest_get_current_part def pytest_generate_tests(metafunc): current_part, part_count = pytest_get_current_part(yatest.common.source_path(__file__)) - return pytest_generate_tests_for_part(metafunc, current_part, part_count, data_path=DATA_PATH) + return pytest_generate_tests_for_part(metafunc, current_part, part_count, data_path=DATA_PATH, mode_expander=mode_expander) -@pytest.mark.parametrize('what', ['Results', 'Debug', 'RunOnOpt', 'LLVM']) -def test(suite, case, cfg, tmpdir, what, yql_http_file_server): +def test(suite, case, cfg, what, tmpdir, yql_http_file_server): return run_test(suite, case, cfg, tmpdir, what, yql_http_file_server) diff --git a/yql/essentials/tests/sql/minirun/part5/canondata/result.json b/yql/essentials/tests/sql/minirun/part5/canondata/result.json index 3b952ed7ec..c69065e43b 100644 --- a/yql/essentials/tests/sql/minirun/part5/canondata/result.json +++ b/yql/essentials/tests/sql/minirun/part5/canondata/result.json @@ -310,6 +310,27 @@ "uri": "file://test.test_bitcast_implicit-mul_bitcast-default.txt-Results_/extracted" } ], + "test.test[blocks-filter-default.txt-Debug]": [ + { + "checksum": "3dea830ec45f78ed78603576b054b4ca", + "size": 219, + "uri": "https://{canondata_backend}/1937150/d87c401e99a94fad9fdc66f7c4a0d9c670b84949/resource.tar.gz#test.test_blocks-filter-default.txt-Debug_/opt.yql" + } + ], + "test.test[blocks-filter-default.txt-Peephole]": [ + { + "checksum": "727b35f5c4d7a0647482ffad550fd950", + "size": 710, + "uri": "https://{canondata_backend}/1942525/3a5c45563e24ce3cec2704463eb5e7a3038772ba/resource.tar.gz#test.test_blocks-filter-default.txt-Peephole_/opt.yql" + } + ], + "test.test[blocks-filter-default.txt-Results]": [ + { + "checksum": "8d6273d5f4540afa8d6303866158700a", + "size": 983, + "uri": "https://{canondata_backend}/1937150/d87c401e99a94fad9fdc66f7c4a0d9c670b84949/resource.tar.gz#test.test_blocks-filter-default.txt-Results_/results.txt" + } + ], "test.test[column_order-union_all_positional-default.txt-Debug]": [ { "checksum": "b136f637022838dde4e7cd35c37911eb", diff --git a/yql/essentials/tests/sql/minirun/part5/test.py b/yql/essentials/tests/sql/minirun/part5/test.py index 9c41b3dbc7..ab149d4b47 100644 --- a/yql/essentials/tests/sql/minirun/part5/test.py +++ b/yql/essentials/tests/sql/minirun/part5/test.py @@ -1,16 +1,14 @@ -import pytest import yatest -from pure import run_test, DATA_PATH +from pure import run_test, DATA_PATH, mode_expander from test_utils import pytest_generate_tests_for_part from yql_utils import pytest_get_current_part def pytest_generate_tests(metafunc): current_part, part_count = pytest_get_current_part(yatest.common.source_path(__file__)) - return pytest_generate_tests_for_part(metafunc, current_part, part_count, data_path=DATA_PATH) + return pytest_generate_tests_for_part(metafunc, current_part, part_count, data_path=DATA_PATH, mode_expander=mode_expander) -@pytest.mark.parametrize('what', ['Results', 'Debug', 'RunOnOpt', 'LLVM']) -def test(suite, case, cfg, tmpdir, what, yql_http_file_server): +def test(suite, case, cfg, what, tmpdir, yql_http_file_server): return run_test(suite, case, cfg, tmpdir, what, yql_http_file_server) diff --git a/yql/essentials/tests/sql/minirun/part6/canondata/result.json b/yql/essentials/tests/sql/minirun/part6/canondata/result.json index 28c8ad4e6e..659875a9f6 100644 --- a/yql/essentials/tests/sql/minirun/part6/canondata/result.json +++ b/yql/essentials/tests/sql/minirun/part6/canondata/result.json @@ -296,6 +296,27 @@ "uri": "file://test.test_bitcast_implicit-mod_bitcast-default.txt-Results_/extracted" } ], + "test.test[blocks-projection_add_ints_filter-default.txt-Debug]": [ + { + "checksum": "0dc7c149ab903b62717b7afe506ab893", + "size": 272, + "uri": "https://{canondata_backend}/1937492/b366687ca1f092d9cdddb9d6e0cc43bb7c82132d/resource.tar.gz#test.test_blocks-projection_add_ints_filter-default.txt-Debug_/opt.yql" + } + ], + "test.test[blocks-projection_add_ints_filter-default.txt-Peephole]": [ + { + "checksum": "f5fb44eaf3fafb816e7d772449908cd6", + "size": 851, + "uri": "https://{canondata_backend}/1784826/964c3d771c51aa808279a968bec9d2485cb85678/resource.tar.gz#test.test_blocks-projection_add_ints_filter-default.txt-Peephole_/opt.yql" + } + ], + "test.test[blocks-projection_add_ints_filter-default.txt-Results]": [ + { + "checksum": "9912317744ccaa40444b2da1c36a1460", + "size": 984, + "uri": "https://{canondata_backend}/1937492/b366687ca1f092d9cdddb9d6e0cc43bb7c82132d/resource.tar.gz#test.test_blocks-projection_add_ints_filter-default.txt-Results_/results.txt" + } + ], "test.test[coalesce-coalesce_symmetry-default.txt-Debug]": [ { "checksum": "569b2ea4c0fca80ab9bb30fafa9a1d47", diff --git a/yql/essentials/tests/sql/minirun/part6/test.py b/yql/essentials/tests/sql/minirun/part6/test.py index 9c41b3dbc7..ab149d4b47 100644 --- a/yql/essentials/tests/sql/minirun/part6/test.py +++ b/yql/essentials/tests/sql/minirun/part6/test.py @@ -1,16 +1,14 @@ -import pytest import yatest -from pure import run_test, DATA_PATH +from pure import run_test, DATA_PATH, mode_expander from test_utils import pytest_generate_tests_for_part from yql_utils import pytest_get_current_part def pytest_generate_tests(metafunc): current_part, part_count = pytest_get_current_part(yatest.common.source_path(__file__)) - return pytest_generate_tests_for_part(metafunc, current_part, part_count, data_path=DATA_PATH) + return pytest_generate_tests_for_part(metafunc, current_part, part_count, data_path=DATA_PATH, mode_expander=mode_expander) -@pytest.mark.parametrize('what', ['Results', 'Debug', 'RunOnOpt', 'LLVM']) -def test(suite, case, cfg, tmpdir, what, yql_http_file_server): +def test(suite, case, cfg, what, tmpdir, yql_http_file_server): return run_test(suite, case, cfg, tmpdir, what, yql_http_file_server) diff --git a/yql/essentials/tests/sql/minirun/part7/test.py b/yql/essentials/tests/sql/minirun/part7/test.py index 9c41b3dbc7..ab149d4b47 100644 --- a/yql/essentials/tests/sql/minirun/part7/test.py +++ b/yql/essentials/tests/sql/minirun/part7/test.py @@ -1,16 +1,14 @@ -import pytest import yatest -from pure import run_test, DATA_PATH +from pure import run_test, DATA_PATH, mode_expander from test_utils import pytest_generate_tests_for_part from yql_utils import pytest_get_current_part def pytest_generate_tests(metafunc): current_part, part_count = pytest_get_current_part(yatest.common.source_path(__file__)) - return pytest_generate_tests_for_part(metafunc, current_part, part_count, data_path=DATA_PATH) + return pytest_generate_tests_for_part(metafunc, current_part, part_count, data_path=DATA_PATH, mode_expander=mode_expander) -@pytest.mark.parametrize('what', ['Results', 'Debug', 'RunOnOpt', 'LLVM']) -def test(suite, case, cfg, tmpdir, what, yql_http_file_server): +def test(suite, case, cfg, what, tmpdir, yql_http_file_server): return run_test(suite, case, cfg, tmpdir, what, yql_http_file_server) diff --git a/yql/essentials/tests/sql/minirun/part8/test.py b/yql/essentials/tests/sql/minirun/part8/test.py index 9c41b3dbc7..ab149d4b47 100644 --- a/yql/essentials/tests/sql/minirun/part8/test.py +++ b/yql/essentials/tests/sql/minirun/part8/test.py @@ -1,16 +1,14 @@ -import pytest import yatest -from pure import run_test, DATA_PATH +from pure import run_test, DATA_PATH, mode_expander from test_utils import pytest_generate_tests_for_part from yql_utils import pytest_get_current_part def pytest_generate_tests(metafunc): current_part, part_count = pytest_get_current_part(yatest.common.source_path(__file__)) - return pytest_generate_tests_for_part(metafunc, current_part, part_count, data_path=DATA_PATH) + return pytest_generate_tests_for_part(metafunc, current_part, part_count, data_path=DATA_PATH, mode_expander=mode_expander) -@pytest.mark.parametrize('what', ['Results', 'Debug', 'RunOnOpt', 'LLVM']) -def test(suite, case, cfg, tmpdir, what, yql_http_file_server): +def test(suite, case, cfg, what, tmpdir, yql_http_file_server): return run_test(suite, case, cfg, tmpdir, what, yql_http_file_server) diff --git a/yql/essentials/tests/sql/minirun/part9/canondata/result.json b/yql/essentials/tests/sql/minirun/part9/canondata/result.json index 75d624df78..b6a64220da 100644 --- a/yql/essentials/tests/sql/minirun/part9/canondata/result.json +++ b/yql/essentials/tests/sql/minirun/part9/canondata/result.json @@ -167,6 +167,27 @@ "uri": "https://{canondata_backend}/1881367/0904eef9815dbdae5991888c9e683a7d755b2751/resource.tar.gz#test.test_bigdate-tzcasts-default.txt-Results_/results.txt" } ], + "test.test[blocks-projection_add_ints-default.txt-Debug]": [ + { + "checksum": "0dc7c149ab903b62717b7afe506ab893", + "size": 272, + "uri": "https://{canondata_backend}/1920236/40cada6092366a126340181e3c5021fcb85c3b83/resource.tar.gz#test.test_blocks-projection_add_ints-default.txt-Debug_/opt.yql" + } + ], + "test.test[blocks-projection_add_ints-default.txt-Peephole]": [ + { + "checksum": "7445b50d8d9dbd3e04e6307b079be6a2", + "size": 662, + "uri": "https://{canondata_backend}/1942525/d498eafe1e3c2e3c1a75f331ab6d923ed8992697/resource.tar.gz#test.test_blocks-projection_add_ints-default.txt-Peephole_/opt.yql" + } + ], + "test.test[blocks-projection_add_ints-default.txt-Results]": [ + { + "checksum": "9912317744ccaa40444b2da1c36a1460", + "size": 984, + "uri": "https://{canondata_backend}/1920236/40cada6092366a126340181e3c5021fcb85c3b83/resource.tar.gz#test.test_blocks-projection_add_ints-default.txt-Results_/results.txt" + } + ], "test.test[case-case_opt_cond-default.txt-Debug]": [ { "checksum": "4fd4460b8d2584006fe79cb1cb1e538e", diff --git a/yql/essentials/tests/sql/minirun/part9/test.py b/yql/essentials/tests/sql/minirun/part9/test.py index 9c41b3dbc7..ab149d4b47 100644 --- a/yql/essentials/tests/sql/minirun/part9/test.py +++ b/yql/essentials/tests/sql/minirun/part9/test.py @@ -1,16 +1,14 @@ -import pytest import yatest -from pure import run_test, DATA_PATH +from pure import run_test, DATA_PATH, mode_expander from test_utils import pytest_generate_tests_for_part from yql_utils import pytest_get_current_part def pytest_generate_tests(metafunc): current_part, part_count = pytest_get_current_part(yatest.common.source_path(__file__)) - return pytest_generate_tests_for_part(metafunc, current_part, part_count, data_path=DATA_PATH) + return pytest_generate_tests_for_part(metafunc, current_part, part_count, data_path=DATA_PATH, mode_expander=mode_expander) -@pytest.mark.parametrize('what', ['Results', 'Debug', 'RunOnOpt', 'LLVM']) -def test(suite, case, cfg, tmpdir, what, yql_http_file_server): +def test(suite, case, cfg, what, tmpdir, yql_http_file_server): return run_test(suite, case, cfg, tmpdir, what, yql_http_file_server) diff --git a/yql/essentials/tests/sql/minirun/pure.py b/yql/essentials/tests/sql/minirun/pure.py index ea5949f698..4c92d6fee5 100644 --- a/yql/essentials/tests/sql/minirun/pure.py +++ b/yql/essentials/tests/sql/minirun/pure.py @@ -20,8 +20,21 @@ ASTDIFF_PATH = yql_binary_path('yql/essentials/tools/astdiff/astdiff') MINIRUN_PATH = yql_binary_path('yql/essentials/tools/minirun/minirun') +def mode_expander(lst): + res = [] + for (suite, case, cfg) in lst: + res.append((suite, case, cfg, 'Results')) + res.append((suite, case, cfg, 'Debug')) + res.append((suite, case, cfg, 'RunOnOpt')) + res.append((suite, case, cfg, 'LLVM')) + if suite == 'blocks': + res.append((suite, case, cfg, 'Blocks')) + res.append((suite, case, cfg, 'Peephole')) + return res + + def run_test(suite, case, cfg, tmpdir, what, yql_http_file_server): - if get_gateway_cfg_suffix() != '' and what not in ('Results','LLVM'): + if get_gateway_cfg_suffix() != '' and what not in ('Results','LLVM','Blocks'): pytest.skip('non-trivial gateways.conf') config = get_config(suite, case, cfg, data_path = DATA_PATH) @@ -44,6 +57,19 @@ def run_test(suite, case, cfg, tmpdir, what, yql_http_file_server): assert xfail or os.path.exists(res.results_file) assert not tables_res + if what == 'Peephole': + canonize_peephole = is_canonize_peephole(config) + if not canonize_peephole: + canonize_peephole = re.search(r"canonize peephole", sql_query) + if not canonize_peephole: + pytest.skip('no peephole canonization requested') + + force_blocks = is_peephole_use_blocks(config) + (res, tables_res) = run_file_no_cache('pure', suite, case, cfg, config, yql_http_file_server, + force_blocks=force_blocks, extra_args=['--peephole'], + data_path=DATA_PATH, yqlrun_binary=MINIRUN_PATH) + return [yatest.common.canonical_file(res.opt_file, diff_tool=ASTDIFF_PATH)] + if what == 'Results': if xfail: return None @@ -59,26 +85,28 @@ def run_test(suite, case, cfg, tmpdir, what, yql_http_file_server): if what == 'Debug': to_canonize = [yatest.common.canonical_file(res.opt_file, diff_tool=ASTDIFF_PATH)] - if what == 'RunOnOpt' or what == 'LLVM': + if what == 'RunOnOpt' or what == 'LLVM' or what == 'Blocks': + is_on_opt = (what == 'RunOnOpt') is_llvm = (what == 'LLVM') + is_blocks = (what == 'Blocks') files = get_files(suite, config, DATA_PATH) http_files = get_http_files(suite, config, DATA_PATH) http_files_urls = yql_http_file_server.register_files({}, http_files) parameters = get_parameters_json(suite, config, DATA_PATH) - query_sql = get_sql_query('pure', suite, case, config, DATA_PATH) if is_llvm else None + query_sql = get_sql_query('pure', suite, case, config, DATA_PATH) if not is_on_opt else None yqlrun = YQLRun( prov='pure', keep_temp=False, - gateway_config=get_gateways_config(http_files, yql_http_file_server, allow_llvm=is_llvm), + gateway_config=get_gateways_config(http_files, yql_http_file_server, allow_llvm=is_llvm, force_blocks=is_blocks), udfs_dir=yql_binary_path('yql/essentials/tests/common/test_framework/udfs_deps'), binary=MINIRUN_PATH ) opt_res, opt_tables_res = execute( yqlrun, - program=res.opt if not is_llvm else query_sql, - run_sql=is_llvm, + program=res.opt if is_on_opt else query_sql, + run_sql=not is_on_opt, files=files, urls=http_files_urls, check_error=True, |