diff options
author | Alexander Smirnov <alex@ydb.tech> | 2024-12-25 19:03:15 +0000 |
---|---|---|
committer | Alexander Smirnov <alex@ydb.tech> | 2024-12-25 19:03:15 +0000 |
commit | c895fcc06300c50afa32558268c113bb8a5ee256 (patch) | |
tree | e26dba6120dc7d0fe4360f2ce3697ff7c63fee32 /yql/essentials | |
parent | e3d3da91e05265f579c3dc5a18faa83a926c3d39 (diff) | |
parent | 117eb6fe09e57cf15c95ea6236718f2f70a7105a (diff) | |
download | ydb-c895fcc06300c50afa32558268c113bb8a5ee256.tar.gz |
Merge pull request #12958 from ydb-platform/merge-libs-241224-2313
Diffstat (limited to 'yql/essentials')
49 files changed, 882 insertions, 44 deletions
diff --git a/yql/essentials/core/dq_integration/yql_dq_integration.h b/yql/essentials/core/dq_integration/yql_dq_integration.h index 1ac9427f65..bc61105a6b 100644 --- a/yql/essentials/core/dq_integration/yql_dq_integration.h +++ b/yql/essentials/core/dq_integration/yql_dq_integration.h @@ -4,6 +4,7 @@ #include <yql/essentials/core/yql_data_provider.h> #include <yql/essentials/core/yql_statistics.h> #include <yql/essentials/core/expr_nodes/yql_expr_nodes.h> +#include <yql/essentials/core/yql_type_annotation.h> #include <yql/essentials/public/issue/yql_issue.h> #include <library/cpp/yson/writer.h> @@ -81,7 +82,7 @@ public: virtual void FillSinkSettings(const TExprNode& node, ::google::protobuf::Any& settings, TString& sinkType) = 0; virtual void FillTransformSettings(const TExprNode& node, ::google::protobuf::Any& settings) = 0; virtual void Annotate(const TExprNode& node, THashMap<TString, TString>& params) = 0; - virtual bool PrepareFullResultTableParams(const TExprNode& root, TExprContext& ctx, THashMap<TString, TString>& params, THashMap<TString, TString>& secureParams) = 0; + virtual bool PrepareFullResultTableParams(const TExprNode& root, TExprContext& ctx, THashMap<TString, TString>& params, THashMap<TString, TString>& secureParams, const TMaybe<TColumnOrder>& = {}) = 0; virtual void WriteFullResultTableRef(NYson::TYsonWriter& writer, const TVector<TString>& columns, const THashMap<TString, TString>& graphParams) = 0; // Fill plan operator properties for sources/sinks diff --git a/yql/essentials/core/yql_type_annotation.h b/yql/essentials/core/yql_type_annotation.h index b508dc6778..ac0f0f36bd 100644 --- a/yql/essentials/core/yql_type_annotation.h +++ b/yql/essentials/core/yql_type_annotation.h @@ -212,6 +212,33 @@ public: const TOrderedItem& back() const { return Order_.back(); } + + TVector<TString> GetLogicalNames() const { + TVector<TString> res; + res.reserve(Order_.size()); + for (const auto &[name, _]: Order_) { + res.emplace_back(name); + } + return res; + } + + TVector<TString> GetPhysicalNames() const { + TVector<TString> res; + res.reserve(Order_.size()); + for (const auto &[_, name]: Order_) { + res.emplace_back(name); + } + return res; + } + + bool HasDuplicates() const { + for (const auto& e: Order_) { + if (e.PhysicalName != e.LogicalName) { + return true; + } + } + return false; + } private: THashMap<TString, TString> GeneratedToOriginal_; THashMap<TString, uint64_t> UseCount_; diff --git a/yql/essentials/providers/common/dq/yql_dq_integration_impl.cpp b/yql/essentials/providers/common/dq/yql_dq_integration_impl.cpp index 4782a79d57..098cc7a266 100644 --- a/yql/essentials/providers/common/dq/yql_dq_integration_impl.cpp +++ b/yql/essentials/providers/common/dq/yql_dq_integration_impl.cpp @@ -74,7 +74,7 @@ void TDqIntegrationBase::FillTransformSettings(const TExprNode&, ::google::proto void TDqIntegrationBase::Annotate(const TExprNode&, THashMap<TString, TString>&) { } -bool TDqIntegrationBase::PrepareFullResultTableParams(const TExprNode&, TExprContext&, THashMap<TString, TString>&, THashMap<TString, TString>&) { +bool TDqIntegrationBase::PrepareFullResultTableParams(const TExprNode&, TExprContext&, THashMap<TString, TString>&, THashMap<TString, TString>&, const TMaybe<TColumnOrder>&) { return false; } diff --git a/yql/essentials/providers/common/dq/yql_dq_integration_impl.h b/yql/essentials/providers/common/dq/yql_dq_integration_impl.h index 1fbcb33d77..d4e1597b85 100644 --- a/yql/essentials/providers/common/dq/yql_dq_integration_impl.h +++ b/yql/essentials/providers/common/dq/yql_dq_integration_impl.h @@ -23,7 +23,7 @@ public: void FillSinkSettings(const TExprNode& node, ::google::protobuf::Any& settings, TString& sinkType) override; void FillTransformSettings(const TExprNode& node, ::google::protobuf::Any& settings) override; void Annotate(const TExprNode& node, THashMap<TString, TString>& params) override; - bool PrepareFullResultTableParams(const TExprNode& root, TExprContext& ctx, THashMap<TString, TString>& params, THashMap<TString, TString>& secureParams) override; + bool PrepareFullResultTableParams(const TExprNode& root, TExprContext& ctx, THashMap<TString, TString>& params, THashMap<TString, TString>& secureParams, const TMaybe<TColumnOrder>& order = {}) override; void WriteFullResultTableRef(NYson::TYsonWriter& writer, const TVector<TString>& columns, const THashMap<TString, TString>& graphParams) override; bool FillSourcePlanProperties(const NNodes::TExprBase& node, TMap<TString, NJson::TJsonValue>& properties) override; bool FillSinkPlanProperties(const NNodes::TExprBase& node, TMap<TString, NJson::TJsonValue>& properties) override; diff --git a/yql/essentials/providers/common/schema/expr/yql_expr_schema.cpp b/yql/essentials/providers/common/schema/expr/yql_expr_schema.cpp index 58a138b764..1ac4c89927 100644 --- a/yql/essentials/providers/common/schema/expr/yql_expr_schema.cpp +++ b/yql/essentials/providers/common/schema/expr/yql_expr_schema.cpp @@ -46,7 +46,7 @@ class TExprTypeSaver: public TSaver<TExprTypeSaver<TSaver>> { struct TMappingOrderedStructAdaptor { TVector<std::pair<TStringBuf, const TTypeAnnotationNode*>> Members; - TMappingOrderedStructAdaptor(const TStructMemberMapper& mapper, const TMaybe<TColumnOrder>& columns, const TStructExprType* type) + TMappingOrderedStructAdaptor(const TStructMemberMapper& mapper, const TMaybe<TColumnOrder>& columns, const TStructExprType* type, bool writePhysical = true) { TMap<TStringBuf, const TTypeAnnotationNode*> members; for (auto& item: type->GetItems()) { @@ -61,7 +61,7 @@ class TExprTypeSaver: public TSaver<TExprTypeSaver<TSaver>> { for (auto& [column, gen_column] : *columns) { auto it = members.find(gen_column); if (it != members.end()) { - Members.emplace_back(column, it->second); + Members.emplace_back(writePhysical ? gen_column : column, it->second); } } } else { @@ -229,9 +229,9 @@ public: } } - void SaveStructType(const TStructExprType* type, const TMaybe<TColumnOrder>& columns, const TStructMemberMapper& mapper) { + void SaveStructType(const TStructExprType* type, const TMaybe<TColumnOrder>& columns, const TStructMemberMapper& mapper, bool physical = true) { if (mapper || columns) { - TBase::SaveStructType(TMappingOrderedStructAdaptor(mapper, columns, type)); + TBase::SaveStructType(TMappingOrderedStructAdaptor(mapper, columns, type, physical)); } else { Save(type); } @@ -240,7 +240,12 @@ public: void SaveStructTypeToYson(NYson::TYsonConsumerBase& writer, const TStructExprType* type, const TMaybe<TColumnOrder>& columns, const TStructMemberMapper& mapper, bool extendedForm) { TExprTypeSaver<TYqlTypeYsonSaverImpl> saver(writer, extendedForm); - saver.SaveStructType(type, columns, mapper); + saver.SaveStructType(type, columns, mapper, true); +} + +void SaveStructTypeToYsonWithLogicalNames(NYson::TYsonConsumerBase& writer, const TStructExprType* type, const TMaybe<TColumnOrder>& columns, const TStructMemberMapper& mapper, bool extendedForm) { + TExprTypeSaver<TYqlTypeYsonSaverImpl> saver(writer, extendedForm); + saver.SaveStructType(type, columns, mapper, false); } void WriteTypeToYson(NYson::TYsonConsumerBase& writer, const TTypeAnnotationNode* type, bool extendedForm) { @@ -315,8 +320,9 @@ struct TExprTypeLoader { } TMaybe<TType> LoadStructType(const TVector<std::pair<TString, TType>>& members, ui32 /*level*/) { TVector<const TItemExprType*> items; + TColumnOrder order; for (auto& member: members) { - items.push_back(Ctx.MakeType<TItemExprType>(member.first, member.second)); + items.push_back(Ctx.MakeType<TItemExprType>(order.AddColumn(member.first), member.second)); } auto ret = Ctx.MakeType<TStructExprType>(items); YQL_ENSURE(ret->Validate(TPosition(), Ctx)); @@ -430,7 +436,7 @@ void WriteResOrPullType(NYson::TYsonConsumerBase& writer, const TTypeAnnotationN writer.OnStringScalar("ListType"); writer.OnListItem(); - SaveStructTypeToYson(writer, type->Cast<TListExprType>()->GetItemType()->Cast<TStructExprType>(), columns, {}, true); + SaveStructTypeToYsonWithLogicalNames(writer, type->Cast<TListExprType>()->GetItemType()->Cast<TStructExprType>(), columns, {}, true); writer.OnEndList(); } diff --git a/yql/essentials/tests/common/test_framework/test_file_common.py b/yql/essentials/tests/common/test_framework/test_file_common.py index 3fc5cfe025..3ff92a8b8b 100644 --- a/yql/essentials/tests/common/test_framework/test_file_common.py +++ b/yql/essentials/tests/common/test_framework/test_file_common.py @@ -14,10 +14,10 @@ from yqlrun import YQLRun from test_utils import get_parameters_json, DATA_PATH, replace_vars -def get_gateways_config(http_files, yql_http_file_server, force_blocks=False, is_hybrid=False): +def get_gateways_config(http_files, yql_http_file_server, force_blocks=False, is_hybrid=False, allow_llvm=True): config = None - if http_files or force_blocks or is_hybrid: + if http_files or force_blocks or is_hybrid or not allow_llvm: config_message = gateways_config_pb2.TGatewaysConfig() if http_files: schema = config_message.Fs.CustomSchemes.add() @@ -34,6 +34,9 @@ def get_gateways_config(http_files, yql_http_file_server, force_blocks=False, is deactivate_dq = config_message.Dq.DefaultSettings.add() deactivate_dq.Name = "AnalyzeQuery" deactivate_dq.Value = "0" + if not allow_llvm: + flags = config_message.YqlCore.Flags.add() + flags.Name = 'LLVM_OFF' config = text_format.MessageToString(config_message) return config @@ -44,6 +47,9 @@ def is_hybrid(provider): def check_provider(provider, config): + if provider == 'pure': + return + if provider not in get_supported_providers(config): pytest.skip('%s provider is not supported here' % provider) @@ -78,7 +84,8 @@ def get_sql_query(provider, suite, case, config): return sql_query -def run_file_no_cache(provider, suite, case, cfg, config, yql_http_file_server, yqlrun_binary=None, extra_args=[], force_blocks=False): +def run_file_no_cache(provider, suite, case, cfg, config, yql_http_file_server, + yqlrun_binary=None, extra_args=[], force_blocks=False, allow_llvm=True): check_provider(provider, config) sql_query = get_sql_query(provider, suite, case, config) @@ -105,7 +112,7 @@ def run_file_no_cache(provider, suite, case, cfg, config, yql_http_file_server, prov=provider, keep_temp=not re.search(r"yt\.ReleaseTempData", sql_query), binary=yqlrun_binary, - gateway_config=get_gateways_config(http_files, yql_http_file_server, force_blocks=force_blocks, is_hybrid=is_hybrid(provider)), + gateway_config=get_gateways_config(http_files, yql_http_file_server, force_blocks=force_blocks, is_hybrid=is_hybrid(provider), allow_llvm=allow_llvm), extra_args=extra_args, udfs_dir=yql_binary_path('yql/essentials/tests/common/test_framework/udfs_deps') ) @@ -142,9 +149,12 @@ def run_file_no_cache(provider, suite, case, cfg, config, yql_http_file_server, return fixed_result, tables_res -def run_file(provider, suite, case, cfg, config, yql_http_file_server, yqlrun_binary=None, extra_args=[], force_blocks=False): +def run_file(provider, suite, case, cfg, config, yql_http_file_server, yqlrun_binary=None, + extra_args=[], force_blocks=False, allow_llvm=True): if (suite, case, cfg) not in run_file.cache: - run_file.cache[(suite, case, cfg)] = run_file_no_cache(provider, suite, case, cfg, config, yql_http_file_server, yqlrun_binary, extra_args, force_blocks=force_blocks) + run_file.cache[(suite, case, cfg)] = \ + run_file_no_cache(provider, suite, case, cfg, config, yql_http_file_server, + yqlrun_binary, extra_args, force_blocks=force_blocks, allow_llvm=allow_llvm) return run_file.cache[(suite, case, cfg)] diff --git a/yql/essentials/tests/common/test_framework/test_utils.py b/yql/essentials/tests/common/test_framework/test_utils.py index 353bcced5c..0c02633558 100644 --- a/yql/essentials/tests/common/test_framework/test_utils.py +++ b/yql/essentials/tests/common/test_framework/test_utils.py @@ -1,7 +1,9 @@ import json import os +import six import re import yatest.common +import zlib from yql_utils import get_param as yql_get_param from google.protobuf import text_format @@ -19,6 +21,12 @@ except BaseException: YQLRUN_PATH = None +def _make_hash(x): + if six.PY2: + return hash(x) + return zlib.crc32(repr(x).encode("utf-8")) + + def get_sql_flags(): gateway_config = gateways_config_pb2.TGatewaysConfig() @@ -86,9 +94,9 @@ def pytest_generate_tests_for_run(metafunc, template='.sql', suites=None, curren if os.path.exists(suite_dir + '/' + case + '.cfg'): configs.append('') for cfg in sorted(configs): - if hash((suite, case, cfg)) % partsCount == currentPart: + if _make_hash((suite, case, cfg)) % partsCount == currentPart: argvalues.append((suite, case, cfg)) - if not configs and hash((suite, case, 'default.txt')) % partsCount == currentPart: + if not configs and _make_hash((suite, case, 'default.txt')) % partsCount == currentPart: argvalues.append((suite, case, 'default.txt')) metafunc.parametrize( diff --git a/yql/essentials/tests/common/test_framework/yql_utils.py b/yql/essentials/tests/common/test_framework/yql_utils.py index 581de69275..84b46d8263 100644 --- a/yql/essentials/tests/common/test_framework/yql_utils.py +++ b/yql/essentials/tests/common/test_framework/yql_utils.py @@ -288,8 +288,7 @@ def normalize_yson(y): return [normalize_yson(i) for i in y] if isinstance(y, dict): return {normalize_yson(k): normalize_yson(v) for k, v in six.iteritems(y)} - s = str(y) if not isinstance(y, six.text_type) else y.encode('utf-8', errors='xmlcharrefreplace') - return s + return y volatile_attrs = {'DataSize', 'ModifyTime', 'Id', 'Revision'} @@ -301,9 +300,13 @@ def _replace_vals_impl(y): return [_replace_vals_impl(i) for i in y] if isinstance(y, dict): return {_replace_vals_impl(k): _replace_vals_impl(v) for k, v in six.iteritems(y) if k not in volatile_attrs} + if isinstance(y, bytes): + s = y.replace(b'tmp/yql/' + current_user.encode('ascii') + b'/', b'tmp/') + s = re.sub(b'tmp/[0-9a-f]+-[0-9a-f]+-[0-9a-f]+-[0-9a-f]+', b'tmp/<temp_table_guid>', s) + return s if isinstance(y, str): s = y.replace('tmp/yql/' + current_user + '/', 'tmp/') - s = re.sub(r'tmp/[0-9a-f]+-[0-9a-f]+-[0-9a-f]+-[0-9a-f]+', 'tmp/<temp_table_guid>', s) + s = re.sub('tmp/[0-9a-f]+-[0-9a-f]+-[0-9a-f]+-[0-9a-f]+', 'tmp/<temp_table_guid>', s) return s return y @@ -953,17 +956,17 @@ def pytest_get_current_part(path): def normalize_result(res, sort): - res = cyson.loads(res) if res else cyson.loads("[]") + res = cyson.loads(res) if res else cyson.loads(b"[]") res = replace_vals(res) for r in res: - for data in r['Write']: - if sort and 'Data' in data: - data['Data'] = sorted(data['Data']) - if 'Ref' in data: - data['Ref'] = [] - data['Truncated'] = True - if 'Data' in data and len(data['Data']) == 0: - del data['Data'] + for data in r[b'Write']: + if sort and b'Data' in data: + data[b'Data'] = sorted(data[b'Data']) + if b'Ref' in data: + data[b'Ref'] = [] + data[b'Truncated'] = True + if b'Data' in data and len(data[b'Data']) == 0: + del data[b'Data'] return res @@ -993,20 +996,20 @@ def stable_write(writer, node): def stable_result_file(res): path = res.results_file assert os.path.exists(path) - with open(path) as f: + with open(path, 'rb') as f: res = f.read() res = cyson.loads(res) res = replace_vals(res) for r in res: - for data in r['Write']: - if 'Unordered' in r and 'Data' in data: - data['Data'] = sorted(data['Data']) - with open(path, 'w') as f: + for data in r[b'Write']: + if b'Unordered' in r and b'Data' in data: + data[b'Data'] = sorted(data[b'Data']) + with open(path, 'wb') as f: writer = cyson.Writer(stream=cyson.OutputStream.from_file(f), format='pretty', mode='node') writer.begin_stream() stable_write(writer, res) writer.end_stream() - with open(path) as f: + with open(path, 'rb') as f: return f.read() @@ -1015,21 +1018,21 @@ def stable_table_file(table): assert os.path.exists(path) assert table.attr is not None is_sorted = False - for column in cyson.loads(table.attr)['schema']: - if 'sort_order' in column: + for column in cyson.loads(table.attr)[b'schema']: + if b'sort_order' in column: is_sorted = True break if not is_sorted: - with open(path) as f: + with open(path, 'rb') as f: r = cyson.Reader(cyson.InputStream.from_file(f), mode='list_fragment') lst = sorted(list(r.list_fragments())) - with open(path, 'w') as f: + with open(path, 'wb') as f: writer = cyson.Writer(stream=cyson.OutputStream.from_file(f), format='pretty', mode='list_fragment') writer.begin_stream() for r in lst: stable_write(writer, r) writer.end_stream() - with open(path) as f: + with open(path, 'rb') as f: return f.read() diff --git a/yql/essentials/tests/common/test_framework/yqlrun.py b/yql/essentials/tests/common/test_framework/yqlrun.py index e23b81c92f..2743449169 100644 --- a/yql/essentials/tests/common/test_framework/yqlrun.py +++ b/yql/essentials/tests/common/test_framework/yqlrun.py @@ -156,17 +156,19 @@ class YQLRun(object): '--err-file=%(err_file)s ' \ '--gateways=%(prov)s ' \ '--syntax-version=%(syntax_version)d ' \ - '--tmp-dir=%(res_dir)s ' \ '--gateways-cfg=%(gateways_cfg_file)s ' \ '--fs-cfg=%(fs_cfg_file)s ' % locals() + if prov != 'pure': + cmd += '--tmp-dir=%(res_dir)s ' % locals() + if self.udfs_path is not None: cmd += '--udfs-dir=%(udfs_dir)s ' % locals() if ansi_lexer: cmd += '--ansi-lexer ' - if self.keep_temp: + if self.keep_temp and prov != 'pure': cmd += '--keep-temp ' if self.extra_args: diff --git a/yql/essentials/tests/sql/minirun/part0/canondata/result.json b/yql/essentials/tests/sql/minirun/part0/canondata/result.json new file mode 100644 index 0000000000..8e84219a87 --- /dev/null +++ b/yql/essentials/tests/sql/minirun/part0/canondata/result.json @@ -0,0 +1,16 @@ +{ + "test.test[compute_range-multiply_limit-default.txt-Debug]": [ + { + "checksum": "0d6c60d2ca8e505c9c96b8e8c7ea2ad5", + "size": 2847, + "uri": "https://{canondata_backend}/1942525/da06b1ba0dff3dd631541e16611d49f4fa053a8f/resource.tar.gz#test.test_compute_range-multiply_limit-default.txt-Debug_/opt.yql" + } + ], + "test.test[compute_range-multiply_limit-default.txt-Results]": [ + { + "checksum": "f487c01561e15c4047d5d539cc579992", + "size": 89787, + "uri": "https://{canondata_backend}/1942525/da06b1ba0dff3dd631541e16611d49f4fa053a8f/resource.tar.gz#test.test_compute_range-multiply_limit-default.txt-Results_/results.txt" + } + ] +} diff --git a/yql/essentials/tests/sql/minirun/part0/test.py b/yql/essentials/tests/sql/minirun/part0/test.py new file mode 100644 index 0000000000..1cb2e7f049 --- /dev/null +++ b/yql/essentials/tests/sql/minirun/part0/test.py @@ -0,0 +1,16 @@ +import pytest +import yatest + +from pure import run_test +from test_utils import pytest_generate_tests_for_part +from yql_utils import pytest_get_current_part + + +def pytest_generate_tests(metafunc): + current_part, part_count = pytest_get_current_part(yatest.common.source_path(__file__)) + return pytest_generate_tests_for_part(metafunc, current_part, part_count) + + +@pytest.mark.parametrize('what', ['Results', 'Debug', 'RunOnOpt', 'LLVM']) +def test(suite, case, cfg, tmpdir, what, yql_http_file_server): + return run_test(suite, case, cfg, tmpdir, what, yql_http_file_server) diff --git a/yql/essentials/tests/sql/minirun/part0/ya.make b/yql/essentials/tests/sql/minirun/part0/ya.make new file mode 100644 index 0000000000..0d703e1e22 --- /dev/null +++ b/yql/essentials/tests/sql/minirun/part0/ya.make @@ -0,0 +1,3 @@ +INCLUDE(../pure.make) + + diff --git a/yql/essentials/tests/sql/minirun/part1/canondata/result.json b/yql/essentials/tests/sql/minirun/part1/canondata/result.json new file mode 100644 index 0000000000..4213c246fe --- /dev/null +++ b/yql/essentials/tests/sql/minirun/part1/canondata/result.json @@ -0,0 +1,16 @@ +{ + "test.test[compute_range-tuples_compare-default.txt-Debug]": [ + { + "checksum": "501d11676cf33c61787adfa1afed5db6", + "size": 4621, + "uri": "https://{canondata_backend}/1942525/bff084eac8b2408f572d038b6b62ea00950a26c0/resource.tar.gz#test.test_compute_range-tuples_compare-default.txt-Debug_/opt.yql" + } + ], + "test.test[compute_range-tuples_compare-default.txt-Results]": [ + { + "checksum": "02e3f84696b07af203d99b540392eb88", + "size": 80850, + "uri": "https://{canondata_backend}/1942525/bff084eac8b2408f572d038b6b62ea00950a26c0/resource.tar.gz#test.test_compute_range-tuples_compare-default.txt-Results_/results.txt" + } + ] +} diff --git a/yql/essentials/tests/sql/minirun/part1/test.py b/yql/essentials/tests/sql/minirun/part1/test.py new file mode 100644 index 0000000000..1cb2e7f049 --- /dev/null +++ b/yql/essentials/tests/sql/minirun/part1/test.py @@ -0,0 +1,16 @@ +import pytest +import yatest + +from pure import run_test +from test_utils import pytest_generate_tests_for_part +from yql_utils import pytest_get_current_part + + +def pytest_generate_tests(metafunc): + current_part, part_count = pytest_get_current_part(yatest.common.source_path(__file__)) + return pytest_generate_tests_for_part(metafunc, current_part, part_count) + + +@pytest.mark.parametrize('what', ['Results', 'Debug', 'RunOnOpt', 'LLVM']) +def test(suite, case, cfg, tmpdir, what, yql_http_file_server): + return run_test(suite, case, cfg, tmpdir, what, yql_http_file_server) diff --git a/yql/essentials/tests/sql/minirun/part1/ya.make b/yql/essentials/tests/sql/minirun/part1/ya.make new file mode 100644 index 0000000000..0d703e1e22 --- /dev/null +++ b/yql/essentials/tests/sql/minirun/part1/ya.make @@ -0,0 +1,3 @@ +INCLUDE(../pure.make) + + diff --git a/yql/essentials/tests/sql/minirun/part2/canondata/result.json b/yql/essentials/tests/sql/minirun/part2/canondata/result.json new file mode 100644 index 0000000000..f7cee622c6 --- /dev/null +++ b/yql/essentials/tests/sql/minirun/part2/canondata/result.json @@ -0,0 +1,44 @@ +{ + "test.test[compute_range-in_literal_nulls-default.txt-Debug]": [ + { + "checksum": "32fad62c2c315c6dad7fe390ba7f40a8", + "size": 1078, + "uri": "https://{canondata_backend}/1942525/befc6ebfd65b57985e99080238980e2a790b9c5a/resource.tar.gz#test.test_compute_range-in_literal_nulls-default.txt-Debug_/opt.yql" + } + ], + "test.test[compute_range-in_literal_nulls-default.txt-Results]": [ + { + "checksum": "defc43980c4e7de97a442d4e8414132e", + "size": 6382, + "uri": "https://{canondata_backend}/1942525/befc6ebfd65b57985e99080238980e2a790b9c5a/resource.tar.gz#test.test_compute_range-in_literal_nulls-default.txt-Results_/results.txt" + } + ], + "test.test[compute_range-multiply_limit_for_single_key-default.txt-Debug]": [ + { + "checksum": "cc44d2b47d1643ca70a240b0c16da1d8", + "size": 1102, + "uri": "https://{canondata_backend}/1942525/befc6ebfd65b57985e99080238980e2a790b9c5a/resource.tar.gz#test.test_compute_range-multiply_limit_for_single_key-default.txt-Debug_/opt.yql" + } + ], + "test.test[compute_range-multiply_limit_for_single_key-default.txt-Results]": [ + { + "checksum": "38505376a6ece0b63b7ac15d7c25fc2c", + "size": 4827, + "uri": "https://{canondata_backend}/1942525/befc6ebfd65b57985e99080238980e2a790b9c5a/resource.tar.gz#test.test_compute_range-multiply_limit_for_single_key-default.txt-Results_/results.txt" + } + ], + "test.test[compute_range-preserve_rest_predicates_order-default.txt-Debug]": [ + { + "checksum": "e8b4617fcbe09473295068e55f477a70", + "size": 1959, + "uri": "https://{canondata_backend}/1942525/befc6ebfd65b57985e99080238980e2a790b9c5a/resource.tar.gz#test.test_compute_range-preserve_rest_predicates_order-default.txt-Debug_/opt.yql" + } + ], + "test.test[compute_range-preserve_rest_predicates_order-default.txt-Results]": [ + { + "checksum": "92254656baf216682150e51de4bbf51d", + "size": 7928, + "uri": "https://{canondata_backend}/1942525/befc6ebfd65b57985e99080238980e2a790b9c5a/resource.tar.gz#test.test_compute_range-preserve_rest_predicates_order-default.txt-Results_/results.txt" + } + ] +} diff --git a/yql/essentials/tests/sql/minirun/part2/test.py b/yql/essentials/tests/sql/minirun/part2/test.py new file mode 100644 index 0000000000..1cb2e7f049 --- /dev/null +++ b/yql/essentials/tests/sql/minirun/part2/test.py @@ -0,0 +1,16 @@ +import pytest +import yatest + +from pure import run_test +from test_utils import pytest_generate_tests_for_part +from yql_utils import pytest_get_current_part + + +def pytest_generate_tests(metafunc): + current_part, part_count = pytest_get_current_part(yatest.common.source_path(__file__)) + return pytest_generate_tests_for_part(metafunc, current_part, part_count) + + +@pytest.mark.parametrize('what', ['Results', 'Debug', 'RunOnOpt', 'LLVM']) +def test(suite, case, cfg, tmpdir, what, yql_http_file_server): + return run_test(suite, case, cfg, tmpdir, what, yql_http_file_server) diff --git a/yql/essentials/tests/sql/minirun/part2/ya.make b/yql/essentials/tests/sql/minirun/part2/ya.make new file mode 100644 index 0000000000..0d703e1e22 --- /dev/null +++ b/yql/essentials/tests/sql/minirun/part2/ya.make @@ -0,0 +1,3 @@ +INCLUDE(../pure.make) + + diff --git a/yql/essentials/tests/sql/minirun/part3/canondata/result.json b/yql/essentials/tests/sql/minirun/part3/canondata/result.json new file mode 100644 index 0000000000..8e47edb372 --- /dev/null +++ b/yql/essentials/tests/sql/minirun/part3/canondata/result.json @@ -0,0 +1,16 @@ +{ + "test.test[compute_range-tztypes-default.txt-Debug]": [ + { + "checksum": "f16da776935c59e7de9005d2e87056e7", + "size": 1605, + "uri": "https://{canondata_backend}/1814674/34855a34556905749369ca6269d86a253802d364/resource.tar.gz#test.test_compute_range-tztypes-default.txt-Debug_/opt.yql" + } + ], + "test.test[compute_range-tztypes-default.txt-Results]": [ + { + "checksum": "b710beeb7841f97c54bf9d130fc2f8ed", + "size": 29342, + "uri": "https://{canondata_backend}/1814674/34855a34556905749369ca6269d86a253802d364/resource.tar.gz#test.test_compute_range-tztypes-default.txt-Results_/results.txt" + } + ] +} diff --git a/yql/essentials/tests/sql/minirun/part3/test.py b/yql/essentials/tests/sql/minirun/part3/test.py new file mode 100644 index 0000000000..1cb2e7f049 --- /dev/null +++ b/yql/essentials/tests/sql/minirun/part3/test.py @@ -0,0 +1,16 @@ +import pytest +import yatest + +from pure import run_test +from test_utils import pytest_generate_tests_for_part +from yql_utils import pytest_get_current_part + + +def pytest_generate_tests(metafunc): + current_part, part_count = pytest_get_current_part(yatest.common.source_path(__file__)) + return pytest_generate_tests_for_part(metafunc, current_part, part_count) + + +@pytest.mark.parametrize('what', ['Results', 'Debug', 'RunOnOpt', 'LLVM']) +def test(suite, case, cfg, tmpdir, what, yql_http_file_server): + return run_test(suite, case, cfg, tmpdir, what, yql_http_file_server) diff --git a/yql/essentials/tests/sql/minirun/part3/ya.make b/yql/essentials/tests/sql/minirun/part3/ya.make new file mode 100644 index 0000000000..0d703e1e22 --- /dev/null +++ b/yql/essentials/tests/sql/minirun/part3/ya.make @@ -0,0 +1,3 @@ +INCLUDE(../pure.make) + + diff --git a/yql/essentials/tests/sql/minirun/part4/canondata/result.json b/yql/essentials/tests/sql/minirun/part4/canondata/result.json new file mode 100644 index 0000000000..9c7c754cc1 --- /dev/null +++ b/yql/essentials/tests/sql/minirun/part4/canondata/result.json @@ -0,0 +1,16 @@ +{ + "test.test[compute_range-yql-12941-default.txt-Debug]": [ + { + "checksum": "6481aaf7fea66b125b864cb9afecaa1f", + "size": 871, + "uri": "https://{canondata_backend}/1936842/98f279a8c97d83ae7e3740bf0c10b4e1b823d89e/resource.tar.gz#test.test_compute_range-yql-12941-default.txt-Debug_/opt.yql" + } + ], + "test.test[compute_range-yql-12941-default.txt-Results]": [ + { + "checksum": "c3ad050fc08649e1fc126e3e9a548e28", + "size": 17026, + "uri": "https://{canondata_backend}/1936842/98f279a8c97d83ae7e3740bf0c10b4e1b823d89e/resource.tar.gz#test.test_compute_range-yql-12941-default.txt-Results_/results.txt" + } + ] +} diff --git a/yql/essentials/tests/sql/minirun/part4/test.py b/yql/essentials/tests/sql/minirun/part4/test.py new file mode 100644 index 0000000000..1cb2e7f049 --- /dev/null +++ b/yql/essentials/tests/sql/minirun/part4/test.py @@ -0,0 +1,16 @@ +import pytest +import yatest + +from pure import run_test +from test_utils import pytest_generate_tests_for_part +from yql_utils import pytest_get_current_part + + +def pytest_generate_tests(metafunc): + current_part, part_count = pytest_get_current_part(yatest.common.source_path(__file__)) + return pytest_generate_tests_for_part(metafunc, current_part, part_count) + + +@pytest.mark.parametrize('what', ['Results', 'Debug', 'RunOnOpt', 'LLVM']) +def test(suite, case, cfg, tmpdir, what, yql_http_file_server): + return run_test(suite, case, cfg, tmpdir, what, yql_http_file_server) diff --git a/yql/essentials/tests/sql/minirun/part4/ya.make b/yql/essentials/tests/sql/minirun/part4/ya.make new file mode 100644 index 0000000000..0d703e1e22 --- /dev/null +++ b/yql/essentials/tests/sql/minirun/part4/ya.make @@ -0,0 +1,3 @@ +INCLUDE(../pure.make) + + diff --git a/yql/essentials/tests/sql/minirun/part5/canondata/result.json b/yql/essentials/tests/sql/minirun/part5/canondata/result.json new file mode 100644 index 0000000000..a4df55619e --- /dev/null +++ b/yql/essentials/tests/sql/minirun/part5/canondata/result.json @@ -0,0 +1,72 @@ +{ + "test.test[compute_range-adjacent_to_point-default.txt-Debug]": [ + { + "checksum": "4dc721da9529f116beebcaa6bc71550c", + "size": 4045, + "uri": "https://{canondata_backend}/1925842/6ebaccad9f032f11fe88822cf39e4932d3c67681/resource.tar.gz#test.test_compute_range-adjacent_to_point-default.txt-Debug_/opt.yql" + } + ], + "test.test[compute_range-adjacent_to_point-default.txt-Results]": [ + { + "checksum": "b4dedb69aa53d7c4bba89661275a4397", + "size": 78774, + "uri": "https://{canondata_backend}/1925842/6ebaccad9f032f11fe88822cf39e4932d3c67681/resource.tar.gz#test.test_compute_range-adjacent_to_point-default.txt-Results_/results.txt" + } + ], + "test.test[compute_range-in-default.txt-Debug]": [ + { + "checksum": "c4aff34ccbcb5965d694d913cd020d32", + "size": 4329, + "uri": "https://{canondata_backend}/1925842/6ebaccad9f032f11fe88822cf39e4932d3c67681/resource.tar.gz#test.test_compute_range-in-default.txt-Debug_/opt.yql" + } + ], + "test.test[compute_range-in-default.txt-Results]": [ + { + "checksum": "04df496c1a24df0cadda30280c91df11", + "size": 41349, + "uri": "https://{canondata_backend}/1925842/6ebaccad9f032f11fe88822cf39e4932d3c67681/resource.tar.gz#test.test_compute_range-in-default.txt-Results_/results.txt" + } + ], + "test.test[compute_range-in3-default.txt-Debug]": [ + { + "checksum": "ff6615c297b5e45e527ce720fbb088d0", + "size": 3531, + "uri": "https://{canondata_backend}/1925842/6ebaccad9f032f11fe88822cf39e4932d3c67681/resource.tar.gz#test.test_compute_range-in3-default.txt-Debug_/opt.yql" + } + ], + "test.test[compute_range-in3-default.txt-Results]": [ + { + "checksum": "1d0173257a9a9f38026accf1a7bc0b19", + "size": 41810, + "uri": "https://{canondata_backend}/1925842/6ebaccad9f032f11fe88822cf39e4932d3c67681/resource.tar.gz#test.test_compute_range-in3-default.txt-Results_/results.txt" + } + ], + "test.test[compute_range-pg_compare-default.txt-Debug]": [ + { + "checksum": "78ea471f6baaec1d6bd885fb7ba87f01", + "size": 1188, + "uri": "https://{canondata_backend}/1925842/6ebaccad9f032f11fe88822cf39e4932d3c67681/resource.tar.gz#test.test_compute_range-pg_compare-default.txt-Debug_/opt.yql" + } + ], + "test.test[compute_range-pg_compare-default.txt-Results]": [ + { + "checksum": "c9f9b0a3709f4c70e7c9e2574e426063", + "size": 19770, + "uri": "https://{canondata_backend}/1925842/6ebaccad9f032f11fe88822cf39e4932d3c67681/resource.tar.gz#test.test_compute_range-pg_compare-default.txt-Results_/results.txt" + } + ], + "test.test[compute_range-startswith-default.txt-Debug]": [ + { + "checksum": "1a839d39041a2abed15fff1df30d905a", + "size": 2724, + "uri": "https://{canondata_backend}/1925842/6ebaccad9f032f11fe88822cf39e4932d3c67681/resource.tar.gz#test.test_compute_range-startswith-default.txt-Debug_/opt.yql" + } + ], + "test.test[compute_range-startswith-default.txt-Results]": [ + { + "checksum": "3c5289cbe736bed101919cbf89fb0203", + "size": 56423, + "uri": "https://{canondata_backend}/1925842/6ebaccad9f032f11fe88822cf39e4932d3c67681/resource.tar.gz#test.test_compute_range-startswith-default.txt-Results_/results.txt" + } + ] +} diff --git a/yql/essentials/tests/sql/minirun/part5/test.py b/yql/essentials/tests/sql/minirun/part5/test.py new file mode 100644 index 0000000000..1cb2e7f049 --- /dev/null +++ b/yql/essentials/tests/sql/minirun/part5/test.py @@ -0,0 +1,16 @@ +import pytest +import yatest + +from pure import run_test +from test_utils import pytest_generate_tests_for_part +from yql_utils import pytest_get_current_part + + +def pytest_generate_tests(metafunc): + current_part, part_count = pytest_get_current_part(yatest.common.source_path(__file__)) + return pytest_generate_tests_for_part(metafunc, current_part, part_count) + + +@pytest.mark.parametrize('what', ['Results', 'Debug', 'RunOnOpt', 'LLVM']) +def test(suite, case, cfg, tmpdir, what, yql_http_file_server): + return run_test(suite, case, cfg, tmpdir, what, yql_http_file_server) diff --git a/yql/essentials/tests/sql/minirun/part5/ya.make b/yql/essentials/tests/sql/minirun/part5/ya.make new file mode 100644 index 0000000000..0d703e1e22 --- /dev/null +++ b/yql/essentials/tests/sql/minirun/part5/ya.make @@ -0,0 +1,3 @@ +INCLUDE(../pure.make) + + diff --git a/yql/essentials/tests/sql/minirun/part6/canondata/result.json b/yql/essentials/tests/sql/minirun/part6/canondata/result.json new file mode 100644 index 0000000000..db3c6ea854 --- /dev/null +++ b/yql/essentials/tests/sql/minirun/part6/canondata/result.json @@ -0,0 +1,44 @@ +{ + "test.test[compute_range-norange-default.txt-Debug]": [ + { + "checksum": "e9bcea49fb8c7f9c47bb470bb745c057", + "size": 381, + "uri": "https://{canondata_backend}/1925842/4728690604955c4954b686e25a953fed04aa8521/resource.tar.gz#test.test_compute_range-norange-default.txt-Debug_/opt.yql" + } + ], + "test.test[compute_range-norange-default.txt-Results]": [ + { + "checksum": "66eca1ced2fe4f8d29dc70632c4fcf0d", + "size": 1287, + "uri": "https://{canondata_backend}/1925842/4728690604955c4954b686e25a953fed04aa8521/resource.tar.gz#test.test_compute_range-norange-default.txt-Results_/results.txt" + } + ], + "test.test[compute_range-pg_equal-default.txt-Debug]": [ + { + "checksum": "9b93782bc546d740e2444d459824ec45", + "size": 819, + "uri": "https://{canondata_backend}/1925842/4728690604955c4954b686e25a953fed04aa8521/resource.tar.gz#test.test_compute_range-pg_equal-default.txt-Debug_/opt.yql" + } + ], + "test.test[compute_range-pg_equal-default.txt-Results]": [ + { + "checksum": "9ac4c6ce9a6fdab0eba801300d39aa9d", + "size": 10526, + "uri": "https://{canondata_backend}/1925842/4728690604955c4954b686e25a953fed04aa8521/resource.tar.gz#test.test_compute_range-pg_equal-default.txt-Results_/results.txt" + } + ], + "test.test[compute_range-pg_exists-default.txt-Debug]": [ + { + "checksum": "ee374bb436483e9c88338e94ec67b409", + "size": 774, + "uri": "https://{canondata_backend}/1925842/4728690604955c4954b686e25a953fed04aa8521/resource.tar.gz#test.test_compute_range-pg_exists-default.txt-Debug_/opt.yql" + } + ], + "test.test[compute_range-pg_exists-default.txt-Results]": [ + { + "checksum": "7fabe5122903e7973848d25692efbcc1", + "size": 9886, + "uri": "https://{canondata_backend}/1925842/4728690604955c4954b686e25a953fed04aa8521/resource.tar.gz#test.test_compute_range-pg_exists-default.txt-Results_/results.txt" + } + ] +} diff --git a/yql/essentials/tests/sql/minirun/part6/test.py b/yql/essentials/tests/sql/minirun/part6/test.py new file mode 100644 index 0000000000..1cb2e7f049 --- /dev/null +++ b/yql/essentials/tests/sql/minirun/part6/test.py @@ -0,0 +1,16 @@ +import pytest +import yatest + +from pure import run_test +from test_utils import pytest_generate_tests_for_part +from yql_utils import pytest_get_current_part + + +def pytest_generate_tests(metafunc): + current_part, part_count = pytest_get_current_part(yatest.common.source_path(__file__)) + return pytest_generate_tests_for_part(metafunc, current_part, part_count) + + +@pytest.mark.parametrize('what', ['Results', 'Debug', 'RunOnOpt', 'LLVM']) +def test(suite, case, cfg, tmpdir, what, yql_http_file_server): + return run_test(suite, case, cfg, tmpdir, what, yql_http_file_server) diff --git a/yql/essentials/tests/sql/minirun/part6/ya.make b/yql/essentials/tests/sql/minirun/part6/ya.make new file mode 100644 index 0000000000..0d703e1e22 --- /dev/null +++ b/yql/essentials/tests/sql/minirun/part6/ya.make @@ -0,0 +1,3 @@ +INCLUDE(../pure.make) + + diff --git a/yql/essentials/tests/sql/minirun/part7/canondata/result.json b/yql/essentials/tests/sql/minirun/part7/canondata/result.json new file mode 100644 index 0000000000..e3400870f7 --- /dev/null +++ b/yql/essentials/tests/sql/minirun/part7/canondata/result.json @@ -0,0 +1,16 @@ +{ + "test.test[compute_range-huge_in-default.txt-Debug]": [ + { + "checksum": "80c96f098a46ff47413c0efdc212ea0d", + "size": 1053, + "uri": "https://{canondata_backend}/1942525/7665133ac15a3f92aef975c99a4dfe3023179a69/resource.tar.gz#test.test_compute_range-huge_in-default.txt-Debug_/opt.yql" + } + ], + "test.test[compute_range-huge_in-default.txt-Results]": [ + { + "checksum": "518dc39b0f56b4461fb4c9f1642444e7", + "size": 4793, + "uri": "https://{canondata_backend}/1942525/7665133ac15a3f92aef975c99a4dfe3023179a69/resource.tar.gz#test.test_compute_range-huge_in-default.txt-Results_/results.txt" + } + ] +} diff --git a/yql/essentials/tests/sql/minirun/part7/test.py b/yql/essentials/tests/sql/minirun/part7/test.py new file mode 100644 index 0000000000..1cb2e7f049 --- /dev/null +++ b/yql/essentials/tests/sql/minirun/part7/test.py @@ -0,0 +1,16 @@ +import pytest +import yatest + +from pure import run_test +from test_utils import pytest_generate_tests_for_part +from yql_utils import pytest_get_current_part + + +def pytest_generate_tests(metafunc): + current_part, part_count = pytest_get_current_part(yatest.common.source_path(__file__)) + return pytest_generate_tests_for_part(metafunc, current_part, part_count) + + +@pytest.mark.parametrize('what', ['Results', 'Debug', 'RunOnOpt', 'LLVM']) +def test(suite, case, cfg, tmpdir, what, yql_http_file_server): + return run_test(suite, case, cfg, tmpdir, what, yql_http_file_server) diff --git a/yql/essentials/tests/sql/minirun/part7/ya.make b/yql/essentials/tests/sql/minirun/part7/ya.make new file mode 100644 index 0000000000..0d703e1e22 --- /dev/null +++ b/yql/essentials/tests/sql/minirun/part7/ya.make @@ -0,0 +1,3 @@ +INCLUDE(../pure.make) + + diff --git a/yql/essentials/tests/sql/minirun/part8/canondata/result.json b/yql/essentials/tests/sql/minirun/part8/canondata/result.json new file mode 100644 index 0000000000..5c45910b50 --- /dev/null +++ b/yql/essentials/tests/sql/minirun/part8/canondata/result.json @@ -0,0 +1,86 @@ +{ + "test.test[compute_range-in2-default.txt-Debug]": [ + { + "checksum": "bd691b0c2a87038767feddb27bbdf790", + "size": 2078, + "uri": "https://{canondata_backend}/1942525/ba8ad8c1da4bf920ccf460436ddcc1cc7df77125/resource.tar.gz#test.test_compute_range-in2-default.txt-Debug_/opt.yql" + } + ], + "test.test[compute_range-in2-default.txt-Results]": [ + { + "checksum": "efe02d47850c6bfa1d3373a4592316a3", + "size": 21540, + "uri": "https://{canondata_backend}/1942525/ba8ad8c1da4bf920ccf460436ddcc1cc7df77125/resource.tar.gz#test.test_compute_range-in2-default.txt-Results_/results.txt" + } + ], + "test.test[compute_range-multiply_limit_with_dups-default.txt-Debug]": [ + { + "checksum": "435700d041ce22aae727de36454e6fb8", + "size": 1526, + "uri": "https://{canondata_backend}/1942525/ba8ad8c1da4bf920ccf460436ddcc1cc7df77125/resource.tar.gz#test.test_compute_range-multiply_limit_with_dups-default.txt-Debug_/opt.yql" + } + ], + "test.test[compute_range-multiply_limit_with_dups-default.txt-Results]": [ + { + "checksum": "a2cd3fa1fe3361fc6a46c3c02a8a32f6", + "size": 6659, + "uri": "https://{canondata_backend}/1942525/ba8ad8c1da4bf920ccf460436ddcc1cc7df77125/resource.tar.gz#test.test_compute_range-multiply_limit_with_dups-default.txt-Results_/results.txt" + } + ], + "test.test[compute_range-pg_sqlin-default.txt-Debug]": [ + { + "checksum": "167b8e6cc99cb2694ccb12bace19006a", + "size": 1440, + "uri": "https://{canondata_backend}/1942525/ba8ad8c1da4bf920ccf460436ddcc1cc7df77125/resource.tar.gz#test.test_compute_range-pg_sqlin-default.txt-Debug_/opt.yql" + } + ], + "test.test[compute_range-pg_sqlin-default.txt-Results]": [ + { + "checksum": "82b78208b7788663c8d6e1ba6d6ea3ff", + "size": 12526, + "uri": "https://{canondata_backend}/1942525/ba8ad8c1da4bf920ccf460436ddcc1cc7df77125/resource.tar.gz#test.test_compute_range-pg_sqlin-default.txt-Results_/results.txt" + } + ], + "test.test[compute_range-pg_startswith-default.txt-Debug]": [ + { + "checksum": "a9d409fd93a872b4f117f8df4e6a8739", + "size": 1264, + "uri": "https://{canondata_backend}/1942525/ba8ad8c1da4bf920ccf460436ddcc1cc7df77125/resource.tar.gz#test.test_compute_range-pg_startswith-default.txt-Debug_/opt.yql" + } + ], + "test.test[compute_range-pg_startswith-default.txt-Results]": [ + { + "checksum": "1f9c71cfb11d01df7ba9be02b7a85def", + "size": 19820, + "uri": "https://{canondata_backend}/1942525/ba8ad8c1da4bf920ccf460436ddcc1cc7df77125/resource.tar.gz#test.test_compute_range-pg_startswith-default.txt-Results_/results.txt" + } + ], + "test.test[compute_range-repeated_keyranges_in_and-default.txt-Debug]": [ + { + "checksum": "d08392fca6dba206b37f2dac105ecf34", + "size": 980, + "uri": "https://{canondata_backend}/1942525/ba8ad8c1da4bf920ccf460436ddcc1cc7df77125/resource.tar.gz#test.test_compute_range-repeated_keyranges_in_and-default.txt-Debug_/opt.yql" + } + ], + "test.test[compute_range-repeated_keyranges_in_and-default.txt-Results]": [ + { + "checksum": "23d062232ed400e90754d4dcf6bdf92a", + "size": 7515, + "uri": "https://{canondata_backend}/1942525/ba8ad8c1da4bf920ccf460436ddcc1cc7df77125/resource.tar.gz#test.test_compute_range-repeated_keyranges_in_and-default.txt-Results_/results.txt" + } + ], + "test.test[compute_range-yql-13489-default.txt-Debug]": [ + { + "checksum": "da3f38c83658879a58f4fd8965f39738", + "size": 1508, + "uri": "https://{canondata_backend}/1942525/ba8ad8c1da4bf920ccf460436ddcc1cc7df77125/resource.tar.gz#test.test_compute_range-yql-13489-default.txt-Debug_/opt.yql" + } + ], + "test.test[compute_range-yql-13489-default.txt-Results]": [ + { + "checksum": "3d7e7fc9f5ebaf22ccec6dca93427e18", + "size": 12234, + "uri": "https://{canondata_backend}/1942525/ba8ad8c1da4bf920ccf460436ddcc1cc7df77125/resource.tar.gz#test.test_compute_range-yql-13489-default.txt-Results_/results.txt" + } + ] +} diff --git a/yql/essentials/tests/sql/minirun/part8/test.py b/yql/essentials/tests/sql/minirun/part8/test.py new file mode 100644 index 0000000000..1cb2e7f049 --- /dev/null +++ b/yql/essentials/tests/sql/minirun/part8/test.py @@ -0,0 +1,16 @@ +import pytest +import yatest + +from pure import run_test +from test_utils import pytest_generate_tests_for_part +from yql_utils import pytest_get_current_part + + +def pytest_generate_tests(metafunc): + current_part, part_count = pytest_get_current_part(yatest.common.source_path(__file__)) + return pytest_generate_tests_for_part(metafunc, current_part, part_count) + + +@pytest.mark.parametrize('what', ['Results', 'Debug', 'RunOnOpt', 'LLVM']) +def test(suite, case, cfg, tmpdir, what, yql_http_file_server): + return run_test(suite, case, cfg, tmpdir, what, yql_http_file_server) diff --git a/yql/essentials/tests/sql/minirun/part8/ya.make b/yql/essentials/tests/sql/minirun/part8/ya.make new file mode 100644 index 0000000000..0d703e1e22 --- /dev/null +++ b/yql/essentials/tests/sql/minirun/part8/ya.make @@ -0,0 +1,3 @@ +INCLUDE(../pure.make) + + diff --git a/yql/essentials/tests/sql/minirun/part9/canondata/result.json b/yql/essentials/tests/sql/minirun/part9/canondata/result.json new file mode 100644 index 0000000000..65e2f4d446 --- /dev/null +++ b/yql/essentials/tests/sql/minirun/part9/canondata/result.json @@ -0,0 +1,44 @@ +{ + "test.test[compute_range-decimal-default.txt-Debug]": [ + { + "checksum": "228d623d782a9f7e31d98bdb46570af9", + "size": 537, + "uri": "https://{canondata_backend}/1936842/1a95fb8259f12b9abef4aeca613b4d2163ce4b69/resource.tar.gz#test.test_compute_range-decimal-default.txt-Debug_/opt.yql" + } + ], + "test.test[compute_range-decimal-default.txt-Results]": [ + { + "checksum": "11783215b1cd57c19d698498ddce9b51", + "size": 5771, + "uri": "https://{canondata_backend}/1936842/1a95fb8259f12b9abef4aeca613b4d2163ce4b69/resource.tar.gz#test.test_compute_range-decimal-default.txt-Results_/results.txt" + } + ], + "test.test[compute_range-merge_adjacent-default.txt-Debug]": [ + { + "checksum": "2d61c161e6ec8f149bf0504b5368082b", + "size": 3465, + "uri": "https://{canondata_backend}/1936842/1a95fb8259f12b9abef4aeca613b4d2163ce4b69/resource.tar.gz#test.test_compute_range-merge_adjacent-default.txt-Debug_/opt.yql" + } + ], + "test.test[compute_range-merge_adjacent-default.txt-Results]": [ + { + "checksum": "a91c71a84b5e629f140e2939d90f8dcd", + "size": 25533, + "uri": "https://{canondata_backend}/1936842/1a95fb8259f12b9abef4aeca613b4d2163ce4b69/resource.tar.gz#test.test_compute_range-merge_adjacent-default.txt-Results_/results.txt" + } + ], + "test.test[compute_range-multiply_limit_with_nulls-default.txt-Debug]": [ + { + "checksum": "e9a01608755bbc6870a0688de6b0f4e5", + "size": 1033, + "uri": "https://{canondata_backend}/1936842/1a95fb8259f12b9abef4aeca613b4d2163ce4b69/resource.tar.gz#test.test_compute_range-multiply_limit_with_nulls-default.txt-Debug_/opt.yql" + } + ], + "test.test[compute_range-multiply_limit_with_nulls-default.txt-Results]": [ + { + "checksum": "f6500e7243d193c711d0f8e70040bd51", + "size": 4787, + "uri": "https://{canondata_backend}/1936842/1a95fb8259f12b9abef4aeca613b4d2163ce4b69/resource.tar.gz#test.test_compute_range-multiply_limit_with_nulls-default.txt-Results_/results.txt" + } + ] +} diff --git a/yql/essentials/tests/sql/minirun/part9/test.py b/yql/essentials/tests/sql/minirun/part9/test.py new file mode 100644 index 0000000000..1cb2e7f049 --- /dev/null +++ b/yql/essentials/tests/sql/minirun/part9/test.py @@ -0,0 +1,16 @@ +import pytest +import yatest + +from pure import run_test +from test_utils import pytest_generate_tests_for_part +from yql_utils import pytest_get_current_part + + +def pytest_generate_tests(metafunc): + current_part, part_count = pytest_get_current_part(yatest.common.source_path(__file__)) + return pytest_generate_tests_for_part(metafunc, current_part, part_count) + + +@pytest.mark.parametrize('what', ['Results', 'Debug', 'RunOnOpt', 'LLVM']) +def test(suite, case, cfg, tmpdir, what, yql_http_file_server): + return run_test(suite, case, cfg, tmpdir, what, yql_http_file_server) diff --git a/yql/essentials/tests/sql/minirun/part9/ya.make b/yql/essentials/tests/sql/minirun/part9/ya.make new file mode 100644 index 0000000000..0d703e1e22 --- /dev/null +++ b/yql/essentials/tests/sql/minirun/part9/ya.make @@ -0,0 +1,3 @@ +INCLUDE(../pure.make) + + diff --git a/yql/essentials/tests/sql/minirun/pure.make b/yql/essentials/tests/sql/minirun/pure.make new file mode 100644 index 0000000000..e76272da52 --- /dev/null +++ b/yql/essentials/tests/sql/minirun/pure.make @@ -0,0 +1,54 @@ +IF (NOT OPENSOURCE) + +PY3TEST() + +TEST_SRCS( + test.py +) + +IF (SANITIZER_TYPE OR NOT OPENSOURCE) + REQUIREMENTS(ram:20) +ENDIF() + +IF (SANITIZER_TYPE OR WITH_VALGRIND) + TIMEOUT(1800) + SIZE(LARGE) + TAG(ya:fat sb:ttl=2) +ELSE() + TIMEOUT(600) + SIZE(MEDIUM) + TAG(sb:ttl=2) +ENDIF() + +#FORK_TESTS() +#FORK_SUBTESTS() +#SPLIT_FACTOR(10) + +DEPENDS( + yql/essentials/tools/astdiff + yql/essentials/tools/minirun + yql/essentials/tests/common/test_framework/udfs_deps + yql/essentials/udfs/test/test_import +) +DATA( + arcadia/yql/essentials/tests/sql/minirun # python files + arcadia/yql/essentials/tests/sql/suites + arcadia/yql/essentials/mount + arcadia/yql/essentials/cfg/tests +) + +PEERDIR( + yql/essentials/tests/common/test_framework + library/python/testing/swag/lib + yql/essentials/core/file_storage/proto +) + +NO_CHECK_IMPORTS() + +IF (SANITIZER_TYPE == "memory") + TAG(ya:not_autocheck) # YQL-15385 +ENDIF() + +END() + +ENDIF() diff --git a/yql/essentials/tests/sql/minirun/pure.py b/yql/essentials/tests/sql/minirun/pure.py new file mode 100644 index 0000000000..e454c7c6e4 --- /dev/null +++ b/yql/essentials/tests/sql/minirun/pure.py @@ -0,0 +1,101 @@ +import codecs +import os +import pytest +import re +import yql_utils + +import yatest.common +from yql_utils import execute, get_tables, get_files, get_http_files, \ + KSV_ATTR, yql_binary_path, is_xfail, is_canonize_peephole, is_peephole_use_blocks, is_canonize_lineage, \ + is_skip_forceblocks, get_param, normalize_source_code_path, replace_vals, get_gateway_cfg_suffix, \ + do_custom_query_check, stable_result_file, stable_table_file, is_with_final_result_issues, \ + normalize_result +from yqlrun import YQLRun + +from test_utils import get_config, get_parameters_json, DATA_PATH +from test_file_common import run_file, run_file_no_cache, get_gateways_config + +ASTDIFF_PATH = yql_binary_path('yql/essentials/tools/astdiff/astdiff') +MINIRUN_PATH = yql_binary_path('yql/essentials/tools/minirun/minirun') + + +def run_test(suite, case, cfg, tmpdir, what, yql_http_file_server): + if get_gateway_cfg_suffix() != '' and what not in ('Results','LLVM'): + pytest.skip('non-trivial gateways.conf') + + if suite != "compute_range": + pytest.skip('TODO:' + suite) + + config = get_config(suite, case, cfg) + + xfail = is_xfail(config) + if xfail and what != 'Results': + pytest.skip('xfail is not supported in this mode') + + program_sql = os.path.join(DATA_PATH, suite, '%s.sql' % case) + with codecs.open(program_sql, encoding='utf-8') as program_file_descr: + sql_query = program_file_descr.read() + + extra_final_args = [] + if is_with_final_result_issues(config): + extra_final_args += ['--with-final-issues'] + (res, tables_res) = run_file('pure', suite, case, cfg, config, yql_http_file_server, MINIRUN_PATH, + extra_args=extra_final_args, allow_llvm=False) + + to_canonize = [] + assert os.path.exists(res.results_file) + assert not tables_res + + if what == 'Results': + if not xfail: + if do_custom_query_check(res, sql_query): + return None + + stable_result_file(res) + to_canonize.append(yatest.common.canonical_file(res.results_file)) + if res.std_err: + to_canonize.append(normalize_source_code_path(res.std_err)) + + if what == 'Debug': + to_canonize = [yatest.common.canonical_file(res.opt_file, diff_tool=ASTDIFF_PATH)] + + if what == 'RunOnOpt' or what == 'LLVM': + is_llvm = (what == 'LLVM') + files = get_files(suite, config, DATA_PATH) + http_files = get_http_files(suite, config, DATA_PATH) + http_files_urls = yql_http_file_server.register_files({}, http_files) + parameters = get_parameters_json(suite, config) + + yqlrun = YQLRun( + prov='pure', + keep_temp=False, + gateway_config=get_gateways_config(http_files, yql_http_file_server, allow_llvm=is_llvm), + udfs_dir=yql_binary_path('yql/essentials/tests/common/test_framework/udfs_deps'), + binary=MINIRUN_PATH + ) + + opt_res, opt_tables_res = execute( + yqlrun, + program=res.opt if not is_llvm else None, + program_file=program_sql if is_llvm else None, + run_sql=is_llvm, + files=files, + urls=http_files_urls, + check_error=True, + verbose=True, + parameters=parameters) + + assert os.path.exists(opt_res.results_file) + assert not opt_tables_res + + base_res_yson = normalize_result(stable_result_file(res), False) + opt_res_yson = normalize_result(stable_result_file(opt_res), False) + + # Compare results + assert opt_res_yson == base_res_yson, 'RESULTS_DIFFER\n' \ + 'Result:\n %(opt_res_yson)s\n\n' \ + 'Base result:\n %(base_res_yson)s\n' % locals() + + return None + + return to_canonize diff --git a/yql/essentials/tests/sql/minirun/ya.make b/yql/essentials/tests/sql/minirun/ya.make new file mode 100644 index 0000000000..cc5451b4fd --- /dev/null +++ b/yql/essentials/tests/sql/minirun/ya.make @@ -0,0 +1,16 @@ +IF (NOT OPENSOURCE) + +RECURSE( + part0 + part1 + part2 + part3 + part4 + part5 + part6 + part7 + part8 + part9 +) + +ENDIF() diff --git a/yql/essentials/tests/sql/sql2yql/canondata/result.json b/yql/essentials/tests/sql/sql2yql/canondata/result.json index da8e4a69ce..715524f3f9 100644 --- a/yql/essentials/tests/sql/sql2yql/canondata/result.json +++ b/yql/essentials/tests/sql/sql2yql/canondata/result.json @@ -15672,6 +15672,13 @@ "uri": "https://{canondata_backend}/1937429/434276f26b2857be3c5ad3fdbbf877d2bf775ac5/resource.tar.gz#test_sql2yql.test_pg_duplicated-ambigous_ref_/sql.yql" } ], + "test_sql2yql.test[pg_duplicated-duplicated_rowspec]": [ + { + "checksum": "70c2665c84dec52c9318ad1b43450c08", + "size": 623, + "uri": "https://{canondata_backend}/1871002/c5bebc3ea8669574f080ea93959c29fb53e4bc67/resource.tar.gz#test_sql2yql.test_pg_duplicated-duplicated_rowspec_/sql.yql" + } + ], "test_sql2yql.test[pg_duplicated-order_by_duplicated_proj_on_column_ref]": [ { "checksum": "e0bd0cd714e708b53afbfb0770cf3e30", diff --git a/yql/essentials/tests/sql/suites/pg_duplicated/duplicated_rowspec.cfg b/yql/essentials/tests/sql/suites/pg_duplicated/duplicated_rowspec.cfg new file mode 100644 index 0000000000..ab354ca5a0 --- /dev/null +++ b/yql/essentials/tests/sql/suites/pg_duplicated/duplicated_rowspec.cfg @@ -0,0 +1 @@ +in input duplicated_rowspec.txt
\ No newline at end of file diff --git a/yql/essentials/tests/sql/suites/pg_duplicated/duplicated_rowspec.sql b/yql/essentials/tests/sql/suites/pg_duplicated/duplicated_rowspec.sql new file mode 100644 index 0000000000..4d7bd0853f --- /dev/null +++ b/yql/essentials/tests/sql/suites/pg_duplicated/duplicated_rowspec.sql @@ -0,0 +1,3 @@ +--!syntax_pg + +select * from plato.Input; diff --git a/yql/essentials/tests/sql/suites/pg_duplicated/duplicated_rowspec.txt b/yql/essentials/tests/sql/suites/pg_duplicated/duplicated_rowspec.txt new file mode 100644 index 0000000000..60840629ed --- /dev/null +++ b/yql/essentials/tests/sql/suites/pg_duplicated/duplicated_rowspec.txt @@ -0,0 +1,2 @@ +{x=1;x_generated_2=333333333333;}; +{x=3;x_generated_2=444444444444;};
\ No newline at end of file diff --git a/yql/essentials/tests/sql/suites/pg_duplicated/duplicated_rowspec.txt.attr b/yql/essentials/tests/sql/suites/pg_duplicated/duplicated_rowspec.txt.attr new file mode 100644 index 0000000000..66ac98585a --- /dev/null +++ b/yql/essentials/tests/sql/suites/pg_duplicated/duplicated_rowspec.txt.attr @@ -0,0 +1,36 @@ +{ + "_yql_row_spec"={ + "ColumnOrder"=["x";"x"]; + "NativeYtTypeFlags"=0u; + "StrictSchema"= %true; + "TypePatch"= [ + "StructType"; [ + ["x"; ["PgType";"int4"]]; + ["x_generated_2"; ["PgType";"int8"]]; + ] + ] + }; + "schema"=< + "strict"=%true; + "unique_keys"=%false; + >[ + { + "name"= "x"; + "required"= %false; + "type"= "int32"; + "type_v3"= { + "item"= "int32"; + "type_name"= "optional"; + } + }; + { + "name"= "x_generated_2"; + "required"= %false; + "type"= "int64"; + "type_v3"= { + "item"= "int64"; + "type_name"= "optional"; + } + }; + ] +}
\ No newline at end of file diff --git a/yql/essentials/tests/sql/ya.make b/yql/essentials/tests/sql/ya.make index b2b9b4089c..87418fb444 100644 --- a/yql/essentials/tests/sql/ya.make +++ b/yql/essentials/tests/sql/ya.make @@ -1,6 +1,7 @@ SUBSCRIBER(g:yql) RECURSE_FOR_TESTS( + minirun sql2yql ) diff --git a/yql/essentials/tools/yql_facade_run/yql_facade_run.cpp b/yql/essentials/tools/yql_facade_run/yql_facade_run.cpp index 2db2f62607..ed01f4ce9c 100644 --- a/yql/essentials/tools/yql_facade_run/yql_facade_run.cpp +++ b/yql/essentials/tools/yql_facade_run/yql_facade_run.cpp @@ -565,7 +565,7 @@ int TFacadeRunner::DoMain(int argc, const char *argv[]) { TModulesTable modules; FillUserDataTableFromFileSystem(*RunOptions_.MountConfig, RunOptions_.DataTable); - if (!CompileLibraries(RunOptions_.DataTable, ctx, modules)) { + if (!CompileLibraries(RunOptions_.DataTable, ctx, modules, RunOptions_.OptimizeLibs && RunOptions_.Mode >= ERunMode::Validate)) { *RunOptions_.ErrStream << "Errors on compile libraries:" << Endl; ctx.IssueManager.GetIssues().PrintTo(*RunOptions_.ErrStream); return -1; |