diff options
| author | vvvv <[email protected]> | 2024-12-17 20:01:01 +0300 |
|---|---|---|
| committer | vvvv <[email protected]> | 2024-12-17 20:24:24 +0300 |
| commit | 25f527db97769d0b47f0d9fd5e8b0df8bba32bcb (patch) | |
| tree | e8fbc3e4715c100baca84edc23f13b5b5c6930af | |
| parent | c1b5b963b8bcc15d4f9074867584bab9338821b6 (diff) | |
split & refactor core/ut
init
commit_hash:6a569cc8a3ace6be991ce981c949eda644f39c0b
| -rw-r--r-- | yql/essentials/core/ut/ya.make | 14 | ||||
| -rw-r--r-- | yql/essentials/core/ut/yql_column_order_ut.cpp | 1 | ||||
| -rw-r--r-- | yql/essentials/core/ut/yql_execution_ut.cpp | 793 | ||||
| -rw-r--r-- | yql/essentials/core/ut/yql_expr_constraint_ut.cpp | 602 | ||||
| -rw-r--r-- | yql/essentials/core/ut/yql_expr_discover_ut.cpp | 151 | ||||
| -rw-r--r-- | yql/essentials/core/ut/yql_expr_optimize_ut.cpp | 88 | ||||
| -rw-r--r-- | yql/essentials/core/ut/yql_expr_providers_ut.cpp | 286 | ||||
| -rw-r--r-- | yql/essentials/core/ut/yql_library_compiler_ut.cpp | 2 | ||||
| -rw-r--r-- | yql/essentials/core/ut/yql_qplayer_ut.cpp | 310 | ||||
| -rw-r--r-- | yql/essentials/public/udf/arrow/ut/ya.make | 1 |
10 files changed, 300 insertions, 1948 deletions
diff --git a/yql/essentials/core/ut/ya.make b/yql/essentials/core/ut/ya.make index c5aa987d204..b5d1e27d6d7 100644 --- a/yql/essentials/core/ut/ya.make +++ b/yql/essentials/core/ut/ya.make @@ -1,19 +1,13 @@ -IF (NOT OPENSOURCE) - UNITTEST_FOR(yql/essentials/core) SRCS( yql_column_order_ut.cpp - yql_execution_ut.cpp yql_expr_constraint_ut.cpp - yql_expr_discover_ut.cpp yql_expr_optimize_ut.cpp - yql_expr_providers_ut.cpp yql_expr_type_annotation_ut.cpp yql_library_compiler_ut.cpp yql_opt_utils_ut.cpp yql_udf_index_ut.cpp - yql_qplayer_ut.cpp ) PEERDIR( @@ -31,16 +25,12 @@ PEERDIR( yql/essentials/public/udf yql/essentials/public/udf/service/exception_policy yql/essentials/core/type_ann - yql/essentials/core/ut_common yql/essentials/providers/common/provider yql/essentials/providers/common/schema/parser yql/essentials/providers/result/provider - yt/yql/providers/yt/gateway/file - yt/yql/providers/yt/provider - yt/yql/providers/yt/codec/codegen - yt/yql/providers/yt/comp_nodes/llvm14 yql/essentials/minikql/comp_nodes/llvm14 yql/essentials/minikql/invoke_builtins/llvm14 + yql/essentials/parser/pg_wrapper yql/essentials/sql/pg yql/essentials/udfs/common/string ) @@ -61,5 +51,3 @@ ENDIF() YQL_LAST_ABI_VERSION() END() - -ENDIF() diff --git a/yql/essentials/core/ut/yql_column_order_ut.cpp b/yql/essentials/core/ut/yql_column_order_ut.cpp index 12669a43990..da74a0118bc 100644 --- a/yql/essentials/core/ut/yql_column_order_ut.cpp +++ b/yql/essentials/core/ut/yql_column_order_ut.cpp @@ -1,5 +1,4 @@ #include <yql/essentials/core/yql_type_annotation.h> -#include <yql/essentials/core/ut_common/yql_ut_common.h> #include <library/cpp/testing/unittest/registar.h> diff --git a/yql/essentials/core/ut/yql_execution_ut.cpp b/yql/essentials/core/ut/yql_execution_ut.cpp deleted file mode 100644 index 01ba31c50d6..00000000000 --- a/yql/essentials/core/ut/yql_execution_ut.cpp +++ /dev/null @@ -1,793 +0,0 @@ -#include "yql_execution.h" - -#include <yql/essentials/core/ut_common/yql_ut_common.h> - -#include <yql/essentials/ast/yql_ast_annotation.h> -#include <yql/essentials/ast/yql_expr.h> -#include <yql/essentials/core/cbo/simple/cbo_simple.h> -#include <yql/essentials/core/type_ann/type_ann_core.h> -#include <yql/essentials/core/yql_expr_optimize.h> -#include <yql/essentials/core/yql_expr_type_annotation.h> -#include <yql/essentials/core/yql_opt_proposed_by_data.h> -#include <yql/essentials/core/yql_opt_rewrite_io.h> -#include <yql/essentials/providers/common/provider/yql_provider_names.h> -#include <yql/essentials/providers/common/schema/parser/yql_type_parser.h> -#include <yql/essentials/providers/result/provider/yql_result_provider.h> - -#include <yql/essentials/core/facade/yql_facade.h> -#include <yt/yql/providers/yt/provider/yql_yt_provider.h> -#include <yt/yql/providers/yt/gateway/file/yql_yt_file.h> -#include <yt/yql/providers/yt/gateway/file/yql_yt_file_services.h> -#include <yql/essentials/minikql/invoke_builtins/mkql_builtins.h> - -#include <library/cpp/testing/unittest/registar.h> - -#include <util/system/user.h> -#include <util/system/tempfile.h> -#include <util/system/defaults.h> -#include <util/system/fstat.h> -#include <util/folder/path.h> -#include <util/folder/tempdir.h> -#include <util/string/cast.h> -#include <util/string/builder.h> -#include <util/system/sanitizers.h> - -namespace NYql { - - static TString BuildFileNameForTmpTable(TStringBuf table, TStringBuf tmpDir) { - return TStringBuilder() << tmpDir << LOCSLASH_C << table.substr(4) << ".tmp"; - } - - struct TRunSingleProgram { - TString Src; - TString TmpDir; - TString Parameters; - IOutputStream& Err; - TVector<TString> Res; - THashMap<TString, TString> Tables; - - TRunSingleProgram(const TString& src, IOutputStream& err) - : Src(src) - , Err(err) - { - } - - bool Run( - const NKikimr::NMiniKQL::IFunctionRegistry* funcReg - ) { - auto yqlNativeServices = NFile::TYtFileServices::Make(funcReg, Tables, {}, TmpDir); - auto ytGateway = CreateYtFileGateway(yqlNativeServices); - - TVector<TDataProviderInitializer> dataProvidersInit; - dataProvidersInit.push_back(GetYtNativeDataProviderInitializer(ytGateway, MakeSimpleCBOOptimizerFactory(), {})); - TProgramFactory factory(true, funcReg, 0ULL, dataProvidersInit, "ut"); - - TProgramPtr program = factory.Create("-stdin-", Src); - program->ConfigureYsonResultFormat(NYson::EYsonFormat::Text); - if (!Parameters.empty()) { - program->SetParametersYson(Parameters); - } - - if (!program->ParseYql() || !program->Compile(GetUsername())) { - program->PrintErrorsTo(Err); - return false; - } - - TProgram::TStatus status = program->Run(GetUsername()); - if (status == TProgram::TStatus::Error) { - program->PrintErrorsTo(Err); - } - Res = program->Results(); - return status == TProgram::TStatus::Ok; - } - - void AddResults(TVector<TString>& res) const { - res.insert(res.end(), Res.begin(), Res.end()); - } - - bool Finished() const { - return true; - } - }; - - struct TRunMultiplePrograms: public TRunSingleProgram { - TVector<TString> Srcs; - size_t Curr; - - TRunMultiplePrograms(const TVector<TString>& srcs, IOutputStream& err) - : TRunSingleProgram(TString(), err) - , Srcs(srcs) - , Curr(0) - { - } - - bool Run( - const NKikimr::NMiniKQL::IFunctionRegistry* funcReg - ) { - TString origTmpDir = TmpDir; - if (TmpDir) { - TFsPath newTmp = TFsPath(TmpDir) / ToString(Curr); - newTmp.MkDirs(); - TmpDir = newTmp.GetPath(); - } - Src = Srcs[Curr]; - if (!TRunSingleProgram::Run(funcReg)) { - return false; - } - ui32 idx = 0; - for (auto& resStr: Res) { - NYT::TNode res; - if (!NCommon::ParseYson(res, resStr, Err)) { - return false; - } - if (!res.IsMap() || !res.HasKey("Write") || !res["Write"].IsList()) { - Err << "Invalid result: " << resStr << Endl; - return false; - } - for (auto& elem: res["Write"].AsList()) { - if (!elem.IsMap()) { - Err << "Invalid result element in result: " << resStr << Endl; - return false; - } - if (elem.HasKey("Ref")) { - if (!elem["Ref"].IsList()) { - Err << "Invalid reference in result: " << resStr << Endl; - return false; - } - for (auto& refElem: elem["Ref"].AsList()) { - if (!refElem.IsMap() || !refElem.HasKey("Reference")) { - Err << "Invalid reference in result: " << resStr << Endl; - return false; - } - if (!refElem["Remove"].AsBool()) { - continue; - } - const auto& ref = refElem["Reference"].AsList(); - TStringStream name; - name << ref[0].AsString() << "." << ref[1].AsString() << ".Result" << Curr << "_" << idx; - Tables[name.Str()] = BuildFileNameForTmpTable(ref[2].AsString(), TmpDir); - ++idx; - } - } - } - } - ++Curr; - origTmpDir.swap(TmpDir); - return true; - } - - bool Finished() const { - return Curr == Srcs.size(); - } - }; - - template <typename TDriver> - TVector<TString> Run(TDriver& driver) { - auto functionRegistry = NKikimr::NMiniKQL::CreateFunctionRegistry(NKikimr::NMiniKQL::CreateBuiltinRegistry()); - - TVector<TString> res; - do { - const bool runRes = driver.Run(functionRegistry.Get()); - UNIT_ASSERT(runRes); - - driver.AddResults(res); - } while (!driver.Finished()); - return res; - } - - TVector<TString> RunProgram(const TString& programSrc, const THashMap<TString, TString>& tables, const TString& tmpDir = TString(), const TString& params = TString()) { - TRunSingleProgram driver(programSrc, Cerr); - driver.Tables = tables; - driver.TmpDir = tmpDir; - driver.Parameters = params; - return Run(driver); - } - - static const TStringBuf KSV_ATTRS = - "{\"_yql_row_spec\" = {\"Type\" = [\"StructType\";[" - "[\"key\";[\"DataType\";\"String\"]];" - "[\"subkey\";[\"DataType\";\"String\"]];" - "[\"value\";[\"DataType\";\"String\"]]" - "]]}}" - ; - - Y_UNIT_TEST_SUITE(ExecutionYqlExpr) { - Y_UNIT_TEST(WriteToResultUsingIsolatedGraph) { - auto s = "(\n" - "(let res_sink (DataSink 'result))\n" - "(let data (AsList (String 'x)))\n" - "(let world (Write! world res_sink (Key) data '()))\n" - "(let world (Commit! world res_sink))\n" - "(return world)\n" - ")\n"; - - auto res = RunProgram(s, THashMap<TString, TString>()); - UNIT_ASSERT_VALUES_EQUAL(res.size(), 1); - UNIT_ASSERT_NO_DIFF("{\"Write\"=[{\"Data\"=[\"x\"]}]}", res[0]); - } - - Y_UNIT_TEST(WriteToResultTableOutput) { - TTempFileHandle inputFile; - TTempFileHandle inputFileAttrs(inputFile.Name() + ".attr"); - - TStringBuf data = - "{\"key\"=\"075\";\"subkey\"=\".\";\"value\"=\"abc\"};\n" - "{\"key\"=\"800\";\"subkey\"=\".\";\"value\"=\"ddd\"};\n" - "{\"key\"=\"020\";\"subkey\"=\".\";\"value\"=\"q\"};\n" - "{\"key\"=\"150\";\"subkey\"=\".\";\"value\"=\"qzz\"};\n"sv - ; - - inputFile.Write(data.data(), data.size()); - inputFile.FlushData(); - inputFileAttrs.Write(KSV_ATTRS.data(), KSV_ATTRS.size()); - inputFileAttrs.FlushData(); - - THashMap<TString, TString> tables; - tables["yt.plato.Input"] = inputFile.Name(); - - auto s = "(\n" - "(let mr_source (DataSource 'yt 'plato))\n" - "(let x (Read! world mr_source (Key '('table (String 'Input))) '('key 'subkey 'value) '()))\n" - "(let world (Left! x))\n" - "(let table1 (Right! x))\n" - "(let res_sink (DataSink 'result))\n" - "(let data (AsList (String 'x)))\n" - "(let world (Write! world res_sink (Key) table1 '()))\n" - "(let world (Commit! world res_sink))\n" - "(return world)\n" - ")\n"; - - auto res = RunProgram(s, tables); - UNIT_ASSERT_VALUES_EQUAL(res.size(), 1); - UNIT_ASSERT_NO_DIFF( - "{\"Write\"=[{\"Data\"=[" - "[\"075\";\".\";\"abc\"];" - "[\"800\";\".\";\"ddd\"];" - "[\"020\";\".\";\"q\"];" - "[\"150\";\".\";\"qzz\"]" - "]}]}", - res[0] - ); - } - - Y_UNIT_TEST(WriteToResultTransformedTable) { - TTempFileHandle inputFile; - TTempFileHandle inputFileAttrs(inputFile.Name() + ".attr"); - - TStringBuf data = - "{\"key\"=\"075\";\"subkey\"=\".\";\"value\"=\"abc\"};\n" - "{\"key\"=\"800\";\"subkey\"=\".\";\"value\"=\"ddd\"};\n" - "{\"key\"=\"020\";\"subkey\"=\".\";\"value\"=\"q\"};\n" - "{\"key\"=\"150\";\"subkey\"=\".\";\"value\"=\"qzz\"};\n"sv - ; - - inputFile.Write(data.data(), data.size()); - inputFile.FlushData(); - inputFileAttrs.Write(KSV_ATTRS.data(), KSV_ATTRS.size()); - inputFileAttrs.FlushData(); - - THashMap<TString, TString> tables; - tables["yt.plato.Input"] = inputFile.Name(); - - auto s = "(\n" - "(let mr_source (DataSource 'yt 'plato))\n" - "(let x (Read! world mr_source (Key '('table (String 'Input))) '('key 'subkey 'value) '()))\n" - "(let world (Left! x))\n" - "(let table1 (Right! x))\n" - "(let table1low (FlatMap table1 (lambda '(item) (block '(\n" - " (let intValueOpt (FromString (Member item 'key) 'Int32))\n" - " (let ret (FlatMap intValueOpt (lambda '(item2) (block '(\n" - " (return (ListIf (< item2 (Int32 '100)) item))\n" - " )))))" - " (return ret)" - ")))))" - "(let res_sink (DataSink 'result))\n" - "(let data (AsList (String 'x)))\n" - "(let world (Write! world res_sink (Key) table1low '()))\n" - "(let world (Commit! world res_sink))\n" - "(return world)\n" - ")\n"; - - auto res = RunProgram(s, tables); - UNIT_ASSERT_VALUES_EQUAL(res.size(), 1); - UNIT_ASSERT_NO_DIFF( - "{\"Write\"=[{\"Data\"=[" - "[\"075\";\".\";\"abc\"];" - "[\"020\";\".\";\"q\"]" - "]}]}", - res[0] - ); - } - - Y_UNIT_TEST(DropTable) { - TTempFileHandle outputFile; - TTempFileHandle outputFileAttrs(outputFile.Name() + ".attr"); - - TStringBuf data = - "{\"key\"=\"075\";\"subkey\"=\".\";\"value\"=\"abc\"};\n"sv - ; - - outputFile.Write(data.data(), data.size()); - outputFile.FlushData(); - outputFile.Close(); - outputFileAttrs.Write(KSV_ATTRS.data(), KSV_ATTRS.size()); - outputFileAttrs.FlushData(); - outputFileAttrs.Close(); - UNIT_ASSERT(TFileStat(outputFile.Name()).IsFile()); - - THashMap<TString, TString> tables; - tables["yt.plato.Output"] = outputFile.Name(); - - auto s = "(\n" - "(let mr_sink (DataSink 'yt 'plato))\n" - "(let world (Write! world mr_sink (Key '('table (String 'Output))) (Void) '('('mode 'drop))))\n" - "(let world (Commit! world mr_sink))\n" - "(return world)\n" - ")\n"; - - RunProgram(s, tables); - - UNIT_ASSERT(!TFileStat(outputFile.Name()).IsFile()); - } - - Y_UNIT_TEST(WriteToResultTableByRef) { - TTempFileHandle inputFile; - TTempFileHandle inputFileAttrs(inputFile.Name() + ".attr"); - - TStringBuf data = - "{\"key\"=\"075\";\"subkey\"=\".\";\"value\"=\"abc\"};\n" - "{\"key\"=\"800\";\"subkey\"=\".\";\"value\"=\"ddd\"};\n" - "{\"key\"=\"020\";\"subkey\"=\".\";\"value\"=\"q\"};\n" - "{\"key\"=\"150\";\"subkey\"=\".\";\"value\"=\"qzz\"};\n"sv - ; - - inputFile.Write(data.data(), data.size()); - inputFile.FlushData(); - inputFileAttrs.Write(KSV_ATTRS.data(), KSV_ATTRS.size()); - inputFileAttrs.FlushData(); - - THashMap<TString, TString> tables; - tables["yt.plato.Input"] = inputFile.Name(); - - auto s = "(\n" - "(let mr_source (DataSource 'yt 'plato))\n" - "(let x (Read! world mr_source (Key '('table (String 'Input))) '('key 'subkey 'value) '()))\n" - "(let world (Left! x))\n" - "(let table1 (Right! x))\n" - "(let res_sink (DataSink 'result))\n" - "(let mr_sink (DataSink 'yt 'plato))\n" - "(let world (Write! world res_sink (Key) table1 '('('ref))))\n" - "(let world (Commit! world mr_sink))\n" - "(let world (Commit! world res_sink))\n" - "(return world)\n" - ")\n"; - - auto res = RunProgram(s, tables); - UNIT_ASSERT_VALUES_EQUAL(res.size(), 1); - UNIT_ASSERT_NO_DIFF( - "{\"Write\"=[{\"Ref\"=[" - "{\"Reference\"=[\"yt\";\"plato\";\"Input\"];\"Columns\"=[\"key\";\"subkey\";\"value\"];\"Remove\"=%false}" - "]}]}", - res[0] - ); - } - - Y_UNIT_TEST(WriteToResultTransformedTableByRef) { - TTempFileHandle inputFile; - TTempFileHandle inputFileAttrs(inputFile.Name() + ".attr"); - TTempDir tmpDir; - - TStringBuf data = - "{\"key\"=\"075\";\"subkey\"=\".\";\"value\"=\"abc\"};\n" - "{\"key\"=\"800\";\"subkey\"=\".\";\"value\"=\"ddd\"};\n" - "{\"key\"=\"020\";\"subkey\"=\".\";\"value\"=\"q\"};\n" - "{\"key\"=\"150\";\"subkey\"=\".\";\"value\"=\"qzz\"};\n"sv - ; - - inputFile.Write(data.data(), data.size()); - inputFile.FlushData(); - inputFileAttrs.Write(KSV_ATTRS.data(), KSV_ATTRS.size()); - inputFileAttrs.FlushData(); - - TVector<TString> progs; - progs.push_back( - "(\n" - "(let mr_source (DataSource 'yt 'plato))\n" - "(let x (Read! world mr_source (Key '('table (String 'Input))) '('key 'subkey 'value) '()))\n" - "(let world (Left! x))\n" - "(let table1 (Right! x))\n" - "(let table1low (FlatMap table1 (lambda '(item) (block '(\n" - " (let intValueOpt (FromString (Member item 'key) 'Int32))\n" - " (let ret (FlatMap intValueOpt (lambda '(item2) (block '(\n" - " (return (ListIf (< item2 (Int32 '100)) item))\n" - " )))))" - " (return ret)" - ")))))" - "(let res_sink (DataSink 'result))\n" - "(let mr_sink (DataSink 'yt 'plato))\n" - "(let world (Write! world res_sink (Key) table1low '('('ref))))\n" - "(let world (Commit! world mr_sink))\n" - "(let world (Commit! world res_sink))\n" - "(return world)\n" - ")\n" - ); - - progs.push_back( - "(\n" - "(let mr_source (DataSource 'yt 'plato))\n" - "(let x (Read! world mr_source (Key '('table (String 'Result0_0))) '('key 'subkey 'value) '()))\n" - "(let world (Left! x))\n" - "(let table1 (Right! x))\n" - "(let res_sink (DataSink 'result))\n" - "(let data (AsList (String 'x)))\n" - "(let world (Write! world res_sink (Key) table1 '()))\n" - "(let world (Commit! world res_sink))\n" - "(return world)\n" - ")\n" - ); - - TRunMultiplePrograms driver(progs, Cerr); - driver.Tables["yt.plato.Input"] = inputFile.Name(); - driver.TmpDir = tmpDir.Name(); - - auto res = Run(driver); - - UNIT_ASSERT_VALUES_EQUAL(res.size(), 2); - UNIT_ASSERT_NO_DIFF( - "{\"Write\"=[{\"Ref\"=[" - "{\"Reference\"=[\"yt\";\"plato\";\"tmp/bb686f68-2245bd5f-2318fa4e-1\"];\"Columns\"=[\"key\";\"subkey\";\"value\"];\"Remove\"=%true}" - "]}]}", - res[0] - ); - UNIT_ASSERT_NO_DIFF( - "{\"Write\"=[{\"Data\"=[" - "[\"075\";\".\";\"abc\"];" - "[\"020\";\".\";\"q\"]" - "]}]}", - res[1] - ); - } - - Y_UNIT_TEST(WriteAndTakeResult) { - TTempFileHandle inputFile; - TTempFileHandle inputFileAttrs(inputFile.Name() + ".attr"); - TTempDir tmpDir; - - TStringBuf data = - "{\"key\"=\"075\";\"subkey\"=\".\";\"value\"=\"abc\"};\n" - "{\"key\"=\"800\";\"subkey\"=\".\";\"value\"=\"ddd\"};\n" - "{\"key\"=\"020\";\"subkey\"=\".\";\"value\"=\"q\"};\n" - "{\"key\"=\"150\";\"subkey\"=\".\";\"value\"=\"qzz\"};\n"sv - ; - - inputFile.Write(data.data(), data.size()); - inputFile.FlushData(); - inputFileAttrs.Write(KSV_ATTRS.data(), KSV_ATTRS.size()); - inputFileAttrs.FlushData(); - - TVector<TString> progs; - progs.push_back( - "(\n" - "(let mr_source (DataSource 'yt 'plato))\n" - "(let x (Read! world mr_source (Key '('table (String 'Input))) '('key 'subkey 'value) '()))\n" - "(let world (Left! x))\n" - "(let table (Right! x))\n" - "(let result (Map table (lambda '(item) (block '(" - " (let res (Struct))" - " (let res (AddMember res 'k (Member item 'key)))" - " (let res (AddMember res 's (Member item 'subkey)))" - " (let res (AddMember res 'v (Member item 'value)))" - " (return res)" - ")))))" - "(let res_sink (DataSink 'result))\n" - "(let mr_sink (DataSink 'yt 'plato))\n" - "(let world (Write! world res_sink (Key) result '('('ref))))\n" - "(let world (Commit! world mr_sink))\n" - "(let world (Commit! world res_sink))\n" - "(return world)\n" - ")\n" - ); - - progs.push_back( - "(\n" - "(let mr_source (DataSource 'yt 'plato))\n" - "(let x (Read! world mr_source (Key '('table (String 'Result0_0))) '('k 's 'v) '()))\n" - "(let world (Left! x))\n" - "(let table (Right! x))\n" - "(let result (Take table (Uint64 '2)))" - "(let res_sink (DataSink 'result))\n" - "(let mr_sink (DataSink 'yt 'plato))\n" - "(let world (Write! world res_sink (Key) result '('('type))))\n" - "(let world (Commit! world mr_sink))\n" - "(let world (Commit! world res_sink))\n" - "(return world)\n" - ")\n" - ); - - TRunMultiplePrograms driver(progs, Cerr); - driver.Tables["yt.plato.Input"] = inputFile.Name(); - driver.TmpDir = tmpDir.Name(); - - auto res = Run(driver); - - UNIT_ASSERT_VALUES_EQUAL(res.size(), 2); - - //~ Cerr << res[0] << Endl; - //~ Cerr << res[1] << Endl; - UNIT_ASSERT_NO_DIFF( - "{\"Write\"=[{\"Ref\"=[" - "{\"Reference\"=[\"yt\";\"plato\";\"tmp/bb686f68-2245bd5f-2318fa4e-1\"];\"Columns\"=[\"k\";\"s\";\"v\"];\"Remove\"=%true}" - "]}]}", - res[0] - ); - UNIT_ASSERT_NO_DIFF( - "{\"Write\"=[{" - "\"Type\"=[\"ListType\";[\"StructType\";[" - "[\"k\";[\"DataType\";\"String\"]];[\"s\";[\"DataType\";\"String\"]];[\"v\";[\"DataType\";\"String\"]]" - "]]];" - "\"Data\"=[" - "[\"075\";\".\";\"abc\"];" - "[\"800\";\".\";\"ddd\"]" - "]}" - "]}", - res[1] - ); - } - - Y_UNIT_TEST(WriteAndReadScheme) { - TTempFileHandle inputFile; - TTempFileHandle inputFileAttrs(inputFile.Name() + ".attr"); - TTempDir tmpDir; - - TStringBuf data = - "{\"key\"=\"075\";\"subkey\"=\".\";\"value\"=\"abc\"};\n" - "{\"key\"=\"800\";\"subkey\"=\".\";\"value\"=\"ddd\"};\n" - "{\"key\"=\"020\";\"subkey\"=\".\";\"value\"=\"q\"};\n" - "{\"key\"=\"150\";\"subkey\"=\".\";\"value\"=\"qzz\"};\n"sv - ; - - inputFile.Write(data.data(), data.size()); - inputFile.FlushData(); - inputFileAttrs.Write(KSV_ATTRS.data(), KSV_ATTRS.size()); - inputFileAttrs.FlushData(); - - TVector<TString> progs; - progs.push_back( - "(\n" - "(let mr_source (DataSource 'yt 'plato))\n" - "(let x (Read! world mr_source (Key '('table (String 'Input))) '('key 'subkey 'value) '()))\n" - "(let world (Left! x))\n" - "(let table (Right! x))\n" - "(let result0 (Map table (lambda '(item) (block '(" - " (return (AsStruct '('bar (Coalesce (FromString (Member item 'key) 'Uint64) (Uint64 '0)))))" - ")))))" - "(let result1 (Map result0 (lambda '(item) (block '(" - " (return (AddMember (Struct) 'foo item))" - ")))))" - "(let res_sink (DataSink 'result))\n" - "(let mr_sink (DataSink 'yt 'plato))\n" - "(let world (Write! world res_sink (Key) result0 '('('ref))))\n" - "(let world (Write! world res_sink (Key) result1 '('('ref))))\n" - "(let world (Commit! world mr_sink))\n" - "(let world (Commit! world res_sink))\n" - "(return world)\n" - ")\n" - ); - - progs.push_back( - "(\n" - "(let mr_source (DataSource 'yt 'plato))\n" - "(let x (Read! world mr_source (Key '('tablescheme (String 'Result0_0))) (Void) '()))\n" - "(let world (Left! x))\n" - "(let scheme (Right! x))\n" - "(let res_sink (DataSink 'result))\n" - "(let mr_sink (DataSink 'yt 'plato))\n" - "(let world (Write! world res_sink (Key) scheme '('('type))))\n" - "(let world (Commit! world mr_sink))\n" - "(let world (Commit! world res_sink))\n" - "(return world)\n" - ")\n" - ); - - progs.push_back( - "(\n" - "(let mr_source (DataSource 'yt 'plato))\n" - "(let x (Read! world mr_source (Key '('tablescheme (String 'Result0_1))) (Void) '()))\n" - "(let world (Left! x))\n" - "(let scheme (Right! x))\n" - "(let res_sink (DataSink 'result))\n" - "(let mr_sink (DataSink 'yt 'plato))\n" - "(let world (Write! world res_sink (Key) scheme '('('type))))\n" - "(let world (Commit! world mr_sink))\n" - "(let world (Commit! world res_sink))\n" - "(return world)\n" - ")\n" - ); - - TRunMultiplePrograms driver(progs, Cerr); - driver.Tables["yt.plato.Input"] = inputFile.Name(); - driver.TmpDir = tmpDir.Name(); - - auto res = Run(driver); - - UNIT_ASSERT_VALUES_EQUAL(res.size(), 3); - - //~ Cerr << res[0] << Endl; - //~ Cerr << res[1] << Endl; - //~ Cerr << res[2] << Endl; - UNIT_ASSERT_NO_DIFF( - "{\"Write\"=[" - "{\"Ref\"=[{\"Reference\"=[\"yt\";\"plato\";\"tmp/bb686f68-2245bd5f-2318fa4e-1\"];\"Columns\"=[\"bar\"];\"Remove\"=%true}]};" - "{\"Ref\"=[{\"Reference\"=[\"yt\";\"plato\";\"tmp/7ae6459a-7382d1e7-7935c08e-2\"];\"Columns\"=[\"foo\"];\"Remove\"=%true}]}" - "]}", - res[0] - ); - UNIT_ASSERT(res[1].find("\"Fields\"=[{\"Name\"=\"bar\"") != TString::npos); - UNIT_ASSERT(res[2].find("\"Fields\"=[{\"Name\"=\"foo\"") != TString::npos); - } - - Y_UNIT_TEST(ExtendSortedWithNonSortedAndRead) { - TTempFileHandle inputFile; - TTempFileHandle inputFileAttrs(inputFile.Name() + ".attr"); - TTempFileHandle outputFile; - TTempFile outputFileAttr(outputFile.Name() + ".attr"); - TTempDir tmpDir; - - TStringBuf data = - "{\"key\"=\"foo\";\"subkey\"=\"wat\";\"value\"=\"222\"};\n" - "{\"key\"=\"bar\";\"subkey\"=\"wat\";\"value\"=\"111\"};\n" - "{\"key\"=\"jar\";\"subkey\"=\"wat\";\"value\"=\"333\"};\n"sv - ; - - inputFile.Write(data.data(), data.size()); - inputFile.FlushData(); - inputFileAttrs.Write(KSV_ATTRS.data(), KSV_ATTRS.size()); - inputFileAttrs.FlushData(); - - TVector<TString> progs; - progs.push_back( - "(\n" - "(let source (DataSource 'yt 'plato))\n" - "(let x (Read! world source (Key '('table (String 'Input))) '('key 'value) '()))\n" - "(let world (Left! x))\n" - "(let table (Right! x))\n" - "(let sorted (Sort table (Bool 'true) (lambda '(item) (Member item 'value))))\n" - "(let result (Extend table sorted))\n" - "(let sink (DataSink 'yt 'plato))\n" - "(let world (Write! world sink (Key '('table (String 'Output))) result '()))\n" - "(let world (Commit! world sink))\n" - "(return world)\n" - ")\n" - ); - - progs.push_back( - "(\n" - "(let mr_source (DataSource 'yt 'plato))\n" - "(let x (Read! world mr_source (Key '('table (String 'Output))) '('key 'value) '()))\n" - "(let world (Left! x))\n" - "(let result (Right! x))\n" - "(let res_sink (DataSink 'result))\n" - "(let mr_sink (DataSink 'yt 'plato))\n" - "(let world (Write! world res_sink (Key) result '('('type))))\n" - "(let world (Commit! world mr_sink))\n" - "(let world (Commit! world res_sink))\n" - "(return world)\n" - ")\n" - ); - - TRunMultiplePrograms driver(progs, Cerr); - driver.TmpDir = tmpDir.Name(); - driver.Tables["yt.plato.Input"] = inputFile.Name(); - driver.Tables["yt.plato.Output"] = outputFile.Name(); - - auto res = Run(driver); - - UNIT_ASSERT_VALUES_EQUAL(res.size(), 1); - - //~ Cerr << res[0] << Endl; - UNIT_ASSERT_NO_DIFF( - "{\"Write\"=[{" - "\"Type\"=[\"ListType\";[\"StructType\";[[\"key\";[\"DataType\";\"String\"]];[\"value\";[\"DataType\";\"String\"]]]]];" - "\"Data\"=[" - "[\"foo\";\"222\"];[\"bar\";\"111\"];[\"jar\";\"333\"];[\"foo\";\"222\"];[\"bar\";\"111\"];[\"jar\";\"333\"]" - "]}]}", - res[0] - ); - } - - Y_UNIT_TEST(OrderedExtendSortedWithNonSortedAndRead) { - TTempFileHandle inputFile; - TTempFileHandle inputFileAttrs(inputFile.Name() + ".attr"); - TTempFileHandle outputFile; - TTempFile outputFileAttr(outputFile.Name() + ".attr"); - TTempDir tmpDir; - - TStringBuf data = - "{\"key\"=\"foo\";\"subkey\"=\"wat\";\"value\"=\"222\"};\n" - "{\"key\"=\"bar\";\"subkey\"=\"wat\";\"value\"=\"111\"};\n" - "{\"key\"=\"jar\";\"subkey\"=\"wat\";\"value\"=\"333\"};\n"sv - ; - - inputFile.Write(data.data(), data.size()); - inputFile.FlushData(); - inputFileAttrs.Write(KSV_ATTRS.data(), KSV_ATTRS.size()); - inputFileAttrs.FlushData(); - - TVector<TString> progs; - progs.push_back( - "(\n" - "(let source (DataSource 'yt 'plato))\n" - "(let x (Read! world source (Key '('table (String 'Input))) '('key 'value) '()))\n" - "(let world (Left! x))\n" - "(let table (Right! x))\n" - "(let sorted (Sort table (Bool 'true) (lambda '(item) (Member item 'value))))\n" - "(let result (OrderedExtend table sorted))\n" - "(let sink (DataSink 'yt 'plato))\n" - "(let world (Write! world sink (Key '('table (String 'Output))) result '()))\n" - "(let world (Commit! world sink))\n" - "(return world)\n" - ")\n" - ); - - progs.push_back( - "(\n" - "(let mr_source (DataSource 'yt 'plato))\n" - "(let x (Read! world mr_source (Key '('table (String 'Output))) '('key 'value) '()))\n" - "(let world (Left! x))\n" - "(let result (Right! x))\n" - "(let res_sink (DataSink 'result))\n" - "(let mr_sink (DataSink 'yt 'plato))\n" - "(let world (Write! world res_sink (Key) result '('('type))))\n" - "(let world (Commit! world mr_sink))\n" - "(let world (Commit! world res_sink))\n" - "(return world)\n" - ")\n" - ); - - TRunMultiplePrograms driver(progs, Cerr); - driver.TmpDir = tmpDir.Name(); - driver.Tables["yt.plato.Input"] = inputFile.Name(); - driver.Tables["yt.plato.Output"] = outputFile.Name(); - - auto res = Run(driver); - - UNIT_ASSERT_VALUES_EQUAL(res.size(), 1); - - //~ Cerr << res[0] << Endl; - UNIT_ASSERT_NO_DIFF( - "{\"Write\"=[{" - "\"Type\"=[\"ListType\";[\"StructType\";[[\"key\";[\"DataType\";\"String\"]];[\"value\";[\"DataType\";\"String\"]]]]];" - "\"Data\"=[" - "[\"foo\";\"222\"];[\"bar\";\"111\"];[\"jar\";\"333\"];[\"bar\";\"111\"];[\"foo\";\"222\"];[\"jar\";\"333\"]" - "]}]}", - res[0] - ); - } - - Y_UNIT_TEST(TestParametersEvaluation) { - auto s = "(\n" - "(let res_sink (DataSink 'result))\n" - "(let data (Parameter '\"$foo\" (ParseType '\"Tuple<String, Int32 ? , List<Uint32>, Dict<Int32, Bool>, Struct<a : Void, b : Double>, Variant<Int32, Bool>>\")))\n" - "(let world (Write! world res_sink (Key) data '('('type))))\n" - "(let world (Commit! world res_sink))\n" - "(return world)\n" - ")\n"; - - auto params = R"__( -{"$foo"={Data=[ - bar; - "33"; - ["1";"2";"3"]; - [["7";%true];["12";%false]]; - [#;"-1.7"]; - ["0";"8"]; -]}} - )__"; - - auto res = RunProgram(s, THashMap<TString, TString>(), "", params); - UNIT_ASSERT_VALUES_EQUAL(res.size(), 1); - UNIT_ASSERT_NO_DIFF(R"__({"Write"=[{"Type"=["TupleType";[["DataType";"String"];["OptionalType";["DataType";"Int32"]];["ListType";["DataType";"Uint32"]];["DictType";["DataType";"Int32"];["DataType";"Bool"]];["StructType";[["a";["VoidType"]];["b";["DataType";"Double"]]]];["VariantType";["TupleType";[["DataType";"Int32"];["DataType";"Bool"]]]]]];"Data"=["bar";["33"];["1";"2";"3"];[["7";%true];["12";%false]];["Void";"-1.7"];["0";"8"]]}]})__", res[0]); - } - } - -} // namespace NYql diff --git a/yql/essentials/core/ut/yql_expr_constraint_ut.cpp b/yql/essentials/core/ut/yql_expr_constraint_ut.cpp index d83c2a15c04..e9b3efd69f1 100644 --- a/yql/essentials/core/ut/yql_expr_constraint_ut.cpp +++ b/yql/essentials/core/ut/yql_expr_constraint_ut.cpp @@ -1,11 +1,9 @@ -#include <yt/yql/providers/yt/provider/yql_yt_provider.h> -#include <yt/yql/providers/yt/gateway/file/yql_yt_file.h> #include <yql/essentials/providers/common/provider/yql_provider_names.h> +#include <yql/essentials/providers/result/provider/yql_result_provider.h> #include <yql/essentials/ast/yql_ast_annotation.h> #include <yql/essentials/ast/yql_expr.h> #include <yql/essentials/core/type_ann/type_ann_core.h> #include <yql/essentials/core/type_ann/type_ann_expr.h> -#include <yql/essentials/core/ut_common/yql_ut_common.h> #include <yql/essentials/core/yql_graph_transformer.h> #include <yql/essentials/core/yql_expr_optimize.h> #include <yql/essentials/core/services/yql_transform_pipeline.h> @@ -26,17 +24,15 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { UNIT_ASSERT(CompileExpr(*astRes.Root, exprRoot, exprCtx, nullptr, nullptr)); auto functionRegistry = NKikimr::NMiniKQL::CreateFunctionRegistry(NKikimr::NMiniKQL::CreateBuiltinRegistry()); - TTestTablesMapping testTables; - auto yqlNativeServices = NFile::TYtFileServices::Make(functionRegistry.Get(), testTables); - auto ytGateway = CreateYtFileGateway(yqlNativeServices); auto typeAnnotationContext = MakeIntrusive<TTypeAnnotationContext>(); typeAnnotationContext->RandomProvider = CreateDeterministicRandomProvider(1); - auto ytState = MakeIntrusive<TYtState>(typeAnnotationContext.Get()); - ytState->Gateway = ytGateway; - InitializeYtGateway(ytGateway, ytState); - typeAnnotationContext->AddDataSink(YtProviderName, CreateYtDataSink(ytState)); - typeAnnotationContext->AddDataSource(YtProviderName, CreateYtDataSource(ytState)); + auto writerFactory = [](){ return CreateYsonResultWriter(NYson::EYsonFormat::Binary); }; + auto resultProviderConfig = MakeIntrusive<TResultProviderConfig>(*typeAnnotationContext, + *functionRegistry, IDataProvider::EResultFormat::Yson, ToString((ui32)NYson::EYsonFormat::Binary), writerFactory); + resultProviderConfig->SupportsResultPosition = true; + auto resultProvider = CreateResultProvider(resultProviderConfig); + typeAnnotationContext->AddDataSink(ResultProviderName, resultProvider); auto transformer = TTransformationPipeline(typeAnnotationContext) .AddServiceTransformers() @@ -77,15 +73,15 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { Y_UNIT_TEST(Sort) { const auto s = R"(( - (let mr_sink (DataSink 'yt (quote plato))) + (let res (DataSink 'result)) (let list (AsList (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value (String 'v))) (AsStruct '('key (String '1)) '('subkey (String 'd)) '('value (String 'v))) (AsStruct '('key (String '3)) '('subkey (String 'b)) '('value (String 'v))) )) (let sorted (Sort list (Bool 'True) (lambda '(item) (Member item 'key)))) - (let world (Write! world mr_sink (Key '('table (String 'Output))) sorted '('('mode 'renew)))) - (let world (Commit! world mr_sink)) + (let world (Write! world res (Key) sorted '())) + (let world (Commit! world res)) (return world) ))"; @@ -96,15 +92,15 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { Y_UNIT_TEST(SortByStablePickle) { const auto s = R"(( - (let mr_sink (DataSink 'yt (quote plato))) + (let res (DataSink 'result)) (let list (AsList (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value (String 'v))) (AsStruct '('key (String '1)) '('subkey (String 'd)) '('value (String 'v))) (AsStruct '('key (String '3)) '('subkey (String 'b)) '('value (String 'v))) )) (let sorted (Sort list '((Bool 'False) (Bool 'True)) (lambda '(item) '((Member item 'key) (StablePickle (Member item 'subkey)))))) - (let world (Write! world mr_sink (Key '('table (String 'Output))) sorted '('('mode 'renew)))) - (let world (Commit! world mr_sink)) + (let world (Write! world res (Key) sorted '())) + (let world (Commit! world res)) (return world) ))"; @@ -115,7 +111,7 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { Y_UNIT_TEST(SortByTranspentIfPresent) { const auto s = R"(( - (let mr_sink (DataSink 'yt (quote plato))) + (let res (DataSink 'result)) (let list (AsList (Just (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value (String 'v)))) (Just (AsStruct '('key (String '1)) '('subkey (String 'd)) '('value (String 'v)))) @@ -123,8 +119,8 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { )) (let sorted (Sort list (Bool 'True) (lambda '(row) (FlatMap row (lambda '(item) (Just (Member item 'key))))))) (let map (OrderedFlatMap sorted (lambda '(item) item))) - (let world (Write! world mr_sink (Key '('table (String 'Output))) map '('('mode 'renew)))) - (let world (Commit! world mr_sink)) + (let world (Write! world res (Key) map '())) + (let world (Commit! world res)) (return world) ))"; @@ -135,15 +131,15 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { Y_UNIT_TEST(SortByDuplicateColumn) { const auto s = R"(( - (let mr_sink (DataSink 'yt (quote plato))) + (let res (DataSink 'result)) (let list (AsList (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value (String 'v))) (AsStruct '('key (String '1)) '('subkey (String 'd)) '('value (String 'v))) (AsStruct '('key (String '3)) '('subkey (String 'b)) '('value (String 'v))) )) (let sorted (Sort list '((Bool 'True) (Bool 'False) (Bool 'True)) (lambda '(item) '((Member item 'key) (Member item 'key) (Member item 'subkey))))) - (let world (Write! world mr_sink (Key '('table (String 'Output))) sorted '('('mode 'renew)))) - (let world (Commit! world mr_sink)) + (let world (Write! world res (Key) sorted '())) + (let world (Commit! world res)) (return world) ))"; @@ -154,12 +150,12 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { Y_UNIT_TEST(SortByFullRow) { const auto s = R"(( - (let mr_sink (DataSink 'yt (quote plato))) + (let res (DataSink 'result)) (let list (AsList (Utf8 's) (Utf8 'o) (Utf8 'r) (Utf8 't))) (let sorted (Sort list (Bool 'True) (lambda '(item) item))) (let map (OrderedMap sorted (lambda '(item) (AsStruct '('key item))))) - (let world (Write! world mr_sink (Key '('table (String 'Output))) map '('('mode 'renew)))) - (let world (Commit! world mr_sink)) + (let world (Write! world res (Key) map '())) + (let world (Commit! world res)) (return world) ))"; @@ -171,15 +167,15 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { Y_UNIT_TEST(SortByTuple) { const auto s = R"(( - (let mr_sink (DataSink 'yt (quote plato))) + (let res (DataSink 'result)) (let list (AsList (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value (String 'v))) (AsStruct '('key (String '1)) '('subkey (String 'd)) '('value (String 'v))) (AsStruct '('key (String '3)) '('subkey (String 'b)) '('value (String 'v))) )) (let sorted (Sort list (Bool 'False) (lambda '(item) '((Member item 'key) (Member item 'subkey))))) - (let world (Write! world mr_sink (Key '('table (String 'Output))) sorted '('('mode 'renew)))) - (let world (Commit! world mr_sink)) + (let world (Write! world res (Key) sorted '())) + (let world (Commit! world res)) (return world) ))"; @@ -190,7 +186,7 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { Y_UNIT_TEST(SortByTupleElements) { const auto s = R"(( - (let mr_sink (DataSink 'yt (quote plato))) + (let res (DataSink 'result)) (let list (AsList (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value '((String 'x) (String 'a) (String 'u)))) (AsStruct '('key (String '1)) '('subkey (String 'd)) '('value '((String 'y) (String 'b) (String 'v)))) @@ -198,8 +194,8 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { )) (let sorted (Sort list '((Not (Bool 'False)) (Bool 'False) (Not (Bool 'True))) (lambda '(item) '((Nth (Member item 'value) '2) (Member item 'key) (Nth (Member item 'value) '0))))) (let map (OrderedMap sorted (lambda '(item) (AsStruct '('tuple '((Nth (Member item 'value) '1) (Nth (Member item 'value) '2) (Nth (Member item 'value) '0) (Member item 'subkey) (Member item 'key))))))) - (let world (Write! world mr_sink (Key '('table (String 'Output))) map '('('mode 'renew)))) - (let world (Commit! world mr_sink)) + (let world (Write! world res (Key) map '())) + (let world (Commit! world res)) (return world) ))"; @@ -211,7 +207,7 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { Y_UNIT_TEST(SortByAllTupleElementsInRightOrder) { const auto s = R"(( - (let mr_sink (DataSink 'yt (quote plato))) + (let res (DataSink 'result)) (let list (AsList (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value '((String 'x) (String 'a) (String 'u)))) (AsStruct '('key (String '1)) '('subkey (String 'd)) '('value '((String 'y) (String 'b) (String 'v)))) @@ -219,8 +215,8 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { )) (let sorted (Sort list (Bool 'False) (lambda '(item) '((Nth (Member item 'value) '0) (Nth (Member item 'value) '1) (Nth (Member item 'value) '2))))) (let map (OrderedMap sorted (lambda '(item) (AsStruct '('tuple (Member item 'value)))))) - (let world (Write! world mr_sink (Key '('table (String 'Output))) map '('('mode 'renew)))) - (let world (Commit! world mr_sink)) + (let world (Write! world res (Key) map '())) + (let world (Commit! world res)) (return world) ))"; @@ -232,7 +228,7 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { Y_UNIT_TEST(SortByAllTupleElementsInWrongOrder) { const auto s = R"(( - (let mr_sink (DataSink 'yt (quote plato))) + (let res (DataSink 'result)) (let list (AsList (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value '((String 'x) (String 'a) (String 'u)))) (AsStruct '('key (String '1)) '('subkey (String 'd)) '('value '((String 'y) (String 'b) (String 'v)))) @@ -240,8 +236,8 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { )) (let sorted (Sort list (Bool 'True) (lambda '(item) '((Nth (Member item 'value) '1) (Nth (Member item 'value) '0) (Nth (Member item 'value) '2))))) (let map (OrderedMap sorted (lambda '(item) (AsStruct '('tuple (Member item 'value)))))) - (let world (Write! world mr_sink (Key '('table (String 'Output))) map '('('mode 'renew)))) - (let world (Commit! world mr_sink)) + (let world (Write! world res (Key) map '())) + (let world (Commit! world res)) (return world) ))"; @@ -253,7 +249,7 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { Y_UNIT_TEST(SortByTupleWithSingleElementAndCopyOfElement) { const auto s = R"(( - (let mr_sink (DataSink 'yt (quote plato))) + (let res (DataSink 'result)) (let list (AsList (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value '((String 'x)))) (AsStruct '('key (String '1)) '('subkey (String 'd)) '('value '((String 'y)))) @@ -261,8 +257,8 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { )) (let sorted (Sort list (Bool 'False) (lambda '(item) '((Nth (Member item 'value) '0))))) (let map (OrderedMap sorted (lambda '(item) (AsStruct '('tuple (Member item 'value)) '('element (Nth (Member item 'value) '0)))))) - (let world (Write! world mr_sink (Key '('table (String 'Output))) map '('('mode 'renew)))) - (let world (Commit! world mr_sink)) + (let world (Write! world res (Key) map '())) + (let world (Commit! world res)) (return world) ))"; @@ -273,7 +269,7 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { Y_UNIT_TEST(SortByFullTupleOnTop) { const auto s = R"(( - (let mr_sink (DataSink 'yt (quote plato))) + (let res (DataSink 'result)) (let list (AsList '((String 'x) (String 'a) (String 'u)) '((String 'y) (String 'b) (String 'v)) @@ -281,8 +277,8 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { )) (let sorted (Sort list (Bool 'False) (lambda '(item) item))) (let map (OrderedMap sorted (lambda '(item) (AsStruct '('one (Nth item '0)) '('two (Nth item '1)) '('three (Nth item '2)))))) - (let world (Write! world mr_sink (Key '('table (String 'Output))) map '('('mode 'renew)))) - (let world (Commit! world mr_sink)) + (let world (Write! world res (Key) map '())) + (let world (Commit! world res)) (return world) ))"; @@ -294,15 +290,15 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { Y_UNIT_TEST(SortByColumnAndExpr) { const auto s = R"(( - (let mr_sink (DataSink 'yt (quote plato))) + (let res (DataSink 'result)) (let list (AsList (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value (String 'v))) (AsStruct '('key (String '1)) '('subkey (String 'd)) '('value (String 'v))) (AsStruct '('key (String '3)) '('subkey (String 'b)) '('value (String 'v))) )) (let sorted (Sort list '((Bool 'True) (Bool 'False) (Bool 'True)) (lambda '(item) '((Member item 'key) (SafeCast (Member item 'value) (DataType 'Utf8)) (Member item 'subkey))))) - (let world (Write! world mr_sink (Key '('table (String 'Output))) sorted '('('mode 'renew)))) - (let world (Commit! world mr_sink)) + (let world (Write! world res (Key) sorted '())) + (let world (Commit! world res)) (return world) ))"; @@ -313,15 +309,15 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { Y_UNIT_TEST(SortDesc) { const auto s = R"(( - (let mr_sink (DataSink 'yt (quote plato))) + (let res (DataSink 'result)) (let list (AsList (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value (String 'v))) (AsStruct '('key (String '1)) '('subkey (String 'd)) '('value (String 'v))) (AsStruct '('key (String '3)) '('subkey (String 'b)) '('value (String 'v))) )) (let sorted (Sort list '((Bool 'True) (Bool 'False)) (lambda '(item) '((Member item 'key) (Member item 'subkey))))) - (let world (Write! world mr_sink (Key '('table (String 'Output))) sorted '('('mode 'renew)))) - (let world (Commit! world mr_sink)) + (let world (Write! world res (Key) sorted '())) + (let world (Commit! world res)) (return world) ))"; @@ -332,7 +328,7 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { Y_UNIT_TEST(SortedOverWideMap) { const auto s = R"(( - (let mr_sink (DataSink 'yt (quote plato))) + (let res (DataSink 'result)) (let list (AsList (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value (String 'v))) (AsStruct '('key (String '1)) '('subkey (String 'd)) '('value (String 'v))) @@ -342,8 +338,8 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { (let expand (ExpandMap sorted (lambda '(item) (Member item 'key) (Member item 'subkey) (Member item 'value)))) (let wide (WideMap expand (lambda '(a b c) c b a))) (let narrow (NarrowMap wide (lambda '(x y z) (AsStruct '('x x) '('y y) '('z z))))) - (let world (Write! world mr_sink (Key '('table (String 'Output))) (Collect narrow) '('('mode 'renew)))) - (let world (Commit! world mr_sink)) + (let world (Write! world res (Key) (Collect narrow) '())) + (let world (Commit! world res)) (return world) ))"; @@ -354,7 +350,7 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { Y_UNIT_TEST(SortedOverWideTopSort) { const auto s = R"(( - (let mr_sink (DataSink 'yt (quote plato))) + (let res (DataSink 'result)) (let list (AsList (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value (String 'v))) (AsStruct '('key (String '1)) '('subkey (String 'd)) '('value (String 'v))) @@ -364,8 +360,8 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { (let expand (ExpandMap flow (lambda '(item) (Member item 'key) (Member item 'subkey) (Member item 'value)))) (let wide (WideTopSort expand (Uint64 '2) '('('2 (Bool 'False)) '('0 (Bool 'True))))) (let narrow (NarrowMap wide (lambda '(x y z) (AsStruct '('x x) '('y y) '('z z))))) - (let world (Write! world mr_sink (Key '('table (String 'Output))) (Collect narrow) '('('mode 'renew)))) - (let world (Commit! world mr_sink)) + (let world (Write! world res (Key) (Collect narrow) '())) + (let world (Commit! world res)) (return world) ))"; @@ -376,7 +372,7 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { Y_UNIT_TEST(SortedOverOrderedMultiMap) { const auto s = R"(( - (let mr_sink (DataSink 'yt (quote plato))) + (let res (DataSink 'result)) (let list (AsList (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value (String 'u))) (AsStruct '('key (String '1)) '('subkey (String 'd)) '('value (String 'v))) @@ -387,8 +383,8 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { (AsStruct '('x (Member row 'key)) '('y (Member row 'subkey)) '('z (Member row 'value))) (AsStruct '('x (Member row 'key)) '('y (String 'subkey_stub)) '('z (Member row 'value))) ))) - (let world (Write! world mr_sink (Key '('table (String 'Output))) map '('('mode 'renew)))) - (let world (Commit! world mr_sink)) + (let world (Write! world res (Key) map '())) + (let world (Commit! world res)) (return world) ))"; @@ -399,7 +395,7 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { Y_UNIT_TEST(SortedOverNestedFlatMap) { const auto s = R"(( - (let mr_sink (DataSink 'yt (quote plato))) + (let res (DataSink 'result)) (let list (AsList (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value (String 'u))) (AsStruct '('key (String '1)) '('subkey (String 'd)) '('value (String 'v))) @@ -409,8 +405,8 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { (let map (OrderedFlatMap sorted (lambda '(row) (FlatMap (ListFromRange (Uint8 '0) (Uint8 '5) (Uint8 '1)) (lambda '(index) (OptionalIf (AggrNotEquals index (Uint8 '3)) (AsStruct '('x (Member row 'key)) '('y (Concat (ToString index) (Member row 'subkey))) '('z (Member row 'value)))) ))))) - (let world (Write! world mr_sink (Key '('table (String 'Output))) (LazyList map) '('('mode 'renew)))) - (let world (Commit! world mr_sink)) + (let world (Write! world res (Key) (LazyList map) '())) + (let world (Commit! world res)) (return world) ))"; @@ -421,7 +417,7 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { Y_UNIT_TEST(SortedOverNestedOrderedFlatMapWithSort) { const auto s = R"(( - (let mr_sink (DataSink 'yt (quote plato))) + (let res (DataSink 'result)) (let list (AsList (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value (String 'u))) (AsStruct '('key (String '1)) '('subkey (String 'd)) '('value (String 'v))) @@ -431,8 +427,8 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { (let map (OrderedFlatMap sorted (lambda '(row) (OrderedFlatMap (Sort (ListFromRange (Uint8 '0) (Uint8 '5) (Uint8 '1)) (Bool 'True) (lambda '(item) item)) (lambda '(index) (OptionalIf (AggrNotEquals index (Uint8 '3)) (AsStruct '('x (Member row 'key)) '('y (Concat (ToString index) (Member row 'subkey))) '('z (Member row 'value)))) ))))) - (let world (Write! world mr_sink (Key '('table (String 'Output))) (LazyList map) '('('mode 'renew)))) - (let world (Commit! world mr_sink)) + (let world (Write! world res (Key) (LazyList map) '())) + (let world (Commit! world res)) (return world) ))"; @@ -443,7 +439,7 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { Y_UNIT_TEST(ExtractSortedTuple) { const auto s = R"(( - (let mr_sink (DataSink 'yt (quote plato))) + (let res (DataSink 'result)) (let list (AsList (AsStruct '('a (String '4)) '('b (String 'c)) '('c (String 'x))) (AsStruct '('a (String '1)) '('b (String 'd)) '('c (String 'y))) @@ -452,8 +448,8 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { (let sorted (Sort list '((Bool 'True) (Bool 'True)) (lambda '(item) '((Member item 'a) (Member item 'b))))) (let map (OrderedFlatMap sorted (lambda '(item) (OptionalIf (== (Member item 'a) (String '1)) (AddMember item 'x '((Member item 'a) (Member item 'b))))))) (let extract (ExtractMembers map '('x))) - (let world (Write! world mr_sink (Key '('table (String 'Output))) extract '('('mode 'renew)))) - (let world (Commit! world mr_sink)) + (let world (Write! world res (Key) extract '())) + (let world (Commit! world res)) (return world) ))"; @@ -465,15 +461,15 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { Y_UNIT_TEST(TopSort) { const auto s = R"(( - (let mr_sink (DataSink 'yt (quote plato))) + (let res (DataSink 'result)) (let list (AsList (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value (String 'v))) (AsStruct '('key (String '1)) '('subkey (String 'd)) '('value (String 'v))) (AsStruct '('key (String '3)) '('subkey (String 'b)) '('value (String 'v))) )) (let sorted (TopSort list (Uint64 '2) '((Bool 'True) (Bool 'True)) (lambda '(item) '((Member item 'key) (Member item 'subkey))))) - (let world (Write! world mr_sink (Key '('table (String 'Output))) sorted '('('mode 'renew)))) - (let world (Commit! world mr_sink)) + (let world (Write! world res (Key) sorted '())) + (let world (Commit! world res)) (return world) ))"; @@ -484,7 +480,7 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { Y_UNIT_TEST(MergeWithFirstEmpty) { const auto s = R"(( - (let mr_sink (DataSink 'yt (quote plato))) + (let res (DataSink 'result)) (let list (AsList (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value (String 'v))) (AsStruct '('key (String '1)) '('subkey (String 'd)) '('value (String 'v))) @@ -493,8 +489,8 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { (let sorted (Sort list (Bool 'True) (lambda '(item) (Member item 'key)))) (let empty (List (ListType (StructType '('key (DataType 'String)) '('subkey (DataType 'String)) '('value (DataType 'String)))))) (let merged (Merge empty sorted)) - (let world (Write! world mr_sink (Key '('table (String 'Output))) merged '('('mode 'renew)))) - (let world (Commit! world mr_sink)) + (let world (Write! world res (Key) merged '())) + (let world (Commit! world res)) (return world) ))"; @@ -505,7 +501,7 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { Y_UNIT_TEST(MergeWithCloneColumns) { const auto s = R"(( - (let mr_sink (DataSink 'yt (quote plato))) + (let res (DataSink 'result)) (let list (AsList (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value (String 'v))) (AsStruct '('key (String '1)) '('subkey (String 'd)) '('value (String 'v))) @@ -514,8 +510,8 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { (let sorted (Sort list (Bool 'False) (lambda '(item) (Member item 'key)))) (let clone (OrderedMap sorted (lambda '(row) (AsStruct '('key (Member row 'key)) '('subkey (Member row 'key)) '('value (Member row 'key)))))) (let merged (Merge clone sorted)) - (let world (Write! world mr_sink (Key '('table (String 'Output))) merged '('('mode 'renew)))) - (let world (Commit! world mr_sink)) + (let world (Write! world res (Key) merged '())) + (let world (Commit! world res)) (return world) ))"; @@ -526,7 +522,7 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { Y_UNIT_TEST(UnionMergeWithDiffTypes) { const auto s = R"(( - (let mr_sink (DataSink 'yt (quote plato))) + (let res (DataSink 'result)) (let list1 (AsList (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value (String 'v))) (AsStruct '('key (String '1)) '('subkey (String 'd)) '('value (String 'v))) @@ -540,8 +536,8 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { (let sorted1 (Sort list1 (Bool 'False) (lambda '(item) '((Member item 'key) (Member item 'subkey) (Member item 'value))))) (let sorted2 (Sort list2 (Bool 'False) (lambda '(item) '((Member item 'key) (Member item 'subkey) (Member item 'value))))) (let merged (UnionMerge sorted1 sorted2)) - (let world (Write! world mr_sink (Key '('table (String 'Output))) merged '('('mode 'renew)))) - (let world (Commit! world mr_sink)) + (let world (Write! world res (Key) merged '())) + (let world (Commit! world res)) (return world) ))"; @@ -552,7 +548,7 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { Y_UNIT_TEST(ExtractMembersKey) { const auto s = R"(( - (let mr_sink (DataSink 'yt (quote plato))) + (let res (DataSink 'result)) (let list (AsList (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value (String 'v))) (AsStruct '('key (String '1)) '('subkey (String 'd)) '('value (String 'w))) @@ -560,8 +556,8 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { )) (let sorted (Sort list '((Bool 'True) (Bool 'True)) (lambda '(item) '((Member item 'key) (Member item 'subkey))))) (let sorted (ExtractMembers sorted '('key))) - (let world (Write! world mr_sink (Key '('table (String 'Output))) sorted '('('mode 'renew)))) - (let world (Commit! world mr_sink)) + (let world (Write! world res (Key) sorted '())) + (let world (Commit! world res)) (return world) ))"; @@ -572,7 +568,7 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { Y_UNIT_TEST(ExtractMembersNonKey) { const auto s = R"(( - (let mr_sink (DataSink 'yt (quote plato))) + (let res (DataSink 'result)) (let list (AsList (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value (String 'v))) (AsStruct '('key (String '1)) '('subkey (String 'd)) '('value (String 'w))) @@ -580,8 +576,8 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { )) (let sorted (Sort list '((Bool 'True) (Bool 'True)) (lambda '(item) '((Member item 'key) (Member item 'subkey))))) (let sorted (ExtractMembers sorted '('value))) - (let world (Write! world mr_sink (Key '('table (String 'Output))) sorted '('('mode 'renew)))) - (let world (Commit! world mr_sink)) + (let world (Write! world res (Key) sorted '())) + (let world (Commit! world res)) (return world) ))"; @@ -592,7 +588,7 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { Y_UNIT_TEST(OrderedLMap) { const auto s = R"(( - (let mr_sink (DataSink 'yt (quote plato))) + (let res (DataSink 'result)) (let list (AsList (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value (String 'v))) (AsStruct '('key (String '1)) '('subkey (String 'd)) '('value (String 'w))) @@ -600,8 +596,8 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { )) (let sorted (Sort list '((Bool 'False) (Bool 'False)) (lambda '(item) '((Member item 'key) (Member item 'subkey))))) (let sorted (OrderedLMap sorted (lambda '(stream) (OrderedFlatMap stream (lambda '(item) (Just item)))))) - (let world (Write! world mr_sink (Key '('table (String 'Output))) sorted '('('mode 'renew)))) - (let world (Commit! world mr_sink)) + (let world (Write! world res (Key) sorted '())) + (let world (Commit! world res)) (return world) ))"; @@ -613,7 +609,7 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { Y_UNIT_TEST(OrderedFlatMap) { const auto s = R"(( - (let mr_sink (DataSink 'yt (quote plato))) + (let res (DataSink 'result)) (let list (AsList (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value (String 'v))) (AsStruct '('key (String '1)) '('subkey (String 'd)) '('value (String 'w))) @@ -623,8 +619,8 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { (let list (AssumeDistinct list '('key 'subkey) '('value))) (let sorted (Sort list '((Bool 'True) (Bool 'True)) (lambda '(item) '((Member item 'key) (Member item 'subkey))))) (let sorted (OrderedFlatMap sorted (lambda '(item) (OptionalIf (== (Member item 'key) (String '1)) (AsStruct '('k (Member item 'key)) '('v (Member item 'value))))))) - (let world (Write! world mr_sink (Key '('table (String 'Output))) sorted '('('mode 'renew)))) - (let world (Commit! world mr_sink)) + (let world (Write! world res (Key) sorted '())) + (let world (Commit! world res)) (return world) ))"; @@ -637,7 +633,7 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { Y_UNIT_TEST(OrderedFlatMapWithEmptyFilter) { const auto s = R"(( - (let mr_sink (DataSink 'yt (quote plato))) + (let res (DataSink 'result)) (let list (AsList (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value (String 'v))) (AsStruct '('key (String '1)) '('subkey (String 'd)) '('value (String 'w))) @@ -647,8 +643,8 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { (let list (AssumeDistinct list '('key 'subkey) '('value))) (let sorted (Sort list '((Bool 'True) (Bool 'True)) (lambda '(item) '((Member item 'key) (Member item 'subkey))))) (let sorted (OrderedFlatMap sorted (lambda '(item) (OptionalIf (Bool 'false) (AsStruct '('k (Member item 'key)) '('v (Member item 'value))))))) - (let world (Write! world mr_sink (Key '('table (String 'Output))) sorted '('('mode 'renew)))) - (let world (Commit! world mr_sink)) + (let world (Write! world res (Key) sorted '())) + (let world (Commit! world res)) (return world) ))"; @@ -661,7 +657,7 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { Y_UNIT_TEST(OrderedFlatMapNonKey) { const auto s = R"(( - (let mr_sink (DataSink 'yt (quote plato))) + (let res (DataSink 'result)) (let list (AsList (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value (String 'v))) (AsStruct '('key (String '1)) '('subkey (String 'd)) '('value (String 'w))) @@ -669,8 +665,8 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { )) (let sorted (Sort list '((Bool 'True) (Bool 'True)) (lambda '(item) '((Member item 'key) (Member item 'subkey))))) (let sorted (OrderedFlatMap sorted (lambda '(item) (OptionalIf (== (Member item 'key) (String '1)) (AsStruct '('key (Member item 'value))))))) - (let world (Write! world mr_sink (Key '('table (String 'Output))) sorted '('('mode 'renew)))) - (let world (Commit! world mr_sink)) + (let world (Write! world res (Key) sorted '())) + (let world (Commit! world res)) (return world) ))"; @@ -681,7 +677,7 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { Y_UNIT_TEST(OrderedFilter) { const auto s = R"(( - (let mr_sink (DataSink 'yt (quote plato))) + (let res (DataSink 'result)) (let list (AsList (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value (String 'v))) (AsStruct '('key (String '1)) '('subkey (String 'd)) '('value (String 'w))) @@ -691,8 +687,8 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { (let list (AssumeDistinct list '('key 'subkey) '('value))) (let sorted (Sort list '((Bool 'True) (Bool 'True)) (lambda '(item) '((Member item 'key) (Member item 'subkey))))) (let sorted (OrderedFilter sorted (lambda '(item) (> (Member item 'key) (String '0))))) - (let world (Write! world mr_sink (Key '('table (String 'Output))) sorted '('('mode 'renew)))) - (let world (Commit! world mr_sink)) + (let world (Write! world res (Key) sorted '())) + (let world (Commit! world res)) (return world) ))"; @@ -705,7 +701,7 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { Y_UNIT_TEST(OrderedMapNullifyOneColumn) { const auto s = R"(( - (let mr_sink (DataSink 'yt (quote plato))) + (let res (DataSink 'result)) (let list (AsList (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value (String 'v))) (AsStruct '('key (String '1)) '('subkey (String 'd)) '('value (String 'w))) @@ -715,8 +711,8 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { (let list (AssumeDistinct list '('key 'subkey) '('value))) (let sorted (Sort list (Bool 'False) (lambda '(row) '((Member row 'key) (Member row 'subkey) (Member row 'value))))) (let map (OrderedMap sorted (lambda '(row) (AsStruct '('k (Member row 'key)) '('s (OptionalIf (AggrNotEquals (Member row 'key) (Member row 'value)) (Member row 'subkey))) '('v (Member row 'value)))))) - (let world (Write! world mr_sink (Key '('table (String 'Output))) map '('('mode 'renew)))) - (let world (Commit! world mr_sink)) + (let world (Write! world res (Key) map '())) + (let world (Commit! world res)) (return world) ))"; @@ -729,7 +725,7 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { Y_UNIT_TEST(NestedFlatMapByOptional) { const auto s = R"(( - (let mr_sink (DataSink 'yt (quote plato))) + (let res (DataSink 'result)) (let list (AsList (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value (String 'v))) (AsStruct '('key (String '1)) '('subkey (String 'd)) '('value (String 'w))) @@ -741,8 +737,8 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { (let mapped (OrderedFlatMap sorted (lambda '(item) (FlatMap (StrictCast (Member item 'key) (OptionalType (DataType 'Uint8))) (lambda '(key) (Just (AsStruct '('k key) '('s (Member item 'subkey)) '('v (Member item 'value))))) )))) - (let world (Write! world mr_sink (Key '('table (String 'Output))) mapped '('('mode 'renew)))) - (let world (Commit! world mr_sink)) + (let world (Write! world res (Key) mapped '())) + (let world (Commit! world res)) (return world) ))"; @@ -755,7 +751,7 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { Y_UNIT_TEST(FlattenMembers) { const auto s = R"(( - (let mr_sink (DataSink 'yt (quote plato))) + (let res (DataSink 'result)) (let list (AsList (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value (String 'v))) (AsStruct '('key (String '1)) '('subkey (String 'd)) '('value (String 'w))) @@ -766,8 +762,8 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { (let sorted (Sort list '((Bool 'True) (Bool 'True)) (lambda '(item) '((Member item 'key) (Member item 'subkey))))) (let mapped (OrderedMap sorted (lambda '(item) '((AsStruct '('key (Member item 'key)) '('subkey (Member item 'subkey))) (AsStruct '('subkey (Member item 'subkey)) '('value (Member item 'value))))))) (let flatten (OrderedMap mapped (lambda '(pair) (FlattenMembers '('one (Nth pair '0)) '('two (Nth pair '1)))))) - (let world (Write! world mr_sink (Key '('table (String 'Output))) (LazyList flatten) '('('mode 'renew)))) - (let world (Commit! world mr_sink)) + (let world (Write! world res (Key) (LazyList flatten) '())) + (let world (Commit! world res)) (return world) ))"; @@ -780,7 +776,7 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { Y_UNIT_TEST(Visit) { const auto s = R"(( -(let mr_sink (DataSink 'yt (quote plato))) +(let res (DataSink 'result)) (let list (AsList (AsStruct '('key (String 'aaa))) @@ -802,10 +798,10 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { (Nothing (OptionalType vt)) )))) -(let res (Map vlist (lambda '(item) (VariantItem item)))) +(let result (Map vlist (lambda '(item) (VariantItem item)))) -(let world (Write! world mr_sink (Key '('table (String 'Output))) res '('('mode 'renew)))) -(let world (Commit! world mr_sink)) +(let world (Write! world res (Key) result '())) +(let world (Commit! world res)) (return world) ))"; @@ -821,7 +817,7 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { Y_UNIT_TEST(SwitchAsFilter) { const auto s = R"(( -(let mr_sink (DataSink 'yt (quote plato))) +(let res (DataSink 'result)) (let list (AsList (AsStruct '('key (String 'aaa))) @@ -838,10 +834,10 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { (Map list (lambda '(x) (Variant (AsStruct '('key (Member x 'key)) '('value (String '2))) '2 vt))) )) -(let res (Switch (Iterator vlist) '0 '('0) (lambda '(item) item))) +(let result (Switch (Iterator vlist) '0 '('0) (lambda '(item) item))) -(let world (Write! world mr_sink (Key '('table (String 'Output))) res '('('mode 'renew)))) -(let world (Commit! world mr_sink)) +(let world (Write! world res (Key) (Collect result) '())) +(let world (Commit! world res)) (return world) ))"; @@ -853,7 +849,7 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { Y_UNIT_TEST(EmptyForMissingMultiOut) { const auto s = R"(( -(let mr_sink (DataSink 'yt (quote plato))) +(let res (DataSink 'result)) (let list (AsList (AsStruct '('key (String 'aaa))) @@ -877,10 +873,10 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { '2 (lambda '(v) (Just (Variant v '1 vt2))) )))) -(let res (Map vlist (lambda '(item) (VariantItem item)))) +(let result (Map vlist (lambda '(item) (VariantItem item)))) -(let world (Write! world mr_sink (Key '('table (String 'Output))) res '('('mode 'renew)))) -(let world (Commit! world mr_sink)) +(let world (Write! world res (Key) result '())) +(let world (Commit! world res)) (return world) ))"; @@ -892,7 +888,7 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { Y_UNIT_TEST(SwitchWithReplicate) { const auto s = R"(( -(let mr_sink (DataSink 'yt (quote plato))) +(let res (DataSink 'result)) (let list (AsList (AsStruct '('key (String 'aaa))) @@ -902,10 +898,10 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { (let vlist (Switch (Iterator list) '0 '('0) (lambda '(item) item) '('0) (lambda '(item) item))) -(let res (Map vlist (lambda '(item) (VariantItem item)))) +(let result (Map vlist (lambda '(item) (VariantItem item)))) -(let world (Write! world mr_sink (Key '('table (String 'Output))) res '('('mode 'renew)))) -(let world (Commit! world mr_sink)) +(let world (Write! world res (Key) (Collect result) '())) +(let world (Commit! world res)) (return world) ))"; @@ -917,7 +913,7 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { Y_UNIT_TEST(SwitchWithMultiOut) { const auto s = R"(( -(let mr_sink (DataSink 'yt (quote plato))) +(let res (DataSink 'result)) (let list (AsList (AsStruct '('key (String 'aaa))) @@ -936,10 +932,10 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { (let vlist (Switch (Iterator vlist) '0 '('0) (lambda '(s) (Map s (lambda '(item) (Variant item '1 vt)))) '('1) (lambda '(s) s))) -(let res (Map vlist (lambda '(item) (VariantItem item)))) +(let result (Map vlist (lambda '(item) (VariantItem item)))) -(let world (Write! world mr_sink (Key '('table (String 'Output))) res '('('mode 'renew)))) -(let world (Commit! world mr_sink)) +(let world (Write! world res (Key) (Collect result) '())) +(let world (Commit! world res)) (return world) ))"; @@ -951,7 +947,7 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { Y_UNIT_TEST(MultiOutLMapWithEmptyInput) { auto s = R"(( -(let mr_sink (DataSink 'yt (quote plato))) +(let res (DataSink 'result)) (let structType (StructType '('key (DataType 'String)) '('value (DataType 'String)))) (let tupleType (TupleType structType structType structType)) @@ -965,10 +961,10 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { (let vlist (Switch (Iterator vlist) '0 '('0) (lambda '(s) s) '('1) (lambda '(s) s))) -(let res (Map vlist (lambda '(item) (VariantItem item)))) +(let result (Map vlist (lambda '(item) (VariantItem item)))) -(let world (Write! world mr_sink (Key '('table (String 'Output))) res '('('mode 'renew)))) -(let world (Commit! world mr_sink)) +(let world (Write! world res (Key) (Collect result) '())) +(let world (Commit! world res)) (return world) ))"; @@ -982,7 +978,7 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { Y_UNIT_TEST(Unique) { const auto s = R"(( - (let mr_sink (DataSink 'yt (quote plato))) + (let res (DataSink 'result)) (let list (AsList (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value (String 'x))) (AsStruct '('key (String '1)) '('subkey (String 'b)) '('value (String 'y))) @@ -1008,8 +1004,8 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { ))) (return res) ))))) - (let world (Write! world mr_sink (Key '('table (String 'Output))) (Take list (Uint64 '2)) '('('mode 'renew)))) - (let world (Commit! world mr_sink)) + (let world (Write! world res (Key) (Take list (Uint64 '2)) '())) + (let world (Commit! world res)) (return world) ))"; @@ -1020,7 +1016,7 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { Y_UNIT_TEST(UniqueOverTuple) { const auto s = R"(( - (let mr_sink (DataSink 'yt (quote plato))) + (let res (DataSink 'result)) (let list (AsList (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value (String 'm))) (AsStruct '('key (String '1)) '('subkey (String 'b)) '('value (String 'm))) @@ -1046,8 +1042,8 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { ))) (return (Just (DivePrefixMembers (Unwrap res) '('p.)))) ))))) - (let world (Write! world mr_sink (Key '('table (String 'Output))) list '('('mode 'renew)))) - (let world (Commit! world mr_sink)) + (let world (Write! world res (Key) list '())) + (let world (Commit! world res)) (return world) ))"; @@ -1060,7 +1056,7 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { Y_UNIT_TEST(UniqueOverWideFlow) { const auto s = R"(( - (let mr_sink (DataSink 'yt (quote plato))) + (let res (DataSink 'result)) (let list (AsList (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value (String 'x))) (AsStruct '('key (String '1)) '('subkey (String 'b)) '('value (String 'y))) @@ -1085,8 +1081,8 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { ))) (return (Map res (lambda '(row) (DivePrefixMembers row '('p.))))) ))))) - (let world (Write! world mr_sink (Key '('table (String 'Output))) list '('('mode 'renew)))) - (let world (Commit! world mr_sink)) + (let world (Write! world res (Key) (Collect list) '())) + (let world (Commit! world res)) (return world) ))"; @@ -1099,7 +1095,7 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { Y_UNIT_TEST(UniqueExOverWideFlow) { const auto s = R"(( - (let mr_sink (DataSink 'yt (quote plato))) + (let res (DataSink 'result)) (let list (AsList (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value (String 'v))) (AsStruct '('key (String '1)) '('subkey (String 'd)) '('value (String 'v))) @@ -1118,8 +1114,8 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { (let res (NarrowMap res (lambda '(t0 s1) (AddMember (AddMember s1 'one (Nth t0 '1)) 'two (Nth t0 '0))))) (return (Collect res)) ))))) - (let world (Write! world mr_sink (Key '('table (String 'Output))) (LazyList list) '('('mode 'renew)))) - (let world (Commit! world mr_sink)) + (let world (Write! world res (Key) (LazyList list) '())) + (let world (Commit! world res)) (return world) ))"; @@ -1130,7 +1126,7 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { Y_UNIT_TEST(Distinct) { const auto s = R"(( - (let mr_sink (DataSink 'yt (quote plato))) + (let res (DataSink 'result)) (let list (AsList (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value (String 'x))) (AsStruct '('key (String '1)) '('subkey (String 'b)) '('value (String 'y))) @@ -1156,8 +1152,8 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { ))) (return res) ))))) - (let world (Write! world mr_sink (Key '('table (String 'Output))) (Take list (Uint64 '2)) '('('mode 'renew)))) - (let world (Commit! world mr_sink)) + (let world (Write! world res (Key) (Take list (Uint64 '2)) '())) + (let world (Commit! world res)) (return world) ))"; @@ -1168,7 +1164,7 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { Y_UNIT_TEST(DistinctOverTuple) { const auto s = R"(( - (let mr_sink (DataSink 'yt (quote plato))) + (let res (DataSink 'result)) (let list (AsList (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value (String 'm))) (AsStruct '('key (String '1)) '('subkey (String 'b)) '('value (String 'm))) @@ -1194,8 +1190,8 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { ))) (return (Just (DivePrefixMembers (Unwrap res) '('p.)))) ))))) - (let world (Write! world mr_sink (Key '('table (String 'Output))) list '('('mode 'renew)))) - (let world (Commit! world mr_sink)) + (let world (Write! world res (Key) list '())) + (let world (Commit! world res)) (return world) ))"; @@ -1208,7 +1204,7 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { Y_UNIT_TEST(DistinctOverWideFlow) { const auto s = R"(( - (let mr_sink (DataSink 'yt (quote plato))) + (let res (DataSink 'result)) (let list (AsList (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value (String 'x))) (AsStruct '('key (String '1)) '('subkey (String 'b)) '('value (String 'y))) @@ -1233,8 +1229,8 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { ))) (return (Map res (lambda '(row) (DivePrefixMembers row '('p.))))) ))))) - (let world (Write! world mr_sink (Key '('table (String 'Output))) list '('('mode 'renew)))) - (let world (Commit! world mr_sink)) + (let world (Write! world res (Key) (Collect list) '())) + (let world (Commit! world res)) (return world) ))"; @@ -1247,7 +1243,7 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { Y_UNIT_TEST(DistinctExOverWideFlow) { const auto s = R"(( - (let mr_sink (DataSink 'yt (quote plato))) + (let res (DataSink 'result)) (let list (AsList (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value (String 'v))) (AsStruct '('key (String '1)) '('subkey (String 'd)) '('value (String 'v))) @@ -1266,8 +1262,8 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { (let res (NarrowMap res (lambda '(t0 s1) (AddMember (AddMember s1 'one (Nth t0 '1)) 'two (Nth t0 '0))))) (return (Collect res)) ))))) - (let world (Write! world mr_sink (Key '('table (String 'Output))) (LazyList list) '('('mode 'renew)))) - (let world (Commit! world mr_sink)) + (let world (Write! world res (Key) (LazyList list) '())) + (let world (Commit! world res)) (return world) ))"; @@ -1279,7 +1275,7 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { Y_UNIT_TEST(PartitionsByKeysWithCondense1) { const auto s = R"( ( - (let mr_sink (DataSink 'yt (quote plato))) + (let res (DataSink 'result)) (let list (AsList (AsStruct '('key (Just (String '4))) '('subkey (Just (String 'c))) '('value (Just (String 'x)))) (AsStruct '('key (Just (String '1))) '('subkey (Just (String 'b))) '('value (Just (String 'y)))) @@ -1292,8 +1288,8 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { (lambda '(row state) (AsStruct '('key (Member row 'key)) '('subkey (Member row 'subkey)) '('value (Coalesce (Member row 'value) (Member state 'value))))) )) )) - (let world (Write! world mr_sink (Key '('table (String 'Output))) (Skip aggr (Uint64 '1)) '('('mode 'renew)))) - (let world (Commit! world mr_sink)) + (let world (Write! world res (Key) (Skip aggr (Uint64 '1)) '())) + (let world (Commit! world res)) (return world) ) )"; @@ -1307,7 +1303,7 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { Y_UNIT_TEST(PartitionsByKeysWithCondense1AndStablePickle) { const auto s = R"( ( - (let mr_sink (DataSink 'yt (quote plato))) + (let res (DataSink 'result)) (let list (AsList (AsStruct '('key (Just (String '4))) '('subkey (Just (String 'c))) '('value (Just (String 'x)))) (AsStruct '('key (Just (String '1))) '('subkey (Just (String 'b))) '('value (Just (String 'y)))) @@ -1320,8 +1316,8 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { (lambda '(row state) (AsStruct '('key (Member row 'key)) '('subkey (Member row 'subkey)) '('value (Coalesce (Member row 'value) (Member state 'value))))) )) )) - (let world (Write! world mr_sink (Key '('table (String 'Output))) (Skip aggr (Uint64 '1)) '('('mode 'renew)))) - (let world (Commit! world mr_sink)) + (let world (Write! world res (Key) (Skip aggr (Uint64 '1)) '())) + (let world (Commit! world res)) (return world) ) )"; @@ -1335,7 +1331,7 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { Y_UNIT_TEST(PartitionsByKeysWithCondense1WithSingleItemTupleKey) { const auto s = R"( ( - (let mr_sink (DataSink 'yt (quote plato))) + (let res (DataSink 'result)) (let list (AsList (AsStruct '('key '((Just (String '4)))) '('subkey (Just (String 'c))) '('value (Just (String 'x)))) (AsStruct '('key '((Just (String '1)))) '('subkey (Just (String 'b))) '('value (Just (String 'y)))) @@ -1350,8 +1346,8 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { (lambda '(item) (AsStruct '('key '(item)))) )) )) - (let world (Write! world mr_sink (Key '('table (String 'Output))) (Skip aggr (Uint64 '1)) '('('mode 'renew)))) - (let world (Commit! world mr_sink)) + (let world (Write! world res (Key) (Skip aggr (Uint64 '1)) '())) + (let world (Commit! world res)) (return world) ) )"; @@ -1365,7 +1361,7 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { Y_UNIT_TEST(PartitionsByKeysWithCondense1WithPairItemsTupleKeyAndDuplicateOut) { const auto s = R"( ( - (let mr_sink (DataSink 'yt (quote plato))) + (let res (DataSink 'result)) (let list (AsList (AsStruct '('key '((Just (String '4)) (Just (String '%)))) '('subkey (Just (String 'c))) '('value (Just (String 'x)))) (AsStruct '('key '((Just (String '1)) (Just (String '€)))) '('subkey (Just (String 'b))) '('value (Just (String 'y)))) @@ -1380,8 +1376,8 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { (lambda '(item) (AsStruct '('one '(item)) '('two '(item)))) )) )) - (let world (Write! world mr_sink (Key '('table (String 'Output))) (Skip aggr (Uint64 '1)) '('('mode 'renew)))) - (let world (Commit! world mr_sink)) + (let world (Write! world res (Key) (Skip aggr (Uint64 '1)) '())) + (let world (Commit! world res)) (return world) ) )"; @@ -1395,7 +1391,7 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { Y_UNIT_TEST(ShuffleByKeysInputUnique) { const auto s = R"( ( - (let mr_sink (DataSink 'yt (quote plato))) + (let res (DataSink 'result)) (let list (AsList (AsStruct '('key (Just (String '4))) '('subkey (Just (String 'c))) '('value (Just (String 'x)))) (AsStruct '('key (Just (String '1))) '('subkey (Just (String 'b))) '('value (Just (String 'y)))) @@ -1407,8 +1403,8 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { (lambda '(item) '((Member item 'key) (Member item 'subkey))) (lambda '(stream) (Take stream (Uint64 '100))) )) - (let world (Write! world mr_sink (Key '('table (String 'Output))) (Skip aggr (Uint64 '1)) '('('mode 'renew)))) - (let world (Commit! world mr_sink)) + (let world (Write! world res (Key) (Skip aggr (Uint64 '1)) '())) + (let world (Commit! world res)) (return world) ) )"; @@ -1422,7 +1418,7 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { Y_UNIT_TEST(ShuffleByKeysHandlerUnique) { const auto s = R"( ( - (let mr_sink (DataSink 'yt (quote plato))) + (let res (DataSink 'result)) (let list (AsList (AsStruct '('key (Just (String '4))) '('subkey (Just (String 'c))) '('value (Just (String 'x)))) (AsStruct '('key (Just (String '1))) '('subkey (Just (String 'b))) '('value (Just (String 'y)))) @@ -1434,8 +1430,8 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { (lambda '(item) '((Member item 'key) (Member item 'subkey))) (lambda '(stream) (AssumeDistinct (AssumeUnique stream '('key) '('subkey)) '('key 'subkey))) )) - (let world (Write! world mr_sink (Key '('table (String 'Output))) (Skip aggr (Uint64 '1)) '('('mode 'renew)))) - (let world (Commit! world mr_sink)) + (let world (Write! world res (Key) (Skip aggr (Uint64 '1)) '())) + (let world (Commit! world res)) (return world) ) )"; @@ -1449,7 +1445,7 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { Y_UNIT_TEST(Reverse) { const auto s = R"( ( - (let mr_sink (DataSink 'yt (quote plato))) + (let res (DataSink 'result)) (let list (AsList (AsStruct '('key (Just (String '4))) '('subkey (Just (String 'c))) '('value (Just (String 'x)))) (AsStruct '('key (Just (String '1))) '('subkey (Just (String 'b))) '('value (Just (String 'y)))) @@ -1459,8 +1455,8 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { (let list (AssumeDistinct list '('key 'subkey))) (let sorted (Sort list '((Bool 'False) (Bool 'True)) (lambda '(item) '((Member item 'key) (Member item 'subkey))))) (let reverse (Reverse sorted)) - (let world (Write! world mr_sink (Key '('table (String 'Output))) reverse '('('mode 'renew)))) - (let world (Commit! world mr_sink)) + (let world (Write! world res (Key) reverse '())) + (let world (Commit! world res)) (return world) ) )"; @@ -1475,7 +1471,7 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { Y_UNIT_TEST(DictItems) { const auto s = R"( ( - (let mr_sink (DataSink 'yt (quote plato))) + (let res (DataSink 'result)) (let list (AsList (AsStruct '('key (Just (String '4))) '('subkey (Just (String 'c))) '('value (Just (String 'x)))) (AsStruct '('key (Just (String '1))) '('subkey (Just (String 'b))) '('value (Just (String 'y)))) @@ -1485,8 +1481,8 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { (let list (AssumeDistinct list '('key 'subkey))) (let dict (ToDict list (lambda '(item) (Member item 'value)) (lambda '(item) (AsStruct '('k (Member item 'key)) '('s (Member item 'subkey)))) '('One 'Hashed))) (let items (Map (DictItems dict) (lambda '(item) (AsStruct '('v (Nth item '0)) '('k (Member (Nth item '1) 'k)) '('s (Member (Nth item '1) 's)))))) - (let world (Write! world mr_sink (Key '('table (String 'Output))) items '('('mode 'renew)))) - (let world (Commit! world mr_sink)) + (let world (Write! world res (Key) items '())) + (let world (Commit! world res)) (return world) ) )"; @@ -1500,7 +1496,7 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { Y_UNIT_TEST(DictKeys) { const auto s = R"( ( - (let mr_sink (DataSink 'yt (quote plato))) + (let res (DataSink 'result)) (let list (AsList (AsStruct '('key (Just (String '4))) '('subkey (Just (String 'c))) '('value (Just (String 'x)))) (AsStruct '('key (Just (String '1))) '('subkey (Just (String 'b))) '('value (Just (String 'y)))) @@ -1508,8 +1504,8 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { )) (let dict (ToDict list (lambda '(item) (Member item 'value)) (lambda '(item) (AsStruct '('k (Member item 'key)) '('s (Member item 'subkey)))) '('One 'Hashed))) (let items (Map (DictKeys dict) (lambda '(row) (AsStruct '('v row))))) - (let world (Write! world mr_sink (Key '('table (String 'Output))) items '('('mode 'renew)))) - (let world (Commit! world mr_sink)) + (let world (Write! world res (Key) items '())) + (let world (Commit! world res)) (return world) ) )"; @@ -1523,7 +1519,7 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { Y_UNIT_TEST(DictPayloads) { const auto s = R"( ( - (let mr_sink (DataSink 'yt (quote plato))) + (let res (DataSink 'result)) (let list (AsList (AsStruct '('key (Just (String '4))) '('subkey (Just (String 'c))) '('value (Just (String 'x)))) (AsStruct '('key (Just (String '1))) '('subkey (Just (String 'b))) '('value (Just (String 'y)))) @@ -1533,8 +1529,8 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { (let list (AssumeDistinct list '('key 'subkey))) (let dict (ToDict list (lambda '(item) (Member item 'value)) (lambda '(item) (AsStruct '('k (Member item 'key)) '('s (Member item 'subkey)))) '('One 'Hashed))) (let items (DictPayloads dict)) - (let world (Write! world mr_sink (Key '('table (String 'Output))) items '('('mode 'renew)))) - (let world (Commit! world mr_sink)) + (let world (Write! world res (Key) items '())) + (let world (Commit! world res)) (return world) ) )"; @@ -1575,10 +1571,10 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { (let join (GraceJoinCore flow1 flow2 'Inner '('0 '1) '('0 '1) '('0 '0 '2 '1) '('0 '2 '1 '3 '2 '4) '() '() '())) (let list (Collect (NarrowMap join (lambda '(lk lv rk rs rv) (AsStruct '('lk lk) '('lv lv) '('rk rk) '('rs rs) '('rv rv)))))) - (let res_sink (DataSink 'yt (quote plato))) - (let world (Write! world res_sink (Key '('table (String 'Output))) list '('('mode 'renew)))) + (let res (DataSink 'result)) + (let world (Write! world res (Key) list '())) - (let world (Commit! world res_sink)) + (let world (Commit! world res)) (return world) ) )"; @@ -1618,10 +1614,10 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { (let join (GraceJoinCore flow1 flow2 'Left '('0 '1) '('0 '1) '('0 '0 '2 '1) '('0 '2 '1 '3 '2 '4) '() '() '())) (let list (Collect (NarrowMap join (lambda '(lk lv rk rs rv) (AsStruct '('lk lk) '('lv lv) '('rk rk) '('rs rs) '('rv rv)))))) - (let res_sink (DataSink 'yt (quote plato))) - (let world (Write! world res_sink (Key '('table (String 'Output))) list '('('mode 'renew)))) + (let res (DataSink 'result)) + (let world (Write! world res (Key) list '())) - (let world (Commit! world res_sink)) + (let world (Commit! world res)) (return world) ) )"; @@ -1661,10 +1657,10 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { (let join (GraceJoinCore flow1 flow2 'Full '('0 '1) '('0 '1) '('0 '0 '2 '1) '('0 '2 '1 '3 '2 '4) '() '() '())) (let list (Collect (NarrowMap join (lambda '(lk lv rk rs rv) (AsStruct '('lk lk) '('lv lv) '('rk rk) '('rs rs) '('rv rv)))))) - (let res_sink (DataSink 'yt (quote plato))) - (let world (Write! world res_sink (Key '('table (String 'Output))) list '('('mode 'renew)))) + (let res (DataSink 'result)) + (let world (Write! world res (Key) list '())) - (let world (Commit! world res_sink)) + (let world (Commit! world res)) (return world) ) )"; @@ -1704,10 +1700,10 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { (let join (GraceJoinCore flow1 flow2 'Right '('0 '1) '('0 '1) '('0 '0 '2 '1) '('0 '2 '1 '3 '2 '4) '() '() '())) (let list (Collect (NarrowMap join (lambda '(lk lv rk rs rv) (AsStruct '('lk lk) '('lv lv) '('rk rk) '('rs rs) '('rv rv)))))) - (let res_sink (DataSink 'yt (quote plato))) - (let world (Write! world res_sink (Key '('table (String 'Output))) list '('('mode 'renew)))) + (let res (DataSink 'result)) + (let world (Write! world res (Key) list '())) - (let world (Commit! world res_sink)) + (let world (Commit! world res)) (return world) ) )"; @@ -1747,10 +1743,10 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { (let join (GraceJoinCore flow1 flow2 'Exclusion '('0 '1) '('0 '1) '('0 '0 '2 '1) '('0 '2 '1 '3 '2 '4) '() '() '())) (let list (Collect (NarrowMap join (lambda '(lk lv rk rs rv) (AsStruct '('lk lk) '('lv lv) '('rk rk) '('rs rs) '('rv rv)))))) - (let res_sink (DataSink 'yt (quote plato))) - (let world (Write! world res_sink (Key '('table (String 'Output))) list '('('mode 'renew)))) + (let res (DataSink 'result)) + (let world (Write! world res (Key) list '())) - (let world (Commit! world res_sink)) + (let world (Commit! world res)) (return world) ) )"; @@ -1790,10 +1786,10 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { (let join (GraceJoinCore flow1 flow2 'LeftSemi '('0 '1) '('0 '1) '('0 '2 '1 '1 '2 '0) '() '() '() '())) (let list (Collect (NarrowMap join (lambda '(lv ls lk) (AsStruct '('lk lk) '('lv lv) '('ls ls)))))) - (let res_sink (DataSink 'yt (quote plato))) - (let world (Write! world res_sink (Key '('table (String 'Output))) list '('('mode 'renew)))) + (let res (DataSink 'result)) + (let world (Write! world res (Key) list '())) - (let world (Commit! world res_sink)) + (let world (Commit! world res)) (return world) ) )"; @@ -1833,10 +1829,10 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { (let join (GraceJoinCore flow1 flow2 'LeftOnly '('0 '1) '('0 '1) '('0 '1 '2 '0) '() '() '() '())) (let list (Collect (NarrowMap join (lambda '(lv lk) (AsStruct '('lk lk) '('lv lv)))))) - (let res_sink (DataSink 'yt (quote plato))) - (let world (Write! world res_sink (Key '('table (String 'Output))) list '('('mode 'renew)))) + (let res (DataSink 'result)) + (let world (Write! world res (Key) list '())) - (let world (Commit! world res_sink)) + (let world (Commit! world res)) (return world) ) )"; @@ -1877,10 +1873,10 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { (let join (GraceJoinCore flow1 flow2 'RightOnly '('0 '1) '('0 '1) '() '('0 '2 '1 '1 '2 '0) '() '() '())) (let list (Collect (NarrowMap join (lambda '(rv rs rk) (AsStruct '('rk rk) '('rv rv) '('rs rs)))))) - (let res_sink (DataSink 'yt (quote plato))) - (let world (Write! world res_sink (Key '('table (String 'Output))) list '('('mode 'renew)))) + (let res (DataSink 'result)) + (let world (Write! world res (Key) list '())) - (let world (Commit! world res_sink)) + (let world (Commit! world res)) (return world) ) )"; @@ -1920,10 +1916,10 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { (let join (GraceJoinCore flow1 flow2 'RightSemi '('0 '1) '('0 '1) '() '('0 '1 '2 '0) '() '() '())) (let list (Collect (NarrowMap join (lambda '(rv rk) (AsStruct '('rk rk) '('rv rv)))))) - (let res_sink (DataSink 'yt (quote plato))) - (let world (Write! world res_sink (Key '('table (String 'Output))) list '('('mode 'renew)))) + (let res (DataSink 'result)) + (let world (Write! world res (Key) list '())) - (let world (Commit! world res_sink)) + (let world (Commit! world res)) (return world) ) )"; @@ -1963,10 +1959,10 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { (let join (GraceJoinCore flow1 flow2 'Inner '('0 '1) '('0 '1) '('0 '0 '1 '1 '2 '2) '('0 '3 '1 '4 '2 '5) '() '() '('LeftAny 'RightAny))) (let list (Collect (NarrowMap join (lambda '(lk ls lv rk rs rv) (AsStruct '('lk lk) '('ls ls) '('lv lv) '('rk rk) '('rs rs) '('rv rv)))))) - (let res_sink (DataSink 'yt (quote plato))) - (let world (Write! world res_sink (Key '('table (String 'Output))) list '('('mode 'renew)))) + (let res (DataSink 'result)) + (let world (Write! world res (Key) list '())) - (let world (Commit! world res_sink)) + (let world (Commit! world res)) (return world) ) )"; @@ -2006,10 +2002,10 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { (let join (MapJoinCore (ToFlow list1) dict 'Inner '('key1 'subkey1) '('key2 'subkey2) '('key1 'key 'subkey1 'subkey 'value1 'value) '('0 's '1 'v) '() '())) (let list (Collect join)) - (let res_sink (DataSink 'yt (quote plato))) - (let world (Write! world res_sink (Key '('table (String 'Output))) list '('('mode 'renew)))) + (let res (DataSink 'result)) + (let world (Write! world res (Key) list '())) - (let world (Commit! world res_sink)) + (let world (Commit! world res)) (return world) ) )"; @@ -2048,10 +2044,10 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { (let join (MapJoinCore (ToFlow list1) dict 'Inner '('key1 'subkey1) '('key2 'subkey2) '('key1 'key 'subkey1 'subkey 'value1 'value) '('0 's '1 'v) '() '())) (let list (Collect join)) - (let res_sink (DataSink 'yt (quote plato))) - (let world (Write! world res_sink (Key '('table (String 'Output))) list '('('mode 'renew)))) + (let res (DataSink 'result)) + (let world (Write! world res (Key) list '())) - (let world (Commit! world res_sink)) + (let world (Commit! world res)) (return world) ) )"; @@ -2090,10 +2086,10 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { (let join (MapJoinCore (ToFlow list1) dict 'Left '('key1 'subkey1) '('key2 'subkey2) '('key1 'key 'subkey1 'subkey 'value1 'value) '('0 's '1 'v) '() '())) (let list (Collect join)) - (let res_sink (DataSink 'yt (quote plato))) - (let world (Write! world res_sink (Key '('table (String 'Output))) list '('('mode 'renew)))) + (let res (DataSink 'result)) + (let world (Write! world res (Key) list '())) - (let world (Commit! world res_sink)) + (let world (Commit! world res)) (return world) ) )"; @@ -2132,10 +2128,10 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { (let join (MapJoinCore (ToFlow list1) dict 'Left '('key1 'subkey1) '('key2 'subkey2) '('key1 'key 'subkey1 'subkey 'value1 'value) '('0 's '1 'v) '() '())) (let list (Collect join)) - (let res_sink (DataSink 'yt (quote plato))) - (let world (Write! world res_sink (Key '('table (String 'Output))) list '('('mode 'renew)))) + (let res (DataSink 'result)) + (let world (Write! world res (Key) list '())) - (let world (Commit! world res_sink)) + (let world (Commit! world res)) (return world) ) )"; @@ -2174,10 +2170,10 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { (let join (MapJoinCore (ToFlow list1) dict 'LeftSemi '('key1 'subkey1) '('key2 'subkey2) '('key1 'key 'subkey1 'subkey 'value1 'value) '() '() '())) (let list (Collect join)) - (let res_sink (DataSink 'yt (quote plato))) - (let world (Write! world res_sink (Key '('table (String 'Output))) list '('('mode 'renew)))) + (let res (DataSink 'result)) + (let world (Write! world res (Key) list '())) - (let world (Commit! world res_sink)) + (let world (Commit! world res)) (return world) ) )"; @@ -2216,10 +2212,10 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { (let join (MapJoinCore (ToFlow list1) dict 'LeftOnly '('key1 'subkey1) '('key2 'subkey2) '('key1 'key 'value1 'value) '() '() '())) (let list (Collect join)) - (let res_sink (DataSink 'yt (quote plato))) - (let world (Write! world res_sink (Key '('table (String 'Output))) list '('('mode 'renew)))) + (let res (DataSink 'result)) + (let world (Write! world res (Key) list '())) - (let world (Commit! world res_sink)) + (let world (Commit! world res)) (return world) ) )"; @@ -2265,10 +2261,10 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { ))) (let lazy (LazyList join)) - (let res_sink (DataSink 'yt (quote plato))) - (let world (Write! world res_sink (Key '('table (String 'Output))) lazy '('('mode 'renew)))) + (let res (DataSink 'result)) + (let world (Write! world res (Key) lazy '())) - (let world (Commit! world res_sink)) + (let world (Commit! world res)) (return world) ) )"; @@ -2311,10 +2307,10 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { ))) (let lazy (LazyList join)) - (let res_sink (DataSink 'yt (quote plato))) - (let world (Write! world res_sink (Key '('table (String 'Output))) lazy '('('mode 'renew)))) + (let res (DataSink 'result)) + (let world (Write! world res (Key) lazy '())) - (let world (Commit! world res_sink)) + (let world (Commit! world res)) (return world) ) )"; @@ -2362,10 +2358,10 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { (let join (EquiJoin '(list1 'a) '(list2 'b) '(list3 'c) '('Inner '('Inner 'a 'b '('a 'key1 'a 'subkey1) '('b 'key2 'b 'subkey2) '()) 'c '('b 'key2 'b 'subkey2) '('c 'key3 'c 'subkey3) '()) '())) (let lazy (LazyList join)) - (let res_sink (DataSink 'yt (quote plato))) - (let world (Write! world res_sink (Key '('table (String 'Output))) lazy '('('mode 'renew)))) + (let res (DataSink 'result)) + (let world (Write! world res (Key) lazy '())) - (let world (Commit! world res_sink)) + (let world (Commit! world res)) (return world) ) )"; @@ -2413,10 +2409,10 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { (let join (EquiJoin '(list1 'a) '(list2 'b) '(list3 'c) '('Inner '('Left 'a 'b '('a 'key1 'a 'subkey1) '('b 'key2 'b 'subkey2) '()) 'c '('b 'key2 'b 'subkey2) '('c 'key3 'c 'subkey3) '()) '())) (let lazy (LazyList join)) - (let res_sink (DataSink 'yt (quote plato))) - (let world (Write! world res_sink (Key '('table (String 'Output))) lazy '('('mode 'renew)))) + (let res (DataSink 'result)) + (let world (Write! world res (Key) lazy '())) - (let world (Commit! world res_sink)) + (let world (Commit! world res)) (return world) ) )"; @@ -2464,10 +2460,10 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { (let join (EquiJoin '(list1 'a) '(list2 'b) '(list3 'c) '('Inner '('Right 'a 'b '('a 'key1 'a 'subkey1) '('b 'key2 'b 'subkey2) '()) 'c '('b 'key2 'b 'subkey2) '('c 'key3 'c 'subkey3) '()) '())) (let lazy (LazyList join)) - (let res_sink (DataSink 'yt (quote plato))) - (let world (Write! world res_sink (Key '('table (String 'Output))) lazy '('('mode 'renew)))) + (let res (DataSink 'result)) + (let world (Write! world res (Key) lazy '())) - (let world (Commit! world res_sink)) + (let world (Commit! world res)) (return world) ) )"; @@ -2515,10 +2511,10 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { (let join (EquiJoin '(list1 'a) '(list2 'b) '(list3 'c) '('Inner '('Full 'a 'b '('a 'key1 'a 'subkey1) '('b 'key2 'b 'subkey2) '()) 'c '('b 'key2 'b 'subkey2) '('c 'key3 'c 'subkey3) '()) '())) (let lazy (LazyList join)) - (let res_sink (DataSink 'yt (quote plato))) - (let world (Write! world res_sink (Key '('table (String 'Output))) lazy '('('mode 'renew)))) + (let res (DataSink 'result)) + (let world (Write! world res (Key) lazy '())) - (let world (Commit! world res_sink)) + (let world (Commit! world res)) (return world) ) )"; @@ -2566,10 +2562,10 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { (let join (EquiJoin '(list1 'a) '(list2 'b) '(list3 'c) '('Inner '('Exclusion 'a 'b '('a 'key1 'a 'subkey1) '('b 'key2 'b 'subkey2) '()) 'c '('b 'key2 'b 'subkey2) '('c 'key3 'c 'subkey3) '()) '())) (let lazy (LazyList join)) - (let res_sink (DataSink 'yt (quote plato))) - (let world (Write! world res_sink (Key '('table (String 'Output))) lazy '('('mode 'renew)))) + (let res (DataSink 'result)) + (let world (Write! world res (Key) lazy '())) - (let world (Commit! world res_sink)) + (let world (Commit! world res)) (return world) ) )"; @@ -2617,10 +2613,10 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { (let join (EquiJoin '(list1 'a) '(list2 'b) '(list3 'c) '('Inner '('LeftOnly 'a 'b '('a 'key1 'a 'subkey1) '('b 'key2 'b 'subkey2) '()) 'c '('a 'key1 'a 'subkey1) '('c 'key3 'c 'subkey3) '()) '())) (let lazy (LazyList join)) - (let res_sink (DataSink 'yt (quote plato))) - (let world (Write! world res_sink (Key '('table (String 'Output))) lazy '('('mode 'renew)))) + (let res (DataSink 'result)) + (let world (Write! world res (Key) lazy '())) - (let world (Commit! world res_sink)) + (let world (Commit! world res)) (return world) ) )"; @@ -2668,10 +2664,10 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { (let join (EquiJoin '(list1 'a) '(list2 'b) '(list3 'c) '('Inner '('LeftSemi 'a 'b '('a 'key1 'a 'subkey1) '('b 'key2 'b 'subkey2) '()) 'c '('a 'key1 'a 'subkey1) '('c 'key3 'c 'subkey3) '()) '())) (let lazy (LazyList join)) - (let res_sink (DataSink 'yt (quote plato))) - (let world (Write! world res_sink (Key '('table (String 'Output))) lazy '('('mode 'renew)))) + (let res (DataSink 'result)) + (let world (Write! world res (Key) lazy '())) - (let world (Commit! world res_sink)) + (let world (Commit! world res)) (return world) ) )"; @@ -2719,10 +2715,10 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { (let join (EquiJoin '(list1 'a) '(list2 'b) '(list3 'c) '('Inner '('RightOnly 'a 'b '('a 'key1 'a 'subkey1) '('b 'key2 'b 'subkey2) '()) 'c '('b 'key2 'b 'subkey2) '('c 'key3 'c 'subkey3) '()) '())) (let lazy (LazyList join)) - (let res_sink (DataSink 'yt (quote plato))) - (let world (Write! world res_sink (Key '('table (String 'Output))) lazy '('('mode 'renew)))) + (let res (DataSink 'result)) + (let world (Write! world res (Key) lazy '())) - (let world (Commit! world res_sink)) + (let world (Commit! world res)) (return world) ) )"; @@ -2770,10 +2766,10 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { (let join (EquiJoin '(list1 'a) '(list2 'b) '(list3 'c) '('Inner '('RightSemi 'a 'b '('a 'key1 'a 'subkey1) '('b 'key2 'b 'subkey2) '()) 'c '('b 'key2 'b 'subkey2) '('c 'key3 'c 'subkey3) '()) '())) (let lazy (LazyList join)) - (let res_sink (DataSink 'yt (quote plato))) - (let world (Write! world res_sink (Key '('table (String 'Output))) lazy '('('mode 'renew)))) + (let res (DataSink 'result)) + (let world (Write! world res (Key) lazy '())) - (let world (Commit! world res_sink)) + (let world (Commit! world res)) (return world) ) )"; @@ -2821,10 +2817,10 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { (let join (EquiJoin '(list1 'a) '(list2 'b) '(list3 'c) '('Left '('Inner 'a 'b '('a 'key1 'a 'subkey1) '('b 'key2 'b 'subkey2) '()) 'c '('b 'key2 'b 'subkey2) '('c 'key3 'c 'subkey3) '()) '())) (let lazy (LazyList join)) - (let res_sink (DataSink 'yt (quote plato))) - (let world (Write! world res_sink (Key '('table (String 'Output))) lazy '('('mode 'renew)))) + (let res (DataSink 'result)) + (let world (Write! world res (Key) lazy '())) - (let world (Commit! world res_sink)) + (let world (Commit! world res)) (return world) ) )"; @@ -2871,10 +2867,10 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { (let join (EquiJoin '(list1 'a) '(list2 'b) '(list3 'c) '('Left '('Left 'a 'b '('a 'key1 'a 'subkey1) '('b 'key2 'b 'subkey2) '()) 'c '('b 'key2 'b 'subkey2) '('c 'key3 'c 'subkey3) '()) '())) (let lazy (LazyList join)) - (let res_sink (DataSink 'yt (quote plato))) - (let world (Write! world res_sink (Key '('table (String 'Output))) lazy '('('mode 'renew)))) + (let res (DataSink 'result)) + (let world (Write! world res (Key) lazy '())) - (let world (Commit! world res_sink)) + (let world (Commit! world res)) (return world) ) )"; @@ -2922,10 +2918,10 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { (let join (EquiJoin '(list1 'a) '(list2 'b) '(list3 'c) '('Left '('Right 'a 'b '('a 'key1 'a 'subkey1) '('b 'key2 'b 'subkey2) '()) 'c '('b 'key2 'b 'subkey2) '('c 'key3 'c 'subkey3) '()) '())) (let lazy (LazyList join)) - (let res_sink (DataSink 'yt (quote plato))) - (let world (Write! world res_sink (Key '('table (String 'Output))) lazy '('('mode 'renew)))) + (let res (DataSink 'result)) + (let world (Write! world res (Key) lazy '())) - (let world (Commit! world res_sink)) + (let world (Commit! world res)) (return world) ) )"; @@ -2973,10 +2969,10 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { (let join (EquiJoin '(list1 'a) '(list2 'b) '(list3 'c) '('Left '('Full 'a 'b '('a 'key1 'a 'subkey1) '('b 'key2 'b 'subkey2) '()) 'c '('b 'key2 'b 'subkey2) '('c 'key3 'c 'subkey3) '()) '())) (let lazy (LazyList join)) - (let res_sink (DataSink 'yt (quote plato))) - (let world (Write! world res_sink (Key '('table (String 'Output))) lazy '('('mode 'renew)))) + (let res (DataSink 'result)) + (let world (Write! world res (Key) lazy '())) - (let world (Commit! world res_sink)) + (let world (Commit! world res)) (return world) ) )"; @@ -3024,10 +3020,10 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { (let join (EquiJoin '(list1 'a) '(list2 'b) '(list3 'c) '('Left '('Exclusion 'a 'b '('a 'key1 'a 'subkey1) '('b 'key2 'b 'subkey2) '()) 'c '('b 'key2 'b 'subkey2) '('c 'key3 'c 'subkey3) '()) '())) (let lazy (LazyList join)) - (let res_sink (DataSink 'yt (quote plato))) - (let world (Write! world res_sink (Key '('table (String 'Output))) lazy '('('mode 'renew)))) + (let res (DataSink 'result)) + (let world (Write! world res (Key) lazy '())) - (let world (Commit! world res_sink)) + (let world (Commit! world res)) (return world) ) )"; @@ -3075,10 +3071,10 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { (let join (EquiJoin '(list1 'a) '(list2 'b) '(list3 'c) '('Left '('LeftOnly 'a 'b '('a 'key1 'a 'subkey1) '('b 'key2 'b 'subkey2) '()) 'c '('a 'key1 'a 'subkey1) '('c 'key3 'c 'subkey3) '()) '())) (let lazy (LazyList join)) - (let res_sink (DataSink 'yt (quote plato))) - (let world (Write! world res_sink (Key '('table (String 'Output))) lazy '('('mode 'renew)))) + (let res (DataSink 'result)) + (let world (Write! world res (Key) lazy '())) - (let world (Commit! world res_sink)) + (let world (Commit! world res)) (return world) ) )"; @@ -3126,10 +3122,10 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { (let join (EquiJoin '(list1 'a) '(list2 'b) '(list3 'c) '('Left '('LeftSemi 'a 'b '('a 'key1 'a 'subkey1) '('b 'key2 'b 'subkey2) '()) 'c '('a 'key1 'a 'subkey1) '('c 'key3 'c 'subkey3) '()) '())) (let lazy (LazyList join)) - (let res_sink (DataSink 'yt (quote plato))) - (let world (Write! world res_sink (Key '('table (String 'Output))) lazy '('('mode 'renew)))) + (let res (DataSink 'result)) + (let world (Write! world res (Key) lazy '())) - (let world (Commit! world res_sink)) + (let world (Commit! world res)) (return world) ) )"; @@ -3177,10 +3173,10 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { (let join (EquiJoin '(list1 'a) '(list2 'b) '(list3 'c) '('Left '('RightOnly 'a 'b '('a 'key1 'a 'subkey1) '('b 'key2 'b 'subkey2) '()) 'c '('b 'key2 'b 'subkey2) '('c 'key3 'c 'subkey3) '()) '())) (let lazy (LazyList join)) - (let res_sink (DataSink 'yt (quote plato))) - (let world (Write! world res_sink (Key '('table (String 'Output))) lazy '('('mode 'renew)))) + (let res (DataSink 'result)) + (let world (Write! world res (Key) lazy '())) - (let world (Commit! world res_sink)) + (let world (Commit! world res)) (return world) ) )"; @@ -3228,10 +3224,10 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { (let join (EquiJoin '(list1 'a) '(list2 'b) '(list3 'c) '('Left '('RightSemi 'a 'b '('a 'key1 'a 'subkey1) '('b 'key2 'b 'subkey2) '()) 'c '('b 'key2 'b 'subkey2) '('c 'key3 'c 'subkey3) '()) '())) (let lazy (LazyList join)) - (let res_sink (DataSink 'yt (quote plato))) - (let world (Write! world res_sink (Key '('table (String 'Output))) lazy '('('mode 'renew)))) + (let res (DataSink 'result)) + (let world (Write! world res (Key) lazy '())) - (let world (Commit! world res_sink)) + (let world (Commit! world res)) (return world) ) )"; @@ -3279,10 +3275,10 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { (let join (EquiJoin '(list1 'a) '(list2 'b) '(list3 'c) '('Inner '('Inner 'a 'b '('a 'key1 'a 'subkey1) '('b 'key2 'b 'subkey2) '()) 'c '('b 'key2 'b 'subkey2) '('c 'key3 'c 'subkey3) '()) '('('flatten)))) (let lazy (LazyList join)) - (let res_sink (DataSink 'yt (quote plato))) - (let world (Write! world res_sink (Key '('table (String 'Output))) lazy '('('mode 'renew)))) + (let res (DataSink 'result)) + (let world (Write! world res (Key) lazy '())) - (let world (Commit! world res_sink)) + (let world (Commit! world res)) (return world) ) )"; @@ -3302,7 +3298,7 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { (let input (FlatMap list (lambda '(row) (Just (AsStruct '('"data" (Member row '"data")) '('group0 (AsList (Member row '"user"))) '('"time" (Member row '"time")) '('"user" (Member row '"user"))))))) (let keySelector (lambda '(row) '((StablePickle (Member row '"data")) (StablePickle (Member row 'group0))))) (let sortKeySelector (lambda '(row) (SafeCast (Member row '"time") (OptionalType (DataType 'Timestamp))))) -(let res (PartitionsByKeys input keySelector (Bool 'true) sortKeySelector (lambda '(row) (block '( +(let result (PartitionsByKeys input keySelector (Bool 'true) sortKeySelector (lambda '(row) (block '( (let interval (Interval '1000000)) (let map (lambda '(item) (AsStruct))) (let reduce (lambda '(lhs rhs) (AsStruct))) @@ -3310,9 +3306,9 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) { (return (ForwardList (FlatMap hopping (lambda '(row) (Just (AsStruct '('_yql_time (Member row '_yql_time)) '('"data" (Unpickle (NullType) (Member row '"data"))) '('group0 (Unpickle (ListType (DataType 'Int32)) (Member row 'group0))))))))) ))))) -(let res_sink (DataSink 'yt (quote plato))) -(let world (Write! world res_sink (Key '('table (String 'Output))) res '('('mode 'renew)))) -(return (Commit! world res_sink)) +(let res (DataSink 'result)) +(let world (Write! world res (Key) result '())) +(return (Commit! world res)) ))"; TExprContext exprCtx; diff --git a/yql/essentials/core/ut/yql_expr_discover_ut.cpp b/yql/essentials/core/ut/yql_expr_discover_ut.cpp deleted file mode 100644 index 59dfd69dc48..00000000000 --- a/yql/essentials/core/ut/yql_expr_discover_ut.cpp +++ /dev/null @@ -1,151 +0,0 @@ -#include "yql_opt_proposed_by_data.h" - -#include <yt/yql/providers/yt/provider/yql_yt_provider.h> -#include <yt/yql/providers/yt/gateway/file/yql_yt_file.h> -#include <yt/yql/providers/yt/gateway/file/yql_yt_file_services.h> -#include <yql/essentials/providers/common/provider/yql_provider_names.h> -#include <yql/essentials/ast/yql_ast_annotation.h> -#include <yql/essentials/ast/yql_expr.h> -#include <yql/essentials/core/type_ann/type_ann_core.h> -#include <yql/essentials/core/ut_common/yql_ut_common.h> -#include <yql/essentials/minikql/invoke_builtins/mkql_builtins.h> -#include <yql/essentials/core/services/yql_eval_expr.h> - -#include <library/cpp/testing/unittest/registar.h> -#include <library/cpp/yson/writer.h> - -namespace NYql { - -Y_UNIT_TEST_SUITE(TDiscoverYqlExpr) { - - static TString Discover(const TString& ast) { - TAstParseResult astRes = ParseAst(ast); - UNIT_ASSERT(astRes.IsOk()); - TExprContext exprCtx; - TExprNode::TPtr exprRoot; - UNIT_ASSERT(CompileExpr(*astRes.Root, exprRoot, exprCtx, nullptr, nullptr)); - - auto functionRegistry = NKikimr::NMiniKQL::CreateFunctionRegistry(NKikimr::NMiniKQL::CreateBuiltinRegistry()); - TTestTablesMapping testTables; - auto yqlNativeServices = NFile::TYtFileServices::Make(functionRegistry.Get(), testTables); - auto ytGateway = CreateYtFileGateway(yqlNativeServices); - auto typeAnnotationContext = MakeIntrusive<TTypeAnnotationContext>(); - typeAnnotationContext->DiscoveryMode = true; - auto ytState = MakeIntrusive<TYtState>(typeAnnotationContext.Get()); - ytState->Gateway = ytGateway; - - InitializeYtGateway(ytGateway, ytState); - auto randomProvider = CreateDeterministicRandomProvider(1); - typeAnnotationContext->AddDataSink(YtProviderName, CreateYtDataSink(ytState)); - auto datasource = CreateYtDataSource(ytState); - typeAnnotationContext->AddDataSource(YtProviderName, datasource); - auto intentTransformer = CreateIntentDeterminationTransformer(*typeAnnotationContext); - TVector<TTransformStage> transformers; - const auto issueCode = TIssuesIds::DEFAULT_ERROR; - transformers.push_back(TTransformStage(CreateFunctorTransformer( - [=](const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) { - return EvaluateExpression(input, output, *typeAnnotationContext, ctx, *functionRegistry, nullptr); - }), - "EvaluateExpression", - issueCode)); - transformers.push_back(TTransformStage( - CreateIODiscoveryTransformer(*typeAnnotationContext), - "IODiscovery", - issueCode)); - transformers.push_back(TTransformStage( - CreateEpochsTransformer(*typeAnnotationContext), - "Epochs", - issueCode)); - transformers.push_back(TTransformStage( - intentTransformer, - "IntentDetermination", - issueCode)); - auto fullTransformer = CreateCompositeGraphTransformer(transformers, true); - - TStringStream str; - if (SyncTransform(*fullTransformer, exprRoot, exprCtx) == IGraphTransformer::TStatus::Ok) { - NYson::TYsonWriter writer(&str, NYson::EYsonFormat::Text); - datasource->CollectDiscoveredData(writer); - } else { - exprCtx.IssueManager.GetIssues().PrintTo(str); - } - return str.Str(); - } - - Y_UNIT_TEST(DiscoverYt) { - auto s = R"(( -(let mr_source (DataSource 'yt 'plato)) -(let x (Read! world mr_source - (Key '('table (String 'Input))) - '('key 'subkey 'value) '())) -(let world (Left! x)) -(let table (Right! x)) - -(let mr_sink (DataSink 'yt 'plato)) -(let world (Write! world mr_sink - (Key '('table (String 'Output))) - table '('('mode 'append)))) - -(let world (Commit! world mr_sink)) -(return world) -))"; - - auto res = Discover(s); - UNIT_ASSERT_VALUES_EQUAL(res, "[[\"plato\";\"Input\";[\"read\"]];[\"plato\";\"Output\";[\"modify\"]]]"); - } - - Y_UNIT_TEST(ErrorOnRange) { - auto s = R"(( -(let mr_source (DataSource 'yt 'plato)) -(let range (MrTableRange '"" (lambda '($i) (And (>= $i (String '"Input1")) (<= $i (String '"Input2")))) '"")) -(let x (Read! world mr_source - (Key '('table range)) - '('key 'subkey 'value) '())) -(let world (Left! x)) -(let table (Right! x)) - -(let mr_sink (DataSink 'yt 'plato)) -(let world (Write! world mr_sink - (Key '('table (String 'Output))) - table '('('mode 'append)))) - -(let world (Commit! world mr_sink)) -(return world) -))"; - - auto res = Discover(s); - UNIT_ASSERT_VALUES_EQUAL(res, R"(<main>: Error: Default error - <main>:5:6: Error: MrTableRange/MrTableRangeStrict is not allowed in Discovery mode, code: 4600 -)"); - } - - Y_UNIT_TEST(ErrorOnTime) { - auto s = R"(( -(let mr_source (DataSource 'yt 'plato)) -(let x (Read! world mr_source - (Key '('table (String (EvaluateAtom (SafeCast (CurrentUtcDate) (DataType 'String)))))) - '('key 'subkey 'value) '())) -(let world (Left! x)) -(let table (Right! x)) - -(let mr_sink (DataSink 'yt 'plato)) -(let world (Write! world mr_sink - (Key '('table (String 'Output))) - table '('('mode 'append)))) - -(let world (Commit! world mr_sink)) -(return world) -))"; - - auto res = Discover(s); - UNIT_ASSERT_VALUES_EQUAL(res, R"(<main>: Error: Default error - <main>:4:28: Error: At function: EvaluateAtom - <main>: Error: Type annotation, code: 1030 - <main>:4:42: Error: At function: SafeCast - <main>:4:52: Error: At function: CurrentUtcDate - <main>:4:52: Error: CurrentUtcDate is not allowed in Discovery mode, code: 4600 -)"); - } -} - -} // namespace NYql diff --git a/yql/essentials/core/ut/yql_expr_optimize_ut.cpp b/yql/essentials/core/ut/yql_expr_optimize_ut.cpp index 3e97406d12e..d7a381ccb60 100644 --- a/yql/essentials/core/ut/yql_expr_optimize_ut.cpp +++ b/yql/essentials/core/ut/yql_expr_optimize_ut.cpp @@ -2,15 +2,11 @@ #include "yql_opt_rewrite_io.h" #include "yql_opt_proposed_by_data.h" -#include <yt/yql/providers/yt/provider/yql_yt_provider.h> -#include <yt/yql/providers/yt/gateway/file/yql_yt_file.h> -#include <yt/yql/providers/yt/gateway/file/yql_yt_file_services.h> #include <yql/essentials/providers/common/provider/yql_provider_names.h> #include <yql/essentials/ast/yql_ast_annotation.h> #include <yql/essentials/ast/yql_expr.h> #include <yql/essentials/core/type_ann/type_ann_core.h> #include <yql/essentials/core/type_ann/type_ann_expr.h> -#include <yql/essentials/core/ut_common/yql_ut_common.h> #include <yql/essentials/core/facade/yql_facade.h> #include <yql/essentials/minikql/invoke_builtins/mkql_builtins.h> @@ -566,90 +562,6 @@ Y_UNIT_TEST_SUITE(TOptimizeYqlExpr) { UNIT_ASSERT_EQUAL(IGraphTransformer::TStatus::Error, ExpandApply(exprRoot, exprRoot, exprCtx).Level); UNIT_ASSERT_VALUES_EQUAL("<main>:2:20: Error: Expected callable, but got: Atom\n", exprCtx.IssueManager.GetIssues().ToString()); } - - Y_UNIT_TEST(CheckIORewrite) { - auto s = "(\n" - "(let mr_source (DataSource 'yt 'plato))\n" - "(let x (Read! world mr_source (Key '('table (String 'Input))) '('key 'subkey 'value) '()))\n" - "(let world (Left! x))\n" - "(let table1 (Right! x))\n" - "(let tresh (String '100))\n" - "(let table1low (Filter table1 (lambda '(item1) (> tresh (Member item1 'key)))))\n" - "(let mr_sink (DataSink 'yt (quote plato)))\n" - "(let world (Write! world mr_sink (Key '('table (String 'Output))) table1low '('('mode 'append))))\n" - "(let world (Commit! world mr_sink))\n" - "(return world)\n" - ")\n"; - - TAstParseResult astRes = ParseAst(s); - UNIT_ASSERT(astRes.IsOk()); - TExprContext exprCtx; - TExprNode::TPtr exprRoot; - UNIT_ASSERT(CompileExpr(*astRes.Root, exprRoot, exprCtx, nullptr, nullptr)); - - auto functionRegistry = NKikimr::NMiniKQL::CreateFunctionRegistry(NKikimr::NMiniKQL::CreateBuiltinRegistry()); - TTestTablesMapping testTables; - auto yqlNativeServices = NFile::TYtFileServices::Make(functionRegistry.Get(), testTables); - auto ytGateway = CreateYtFileGateway(yqlNativeServices); - auto typeAnnotationContext = MakeIntrusive<TTypeAnnotationContext>(); - auto ytState = MakeIntrusive<TYtState>(typeAnnotationContext.Get()); - ytState->Gateway = ytGateway; - - InitializeYtGateway(ytGateway, ytState); - auto randomProvider = CreateDeterministicRandomProvider(1); - typeAnnotationContext->AddDataSink(YtProviderName, CreateYtDataSink(ytState)); - typeAnnotationContext->AddDataSource(YtProviderName, CreateYtDataSource(ytState)); - typeAnnotationContext->RandomProvider = randomProvider; - auto intentTransformer = CreateIntentDeterminationTransformer(*typeAnnotationContext); - TVector<TTransformStage> transformers; - const auto issueCode = TIssuesIds::DEFAULT_ERROR; - transformers.push_back(TTransformStage( - CreateIODiscoveryTransformer(*typeAnnotationContext), - "IODiscovery", - issueCode)); - transformers.push_back(TTransformStage( - CreateEpochsTransformer(*typeAnnotationContext), - "Epochs", - issueCode)); - - // NOTE: add fake EvaluateExpression step to break infinite loop - // (created by Repeat on ExprEval step after RewriteIO completion) - transformers.push_back(TTransformStage( - CreateFunctorTransformer( - [](const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) { - output = input; - ctx.Step.Done(TExprStep::ExprEval); - return IGraphTransformer::TStatus::Ok; - } - ), - "EvaluateExpression", - issueCode)); - - transformers.push_back(TTransformStage( - intentTransformer, - "IntentDetermination", - issueCode)); - transformers.push_back(TTransformStage( - CreateTableMetadataLoader(*typeAnnotationContext), - "MetadataLoader", - issueCode)); - // NOTE: metadata loader unconditionally drops ExpandApplyForLambdas flag - // in DoApplyAsyncChanges, so ExpandApply transformation is in such an unusual place - transformers.push_back(TTransformStage( - CreateFunctorTransformer(&ExpandApply), - "ExpandApply", - issueCode)); - auto fullTransformer = CreateCompositeGraphTransformer(transformers, true); - bool success = SyncTransform(*fullTransformer, exprRoot, exprCtx) == IGraphTransformer::TStatus::Ok; - UNIT_ASSERT(success); - - UNIT_ASSERT(RewriteIO(exprRoot, exprRoot, *typeAnnotationContext, exprCtx).Level == IGraphTransformer::TStatus::Ok); - - auto ast = ConvertToAst(*exprRoot, exprCtx, TExprAnnotationFlags::None, true); - auto strRes = ast.Root->ToString(TAstPrintFlags::PerLine | TAstPrintFlags::ShortQuote); - UNIT_ASSERT(strRes.find("(YtReadTable!") != TString::npos); - UNIT_ASSERT(strRes.find("(Read!") == TString::npos); - } } } // namespace NYql diff --git a/yql/essentials/core/ut/yql_expr_providers_ut.cpp b/yql/essentials/core/ut/yql_expr_providers_ut.cpp deleted file mode 100644 index ba98ccc8f97..00000000000 --- a/yql/essentials/core/ut/yql_expr_providers_ut.cpp +++ /dev/null @@ -1,286 +0,0 @@ -#include <yql/essentials/core/type_ann/type_ann_core.h> -#include <yql/essentials/core/type_ann/type_ann_expr.h> - -#include <yql/essentials/ast/yql_ast_annotation.h> -#include <yql/essentials/ast/yql_expr.h> -#include <yql/essentials/core/ut_common/yql_ut_common.h> -#include <yql/essentials/core/yql_expr_type_annotation.h> -#include <yql/essentials/core/facade/yql_facade.h> -#include <yql/essentials/providers/common/provider/yql_provider_names.h> -#include <yt/yql/providers/yt/provider/yql_yt_provider.h> -#include <yt/yql/providers/yt/gateway/file/yql_yt_file.h> -#include <yt/yql/providers/yt/gateway/file/yql_yt_file_services.h> -#include <yql/essentials/minikql/invoke_builtins/mkql_builtins.h> - -#include <library/cpp/testing/unittest/registar.h> - -#include <util/string/hex.h> -#include <util/random/random.h> -#include <util/system/sanitizers.h> - -namespace NYql { - -Y_UNIT_TEST_SUITE(TCompileWithProvidersYqlExpr) { - static TAstParseResult ParseAstWithCheck(const TStringBuf& s) { - TAstParseResult res = ParseAst(s); - res.Issues.PrintTo(Cout); - UNIT_ASSERT(res.IsOk()); - return res; - } - - static void CompileExprWithCheck(TAstNode& root, TExprNode::TPtr& exprRoot, TExprContext& exprCtx, - ui32 annotationFlags = TExprAnnotationFlags::None) - { - const bool success = CompileExpr(root, exprRoot, exprCtx, nullptr, nullptr, annotationFlags); - exprCtx.IssueManager.GetIssues().PrintTo(Cout); - - UNIT_ASSERT(success); - UNIT_ASSERT_VALUES_EQUAL(exprRoot->GetState(), annotationFlags != TExprAnnotationFlags::None - ? TExprNode::EState::TypeComplete - : TExprNode::EState::Initial); - } - - static void AnnotateExprWithCheck(TExprNode::TPtr& root, TExprContext& ctx, bool wholeProgram, TTypeAnnotationContextPtr typeAnnotationContext) { - const bool success = SyncAnnotateTypes(root, ctx, wholeProgram, *typeAnnotationContext); - ctx.IssueManager.GetIssues().PrintTo(Cout); - UNIT_ASSERT(success); - } - - static void VerifyProgram(const TString& s) { - TAstParseResult astRes = ParseAstWithCheck(s); - TExprContext exprCtx; - TExprNode::TPtr exprRoot; - CompileExprWithCheck(*astRes.Root, exprRoot, exprCtx); - UNIT_ASSERT(!exprRoot->GetTypeAnn()); - - auto functionRegistry = NKikimr::NMiniKQL::CreateFunctionRegistry(NKikimr::NMiniKQL::CreateBuiltinRegistry()); - TTestTablesMapping testTables; - - auto yqlNativeServices = NFile::TYtFileServices::Make(functionRegistry.Get(), testTables); - auto ytGateway = CreateYtFileGateway(yqlNativeServices); - auto typeAnnotationContext = MakeIntrusive<TTypeAnnotationContext>(); - auto ytState = MakeIntrusive<TYtState>(typeAnnotationContext.Get()); - ytState->Gateway = ytGateway; - - InitializeYtGateway(ytGateway, ytState); - typeAnnotationContext->AddDataSink(YtProviderName, CreateYtDataSink(ytState)); - typeAnnotationContext->AddDataSource(YtProviderName, CreateYtDataSource(ytState)); - auto randomProvider = CreateDeterministicRandomProvider(1); - typeAnnotationContext->RandomProvider = randomProvider; - AnnotateExprWithCheck(exprRoot, exprCtx, true, typeAnnotationContext); - UNIT_ASSERT(exprRoot->GetTypeAnn()); - //PrintExpr(Cout, *exprRoot, exprRoot, true); - - auto convertedAst = ConvertToAst(*exprRoot, exprCtx, TExprAnnotationFlags::None, true); - //PrintAst(Cout, *convertedAst, TAstPrintFlags::PerLine | TAstPrintFlags::ShortQuote); - TStringStream strOrig; - convertedAst.Root->PrettyPrintTo(strOrig, TAstPrintFlags::PerLine | TAstPrintFlags::ShortQuote); - - //PrintAst(Cout, *AnnotatePositions(*convertedAst), TAstPrintFlags::PerLine | TAstPrintFlags::ShortQuote); - - auto convertedAstWithAnnotations1 = ConvertToAst(*exprRoot, exprCtx, TExprAnnotationFlags::Position, true); - TString strAnn1res = convertedAstWithAnnotations1.Root->ToString(TAstPrintFlags::ShortQuote); - TAstParseResult astResAnn1 = ParseAstWithCheck(strAnn1res); - UNIT_ASSERT(astResAnn1.IsOk()); - - TMemoryPool pool(4096); - auto cleanAst1 = RemoveAnnotations(*convertedAstWithAnnotations1.Root, pool); - UNIT_ASSERT(cleanAst1); - TStringStream strClean1; - cleanAst1->PrettyPrintTo(strClean1, TAstPrintFlags::PerLine | TAstPrintFlags::ShortQuote); - UNIT_ASSERT_VALUES_EQUAL(strClean1.Str(), strOrig.Str()); - - auto convertedAstWithAnnotations2 = ConvertToAst(*exprRoot, exprCtx, TExprAnnotationFlags::Types, true); - TString strAnn2res = convertedAstWithAnnotations2.Root->ToString(TAstPrintFlags::ShortQuote); - - TAstParseResult astResAnn2 = ParseAstWithCheck(strAnn2res); - UNIT_ASSERT(astResAnn2.IsOk()); - - auto cleanAst2 = RemoveAnnotations(*convertedAstWithAnnotations2.Root, pool); - UNIT_ASSERT(cleanAst2); - TStringStream strClean2; - cleanAst2->PrettyPrintTo(strClean2, TAstPrintFlags::PerLine | TAstPrintFlags::ShortQuote); - UNIT_ASSERT_VALUES_EQUAL(strClean2.Str(), strOrig.Str()); - - TExprContext exprCtxAnnTypes; - TExprNode::TPtr exprRootAnnTypes; - CompileExprWithCheck(*astResAnn2.Root, exprRootAnnTypes, exprCtxAnnTypes, (ui32)TExprAnnotationFlags::Types); - UNIT_ASSERT(exprRootAnnTypes->GetTypeAnn()); - - auto convertedAstWithAnnotations3 = ConvertToAst(*exprRootAnnTypes, exprCtxAnnTypes, TExprAnnotationFlags::Types, true); - TString strAnn3res = convertedAstWithAnnotations3.Root->ToString(TAstPrintFlags::ShortQuote); - UNIT_ASSERT_EQUAL(strAnn2res, strAnn3res); - - const ui32 flagsToParseBack[] = { - 0, - TAstPrintFlags::PerLine, - // TAstPrintFlags::ShortQuote, // -- generates invalid AST - TAstPrintFlags::PerLine | TAstPrintFlags::ShortQuote - }; - - for (ui32 index = 0; index < Y_ARRAY_SIZE(flagsToParseBack); ++index) { - auto flag = flagsToParseBack[index]; - auto s2 = convertedAst.Root->ToString(flag); - - TAstParseResult res2 = ParseAst(s2); - res2.Issues.PrintTo(Cout); - UNIT_ASSERT(res2.IsOk()); - - auto s3 = res2.Root->ToString(flag); - UNIT_ASSERT_STRINGS_EQUAL(s2, s3); - - TExprContext exprCtx2; - TExprNode::TPtr exprRoot2; - CompileExprWithCheck(*res2.Root, exprRoot2, exprCtx2); - - auto convertedAst2 = ConvertToAst(*exprRoot2, exprCtx2, TExprAnnotationFlags::None, true); - auto s4 = convertedAst2.Root->ToString(flag); - UNIT_ASSERT_STRINGS_EQUAL(s2, s4); - } - } - - Y_UNIT_TEST(TestComplexProgramWithLamda) { - auto s = "(\n" - "#comment\n" - "(let mr_source (DataSource 'yt 'plato))\n" - "(let x (Read! world mr_source (Key '('table (String 'Input))) '('key 'subkey 'value) '()))\n" - "(let world (Left! x))\n" - "(let table1 (Right! x))\n" - "(let tresh (String '100))\n" - "(let table1low (Filter table1 (lambda '(item) (< (Member item 'key) tresh))))\n" - "(let mr_sink (DataSink 'yt (quote plato)))\n" - "(let world (Write! world mr_sink (Key '('table (String 'Output))) table1low '('('mode 'append))))\n" - "(let world (Commit! world mr_sink))\n" - "(return world)\n" - ")\n"; - - VerifyProgram(s); - } - - Y_UNIT_TEST(TestSerializeAtomAsPartialExpression) { - auto s = "(\n" - "(let x 'a)\n" - "(let y 'b)\n" - "(let z (quote (x y)))\n" - "(return z)\n" - ")\n"; - - TAstParseResult astRes = ParseAstWithCheck(s); - UNIT_ASSERT(astRes.IsOk()); - - TExprContext exprCtx; - TExprNode::TPtr exprRoot; - CompileExprWithCheck(*astRes.Root, exprRoot, exprCtx); - - auto typeAnnotationContext = MakeIntrusive<TTypeAnnotationContext>(); - auto randomProvider = CreateDeterministicRandomProvider(1); - typeAnnotationContext->RandomProvider = randomProvider; - AnnotateExprWithCheck(exprRoot, exprCtx, false, typeAnnotationContext); - UNIT_ASSERT(exprRoot->GetTypeAnn()); - UNIT_ASSERT_VALUES_EQUAL(exprRoot->Child(0)->Content(), "a"); - - auto convertedAstWithTypes = ConvertToAst(*exprRoot->Child(0), exprCtx, TExprAnnotationFlags::Types, true); - TString strAnnRes = convertedAstWithTypes.Root->ToString(TAstPrintFlags::ShortQuote); - TAstParseResult astRes2 = ParseAstWithCheck(strAnnRes); - UNIT_ASSERT(astRes2.IsOk()); - - TExprContext exprCtx2; - TExprNode::TPtr exprRoot2; - CompileExprWithCheck(*astRes2.Root, exprRoot2, exprCtx2, (ui32)TExprAnnotationFlags::Types); - UNIT_ASSERT(exprRoot2->GetTypeAnn()); - UNIT_ASSERT_VALUES_EQUAL(exprRoot2->Content(), "a"); - - ui32 annotationFlags = TExprAnnotationFlags::Types | TExprAnnotationFlags::Position; - auto convertedAstWithTypesAndPos = ConvertToAst(*exprRoot->Child(0), exprCtx, annotationFlags, true); - TString strPosAnnRes = convertedAstWithTypesAndPos.Root->ToString(TAstPrintFlags::ShortQuote); - TAstParseResult astRes3 = ParseAstWithCheck(strPosAnnRes); - UNIT_ASSERT(astRes3.IsOk()); - - TExprContext exprCtx3; - TExprNode::TPtr exprRoot3; - CompileExprWithCheck(*astRes3.Root, exprRoot3, exprCtx3, annotationFlags); - UNIT_ASSERT(exprRoot2->GetTypeAnn()); - UNIT_ASSERT_VALUES_EQUAL(exprRoot2->Content(), "a"); - } - - Y_UNIT_TEST(TestComplexProgramWithLamdaAsBlock) { - auto s = "(\n" - "#comment\n" - "(let mr_source (DataSource 'yt 'plato))\n" - "(let x (Read! world mr_source (Key '('table (String 'Input))) '('key 'subkey 'value) '()))\n" - "(let world (Left! x))\n" - "(let table1 (Right! x))\n" - "(let table1low (Filter table1 (lambda '(item) (block '(\n" - " (let tresh (String '100))\n" - " (let predicate (< (Member item 'key) tresh))\n" - " (return predicate)\n" - ")))))\n" - "(let mr_sink (DataSink 'yt (quote plato)))\n" - "(let world (Write! world mr_sink (Key '('table (String 'Output))) table1low '('('mode 'append))))\n" - "(let world (Commit! world mr_sink))\n" - "(return world)\n" - ")\n"; - - VerifyProgram(s); - } - - Y_UNIT_TEST(TestComplexProgramWithBlockAndLambdaAsBlock) { - auto s = "(\n" - "#comment\n" - "(let mr_source (DataSource 'yt 'plato))\n" - "(let data (block '(\n" - " (let x (Read! world mr_source (Key '('table (String 'Input))) '('key 'subkey 'value) '()))\n" - " (let world (Left! x))\n" - " (let table1 (Right! x))\n" - " (let table1low (Filter table1 (lambda '(item) (block '(\n" - " (let tresh (String '100))\n" - " (let predicate (< (Member item 'key) tresh))\n" - " (return predicate)\n" - " )))))\n" - " (return table1low)\n" - ")))\n" - "(let mr_sink (DataSink 'yt (quote plato)))\n" - "(let world (Write! world mr_sink (Key '('table (String 'Output))) data '('('mode 'append))))\n" - "(let world (Commit! world mr_sink))\n" - "(return world)\n" - ")\n"; - - VerifyProgram(s); - } - - Y_UNIT_TEST(TestPerfCompile) { - auto s = "(\n" - "#comment\n" - "(let mr_source (DataSource 'yt 'plato))\n" - "(let data (block '(\n" - " (let x (Read! world mr_source (Key '('table (String 'Input))) '('key 'subkey 'value) '()))\n" - " (let world (Left! x))\n" - " (let table1 (Right! x))\n" - " (let table1low (Filter table1 (lambda '(item) (block '(\n" - " (let tresh (String '100))\n" - " (let predicate (< (Member item 'key) tresh))\n" - " (return predicate)\n" - " )))))\n" - " (return table1low)\n" - ")))\n" - "(let mr_sink (DataSink 'yt (quote plato)))\n" - "(let world (Write! world mr_sink (Key '('table (String 'Output))) data '('('mode 'append))))\n" - "(let world (Commit! world mr_sink))\n" - "(return world)\n" - ")\n"; - - TAstParseResult astRes = ParseAstWithCheck(s); - const ui32 n = NSan::PlainOrUnderSanitizer(10000, 1000); - auto t1 = TInstant::Now(); - for (ui32 i = 0; i < n; ++i) { - TExprContext exprCtx; - TExprNode::TPtr exprRoot; - CompileExprWithCheck(*astRes.Root, exprRoot, exprCtx); - } - - auto t2 = TInstant::Now(); - Cout << t2 - t1 << Endl; - } -} - -} // namespace NYql diff --git a/yql/essentials/core/ut/yql_library_compiler_ut.cpp b/yql/essentials/core/ut/yql_library_compiler_ut.cpp index 0634578f963..0942dfc424b 100644 --- a/yql/essentials/core/ut/yql_library_compiler_ut.cpp +++ b/yql/essentials/core/ut/yql_library_compiler_ut.cpp @@ -1,7 +1,5 @@ #include <library/cpp/testing/unittest/registar.h> -#include <yql/essentials/core/ut_common/yql_ut_common.h> - #include "yql_library_compiler.h" namespace NYql { diff --git a/yql/essentials/core/ut/yql_qplayer_ut.cpp b/yql/essentials/core/ut/yql_qplayer_ut.cpp deleted file mode 100644 index 11dca7b68c7..00000000000 --- a/yql/essentials/core/ut/yql_qplayer_ut.cpp +++ /dev/null @@ -1,310 +0,0 @@ -#include <yql/essentials/minikql/invoke_builtins/mkql_builtins.h> -#include <yt/yql/providers/yt/provider/yql_yt_provider.h> -#include <yt/yql/providers/yt/gateway/file/yql_yt_file.h> -#include <yt/yql/providers/yt/gateway/file/yql_yt_file_services.h> -#include <yql/essentials/core/cbo/simple/cbo_simple.h> -#include <yql/essentials/core/facade/yql_facade.h> -#include <yql/essentials/core/qplayer/storage/memory/yql_qstorage_memory.h> -#include <yql/essentials/providers/common/udf_resolve/yql_simple_udf_resolver.h> -#include <yql/essentials/providers/common/provider/yql_provider_names.h> -#include <yql/essentials/core/file_storage/file_storage.h> -#include <yql/essentials/core/services/mounts/yql_mounts.h> - -#include <library/cpp/yson/node/node_io.h> - -#include <library/cpp/testing/unittest/registar.h> - -#include <util/system/user.h> -#include <util/system/tempfile.h> - -using namespace NYql; - -template <typename F> -void WithTables(const F&& f) { - static const TStringBuf KSV_ATTRS = - "{\"_yql_row_spec\" = {\"Type\" = [\"StructType\";[" - "[\"key\";[\"DataType\";\"String\"]];" - "[\"subkey\";[\"DataType\";\"String\"]];" - "[\"value\";[\"DataType\";\"String\"]]" - "]]}}" - ; - - TTempFileHandle inputFile; - TTempFileHandle inputFileAttrs(inputFile.Name() + ".attr"); - - TStringBuf data = - "{\"key\"=\"075\";\"subkey\"=\".\";\"value\"=\"abc\"};\n" - "{\"key\"=\"800\";\"subkey\"=\".\";\"value\"=\"ddd\"};\n" - "{\"key\"=\"020\";\"subkey\"=\".\";\"value\"=\"q\"};\n" - "{\"key\"=\"150\";\"subkey\"=\".\";\"value\"=\"qzz\"};\n"sv - ; - - inputFile.Write(data.data(), data.size()); - inputFile.FlushData(); - inputFileAttrs.Write(KSV_ATTRS.data(), KSV_ATTRS.size()); - inputFileAttrs.FlushData(); - - THashMap<TString, TString> tables; - tables["yt.plato.Input"] = inputFile.Name(); - f(tables); -} - -struct TRunSettings { - bool IsSql = true; - bool IsPg = false; - THashMap<TString, TString> Tables; - TMaybe<TString> ParametersYson; - THashMap<TString, TString> StaticFiles, DynamicFiles; - bool ChangeTime = false; - TVector<std::pair<TString, TCredential>> Credentials; -}; - -TUserDataTable MakeUserTables(const THashMap<TString, TString>& map) { - TUserDataTable userDataTable; - for (const auto& f : map) { - TUserDataBlock block; - block.Type = EUserDataType::RAW_INLINE_DATA; - block.Data = f.second; - userDataTable[TUserDataKey::File(GetDefaultFilePrefix() + f.first)] = block; - } - - return userDataTable; -}; - -bool RunProgram(bool replay, const TString& query, const TQContext& qContext, const TRunSettings& runSettings) { - auto functionRegistry = NKikimr::NMiniKQL::CreateFunctionRegistry(NKikimr::NMiniKQL::CreateBuiltinRegistry()); - if (!replay) { - auto cloned = functionRegistry->Clone(); - NKikimr::NMiniKQL::FillStaticModules(*cloned); - functionRegistry = cloned; - } - - auto yqlNativeServices = NFile::TYtFileServices::Make(functionRegistry.Get(), runSettings.Tables, {}, ""); - auto ytGateway = CreateYtFileGateway(yqlNativeServices); - - TVector<TDataProviderInitializer> dataProvidersInit; - dataProvidersInit.push_back(GetYtNativeDataProviderInitializer(ytGateway, MakeSimpleCBOOptimizerFactory(), {})); - - TExprContext modulesCtx; - IModuleResolver::TPtr moduleResolver; - if (!GetYqlDefaultModuleResolver(modulesCtx, moduleResolver)) { - Cerr << "Errors loading default YQL libraries:" << Endl; - modulesCtx.IssueManager.GetIssues().PrintTo(Cerr); - return false; - } - TExprContext::TFreezeGuard freezeGuard(modulesCtx); - - TProgramFactory factory(!runSettings.ChangeTime, functionRegistry.Get(), 0ULL, dataProvidersInit, "ut"); - factory.SetUdfResolver(NCommon::CreateSimpleUdfResolver(functionRegistry.Get())); - factory.SetModules(moduleResolver); - - if (!replay && (!runSettings.StaticFiles.empty() || !runSettings.DynamicFiles.empty())) { - TFileStorageConfig fsConfig; - LoadFsConfigFromResource("fs.conf", fsConfig); - auto storage = WithAsync(CreateFileStorage(fsConfig)); - factory.SetFileStorage(storage); - } - - if (!replay && !runSettings.StaticFiles.empty()) { - factory.AddUserDataTable(MakeUserTables(runSettings.StaticFiles)); - } - - if (!replay && !runSettings.Credentials.empty()) { - auto credentials = MakeIntrusive<TCredentials>(); - for (const auto& x : runSettings.Credentials) { - credentials->AddCredential(x.first, x.second); - } - - factory.SetCredentials(credentials); - } - - TProgramPtr program = factory.Create("-stdin-", query, "", EHiddenMode::Disable, qContext); - if (!replay && runSettings.ParametersYson) { - program->SetParametersYson(*runSettings.ParametersYson); - } - - if (!replay && !runSettings.DynamicFiles.empty()) { - program->AddUserDataTable(MakeUserTables(runSettings.DynamicFiles)); - } - - if (runSettings.IsSql || runSettings.IsPg) { - NSQLTranslation::TTranslationSettings settings; - settings.PgParser = runSettings.IsPg; - if (!replay) { - settings.ClusterMapping["plato"] = TString(YtProviderName); - } - - if (!program->ParseSql(settings)) { - program->PrintErrorsTo(Cerr); - return false; - } - } else if (!program->ParseYql()) { - program->PrintErrorsTo(Cerr); - return false; - } - - if (!program->Compile(GetUsername())) { - program->PrintErrorsTo(Cerr); - return false; - } - - TProgram::TStatus status = replay ? - program->Optimize(GetUsername()) : - program->Run(GetUsername()); - if (status == TProgram::TStatus::Error) { - program->PrintErrorsTo(Cerr); - } - - return status == TProgram::TStatus::Ok; -} - -void CheckProgram(const TString& query, const TRunSettings& runSettings) { - auto qStorage = MakeMemoryQStorage(); - TQContext savingCtx(qStorage->MakeWriter("foo", {})); - UNIT_ASSERT(RunProgram(false, query, savingCtx, runSettings)); - savingCtx.GetWriter()->Commit().GetValueSync(); - TQContext loadingCtx(qStorage->MakeReader("foo", {})); - UNIT_ASSERT(RunProgram(true, "", loadingCtx, runSettings)); -} - -Y_UNIT_TEST_SUITE(QPlayerTests) { - Y_UNIT_TEST(SimpleYql) { - auto s = R"( -( -(let world (block '( - (let output (block '( - (let select (block '( - (let core (AsList (AsStruct))) - (let core (PersistableRepr (block '( - (let projectCoreType (TypeOf core)) - (let core (OrderedSqlProject core '((SqlProjectItem projectCoreType '"" (lambda '(row) (block '( - (let res (Int32 '"1")) - (return res) - ))) '('('autoName)))))) - (return core) - )))) - (return core) - ))) - (return select) - ))) - (let output (Unordered output)) - (let world (block '( - (let result_sink (DataSink 'result)) - (let world (Write! world result_sink (Key) output '('('type) '('autoref) '('columns '('('auto))) '('unordered)))) - (return (Commit! world result_sink)) - ))) - (return world) -))) -(let world (block '( - (let world (CommitAll! world)) - (return world) -))) -(return world) -) - )"; - - TRunSettings runSettings; - runSettings.IsSql = false; - CheckProgram(s, runSettings); - } - - Y_UNIT_TEST(SimpleSql) { - auto s = "select 1"; - TRunSettings runSettings; - CheckProgram(s, runSettings); - } - - Y_UNIT_TEST(SimplePg) { - auto s = "select 1::text"; - TRunSettings runSettings; - runSettings.IsPg = true; - CheckProgram(s, runSettings); - } - - Y_UNIT_TEST(Udf) { - auto s = "select String::AsciiToUpper('a')"; - TRunSettings runSettings; - CheckProgram(s, runSettings); - } - - Y_UNIT_TEST(YtGetFolder) { - auto s = "select * from plato.folder('','_yql_row_spec')"; - WithTables([&](const auto& tables) { - TRunSettings runSettings; - runSettings.Tables = tables; - CheckProgram(s, runSettings); - }); - } - - Y_UNIT_TEST(YtGetTableInfo) { - auto s = "select * from plato.Input"; - WithTables([&](const auto& tables) { - TRunSettings runSettings; - runSettings.Tables = tables; - CheckProgram(s, runSettings); - }); - } - - Y_UNIT_TEST(Parameters) { - auto s = "declare $x as String; select $x"; - TRunSettings runSettings; - runSettings.ParametersYson = NYT::NodeToYsonString(NYT::TNode() - ("$x", NYT::TNode()("Data", "value"))); - CheckProgram(s, runSettings); - } - - Y_UNIT_TEST(Files) { - auto s = "select FileContent('a'), FileContent('b')"; - TRunSettings runSettings; - runSettings.StaticFiles["a"] = "1"; - runSettings.DynamicFiles["b"] = "2"; - CheckProgram(s, runSettings); - } - - Y_UNIT_TEST(Libraries) { - auto s = "pragma library('a.sql'); import a symbols $f; select $f(1)"; - TRunSettings runSettings; - runSettings.StaticFiles["a.sql"] = "$f = ($x)->($x+1); export $f"; - CheckProgram(s, runSettings); - } - - Y_UNIT_TEST(Evaluation) { - auto s = "$c = select count(*) from plato.Input; select EvaluateExpr($c)"; - WithTables([&](const auto& tables) { - TRunSettings runSettings; - runSettings.Tables = tables; - CheckProgram(s, runSettings); - }); - } - - Y_UNIT_TEST(WalkFolders) { - auto s = R"( -$postHandler = ($nodes, $state, $level) -> { - $tables = ListFilter($nodes, ($x)->($x.Type = "table")); - return ListExtend($state, ListExtract($tables, "Path")); -}; - -SELECT State FROM plato.WalkFolders(``, $postHandler AS PostHandler); - )"; - - WithTables([&](const auto& tables) { - TRunSettings runSettings; - runSettings.Tables = tables; - CheckProgram(s, runSettings); - }); - } - - Y_UNIT_TEST(EvaluateNow) { - auto s = "select EvaluateExpr(CurrentUtcDate())"; - TRunSettings runSettings; - runSettings.ChangeTime = true; - CheckProgram(s, runSettings); - } - - Y_UNIT_TEST(Credentials) { - auto s = "select SecureParam('token:foo');"; - TRunSettings runSettings; - runSettings.Credentials.push_back(std::make_pair("foo",TCredential("C","S","V"))); - CheckProgram(s, runSettings); - } -} diff --git a/yql/essentials/public/udf/arrow/ut/ya.make b/yql/essentials/public/udf/arrow/ut/ya.make index 4515dbe26a4..da52d50ded5 100644 --- a/yql/essentials/public/udf/arrow/ut/ya.make +++ b/yql/essentials/public/udf/arrow/ut/ya.make @@ -6,7 +6,6 @@ SRCS( PEERDIR( yql/essentials/public/udf/arrow - yql/essentials/core/ut_common yql/essentials/public/udf/service/exception_policy yql/essentials/sql/pg_dummy yql/essentials/minikql |