diff options
| author | a-romanov <Anton.Romanov@ydb.tech> | 2023-08-08 19:20:14 +0300 |
|---|---|---|
| committer | a-romanov <Anton.Romanov@ydb.tech> | 2023-08-08 20:24:28 +0300 |
| commit | 3803a3805b0c8ec3c7cd373f9d5b27d17acb318b (patch) | |
| tree | 49c787d3b4e6dea681174d63f23870d41439d8d4 | |
| parent | c1852ff797428c5f31b5f69d74d96b79fa79e8a7 (diff) | |
| download | ydb-3803a3805b0c8ec3c7cd373f9d5b27d17acb318b.tar.gz | |
Move core ut to common place.
30 files changed, 6586 insertions, 32 deletions
diff --git a/ydb/library/yql/core/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/core/CMakeLists.darwin-x86_64.txt index 0b4b5ebc1d8..151c88bb9d9 100644 --- a/ydb/library/yql/core/CMakeLists.darwin-x86_64.txt +++ b/ydb/library/yql/core/CMakeLists.darwin-x86_64.txt @@ -24,6 +24,7 @@ add_subdirectory(url_lister) add_subdirectory(url_preprocessing) add_subdirectory(user_data) add_subdirectory(ut) +add_subdirectory(ut_common) get_built_tool_path( TOOL_enum_parser_bin TOOL_enum_parser_dependency diff --git a/ydb/library/yql/core/CMakeLists.linux-aarch64.txt b/ydb/library/yql/core/CMakeLists.linux-aarch64.txt index 66ef852bbe6..bcf8ef8a85c 100644 --- a/ydb/library/yql/core/CMakeLists.linux-aarch64.txt +++ b/ydb/library/yql/core/CMakeLists.linux-aarch64.txt @@ -24,6 +24,7 @@ add_subdirectory(url_lister) add_subdirectory(url_preprocessing) add_subdirectory(user_data) add_subdirectory(ut) +add_subdirectory(ut_common) get_built_tool_path( TOOL_enum_parser_bin TOOL_enum_parser_dependency diff --git a/ydb/library/yql/core/CMakeLists.linux-x86_64.txt b/ydb/library/yql/core/CMakeLists.linux-x86_64.txt index 66ef852bbe6..bcf8ef8a85c 100644 --- a/ydb/library/yql/core/CMakeLists.linux-x86_64.txt +++ b/ydb/library/yql/core/CMakeLists.linux-x86_64.txt @@ -24,6 +24,7 @@ add_subdirectory(url_lister) add_subdirectory(url_preprocessing) add_subdirectory(user_data) add_subdirectory(ut) +add_subdirectory(ut_common) get_built_tool_path( TOOL_enum_parser_bin TOOL_enum_parser_dependency diff --git a/ydb/library/yql/core/CMakeLists.windows-x86_64.txt b/ydb/library/yql/core/CMakeLists.windows-x86_64.txt index 0b4b5ebc1d8..151c88bb9d9 100644 --- a/ydb/library/yql/core/CMakeLists.windows-x86_64.txt +++ b/ydb/library/yql/core/CMakeLists.windows-x86_64.txt @@ -24,6 +24,7 @@ add_subdirectory(url_lister) add_subdirectory(url_preprocessing) add_subdirectory(user_data) add_subdirectory(ut) +add_subdirectory(ut_common) get_built_tool_path( TOOL_enum_parser_bin TOOL_enum_parser_dependency diff --git a/ydb/library/yql/core/ut/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/core/ut/CMakeLists.darwin-x86_64.txt index 4e6e7690bc1..8ec2ef772a4 100644 --- a/ydb/library/yql/core/ut/CMakeLists.darwin-x86_64.txt +++ b/ydb/library/yql/core/ut/CMakeLists.darwin-x86_64.txt @@ -20,11 +20,20 @@ target_link_libraries(ydb-library-yql-core-ut PUBLIC library-cpp-cpuid_check cpp-testing-unittest_main library-yql-core - udf-service-terminate_policy - yql-sql-pg_dummy + library-cpp-yson library-yql-ast - library-yql-sql - yql-sql-v1 + yql-core-facade + yql-core-services + yql-public-udf + udf-service-exception_policy + yql-core-type_ann + yql-core-ut_common + providers-common-provider + common-schema-parser + providers-result-provider + yt-gateway-file + providers-yt-provider + yql-sql-pg ) target_link_options(ydb-library-yql-core-ut PRIVATE -Wl,-platform_version,macos,11.0,11.0 @@ -34,7 +43,16 @@ target_link_options(ydb-library-yql-core-ut PRIVATE CoreFoundation ) target_sources(ydb-library-yql-core-ut PRIVATE - ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/yql_opt_utils_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/ut/yql_csv_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/ut/yql_execution_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/ut/yql_expr_constraint_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/ut/yql_expr_discover_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/ut/yql_expr_optimize_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/ut/yql_expr_providers_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/ut/yql_expr_type_annotation_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/ut/yql_library_compiler_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/ut/yql_opt_utils_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/ut/yql_udf_index_ut.cpp ) set_property( TARGET @@ -60,7 +78,7 @@ set_yunittest_property( ydb-library-yql-core-ut PROPERTY LABELS - SMALL + MEDIUM ) set_yunittest_property( TEST @@ -69,6 +87,13 @@ set_yunittest_property( PROCESSORS 1 ) +set_yunittest_property( + TEST + ydb-library-yql-core-ut + PROPERTY + TIMEOUT + 600 +) target_allocator(ydb-library-yql-core-ut system_allocator ) diff --git a/ydb/library/yql/core/ut/CMakeLists.linux-aarch64.txt b/ydb/library/yql/core/ut/CMakeLists.linux-aarch64.txt index 00c63669afb..3e71688c69b 100644 --- a/ydb/library/yql/core/ut/CMakeLists.linux-aarch64.txt +++ b/ydb/library/yql/core/ut/CMakeLists.linux-aarch64.txt @@ -20,11 +20,20 @@ target_link_libraries(ydb-library-yql-core-ut PUBLIC yutil cpp-testing-unittest_main library-yql-core - udf-service-terminate_policy - yql-sql-pg_dummy + library-cpp-yson library-yql-ast - library-yql-sql - yql-sql-v1 + yql-core-facade + yql-core-services + yql-public-udf + udf-service-exception_policy + yql-core-type_ann + yql-core-ut_common + providers-common-provider + common-schema-parser + providers-result-provider + yt-gateway-file + providers-yt-provider + yql-sql-pg ) target_link_options(ydb-library-yql-core-ut PRIVATE -ldl @@ -35,9 +44,19 @@ target_link_options(ydb-library-yql-core-ut PRIVATE -lpthread -lrt -ldl + -lutil ) target_sources(ydb-library-yql-core-ut PRIVATE - ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/yql_opt_utils_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/ut/yql_csv_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/ut/yql_execution_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/ut/yql_expr_constraint_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/ut/yql_expr_discover_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/ut/yql_expr_optimize_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/ut/yql_expr_providers_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/ut/yql_expr_type_annotation_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/ut/yql_library_compiler_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/ut/yql_opt_utils_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/ut/yql_udf_index_ut.cpp ) set_property( TARGET @@ -63,7 +82,7 @@ set_yunittest_property( ydb-library-yql-core-ut PROPERTY LABELS - SMALL + MEDIUM ) set_yunittest_property( TEST @@ -72,6 +91,13 @@ set_yunittest_property( PROCESSORS 1 ) +set_yunittest_property( + TEST + ydb-library-yql-core-ut + PROPERTY + TIMEOUT + 600 +) target_allocator(ydb-library-yql-core-ut cpp-malloc-jemalloc ) diff --git a/ydb/library/yql/core/ut/CMakeLists.linux-x86_64.txt b/ydb/library/yql/core/ut/CMakeLists.linux-x86_64.txt index 2c2bd9c84d9..70924389dda 100644 --- a/ydb/library/yql/core/ut/CMakeLists.linux-x86_64.txt +++ b/ydb/library/yql/core/ut/CMakeLists.linux-x86_64.txt @@ -21,11 +21,20 @@ target_link_libraries(ydb-library-yql-core-ut PUBLIC library-cpp-cpuid_check cpp-testing-unittest_main library-yql-core - udf-service-terminate_policy - yql-sql-pg_dummy + library-cpp-yson library-yql-ast - library-yql-sql - yql-sql-v1 + yql-core-facade + yql-core-services + yql-public-udf + udf-service-exception_policy + yql-core-type_ann + yql-core-ut_common + providers-common-provider + common-schema-parser + providers-result-provider + yt-gateway-file + providers-yt-provider + yql-sql-pg ) target_link_options(ydb-library-yql-core-ut PRIVATE -ldl @@ -36,9 +45,19 @@ target_link_options(ydb-library-yql-core-ut PRIVATE -lpthread -lrt -ldl + -lutil ) target_sources(ydb-library-yql-core-ut PRIVATE - ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/yql_opt_utils_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/ut/yql_csv_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/ut/yql_execution_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/ut/yql_expr_constraint_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/ut/yql_expr_discover_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/ut/yql_expr_optimize_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/ut/yql_expr_providers_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/ut/yql_expr_type_annotation_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/ut/yql_library_compiler_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/ut/yql_opt_utils_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/ut/yql_udf_index_ut.cpp ) set_property( TARGET @@ -64,7 +83,7 @@ set_yunittest_property( ydb-library-yql-core-ut PROPERTY LABELS - SMALL + MEDIUM ) set_yunittest_property( TEST @@ -73,6 +92,13 @@ set_yunittest_property( PROCESSORS 1 ) +set_yunittest_property( + TEST + ydb-library-yql-core-ut + PROPERTY + TIMEOUT + 600 +) target_allocator(ydb-library-yql-core-ut cpp-malloc-tcmalloc libs-tcmalloc-no_percpu_cache diff --git a/ydb/library/yql/core/ut/CMakeLists.windows-x86_64.txt b/ydb/library/yql/core/ut/CMakeLists.windows-x86_64.txt index 7584c1ba257..193dc53ccd5 100644 --- a/ydb/library/yql/core/ut/CMakeLists.windows-x86_64.txt +++ b/ydb/library/yql/core/ut/CMakeLists.windows-x86_64.txt @@ -20,14 +20,32 @@ target_link_libraries(ydb-library-yql-core-ut PUBLIC library-cpp-cpuid_check cpp-testing-unittest_main library-yql-core - udf-service-terminate_policy - yql-sql-pg_dummy + library-cpp-yson library-yql-ast - library-yql-sql - yql-sql-v1 + yql-core-facade + yql-core-services + yql-public-udf + udf-service-exception_policy + yql-core-type_ann + yql-core-ut_common + providers-common-provider + common-schema-parser + providers-result-provider + yt-gateway-file + providers-yt-provider + yql-sql-pg ) target_sources(ydb-library-yql-core-ut PRIVATE - ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/yql_opt_utils_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/ut/yql_csv_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/ut/yql_execution_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/ut/yql_expr_constraint_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/ut/yql_expr_discover_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/ut/yql_expr_optimize_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/ut/yql_expr_providers_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/ut/yql_expr_type_annotation_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/ut/yql_library_compiler_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/ut/yql_opt_utils_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/ut/yql_udf_index_ut.cpp ) set_property( TARGET @@ -53,7 +71,7 @@ set_yunittest_property( ydb-library-yql-core-ut PROPERTY LABELS - SMALL + MEDIUM ) set_yunittest_property( TEST @@ -62,6 +80,13 @@ set_yunittest_property( PROCESSORS 1 ) +set_yunittest_property( + TEST + ydb-library-yql-core-ut + PROPERTY + TIMEOUT + 600 +) target_allocator(ydb-library-yql-core-ut system_allocator ) diff --git a/ydb/library/yql/core/ut/ya.make b/ydb/library/yql/core/ut/ya.make index c146ed5148e..9103e4d88fd 100644 --- a/ydb/library/yql/core/ut/ya.make +++ b/ydb/library/yql/core/ut/ya.make @@ -1,20 +1,45 @@ UNITTEST_FOR(ydb/library/yql/core) -SIZE(SMALL) - SRCS( - ../yql_opt_utils_ut.cpp + yql_csv_ut.cpp + yql_execution_ut.cpp + yql_expr_constraint_ut.cpp + yql_expr_discover_ut.cpp + yql_expr_optimize_ut.cpp + yql_expr_providers_ut.cpp + yql_expr_type_annotation_ut.cpp + yql_library_compiler_ut.cpp + yql_opt_utils_ut.cpp + yql_udf_index_ut.cpp ) PEERDIR( - ydb/library/yql/core - ydb/library/yql/public/udf/service/terminate_policy - ydb/library/yql/sql/pg_dummy + library/cpp/yson ydb/library/yql/ast - ydb/library/yql/sql - ydb/library/yql/sql/v1 + ydb/library/yql/core + ydb/library/yql/core/facade + ydb/library/yql/core/services + ydb/library/yql/public/udf + ydb/library/yql/public/udf/service/exception_policy + ydb/library/yql/core/type_ann + ydb/library/yql/core/ut_common + ydb/library/yql/providers/common/provider + ydb/library/yql/providers/common/schema/parser + ydb/library/yql/providers/result/provider + ydb/library/yql/providers/yt/gateway/file + ydb/library/yql/providers/yt/provider + ydb/library/yql/sql/pg ) +IF (SANITIZER_TYPE OR WITH_VALGRIND) + TIMEOUT(1800) + SIZE(LARGE) + TAG(ya:fat) +ELSE() + TIMEOUT(600) + SIZE(MEDIUM) +ENDIF() + YQL_LAST_ABI_VERSION() END() diff --git a/ydb/library/yql/core/ut/yql_csv_ut.cpp b/ydb/library/yql/core/ut/yql_csv_ut.cpp new file mode 100644 index 00000000000..2ae83f90966 --- /dev/null +++ b/ydb/library/yql/core/ut/yql_csv_ut.cpp @@ -0,0 +1,312 @@ +#include "yql_csv.h" +#include <ydb/library/yql/core/ut_common/yql_ut_common.h> + +#include <library/cpp/testing/unittest/registar.h> + +#include <util/stream/mem.h> + +using namespace NYql; +using namespace NUtils; + +TVector<TString> ReadOneLineCsv(const TString& csv) +{ + TMemoryInput memIn(csv); + TCsvInputStream in(memIn); + return in.ReadLine(); +} + +TVector<TString> ReadOneLineCsvData(const TStringBuf& csvdata) +{ + TCsvInputBuffer in(csvdata); + return in.ReadLine(); +} + + +template<typename T> +TString WriteCsv(T&& functor) { + TStringStream ss; + TCsvOutputStream out(ss); + functor(out); + return ss.Str(); +} + +Y_UNIT_TEST_SUITE(TYqlUtils) { + + Y_UNIT_TEST(CsvReadTest) { + { + TVector<TString> columns = ReadOneLineCsv(""); + UNIT_ASSERT_EQUAL(columns.size(), 0); + } + + { + TVector<TString> columns = ReadOneLineCsv("\"\""); + UNIT_ASSERT_EQUAL(columns.size(), 1); + UNIT_ASSERT_STRINGS_EQUAL(columns[0], ""); + } + + { + TVector<TString> columns = ReadOneLineCsv("one"); + UNIT_ASSERT_EQUAL(columns.size(), 1); + UNIT_ASSERT_STRINGS_EQUAL(columns[0], "one"); + } + + { + TVector<TString> columns = ReadOneLineCsv("\"one\""); + UNIT_ASSERT_EQUAL(columns.size(), 1); + UNIT_ASSERT_STRINGS_EQUAL(columns[0], "one"); + } + + { + TVector<TString> columns = ReadOneLineCsv("one;two;three"); + UNIT_ASSERT_EQUAL(columns.size(), 3); + UNIT_ASSERT_STRINGS_EQUAL(columns[0], "one"); + UNIT_ASSERT_STRINGS_EQUAL(columns[1], "two"); + UNIT_ASSERT_STRINGS_EQUAL(columns[2], "three"); + } + + { + TVector<TString> columns = ReadOneLineCsv("\"one\";\"two\";\"three\""); + UNIT_ASSERT_EQUAL(columns.size(), 3); + UNIT_ASSERT_STRINGS_EQUAL(columns[0], "one"); + UNIT_ASSERT_STRINGS_EQUAL(columns[1], "two"); + UNIT_ASSERT_STRINGS_EQUAL(columns[2], "three"); + } + + { + TVector<TString> columns = ReadOneLineCsv("\"one\";two;three"); + UNIT_ASSERT_EQUAL(columns.size(), 3); + UNIT_ASSERT_STRINGS_EQUAL(columns[0], "one"); + UNIT_ASSERT_STRINGS_EQUAL(columns[1], "two"); + UNIT_ASSERT_STRINGS_EQUAL(columns[2], "three"); + } + + { + TVector<TString> columns = ReadOneLineCsv("one;\"two\";three"); + UNIT_ASSERT_EQUAL(columns.size(), 3); + UNIT_ASSERT_STRINGS_EQUAL(columns[0], "one"); + UNIT_ASSERT_STRINGS_EQUAL(columns[1], "two"); + UNIT_ASSERT_STRINGS_EQUAL(columns[2], "three"); + } + + { + TVector<TString> columns = ReadOneLineCsv("one;two;\"three\""); + UNIT_ASSERT_EQUAL(columns.size(), 3); + UNIT_ASSERT_STRINGS_EQUAL(columns[0], "one"); + UNIT_ASSERT_STRINGS_EQUAL(columns[1], "two"); + UNIT_ASSERT_STRINGS_EQUAL(columns[2], "three"); + } + + { + TVector<TString> columns = ReadOneLineCsv(";two;three"); + UNIT_ASSERT_EQUAL(columns.size(), 3); + UNIT_ASSERT_STRINGS_EQUAL(columns[0], ""); + UNIT_ASSERT_STRINGS_EQUAL(columns[1], "two"); + UNIT_ASSERT_STRINGS_EQUAL(columns[2], "three"); + } + + { + TVector<TString> columns = ReadOneLineCsv("one;two;"); + UNIT_ASSERT_EQUAL(columns.size(), 3); + UNIT_ASSERT_STRINGS_EQUAL(columns[0], "one"); + UNIT_ASSERT_STRINGS_EQUAL(columns[1], "two"); + UNIT_ASSERT_STRINGS_EQUAL(columns[2], ""); + } + + { + TVector<TString> columns = ReadOneLineCsv(";"); + UNIT_ASSERT_EQUAL(columns.size(), 2); + UNIT_ASSERT_STRINGS_EQUAL(columns[0], ""); + UNIT_ASSERT_STRINGS_EQUAL(columns[1], ""); + } + + { + TVector<TString> columns = ReadOneLineCsv("\"\";"); + UNIT_ASSERT_EQUAL(columns.size(), 2); + UNIT_ASSERT_STRINGS_EQUAL(columns[0], ""); + UNIT_ASSERT_STRINGS_EQUAL(columns[1], ""); + } + + { + TVector<TString> columns = ReadOneLineCsv(";\"\""); + UNIT_ASSERT_EQUAL(columns.size(), 2); + UNIT_ASSERT_STRINGS_EQUAL(columns[0], ""); + UNIT_ASSERT_STRINGS_EQUAL(columns[1], ""); + } + + { + TVector<TString> columns = ReadOneLineCsv("\"ab;cd\""); + UNIT_ASSERT_EQUAL(columns.size(), 1); + UNIT_ASSERT_STRINGS_EQUAL(columns[0], "ab;cd"); + } + + { + TVector<TString> columns = ReadOneLineCsv( + "\\\";" + "\"\\\"\";" + "\\\"\\\";" + "\"\\\"\\\"\";" + "\\\"\\\"\\\"\\\";" + "\"\\\"\\\"\\\"\\\"\";" + "\\\\\";" + "\"\\\\\"\";" + "\\\\\"\\\\\";" + "\"\\\\\"\\\\\"\";" + "\\\\\"\\\\\"\\\\\"\\\\\";" + "\"\\\\\"\\\\\"\\\\\"\\\\\"\""); + UNIT_ASSERT_EQUAL(columns.size(), 12); + UNIT_ASSERT_STRINGS_EQUAL(columns[0], "\""); + UNIT_ASSERT_STRINGS_EQUAL(columns[1], "\""); + UNIT_ASSERT_STRINGS_EQUAL(columns[2], "\"\""); + UNIT_ASSERT_STRINGS_EQUAL(columns[3], "\"\""); + UNIT_ASSERT_STRINGS_EQUAL(columns[4], "\"\"\"\""); + UNIT_ASSERT_STRINGS_EQUAL(columns[5], "\"\"\"\""); + UNIT_ASSERT_STRINGS_EQUAL(columns[6], "\\\""); + UNIT_ASSERT_STRINGS_EQUAL(columns[7], "\\\""); + UNIT_ASSERT_STRINGS_EQUAL(columns[8], "\\\"\\\""); + UNIT_ASSERT_STRINGS_EQUAL(columns[9], "\\\"\\\""); + UNIT_ASSERT_STRINGS_EQUAL(columns[10], "\\\"\\\"\\\"\\\""); + UNIT_ASSERT_STRINGS_EQUAL(columns[11], "\\\"\\\"\\\"\\\""); + } + } + + Y_UNIT_TEST(CsvReadFromBufferTest) { + { + TVector<TString> columns = ReadOneLineCsvData(""); + UNIT_ASSERT_EQUAL(columns.size(), 0); + } + + { + TVector<TString> columns = ReadOneLineCsvData("\"\""); + UNIT_ASSERT_EQUAL(columns.size(), 1); + UNIT_ASSERT_STRINGS_EQUAL(columns[0], ""); + } + + { + TVector<TString> columns = ReadOneLineCsvData("one"); + UNIT_ASSERT_EQUAL(columns.size(), 1); + UNIT_ASSERT_STRINGS_EQUAL(columns[0], "one"); + } + + { + TVector<TString> columns = ReadOneLineCsvData("\"one\""); + UNIT_ASSERT_EQUAL(columns.size(), 1); + UNIT_ASSERT_STRINGS_EQUAL(columns[0], "one"); + } + + { + TVector<TString> columns = ReadOneLineCsvData("one;two;three"); + UNIT_ASSERT_EQUAL(columns.size(), 3); + UNIT_ASSERT_STRINGS_EQUAL(columns[0], "one"); + UNIT_ASSERT_STRINGS_EQUAL(columns[1], "two"); + UNIT_ASSERT_STRINGS_EQUAL(columns[2], "three"); + } + + { + TVector<TString> columns = ReadOneLineCsvData("\"one\";\"two\";\"three\""); + UNIT_ASSERT_EQUAL(columns.size(), 3); + UNIT_ASSERT_STRINGS_EQUAL(columns[0], "one"); + UNIT_ASSERT_STRINGS_EQUAL(columns[1], "two"); + UNIT_ASSERT_STRINGS_EQUAL(columns[2], "three"); + } + + { + TVector<TString> columns = ReadOneLineCsvData("\"one\";two;three"); + UNIT_ASSERT_EQUAL(columns.size(), 3); + UNIT_ASSERT_STRINGS_EQUAL(columns[0], "one"); + UNIT_ASSERT_STRINGS_EQUAL(columns[1], "two"); + UNIT_ASSERT_STRINGS_EQUAL(columns[2], "three"); + } + + { + TVector<TString> columns = ReadOneLineCsvData("one;\"two\";three"); + UNIT_ASSERT_EQUAL(columns.size(), 3); + UNIT_ASSERT_STRINGS_EQUAL(columns[0], "one"); + UNIT_ASSERT_STRINGS_EQUAL(columns[1], "two"); + UNIT_ASSERT_STRINGS_EQUAL(columns[2], "three"); + } + + { + TVector<TString> columns = ReadOneLineCsvData("one;two;\"three\""); + UNIT_ASSERT_EQUAL(columns.size(), 3); + UNIT_ASSERT_STRINGS_EQUAL(columns[0], "one"); + UNIT_ASSERT_STRINGS_EQUAL(columns[1], "two"); + UNIT_ASSERT_STRINGS_EQUAL(columns[2], "three"); + } + + { + TVector<TString> columns = ReadOneLineCsvData(";two;three"); + UNIT_ASSERT_EQUAL(columns.size(), 3); + UNIT_ASSERT_STRINGS_EQUAL(columns[0], ""); + UNIT_ASSERT_STRINGS_EQUAL(columns[1], "two"); + UNIT_ASSERT_STRINGS_EQUAL(columns[2], "three"); + } + + { + TVector<TString> columns = ReadOneLineCsvData("one;two;"); + UNIT_ASSERT_EQUAL(columns.size(), 3); + UNIT_ASSERT_STRINGS_EQUAL(columns[0], "one"); + UNIT_ASSERT_STRINGS_EQUAL(columns[1], "two"); + UNIT_ASSERT_STRINGS_EQUAL(columns[2], ""); + } + + { + TVector<TString> columns = ReadOneLineCsvData(";"); + UNIT_ASSERT_EQUAL(columns.size(), 2); + UNIT_ASSERT_STRINGS_EQUAL(columns[0], ""); + UNIT_ASSERT_STRINGS_EQUAL(columns[1], ""); + } + + { + TVector<TString> columns = ReadOneLineCsvData("\"\";"); + UNIT_ASSERT_EQUAL(columns.size(), 2); + UNIT_ASSERT_STRINGS_EQUAL(columns[0], ""); + UNIT_ASSERT_STRINGS_EQUAL(columns[1], ""); + } + + { + TVector<TString> columns = ReadOneLineCsvData(";\"\""); + UNIT_ASSERT_EQUAL(columns.size(), 2); + UNIT_ASSERT_STRINGS_EQUAL(columns[0], ""); + UNIT_ASSERT_STRINGS_EQUAL(columns[1], ""); + } + + { + TVector<TString> columns = ReadOneLineCsvData("\"ab;cd\""); + UNIT_ASSERT_EQUAL(columns.size(), 1); + UNIT_ASSERT_STRINGS_EQUAL(columns[0], "ab;cd"); + } + } + + Y_UNIT_TEST(CsvWriteTest) { + { + TString str = WriteCsv([](TCsvOutputStream& out) { + out << ""; + }); + UNIT_ASSERT_STRINGS_EQUAL(str, "\"\""); + } + + { + TString str = WriteCsv([](TCsvOutputStream& out) { + out << "one"; + }); + UNIT_ASSERT_STRINGS_EQUAL(str, "\"one\""); + } + + { + TString str = WriteCsv([](TCsvOutputStream& out) { + out << "one" << "two"; + }); + UNIT_ASSERT_STRINGS_EQUAL(str, "\"one\";\"two\""); + } + + { + TString str = WriteCsv([](TCsvOutputStream& out) { + out << "one" << "two" << Endl + << 1 << 2; + }); + UNIT_ASSERT_STRINGS_EQUAL(str, "\"one\";\"two\"\n" + "\"1\";\"2\""); + } + } +} + diff --git a/ydb/library/yql/core/ut/yql_execution_ut.cpp b/ydb/library/yql/core/ut/yql_execution_ut.cpp new file mode 100644 index 00000000000..3c405952b07 --- /dev/null +++ b/ydb/library/yql/core/ut/yql_execution_ut.cpp @@ -0,0 +1,792 @@ +#include "yql_execution.h" + +#include <ydb/library/yql/core/ut_common/yql_ut_common.h> + +#include <ydb/library/yql/ast/yql_ast_annotation.h> +#include <ydb/library/yql/ast/yql_expr.h> +#include <ydb/library/yql/core/type_ann/type_ann_core.h> +#include <ydb/library/yql/core/yql_expr_optimize.h> +#include <ydb/library/yql/core/yql_expr_type_annotation.h> +#include <ydb/library/yql/core/yql_opt_proposed_by_data.h> +#include <ydb/library/yql/core/yql_opt_rewrite_io.h> +#include <ydb/library/yql/providers/common/provider/yql_provider_names.h> +#include <ydb/library/yql/providers/common/schema/parser/yql_type_parser.h> +#include <ydb/library/yql/providers/result/provider/yql_result_provider.h> + +#include <ydb/library/yql/core/facade/yql_facade.h> +#include <ydb/library/yql/providers/yt/provider/yql_yt_provider.h> +#include <ydb/library/yql/providers/yt/gateway/file/yql_yt_file.h> +#include <ydb/library/yql/providers/yt/gateway/file/yql_yt_file_services.h> +#include <ydb/library/yql/minikql/invoke_builtins/mkql_builtins.h> + +#include <library/cpp/testing/unittest/registar.h> + +#include <util/system/user.h> +#include <util/system/tempfile.h> +#include <util/system/defaults.h> +#include <util/system/fstat.h> +#include <util/folder/path.h> +#include <util/folder/tempdir.h> +#include <util/string/cast.h> +#include <util/string/builder.h> +#include <util/system/sanitizers.h> + +namespace NYql { + + static TString BuildFileNameForTmpTable(TStringBuf table, TStringBuf tmpDir) { + return TStringBuilder() << tmpDir << LOCSLASH_C << table.substr(4) << ".tmp"; + } + + struct TRunSingleProgram { + TString Src; + TString TmpDir; + TString Parameters; + IOutputStream& Err; + TVector<TString> Res; + THashMap<TString, TString> Tables; + + TRunSingleProgram(const TString& src, IOutputStream& err) + : Src(src) + , Err(err) + { + } + + bool Run( + const NKikimr::NMiniKQL::IFunctionRegistry* funcReg + ) { + auto yqlNativeServices = NFile::TYtFileServices::Make(funcReg, Tables, {}, TmpDir); + auto ytGateway = CreateYtFileGateway(yqlNativeServices); + + TVector<TDataProviderInitializer> dataProvidersInit; + dataProvidersInit.push_back(GetYtNativeDataProviderInitializer(ytGateway)); + TProgramFactory factory(true, funcReg, 0ULL, dataProvidersInit, "ut"); + + TProgramPtr program = factory.Create("-stdin-", Src); + program->ConfigureYsonResultFormat(NYson::EYsonFormat::Text); + if (!Parameters.empty()) { + program->SetParametersYson(Parameters); + } + + if (!program->ParseYql() || !program->Compile(GetUsername())) { + program->PrintErrorsTo(Err); + return false; + } + + TProgram::TStatus status = program->Run(GetUsername()); + if (status == TProgram::TStatus::Error) { + program->PrintErrorsTo(Err); + } + Res = program->Results(); + return status == TProgram::TStatus::Ok; + } + + void AddResults(TVector<TString>& res) const { + res.insert(res.end(), Res.begin(), Res.end()); + } + + bool Finished() const { + return true; + } + }; + + struct TRunMultiplePrograms: public TRunSingleProgram { + TVector<TString> Srcs; + size_t Curr; + + TRunMultiplePrograms(const TVector<TString>& srcs, IOutputStream& err) + : TRunSingleProgram(TString(), err) + , Srcs(srcs) + , Curr(0) + { + } + + bool Run( + const NKikimr::NMiniKQL::IFunctionRegistry* funcReg + ) { + TString origTmpDir = TmpDir; + if (TmpDir) { + TFsPath newTmp = TFsPath(TmpDir) / ToString(Curr); + newTmp.MkDirs(); + TmpDir = newTmp.GetPath(); + } + Src = Srcs[Curr]; + if (!TRunSingleProgram::Run(funcReg)) { + return false; + } + ui32 idx = 0; + for (auto& resStr: Res) { + NYT::TNode res; + if (!NCommon::ParseYson(res, resStr, Err)) { + return false; + } + if (!res.IsMap() || !res.HasKey("Write") || !res["Write"].IsList()) { + Err << "Invalid result: " << resStr << Endl; + return false; + } + for (auto& elem: res["Write"].AsList()) { + if (!elem.IsMap()) { + Err << "Invalid result element in result: " << resStr << Endl; + return false; + } + if (elem.HasKey("Ref")) { + if (!elem["Ref"].IsList()) { + Err << "Invalid reference in result: " << resStr << Endl; + return false; + } + for (auto& refElem: elem["Ref"].AsList()) { + if (!refElem.IsMap() || !refElem.HasKey("Reference")) { + Err << "Invalid reference in result: " << resStr << Endl; + return false; + } + if (!refElem["Remove"].AsBool()) { + continue; + } + const auto& ref = refElem["Reference"].AsList(); + TStringStream name; + name << ref[0].AsString() << "." << ref[1].AsString() << ".Result" << Curr << "_" << idx; + Tables[name.Str()] = BuildFileNameForTmpTable(ref[2].AsString(), TmpDir); + ++idx; + } + } + } + } + ++Curr; + origTmpDir.swap(TmpDir); + return true; + } + + bool Finished() const { + return Curr == Srcs.size(); + } + }; + + template <typename TDriver> + TVector<TString> Run(TDriver& driver) { + auto functionRegistry = NKikimr::NMiniKQL::CreateFunctionRegistry(NKikimr::NMiniKQL::CreateBuiltinRegistry()); + + TVector<TString> res; + do { + const bool runRes = driver.Run(functionRegistry.Get()); + UNIT_ASSERT(runRes); + + driver.AddResults(res); + } while (!driver.Finished()); + return res; + } + + TVector<TString> RunProgram(const TString& programSrc, const THashMap<TString, TString>& tables, const TString& tmpDir = TString(), const TString& params = TString()) { + TRunSingleProgram driver(programSrc, Cerr); + driver.Tables = tables; + driver.TmpDir = tmpDir; + driver.Parameters = params; + return Run(driver); + } + + static const TStringBuf KSV_ATTRS = + "{\"_yql_row_spec\" = {\"Type\" = [\"StructType\";[" + "[\"key\";[\"DataType\";\"String\"]];" + "[\"subkey\";[\"DataType\";\"String\"]];" + "[\"value\";[\"DataType\";\"String\"]]" + "]]}}" + ; + + Y_UNIT_TEST_SUITE(ExecutionYqlExpr) { + Y_UNIT_TEST(WriteToResultUsingIsolatedGraph) { + auto s = "(\n" + "(let res_sink (DataSink 'result))\n" + "(let data (AsList (String 'x)))\n" + "(let world (Write! world res_sink (Key) data '()))\n" + "(let world (Commit! world res_sink))\n" + "(return world)\n" + ")\n"; + + auto res = RunProgram(s, THashMap<TString, TString>()); + UNIT_ASSERT_VALUES_EQUAL(res.size(), 1); + UNIT_ASSERT_NO_DIFF("{\"Write\"=[{\"Data\"=[\"x\"]}]}", res[0]); + } + + Y_UNIT_TEST(WriteToResultTableOutput) { + TTempFileHandle inputFile; + TTempFileHandle inputFileAttrs(inputFile.Name() + ".attr"); + + TStringBuf data = + "{\"key\"=\"075\";\"subkey\"=\".\";\"value\"=\"abc\"};\n" + "{\"key\"=\"800\";\"subkey\"=\".\";\"value\"=\"ddd\"};\n" + "{\"key\"=\"020\";\"subkey\"=\".\";\"value\"=\"q\"};\n" + "{\"key\"=\"150\";\"subkey\"=\".\";\"value\"=\"qzz\"};\n"sv + ; + + inputFile.Write(data.data(), data.size()); + inputFile.FlushData(); + inputFileAttrs.Write(KSV_ATTRS.data(), KSV_ATTRS.size()); + inputFileAttrs.FlushData(); + + THashMap<TString, TString> tables; + tables["yt.plato.Input"] = inputFile.Name(); + + auto s = "(\n" + "(let mr_source (DataSource 'yt 'plato))\n" + "(let x (Read! world mr_source (Key '('table (String 'Input))) '('key 'subkey 'value) '()))\n" + "(let world (Left! x))\n" + "(let table1 (Right! x))\n" + "(let res_sink (DataSink 'result))\n" + "(let data (AsList (String 'x)))\n" + "(let world (Write! world res_sink (Key) table1 '()))\n" + "(let world (Commit! world res_sink))\n" + "(return world)\n" + ")\n"; + + auto res = RunProgram(s, tables); + UNIT_ASSERT_VALUES_EQUAL(res.size(), 1); + UNIT_ASSERT_NO_DIFF( + "{\"Write\"=[{\"Data\"=[" + "[\"075\";\".\";\"abc\"];" + "[\"800\";\".\";\"ddd\"];" + "[\"020\";\".\";\"q\"];" + "[\"150\";\".\";\"qzz\"]" + "]}]}", + res[0] + ); + } + + Y_UNIT_TEST(WriteToResultTransformedTable) { + TTempFileHandle inputFile; + TTempFileHandle inputFileAttrs(inputFile.Name() + ".attr"); + + TStringBuf data = + "{\"key\"=\"075\";\"subkey\"=\".\";\"value\"=\"abc\"};\n" + "{\"key\"=\"800\";\"subkey\"=\".\";\"value\"=\"ddd\"};\n" + "{\"key\"=\"020\";\"subkey\"=\".\";\"value\"=\"q\"};\n" + "{\"key\"=\"150\";\"subkey\"=\".\";\"value\"=\"qzz\"};\n"sv + ; + + inputFile.Write(data.data(), data.size()); + inputFile.FlushData(); + inputFileAttrs.Write(KSV_ATTRS.data(), KSV_ATTRS.size()); + inputFileAttrs.FlushData(); + + THashMap<TString, TString> tables; + tables["yt.plato.Input"] = inputFile.Name(); + + auto s = "(\n" + "(let mr_source (DataSource 'yt 'plato))\n" + "(let x (Read! world mr_source (Key '('table (String 'Input))) '('key 'subkey 'value) '()))\n" + "(let world (Left! x))\n" + "(let table1 (Right! x))\n" + "(let table1low (FlatMap table1 (lambda '(item) (block '(\n" + " (let intValueOpt (FromString (Member item 'key) 'Int32))\n" + " (let ret (FlatMap intValueOpt (lambda '(item2) (block '(\n" + " (return (ListIf (< item2 (Int32 '100)) item))\n" + " )))))" + " (return ret)" + ")))))" + "(let res_sink (DataSink 'result))\n" + "(let data (AsList (String 'x)))\n" + "(let world (Write! world res_sink (Key) table1low '()))\n" + "(let world (Commit! world res_sink))\n" + "(return world)\n" + ")\n"; + + auto res = RunProgram(s, tables); + UNIT_ASSERT_VALUES_EQUAL(res.size(), 1); + UNIT_ASSERT_NO_DIFF( + "{\"Write\"=[{\"Data\"=[" + "[\"075\";\".\";\"abc\"];" + "[\"020\";\".\";\"q\"]" + "]}]}", + res[0] + ); + } + + Y_UNIT_TEST(DropTable) { + TTempFileHandle outputFile; + TTempFileHandle outputFileAttrs(outputFile.Name() + ".attr"); + + TStringBuf data = + "{\"key\"=\"075\";\"subkey\"=\".\";\"value\"=\"abc\"};\n"sv + ; + + outputFile.Write(data.data(), data.size()); + outputFile.FlushData(); + outputFile.Close(); + outputFileAttrs.Write(KSV_ATTRS.data(), KSV_ATTRS.size()); + outputFileAttrs.FlushData(); + outputFileAttrs.Close(); + UNIT_ASSERT(TFileStat(outputFile.Name()).IsFile()); + + THashMap<TString, TString> tables; + tables["yt.plato.Output"] = outputFile.Name(); + + auto s = "(\n" + "(let mr_sink (DataSink 'yt 'plato))\n" + "(let world (Write! world mr_sink (Key '('table (String 'Output))) (Void) '('('mode 'drop))))\n" + "(let world (Commit! world mr_sink))\n" + "(return world)\n" + ")\n"; + + RunProgram(s, tables); + + UNIT_ASSERT(!TFileStat(outputFile.Name()).IsFile()); + } + + Y_UNIT_TEST(WriteToResultTableByRef) { + TTempFileHandle inputFile; + TTempFileHandle inputFileAttrs(inputFile.Name() + ".attr"); + + TStringBuf data = + "{\"key\"=\"075\";\"subkey\"=\".\";\"value\"=\"abc\"};\n" + "{\"key\"=\"800\";\"subkey\"=\".\";\"value\"=\"ddd\"};\n" + "{\"key\"=\"020\";\"subkey\"=\".\";\"value\"=\"q\"};\n" + "{\"key\"=\"150\";\"subkey\"=\".\";\"value\"=\"qzz\"};\n"sv + ; + + inputFile.Write(data.data(), data.size()); + inputFile.FlushData(); + inputFileAttrs.Write(KSV_ATTRS.data(), KSV_ATTRS.size()); + inputFileAttrs.FlushData(); + + THashMap<TString, TString> tables; + tables["yt.plato.Input"] = inputFile.Name(); + + auto s = "(\n" + "(let mr_source (DataSource 'yt 'plato))\n" + "(let x (Read! world mr_source (Key '('table (String 'Input))) '('key 'subkey 'value) '()))\n" + "(let world (Left! x))\n" + "(let table1 (Right! x))\n" + "(let res_sink (DataSink 'result))\n" + "(let mr_sink (DataSink 'yt 'plato))\n" + "(let world (Write! world res_sink (Key) table1 '('('ref))))\n" + "(let world (Commit! world mr_sink))\n" + "(let world (Commit! world res_sink))\n" + "(return world)\n" + ")\n"; + + auto res = RunProgram(s, tables); + UNIT_ASSERT_VALUES_EQUAL(res.size(), 1); + UNIT_ASSERT_NO_DIFF( + "{\"Write\"=[{\"Ref\"=[" + "{\"Reference\"=[\"yt\";\"plato\";\"Input\"];\"Columns\"=[\"key\";\"subkey\";\"value\"];\"Remove\"=%false}" + "]}]}", + res[0] + ); + } + + Y_UNIT_TEST(WriteToResultTransformedTableByRef) { + TTempFileHandle inputFile; + TTempFileHandle inputFileAttrs(inputFile.Name() + ".attr"); + TTempDir tmpDir; + + TStringBuf data = + "{\"key\"=\"075\";\"subkey\"=\".\";\"value\"=\"abc\"};\n" + "{\"key\"=\"800\";\"subkey\"=\".\";\"value\"=\"ddd\"};\n" + "{\"key\"=\"020\";\"subkey\"=\".\";\"value\"=\"q\"};\n" + "{\"key\"=\"150\";\"subkey\"=\".\";\"value\"=\"qzz\"};\n"sv + ; + + inputFile.Write(data.data(), data.size()); + inputFile.FlushData(); + inputFileAttrs.Write(KSV_ATTRS.data(), KSV_ATTRS.size()); + inputFileAttrs.FlushData(); + + TVector<TString> progs; + progs.push_back( + "(\n" + "(let mr_source (DataSource 'yt 'plato))\n" + "(let x (Read! world mr_source (Key '('table (String 'Input))) '('key 'subkey 'value) '()))\n" + "(let world (Left! x))\n" + "(let table1 (Right! x))\n" + "(let table1low (FlatMap table1 (lambda '(item) (block '(\n" + " (let intValueOpt (FromString (Member item 'key) 'Int32))\n" + " (let ret (FlatMap intValueOpt (lambda '(item2) (block '(\n" + " (return (ListIf (< item2 (Int32 '100)) item))\n" + " )))))" + " (return ret)" + ")))))" + "(let res_sink (DataSink 'result))\n" + "(let mr_sink (DataSink 'yt 'plato))\n" + "(let world (Write! world res_sink (Key) table1low '('('ref))))\n" + "(let world (Commit! world mr_sink))\n" + "(let world (Commit! world res_sink))\n" + "(return world)\n" + ")\n" + ); + + progs.push_back( + "(\n" + "(let mr_source (DataSource 'yt 'plato))\n" + "(let x (Read! world mr_source (Key '('table (String 'Result0_0))) '('key 'subkey 'value) '()))\n" + "(let world (Left! x))\n" + "(let table1 (Right! x))\n" + "(let res_sink (DataSink 'result))\n" + "(let data (AsList (String 'x)))\n" + "(let world (Write! world res_sink (Key) table1 '()))\n" + "(let world (Commit! world res_sink))\n" + "(return world)\n" + ")\n" + ); + + TRunMultiplePrograms driver(progs, Cerr); + driver.Tables["yt.plato.Input"] = inputFile.Name(); + driver.TmpDir = tmpDir.Name(); + + auto res = Run(driver); + + UNIT_ASSERT_VALUES_EQUAL(res.size(), 2); + UNIT_ASSERT_NO_DIFF( + "{\"Write\"=[{\"Ref\"=[" + "{\"Reference\"=[\"yt\";\"plato\";\"tmp/bb686f68-2245bd5f-2318fa4e-1\"];\"Columns\"=[\"key\";\"subkey\";\"value\"];\"Remove\"=%true}" + "]}]}", + res[0] + ); + UNIT_ASSERT_NO_DIFF( + "{\"Write\"=[{\"Data\"=[" + "[\"075\";\".\";\"abc\"];" + "[\"020\";\".\";\"q\"]" + "]}]}", + res[1] + ); + } + + Y_UNIT_TEST(WriteAndTakeResult) { + TTempFileHandle inputFile; + TTempFileHandle inputFileAttrs(inputFile.Name() + ".attr"); + TTempDir tmpDir; + + TStringBuf data = + "{\"key\"=\"075\";\"subkey\"=\".\";\"value\"=\"abc\"};\n" + "{\"key\"=\"800\";\"subkey\"=\".\";\"value\"=\"ddd\"};\n" + "{\"key\"=\"020\";\"subkey\"=\".\";\"value\"=\"q\"};\n" + "{\"key\"=\"150\";\"subkey\"=\".\";\"value\"=\"qzz\"};\n"sv + ; + + inputFile.Write(data.data(), data.size()); + inputFile.FlushData(); + inputFileAttrs.Write(KSV_ATTRS.data(), KSV_ATTRS.size()); + inputFileAttrs.FlushData(); + + TVector<TString> progs; + progs.push_back( + "(\n" + "(let mr_source (DataSource 'yt 'plato))\n" + "(let x (Read! world mr_source (Key '('table (String 'Input))) '('key 'subkey 'value) '()))\n" + "(let world (Left! x))\n" + "(let table (Right! x))\n" + "(let result (Map table (lambda '(item) (block '(" + " (let res (Struct))" + " (let res (AddMember res 'k (Member item 'key)))" + " (let res (AddMember res 's (Member item 'subkey)))" + " (let res (AddMember res 'v (Member item 'value)))" + " (return res)" + ")))))" + "(let res_sink (DataSink 'result))\n" + "(let mr_sink (DataSink 'yt 'plato))\n" + "(let world (Write! world res_sink (Key) result '('('ref))))\n" + "(let world (Commit! world mr_sink))\n" + "(let world (Commit! world res_sink))\n" + "(return world)\n" + ")\n" + ); + + progs.push_back( + "(\n" + "(let mr_source (DataSource 'yt 'plato))\n" + "(let x (Read! world mr_source (Key '('table (String 'Result0_0))) '('k 's 'v) '()))\n" + "(let world (Left! x))\n" + "(let table (Right! x))\n" + "(let result (Take table (Uint64 '2)))" + "(let res_sink (DataSink 'result))\n" + "(let mr_sink (DataSink 'yt 'plato))\n" + "(let world (Write! world res_sink (Key) result '('('type))))\n" + "(let world (Commit! world mr_sink))\n" + "(let world (Commit! world res_sink))\n" + "(return world)\n" + ")\n" + ); + + TRunMultiplePrograms driver(progs, Cerr); + driver.Tables["yt.plato.Input"] = inputFile.Name(); + driver.TmpDir = tmpDir.Name(); + + auto res = Run(driver); + + UNIT_ASSERT_VALUES_EQUAL(res.size(), 2); + + //~ Cerr << res[0] << Endl; + //~ Cerr << res[1] << Endl; + UNIT_ASSERT_NO_DIFF( + "{\"Write\"=[{\"Ref\"=[" + "{\"Reference\"=[\"yt\";\"plato\";\"tmp/bb686f68-2245bd5f-2318fa4e-1\"];\"Columns\"=[\"k\";\"s\";\"v\"];\"Remove\"=%true}" + "]}]}", + res[0] + ); + UNIT_ASSERT_NO_DIFF( + "{\"Write\"=[{" + "\"Type\"=[\"ListType\";[\"StructType\";[" + "[\"k\";[\"DataType\";\"String\"]];[\"s\";[\"DataType\";\"String\"]];[\"v\";[\"DataType\";\"String\"]]" + "]]];" + "\"Data\"=[" + "[\"075\";\".\";\"abc\"];" + "[\"800\";\".\";\"ddd\"]" + "]}" + "]}", + res[1] + ); + } + + Y_UNIT_TEST(WriteAndReadScheme) { + TTempFileHandle inputFile; + TTempFileHandle inputFileAttrs(inputFile.Name() + ".attr"); + TTempDir tmpDir; + + TStringBuf data = + "{\"key\"=\"075\";\"subkey\"=\".\";\"value\"=\"abc\"};\n" + "{\"key\"=\"800\";\"subkey\"=\".\";\"value\"=\"ddd\"};\n" + "{\"key\"=\"020\";\"subkey\"=\".\";\"value\"=\"q\"};\n" + "{\"key\"=\"150\";\"subkey\"=\".\";\"value\"=\"qzz\"};\n"sv + ; + + inputFile.Write(data.data(), data.size()); + inputFile.FlushData(); + inputFileAttrs.Write(KSV_ATTRS.data(), KSV_ATTRS.size()); + inputFileAttrs.FlushData(); + + TVector<TString> progs; + progs.push_back( + "(\n" + "(let mr_source (DataSource 'yt 'plato))\n" + "(let x (Read! world mr_source (Key '('table (String 'Input))) '('key 'subkey 'value) '()))\n" + "(let world (Left! x))\n" + "(let table (Right! x))\n" + "(let result0 (Map table (lambda '(item) (block '(" + " (return (AsStruct '('bar (Coalesce (FromString (Member item 'key) 'Uint64) (Uint64 '0)))))" + ")))))" + "(let result1 (Map result0 (lambda '(item) (block '(" + " (return (AddMember (Struct) 'foo item))" + ")))))" + "(let res_sink (DataSink 'result))\n" + "(let mr_sink (DataSink 'yt 'plato))\n" + "(let world (Write! world res_sink (Key) result0 '('('ref))))\n" + "(let world (Write! world res_sink (Key) result1 '('('ref))))\n" + "(let world (Commit! world mr_sink))\n" + "(let world (Commit! world res_sink))\n" + "(return world)\n" + ")\n" + ); + + progs.push_back( + "(\n" + "(let mr_source (DataSource 'yt 'plato))\n" + "(let x (Read! world mr_source (Key '('tablescheme (String 'Result0_0))) (Void) '()))\n" + "(let world (Left! x))\n" + "(let scheme (Right! x))\n" + "(let res_sink (DataSink 'result))\n" + "(let mr_sink (DataSink 'yt 'plato))\n" + "(let world (Write! world res_sink (Key) scheme '('('type))))\n" + "(let world (Commit! world mr_sink))\n" + "(let world (Commit! world res_sink))\n" + "(return world)\n" + ")\n" + ); + + progs.push_back( + "(\n" + "(let mr_source (DataSource 'yt 'plato))\n" + "(let x (Read! world mr_source (Key '('tablescheme (String 'Result0_1))) (Void) '()))\n" + "(let world (Left! x))\n" + "(let scheme (Right! x))\n" + "(let res_sink (DataSink 'result))\n" + "(let mr_sink (DataSink 'yt 'plato))\n" + "(let world (Write! world res_sink (Key) scheme '('('type))))\n" + "(let world (Commit! world mr_sink))\n" + "(let world (Commit! world res_sink))\n" + "(return world)\n" + ")\n" + ); + + TRunMultiplePrograms driver(progs, Cerr); + driver.Tables["yt.plato.Input"] = inputFile.Name(); + driver.TmpDir = tmpDir.Name(); + + auto res = Run(driver); + + UNIT_ASSERT_VALUES_EQUAL(res.size(), 3); + + //~ Cerr << res[0] << Endl; + //~ Cerr << res[1] << Endl; + //~ Cerr << res[2] << Endl; + UNIT_ASSERT_NO_DIFF( + "{\"Write\"=[" + "{\"Ref\"=[{\"Reference\"=[\"yt\";\"plato\";\"tmp/bb686f68-2245bd5f-2318fa4e-1\"];\"Columns\"=[\"bar\"];\"Remove\"=%true}]};" + "{\"Ref\"=[{\"Reference\"=[\"yt\";\"plato\";\"tmp/7ae6459a-7382d1e7-7935c08e-2\"];\"Columns\"=[\"foo\"];\"Remove\"=%true}]}" + "]}", + res[0] + ); + UNIT_ASSERT(res[1].find("\"Fields\"=[{\"Name\"=\"bar\"") != TString::npos); + UNIT_ASSERT(res[2].find("\"Fields\"=[{\"Name\"=\"foo\"") != TString::npos); + } + + Y_UNIT_TEST(ExtendSortedWithNonSortedAndRead) { + TTempFileHandle inputFile; + TTempFileHandle inputFileAttrs(inputFile.Name() + ".attr"); + TTempFileHandle outputFile; + TTempFile outputFileAttr(outputFile.Name() + ".attr"); + TTempDir tmpDir; + + TStringBuf data = + "{\"key\"=\"foo\";\"subkey\"=\"wat\";\"value\"=\"222\"};\n" + "{\"key\"=\"bar\";\"subkey\"=\"wat\";\"value\"=\"111\"};\n" + "{\"key\"=\"jar\";\"subkey\"=\"wat\";\"value\"=\"333\"};\n"sv + ; + + inputFile.Write(data.data(), data.size()); + inputFile.FlushData(); + inputFileAttrs.Write(KSV_ATTRS.data(), KSV_ATTRS.size()); + inputFileAttrs.FlushData(); + + TVector<TString> progs; + progs.push_back( + "(\n" + "(let source (DataSource 'yt 'plato))\n" + "(let x (Read! world source (Key '('table (String 'Input))) '('key 'value) '()))\n" + "(let world (Left! x))\n" + "(let table (Right! x))\n" + "(let sorted (Sort table (Bool 'true) (lambda '(item) (Member item 'value))))\n" + "(let result (Extend table sorted))\n" + "(let sink (DataSink 'yt 'plato))\n" + "(let world (Write! world sink (Key '('table (String 'Output))) result '()))\n" + "(let world (Commit! world sink))\n" + "(return world)\n" + ")\n" + ); + + progs.push_back( + "(\n" + "(let mr_source (DataSource 'yt 'plato))\n" + "(let x (Read! world mr_source (Key '('table (String 'Output))) '('key 'value) '()))\n" + "(let world (Left! x))\n" + "(let result (Right! x))\n" + "(let res_sink (DataSink 'result))\n" + "(let mr_sink (DataSink 'yt 'plato))\n" + "(let world (Write! world res_sink (Key) result '('('type))))\n" + "(let world (Commit! world mr_sink))\n" + "(let world (Commit! world res_sink))\n" + "(return world)\n" + ")\n" + ); + + TRunMultiplePrograms driver(progs, Cerr); + driver.TmpDir = tmpDir.Name(); + driver.Tables["yt.plato.Input"] = inputFile.Name(); + driver.Tables["yt.plato.Output"] = outputFile.Name(); + + auto res = Run(driver); + + UNIT_ASSERT_VALUES_EQUAL(res.size(), 1); + + //~ Cerr << res[0] << Endl; + UNIT_ASSERT_NO_DIFF( + "{\"Write\"=[{" + "\"Type\"=[\"ListType\";[\"StructType\";[[\"key\";[\"DataType\";\"String\"]];[\"value\";[\"DataType\";\"String\"]]]]];" + "\"Data\"=[" + "[\"foo\";\"222\"];[\"bar\";\"111\"];[\"jar\";\"333\"];[\"foo\";\"222\"];[\"bar\";\"111\"];[\"jar\";\"333\"]" + "]}]}", + res[0] + ); + } + + Y_UNIT_TEST(OrderedExtendSortedWithNonSortedAndRead) { + TTempFileHandle inputFile; + TTempFileHandle inputFileAttrs(inputFile.Name() + ".attr"); + TTempFileHandle outputFile; + TTempFile outputFileAttr(outputFile.Name() + ".attr"); + TTempDir tmpDir; + + TStringBuf data = + "{\"key\"=\"foo\";\"subkey\"=\"wat\";\"value\"=\"222\"};\n" + "{\"key\"=\"bar\";\"subkey\"=\"wat\";\"value\"=\"111\"};\n" + "{\"key\"=\"jar\";\"subkey\"=\"wat\";\"value\"=\"333\"};\n"sv + ; + + inputFile.Write(data.data(), data.size()); + inputFile.FlushData(); + inputFileAttrs.Write(KSV_ATTRS.data(), KSV_ATTRS.size()); + inputFileAttrs.FlushData(); + + TVector<TString> progs; + progs.push_back( + "(\n" + "(let source (DataSource 'yt 'plato))\n" + "(let x (Read! world source (Key '('table (String 'Input))) '('key 'value) '()))\n" + "(let world (Left! x))\n" + "(let table (Right! x))\n" + "(let sorted (Sort table (Bool 'true) (lambda '(item) (Member item 'value))))\n" + "(let result (OrderedExtend table sorted))\n" + "(let sink (DataSink 'yt 'plato))\n" + "(let world (Write! world sink (Key '('table (String 'Output))) result '()))\n" + "(let world (Commit! world sink))\n" + "(return world)\n" + ")\n" + ); + + progs.push_back( + "(\n" + "(let mr_source (DataSource 'yt 'plato))\n" + "(let x (Read! world mr_source (Key '('table (String 'Output))) '('key 'value) '()))\n" + "(let world (Left! x))\n" + "(let result (Right! x))\n" + "(let res_sink (DataSink 'result))\n" + "(let mr_sink (DataSink 'yt 'plato))\n" + "(let world (Write! world res_sink (Key) result '('('type))))\n" + "(let world (Commit! world mr_sink))\n" + "(let world (Commit! world res_sink))\n" + "(return world)\n" + ")\n" + ); + + TRunMultiplePrograms driver(progs, Cerr); + driver.TmpDir = tmpDir.Name(); + driver.Tables["yt.plato.Input"] = inputFile.Name(); + driver.Tables["yt.plato.Output"] = outputFile.Name(); + + auto res = Run(driver); + + UNIT_ASSERT_VALUES_EQUAL(res.size(), 1); + + //~ Cerr << res[0] << Endl; + UNIT_ASSERT_NO_DIFF( + "{\"Write\"=[{" + "\"Type\"=[\"ListType\";[\"StructType\";[[\"key\";[\"DataType\";\"String\"]];[\"value\";[\"DataType\";\"String\"]]]]];" + "\"Data\"=[" + "[\"foo\";\"222\"];[\"bar\";\"111\"];[\"jar\";\"333\"];[\"bar\";\"111\"];[\"foo\";\"222\"];[\"jar\";\"333\"]" + "]}]}", + res[0] + ); + } + + Y_UNIT_TEST(TestParametersEvaluation) { + auto s = "(\n" + "(let res_sink (DataSink 'result))\n" + "(let data (Parameter '\"$foo\" (ParseType '\"Tuple<String, Int32 ? , List<Uint32>, Dict<Int32, Bool>, Struct<a : Void, b : Double>, Variant<Int32, Bool>>\")))\n" + "(let world (Write! world res_sink (Key) data '('('type))))\n" + "(let world (Commit! world res_sink))\n" + "(return world)\n" + ")\n"; + + auto params = R"__( +{"$foo"={Data=[ + bar; + "33"; + ["1";"2";"3"]; + [["7";%true];["12";%false]]; + [#;"-1.7"]; + ["0";"8"]; +]}} + )__"; + + auto res = RunProgram(s, THashMap<TString, TString>(), "", params); + UNIT_ASSERT_VALUES_EQUAL(res.size(), 1); + UNIT_ASSERT_NO_DIFF(R"__({"Write"=[{"Type"=["TupleType";[["DataType";"String"];["OptionalType";["DataType";"Int32"]];["ListType";["DataType";"Uint32"]];["DictType";["DataType";"Int32"];["DataType";"Bool"]];["StructType";[["a";["VoidType"]];["b";["DataType";"Double"]]]];["VariantType";["TupleType";[["DataType";"Int32"];["DataType";"Bool"]]]]]];"Data"=["bar";["33"];["1";"2";"3"];[["7";%true];["12";%false]];["Void";"-1.7"];["0";"8"]]}]})__", res[0]); + } + } + +} // namespace NYql diff --git a/ydb/library/yql/core/ut/yql_expr_constraint_ut.cpp b/ydb/library/yql/core/ut/yql_expr_constraint_ut.cpp new file mode 100644 index 00000000000..431beabaa95 --- /dev/null +++ b/ydb/library/yql/core/ut/yql_expr_constraint_ut.cpp @@ -0,0 +1,3217 @@ +#include <ydb/library/yql/providers/yt/provider/yql_yt_provider.h> +#include <ydb/library/yql/providers/yt/gateway/file/yql_yt_file.h> +#include <ydb/library/yql/providers/common/provider/yql_provider_names.h> +#include <ydb/library/yql/ast/yql_ast_annotation.h> +#include <ydb/library/yql/ast/yql_expr.h> +#include <ydb/library/yql/core/type_ann/type_ann_core.h> +#include <ydb/library/yql/core/type_ann/type_ann_expr.h> +#include <ydb/library/yql/core/ut_common/yql_ut_common.h> +#include <ydb/library/yql/core/yql_graph_transformer.h> +#include <ydb/library/yql/core/yql_expr_optimize.h> +#include <ydb/library/yql/core/services/yql_transform_pipeline.h> +#include <ydb/library/yql/minikql/mkql_function_registry.h> +#include <ydb/library/yql/minikql/invoke_builtins/mkql_builtins.h> + +#include <library/cpp/testing/unittest/registar.h> +#include <util/string/cast.h> + +namespace NYql { + +Y_UNIT_TEST_SUITE(TYqlExprConstraints) { + + static TExprNode::TPtr ParseAndAnnotate(const TStringBuf program, TExprContext& exprCtx) { + TAstParseResult astRes = ParseAst(program); + UNIT_ASSERT(astRes.IsOk()); + TExprNode::TPtr exprRoot; + UNIT_ASSERT(CompileExpr(*astRes.Root, exprRoot, exprCtx, nullptr, nullptr)); + + auto functionRegistry = NKikimr::NMiniKQL::CreateFunctionRegistry(NKikimr::NMiniKQL::CreateBuiltinRegistry()); + TTestTablesMapping testTables; + auto yqlNativeServices = NFile::TYtFileServices::Make(functionRegistry.Get(), testTables); + auto ytGateway = CreateYtFileGateway(yqlNativeServices); + auto typeAnnotationContext = MakeIntrusive<TTypeAnnotationContext>(); + typeAnnotationContext->RandomProvider = CreateDeterministicRandomProvider(1); + auto ytState = MakeIntrusive<TYtState>(); + ytState->Gateway = ytGateway; + ytState->Types = typeAnnotationContext.Get(); + + InitializeYtGateway(ytGateway, ytState); + typeAnnotationContext->AddDataSink(YtProviderName, CreateYtDataSink(ytState)); + typeAnnotationContext->AddDataSource(YtProviderName, CreateYtDataSource(ytState)); + + auto transformer = TTransformationPipeline(typeAnnotationContext) + .AddServiceTransformers() + .AddPreTypeAnnotation() + .AddExpressionEvaluation(*functionRegistry) + .AddIOAnnotation() + .AddTypeAnnotation() + .AddPostTypeAnnotation() + .Build(); + + const auto status = SyncTransform(*transformer, exprRoot, exprCtx); + if (status == IGraphTransformer::TStatus::Error) + Cerr << exprCtx.IssueManager.GetIssues().ToString() << Endl; + UNIT_ASSERT(status == IGraphTransformer::TStatus::Ok); + return exprRoot; + } + + template <class TConstraint> + static void CheckConstraint(const TExprNode::TPtr& exprRoot, const TStringBuf nodeName, const TStringBuf constrStr) { + TExprNode* nodeToCheck = nullptr; + VisitExpr(exprRoot, [nodeName, &nodeToCheck] (const TExprNode::TPtr& node) { + if (node->IsCallable(nodeName)) { + nodeToCheck = node.Get(); + } + return !nodeToCheck; + }); + UNIT_ASSERT(nodeToCheck); + UNIT_ASSERT(nodeToCheck->GetState() == TExprNode::EState::ConstrComplete); + const auto constr = nodeToCheck->GetConstraint<TConstraint>(); + if (constrStr.empty()) { + UNIT_ASSERT(!constr); + } else { + UNIT_ASSERT(constr); + UNIT_ASSERT(constr->IsApplicableToType(*nodeToCheck->GetTypeAnn())); + UNIT_ASSERT_VALUES_EQUAL(ToString(*constr), constrStr); + } + } + + Y_UNIT_TEST(Sort) { + const auto s = R"(( + (let mr_sink (DataSink 'yt (quote plato))) + (let list (AsList + (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value (String 'v))) + (AsStruct '('key (String '1)) '('subkey (String 'd)) '('value (String 'v))) + (AsStruct '('key (String '3)) '('subkey (String 'b)) '('value (String 'v))) + )) + (let sorted (Sort list (Bool 'True) (lambda '(item) (Member item 'key)))) + (let world (Write! world mr_sink (Key '('table (String 'Output))) sorted '('('mode 'renew)))) + (let world (Commit! world mr_sink)) + (return world) + ))"; + + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TSortedConstraintNode>(exprRoot, "Sort", "Sorted(key[asc])"); + } + + Y_UNIT_TEST(SortByTranspentIfPresent) { + const auto s = R"(( + (let mr_sink (DataSink 'yt (quote plato))) + (let list (AsList + (Just (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value (String 'v)))) + (Just (AsStruct '('key (String '1)) '('subkey (String 'd)) '('value (String 'v)))) + (Just (AsStruct '('key (String '3)) '('subkey (String 'b)) '('value (String 'v)))) + )) + (let sorted (Sort list (Bool 'True) (lambda '(row) (FlatMap row (lambda '(item) (Just (Member item 'key))))))) + (let map (OrderedFlatMap sorted (lambda '(item) item))) + (let world (Write! world mr_sink (Key '('table (String 'Output))) map '('('mode 'renew)))) + (let world (Commit! world mr_sink)) + (return world) + ))"; + + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TSortedConstraintNode>(exprRoot, "OrderedFlatMap", "Sorted(key[asc])"); + } + + Y_UNIT_TEST(SortByDuplicateColumn) { + const auto s = R"(( + (let mr_sink (DataSink 'yt (quote plato))) + (let list (AsList + (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value (String 'v))) + (AsStruct '('key (String '1)) '('subkey (String 'd)) '('value (String 'v))) + (AsStruct '('key (String '3)) '('subkey (String 'b)) '('value (String 'v))) + )) + (let sorted (Sort list '((Bool 'True) (Bool 'False) (Bool 'True)) (lambda '(item) '((Member item 'key) (Member item 'key) (Member item 'subkey))))) + (let world (Write! world mr_sink (Key '('table (String 'Output))) sorted '('('mode 'renew)))) + (let world (Commit! world mr_sink)) + (return world) + ))"; + + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TSortedConstraintNode>(exprRoot, "Sort", "Sorted(key[asc];subkey[asc])"); + } + + Y_UNIT_TEST(SortByFullRow) { + const auto s = R"(( + (let mr_sink (DataSink 'yt (quote plato))) + (let list (AsList (Utf8 's) (Utf8 'o) (Utf8 'r) (Utf8 't))) + (let sorted (Sort list (Bool 'True) (lambda '(item) item))) + (let map (OrderedMap sorted (lambda '(item) (AsStruct '('key item))))) + (let world (Write! world mr_sink (Key '('table (String 'Output))) map '('('mode 'renew)))) + (let world (Commit! world mr_sink)) + (return world) + ))"; + + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TSortedConstraintNode>(exprRoot, "Sort", "Sorted(/[asc])"); + CheckConstraint<TSortedConstraintNode>(exprRoot, "OrderedMap", "Sorted(key[asc])"); + } + + Y_UNIT_TEST(SortByTuple) { + const auto s = R"(( + (let mr_sink (DataSink 'yt (quote plato))) + (let list (AsList + (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value (String 'v))) + (AsStruct '('key (String '1)) '('subkey (String 'd)) '('value (String 'v))) + (AsStruct '('key (String '3)) '('subkey (String 'b)) '('value (String 'v))) + )) + (let sorted (Sort list (Bool 'False) (lambda '(item) '((Member item 'key) (Member item 'subkey))))) + (let world (Write! world mr_sink (Key '('table (String 'Output))) sorted '('('mode 'renew)))) + (let world (Commit! world mr_sink)) + (return world) + ))"; + + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TSortedConstraintNode>(exprRoot, "Sort", "Sorted(key[desc];subkey[desc])"); + } + + Y_UNIT_TEST(SortByTupleElements) { + const auto s = R"(( + (let mr_sink (DataSink 'yt (quote plato))) + (let list (AsList + (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value '((String 'x) (String 'a) (String 'u)))) + (AsStruct '('key (String '1)) '('subkey (String 'd)) '('value '((String 'y) (String 'b) (String 'v)))) + (AsStruct '('key (String '3)) '('subkey (String 'b)) '('value '((String 'z) (String 'c) (String 'w)))) + )) + (let sorted (Sort list '((Not (Bool 'False)) (Bool 'False) (Not (Bool 'True))) (lambda '(item) '((Nth (Member item 'value) '2) (Member item 'key) (Nth (Member item 'value) '0))))) + (let map (OrderedMap sorted (lambda '(item) (AsStruct '('tuple '((Nth (Member item 'value) '1) (Nth (Member item 'value) '2) (Nth (Member item 'value) '0) (Member item 'subkey) (Member item 'key))))))) + (let world (Write! world mr_sink (Key '('table (String 'Output))) map '('('mode 'renew)))) + (let world (Commit! world mr_sink)) + (return world) + ))"; + + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TSortedConstraintNode>(exprRoot, "Sort", "Sorted(value/2[asc];key[desc];value/0[desc])"); + CheckConstraint<TSortedConstraintNode>(exprRoot, "OrderedMap", "Sorted(tuple/1[asc];tuple/4[desc];tuple/2[desc])"); + } + + Y_UNIT_TEST(SortByColumnAndExpr) { + const auto s = R"(( + (let mr_sink (DataSink 'yt (quote plato))) + (let list (AsList + (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value (String 'v))) + (AsStruct '('key (String '1)) '('subkey (String 'd)) '('value (String 'v))) + (AsStruct '('key (String '3)) '('subkey (String 'b)) '('value (String 'v))) + )) + (let sorted (Sort list '((Bool 'True) (Bool 'False) (Bool 'True)) (lambda '(item) '((Member item 'key) (SafeCast (Member item 'value) (DataType 'Utf8)) (Member item 'subkey))))) + (let world (Write! world mr_sink (Key '('table (String 'Output))) sorted '('('mode 'renew)))) + (let world (Commit! world mr_sink)) + (return world) + ))"; + + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TSortedConstraintNode>(exprRoot, "Sort", "Sorted(key[asc])"); + } + + Y_UNIT_TEST(SortDesc) { + const auto s = R"(( + (let mr_sink (DataSink 'yt (quote plato))) + (let list (AsList + (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value (String 'v))) + (AsStruct '('key (String '1)) '('subkey (String 'd)) '('value (String 'v))) + (AsStruct '('key (String '3)) '('subkey (String 'b)) '('value (String 'v))) + )) + (let sorted (Sort list '((Bool 'True) (Bool 'False)) (lambda '(item) '((Member item 'key) (Member item 'subkey))))) + (let world (Write! world mr_sink (Key '('table (String 'Output))) sorted '('('mode 'renew)))) + (let world (Commit! world mr_sink)) + (return world) + ))"; + + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TSortedConstraintNode>(exprRoot, "Sort", "Sorted(key[asc];subkey[desc])"); + } + + Y_UNIT_TEST(SortedOverWideMap) { + const auto s = R"(( + (let mr_sink (DataSink 'yt (quote plato))) + (let list (AsList + (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value (String 'v))) + (AsStruct '('key (String '1)) '('subkey (String 'd)) '('value (String 'v))) + (AsStruct '('key (String '3)) '('subkey (String 'b)) '('value (String 'v))) + )) + (let sorted (ToFlow (Sort list '((Bool 'True) (Bool 'False)) (lambda '(item) '((Member item 'key) (Member item 'subkey)))))) + (let expand (ExpandMap sorted (lambda '(item) (Member item 'key) (Member item 'subkey) (Member item 'value)))) + (let wide (WideMap expand (lambda '(a b c) c b a))) + (let narrow (NarrowMap wide (lambda '(x y z) (AsStruct '('x x) '('y y) '('z z))))) + (let world (Write! world mr_sink (Key '('table (String 'Output))) (Collect narrow) '('('mode 'renew)))) + (let world (Commit! world mr_sink)) + (return world) + ))"; + + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TSortedConstraintNode>(exprRoot, "Collect", "Sorted(z[asc];y[desc])"); + } + + Y_UNIT_TEST(SortedOverWideTopSort) { + const auto s = R"(( + (let mr_sink (DataSink 'yt (quote plato))) + (let list (AsList + (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value (String 'v))) + (AsStruct '('key (String '1)) '('subkey (String 'd)) '('value (String 'v))) + (AsStruct '('key (String '3)) '('subkey (String 'b)) '('value (String 'v))) + )) + (let flow (ToFlow list)) + (let expand (ExpandMap flow (lambda '(item) (Member item 'key) (Member item 'subkey) (Member item 'value)))) + (let wide (WideTopSort expand (Uint64 '2) '('('2 (Bool 'False)) '('0 (Bool 'True))))) + (let narrow (NarrowMap wide (lambda '(x y z) (AsStruct '('x x) '('y y) '('z z))))) + (let world (Write! world mr_sink (Key '('table (String 'Output))) (Collect narrow) '('('mode 'renew)))) + (let world (Commit! world mr_sink)) + (return world) + ))"; + + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TSortedConstraintNode>(exprRoot, "Collect", "Sorted(z[desc];x[asc])"); + } + + Y_UNIT_TEST(TopSort) { + const auto s = R"(( + (let mr_sink (DataSink 'yt (quote plato))) + (let list (AsList + (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value (String 'v))) + (AsStruct '('key (String '1)) '('subkey (String 'd)) '('value (String 'v))) + (AsStruct '('key (String '3)) '('subkey (String 'b)) '('value (String 'v))) + )) + (let sorted (TopSort list (Uint64 '2) '((Bool 'True) (Bool 'True)) (lambda '(item) '((Member item 'key) (Member item 'subkey))))) + (let world (Write! world mr_sink (Key '('table (String 'Output))) sorted '('('mode 'renew)))) + (let world (Commit! world mr_sink)) + (return world) + ))"; + + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TSortedConstraintNode>(exprRoot, "TopSort", "Sorted(key[asc];subkey[asc])"); + } + + Y_UNIT_TEST(MergeWithFirstEmpty) { + const auto s = R"(( + (let mr_sink (DataSink 'yt (quote plato))) + (let list (AsList + (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value (String 'v))) + (AsStruct '('key (String '1)) '('subkey (String 'd)) '('value (String 'v))) + (AsStruct '('key (String '3)) '('subkey (String 'b)) '('value (String 'v))) + )) + (let sorted (Sort list (Bool 'True) (lambda '(item) (Member item 'key)))) + (let empty (List (ListType (StructType '('key (DataType 'String)) '('subkey (DataType 'String)) '('value (DataType 'String)))))) + (let merged (Merge empty sorted)) + (let world (Write! world mr_sink (Key '('table (String 'Output))) merged '('('mode 'renew)))) + (let world (Commit! world mr_sink)) + (return world) + ))"; + + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TSortedConstraintNode>(exprRoot, "Merge", "Sorted(key[asc])"); + } + + Y_UNIT_TEST(MergeWithCloneColumns) { + const auto s = R"(( + (let mr_sink (DataSink 'yt (quote plato))) + (let list (AsList + (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value (String 'v))) + (AsStruct '('key (String '1)) '('subkey (String 'd)) '('value (String 'v))) + (AsStruct '('key (String '3)) '('subkey (String 'b)) '('value (String 'v))) + )) + (let sorted (Sort list (Bool 'False) (lambda '(item) (Member item 'key)))) + (let clone (OrderedMap sorted (lambda '(row) (AsStruct '('key (Member row 'key)) '('subkey (Member row 'key)) '('value (Member row 'key)))))) + (let merged (Merge clone sorted)) + (let world (Write! world mr_sink (Key '('table (String 'Output))) merged '('('mode 'renew)))) + (let world (Commit! world mr_sink)) + (return world) + ))"; + + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TSortedConstraintNode>(exprRoot, "Merge", "Sorted(key[desc])"); + } + + Y_UNIT_TEST(UnionMergeWithDiffTypes) { + const auto s = R"(( + (let mr_sink (DataSink 'yt (quote plato))) + (let list1 (AsList + (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value (String 'v))) + (AsStruct '('key (String '1)) '('subkey (String 'd)) '('value (String 'v))) + (AsStruct '('key (String '3)) '('subkey (String 'b)) '('value (String 'v))) + )) + (let list2 (AsList + (AsStruct '('key (String '4)) '('subkey (Utf8 'c)) '('value (String 'v))) + (AsStruct '('key (String '1)) '('subkey (Utf8 'd)) '('value (String 'v))) + (AsStruct '('key (String '3)) '('subkey (Utf8 'b)) '('value (String 'v))) + )) + (let sorted1 (Sort list1 (Bool 'False) (lambda '(item) '((Member item 'key) (Member item 'subkey) (Member item 'value))))) + (let sorted2 (Sort list2 (Bool 'False) (lambda '(item) '((Member item 'key) (Member item 'subkey) (Member item 'value))))) + (let merged (UnionMerge sorted1 sorted2)) + (let world (Write! world mr_sink (Key '('table (String 'Output))) merged '('('mode 'renew)))) + (let world (Commit! world mr_sink)) + (return world) + ))"; + + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TSortedConstraintNode>(exprRoot, "UnionMerge", "Sorted(key[desc])"); + } + + Y_UNIT_TEST(ExtractMembersKey) { + const auto s = R"(( + (let mr_sink (DataSink 'yt (quote plato))) + (let list (AsList + (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value (String 'v))) + (AsStruct '('key (String '1)) '('subkey (String 'd)) '('value (String 'w))) + (AsStruct '('key (String '3)) '('subkey (String 'b)) '('value (String 'u))) + )) + (let sorted (Sort list '((Bool 'True) (Bool 'True)) (lambda '(item) '((Member item 'key) (Member item 'subkey))))) + (let sorted (ExtractMembers sorted '('key))) + (let world (Write! world mr_sink (Key '('table (String 'Output))) sorted '('('mode 'renew)))) + (let world (Commit! world mr_sink)) + (return world) + ))"; + + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TSortedConstraintNode>(exprRoot, "ExtractMembers", "Sorted(key[asc])"); + } + + Y_UNIT_TEST(ExtractMembersNonKey) { + const auto s = R"(( + (let mr_sink (DataSink 'yt (quote plato))) + (let list (AsList + (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value (String 'v))) + (AsStruct '('key (String '1)) '('subkey (String 'd)) '('value (String 'w))) + (AsStruct '('key (String '3)) '('subkey (String 'b)) '('value (String 'u))) + )) + (let sorted (Sort list '((Bool 'True) (Bool 'True)) (lambda '(item) '((Member item 'key) (Member item 'subkey))))) + (let sorted (ExtractMembers sorted '('value))) + (let world (Write! world mr_sink (Key '('table (String 'Output))) sorted '('('mode 'renew)))) + (let world (Commit! world mr_sink)) + (return world) + ))"; + + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TSortedConstraintNode>(exprRoot, "ExtractMembers", ""); + } + + Y_UNIT_TEST(OrderedLMap) { + const auto s = R"(( + (let mr_sink (DataSink 'yt (quote plato))) + (let list (AsList + (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value (String 'v))) + (AsStruct '('key (String '1)) '('subkey (String 'd)) '('value (String 'w))) + (AsStruct '('key (String '3)) '('subkey (String 'b)) '('value (String 'u))) + )) + (let sorted (Sort list '((Bool 'False) (Bool 'False)) (lambda '(item) '((Member item 'key) (Member item 'subkey))))) + (let sorted (OrderedLMap sorted (lambda '(stream) (OrderedFlatMap stream (lambda '(item) (Just item)))))) + (let world (Write! world mr_sink (Key '('table (String 'Output))) sorted '('('mode 'renew)))) + (let world (Commit! world mr_sink)) + (return world) + ))"; + + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TSortedConstraintNode>(exprRoot, "OrderedLMap", "Sorted(key[desc];subkey[desc])"); + CheckConstraint<TSortedConstraintNode>(exprRoot, "OrderedFlatMap", "Sorted(key[desc];subkey[desc])"); + } + + Y_UNIT_TEST(OrderedFlatMap) { + const auto s = R"(( + (let mr_sink (DataSink 'yt (quote plato))) + (let list (AsList + (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value (String 'v))) + (AsStruct '('key (String '1)) '('subkey (String 'd)) '('value (String 'w))) + (AsStruct '('key (String '3)) '('subkey (String 'b)) '('value (String 'u))) + )) + (let list (AssumeUnique list '('key) '('subkey 'value))) + (let list (AssumeDistinct list '('key 'subkey) '('value))) + (let sorted (Sort list '((Bool 'True) (Bool 'True)) (lambda '(item) '((Member item 'key) (Member item 'subkey))))) + (let sorted (OrderedFlatMap sorted (lambda '(item) (OptionalIf (== (Member item 'key) (String '1)) (AsStruct '('k (Member item 'key)) '('v (Member item 'value))))))) + (let world (Write! world mr_sink (Key '('table (String 'Output))) sorted '('('mode 'renew)))) + (let world (Commit! world mr_sink)) + (return world) + ))"; + + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TSortedConstraintNode>(exprRoot, "OrderedFlatMap", "Sorted(k[asc])"); + CheckConstraint<TUniqueConstraintNode>(exprRoot, "OrderedFlatMap", "Unique((k))"); + CheckConstraint<TDistinctConstraintNode>(exprRoot, "OrderedFlatMap", "Distinct((v))"); + } + + Y_UNIT_TEST(OrderedFlatMapWithEmptyFilter) { + const auto s = R"(( + (let mr_sink (DataSink 'yt (quote plato))) + (let list (AsList + (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value (String 'v))) + (AsStruct '('key (String '1)) '('subkey (String 'd)) '('value (String 'w))) + (AsStruct '('key (String '3)) '('subkey (String 'b)) '('value (String 'u))) + )) + (let list (AssumeUnique list '('key) '('subkey 'value))) + (let list (AssumeDistinct list '('key 'subkey) '('value))) + (let sorted (Sort list '((Bool 'True) (Bool 'True)) (lambda '(item) '((Member item 'key) (Member item 'subkey))))) + (let sorted (OrderedFlatMap sorted (lambda '(item) (OptionalIf (Bool 'false) (AsStruct '('k (Member item 'key)) '('v (Member item 'value))))))) + (let world (Write! world mr_sink (Key '('table (String 'Output))) sorted '('('mode 'renew)))) + (let world (Commit! world mr_sink)) + (return world) + ))"; + + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TSortedConstraintNode>(exprRoot, "OrderedFlatMap", "Sorted(k[asc])"); + CheckConstraint<TUniqueConstraintNode>(exprRoot, "OrderedFlatMap", "Unique((k))"); + CheckConstraint<TDistinctConstraintNode>(exprRoot, "OrderedFlatMap", "Distinct((v))"); + } + + Y_UNIT_TEST(OrderedFlatMapNonKey) { + const auto s = R"(( + (let mr_sink (DataSink 'yt (quote plato))) + (let list (AsList + (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value (String 'v))) + (AsStruct '('key (String '1)) '('subkey (String 'd)) '('value (String 'w))) + (AsStruct '('key (String '3)) '('subkey (String 'b)) '('value (String 'u))) + )) + (let sorted (Sort list '((Bool 'True) (Bool 'True)) (lambda '(item) '((Member item 'key) (Member item 'subkey))))) + (let sorted (OrderedFlatMap sorted (lambda '(item) (OptionalIf (== (Member item 'key) (String '1)) (AsStruct '('key (Member item 'value))))))) + (let world (Write! world mr_sink (Key '('table (String 'Output))) sorted '('('mode 'renew)))) + (let world (Commit! world mr_sink)) + (return world) + ))"; + + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TSortedConstraintNode>(exprRoot, "OrderedFlatMap", ""); + } + + Y_UNIT_TEST(OrderedFilter) { + const auto s = R"(( + (let mr_sink (DataSink 'yt (quote plato))) + (let list (AsList + (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value (String 'v))) + (AsStruct '('key (String '1)) '('subkey (String 'd)) '('value (String 'w))) + (AsStruct '('key (String '3)) '('subkey (String 'b)) '('value (String 'u))) + )) + (let list (AssumeUnique list '('key) '('subkey 'value))) + (let list (AssumeDistinct list '('key 'subkey) '('value))) + (let sorted (Sort list '((Bool 'True) (Bool 'True)) (lambda '(item) '((Member item 'key) (Member item 'subkey))))) + (let sorted (OrderedFilter sorted (lambda '(item) (> (Member item 'key) (String '0))))) + (let world (Write! world mr_sink (Key '('table (String 'Output))) sorted '('('mode 'renew)))) + (let world (Commit! world mr_sink)) + (return world) + ))"; + + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TSortedConstraintNode>(exprRoot, "OrderedFilter", "Sorted(key[asc];subkey[asc])"); + CheckConstraint<TUniqueConstraintNode>(exprRoot, "OrderedFilter", "Unique((key)(subkey,value))"); + CheckConstraint<TDistinctConstraintNode>(exprRoot, "OrderedFilter", "Distinct((key,subkey)(value))"); + } + + Y_UNIT_TEST(OrderedMapNullifyOneColumn) { + const auto s = R"(( + (let mr_sink (DataSink 'yt (quote plato))) + (let list (AsList + (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value (String 'v))) + (AsStruct '('key (String '1)) '('subkey (String 'd)) '('value (String 'w))) + (AsStruct '('key (String '3)) '('subkey (String 'b)) '('value (String 'u))) + )) + (let list (AssumeUnique list '('key) '('subkey 'value))) + (let list (AssumeDistinct list '('key 'subkey) '('value))) + (let sorted (Sort list (Bool 'False) (lambda '(row) '((Member row 'key) (Member row 'subkey) (Member row 'value))))) + (let map (OrderedMap sorted (lambda '(row) (AsStruct '('k (Member row 'key)) '('s (OptionalIf (AggrNotEquals (Member row 'key) (Member row 'value)) (Member row 'subkey))) '('v (Member row 'value)))))) + (let world (Write! world mr_sink (Key '('table (String 'Output))) map '('('mode 'renew)))) + (let world (Commit! world mr_sink)) + (return world) + ))"; + + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TSortedConstraintNode>(exprRoot, "OrderedMap", "Sorted(k[desc])"); + CheckConstraint<TUniqueConstraintNode>(exprRoot, "OrderedMap", "Unique((k)(s,v))"); + CheckConstraint<TDistinctConstraintNode>(exprRoot, "OrderedMap", "Distinct((v))"); + } + + Y_UNIT_TEST(FlattenMembers) { + const auto s = R"(( + (let mr_sink (DataSink 'yt (quote plato))) + (let list (AsList + (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value (String 'v))) + (AsStruct '('key (String '1)) '('subkey (String 'd)) '('value (String 'w))) + (AsStruct '('key (String '3)) '('subkey (String 'b)) '('value (String 'u))) + )) + (let list (AssumeUnique list '('key) '('subkey 'value))) + (let list (AssumeDistinct list '('key 'subkey) '('value))) + (let sorted (Sort list '((Bool 'True) (Bool 'True)) (lambda '(item) '((Member item 'key) (Member item 'subkey))))) + (let mapped (OrderedMap sorted (lambda '(item) '((AsStruct '('key (Member item 'key)) '('subkey (Member item 'subkey))) (AsStruct '('subkey (Member item 'subkey)) '('value (Member item 'value))))))) + (let flatten (OrderedMap mapped (lambda '(pair) (FlattenMembers '('one (Nth pair '0)) '('two (Nth pair '1)))))) + (let world (Write! world mr_sink (Key '('table (String 'Output))) (LazyList flatten) '('('mode 'renew)))) + (let world (Commit! world mr_sink)) + (return world) + ))"; + + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TSortedConstraintNode>(exprRoot, "LazyList", "Sorted(onekey[asc];onesubkey,twosubkey[asc])"); + CheckConstraint<TUniqueConstraintNode>(exprRoot, "LazyList", "Unique((onekey)({onesubkey,twosubkey},twovalue))"); + CheckConstraint<TDistinctConstraintNode>(exprRoot, "LazyList", "Distinct((onekey,{onesubkey,twosubkey})(twovalue))"); + } + + Y_UNIT_TEST(Passthrough) { + const auto s = R"(( + (let mr_sink (DataSink 'yt (quote plato))) + (let list (AsList + (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value (String 'v))) + (AsStruct '('key (String '1)) '('subkey (String 'd)) '('value (String 'w))) + (AsStruct '('key (String '3)) '('subkey (String 'b)) '('value (String 'u))) + )) + (let list (FlatMap list (lambda '(item) (block '( + (let res (Map (Just item) (lambda '(m) + (AsStruct + '('key1 (Member m 'key)) + '('key2 (Member m 'key)) + '('subkey (Member m 'subkey)) + '('value (Member m 'value)) + ) + ))) + (let res (Map res (lambda '(m) + (AsStruct + '('p.key1 (Member m 'key1)) + '('p.key2 (Member m 'key2)) + '('p.subkey (Member m 'subkey)) + '('value (Member m 'value)) + ) + ))) + (return (Just (DivePrefixMembers (Unwrap res) '('p.)))) + ))))) + (let world (Write! world mr_sink (Key '('table (String 'Output))) list '('('mode 'renew)))) + (let world (Commit! world mr_sink)) + (return world) +))"; + + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TPassthroughConstraintNode>(exprRoot, "DivePrefixMembers", "Passthrough(key1:key,key2:key,subkey:subkey)"); + CheckConstraint<TPassthroughConstraintNode>(exprRoot, "Unwrap", "Passthrough(p.key1:key,p.key2:key,p.subkey:subkey,value:value)"); + + } + + Y_UNIT_TEST(PassthroughOverTuple) { + const auto s = R"(( + (let mr_sink (DataSink 'yt (quote plato))) + (let list (AsList + (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value (String 'v))) + (AsStruct '('key (String '1)) '('subkey (String 'd)) '('value (String 'v))) + (AsStruct '('key (String '3)) '('subkey (String 'b)) '('value (String 'v))) + )) + (let list (FlatMap list (lambda '(item) (block '( + (let res (Map (Just item) (lambda '(m) + '( + (Member m 'key) + (Member m 'key) + (Member m 'subkey) + (Member m 'value) + ) + ))) + (let res (Map res (lambda '(m) + (AsStruct + '('p.key1 (Nth m '0)) + '('p.key2 (Nth m '1)) + '('p.subkey (Nth m '2)) + '('value (Nth m '3)) + ) + ))) + (return (Just (DivePrefixMembers (Unwrap res) '('p.)))) + ))))) + (let world (Write! world mr_sink (Key '('table (String 'Output))) list '('('mode 'renew)))) + (let world (Commit! world mr_sink)) + (return world) +))"; + + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TPassthroughConstraintNode>(exprRoot, "DivePrefixMembers", "Passthrough(key1:key,key2:key,subkey:subkey)"); + CheckConstraint<TPassthroughConstraintNode>(exprRoot, "Unwrap", "Passthrough(p.key1:key,p.key2:key,p.subkey:subkey,value:value)"); + } + + Y_UNIT_TEST(PassthroughOverWideFlow) { + const auto s = R"(( + (let mr_sink (DataSink 'yt (quote plato))) + (let list (AsList + (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value (String 'v))) + (AsStruct '('key (String '1)) '('subkey (String 'd)) '('value (String 'v))) + (AsStruct '('key (String '3)) '('subkey (String 'b)) '('value (String 'v))) + )) + (let list (FlatMap list (lambda '(item) (block '( + (let res (ExpandMap (ToFlow (Just item)) (lambda '(m) + (Member m 'key) + (Member m 'key) + (Member m 'subkey) + (Member m 'value) + ))) + (let res (WideMap res (lambda '(m0 m1 m2 m3) m0 m2 m3 m1))) + (let res (NarrowMap res (lambda '(m0 m1 m2 m3) + (AsStruct + '('p.key1 m0) + '('p.key2 m3) + '('p.subkey m1) + '('value m2) + ) + ))) + (return (Just (DivePrefixMembers (Unwrap (Head (Collect res))) '('p.)))) + ))))) + (let world (Write! world mr_sink (Key '('table (String 'Output))) list '('('mode 'renew)))) + (let world (Commit! world mr_sink)) + (return world) +))"; + + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TPassthroughConstraintNode>(exprRoot, "DivePrefixMembers", "Passthrough(key1:key,key2:key,subkey:subkey)"); + CheckConstraint<TPassthroughConstraintNode>(exprRoot, "Collect", "Passthrough(p.key1:key,p.key2:key,p.subkey:subkey,value:value)"); + } + + Y_UNIT_TEST(PassthroughExOverWideFlow) { + const auto s = R"(( + (let mr_sink (DataSink 'yt (quote plato))) + (let list (AsList + (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value (String 'v))) + (AsStruct '('key (String '1)) '('subkey (String 'd)) '('value (String 'v))) + (AsStruct '('key (String '3)) '('subkey (String 'b)) '('value (String 'v))) + )) + (let list (FlatMap list (lambda '(item) (block '( + (let res (ExpandMap (ToFlow (Just item)) (lambda '(m) + (Member m 'key) + (Member m 'key) + (Member m 'subkey) + (Member m 'value) + ))) + (let res (WideMap res (lambda '(m0 m1 m2 m3) (AsStruct '('xxx m0) '('yyy m2)) '(m1 m3)))) + (let res (WideMap res (lambda '(s0 t1) '((Member s0 'xxx) (Nth t1 '1)) (ReplaceMember s0 'xxx (Nth t1 '0))))) + (let res (NarrowMap res (lambda '(t0 s1) (AddMember (AddMember s1 'one (Nth t0 '1)) 'two (Nth t0 '0))))) + (return (Just (Unwrap (Head (Collect res))))) + ))))) + (let world (Write! world mr_sink (Key '('table (String 'Output))) list '('('mode 'renew)))) + (let world (Commit! world mr_sink)) + (return world) +))"; + + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TPassthroughConstraintNode>(exprRoot, "Unwrap", "Passthrough(one:value,two:key,xxx:key,yyy:subkey)"); + } + + Y_UNIT_TEST(PassthroughCondense1) { + const auto s = R"(( + (let mr_sink (DataSink 'yt (quote plato))) + (let list (AsList + (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value (String 'v))) + (AsStruct '('key (String '1)) '('subkey (String 'd)) '('value (String 'v))) + (AsStruct '('key (String '3)) '('subkey (String 'b)) '('value (String 'v))) + )) + (let list (FlatMap list (lambda '(item) (block '( + (let res (Condense1 (ToFlow (Just item)) + (lambda '(m) + (AsStruct + '('p.key1 (Member m 'key)) + '('p.key2 (Member m 'key)) + '('p.subkey (Member m 'subkey)) + '('value (Member m 'value)) + ) + ) + (lambda '(i s) (Bool 'false)) + (lambda '(i s) + (AsStruct + '('p.key1 (Member i 'key)) + '('p.key2 (Member s 'p.key2)) + '('p.subkey (Member i 'subkey)) + '('value (Member s 'value)) + ) + ) + )) + (return (Just (DivePrefixMembers (Unwrap (Last (Collect res))) '('p.)))) + ))))) + (let world (Write! world mr_sink (Key '('table (String 'Output))) list '('('mode 'renew)))) + (let world (Commit! world mr_sink)) + (return world) +))"; + + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TPassthroughConstraintNode>(exprRoot, "DivePrefixMembers", "Passthrough(key1:key,key2:key,subkey:subkey)"); + CheckConstraint<TPassthroughConstraintNode>(exprRoot, "Collect", "Passthrough(p.key1:key,p.key2:key,p.subkey:subkey,value:value)"); + } + + Y_UNIT_TEST(PassthroughWideCondense1) { + const auto s = R"(( + (let mr_sink (DataSink 'yt (quote plato))) + (let list (AsList + (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value (String 'v))) + (AsStruct '('key (String '1)) '('subkey (String 'd)) '('value (String 'v))) + (AsStruct '('key (String '3)) '('subkey (String 'b)) '('value (String 'v))) + )) + (let list (FlatMap list (lambda '(item) (block '( + (let res (ExpandMap (ToFlow (Just item)) (lambda '(m) + (Member m 'key) + (Member m 'key) + (Member m 'subkey) + (Member m 'value) + ))) + (let res (WideCondense1 res + (lambda '(m0 m1 m2 m3) m0 m1 m2 m3) + (lambda '(i0 i1 i2 i3 s0 s1 s2 s3) (Bool 'false)) + (lambda '(i0 i1 i2 i3 s0 s1 s2 s3) i0 s1 i2 s3) + )) + (let res (NarrowMap res (lambda '(m0 m1 m2 m3) + (AsStruct + '('p.key1 m0) + '('p.key2 m1) + '('p.subkey m2) + '('value m3) + ) + ))) + (return (Just (DivePrefixMembers (Unwrap (Head (Collect res))) '('p.)))) + ))))) + (let world (Write! world mr_sink (Key '('table (String 'Output))) list '('('mode 'renew)))) + (let world (Commit! world mr_sink)) + (return world) +))"; + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TPassthroughConstraintNode>(exprRoot, "DivePrefixMembers", "Passthrough(key1:key,key2:key,subkey:subkey)"); + CheckConstraint<TPassthroughConstraintNode>(exprRoot, "Collect", "Passthrough(p.key1:key,p.key2:key,p.subkey:subkey,value:value)"); + } + + Y_UNIT_TEST(Visit) { + const auto s = R"(( +(let mr_sink (DataSink 'yt (quote plato))) + +(let list (AsList + (AsStruct '('key (String 'aaa))) + (AsStruct '('key (String 'bbb))) + (AsStruct '('key (String 'ccc))) +)) + +(let structType (StructType '('key (DataType 'String)) '('value (DataType 'String)))) +(let tupleType (TupleType structType structType structType)) +(let vt (VariantType tupleType)) +(let vlist (Extend + (Map list (lambda '(x) (Variant (AsStruct '('key (Member x 'key)) '('value (String '0))) '0 vt))) + (Map list (lambda '(x) (Variant (AsStruct '('key (Member x 'key)) '('value (String '1))) '1 vt))) + (Map list (lambda '(x) (Variant (AsStruct '('key (Member x 'key)) '('value (String '2))) '2 vt))) +)) + +(let vlist (FlatMap vlist (lambda '(item) (Visit item + '0 (lambda '(v) (Just (Variant (AsStruct '('key (Member v 'key)) '('value (Member v 'value))) '1 vt))) + (Nothing (OptionalType vt)) +)))) + +(let res (Map vlist (lambda '(item) (VariantItem item)))) + +(let world (Write! world mr_sink (Key '('table (String 'Output))) res '('('mode 'renew)))) +(let world (Commit! world mr_sink)) +(return world) +))"; + + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TMultiConstraintNode>(exprRoot, "Extend", "Multi(0:{},1:{},2:{})"); + CheckConstraint<TMultiConstraintNode>(exprRoot, "Visit", "Multi(1:{Passthrough(key:key,value:value)})"); + CheckConstraint<TVarIndexConstraintNode>(exprRoot, "Visit", "VarIndex(1:0)"); + CheckConstraint<TMultiConstraintNode>(exprRoot, "FlatMap", "Multi(1:{})"); + CheckConstraint<TMultiConstraintNode>(exprRoot, "VariantItem", ""); + CheckConstraint<TMultiConstraintNode>(exprRoot, "Map", ""); + } + + Y_UNIT_TEST(SwitchAsFilter) { + const auto s = R"(( +(let mr_sink (DataSink 'yt (quote plato))) + +(let list (AsList + (AsStruct '('key (String 'aaa))) + (AsStruct '('key (String 'bbb))) + (AsStruct '('key (String 'ccc))) +)) + +(let structType (StructType '('key (DataType 'String)) '('value (DataType 'String)))) +(let tupleType (TupleType structType structType structType)) +(let vt (VariantType tupleType)) +(let vlist (Extend + (Map list (lambda '(x) (Variant (AsStruct '('key (Member x 'key)) '('value (String '0))) '0 vt))) + (Map list (lambda '(x) (Variant (AsStruct '('key (Member x 'key)) '('value (String '1))) '1 vt))) + (Map list (lambda '(x) (Variant (AsStruct '('key (Member x 'key)) '('value (String '2))) '2 vt))) +)) + +(let res (Switch (Iterator vlist) '0 '('0) (lambda '(item) item))) + +(let world (Write! world mr_sink (Key '('table (String 'Output))) res '('('mode 'renew)))) +(let world (Commit! world mr_sink)) +(return world) +))"; + + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TMultiConstraintNode>(exprRoot, "Extend", "Multi(0:{},1:{},2:{})"); + CheckConstraint<TMultiConstraintNode>(exprRoot, "Switch", ""); + } + + Y_UNIT_TEST(EmptyForMissingMultiOut) { + const auto s = R"(( +(let mr_sink (DataSink 'yt (quote plato))) + +(let list (AsList + (AsStruct '('key (String 'aaa))) + (AsStruct '('key (String 'bbb))) + (AsStruct '('key (String 'ccc))) +)) + +(let structType (StructType '('key (DataType 'String)) '('value (DataType 'String)))) +(let tupleType3 (TupleType structType structType structType)) +(let vt3 (VariantType tupleType3)) +(let tupleType2 (TupleType structType structType)) +(let vt2 (VariantType tupleType2)) +(let vlist (Extend + (Map list (lambda '(x) (Variant (AsStruct '('key (Member x 'key)) '('value (String '0))) '0 vt3))) + (Map list (lambda '(x) (Variant (AsStruct '('key (Member x 'key)) '('value (String '1))) '1 vt3))) +)) + +(let vlist (FlatMap vlist (lambda '(item) (Visit item + '0 (lambda '(v) (Just (Variant v '0 vt2))) + '1 (lambda '(v) (Just (Variant v '0 vt2))) + '2 (lambda '(v) (Just (Variant v '1 vt2))) +)))) + +(let res (Map vlist (lambda '(item) (VariantItem item)))) + +(let world (Write! world mr_sink (Key '('table (String 'Output))) res '('('mode 'renew)))) +(let world (Commit! world mr_sink)) +(return world) +))"; + + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TMultiConstraintNode>(exprRoot, "Extend", "Multi(0:{},1:{})"); + CheckConstraint<TMultiConstraintNode>(exprRoot, "FlatMap", "Multi(0:{})"); + } + + Y_UNIT_TEST(SwitchWithReplicate) { + const auto s = R"(( +(let mr_sink (DataSink 'yt (quote plato))) + +(let list (AsList + (AsStruct '('key (String 'aaa))) + (AsStruct '('key (String 'bbb))) + (AsStruct '('key (String 'ccc))) +)) + +(let vlist (Switch (Iterator list) '0 '('0) (lambda '(item) item) '('0) (lambda '(item) item))) + +(let res (Map vlist (lambda '(item) (VariantItem item)))) + +(let world (Write! world mr_sink (Key '('table (String 'Output))) res '('('mode 'renew)))) +(let world (Commit! world mr_sink)) +(return world) +))"; + + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TMultiConstraintNode>(exprRoot, "Switch", "Multi(0:{},1:{})"); + CheckConstraint<TMultiConstraintNode>(exprRoot, "Map", ""); + } + + Y_UNIT_TEST(SwitchWithMultiOut) { + const auto s = R"(( +(let mr_sink (DataSink 'yt (quote plato))) + +(let list (AsList + (AsStruct '('key (String 'aaa))) + (AsStruct '('key (String 'bbb))) + (AsStruct '('key (String 'ccc))) +)) + +(let structType (StructType '('key (DataType 'String)) '('value (DataType 'String)))) +(let tupleType (TupleType structType structType structType)) +(let vt (VariantType tupleType)) +(let vlist (Extend + (Map list (lambda '(x) (Variant (AsStruct '('key (Member x 'key)) '('value (String '0))) '0 vt))) + (Map list (lambda '(x) (Variant (AsStruct '('key (Member x 'key)) '('value (String '1))) '1 vt))) + (Map list (lambda '(x) (Variant (AsStruct '('key (Member x 'key)) '('value (String '2))) '2 vt))) +)) + +(let vlist (Switch (Iterator vlist) '0 '('0) (lambda '(s) (Map s (lambda '(item) (Variant item '1 vt)))) '('1) (lambda '(s) s))) + +(let res (Map vlist (lambda '(item) (VariantItem item)))) + +(let world (Write! world mr_sink (Key '('table (String 'Output))) res '('('mode 'renew)))) +(let world (Commit! world mr_sink)) +(return world) +))"; + + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TMultiConstraintNode>(exprRoot, "Extend", "Multi(0:{},1:{},2:{})"); + CheckConstraint<TMultiConstraintNode>(exprRoot, "Switch", "Multi(1:{},3:{})"); + } + + Y_UNIT_TEST(MultiOutLMapWithEmptyInput) { + auto s = R"(( +(let mr_sink (DataSink 'yt (quote plato))) + +(let structType (StructType '('key (DataType 'String)) '('value (DataType 'String)))) +(let tupleType (TupleType structType structType structType)) +(let vt (VariantType tupleType)) + +(let vlist (LMap (List (ListType structType)) (lambda '(stream) + (FlatMap stream (lambda '(item) + (Extend (AsList (Variant item '0 vt)) (AsList (Variant item '1 vt))) + )) +))) + +(let vlist (Switch (Iterator vlist) '0 '('0) (lambda '(s) s) '('1) (lambda '(s) s))) + +(let res (Map vlist (lambda '(item) (VariantItem item)))) + +(let world (Write! world mr_sink (Key '('table (String 'Output))) res '('('mode 'renew)))) +(let world (Commit! world mr_sink)) +(return world) +))"; + + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TEmptyConstraintNode>(exprRoot, "LMap", "Empty"); + CheckConstraint<TMultiConstraintNode>(exprRoot, "LMap", ""); + CheckConstraint<TEmptyConstraintNode>(exprRoot, "Switch", "Empty"); + CheckConstraint<TMultiConstraintNode>(exprRoot, "Switch", ""); + } + + Y_UNIT_TEST(Unique) { + const auto s = R"(( + (let mr_sink (DataSink 'yt (quote plato))) + (let list (AsList + (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value (String 'x))) + (AsStruct '('key (String '1)) '('subkey (String 'b)) '('value (String 'y))) + (AsStruct '('key (String '4)) '('subkey (String 'b)) '('value (String 'z))) + )) + (let list (AssumeUnique list '('key 'subkey) '('value))) + (let list (FlatMap list (lambda '(item) (block '( + (let res (Map (Just item) (lambda '(m) + (AsStruct + '('key1 (Member m 'key)) + '('key2 (Member m 'key)) + '('subkey (Member m 'subkey)) + '('value (Member m 'value)) + ) + ))) + (let res (Map res (lambda '(m) + (AsStruct + '('p.key1 (Member m 'key1)) + '('p.key2 (Member m 'key2)) + '('p.subkey (Member m 'subkey)) + '('value (Member m 'value)) + ) + ))) + (return res) + ))))) + (let world (Write! world mr_sink (Key '('table (String 'Output))) (Take list (Uint64 '2)) '('('mode 'renew)))) + (let world (Commit! world mr_sink)) + (return world) +))"; + + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TUniqueConstraintNode>(exprRoot, "Take", "Unique(({p.key1,p.key2},p.subkey)(value))"); + } + + Y_UNIT_TEST(UniqueOverTuple) { + const auto s = R"(( + (let mr_sink (DataSink 'yt (quote plato))) + (let list (AsList + (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value (String 'm))) + (AsStruct '('key (String '1)) '('subkey (String 'b)) '('value (String 'm))) + (AsStruct '('key (String '3)) '('subkey (String 'b)) '('value (String 'n))) + )) + (let list (AssumeUnique list '('key) '('subkey 'value))) + (let list (FlatMap list (lambda '(item) (block '( + (let res (Map (Just item) (lambda '(m) + '( + (Member m 'key) + (Member m 'key) + (Member m 'subkey) + (Member m 'value) + ) + ))) + (let res (Map res (lambda '(m) + (AsStruct + '('p.key1 (Nth m '0)) + '('p.key2 (Nth m '1)) + '('p.subkey (Nth m '2)) + '('value (Nth m '3)) + ) + ))) + (return (Just (DivePrefixMembers (Unwrap res) '('p.)))) + ))))) + (let world (Write! world mr_sink (Key '('table (String 'Output))) list '('('mode 'renew)))) + (let world (Commit! world mr_sink)) + (return world) +))"; + + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TPartOfUniqueConstraintNode>(exprRoot, "Unwrap", "PartOfUnique(p.key1:key,p.key2:key,p.subkey:subkey,value:value)"); + CheckConstraint<TPartOfUniqueConstraintNode>(exprRoot, "DivePrefixMembers", "PartOfUnique(key1:key,key2:key,subkey:subkey)"); + CheckConstraint<TUniqueConstraintNode>(exprRoot, "FlatMap", "Unique(({key1,key2}))"); + } + + Y_UNIT_TEST(UniqueOverWideFlow) { + const auto s = R"(( + (let mr_sink (DataSink 'yt (quote plato))) + (let list (AsList + (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value (String 'x))) + (AsStruct '('key (String '1)) '('subkey (String 'b)) '('value (String 'y))) + (AsStruct '('key (String '4)) '('subkey (String 'b)) '('value (String 'z))) + )) + (let list (AssumeUnique list '('key 'subkey) '('value))) + (let list (FlatMap list (lambda '(item) (block '( + (let res (ExpandMap (ToFlow (Just item)) (lambda '(m) + (Member m 'key) + (Member m 'key) + (Member m 'subkey) + (Member m 'value) + ))) + (let res (WideMap res (lambda '(m0 m1 m2 m3) m0 m2 m3 m1))) + (let res (NarrowMap res (lambda '(m0 m1 m2 m3) + (AsStruct + '('p.key1 m0) + '('p.key2 m3) + '('p.subkey m1) + '('value m2) + ) + ))) + (return (Map res (lambda '(row) (DivePrefixMembers row '('p.))))) + ))))) + (let world (Write! world mr_sink (Key '('table (String 'Output))) list '('('mode 'renew)))) + (let world (Commit! world mr_sink)) + (return world) +))"; + + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TPartOfUniqueConstraintNode>(exprRoot, "NarrowMap", "PartOfUnique(p.key1:key,p.key2:key,p.subkey:subkey,value:value)"); + CheckConstraint<TPartOfUniqueConstraintNode>(exprRoot, "Map", "PartOfUnique(key1:key,key2:key,subkey:subkey)"); + CheckConstraint<TUniqueConstraintNode>(exprRoot, "FlatMap", "Unique(({key1,key2},subkey))"); + } + + Y_UNIT_TEST(UniqueExOverWideFlow) { + const auto s = R"(( + (let mr_sink (DataSink 'yt (quote plato))) + (let list (AsList + (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value (String 'v))) + (AsStruct '('key (String '1)) '('subkey (String 'd)) '('value (String 'v))) + (AsStruct '('key (String '3)) '('subkey (String 'b)) '('value (String 'v))) + )) + (let list (AssumeUnique list)) + (let list (FlatMap list (lambda '(item) (block '( + (let res (ExpandMap (ToFlow (Just item)) (lambda '(m) + (Member m 'key) + (Member m 'key) + (Member m 'subkey) + (Member m 'value) + ))) + (let res (WideMap res (lambda '(m0 m1 m2 m3) (AsStruct '('xxx m0) '('yyy m2)) '(m1 m3)))) + (let res (WideMap res (lambda '(s0 t1) '((Member s0 'xxx) (Nth t1 '1)) (ReplaceMember s0 'xxx (Nth t1 '0))))) + (let res (NarrowMap res (lambda '(t0 s1) (AddMember (AddMember s1 'one (Nth t0 '1)) 'two (Nth t0 '0))))) + (return (Collect res)) + ))))) + (let world (Write! world mr_sink (Key '('table (String 'Output))) (LazyList list) '('('mode 'renew)))) + (let world (Commit! world mr_sink)) + (return world) +))"; + + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TUniqueConstraintNode>(exprRoot, "LazyList", "Unique((one,{two,xxx},yyy))"); + } + + Y_UNIT_TEST(Distinct) { + const auto s = R"(( + (let mr_sink (DataSink 'yt (quote plato))) + (let list (AsList + (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value (String 'x))) + (AsStruct '('key (String '1)) '('subkey (String 'b)) '('value (String 'y))) + (AsStruct '('key (String '4)) '('subkey (String 'b)) '('value (String 'z))) + )) + (let list (AssumeDistinct list '('key 'subkey) '('value))) + (let list (FlatMap list (lambda '(item) (block '( + (let res (Map (Just item) (lambda '(m) + (AsStruct + '('key1 (Member m 'key)) + '('key2 (Member m 'key)) + '('subkey (Member m 'subkey)) + '('value (Member m 'value)) + ) + ))) + (let res (Map res (lambda '(m) + (AsStruct + '('p.key1 (Member m 'key1)) + '('p.key2 (Member m 'key2)) + '('p.subkey (Member m 'subkey)) + '('value (Member m 'value)) + ) + ))) + (return res) + ))))) + (let world (Write! world mr_sink (Key '('table (String 'Output))) (Take list (Uint64 '2)) '('('mode 'renew)))) + (let world (Commit! world mr_sink)) + (return world) +))"; + + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TDistinctConstraintNode>(exprRoot, "Take", "Distinct(({p.key1,p.key2},p.subkey)(value))"); + } + + Y_UNIT_TEST(DistinctOverTuple) { + const auto s = R"(( + (let mr_sink (DataSink 'yt (quote plato))) + (let list (AsList + (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value (String 'm))) + (AsStruct '('key (String '1)) '('subkey (String 'b)) '('value (String 'm))) + (AsStruct '('key (String '3)) '('subkey (String 'b)) '('value (String 'n))) + )) + (let list (AssumeDistinct list '('key) '('subkey 'value))) + (let list (FlatMap list (lambda '(item) (block '( + (let res (Map (Just item) (lambda '(m) + '( + (Member m 'key) + (Member m 'key) + (Member m 'subkey) + (Member m 'value) + ) + ))) + (let res (Map res (lambda '(m) + (AsStruct + '('p.key1 (Nth m '0)) + '('p.key2 (Nth m '1)) + '('p.subkey (Nth m '2)) + '('value (Nth m '3)) + ) + ))) + (return (Just (DivePrefixMembers (Unwrap res) '('p.)))) + ))))) + (let world (Write! world mr_sink (Key '('table (String 'Output))) list '('('mode 'renew)))) + (let world (Commit! world mr_sink)) + (return world) +))"; + + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TPartOfDistinctConstraintNode>(exprRoot, "Unwrap", "PartOfDistinct(p.key1:key,p.key2:key,p.subkey:subkey,value:value)"); + CheckConstraint<TPartOfDistinctConstraintNode>(exprRoot, "DivePrefixMembers", "PartOfDistinct(key1:key,key2:key,subkey:subkey)"); + CheckConstraint<TDistinctConstraintNode>(exprRoot, "FlatMap", "Distinct(({key1,key2}))"); + } + + Y_UNIT_TEST(DistinctOverWideFlow) { + const auto s = R"(( + (let mr_sink (DataSink 'yt (quote plato))) + (let list (AsList + (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value (String 'x))) + (AsStruct '('key (String '1)) '('subkey (String 'b)) '('value (String 'y))) + (AsStruct '('key (String '4)) '('subkey (String 'b)) '('value (String 'z))) + )) + (let list (AssumeDistinct list '('key 'subkey) '('value))) + (let list (FlatMap list (lambda '(item) (block '( + (let res (ExpandMap (ToFlow (Just item)) (lambda '(m) + (Member m 'key) + (Member m 'key) + (Member m 'subkey) + (Member m 'value) + ))) + (let res (WideMap res (lambda '(m0 m1 m2 m3) m0 m2 m3 m1))) + (let res (NarrowMap res (lambda '(m0 m1 m2 m3) + (AsStruct + '('p.key1 m0) + '('p.key2 m3) + '('p.subkey m1) + '('value m2) + ) + ))) + (return (Map res (lambda '(row) (DivePrefixMembers row '('p.))))) + ))))) + (let world (Write! world mr_sink (Key '('table (String 'Output))) list '('('mode 'renew)))) + (let world (Commit! world mr_sink)) + (return world) +))"; + + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TPartOfDistinctConstraintNode>(exprRoot, "NarrowMap", "PartOfDistinct(p.key1:key,p.key2:key,p.subkey:subkey,value:value)"); + CheckConstraint<TPartOfDistinctConstraintNode>(exprRoot, "Map", "PartOfDistinct(key1:key,key2:key,subkey:subkey)"); + CheckConstraint<TDistinctConstraintNode>(exprRoot, "FlatMap", "Distinct(({key1,key2},subkey))"); + } + + Y_UNIT_TEST(DistinctExOverWideFlow) { + const auto s = R"(( + (let mr_sink (DataSink 'yt (quote plato))) + (let list (AsList + (AsStruct '('key (String '4)) '('subkey (String 'c)) '('value (String 'v))) + (AsStruct '('key (String '1)) '('subkey (String 'd)) '('value (String 'v))) + (AsStruct '('key (String '3)) '('subkey (String 'b)) '('value (String 'v))) + )) + (let list (AssumeDistinct list)) + (let list (FlatMap list (lambda '(item) (block '( + (let res (ExpandMap (ToFlow (Just item)) (lambda '(m) + (Member m 'key) + (Member m 'key) + (Member m 'subkey) + (Member m 'value) + ))) + (let res (WideMap res (lambda '(m0 m1 m2 m3) (AsStruct '('xxx m0) '('yyy m2)) '(m1 m3)))) + (let res (WideMap res (lambda '(s0 t1) '((Member s0 'xxx) (Nth t1 '1)) (ReplaceMember s0 'xxx (Nth t1 '0))))) + (let res (NarrowMap res (lambda '(t0 s1) (AddMember (AddMember s1 'one (Nth t0 '1)) 'two (Nth t0 '0))))) + (return (Collect res)) + ))))) + (let world (Write! world mr_sink (Key '('table (String 'Output))) (LazyList list) '('('mode 'renew)))) + (let world (Commit! world mr_sink)) + (return world) +))"; + + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TDistinctConstraintNode>(exprRoot, "LazyList", "Distinct((one,{two,xxx},yyy))"); + } + + Y_UNIT_TEST(PartitionsByKeysWithCondense1) { + const auto s = R"( +( + (let mr_sink (DataSink 'yt (quote plato))) + (let list (AsList + (AsStruct '('key (Just (String '4))) '('subkey (Just (String 'c))) '('value (Just (String 'x)))) + (AsStruct '('key (Just (String '1))) '('subkey (Just (String 'b))) '('value (Just (String 'y)))) + (AsStruct '('key (Just (String '4))) '('subkey (Just (String 'b))) '('value (Just (String 'z)))) + )) + (let extractor (lambda '(item) '((Member item 'key) (Member item 'subkey)))) + (let aggr (PartitionsByKeys list extractor (Void) (Void) + (lambda '(stream) (Condense1 stream (lambda '(row) row) + (lambda '(row state) (IsKeySwitch row state extractor extractor)) + (lambda '(row state) (AsStruct '('key (Member row 'key)) '('subkey (Member row 'subkey)) '('value (Coalesce (Member row 'value) (Member state 'value))))) + )) + )) + (let world (Write! world mr_sink (Key '('table (String 'Output))) (Skip aggr (Uint64 '1)) '('('mode 'renew)))) + (let world (Commit! world mr_sink)) + (return world) +) + )"; + + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TUniqueConstraintNode>(exprRoot, "Skip", "Unique((key,subkey))"); + CheckConstraint<TDistinctConstraintNode>(exprRoot, "Skip", "Distinct((key,subkey))"); + } + + Y_UNIT_TEST(ShuffleByKeysInputUnique) { + const auto s = R"( +( + (let mr_sink (DataSink 'yt (quote plato))) + (let list (AsList + (AsStruct '('key (Just (String '4))) '('subkey (Just (String 'c))) '('value (Just (String 'x)))) + (AsStruct '('key (Just (String '1))) '('subkey (Just (String 'b))) '('value (Just (String 'y)))) + (AsStruct '('key (Just (String '4))) '('subkey (Just (String 'b))) '('value (Just (String 'z)))) + )) + (let list (AssumeUnique list '('key) '('subkey))) + (let list (AssumeDistinct list '('key 'subkey))) + (let aggr (ShuffleByKeys list + (lambda '(item) '((Member item 'key) (Member item 'subkey))) + (lambda '(stream) (Take stream (Uint64 '100))) + )) + (let world (Write! world mr_sink (Key '('table (String 'Output))) (Skip aggr (Uint64 '1)) '('('mode 'renew)))) + (let world (Commit! world mr_sink)) + (return world) +) + )"; + + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TUniqueConstraintNode>(exprRoot, "Skip", "Unique((key)(subkey))"); + CheckConstraint<TDistinctConstraintNode>(exprRoot, "Skip", "Distinct((key,subkey))"); + } + + Y_UNIT_TEST(ShuffleByKeysHandlerUnique) { + const auto s = R"( +( + (let mr_sink (DataSink 'yt (quote plato))) + (let list (AsList + (AsStruct '('key (Just (String '4))) '('subkey (Just (String 'c))) '('value (Just (String 'x)))) + (AsStruct '('key (Just (String '1))) '('subkey (Just (String 'b))) '('value (Just (String 'y)))) + (AsStruct '('key (Just (String '4))) '('subkey (Just (String 'b))) '('value (Just (String 'z)))) + )) + (let list (AssumeUnique list '('key) '('subkey))) + (let list (AssumeDistinct list '('key 'subkey))) + (let aggr (ShuffleByKeys list + (lambda '(item) '((Member item 'key) (Member item 'subkey))) + (lambda '(stream) (AssumeDistinct (AssumeUnique stream '('key) '('subkey)) '('key 'subkey))) + )) + (let world (Write! world mr_sink (Key '('table (String 'Output))) (Skip aggr (Uint64 '1)) '('('mode 'renew)))) + (let world (Commit! world mr_sink)) + (return world) +) + )"; + + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TUniqueConstraintNode>(exprRoot, "Skip", "Unique((key)(subkey))"); + CheckConstraint<TDistinctConstraintNode>(exprRoot, "Skip", "Distinct((key,subkey))"); + } + + Y_UNIT_TEST(Reverse) { + const auto s = R"( +( + (let mr_sink (DataSink 'yt (quote plato))) + (let list (AsList + (AsStruct '('key (Just (String '4))) '('subkey (Just (String 'c))) '('value (Just (String 'x)))) + (AsStruct '('key (Just (String '1))) '('subkey (Just (String 'b))) '('value (Just (String 'y)))) + (AsStruct '('key (Just (String '4))) '('subkey (Just (String 'b))) '('value (Just (String 'z)))) + )) + (let list (AssumeUnique list '('key) '('subkey))) + (let list (AssumeDistinct list '('key 'subkey))) + (let sorted (Sort list '((Bool 'False) (Bool 'True)) (lambda '(item) '((Member item 'key) (Member item 'subkey))))) + (let reverse (Reverse sorted)) + (let world (Write! world mr_sink (Key '('table (String 'Output))) reverse '('('mode 'renew)))) + (let world (Commit! world mr_sink)) + (return world) +) + )"; + + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TUniqueConstraintNode>(exprRoot, "Reverse", "Unique((key)(subkey))"); + CheckConstraint<TDistinctConstraintNode>(exprRoot, "Reverse", "Distinct((key,subkey))"); + CheckConstraint<TSortedConstraintNode>(exprRoot, "Reverse", "Sorted(key[asc];subkey[desc])"); + } + + Y_UNIT_TEST(DictItems) { + const auto s = R"( +( + (let mr_sink (DataSink 'yt (quote plato))) + (let list (AsList + (AsStruct '('key (Just (String '4))) '('subkey (Just (String 'c))) '('value (Just (String 'x)))) + (AsStruct '('key (Just (String '1))) '('subkey (Just (String 'b))) '('value (Just (String 'y)))) + (AsStruct '('key (Just (String '4))) '('subkey (Just (String 'b))) '('value (Just (String 'z)))) + )) + (let list (AssumeUnique list '('key) '('subkey))) + (let list (AssumeDistinct list '('key 'subkey))) + (let dict (ToDict list (lambda '(item) (Member item 'value)) (lambda '(item) (AsStruct '('k (Member item 'key)) '('s (Member item 'subkey)))) '('One 'Hashed))) + (let items (Map (DictItems dict) (lambda '(item) (AsStruct '('v (Nth item '0)) '('k (Member (Nth item '1) 'k)) '('s (Member (Nth item '1) 's)))))) + (let world (Write! world mr_sink (Key '('table (String 'Output))) items '('('mode 'renew)))) + (let world (Commit! world mr_sink)) + (return world) +) + )"; + + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TUniqueConstraintNode>(exprRoot, "Map", "Unique((k)(s)(v))"); + CheckConstraint<TDistinctConstraintNode>(exprRoot, "Map", "Distinct((k,s)(v))"); + } + + Y_UNIT_TEST(DictKeys) { + const auto s = R"( +( + (let mr_sink (DataSink 'yt (quote plato))) + (let list (AsList + (AsStruct '('key (Just (String '4))) '('subkey (Just (String 'c))) '('value (Just (String 'x)))) + (AsStruct '('key (Just (String '1))) '('subkey (Just (String 'b))) '('value (Just (String 'y)))) + (AsStruct '('key (Just (String '4))) '('subkey (Just (String 'b))) '('value (Just (String 'z)))) + )) + (let dict (ToDict list (lambda '(item) (Member item 'value)) (lambda '(item) (AsStruct '('k (Member item 'key)) '('s (Member item 'subkey)))) '('One 'Hashed))) + (let items (Map (DictKeys dict) (lambda '(row) (AsStruct '('v row))))) + (let world (Write! world mr_sink (Key '('table (String 'Output))) items '('('mode 'renew)))) + (let world (Commit! world mr_sink)) + (return world) +) + )"; + + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TUniqueConstraintNode>(exprRoot, "Map", "Unique((v))"); + CheckConstraint<TDistinctConstraintNode>(exprRoot, "Map", "Distinct((v))"); + } + + Y_UNIT_TEST(DictPayloads) { + const auto s = R"( +( + (let mr_sink (DataSink 'yt (quote plato))) + (let list (AsList + (AsStruct '('key (Just (String '4))) '('subkey (Just (String 'c))) '('value (Just (String 'x)))) + (AsStruct '('key (Just (String '1))) '('subkey (Just (String 'b))) '('value (Just (String 'y)))) + (AsStruct '('key (Just (String '4))) '('subkey (Just (String 'b))) '('value (Just (String 'z)))) + )) + (let list (AssumeUnique list '('key) '('subkey))) + (let list (AssumeDistinct list '('key 'subkey))) + (let dict (ToDict list (lambda '(item) (Member item 'value)) (lambda '(item) (AsStruct '('k (Member item 'key)) '('s (Member item 'subkey)))) '('One 'Hashed))) + (let items (DictPayloads dict)) + (let world (Write! world mr_sink (Key '('table (String 'Output))) items '('('mode 'renew)))) + (let world (Commit! world mr_sink)) + (return world) +) + )"; + + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TUniqueConstraintNode>(exprRoot, "DictPayloads", "Unique((k)(s))"); + CheckConstraint<TDistinctConstraintNode>(exprRoot, "DictPayloads", "Distinct((k,s))"); + } + + Y_UNIT_TEST(GraceJoinInner) { + const auto s = R"( +( + (let list1 (AsList + (AsStruct '('key1 (Int32 '1)) '('subkey1 (Uint8 '0)) '('value1 (String 'A))) + (AsStruct '('key1 (Int32 '7)) '('subkey1 (Uint8 '0)) '('value1 (String 'B))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '0)) '('value1 (String 'C))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '1)) '('value1 (String 'D))) + )) + + (let list1 (AssumeUnique list1 '('key1 'subkey1) '('value1))) + (let list1 (AssumeDistinct list1 '('key1 'subkey1) '('value1))) + + (let list2 (AsList + (AsStruct '('key2 (Int32 '9)) '('subkey2 (Uint8 '0)) '('value2 (String 'Z))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '0)) '('value2 (String 'Y))) + (AsStruct '('key2 (Int32 '3)) '('subkey2 (Uint8 '1)) '('value2 (String 'X))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '1)) '('value2 (String 'W))) + (AsStruct '('key2 (Int32 '8)) '('subkey2 (Uint8 '1)) '('value2 (String 'V))) + )) + + (let list2 (AssumeUnique list2 '('key2 'subkey2) '('value2))) + (let list2 (AssumeDistinct list2 '('key2 'subkey2) '('value2))) + + (let flow1 (ExpandMap (ToFlow list1) (lambda '(item) (Member item 'key1) (Member item 'subkey1) (Member item 'value1)))) + (let flow2 (ExpandMap (ToFlow list2) (lambda '(item) (Member item 'key2) (Member item 'subkey2) (Member item 'value2)))) + + (let join (GraceJoinCore flow1 flow2 'Inner '('0 '1) '('0 '1) '('0 '0 '2 '1) '('0 '2 '1 '3 '2 '4) '())) + (let list (Collect (NarrowMap join (lambda '(lk lv rk rs rv) (AsStruct '('lk lk) '('lv lv) '('rk rk) '('rs rs) '('rv rv)))))) + + (let res_sink (DataSink 'yt (quote plato))) + (let world (Write! world res_sink (Key '('table (String 'Output))) list '('('mode 'renew)))) + + (let world (Commit! world res_sink)) + (return world) +) + )"; + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TUniqueConstraintNode>(exprRoot, "Collect", "Unique((lv)(rk,rs)(rv))"); + CheckConstraint<TDistinctConstraintNode>(exprRoot, "Collect", "Distinct((lv)(rk,rs)(rv))"); + } + + Y_UNIT_TEST(GraceJoinLeft) { + const auto s = R"( +( + (let list1 (AsList + (AsStruct '('key1 (Int32 '1)) '('subkey1 (Uint8 '0)) '('value1 (String 'A))) + (AsStruct '('key1 (Int32 '7)) '('subkey1 (Uint8 '0)) '('value1 (String 'B))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '0)) '('value1 (String 'C))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '1)) '('value1 (String 'D))) + )) + + (let list1 (AssumeUnique list1 '('key1 'subkey1) '('value1))) + (let list1 (AssumeDistinct list1 '('key1 'subkey1) '('value1))) + + (let list2 (AsList + (AsStruct '('key2 (Int32 '9)) '('subkey2 (Uint8 '0)) '('value2 (String 'Z))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '0)) '('value2 (String 'Y))) + (AsStruct '('key2 (Int32 '3)) '('subkey2 (Uint8 '1)) '('value2 (String 'X))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '1)) '('value2 (String 'W))) + (AsStruct '('key2 (Int32 '8)) '('subkey2 (Uint8 '1)) '('value2 (String 'V))) + )) + + (let list2 (AssumeUnique list2 '('key2 'subkey2) '('value2))) + (let list2 (AssumeDistinct list2 '('key2 'subkey2) '('value2))) + + (let flow1 (ExpandMap (ToFlow list1) (lambda '(item) (Member item 'key1) (Member item 'subkey1) (Member item 'value1)))) + (let flow2 (ExpandMap (ToFlow list2) (lambda '(item) (Member item 'key2) (Member item 'subkey2) (Member item 'value2)))) + + (let join (GraceJoinCore flow1 flow2 'Left '('0 '1) '('0 '1) '('0 '0 '2 '1) '('0 '2 '1 '3 '2 '4) '())) + (let list (Collect (NarrowMap join (lambda '(lk lv rk rs rv) (AsStruct '('lk lk) '('lv lv) '('rk rk) '('rs rs) '('rv rv)))))) + + (let res_sink (DataSink 'yt (quote plato))) + (let world (Write! world res_sink (Key '('table (String 'Output))) list '('('mode 'renew)))) + + (let world (Commit! world res_sink)) + (return world) +) + )"; + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TUniqueConstraintNode>(exprRoot, "Collect", "Unique((lv)(rk,rs)(rv))"); + CheckConstraint<TDistinctConstraintNode>(exprRoot, "Collect", "Distinct((lv))"); + } + + Y_UNIT_TEST(GraceJoinFull) { + const auto s = R"( +( + (let list1 (AsList + (AsStruct '('key1 (Int32 '1)) '('subkey1 (Uint8 '0)) '('value1 (String 'A))) + (AsStruct '('key1 (Int32 '7)) '('subkey1 (Uint8 '0)) '('value1 (String 'B))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '0)) '('value1 (String 'C))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '1)) '('value1 (String 'D))) + )) + + (let list1 (AssumeUnique list1 '('key1 'subkey1) '('value1))) + (let list1 (AssumeDistinct list1 '('key1 'subkey1) '('value1))) + + (let list2 (AsList + (AsStruct '('key2 (Int32 '9)) '('subkey2 (Uint8 '0)) '('value2 (String 'Z))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '0)) '('value2 (String 'Y))) + (AsStruct '('key2 (Int32 '3)) '('subkey2 (Uint8 '1)) '('value2 (String 'X))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '1)) '('value2 (String 'W))) + (AsStruct '('key2 (Int32 '8)) '('subkey2 (Uint8 '1)) '('value2 (String 'V))) + )) + + (let list2 (AssumeUnique list2 '('key2 'subkey2) '('value2))) + (let list2 (AssumeDistinct list2 '('key2 'subkey2) '('value2))) + + (let flow1 (ExpandMap (ToFlow list1) (lambda '(item) (Member item 'key1) (Member item 'subkey1) (Member item 'value1)))) + (let flow2 (ExpandMap (ToFlow list2) (lambda '(item) (Member item 'key2) (Member item 'subkey2) (Member item 'value2)))) + + (let join (GraceJoinCore flow1 flow2 'Full '('0 '1) '('0 '1) '('0 '0 '2 '1) '('0 '2 '1 '3 '2 '4) '())) + (let list (Collect (NarrowMap join (lambda '(lk lv rk rs rv) (AsStruct '('lk lk) '('lv lv) '('rk rk) '('rs rs) '('rv rv)))))) + + (let res_sink (DataSink 'yt (quote plato))) + (let world (Write! world res_sink (Key '('table (String 'Output))) list '('('mode 'renew)))) + + (let world (Commit! world res_sink)) + (return world) +) + )"; + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TUniqueConstraintNode>(exprRoot, "Collect", "Unique((lv)(rk,rs)(rv))"); + CheckConstraint<TDistinctConstraintNode>(exprRoot, "Collect", ""); + } + + Y_UNIT_TEST(GraceJoinRight) { + const auto s = R"( +( + (let list1 (AsList + (AsStruct '('key1 (Int32 '1)) '('subkey1 (Uint8 '0)) '('value1 (String 'A))) + (AsStruct '('key1 (Int32 '7)) '('subkey1 (Uint8 '0)) '('value1 (String 'B))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '0)) '('value1 (String 'C))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '1)) '('value1 (String 'D))) + )) + + (let list1 (AssumeUnique list1 '('key1 'subkey1) '('value1))) + (let list1 (AssumeDistinct list1 '('key1 'subkey1) '('value1))) + + (let list2 (AsList + (AsStruct '('key2 (Int32 '9)) '('subkey2 (Uint8 '0)) '('value2 (String 'Z))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '0)) '('value2 (String 'Y))) + (AsStruct '('key2 (Int32 '3)) '('subkey2 (Uint8 '1)) '('value2 (String 'X))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '1)) '('value2 (String 'W))) + (AsStruct '('key2 (Int32 '8)) '('subkey2 (Uint8 '1)) '('value2 (String 'V))) + )) + + (let list2 (AssumeUnique list2 '('key2 'subkey2) '('value2))) + (let list2 (AssumeDistinct list2 '('key2 'subkey2) '('value2))) + + (let flow1 (ExpandMap (ToFlow list1) (lambda '(item) (Member item 'key1) (Member item 'subkey1) (Member item 'value1)))) + (let flow2 (ExpandMap (ToFlow list2) (lambda '(item) (Member item 'key2) (Member item 'subkey2) (Member item 'value2)))) + + (let join (GraceJoinCore flow1 flow2 'Right '('0 '1) '('0 '1) '('0 '0 '2 '1) '('0 '2 '1 '3 '2 '4) '())) + (let list (Collect (NarrowMap join (lambda '(lk lv rk rs rv) (AsStruct '('lk lk) '('lv lv) '('rk rk) '('rs rs) '('rv rv)))))) + + (let res_sink (DataSink 'yt (quote plato))) + (let world (Write! world res_sink (Key '('table (String 'Output))) list '('('mode 'renew)))) + + (let world (Commit! world res_sink)) + (return world) +) + )"; + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TUniqueConstraintNode>(exprRoot, "Collect", "Unique((lv)(rk,rs)(rv))"); + CheckConstraint<TDistinctConstraintNode>(exprRoot, "Collect", "Distinct((rk,rs)(rv))"); + } + + Y_UNIT_TEST(GraceJoinExclusion) { + const auto s = R"( +( + (let list1 (AsList + (AsStruct '('key1 (Int32 '1)) '('subkey1 (Uint8 '0)) '('value1 (String 'A))) + (AsStruct '('key1 (Int32 '7)) '('subkey1 (Uint8 '0)) '('value1 (String 'B))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '0)) '('value1 (String 'C))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '1)) '('value1 (String 'D))) + )) + + (let list1 (AssumeUnique list1 '('key1 'subkey1) '('value1))) + (let list1 (AssumeDistinct list1 '('key1 'subkey1) '('value1))) + + (let list2 (AsList + (AsStruct '('key2 (Int32 '9)) '('subkey2 (Uint8 '0)) '('value2 (String 'Z))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '0)) '('value2 (String 'Y))) + (AsStruct '('key2 (Int32 '3)) '('subkey2 (Uint8 '1)) '('value2 (String 'X))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '1)) '('value2 (String 'W))) + (AsStruct '('key2 (Int32 '8)) '('subkey2 (Uint8 '1)) '('value2 (String 'V))) + )) + + (let list2 (AssumeUnique list2 '('key2 'subkey2) '('value2))) + (let list2 (AssumeDistinct list2 '('key2 'subkey2) '('value2))) + + (let flow1 (ExpandMap (ToFlow list1) (lambda '(item) (Member item 'key1) (Member item 'subkey1) (Member item 'value1)))) + (let flow2 (ExpandMap (ToFlow list2) (lambda '(item) (Member item 'key2) (Member item 'subkey2) (Member item 'value2)))) + + (let join (GraceJoinCore flow1 flow2 'Exclusion '('0 '1) '('0 '1) '('0 '0 '2 '1) '('0 '2 '1 '3 '2 '4) '())) + (let list (Collect (NarrowMap join (lambda '(lk lv rk rs rv) (AsStruct '('lk lk) '('lv lv) '('rk rk) '('rs rs) '('rv rv)))))) + + (let res_sink (DataSink 'yt (quote plato))) + (let world (Write! world res_sink (Key '('table (String 'Output))) list '('('mode 'renew)))) + + (let world (Commit! world res_sink)) + (return world) +) + )"; + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TUniqueConstraintNode>(exprRoot, "Collect", "Unique((lv)(rk,rs)(rv))"); + CheckConstraint<TDistinctConstraintNode>(exprRoot, "Collect", ""); + } + + Y_UNIT_TEST(GraceJoinLeftSemi) { + const auto s = R"( +( + (let list1 (AsList + (AsStruct '('key1 (Int32 '1)) '('subkey1 (Uint8 '0)) '('value1 (String 'A))) + (AsStruct '('key1 (Int32 '7)) '('subkey1 (Uint8 '0)) '('value1 (String 'B))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '0)) '('value1 (String 'C))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '1)) '('value1 (String 'D))) + )) + + (let list1 (AssumeUnique list1 '('key1 'subkey1) '('value1))) + (let list1 (AssumeDistinct list1 '('key1 'subkey1) '('value1))) + + (let list2 (AsList + (AsStruct '('key2 (Int32 '9)) '('subkey2 (Uint8 '0)) '('value2 (String 'Z))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '0)) '('value2 (String 'Y))) + (AsStruct '('key2 (Int32 '3)) '('subkey2 (Uint8 '1)) '('value2 (String 'X))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '1)) '('value2 (String 'W))) + (AsStruct '('key2 (Int32 '8)) '('subkey2 (Uint8 '1)) '('value2 (String 'V))) + )) + + (let list2 (AssumeUnique list2 '('key2 'subkey2) '('value2))) + (let list2 (AssumeDistinct list2 '('key2 'subkey2) '('value2))) + + (let flow1 (ExpandMap (ToFlow list1) (lambda '(item) (Member item 'key1) (Member item 'subkey1) (Member item 'value1)))) + (let flow2 (ExpandMap (ToFlow list2) (lambda '(item) (Member item 'key2) (Member item 'subkey2) (Member item 'value2)))) + + (let join (GraceJoinCore flow1 flow2 'LeftSemi '('0 '1) '('0 '1) '('0 '2 '1 '1 '2 '0) '() '())) + (let list (Collect (NarrowMap join (lambda '(lv ls lk) (AsStruct '('lk lk) '('lv lv) '('ls ls)))))) + + (let res_sink (DataSink 'yt (quote plato))) + (let world (Write! world res_sink (Key '('table (String 'Output))) list '('('mode 'renew)))) + + (let world (Commit! world res_sink)) + (return world) +) + )"; + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TUniqueConstraintNode>(exprRoot, "Collect", "Unique((lk,ls)(lv))"); + CheckConstraint<TDistinctConstraintNode>(exprRoot, "Collect", "Distinct((lk,ls)(lv))"); + } + + Y_UNIT_TEST(GraceJoinLeftOnly) { + const auto s = R"( +( + (let list1 (AsList + (AsStruct '('key1 (Int32 '1)) '('subkey1 (Uint8 '0)) '('value1 (String 'A))) + (AsStruct '('key1 (Int32 '7)) '('subkey1 (Uint8 '0)) '('value1 (String 'B))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '0)) '('value1 (String 'C))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '1)) '('value1 (String 'D))) + )) + + (let list1 (AssumeUnique list1 '('key1 'subkey1) '('value1))) + (let list1 (AssumeDistinct list1 '('key1 'subkey1) '('value1))) + + (let list2 (AsList + (AsStruct '('key2 (Int32 '9)) '('subkey2 (Uint8 '0)) '('value2 (String 'Z))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '0)) '('value2 (String 'Y))) + (AsStruct '('key2 (Int32 '3)) '('subkey2 (Uint8 '1)) '('value2 (String 'X))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '1)) '('value2 (String 'W))) + (AsStruct '('key2 (Int32 '8)) '('subkey2 (Uint8 '1)) '('value2 (String 'V))) + )) + + (let list2 (AssumeUnique list2 '('key2 'subkey2) '('value2))) + (let list2 (AssumeDistinct list2 '('key2 'subkey2) '('value2))) + + (let flow1 (ExpandMap (ToFlow list1) (lambda '(item) (Member item 'key1) (Member item 'subkey1) (Member item 'value1)))) + (let flow2 (ExpandMap (ToFlow list2) (lambda '(item) (Member item 'key2) (Member item 'subkey2) (Member item 'value2)))) + + (let join (GraceJoinCore flow1 flow2 'LeftOnly '('0 '1) '('0 '1) '('0 '1 '2 '0) '() '())) + (let list (Collect (NarrowMap join (lambda '(lv lk) (AsStruct '('lk lk) '('lv lv)))))) + + (let res_sink (DataSink 'yt (quote plato))) + (let world (Write! world res_sink (Key '('table (String 'Output))) list '('('mode 'renew)))) + + (let world (Commit! world res_sink)) + (return world) +) + )"; + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TUniqueConstraintNode>(exprRoot, "Collect", "Unique((lv))"); + CheckConstraint<TDistinctConstraintNode>(exprRoot, "Collect", "Distinct((lv))"); + } + + + Y_UNIT_TEST(GraceJoinRightOnly) { + const auto s = R"( +( + (let list1 (AsList + (AsStruct '('key1 (Int32 '1)) '('subkey1 (Uint8 '0)) '('value1 (String 'A))) + (AsStruct '('key1 (Int32 '7)) '('subkey1 (Uint8 '0)) '('value1 (String 'B))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '0)) '('value1 (String 'C))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '1)) '('value1 (String 'D))) + )) + + (let list1 (AssumeUnique list1 '('key1 'subkey1) '('value1))) + (let list1 (AssumeDistinct list1 '('key1 'subkey1) '('value1))) + + (let list2 (AsList + (AsStruct '('key2 (Int32 '9)) '('subkey2 (Uint8 '0)) '('value2 (String 'Z))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '0)) '('value2 (String 'Y))) + (AsStruct '('key2 (Int32 '3)) '('subkey2 (Uint8 '1)) '('value2 (String 'X))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '1)) '('value2 (String 'W))) + (AsStruct '('key2 (Int32 '8)) '('subkey2 (Uint8 '1)) '('value2 (String 'V))) + )) + + (let list2 (AssumeUnique list2 '('key2 'subkey2) '('value2))) + (let list2 (AssumeDistinct list2 '('key2 'subkey2) '('value2))) + + (let flow1 (ExpandMap (ToFlow list1) (lambda '(item) (Member item 'key1) (Member item 'subkey1) (Member item 'value1)))) + (let flow2 (ExpandMap (ToFlow list2) (lambda '(item) (Member item 'key2) (Member item 'subkey2) (Member item 'value2)))) + + (let join (GraceJoinCore flow1 flow2 'RightOnly '('0 '1) '('0 '1) '() '('0 '2 '1 '1 '2 '0) '())) + (let list (Collect (NarrowMap join (lambda '(rv rs rk) (AsStruct '('rk rk) '('rv rv) '('rs rs)))))) + + (let res_sink (DataSink 'yt (quote plato))) + (let world (Write! world res_sink (Key '('table (String 'Output))) list '('('mode 'renew)))) + + (let world (Commit! world res_sink)) + (return world) +) + )"; + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TUniqueConstraintNode>(exprRoot, "Collect", "Unique((rk,rs)(rv))"); + CheckConstraint<TDistinctConstraintNode>(exprRoot, "Collect", "Distinct((rk,rs)(rv))"); + } + + Y_UNIT_TEST(GraceJoinRightSemi) { + const auto s = R"( +( + (let list1 (AsList + (AsStruct '('key1 (Int32 '1)) '('subkey1 (Uint8 '0)) '('value1 (String 'A))) + (AsStruct '('key1 (Int32 '7)) '('subkey1 (Uint8 '0)) '('value1 (String 'B))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '0)) '('value1 (String 'C))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '1)) '('value1 (String 'D))) + )) + + (let list1 (AssumeUnique list1 '('key1 'subkey1) '('value1))) + (let list1 (AssumeDistinct list1 '('key1 'subkey1) '('value1))) + + (let list2 (AsList + (AsStruct '('key2 (Int32 '9)) '('subkey2 (Uint8 '0)) '('value2 (String 'Z))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '0)) '('value2 (String 'Y))) + (AsStruct '('key2 (Int32 '3)) '('subkey2 (Uint8 '1)) '('value2 (String 'X))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '1)) '('value2 (String 'W))) + (AsStruct '('key2 (Int32 '8)) '('subkey2 (Uint8 '1)) '('value2 (String 'V))) + )) + + (let list2 (AssumeUnique list2 '('key2 'subkey2) '('value2))) + (let list2 (AssumeDistinct list2 '('key2 'subkey2) '('value2))) + + (let flow1 (ExpandMap (ToFlow list1) (lambda '(item) (Member item 'key1) (Member item 'subkey1) (Member item 'value1)))) + (let flow2 (ExpandMap (ToFlow list2) (lambda '(item) (Member item 'key2) (Member item 'subkey2) (Member item 'value2)))) + + (let join (GraceJoinCore flow1 flow2 'RightSemi '('0 '1) '('0 '1) '() '('0 '1 '2 '0) '())) + (let list (Collect (NarrowMap join (lambda '(rv rk) (AsStruct '('rk rk) '('rv rv)))))) + + (let res_sink (DataSink 'yt (quote plato))) + (let world (Write! world res_sink (Key '('table (String 'Output))) list '('('mode 'renew)))) + + (let world (Commit! world res_sink)) + (return world) +) + )"; + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TUniqueConstraintNode>(exprRoot, "Collect", "Unique((rv))"); + CheckConstraint<TDistinctConstraintNode>(exprRoot, "Collect", "Distinct((rv))"); + } + + Y_UNIT_TEST(GraceJoinInnerBothAny) { + const auto s = R"( +( + (let list1 (AsList + (AsStruct '('key1 (Int32 '1)) '('subkey1 (Uint8 '1)) '('value1 (String 'A))) + (AsStruct '('key1 (Int32 '7)) '('subkey1 (Uint8 '2)) '('value1 (String 'B))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '3)) '('value1 (String 'C))) + (AsStruct '('key1 (Int32 '8)) '('subkey1 (Uint8 '4)) '('value1 (String 'D))) + )) + + (let list1 (AssumeUnique list1 '('key1) '('value1))) + (let list1 (AssumeDistinct list1 '('subkey1) '('value1))) + + (let list2 (AsList + (AsStruct '('key2 (Int32 '0)) '('subkey2 (Uint8 '2)) '('value2 (String 'Z))) + (AsStruct '('key2 (Int32 '0)) '('subkey2 (Uint8 '2)) '('value2 (String 'Y))) + (AsStruct '('key2 (Int32 '0)) '('subkey2 (Uint8 '3)) '('value2 (String 'X))) + (AsStruct '('key2 (Int32 '3)) '('subkey2 (Uint8 '3)) '('value2 (String 'W))) + (AsStruct '('key2 (Int32 '3)) '('subkey2 (Uint8 '3)) '('value2 (String 'V))) + )) + + (let list2 (AssumeUnique list2 '('key2 'value2))) + (let list2 (AssumeDistinct list2 '('subkey2 'value2))) + + (let flow1 (ExpandMap (ToFlow list1) (lambda '(item) (Member item 'key1) (Member item 'subkey1) (Member item 'value1)))) + (let flow2 (ExpandMap (ToFlow list2) (lambda '(item) (Member item 'key2) (Member item 'subkey2) (Member item 'value2)))) + + (let join (GraceJoinCore flow1 flow2 'Inner '('0 '1) '('0 '1) '('0 '0 '1 '1 '2 '2) '('0 '3 '1 '4 '2 '5) '('LeftAny 'RightAny))) + (let list (Collect (NarrowMap join (lambda '(lk ls lv rk rs rv) (AsStruct '('lk lk) '('ls ls) '('lv lv) '('rk rk) '('rs rs) '('rv rv)))))) + + (let res_sink (DataSink 'yt (quote plato))) + (let world (Write! world res_sink (Key '('table (String 'Output))) list '('('mode 'renew)))) + + (let world (Commit! world res_sink)) + (return world) +) + )"; + + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TUniqueConstraintNode>(exprRoot, "Collect", "Unique((lk)(lv)(rk,rv))"); + CheckConstraint<TDistinctConstraintNode>(exprRoot, "Collect", "Distinct((ls)(lv)(rs,rv))"); + } + + Y_UNIT_TEST(MapJoinInnerOne) { + const auto s = R"( +( + (let list1 (AsList + (AsStruct '('key1 (Int32 '1)) '('subkey1 (Uint8 '0)) '('value1 (String 'A))) + (AsStruct '('key1 (Int32 '7)) '('subkey1 (Uint8 '0)) '('value1 (String 'B))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '0)) '('value1 (String 'C))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '1)) '('value1 (String 'D))) + )) + + (let list1 (AssumeUnique list1 '('key1 'subkey1) '('value1))) + (let list1 (AssumeDistinct list1 '('key1 'subkey1) '('value1))) + + (let list2 (AsList + (AsStruct '('key2 (Int32 '9)) '('subkey2 (Uint8 '0)) '('value2 (String 'Z))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '0)) '('value2 (String 'Y))) + (AsStruct '('key2 (Int32 '3)) '('subkey2 (Uint8 '1)) '('value2 (String 'X))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '1)) '('value2 (String 'W))) + (AsStruct '('key2 (Int32 '8)) '('subkey2 (Uint8 '1)) '('value2 (String 'V))) + )) + + (let list2 (AssumeUnique list2 '('key2 'subkey2) '('value2))) + (let list2 (AssumeDistinct list2 '('key2 'subkey2) '('value2))) + + (let dict (ToDict list2 (lambda '(item) '((Member item 'key2) (Member item 'subkey2))) (lambda '(item) '((Member item 'subkey2) (Member item 'value2))) '('One 'Hashed))) + + (let join (MapJoinCore (ToFlow list1) dict 'Inner '('key1 'subkey1) '('key1 'key 'subkey1 'subkey 'value1 'value) '('0 's '1 'v))) + (let list (Collect join)) + + (let res_sink (DataSink 'yt (quote plato))) + (let world (Write! world res_sink (Key '('table (String 'Output))) list '('('mode 'renew)))) + + (let world (Commit! world res_sink)) + (return world) +) + )"; + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TUniqueConstraintNode>(exprRoot, "Collect", "Unique((key,subkey)(v)(value))"); + CheckConstraint<TDistinctConstraintNode>(exprRoot, "Collect", "Distinct((key,subkey)(v)(value))"); + } + + Y_UNIT_TEST(MapJoinInnerMany) { + const auto s = R"( +( + (let list1 (AsList + (AsStruct '('key1 (Int32 '1)) '('subkey1 (Uint8 '0)) '('value1 (String 'A))) + (AsStruct '('key1 (Int32 '7)) '('subkey1 (Uint8 '0)) '('value1 (String 'B))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '0)) '('value1 (String 'C))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '1)) '('value1 (String 'D))) + )) + + (let list1 (AssumeUnique list1 '('key1 'subkey1) '('value1))) + (let list1 (AssumeDistinct list1 '('key1 'subkey1) '('value1))) + + (let list2 (AsList + (AsStruct '('key2 (Int32 '9)) '('subkey2 (Uint8 '0)) '('value2 (String 'Z))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '0)) '('value2 (String 'Y))) + (AsStruct '('key2 (Int32 '3)) '('subkey2 (Uint8 '1)) '('value2 (String 'X))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '1)) '('value2 (String 'W))) + (AsStruct '('key2 (Int32 '8)) '('subkey2 (Uint8 '1)) '('value2 (String 'V))) + )) + + (let list2 (AssumeUnique list2 '('key2 'subkey2) '('value2))) + (let list2 (AssumeDistinct list2 '('key2 'subkey2) '('value2))) + + (let dict (ToDict list2 (lambda '(item) '((Member item 'key2) (Member item 'subkey2))) (lambda '(item) '((Member item 'subkey2) (Member item 'value2))) '('Many 'Hashed))) + + (let join (MapJoinCore (ToFlow list1) dict 'Inner '('key1 'subkey1) '('key1 'key 'subkey1 'subkey 'value1 'value) '('0 's '1 'v))) + (let list (Collect join)) + + (let res_sink (DataSink 'yt (quote plato))) + (let world (Write! world res_sink (Key '('table (String 'Output))) list '('('mode 'renew)))) + + (let world (Commit! world res_sink)) + (return world) +) + )"; + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TUniqueConstraintNode>(exprRoot, "Collect", ""); + CheckConstraint<TDistinctConstraintNode>(exprRoot, "Collect", ""); + } + + Y_UNIT_TEST(MapJoinLeftOne) { + const auto s = R"( +( + (let list1 (AsList + (AsStruct '('key1 (Int32 '1)) '('subkey1 (Uint8 '0)) '('value1 (String 'A))) + (AsStruct '('key1 (Int32 '7)) '('subkey1 (Uint8 '0)) '('value1 (String 'B))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '0)) '('value1 (String 'C))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '1)) '('value1 (String 'D))) + )) + + (let list1 (AssumeUnique list1 '('key1 'subkey1) '('value1))) + (let list1 (AssumeDistinct list1 '('key1 'subkey1) '('value1))) + + (let list2 (AsList + (AsStruct '('key2 (Int32 '9)) '('subkey2 (Uint8 '0)) '('value2 (String 'Z))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '0)) '('value2 (String 'Y))) + (AsStruct '('key2 (Int32 '3)) '('subkey2 (Uint8 '1)) '('value2 (String 'X))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '1)) '('value2 (String 'W))) + (AsStruct '('key2 (Int32 '8)) '('subkey2 (Uint8 '1)) '('value2 (String 'V))) + )) + + (let list2 (AssumeUnique list2 '('key2 'subkey2) '('value2))) + (let list2 (AssumeDistinct list2 '('key2 'subkey2) '('value2))) + + (let dict (ToDict list2 (lambda '(item) '((Member item 'key2) (Member item 'subkey2))) (lambda '(item) '((Member item 'subkey2) (Member item 'value2))) '('One 'Hashed))) + + (let join (MapJoinCore (ToFlow list1) dict 'Left '('key1 'subkey1) '('key1 'key 'subkey1 'subkey 'value1 'value) '('0 's '1 'v))) + (let list (Collect join)) + + (let res_sink (DataSink 'yt (quote plato))) + (let world (Write! world res_sink (Key '('table (String 'Output))) list '('('mode 'renew)))) + + (let world (Commit! world res_sink)) + (return world) +) + )"; + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TUniqueConstraintNode>(exprRoot, "Collect", "Unique((key,subkey)(v)(value))"); + CheckConstraint<TDistinctConstraintNode>(exprRoot, "Collect", "Distinct((key,subkey)(value))"); + } + + Y_UNIT_TEST(MapJoinLeftMany) { + const auto s = R"( +( + (let list1 (AsList + (AsStruct '('key1 (Int32 '1)) '('subkey1 (Uint8 '0)) '('value1 (String 'A))) + (AsStruct '('key1 (Int32 '7)) '('subkey1 (Uint8 '0)) '('value1 (String 'B))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '0)) '('value1 (String 'C))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '1)) '('value1 (String 'D))) + )) + + (let list1 (AssumeUnique list1 '('key1 'subkey1) '('value1))) + (let list1 (AssumeDistinct list1 '('key1 'subkey1) '('value1))) + + (let list2 (AsList + (AsStruct '('key2 (Int32 '9)) '('subkey2 (Uint8 '0)) '('value2 (String 'Z))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '0)) '('value2 (String 'Y))) + (AsStruct '('key2 (Int32 '3)) '('subkey2 (Uint8 '1)) '('value2 (String 'X))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '1)) '('value2 (String 'W))) + (AsStruct '('key2 (Int32 '8)) '('subkey2 (Uint8 '1)) '('value2 (String 'V))) + )) + + (let list2 (AssumeUnique list2 '('key2 'subkey2) '('value2))) + (let list2 (AssumeDistinct list2 '('key2 'subkey2) '('value2))) + + (let dict (ToDict list2 (lambda '(item) '((Member item 'key2) (Member item 'subkey2))) (lambda '(item) '((Member item 'subkey2) (Member item 'value2))) '('Many 'Hashed))) + + (let join (MapJoinCore (ToFlow list1) dict 'Left '('key1 'subkey1) '('key1 'key 'subkey1 'subkey 'value1 'value) '('0 's '1 'v))) + (let list (Collect join)) + + (let res_sink (DataSink 'yt (quote plato))) + (let world (Write! world res_sink (Key '('table (String 'Output))) list '('('mode 'renew)))) + + (let world (Commit! world res_sink)) + (return world) +) + )"; + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TUniqueConstraintNode>(exprRoot, "Collect", ""); + CheckConstraint<TDistinctConstraintNode>(exprRoot, "Collect", ""); + } + + Y_UNIT_TEST(MapJoinLeftSemi) { + const auto s = R"( +( + (let list1 (AsList + (AsStruct '('key1 (Int32 '1)) '('subkey1 (Uint8 '0)) '('value1 (String 'A))) + (AsStruct '('key1 (Int32 '7)) '('subkey1 (Uint8 '0)) '('value1 (String 'B))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '0)) '('value1 (String 'C))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '1)) '('value1 (String 'D))) + )) + + (let list1 (AssumeUnique list1 '('key1 'subkey1) '('value1))) + (let list1 (AssumeDistinct list1 '('key1 'subkey1) '('value1))) + + (let list2 (AsList + (AsStruct '('key2 (Int32 '9)) '('subkey2 (Uint8 '0)) '('value2 (String 'Z))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '0)) '('value2 (String 'Y))) + (AsStruct '('key2 (Int32 '3)) '('subkey2 (Uint8 '1)) '('value2 (String 'X))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '1)) '('value2 (String 'W))) + (AsStruct '('key2 (Int32 '8)) '('subkey2 (Uint8 '1)) '('value2 (String 'V))) + )) + + (let list2 (AssumeUnique list2 '('key2 'subkey2) '('value2))) + (let list2 (AssumeDistinct list2 '('key2 'subkey2) '('value2))) + + (let dict (ToDict list2 (lambda '(item) '((Member item 'key2) (Member item 'subkey2))) (lambda '(item) '()) '('One 'Hashed))) + + (let join (MapJoinCore (ToFlow list1) dict 'LeftSemi '('key1 'subkey1) '('key1 'key 'subkey1 'subkey 'value1 'value) '())) + (let list (Collect join)) + + (let res_sink (DataSink 'yt (quote plato))) + (let world (Write! world res_sink (Key '('table (String 'Output))) list '('('mode 'renew)))) + + (let world (Commit! world res_sink)) + (return world) +) + )"; + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TUniqueConstraintNode>(exprRoot, "Collect", "Unique((key,subkey)(value))"); + CheckConstraint<TDistinctConstraintNode>(exprRoot, "Collect", "Distinct((key,subkey)(value))"); + } + + Y_UNIT_TEST(MapJoinLeftOnly) { + const auto s = R"( +( + (let list1 (AsList + (AsStruct '('key1 (Int32 '1)) '('subkey1 (Uint8 '0)) '('value1 (String 'A))) + (AsStruct '('key1 (Int32 '7)) '('subkey1 (Uint8 '0)) '('value1 (String 'B))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '0)) '('value1 (String 'C))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '1)) '('value1 (String 'D))) + )) + + (let list1 (AssumeUnique list1 '('key1 'subkey1) '('value1))) + (let list1 (AssumeDistinct list1 '('key1 'subkey1) '('value1))) + + (let list2 (AsList + (AsStruct '('key2 (Int32 '9)) '('subkey2 (Uint8 '0)) '('value2 (String 'Z))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '0)) '('value2 (String 'Y))) + (AsStruct '('key2 (Int32 '3)) '('subkey2 (Uint8 '1)) '('value2 (String 'X))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '1)) '('value2 (String 'W))) + (AsStruct '('key2 (Int32 '8)) '('subkey2 (Uint8 '1)) '('value2 (String 'V))) + )) + + (let list2 (AssumeUnique list2 '('key2 'subkey2) '('value2))) + (let list2 (AssumeDistinct list2 '('key2 'subkey2) '('value2))) + + (let dict (ToDict list2 (lambda '(item) '((Member item 'key2) (Member item 'subkey2))) (lambda '(item) '()) '('One 'Hashed))) + + (let join (MapJoinCore (ToFlow list1) dict 'LeftOnly '('key1 'subkey1) '('key1 'key 'value1 'value) '())) + (let list (Collect join)) + + (let res_sink (DataSink 'yt (quote plato))) + (let world (Write! world res_sink (Key '('table (String 'Output))) list '('('mode 'renew)))) + + (let world (Commit! world res_sink)) + (return world) +) + )"; + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TUniqueConstraintNode>(exprRoot, "Collect", "Unique((value))"); + CheckConstraint<TDistinctConstraintNode>(exprRoot, "Collect", "Distinct((value))"); + } + + Y_UNIT_TEST(EquiJoinWithRenames) { + const auto s = R"( +( + (let list1 (AsList + (AsStruct '('key1 (Int32 '1)) '('subkey1 (Uint8 '0)) '('value1 (String 'A))) + (AsStruct '('key1 (Int32 '7)) '('subkey1 (Uint8 '0)) '('value1 (String 'B))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '0)) '('value1 (String 'C))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '1)) '('value1 (String 'D))) + )) + + (let list1 (AssumeUnique list1 '('key1 'subkey1) '('value1))) + (let list1 (AssumeDistinct list1 '('key1 'subkey1) '('value1))) + + (let list2 (AsList + (AsStruct '('key2 (Int32 '9)) '('subkey2 (Uint8 '0)) '('value2 (String 'Z))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '0)) '('value2 (String 'Y))) + (AsStruct '('key2 (Int32 '3)) '('subkey2 (Uint8 '1)) '('value2 (String 'X))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '1)) '('value2 (String 'W))) + (AsStruct '('key2 (Int32 '8)) '('subkey2 (Uint8 '1)) '('value2 (String 'V))) + )) + + (let list2 (AssumeUnique list2 '('key2 'subkey2) '('value2))) + (let list2 (AssumeDistinct list2 '('key2 'subkey2) '('value2))) + + (let join (EquiJoin '(list1 'a) '(list2 'b) '('Inner 'a 'b '('a 'key1 'a 'subkey1) '('b 'key2 'b 'subkey2) '()) '( + '('rename 'a.key1 'key_1) + '('rename 'a.key1 'key_2) + '('rename 'a.subkey1 'subkey_1) + '('rename 'a.subkey1 'subkey_2) + '('rename 'a.value1 '"") + '('rename 'b.key2 '"") + '('rename 'b.value2 'value_1) + '('rename 'b.value2 'value_2) + ))) + (let lazy (LazyList join)) + + (let res_sink (DataSink 'yt (quote plato))) + (let world (Write! world res_sink (Key '('table (String 'Output))) lazy '('('mode 'renew)))) + + (let world (Commit! world res_sink)) + (return world) +) + )"; + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TUniqueConstraintNode>(exprRoot, "LazyList", "Unique(({key_1,key_2},{subkey_1,subkey_2})({value_1,value_2}))"); + CheckConstraint<TDistinctConstraintNode>(exprRoot, "LazyList", "Distinct(({key_1,key_2},{subkey_1,subkey_2})({value_1,value_2}))"); + } + + Y_UNIT_TEST(EquiJoinWithPartialRenames) { + const auto s = R"( +( + (let list1 (AsList + (AsStruct '('key1 (Int32 '1)) '('subkey1 (Uint8 '0)) '('value1 (String 'A))) + (AsStruct '('key1 (Int32 '7)) '('subkey1 (Uint8 '0)) '('value1 (String 'B))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '0)) '('value1 (String 'C))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '1)) '('value1 (String 'D))) + )) + + (let list1 (AssumeUnique list1 '('key1 'subkey1) '('value1))) + (let list1 (AssumeDistinct list1 '('key1 'subkey1) '('value1))) + + (let list2 (AsList + (AsStruct '('key2 (Int32 '9)) '('subkey2 (Uint8 '0)) '('value2 (String 'Z))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '0)) '('value2 (String 'Y))) + (AsStruct '('key2 (Int32 '3)) '('subkey2 (Uint8 '1)) '('value2 (String 'X))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '1)) '('value2 (String 'W))) + (AsStruct '('key2 (Int32 '8)) '('subkey2 (Uint8 '1)) '('value2 (String 'V))) + )) + + (let list2 (AssumeUnique list2 '('key2 'subkey2) '('value2))) + (let list2 (AssumeDistinct list2 '('key2 'subkey2) '('value2))) + + (let join (EquiJoin '(list1 'a) '(list2 'b) '('Left 'a 'b '('a 'key1 'a 'subkey1) '('b 'key2 'b 'subkey2) '()) '( + '('rename 'a.key1 'key_1) + '('rename 'a.key1 'key_2) + '('rename 'a.value1 '"") + '('rename 'b.key2 '"") + '('rename 'b.value2 'value) + ))) + (let lazy (LazyList join)) + + (let res_sink (DataSink 'yt (quote plato))) + (let world (Write! world res_sink (Key '('table (String 'Output))) lazy '('('mode 'renew)))) + + (let world (Commit! world res_sink)) + (return world) +) + )"; + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TUniqueConstraintNode>(exprRoot, "LazyList", "Unique((a.subkey1,{key_1,key_2})(value))"); + CheckConstraint<TDistinctConstraintNode>(exprRoot, "LazyList", "Distinct((a.subkey1,{key_1,key_2}))"); + } + + Y_UNIT_TEST(EquiJoinInnerInner) { + const auto s = R"( +( + (let list1 (AsList + (AsStruct '('key1 (Int32 '1)) '('subkey1 (Uint8 '0)) '('value1 (String 'A))) + (AsStruct '('key1 (Int32 '7)) '('subkey1 (Uint8 '0)) '('value1 (String 'B))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '0)) '('value1 (String 'C))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '1)) '('value1 (String 'D))) + )) + + (let list1 (AssumeUnique list1 '('key1 'subkey1) '('value1))) + (let list1 (AssumeDistinct list1 '('key1 'subkey1) '('value1))) + + (let list2 (AsList + (AsStruct '('key2 (Int32 '9)) '('subkey2 (Uint8 '0)) '('value2 (String 'Z))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '0)) '('value2 (String 'Y))) + (AsStruct '('key2 (Int32 '3)) '('subkey2 (Uint8 '1)) '('value2 (String 'X))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '1)) '('value2 (String 'W))) + (AsStruct '('key2 (Int32 '8)) '('subkey2 (Uint8 '1)) '('value2 (String 'V))) + )) + + (let list2 (AssumeUnique list2 '('key2 'subkey2) '('value2))) + (let list2 (AssumeDistinct list2 '('key2 'subkey2) '('value2))) + + (let list3 (AsList + (AsStruct '('key3 (Int32 '1)) '('subkey3 (Uint8 '0)) '('value3 (String 'G))) + (AsStruct '('key3 (Int32 '4)) '('subkey3 (Uint8 '1)) '('value3 (String 'H))) + (AsStruct '('key3 (Int32 '2)) '('subkey3 (Uint8 '0)) '('value3 (String 'I))) + (AsStruct '('key3 (Int32 '3)) '('subkey3 (Uint8 '1)) '('value3 (String 'J))) + (AsStruct '('key3 (Int32 '3)) '('subkey3 (Uint8 '0)) '('value3 (String 'K))) + )) + + (let list3 (AssumeUnique list3 '('key3 'subkey3) '('value3))) + (let list3 (AssumeDistinct list3 '('key3 'subkey3) '('value3))) + + (let join (EquiJoin '(list1 'a) '(list2 'b) '(list3 'c) '('Inner '('Inner 'a 'b '('a 'key1 'a 'subkey1) '('b 'key2 'b 'subkey2) '()) 'c '('b 'key2 'b 'subkey2) '('c 'key3 'c 'subkey3) '()) '())) + (let lazy (LazyList join)) + + (let res_sink (DataSink 'yt (quote plato))) + (let world (Write! world res_sink (Key '('table (String 'Output))) lazy '('('mode 'renew)))) + + (let world (Commit! world res_sink)) + (return world) +) + )"; + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TUniqueConstraintNode>(exprRoot, "LazyList", "Unique((a.key1,a.subkey1)(a.value1)(b.key2,b.subkey2)(b.value2)(c.key3,c.subkey3)(c.value3))"); + CheckConstraint<TDistinctConstraintNode>(exprRoot, "LazyList", "Distinct((a.key1,a.subkey1)(a.value1)(b.key2,b.subkey2)(b.value2)(c.key3,c.subkey3)(c.value3))"); + } + + Y_UNIT_TEST(EquiJoinInnerLeft) { + const auto s = R"( +( + (let list1 (AsList + (AsStruct '('key1 (Int32 '1)) '('subkey1 (Uint8 '0)) '('value1 (String 'A))) + (AsStruct '('key1 (Int32 '7)) '('subkey1 (Uint8 '0)) '('value1 (String 'B))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '0)) '('value1 (String 'C))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '1)) '('value1 (String 'D))) + )) + + (let list1 (AssumeUnique list1 '('key1 'subkey1) '('value1))) + (let list1 (AssumeDistinct list1 '('key1 'subkey1) '('value1))) + + (let list2 (AsList + (AsStruct '('key2 (Int32 '9)) '('subkey2 (Uint8 '0)) '('value2 (String 'Z))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '0)) '('value2 (String 'Y))) + (AsStruct '('key2 (Int32 '3)) '('subkey2 (Uint8 '1)) '('value2 (String 'X))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '1)) '('value2 (String 'W))) + (AsStruct '('key2 (Int32 '8)) '('subkey2 (Uint8 '1)) '('value2 (String 'V))) + )) + + (let list2 (AssumeUnique list2 '('key2 'subkey2) '('value2))) + (let list2 (AssumeDistinct list2 '('key2 'subkey2) '('value2))) + + (let list3 (AsList + (AsStruct '('key3 (Int32 '1)) '('subkey3 (Uint8 '0)) '('value3 (String 'G))) + (AsStruct '('key3 (Int32 '4)) '('subkey3 (Uint8 '1)) '('value3 (String 'H))) + (AsStruct '('key3 (Int32 '2)) '('subkey3 (Uint8 '0)) '('value3 (String 'I))) + (AsStruct '('key3 (Int32 '3)) '('subkey3 (Uint8 '1)) '('value3 (String 'J))) + (AsStruct '('key3 (Int32 '3)) '('subkey3 (Uint8 '0)) '('value3 (String 'K))) + )) + + (let list3 (AssumeUnique list3 '('key3 'subkey3) '('value3))) + (let list3 (AssumeDistinct list3 '('key3 'subkey3) '('value3))) + + (let join (EquiJoin '(list1 'a) '(list2 'b) '(list3 'c) '('Inner '('Left 'a 'b '('a 'key1 'a 'subkey1) '('b 'key2 'b 'subkey2) '()) 'c '('b 'key2 'b 'subkey2) '('c 'key3 'c 'subkey3) '()) '())) + (let lazy (LazyList join)) + + (let res_sink (DataSink 'yt (quote plato))) + (let world (Write! world res_sink (Key '('table (String 'Output))) lazy '('('mode 'renew)))) + + (let world (Commit! world res_sink)) + (return world) +) + )"; + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TUniqueConstraintNode>(exprRoot, "LazyList", "Unique((a.key1,a.subkey1)(a.value1)(b.key2,b.subkey2)(b.value2)(c.key3,c.subkey3)(c.value3))"); + CheckConstraint<TDistinctConstraintNode>(exprRoot, "LazyList", "Distinct((a.key1,a.subkey1)(a.value1)(c.key3,c.subkey3)(c.value3))"); + } + + Y_UNIT_TEST(EquiJoinInnerRight) { + const auto s = R"( +( + (let list1 (AsList + (AsStruct '('key1 (Int32 '1)) '('subkey1 (Uint8 '0)) '('value1 (String 'A))) + (AsStruct '('key1 (Int32 '7)) '('subkey1 (Uint8 '0)) '('value1 (String 'B))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '0)) '('value1 (String 'C))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '1)) '('value1 (String 'D))) + )) + + (let list1 (AssumeUnique list1 '('key1 'subkey1) '('value1))) + (let list1 (AssumeDistinct list1 '('key1 'subkey1) '('value1))) + + (let list2 (AsList + (AsStruct '('key2 (Int32 '9)) '('subkey2 (Uint8 '0)) '('value2 (String 'Z))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '0)) '('value2 (String 'Y))) + (AsStruct '('key2 (Int32 '3)) '('subkey2 (Uint8 '1)) '('value2 (String 'X))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '1)) '('value2 (String 'W))) + (AsStruct '('key2 (Int32 '8)) '('subkey2 (Uint8 '1)) '('value2 (String 'V))) + )) + + (let list2 (AssumeUnique list2 '('key2 'subkey2) '('value2))) + (let list2 (AssumeDistinct list2 '('key2 'subkey2) '('value2))) + + (let list3 (AsList + (AsStruct '('key3 (Int32 '1)) '('subkey3 (Uint8 '0)) '('value3 (String 'G))) + (AsStruct '('key3 (Int32 '4)) '('subkey3 (Uint8 '1)) '('value3 (String 'H))) + (AsStruct '('key3 (Int32 '2)) '('subkey3 (Uint8 '0)) '('value3 (String 'I))) + (AsStruct '('key3 (Int32 '3)) '('subkey3 (Uint8 '1)) '('value3 (String 'J))) + (AsStruct '('key3 (Int32 '3)) '('subkey3 (Uint8 '0)) '('value3 (String 'K))) + )) + + (let list3 (AssumeUnique list3 '('key3 'subkey3) '('value3))) + (let list3 (AssumeDistinct list3 '('key3 'subkey3) '('value3))) + + (let join (EquiJoin '(list1 'a) '(list2 'b) '(list3 'c) '('Inner '('Right 'a 'b '('a 'key1 'a 'subkey1) '('b 'key2 'b 'subkey2) '()) 'c '('b 'key2 'b 'subkey2) '('c 'key3 'c 'subkey3) '()) '())) + (let lazy (LazyList join)) + + (let res_sink (DataSink 'yt (quote plato))) + (let world (Write! world res_sink (Key '('table (String 'Output))) lazy '('('mode 'renew)))) + + (let world (Commit! world res_sink)) + (return world) +) + )"; + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TUniqueConstraintNode>(exprRoot, "LazyList", "Unique((a.key1,a.subkey1)(a.value1)(b.key2,b.subkey2)(b.value2)(c.key3,c.subkey3)(c.value3))"); + CheckConstraint<TDistinctConstraintNode>(exprRoot, "LazyList", "Distinct((b.key2,b.subkey2)(b.value2)(c.key3,c.subkey3)(c.value3))"); + } + + Y_UNIT_TEST(EquiJoinInnerFull) { + const auto s = R"( +( + (let list1 (AsList + (AsStruct '('key1 (Int32 '1)) '('subkey1 (Uint8 '0)) '('value1 (String 'A))) + (AsStruct '('key1 (Int32 '7)) '('subkey1 (Uint8 '0)) '('value1 (String 'B))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '0)) '('value1 (String 'C))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '1)) '('value1 (String 'D))) + )) + + (let list1 (AssumeUnique list1 '('key1 'subkey1) '('value1))) + (let list1 (AssumeDistinct list1 '('key1 'subkey1) '('value1))) + + (let list2 (AsList + (AsStruct '('key2 (Int32 '9)) '('subkey2 (Uint8 '0)) '('value2 (String 'Z))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '0)) '('value2 (String 'Y))) + (AsStruct '('key2 (Int32 '3)) '('subkey2 (Uint8 '1)) '('value2 (String 'X))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '1)) '('value2 (String 'W))) + (AsStruct '('key2 (Int32 '8)) '('subkey2 (Uint8 '1)) '('value2 (String 'V))) + )) + + (let list2 (AssumeUnique list2 '('key2 'subkey2) '('value2))) + (let list2 (AssumeDistinct list2 '('key2 'subkey2) '('value2))) + + (let list3 (AsList + (AsStruct '('key3 (Int32 '1)) '('subkey3 (Uint8 '0)) '('value3 (String 'G))) + (AsStruct '('key3 (Int32 '4)) '('subkey3 (Uint8 '1)) '('value3 (String 'H))) + (AsStruct '('key3 (Int32 '2)) '('subkey3 (Uint8 '0)) '('value3 (String 'I))) + (AsStruct '('key3 (Int32 '3)) '('subkey3 (Uint8 '1)) '('value3 (String 'J))) + (AsStruct '('key3 (Int32 '3)) '('subkey3 (Uint8 '0)) '('value3 (String 'K))) + )) + + (let list3 (AssumeUnique list3 '('key3 'subkey3) '('value3))) + (let list3 (AssumeDistinct list3 '('key3 'subkey3) '('value3))) + + (let join (EquiJoin '(list1 'a) '(list2 'b) '(list3 'c) '('Inner '('Full 'a 'b '('a 'key1 'a 'subkey1) '('b 'key2 'b 'subkey2) '()) 'c '('b 'key2 'b 'subkey2) '('c 'key3 'c 'subkey3) '()) '())) + (let lazy (LazyList join)) + + (let res_sink (DataSink 'yt (quote plato))) + (let world (Write! world res_sink (Key '('table (String 'Output))) lazy '('('mode 'renew)))) + + (let world (Commit! world res_sink)) + (return world) +) + )"; + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TUniqueConstraintNode>(exprRoot, "LazyList", "Unique((a.key1,a.subkey1)(a.value1)(b.key2,b.subkey2)(b.value2)(c.key3,c.subkey3)(c.value3))"); + CheckConstraint<TDistinctConstraintNode>(exprRoot, "LazyList", "Distinct((c.key3,c.subkey3)(c.value3))"); + } + + Y_UNIT_TEST(EquiJoinInnerExclusion) { + const auto s = R"( +( + (let list1 (AsList + (AsStruct '('key1 (Int32 '1)) '('subkey1 (Uint8 '0)) '('value1 (String 'A))) + (AsStruct '('key1 (Int32 '7)) '('subkey1 (Uint8 '0)) '('value1 (String 'B))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '0)) '('value1 (String 'C))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '1)) '('value1 (String 'D))) + )) + + (let list1 (AssumeUnique list1 '('key1 'subkey1) '('value1))) + (let list1 (AssumeDistinct list1 '('key1 'subkey1) '('value1))) + + (let list2 (AsList + (AsStruct '('key2 (Int32 '9)) '('subkey2 (Uint8 '0)) '('value2 (String 'Z))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '0)) '('value2 (String 'Y))) + (AsStruct '('key2 (Int32 '3)) '('subkey2 (Uint8 '1)) '('value2 (String 'X))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '1)) '('value2 (String 'W))) + (AsStruct '('key2 (Int32 '8)) '('subkey2 (Uint8 '1)) '('value2 (String 'V))) + )) + + (let list2 (AssumeUnique list2 '('key2 'subkey2) '('value2))) + (let list2 (AssumeDistinct list2 '('key2 'subkey2) '('value2))) + + (let list3 (AsList + (AsStruct '('key3 (Int32 '1)) '('subkey3 (Uint8 '0)) '('value3 (String 'G))) + (AsStruct '('key3 (Int32 '4)) '('subkey3 (Uint8 '1)) '('value3 (String 'H))) + (AsStruct '('key3 (Int32 '2)) '('subkey3 (Uint8 '0)) '('value3 (String 'I))) + (AsStruct '('key3 (Int32 '3)) '('subkey3 (Uint8 '1)) '('value3 (String 'J))) + (AsStruct '('key3 (Int32 '3)) '('subkey3 (Uint8 '0)) '('value3 (String 'K))) + )) + + (let list3 (AssumeUnique list3 '('key3 'subkey3) '('value3))) + (let list3 (AssumeDistinct list3 '('key3 'subkey3) '('value3))) + + (let join (EquiJoin '(list1 'a) '(list2 'b) '(list3 'c) '('Inner '('Exclusion 'a 'b '('a 'key1 'a 'subkey1) '('b 'key2 'b 'subkey2) '()) 'c '('b 'key2 'b 'subkey2) '('c 'key3 'c 'subkey3) '()) '())) + (let lazy (LazyList join)) + + (let res_sink (DataSink 'yt (quote plato))) + (let world (Write! world res_sink (Key '('table (String 'Output))) lazy '('('mode 'renew)))) + + (let world (Commit! world res_sink)) + (return world) +) + )"; + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TUniqueConstraintNode>(exprRoot, "LazyList", "Unique((a.key1,a.subkey1)(a.value1)(b.key2,b.subkey2)(b.value2)(c.key3,c.subkey3)(c.value3))"); + CheckConstraint<TDistinctConstraintNode>(exprRoot, "LazyList", "Distinct((c.key3,c.subkey3)(c.value3))"); + } + + Y_UNIT_TEST(EquiJoinInnerLeftOnly) { + const auto s = R"( +( + (let list1 (AsList + (AsStruct '('key1 (Int32 '1)) '('subkey1 (Uint8 '0)) '('value1 (String 'A))) + (AsStruct '('key1 (Int32 '7)) '('subkey1 (Uint8 '0)) '('value1 (String 'B))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '0)) '('value1 (String 'C))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '1)) '('value1 (String 'D))) + )) + + (let list1 (AssumeUnique list1 '('key1 'subkey1) '('value1))) + (let list1 (AssumeDistinct list1 '('key1 'subkey1) '('value1))) + + (let list2 (AsList + (AsStruct '('key2 (Int32 '9)) '('subkey2 (Uint8 '0)) '('value2 (String 'Z))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '0)) '('value2 (String 'Y))) + (AsStruct '('key2 (Int32 '3)) '('subkey2 (Uint8 '1)) '('value2 (String 'X))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '1)) '('value2 (String 'W))) + (AsStruct '('key2 (Int32 '8)) '('subkey2 (Uint8 '1)) '('value2 (String 'V))) + )) + + (let list2 (AssumeUnique list2 '('key2 'subkey2) '('value2))) + (let list2 (AssumeDistinct list2 '('key2 'subkey2) '('value2))) + + (let list3 (AsList + (AsStruct '('key3 (Int32 '1)) '('subkey3 (Uint8 '0)) '('value3 (String 'G))) + (AsStruct '('key3 (Int32 '4)) '('subkey3 (Uint8 '1)) '('value3 (String 'H))) + (AsStruct '('key3 (Int32 '2)) '('subkey3 (Uint8 '0)) '('value3 (String 'I))) + (AsStruct '('key3 (Int32 '3)) '('subkey3 (Uint8 '1)) '('value3 (String 'J))) + (AsStruct '('key3 (Int32 '3)) '('subkey3 (Uint8 '0)) '('value3 (String 'K))) + )) + + (let list3 (AssumeUnique list3 '('key3 'subkey3) '('value3))) + (let list3 (AssumeDistinct list3 '('key3 'subkey3) '('value3))) + + (let join (EquiJoin '(list1 'a) '(list2 'b) '(list3 'c) '('Inner '('LeftOnly 'a 'b '('a 'key1 'a 'subkey1) '('b 'key2 'b 'subkey2) '()) 'c '('a 'key1 'a 'subkey1) '('c 'key3 'c 'subkey3) '()) '())) + (let lazy (LazyList join)) + + (let res_sink (DataSink 'yt (quote plato))) + (let world (Write! world res_sink (Key '('table (String 'Output))) lazy '('('mode 'renew)))) + + (let world (Commit! world res_sink)) + (return world) +) + )"; + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TUniqueConstraintNode>(exprRoot, "LazyList", "Unique((a.key1,a.subkey1)(a.value1)(c.key3,c.subkey3)(c.value3))"); + CheckConstraint<TDistinctConstraintNode>(exprRoot, "LazyList", "Distinct((a.key1,a.subkey1)(a.value1)(c.key3,c.subkey3)(c.value3))"); + } + + Y_UNIT_TEST(EquiJoinInnerLeftSemi) { + const auto s = R"( +( + (let list1 (AsList + (AsStruct '('key1 (Int32 '1)) '('subkey1 (Uint8 '0)) '('value1 (String 'A))) + (AsStruct '('key1 (Int32 '7)) '('subkey1 (Uint8 '0)) '('value1 (String 'B))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '0)) '('value1 (String 'C))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '1)) '('value1 (String 'D))) + )) + + (let list1 (AssumeUnique list1 '('key1 'subkey1) '('value1))) + (let list1 (AssumeDistinct list1 '('key1 'subkey1) '('value1))) + + (let list2 (AsList + (AsStruct '('key2 (Int32 '9)) '('subkey2 (Uint8 '0)) '('value2 (String 'Z))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '0)) '('value2 (String 'Y))) + (AsStruct '('key2 (Int32 '3)) '('subkey2 (Uint8 '1)) '('value2 (String 'X))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '1)) '('value2 (String 'W))) + (AsStruct '('key2 (Int32 '8)) '('subkey2 (Uint8 '1)) '('value2 (String 'V))) + )) + + (let list2 (AssumeUnique list2 '('key2 'subkey2) '('value2))) + (let list2 (AssumeDistinct list2 '('key2 'subkey2) '('value2))) + + (let list3 (AsList + (AsStruct '('key3 (Int32 '1)) '('subkey3 (Uint8 '0)) '('value3 (String 'G))) + (AsStruct '('key3 (Int32 '4)) '('subkey3 (Uint8 '1)) '('value3 (String 'H))) + (AsStruct '('key3 (Int32 '2)) '('subkey3 (Uint8 '0)) '('value3 (String 'I))) + (AsStruct '('key3 (Int32 '3)) '('subkey3 (Uint8 '1)) '('value3 (String 'J))) + (AsStruct '('key3 (Int32 '3)) '('subkey3 (Uint8 '0)) '('value3 (String 'K))) + )) + + (let list3 (AssumeUnique list3 '('key3 'subkey3) '('value3))) + (let list3 (AssumeDistinct list3 '('key3 'subkey3) '('value3))) + + (let join (EquiJoin '(list1 'a) '(list2 'b) '(list3 'c) '('Inner '('LeftSemi 'a 'b '('a 'key1 'a 'subkey1) '('b 'key2 'b 'subkey2) '()) 'c '('a 'key1 'a 'subkey1) '('c 'key3 'c 'subkey3) '()) '())) + (let lazy (LazyList join)) + + (let res_sink (DataSink 'yt (quote plato))) + (let world (Write! world res_sink (Key '('table (String 'Output))) lazy '('('mode 'renew)))) + + (let world (Commit! world res_sink)) + (return world) +) + )"; + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TUniqueConstraintNode>(exprRoot, "LazyList", "Unique((a.key1,a.subkey1)(a.value1)(c.key3,c.subkey3)(c.value3))"); + CheckConstraint<TDistinctConstraintNode>(exprRoot, "LazyList", "Distinct((a.key1,a.subkey1)(a.value1)(c.key3,c.subkey3)(c.value3))"); + } + + Y_UNIT_TEST(EquiJoinInnerRightOnly) { + const auto s = R"( +( + (let list1 (AsList + (AsStruct '('key1 (Int32 '1)) '('subkey1 (Uint8 '0)) '('value1 (String 'A))) + (AsStruct '('key1 (Int32 '7)) '('subkey1 (Uint8 '0)) '('value1 (String 'B))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '0)) '('value1 (String 'C))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '1)) '('value1 (String 'D))) + )) + + (let list1 (AssumeUnique list1 '('key1 'subkey1) '('value1))) + (let list1 (AssumeDistinct list1 '('key1 'subkey1) '('value1))) + + (let list2 (AsList + (AsStruct '('key2 (Int32 '9)) '('subkey2 (Uint8 '0)) '('value2 (String 'Z))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '0)) '('value2 (String 'Y))) + (AsStruct '('key2 (Int32 '3)) '('subkey2 (Uint8 '1)) '('value2 (String 'X))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '1)) '('value2 (String 'W))) + (AsStruct '('key2 (Int32 '8)) '('subkey2 (Uint8 '1)) '('value2 (String 'V))) + )) + + (let list2 (AssumeUnique list2 '('key2 'subkey2) '('value2))) + (let list2 (AssumeDistinct list2 '('key2 'subkey2) '('value2))) + + (let list3 (AsList + (AsStruct '('key3 (Int32 '1)) '('subkey3 (Uint8 '0)) '('value3 (String 'G))) + (AsStruct '('key3 (Int32 '4)) '('subkey3 (Uint8 '1)) '('value3 (String 'H))) + (AsStruct '('key3 (Int32 '2)) '('subkey3 (Uint8 '0)) '('value3 (String 'I))) + (AsStruct '('key3 (Int32 '3)) '('subkey3 (Uint8 '1)) '('value3 (String 'J))) + (AsStruct '('key3 (Int32 '3)) '('subkey3 (Uint8 '0)) '('value3 (String 'K))) + )) + + (let list3 (AssumeUnique list3 '('key3 'subkey3) '('value3))) + (let list3 (AssumeDistinct list3 '('key3 'subkey3) '('value3))) + + (let join (EquiJoin '(list1 'a) '(list2 'b) '(list3 'c) '('Inner '('RightOnly 'a 'b '('a 'key1 'a 'subkey1) '('b 'key2 'b 'subkey2) '()) 'c '('b 'key2 'b 'subkey2) '('c 'key3 'c 'subkey3) '()) '())) + (let lazy (LazyList join)) + + (let res_sink (DataSink 'yt (quote plato))) + (let world (Write! world res_sink (Key '('table (String 'Output))) lazy '('('mode 'renew)))) + + (let world (Commit! world res_sink)) + (return world) +) + )"; + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TUniqueConstraintNode>(exprRoot, "LazyList", "Unique((b.key2,b.subkey2)(b.value2)(c.key3,c.subkey3)(c.value3))"); + CheckConstraint<TDistinctConstraintNode>(exprRoot, "LazyList", "Distinct((b.key2,b.subkey2)(b.value2)(c.key3,c.subkey3)(c.value3))"); + } + + Y_UNIT_TEST(EquiJoinInnerRightSemi) { + const auto s = R"( +( + (let list1 (AsList + (AsStruct '('key1 (Int32 '1)) '('subkey1 (Uint8 '0)) '('value1 (String 'A))) + (AsStruct '('key1 (Int32 '7)) '('subkey1 (Uint8 '0)) '('value1 (String 'B))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '0)) '('value1 (String 'C))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '1)) '('value1 (String 'D))) + )) + + (let list1 (AssumeUnique list1 '('key1 'subkey1) '('value1))) + (let list1 (AssumeDistinct list1 '('key1 'subkey1) '('value1))) + + (let list2 (AsList + (AsStruct '('key2 (Int32 '9)) '('subkey2 (Uint8 '0)) '('value2 (String 'Z))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '0)) '('value2 (String 'Y))) + (AsStruct '('key2 (Int32 '3)) '('subkey2 (Uint8 '1)) '('value2 (String 'X))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '1)) '('value2 (String 'W))) + (AsStruct '('key2 (Int32 '8)) '('subkey2 (Uint8 '1)) '('value2 (String 'V))) + )) + + (let list2 (AssumeUnique list2 '('key2 'subkey2) '('value2))) + (let list2 (AssumeDistinct list2 '('key2 'subkey2) '('value2))) + + (let list3 (AsList + (AsStruct '('key3 (Int32 '1)) '('subkey3 (Uint8 '0)) '('value3 (String 'G))) + (AsStruct '('key3 (Int32 '4)) '('subkey3 (Uint8 '1)) '('value3 (String 'H))) + (AsStruct '('key3 (Int32 '2)) '('subkey3 (Uint8 '0)) '('value3 (String 'I))) + (AsStruct '('key3 (Int32 '3)) '('subkey3 (Uint8 '1)) '('value3 (String 'J))) + (AsStruct '('key3 (Int32 '3)) '('subkey3 (Uint8 '0)) '('value3 (String 'K))) + )) + + (let list3 (AssumeUnique list3 '('key3 'subkey3) '('value3))) + (let list3 (AssumeDistinct list3 '('key3 'subkey3) '('value3))) + + (let join (EquiJoin '(list1 'a) '(list2 'b) '(list3 'c) '('Inner '('RightSemi 'a 'b '('a 'key1 'a 'subkey1) '('b 'key2 'b 'subkey2) '()) 'c '('b 'key2 'b 'subkey2) '('c 'key3 'c 'subkey3) '()) '())) + (let lazy (LazyList join)) + + (let res_sink (DataSink 'yt (quote plato))) + (let world (Write! world res_sink (Key '('table (String 'Output))) lazy '('('mode 'renew)))) + + (let world (Commit! world res_sink)) + (return world) +) + )"; + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TUniqueConstraintNode>(exprRoot, "LazyList", "Unique((b.key2,b.subkey2)(b.value2)(c.key3,c.subkey3)(c.value3))"); + CheckConstraint<TDistinctConstraintNode>(exprRoot, "LazyList", "Distinct((b.key2,b.subkey2)(b.value2)(c.key3,c.subkey3)(c.value3))"); + } + + Y_UNIT_TEST(EquiJoinLeftInner) { + const auto s = R"( +( + (let list1 (AsList + (AsStruct '('key1 (Int32 '1)) '('subkey1 (Uint8 '0)) '('value1 (String 'A))) + (AsStruct '('key1 (Int32 '7)) '('subkey1 (Uint8 '0)) '('value1 (String 'B))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '0)) '('value1 (String 'C))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '1)) '('value1 (String 'D))) + )) + + (let list1 (AssumeUnique list1 '('key1 'subkey1) '('value1))) + (let list1 (AssumeDistinct list1 '('key1 'subkey1) '('value1))) + + (let list2 (AsList + (AsStruct '('key2 (Int32 '9)) '('subkey2 (Uint8 '0)) '('value2 (String 'Z))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '0)) '('value2 (String 'Y))) + (AsStruct '('key2 (Int32 '3)) '('subkey2 (Uint8 '1)) '('value2 (String 'X))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '1)) '('value2 (String 'W))) + (AsStruct '('key2 (Int32 '8)) '('subkey2 (Uint8 '1)) '('value2 (String 'V))) + )) + + (let list2 (AssumeUnique list2 '('key2 'subkey2) '('value2))) + (let list2 (AssumeDistinct list2 '('key2 'subkey2) '('value2))) + + (let list3 (AsList + (AsStruct '('key3 (Int32 '1)) '('subkey3 (Uint8 '0)) '('value3 (String 'G))) + (AsStruct '('key3 (Int32 '4)) '('subkey3 (Uint8 '1)) '('value3 (String 'H))) + (AsStruct '('key3 (Int32 '2)) '('subkey3 (Uint8 '0)) '('value3 (String 'I))) + (AsStruct '('key3 (Int32 '3)) '('subkey3 (Uint8 '1)) '('value3 (String 'J))) + (AsStruct '('key3 (Int32 '3)) '('subkey3 (Uint8 '0)) '('value3 (String 'K))) + )) + + (let list3 (AssumeUnique list3 '('key3 'subkey3) '('value3))) + (let list3 (AssumeDistinct list3 '('key3 'subkey3) '('value3))) + + (let join (EquiJoin '(list1 'a) '(list2 'b) '(list3 'c) '('Left '('Inner 'a 'b '('a 'key1 'a 'subkey1) '('b 'key2 'b 'subkey2) '()) 'c '('b 'key2 'b 'subkey2) '('c 'key3 'c 'subkey3) '()) '())) + (let lazy (LazyList join)) + + (let res_sink (DataSink 'yt (quote plato))) + (let world (Write! world res_sink (Key '('table (String 'Output))) lazy '('('mode 'renew)))) + + (let world (Commit! world res_sink)) + (return world) +) + )"; + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TDistinctConstraintNode>(exprRoot, "LazyList", "Distinct((a.key1,a.subkey1)(a.value1)(b.key2,b.subkey2)(b.value2))"); + } + + Y_UNIT_TEST(EquiJoinLeftLeft) { + const auto s = R"( +( + (let list1 (AsList + (AsStruct '('key1 (Int32 '1)) '('subkey1 (Uint8 '0)) '('value1 (String 'A))) + (AsStruct '('key1 (Int32 '7)) '('subkey1 (Uint8 '0)) '('value1 (String 'B))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '0)) '('value1 (String 'C))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '1)) '('value1 (String 'D))) + )) + + (let list1 (AssumeUnique list1 '('key1 'subkey1) '('value1))) + (let list1 (AssumeDistinct list1 '('key1 'subkey1) '('value1))) + + (let list2 (AsList + (AsStruct '('key2 (Int32 '9)) '('subkey2 (Uint8 '0)) '('value2 (String 'Z))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '0)) '('value2 (String 'Y))) + (AsStruct '('key2 (Int32 '3)) '('subkey2 (Uint8 '1)) '('value2 (String 'X))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '1)) '('value2 (String 'W))) + (AsStruct '('key2 (Int32 '8)) '('subkey2 (Uint8 '1)) '('value2 (String 'V))) + )) + + (let list2 (AssumeUnique list2 '('key2 'subkey2) '('value2))) + (let list2 (AssumeDistinct list2 '('key2 'subkey2) '('value2))) + + (let list3 (AsList + (AsStruct '('key3 (Int32 '1)) '('subkey3 (Uint8 '0)) '('value3 (String 'G))) + (AsStruct '('key3 (Int32 '4)) '('subkey3 (Uint8 '1)) '('value3 (String 'H))) + (AsStruct '('key3 (Int32 '2)) '('subkey3 (Uint8 '0)) '('value3 (String 'I))) + (AsStruct '('key3 (Int32 '3)) '('subkey3 (Uint8 '1)) '('value3 (String 'J))) + (AsStruct '('key3 (Int32 '3)) '('subkey3 (Uint8 '0)) '('value3 (String 'K))) + )) + + (let list3 (AssumeUnique list3 '('key3 'subkey3) '('value3))) + (let list3 (AssumeDistinct list3 '('key3 'subkey3) '('value3))) + + (let join (EquiJoin '(list1 'a) '(list2 'b) '(list3 'c) '('Left '('Left 'a 'b '('a 'key1 'a 'subkey1) '('b 'key2 'b 'subkey2) '()) 'c '('b 'key2 'b 'subkey2) '('c 'key3 'c 'subkey3) '()) '())) + (let lazy (LazyList join)) + + (let res_sink (DataSink 'yt (quote plato))) + (let world (Write! world res_sink (Key '('table (String 'Output))) lazy '('('mode 'renew)))) + + (let world (Commit! world res_sink)) + (return world) +) + )"; + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TUniqueConstraintNode>(exprRoot, "LazyList", "Unique((a.key1,a.subkey1)(a.value1)(b.key2,b.subkey2)(b.value2)(c.key3,c.subkey3)(c.value3))"); + CheckConstraint<TDistinctConstraintNode>(exprRoot, "LazyList", "Distinct((a.key1,a.subkey1)(a.value1))"); + } + + Y_UNIT_TEST(EquiJoinLeftRight) { + const auto s = R"( +( + (let list1 (AsList + (AsStruct '('key1 (Int32 '1)) '('subkey1 (Uint8 '0)) '('value1 (String 'A))) + (AsStruct '('key1 (Int32 '7)) '('subkey1 (Uint8 '0)) '('value1 (String 'B))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '0)) '('value1 (String 'C))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '1)) '('value1 (String 'D))) + )) + + (let list1 (AssumeUnique list1 '('key1 'subkey1) '('value1))) + (let list1 (AssumeDistinct list1 '('key1 'subkey1) '('value1))) + + (let list2 (AsList + (AsStruct '('key2 (Int32 '9)) '('subkey2 (Uint8 '0)) '('value2 (String 'Z))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '0)) '('value2 (String 'Y))) + (AsStruct '('key2 (Int32 '3)) '('subkey2 (Uint8 '1)) '('value2 (String 'X))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '1)) '('value2 (String 'W))) + (AsStruct '('key2 (Int32 '8)) '('subkey2 (Uint8 '1)) '('value2 (String 'V))) + )) + + (let list2 (AssumeUnique list2 '('key2 'subkey2) '('value2))) + (let list2 (AssumeDistinct list2 '('key2 'subkey2) '('value2))) + + (let list3 (AsList + (AsStruct '('key3 (Int32 '1)) '('subkey3 (Uint8 '0)) '('value3 (String 'G))) + (AsStruct '('key3 (Int32 '4)) '('subkey3 (Uint8 '1)) '('value3 (String 'H))) + (AsStruct '('key3 (Int32 '2)) '('subkey3 (Uint8 '0)) '('value3 (String 'I))) + (AsStruct '('key3 (Int32 '3)) '('subkey3 (Uint8 '1)) '('value3 (String 'J))) + (AsStruct '('key3 (Int32 '3)) '('subkey3 (Uint8 '0)) '('value3 (String 'K))) + )) + + (let list3 (AssumeUnique list3 '('key3 'subkey3) '('value3))) + (let list3 (AssumeDistinct list3 '('key3 'subkey3) '('value3))) + + (let join (EquiJoin '(list1 'a) '(list2 'b) '(list3 'c) '('Left '('Right 'a 'b '('a 'key1 'a 'subkey1) '('b 'key2 'b 'subkey2) '()) 'c '('b 'key2 'b 'subkey2) '('c 'key3 'c 'subkey3) '()) '())) + (let lazy (LazyList join)) + + (let res_sink (DataSink 'yt (quote plato))) + (let world (Write! world res_sink (Key '('table (String 'Output))) lazy '('('mode 'renew)))) + + (let world (Commit! world res_sink)) + (return world) +) + )"; + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TUniqueConstraintNode>(exprRoot, "LazyList", "Unique((a.key1,a.subkey1)(a.value1)(b.key2,b.subkey2)(b.value2)(c.key3,c.subkey3)(c.value3))"); + CheckConstraint<TDistinctConstraintNode>(exprRoot, "LazyList", "Distinct((b.key2,b.subkey2)(b.value2))"); + } + + Y_UNIT_TEST(EquiJoinLeftFull) { + const auto s = R"( +( + (let list1 (AsList + (AsStruct '('key1 (Int32 '1)) '('subkey1 (Uint8 '0)) '('value1 (String 'A))) + (AsStruct '('key1 (Int32 '7)) '('subkey1 (Uint8 '0)) '('value1 (String 'B))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '0)) '('value1 (String 'C))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '1)) '('value1 (String 'D))) + )) + + (let list1 (AssumeUnique list1 '('key1 'subkey1) '('value1))) + (let list1 (AssumeDistinct list1 '('key1 'subkey1) '('value1))) + + (let list2 (AsList + (AsStruct '('key2 (Int32 '9)) '('subkey2 (Uint8 '0)) '('value2 (String 'Z))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '0)) '('value2 (String 'Y))) + (AsStruct '('key2 (Int32 '3)) '('subkey2 (Uint8 '1)) '('value2 (String 'X))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '1)) '('value2 (String 'W))) + (AsStruct '('key2 (Int32 '8)) '('subkey2 (Uint8 '1)) '('value2 (String 'V))) + )) + + (let list2 (AssumeUnique list2 '('key2 'subkey2) '('value2))) + (let list2 (AssumeDistinct list2 '('key2 'subkey2) '('value2))) + + (let list3 (AsList + (AsStruct '('key3 (Int32 '1)) '('subkey3 (Uint8 '0)) '('value3 (String 'G))) + (AsStruct '('key3 (Int32 '4)) '('subkey3 (Uint8 '1)) '('value3 (String 'H))) + (AsStruct '('key3 (Int32 '2)) '('subkey3 (Uint8 '0)) '('value3 (String 'I))) + (AsStruct '('key3 (Int32 '3)) '('subkey3 (Uint8 '1)) '('value3 (String 'J))) + (AsStruct '('key3 (Int32 '3)) '('subkey3 (Uint8 '0)) '('value3 (String 'K))) + )) + + (let list3 (AssumeUnique list3 '('key3 'subkey3) '('value3))) + (let list3 (AssumeDistinct list3 '('key3 'subkey3) '('value3))) + + (let join (EquiJoin '(list1 'a) '(list2 'b) '(list3 'c) '('Left '('Full 'a 'b '('a 'key1 'a 'subkey1) '('b 'key2 'b 'subkey2) '()) 'c '('b 'key2 'b 'subkey2) '('c 'key3 'c 'subkey3) '()) '())) + (let lazy (LazyList join)) + + (let res_sink (DataSink 'yt (quote plato))) + (let world (Write! world res_sink (Key '('table (String 'Output))) lazy '('('mode 'renew)))) + + (let world (Commit! world res_sink)) + (return world) +) + )"; + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TUniqueConstraintNode>(exprRoot, "LazyList", "Unique((a.key1,a.subkey1)(a.value1)(b.key2,b.subkey2)(b.value2)(c.key3,c.subkey3)(c.value3))"); + CheckConstraint<TDistinctConstraintNode>(exprRoot, "LazyList", ""); + } + + Y_UNIT_TEST(EquiJoinLeftExclusion) { + const auto s = R"( +( + (let list1 (AsList + (AsStruct '('key1 (Int32 '1)) '('subkey1 (Uint8 '0)) '('value1 (String 'A))) + (AsStruct '('key1 (Int32 '7)) '('subkey1 (Uint8 '0)) '('value1 (String 'B))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '0)) '('value1 (String 'C))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '1)) '('value1 (String 'D))) + )) + + (let list1 (AssumeUnique list1 '('key1 'subkey1) '('value1))) + (let list1 (AssumeDistinct list1 '('key1 'subkey1) '('value1))) + + (let list2 (AsList + (AsStruct '('key2 (Int32 '9)) '('subkey2 (Uint8 '0)) '('value2 (String 'Z))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '0)) '('value2 (String 'Y))) + (AsStruct '('key2 (Int32 '3)) '('subkey2 (Uint8 '1)) '('value2 (String 'X))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '1)) '('value2 (String 'W))) + (AsStruct '('key2 (Int32 '8)) '('subkey2 (Uint8 '1)) '('value2 (String 'V))) + )) + + (let list2 (AssumeUnique list2 '('key2 'subkey2) '('value2))) + (let list2 (AssumeDistinct list2 '('key2 'subkey2) '('value2))) + + (let list3 (AsList + (AsStruct '('key3 (Int32 '1)) '('subkey3 (Uint8 '0)) '('value3 (String 'G))) + (AsStruct '('key3 (Int32 '4)) '('subkey3 (Uint8 '1)) '('value3 (String 'H))) + (AsStruct '('key3 (Int32 '2)) '('subkey3 (Uint8 '0)) '('value3 (String 'I))) + (AsStruct '('key3 (Int32 '3)) '('subkey3 (Uint8 '1)) '('value3 (String 'J))) + (AsStruct '('key3 (Int32 '3)) '('subkey3 (Uint8 '0)) '('value3 (String 'K))) + )) + + (let list3 (AssumeUnique list3 '('key3 'subkey3) '('value3))) + (let list3 (AssumeDistinct list3 '('key3 'subkey3) '('value3))) + + (let join (EquiJoin '(list1 'a) '(list2 'b) '(list3 'c) '('Left '('Exclusion 'a 'b '('a 'key1 'a 'subkey1) '('b 'key2 'b 'subkey2) '()) 'c '('b 'key2 'b 'subkey2) '('c 'key3 'c 'subkey3) '()) '())) + (let lazy (LazyList join)) + + (let res_sink (DataSink 'yt (quote plato))) + (let world (Write! world res_sink (Key '('table (String 'Output))) lazy '('('mode 'renew)))) + + (let world (Commit! world res_sink)) + (return world) +) + )"; + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TUniqueConstraintNode>(exprRoot, "LazyList", "Unique((a.key1,a.subkey1)(a.value1)(b.key2,b.subkey2)(b.value2)(c.key3,c.subkey3)(c.value3))"); + CheckConstraint<TDistinctConstraintNode>(exprRoot, "LazyList", ""); + } + + Y_UNIT_TEST(EquiJoinLeftLeftOnly) { + const auto s = R"( +( + (let list1 (AsList + (AsStruct '('key1 (Int32 '1)) '('subkey1 (Uint8 '0)) '('value1 (String 'A))) + (AsStruct '('key1 (Int32 '7)) '('subkey1 (Uint8 '0)) '('value1 (String 'B))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '0)) '('value1 (String 'C))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '1)) '('value1 (String 'D))) + )) + + (let list1 (AssumeUnique list1 '('key1 'subkey1) '('value1))) + (let list1 (AssumeDistinct list1 '('key1 'subkey1) '('value1))) + + (let list2 (AsList + (AsStruct '('key2 (Int32 '9)) '('subkey2 (Uint8 '0)) '('value2 (String 'Z))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '0)) '('value2 (String 'Y))) + (AsStruct '('key2 (Int32 '3)) '('subkey2 (Uint8 '1)) '('value2 (String 'X))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '1)) '('value2 (String 'W))) + (AsStruct '('key2 (Int32 '8)) '('subkey2 (Uint8 '1)) '('value2 (String 'V))) + )) + + (let list2 (AssumeUnique list2 '('key2 'subkey2) '('value2))) + (let list2 (AssumeDistinct list2 '('key2 'subkey2) '('value2))) + + (let list3 (AsList + (AsStruct '('key3 (Int32 '1)) '('subkey3 (Uint8 '0)) '('value3 (String 'G))) + (AsStruct '('key3 (Int32 '4)) '('subkey3 (Uint8 '1)) '('value3 (String 'H))) + (AsStruct '('key3 (Int32 '2)) '('subkey3 (Uint8 '0)) '('value3 (String 'I))) + (AsStruct '('key3 (Int32 '3)) '('subkey3 (Uint8 '1)) '('value3 (String 'J))) + (AsStruct '('key3 (Int32 '3)) '('subkey3 (Uint8 '0)) '('value3 (String 'K))) + )) + + (let list3 (AssumeUnique list3 '('key3 'subkey3) '('value3))) + (let list3 (AssumeDistinct list3 '('key3 'subkey3) '('value3))) + + (let join (EquiJoin '(list1 'a) '(list2 'b) '(list3 'c) '('Left '('LeftOnly 'a 'b '('a 'key1 'a 'subkey1) '('b 'key2 'b 'subkey2) '()) 'c '('a 'key1 'a 'subkey1) '('c 'key3 'c 'subkey3) '()) '())) + (let lazy (LazyList join)) + + (let res_sink (DataSink 'yt (quote plato))) + (let world (Write! world res_sink (Key '('table (String 'Output))) lazy '('('mode 'renew)))) + + (let world (Commit! world res_sink)) + (return world) +) + )"; + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TUniqueConstraintNode>(exprRoot, "LazyList", "Unique((a.key1,a.subkey1)(a.value1)(c.key3,c.subkey3)(c.value3))"); + CheckConstraint<TDistinctConstraintNode>(exprRoot, "LazyList", "Distinct((a.key1,a.subkey1)(a.value1))"); + } + + Y_UNIT_TEST(EquiJoinLeftLeftSemi) { + const auto s = R"( +( + (let list1 (AsList + (AsStruct '('key1 (Int32 '1)) '('subkey1 (Uint8 '0)) '('value1 (String 'A))) + (AsStruct '('key1 (Int32 '7)) '('subkey1 (Uint8 '0)) '('value1 (String 'B))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '0)) '('value1 (String 'C))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '1)) '('value1 (String 'D))) + )) + + (let list1 (AssumeUnique list1 '('key1 'subkey1) '('value1))) + (let list1 (AssumeDistinct list1 '('key1 'subkey1) '('value1))) + + (let list2 (AsList + (AsStruct '('key2 (Int32 '9)) '('subkey2 (Uint8 '0)) '('value2 (String 'Z))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '0)) '('value2 (String 'Y))) + (AsStruct '('key2 (Int32 '3)) '('subkey2 (Uint8 '1)) '('value2 (String 'X))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '1)) '('value2 (String 'W))) + (AsStruct '('key2 (Int32 '8)) '('subkey2 (Uint8 '1)) '('value2 (String 'V))) + )) + + (let list2 (AssumeUnique list2 '('key2 'subkey2) '('value2))) + (let list2 (AssumeDistinct list2 '('key2 'subkey2) '('value2))) + + (let list3 (AsList + (AsStruct '('key3 (Int32 '1)) '('subkey3 (Uint8 '0)) '('value3 (String 'G))) + (AsStruct '('key3 (Int32 '4)) '('subkey3 (Uint8 '1)) '('value3 (String 'H))) + (AsStruct '('key3 (Int32 '2)) '('subkey3 (Uint8 '0)) '('value3 (String 'I))) + (AsStruct '('key3 (Int32 '3)) '('subkey3 (Uint8 '1)) '('value3 (String 'J))) + (AsStruct '('key3 (Int32 '3)) '('subkey3 (Uint8 '0)) '('value3 (String 'K))) + )) + + (let list3 (AssumeUnique list3 '('key3 'subkey3) '('value3))) + (let list3 (AssumeDistinct list3 '('key3 'subkey3) '('value3))) + + (let join (EquiJoin '(list1 'a) '(list2 'b) '(list3 'c) '('Left '('LeftSemi 'a 'b '('a 'key1 'a 'subkey1) '('b 'key2 'b 'subkey2) '()) 'c '('a 'key1 'a 'subkey1) '('c 'key3 'c 'subkey3) '()) '())) + (let lazy (LazyList join)) + + (let res_sink (DataSink 'yt (quote plato))) + (let world (Write! world res_sink (Key '('table (String 'Output))) lazy '('('mode 'renew)))) + + (let world (Commit! world res_sink)) + (return world) +) + )"; + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TUniqueConstraintNode>(exprRoot, "LazyList", "Unique((a.key1,a.subkey1)(a.value1)(c.key3,c.subkey3)(c.value3))"); + CheckConstraint<TDistinctConstraintNode>(exprRoot, "LazyList", "Distinct((a.key1,a.subkey1)(a.value1))"); + } + + Y_UNIT_TEST(EquiJoinLeftRightOnly) { + const auto s = R"( +( + (let list1 (AsList + (AsStruct '('key1 (Int32 '1)) '('subkey1 (Uint8 '0)) '('value1 (String 'A))) + (AsStruct '('key1 (Int32 '7)) '('subkey1 (Uint8 '0)) '('value1 (String 'B))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '0)) '('value1 (String 'C))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '1)) '('value1 (String 'D))) + )) + + (let list1 (AssumeUnique list1 '('key1 'subkey1) '('value1))) + (let list1 (AssumeDistinct list1 '('key1 'subkey1) '('value1))) + + (let list2 (AsList + (AsStruct '('key2 (Int32 '9)) '('subkey2 (Uint8 '0)) '('value2 (String 'Z))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '0)) '('value2 (String 'Y))) + (AsStruct '('key2 (Int32 '3)) '('subkey2 (Uint8 '1)) '('value2 (String 'X))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '1)) '('value2 (String 'W))) + (AsStruct '('key2 (Int32 '8)) '('subkey2 (Uint8 '1)) '('value2 (String 'V))) + )) + + (let list2 (AssumeUnique list2 '('key2 'subkey2) '('value2))) + (let list2 (AssumeDistinct list2 '('key2 'subkey2) '('value2))) + + (let list3 (AsList + (AsStruct '('key3 (Int32 '1)) '('subkey3 (Uint8 '0)) '('value3 (String 'G))) + (AsStruct '('key3 (Int32 '4)) '('subkey3 (Uint8 '1)) '('value3 (String 'H))) + (AsStruct '('key3 (Int32 '2)) '('subkey3 (Uint8 '0)) '('value3 (String 'I))) + (AsStruct '('key3 (Int32 '3)) '('subkey3 (Uint8 '1)) '('value3 (String 'J))) + (AsStruct '('key3 (Int32 '3)) '('subkey3 (Uint8 '0)) '('value3 (String 'K))) + )) + + (let list3 (AssumeUnique list3 '('key3 'subkey3) '('value3))) + (let list3 (AssumeDistinct list3 '('key3 'subkey3) '('value3))) + + (let join (EquiJoin '(list1 'a) '(list2 'b) '(list3 'c) '('Left '('RightOnly 'a 'b '('a 'key1 'a 'subkey1) '('b 'key2 'b 'subkey2) '()) 'c '('b 'key2 'b 'subkey2) '('c 'key3 'c 'subkey3) '()) '())) + (let lazy (LazyList join)) + + (let res_sink (DataSink 'yt (quote plato))) + (let world (Write! world res_sink (Key '('table (String 'Output))) lazy '('('mode 'renew)))) + + (let world (Commit! world res_sink)) + (return world) +) + )"; + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TUniqueConstraintNode>(exprRoot, "LazyList", "Unique((b.key2,b.subkey2)(b.value2)(c.key3,c.subkey3)(c.value3))"); + CheckConstraint<TDistinctConstraintNode>(exprRoot, "LazyList", "Distinct((b.key2,b.subkey2)(b.value2))"); + } + + Y_UNIT_TEST(EquiJoinLeftRightSemi) { + const auto s = R"( +( + (let list1 (AsList + (AsStruct '('key1 (Int32 '1)) '('subkey1 (Uint8 '0)) '('value1 (String 'A))) + (AsStruct '('key1 (Int32 '7)) '('subkey1 (Uint8 '0)) '('value1 (String 'B))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '0)) '('value1 (String 'C))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '1)) '('value1 (String 'D))) + )) + + (let list1 (AssumeUnique list1 '('key1 'subkey1) '('value1))) + (let list1 (AssumeDistinct list1 '('key1 'subkey1) '('value1))) + + (let list2 (AsList + (AsStruct '('key2 (Int32 '9)) '('subkey2 (Uint8 '0)) '('value2 (String 'Z))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '0)) '('value2 (String 'Y))) + (AsStruct '('key2 (Int32 '3)) '('subkey2 (Uint8 '1)) '('value2 (String 'X))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '1)) '('value2 (String 'W))) + (AsStruct '('key2 (Int32 '8)) '('subkey2 (Uint8 '1)) '('value2 (String 'V))) + )) + + (let list2 (AssumeUnique list2 '('key2 'subkey2) '('value2))) + (let list2 (AssumeDistinct list2 '('key2 'subkey2) '('value2))) + + (let list3 (AsList + (AsStruct '('key3 (Int32 '1)) '('subkey3 (Uint8 '0)) '('value3 (String 'G))) + (AsStruct '('key3 (Int32 '4)) '('subkey3 (Uint8 '1)) '('value3 (String 'H))) + (AsStruct '('key3 (Int32 '2)) '('subkey3 (Uint8 '0)) '('value3 (String 'I))) + (AsStruct '('key3 (Int32 '3)) '('subkey3 (Uint8 '1)) '('value3 (String 'J))) + (AsStruct '('key3 (Int32 '3)) '('subkey3 (Uint8 '0)) '('value3 (String 'K))) + )) + + (let list3 (AssumeUnique list3 '('key3 'subkey3) '('value3))) + (let list3 (AssumeDistinct list3 '('key3 'subkey3) '('value3))) + + (let join (EquiJoin '(list1 'a) '(list2 'b) '(list3 'c) '('Left '('RightSemi 'a 'b '('a 'key1 'a 'subkey1) '('b 'key2 'b 'subkey2) '()) 'c '('b 'key2 'b 'subkey2) '('c 'key3 'c 'subkey3) '()) '())) + (let lazy (LazyList join)) + + (let res_sink (DataSink 'yt (quote plato))) + (let world (Write! world res_sink (Key '('table (String 'Output))) lazy '('('mode 'renew)))) + + (let world (Commit! world res_sink)) + (return world) +) + )"; + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TUniqueConstraintNode>(exprRoot, "LazyList", "Unique((b.key2,b.subkey2)(b.value2)(c.key3,c.subkey3)(c.value3))"); + CheckConstraint<TDistinctConstraintNode>(exprRoot, "LazyList", "Distinct((b.key2,b.subkey2)(b.value2))"); + } + + Y_UNIT_TEST(EquiJoinFlatten) { + const auto s = R"( +( + (let list1 (AsList + (AsStruct '('key1 (Int32 '1)) '('subkey1 (Uint8 '0)) '('value1 (String 'A))) + (AsStruct '('key1 (Int32 '7)) '('subkey1 (Uint8 '0)) '('value1 (String 'B))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '0)) '('value1 (String 'C))) + (AsStruct '('key1 (Int32 '4)) '('subkey1 (Uint8 '1)) '('value1 (String 'D))) + )) + + (let list1 (AssumeUnique list1 '('key1 'subkey1) '('value1))) + (let list1 (AssumeDistinct list1 '('key1 'subkey1) '('value1))) + + (let list2 (AsList + (AsStruct '('key2 (Int32 '9)) '('subkey2 (Uint8 '0)) '('value2 (String 'Z))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '0)) '('value2 (String 'Y))) + (AsStruct '('key2 (Int32 '3)) '('subkey2 (Uint8 '1)) '('value2 (String 'X))) + (AsStruct '('key2 (Int32 '4)) '('subkey2 (Uint8 '1)) '('value2 (String 'W))) + (AsStruct '('key2 (Int32 '8)) '('subkey2 (Uint8 '1)) '('value2 (String 'V))) + )) + + (let list2 (AssumeUnique list2 '('key2 'subkey2) '('value2))) + (let list2 (AssumeDistinct list2 '('key2 'subkey2) '('value2))) + + (let list3 (AsList + (AsStruct '('key3 (Int32 '1)) '('subkey3 (Uint8 '0)) '('value3 (String 'G))) + (AsStruct '('key3 (Int32 '4)) '('subkey3 (Uint8 '1)) '('value3 (String 'H))) + (AsStruct '('key3 (Int32 '2)) '('subkey3 (Uint8 '0)) '('value3 (String 'I))) + (AsStruct '('key3 (Int32 '3)) '('subkey3 (Uint8 '1)) '('value3 (String 'J))) + (AsStruct '('key3 (Int32 '3)) '('subkey3 (Uint8 '0)) '('value3 (String 'K))) + )) + + (let list3 (AssumeUnique list3 '('key3 'subkey3) '('value3))) + (let list3 (AssumeDistinct list3 '('key3 'subkey3) '('value3))) + + (let join (EquiJoin '(list1 'a) '(list2 'b) '(list3 'c) '('Inner '('Inner 'a 'b '('a 'key1 'a 'subkey1) '('b 'key2 'b 'subkey2) '()) 'c '('b 'key2 'b 'subkey2) '('c 'key3 'c 'subkey3) '()) '('('flatten)))) + (let lazy (LazyList join)) + + (let res_sink (DataSink 'yt (quote plato))) + (let world (Write! world res_sink (Key '('table (String 'Output))) lazy '('('mode 'renew)))) + + (let world (Commit! world res_sink)) + (return world) +) + )"; + TExprContext exprCtx; + const auto exprRoot = ParseAndAnnotate(s, exprCtx); + CheckConstraint<TUniqueConstraintNode>(exprRoot, "LazyList", ""); + CheckConstraint<TDistinctConstraintNode>(exprRoot, "LazyList", ""); + } +} + +} // namespace NYql diff --git a/ydb/library/yql/core/ut/yql_expr_discover_ut.cpp b/ydb/library/yql/core/ut/yql_expr_discover_ut.cpp new file mode 100644 index 00000000000..b754ee74433 --- /dev/null +++ b/ydb/library/yql/core/ut/yql_expr_discover_ut.cpp @@ -0,0 +1,152 @@ +#include "yql_opt_proposed_by_data.h" + +#include <ydb/library/yql/providers/yt/provider/yql_yt_provider.h> +#include <ydb/library/yql/providers/yt/gateway/file/yql_yt_file.h> +#include <ydb/library/yql/providers/yt/gateway/file/yql_yt_file_services.h> +#include <ydb/library/yql/providers/common/provider/yql_provider_names.h> +#include <ydb/library/yql/ast/yql_ast_annotation.h> +#include <ydb/library/yql/ast/yql_expr.h> +#include <ydb/library/yql/core/type_ann/type_ann_core.h> +#include <ydb/library/yql/core/ut_common/yql_ut_common.h> +#include <ydb/library/yql/minikql/invoke_builtins/mkql_builtins.h> +#include <ydb/library/yql/core/services/yql_eval_expr.h> + +#include <library/cpp/testing/unittest/registar.h> +#include <library/cpp/yson/writer.h> + +namespace NYql { + +Y_UNIT_TEST_SUITE(TDiscoverYqlExpr) { + + static TString Discover(const TString& ast) { + TAstParseResult astRes = ParseAst(ast); + UNIT_ASSERT(astRes.IsOk()); + TExprContext exprCtx; + TExprNode::TPtr exprRoot; + UNIT_ASSERT(CompileExpr(*astRes.Root, exprRoot, exprCtx, nullptr, nullptr)); + + auto functionRegistry = NKikimr::NMiniKQL::CreateFunctionRegistry(NKikimr::NMiniKQL::CreateBuiltinRegistry()); + TTestTablesMapping testTables; + auto yqlNativeServices = NFile::TYtFileServices::Make(functionRegistry.Get(), testTables); + auto ytGateway = CreateYtFileGateway(yqlNativeServices); + auto typeAnnotationContext = MakeIntrusive<TTypeAnnotationContext>(); + typeAnnotationContext->DiscoveryMode = true; + auto ytState = MakeIntrusive<TYtState>(); + ytState->Gateway = ytGateway; + ytState->Types = typeAnnotationContext.Get(); + + InitializeYtGateway(ytGateway, ytState); + auto randomProvider = CreateDeterministicRandomProvider(1); + typeAnnotationContext->AddDataSink(YtProviderName, CreateYtDataSink(ytState)); + auto datasource = CreateYtDataSource(ytState); + typeAnnotationContext->AddDataSource(YtProviderName, datasource); + auto intentTransformer = CreateIntentDeterminationTransformer(*typeAnnotationContext); + TVector<TTransformStage> transformers; + const auto issueCode = TIssuesIds::DEFAULT_ERROR; + transformers.push_back(TTransformStage(CreateFunctorTransformer( + [=](const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) { + return EvaluateExpression(input, output, *typeAnnotationContext, ctx, *functionRegistry, nullptr); + }), + "EvaluateExpression", + issueCode)); + transformers.push_back(TTransformStage( + CreateIODiscoveryTransformer(*typeAnnotationContext), + "IODiscovery", + issueCode)); + transformers.push_back(TTransformStage( + CreateEpochsTransformer(*typeAnnotationContext), + "Epochs", + issueCode)); + transformers.push_back(TTransformStage( + intentTransformer, + "IntentDetermination", + issueCode)); + auto fullTransformer = CreateCompositeGraphTransformer(transformers, true); + + TStringStream str; + if (SyncTransform(*fullTransformer, exprRoot, exprCtx) == IGraphTransformer::TStatus::Ok) { + NYson::TYsonWriter writer(&str, NYson::EYsonFormat::Text); + datasource->CollectDiscoveredData(writer); + } else { + exprCtx.IssueManager.GetIssues().PrintTo(str); + } + return str.Str(); + } + + Y_UNIT_TEST(DiscoverYt) { + auto s = R"(( +(let mr_source (DataSource 'yt 'plato)) +(let x (Read! world mr_source + (Key '('table (String 'Input))) + '('key 'subkey 'value) '())) +(let world (Left! x)) +(let table (Right! x)) + +(let mr_sink (DataSink 'yt 'plato)) +(let world (Write! world mr_sink + (Key '('table (String 'Output))) + table '('('mode 'append)))) + +(let world (Commit! world mr_sink)) +(return world) +))"; + + auto res = Discover(s); + UNIT_ASSERT_VALUES_EQUAL(res, "[[\"plato\";\"Input\";[\"read\"]];[\"plato\";\"Output\";[\"modify\"]]]"); + } + + Y_UNIT_TEST(ErrorOnRange) { + auto s = R"(( +(let mr_source (DataSource 'yt 'plato)) +(let range (MrTableRange '"" (lambda '($i) (And (>= $i (String '"Input1")) (<= $i (String '"Input2")))) '"")) +(let x (Read! world mr_source + (Key '('table range)) + '('key 'subkey 'value) '())) +(let world (Left! x)) +(let table (Right! x)) + +(let mr_sink (DataSink 'yt 'plato)) +(let world (Write! world mr_sink + (Key '('table (String 'Output))) + table '('('mode 'append)))) + +(let world (Commit! world mr_sink)) +(return world) +))"; + + auto res = Discover(s); + UNIT_ASSERT_VALUES_EQUAL(res, R"(<main>: Error: Default error + <main>:5:6: Error: MrTableRange/MrTableRangeStrict is not allowed in Discovery mode, code: 4600 +)"); + } + + Y_UNIT_TEST(ErrorOnTime) { + auto s = R"(( +(let mr_source (DataSource 'yt 'plato)) +(let x (Read! world mr_source + (Key '('table (String (EvaluateAtom (SafeCast (CurrentUtcDate) (DataType 'String)))))) + '('key 'subkey 'value) '())) +(let world (Left! x)) +(let table (Right! x)) + +(let mr_sink (DataSink 'yt 'plato)) +(let world (Write! world mr_sink + (Key '('table (String 'Output))) + table '('('mode 'append)))) + +(let world (Commit! world mr_sink)) +(return world) +))"; + + auto res = Discover(s); + UNIT_ASSERT_VALUES_EQUAL(res, R"(<main>: Error: Default error + <main>:4:28: Error: At function: EvaluateAtom + <main>: Error: Type annotation, code: 1030 + <main>:4:42: Error: At function: SafeCast + <main>:4:52: Error: At function: CurrentUtcDate + <main>:4:52: Error: CurrentUtcDate is not allowed in Discovery mode, code: 4600 +)"); + } +} + +} // namespace NYql diff --git a/ydb/library/yql/core/ut/yql_expr_optimize_ut.cpp b/ydb/library/yql/core/ut/yql_expr_optimize_ut.cpp new file mode 100644 index 00000000000..024e0b437b1 --- /dev/null +++ b/ydb/library/yql/core/ut/yql_expr_optimize_ut.cpp @@ -0,0 +1,656 @@ +#include "yql_expr_optimize.h" +#include "yql_opt_rewrite_io.h" +#include "yql_opt_proposed_by_data.h" + +#include <ydb/library/yql/providers/yt/provider/yql_yt_provider.h> +#include <ydb/library/yql/providers/yt/gateway/file/yql_yt_file.h> +#include <ydb/library/yql/providers/yt/gateway/file/yql_yt_file_services.h> +#include <ydb/library/yql/providers/common/provider/yql_provider_names.h> +#include <ydb/library/yql/ast/yql_ast_annotation.h> +#include <ydb/library/yql/ast/yql_expr.h> +#include <ydb/library/yql/core/type_ann/type_ann_core.h> +#include <ydb/library/yql/core/type_ann/type_ann_expr.h> +#include <ydb/library/yql/core/ut_common/yql_ut_common.h> +#include <ydb/library/yql/core/facade/yql_facade.h> +#include <ydb/library/yql/minikql/invoke_builtins/mkql_builtins.h> + +#include <library/cpp/testing/unittest/registar.h> + +namespace NYql { + +Y_UNIT_TEST_SUITE(TOptimizeYqlExpr) { + Y_UNIT_TEST(CombineAtoms) { + const auto s = "(\n" + "(let x (Combine '11 '333 '7))\n" + "(return x)\n" + ")\n"; + + const auto astRes = ParseAst(s); + UNIT_ASSERT(astRes.IsOk()); + TExprContext exprCtx; + TExprNode::TPtr exprRoot; + UNIT_ASSERT(CompileExpr(*astRes.Root, exprRoot, exprCtx, nullptr, nullptr)); + UNIT_ASSERT_EQUAL(IGraphTransformer::TStatus::Repeat, ExpandApply(exprRoot, exprRoot, exprCtx).Level); + + auto ast = ConvertToAst(*exprRoot, exprCtx, TExprAnnotationFlags::None, true); + auto strRes = ast.Root->ToString(TAstPrintFlags::PerLine | TAstPrintFlags::ShortQuote); + UNIT_ASSERT(strRes.find("(return '113337)") != TString::npos); + } + + Y_UNIT_TEST(RecursiveLambda) { + const auto s = + R"( + ( + (let f**k (lambda '(x l) (+ x (Apply l x l)))) + (return (Apply f**k (Uint32 '1) f**k)) + ) + )"; + + const auto astRes = ParseAst(s); + UNIT_ASSERT(astRes.IsOk()); + TExprContext exprCtx; + TExprNode::TPtr exprRoot; + UNIT_ASSERT(CompileExpr(*astRes.Root, exprRoot, exprCtx, nullptr, nullptr)); + + for (size_t i = 0U; i < 0x100; ++i) + UNIT_ASSERT_EQUAL(IGraphTransformer::TStatus::Repeat, ExpandApply(exprRoot, exprRoot, exprCtx).Level); + + const auto ast = ConvertToAst(*exprRoot, exprCtx, TExprAnnotationFlags::None, true); + const auto strRes = ast.Root->ToString(TAstPrintFlags::PerLine | TAstPrintFlags::ShortQuote); + UNIT_ASSERT_EQUAL(0x101, std::count(strRes.cbegin(), strRes.cend(), '+')); + } + + Y_UNIT_TEST(ApplyWideLambda) { + const auto s = + R"( + ( + (let wide (lambda '(x y) (+ x y) (* x y) x y)) + (return '('1 (Apply wide (Int32 '3) (Int32 '7)) '9)) + ) + )"; + + const auto astRes = ParseAst(s); + UNIT_ASSERT(astRes.IsOk()); + TExprContext exprCtx; + TExprNode::TPtr exprRoot; + UNIT_ASSERT(CompileExpr(*astRes.Root, exprRoot, exprCtx, nullptr, nullptr)); + UNIT_ASSERT_EQUAL(IGraphTransformer::TStatus::Repeat, ExpandApply(exprRoot, exprRoot, exprCtx).Level); + + auto ast = ConvertToAst(*exprRoot, exprCtx, TExprAnnotationFlags::None, true); + auto strRes = ast.Root->ToString(TAstPrintFlags::PerLine | TAstPrintFlags::ShortQuote); + UNIT_ASSERT(strRes.find("(return '('1 (+ $1 $2) (* $1 $2) $1 $2 '9))") != TString::npos); + } + + Y_UNIT_TEST(ApplyThinLambda) { + const auto s = + R"( + ( + (let wide (lambda '(x y))) + (return '('1 (Apply wide (Int32 '3) (Int32 '7)) '9)) + ) + )"; + + const auto astRes = ParseAst(s); + UNIT_ASSERT(astRes.IsOk()); + TExprContext exprCtx; + TExprNode::TPtr exprRoot; + UNIT_ASSERT(CompileExpr(*astRes.Root, exprRoot, exprCtx, nullptr, nullptr)); + UNIT_ASSERT_EQUAL(IGraphTransformer::TStatus::Repeat, ExpandApply(exprRoot, exprRoot, exprCtx).Level); + + auto ast = ConvertToAst(*exprRoot, exprCtx, TExprAnnotationFlags::None, true); + auto strRes = ast.Root->ToString(TAstPrintFlags::PerLine | TAstPrintFlags::ShortQuote); + UNIT_ASSERT(strRes.find("(return '('1 '9))") != TString::npos); + } + + Y_UNIT_TEST(ApplyDeepLambda) { + const auto s = "# program\n" + "(\n" + "(let x (Uint64 '42))\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "(let l (lambda '(y) (block '(\n" + "\n" + "(let l (lambda '(y) (+ x y)))\n" + "\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "(return (Apply l (+ y x)))))))\n" + "\n" + "(let res_sink (DataSink 'result))\n" + "(let resKey (Apply l (Int64 '7)))\n" + "(let world (Write! world res_sink (Key) resKey '('('type))))\n" + "(let world (Commit! world res_sink))\n" + "(return world)\n" + ")\n"; + + const auto astRes = ParseAst(s); + UNIT_ASSERT(astRes.IsOk()); + TExprContext exprCtx; + TExprNode::TPtr exprRoot; + UNIT_ASSERT(CompileExpr(*astRes.Root, exprRoot, exprCtx, nullptr, nullptr)); + + while (true) { + const auto ret = ExpandApply(exprRoot, exprRoot, exprCtx); + if (ret.Level != IGraphTransformer::TStatus::Repeat) { + UNIT_ASSERT_EQUAL(ret.Level, IGraphTransformer::TStatus::Ok); + break; + } + } + + auto ast = ConvertToAst(*exprRoot, exprCtx, TExprAnnotationFlags::None, true); + auto strRes = ast.Root->ToString(TAstPrintFlags::PerLine | TAstPrintFlags::ShortQuote); + UNIT_ASSERT_EQUAL(strRes.find("lambda"), TString::npos); + } + + Y_UNIT_TEST(Nth) { + const auto s = "(\n" + "(let x '('11 '333 '7))\n" + "(return (Nth x '2))\n" + ")\n"; + + const auto astRes = ParseAst(s); + UNIT_ASSERT(astRes.IsOk()); + TExprContext exprCtx; + TExprNode::TPtr exprRoot; + UNIT_ASSERT(CompileExpr(*astRes.Root, exprRoot, exprCtx, nullptr, nullptr)); + UNIT_ASSERT_EQUAL(IGraphTransformer::TStatus::Repeat, ExpandApply(exprRoot, exprRoot, exprCtx).Level); + + auto ast = ConvertToAst(*exprRoot, exprCtx, TExprAnnotationFlags::None, true); + auto strRes = ast.Root->ToString(TAstPrintFlags::PerLine | TAstPrintFlags::ShortQuote); + UNIT_ASSERT(strRes.find("(return '7)") != TString::npos); + } + + Y_UNIT_TEST(NthLargeIndex) { + const auto s = "(\n" + "(let x '('11 '333 '7))\n" + "(return (Nth x '3))\n" + ")\n"; + + const auto astRes = ParseAst(s); + UNIT_ASSERT(astRes.IsOk()); + TExprContext exprCtx; + TExprNode::TPtr exprRoot; + UNIT_ASSERT(CompileExpr(*astRes.Root, exprRoot, exprCtx, nullptr, nullptr)); + UNIT_ASSERT_EQUAL(IGraphTransformer::TStatus::Error, ExpandApply(exprRoot, exprRoot, exprCtx).Level); + UNIT_ASSERT_VALUES_EQUAL("<main>:3:17: Error: Index too large: (3 >= 3).\n", exprCtx.IssueManager.GetIssues().ToString()); + } + + Y_UNIT_TEST(NthWrongIndex) { + const auto s = "(\n" + "(let x '('11 '333 '7))\n" + "(return (Nth x 'Z))\n" + ")\n"; + + const auto astRes = ParseAst(s); + UNIT_ASSERT(astRes.IsOk()); + TExprContext exprCtx; + TExprNode::TPtr exprRoot; + UNIT_ASSERT(CompileExpr(*astRes.Root, exprRoot, exprCtx, nullptr, nullptr)); + UNIT_ASSERT_EQUAL(IGraphTransformer::TStatus::Error, ExpandApply(exprRoot, exprRoot, exprCtx).Level); + UNIT_ASSERT_VALUES_EQUAL("<main>:3:17: Error: Index 'Z' isn't UI32.\n", exprCtx.IssueManager.GetIssues().ToString()); + } + + Y_UNIT_TEST(NthArg) { + const auto s = "(\n" + "(let x (NthArg '1 (+ '37 '42)))\n" + "(return x)\n" + ")\n"; + + const auto astRes = ParseAst(s); + UNIT_ASSERT(astRes.IsOk()); + TExprContext exprCtx; + TExprNode::TPtr exprRoot; + UNIT_ASSERT(CompileExpr(*astRes.Root, exprRoot, exprCtx, nullptr, nullptr)); + UNIT_ASSERT_EQUAL(IGraphTransformer::TStatus::Repeat, ExpandApply(exprRoot, exprRoot, exprCtx).Level); + + auto ast = ConvertToAst(*exprRoot, exprCtx, TExprAnnotationFlags::None, true); + auto strRes = ast.Root->ToString(TAstPrintFlags::PerLine | TAstPrintFlags::ShortQuote); + UNIT_ASSERT(strRes.find("(return '42)") != TString::npos); + } + + Y_UNIT_TEST(NthArgLargeIndex) { + const auto s = "(\n" + "(let x (NthArg '2 (- '37 '42)))\n" + "(return x)\n" + ")\n"; + + const auto astRes = ParseAst(s); + UNIT_ASSERT(astRes.IsOk()); + TExprContext exprCtx; + TExprNode::TPtr exprRoot; + UNIT_ASSERT(CompileExpr(*astRes.Root, exprRoot, exprCtx, nullptr, nullptr)); + UNIT_ASSERT_EQUAL(IGraphTransformer::TStatus::Error, ExpandApply(exprRoot, exprRoot, exprCtx).Level); + UNIT_ASSERT_VALUES_EQUAL("<main>:2:17: Error: Index too large: (2 >= 2).\n", exprCtx.IssueManager.GetIssues().ToString()); + } + + Y_UNIT_TEST(NthArgWrongIndex) { + const auto s = "(\n" + "(let x (NthArg 'bad (* '37 '42)))\n" + "(return x)\n" + ")\n"; + + const auto astRes = ParseAst(s); + UNIT_ASSERT(astRes.IsOk()); + TExprContext exprCtx; + TExprNode::TPtr exprRoot; + UNIT_ASSERT(CompileExpr(*astRes.Root, exprRoot, exprCtx, nullptr, nullptr)); + UNIT_ASSERT_EQUAL(IGraphTransformer::TStatus::Error, ExpandApply(exprRoot, exprRoot, exprCtx).Level); + UNIT_ASSERT_VALUES_EQUAL("<main>:2:17: Error: Index 'bad' isn't UI32.\n", exprCtx.IssueManager.GetIssues().ToString()); + } + + Y_UNIT_TEST(NthArgNotCallable) { + const auto s = "(\n" + "(let x (NthArg '0 'bad))\n" + "(return x)\n" + ")\n"; + + const auto astRes = ParseAst(s); + UNIT_ASSERT(astRes.IsOk()); + TExprContext exprCtx; + TExprNode::TPtr exprRoot; + UNIT_ASSERT(CompileExpr(*astRes.Root, exprRoot, exprCtx, nullptr, nullptr)); + UNIT_ASSERT_EQUAL(IGraphTransformer::TStatus::Error, ExpandApply(exprRoot, exprRoot, exprCtx).Level); + UNIT_ASSERT_VALUES_EQUAL("<main>:2:20: Error: Expected callable, but got: Atom\n", exprCtx.IssueManager.GetIssues().ToString()); + } + + Y_UNIT_TEST(CheckIORewrite) { + auto s = "(\n" + "(let mr_source (DataSource 'yt 'plato))\n" + "(let x (Read! world mr_source (Key '('table (String 'Input))) '('key 'subkey 'value) '()))\n" + "(let world (Left! x))\n" + "(let table1 (Right! x))\n" + "(let tresh (String '100))\n" + "(let table1low (Filter table1 (lambda '(item1) (> tresh (Member item1 'key)))))\n" + "(let mr_sink (DataSink 'yt (quote plato)))\n" + "(let world (Write! world mr_sink (Key '('table (String 'Output))) table1low '('('mode 'append))))\n" + "(let world (Commit! world mr_sink))\n" + "(return world)\n" + ")\n"; + + TAstParseResult astRes = ParseAst(s); + UNIT_ASSERT(astRes.IsOk()); + TExprContext exprCtx; + TExprNode::TPtr exprRoot; + UNIT_ASSERT(CompileExpr(*astRes.Root, exprRoot, exprCtx, nullptr, nullptr)); + + auto functionRegistry = NKikimr::NMiniKQL::CreateFunctionRegistry(NKikimr::NMiniKQL::CreateBuiltinRegistry()); + TTestTablesMapping testTables; + auto yqlNativeServices = NFile::TYtFileServices::Make(functionRegistry.Get(), testTables); + auto ytGateway = CreateYtFileGateway(yqlNativeServices); + auto typeAnnotationContext = MakeIntrusive<TTypeAnnotationContext>(); + auto ytState = MakeIntrusive<TYtState>(); + ytState->Gateway = ytGateway; + ytState->Types = typeAnnotationContext.Get(); + + InitializeYtGateway(ytGateway, ytState); + auto randomProvider = CreateDeterministicRandomProvider(1); + typeAnnotationContext->AddDataSink(YtProviderName, CreateYtDataSink(ytState)); + typeAnnotationContext->AddDataSource(YtProviderName, CreateYtDataSource(ytState)); + typeAnnotationContext->RandomProvider = randomProvider; + auto intentTransformer = CreateIntentDeterminationTransformer(*typeAnnotationContext); + TVector<TTransformStage> transformers; + const auto issueCode = TIssuesIds::DEFAULT_ERROR; + transformers.push_back(TTransformStage( + CreateIODiscoveryTransformer(*typeAnnotationContext), + "IODiscovery", + issueCode)); + transformers.push_back(TTransformStage( + CreateEpochsTransformer(*typeAnnotationContext), + "Epochs", + issueCode)); + + // NOTE: add fake EvaluateExpression step to break infinite loop + // (created by Repeat on ExprEval step after RewriteIO completion) + transformers.push_back(TTransformStage( + CreateFunctorTransformer( + [](const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) { + output = input; + ctx.Step.Done(TExprStep::ExprEval); + return IGraphTransformer::TStatus::Ok; + } + ), + "EvaluateExpression", + issueCode)); + + transformers.push_back(TTransformStage( + intentTransformer, + "IntentDetermination", + issueCode)); + transformers.push_back(TTransformStage( + CreateTableMetadataLoader(*typeAnnotationContext), + "MetadataLoader", + issueCode)); + // NOTE: metadata loader unconditionally drops ExpandApplyForLambdas flag + // in DoApplyAsyncChanges, so ExpandApply transformation is in such an unusual place + transformers.push_back(TTransformStage( + CreateFunctorTransformer(&ExpandApply), + "ExpandApply", + issueCode)); + auto fullTransformer = CreateCompositeGraphTransformer(transformers, true); + bool success = SyncTransform(*fullTransformer, exprRoot, exprCtx) == IGraphTransformer::TStatus::Ok; + UNIT_ASSERT(success); + + UNIT_ASSERT(RewriteIO(exprRoot, exprRoot, *typeAnnotationContext, exprCtx).Level == IGraphTransformer::TStatus::Ok); + + auto ast = ConvertToAst(*exprRoot, exprCtx, TExprAnnotationFlags::None, true); + auto strRes = ast.Root->ToString(TAstPrintFlags::PerLine | TAstPrintFlags::ShortQuote); + UNIT_ASSERT(strRes.find("(YtReadTable!") != TString::npos); + UNIT_ASSERT(strRes.find("(Read!") == TString::npos); + } +} + +} // namespace NYql diff --git a/ydb/library/yql/core/ut/yql_expr_providers_ut.cpp b/ydb/library/yql/core/ut/yql_expr_providers_ut.cpp new file mode 100644 index 00000000000..9cc6153e860 --- /dev/null +++ b/ydb/library/yql/core/ut/yql_expr_providers_ut.cpp @@ -0,0 +1,287 @@ +#include <ydb/library/yql/core/type_ann/type_ann_core.h> +#include <ydb/library/yql/core/type_ann/type_ann_expr.h> + +#include <ydb/library/yql/ast/yql_ast_annotation.h> +#include <ydb/library/yql/ast/yql_expr.h> +#include <ydb/library/yql/core/ut_common/yql_ut_common.h> +#include <ydb/library/yql/core/yql_expr_type_annotation.h> +#include <ydb/library/yql/core/facade/yql_facade.h> +#include <ydb/library/yql/providers/common/provider/yql_provider_names.h> +#include <ydb/library/yql/providers/yt/provider/yql_yt_provider.h> +#include <ydb/library/yql/providers/yt/gateway/file/yql_yt_file.h> +#include <ydb/library/yql/providers/yt/gateway/file/yql_yt_file_services.h> +#include <ydb/library/yql/minikql/invoke_builtins/mkql_builtins.h> + +#include <library/cpp/testing/unittest/registar.h> + +#include <util/string/hex.h> +#include <util/random/random.h> +#include <util/system/sanitizers.h> + +namespace NYql { + +Y_UNIT_TEST_SUITE(TCompileWithProvidersYqlExpr) { + static TAstParseResult ParseAstWithCheck(const TStringBuf& s) { + TAstParseResult res = ParseAst(s); + res.Issues.PrintTo(Cout); + UNIT_ASSERT(res.IsOk()); + return res; + } + + static void CompileExprWithCheck(TAstNode& root, TExprNode::TPtr& exprRoot, TExprContext& exprCtx, + ui32 annotationFlags = TExprAnnotationFlags::None) + { + const bool success = CompileExpr(root, exprRoot, exprCtx, nullptr, nullptr, annotationFlags); + exprCtx.IssueManager.GetIssues().PrintTo(Cout); + + UNIT_ASSERT(success); + UNIT_ASSERT_VALUES_EQUAL(exprRoot->GetState(), annotationFlags != TExprAnnotationFlags::None + ? TExprNode::EState::TypeComplete + : TExprNode::EState::Initial); + } + + static void AnnotateExprWithCheck(TExprNode::TPtr& root, TExprContext& ctx, bool wholeProgram, TTypeAnnotationContextPtr typeAnnotationContext) { + const bool success = SyncAnnotateTypes(root, ctx, wholeProgram, *typeAnnotationContext); + ctx.IssueManager.GetIssues().PrintTo(Cout); + UNIT_ASSERT(success); + } + + static void VerifyProgram(const TString& s) { + TAstParseResult astRes = ParseAstWithCheck(s); + TExprContext exprCtx; + TExprNode::TPtr exprRoot; + CompileExprWithCheck(*astRes.Root, exprRoot, exprCtx); + UNIT_ASSERT(!exprRoot->GetTypeAnn()); + + auto functionRegistry = NKikimr::NMiniKQL::CreateFunctionRegistry(NKikimr::NMiniKQL::CreateBuiltinRegistry()); + TTestTablesMapping testTables; + + auto yqlNativeServices = NFile::TYtFileServices::Make(functionRegistry.Get(), testTables); + auto ytGateway = CreateYtFileGateway(yqlNativeServices); + auto typeAnnotationContext = MakeIntrusive<TTypeAnnotationContext>(); + auto ytState = MakeIntrusive<TYtState>(); + ytState->Gateway = ytGateway; + ytState->Types = typeAnnotationContext.Get(); + + InitializeYtGateway(ytGateway, ytState); + typeAnnotationContext->AddDataSink(YtProviderName, CreateYtDataSink(ytState)); + typeAnnotationContext->AddDataSource(YtProviderName, CreateYtDataSource(ytState)); + auto randomProvider = CreateDeterministicRandomProvider(1); + typeAnnotationContext->RandomProvider = randomProvider; + AnnotateExprWithCheck(exprRoot, exprCtx, true, typeAnnotationContext); + UNIT_ASSERT(exprRoot->GetTypeAnn()); + //PrintExpr(Cout, *exprRoot, exprRoot, true); + + auto convertedAst = ConvertToAst(*exprRoot, exprCtx, TExprAnnotationFlags::None, true); + //PrintAst(Cout, *convertedAst, TAstPrintFlags::PerLine | TAstPrintFlags::ShortQuote); + TStringStream strOrig; + convertedAst.Root->PrettyPrintTo(strOrig, TAstPrintFlags::PerLine | TAstPrintFlags::ShortQuote); + + //PrintAst(Cout, *AnnotatePositions(*convertedAst), TAstPrintFlags::PerLine | TAstPrintFlags::ShortQuote); + + auto convertedAstWithAnnotations1 = ConvertToAst(*exprRoot, exprCtx, TExprAnnotationFlags::Position, true); + TString strAnn1res = convertedAstWithAnnotations1.Root->ToString(TAstPrintFlags::ShortQuote); + TAstParseResult astResAnn1 = ParseAstWithCheck(strAnn1res); + UNIT_ASSERT(astResAnn1.IsOk()); + + TMemoryPool pool(4096); + auto cleanAst1 = RemoveAnnotations(*convertedAstWithAnnotations1.Root, pool); + UNIT_ASSERT(cleanAst1); + TStringStream strClean1; + cleanAst1->PrettyPrintTo(strClean1, TAstPrintFlags::PerLine | TAstPrintFlags::ShortQuote); + UNIT_ASSERT_VALUES_EQUAL(strClean1.Str(), strOrig.Str()); + + auto convertedAstWithAnnotations2 = ConvertToAst(*exprRoot, exprCtx, TExprAnnotationFlags::Types, true); + TString strAnn2res = convertedAstWithAnnotations2.Root->ToString(TAstPrintFlags::ShortQuote); + + TAstParseResult astResAnn2 = ParseAstWithCheck(strAnn2res); + UNIT_ASSERT(astResAnn2.IsOk()); + + auto cleanAst2 = RemoveAnnotations(*convertedAstWithAnnotations2.Root, pool); + UNIT_ASSERT(cleanAst2); + TStringStream strClean2; + cleanAst2->PrettyPrintTo(strClean2, TAstPrintFlags::PerLine | TAstPrintFlags::ShortQuote); + UNIT_ASSERT_VALUES_EQUAL(strClean2.Str(), strOrig.Str()); + + TExprContext exprCtxAnnTypes; + TExprNode::TPtr exprRootAnnTypes; + CompileExprWithCheck(*astResAnn2.Root, exprRootAnnTypes, exprCtxAnnTypes, (ui32)TExprAnnotationFlags::Types); + UNIT_ASSERT(exprRootAnnTypes->GetTypeAnn()); + + auto convertedAstWithAnnotations3 = ConvertToAst(*exprRootAnnTypes, exprCtxAnnTypes, TExprAnnotationFlags::Types, true); + TString strAnn3res = convertedAstWithAnnotations3.Root->ToString(TAstPrintFlags::ShortQuote); + UNIT_ASSERT_EQUAL(strAnn2res, strAnn3res); + + const ui32 flagsToParseBack[] = { + 0, + TAstPrintFlags::PerLine, + // TAstPrintFlags::ShortQuote, // -- generates invalid AST + TAstPrintFlags::PerLine | TAstPrintFlags::ShortQuote + }; + + for (ui32 index = 0; index < Y_ARRAY_SIZE(flagsToParseBack); ++index) { + auto flag = flagsToParseBack[index]; + auto s2 = convertedAst.Root->ToString(flag); + + TAstParseResult res2 = ParseAst(s2); + res2.Issues.PrintTo(Cout); + UNIT_ASSERT(res2.IsOk()); + + auto s3 = res2.Root->ToString(flag); + UNIT_ASSERT_STRINGS_EQUAL(s2, s3); + + TExprContext exprCtx2; + TExprNode::TPtr exprRoot2; + CompileExprWithCheck(*res2.Root, exprRoot2, exprCtx2); + + auto convertedAst2 = ConvertToAst(*exprRoot2, exprCtx2, TExprAnnotationFlags::None, true); + auto s4 = convertedAst2.Root->ToString(flag); + UNIT_ASSERT_STRINGS_EQUAL(s2, s4); + } + } + + Y_UNIT_TEST(TestComplexProgramWithLamda) { + auto s = "(\n" + "#comment\n" + "(let mr_source (DataSource 'yt 'plato))\n" + "(let x (Read! world mr_source (Key '('table (String 'Input))) '('key 'subkey 'value) '()))\n" + "(let world (Left! x))\n" + "(let table1 (Right! x))\n" + "(let tresh (String '100))\n" + "(let table1low (Filter table1 (lambda '(item) (< (Member item 'key) tresh))))\n" + "(let mr_sink (DataSink 'yt (quote plato)))\n" + "(let world (Write! world mr_sink (Key '('table (String 'Output))) table1low '('('mode 'append))))\n" + "(let world (Commit! world mr_sink))\n" + "(return world)\n" + ")\n"; + + VerifyProgram(s); + } + + Y_UNIT_TEST(TestSerializeAtomAsPartialExpression) { + auto s = "(\n" + "(let x 'a)\n" + "(let y 'b)\n" + "(let z (quote (x y)))\n" + "(return z)\n" + ")\n"; + + TAstParseResult astRes = ParseAstWithCheck(s); + UNIT_ASSERT(astRes.IsOk()); + + TExprContext exprCtx; + TExprNode::TPtr exprRoot; + CompileExprWithCheck(*astRes.Root, exprRoot, exprCtx); + + auto typeAnnotationContext = MakeIntrusive<TTypeAnnotationContext>(); + auto randomProvider = CreateDeterministicRandomProvider(1); + typeAnnotationContext->RandomProvider = randomProvider; + AnnotateExprWithCheck(exprRoot, exprCtx, false, typeAnnotationContext); + UNIT_ASSERT(exprRoot->GetTypeAnn()); + UNIT_ASSERT_VALUES_EQUAL(exprRoot->Child(0)->Content(), "a"); + + auto convertedAstWithTypes = ConvertToAst(*exprRoot->Child(0), exprCtx, TExprAnnotationFlags::Types, true); + TString strAnnRes = convertedAstWithTypes.Root->ToString(TAstPrintFlags::ShortQuote); + TAstParseResult astRes2 = ParseAstWithCheck(strAnnRes); + UNIT_ASSERT(astRes2.IsOk()); + + TExprContext exprCtx2; + TExprNode::TPtr exprRoot2; + CompileExprWithCheck(*astRes2.Root, exprRoot2, exprCtx2, (ui32)TExprAnnotationFlags::Types); + UNIT_ASSERT(exprRoot2->GetTypeAnn()); + UNIT_ASSERT_VALUES_EQUAL(exprRoot2->Content(), "a"); + + ui32 annotationFlags = TExprAnnotationFlags::Types | TExprAnnotationFlags::Position; + auto convertedAstWithTypesAndPos = ConvertToAst(*exprRoot->Child(0), exprCtx, annotationFlags, true); + TString strPosAnnRes = convertedAstWithTypesAndPos.Root->ToString(TAstPrintFlags::ShortQuote); + TAstParseResult astRes3 = ParseAstWithCheck(strPosAnnRes); + UNIT_ASSERT(astRes3.IsOk()); + + TExprContext exprCtx3; + TExprNode::TPtr exprRoot3; + CompileExprWithCheck(*astRes3.Root, exprRoot3, exprCtx3, annotationFlags); + UNIT_ASSERT(exprRoot2->GetTypeAnn()); + UNIT_ASSERT_VALUES_EQUAL(exprRoot2->Content(), "a"); + } + + Y_UNIT_TEST(TestComplexProgramWithLamdaAsBlock) { + auto s = "(\n" + "#comment\n" + "(let mr_source (DataSource 'yt 'plato))\n" + "(let x (Read! world mr_source (Key '('table (String 'Input))) '('key 'subkey 'value) '()))\n" + "(let world (Left! x))\n" + "(let table1 (Right! x))\n" + "(let table1low (Filter table1 (lambda '(item) (block '(\n" + " (let tresh (String '100))\n" + " (let predicate (< (Member item 'key) tresh))\n" + " (return predicate)\n" + ")))))\n" + "(let mr_sink (DataSink 'yt (quote plato)))\n" + "(let world (Write! world mr_sink (Key '('table (String 'Output))) table1low '('('mode 'append))))\n" + "(let world (Commit! world mr_sink))\n" + "(return world)\n" + ")\n"; + + VerifyProgram(s); + } + + Y_UNIT_TEST(TestComplexProgramWithBlockAndLambdaAsBlock) { + auto s = "(\n" + "#comment\n" + "(let mr_source (DataSource 'yt 'plato))\n" + "(let data (block '(\n" + " (let x (Read! world mr_source (Key '('table (String 'Input))) '('key 'subkey 'value) '()))\n" + " (let world (Left! x))\n" + " (let table1 (Right! x))\n" + " (let table1low (Filter table1 (lambda '(item) (block '(\n" + " (let tresh (String '100))\n" + " (let predicate (< (Member item 'key) tresh))\n" + " (return predicate)\n" + " )))))\n" + " (return table1low)\n" + ")))\n" + "(let mr_sink (DataSink 'yt (quote plato)))\n" + "(let world (Write! world mr_sink (Key '('table (String 'Output))) data '('('mode 'append))))\n" + "(let world (Commit! world mr_sink))\n" + "(return world)\n" + ")\n"; + + VerifyProgram(s); + } + + Y_UNIT_TEST(TestPerfCompile) { + auto s = "(\n" + "#comment\n" + "(let mr_source (DataSource 'yt 'plato))\n" + "(let data (block '(\n" + " (let x (Read! world mr_source (Key '('table (String 'Input))) '('key 'subkey 'value) '()))\n" + " (let world (Left! x))\n" + " (let table1 (Right! x))\n" + " (let table1low (Filter table1 (lambda '(item) (block '(\n" + " (let tresh (String '100))\n" + " (let predicate (< (Member item 'key) tresh))\n" + " (return predicate)\n" + " )))))\n" + " (return table1low)\n" + ")))\n" + "(let mr_sink (DataSink 'yt (quote plato)))\n" + "(let world (Write! world mr_sink (Key '('table (String 'Output))) data '('('mode 'append))))\n" + "(let world (Commit! world mr_sink))\n" + "(return world)\n" + ")\n"; + + TAstParseResult astRes = ParseAstWithCheck(s); + const ui32 n = NSan::PlainOrUnderSanitizer(10000, 1000); + auto t1 = TInstant::Now(); + for (ui32 i = 0; i < n; ++i) { + TExprContext exprCtx; + TExprNode::TPtr exprRoot; + CompileExprWithCheck(*astRes.Root, exprRoot, exprCtx); + } + + auto t2 = TInstant::Now(); + Cout << t2 - t1 << Endl; + } +} + +} // namespace NYql diff --git a/ydb/library/yql/core/ut/yql_expr_type_annotation_ut.cpp b/ydb/library/yql/core/ut/yql_expr_type_annotation_ut.cpp new file mode 100644 index 00000000000..09ce7fd838b --- /dev/null +++ b/ydb/library/yql/core/ut/yql_expr_type_annotation_ut.cpp @@ -0,0 +1,53 @@ +#include "yql_expr_type_annotation.h" + +#include <library/cpp/testing/unittest/registar.h> + + +namespace NYql { + +Y_UNIT_TEST_SUITE(Misc) { + Y_UNIT_TEST(NormalizeName) { + auto CheckNoIssues = [](TString name, const TString& expected) { + UNIT_ASSERT_C(!NormalizeName(TPosition(), name), name); + UNIT_ASSERT_VALUES_EQUAL(name, expected); + }; + + auto CheckIssues = [](TString name, const TString& expected) { + UNIT_ASSERT_C(NormalizeName(TPosition(), name), name); + UNIT_ASSERT_VALUES_EQUAL(name, expected); + }; + + CheckNoIssues("", ""); + CheckNoIssues("_", "_"); + CheckNoIssues("__", "__"); + CheckNoIssues("a", "a"); + CheckNoIssues("abc", "abc"); + CheckNoIssues("aBc", "abc"); + CheckNoIssues("_aBc", "_abc"); + CheckNoIssues("__aBc", "__abc"); + CheckNoIssues("___aBc", "___abc"); + CheckNoIssues("______aBc", "______abc"); + CheckNoIssues("_a_Bc", "_abc"); + CheckIssues("_a__Bc", "_a__Bc"); + CheckNoIssues("_aBc_", "_abc"); + CheckIssues("_aBc__", "_aBc__"); + CheckNoIssues("aBc_", "abc"); + CheckIssues("aBc__", "aBc__"); + CheckIssues("aBc___", "aBc___"); + CheckNoIssues("aB_c", "abc"); + CheckIssues("aB__c", "aB__c"); + CheckNoIssues("a_B_c", "abc"); + CheckNoIssues("_a_B_c", "_abc"); + CheckNoIssues("__a_B_c", "__abc"); + CheckNoIssues("a_B_c_", "abc"); + CheckNoIssues("a_B_c_d", "abcd"); + CheckNoIssues("a_B_c_d_", "abcd"); + CheckNoIssues("a_B_c_d_e", "abcde"); + CheckNoIssues("_a_B_c_d_e", "_abcde"); + CheckIssues("a_B_c_d_e_", "a_B_c_d_e_"); + CheckIssues("a_B_c_d_e_f", "a_B_c_d_e_f"); + CheckIssues("_a_B_c_d_e_f", "_a_B_c_d_e_f"); + } +} + +} // namespace NYql diff --git a/ydb/library/yql/core/ut/yql_library_compiler_ut.cpp b/ydb/library/yql/core/ut/yql_library_compiler_ut.cpp new file mode 100644 index 00000000000..68dcd766931 --- /dev/null +++ b/ydb/library/yql/core/ut/yql_library_compiler_ut.cpp @@ -0,0 +1,223 @@ +#include <library/cpp/testing/unittest/registar.h> + +#include <ydb/library/yql/core/ut_common/yql_ut_common.h> + +#include "yql_library_compiler.h" + +namespace NYql { + +Y_UNIT_TEST_SUITE(TLibraryCompilerTests) { + + static const char* alias = "/lib/ut.yql"; + + static bool CompileAndLink(const THashMap<TString, TString>& libs, TExprContext& ctx) { + THashMap<TString, TLibraryCohesion> compiled; + for (const auto& lib : libs) + if (!CompileLibrary(alias, lib.second, ctx, compiled[lib.first])) + return false; + + return LinkLibraries(compiled, ctx, ctx); + } + + Y_UNIT_TEST(OnlyExportsTest) { + const auto s = "(\n" + "(let X 'Y)\n" + "(let ex '42)\n" + "(export ex)\n" + "(export X)\n" + ")\n"; + + TExprContext ctx; + TLibraryCohesion cohesion; + UNIT_ASSERT(CompileLibrary(alias, s, ctx, cohesion)); + UNIT_ASSERT_VALUES_EQUAL(2, cohesion.Exports.Symbols().size()); + UNIT_ASSERT(cohesion.Imports.empty()); + } + + Y_UNIT_TEST(ExportAndImportsTest) { + const auto s = "(\n" + "(import math_module '""/lib/yql/math.yql"")\n" + "(let mySqr2 (bind math_module 'sqr2))\n" + "(let mySqr3 (bind math_module 'sqr3))\n" + "(let ex '42)\n" + "(export ex)\n" + ")\n"; + + TExprContext ctx; + TLibraryCohesion cohesion; + UNIT_ASSERT(CompileLibrary(alias, s, ctx, cohesion)); + UNIT_ASSERT_VALUES_EQUAL(1, cohesion.Exports.Symbols().size()); + UNIT_ASSERT_VALUES_EQUAL(2, cohesion.Imports.size()); + } + + Y_UNIT_TEST(TestImportSelf) { + const auto xxx = "(\n" + "(import math_module '""/lib/yql/xxx.yql"")\n" + "(let myXxx (bind math_module 'xxx))\n" + "(let sqr (lambda '(x) (Apply myXxx x)))\n" + "(export sqr)\n" + ")\n"; + + const THashMap<TString, TString> libs = { + {"/lib/yql/xxx.yql", xxx} + }; + + TExprContext ctx; + UNIT_ASSERT(!CompileAndLink(libs, ctx)); + UNIT_ASSERT_VALUES_EQUAL("/lib/ut.yql:3:13: Error: Library '/lib/yql/xxx.yql' tries to import itself.\n", ctx.IssueManager.GetIssues().ToString()); + } + + Y_UNIT_TEST(TestMissedModule) { + const auto aaa = "(\n" + "(import math_module '""/lib/yql/xxx.yql"")\n" + "(let myXxx (bind math_module 'xxx))\n" + "(let sqr (lambda '(x) (Apply x myXxx)))\n" + "(export sqr)\n" + ")\n"; + + const THashMap<TString, TString> libs = { + {"/lib/yql/aaa.yql", aaa} + }; + + TExprContext ctx; + UNIT_ASSERT(!CompileAndLink(libs, ctx)); + UNIT_ASSERT_VALUES_EQUAL("/lib/ut.yql:3:13: Error: Library '/lib/yql/aaa.yql' has unresolved dependency from '/lib/yql/xxx.yql'.\n", ctx.IssueManager.GetIssues().ToString()); + } + + Y_UNIT_TEST(TestUnresolvedSymbol) { + const auto one = "(\n" + "(import math_module '""/lib/yql/two.yql"")\n" + "(let myTwo (bind math_module 'zzz))\n" + "(let one (lambda '(x) (Apply myTwo x)))\n" + "(export one)\n" + ")\n"; + + const auto two = "(\n" + "(let two (lambda '(x) (+ x x)))\n" + "(export two)\n" + ")\n"; + + const THashMap<TString, TString> libs = { + {"/lib/yql/one.yql", one}, + {"/lib/yql/two.yql", two} + }; + + TExprContext ctx; + UNIT_ASSERT(!CompileAndLink(libs, ctx)); + UNIT_ASSERT_VALUES_EQUAL("/lib/ut.yql:3:13: Error: Library '/lib/yql/one.yql' has unresolved symbol 'zzz' from '/lib/yql/two.yql'.\n", ctx.IssueManager.GetIssues().ToString()); + } + + Y_UNIT_TEST(TestCrossReference) { + const auto one = "(\n" + "(import math_module '""/lib/yql/two.yql"")\n" + "(let myTwo (bind math_module 'two))\n" + "(let one (lambda '(x) (Apply myTwo x)))\n" + "(export one)\n" + ")\n"; + + const auto two = "(\n" + "(import math_module '""/lib/yql/one.yql"")\n" + "(let myOne (bind math_module 'one))\n" + "(let two (lambda '(x) (Apply myOne x)))\n" + "(export two)\n" + ")\n"; + + const THashMap<TString, TString> libs = { + {"/lib/yql/one.yql", one}, + {"/lib/yql/two.yql", two} + }; + + TExprContext ctx; + UNIT_ASSERT(!CompileAndLink(libs, ctx)); + UNIT_ASSERT(ctx.IssueManager.GetIssues().ToString().Contains("Cross reference detected")); + } + + Y_UNIT_TEST(TestCrorssDependencyWithoutCrossReference) { + const auto one = "(\n" + "(import math_module '""/lib/yql/two.yql"")\n" + "(let myTwo (bind math_module 'two))\n" + "(let one (lambda '(x) (Apply myTwo x)))\n" + "(export one)\n" + ")\n"; + + const auto two = "(\n" + "(import math_module '""/lib/yql/one.yql"")\n" + "(let myOne (bind math_module 'one))\n" + "(let two (lambda '(x) (+ x x)))\n" + "(export two)\n" + "(let exp (lambda '(x) (Apply myOne x)))\n" + "(export exp)\n" + ")\n"; + + const THashMap<TString, TString> libs = { + {"/lib/yql/one.yql", one}, + {"/lib/yql/two.yql", two} + }; + + TExprContext ctx; + UNIT_ASSERT(CompileAndLink(libs, ctx)); + } + + Y_UNIT_TEST(TestCircleReference) { + const auto one = "(\n" + "(import math_module '""/lib/yql/two.yql"")\n" + "(let myTwo (bind math_module 'two))\n" + "(let one (lambda '(x) (Apply myTwo x)))\n" + "(export one)\n" + ")\n"; + + const auto two = "(\n" + "(import math_module '""/lib/yql/xxx.yql"")\n" + "(let myXxx (bind math_module 'xxx))\n" + "(let two (lambda '(x) (Apply myXxx x)))\n" + "(export two)\n" + ")\n"; + + const auto xxx = "(\n" + "(import math_module '""/lib/yql/one.yql"")\n" + "(let myOne (bind math_module 'one))\n" + "(let xxx (lambda '(x) (Apply myOne x)))\n" + "(export xxx)\n" + ")\n"; + + const THashMap<TString, TString> libs = { + {"/lib/yql/one.yql", one}, + {"/lib/yql/two.yql", two}, + {"/lib/yql/xxx.yql", xxx} + }; + + TExprContext ctx; + UNIT_ASSERT(!CompileAndLink(libs, ctx)); + UNIT_ASSERT(ctx.IssueManager.GetIssues().ToString().Contains("Cross reference detected")); + } + + Y_UNIT_TEST(TestForwarding) { + const auto one = "(\n" + "(let one '1)\n" + "(export one)\n" + ")\n"; + + const auto two = "(\n" + "(import math_module '""/lib/yql/one.yql"")\n" + "(let myOne (bind math_module 'one))\n" + "(export myOne)\n" + ")\n"; + + const auto xxx = "(\n" + "(import math_module '""/lib/yql/two.yql"")\n" + "(let xxx (bind math_module 'myOne))\n" + "(export xxx)\n" + ")\n"; + + const THashMap<TString, TString> libs = { + {"/lib/yql/one.yql", one}, + {"/lib/yql/two.yql", two}, + {"/lib/yql/xxx.yql", xxx} + }; + + TExprContext ctx; + UNIT_ASSERT(CompileAndLink(libs, ctx)); + } +} + +} diff --git a/ydb/library/yql/core/yql_opt_utils_ut.cpp b/ydb/library/yql/core/ut/yql_opt_utils_ut.cpp index c3f3058ab3b..c3f3058ab3b 100644 --- a/ydb/library/yql/core/yql_opt_utils_ut.cpp +++ b/ydb/library/yql/core/ut/yql_opt_utils_ut.cpp diff --git a/ydb/library/yql/core/ut/yql_udf_index_ut.cpp b/ydb/library/yql/core/ut/yql_udf_index_ut.cpp new file mode 100644 index 00000000000..c0c6fc93975 --- /dev/null +++ b/ydb/library/yql/core/ut/yql_udf_index_ut.cpp @@ -0,0 +1,368 @@ +#include "yql_udf_index.h" +#include <ydb/library/yql/minikql/mkql_function_registry.h> +#include <library/cpp/testing/unittest/registar.h> + +using namespace NYql; +namespace { +class TResourceBuilder { + TIntrusivePtr<TResourceInfo> Resource_ = new TResourceInfo(); + +public: + explicit TResourceBuilder(const TDownloadLink& link) { + Resource_->Link = link; + } + + TResourceBuilder& AddFunction(const TFunctionInfo& f) { + auto module = TString(NKikimr::NMiniKQL::ModuleName(TStringBuf(f.Name))); + Resource_->Modules.insert(module); + Resource_->SetFunctions({ f }); + return *this; + } + + TResourceInfo::TPtr Build() const { + return Resource_; + } +}; + +TFunctionInfo BuildFunctionInfo(const TString& name, int argCount) { + TFunctionInfo result; + result.Name = name; + result.ArgCount = argCount; + return result; +} + +void EnsureFunctionsEqual(const TFunctionInfo& f1, const TFunctionInfo& f2) { + UNIT_ASSERT_VALUES_EQUAL(f1.Name, f2.Name); + UNIT_ASSERT_VALUES_EQUAL(f1.ArgCount, f2.ArgCount); +} + +void EnsureLinksEqual(const TDownloadLink& link1, const TDownloadLink& link2) { + UNIT_ASSERT_VALUES_EQUAL(link1.IsUrl, link2.IsUrl); + UNIT_ASSERT_VALUES_EQUAL(link1.Path, link2.Path); +} + +void EnsureContainsFunction(TUdfIndex::TPtr index, TString module, const TFunctionInfo& f) { + TFunctionInfo existingFunc; + UNIT_ASSERT(index->FindFunction(module, f.Name, existingFunc)); + EnsureFunctionsEqual(f, existingFunc); +} +} + +Y_UNIT_TEST_SUITE(TUdfIndexTests) { + Y_UNIT_TEST(Empty) { + auto index1 = MakeIntrusive<TUdfIndex>(); + + UNIT_ASSERT(!index1->ContainsModule("M1")); + UNIT_ASSERT(index1->FindResourceByModule("M1") == nullptr); + TFunctionInfo f1; + UNIT_ASSERT(!index1->FindFunction("M1", "M1.F1", f1)); + + auto index2 = index1->Clone(); + UNIT_ASSERT(!index2->ContainsModule("M1")); + UNIT_ASSERT(index2->FindResourceByModule("M1") == nullptr); + UNIT_ASSERT(!index2->FindFunction("M1", "M1.F1", f1)); + } + + Y_UNIT_TEST(SingleModuleAndFunction) { + auto index1 = MakeIntrusive<TUdfIndex>(); + auto func1 = BuildFunctionInfo("M1.F1", 1); + auto link1 = TDownloadLink::File("file1"); + + TResourceBuilder b(link1); + b.AddFunction(func1); + + index1->RegisterResource(b.Build(), TUdfIndex::EOverrideMode::RaiseError); + UNIT_ASSERT(index1->ContainsModule("M1")); + UNIT_ASSERT(!index1->ContainsModule("M2")); + + UNIT_ASSERT(index1->FindResourceByModule("M2") == nullptr); + auto resource1 = index1->FindResourceByModule("M1"); + UNIT_ASSERT(resource1 != nullptr); + EnsureLinksEqual(resource1->Link, link1); + + TFunctionInfo f1; + UNIT_ASSERT(!index1->FindFunction("M2", "M2.F1", f1)); + + UNIT_ASSERT(index1->FindFunction("M1", "M1.F1", f1)); + EnsureFunctionsEqual(f1, func1); + + // ensure both indexes contain the same info + auto index2 = index1->Clone(); + + UNIT_ASSERT(index1->ContainsModule("M1")); + UNIT_ASSERT(index2->ContainsModule("M1")); + + TFunctionInfo f2; + UNIT_ASSERT(index2->FindFunction("M1", "M1.F1", f2)); + EnsureFunctionsEqual(f1, f2); + + auto resource2 = index2->FindResourceByModule("M1"); + UNIT_ASSERT(resource2 != nullptr); + } + + Y_UNIT_TEST(SeveralModulesAndFunctions) { + auto index1 = MakeIntrusive<TUdfIndex>(); + auto func11 = BuildFunctionInfo("M1.F1", 1); + auto func12 = BuildFunctionInfo("M1.F2", 2); + auto func13 = BuildFunctionInfo("M2.F1", 3); + + auto link1 = TDownloadLink::File("file1"); + auto resource1 = TResourceBuilder(link1) + .AddFunction(func11) + .AddFunction(func12) + .AddFunction(func13) + .Build(); + + auto func21 = BuildFunctionInfo("M3.F1", 4); + auto func22 = BuildFunctionInfo("M4.F2", 5); + + auto link2 = TDownloadLink::Url("url1"); + auto resource2 = TResourceBuilder(link2) + .AddFunction(func21) + .AddFunction(func22) + .Build(); + + index1->RegisterResource(resource1, TUdfIndex::EOverrideMode::RaiseError); + index1->RegisterResource(resource2, TUdfIndex::EOverrideMode::RaiseError); + + // check resources by module + UNIT_ASSERT(index1->FindResourceByModule("M5") == nullptr); + auto r11 = index1->FindResourceByModule("M1"); + auto r12 = index1->FindResourceByModule("M2"); + UNIT_ASSERT(r11 != nullptr && r12 != nullptr); + EnsureLinksEqual(r11->Link, link1); + EnsureLinksEqual(r12->Link, link1); + + auto r21 = index1->FindResourceByModule("M3"); + auto r22 = index1->FindResourceByModule("M4"); + UNIT_ASSERT(r21 != nullptr && r22 != nullptr); + EnsureLinksEqual(r21->Link, link2); + EnsureLinksEqual(r22->Link, link2); + + // check modules + UNIT_ASSERT(index1->ContainsModule("M1")); + UNIT_ASSERT(index1->ContainsModule("M2")); + UNIT_ASSERT(index1->ContainsModule("M3")); + UNIT_ASSERT(index1->ContainsModule("M4")); + UNIT_ASSERT(!index1->ContainsModule("M5")); + + EnsureContainsFunction(index1, "M1", func11); + EnsureContainsFunction(index1, "M1", func12); + EnsureContainsFunction(index1, "M2", func13); + EnsureContainsFunction(index1, "M3", func21); + EnsureContainsFunction(index1, "M4", func22); + + // works because M2 refers to the same resource + EnsureContainsFunction(index1, "M2", func11); + + TFunctionInfo f; + // known func, but non-existent module + UNIT_ASSERT(!index1->FindFunction("M5", "M1.F1", f)); + UNIT_ASSERT(!index1->FindFunction("M2", "M3.F1", f)); + } + + Y_UNIT_TEST(ConflictRaiseError) { + auto index1 = MakeIntrusive<TUdfIndex>(); + auto func11 = BuildFunctionInfo("M1.F1", 1); + auto func12 = BuildFunctionInfo("M1.F2", 2); + auto func13 = BuildFunctionInfo("M2.F1", 3); + + auto link1 = TDownloadLink::File("file1"); + auto resource1 = TResourceBuilder(link1) + .AddFunction(func11) + .AddFunction(func12) + .AddFunction(func13) + .Build(); + + auto func21 = BuildFunctionInfo("M3.F1", 4); + auto func22 = BuildFunctionInfo("M2.F1", 5); + + auto link2 = TDownloadLink::Url("url1"); + auto resource2 = TResourceBuilder(link2) + .AddFunction(func21) + .AddFunction(func22) + .Build(); + + index1->RegisterResource(resource1, TUdfIndex::EOverrideMode::RaiseError); + UNIT_ASSERT_EXCEPTION_CONTAINS(index1->RegisterResource(resource2, TUdfIndex::EOverrideMode::RaiseError), std::exception, "Conflict during resource url1 registration"); + + // ensure state untouched + UNIT_ASSERT(index1->FindResourceByModule("M3") == nullptr); + auto r1 = index1->FindResourceByModule("M1"); + auto r2 = index1->FindResourceByModule("M2"); + UNIT_ASSERT(r1 != nullptr && r2 != nullptr); + EnsureLinksEqual(r1->Link, link1); + EnsureLinksEqual(r2->Link, link1); + + EnsureContainsFunction(index1, "M1", func11); + EnsureContainsFunction(index1, "M1", func12); + EnsureContainsFunction(index1, "M2", func13); + + TFunctionInfo f; + UNIT_ASSERT(!index1->FindFunction("M3", "M3.F1", f)); + } + + Y_UNIT_TEST(ConflictPreserveExisting) { + auto index1 = MakeIntrusive<TUdfIndex>(); + auto func11 = BuildFunctionInfo("M1.F1", 1); + auto func12 = BuildFunctionInfo("M1.F2", 2); + auto func13 = BuildFunctionInfo("M2.F1", 3); + + auto link1 = TDownloadLink::File("file1"); + auto resource1 = TResourceBuilder(link1) + .AddFunction(func11) + .AddFunction(func12) + .AddFunction(func13) + .Build(); + + auto func21 = BuildFunctionInfo("M3.F1", 4); + auto func22 = BuildFunctionInfo("M2.F1", 5); + + auto link2 = TDownloadLink::Url("url1"); + auto resource2 = TResourceBuilder(link2) + .AddFunction(func21) + .AddFunction(func22) + .Build(); + + index1->RegisterResource(resource1, TUdfIndex::EOverrideMode::RaiseError); + index1->RegisterResource(resource2, TUdfIndex::EOverrideMode::PreserveExisting); + + // ensure state untouched + UNIT_ASSERT(index1->FindResourceByModule("M3") == nullptr); + auto r1 = index1->FindResourceByModule("M1"); + auto r2 = index1->FindResourceByModule("M2"); + UNIT_ASSERT(r1 != nullptr && r2 != nullptr); + EnsureLinksEqual(r1->Link, link1); + EnsureLinksEqual(r2->Link, link1); + + EnsureContainsFunction(index1, "M1", func11); + EnsureContainsFunction(index1, "M1", func12); + EnsureContainsFunction(index1, "M2", func13); + + TFunctionInfo f; + UNIT_ASSERT(!index1->FindFunction("M3", "M3.F1", f)); + } + + Y_UNIT_TEST(ConflictReplace1WithNew) { + // single resource will be replaced + auto index1 = MakeIntrusive<TUdfIndex>(); + auto func11 = BuildFunctionInfo("M1.F1", 1); + auto func12 = BuildFunctionInfo("M1.F2", 2); + auto func13 = BuildFunctionInfo("M2.F1", 3); + + auto link1 = TDownloadLink::File("file1"); + auto resource1 = TResourceBuilder(link1) + .AddFunction(func11) + .AddFunction(func12) + .AddFunction(func13) + .Build(); + + auto func21 = BuildFunctionInfo("M3.F3", 4); + auto func22 = BuildFunctionInfo("M4.F4", 5); + + auto link2 = TDownloadLink::Url("url1"); + auto resource2 = TResourceBuilder(link2) + .AddFunction(func21) + .AddFunction(func22) + .Build(); + + + auto func31 = BuildFunctionInfo("M5.F5", 6); + // conflict by module name + auto func32 = BuildFunctionInfo("M1.F7", 7); + + auto link3 = TDownloadLink::Url("url3"); + auto resource3 = TResourceBuilder(link3) + .AddFunction(func31) + .AddFunction(func32) + .Build(); + + index1->RegisterResource(resource1, TUdfIndex::EOverrideMode::RaiseError); + index1->RegisterResource(resource2, TUdfIndex::EOverrideMode::RaiseError); + index1->RegisterResource(resource3, TUdfIndex::EOverrideMode::ReplaceWithNew); + + UNIT_ASSERT(index1->FindResourceByModule("M2") == nullptr); + auto r1 = index1->FindResourceByModule("M3"); + UNIT_ASSERT(r1 != nullptr); + EnsureLinksEqual(r1->Link, link2); + + auto r2 = index1->FindResourceByModule("M1"); + UNIT_ASSERT(r2 != nullptr); + EnsureLinksEqual(r2->Link, link3); + + // ensure untouched + EnsureContainsFunction(index1, "M3", func21); + EnsureContainsFunction(index1, "M4", func22); + + EnsureContainsFunction(index1, "M5", func31); + EnsureContainsFunction(index1, "M1", func32); + + // not here anymore + TFunctionInfo f; + UNIT_ASSERT(!index1->FindFunction("M1", "M1.F1", f)); + UNIT_ASSERT(!index1->FindFunction("M1", "M1.F2", f)); + UNIT_ASSERT(!index1->FindFunction("M2", "M2.F1", f)); + } + + Y_UNIT_TEST(ConflictReplace2WithNew) { + // both resources will be replaced + auto index1 = MakeIntrusive<TUdfIndex>(); + auto func11 = BuildFunctionInfo("M1.F1", 1); + auto func12 = BuildFunctionInfo("M1.F2", 2); + auto func13 = BuildFunctionInfo("M2.F1", 3); + + auto link1 = TDownloadLink::File("file1"); + auto resource1 = TResourceBuilder(link1) + .AddFunction(func11) + .AddFunction(func12) + .AddFunction(func13) + .Build(); + + auto func21 = BuildFunctionInfo("M3.F3", 4); + auto func22 = BuildFunctionInfo("M4.F4", 5); + + auto link2 = TDownloadLink::Url("url1"); + auto resource2 = TResourceBuilder(link2) + .AddFunction(func21) + .AddFunction(func22) + .Build(); + + + // conflict by module name + auto func31 = BuildFunctionInfo("M3.F7", 6); + // conflict by func name + auto func32 = BuildFunctionInfo("M1.F1", 7); + + auto link3 = TDownloadLink::Url("url3"); + auto resource3 = TResourceBuilder(link3) + .AddFunction(func31) + .AddFunction(func32) + .Build(); + + index1->RegisterResource(resource1, TUdfIndex::EOverrideMode::RaiseError); + index1->RegisterResource(resource2, TUdfIndex::EOverrideMode::RaiseError); + index1->RegisterResource(resource3, TUdfIndex::EOverrideMode::ReplaceWithNew); + + UNIT_ASSERT(index1->FindResourceByModule("M2") == nullptr); + UNIT_ASSERT(index1->FindResourceByModule("M4") == nullptr); + auto r1 = index1->FindResourceByModule("M3"); + UNIT_ASSERT(r1 != nullptr); + EnsureLinksEqual(r1->Link, link3); + + auto r2 = index1->FindResourceByModule("M1"); + UNIT_ASSERT(r2 != nullptr); + EnsureLinksEqual(r2->Link, link3); + + // ensure untouched + EnsureContainsFunction(index1, "M3", func31); + EnsureContainsFunction(index1, "M1", func32); + + // not here anymore + TFunctionInfo f; + UNIT_ASSERT(!index1->FindFunction("M1", "M1.F2", f)); + UNIT_ASSERT(!index1->FindFunction("M2", "M2.F1", f)); + + UNIT_ASSERT(!index1->FindFunction("M3", "M3.F3", f)); + UNIT_ASSERT(!index1->FindFunction("M4", "M4.F4", f)); + } +} diff --git a/ydb/library/yql/core/ut_common/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/core/ut_common/CMakeLists.darwin-x86_64.txt new file mode 100644 index 00000000000..0ffc0edcc65 --- /dev/null +++ b/ydb/library/yql/core/ut_common/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,24 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(yql-core-ut_common) +target_compile_options(yql-core-ut_common PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(yql-core-ut_common PUBLIC + contrib-libs-cxxsupp + yutil + library-yql-core + yql-core-expr_nodes + yql-dq-proto + yql-dq-expr_nodes +) +target_sources(yql-core-ut_common PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/ut_common/yql_ut_common.cpp +) diff --git a/ydb/library/yql/core/ut_common/CMakeLists.linux-aarch64.txt b/ydb/library/yql/core/ut_common/CMakeLists.linux-aarch64.txt new file mode 100644 index 00000000000..8eb14de93b0 --- /dev/null +++ b/ydb/library/yql/core/ut_common/CMakeLists.linux-aarch64.txt @@ -0,0 +1,25 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(yql-core-ut_common) +target_compile_options(yql-core-ut_common PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(yql-core-ut_common PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + library-yql-core + yql-core-expr_nodes + yql-dq-proto + yql-dq-expr_nodes +) +target_sources(yql-core-ut_common PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/ut_common/yql_ut_common.cpp +) diff --git a/ydb/library/yql/core/ut_common/CMakeLists.linux-x86_64.txt b/ydb/library/yql/core/ut_common/CMakeLists.linux-x86_64.txt new file mode 100644 index 00000000000..8eb14de93b0 --- /dev/null +++ b/ydb/library/yql/core/ut_common/CMakeLists.linux-x86_64.txt @@ -0,0 +1,25 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(yql-core-ut_common) +target_compile_options(yql-core-ut_common PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(yql-core-ut_common PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + library-yql-core + yql-core-expr_nodes + yql-dq-proto + yql-dq-expr_nodes +) +target_sources(yql-core-ut_common PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/ut_common/yql_ut_common.cpp +) diff --git a/ydb/library/yql/core/ut_common/CMakeLists.txt b/ydb/library/yql/core/ut_common/CMakeLists.txt new file mode 100644 index 00000000000..f8b31df0c11 --- /dev/null +++ b/ydb/library/yql/core/ut_common/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/ydb/library/yql/core/ut_common/CMakeLists.windows-x86_64.txt b/ydb/library/yql/core/ut_common/CMakeLists.windows-x86_64.txt new file mode 100644 index 00000000000..0ffc0edcc65 --- /dev/null +++ b/ydb/library/yql/core/ut_common/CMakeLists.windows-x86_64.txt @@ -0,0 +1,24 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(yql-core-ut_common) +target_compile_options(yql-core-ut_common PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(yql-core-ut_common PUBLIC + contrib-libs-cxxsupp + yutil + library-yql-core + yql-core-expr_nodes + yql-dq-proto + yql-dq-expr_nodes +) +target_sources(yql-core-ut_common PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/ut_common/yql_ut_common.cpp +) diff --git a/ydb/library/yql/providers/yt/gateway/CMakeLists.txt b/ydb/library/yql/providers/yt/gateway/CMakeLists.txt index e4353f2ae03..39d0a638664 100644 --- a/ydb/library/yql/providers/yt/gateway/CMakeLists.txt +++ b/ydb/library/yql/providers/yt/gateway/CMakeLists.txt @@ -6,5 +6,6 @@ # original buildsystem will not be accepted. +add_subdirectory(file) add_subdirectory(lib) add_subdirectory(native) diff --git a/ydb/library/yql/providers/yt/gateway/file/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/providers/yt/gateway/file/CMakeLists.darwin-x86_64.txt new file mode 100644 index 00000000000..323acc09e87 --- /dev/null +++ b/ydb/library/yql/providers/yt/gateway/file/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,57 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(yt-gateway-file) +target_compile_options(yt-gateway-file PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(yt-gateway-file PUBLIC + contrib-libs-cxxsupp + yutil + library-cpp-yson + cpp-yson-node + cpp-mapreduce-common + cpp-mapreduce-interface + yql-core-file_storage + core-file_storage-proto + core-file_storage-http_download + minikql-comp_nodes-llvm + yql-public-udf + library-yql-utils + yql-utils-threading + library-yql-core + yql-core-expr_nodes + yql-core-type_ann + providers-common-codec + providers-common-comp_nodes + providers-common-gateway + providers-common-mkql + providers-common-provider + common-schema-expr + common-schema-mkql + providers-result-expr_nodes + providers-yt-common + providers-yt-comp_nodes + providers-yt-expr_nodes + yt-gateway-lib + yt-lib-infer_schema + yt-lib-lambda_builder + yt-lib-mkql_helpers + yt-lib-res_pull + yt-lib-schema + yt-lib-yson_helpers + providers-yt-provider + yql-parser-pg_wrapper +) +target_sources(yt-gateway-file PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/yt/gateway/file/yql_yt_file_comp_nodes.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/yt/gateway/file/yql_yt_file_mkql_compiler.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/yt/gateway/file/yql_yt_file_services.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/yt/gateway/file/yql_yt_file.cpp +) diff --git a/ydb/library/yql/providers/yt/gateway/file/CMakeLists.linux-aarch64.txt b/ydb/library/yql/providers/yt/gateway/file/CMakeLists.linux-aarch64.txt new file mode 100644 index 00000000000..33f1a8c2fcc --- /dev/null +++ b/ydb/library/yql/providers/yt/gateway/file/CMakeLists.linux-aarch64.txt @@ -0,0 +1,58 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(yt-gateway-file) +target_compile_options(yt-gateway-file PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(yt-gateway-file PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + library-cpp-yson + cpp-yson-node + cpp-mapreduce-common + cpp-mapreduce-interface + yql-core-file_storage + core-file_storage-proto + core-file_storage-http_download + minikql-comp_nodes-llvm + yql-public-udf + library-yql-utils + yql-utils-threading + library-yql-core + yql-core-expr_nodes + yql-core-type_ann + providers-common-codec + providers-common-comp_nodes + providers-common-gateway + providers-common-mkql + providers-common-provider + common-schema-expr + common-schema-mkql + providers-result-expr_nodes + providers-yt-common + providers-yt-comp_nodes + providers-yt-expr_nodes + yt-gateway-lib + yt-lib-infer_schema + yt-lib-lambda_builder + yt-lib-mkql_helpers + yt-lib-res_pull + yt-lib-schema + yt-lib-yson_helpers + providers-yt-provider + yql-parser-pg_wrapper +) +target_sources(yt-gateway-file PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/yt/gateway/file/yql_yt_file_comp_nodes.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/yt/gateway/file/yql_yt_file_mkql_compiler.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/yt/gateway/file/yql_yt_file_services.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/yt/gateway/file/yql_yt_file.cpp +) diff --git a/ydb/library/yql/providers/yt/gateway/file/CMakeLists.linux-x86_64.txt b/ydb/library/yql/providers/yt/gateway/file/CMakeLists.linux-x86_64.txt new file mode 100644 index 00000000000..33f1a8c2fcc --- /dev/null +++ b/ydb/library/yql/providers/yt/gateway/file/CMakeLists.linux-x86_64.txt @@ -0,0 +1,58 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(yt-gateway-file) +target_compile_options(yt-gateway-file PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(yt-gateway-file PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + library-cpp-yson + cpp-yson-node + cpp-mapreduce-common + cpp-mapreduce-interface + yql-core-file_storage + core-file_storage-proto + core-file_storage-http_download + minikql-comp_nodes-llvm + yql-public-udf + library-yql-utils + yql-utils-threading + library-yql-core + yql-core-expr_nodes + yql-core-type_ann + providers-common-codec + providers-common-comp_nodes + providers-common-gateway + providers-common-mkql + providers-common-provider + common-schema-expr + common-schema-mkql + providers-result-expr_nodes + providers-yt-common + providers-yt-comp_nodes + providers-yt-expr_nodes + yt-gateway-lib + yt-lib-infer_schema + yt-lib-lambda_builder + yt-lib-mkql_helpers + yt-lib-res_pull + yt-lib-schema + yt-lib-yson_helpers + providers-yt-provider + yql-parser-pg_wrapper +) +target_sources(yt-gateway-file PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/yt/gateway/file/yql_yt_file_comp_nodes.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/yt/gateway/file/yql_yt_file_mkql_compiler.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/yt/gateway/file/yql_yt_file_services.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/yt/gateway/file/yql_yt_file.cpp +) diff --git a/ydb/library/yql/providers/yt/gateway/file/CMakeLists.txt b/ydb/library/yql/providers/yt/gateway/file/CMakeLists.txt new file mode 100644 index 00000000000..f8b31df0c11 --- /dev/null +++ b/ydb/library/yql/providers/yt/gateway/file/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/ydb/library/yql/providers/yt/gateway/file/CMakeLists.windows-x86_64.txt b/ydb/library/yql/providers/yt/gateway/file/CMakeLists.windows-x86_64.txt new file mode 100644 index 00000000000..323acc09e87 --- /dev/null +++ b/ydb/library/yql/providers/yt/gateway/file/CMakeLists.windows-x86_64.txt @@ -0,0 +1,57 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(yt-gateway-file) +target_compile_options(yt-gateway-file PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(yt-gateway-file PUBLIC + contrib-libs-cxxsupp + yutil + library-cpp-yson + cpp-yson-node + cpp-mapreduce-common + cpp-mapreduce-interface + yql-core-file_storage + core-file_storage-proto + core-file_storage-http_download + minikql-comp_nodes-llvm + yql-public-udf + library-yql-utils + yql-utils-threading + library-yql-core + yql-core-expr_nodes + yql-core-type_ann + providers-common-codec + providers-common-comp_nodes + providers-common-gateway + providers-common-mkql + providers-common-provider + common-schema-expr + common-schema-mkql + providers-result-expr_nodes + providers-yt-common + providers-yt-comp_nodes + providers-yt-expr_nodes + yt-gateway-lib + yt-lib-infer_schema + yt-lib-lambda_builder + yt-lib-mkql_helpers + yt-lib-res_pull + yt-lib-schema + yt-lib-yson_helpers + providers-yt-provider + yql-parser-pg_wrapper +) +target_sources(yt-gateway-file PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/yt/gateway/file/yql_yt_file_comp_nodes.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/yt/gateway/file/yql_yt_file_mkql_compiler.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/yt/gateway/file/yql_yt_file_services.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/yt/gateway/file/yql_yt_file.cpp +) |