diff options
| author | Alexander Smirnov <alex@ydb.tech> | 2025-03-04 13:27:53 +0000 |
|---|---|---|
| committer | Alexander Smirnov <alex@ydb.tech> | 2025-03-04 13:27:53 +0000 |
| commit | ad0b83372abbcedc2887412cfdd967ea22b7148e (patch) | |
| tree | d24fee47ed2aef834af220ac137f5a0e26ed4678 | |
| parent | 31ac77ec345d18126d79b7797bef365c9068d999 (diff) | |
| parent | 040bd4000eeec19eab31a023f72c3c06494ce92d (diff) | |
| download | ydb-ad0b83372abbcedc2887412cfdd967ea22b7148e.tar.gz | |
Merge pull request #15226 from ydb-platform/merge-libs-250302-1120
307 files changed, 14150 insertions, 17715 deletions
diff --git a/.github/config/muted_ya.txt b/.github/config/muted_ya.txt index 591d7b665c..52651e4221 100644 --- a/.github/config/muted_ya.txt +++ b/.github/config/muted_ya.txt @@ -85,6 +85,7 @@ ydb/library/actors/http/ut sole chunk chunk ydb/library/actors/http/ut sole+chunk+chunk ydb/library/actors/interconnect/ut_huge_cluster HugeCluster.AllToAll ydb/library/actors/interconnect/ut_huge_cluster sole chunk chunk +ydb/library/yaml_config/ut_transform test_transform.py.TestYamlConfigTransformations.test_basic[args1-dump_ds_init] ydb/library/yql/dq/opt/ut HypergraphBuild.JoinTopologiesBenchmark ydb/library/yql/dq/opt/ut sole chunk chunk ydb/library/yql/providers/generic/connector/tests/datasource/ydb test.py.test_select_positive[column_selection_col2_COL1-kqprun] @@ -179,8 +180,8 @@ ydb/tests/olap/ttl_tiering data_correctness.py.TestDataCorrectness.test ydb/tests/olap/ttl_tiering data_migration_when_alter_ttl.py.TestDataMigrationWhenAlterTtl.test ydb/tests/olap/ttl_tiering sole chunk chunk ydb/tests/olap/ttl_tiering ttl_delete_s3.py.TestDeleteS3Ttl.test_data_unchanged_after_ttl_change -ydb/tests/olap/ttl_tiering ttl_delete_s3.py.TestDeleteS3Ttl.test_ttl_delete ydb/tests/olap/ttl_tiering ttl_delete_s3.py.TestDeleteS3Ttl.test_delete_s3_tiering +ydb/tests/olap/ttl_tiering ttl_delete_s3.py.TestDeleteS3Ttl.test_ttl_delete ydb/tests/olap/ttl_tiering ttl_unavailable_s3.py.TestUnavailableS3.test ydb/tests/olap/ttl_tiering unstable_connection.py.TestUnstableConnection.test ydb/tests/postgres_integrations/go-libpq [docker_wrapper_test.py] chunk chunk diff --git a/build/conf/docs.conf b/build/conf/docs.conf index 28d0cf0d7c..c0db3f454e 100644 --- a/build/conf/docs.conf +++ b/build/conf/docs.conf @@ -53,7 +53,8 @@ _DOCS_YFM_CMD=$_DOCS_YFM_CMD_IMPL($_DOCS_CONFIG_VALUE EXTRA_INPUTS $_DOCS_EXTRA_ ### should be defined here. module _DOCS_BARE_UNIT: _BARE_UNIT { .ALLOWED=DOCS_DIR DOCS_CONFIG DOCS_VARS - .CMD=TOUCH_DOCS_MF + .CMD=$TOUCH_DOCS_MF + .STRUCT_CMD=yes .FINAL_TARGET=no .NODE_TYPE=Bundle .PEERDIR_POLICY=as_include @@ -82,7 +83,7 @@ _DOCS_LIBRARY_CMD=$_DOCS_LIBRARY_CMD_IMPL(SOURCES $_DOCS_SOURCES_VALUE INCLUDE_S # tag:docs module DOCS_LIBRARY: _DOCS_BARE_UNIT { - .CMD=_DOCS_LIBRARY_CMD + .CMD=$_DOCS_LIBRARY_CMD .ALIASES=SRCS=_DOCS_SRCS DOCS_DIR=_YFM_DOCS_DIR .EPILOGUE=_DOCS_LIBRARY_EPILOGUE .NODE_TYPE=Library @@ -150,7 +151,7 @@ macro _DOCS_YFM_USE_PLANTUML() { ### @see: [DOCS_DIR()](#macro_DOCS_DIR), [DOCS_CONFIG()](#macro_DOCS_CONFIG), [DOCS_VARS()](#macro_DOCS_VARS). multimodule DOCS { module DOCSBOOK: _DOCS_BASE_UNIT { - .CMD=_DOCS_YFM_CMD + .CMD=$_DOCS_YFM_CMD .FINAL_TARGET=yes .PEERDIR_POLICY=as_build_from .IGNORED=DOCS_DIR DOCS_INCLUDE_SOURCES DOCS_COPY_FILES PEERDIR PYTHON RUN_PROGRAM RUN_PYTHON3 RUN_LUA RUN_JAVA_PROGRAM FROM_SANDBOX SRCS COPY COPY_FILE FILES @@ -167,7 +168,7 @@ multimodule DOCS { } module DOCSLIB: _DOCS_BASE_UNIT { - .CMD=_DOCS_YFM_CMD + .CMD=$_DOCS_YFM_CMD .PEERDIR_POLICY=as_build_from .IGNORED=DOCS_DIR DOCS_INCLUDE_SOURCES DOCS_COPY_FILES PEERDIR PYTHON RUN_PROGRAM RUN_PYTHON3 RUN_LUA RUN_JAVA_PROGRAM FROM_SANDBOX SRCS COPY COPY_FILE FILES .PEERDIRSELF=DOCSLIB_INTERNAL diff --git a/build/conf/go.conf b/build/conf/go.conf index 83b03e753c..9776f39287 100644 --- a/build/conf/go.conf +++ b/build/conf/go.conf @@ -1022,5 +1022,5 @@ macro GO_MOCKGEN_MOCKS() { PEERDIR(vendor/go.uber.org/mock/gomock) # Unfortunately ${rootrel;tool:TOOL} doesn't work currently, so we use this ugly workaround $MODDIR/gen/gen$_GO_EXE_SUFFIX - RUN_PROGRAM(vendor/go.uber.org/mock/mockgen -package mocks -exec_only $MODDIR/gen/gen$_GO_EXE_SUFFIX ${GO_ARCADIA_PROJECT_PREFIX}${MOCKGEN_FROM} $MOCKGEN_TYPES STDOUT main.go TOOL $MODDIR/gen CWD $ARCADIA_BUILD_ROOT) + RUN_PROGRAM(vendor/go.uber.org/mock/mockgen -package mocks -exec_only $MODDIR/gen/gen$_GO_EXE_SUFFIX ${GO_ARCADIA_PROJECT_PREFIX}${MOCKGEN_FROM} $MOCKGEN_TYPES STDOUT main.go TOOL $MODDIR/gen CWD $ARCADIA_BUILD_ROOT ENV PATH=${GO_TOOLS_ROOT}/bin GOROOT=${GO_TOOLS_ROOT} GOCACHE=${BINDIR}/.gocache) } diff --git a/build/conf/java.conf b/build/conf/java.conf index dbc53bb371..f76b0cfc90 100644 --- a/build/conf/java.conf +++ b/build/conf/java.conf @@ -451,11 +451,6 @@ macro DEFAULT_JUNIT_JAVA_SRCS_LAYOUT() { FULL_JAVA_SRCS(RESOURCES SRCDIR resources **/* SKIP_CHECK_SRCDIR) } -macro _HASH_HELPER(Args...) { - .CMD=${hash:Args} - .SEM=${hash:Args} -} - macro _GENTAR_HELPER(HASH_SUF="hash_suf", OUT_DIR[]) { .CMD=${cwd:BINDIR} $YMAKE_PYTHON ${input:"build/scripts/autotar_gendirs.py"} --pack ${OUT_DIR} --outs ${output;tared;suf=$HASH_SUF:OUT_DIR} ${hide;kv:"tared_kind nodir"} .SEM=${hide;suf=$HASH_SUF;tared;output:OUT_DIR} diff --git a/build/conf/opensource.conf b/build/conf/opensource.conf index ee6dd6cb74..067aacf3cb 100644 --- a/build/conf/opensource.conf +++ b/build/conf/opensource.conf @@ -41,6 +41,9 @@ when ($OPENSOURCE == "yes" && $EXPORT_GRADLE == "yes") { EXPORT_SEM=yes EXPORTED_BUILD_SYSTEM_SOURCE_ROOT=${"$"}{PROJECT_SOURCE_DIR} EXPORTED_BUILD_SYSTEM_BUILD_ROOT=${"$"}{PROJECT_BINARY_DIR} +} + +when ($EXPORT_GRADLE == "yes") { EXPORT_LANG=JAVA } @@ -58,6 +61,9 @@ when ($OPENSOURCE == "yes" && $EXPORT_CMAKE == "yes") { EXPORT_SEM=yes EXPORTED_BUILD_SYSTEM_SOURCE_ROOT="${PROJECT_SOURCE_DIR}" EXPORTED_BUILD_SYSTEM_BUILD_ROOT="${PROJECT_BINARY_DIR}" +} + +when ($EXPORT_CMAKE == "yes") { EXPORT_LANG=CPP } diff --git a/build/conf/python.conf b/build/conf/python.conf index 8c0ca876a8..bfa42ee36d 100644 --- a/build/conf/python.conf +++ b/build/conf/python.conf @@ -267,6 +267,7 @@ multimodule PY3_PROGRAM { # Notify pybuild to skip all python main function definitions ENABLE(IGNORE_PY_MAIN) NO_CLANG_TIDY() + DISABLE(START_TARGET) } } @@ -435,6 +436,7 @@ multimodule PY3TEST { PEERDIR+=library/python/pytest _DONT_REQUIRE_LICENSE() WITHOUT_VERSION() + DISABLE(START_TARGET) } } diff --git a/build/conf/settings.conf b/build/conf/settings.conf index 3f0ba1ef70..761d9d83af 100644 --- a/build/conf/settings.conf +++ b/build/conf/settings.conf @@ -1,19 +1,14 @@ JSON_CACHE_IS_ATTACHED=yes LANGS_REQUIRE_BUILD_AND_SRC_ROOTS=c asm cython proto flatc swig ydl nlg -CHECK_GO_INCORRECT_DEPS=yes USE_PREBUILT_TOOLS=yes RESOLVE_FORCE_LISTDIR=no -ENABLE_NODE_SELF_UID=yes REPORT_CONFIGURE_PROGRESS=yes -FORCE_RESOLVE_MACRO_INCLS=yes -ENABLE_RERESOLVE_FOR_GENERATED_FILES=yes REPORT_ALL_DUPSRC=yes DEPS_CACHE_CONTROL_UIDS_CACHE=yes USE_GLOBAL_CMD=yes PIC_NO_PIE=no FAIL_PY2=no MAIN_OUTPUT_AS_EXTRA=yes -USE_REACHABILITY_TO_REPORT_CONF_ERRORS = yes DISABLE_ATD=yes when ($OS_WINDOWS == "yes") { diff --git a/build/plugins/_dart_fields.py b/build/plugins/_dart_fields.py index 8504891586..e102154a34 100644 --- a/build/plugins/_dart_fields.py +++ b/build/plugins/_dart_fields.py @@ -1137,6 +1137,20 @@ class TestFiles: # https://a.yandex-team.ru/arcadia/devtools/ya/test/dartfile/__init__.py?rev=r14292146#L10 KEY2 = 'FILES' + _GRUT_PREFIX = 'grut' + _GRUT_INCLUDE_LINTER_TEST_PATHS = ( + 'grut/libs/bigrt/clients', + 'grut/libs/bigrt/common', + 'grut/libs/bigrt/data', + 'grut/libs/bigrt/event_filter', + 'grut/libs/bigrt/info_keepers', + 'grut/libs/bigrt/processor', + 'grut/libs/bigrt/profile', + 'grut/libs/bigrt/profiles', + 'grut/libs/bigrt/queue_info_config', + 'grut/libs/shooter', + ) + @classmethod def value(cls, unit, flat_args, spec_args): data_re = re.compile(r"sbr:/?/?(\d+)=?.*") @@ -1230,6 +1244,13 @@ class TestFiles: @classmethod def cpp_linter_files(cls, unit, flat_args, spec_args): + upath = unit.path()[3:] + if upath.startswith(cls._GRUT_PREFIX): + for path in cls._GRUT_INCLUDE_LINTER_TEST_PATHS: + if os.path.commonpath([upath, path]) == path: + break + else: + raise DartValueError() files_dart = _reference_group_var("ALL_SRCS", consts.STYLE_CPP_ALL_EXTS) return {cls.KEY: files_dart, cls.KEY2: files_dart} diff --git a/build/plugins/lib/test_const/__init__.py b/build/plugins/lib/test_const/__init__.py index cd7534926c..293b514d01 100644 --- a/build/plugins/lib/test_const/__init__.py +++ b/build/plugins/lib/test_const/__init__.py @@ -181,6 +181,7 @@ NODEJS_RESOURCE = 'NODEJS_RESOURCE_GLOBAL' NYC_RESOURCE = 'NYC_RESOURCE_GLOBAL' RUFF_RESOURCE = 'RUFF_RESOURCE_GLOBAL' CLANG_FORMAT_RESOURCE = 'CLANG_FORMAT_RESOURCE_GLOBAL' +CLANG_FORMAT_15_RESOURCE = 'CLANG_FORMAT_15_RESOURCE_GLOBAL' # test_tool resource for host platform. # source - build/platform/test_tool/host.ya.make.inc. diff --git a/build/plugins/ytest.py b/build/plugins/ytest.py index 7adb582a91..4ae288e062 100644 --- a/build/plugins/ytest.py +++ b/build/plugins/ytest.py @@ -141,7 +141,18 @@ def validate_test(unit, kw): if valid_kw.get('SCRIPT-REL-PATH') == 'boost.test': project_path = valid_kw.get('BUILD-FOLDER-PATH', "") if not project_path.startswith( - ("contrib", "mail", "maps", "tools/idl", "metrika", "devtools", "mds", "yandex_io", "smart_devices") + ( + "contrib", + "mail", + "maps", + "mobile/geo/maps", + "tools/idl", + "metrika", + "devtools", + "mds", + "yandex_io", + "smart_devices", + ) ): errors.append("BOOSTTEST is not allowed here") diff --git a/build/scripts/fetch_resource.py b/build/scripts/fetch_resource.py deleted file mode 100644 index d5af311e5d..0000000000 --- a/build/scripts/fetch_resource.py +++ /dev/null @@ -1,43 +0,0 @@ -import urllib2 -import argparse -import xmlrpclib - - -def parse_args(): - parser = argparse.ArgumentParser() - parser.add_argument('-r', '--resource-id', type=int, required=True) - parser.add_argument('-o', '--output', required=True) - return parser.parse_args() - - -def fetch(url, retries=4, timeout=5): - for i in xrange(retries): - try: - return urllib2.urlopen(url, timeout=timeout).read() - - except Exception: - if i + 1 < retries: - continue - - else: - raise - - -def fetch_resource(id_): - urls = xmlrpclib.ServerProxy("https://sandbox.yandex-team.ru/sandbox/xmlrpc").get_resource_http_links(id_) - - for u in urls: - try: - return fetch(u) - - except Exception: - continue - - raise Exception('Cannot fetch resource {}'.format(id_)) - - -if __name__ == '__main__': - args = parse_args() - - with open(args.output, 'wb') as f: - f.write(fetch_resource(int(args.resource_id))) diff --git a/build/scripts/ya.make b/build/scripts/ya.make index e59871172b..165a8da6db 100644 --- a/build/scripts/ya.make +++ b/build/scripts/ya.make @@ -16,7 +16,6 @@ IF (PY2) fetch_from_archive.py fetch_from_mds.py fetch_from_sandbox.py - fetch_resource.py gen_java_codenav_entry.py gen_py3_reg.py go_tool.py diff --git a/build/sysincl/emscripten.yml b/build/sysincl/emscripten.yml index 2ac68cc16c..8c637abebe 100644 --- a/build/sysincl/emscripten.yml +++ b/build/sysincl/emscripten.yml @@ -108,6 +108,9 @@ - contrib/restricted/emscripten/system/lib/libc/musl/src/internal/syscall.h - contrib/restricted/emscripten/system/lib/libc/musl/arch/emscripten/bits/syscall.h + - syscall_arch.h: + - contrib/restricted/emscripten/system/lib/libc/musl/arch/emscripten/syscall_arch.h + - inttypes.h: contrib/restricted/emscripten/system/lib/libc/musl/include/inttypes.h - stddef.h: contrib/restricted/emscripten/system/lib/libc/musl/include/stddef.h - stdint.h: contrib/restricted/emscripten/system/lib/libc/musl/include/stdint.h diff --git a/build/ymake.core.conf b/build/ymake.core.conf index 1ed20b6aab..b96a5a412b 100644 --- a/build/ymake.core.conf +++ b/build/ymake.core.conf @@ -487,8 +487,13 @@ macro CHECK_DEPENDENT_DIRS(TYPE, ALL?"UNUSED":"", PEERDIRS?"PEERDIRS":"ALL", RES SET_APPEND(CHECK_DEPENDENT_DIRS_TYPES $TYPE) } +macro _HASH_HELPER(Args...) { + .CMD=${hash:Args} + .SEM=${hash:Args} +} + macro _RESOURCE_SEM(INPUTS[], KEYS[], OPTS[]) { - SET(RESOURCE_OUTPUT ${hash:INPUTS}.cpp) + SET(RESOURCE_OUTPUT $_HASH_HELPER($INPUTS $KEYS $OPTS).cpp) .SEM=target_macroses-ITEM && target_macroses-macro resources && target_macroses-args ${output;global:RESOURCE_OUTPUT} INPUTS ${input:INPUTS} KEYS $KEYS OPTS $OPTS ${hide;tool:"tools/rescompiler/bin"} } @@ -775,9 +780,15 @@ module _BASE_UNIT: _BARE_UNIT { when ($OS_EMSCRIPTEN == "yes") { when ($NOLIBC != "yes") { PEERDIR+=contrib/restricted/emscripten/include + # Important: In arcadia we usually build standalone WASM applications, + # so standalone library must go before lib/c in linker command-line. + # Emscripten uses two weak definitions for __clock_gettime function, + # so linker will pick one from the first library present in command + # line. + PEERDIR+=contrib/restricted/emscripten/system/lib/standalonewasm + PEERDIR+=contrib/restricted/emscripten/system/lib/stubs PEERDIR+=contrib/restricted/emscripten/system/lib/c PEERDIR+=contrib/restricted/emscripten/system/lib/dlmalloc - PEERDIR+=contrib/restricted/emscripten/system/lib/standalonewasm } } diff --git a/contrib/libs/cxxsupp/libcxxrt/.yandex_meta/override.nix b/contrib/libs/cxxsupp/libcxxrt/.yandex_meta/override.nix index d80f39e895..af8de9dc63 100644 --- a/contrib/libs/cxxsupp/libcxxrt/.yandex_meta/override.nix +++ b/contrib/libs/cxxsupp/libcxxrt/.yandex_meta/override.nix @@ -1,12 +1,12 @@ pkgs: attrs: with pkgs; rec { - version = "2024-10-14"; - revision = "76435c4451aeb5e04e9500b090293347a38cef8d"; + version = "2025-02-25"; + revision = "a6f71cbc3a1e1b8b9df241e081fa0ffdcde96249"; src = fetchFromGitHub { owner = "libcxxrt"; repo = "libcxxrt"; rev = "${revision}"; - hash = "sha256-U7mq79/0xbyRr2+KUMKgEvyd2lfr3Q5GrByt/8J9sC8="; + hash = "sha256-+oTjU/DgOEIwJebSVkSEt22mJSdeONozB8FfzEiESHU="; }; nativeBuildInputs = [ cmake ]; diff --git a/contrib/libs/cxxsupp/libcxxrt/exception.cc b/contrib/libs/cxxsupp/libcxxrt/exception.cc index 088e62e449..49dbeff5d8 100644 --- a/contrib/libs/cxxsupp/libcxxrt/exception.cc +++ b/contrib/libs/cxxsupp/libcxxrt/exception.cc @@ -215,7 +215,7 @@ static_assert(offsetof(__cxa_dependent_exception, unwindHeader) == namespace std { - void unexpected(); + [[noreturn]] void unexpected(); class exception { public: @@ -287,12 +287,16 @@ namespace std using namespace ABI_NAMESPACE; +#ifdef LIBCXXRT_NO_DEFAULT_TERMINATE_DIAGNOSTICS +/** The global termination handler. */ +static atomic<terminate_handler> terminateHandler = abort; +#else /** * Callback function used with _Unwind_Backtrace(). * * Prints a stack trace. Used only for debugging help. * - * Note: As of FreeBSD 8.1, dladd() still doesn't work properly, so this only + * Note: As of FreeBSD 8.1, dladdr() still doesn't work properly, so this only * correctly prints function names from public, relocatable, symbols. */ static _Unwind_Reason_Code trace(struct _Unwind_Context *context, void *c) @@ -313,24 +317,20 @@ static _Unwind_Reason_Code trace(struct _Unwind_Context *context, void *c) } static void terminate_with_diagnostics() { - __cxa_eh_globals *globals = __cxa_get_globals(); - __cxa_exception *ex = globals->caughtExceptions; + __cxa_eh_globals *globals = __cxa_get_globals(); + __cxa_exception *ex = globals->caughtExceptions; - if (ex != nullptr) { - fprintf(stderr, "uncaught exception:\n address -> %p\n", (void*)ex); + if (ex != nullptr) { + fprintf(stderr, "Terminating due to uncaught exception %p", static_cast<void*>(ex)); ex = realExceptionFromException(ex); - - const __class_type_info *e_ti = + const __class_type_info *e_ti = static_cast<const __class_type_info*>(&typeid(std::exception)); - const __class_type_info *throw_ti = + const __class_type_info *throw_ti = dynamic_cast<const __class_type_info*>(ex->exceptionType); - if (throw_ti) { void* ptr = ex + 1; - if (throw_ti->__do_upcast(e_ti, &ptr)) { std::exception* e = static_cast<std::exception*>(ptr); - if (e) { fprintf(stderr, " what() -> \"%s\"\n", e->what()); } @@ -342,14 +342,19 @@ static void terminate_with_diagnostics() { const char *mangled = ex->exceptionType->name(); int status; demangled = __cxa_demangle(mangled, demangled, &bufferSize, &status); - fprintf(stderr, " type -> %s\n", status == 0 ? demangled : mangled); + fprintf(stderr, " of type %s\n", status == 0 ? demangled : mangled); if (status == 0) { free(demangled); } + + _Unwind_Backtrace(trace, 0); } - abort(); + + abort(); } /** The global termination handler. */ static atomic<terminate_handler> terminateHandler = terminate_with_diagnostics; +#endif + /** The global unexpected exception handler. */ static atomic<unexpected_handler> unexpectedHandler = std::terminate; @@ -788,7 +793,8 @@ void __cxa_free_dependent_exception(void *thrown_exception) * Report a failure that occurred when attempting to throw an exception. * * If the failure happened by falling off the end of the stack without finding - * a handler, prints a back trace before aborting. + * a handler, catch the exception before calling terminate. The default + * terminate handler will print a backtrace before aborting. */ #if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 4) extern "C" void *__cxa_begin_catch(void *e) _LIBCXXRT_NOEXCEPT; @@ -810,39 +816,6 @@ static void report_failure(_Unwind_Reason_Code err, __cxa_exception *thrown_exce #endif case _URC_END_OF_STACK: __cxa_begin_catch (&(thrown_exception->unwindHeader)); - std::terminate(); - fprintf(stderr, "Terminating due to uncaught exception %p", - static_cast<void*>(thrown_exception)); - thrown_exception = realExceptionFromException(thrown_exception); - static const __class_type_info *e_ti = - static_cast<const __class_type_info*>(&typeid(std::exception)); - const __class_type_info *throw_ti = - dynamic_cast<const __class_type_info*>(thrown_exception->exceptionType); - if (throw_ti) - { - std::exception *e = - static_cast<std::exception*>(e_ti->cast_to(static_cast<void*>(thrown_exception+1), - throw_ti)); - if (e) - { - fprintf(stderr, " '%s'", e->what()); - } - } - - size_t bufferSize = 128; - char *demangled = static_cast<char*>(malloc(bufferSize)); - const char *mangled = thrown_exception->exceptionType->name(); - int status; - demangled = __cxa_demangle(mangled, demangled, &bufferSize, &status); - fprintf(stderr, " of type %s\n", - status == 0 ? demangled : mangled); - if (status == 0) { free(demangled); } - // Print a back trace if no handler is found. - // TODO: Make this optional - _Unwind_Backtrace(trace, 0); - - // Just abort. No need to call std::terminate for the second time - abort(); break; } std::terminate(); @@ -1642,28 +1615,34 @@ namespace std if (0 != info && 0 != info->terminateHandler) { info->terminateHandler(); - // Should not be reached - a terminate handler is not expected to - // return. - abort(); } - terminateHandler.load()(); + else + { + terminateHandler.load()(); + } + // Should not be reached - a terminate handler is not expected + // to return. + abort(); } /** * Called when an unexpected exception is encountered (i.e. an exception * violates an exception specification). This calls abort() unless a * custom handler has been set.. */ - void unexpected() + [[noreturn]] void unexpected() { static __cxa_thread_info *info = thread_info(); if (0 != info && 0 != info->unexpectedHandler) { info->unexpectedHandler(); - // Should not be reached - a terminate handler is not expected to - // return. - abort(); } - unexpectedHandler.load()(); + else + { + unexpectedHandler.load()(); + } + // Should not be reached - a unexpected handler is not expected + // to return. + abort(); } /** * Returns whether there are any exceptions currently being thrown that diff --git a/contrib/libs/cxxsupp/libcxxrt/patches/do-upcast-not-cast-to.patch b/contrib/libs/cxxsupp/libcxxrt/patches/do-upcast-not-cast-to.patch new file mode 100644 index 0000000000..93fce488a5 --- /dev/null +++ b/contrib/libs/cxxsupp/libcxxrt/patches/do-upcast-not-cast-to.patch @@ -0,0 +1,17 @@ +--- contrib/libs/cxxsupp/libcxxrt/exception.cc (index) ++++ contrib/libs/cxxsupp/libcxxrt/exception.cc (working tree) +@@ -321,7 +321,7 @@ static void terminate_with_diagnostics() { +- if (throw_ti) +- { +- std::exception *e = +- static_cast<std::exception*>(e_ti->cast_to(static_cast<void*>(ex+1), throw_ti)); +- if (e) +- { +- fprintf(stderr, " '%s'", e->what()); ++ if (throw_ti) { ++ void* ptr = ex + 1; ++ if (throw_ti->__do_upcast(e_ti, &ptr)) { ++ std::exception* e = static_cast<std::exception*>(ptr); ++ if (e) { ++ fprintf(stderr, " what() -> \"%s\"\n", e->what()); ++ } diff --git a/contrib/libs/cxxsupp/libcxxrt/patches/pr41.1-move-trace.patch b/contrib/libs/cxxsupp/libcxxrt/patches/pr41.1-move-trace.patch deleted file mode 100644 index b1331c9445..0000000000 --- a/contrib/libs/cxxsupp/libcxxrt/patches/pr41.1-move-trace.patch +++ /dev/null @@ -1,74 +0,0 @@ -commit 690915810d43430667f8d7b84fc7f88784f68fdf (HEAD -> update-libcxxrt) -author: thegeorg -date: 2025-02-26T13:56:00+03:00 - - Revert "Simplify libcxxrt patches" - - This reverts commit 4358151e7723e37a39cf6f5478bfa336aa687fd4, reversing - changes made to ad7618a2219d22bcd89a67fe082f290b4a9656ef. - ---- contrib/libs/cxxsupp/libcxxrt/exception.cc (27ffe99a31d7fb0dd8a933c936f70853dac6041a) -+++ contrib/libs/cxxsupp/libcxxrt/exception.cc (690915810d43430667f8d7b84fc7f88784f68fdf) -@@ -287,6 +287,30 @@ namespace std - - using namespace ABI_NAMESPACE; - -+/** -+ * Callback function used with _Unwind_Backtrace(). -+ * -+ * Prints a stack trace. Used only for debugging help. -+ * -+ * Note: As of FreeBSD 8.1, dladd() still doesn't work properly, so this only -+ * correctly prints function names from public, relocatable, symbols. -+ */ -+static _Unwind_Reason_Code trace(struct _Unwind_Context *context, void *c) -+{ -+ Dl_info myinfo; -+ int mylookup = -+ dladdr(reinterpret_cast<void *>(__cxa_current_exception_type), &myinfo); -+ void *ip = reinterpret_cast<void*>(_Unwind_GetIP(context)); -+ Dl_info info; -+ if (dladdr(ip, &info) != 0) -+ { -+ if (mylookup == 0 || strcmp(info.dli_fname, myinfo.dli_fname) != 0) -+ { -+ printf("%p:%s() in %s\n", ip, info.dli_sname, info.dli_fname); -+ } -+ } -+ return _URC_CONTINUE_UNWIND; -+} - - - /** The global termination handler. */ -@@ -760,31 +785,6 @@ void __cxa_free_dependent_exception(void *thrown_exception) - } - - /** -- * Callback function used with _Unwind_Backtrace(). -- * -- * Prints a stack trace. Used only for debugging help. -- * -- * Note: As of FreeBSD 8.1, dladd() still doesn't work properly, so this only -- * correctly prints function names from public, relocatable, symbols. -- */ --static _Unwind_Reason_Code trace(struct _Unwind_Context *context, void *c) --{ -- Dl_info myinfo; -- int mylookup = -- dladdr(reinterpret_cast<void *>(__cxa_current_exception_type), &myinfo); -- void *ip = reinterpret_cast<void*>(_Unwind_GetIP(context)); -- Dl_info info; -- if (dladdr(ip, &info) != 0) -- { -- if (mylookup == 0 || strcmp(info.dli_fname, myinfo.dli_fname) != 0) -- { -- printf("%p:%s() in %s\n", ip, info.dli_sname, info.dli_fname); -- } -- } -- return _URC_CONTINUE_UNWIND; --} -- --/** - * Report a failure that occurred when attempting to throw an exception. - * - * If the failure happened by falling off the end of the stack without finding diff --git a/contrib/libs/cxxsupp/libcxxrt/patches/pr41.2-fix-trace-format.patch b/contrib/libs/cxxsupp/libcxxrt/patches/pr41.2-fix-trace-format.patch deleted file mode 100644 index a70c234b4f..0000000000 --- a/contrib/libs/cxxsupp/libcxxrt/patches/pr41.2-fix-trace-format.patch +++ /dev/null @@ -1,48 +0,0 @@ ---- contrib/libs/cxxsupp/libcxxrt/exception.cc (index) -+++ contrib/libs/cxxsupp/libcxxrt/exception.cc (working tree) -@@ -312,9 +312,44 @@ static _Unwind_Reason_Code trace(struct _Unwind_Context *context, void *c) - return _URC_CONTINUE_UNWIND; - } - -+static void terminate_with_diagnostics() { -+ __cxa_eh_globals *globals = __cxa_get_globals(); -+ __cxa_exception *ex = globals->caughtExceptions; -+ -+ if (ex != nullptr) { -+ fprintf(stderr, "uncaught exception:\n address -> %p\n", (void*)ex); -+ ex = realExceptionFromException(ex); -+ -+ const __class_type_info *e_ti = -+ static_cast<const __class_type_info*>(&typeid(std::exception)); -+ const __class_type_info *throw_ti = -+ dynamic_cast<const __class_type_info*>(ex->exceptionType); -+ -+ if (throw_ti) { -+ void* ptr = ex + 1; -+ -+ if (throw_ti->__do_upcast(e_ti, &ptr)) { -+ std::exception* e = static_cast<std::exception*>(ptr); -+ -+ if (e) { -+ fprintf(stderr, " what() -> \"%s\"\n", e->what()); -+ } -+ } -+ } -+ -+ size_t bufferSize = 128; -+ char *demangled = static_cast<char*>(malloc(bufferSize)); -+ const char *mangled = ex->exceptionType->name(); -+ int status; -+ demangled = __cxa_demangle(mangled, demangled, &bufferSize, &status); -+ fprintf(stderr, " type -> %s\n", status == 0 ? demangled : mangled); -+ if (status == 0) { free(demangled); } -+ } -+ abort(); -+} - - /** The global termination handler. */ --static atomic<terminate_handler> terminateHandler = abort; -+static atomic<terminate_handler> terminateHandler = terminate_with_diagnostics; - /** The global unexpected exception handler. */ - static atomic<unexpected_handler> unexpectedHandler = std::terminate; - diff --git a/contrib/libs/cxxsupp/libcxxrt/patches/pr49-cosmetics.patch b/contrib/libs/cxxsupp/libcxxrt/patches/pr49-cosmetics.patch new file mode 100644 index 0000000000..740aed0c11 --- /dev/null +++ b/contrib/libs/cxxsupp/libcxxrt/patches/pr49-cosmetics.patch @@ -0,0 +1,24 @@ +From 9a9bcd8ef3e635194f9bfaa1085b8ed2b88455e3 Mon Sep 17 00:00:00 2001 +From: Yuriy Chernyshov <thegeorg@yandex-team.com> +Date: Thu, 27 Feb 2025 16:38:01 +0100 +Subject: [PATCH] Merge short lines into one + +We still need to reduce the diff with our downstream, as #41 does not work for us out of the box. +--- + src/exception.cc | 3 +-- + 1 file changed, 1 insertion(+), 2 deletions(-) + +diff --git a/src/exception.cc b/src/exception.cc +index 9bcf903..ccf32de 100644 +--- a/exception.cc ++++ b/exception.cc +@@ -356,8 +356,7 @@ static void terminate_with_diagnostics() { + const char *mangled = ex->exceptionType->name(); + int status; + demangled = __cxa_demangle(mangled, demangled, &bufferSize, &status); +- fprintf(stderr, " of type %s\n", +- status == 0 ? demangled : mangled); ++ fprintf(stderr, " of type %s\n", status == 0 ? demangled : mangled); + if (status == 0) { free(demangled); } + + _Unwind_Backtrace(trace, 0); diff --git a/contrib/libs/cxxsupp/libcxxrt/ya.make b/contrib/libs/cxxsupp/libcxxrt/ya.make index f9eb8743d8..f9b1964b1f 100644 --- a/contrib/libs/cxxsupp/libcxxrt/ya.make +++ b/contrib/libs/cxxsupp/libcxxrt/ya.make @@ -11,9 +11,9 @@ LICENSE( LICENSE_TEXTS(.yandex_meta/licenses.list.txt) -VERSION(2024-10-14) +VERSION(2025-02-25) -ORIGINAL_SOURCE(https://github.com/libcxxrt/libcxxrt/archive/76435c4451aeb5e04e9500b090293347a38cef8d.tar.gz) +ORIGINAL_SOURCE(https://github.com/libcxxrt/libcxxrt/archive/a6f71cbc3a1e1b8b9df241e081fa0ffdcde96249.tar.gz) PEERDIR( contrib/libs/libunwind diff --git a/contrib/libs/llvm14/lib/CodeGen/MIRParser/ya.make b/contrib/libs/llvm14/lib/CodeGen/MIRParser/ya.make index 1cc779f6a6..b946280f8b 100644 --- a/contrib/libs/llvm14/lib/CodeGen/MIRParser/ya.make +++ b/contrib/libs/llvm14/lib/CodeGen/MIRParser/ya.make @@ -2,6 +2,8 @@ LIBRARY() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/lib/DWARFLinker/ya.make b/contrib/libs/llvm14/lib/DWARFLinker/ya.make index 1c8ae4a195..8c5d3ae624 100644 --- a/contrib/libs/llvm14/lib/DWARFLinker/ya.make +++ b/contrib/libs/llvm14/lib/DWARFLinker/ya.make @@ -2,6 +2,8 @@ LIBRARY() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/lib/DWP/ya.make b/contrib/libs/llvm14/lib/DWP/ya.make index 4c42a8d7d8..013db9379a 100644 --- a/contrib/libs/llvm14/lib/DWP/ya.make +++ b/contrib/libs/llvm14/lib/DWP/ya.make @@ -2,6 +2,8 @@ LIBRARY() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/lib/DebugInfo/GSYM/ya.make b/contrib/libs/llvm14/lib/DebugInfo/GSYM/ya.make index 71b4a338a4..f62cf1c9be 100644 --- a/contrib/libs/llvm14/lib/DebugInfo/GSYM/ya.make +++ b/contrib/libs/llvm14/lib/DebugInfo/GSYM/ya.make @@ -2,6 +2,8 @@ LIBRARY() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/lib/DebugInfo/PDB/ya.make b/contrib/libs/llvm14/lib/DebugInfo/PDB/ya.make index e65f185591..efd2086347 100644 --- a/contrib/libs/llvm14/lib/DebugInfo/PDB/ya.make +++ b/contrib/libs/llvm14/lib/DebugInfo/PDB/ya.make @@ -2,6 +2,8 @@ LIBRARY() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/lib/DebugInfo/Symbolize/ya.make b/contrib/libs/llvm14/lib/DebugInfo/Symbolize/ya.make index 1e9ece62ec..e8f8472996 100644 --- a/contrib/libs/llvm14/lib/DebugInfo/Symbolize/ya.make +++ b/contrib/libs/llvm14/lib/DebugInfo/Symbolize/ya.make @@ -2,6 +2,8 @@ LIBRARY() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/lib/ExecutionEngine/Interpreter/ya.make b/contrib/libs/llvm14/lib/ExecutionEngine/Interpreter/ya.make index cf474449bf..2dcc00b4e9 100644 --- a/contrib/libs/llvm14/lib/ExecutionEngine/Interpreter/ya.make +++ b/contrib/libs/llvm14/lib/ExecutionEngine/Interpreter/ya.make @@ -2,6 +2,8 @@ LIBRARY() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/lib/ExecutionEngine/JITLink/ya.make b/contrib/libs/llvm14/lib/ExecutionEngine/JITLink/ya.make index e1844bf4e7..f382e26e31 100644 --- a/contrib/libs/llvm14/lib/ExecutionEngine/JITLink/ya.make +++ b/contrib/libs/llvm14/lib/ExecutionEngine/JITLink/ya.make @@ -2,6 +2,8 @@ LIBRARY() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE( diff --git a/contrib/libs/llvm14/lib/ExecutionEngine/Orc/ya.make b/contrib/libs/llvm14/lib/ExecutionEngine/Orc/ya.make index 64d9456302..062953b4fb 100644 --- a/contrib/libs/llvm14/lib/ExecutionEngine/Orc/ya.make +++ b/contrib/libs/llvm14/lib/ExecutionEngine/Orc/ya.make @@ -2,6 +2,8 @@ LIBRARY() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/lib/Extensions/ya.make b/contrib/libs/llvm14/lib/Extensions/ya.make index 87ebf15124..5a55406f1a 100644 --- a/contrib/libs/llvm14/lib/Extensions/ya.make +++ b/contrib/libs/llvm14/lib/Extensions/ya.make @@ -4,6 +4,8 @@ LIBRARY() WITHOUT_LICENSE_TEXTS() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(NCSA) diff --git a/contrib/libs/llvm14/lib/FileCheck/ya.make b/contrib/libs/llvm14/lib/FileCheck/ya.make index 3c23b3ec2e..1cc8e15030 100644 --- a/contrib/libs/llvm14/lib/FileCheck/ya.make +++ b/contrib/libs/llvm14/lib/FileCheck/ya.make @@ -2,6 +2,8 @@ LIBRARY() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/lib/Frontend/OpenACC/ya.make b/contrib/libs/llvm14/lib/Frontend/OpenACC/ya.make index 8bcda0013e..7632111f27 100644 --- a/contrib/libs/llvm14/lib/Frontend/OpenACC/ya.make +++ b/contrib/libs/llvm14/lib/Frontend/OpenACC/ya.make @@ -2,6 +2,8 @@ LIBRARY() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/lib/FuzzMutate/ya.make b/contrib/libs/llvm14/lib/FuzzMutate/ya.make index 3b5aa6acc4..e0abb51d03 100644 --- a/contrib/libs/llvm14/lib/FuzzMutate/ya.make +++ b/contrib/libs/llvm14/lib/FuzzMutate/ya.make @@ -2,6 +2,8 @@ LIBRARY() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/lib/InterfaceStub/ya.make b/contrib/libs/llvm14/lib/InterfaceStub/ya.make index 3e94c6c1dc..53ede22468 100644 --- a/contrib/libs/llvm14/lib/InterfaceStub/ya.make +++ b/contrib/libs/llvm14/lib/InterfaceStub/ya.make @@ -2,6 +2,8 @@ LIBRARY() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/lib/LTO/ya.make b/contrib/libs/llvm14/lib/LTO/ya.make index c9329a9e53..0175c3eec4 100644 --- a/contrib/libs/llvm14/lib/LTO/ya.make +++ b/contrib/libs/llvm14/lib/LTO/ya.make @@ -2,6 +2,8 @@ LIBRARY() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/lib/LineEditor/ya.make b/contrib/libs/llvm14/lib/LineEditor/ya.make index 9eb4ef0191..f3325b9427 100644 --- a/contrib/libs/llvm14/lib/LineEditor/ya.make +++ b/contrib/libs/llvm14/lib/LineEditor/ya.make @@ -2,6 +2,8 @@ LIBRARY() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/lib/MCA/ya.make b/contrib/libs/llvm14/lib/MCA/ya.make index bfb93027dc..93678d8535 100644 --- a/contrib/libs/llvm14/lib/MCA/ya.make +++ b/contrib/libs/llvm14/lib/MCA/ya.make @@ -2,6 +2,8 @@ LIBRARY() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/lib/ObjectYAML/ya.make b/contrib/libs/llvm14/lib/ObjectYAML/ya.make index 7f6a7c0a03..94cbbc66f5 100644 --- a/contrib/libs/llvm14/lib/ObjectYAML/ya.make +++ b/contrib/libs/llvm14/lib/ObjectYAML/ya.make @@ -2,6 +2,8 @@ LIBRARY() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/lib/Option/ya.make b/contrib/libs/llvm14/lib/Option/ya.make index d981c842bb..71c473b2e5 100644 --- a/contrib/libs/llvm14/lib/Option/ya.make +++ b/contrib/libs/llvm14/lib/Option/ya.make @@ -2,6 +2,8 @@ LIBRARY() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/lib/Passes/ya.make b/contrib/libs/llvm14/lib/Passes/ya.make index 195eb5f388..d5598b9329 100644 --- a/contrib/libs/llvm14/lib/Passes/ya.make +++ b/contrib/libs/llvm14/lib/Passes/ya.make @@ -2,6 +2,8 @@ LIBRARY() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/lib/ProfileData/Coverage/ya.make b/contrib/libs/llvm14/lib/ProfileData/Coverage/ya.make index 72bc2af83d..002c263931 100644 --- a/contrib/libs/llvm14/lib/ProfileData/Coverage/ya.make +++ b/contrib/libs/llvm14/lib/ProfileData/Coverage/ya.make @@ -2,6 +2,8 @@ LIBRARY() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/lib/Target/AArch64/AsmParser/ya.make b/contrib/libs/llvm14/lib/Target/AArch64/AsmParser/ya.make index 86b0033a93..b4e2198098 100644 --- a/contrib/libs/llvm14/lib/Target/AArch64/AsmParser/ya.make +++ b/contrib/libs/llvm14/lib/Target/AArch64/AsmParser/ya.make @@ -2,6 +2,8 @@ LIBRARY() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/lib/Target/AArch64/Disassembler/ya.make b/contrib/libs/llvm14/lib/Target/AArch64/Disassembler/ya.make index 0153cb900a..b5c1b176e2 100644 --- a/contrib/libs/llvm14/lib/Target/AArch64/Disassembler/ya.make +++ b/contrib/libs/llvm14/lib/Target/AArch64/Disassembler/ya.make @@ -2,6 +2,8 @@ LIBRARY() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/lib/Target/AArch64/MCTargetDesc/ya.make b/contrib/libs/llvm14/lib/Target/AArch64/MCTargetDesc/ya.make index 3996127ec9..5774bc4bc2 100644 --- a/contrib/libs/llvm14/lib/Target/AArch64/MCTargetDesc/ya.make +++ b/contrib/libs/llvm14/lib/Target/AArch64/MCTargetDesc/ya.make @@ -2,6 +2,8 @@ LIBRARY() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/lib/Target/AArch64/TargetInfo/ya.make b/contrib/libs/llvm14/lib/Target/AArch64/TargetInfo/ya.make index 33f279fb20..321ca1c837 100644 --- a/contrib/libs/llvm14/lib/Target/AArch64/TargetInfo/ya.make +++ b/contrib/libs/llvm14/lib/Target/AArch64/TargetInfo/ya.make @@ -2,6 +2,8 @@ LIBRARY() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/lib/Target/AArch64/Utils/ya.make b/contrib/libs/llvm14/lib/Target/AArch64/Utils/ya.make index 86a6860e41..472e3f3fdd 100644 --- a/contrib/libs/llvm14/lib/Target/AArch64/Utils/ya.make +++ b/contrib/libs/llvm14/lib/Target/AArch64/Utils/ya.make @@ -2,6 +2,8 @@ LIBRARY() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/lib/Target/AArch64/ya.make b/contrib/libs/llvm14/lib/Target/AArch64/ya.make index a773c70646..dcaa68a65d 100644 --- a/contrib/libs/llvm14/lib/Target/AArch64/ya.make +++ b/contrib/libs/llvm14/lib/Target/AArch64/ya.make @@ -2,6 +2,8 @@ LIBRARY() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/lib/Target/ARM/AsmParser/ya.make b/contrib/libs/llvm14/lib/Target/ARM/AsmParser/ya.make index f66c2b4b0f..74d91274cc 100644 --- a/contrib/libs/llvm14/lib/Target/ARM/AsmParser/ya.make +++ b/contrib/libs/llvm14/lib/Target/ARM/AsmParser/ya.make @@ -2,6 +2,8 @@ LIBRARY() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/lib/Target/ARM/Disassembler/ya.make b/contrib/libs/llvm14/lib/Target/ARM/Disassembler/ya.make index a66db9c90f..3ef6e75115 100644 --- a/contrib/libs/llvm14/lib/Target/ARM/Disassembler/ya.make +++ b/contrib/libs/llvm14/lib/Target/ARM/Disassembler/ya.make @@ -2,6 +2,8 @@ LIBRARY() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/lib/Target/ARM/MCTargetDesc/ya.make b/contrib/libs/llvm14/lib/Target/ARM/MCTargetDesc/ya.make index 53be40e3a6..4c4d22f68c 100644 --- a/contrib/libs/llvm14/lib/Target/ARM/MCTargetDesc/ya.make +++ b/contrib/libs/llvm14/lib/Target/ARM/MCTargetDesc/ya.make @@ -2,6 +2,8 @@ LIBRARY() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/lib/Target/ARM/TargetInfo/ya.make b/contrib/libs/llvm14/lib/Target/ARM/TargetInfo/ya.make index c63a083fb5..53f5d6c169 100644 --- a/contrib/libs/llvm14/lib/Target/ARM/TargetInfo/ya.make +++ b/contrib/libs/llvm14/lib/Target/ARM/TargetInfo/ya.make @@ -2,6 +2,8 @@ LIBRARY() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/lib/Target/ARM/Utils/ya.make b/contrib/libs/llvm14/lib/Target/ARM/Utils/ya.make index 74bef2cc19..876f51dd82 100644 --- a/contrib/libs/llvm14/lib/Target/ARM/Utils/ya.make +++ b/contrib/libs/llvm14/lib/Target/ARM/Utils/ya.make @@ -2,6 +2,8 @@ LIBRARY() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/lib/Target/ARM/ya.make b/contrib/libs/llvm14/lib/Target/ARM/ya.make index 39a7c2e9c4..b48e06a152 100644 --- a/contrib/libs/llvm14/lib/Target/ARM/ya.make +++ b/contrib/libs/llvm14/lib/Target/ARM/ya.make @@ -2,6 +2,8 @@ LIBRARY() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/lib/Target/BPF/AsmParser/ya.make b/contrib/libs/llvm14/lib/Target/BPF/AsmParser/ya.make index a02ef07bc6..65636826d5 100644 --- a/contrib/libs/llvm14/lib/Target/BPF/AsmParser/ya.make +++ b/contrib/libs/llvm14/lib/Target/BPF/AsmParser/ya.make @@ -2,6 +2,8 @@ LIBRARY() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/lib/Target/BPF/Disassembler/ya.make b/contrib/libs/llvm14/lib/Target/BPF/Disassembler/ya.make index 284989d3ea..7306e77e8b 100644 --- a/contrib/libs/llvm14/lib/Target/BPF/Disassembler/ya.make +++ b/contrib/libs/llvm14/lib/Target/BPF/Disassembler/ya.make @@ -2,6 +2,8 @@ LIBRARY() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/lib/Target/BPF/MCTargetDesc/ya.make b/contrib/libs/llvm14/lib/Target/BPF/MCTargetDesc/ya.make index d9964ea5f8..cd189ea70d 100644 --- a/contrib/libs/llvm14/lib/Target/BPF/MCTargetDesc/ya.make +++ b/contrib/libs/llvm14/lib/Target/BPF/MCTargetDesc/ya.make @@ -2,6 +2,8 @@ LIBRARY() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/lib/Target/BPF/TargetInfo/ya.make b/contrib/libs/llvm14/lib/Target/BPF/TargetInfo/ya.make index a0aeeff1a3..83678a4b99 100644 --- a/contrib/libs/llvm14/lib/Target/BPF/TargetInfo/ya.make +++ b/contrib/libs/llvm14/lib/Target/BPF/TargetInfo/ya.make @@ -2,6 +2,8 @@ LIBRARY() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/lib/Target/BPF/ya.make b/contrib/libs/llvm14/lib/Target/BPF/ya.make index 59fd10bf60..72ed0c56d0 100644 --- a/contrib/libs/llvm14/lib/Target/BPF/ya.make +++ b/contrib/libs/llvm14/lib/Target/BPF/ya.make @@ -2,6 +2,8 @@ LIBRARY() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/lib/Target/NVPTX/MCTargetDesc/ya.make b/contrib/libs/llvm14/lib/Target/NVPTX/MCTargetDesc/ya.make index 4fac6e4b95..71adacfd36 100644 --- a/contrib/libs/llvm14/lib/Target/NVPTX/MCTargetDesc/ya.make +++ b/contrib/libs/llvm14/lib/Target/NVPTX/MCTargetDesc/ya.make @@ -2,6 +2,8 @@ LIBRARY() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/lib/Target/NVPTX/TargetInfo/ya.make b/contrib/libs/llvm14/lib/Target/NVPTX/TargetInfo/ya.make index 912c02bbb8..0db3b7c935 100644 --- a/contrib/libs/llvm14/lib/Target/NVPTX/TargetInfo/ya.make +++ b/contrib/libs/llvm14/lib/Target/NVPTX/TargetInfo/ya.make @@ -2,6 +2,8 @@ LIBRARY() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/lib/Target/NVPTX/ya.make b/contrib/libs/llvm14/lib/Target/NVPTX/ya.make index b2ade7229f..0a7e509514 100644 --- a/contrib/libs/llvm14/lib/Target/NVPTX/ya.make +++ b/contrib/libs/llvm14/lib/Target/NVPTX/ya.make @@ -2,6 +2,8 @@ LIBRARY() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/lib/Target/PowerPC/AsmParser/ya.make b/contrib/libs/llvm14/lib/Target/PowerPC/AsmParser/ya.make index a24a8f8ec1..13882a7b74 100644 --- a/contrib/libs/llvm14/lib/Target/PowerPC/AsmParser/ya.make +++ b/contrib/libs/llvm14/lib/Target/PowerPC/AsmParser/ya.make @@ -2,6 +2,8 @@ LIBRARY() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/lib/Target/PowerPC/Disassembler/ya.make b/contrib/libs/llvm14/lib/Target/PowerPC/Disassembler/ya.make index 95c0eeeff7..e2a6fd59a6 100644 --- a/contrib/libs/llvm14/lib/Target/PowerPC/Disassembler/ya.make +++ b/contrib/libs/llvm14/lib/Target/PowerPC/Disassembler/ya.make @@ -2,6 +2,8 @@ LIBRARY() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/lib/Target/PowerPC/MCTargetDesc/ya.make b/contrib/libs/llvm14/lib/Target/PowerPC/MCTargetDesc/ya.make index 17d422af3d..a2c1a0b658 100644 --- a/contrib/libs/llvm14/lib/Target/PowerPC/MCTargetDesc/ya.make +++ b/contrib/libs/llvm14/lib/Target/PowerPC/MCTargetDesc/ya.make @@ -2,6 +2,8 @@ LIBRARY() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/lib/Target/PowerPC/TargetInfo/ya.make b/contrib/libs/llvm14/lib/Target/PowerPC/TargetInfo/ya.make index b035521fd7..8e7c500147 100644 --- a/contrib/libs/llvm14/lib/Target/PowerPC/TargetInfo/ya.make +++ b/contrib/libs/llvm14/lib/Target/PowerPC/TargetInfo/ya.make @@ -2,6 +2,8 @@ LIBRARY() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/lib/Target/PowerPC/ya.make b/contrib/libs/llvm14/lib/Target/PowerPC/ya.make index 347a8b1c3f..8ff7a52ea4 100644 --- a/contrib/libs/llvm14/lib/Target/PowerPC/ya.make +++ b/contrib/libs/llvm14/lib/Target/PowerPC/ya.make @@ -2,6 +2,8 @@ LIBRARY() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/lib/ToolDrivers/llvm-dlltool/ya.make b/contrib/libs/llvm14/lib/ToolDrivers/llvm-dlltool/ya.make index fa7ac3830d..47d9c0ef26 100644 --- a/contrib/libs/llvm14/lib/ToolDrivers/llvm-dlltool/ya.make +++ b/contrib/libs/llvm14/lib/ToolDrivers/llvm-dlltool/ya.make @@ -2,6 +2,8 @@ LIBRARY() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/lib/ToolDrivers/llvm-lib/ya.make b/contrib/libs/llvm14/lib/ToolDrivers/llvm-lib/ya.make index 7bf1853d0b..da2b07fa19 100644 --- a/contrib/libs/llvm14/lib/ToolDrivers/llvm-lib/ya.make +++ b/contrib/libs/llvm14/lib/ToolDrivers/llvm-lib/ya.make @@ -2,6 +2,8 @@ LIBRARY() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/lib/Transforms/Coroutines/ya.make b/contrib/libs/llvm14/lib/Transforms/Coroutines/ya.make index 1359dd6df0..807a94985a 100644 --- a/contrib/libs/llvm14/lib/Transforms/Coroutines/ya.make +++ b/contrib/libs/llvm14/lib/Transforms/Coroutines/ya.make @@ -2,6 +2,8 @@ LIBRARY() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/lib/WindowsManifest/ya.make b/contrib/libs/llvm14/lib/WindowsManifest/ya.make index 08c6ad8e33..9212603975 100644 --- a/contrib/libs/llvm14/lib/WindowsManifest/ya.make +++ b/contrib/libs/llvm14/lib/WindowsManifest/ya.make @@ -2,6 +2,8 @@ LIBRARY() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/lib/XRay/ya.make b/contrib/libs/llvm14/lib/XRay/ya.make index 451ce68256..5b3919203c 100644 --- a/contrib/libs/llvm14/lib/XRay/ya.make +++ b/contrib/libs/llvm14/lib/XRay/ya.make @@ -2,6 +2,8 @@ LIBRARY() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/tools/bugpoint/ya.make b/contrib/libs/llvm14/tools/bugpoint/ya.make index b6ccc9cc6c..2c92dc3f78 100644 --- a/contrib/libs/llvm14/tools/bugpoint/ya.make +++ b/contrib/libs/llvm14/tools/bugpoint/ya.make @@ -2,6 +2,8 @@ PROGRAM() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/tools/dsymutil/ya.make b/contrib/libs/llvm14/tools/dsymutil/ya.make index e5b28b06d4..1fd7a3794b 100644 --- a/contrib/libs/llvm14/tools/dsymutil/ya.make +++ b/contrib/libs/llvm14/tools/dsymutil/ya.make @@ -2,6 +2,8 @@ PROGRAM() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/tools/gold/ya.make b/contrib/libs/llvm14/tools/gold/ya.make index 78e06212fc..a1df0fa646 100644 --- a/contrib/libs/llvm14/tools/gold/ya.make +++ b/contrib/libs/llvm14/tools/gold/ya.make @@ -2,6 +2,8 @@ DLL(LLVMgold PREFIX "") +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/tools/llc/ya.make b/contrib/libs/llvm14/tools/llc/ya.make index e8afa15539..57c55653ca 100644 --- a/contrib/libs/llvm14/tools/llc/ya.make +++ b/contrib/libs/llvm14/tools/llc/ya.make @@ -2,6 +2,8 @@ PROGRAM() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/tools/lli/ChildTarget/ya.make b/contrib/libs/llvm14/tools/lli/ChildTarget/ya.make index 3418e04553..13086fcf46 100644 --- a/contrib/libs/llvm14/tools/lli/ChildTarget/ya.make +++ b/contrib/libs/llvm14/tools/lli/ChildTarget/ya.make @@ -2,6 +2,8 @@ PROGRAM(lli-child-target) +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/tools/lli/ya.make b/contrib/libs/llvm14/tools/lli/ya.make index dc56e17046..8b57788f71 100644 --- a/contrib/libs/llvm14/tools/lli/ya.make +++ b/contrib/libs/llvm14/tools/lli/ya.make @@ -2,6 +2,8 @@ PROGRAM() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/tools/llvm-ar/ya.make b/contrib/libs/llvm14/tools/llvm-ar/ya.make index 1ce94bcb6f..28c9072a27 100644 --- a/contrib/libs/llvm14/tools/llvm-ar/ya.make +++ b/contrib/libs/llvm14/tools/llvm-ar/ya.make @@ -2,6 +2,8 @@ PROGRAM() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/tools/llvm-as/ya.make b/contrib/libs/llvm14/tools/llvm-as/ya.make index 48d5e6e683..28577ac80b 100644 --- a/contrib/libs/llvm14/tools/llvm-as/ya.make +++ b/contrib/libs/llvm14/tools/llvm-as/ya.make @@ -2,6 +2,8 @@ PROGRAM() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/tools/llvm-bcanalyzer/ya.make b/contrib/libs/llvm14/tools/llvm-bcanalyzer/ya.make index 3d57161d8d..0734332b00 100644 --- a/contrib/libs/llvm14/tools/llvm-bcanalyzer/ya.make +++ b/contrib/libs/llvm14/tools/llvm-bcanalyzer/ya.make @@ -2,6 +2,8 @@ PROGRAM() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/tools/llvm-cat/ya.make b/contrib/libs/llvm14/tools/llvm-cat/ya.make index 2101c805ee..9e2b10ff9d 100644 --- a/contrib/libs/llvm14/tools/llvm-cat/ya.make +++ b/contrib/libs/llvm14/tools/llvm-cat/ya.make @@ -2,6 +2,8 @@ PROGRAM() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/tools/llvm-cfi-verify/lib/ya.make b/contrib/libs/llvm14/tools/llvm-cfi-verify/lib/ya.make index d3b196a413..ae93f02c0d 100644 --- a/contrib/libs/llvm14/tools/llvm-cfi-verify/lib/ya.make +++ b/contrib/libs/llvm14/tools/llvm-cfi-verify/lib/ya.make @@ -2,6 +2,8 @@ LIBRARY() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/tools/llvm-cfi-verify/ya.make b/contrib/libs/llvm14/tools/llvm-cfi-verify/ya.make index cef9ba064d..b42fbc18e0 100644 --- a/contrib/libs/llvm14/tools/llvm-cfi-verify/ya.make +++ b/contrib/libs/llvm14/tools/llvm-cfi-verify/ya.make @@ -2,6 +2,8 @@ PROGRAM() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/tools/llvm-config/ya.make b/contrib/libs/llvm14/tools/llvm-config/ya.make index 81523992ae..1162e8d0d2 100644 --- a/contrib/libs/llvm14/tools/llvm-config/ya.make +++ b/contrib/libs/llvm14/tools/llvm-config/ya.make @@ -2,6 +2,8 @@ PROGRAM() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/tools/llvm-cov/ya.make b/contrib/libs/llvm14/tools/llvm-cov/ya.make index 9bc4c3debe..bbdd860fd3 100644 --- a/contrib/libs/llvm14/tools/llvm-cov/ya.make +++ b/contrib/libs/llvm14/tools/llvm-cov/ya.make @@ -2,6 +2,8 @@ PROGRAM() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/tools/llvm-cvtres/ya.make b/contrib/libs/llvm14/tools/llvm-cvtres/ya.make index 706e0a9876..c0a91d3a0a 100644 --- a/contrib/libs/llvm14/tools/llvm-cvtres/ya.make +++ b/contrib/libs/llvm14/tools/llvm-cvtres/ya.make @@ -2,6 +2,8 @@ PROGRAM() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/tools/llvm-cxxdump/ya.make b/contrib/libs/llvm14/tools/llvm-cxxdump/ya.make index 00af0e4b02..cd69d47245 100644 --- a/contrib/libs/llvm14/tools/llvm-cxxdump/ya.make +++ b/contrib/libs/llvm14/tools/llvm-cxxdump/ya.make @@ -2,6 +2,8 @@ PROGRAM() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/tools/llvm-cxxfilt/ya.make b/contrib/libs/llvm14/tools/llvm-cxxfilt/ya.make index 77f0e4f186..5d36a25c2a 100644 --- a/contrib/libs/llvm14/tools/llvm-cxxfilt/ya.make +++ b/contrib/libs/llvm14/tools/llvm-cxxfilt/ya.make @@ -2,6 +2,8 @@ PROGRAM() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/tools/llvm-cxxmap/ya.make b/contrib/libs/llvm14/tools/llvm-cxxmap/ya.make index 4c05475986..1761466b26 100644 --- a/contrib/libs/llvm14/tools/llvm-cxxmap/ya.make +++ b/contrib/libs/llvm14/tools/llvm-cxxmap/ya.make @@ -2,6 +2,8 @@ PROGRAM() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/tools/llvm-diff/lib/ya.make b/contrib/libs/llvm14/tools/llvm-diff/lib/ya.make index 4dd6acd33b..36fd72939d 100644 --- a/contrib/libs/llvm14/tools/llvm-diff/lib/ya.make +++ b/contrib/libs/llvm14/tools/llvm-diff/lib/ya.make @@ -2,6 +2,8 @@ LIBRARY() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/tools/llvm-diff/ya.make b/contrib/libs/llvm14/tools/llvm-diff/ya.make index 86189b0ff3..f0f3f8bd8a 100644 --- a/contrib/libs/llvm14/tools/llvm-diff/ya.make +++ b/contrib/libs/llvm14/tools/llvm-diff/ya.make @@ -2,6 +2,8 @@ PROGRAM() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/tools/llvm-dis/ya.make b/contrib/libs/llvm14/tools/llvm-dis/ya.make index 46629d7e61..33d964c4a6 100644 --- a/contrib/libs/llvm14/tools/llvm-dis/ya.make +++ b/contrib/libs/llvm14/tools/llvm-dis/ya.make @@ -2,6 +2,8 @@ PROGRAM() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/tools/llvm-dwarfdump/ya.make b/contrib/libs/llvm14/tools/llvm-dwarfdump/ya.make index 924e23a023..680a091c45 100644 --- a/contrib/libs/llvm14/tools/llvm-dwarfdump/ya.make +++ b/contrib/libs/llvm14/tools/llvm-dwarfdump/ya.make @@ -2,6 +2,8 @@ PROGRAM() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/tools/llvm-dwp/ya.make b/contrib/libs/llvm14/tools/llvm-dwp/ya.make index 68226e677b..6ffcf3cec9 100644 --- a/contrib/libs/llvm14/tools/llvm-dwp/ya.make +++ b/contrib/libs/llvm14/tools/llvm-dwp/ya.make @@ -2,6 +2,8 @@ PROGRAM() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/tools/llvm-exegesis/lib/AArch64/ya.make b/contrib/libs/llvm14/tools/llvm-exegesis/lib/AArch64/ya.make index cc0d66211f..bbe60d6281 100644 --- a/contrib/libs/llvm14/tools/llvm-exegesis/lib/AArch64/ya.make +++ b/contrib/libs/llvm14/tools/llvm-exegesis/lib/AArch64/ya.make @@ -2,6 +2,8 @@ LIBRARY() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/tools/llvm-exegesis/lib/PowerPC/ya.make b/contrib/libs/llvm14/tools/llvm-exegesis/lib/PowerPC/ya.make index 953fd9824c..1928fc0f8d 100644 --- a/contrib/libs/llvm14/tools/llvm-exegesis/lib/PowerPC/ya.make +++ b/contrib/libs/llvm14/tools/llvm-exegesis/lib/PowerPC/ya.make @@ -2,6 +2,8 @@ LIBRARY() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/tools/llvm-exegesis/lib/X86/ya.make b/contrib/libs/llvm14/tools/llvm-exegesis/lib/X86/ya.make index 88c500fa8f..568158046d 100644 --- a/contrib/libs/llvm14/tools/llvm-exegesis/lib/X86/ya.make +++ b/contrib/libs/llvm14/tools/llvm-exegesis/lib/X86/ya.make @@ -2,6 +2,8 @@ LIBRARY() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/tools/llvm-exegesis/lib/ya.make b/contrib/libs/llvm14/tools/llvm-exegesis/lib/ya.make index 6b535feb96..61083d3dde 100644 --- a/contrib/libs/llvm14/tools/llvm-exegesis/lib/ya.make +++ b/contrib/libs/llvm14/tools/llvm-exegesis/lib/ya.make @@ -2,6 +2,8 @@ LIBRARY() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/tools/llvm-exegesis/ya.make b/contrib/libs/llvm14/tools/llvm-exegesis/ya.make index 6e9251c81f..afdb24ff6a 100644 --- a/contrib/libs/llvm14/tools/llvm-exegesis/ya.make +++ b/contrib/libs/llvm14/tools/llvm-exegesis/ya.make @@ -2,6 +2,8 @@ PROGRAM() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/tools/llvm-extract/ya.make b/contrib/libs/llvm14/tools/llvm-extract/ya.make index 619d1cce51..20eac43f24 100644 --- a/contrib/libs/llvm14/tools/llvm-extract/ya.make +++ b/contrib/libs/llvm14/tools/llvm-extract/ya.make @@ -2,6 +2,8 @@ PROGRAM() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/tools/llvm-gsymutil/ya.make b/contrib/libs/llvm14/tools/llvm-gsymutil/ya.make index ea67d249d4..ed0d20db16 100644 --- a/contrib/libs/llvm14/tools/llvm-gsymutil/ya.make +++ b/contrib/libs/llvm14/tools/llvm-gsymutil/ya.make @@ -2,6 +2,8 @@ PROGRAM() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/tools/llvm-ifs/ya.make b/contrib/libs/llvm14/tools/llvm-ifs/ya.make index a1b79aab42..1fb1e6e5c6 100644 --- a/contrib/libs/llvm14/tools/llvm-ifs/ya.make +++ b/contrib/libs/llvm14/tools/llvm-ifs/ya.make @@ -2,6 +2,8 @@ PROGRAM() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/tools/llvm-jitlink/llvm-jitlink-executor/ya.make b/contrib/libs/llvm14/tools/llvm-jitlink/llvm-jitlink-executor/ya.make index dd0107ba99..07bba6dc4a 100644 --- a/contrib/libs/llvm14/tools/llvm-jitlink/llvm-jitlink-executor/ya.make +++ b/contrib/libs/llvm14/tools/llvm-jitlink/llvm-jitlink-executor/ya.make @@ -2,6 +2,8 @@ PROGRAM() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/tools/llvm-jitlink/ya.make b/contrib/libs/llvm14/tools/llvm-jitlink/ya.make index 132e84dfd0..1d9421c566 100644 --- a/contrib/libs/llvm14/tools/llvm-jitlink/ya.make +++ b/contrib/libs/llvm14/tools/llvm-jitlink/ya.make @@ -2,6 +2,8 @@ PROGRAM() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/tools/llvm-libtool-darwin/ya.make b/contrib/libs/llvm14/tools/llvm-libtool-darwin/ya.make index 4cffcb76bc..99ad2efb1d 100644 --- a/contrib/libs/llvm14/tools/llvm-libtool-darwin/ya.make +++ b/contrib/libs/llvm14/tools/llvm-libtool-darwin/ya.make @@ -2,6 +2,8 @@ PROGRAM() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/tools/llvm-link/ya.make b/contrib/libs/llvm14/tools/llvm-link/ya.make index de434f851c..1d7b183a51 100644 --- a/contrib/libs/llvm14/tools/llvm-link/ya.make +++ b/contrib/libs/llvm14/tools/llvm-link/ya.make @@ -2,6 +2,8 @@ PROGRAM() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/tools/llvm-lipo/ya.make b/contrib/libs/llvm14/tools/llvm-lipo/ya.make index 97fd93bc16..3d987c814d 100644 --- a/contrib/libs/llvm14/tools/llvm-lipo/ya.make +++ b/contrib/libs/llvm14/tools/llvm-lipo/ya.make @@ -2,6 +2,8 @@ PROGRAM() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/tools/llvm-lto/ya.make b/contrib/libs/llvm14/tools/llvm-lto/ya.make index b9d03c9218..1b3aac66f9 100644 --- a/contrib/libs/llvm14/tools/llvm-lto/ya.make +++ b/contrib/libs/llvm14/tools/llvm-lto/ya.make @@ -2,6 +2,8 @@ PROGRAM() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/tools/llvm-lto2/ya.make b/contrib/libs/llvm14/tools/llvm-lto2/ya.make index 1869e4a344..51450ac778 100644 --- a/contrib/libs/llvm14/tools/llvm-lto2/ya.make +++ b/contrib/libs/llvm14/tools/llvm-lto2/ya.make @@ -2,6 +2,8 @@ PROGRAM() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/tools/llvm-mc/ya.make b/contrib/libs/llvm14/tools/llvm-mc/ya.make index 0e71f18028..515e55259f 100644 --- a/contrib/libs/llvm14/tools/llvm-mc/ya.make +++ b/contrib/libs/llvm14/tools/llvm-mc/ya.make @@ -2,6 +2,8 @@ PROGRAM() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/tools/llvm-mca/ya.make b/contrib/libs/llvm14/tools/llvm-mca/ya.make index 0346c0a5cc..df21987ed5 100644 --- a/contrib/libs/llvm14/tools/llvm-mca/ya.make +++ b/contrib/libs/llvm14/tools/llvm-mca/ya.make @@ -2,6 +2,8 @@ PROGRAM() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/tools/llvm-ml/ya.make b/contrib/libs/llvm14/tools/llvm-ml/ya.make index 4c2e95de0d..718ed5f505 100644 --- a/contrib/libs/llvm14/tools/llvm-ml/ya.make +++ b/contrib/libs/llvm14/tools/llvm-ml/ya.make @@ -2,6 +2,8 @@ PROGRAM() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/tools/llvm-modextract/ya.make b/contrib/libs/llvm14/tools/llvm-modextract/ya.make index 3e98583ba1..3ec9981389 100644 --- a/contrib/libs/llvm14/tools/llvm-modextract/ya.make +++ b/contrib/libs/llvm14/tools/llvm-modextract/ya.make @@ -2,6 +2,8 @@ PROGRAM() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/tools/llvm-mt/ya.make b/contrib/libs/llvm14/tools/llvm-mt/ya.make index c10b978f5b..1b7f48ed79 100644 --- a/contrib/libs/llvm14/tools/llvm-mt/ya.make +++ b/contrib/libs/llvm14/tools/llvm-mt/ya.make @@ -2,6 +2,8 @@ PROGRAM() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/tools/llvm-nm/ya.make b/contrib/libs/llvm14/tools/llvm-nm/ya.make index bb57fc4cf6..4ab4c7c0ed 100644 --- a/contrib/libs/llvm14/tools/llvm-nm/ya.make +++ b/contrib/libs/llvm14/tools/llvm-nm/ya.make @@ -2,6 +2,8 @@ PROGRAM() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/tools/llvm-objcopy/ya.make b/contrib/libs/llvm14/tools/llvm-objcopy/ya.make index 0be7feaf39..92e1cde2a2 100644 --- a/contrib/libs/llvm14/tools/llvm-objcopy/ya.make +++ b/contrib/libs/llvm14/tools/llvm-objcopy/ya.make @@ -2,6 +2,8 @@ PROGRAM() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/tools/llvm-objdump/ya.make b/contrib/libs/llvm14/tools/llvm-objdump/ya.make index d4ef19c193..b8da19c89b 100644 --- a/contrib/libs/llvm14/tools/llvm-objdump/ya.make +++ b/contrib/libs/llvm14/tools/llvm-objdump/ya.make @@ -2,6 +2,8 @@ PROGRAM() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/tools/llvm-opt-report/ya.make b/contrib/libs/llvm14/tools/llvm-opt-report/ya.make index c747b91c42..b70342853e 100644 --- a/contrib/libs/llvm14/tools/llvm-opt-report/ya.make +++ b/contrib/libs/llvm14/tools/llvm-opt-report/ya.make @@ -2,6 +2,8 @@ PROGRAM() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/tools/llvm-pdbutil/ya.make b/contrib/libs/llvm14/tools/llvm-pdbutil/ya.make index 2d8b043b40..6e10940bfb 100644 --- a/contrib/libs/llvm14/tools/llvm-pdbutil/ya.make +++ b/contrib/libs/llvm14/tools/llvm-pdbutil/ya.make @@ -2,6 +2,8 @@ PROGRAM() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/tools/llvm-profdata/ya.make b/contrib/libs/llvm14/tools/llvm-profdata/ya.make index 628b163b23..64d113f31b 100644 --- a/contrib/libs/llvm14/tools/llvm-profdata/ya.make +++ b/contrib/libs/llvm14/tools/llvm-profdata/ya.make @@ -2,6 +2,8 @@ PROGRAM() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/tools/llvm-profgen/ya.make b/contrib/libs/llvm14/tools/llvm-profgen/ya.make index f4518c571f..ef8a72c5e3 100644 --- a/contrib/libs/llvm14/tools/llvm-profgen/ya.make +++ b/contrib/libs/llvm14/tools/llvm-profgen/ya.make @@ -2,6 +2,8 @@ PROGRAM() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/tools/llvm-rc/ya.make b/contrib/libs/llvm14/tools/llvm-rc/ya.make index 0734abe2dc..b757795675 100644 --- a/contrib/libs/llvm14/tools/llvm-rc/ya.make +++ b/contrib/libs/llvm14/tools/llvm-rc/ya.make @@ -2,6 +2,8 @@ PROGRAM() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/tools/llvm-readobj/ya.make b/contrib/libs/llvm14/tools/llvm-readobj/ya.make index b09c1b9dae..59af4fb196 100644 --- a/contrib/libs/llvm14/tools/llvm-readobj/ya.make +++ b/contrib/libs/llvm14/tools/llvm-readobj/ya.make @@ -2,6 +2,8 @@ PROGRAM() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/tools/llvm-reduce/ya.make b/contrib/libs/llvm14/tools/llvm-reduce/ya.make index 4dfeb1339d..f331598c08 100644 --- a/contrib/libs/llvm14/tools/llvm-reduce/ya.make +++ b/contrib/libs/llvm14/tools/llvm-reduce/ya.make @@ -2,6 +2,8 @@ PROGRAM() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/tools/llvm-rtdyld/ya.make b/contrib/libs/llvm14/tools/llvm-rtdyld/ya.make index 06ab2d2c88..f163718ce8 100644 --- a/contrib/libs/llvm14/tools/llvm-rtdyld/ya.make +++ b/contrib/libs/llvm14/tools/llvm-rtdyld/ya.make @@ -2,6 +2,8 @@ PROGRAM() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/tools/llvm-size/ya.make b/contrib/libs/llvm14/tools/llvm-size/ya.make index 88731a7476..60e3ed8e61 100644 --- a/contrib/libs/llvm14/tools/llvm-size/ya.make +++ b/contrib/libs/llvm14/tools/llvm-size/ya.make @@ -2,6 +2,8 @@ PROGRAM() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/tools/llvm-split/ya.make b/contrib/libs/llvm14/tools/llvm-split/ya.make index 7099375aa9..2c028e36ba 100644 --- a/contrib/libs/llvm14/tools/llvm-split/ya.make +++ b/contrib/libs/llvm14/tools/llvm-split/ya.make @@ -2,6 +2,8 @@ PROGRAM() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/tools/llvm-stress/ya.make b/contrib/libs/llvm14/tools/llvm-stress/ya.make index a6a10a3323..e8fdd4971a 100644 --- a/contrib/libs/llvm14/tools/llvm-stress/ya.make +++ b/contrib/libs/llvm14/tools/llvm-stress/ya.make @@ -2,6 +2,8 @@ PROGRAM() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/tools/llvm-strings/ya.make b/contrib/libs/llvm14/tools/llvm-strings/ya.make index 85ae64afaf..5dc8f557f8 100644 --- a/contrib/libs/llvm14/tools/llvm-strings/ya.make +++ b/contrib/libs/llvm14/tools/llvm-strings/ya.make @@ -2,6 +2,8 @@ PROGRAM() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/tools/llvm-symbolizer/ya.make b/contrib/libs/llvm14/tools/llvm-symbolizer/ya.make index ca777b10f3..3a7acbf26b 100644 --- a/contrib/libs/llvm14/tools/llvm-symbolizer/ya.make +++ b/contrib/libs/llvm14/tools/llvm-symbolizer/ya.make @@ -2,6 +2,8 @@ PROGRAM() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/tools/llvm-undname/ya.make b/contrib/libs/llvm14/tools/llvm-undname/ya.make index 439ce5e551..593682721b 100644 --- a/contrib/libs/llvm14/tools/llvm-undname/ya.make +++ b/contrib/libs/llvm14/tools/llvm-undname/ya.make @@ -2,6 +2,8 @@ PROGRAM() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/tools/llvm-xray/ya.make b/contrib/libs/llvm14/tools/llvm-xray/ya.make index 06e0b0bb2d..d905463b20 100644 --- a/contrib/libs/llvm14/tools/llvm-xray/ya.make +++ b/contrib/libs/llvm14/tools/llvm-xray/ya.make @@ -2,6 +2,8 @@ PROGRAM() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/tools/lto/ya.make b/contrib/libs/llvm14/tools/lto/ya.make index e1bc27df39..e149d3b2ce 100644 --- a/contrib/libs/llvm14/tools/lto/ya.make +++ b/contrib/libs/llvm14/tools/lto/ya.make @@ -2,6 +2,8 @@ LIBRARY() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/tools/obj2yaml/ya.make b/contrib/libs/llvm14/tools/obj2yaml/ya.make index b3f0522cce..9b4752d911 100644 --- a/contrib/libs/llvm14/tools/obj2yaml/ya.make +++ b/contrib/libs/llvm14/tools/obj2yaml/ya.make @@ -2,6 +2,8 @@ PROGRAM() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/tools/opt/ya.make b/contrib/libs/llvm14/tools/opt/ya.make index 4faf896fd9..b4b0d02440 100644 --- a/contrib/libs/llvm14/tools/opt/ya.make +++ b/contrib/libs/llvm14/tools/opt/ya.make @@ -2,6 +2,8 @@ PROGRAM() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/tools/polly/lib/External/isl/ya.make b/contrib/libs/llvm14/tools/polly/lib/External/isl/ya.make index bc6e9dd500..29d7e550fd 100644 --- a/contrib/libs/llvm14/tools/polly/lib/External/isl/ya.make +++ b/contrib/libs/llvm14/tools/polly/lib/External/isl/ya.make @@ -2,6 +2,8 @@ LIBRARY() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE( diff --git a/contrib/libs/llvm14/tools/polly/lib/External/ppcg/ya.make b/contrib/libs/llvm14/tools/polly/lib/External/ppcg/ya.make index 795b5afca3..9afe1882d9 100644 --- a/contrib/libs/llvm14/tools/polly/lib/External/ppcg/ya.make +++ b/contrib/libs/llvm14/tools/polly/lib/External/ppcg/ya.make @@ -2,6 +2,8 @@ LIBRARY() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(MIT) diff --git a/contrib/libs/llvm14/tools/polly/lib/ya.make b/contrib/libs/llvm14/tools/polly/lib/ya.make index 28503484fa..959f2a0c9b 100644 --- a/contrib/libs/llvm14/tools/polly/lib/ya.make +++ b/contrib/libs/llvm14/tools/polly/lib/ya.make @@ -2,6 +2,8 @@ LIBRARY() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE( diff --git a/contrib/libs/llvm14/tools/remarks-shlib/ya.make b/contrib/libs/llvm14/tools/remarks-shlib/ya.make index 9c08142a76..966960eced 100644 --- a/contrib/libs/llvm14/tools/remarks-shlib/ya.make +++ b/contrib/libs/llvm14/tools/remarks-shlib/ya.make @@ -2,6 +2,8 @@ LIBRARY() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/tools/sancov/ya.make b/contrib/libs/llvm14/tools/sancov/ya.make index 8f2b4ecca3..cc1aa1b265 100644 --- a/contrib/libs/llvm14/tools/sancov/ya.make +++ b/contrib/libs/llvm14/tools/sancov/ya.make @@ -2,6 +2,8 @@ PROGRAM() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/tools/sanstats/ya.make b/contrib/libs/llvm14/tools/sanstats/ya.make index b15632ef0c..f1eccea96c 100644 --- a/contrib/libs/llvm14/tools/sanstats/ya.make +++ b/contrib/libs/llvm14/tools/sanstats/ya.make @@ -2,6 +2,8 @@ PROGRAM() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/tools/split-file/ya.make b/contrib/libs/llvm14/tools/split-file/ya.make index b3ed0665e8..cc28284df4 100644 --- a/contrib/libs/llvm14/tools/split-file/ya.make +++ b/contrib/libs/llvm14/tools/split-file/ya.make @@ -2,6 +2,8 @@ PROGRAM() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/tools/verify-uselistorder/ya.make b/contrib/libs/llvm14/tools/verify-uselistorder/ya.make index 0cfb5fb2ad..93f4fb4b2b 100644 --- a/contrib/libs/llvm14/tools/verify-uselistorder/ya.make +++ b/contrib/libs/llvm14/tools/verify-uselistorder/ya.make @@ -2,6 +2,8 @@ PROGRAM() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/libs/llvm14/tools/yaml2obj/ya.make b/contrib/libs/llvm14/tools/yaml2obj/ya.make index 9834e7075e..feb01e29ee 100644 --- a/contrib/libs/llvm14/tools/yaml2obj/ya.make +++ b/contrib/libs/llvm14/tools/yaml2obj/ya.make @@ -2,6 +2,8 @@ PROGRAM() +SUBSCRIBER(g:cpp-contrib) + VERSION(14.0.6) LICENSE(Apache-2.0 WITH LLVM-exception) diff --git a/contrib/restricted/googletest/googletest/include/gtest/gtest-printers.h b/contrib/restricted/googletest/googletest/include/gtest/gtest-printers.h index 6fc97d9143..e3df46b74f 100644 --- a/contrib/restricted/googletest/googletest/include/gtest/gtest-printers.h +++ b/contrib/restricted/googletest/googletest/include/gtest/gtest-printers.h @@ -829,6 +829,11 @@ void PrintTo(const ::std::pair<T1, T2>& value, ::std::ostream* os) { *os << ')'; } +template <typename C, typename D> +void PrintTo(const ::std::chrono::time_point<C, D>& value, ::std::ostream* os) { + *os << "TimeStamp(" << static_cast<int64_t>(value.time_since_epoch().count()) << ")"; +} + // Implements printing a non-reference type T by letting the compiler // pick the right overload of PrintTo() for T. template <typename T> diff --git a/library/cpp/http/simple/http_client.h b/library/cpp/http/simple/http_client.h index 224be58a24..86a8cb4e99 100644 --- a/library/cpp/http/simple/http_client.h +++ b/library/cpp/http/simple/http_client.h @@ -267,11 +267,13 @@ TKeepAliveHttpClient::THttpCode TKeepAliveHttpClient::DoRequestReliable(const T& const bool haveNewConnection = CreateNewConnectionIfNeeded(); const bool couldRetry = !haveNewConnection && i == 0; // Actually old connection could be already closed by server, // so we should try one more time in this case. - try { - cancellation.Future().Subscribe([&](auto&) { - Connection->Shutdown(); - }); + TManualEvent cancellationEndEvent; + cancellation.Future().Subscribe([&](auto&) { + Connection->Shutdown(); + cancellationEndEvent.Signal(); + }); + try { Connection->Write(raw); THttpCode code = ReadAndTransferHttp(*Connection->GetHttpInput(), output, outHeaders); @@ -280,20 +282,29 @@ TKeepAliveHttpClient::THttpCode TKeepAliveHttpClient::DoRequestReliable(const T& } return code; } catch (const TSystemError& e) { + if (cancellation.IsCancellationRequested()) { + cancellationEndEvent.WaitI(); + cancellation.ThrowIfCancellationRequested(); + } Connection.Reset(); - cancellation.ThrowIfCancellationRequested(); if (!couldRetry || e.Status() != EPIPE) { throw; } } catch (const THttpReadException&) { // Actually old connection is already closed by server + if (cancellation.IsCancellationRequested()) { + cancellationEndEvent.WaitI(); + cancellation.ThrowIfCancellationRequested(); + } Connection.Reset(); - cancellation.ThrowIfCancellationRequested(); if (!couldRetry) { throw; } } catch (const std::exception&) { + if (cancellation.IsCancellationRequested()) { + cancellationEndEvent.WaitI(); + cancellation.ThrowIfCancellationRequested(); + } Connection.Reset(); - cancellation.ThrowIfCancellationRequested(); throw; } } diff --git a/library/cpp/neh/http_common.cpp b/library/cpp/neh/http_common.cpp index 7ae466c31a..23ed6a2889 100644 --- a/library/cpp/neh/http_common.cpp +++ b/library/cpp/neh/http_common.cpp @@ -231,5 +231,32 @@ namespace NNeh { return NeedGetRequestFor(scheme) || NeedPostRequestFor(scheme); } } + + HttpCodes GetHttpCode(const IRequest::TResponseError& error) { + switch (error) { + case IRequest::TResponseError::BadRequest: + return HttpCodes::HTTP_BAD_REQUEST; + case IRequest::TResponseError::Forbidden: + return HttpCodes::HTTP_FORBIDDEN; + case IRequest::TResponseError::NotExistService: + return HttpCodes::HTTP_NOT_FOUND; + case IRequest::TResponseError::TooManyRequests: + return HttpCodes::HTTP_TOO_MANY_REQUESTS; + case IRequest::TResponseError::InternalError: + return HttpCodes::HTTP_INTERNAL_SERVER_ERROR; + case IRequest::TResponseError::NotImplemented: + return HttpCodes::HTTP_NOT_IMPLEMENTED; + case IRequest::TResponseError::BadGateway: + return HttpCodes::HTTP_BAD_GATEWAY; + case IRequest::TResponseError::ServiceUnavailable: + return HttpCodes::HTTP_SERVICE_UNAVAILABLE; + case IRequest::TResponseError::BandwidthLimitExceeded: + return HttpCodes::HTTP_BANDWIDTH_LIMIT_EXCEEDED; + case IRequest::TResponseError::MaxResponseError: + ythrow yexception() << TStringBuf("unknow type of error"); + + Y_UNREACHABLE(); + } + } } diff --git a/library/cpp/neh/http_common.h b/library/cpp/neh/http_common.h index 69659ba907..acefe80c37 100644 --- a/library/cpp/neh/http_common.h +++ b/library/cpp/neh/http_common.h @@ -7,6 +7,7 @@ #include <util/stream/mem.h> #include <util/stream/output.h> #include <library/cpp/deprecated/atomic/atomic.h> +#include <library/cpp/http/misc/httpcodes.h> #include "location.h" #include "neh.h" @@ -298,4 +299,6 @@ namespace NNeh { bool IsHttpScheme(TStringBuf scheme); } + + HttpCodes GetHttpCode(const IRequest::TResponseError&); } diff --git a/library/cpp/neh/https.cpp b/library/cpp/neh/https.cpp index f1a2ec9e7e..ece7d3cf2d 100644 --- a/library/cpp/neh/https.cpp +++ b/library/cpp/neh/https.cpp @@ -1394,37 +1394,7 @@ namespace NNeh { return; } - switch (*error) { - case IRequest::TResponseError::BadRequest: - os << HttpCodeStrEx(HttpCodes::HTTP_BAD_REQUEST); - break; - case IRequest::TResponseError::Forbidden: - os << HttpCodeStrEx(HttpCodes::HTTP_FORBIDDEN); - break; - case IRequest::TResponseError::NotExistService: - os << HttpCodeStrEx(HttpCodes::HTTP_NOT_FOUND); - break; - case IRequest::TResponseError::TooManyRequests: - os << HttpCodeStrEx(HttpCodes::HTTP_TOO_MANY_REQUESTS); - break; - case IRequest::TResponseError::InternalError: - os << HttpCodeStrEx(HttpCodes::HTTP_INTERNAL_SERVER_ERROR); - break; - case IRequest::TResponseError::NotImplemented: - os << HttpCodeStrEx(HttpCodes::HTTP_NOT_IMPLEMENTED); - break; - case IRequest::TResponseError::BadGateway: - os << HttpCodeStrEx(HttpCodes::HTTP_BAD_GATEWAY); - break; - case IRequest::TResponseError::ServiceUnavailable: - os << HttpCodeStrEx(HttpCodes::HTTP_SERVICE_UNAVAILABLE); - break; - case IRequest::TResponseError::BandwidthLimitExceeded: - os << HttpCodeStrEx(HttpCodes::HTTP_BANDWIDTH_LIMIT_EXCEEDED); - break; - case IRequest::TResponseError::MaxResponseError: - ythrow yexception() << TStringBuf("unknow type of error"); - } + os << HttpCodeStrEx(GetHttpCode(*error)); } public: diff --git a/library/cpp/tdigest/tdigest.cpp b/library/cpp/tdigest/tdigest.cpp index 145cef78e1..3d3772a9de 100644 --- a/library/cpp/tdigest/tdigest.cpp +++ b/library/cpp/tdigest/tdigest.cpp @@ -3,45 +3,52 @@ #include <library/cpp/tdigest/tdigest.pb.h> #include <cmath> +#include <util/generic/yexception.h> // TODO: rewrite to https://github.com/tdunning/t-digest/blob/master/src/main/java/com/tdunning/math/stats/MergingDigest.java -TDigest::TDigest(double delta, double k) +TDigest::TDigest(double delta, double k, bool supportsNaN) : N(0) , Delta(delta) , K(k) + , SupportsNaN(supportsNaN) { } -TDigest::TDigest(double delta, double k, double firstValue) - : TDigest(delta, k) +TDigest::TDigest(double delta, double k, double firstValue, bool supportsNaN) + : TDigest(delta, k, supportsNaN) { AddValue(firstValue); } -TDigest::TDigest(TStringBuf serializedDigest) +TDigest::TDigest(TStringBuf serializedDigest, bool supportsNaN) : N(0) + , SupportsNaN(supportsNaN) { NTDigest::TDigest digest; Y_ABORT_UNLESS(digest.ParseFromArray(serializedDigest.data(), serializedDigest.size())); Delta = digest.delta(); K = digest.k(); + HasNaN = SupportsNaN && digest.nans(); for (int i = 0; i < digest.centroids_size(); ++i) { const NTDigest::TDigest::TCentroid& centroid = digest.centroids(i); Update(centroid.mean(), centroid.weight()); } } -TDigest::TDigest(const TDigest* digest1, const TDigest* digest2) +TDigest::TDigest(const TDigest* digest1, const TDigest* digest2, bool supportsNaN) : N(0) , Delta(std::min(digest1->Delta, digest2->Delta)) , K(std::max(digest1->K, digest2->K)) + , SupportsNaN(supportsNaN) + , HasNaN(supportsNaN && (digest1->HasNaN || digest2->HasNaN)) { Add(*digest1); Add(*digest2); } void TDigest::Add(const TDigest& otherDigest) { + Y_ENSURE(SupportsNaN == otherDigest.SupportsNaN); for (auto& it : otherDigest.Centroids) Update(it.Mean, it.Count); for (auto& it : otherDigest.Unmerged) @@ -49,7 +56,8 @@ void TDigest::Add(const TDigest& otherDigest) { } TDigest TDigest::operator+(const TDigest& other) { - TDigest T(Delta, K); + Y_ENSURE(SupportsNaN == other.SupportsNaN); + TDigest T(Delta, K, SupportsNaN); T.Add(*this); T.Add(other); return T; @@ -92,6 +100,12 @@ void TDigest::MergeCentroid(TVector<TCentroid>& merged, double& sum, const TCent } void TDigest::Update(double x, double w) { + if (SupportsNaN) { + if (std::isnan(x)) { + HasNaN = true; + return; + } + } AddCentroid(TCentroid(x, w)); if (Unmerged.size() >= K / Delta) { Compress(); @@ -136,8 +150,17 @@ void TDigest::AddValue(double value) { double TDigest::GetPercentile(double percentile) { Compress(); - if (Centroids.empty()) + if (Centroids.empty()) { + if (HasNaN) { + return std::numeric_limits<double>::quiet_NaN(); + } return 0.0; + } + + if (HasNaN && percentile >= 1.0) { + return std::numeric_limits<double>::quiet_NaN(); + } + // This algorithm uses C=1/2 with 0.5 optimized away // See https://en.wikipedia.org/wiki/Percentile#First_Variant.2C double x = percentile * N; @@ -159,6 +182,9 @@ double TDigest::GetPercentile(double percentile) { double TDigest::GetRank(double value) { Compress(); + if (SupportsNaN && std::isnan(value)) { + return 1.0; + } if (Centroids.empty()) { return 0.0; } @@ -189,6 +215,10 @@ TString TDigest::Serialize() { NTDigest::TDigest digest; digest.set_delta(Delta); digest.set_k(K); + if (HasNaN) { + digest.set_nans(HasNaN); + } + for (const auto& it : Centroids) { NTDigest::TDigest::TCentroid* centroid = digest.add_centroids(); centroid->set_mean(it.Mean); diff --git a/library/cpp/tdigest/tdigest.h b/library/cpp/tdigest/tdigest.h index 715620258c..22c87e63c3 100644 --- a/library/cpp/tdigest/tdigest.h +++ b/library/cpp/tdigest/tdigest.h @@ -36,6 +36,8 @@ class TDigest { double N; double Delta; double K; + bool SupportsNaN = false; + bool HasNaN = false; void Add(const TDigest& otherDigest); void AddCentroid(const TCentroid& centroid); @@ -47,10 +49,10 @@ protected: void Update(double x, double w = 1.0); public: - TDigest(double delta = 0.01, double k = 25); - TDigest(double delta, double k, double firstValue); - TDigest(TStringBuf serializedDigest); - TDigest(const TDigest* digest1, const TDigest* digest2); // merge + TDigest(double delta = 0.01, double k = 25, bool supportsNaN = false); + TDigest(double delta, double k, double firstValue, bool supportsNaN = false); + TDigest(TStringBuf serializedDigest, bool supportsNaN = false); + TDigest(const TDigest* digest1, const TDigest* digest2, bool supportsNaN = false); // merge TString Serialize(); diff --git a/library/cpp/tdigest/tdigest.proto b/library/cpp/tdigest/tdigest.proto index 4a2db3e638..abd8b821cc 100644 --- a/library/cpp/tdigest/tdigest.proto +++ b/library/cpp/tdigest/tdigest.proto @@ -8,4 +8,5 @@ message TDigest { optional double Weight = 2; } repeated TCentroid Centroids = 3; + optional bool Nans = 4; } diff --git a/library/cpp/tld/tlds-alpha-by-domain.txt b/library/cpp/tld/tlds-alpha-by-domain.txt index fc48570eaf..b534e60ce4 100644 --- a/library/cpp/tld/tlds-alpha-by-domain.txt +++ b/library/cpp/tld/tlds-alpha-by-domain.txt @@ -1,4 +1,4 @@ -# Version 2025022200, Last Updated Sat Feb 22 07:07:01 2025 UTC +# Version 2025030100, Last Updated Sat Mar 1 07:07:02 2025 UTC AAA AARP ABB @@ -627,7 +627,6 @@ KAUFEN KDDI KE KERRYHOTELS -KERRYLOGISTICS KERRYPROPERTIES KFH KG @@ -691,7 +690,6 @@ LIMITED LIMO LINCOLN LINK -LIPSY LIVE LIVING LK diff --git a/library/cpp/yt/error/error.cpp b/library/cpp/yt/error/error.cpp index 636d7d81fc..f097697cd6 100644 --- a/library/cpp/yt/error/error.cpp +++ b/library/cpp/yt/error/error.cpp @@ -29,6 +29,8 @@ void FormatValue(TStringBuilderBase* builder, TErrorCode code, TStringBuf spec) constexpr TStringBuf ErrorMessageTruncatedSuffix = "...<message truncated>"; +TError::TEnricher TError::Enricher_; + //////////////////////////////////////////////////////////////////////////////// class TError::TImpl @@ -275,15 +277,20 @@ TError::TErrorOr(const std::exception& ex) *this = TError(NYT::EErrorCode::Generic, TRuntimeFormat{ex.what()}); } YT_VERIFY(!IsOK()); + Enrich(); } TError::TErrorOr(std::string message, TDisableFormat) : Impl_(std::make_unique<TImpl>(std::move(message))) -{ } +{ + Enrich(); +} TError::TErrorOr(TErrorCode code, std::string message, TDisableFormat) : Impl_(std::make_unique<TImpl>(code, std::move(message))) -{ } +{ + Enrich(); +} TError& TError::operator = (const TError& other) { @@ -632,6 +639,20 @@ std::optional<TError> TError::FindMatching(const THashSet<TErrorCode>& codes) co }); } +void TError::RegisterEnricher(TEnricher enricher) +{ + // NB: This daisy-chaining strategy is optimal when there's O(1) callbacks. Convert to a vector + // if the number grows. + if (Enricher_) { + Enricher_ = [first = std::move(Enricher_), second = std::move(enricher)] (TError& error) { + first(error); + second(error); + }; + } else { + Enricher_ = std::move(enricher); + } +} + TError::TErrorOr(std::unique_ptr<TImpl> impl) : Impl_(std::move(impl)) { } @@ -643,6 +664,13 @@ void TError::MakeMutable() } } +void TError::Enrich() +{ + if (Enricher_) { + Enricher_(*this); + } +} + //////////////////////////////////////////////////////////////////////////////// TError& TError::operator <<= (const TErrorAttribute& attribute) & diff --git a/library/cpp/yt/error/error.h b/library/cpp/yt/error/error.h index 8b459d0beb..92f19bc398 100644 --- a/library/cpp/yt/error/error.h +++ b/library/cpp/yt/error/error.h @@ -219,6 +219,13 @@ public: template <CErrorNestable TValue> TError operator << (const std::optional<TValue>& rhs) const &; + // The |enricher| is called during TError construction and before TErrorOr<> construction. Meant + // to enrich the error, e.g. by setting generic attributes. The |RegisterEnricher| method is not + // threadsafe and is meant to be called from single-threaded bootstrapping code. Multiple + // enrichers are supported and will be called in order of registration. + using TEnricher = std::function<void(TError&)>; + static void RegisterEnricher(TEnricher enricher); + private: class TImpl; std::unique_ptr<TImpl> Impl_; @@ -226,8 +233,11 @@ private: explicit TErrorOr(std::unique_ptr<TImpl> impl); void MakeMutable(); + void Enrich(); friend class TErrorAttributes; + + static TEnricher Enricher_; }; //////////////////////////////////////////////////////////////////////////////// diff --git a/library/python/runtime_py3/main/main.c b/library/python/runtime_py3/main/main.c index 0c4aa23013..21742d263c 100644 --- a/library/python/runtime_py3/main/main.c +++ b/library/python/runtime_py3/main/main.c @@ -74,11 +74,20 @@ static int RunModule(const char* modname) return 0; } -static int pymain(int argc, char** argv) { +#ifdef MS_WINDOWS +static int pymain(int argc, wchar_t** argv) +#else +static int pymain(int argc, char** argv) +#endif +{ PyStatus status; if (IsYaIdeVenv()) { +#ifdef MS_WINDOWS + return Py_Main(argc, argv); +#else return Py_BytesMain(argc, argv); +#endif } status = _PyRuntime_Initialize(); @@ -115,12 +124,20 @@ static int pymain(int argc, char** argv) { } if (argc > 0 && argv) { +#ifdef MS_WINDOWS + status = PyConfig_SetString(&config, &config.program_name, argv[0]); +#else status = PyConfig_SetBytesString(&config, &config.program_name, argv[0]); +#endif if (PyStatus_Exception(status)) { goto error; } +#ifdef MS_WINDOWS + status = PyConfig_SetArgv(&config, argc, argv); +#else status = PyConfig_SetBytesArgv(&config, argc, argv); +#endif if (PyStatus_Exception(status)) { goto error; } @@ -155,7 +172,11 @@ static int pymain(int argc, char** argv) { } if (entry_point_copy && !strcmp(entry_point_copy, main_entry_point)) { +#ifdef MS_WINDOWS + sts = Py_Main(argc, argv); +#else sts = Py_BytesMain(argc, argv); +#endif free(entry_point_copy); return sts; } @@ -217,8 +238,16 @@ error: return sts; } +#ifdef MS_WINDOWS +int (*mainptr)(int argc, wchar_t** argv) = pymain; + +int wmain(int argc, wchar_t** argv) { + return mainptr(argc, argv); +} +#else int (*mainptr)(int argc, char** argv) = pymain; int main(int argc, char** argv) { return mainptr(argc, argv); } +#endif diff --git a/util/system/backtrace.cpp b/util/system/backtrace.cpp index 7e3ca7ffcd..97a798e9bb 100644 --- a/util/system/backtrace.cpp +++ b/util/system/backtrace.cpp @@ -32,7 +32,7 @@ extern "C" __stdcall unsigned short CaptureStackBackTrace(unsigned long FramesTo #define HAVE_BACKTRACE #endif - #if !defined(HAVE_BACKTRACE) && defined(__GNUC__) + #if !defined(HAVE_BACKTRACE) && defined(__GNUC__) && !defined(__EMSCRIPTEN__) #define USE_GCC_BACKTRACE #define HAVE_BACKTRACE #endif diff --git a/util/system/thread.cpp b/util/system/thread.cpp index cdde02fa3e..7c880a2909 100644 --- a/util/system/thread.cpp +++ b/util/system/thread.cpp @@ -535,11 +535,11 @@ TCurrentThreadLimits::TCurrentThreadLimits() noexcept : StackBegin(nullptr) , StackLength(0) { -#if defined(_linux_) || defined(_cygwin_) || defined(_freebsd_) +#if defined(_linux_) || defined(_cygwin_) || defined(_freebsd_) || defined(__EMSCRIPTEN__) pthread_attr_t attr; pthread_attr_init(&attr); - #if defined(_linux_) || defined(_cygwin_) + #if defined(_linux_) || defined(_cygwin_) || defined(__EMSCRIPTEN__) Y_ABORT_UNLESS(pthread_getattr_np(pthread_self(), &attr) == 0, "pthread_getattr failed"); #else Y_ABORT_UNLESS(pthread_attr_get_np(pthread_self(), &attr) == 0, "pthread_attr_get_np failed"); diff --git a/util/ya.make b/util/ya.make index 943d8b16cd..dea1f1f721 100644 --- a/util/ya.make +++ b/util/ya.make @@ -322,6 +322,7 @@ JOIN_SRCS( system/sys_alloc.cpp system/sysstat.cpp system/tempfile.cpp + system/thread.cpp system/tls.cpp system/type_name.cpp system/unaligned_mem.cpp @@ -335,7 +336,6 @@ JOIN_SRCS( all_system_4.cpp system/mem_info.cpp system/sem.cpp - system/thread.cpp system/types.cpp ) ENDIF() diff --git a/ydb/ci/rightlib.txt b/ydb/ci/rightlib.txt index 98e9fec1c8..b7b70c3a1b 100644 --- a/ydb/ci/rightlib.txt +++ b/ydb/ci/rightlib.txt @@ -1 +1 @@ -56146bfad396d685118eaaec9cfff0c0675eae75 +6678165e016ba474f1b8dd6d49af92b0d46350b9 diff --git a/ydb/library/yaml_config/ut_transform/canondata/test_transform.TestYamlConfigTransformations.test_basic_args1-dump_ds_init_/nvme.yaml.result.json b/ydb/library/yaml_config/ut_transform/canondata/test_transform.TestYamlConfigTransformations.test_basic_args1-dump_ds_init_/nvme.yaml.result.json index df02ee3be2..5b916ee9d9 100644 --- a/ydb/library/yaml_config/ut_transform/canondata/test_transform.TestYamlConfigTransformations.test_basic_args1-dump_ds_init_/nvme.yaml.result.json +++ b/ydb/library/yaml_config/ut_transform/canondata/test_transform.TestYamlConfigTransformations.test_basic_args1-dump_ds_init_/nvme.yaml.result.json @@ -1 +1 @@ -{"error": true, "stderr": "uncaught exception:\n address -> REDACTED\n what() -> \"ydb/library/yaml_config/deprecated/yaml_config_parser.cpp:855: Specify host_configs to use blobstorage init command\"\n type -> TWithBackTrace<yexception>\n"}
\ No newline at end of file +{"error": true, "stderr": "Terminating due to uncaught exception 0x59e3f140410 what() -> \"ydb/library/yaml_config/deprecated/yaml_config_parser.cpp:855: Specify host_configs to use blobstorage init command\"\n of type TWithBackTrace<yexception>\n"}
\ No newline at end of file diff --git a/yql/essentials/core/facade/ya.make b/yql/essentials/core/facade/ya.make index 756ed88de4..6164039fa8 100644 --- a/yql/essentials/core/facade/ya.make +++ b/yql/essentials/core/facade/ya.make @@ -26,12 +26,6 @@ PEERDIR( yql/essentials/sql/v1/lexer/antlr4_ansi yql/essentials/sql/v1/proto_parser/antlr4 yql/essentials/sql/v1/proto_parser/antlr4_ansi - #FIXME { - yql/essentials/sql/v1/lexer/antlr3 - yql/essentials/sql/v1/lexer/antlr3_ansi - yql/essentials/sql/v1/proto_parser/antlr3 - yql/essentials/sql/v1/proto_parser/antlr3_ansi - #} yql/essentials/parser/pg_wrapper/interface yql/essentials/utils/log yql/essentials/core diff --git a/yql/essentials/core/facade/yql_facade.cpp b/yql/essentials/core/facade/yql_facade.cpp index 97a6f54d0b..f8d9c28ba1 100644 --- a/yql/essentials/core/facade/yql_facade.cpp +++ b/yql/essentials/core/facade/yql_facade.cpp @@ -12,12 +12,6 @@ #include <yql/essentials/core/services/yql_eval_params.h> #include <yql/essentials/sql/sql.h> #include <yql/essentials/sql/v1/sql.h> -//FIXME { -#include <yql/essentials/sql/v1/lexer/antlr3/lexer.h> -#include <yql/essentials/sql/v1/lexer/antlr3_ansi/lexer.h> -#include <yql/essentials/sql/v1/proto_parser/antlr3/proto_parser.h> -#include <yql/essentials/sql/v1/proto_parser/antlr3_ansi/proto_parser.h> -//} #include <yql/essentials/sql/v1/lexer/antlr4/lexer.h> #include <yql/essentials/sql/v1/lexer/antlr4_ansi/lexer.h> #include <yql/essentials/sql/v1/proto_parser/antlr4/proto_parser.h> @@ -742,13 +736,9 @@ bool TProgram::ParseSql(const NSQLTranslation::TTranslationSettings& settings) currentSettings->EmitReadsForExists = true; NSQLTranslationV1::TLexers lexers; - lexers.Antlr3 = NSQLTranslationV1::MakeAntlr3LexerFactory(); - lexers.Antlr3Ansi = NSQLTranslationV1::MakeAntlr3AnsiLexerFactory(); lexers.Antlr4 = NSQLTranslationV1::MakeAntlr4LexerFactory(); lexers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiLexerFactory(); NSQLTranslationV1::TParsers parsers; - parsers.Antlr3 = NSQLTranslationV1::MakeAntlr3ParserFactory(); - parsers.Antlr3Ansi = NSQLTranslationV1::MakeAntlr3AnsiParserFactory(); parsers.Antlr4 = NSQLTranslationV1::MakeAntlr4ParserFactory(); parsers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiParserFactory(); diff --git a/yql/essentials/minikql/computation/llvm16/ya.make b/yql/essentials/minikql/computation/llvm16/ya.make index 98e800970f..2058603ae1 100644 --- a/yql/essentials/minikql/computation/llvm16/ya.make +++ b/yql/essentials/minikql/computation/llvm16/ya.make @@ -6,13 +6,24 @@ PEERDIR( contrib/libs/llvm16/lib/ExecutionEngine/MCJIT contrib/libs/llvm16/lib/Linker contrib/libs/llvm16/lib/Passes - contrib/libs/llvm16/lib/Target/X86 - contrib/libs/llvm16/lib/Target/X86/AsmParser - contrib/libs/llvm16/lib/Target/X86/Disassembler contrib/libs/llvm16/lib/Transforms/IPO contrib/libs/llvm16/lib/Transforms/ObjCARC ) +IF (ARCH_X86_64) + PEERDIR( + contrib/libs/llvm16/lib/Target/X86 + contrib/libs/llvm16/lib/Target/X86/AsmParser + contrib/libs/llvm16/lib/Target/X86/Disassembler + ) +ELSEIF (ARCH_AARCH64) + PEERDIR( + contrib/libs/llvm16/lib/Target/AArch64 + contrib/libs/llvm16/lib/Target/AArch64/AsmParser + contrib/libs/llvm16/lib/Target/AArch64/Disassembler + ) +ENDIF() + INCLUDE(../ya.make.inc) END() diff --git a/yql/essentials/parser/antlr_ast/gen/v1_ansi_antlr4/epilogue.cmake b/yql/essentials/parser/antlr_ast/gen/v1_ansi_antlr4/epilogue.cmake new file mode 100644 index 0000000000..e039c0caf6 --- /dev/null +++ b/yql/essentials/parser/antlr_ast/gen/v1_ansi_antlr4/epilogue.cmake @@ -0,0 +1,8 @@ +set(GRAMMAR_STRING_CORE_SINGLE "~([']) | (QUOTE_SINGLE QUOTE_SINGLE)") +set(GRAMMAR_STRING_CORE_DOUBLE "~([\"]) | (QUOTE_DOUBLE QUOTE_DOUBLE)") +set(GRAMMAR_MULTILINE_COMMENT_CORE "MULTILINE_COMMENT | .") + +configure_file( + ${CMAKE_SOURCE_DIR}/yql/essentials/sql/v1/SQLv1Antlr4.g.in + ${CMAKE_BINARY_DIR}/yql/essentials/parser/antlr_ast/gen/v1_ansi_antlr4/SQLv1Antlr4.g +) diff --git a/yql/essentials/parser/antlr_ast/gen/v1_ansi_antlr4/ya.make b/yql/essentials/parser/antlr_ast/gen/v1_ansi_antlr4/ya.make new file mode 100644 index 0000000000..d0b36ae35a --- /dev/null +++ b/yql/essentials/parser/antlr_ast/gen/v1_ansi_antlr4/ya.make @@ -0,0 +1,43 @@ +LIBRARY() + +SET(SQL_GRAMMAR ${ARCADIA_BUILD_ROOT}/${MODDIR}/SQLv1Antlr4.g4) + +IF(EXPORT_CMAKE) + MANUAL_GENERATION(${SQL_GRAMMAR}) +ELSE() + SET(GRAMMAR_STRING_CORE_SINGLE "\"~([']) | (QUOTE_SINGLE QUOTE_SINGLE)\"") + SET(GRAMMAR_STRING_CORE_DOUBLE "\"~([#DOUBLE_QUOTE#]) | (QUOTE_DOUBLE QUOTE_DOUBLE)\"") + SET(GRAMMAR_MULTILINE_COMMENT_CORE "\"MULTILINE_COMMENT | .\"") + + CONFIGURE_FILE(${ARCADIA_ROOT}/yql/essentials/sql/v1/SQLv1Antlr4.g.in ${SQL_GRAMMAR}) +ENDIF() + +COPY_FILE( + ${ARCADIA_ROOT}/yql/essentials/parser/antlr_ast/org/antlr/v4/tool/templates/codegen/Cpp/Cpp.stg + ${ARCADIA_BUILD_ROOT}/${MODDIR}/org/antlr/v4/tool/templates/codegen/Cpp/Cpp.stg +) + +COPY_FILE( + ${ARCADIA_ROOT}/yql/essentials/parser/antlr_ast/org/antlr/v4/tool/templates/codegen/Cpp/Files.stg + ${ARCADIA_BUILD_ROOT}/${MODDIR}/org/antlr/v4/tool/templates/codegen/Cpp/Files.stg +) + +RUN_ANTLR4( + ${SQL_GRAMMAR} + -no-listener + -package NALAAnsiAntlr4 + -lib . + IN + ${SQL_GRAMMAR} + ${ARCADIA_BUILD_ROOT}/${MODDIR}/org/antlr/v4/tool/templates/codegen/Cpp/Cpp.stg + ${ARCADIA_BUILD_ROOT}/${MODDIR}/org/antlr/v4/tool/templates/codegen/Cpp/Files.stg + OUT SQLv1Antlr4Parser.cpp SQLv1Antlr4Lexer.cpp SQLv1Antlr4Parser.h SQLv1Antlr4Lexer.h + OUTPUT_INCLUDES contrib/libs/antlr4_cpp_runtime/src/antlr4-runtime.h + CWD ${ARCADIA_BUILD_ROOT}/${MODDIR} +) + +PEERDIR( + contrib/libs/antlr4_cpp_runtime +) + +END() diff --git a/yql/essentials/parser/antlr_ast/gen/v1_antlr4/epilogue.cmake b/yql/essentials/parser/antlr_ast/gen/v1_antlr4/epilogue.cmake new file mode 100644 index 0000000000..1fb34e7415 --- /dev/null +++ b/yql/essentials/parser/antlr_ast/gen/v1_antlr4/epilogue.cmake @@ -0,0 +1,8 @@ +set(GRAMMAR_STRING_CORE_SINGLE "~(['\\\\]) | (BACKSLASH .)") +set(GRAMMAR_STRING_CORE_DOUBLE "~([\"\\\\]) | (BACKSLASH .)") +set(GRAMMAR_MULTILINE_COMMENT_CORE ".") + +configure_file( + ${CMAKE_SOURCE_DIR}/yql/essentials/sql/v1/SQLv1Antlr4.g.in + ${CMAKE_BINARY_DIR}/yql/essentials/parser/antlr_ast/gen/v1_antlr4/SQLv1Antlr4.g +) diff --git a/yql/essentials/parser/antlr_ast/gen/v1_antlr4/ya.make b/yql/essentials/parser/antlr_ast/gen/v1_antlr4/ya.make new file mode 100644 index 0000000000..2ef05bd592 --- /dev/null +++ b/yql/essentials/parser/antlr_ast/gen/v1_antlr4/ya.make @@ -0,0 +1,43 @@ +LIBRARY() + +SET(SQL_GRAMMAR ${ARCADIA_BUILD_ROOT}/${MODDIR}/SQLv1Antlr4.g) + +IF(EXPORT_CMAKE) + MANUAL_GENERATION(${SQL_GRAMMAR}) +ELSE() + SET(GRAMMAR_STRING_CORE_SINGLE "\"~(['#BACKSLASH#]) | (BACKSLASH .)\"") + SET(GRAMMAR_STRING_CORE_DOUBLE "\"~([#DOUBLE_QUOTE##BACKSLASH#]) | (BACKSLASH .)\"") + SET(GRAMMAR_MULTILINE_COMMENT_CORE "\".\"") + + CONFIGURE_FILE(${ARCADIA_ROOT}/yql/essentials/sql/v1/SQLv1Antlr4.g.in ${SQL_GRAMMAR}) +ENDIF() + +COPY_FILE( + ${ARCADIA_ROOT}/yql/essentials/parser/antlr_ast/org/antlr/v4/tool/templates/codegen/Cpp/Cpp.stg + ${ARCADIA_BUILD_ROOT}/${MODDIR}/org/antlr/v4/tool/templates/codegen/Cpp/Cpp.stg +) + +COPY_FILE( + ${ARCADIA_ROOT}/yql/essentials/parser/antlr_ast/org/antlr/v4/tool/templates/codegen/Cpp/Files.stg + ${ARCADIA_BUILD_ROOT}/${MODDIR}/org/antlr/v4/tool/templates/codegen/Cpp/Files.stg +) + +RUN_ANTLR4( + ${SQL_GRAMMAR} + -no-listener + -package NALADefaultAntlr4 + -lib . + IN + ${SQL_GRAMMAR} + ${ARCADIA_BUILD_ROOT}/${MODDIR}/org/antlr/v4/tool/templates/codegen/Cpp/Cpp.stg + ${ARCADIA_BUILD_ROOT}/${MODDIR}/org/antlr/v4/tool/templates/codegen/Cpp/Files.stg + OUT SQLv1Antlr4Parser.cpp SQLv1Antlr4Lexer.cpp SQLv1Antlr4Parser.h SQLv1Antlr4Lexer.h + OUTPUT_INCLUDES contrib/libs/antlr4_cpp_runtime/src/antlr4-runtime.h + CWD ${ARCADIA_BUILD_ROOT}/${MODDIR} +) + +PEERDIR( + contrib/libs/antlr4_cpp_runtime +) + +END() diff --git a/yql/essentials/parser/antlr_ast/gen/ya.make b/yql/essentials/parser/antlr_ast/gen/ya.make new file mode 100644 index 0000000000..2f94911b40 --- /dev/null +++ b/yql/essentials/parser/antlr_ast/gen/ya.make @@ -0,0 +1,4 @@ +RECURSE( + v1_antlr4 + v1_ansi_antlr4 +) diff --git a/yql/essentials/parser/antlr_ast/org/antlr/v4/tool/templates/codegen/Cpp/Cpp.stg b/yql/essentials/parser/antlr_ast/org/antlr/v4/tool/templates/codegen/Cpp/Cpp.stg new file mode 100644 index 0000000000..c6592680f7 --- /dev/null +++ b/yql/essentials/parser/antlr_ast/org/antlr/v4/tool/templates/codegen/Cpp/Cpp.stg @@ -0,0 +1,1176 @@ +/* + * [The "BSD license"] + * Copyright (c) 2015 Dan McLaughlin, Mike Lischke + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import "Files.stg" // All file specific stuff. + +cppTypeInitMap ::= [ + "int":"0", + "long":"0", + "float":"0.0f", + "double":"0.0", + "bool":"false", + "short":"0", + "char":"0", + default: "nullptr" // anything other than a primitive type is an object +] + +LexerHeader(lexer, atn, actionFuncs, sempredFuncs, superClass = {antlr4::Lexer}) ::= << +<namedActions.context> + +class <file.exportMacro> <lexer.name> : public <superClass> { +public: +<if (lexer.tokens)> + enum { + <lexer.tokens: {k | TOKEN_<k> = <lexer.tokens.(k)>}; separator=", ", wrap, anchor> + }; +<endif> + +<if (lexer.escapedChannels)> + enum { + <lexer.escapedChannels: {k | <k> = <lexer.escapedChannels.(k)>}; separator=", ", wrap, anchor> + }; +<endif> + +<if (rest(lexer.escapedModeNames))> + enum { + <rest(lexer.escapedModeNames): {m | TOKEN_<m> = <i>}; separator=", ", wrap, anchor> + }; +<endif> + + explicit <lexer.name>(antlr4::CharStream *input); + + ~<lexer.name>() override; + + <namedActions.members> + + std::string getGrammarFileName() const override; + + const std::vector\<std::string>& getRuleNames() const override; + + const std::vector\<std::string>& getChannelNames() const override; + + const std::vector\<std::string>& getModeNames() const override; + + const antlr4::dfa::Vocabulary& getVocabulary() const override; + + antlr4::atn::SerializedATNView getSerializedATN() const override; + + const antlr4::atn::ATN& getATN() const override; + + <if (actionFuncs)> + void action(antlr4::RuleContext *context, size_t ruleIndex, size_t actionIndex) override; + <endif> + + <if (sempredFuncs)> + bool sempred(antlr4::RuleContext *_localctx, size_t ruleIndex, size_t predicateIndex) override; + <endif> + + // By default the static state used to implement the lexer is lazily initialized during the first + // call to the constructor. You can call this function if you wish to initialize the static state + // ahead of time. + static void initialize(); + +private: + <namedActions.declarations> + + // Individual action functions triggered by action() above. + <actionFuncs.values; separator="\n"> + + // Individual semantic predicate functions triggered by sempred() above. + <sempredFuncs.values; separator="\n"> + + <atn> +}; +>> + +Lexer(lexer, atn, actionFuncs, sempredFuncs, superClass = {Lexer}) ::= << + +using namespace antlr4; + +namespace { + +struct <lexer.name; format = "cap">StaticData final { + <lexer.name; format = "cap">StaticData(std::vector\<std::string> ruleNames, + std::vector\<std::string> channelNames, + std::vector\<std::string> modeNames, + std::vector\<std::string> literalNames, + std::vector\<std::string> symbolicNames) + : ruleNames(std::move(ruleNames)), channelNames(std::move(channelNames)), + modeNames(std::move(modeNames)), literalNames(std::move(literalNames)), + symbolicNames(std::move(symbolicNames)), + vocabulary(this->literalNames, this->symbolicNames) {} + + <lexer.name; format = "cap">StaticData(const <lexer.name; format = "cap">StaticData&) = delete; + <lexer.name; format = "cap">StaticData(<lexer.name; format = "cap">StaticData&&) = delete; + <lexer.name; format = "cap">StaticData& operator=(const <lexer.name; format = "cap">StaticData&) = delete; + <lexer.name; format = "cap">StaticData& operator=(<lexer.name; format = "cap">StaticData&&) = delete; + + std::vector\<antlr4::dfa::DFA> decisionToDFA; + antlr4::atn::PredictionContextCache sharedContextCache; + const std::vector\<std::string> ruleNames; + const std::vector\<std::string> channelNames; + const std::vector\<std::string> modeNames; + const std::vector\<std::string> literalNames; + const std::vector\<std::string> symbolicNames; + const antlr4::dfa::Vocabulary vocabulary; + antlr4::atn::SerializedATNView serializedATN; + std::unique_ptr\<antlr4::atn::ATN> atn; +}; + +::antlr4::internal::OnceFlag <lexer.grammarName; format = "lower">LexerOnceFlag; +#if ANTLR4_USE_THREAD_LOCAL_CACHE +static thread_local +#endif +std::unique_ptr\<<lexer.name; format = "cap">StaticData> <lexer.grammarName; format = "lower">LexerStaticData = nullptr; + +void <lexer.grammarName; format = "lower">LexerInitialize() { +#if ANTLR4_USE_THREAD_LOCAL_CACHE + if (<lexer.grammarName; format = "lower">LexerStaticData != nullptr) { + return; + } +#else + assert(<lexer.grammarName; format = "lower">LexerStaticData == nullptr); +#endif + auto staticData = std::make_unique\<<lexer.name; format = "cap">StaticData>( + std::vector\<std::string>{ + <lexer.ruleNames: {r | "<r>"}; separator = ", ", wrap, anchor> + }, + std::vector\<std::string>{ + "DEFAULT_TOKEN_CHANNEL", "HIDDEN"<if (lexer.channels)>, <lexer.channels: {c | "<c>"}; separator = ", ", wrap, anchor><endif> + }, + std::vector\<std::string>{ + <lexer.modes: {m | "<m>"}; separator = ", ", wrap, anchor> + }, + std::vector\<std::string>{ + <lexer.literalNames: {t | <t>}; null = "\"\"", separator = ", ", wrap, anchor> + }, + std::vector\<std::string>{ + <lexer.symbolicNames: {t | <t>}; null = "\"\"", separator = ", ", wrap, anchor> + } + ); + <atn> + <lexer.grammarName; format = "lower">LexerStaticData = std::move(staticData); +} + +} + +<lexer.name>::<lexer.name>(CharStream *input) : <superClass>(input) { + <lexer.name>::initialize(); + _interpreter = new atn::LexerATNSimulator(this, *<lexer.grammarName; format = "lower">LexerStaticData->atn, <lexer.grammarName; format = "lower">LexerStaticData->decisionToDFA, <lexer.grammarName; format = "lower">LexerStaticData->sharedContextCache); +} + +<lexer.name>::~<lexer.name>() { + delete _interpreter; +} + +std::string <lexer.name>::getGrammarFileName() const { + return "<lexer.grammarFileName>"; +} + +const std::vector\<std::string>& <lexer.name>::getRuleNames() const { + return <lexer.grammarName; format = "lower">LexerStaticData->ruleNames; +} + +const std::vector\<std::string>& <lexer.name>::getChannelNames() const { + return <lexer.grammarName; format = "lower">LexerStaticData->channelNames; +} + +const std::vector\<std::string>& <lexer.name>::getModeNames() const { + return <lexer.grammarName; format = "lower">LexerStaticData->modeNames; +} + +const dfa::Vocabulary& <lexer.name>::getVocabulary() const { + return <lexer.grammarName; format = "lower">LexerStaticData->vocabulary; +} + +antlr4::atn::SerializedATNView <lexer.name>::getSerializedATN() const { + return <lexer.grammarName; format = "lower">LexerStaticData->serializedATN; +} + +const atn::ATN& <lexer.name>::getATN() const { + return *<lexer.grammarName; format = "lower">LexerStaticData->atn; +} + +<namedActions.definitions> + +<if (actionFuncs)> +void <lexer.name>::action(RuleContext *context, size_t ruleIndex, size_t actionIndex) { + switch (ruleIndex) { + <lexer.actionFuncs.values: {f | case <f.ruleIndex>: <f.name>Action(antlrcpp::downCast\<<f.ctxType> *>(context), actionIndex); break;}; separator="\n"> + + default: + break; + } +} +<endif> + +<if (sempredFuncs)> +bool <lexer.name>::sempred(RuleContext *context, size_t ruleIndex, size_t predicateIndex) { + switch (ruleIndex) { + <lexer.sempredFuncs.values: {f | case <f.ruleIndex>: return <f.name>Sempred(antlrcpp::downCast\<<f.ctxType> *>(context), predicateIndex);}; separator="\n"> + + default: + break; + } + return true; +} +<endif> + +<actionFuncs.values; separator="\n"> + +<sempredFuncs.values; separator="\n"> + +void <lexer.name>::initialize() { +#if ANTLR4_USE_THREAD_LOCAL_CACHE + <lexer.grammarName; format = "lower">LexerInitialize(); +#else + ::antlr4::internal::call_once(<lexer.grammarName; format = "lower">LexerOnceFlag, <lexer.grammarName; format = "lower">LexerInitialize); +#endif +} +>> + +RuleActionFunctionHeader(r, actions) ::= << +void <r.name>Action(<r.ctxType> *context, size_t actionIndex); +>> + +RuleActionFunction(r, actions) ::= << +void <r.factory.grammar.name>::<r.name>Action(<r.ctxType> *context, size_t actionIndex) { + switch (actionIndex) { + <actions: {index | case <index>: <actions.(index)> break;}; separator="\n"> + + default: + break; + } +} + +>> + +RuleSempredFunctionHeader(r, actions) ::= << +bool <r.name>Sempred(<r.ctxType> *_localctx, size_t predicateIndex); +>> + +RuleSempredFunction(r, actions) ::= << +<! Called for both lexer and parser. But only one of them is actually available. Testing for the parser directly + generates a warning, however. So do the check via the factory instead. !> +bool <if (r.factory.g.lexer)><lexer.name><else><parser.name><endif>::<r.name>Sempred(<r.ctxType> *_localctx, size_t predicateIndex) { + switch (predicateIndex) { + <actions: {index | case <index>: return <actions.(index)>}; separator=";\n">; + + default: + break; + } + return true; +} + +>> + +//-------------------------------------------------------------------------------------------------- + +ParserHeader(parser, funcs, atn, sempredFuncs, superClass = {antlr4::Parser}) ::= << +<namedActions.context> + +class <file.exportMacro> <parser.name> : public <superClass> { +public: +<if (parser.tokens)> + enum { + <parser.tokens: {k | TOKEN_<k> = <parser.tokens.(k)>}; separator=", ", wrap, anchor> + }; +<endif> + +<if (parser.rules)> + enum { + <parser.rules: {r | Rule<r.name; format="cap"> = <r.index>}; separator=", ", wrap, anchor> + }; +<endif> + + explicit <parser.name>(antlr4::TokenStream *input); + + <parser.name>(antlr4::TokenStream *input, const antlr4::atn::ParserATNSimulatorOptions &options); + + ~<parser.name>() override; + + std::string getGrammarFileName() const override; + + const antlr4::atn::ATN& getATN() const override; + + const std::vector\<std::string>& getRuleNames() const override; + + const antlr4::dfa::Vocabulary& getVocabulary() const override; + + antlr4::atn::SerializedATNView getSerializedATN() const override; + + <namedActions.members> + + <parser.funcs: {f | class <f.name; format = "cap">Context;}; separator = "\n"> <! Forward declare context classes. !> + + <funcs; separator = "\n"> + + <if (sempredFuncs)> + bool sempred(antlr4::RuleContext *_localctx, size_t ruleIndex, size_t predicateIndex) override; + + <sempredFuncs.values; separator = "\n"> + <endif> + + // By default the static state used to implement the parser is lazily initialized during the first + // call to the constructor. You can call this function if you wish to initialize the static state + // ahead of time. + static void initialize(); + + static const size_t TOKEN_EOF = antlr4::Token::EOF; + +private: + <namedActions.declarations> +}; +>> + +Parser(parser, funcs, atn, sempredFuncs, superClass = {Parser}) ::= << + +using namespace antlr4; + +namespace { + +struct <parser.name; format = "cap">StaticData final { + <parser.name; format = "cap">StaticData(std::vector\<std::string> ruleNames, + std::vector\<std::string> literalNames, + std::vector\<std::string> symbolicNames) + : ruleNames(std::move(ruleNames)), literalNames(std::move(literalNames)), + symbolicNames(std::move(symbolicNames)), + vocabulary(this->literalNames, this->symbolicNames) {} + + <parser.name; format = "cap">StaticData(const <parser.name; format = "cap">StaticData&) = delete; + <parser.name; format = "cap">StaticData(<parser.name; format = "cap">StaticData&&) = delete; + <parser.name; format = "cap">StaticData& operator=(const <parser.name; format = "cap">StaticData&) = delete; + <parser.name; format = "cap">StaticData& operator=(<parser.name; format = "cap">StaticData&&) = delete; + + std::vector\<antlr4::dfa::DFA> decisionToDFA; + antlr4::atn::PredictionContextCache sharedContextCache; + const std::vector\<std::string> ruleNames; + const std::vector\<std::string> literalNames; + const std::vector\<std::string> symbolicNames; + const antlr4::dfa::Vocabulary vocabulary; + antlr4::atn::SerializedATNView serializedATN; + std::unique_ptr\<antlr4::atn::ATN> atn; +}; + +::antlr4::internal::OnceFlag <parser.grammarName; format = "lower">ParserOnceFlag; +#if ANTLR4_USE_THREAD_LOCAL_CACHE +static thread_local +#endif +std::unique_ptr\<<parser.name; format = "cap">StaticData> <parser.grammarName; format = "lower">ParserStaticData = nullptr; + +void <parser.grammarName; format = "lower">ParserInitialize() { +#if ANTLR4_USE_THREAD_LOCAL_CACHE + if (<parser.grammarName; format = "lower">ParserStaticData != nullptr) { + return; + } +#else + assert(<parser.grammarName; format = "lower">ParserStaticData == nullptr); +#endif + auto staticData = std::make_unique\<<parser.name; format = "cap">StaticData>( + std::vector\<std::string>{ + <parser.ruleNames: {r | "<r>"}; separator = ", ", wrap, anchor> + }, + std::vector\<std::string>{ + <parser.literalNames: {t | <t>}; null = "\"\"", separator = ", ", wrap, anchor> + }, + std::vector\<std::string>{ + <parser.symbolicNames: {t | <t>}; null = "\"\"", separator = ", ", wrap, anchor> + } + ); + <atn> + <parser.grammarName; format = "lower">ParserStaticData = std::move(staticData); +} + +} + +<parser.name>::<parser.name>(TokenStream *input) : <parser.name>(input, antlr4::atn::ParserATNSimulatorOptions()) {} + +<parser.name>::<parser.name>(TokenStream *input, const antlr4::atn::ParserATNSimulatorOptions &options) : <superClass>(input) { + <parser.name>::initialize(); + _interpreter = new atn::ParserATNSimulator(this, *<parser.grammarName; format = "lower">ParserStaticData->atn, <parser.grammarName; format = "lower">ParserStaticData->decisionToDFA, <parser.grammarName; format = "lower">ParserStaticData->sharedContextCache, options); +} + +<parser.name>::~<parser.name>() { + delete _interpreter; +} + +const atn::ATN& <parser.name>::getATN() const { + return *<parser.grammarName; format = "lower">ParserStaticData->atn; +} + +std::string <parser.name>::getGrammarFileName() const { + return "<parser.grammarFileName>"; +} + +const std::vector\<std::string>& <parser.name>::getRuleNames() const { + return <parser.grammarName; format = "lower">ParserStaticData->ruleNames; +} + +const dfa::Vocabulary& <parser.name>::getVocabulary() const { + return <parser.grammarName; format = "lower">ParserStaticData->vocabulary; +} + +antlr4::atn::SerializedATNView <parser.name>::getSerializedATN() const { + return <parser.grammarName; format = "lower">ParserStaticData->serializedATN; +} + +<namedActions.definitions> + +<funcs; separator = "\n\n"> + +<if (sempredFuncs)> +bool <parser.name>::sempred(RuleContext *context, size_t ruleIndex, size_t predicateIndex) { + switch (ruleIndex) { + <parser.sempredFuncs.values: {f | + case <f.ruleIndex>: return <f.name>Sempred(antlrcpp::downCast\<<f.ctxType> *>(context), predicateIndex);}; separator="\n"> + + default: + break; + } + return true; +} + +<sempredFuncs.values; separator="\n"><endif> + +void <parser.name>::initialize() { +#if ANTLR4_USE_THREAD_LOCAL_CACHE + <parser.grammarName; format = "lower">ParserInitialize(); +#else + ::antlr4::internal::call_once(<parser.grammarName; format = "lower">ParserOnceFlag, <parser.grammarName; format = "lower">ParserInitialize); +#endif +} +>> + +SerializedATNHeader(model) ::= << +>> + +SerializedATN(model) ::= << +static const int32_t serializedATNSegment[] = { + <model.serialized: {s | <s>}; separator=",", wrap> +}; +staticData->serializedATN = antlr4::atn::SerializedATNView(serializedATNSegment, sizeof(serializedATNSegment) / sizeof(serializedATNSegment[0])); + +antlr4::atn::ATNDeserializer deserializer; +staticData->atn = deserializer.deserialize(staticData->serializedATN); + +const size_t count = staticData->atn->getNumberOfDecisions(); +staticData->decisionToDFA.reserve(count); +for (size_t i = 0; i \< count; i++) { <! Rework class ATN to allow standard iterations. !> + staticData->decisionToDFA.emplace_back(staticData->atn->getDecisionState(i), i); +} +>> + +RuleFunctionHeader(currentRule, args, code, locals, ruleCtx, altLabelCtxs, namedActions, finallyAction, postamble, exceptions) ::= << +<ruleCtx> +<! TODO: untested !><if (altLabelCtxs)><altLabelCtxs: {l | <altLabelCtxs.(l)>}; separator="\n"><endif> +<currentRule.ctxType>* <currentRule.escapedName>(<args; separator=",">); + +>> + +RuleFunction(currentRule, args, code, locals, ruleCtx, altLabelCtxs, namedActions, finallyAction, postamble, exceptions) ::= << +<ruleCtx> +<! TODO: untested !><altLabelCtxs: {l | <altLabelCtxs.(l)>}; separator = "\n"> +<parser.name>::<currentRule.ctxType>* <parser.name>::<currentRule.escapedName>(<args; separator=",">) { + <currentRule.ctxType> *_localctx = _tracker.createInstance\<<currentRule.ctxType>\>(_ctx, getState()<currentRule.args:{a | , <a.escapedName>}>); + enterRule(_localctx, <currentRule.startState>, <parser.name>::Rule<currentRule.name; format = "cap">); + <namedActions.init> + <locals; separator = "\n"> + +#if __cplusplus > 201703L + auto onExit = finally([=, this] { +#else + auto onExit = finally([=] { +#endif + <finallyAction> + exitRule(); + }); + try { +<! TODO: untested !><if (currentRule.hasLookaheadBlock)> + size_t alt; + <endif> + <code> +<! TODO: untested !> <postamble; separator = "\n"> + <namedActions.after> + } + <if (exceptions)> + <exceptions; separator="\n"> + <else> + catch (RecognitionException &e) { + _errHandler->reportError(this, e); + _localctx->exception = std::current_exception(); + _errHandler->recover(this, _localctx->exception); + } + <endif> + + return _localctx; +} +>> + +LeftRecursiveRuleFunctionHeader(currentRule, args, code, locals, ruleCtx, altLabelCtxs, namedActions, finallyAction, postamble) ::= << +<ruleCtx> +<! TODO: untested !><altLabelCtxs: {l | <altLabelCtxs.(l)>}; separator="\n"> +<currentRule.ctxType>* <currentRule.escapedName>(<currentRule.args; separator = ", ">); +<currentRule.ctxType>* <currentRule.escapedName>(int precedence<currentRule.args: {a | , <a>}>); +>> + +LeftRecursiveRuleFunction(currentRule, args, code, locals, ruleCtx, altLabelCtxs, namedActions, finallyAction, postamble) ::= << +<ruleCtx> +<altLabelCtxs: {l | <altLabelCtxs.(l)>}; separator="\n"> + +<parser.name>::<currentRule.ctxType>* <parser.name>::<currentRule.escapedName>(<currentRule.args; separator=", ">) { +<! TODO: currentRule.args untested !> return <currentRule.escapedName>(0<currentRule.args: {a | , <a.escapedName>}>); +} + +<parser.name>::<currentRule.ctxType>* <parser.name>::<currentRule.escapedName>(int precedence<currentRule.args:{a | , <a>}>) { + ParserRuleContext *parentContext = _ctx; + size_t parentState = getState(); + <parser.name>::<currentRule.ctxType> *_localctx = _tracker.createInstance\<<currentRule.ctxType>\>(_ctx, parentState<currentRule.args: {a | , <a.escapedName>}>); + <parser.name>::<currentRule.ctxType> *previousContext = _localctx; + (void)previousContext; // Silence compiler, in case the context is not used by generated code. + size_t startState = <currentRule.startState>; + enterRecursionRule(_localctx, <currentRule.startState>, <parser.name>::Rule<currentRule.name; format = "cap">, precedence); + + <namedActions.init> +<! TODO: untested !> <locals; separator = "\n"> + +#if __cplusplus > 201703L + auto onExit = finally([=, this] { +#else + auto onExit = finally([=] { +#endif + <if (finallyAction)><finallyAction><endif> + unrollRecursionContexts(parentContext); + }); + try { + <if (currentRule.hasLookaheadBlock)>size_t alt;<endif> + <code> +<! TODO: untested !><postamble; separator = "\n"> + <namedActions.after> + } + catch (RecognitionException &e) { + _errHandler->reportError(this, e); + _localctx->exception = std::current_exception(); + _errHandler->recover(this, _localctx->exception); + } + return _localctx; +} +>> + +StructDeclHeader(struct, ctorAttrs, attrs, getters, dispatchMethods, interfaces, extensionMembers) ::= << +class <file.exportMacro> <struct.escapedName> : public <if (contextSuperClass)><contextSuperClass><else>antlr4::ParserRuleContext<endif><if(interfaces)>, <interfaces; separator=", "><endif> { +public: + <attrs: {a | <a>;}; separator = "\n"> + <if (ctorAttrs)><struct.escapedName>(antlr4::ParserRuleContext *parent, size_t invokingState);<endif> + <struct.escapedName>(antlr4::ParserRuleContext *parent, size_t invokingState<ctorAttrs: {a | , <a>}>); +<if (struct.provideCopyFrom)> <! don't need copy unless we have subclasses !> + <struct.escapedName>() = default; + void copyFrom(<struct.escapedName> *context); + using antlr4::ParserRuleContext::copyFrom; +<endif> + + virtual size_t getRuleIndex() const override; + <getters: {g | <g>}; separator = "\n"> + + <dispatchMethods; separator = "\n"> +<! TODO: untested !> <extensionMembers; separator = "\n"> +}; + +>> + +StructDecl(struct, ctorAttrs, attrs, getters, dispatchMethods, interfaces, extensionMembers, signatures) ::= << +//----------------- <struct.escapedName> ------------------------------------------------------------------ + +<if (ctorAttrs)> +<parser.name>::<struct.escapedName>::<struct.escapedName>(ParserRuleContext *parent, size_t invokingState) + : <if (contextSuperClass)><contextSuperClass><else>ParserRuleContext<endif>(parent, invokingState) { +} +<endif> + +<parser.name>::<struct.escapedName>::<struct.escapedName>(ParserRuleContext *parent, size_t invokingState<ctorAttrs: {a | , <a>}>) + : <if (contextSuperClass)><contextSuperClass><else>ParserRuleContext<endif>(parent, invokingState) { + <struct.ctorAttrs: {a | this-><a.escapedName> = <a.escapedName>;}; separator="\n"> +} + +<getters: {g | <g>}; separator = "\n"> + +size_t <parser.name>::<struct.escapedName>::getRuleIndex() const { + return <parser.name>::Rule<struct.derivedFromName; format = "cap">; +} + +<if (struct.provideCopyFrom)> +void <parser.name>::<struct.escapedName>::copyFrom(<struct.escapedName> *ctx) { + <if (contextSuperClass)><contextSuperClass><else>ParserRuleContext<endif>::copyFrom(ctx); + <struct.attrs: {a | this-><a.escapedName> = ctx-><a.escapedName>;}; separator = "\n"> +} +<endif> +<dispatchMethods; separator = "\n\n"> +<! TODO: untested !><extensionMembers; separator = "\n\n"> + +>> + +AltLabelStructDeclHeader(struct, attrs, getters, dispatchMethods) ::= << +class <file.exportMacro> <struct.escapedName> : public <currentRule.name; format = "cap">Context { +public: + <struct.escapedName>(<currentRule.name; format = "cap">Context *ctx); + + <if (attrs)><attrs: {a | <a>;}; separator = "\n"><endif> + <getters: {g | <g>}; separator = "\n"> + <dispatchMethods; separator = "\n"> +}; + +>> + +AltLabelStructDecl(struct, attrs, getters, dispatchMethods) ::= << +//----------------- <struct.escapedName> ------------------------------------------------------------------ + +<! TODO: untested !><if (attrs)><attrs: {a | <a>}; separator = "\n"><endif> +<getters: {g | <g>}; separator = "\n"> +<parser.name>::<struct.escapedName>::<struct.escapedName>(<currentRule.name; format = "cap">Context *ctx) { copyFrom(ctx); } + +<dispatchMethods; separator="\n"> +>> + + +CodeBlockForOuterMostAltHeader(currentOuterMostAltCodeBlock, locals, preamble, ops) ::= "<! Required to exist, but unused. !>" +CodeBlockForOuterMostAlt(currentOuterMostAltCodeBlock, locals, preamble, ops) ::= << +<if (currentOuterMostAltCodeBlock.altLabel)> +_localctx = _tracker.createInstance\<<parser.name>::<currentOuterMostAltCodeBlock.altLabel; format = "cap">Context>(_localctx); +<endif> +enterOuterAlt(_localctx, <currentOuterMostAltCodeBlock.alt.altNum>); +<CodeBlockForAlt(currentAltCodeBlock = currentOuterMostAltCodeBlock, ...)> +>> + +CodeBlockForAltHeader(currentAltCodeBlock, locals, preamble, ops) ::= "<! Required to exist, but unused. !>" +CodeBlockForAlt(currentAltCodeBlock, locals, preamble, ops) ::= << +<! TODO: untested !><locals; separator = "\n"> +<! TODO: untested !><preamble; separator = "\n"> +<ops; separator = "\n"> +>> + +LL1AltBlockHeader(choice, preamble, alts, error) ::= "<! Required to exist, but unused. !>" +LL1AltBlock(choice, preamble, alts, error) ::= << +setState(<choice.stateNumber>); +_errHandler->sync(this); +<! TODO: untested !><if (choice.label)>LL1AltBlock(choice, preamble, alts, error) <labelref(choice.label)> = _input->LT(1);<endif> +<preamble; separator="\n"> +switch (_input->LA(1)) { + <choice.altLook, alts: {look, alt | <cases(tokens = look)> { + <alt> + break; +\} +}; separator = "\n"> +default: + <error> +} +>> + +LL1OptionalBlockHeader(choice, alts, error) ::= "<! Required but unused. !>" +LL1OptionalBlock(choice, alts, error) ::= << +setState(<choice.stateNumber>); +_errHandler->sync(this); +switch (_input->LA(1)) { + <choice.altLook, alts: {look, alt | <cases(tokens = look)> { + <alt> + break; +\} +}; separator="\n"> +default: + break; +} +>> + +LL1OptionalBlockSingleAltHeader(choice, expr, alts, preamble, error, followExpr) ::= "<! Required but unused. !>" +LL1OptionalBlockSingleAlt(choice, expr, alts, preamble, error, followExpr) ::= << +setState(<choice.stateNumber>); +_errHandler->sync(this); + +<preamble; separator = "\n"> +if (<expr>) { + <alts; separator = "\n"> +} +>> + +LL1StarBlockSingleAltHeader(choice, loopExpr, alts, preamble, iteration) ::= "<! Required but unused. !>" +LL1StarBlockSingleAlt(choice, loopExpr, alts, preamble, iteration) ::= << +setState(<choice.stateNumber>); +_errHandler->sync(this); +<preamble; separator="\n"> +while (<loopExpr>) { + <alts; separator="\n"> + setState(<choice.loopBackStateNumber>); + _errHandler->sync(this); + <iteration> +} +>> + +LL1PlusBlockSingleAltHeader(choice, loopExpr, alts, preamble, iteration) ::= "<! Required but unused. !>" +LL1PlusBlockSingleAlt(choice, loopExpr, alts, preamble, iteration) ::= << +setState(<choice.blockStartStateNumber>); <! alt block decision !> +_errHandler->sync(this); +<preamble; separator="\n"> +do { + <alts; separator="\n"> + setState(<choice.stateNumber>); <! loopback/exit decision !> + _errHandler->sync(this); + <iteration> +} while (<loopExpr>); +>> + +// LL(*) stuff + +AltBlockHeader(choice, preamble, alts, error) ::= "<! Unused but must be present. !>" +AltBlock(choice, preamble, alts, error) ::= << +setState(<choice.stateNumber>); +_errHandler->sync(this); +<! TODO: untested !><if (choice.label)><labelref(choice.label)> = _input->LT(1);<endif> +<! TODO: untested !><preamble; separator = "\n"> +switch (getInterpreter\<atn::ParserATNSimulator>()->adaptivePredict(_input, <choice.decision>, _ctx)) { +<alts: {alt | case <i>: { + <alt> + break; +\} +}; separator="\n"> +default: + break; +} +>> + +OptionalBlockHeader(choice, alts, error) ::= "<! Unused but must be present. !>" +OptionalBlock(choice, alts, error) ::= << +setState(<choice.stateNumber>); +_errHandler->sync(this); + +switch (getInterpreter\<atn::ParserATNSimulator>()->adaptivePredict(_input, <choice.decision>, _ctx)) { +<alts: {alt | case <i><if (!choice.ast.greedy)> + 1<endif>: { + <alt> + break; +\} +}; separator = "\n"> +default: + break; +} +>> + +StarBlockHeader(choice, alts, sync, iteration) ::= "<! Unused but must be present. !>" +StarBlock(choice, alts, sync, iteration) ::= << +setState(<choice.stateNumber>); +_errHandler->sync(this); +alt = getInterpreter\<atn::ParserATNSimulator>()->adaptivePredict(_input, <choice.decision>, _ctx); +while (alt != <choice.exitAlt> && alt != atn::ATN::INVALID_ALT_NUMBER) { + if (alt == 1<if(!choice.ast.greedy)> + 1<endif>) { + <iteration> + <alts> <! should only be one !> + } + setState(<choice.loopBackStateNumber>); + _errHandler->sync(this); + alt = getInterpreter\<atn::ParserATNSimulator>()->adaptivePredict(_input, <choice.decision>, _ctx); +} +>> + +PlusBlockHeader(choice, alts, error) ::= "<! Required to exist, but unused. !>" +PlusBlock(choice, alts, error) ::= << +setState(<choice.blockStartStateNumber>); <! alt block decision !> +_errHandler->sync(this); +alt = 1<if(!choice.ast.greedy)> + 1<endif>; +do { + switch (alt) { + <alts: {alt | case <i><if (!choice.ast.greedy)> + 1<endif>: { + <alt> + break; + \} +}; separator="\n"> + default: + <error> + } + setState(<choice.loopBackStateNumber>); <! loopback/exit decision !> + _errHandler->sync(this); + alt = getInterpreter\<atn::ParserATNSimulator>()->adaptivePredict(_input, <choice.decision>, _ctx); +} while (alt != <choice.exitAlt> && alt != atn::ATN::INVALID_ALT_NUMBER); +>> + +Sync(s) ::= "Sync(s) sync(<s.expecting.name>);" + +ThrowNoViableAltHeader(t) ::= "<! Unused but must be present. !>" +ThrowNoViableAlt(t) ::= "throw NoViableAltException(this);" + +TestSetInlineHeader(s) ::= "<! Required but unused. !>" +TestSetInline(s) ::= << +<s.bitsets: {bits | <if (rest(rest(bits.tokens)))><bitsetBitfieldComparison(s, bits)><else><bitsetInlineComparison(s, bits)><endif>}; separator=" || "> +>> + +// Java language spec 15.19 - shift operators mask operands rather than overflow to 0... need range test +testShiftInRange(shiftAmount) ::= << +((<shiftAmount> & ~ 0x3fULL) == 0) +>> + +bitsetBitfieldComparison(s, bits) ::= << +(<testShiftInRange({<offsetShift(s.varName, bits.shift)>})> && + ((1ULL \<\< <offsetShift(s.varName, bits.shift)>) & <bits.calculated>) != 0) +>> + +isZero ::= [ + "0": true, + default: false +] + +offsetShift(shiftAmount, offset, prefix = false) ::= <% +<if (!isZero.(offset))>(<if (prefix)><parser.name>::TOKEN_<endif><shiftAmount> - <offset>)<else><if (prefix)><parser.name>::TOKEN_<endif><shiftAmount><endif> +%> + +bitsetInlineComparison(s, bits) ::= <% +<bits.tokens: {t | <s.varName> == <parser.name>::TOKEN_<t.name>}; separator = "\n\n|| "> +%> + +cases(tokens) ::= << +<tokens: {t | case <parser.name>::TOKEN_<t.name>:}; separator="\n"> +>> + +InvokeRuleHeader(r, argExprsChunks) ::= "InvokeRuleHeader" +InvokeRule(r, argExprsChunks) ::= << +setState(<r.stateNumber>); +<if(r.labels)><r.labels: {l | <labelref(l)> = }><endif><r.escapedName>(<if(r.ast.options.p)><r.ast.options.p><if(argExprsChunks)>,<endif><endif><argExprsChunks>); +>> + +MatchTokenHeader(m) ::= "<! Required but unused. !>" +MatchToken(m) ::= << +setState(<m.stateNumber>); +<if (m.labels)><m.labels: {l | <labelref(l)> = }><endif>match(<parser.name>::TOKEN_<m.escapedName>); +>> + +MatchSetHeader(m, expr, capture) ::= "<! Required but unused. !>" +MatchSet(m, expr, capture) ::= "<CommonSetStuff(m, expr, capture, false)>" + +MatchNotSetHeader(m, expr, capture) ::= "<! Required but unused. !>" +MatchNotSet(m, expr, capture) ::= "<CommonSetStuff(m, expr, capture, true)>" + +CommonSetStuff(m, expr, capture, invert) ::= << +setState(<m.stateNumber>); +<if (m.labels)><m.labels: {l | <labelref(l)> = }>_input->LT(1);<endif> +<capture> +if (<if (invert)><m.varName> == 0 || <m.varName> == Token::EOF || <else>!<endif>(<expr>)) { + <if (m.labels)><m.labels: {l | <labelref(l)> = }><endif>_errHandler->recoverInline(this); +} +else { + _errHandler->reportMatch(this); + consume(); +} +>> + +WildcardHeader(w) ::= "<! Required but unused. !>" +Wildcard(w) ::= << +setState(<w.stateNumber>); +<if (w.labels)><w.labels: {l | <labelref(l)> = }><endif>matchWildcard(); +>> + +// ACTION STUFF + +ActionHeader(a, foo, chunks) ::= "<chunks>" +Action(a, foo, chunks) ::= "<chunks>" + +ArgAction(a, chunks) ::= "ArgAction(a, chunks) <chunks>" + +SemPredHeader(p, chunks, failChunks) ::= "<! Required but unused. !>" +SemPred(p, chunks, failChunks) ::= << +setState(<p.stateNumber>); + +if (!(<chunks>)) throw FailedPredicateException(this, <p.predicate><if (failChunks)>, <failChunks><elseif (p.msg)>, <p.msg><endif>); +>> + +ExceptionClauseHeader(e, catchArg, catchAction) ::= "<! Required but unused. !>" +ExceptionClause(e, catchArg, catchAction) ::= << +catch (<catchArg>) { + <catchAction> +} +>> + +// Lexer actions are not associated with model objects. + +LexerSkipCommand() ::= "skip();" +LexerMoreCommand() ::= "more();" +LexerPopModeCommand() ::= "popMode();" + +LexerTypeCommand(arg, grammar) ::= "type = <grammar.name>::<arg>;" +LexerChannelCommand(arg, grammar) ::= "channel = <arg>;" +LexerModeCommand(arg, grammar) ::= "mode = <grammar.name>Mode::<arg>;" +LexerPushModeCommand(arg, grammar) ::= "pushMode(<grammar.name>Mode::<arg>);" + +ActionTextHeader(t) ::= "<t.text>" +ActionText(t) ::= "<t.text>" + +ActionTemplateHeader(t) ::= "<! Required but unused. !>" +ActionTemplate(t) ::= "<t.st>" + +ArgRefHeader(t) ::= "<! Required but unused. !>" +ArgRef(a) ::= "_localctx-><a.escapedName>" + +LocalRefHeader(t) ::= "<! Required but unused. !>" +LocalRef(a) ::= "_localctx-><a.escapedName>" + +RetValueRefHeader(t) ::= "<! Required but unused. !>" +RetValueRef(a) ::= "_localctx-><a.escapedName>" + +QRetValueRefHeader(t) ::= "<! Required but unused. !>" +QRetValueRef(a) ::= "<ctx(a)>-><a.dict>-><a.escapedName>" +/** How to translate $tokenLabel */ + +TokenRefHeader(t) ::= "<! Required but unused. !>" +TokenRef(t) ::= "<ctx(t)>-><t.escapedName>" + +LabelRefHeader(t) ::= "<! Required but unused. !>" +LabelRef(t) ::= "<ctx(t)>-><t.escapedName>" + +ListLabelRefHeader(t) ::= "<! Required but unused. !>" +ListLabelRef(t) ::= "<ctx(t)>-><ListLabelName(t.escapedName)>" + +SetAttrHeader(t) ::= "<! Required but unused. !>" +SetAttr(s,rhsChunks) ::= "<ctx(s)>-><s.escapedName> = <rhsChunks>;" + +InputSymbolType() ::= "<file.InputSymbolType; null = {Token}> *" + +TokenPropertyRef_textHeader(t) ::= "<! Required but unused. !>" +TokenPropertyRef_text(t) ::= <<(<ctx(t)>-><t.label> != nullptr ? <ctx(t)>-><t.label>->getText() : "")>> + +TokenPropertyRef_typeHeader(t) ::= "<! Required but unused. !>" +TokenPropertyRef_type(t) ::= "(<ctx(t)>-><t.label> != nullptr ? <ctx(t)>-><t.label>->getType() : 0)" + +TokenPropertyRef_lineHeader(t) ::= "<! Required but unused. !>" +TokenPropertyRef_line(t) ::= "(<ctx(t)>-><t.label> != nullptr ? <ctx(t)>-><t.label>->getLine() : 0)" + +TokenPropertyRef_posHeader(t) ::= "<! Required but unused. !>" +TokenPropertyRef_pos(t) ::= "(<ctx(t)>-><t.label> != nullptr ? <ctx(t)>-><t.label>->getCharPositionInLine() : 0)" + +TokenPropertyRef_channelHeader(t) ::= "<! Required but unused. !>" +TokenPropertyRef_channel(t) ::= "(<ctx(t)>-><t.label> != nullptr ? <ctx(t)>-><t.label>->getChannel() : 0)" + +TokenPropertyRef_indexHeader(t) ::= "<! Required but unused. !>" +TokenPropertyRef_index(t) ::= "(<ctx(t)>-><t.label> != nullptr ? <ctx(t)>-><t.label>->getTokenIndex() : 0)" + +TokenPropertyRef_intHeader(t) ::= "<! Required but unused. !>" +TokenPropertyRef_int(t) ::= "(<ctx(t)>-><t.label> != nullptr ? std::stoi(<ctx(t)>-><t.label>->getText()) : 0)" + +RulePropertyRef_startHeader(r) ::= "<! Required but unused. !>" +RulePropertyRef_start(r) ::= "(<ctx(r)>-><r.label> != nullptr ? (<ctx(r)>-><r.label>->start) : nullptr)" + +RulePropertyRef_stopHeader(r) ::= "<! Required but unused. !>" +RulePropertyRef_stop(r) ::= "(<ctx(r)>-><r.label> != nullptr ? (<ctx(r)>-><r.label>->stop) : nullptr)" + +RulePropertyRef_textHeader(r) ::= "<! Required but unused. !>" +RulePropertyRef_text(r) ::= "(<ctx(r)>-><r.label> != nullptr ? _input->getText(<ctx(r)>-><r.label>->start, <ctx(r)>-><r.label>->stop) : nullptr)" + +RulePropertyRef_ctxHeader(r) ::= "<! Required but unused. !>" +RulePropertyRef_ctx(r) ::= "<ctx(r)>-><r.label>" + +ThisRulePropertyRef_start(r) ::= "ThisRulePropertyRef_start(r) _localctx->start" +ThisRulePropertyRef_stop(r) ::= "ThisRulePropertyRef_stop(r) _localctx->stop" + +ThisRulePropertyRef_textHeader(r) ::= "<! Required but unused. !>" +ThisRulePropertyRef_text(r) ::= "_input->getText(_localctx->start, _input->LT(-1))" + +ThisRulePropertyRef_ctxHeader(r) ::= "<! Required but unused. !>" +ThisRulePropertyRef_ctx(r) ::= "_localctx" + +ThisRulePropertyRef_parserHeader(r) ::= "<! Required but unused. !>" +ThisRulePropertyRef_parser(r) ::= "this" + +NonLocalAttrRef(s) ::= "NonLocalAttrRef(s) ((<s.ruleName; format=\"cap\">Context)getInvokingContext(<s.ruleIndex>)).<s.escapedName>" +SetNonLocalAttr(s, rhsChunks) ::= + "SetNonLocalAttr(s, rhsChunks) ((<s.ruleName; format=\"cap\">Context)getInvokingContext(<s.ruleIndex>)).<s.escapedName> = <rhsChunks>;" + +AddToLabelListHeader(a) ::= "<! Required but unused. !>" +AddToLabelList(a) ::= << +<ctx(a.label)>-><a.listName>.push_back(<labelref(a.label)>); +>> + +TokenLabelType() ::= "<file.TokenLabelType; null = {Token}> *" + +TokenDeclHeader(t) ::= "antlr4::<TokenLabelType()><t.escapedName> = nullptr" +TokenDecl(t) ::= "<! Variable Declaration !>" + +TokenTypeDeclHeader(t) ::= "<! Local Variable !>" +TokenTypeDecl(t) ::= "size_t <t.escapedName> = 0;" + +TokenListDeclHeader(t) ::= "std::vector\<antlr4::Token *> <t.escapedName>" +TokenListDecl(t) ::= "<! Variable Declaration !>" + +RuleContextDeclHeader(r) ::= "<parser.name>::<r.ctxName> *<r.escapedName> = nullptr" +RuleContextDecl(r) ::= "<! Variable Declaration !>" + +RuleContextListDeclHeader(rdecl) ::= "std::vector\<<rdecl.ctxName> *> <rdecl.escapedName>" +RuleContextListDecl(rdecl) ::= "<! Variable Declaration !>" + +ContextTokenGetterDeclHeader(t) ::= "antlr4::tree::TerminalNode *TOKEN_<t.escapedName>();" +ContextTokenGetterDecl(t) ::= << +tree::TerminalNode* <parser.name>::<t.ctx.name>::TOKEN_<t.escapedName>() { + return getToken(<parser.name>::TOKEN_<t.escapedName>, 0); +} + +>> + +ContextTokenListGetterDeclHeader(t) ::= "std::vector\<antlr4::tree::TerminalNode *> TOKEN_<t.escapedName>();" +ContextTokenListGetterDecl(t) ::= << +std::vector\<tree::TerminalNode *> <parser.name>::<t.ctx.name>::TOKEN_<t.escapedName>() { + return getTokens(<parser.name>::TOKEN_<t.escapedName>); +} + +>> + +ContextTokenListIndexedGetterDeclHeader(t) ::= "antlr4::tree::TerminalNode* TOKEN_<t.escapedName>(size_t i);" +ContextTokenListIndexedGetterDecl(t) ::= << +tree::TerminalNode* <parser.name>::<t.ctx.name>::TOKEN_<t.escapedName>(size_t i) { + return getToken(<parser.name>::TOKEN_<t.escapedName>, i); +} + +>> + +ContextRuleGetterDeclHeader(r) ::= "<r.ctxName> *<r.escapedName>();" +ContextRuleGetterDecl(r) ::= << +<! Note: ctxName is the name of the context to return, while ctx is the owning context. !> +<parser.name>::<r.ctxName>* <parser.name>::<r.ctx.name>::<r.escapedName>() { + return getRuleContext\<<parser.name>::<r.ctxName>\>(0); +} + +>> + +ContextRuleListGetterDeclHeader(r) ::= "std::vector\<<r.ctxName> *> <r.escapedName>();" +ContextRuleListGetterDecl(r) ::= << +std::vector\<<parser.name>::<r.ctxName> *> <parser.name>::<r.ctx.name>::<r.escapedName>() { + return getRuleContexts\<<parser.name>::<r.ctxName>\>(); +} + +>> + +ContextRuleListIndexedGetterDeclHeader(r) ::= "<r.ctxName>* <r.escapedName>(size_t i);" +ContextRuleListIndexedGetterDecl(r) ::= << +<parser.name>::<r.ctxName>* <parser.name>::<r.ctx.name>::<r.escapedName>(size_t i) { + return getRuleContext\<<parser.name>::<r.ctxName>\>(i); +} + +>> + +LexerRuleContext() ::= "antlr4::RuleContext" + +// The rule context name is the rule followed by a suffix; e.g. r becomes rContext. +RuleContextNameSuffix() ::= "Context" + +ImplicitTokenLabel(tokenName) ::= << +<tokenName; format = "lower">Token +>> + +ImplicitRuleLabel(ruleName) ::= "<ruleName>Context" +ImplicitSetLabel(id) ::= "_tset<id>" +ListLabelName(label) ::= "<label>" + +CaptureNextToken(d) ::= "CaptureNextToken(d) <d.varName> = _input->LT(1);" + +CaptureNextTokenTypeHeader(d) ::= "<! Required but unused. !>" +CaptureNextTokenType(d) ::= "<d.varName> = _input->LA(1);" + +ListenerDispatchMethodHeader(method) ::= << +virtual void <if (method.isEnter)>enter<else>exit<endif>Rule(antlr4::tree::ParseTreeListener *listener) override; +>> +ListenerDispatchMethod(method) ::= << +void <parser.name>::<struct.escapedName>::<if (method.isEnter)>enter<else>exit<endif>Rule(tree::ParseTreeListener *listener) { + auto parserListener = dynamic_cast\<<parser.grammarName>Listener *>(listener); + if (parserListener != nullptr) + parserListener-><if(method.isEnter)>enter<else>exit<endif><struct.derivedFromName; format="cap">(this); +} +>> + +VisitorDispatchMethodHeader(method) ::= << + +virtual std::any accept(antlr4::tree::ParseTreeVisitor *visitor) override; +>> +VisitorDispatchMethod(method) ::= << + +std::any <parser.name>::<struct.escapedName>::accept(tree::ParseTreeVisitor *visitor) { + if (auto parserVisitor = dynamic_cast\<<parser.grammarName>Visitor*>(visitor)) + return parserVisitor->visit<struct.derivedFromName; format="cap">(this); + else + return visitor->visitChildren(this); +} +>> + +AttributeDeclHeader(d) ::= "<d.type> <d.escapedName><if(d.initValue)> = <d.initValue><endif>" +AttributeDecl(d) ::= "<d.type> <d.escapedName>" + +/** If we don't know location of label def x, use this template */ +labelref(x) ::= "<if (!x.isLocal)>antlrcpp::downCast\<<x.ctx.name> *>(_localctx)-><endif><x.escapedName>" + +/** For any action chunk, what is correctly-typed context struct ptr? */ +ctx(actionChunk) ::= "antlrcpp::downCast\<<actionChunk.ctx.name> *>(_localctx)" + +// used for left-recursive rules +recRuleAltPredicate(ruleName,opPrec) ::= "precpred(_ctx, <opPrec>)" +recRuleSetReturnAction(src,name) ::= "recRuleSetReturnAction(src,name) $<name>=$<src>.<name>;" +recRuleSetStopToken() ::= "_ctx->stop = _input->LT(-1);" + +recRuleAltStartAction(ruleName, ctxName, label, isListLabel) ::= << +_localctx = _tracker.createInstance\<<ctxName>Context>(parentContext, parentState); +<if(label)> +<if(isListLabel)> +_localctx-><label>.push_back(previousContext); +<else> +_localctx-><label> = previousContext; +<endif> +<endif> +pushNewRecursionContext(_localctx, startState, Rule<ruleName; format = "cap">); +>> + +// Separate context variable to avoid frequent pointer type casts. +recRuleLabeledAltStartAction(ruleName, currentAltLabel, label, isListLabel) ::= << +auto newContext = _tracker.createInstance\<<currentAltLabel; format = "cap">Context>(_tracker.createInstance\<<ruleName; format="cap">Context>(parentContext, parentState)); +_localctx = newContext; +<if(label)> +<if(isListLabel)> +newContext-><label>.push_back(previousContext); +<else> +newContext-><label> = previousContext; +<endif> +<endif> +pushNewRecursionContext(newContext, startState, Rule<ruleName; format = "cap">); +>> + +recRuleReplaceContext(ctxName) ::= << +_localctx = _tracker.createInstance\<<ctxName>Context>(_localctx); +_ctx = _localctx; +previousContext = _localctx; +>> + +recRuleSetPrevCtx() ::= << +if (!_parseListeners.empty()) + triggerExitRuleEvent(); +previousContext = _localctx; +>> + +/** Using a type to init value map, try to init a type; if not in table + * must be an object, default value is "null". + */ +initValue(typeName) ::= << +<cppTypeInitMap.(typeName)> +>>
\ No newline at end of file diff --git a/yql/essentials/parser/antlr_ast/org/antlr/v4/tool/templates/codegen/Cpp/Files.stg b/yql/essentials/parser/antlr_ast/org/antlr/v4/tool/templates/codegen/Cpp/Files.stg new file mode 100644 index 0000000000..c2a36eb758 --- /dev/null +++ b/yql/essentials/parser/antlr_ast/org/antlr/v4/tool/templates/codegen/Cpp/Files.stg @@ -0,0 +1,344 @@ +/* + * [The "BSD license"] + * Copyright (c) 2016, Mike Lischke + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +codeFileExtension() ::= ".cpp" +headerFileExtension() ::= ".h" + +fileHeader(grammarFileName, ANTLRVersion, header) ::= << +<header> + +// Generated from <grammarFileName> by ANTLR <ANTLRVersion> +>> + +LexerFileHeader(file, lexer, namedActions) ::= << +<fileHeader(file.grammarFileName, file.ANTLRVersion, namedActions.header)> + +#pragma once + +<namedActions.preinclude> + +#include "antlr4-runtime.h" + +<namedActions.postinclude> + +<if(file.genPackage)>namespace <file.genPackage> {<endif> + +<lexer> + +<if (file.genPackage)> +} // namespace <file.genPackage> +<endif> +>> + +LexerFile(file, lexer, namedActions) ::= << +<fileHeader(file.grammarFileName, file.ANTLRVersion, namedActions.header)> + +<namedActions.preinclude> + +#include "<file.lexer.name>.h" + +<namedActions.postinclude> + +using namespace antlr4; + +<if (file.genPackage)>using namespace <file.genPackage>;<endif> + +<lexer> + +>> + +ParserFileHeader(file, parser, namedActions, contextSuperClass) ::= << +<fileHeader(file.grammarFileName, file.ANTLRVersion, namedActions.header)> + +#pragma once + +<namedActions.preinclude> + +#include "antlr4-runtime.h" + +<namedActions.postinclude> + +<if (file.genPackage)>namespace <file.genPackage> {<endif> + +<parser> + +<if (file.genPackage)> +} // namespace <file.genPackage> +<endif> +>> + +ParserFile(file, parser, namedActions, contextSuperClass) ::= << +<fileHeader(file.grammarFileName, file.ANTLRVersion, namedActions.header)> + +<namedActions.preinclude> + +<if (file.genListener)>#include "<file.grammarName>Listener.h"<endif> +<if (file.genVisitor)>#include "<file.grammarName>Visitor.h"<endif> + +#include "<file.parser.name>.h" + +<namedActions.postinclude> + +using namespace antlrcpp; +<if (file.genPackage)>using namespace <file.genPackage>;<endif> +<parser> + +>> + +BaseListenerFileHeader(file, header, namedActions) ::= << +<fileHeader(file.grammarFileName, file.ANTLRVersion, header)> + +#pragma once + +<namedActions.baselistenerpreinclude> + +#include "antlr4-runtime.h" +#include "<file.grammarName>Listener.h" + +<namedActions.baselistenerpostinclude> + +<if(file.genPackage)> +namespace <file.genPackage> { +<endif> + +/** + * This class provides an empty implementation of <file.grammarName>Listener, + * which can be extended to create a listener which only needs to handle a subset + * of the available methods. + */ +class <file.exportMacro> <file.grammarName>BaseListener : public <file.grammarName>Listener { +public: +<namedActions.baselistenerdeclarations> + +<file.listenerNames: {lname | + virtual void enter<lname; format="cap">(<file.parserName>::<lname; format = "cap">Context * /*ctx*/) override { \} + virtual void exit<lname; format="cap">(<file.parserName>::<lname; format = "cap">Context * /*ctx*/) override { \} +}; separator="\n"> + + virtual void enterEveryRule(antlr4::ParserRuleContext * /*ctx*/) override { } + virtual void exitEveryRule(antlr4::ParserRuleContext * /*ctx*/) override { } + virtual void visitTerminal(antlr4::tree::TerminalNode * /*node*/) override { } + virtual void visitErrorNode(antlr4::tree::ErrorNode * /*node*/) override { } + +<if (namedActions.baselistenermembers)> +private: +<namedActions.baselistenermembers> +<endif> +}; + +<if (file.genPackage)> +} // namespace <file.genPackage> +<endif> +>> + +BaseListenerFile(file, header, namedActions) ::= << +<fileHeader(file.grammarFileName, file.ANTLRVersion, header)> + +<namedActions.baselistenerpreinclude> + +#include "<file.grammarName>BaseListener.h" + +<namedActions.baselistenerpostinclude> + +<if(file.genPackage)> +using namespace <file.genPackage>; +<endif> + +<namedActions.baselistenerdefinitions> +>> + +ListenerFileHeader(file, header, namedActions) ::= << +<fileHeader(file.grammarFileName, file.ANTLRVersion, header)> + +#pragma once + +<namedActions.listenerpreinclude> + +#include "antlr4-runtime.h" +#include "<file.parserName>.h" + +<namedActions.listenerpostinclude> + +<if(file.genPackage)> +namespace <file.genPackage> { +<endif> + +/** + * This interface defines an abstract listener for a parse tree produced by <file.parserName>. + */ +class <file.exportMacro> <file.grammarName>Listener : public antlr4::tree::ParseTreeListener { +public: +<namedActions.listenerdeclarations> + +<file.listenerNames: {lname | + virtual void enter<lname; format = "cap">(<file.parserName>::<lname; format ="cap">Context *ctx) = 0; + virtual void exit<lname; format = "cap">(<file.parserName>::<lname; format="cap">Context *ctx) = 0; +}; separator = "\n"> + +<if (namedActions.listenermembers)> +private: +<namedActions.listenermembers> +<endif> +}; + +<if(file.genPackage)> +} // namespace <file.genPackage> +<endif> +>> + +ListenerFile(file, header, namedActions) ::= << +<fileHeader(file.grammarFileName, file.ANTLRVersion, header)> + +<namedActions.listenerpreinclude> + +#include "<file.grammarName>Listener.h" + +<namedActions.listenerpostinclude> + +<if(file.genPackage)> +using namespace <file.genPackage>; +<endif> + +<namedActions.listenerdefinitions> +>> + +BaseVisitorFileHeader(file, header, namedActions) ::= << +<fileHeader(file.grammarFileName, file.ANTLRVersion, header)> + +#pragma once + +<namedActions.basevisitorpreinclude> + +#include "antlr4-runtime.h" +#include "<file.grammarName>Visitor.h" + +<namedActions.basevisitorpostinclude> + +<if(file.genPackage)> +namespace <file.genPackage> { +<endif> + +/** + * This class provides an empty implementation of <file.grammarName>Visitor, which can be + * extended to create a visitor which only needs to handle a subset of the available methods. + */ +class <file.exportMacro> <file.grammarName>BaseVisitor : public <file.grammarName>Visitor { +public: +<namedActions.basevisitordeclarations> + +<file.visitorNames: { lname | + virtual std::any visit<lname; format = "cap">(<file.parserName>::<lname; format = "cap">Context *ctx) override { + return visitChildren(ctx); + \} +}; separator="\n"> + +<if (namedActions.basevisitormembers)> +private: +<namedActions.basevisitormembers> +<endif> +}; + +<if(file.genPackage)> +} // namespace <file.genPackage> +<endif> +>> + +BaseVisitorFile(file, header, namedActions) ::= << +<fileHeader(file.grammarFileName, file.ANTLRVersion, header)> + +<namedActions.basevisitorpreinclude> + +#include "<file.grammarName>BaseVisitor.h" + +<namedActions.basevisitorpostinclude> + +<if(file.genPackage)> +using namespace <file.genPackage>; +<endif> + +<namedActions.basevisitordefinitions> + +>> + +VisitorFileHeader(file, header, namedActions) ::= << +<fileHeader(file.grammarFileName, file.ANTLRVersion, header)> + +#pragma once + +<namedActions.visitorpreinclude> + +#include "antlr4-runtime.h" +#include "<file.parserName>.h" + +<namedActions.visitorpostinclude> + +<if(file.genPackage)>namespace <file.genPackage> {<endif> + +/** + * This class defines an abstract visitor for a parse tree + * produced by <file.parserName>. + */ +class <file.exportMacro> <file.grammarName>Visitor : public antlr4::tree::AbstractParseTreeVisitor { +public: + <namedActions.visitordeclarations> + + /** + * Visit parse trees produced by <file.parserName>. + */ + <file.visitorNames: {lname | + virtual std::any visit<lname; format = "cap">(<file.parserName>::<lname; format = "cap">Context *context) = 0; + }; separator="\n"> + +<if (namedActions.visitormembers)> +private: +<namedActions.visitormembers> +<endif> +}; + +<if(file.genPackage)> +} // namespace <file.genPackage> +<endif> +>> + +VisitorFile(file, header, namedActions) ::= << +<fileHeader(file.grammarFileName, file.ANTLRVersion, header)> + +<namedActions.visitorpreinclude> + +#include "<file.grammarName>Visitor.h" + +<namedActions.visitorpostinclude> + +<if(file.genPackage)> +using namespace <file.genPackage>; +<endif> + +<namedActions.visitordefinitions> + +>>
\ No newline at end of file diff --git a/yql/essentials/parser/antlr_ast/ya.make b/yql/essentials/parser/antlr_ast/ya.make new file mode 100644 index 0000000000..d3b33a2dfa --- /dev/null +++ b/yql/essentials/parser/antlr_ast/ya.make @@ -0,0 +1,4 @@ +RECURSE( + gen +) + diff --git a/yql/essentials/parser/ya.make b/yql/essentials/parser/ya.make index 7f1b0fe700..c1cacaa6a8 100644 --- a/yql/essentials/parser/ya.make +++ b/yql/essentials/parser/ya.make @@ -1,4 +1,5 @@ RECURSE( + antlr_ast lexer_common pg_catalog pg_wrapper diff --git a/yql/essentials/public/fastcheck/translator.cpp b/yql/essentials/public/fastcheck/translator.cpp index eec337be86..8350dd9b4f 100644 --- a/yql/essentials/public/fastcheck/translator.cpp +++ b/yql/essentials/public/fastcheck/translator.cpp @@ -61,6 +61,17 @@ private: settings.Antlr4Parser = true; settings.AnsiLexer = request.IsAnsiLexer; settings.SyntaxVersion = request.SyntaxVersion; + settings.Flags.insert({ + "AnsiOrderByLimitInUnionAll", + "DisableCoalesceJoinKeysOnQualifiedAll", + "AnsiRankForNullableKeys", + "DisableUnorderedSubqueries", + "DisableAnsiOptionalAs", + "FlexibleTypes", + "CompactNamedExprs", + "DistinctOverWindow" + }); + switch (request.Mode) { case EMode::Default: settings.AlwaysAllowExports = true; diff --git a/yql/essentials/public/purecalc/common/worker_factory.cpp b/yql/essentials/public/purecalc/common/worker_factory.cpp index 50b85ed05a..7597c7fdd2 100644 --- a/yql/essentials/public/purecalc/common/worker_factory.cpp +++ b/yql/essentials/public/purecalc/common/worker_factory.cpp @@ -7,13 +7,6 @@ #include <yql/essentials/sql/sql.h> #include <yql/essentials/sql/v1/sql.h> -//FIXME { -#include <yql/essentials/sql/v1/lexer/antlr3/lexer.h> -#include <yql/essentials/sql/v1/lexer/antlr3_ansi/lexer.h> -#include <yql/essentials/sql/v1/proto_parser/antlr3/proto_parser.h> -#include <yql/essentials/sql/v1/proto_parser/antlr3_ansi/proto_parser.h> -//} - #include <yql/essentials/sql/v1/lexer/antlr4/lexer.h> #include <yql/essentials/sql/v1/lexer/antlr4_ansi/lexer.h> #include <yql/essentials/sql/v1/proto_parser/antlr4/proto_parser.h> @@ -152,6 +145,7 @@ TExprNode::TPtr TWorkerFactory<TBase>::Compile( bool useAntlr4, EProcessorMode processorMode ) { + Y_ENSURE(useAntlr4, "Antlr3 support is dropped"); if (mode == ETranslationMode::PG && processorMode != EProcessorMode::PullList) { ythrow TCompileError("", "") << "only PullList mode is compatible to PostgreSQL syntax"; } @@ -221,13 +215,9 @@ TExprNode::TPtr TWorkerFactory<TBase>::Compile( } NSQLTranslationV1::TLexers lexers; - lexers.Antlr3 = NSQLTranslationV1::MakeAntlr3LexerFactory(); - lexers.Antlr3Ansi = NSQLTranslationV1::MakeAntlr3AnsiLexerFactory(); lexers.Antlr4 = NSQLTranslationV1::MakeAntlr4LexerFactory(); lexers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiLexerFactory(); NSQLTranslationV1::TParsers parsers; - parsers.Antlr3 = NSQLTranslationV1::MakeAntlr3ParserFactory(); - parsers.Antlr3Ansi = NSQLTranslationV1::MakeAntlr3AnsiParserFactory(); parsers.Antlr4 = NSQLTranslationV1::MakeAntlr4ParserFactory(); parsers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiParserFactory(); diff --git a/yql/essentials/public/purecalc/common/ya.make.inc b/yql/essentials/public/purecalc/common/ya.make.inc index 4059a9e78d..0f3e3d9370 100644 --- a/yql/essentials/public/purecalc/common/ya.make.inc +++ b/yql/essentials/public/purecalc/common/ya.make.inc @@ -43,12 +43,6 @@ PEERDIR( yql/essentials/sql/v1/lexer/antlr4_ansi yql/essentials/sql/v1/proto_parser/antlr4 yql/essentials/sql/v1/proto_parser/antlr4_ansi - #{ FIXME - yql/essentials/sql/v1/lexer/antlr3 - yql/essentials/sql/v1/lexer/antlr3_ansi - yql/essentials/sql/v1/proto_parser/antlr3 - yql/essentials/sql/v1/proto_parser/antlr3_ansi - #} yql/essentials/parser/pg_wrapper/interface yql/essentials/providers/common/codec yql/essentials/providers/common/comp_nodes diff --git a/yql/essentials/sql/v1/SQLv1.g.in b/yql/essentials/sql/v1/SQLv1.g.in index 1d7dca5185..dbec682803 100644 --- a/yql/essentials/sql/v1/SQLv1.g.in +++ b/yql/essentials/sql/v1/SQLv1.g.in @@ -845,8 +845,9 @@ role_name: an_id_or_type | bind_parameter; user_option: authentication_option | login_option; authentication_option: password_option | hash_option; -password_option: ENCRYPTED? PASSWORD expr; -hash_option: HASH expr; +password_option: ENCRYPTED? PASSWORD password_value; +password_value: STRING_VALUE | NULL; +hash_option: HASH STRING_VALUE; login_option: LOGIN | NOLOGIN; grant_permissions_stmt: GRANT permission_name_target ON an_id_schema (COMMA an_id_schema)* TO role_name (COMMA role_name)* COMMA? (WITH GRANT OPTION)?; diff --git a/yql/essentials/sql/v1/SQLv1Antlr4.g.in b/yql/essentials/sql/v1/SQLv1Antlr4.g.in index 8aff2bb89a..fb92a68f9a 100644 --- a/yql/essentials/sql/v1/SQLv1Antlr4.g.in +++ b/yql/essentials/sql/v1/SQLv1Antlr4.g.in @@ -844,8 +844,9 @@ role_name: an_id_or_type | bind_parameter; user_option: authentication_option | login_option; authentication_option: password_option | hash_option; -password_option: ENCRYPTED? PASSWORD expr; -hash_option: HASH expr; +password_option: ENCRYPTED? PASSWORD password_value; +password_value: STRING_VALUE | NULL; +hash_option: HASH STRING_VALUE; login_option: LOGIN | NOLOGIN; grant_permissions_stmt: GRANT permission_name_target ON an_id_schema (COMMA an_id_schema)* TO role_name (COMMA role_name)* COMMA? (WITH GRANT OPTION)?; diff --git a/yql/essentials/sql/v1/complete/c3_engine.h b/yql/essentials/sql/v1/complete/c3_engine.h new file mode 100644 index 0000000000..8b729f880b --- /dev/null +++ b/yql/essentials/sql/v1/complete/c3_engine.h @@ -0,0 +1,116 @@ +#pragma once + +#include "sql_antlr4.h" +#include "string_util.h" + +#include <contrib/libs/antlr4_cpp_runtime/src/ANTLRInputStream.h> +#include <contrib/libs/antlr4_cpp_runtime/src/BufferedTokenStream.h> +#include <contrib/libs/antlr4_cpp_runtime/src/Vocabulary.h> +#include <contrib/libs/antlr4-c3/src/CodeCompletionCore.hpp> + +#include <util/generic/string.h> +#include <util/generic/vector.h> + +#include <vector> +#include <unordered_set> + +namespace NSQLComplete { + + // std::vector is used to prevent copying from c3 results + struct TSuggestedToken { + TTokenId Number; + std::vector<TRuleId> ParserCallStack; + }; + + class IC3Engine { + public: + using TPtr = THolder<IC3Engine>; + + // std::unordered_set is used to prevent copying into c3 core + struct TConfig { + std::unordered_set<TTokenId> IgnoredTokens; + std::unordered_set<TRuleId> PreferredRules; + }; + + virtual TVector<TSuggestedToken> Complete(TStringBuf queryPrefix) = 0; + virtual const antlr4::dfa::Vocabulary& GetVocabulary() const = 0; + virtual ~IC3Engine() = default; + }; + + template <class Lexer, class Parser> + struct TAntlrGrammar { + using TLexer = Lexer; + using TParser = Parser; + + TAntlrGrammar() = delete; + }; + + template <class G> + class TC3Engine: public IC3Engine { + public: + explicit TC3Engine(TConfig config) + : Chars() + , Lexer(&Chars) + , Tokens(&Lexer) + , Parser(&Tokens) + , CompletionCore(&Parser) + { + Lexer.removeErrorListeners(); + Parser.removeErrorListeners(); + + CompletionCore.ignoredTokens = std::move(config.IgnoredTokens); + CompletionCore.preferredRules = std::move(config.PreferredRules); + } + + TVector<TSuggestedToken> Complete(TStringBuf queryPrefix) override { + Assign(queryPrefix); + const auto caretTokenIndex = CaretTokenIndex(queryPrefix); + auto candidates = CompletionCore.collectCandidates(caretTokenIndex); + return Converted(std::move(candidates)); + } + + const antlr4::dfa::Vocabulary& GetVocabulary() const override { + return Lexer.getVocabulary(); + } + + private: + void Assign(TStringBuf queryPrefix) { + Chars.load(queryPrefix.Data(), queryPrefix.Size(), /* lenient = */ false); + Lexer.reset(); + Tokens.setTokenSource(&Lexer); + + Tokens.fill(); + } + + size_t CaretTokenIndex(TStringBuf queryPrefix) { + const auto tokensCount = Tokens.size(); + if (2 <= tokensCount && !LastWord(queryPrefix).Empty()) { + return tokensCount - 2; + } + return tokensCount - 1; + } + + static TVector<TSuggestedToken> Converted(c3::CandidatesCollection candidates) { + TVector<TSuggestedToken> converted; + for (const auto& [token, _] : candidates.tokens) { + std::vector<TRuleId> parserCallStack; + + if ( + auto rules = candidates.rules.find(token); + rules != std::end(candidates.rules)) { + parserCallStack = std::move(rules->second.ruleList); + } + + converted.emplace_back(token, std::move(parserCallStack)); + } + return converted; + } + + antlr4::ANTLRInputStream Chars; + G::TLexer Lexer; + antlr4::BufferedTokenStream Tokens; + G::TParser Parser; + c3::CodeCompletionCore CompletionCore; + }; + +} // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/sql_antlr4.cpp b/yql/essentials/sql/v1/complete/sql_antlr4.cpp new file mode 100644 index 0000000000..33c847f3e2 --- /dev/null +++ b/yql/essentials/sql/v1/complete/sql_antlr4.cpp @@ -0,0 +1,116 @@ +#include "sql_antlr4.h" + +#include <yql/essentials/sql/v1/format/sql_format.h> + +#include <yql/essentials/parser/antlr_ast/gen/v1_antlr4/SQLv1Antlr4Lexer.h> +#include <yql/essentials/parser/antlr_ast/gen/v1_antlr4/SQLv1Antlr4Parser.h> +#include <yql/essentials/parser/antlr_ast/gen/v1_ansi_antlr4/SQLv1Antlr4Lexer.h> +#include <yql/essentials/parser/antlr_ast/gen/v1_ansi_antlr4/SQLv1Antlr4Parser.h> + +#define RULE_(mode, name) NALA##mode##Antlr4::SQLv1Antlr4Parser::Rule##name + +#define RULE(name) RULE_(Default, name) + +#define STATIC_ASSERT_RULE_ID_MODE_INDEPENDENT(name) \ + static_assert(RULE_(Default, name) == RULE_(Ansi, name)) + +namespace NSQLComplete { + + class TSqlGrammar: public ISqlGrammar { + public: + TSqlGrammar(bool isAnsiLexer) + : Vocabulary(GetVocabulary(isAnsiLexer)) + , AllTokens(ComputeAllTokens()) + , KeywordTokens(ComputeKeywordTokens()) + { + } + + const antlr4::dfa::Vocabulary& GetVocabulary() const override { + return *Vocabulary; + } + + const std::unordered_set<TTokenId>& GetAllTokens() const override { + return AllTokens; + } + + const std::unordered_set<TTokenId>& GetKeywordTokens() const override { + return KeywordTokens; + } + + const TVector<TRuleId>& GetKeywordRules() const override { + static const TVector<TRuleId> KeywordRules = { + RULE(Keyword), + RULE(Keyword_expr_uncompat), + RULE(Keyword_table_uncompat), + RULE(Keyword_select_uncompat), + RULE(Keyword_alter_uncompat), + RULE(Keyword_in_uncompat), + RULE(Keyword_window_uncompat), + RULE(Keyword_hint_uncompat), + RULE(Keyword_as_compat), + RULE(Keyword_compat), + }; + + STATIC_ASSERT_RULE_ID_MODE_INDEPENDENT(Keyword); + STATIC_ASSERT_RULE_ID_MODE_INDEPENDENT(Keyword_expr_uncompat); + STATIC_ASSERT_RULE_ID_MODE_INDEPENDENT(Keyword_table_uncompat); + STATIC_ASSERT_RULE_ID_MODE_INDEPENDENT(Keyword_select_uncompat); + STATIC_ASSERT_RULE_ID_MODE_INDEPENDENT(Keyword_alter_uncompat); + STATIC_ASSERT_RULE_ID_MODE_INDEPENDENT(Keyword_in_uncompat); + STATIC_ASSERT_RULE_ID_MODE_INDEPENDENT(Keyword_window_uncompat); + STATIC_ASSERT_RULE_ID_MODE_INDEPENDENT(Keyword_hint_uncompat); + STATIC_ASSERT_RULE_ID_MODE_INDEPENDENT(Keyword_as_compat); + STATIC_ASSERT_RULE_ID_MODE_INDEPENDENT(Keyword_compat); + + return KeywordRules; + } + + private: + static const antlr4::dfa::Vocabulary* GetVocabulary(bool isAnsiLexer) { + if (isAnsiLexer) { // Taking a reference is okay as vocabulary storage is static + return &NALAAnsiAntlr4::SQLv1Antlr4Parser(nullptr).getVocabulary(); + } + return &NALADefaultAntlr4::SQLv1Antlr4Parser(nullptr).getVocabulary(); + } + + std::unordered_set<TTokenId> ComputeAllTokens() { + const auto& vocabulary = GetVocabulary(); + + std::unordered_set<TTokenId> allTokens; + + for (size_t type = 1; type <= vocabulary.getMaxTokenType(); ++type) { + allTokens.emplace(type); + } + + return allTokens; + } + + std::unordered_set<TTokenId> ComputeKeywordTokens() { + const auto& vocabulary = GetVocabulary(); + const auto keywords = NSQLFormat::GetKeywords(); + + auto keywordTokens = GetAllTokens(); + std::erase_if(keywordTokens, [&](TTokenId token) { + return !keywords.contains(vocabulary.getSymbolicName(token)); + }); + keywordTokens.erase(TOKEN_EOF); + + return keywordTokens; + } + + const antlr4::dfa::Vocabulary* Vocabulary; + const std::unordered_set<TTokenId> AllTokens; + const std::unordered_set<TTokenId> KeywordTokens; + }; + + const ISqlGrammar& GetSqlGrammar(bool isAnsiLexer) { + const static TSqlGrammar DefaultSqlGrammar(/* isAnsiLexer = */ false); + const static TSqlGrammar AnsiSqlGrammar(/* isAnsiLexer = */ true); + + if (isAnsiLexer) { + return AnsiSqlGrammar; + } + return DefaultSqlGrammar; + } + +} // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/sql_antlr4.h b/yql/essentials/sql/v1/complete/sql_antlr4.h new file mode 100644 index 0000000000..3d3c4c024a --- /dev/null +++ b/yql/essentials/sql/v1/complete/sql_antlr4.h @@ -0,0 +1,28 @@ +#pragma once + +#include "sql_syntax.h" + +#include <contrib/libs/antlr4_cpp_runtime/src/Token.h> +#include <contrib/libs/antlr4_cpp_runtime/src/Vocabulary.h> + +#include <unordered_set> + +namespace NSQLComplete { + + using TTokenId = size_t; + using TRuleId = size_t; + + constexpr TTokenId TOKEN_EOF = antlr4::Token::EOF; + + class ISqlGrammar { + public: + virtual const antlr4::dfa::Vocabulary& GetVocabulary() const = 0; + virtual const std::unordered_set<TTokenId>& GetAllTokens() const = 0; + virtual const std::unordered_set<TTokenId>& GetKeywordTokens() const = 0; + virtual const TVector<TRuleId>& GetKeywordRules() const = 0; + virtual ~ISqlGrammar() = default; + }; + + const ISqlGrammar& GetSqlGrammar(bool isAnsiLexer); + +} // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/sql_complete.cpp b/yql/essentials/sql/v1/complete/sql_complete.cpp new file mode 100644 index 0000000000..2a16a250e5 --- /dev/null +++ b/yql/essentials/sql/v1/complete/sql_complete.cpp @@ -0,0 +1,89 @@ +#include "sql_complete.h" + +#include "sql_context.h" +#include "string_util.h" + +#include <util/generic/algorithm.h> +#include <util/charset/utf8.h> + +namespace NSQLComplete { + + class TSqlCompletionEngine: public ISqlCompletionEngine { + public: + TSqlCompletionEngine() + : ContextInference(MakeSqlContextInference()) + { + } + + TCompletion Complete(TCompletionInput input) { + auto prefix = input.Text.Head(input.CursorPosition); + auto completedToken = GetCompletedToken(prefix); + + auto context = ContextInference->Analyze(input); + + TVector<TCandidate> candidates; + EnrichWithKeywords(candidates, context.Keywords); + + FilterByContent(candidates, completedToken.Content); + + RankingSort(candidates); + + return { + .CompletedToken = std::move(completedToken), + .Candidates = std::move(candidates), + }; + } + + private: + TCompletedToken GetCompletedToken(TStringBuf prefix) { + return { + .Content = LastWord(prefix), + .SourcePosition = LastWordIndex(prefix), + }; + } + + void EnrichWithKeywords(TVector<TCandidate>& candidates, TVector<TString> keywords) { + for (auto keyword : keywords) { + candidates.push_back({ + .Kind = ECandidateKind::Keyword, + .Content = std::move(keyword), + }); + } + } + + void FilterByContent(TVector<TCandidate>& candidates, TStringBuf prefix) { + const auto lowerPrefix = ToLowerUTF8(prefix); + auto removed = std::ranges::remove_if(candidates, [&](const auto& candidate) { + return !ToLowerUTF8(candidate.Content).StartsWith(lowerPrefix); + }); + candidates.erase(std::begin(removed), std::end(removed)); + } + + void RankingSort(TVector<TCandidate>& candidates) { + Sort(candidates, [](const TCandidate& lhs, const TCandidate& rhs) { + return std::tie(lhs.Kind, lhs.Content) < std::tie(rhs.Kind, rhs.Content); + }); + } + + ISqlContextInference::TPtr ContextInference; + }; + + ISqlCompletionEngine::TPtr MakeSqlCompletionEngine() { + return ISqlCompletionEngine::TPtr(new TSqlCompletionEngine()); + } + +} // namespace NSQLComplete + +template <> +void Out<NSQLComplete::ECandidateKind>(IOutputStream& out, NSQLComplete::ECandidateKind kind) { + switch (kind) { + case NSQLComplete::ECandidateKind::Keyword: + out << "Keyword"; + break; + } +} + +template <> +void Out<NSQLComplete::TCandidate>(IOutputStream& out, const NSQLComplete::TCandidate& candidate) { + out << "(" << candidate.Kind << ": " << candidate.Content << ")"; +} diff --git a/yql/essentials/sql/v1/complete/sql_complete.h b/yql/essentials/sql/v1/complete/sql_complete.h new file mode 100644 index 0000000000..99e74cce7a --- /dev/null +++ b/yql/essentials/sql/v1/complete/sql_complete.h @@ -0,0 +1,44 @@ +#pragma once + +#include <util/generic/string.h> +#include <util/generic/vector.h> + +namespace NSQLComplete { + + struct TCompletionInput { + TStringBuf Text; + size_t CursorPosition = Text.length(); + }; + + struct TCompletedToken { + TStringBuf Content; + size_t SourcePosition; + }; + + enum class ECandidateKind { + Keyword, + }; + + struct TCandidate { + ECandidateKind Kind; + TString Content; + + friend bool operator==(const TCandidate& lhs, const TCandidate& rhs) = default; + }; + + struct TCompletion { + TCompletedToken CompletedToken; + TVector<TCandidate> Candidates; + }; + + class ISqlCompletionEngine { + public: + using TPtr = THolder<ISqlCompletionEngine>; + + virtual TCompletion Complete(TCompletionInput input) = 0; + virtual ~ISqlCompletionEngine() = default; + }; + + ISqlCompletionEngine::TPtr MakeSqlCompletionEngine(); + +} // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/sql_complete_ut.cpp b/yql/essentials/sql/v1/complete/sql_complete_ut.cpp new file mode 100644 index 0000000000..e0a012f9f6 --- /dev/null +++ b/yql/essentials/sql/v1/complete/sql_complete_ut.cpp @@ -0,0 +1,323 @@ +#include "sql_complete.h" + +#include <library/cpp/testing/unittest/registar.h> + +using namespace NSQLComplete; + +Y_UNIT_TEST_SUITE(SqlCompleteTests) { + using ECandidateKind::Keyword; + + TVector<TCandidate> Complete(ISqlCompletionEngine::TPtr& engine, TStringBuf prefix) { + return engine->Complete({prefix}).Candidates; + } + + Y_UNIT_TEST(Beginning) { + TVector<TCandidate> expected = { + {Keyword, "ALTER"}, + {Keyword, "ANALYZE"}, + {Keyword, "BACKUP"}, + {Keyword, "BATCH"}, + {Keyword, "COMMIT"}, + {Keyword, "CREATE"}, + {Keyword, "DECLARE"}, + {Keyword, "DEFINE"}, + {Keyword, "DELETE"}, + {Keyword, "DISCARD"}, + {Keyword, "DO"}, + {Keyword, "DROP"}, + {Keyword, "EVALUATE"}, + {Keyword, "EXPLAIN"}, + {Keyword, "EXPORT"}, + {Keyword, "FOR"}, + {Keyword, "FROM"}, + {Keyword, "GRANT"}, + {Keyword, "IF"}, + {Keyword, "IMPORT"}, + {Keyword, "INSERT"}, + {Keyword, "PARALLEL"}, + {Keyword, "PRAGMA"}, + {Keyword, "PROCESS"}, + {Keyword, "REDUCE"}, + {Keyword, "REPLACE"}, + {Keyword, "RESTORE"}, + {Keyword, "REVOKE"}, + {Keyword, "ROLLBACK"}, + {Keyword, "SELECT"}, + {Keyword, "SHOW"}, + {Keyword, "UPDATE"}, + {Keyword, "UPSERT"}, + {Keyword, "USE"}, + {Keyword, "VALUES"}, + }; + + auto engine = MakeSqlCompletionEngine(); + UNIT_ASSERT_VALUES_EQUAL(Complete(engine, {""}), expected); + UNIT_ASSERT_VALUES_EQUAL(Complete(engine, {" "}), expected); + UNIT_ASSERT_VALUES_EQUAL(Complete(engine, {" "}), expected); + UNIT_ASSERT_VALUES_EQUAL(Complete(engine, {";"}), expected); + UNIT_ASSERT_VALUES_EQUAL(Complete(engine, {"; "}), expected); + UNIT_ASSERT_VALUES_EQUAL(Complete(engine, {" ; "}), expected); + } + + Y_UNIT_TEST(Alter) { + TVector<TCandidate> expected = { + {Keyword, "ASYNC"}, + {Keyword, "BACKUP"}, + {Keyword, "DATABASE"}, + {Keyword, "EXTERNAL"}, + {Keyword, "GROUP"}, + {Keyword, "OBJECT"}, + {Keyword, "RESOURCE"}, + {Keyword, "SEQUENCE"}, + {Keyword, "TABLE"}, + {Keyword, "TABLESTORE"}, + {Keyword, "TOPIC"}, + {Keyword, "TRANSFER"}, + {Keyword, "USER"}, + }; + + auto engine = MakeSqlCompletionEngine(); + UNIT_ASSERT_VALUES_EQUAL(Complete(engine, {"ALTER "}), expected); + } + + Y_UNIT_TEST(Create) { + TVector<TCandidate> expected = { + {Keyword, "ASYNC"}, + {Keyword, "BACKUP"}, + {Keyword, "EXTERNAL"}, + {Keyword, "GROUP"}, + {Keyword, "OBJECT"}, + {Keyword, "OR"}, + {Keyword, "RESOURCE"}, + {Keyword, "TABLE"}, + {Keyword, "TABLESTORE"}, + {Keyword, "TEMP"}, + {Keyword, "TEMPORARY"}, + {Keyword, "TOPIC"}, + {Keyword, "TRANSFER"}, + {Keyword, "USER"}, + {Keyword, "VIEW"}, + }; + + auto engine = MakeSqlCompletionEngine(); + UNIT_ASSERT_VALUES_EQUAL(Complete(engine, {"CREATE "}), expected); + } + + Y_UNIT_TEST(Delete) { + TVector<TCandidate> expected = { + {Keyword, "FROM"}, + }; + + auto engine = MakeSqlCompletionEngine(); + UNIT_ASSERT_VALUES_EQUAL(Complete(engine, {"DELETE "}), expected); + } + + Y_UNIT_TEST(Drop) { + TVector<TCandidate> expected = { + {Keyword, "ASYNC"}, + {Keyword, "BACKUP"}, + {Keyword, "EXTERNAL"}, + {Keyword, "GROUP"}, + {Keyword, "OBJECT"}, + {Keyword, "RESOURCE"}, + {Keyword, "TABLE"}, + {Keyword, "TABLESTORE"}, + {Keyword, "TOPIC"}, + {Keyword, "TRANSFER"}, + {Keyword, "USER"}, + {Keyword, "VIEW"}, + }; + + auto engine = MakeSqlCompletionEngine(); + UNIT_ASSERT_VALUES_EQUAL(Complete(engine, {"DROP "}), expected); + } + + Y_UNIT_TEST(Explain) { + TVector<TCandidate> expected = { + {Keyword, "ALTER"}, + {Keyword, "ANALYZE"}, + {Keyword, "BACKUP"}, + {Keyword, "BATCH"}, + {Keyword, "COMMIT"}, + {Keyword, "CREATE"}, + {Keyword, "DECLARE"}, + {Keyword, "DEFINE"}, + {Keyword, "DELETE"}, + {Keyword, "DISCARD"}, + {Keyword, "DO"}, + {Keyword, "DROP"}, + {Keyword, "EVALUATE"}, + {Keyword, "EXPORT"}, + {Keyword, "FOR"}, + {Keyword, "FROM"}, + {Keyword, "GRANT"}, + {Keyword, "IF"}, + {Keyword, "IMPORT"}, + {Keyword, "INSERT"}, + {Keyword, "PARALLEL"}, + {Keyword, "PRAGMA"}, + {Keyword, "PROCESS"}, + {Keyword, "QUERY"}, + {Keyword, "REDUCE"}, + {Keyword, "REPLACE"}, + {Keyword, "RESTORE"}, + {Keyword, "REVOKE"}, + {Keyword, "ROLLBACK"}, + {Keyword, "SELECT"}, + {Keyword, "SHOW"}, + {Keyword, "UPDATE"}, + {Keyword, "UPSERT"}, + {Keyword, "USE"}, + {Keyword, "VALUES"}, + }; + + auto engine = MakeSqlCompletionEngine(); + UNIT_ASSERT_VALUES_EQUAL(Complete(engine, {"EXPLAIN "}), expected); + } + + Y_UNIT_TEST(Grant) { + TVector<TCandidate> expected = { + {Keyword, "ALL"}, + {Keyword, "ALTER"}, + {Keyword, "CONNECT"}, + {Keyword, "CREATE"}, + {Keyword, "DESCRIBE"}, + {Keyword, "DROP"}, + {Keyword, "ERASE"}, + {Keyword, "FULL"}, + {Keyword, "GRANT"}, + {Keyword, "INSERT"}, + {Keyword, "LIST"}, + {Keyword, "MANAGE"}, + {Keyword, "MODIFY"}, + {Keyword, "REMOVE"}, + {Keyword, "SELECT"}, + {Keyword, "UPDATE"}, + {Keyword, "USE"}, + }; + + auto engine = MakeSqlCompletionEngine(); + UNIT_ASSERT_VALUES_EQUAL(Complete(engine, {"GRANT "}), expected); + } + + Y_UNIT_TEST(Insert) { + TVector<TCandidate> expected = { + {Keyword, "INTO"}, + {Keyword, "OR"}, + }; + + auto engine = MakeSqlCompletionEngine(); + UNIT_ASSERT_VALUES_EQUAL(Complete(engine, {"INSERT "}), expected); + } + + Y_UNIT_TEST(Pragma) { + TVector<TCandidate> expected = { + {Keyword, "ANSI"}, + {Keyword, "CALLABLE"}, + {Keyword, "DICT"}, + {Keyword, "ENUM"}, + {Keyword, "FLOW"}, + {Keyword, "LIST"}, + {Keyword, "OPTIONAL"}, + {Keyword, "RESOURCE"}, + {Keyword, "SET"}, + {Keyword, "STRUCT"}, + {Keyword, "TAGGED"}, + {Keyword, "TUPLE"}, + {Keyword, "VARIANT"}, + }; + + auto engine = MakeSqlCompletionEngine(); + UNIT_ASSERT_VALUES_EQUAL(Complete(engine, {"PRAGMA "}), expected); + } + + Y_UNIT_TEST(Select) { + TVector<TCandidate> expected = { + {Keyword, "ALL"}, + {Keyword, "BITCAST"}, + {Keyword, "CALLABLE"}, + {Keyword, "CASE"}, + {Keyword, "CAST"}, + {Keyword, "CURRENT_DATE"}, + {Keyword, "CURRENT_TIME"}, + {Keyword, "CURRENT_TIMESTAMP"}, + {Keyword, "DICT"}, + {Keyword, "DISTINCT"}, + {Keyword, "EMPTY_ACTION"}, + {Keyword, "ENUM"}, + {Keyword, "EXISTS"}, + {Keyword, "FALSE"}, + {Keyword, "FLOW"}, + {Keyword, "JSON_EXISTS"}, + {Keyword, "JSON_QUERY"}, + {Keyword, "JSON_VALUE"}, + {Keyword, "LIST"}, + {Keyword, "NOT"}, + {Keyword, "NULL"}, + {Keyword, "OPTIONAL"}, + {Keyword, "RESOURCE"}, + {Keyword, "SET"}, + {Keyword, "STREAM"}, + {Keyword, "STRUCT"}, + {Keyword, "TAGGED"}, + {Keyword, "TRUE"}, + {Keyword, "TUPLE"}, + {Keyword, "VARIANT"}, + }; + + auto engine = MakeSqlCompletionEngine(); + UNIT_ASSERT_VALUES_EQUAL(Complete(engine, {"SELECT "}), expected); + } + + Y_UNIT_TEST(Upsert) { + TVector<TCandidate> expected = { + {Keyword, "INTO"}, + {Keyword, "OBJECT"}, + }; + + auto engine = MakeSqlCompletionEngine(); + UNIT_ASSERT_VALUES_EQUAL(Complete(engine, {"UPSERT "}), expected); + } + + Y_UNIT_TEST(UTF8Wide) { + auto engine = MakeSqlCompletionEngine(); + UNIT_ASSERT_VALUES_EQUAL(Complete(engine, {"\xF0\x9F\x98\x8A"}).size(), 0); + UNIT_ASSERT_VALUES_EQUAL(Complete(engine, {"编码"}).size(), 0); + } + + Y_UNIT_TEST(WordBreak) { + auto engine = MakeSqlCompletionEngine(); + UNIT_ASSERT_VALUES_EQUAL(Complete(engine, {"SELECT ("}).size(), 28); + UNIT_ASSERT_VALUES_EQUAL(Complete(engine, {"SELECT (1)"}).size(), 30); + UNIT_ASSERT_VALUES_EQUAL(Complete(engine, {"SELECT 1;"}).size(), 35); + } + + Y_UNIT_TEST(Typing) { + const auto queryUtf16 = TUtf16String::FromUtf8( + "SELECT \n" + " 123467, \"Hello, {name}! 编码\"}, \n" + " (1 + (5 * 1 / 0)), MIN(identifier), \n" + " Bool(field), Math::Sin(var) \n" + "FROM `local/test/space/table` JOIN test;"); + + auto engine = MakeSqlCompletionEngine(); + + for (std::size_t size = 0; size <= queryUtf16.size(); ++size) { + const TWtringBuf prefixUtf16(queryUtf16, 0, size); + auto completion = engine->Complete({TString::FromUtf16(prefixUtf16)}); + Y_DO_NOT_OPTIMIZE_AWAY(completion); + } + } + + Y_UNIT_TEST(CaseInsensitivity) { + TVector<TCandidate> expected = { + {Keyword, "SELECT"}, + }; + + auto engine = MakeSqlCompletionEngine(); + UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "se"), expected); + UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "sE"), expected); + UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "Se"), expected); + UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "SE"), expected); + } +} // Y_UNIT_TEST_SUITE(SqlCompleteTests) diff --git a/yql/essentials/sql/v1/complete/sql_context.cpp b/yql/essentials/sql/v1/complete/sql_context.cpp new file mode 100644 index 0000000000..18f676e40b --- /dev/null +++ b/yql/essentials/sql/v1/complete/sql_context.cpp @@ -0,0 +1,123 @@ +#include "sql_context.h" + +#include "c3_engine.h" +#include "sql_syntax.h" + +#include <yql/essentials/parser/antlr_ast/gen/v1_antlr4/SQLv1Antlr4Lexer.h> +#include <yql/essentials/parser/antlr_ast/gen/v1_antlr4/SQLv1Antlr4Parser.h> +#include <yql/essentials/parser/antlr_ast/gen/v1_ansi_antlr4/SQLv1Antlr4Lexer.h> +#include <yql/essentials/parser/antlr_ast/gen/v1_ansi_antlr4/SQLv1Antlr4Parser.h> + +#include <util/generic/algorithm.h> +#include <util/stream/output.h> + +namespace NSQLComplete { + + template <bool IsAnsiLexer> + class TSpecializedSqlContextInference: public ISqlContextInference { + private: + using TDefaultYQLGrammar = TAntlrGrammar< + NALADefaultAntlr4::SQLv1Antlr4Lexer, + NALADefaultAntlr4::SQLv1Antlr4Parser>; + + using TAnsiYQLGrammar = TAntlrGrammar< + NALAAnsiAntlr4::SQLv1Antlr4Lexer, + NALAAnsiAntlr4::SQLv1Antlr4Parser>; + + using G = std::conditional_t< + IsAnsiLexer, + TAnsiYQLGrammar, + TDefaultYQLGrammar>; + + public: + TSpecializedSqlContextInference() + : Grammar(&GetSqlGrammar(IsAnsiLexer)) + , C3(ComputeC3Config()) + { + } + + TCompletionContext Analyze(TCompletionInput input) override { + auto prefix = input.Text.Head(input.CursorPosition); + auto tokens = C3.Complete(prefix); + FilterIdKeywords(tokens); + return { + .Keywords = SiftedKeywords(tokens), + }; + } + + private: + IC3Engine::TConfig ComputeC3Config() { + return { + .IgnoredTokens = ComputeIgnoredTokens(), + .PreferredRules = ComputePreferredRules(), + }; + } + + std::unordered_set<TTokenId> ComputeIgnoredTokens() { + auto ignoredTokens = Grammar->GetAllTokens(); + for (auto keywordToken : Grammar->GetKeywordTokens()) { + ignoredTokens.erase(keywordToken); + } + return ignoredTokens; + } + + std::unordered_set<TRuleId> ComputePreferredRules() { + const auto& keywordRules = Grammar->GetKeywordRules(); + + std::unordered_set<TRuleId> preferredRules; + preferredRules.insert(std::begin(keywordRules), std::end(keywordRules)); + return preferredRules; + } + + void FilterIdKeywords(TVector<TSuggestedToken>& tokens) { + const auto& keywordRules = Grammar->GetKeywordRules(); + auto [first, last] = std::ranges::remove_if(tokens, [&](const TSuggestedToken& token) { + return AnyOf(token.ParserCallStack, [&](TRuleId rule) { + return Find(keywordRules, rule) != std::end(keywordRules); + }); + }); + tokens.erase(first, last); + } + + TVector<TString> SiftedKeywords(const TVector<TSuggestedToken>& tokens) { + const auto& vocabulary = Grammar->GetVocabulary(); + const auto& keywordTokens = Grammar->GetKeywordTokens(); + + TVector<TString> keywords; + for (const auto& token : tokens) { + if (keywordTokens.contains(token.Number)) { + keywords.emplace_back(vocabulary.getDisplayName(token.Number)); + } + } + return keywords; + } + + const ISqlGrammar* Grammar; + TC3Engine<G> C3; + }; + + class TSqlContextInference: public ISqlContextInference { + public: + TCompletionContext Analyze(TCompletionInput input) override { + auto isAnsiLexer = IsAnsiQuery(TString(input.Text)); + auto& engine = GetSpecializedEngine(isAnsiLexer); + return engine.Analyze(std::move(input)); + } + + private: + ISqlContextInference& GetSpecializedEngine(bool isAnsiLexer) { + if (isAnsiLexer) { + return AnsiEngine; + } + return DefaultEngine; + } + + TSpecializedSqlContextInference</* IsAnsiLexer = */ false> DefaultEngine; + TSpecializedSqlContextInference</* IsAnsiLexer = */ true> AnsiEngine; + }; + + ISqlContextInference::TPtr MakeSqlContextInference() { + return TSqlContextInference::TPtr(new TSqlContextInference()); + } + +} // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/sql_context.h b/yql/essentials/sql/v1/complete/sql_context.h new file mode 100644 index 0000000000..bc3b8d4840 --- /dev/null +++ b/yql/essentials/sql/v1/complete/sql_context.h @@ -0,0 +1,23 @@ +#pragma once + +#include "sql_complete.h" + +#include <util/generic/string.h> + +namespace NSQLComplete { + + struct TCompletionContext { + TVector<TString> Keywords; + }; + + class ISqlContextInference { + public: + using TPtr = THolder<ISqlContextInference>; + + virtual TCompletionContext Analyze(TCompletionInput input) = 0; + virtual ~ISqlContextInference() = default; + }; + + ISqlContextInference::TPtr MakeSqlContextInference(); + +} // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/sql_syntax.cpp b/yql/essentials/sql/v1/complete/sql_syntax.cpp new file mode 100644 index 0000000000..ba5a08d371 --- /dev/null +++ b/yql/essentials/sql/v1/complete/sql_syntax.cpp @@ -0,0 +1,19 @@ +#include "sql_syntax.h" + +#include <yql/essentials/public/issue/yql_issue.h> +#include <yql/essentials/sql/settings/translation_settings.h> + +namespace NSQLComplete { + + using NSQLTranslation::ParseTranslationSettings; + using NSQLTranslation::TTranslationSettings; + using NYql::TIssues; + + bool IsAnsiQuery(const TString& query) { + TTranslationSettings settings; + TIssues issues; + ParseTranslationSettings(query, settings, issues); + return settings.AnsiLexer; + } + +} // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/sql_syntax.h b/yql/essentials/sql/v1/complete/sql_syntax.h new file mode 100644 index 0000000000..f03cbc9fb9 --- /dev/null +++ b/yql/essentials/sql/v1/complete/sql_syntax.h @@ -0,0 +1,10 @@ +#pragma once + +#include <util/generic/fwd.h> + +namespace NSQLComplete { + + // Permits invalid special comments + bool IsAnsiQuery(const TString& query); + +} // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/string_util.cpp b/yql/essentials/sql/v1/complete/string_util.cpp new file mode 100644 index 0000000000..12a6701065 --- /dev/null +++ b/yql/essentials/sql/v1/complete/string_util.cpp @@ -0,0 +1,29 @@ +#include "string_util.h" + +#include <util/generic/strbuf.h> + +namespace NSQLComplete { + + bool IsWordBoundary(char ch) { // Is optimized into table lookup by clang + for (size_t i = 0; i < sizeof(WordBreakCharacters) - 1; ++i) { + if (WordBreakCharacters[i] == ch) { + return true; + } + } + return false; + } + + size_t LastWordIndex(TStringBuf text) { + for (auto it = std::rbegin(text); it != std::rend(text); std::advance(it, 1)) { + if (IsWordBoundary(*it)) { + return std::distance(it, std::rend(text)); + } + } + return 0; + } + + TStringBuf LastWord(TStringBuf text) { + return text.SubStr(LastWordIndex(text)); + } + +} // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/string_util.h b/yql/essentials/sql/v1/complete/string_util.h new file mode 100644 index 0000000000..bafc578d82 --- /dev/null +++ b/yql/essentials/sql/v1/complete/string_util.h @@ -0,0 +1,17 @@ +#pragma once + +#include <util/charset/unidata.h> + +#include <string_view> + +namespace NSQLComplete { + + static const char WordBreakCharacters[] = " \t\v\f\a\b\r\n`~!@#$%^&*-=+[](){}\\|;:'\".,<>/?"; + + bool IsWordBoundary(char ch); + + size_t LastWordIndex(TStringBuf text); + + TStringBuf LastWord(TStringBuf text); + +} // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/string_util_ut.cpp b/yql/essentials/sql/v1/complete/string_util_ut.cpp new file mode 100644 index 0000000000..ca3ed546a3 --- /dev/null +++ b/yql/essentials/sql/v1/complete/string_util_ut.cpp @@ -0,0 +1,21 @@ +#include "string_util.h" + +#include <library/cpp/testing/unittest/registar.h> + +using namespace NSQLComplete; + +Y_UNIT_TEST_SUITE(StringUtilTest) { + Y_UNIT_TEST(Blank) { + UNIT_ASSERT_VALUES_EQUAL(LastWord(""), ""); + UNIT_ASSERT_VALUES_EQUAL(LastWord(" "), ""); + UNIT_ASSERT_VALUES_EQUAL(LastWord(" "), ""); + UNIT_ASSERT_VALUES_EQUAL(LastWord(" "), ""); + } + + Y_UNIT_TEST(Space) { + UNIT_ASSERT_VALUES_EQUAL(LastWord("two "), ""); + UNIT_ASSERT_VALUES_EQUAL(LastWord("one two "), ""); + UNIT_ASSERT_VALUES_EQUAL(LastWord("two"), "two"); + UNIT_ASSERT_VALUES_EQUAL(LastWord("one two"), "two"); + } +} // Y_UNIT_TEST_SUITE(StringUtilTest) diff --git a/yql/essentials/sql/v1/complete/ut/ya.make b/yql/essentials/sql/v1/complete/ut/ya.make new file mode 100644 index 0000000000..91f7da1361 --- /dev/null +++ b/yql/essentials/sql/v1/complete/ut/ya.make @@ -0,0 +1,8 @@ +UNITTEST_FOR(yql/essentials/sql/v1/complete) + +SRCS( + sql_complete_ut.cpp + string_util_ut.cpp +) + +END() diff --git a/yql/essentials/sql/v1/complete/ya.make b/yql/essentials/sql/v1/complete/ya.make new file mode 100644 index 0000000000..70189e5f50 --- /dev/null +++ b/yql/essentials/sql/v1/complete/ya.make @@ -0,0 +1,24 @@ +LIBRARY() + +SRCS( + sql_antlr4.cpp + sql_complete.cpp + sql_context.cpp + sql_syntax.cpp + string_util.cpp +) + +PEERDIR( + contrib/libs/antlr4_cpp_runtime + contrib/libs/antlr4-c3 + yql/essentials/sql/settings + yql/essentials/sql/v1/format + yql/essentials/parser/antlr_ast/gen/v1_ansi_antlr4 + yql/essentials/parser/antlr_ast/gen/v1_antlr4 +) + +END() + +RECURSE_FOR_TESTS( + ut +) diff --git a/yql/essentials/sql/v1/sql_translation.cpp b/yql/essentials/sql/v1/sql_translation.cpp index 71dfafb3c4..af492431c8 100644 --- a/yql/essentials/sql/v1/sql_translation.cpp +++ b/yql/essentials/sql/v1/sql_translation.cpp @@ -3819,35 +3819,44 @@ bool TSqlTranslation::RoleNameClause(const TRule_role_name& node, TDeferredAtom& } bool TSqlTranslation::PasswordParameter(const TRule_password_option& passwordOption, TUserParameters& result) { - // password_option: ENCRYPTED? PASSWORD expr; - TSqlExpression expr(Ctx, Mode); - TNodePtr password = expr.Build(passwordOption.GetRule_expr3()); - if (!password) { - Error() << "Couldn't parse the password"; - return false; + // password_option: ENCRYPTED? PASSWORD password_value; + // password_value: STRING_VALUE | NULL; + + const auto& token = passwordOption.GetRule_password_value3().GetToken1(); + TString stringValue(Ctx.Token(token)); + + if (to_lower(stringValue) == "null") { + // result.Password = default value + } else { + auto password = StringContent(Ctx, Ctx.Pos(), stringValue); + + if (!password) { + Error() << "Password should be enclosed into quotation marks."; + return false; + } + + result.Password = TDeferredAtom(Ctx.Pos(), std::move(password->Content)); } result.IsPasswordEncrypted = passwordOption.HasBlock1(); - if (!password->IsNull()) { - result.Password = MakeAtomFromExpression(Ctx.Pos(), Ctx, password); - } return true; } bool TSqlTranslation::HashParameter(const TRule_hash_option& hashOption, TUserParameters& result) { - // hash_option: HASH expr; - TSqlExpression expr(Ctx, Mode); - TNodePtr hash = expr.Build(hashOption.GetRule_expr2()); + // hash_option: HASH STRING_VALUE; + + const auto& token = hashOption.GetToken2(); + TString stringValue(Ctx.Token(token)); + + auto hash = StringContent(Ctx, Ctx.Pos(), stringValue); if (!hash) { - Error() << "Couldn't parse the hash of password"; + Error() << "Hash should be enclosed into quotation marks."; return false; } - if (!hash->IsNull()) { - result.Hash = MakeAtomFromExpression(Ctx.Pos(), Ctx, hash); - } + result.Hash = TDeferredAtom(Ctx.Pos(), std::move(hash->Content)); return true; } diff --git a/yql/essentials/sql/v1/sql_ut.cpp b/yql/essentials/sql/v1/sql_ut.cpp index aa1e17e53f..66c9a159fb 100644 --- a/yql/essentials/sql/v1/sql_ut.cpp +++ b/yql/essentials/sql/v1/sql_ut.cpp @@ -4,7 +4,6 @@ #include <yql/essentials/providers/common/provider/yql_provider_names.h> #include <yql/essentials/sql/sql.h> -#include <yql/essentials/sql/v1/sql.h> #include <yql/essentials/sql/v1/lexer/antlr3/lexer.h> #include <util/generic/map.h> @@ -30,8121 +29,10 @@ TParsedTokenList Tokenize(const TString& query) { return tokens; } -TString ToString(const TParsedTokenList& tokens) { - TStringBuilder reconstructedQuery; - for (const auto& token : tokens) { - if (token.Name == "WS" || token.Name == "EOF") { - continue; - } - if (!reconstructedQuery.empty()) { - reconstructedQuery << ' '; - } - reconstructedQuery << token.Content; - } - return reconstructedQuery; -} - -} - -Y_UNIT_TEST_SUITE(AnsiMode) { - Y_UNIT_TEST(PragmaAnsi) { - UNIT_ASSERT(SqlToYql("PRAGMA ANSI 2016;").IsOk()); - } -} - -Y_UNIT_TEST_SUITE(SqlParsingOnly) { - ///This function is used in BACKWARD COMPATIBILITY tests below that LIMIT the sets of token that CAN NOT be used - ///as identifiers in different contexts in a SQL request - ///\return list of tokens that failed this check - TVector<TString> ValidateTokens(const THashSet<TString>& forbidden, const std::function<TString (const TString& )>& makeRequest) { - THashMap<TString, bool> allTokens; - for (const auto& t: NSQLFormat::GetKeywords()) { - allTokens[t] = !forbidden.contains((t)); - } - for (const auto& f: forbidden) { - UNIT_ASSERT(allTokens.contains(f)); //check that forbidden list contains tokens only(argument check) - } - TVector<TString> failed; - for (const auto& [token, allowed]: allTokens) { - if (SqlToYql(makeRequest(token)).IsOk() != allowed) - failed.push_back(token); - } - return failed; - } - - Y_UNIT_TEST(TokensAsColumnName) { //id_expr - auto failed = ValidateTokens({ - "ALL", "ANY", "AS", "ASSUME", "ASYMMETRIC", "AUTOMAP", "BETWEEN", "BITCAST", - "CALLABLE", "CASE", "CAST", "CUBE", "CURRENT_DATE", "CURRENT_TIME", "CURRENT_TIMESTAMP", - "DICT", "DISTINCT", "ENUM", "ERASE", "EXCEPT", "EXISTS", "FLOW", "FROM", "FULL", - "HAVING", "HOP", "INTERSECT", "JSON_EXISTS", "JSON_QUERY", "JSON_VALUE", "LIMIT", "LIST", - "NOT", "OPTIONAL", "PROCESS", "REDUCE", "REPEATABLE", "RESOURCE", "RETURN", "RETURNING", "ROLLUP", - "SELECT", "SET", "STREAM", "STRUCT", "SYMMETRIC", "TAGGED", "TUPLE", "UNBOUNDED", - "UNION", "VARIANT", "WHEN", "WHERE", "WINDOW", "WITHOUT" - }, - [](const TString& token){ - TStringBuilder req; - req << "SELECT " << token << " FROM Plato.Input"; - return req; - } - ); - UNIT_ASSERT_VALUES_EQUAL(failed, TVector<TString>{}); - } - - Y_UNIT_TEST(TokensAsWithoutColumnName) { //id_without - auto failed = ValidateTokens({ - "ALL", "AS", "ASSUME", "ASYMMETRIC", "AUTOMAP", "BETWEEN", "BITCAST", - "CALLABLE", "CASE", "CAST", "CUBE", "CURRENT_DATE", "CURRENT_TIME", "CURRENT_TIMESTAMP", - "DICT", "DISTINCT", "EMPTY_ACTION", "ENUM", "EXCEPT", "EXISTS", "FALSE", "FLOW", "FROM", "FULL", - "HAVING", "HOP", "INTERSECT", "JSON_EXISTS", "JSON_QUERY", "JSON_VALUE", "LIMIT", "LIST", - "NOT", "NULL", "OPTIONAL", "PROCESS", "REDUCE", "REPEATABLE", "RESOURCE", "RETURN", "RETURNING", "ROLLUP", - "SELECT", "SET", "STRUCT", "SYMMETRIC", "TAGGED", "TRUE", "TUPLE", "UNBOUNDED", - "UNION", "VARIANT", "WHEN", "WHERE", "WINDOW", "WITHOUT" - }, - [](const TString& token){ - TStringBuilder req; - req << "SELECT * WITHOUT " << token << " FROM Plato.Input"; - return req; - } - ); - UNIT_ASSERT_VALUES_EQUAL(failed, TVector<TString>{}); - } - - Y_UNIT_TEST(TokensAsColumnNameInAddColumn) { //id_schema - auto failed = ValidateTokens({ - "ANY", "AUTOMAP", "CALLABLE", "COLUMN", "DICT", "ENUM", "ERASE", "FALSE", "FLOW", - "LIST", "OPTIONAL", "REPEATABLE", "RESOURCE", - "SET", "STREAM", "STRUCT", "TAGGED", "TRUE", "TUPLE", "VARIANT" - }, - [](const TString& token){ - TStringBuilder req; - req << "ALTER TABLE Plato.Input ADD COLUMN " << token << " Bool"; - return req; - } - ); - UNIT_ASSERT_VALUES_EQUAL(failed, TVector<TString>{}); - } - - Y_UNIT_TEST(TokensAsColumnAlias) { - auto failed = ValidateTokens({ - "AUTOMAP", "FALSE", - "REPEATABLE", "TRUE" - }, - [](const TString& token){ - TStringBuilder req; - req << "SELECT Col as " << token << " FROM Plato.Input"; - return req; - } - ); - UNIT_ASSERT_VALUES_EQUAL(failed, TVector<TString>{}); - } - - Y_UNIT_TEST(TokensAsTableName) { //id_table_or_type - auto failed = ValidateTokens({ - "ANY", "AUTOMAP", "COLUMN", "ERASE", "FALSE", - "REPEATABLE", "STREAM", "TRUE" - }, - [](const TString& token){ - TStringBuilder req; - req << "SELECT * FROM Plato." << token; - return req; - } - ); - UNIT_ASSERT_VALUES_EQUAL(failed, TVector<TString>{}); - } - - Y_UNIT_TEST(TokensAsTableAlias) { //id_table - auto failed = ValidateTokens({ - "AUTOMAP", "CALLABLE", "DICT", "ENUM","FALSE", "FLOW", - "LIST", "OPTIONAL", "REPEATABLE", "RESOURCE", - "SET", "STRUCT", "TAGGED", "TRUE", "TUPLE", "VARIANT" - }, - [](const TString& token){ - TStringBuilder req; - req << "SELECT * FROM Plato.Input AS " << token; - return req; - } - ); - UNIT_ASSERT_VALUES_EQUAL(failed, TVector<TString>{}); - } - - Y_UNIT_TEST(TokensAsHints) { //id_hint - auto failed = ValidateTokens({ - "AUTOMAP", "CALLABLE", "COLUMNS", "DICT", "ENUM", "FALSE", "FLOW", - "LIST", "OPTIONAL", "REPEATABLE", "RESOURCE", - "SCHEMA", "SET", "STRUCT", "TAGGED", "TRUE", "TUPLE", "VARIANT" - }, - [](const TString& token){ - TStringBuilder req; - req << "SELECT * FROM Plato.Input WITH " << token; - return req; - } - ); - UNIT_ASSERT_VALUES_EQUAL(failed, TVector<TString>{}); - } - - Y_UNIT_TEST(TokensAsWindow) { //id_window - auto failed = ValidateTokens({ - "AUTOMAP", "CALLABLE", "DICT", "ENUM", "FALSE", "FLOW", "GROUPS", "LIST", "OPTIONAL", - "RANGE", "REPEATABLE", "RESOURCE", "ROWS", "SET", "STRUCT", "TAGGED" ,"TRUE", "TUPLE", "VARIANT" - }, - [](const TString& token){ - TStringBuilder req; - req << "SELECT * FROM Plato.Input WINDOW " << token << " AS ()"; - return req; - } - ); - UNIT_ASSERT_VALUES_EQUAL(failed, TVector<TString>{}); - } - - Y_UNIT_TEST(TokensAsIdExprIn) { //id_expr_in - auto failed = ValidateTokens({ - "ALL", "ANY", "AS", "ASSUME", "ASYMMETRIC", "AUTOMAP", "BETWEEN", "BITCAST", - "CALLABLE", "CASE", "CAST", "COMPACT", "CUBE", "CURRENT_DATE", "CURRENT_TIME", "CURRENT_TIMESTAMP", - "DICT", "DISTINCT", "ENUM", "ERASE", "EXCEPT", "EXISTS", "FLOW", "FROM", "FULL", - "HAVING", "HOP", "INTERSECT", "JSON_EXISTS", "JSON_QUERY", "JSON_VALUE", "LIMIT", "LIST", - "NOT", "OPTIONAL", "PROCESS", "REDUCE", "REPEATABLE", "RESOURCE", "RETURN", "RETURNING", "ROLLUP", - "SELECT", "SET", "STREAM", "STRUCT", "SYMMETRIC", "TAGGED", "TUPLE", "UNBOUNDED", - "UNION", "VARIANT", "WHEN", "WHERE", "WINDOW", "WITHOUT" - }, - [](const TString& token){ - TStringBuilder req; - req << "SELECT * FROM Plato.Input WHERE q IN " << token; - return req; - } - ); - UNIT_ASSERT_VALUES_EQUAL(failed, TVector<TString>{}); - } - - Y_UNIT_TEST(TableHints) { - UNIT_ASSERT(SqlToYql("SELECT * FROM plato.Input WITH INFER_SCHEMA").IsOk()); - UNIT_ASSERT(SqlToYql("SELECT * FROM plato.Input WITH (INFER_SCHEMA)").IsOk()); - } - - Y_UNIT_TEST(InNoHints) { - TString query = "SELECT * FROM plato.Input WHERE key IN (1,2,3)"; - - VerifySqlInHints(query, { "'('('warnNoAnsi))" }, {}); - VerifySqlInHints(query, { "'()" }, false); - VerifySqlInHints(query, { "'('('ansi))" }, true); - } - - Y_UNIT_TEST(InHintCompact) { - // should parse COMPACT as hint - TString query = "SELECT * FROM plato.Input WHERE key IN COMPACT(1, 2, 3)"; - - VerifySqlInHints(query, { "'('isCompact)" }); - } - - Y_UNIT_TEST(InHintSubquery) { - // should parse tableSource as hint - TString query = "$subq = (SELECT key FROM plato.Input); SELECT * FROM plato.Input WHERE key IN $subq"; - - VerifySqlInHints(query, { "'('tableSource)" }); - } - - Y_UNIT_TEST(InHintCompactSubquery) { - TString query = "$subq = (SELECT key FROM plato.Input); SELECT * FROM plato.Input WHERE key IN COMPACT $subq"; - - VerifySqlInHints(query, { "'('isCompact)", "'('tableSource)" }); - } - - Y_UNIT_TEST(CompactKeywordNotReservedForNames) { - UNIT_ASSERT(SqlToYql("SELECT COMPACT FROM plato.Input WHERE COMPACT IN COMPACT(1, 2, 3)").IsOk()); - UNIT_ASSERT(SqlToYql("USE plato; SELECT * FROM COMPACT").IsOk()); - } - - Y_UNIT_TEST(FamilyKeywordNotReservedForNames) { - // FIXME: check if we can get old behaviour - //UNIT_ASSERT(SqlToYql("USE plato; CREATE TABLE FAMILY (FAMILY Uint32, PRIMARY KEY (FAMILY));").IsOk()); - //UNIT_ASSERT(SqlToYql("USE plato; SELECT FAMILY FROM FAMILY").IsOk()); - UNIT_ASSERT(SqlToYql("USE plato; SELECT FAMILY FROM Input").IsOk()); - } - - Y_UNIT_TEST(ResetKeywordNotReservedForNames) { - UNIT_ASSERT(SqlToYql("USE plato; CREATE TABLE RESET (RESET Uint32, PRIMARY KEY (RESET));").IsOk()); - UNIT_ASSERT(SqlToYql("USE plato; SELECT RESET FROM RESET").IsOk()); - } - - Y_UNIT_TEST(SyncKeywordNotReservedForNames) { - UNIT_ASSERT(SqlToYql("USE plato; CREATE TABLE SYNC (SYNC Uint32, PRIMARY KEY (SYNC));").IsOk()); - UNIT_ASSERT(SqlToYql("USE plato; SELECT SYNC FROM SYNC").IsOk()); - } - - Y_UNIT_TEST(AsyncKeywordNotReservedForNames) { - UNIT_ASSERT(SqlToYql("USE plato; CREATE TABLE ASYNC (ASYNC Uint32, PRIMARY KEY (ASYNC));").IsOk()); - UNIT_ASSERT(SqlToYql("USE plato; SELECT ASYNC FROM ASYNC").IsOk()); - } - - Y_UNIT_TEST(DisableKeywordNotReservedForNames) { - UNIT_ASSERT(SqlToYql("USE plato; CREATE TABLE DISABLE (DISABLE Uint32, PRIMARY KEY (DISABLE));").IsOk()); - UNIT_ASSERT(SqlToYql("USE plato; SELECT DISABLE FROM DISABLE").IsOk()); - } - - Y_UNIT_TEST(ChangefeedKeywordNotReservedForNames) { - UNIT_ASSERT(SqlToYql("USE plato; CREATE TABLE CHANGEFEED (CHANGEFEED Uint32, PRIMARY KEY (CHANGEFEED));").IsOk()); - UNIT_ASSERT(SqlToYql("USE plato; SELECT CHANGEFEED FROM CHANGEFEED").IsOk()); - } - - Y_UNIT_TEST(ReplicationKeywordNotReservedForNames) { - UNIT_ASSERT(SqlToYql("USE plato; CREATE TABLE REPLICATION (REPLICATION Uint32, PRIMARY KEY (REPLICATION));").IsOk()); - UNIT_ASSERT(SqlToYql("USE plato; SELECT REPLICATION FROM REPLICATION").IsOk()); - } - - Y_UNIT_TEST(TransferKeywordNotReservedForNames) { - UNIT_ASSERT(SqlToYql("USE plato; CREATE TABLE TRANSFER (TRANSFER Uint32, PRIMARY KEY (TRANSFER));").IsOk()); - UNIT_ASSERT(SqlToYql("USE plato; SELECT TRANSFER FROM TRANSFER").IsOk()); - } - - Y_UNIT_TEST(SecondsKeywordNotReservedForNames) { - UNIT_ASSERT(SqlToYql("USE plato; CREATE TABLE SECONDS (SECONDS Uint32, PRIMARY KEY (SECONDS));").IsOk()); - UNIT_ASSERT(SqlToYql("USE plato; SELECT SECONDS FROM SECONDS").IsOk()); - } - - Y_UNIT_TEST(MillisecondsKeywordNotReservedForNames) { - UNIT_ASSERT(SqlToYql("USE plato; CREATE TABLE MILLISECONDS (MILLISECONDS Uint32, PRIMARY KEY (MILLISECONDS));").IsOk()); - UNIT_ASSERT(SqlToYql("USE plato; SELECT MILLISECONDS FROM MILLISECONDS").IsOk()); - } - - Y_UNIT_TEST(MicrosecondsKeywordNotReservedForNames) { - UNIT_ASSERT(SqlToYql("USE plato; CREATE TABLE MICROSECONDS (MICROSECONDS Uint32, PRIMARY KEY (MICROSECONDS));").IsOk()); - UNIT_ASSERT(SqlToYql("USE plato; SELECT MICROSECONDS FROM MICROSECONDS").IsOk()); - } - - Y_UNIT_TEST(NanosecondsKeywordNotReservedForNames) { - UNIT_ASSERT(SqlToYql("USE plato; CREATE TABLE NANOSECONDS (NANOSECONDS Uint32, PRIMARY KEY (NANOSECONDS));").IsOk()); - UNIT_ASSERT(SqlToYql("USE plato; SELECT NANOSECONDS FROM NANOSECONDS").IsOk()); - } - - Y_UNIT_TEST(Jubilee) { - NYql::TAstParseResult res = SqlToYql("USE plato; INSERT INTO Arcadia (r2000000) VALUES (\"2M GET!!!\");"); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(QualifiedAsteriskBefore) { - NYql::TAstParseResult res = SqlToYql( - "PRAGMA DisableSimpleColumns;" - "select interested_table.*, LENGTH(value) AS megahelpful_len from plato.Input as interested_table;" - ); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - static bool seenStar = false; - if (word == "FlattenMembers") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("interested_table.")); - } else if (word == "SqlProjectItem") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(Quote("megahelpful_len"))); - UNIT_ASSERT_VALUES_EQUAL(seenStar, true); - } else if (word == "SqlProjectStarItem") { - seenStar = true; - } - }; - TWordCountHive elementStat = {{TString("FlattenMembers"), 0}, {TString("SqlProjectItem"), 0}, {TString("SqlProjectStarItem"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["FlattenMembers"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SqlProjectItem"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SqlProjectStarItem"]); - } - - Y_UNIT_TEST(QualifiedAsteriskAfter) { - NYql::TAstParseResult res = SqlToYql( - "PRAGMA DisableSimpleColumns;" - "select LENGTH(value) AS megahelpful_len, interested_table.* from plato.Input as interested_table;" - ); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - static bool seenStar = false; - if (word == "FlattenMembers") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("interested_table.")); - } else if (word == "SqlProjectItem") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(Quote("megahelpful_len"))); - UNIT_ASSERT_VALUES_EQUAL(seenStar, false); - } else if (word == "SqlProjectStarItem") { - seenStar = true; - } - }; - TWordCountHive elementStat = {{TString("FlattenMembers"), 0}, {TString("SqlProjectItem"), 0}, {TString("SqlProjectStarItem"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["FlattenMembers"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SqlProjectItem"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SqlProjectStarItem"]); - } - - Y_UNIT_TEST(QualifiedMembers) { - NYql::TAstParseResult res = SqlToYql("select interested_table.key, interested_table.value from plato.Input as interested_table;"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - const bool fieldKey = TString::npos != line.find(Quote("key")); - const bool fieldValue = TString::npos != line.find(Quote("value")); - const bool refOnTable = TString::npos != line.find("interested_table."); - if (word == "SqlProjectItem") { - UNIT_ASSERT(fieldKey || fieldValue); - UNIT_ASSERT(!refOnTable); - } else if (word == "Write!") { - UNIT_ASSERT(fieldKey && fieldValue && !refOnTable); - } - }; - TWordCountHive elementStat = {{TString("SqlProjectStarItem"), 0}, {TString("SqlProjectItem"), 0}, {TString("Write!"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - UNIT_ASSERT_VALUES_EQUAL(0, elementStat["SqlProjectStarItem"]); - UNIT_ASSERT_VALUES_EQUAL(2, elementStat["SqlProjectItem"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write!"]); - } - - Y_UNIT_TEST(ExplainQueryPlan) { - UNIT_ASSERT(SqlToYql("EXPLAIN SELECT 1;").IsOk()); - UNIT_ASSERT(SqlToYql("EXPLAIN QUERY PLAN SELECT 1;").IsOk()); - } - - Y_UNIT_TEST(JoinParseCorrect) { - NYql::TAstParseResult res = SqlToYql( - "PRAGMA DisableSimpleColumns;" - " SELECT table_bb.*, table_aa.key as megakey" - " FROM plato.Input AS table_aa" - " JOIN plato.Input AS table_bb" - " ON table_aa.value == table_bb.value;" - ); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "SelectMembers") { - UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("table_aa.")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("table_bb.")); - } else if (word == "SqlProjectItem") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(Quote("megakey"))); - } else if (word == "SqlColumn") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(Quote("table_aa"))); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(Quote("key"))); - } - }; - TWordCountHive elementStat = {{TString("SqlProjectItem"), 0}, {TString("SqlProjectStarItem"), 0}, {TString("SelectMembers"), 0}, {TString("SqlColumn"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SqlProjectItem"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SqlProjectStarItem"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SelectMembers"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SqlColumn"]); - } - - Y_UNIT_TEST(Join3Table) { - NYql::TAstParseResult res = SqlToYql( - " PRAGMA DisableSimpleColumns;" - " SELECT table_bb.*, table_aa.key as gigakey, table_cc.* " - " FROM plato.Input AS table_aa" - " JOIN plato.Input AS table_bb ON table_aa.key == table_bb.key" - " JOIN plato.Input AS table_cc ON table_aa.subkey == table_cc.subkey;" - ); - Err2Str(res); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "SelectMembers") { - UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("table_aa.")); - UNIT_ASSERT(line.find("table_bb.") != TString::npos || line.find("table_cc.") != TString::npos); - } else if (word == "SqlProjectItem") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(Quote("gigakey"))); - } else if (word == "SqlColumn") { - const auto posTableAA = line.find(Quote("table_aa")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, posTableAA); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(Quote("key"))); - UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("table_aa", posTableAA + 3)); - } - }; - TWordCountHive elementStat = {{TString("SqlProjectItem"), 0}, {TString("SqlProjectStarItem"), 0}, {TString("SelectMembers"), 0}, {TString("SqlColumn"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SqlProjectItem"]); - UNIT_ASSERT_VALUES_EQUAL(2, elementStat["SqlProjectStarItem"]); - UNIT_ASSERT_VALUES_EQUAL(2, elementStat["SelectMembers"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SqlColumn"]); - } - - Y_UNIT_TEST(DisabledJoinCartesianProduct) { - NYql::TAstParseResult res = SqlToYql("pragma DisableAnsiImplicitCrossJoin; use plato; select * from A,B,C"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_STRINGS_EQUAL(res.Issues.ToString(), "<main>:1:67: Error: Cartesian product of tables is disabled. Please use explicit CROSS JOIN or enable it via PRAGMA AnsiImplicitCrossJoin\n"); - } - - Y_UNIT_TEST(JoinCartesianProduct) { - NYql::TAstParseResult res = SqlToYql("pragma AnsiImplicitCrossJoin; use plato; select * from A,B,C"); - UNIT_ASSERT(res.Root); - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "EquiJoin") { - auto pos = line.find("Cross"); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, pos); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("Cross", pos + 1)); - } - }; - TWordCountHive elementStat = {{TString("EquiJoin"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["EquiJoin"]); - } - - Y_UNIT_TEST(JoinWithoutConcreteColumns) { - NYql::TAstParseResult res = SqlToYql( - " use plato;" - " SELECT a.v, b.value" - " FROM `Input1` VIEW `ksv` AS a" - " JOIN `Input2` AS b" - " ON a.k == b.key;" - ); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "SqlProjectItem") { - UNIT_ASSERT(line.find(Quote("a.v")) != TString::npos || line.find(Quote("b.value")) != TString::npos); - } else if (word == "SqlColumn") { - const auto posTableA = line.find(Quote("a")); - const auto posTableB = line.find(Quote("b")); - if (posTableA != TString::npos) { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(Quote("v"))); - } else { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, posTableB); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(Quote("value"))); - } - } - }; - TWordCountHive elementStat = {{TString("SqlProjectStarItem"), 0}, {TString("SqlProjectItem"), 0}, {TString("SqlColumn"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - UNIT_ASSERT_VALUES_EQUAL(0, elementStat["SqlProjectStarItem"]); - UNIT_ASSERT_VALUES_EQUAL(2, elementStat["SqlProjectItem"]); - UNIT_ASSERT_VALUES_EQUAL(2, elementStat["SqlColumn"]); - } - - Y_UNIT_TEST(JoinWithSameValues) { - NYql::TAstParseResult res = SqlToYql("SELECT a.value, b.value FROM plato.Input AS a JOIN plato.Input as b ON a.key == b.key;"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "SqlProjectItem") { - const bool isValueFromA = TString::npos != line.find(Quote("a.value")); - const bool isValueFromB = TString::npos != line.find(Quote("b.value")); - UNIT_ASSERT(isValueFromA || isValueFromB); - } if (word == "Write!") { - UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("a.a.")); - UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("b.b.")); - } - }; - TWordCountHive elementStat = {{TString("SqlProjectStarItem"), 0}, {TString("SqlProjectItem"), 0}, {"Write!", 0}}; - VerifyProgram(res, elementStat, verifyLine); - UNIT_ASSERT_VALUES_EQUAL(0, elementStat["SqlProjectStarItem"]); - UNIT_ASSERT_VALUES_EQUAL(2, elementStat["SqlProjectItem"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write!"]); - } - - Y_UNIT_TEST(SameColumnsForDifferentTables) { - NYql::TAstParseResult res = SqlToYql("SELECT a.key, b.key FROM plato.Input as a JOIN plato.Input as b on a.key==b.key;"); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(SameColumnsForDifferentTablesFullJoin) { - NYql::TAstParseResult res = SqlToYql("SELECT a.key, b.key, a.value, b.value FROM plato.Input AS a FULL JOIN plato.Input AS b USING(key);"); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(JoinStreamLookupStrategyHint) { - { - NYql::TAstParseResult res = SqlToYql("SELECT * FROM plato.Input AS a JOIN /*+ StreamLookup() */ plato.Input AS b USING(key);"); - UNIT_ASSERT(res.Root); - } - //case insensitive - { - NYql::TAstParseResult res = SqlToYql("SELECT * FROM plato.Input AS a JOIN /*+ streamlookup() */ plato.Input AS b USING(key);"); - UNIT_ASSERT(res.Root); - } - } - - Y_UNIT_TEST(JoinConflictingStrategyHint) { - { - NYql::TAstParseResult res = SqlToYql("SELECT * FROM plato.Input AS a JOIN /*+ StreamLookup() */ /*+ Merge() */ plato.Input AS b USING(key);"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_STRINGS_EQUAL(res.Issues.ToString(), "<main>:1:91: Error: Conflicting join strategy hints\n"); - } - } - - Y_UNIT_TEST(JoinDuplicatingStrategyHint) { - { - NYql::TAstParseResult res = SqlToYql("SELECT * FROM plato.Input AS a JOIN /*+ StreamLookup() */ /*+ StreamLookup() */ plato.Input AS b USING(key);"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_STRINGS_EQUAL(res.Issues.ToString(), "<main>:1:98: Error: Duplicate join strategy hint\n"); - } - } - - Y_UNIT_TEST(WarnCrossJoinStrategyHint) { - NYql::TAstParseResult res = SqlToYql("SELECT * FROM plato.Input AS a CROSS JOIN /*+ merge() */ plato.Input AS b;"); - UNIT_ASSERT(res.Root); - UNIT_ASSERT_STRINGS_EQUAL(res.Issues.ToString(), "<main>:1:32: Warning: Non-default join strategy will not be used for CROSS JOIN, code: 4534\n"); - } - - Y_UNIT_TEST(WarnCartesianProductStrategyHint) { - NYql::TAstParseResult res = SqlToYql("pragma AnsiImplicitCrossJoin; use plato; SELECT * FROM A, /*+ merge() */ B;"); - UNIT_ASSERT(res.Root); - UNIT_ASSERT_STRINGS_EQUAL(res.Issues.ToString(), "<main>:1:74: Warning: Non-default join strategy will not be used for CROSS JOIN, code: 4534\n"); - } - - Y_UNIT_TEST(WarnUnknownJoinStrategyHint) { - NYql::TAstParseResult res = SqlToYql("SELECT * FROM plato.Input AS a JOIN /*+ xmerge() */ plato.Input AS b USING (key);"); - UNIT_ASSERT(res.Root); - UNIT_ASSERT_STRINGS_EQUAL(res.Issues.ToString(), "<main>:1:41: Warning: Unsupported join hint: xmerge, code: 4534\n"); - } - - Y_UNIT_TEST(ReverseLabels) { - NYql::TAstParseResult res = SqlToYql("select in.key as subkey, subkey as key from plato.Input as in;"); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(AutogenerationAliasWithoutCollisionConflict1) { - NYql::TAstParseResult res = SqlToYql("select LENGTH(Value), key as column1 from plato.Input;"); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(AutogenerationAliasWithoutCollision2Conflict2) { - NYql::TAstParseResult res = SqlToYql("select key as column0, LENGTH(Value) from plato.Input;"); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(InputAliasForQualifiedAsterisk) { - NYql::TAstParseResult res = SqlToYql("use plato; select zyuzya.*, key from plato.Input as zyuzya;"); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(SelectSupportsResultColumnsWithTrailingComma) { - NYql::TAstParseResult res = SqlToYql("select a, b, c, from plato.Input;"); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(SelectOrderByLabeledColumn) { - NYql::TAstParseResult res = SqlToYql("pragma DisableOrderedColumns; select key as goal from plato.Input order by goal"); - UNIT_ASSERT(res.Root); - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "DataSource") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("plato")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("Input")); - - UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("goal")); - } else if (word == "Sort") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("goal")); - - UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("key")); - } - }; - TWordCountHive elementStat = {{TString("DataSource"), 0}, {TString("Sort"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["DataSource"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Sort"]); - } - - Y_UNIT_TEST(SelectOrderBySimpleExpr) { - NYql::TAstParseResult res = SqlToYql("select a from plato.Input order by a + a"); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(SelectAssumeOrderByTableRowAccess) { - NYql::TAstParseResult res = SqlToYql("$key = 'foo';select * from plato.Input assume order by TableRow().$key"); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(SelectOrderByDuplicateLabels) { - NYql::TAstParseResult res = SqlToYql("select a from plato.Input order by a, a"); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(SelectOrderByExpression) { - NYql::TAstParseResult res = SqlToYql("select * from plato.Input as i order by cast(key as uint32) + cast(subkey as uint32)"); - UNIT_ASSERT(res.Root); - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Sort") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("\"+MayWarn\"")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("key")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("subkey")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("(Bool 'true)")); - - UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("i.key")); - UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("i.subkey")); - } - }; - TWordCountHive elementStat = {{TString("Sort"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Sort"]); - } - - Y_UNIT_TEST(SelectOrderByExpressionDesc) { - NYql::TAstParseResult res = SqlToYql("pragma disablesimplecolumns; select i.*, key, subkey from plato.Input as i order by cast(i.key as uint32) - cast(i.subkey as uint32) desc"); - UNIT_ASSERT(res.Root); - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Sort") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("\"-MayWarn\"")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("\"key\"")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("\"subkey\"")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("(Bool 'false)")); - } else if (word == "Write!") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'columns")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("\"key\"")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("\"subkey\"")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("prefix")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("\"i.\"")); - } - }; - TWordCountHive elementStat = {{TString("Sort"), 0}, {TString("Write!"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Sort"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write!"]); - } - - Y_UNIT_TEST(SelectOrderByExpressionAsc) { - NYql::TAstParseResult res = SqlToYql("select i.key, i.subkey from plato.Input as i order by cast(key as uint32) % cast(i.subkey as uint32) asc"); - UNIT_ASSERT(res.Root); - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Sort") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("\"%MayWarn\"")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("\"key\"")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("\"subkey\"")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("(Bool 'true)")); - } else if (word == "Write!") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'columns")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("\"key\"")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("\"subkey\"")); - UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("i.")); - } - }; - TWordCountHive elementStat = {{TString("Sort"), 0}, {TString("Write!"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Sort"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write!"]); - } - - Y_UNIT_TEST(ReferenceToKeyInSubselect) { - NYql::TAstParseResult res = SqlToYql("select b.key from (select a.key from plato.Input as a) as b;"); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(OrderByCastValue) { - NYql::TAstParseResult res = SqlToYql("select i.key, i.subkey from plato.Input as i order by cast(key as uint32) desc;"); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(GroupByCastValue) { - NYql::TAstParseResult res = SqlToYql("select count(1) from plato.Input as i group by cast(key as uint8);"); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(KeywordInSelectColumns) { - NYql::TAstParseResult res = SqlToYql("select in, s.check from (select 1 as in, \"test\" as check) as s;"); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(SelectAllGroupBy) { - NYql::TAstParseResult res = SqlToYql("select * from plato.Input group by subkey;"); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(CreateObjectWithFeatures) { - NYql::TAstParseResult res = SqlToYql("USE plato; CREATE OBJECT secretId (TYPE SECRET) WITH (Key1=Value1, K2=V2);"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'('\"K2\" '\"V2\") '('\"Key1\" '\"Value1\")")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("createObject")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0}, {TString("SECRET"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SECRET"]); - } - - Y_UNIT_TEST(CreateObjectIfNotExists) { - NYql::TAstParseResult res = SqlToYql("USE plato; CREATE OBJECT IF NOT EXISTS secretId (TYPE SECRET) WITH (Key1=Value1, K2=V2);"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("createObjectIfNotExists")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0}, {TString("SECRET"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SECRET"]); - } - - Y_UNIT_TEST(CreateObjectWithFeaturesStrings) { - NYql::TAstParseResult res = SqlToYql("USE plato; CREATE OBJECT secretId (TYPE SECRET) WITH (Key1=\"Value1\", K2='V2', K3=V3, K4='', K5=`aaa`, K6='a\\'aa');"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'('\"K2\" '\"V2\") '('\"K3\" '\"V3\") '('\"K4\" '\"\") '('\"K5\" '\"aaa\") '('\"K6\" '\"a'aa\") '('\"Key1\" '\"Value1\")")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("createObject")); - } - }; - - TWordCountHive elementStat = {{TString("Write"), 0}, {TString("SECRET"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - } - - Y_UNIT_TEST(UpsertObjectWithFeatures) { - NYql::TAstParseResult res = SqlToYql("USE plato; UPSERT OBJECT secretId (TYPE SECRET) WITH (Key1=Value1, K2=V2);"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'('\"K2\" '\"V2\") '('\"Key1\" '\"Value1\")")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("upsertObject")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0}, {TString("SECRET"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SECRET"]); - } - - Y_UNIT_TEST(CreateObjectWithFeaturesAndFlags) { - NYql::TAstParseResult res = SqlToYql("USE plato; CREATE OBJECT secretId (TYPE SECRET) WITH (Key1=Value1, K2=V2, RECURSE);"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'('\"Key1\" '\"Value1\") '('\"RECURSE\")")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("createObject")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0}, {TString("SECRET"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SECRET"]); - } - - Y_UNIT_TEST(Select1Type) { - NYql::TAstParseResult res = SqlToYql("SELECT 1 type;"); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(SelectTableType) { - NYql::TAstParseResult res = SqlToYql("USE plato; SELECT * from T type;"); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(CreateObjectNoFeatures) { - NYql::TAstParseResult res = SqlToYql("USE plato; CREATE OBJECT secretId (TYPE SECRET);"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("'features")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("createObject")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0}, {TString("SECRET"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SECRET"]); - } - - Y_UNIT_TEST(AlterObjectWithFeatures) { - NYql::TAstParseResult res = SqlToYql( - "USE plato;\n" - "declare $path as String;\n" - "ALTER OBJECT secretId (TYPE SECRET) SET (Key1=$path, K2=V2);" - ); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'features")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'\"Key1\" (EvaluateAtom \"$path\"")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'\"K2\" '\"V2\"")); - - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("alterObject")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0}, {TString("SECRET"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SECRET"]); - } - - Y_UNIT_TEST(AlterObjectNoFeatures) { - NYql::TAstParseResult res = SqlToYql("USE plato; ALTER OBJECT secretId (TYPE SECRET);"); - UNIT_ASSERT(!res.Root); - } - - Y_UNIT_TEST(DropObjectNoFeatures) { - NYql::TAstParseResult res = SqlToYql("USE plato; DROP OBJECT secretId (TYPE SECRET);"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("'features")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("dropObject")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0}, {TString("SECRET"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SECRET"]); - } - - Y_UNIT_TEST(DropObjectWithFeatures) { - NYql::TAstParseResult res = SqlToYql("USE plato; DROP OBJECT secretId (TYPE SECRET) WITH (A, B, C);"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'features")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("dropObject")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0}, {TString("SECRET"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SECRET"]); - } - - Y_UNIT_TEST(DropObjectWithOneOption) { - NYql::TAstParseResult res = SqlToYql("USE plato; DROP OBJECT secretId (TYPE SECRET) WITH OVERRIDE;"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'features")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'\"OVERRIDE\"")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("dropObject")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0}, {TString("SECRET"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SECRET"]); - } - - Y_UNIT_TEST(DropObjectIfExists) { - NYql::TAstParseResult res = SqlToYql("USE plato; DROP OBJECT IF EXISTS secretId (TYPE SECRET);"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("dropObjectIfExists")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0}, {TString("SECRET"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SECRET"]); - } - - Y_UNIT_TEST(PrimaryKeyParseCorrect) { - NYql::TAstParseResult res = SqlToYql("USE plato; CREATE TABLE tableName (Key Uint32, Subkey Int64, Value String, PRIMARY KEY (Key, Subkey));"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("\"Key\"")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("\"Subkey\"")); - } - }; - - TWordCountHive elementStat = {{TString("Write"), 0}, {TString("primarykey"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["primarykey"]); - } - - Y_UNIT_TEST(AlterDatabaseAst) { - NYql::TAstParseResult request = SqlToYql("USE plato; ALTER DATABASE `/Root/test` OWNER TO user1;"); - UNIT_ASSERT(request.IsOk()); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - Y_UNUSED(word); - - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find( - R"(let world (Write! world sink (Key '('databasePath (String '"/Root/test"))) (Void) '('('mode 'alterDatabase) '('owner '"user1"))))" - )); - }; - - TWordCountHive elementStat({TString("\'mode \'alterDatabase")}); - VerifyProgram(request, elementStat, verifyLine); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["\'mode \'alterDatabase"]); - } - - Y_UNIT_TEST(CreateTableNonNullableYqlTypeAstCorrect) { - NYql::TAstParseResult res = SqlToYql("USE plato; CREATE TABLE t (a int32 not null);"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write!") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, - line.find(R"__((Write! world sink (Key '('tablescheme (String '"t"))) values '('('mode 'create) '('columns '('('"a" (DataType 'Int32) '('columnConstrains '('('not_null))) '())))))))__")); - } - }; - - TWordCountHive elementStat = {{TString("Write!"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write!"]); - } - - Y_UNIT_TEST(CreateTableNullableYqlTypeAstCorrect) { - NYql::TAstParseResult res = SqlToYql("USE plato; CREATE TABLE t (a int32);"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write!") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, - line.find(R"__((Write! world sink (Key '('tablescheme (String '"t"))) values '('('mode 'create) '('columns '('('"a" (AsOptionalType (DataType 'Int32)) '('columnConstrains '()) '()))))))__")); - } - }; - - TWordCountHive elementStat = {{TString("Write!"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write!"]); - } - - Y_UNIT_TEST(CreateTableNonNullablePgTypeAstCorrect) { - NYql::TAstParseResult res = SqlToYql("USE plato; CREATE TABLE t (a pg_int4 not null);"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write!") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, - line.find(R"__((Write! world sink (Key '('tablescheme (String '"t"))) values '('('mode 'create) '('columns '('('"a" (PgType '_int4) '('columnConstrains '('('not_null))) '())))))))__")); - } - }; - - TWordCountHive elementStat = {{TString("Write!"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write!"]); - } - - Y_UNIT_TEST(CreateTableNullablePgTypeAstCorrect) { - NYql::TAstParseResult res = SqlToYql("USE plato; CREATE TABLE t (a pg_int4);"); - UNIT_ASSERT(res.Root); - - res.Root->PrettyPrintTo(Cout, PRETTY_FLAGS); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write!") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, - line.find(R"__((Write! world sink (Key '('tablescheme (String '"t"))) values '('('mode 'create) '('columns '('('"a" (AsOptionalType (PgType '_int4)) '('columnConstrains '()) '()))))))__")); - } - }; - - TWordCountHive elementStat = {{TString("Write!"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write!"]); - } - - Y_UNIT_TEST(CreateTableNullPkColumnsAreAllowed) { - NYql::TAstParseResult res = SqlToYql("USE plato; CREATE TABLE t (a int32, primary key(a));"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write!") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, - line.find(R"__((Write! world sink (Key '('tablescheme (String '"t"))) values '('('mode 'create) '('columns '('('"a" (AsOptionalType (DataType 'Int32)) '('columnConstrains '()) '()))) '('primarykey '('"a")))))__")); - } - }; - - TWordCountHive elementStat = {{TString("Write!"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write!"]); - } - - Y_UNIT_TEST(CreateTableNotNullPkColumnsAreIdempotentAstCorrect) { - NYql::TAstParseResult res = SqlToYql("USE plato; CREATE TABLE t (a int32 not null, primary key(a));"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write!") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, - line.find(R"__((Write! world sink (Key '('tablescheme (String '"t"))) values '('('mode 'create) '('columns '('('"a" (DataType 'Int32) '('columnConstrains '('('not_null))) '()))) '('primarykey '('"a"))))))__")); - } - }; - - TWordCountHive elementStat = {{TString("Write!"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write!"]); - } - - Y_UNIT_TEST(CreateTableWithIfNotExists) { - NYql::TAstParseResult res = SqlToYql("USE plato; CREATE TABLE IF NOT EXISTS t (a int32, primary key(a));"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write!") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, - line.find(R"__((Write! world sink (Key '('tablescheme (String '"t"))) values '('('mode 'create_if_not_exists) '('columns '('('"a" (AsOptionalType (DataType 'Int32)) '('columnConstrains '()) '()))) '('primarykey '('"a")))))__")); - } - }; - - TWordCountHive elementStat = {{TString("Write!"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write!"]); - } - - Y_UNIT_TEST(CreateTempTable) { - NYql::TAstParseResult res = SqlToYql("USE plato; CREATE TEMP TABLE t (a int32, primary key(a));"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write!") { - UNIT_ASSERT_VALUES_UNEQUAL_C(TString::npos, - line.find(R"__((Write! world sink (Key '('tablescheme (String '"t"))) values '('('mode 'create) '('columns '('('"a" (AsOptionalType (DataType 'Int32)) '('columnConstrains '()) '()))) '('primarykey '('"a")) '('temporary))))__"), line); - } - }; - - TWordCountHive elementStat = {{TString("Write!"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write!"]); - } - - Y_UNIT_TEST(CreateTemporaryTable) { - NYql::TAstParseResult res = SqlToYql("USE plato; CREATE TEMPORARY TABLE t (a int32, primary key(a));"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write!") { - UNIT_ASSERT_VALUES_UNEQUAL_C(TString::npos, - line.find(R"__((Write! world sink (Key '('tablescheme (String '"t"))) values '('('mode 'create) '('columns '('('"a" (AsOptionalType (DataType 'Int32)) '('columnConstrains '()) '()))) '('primarykey '('"a")) '('temporary))))__"), line); - } - }; - - TWordCountHive elementStat = {{TString("Write!"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write!"]); - } - - Y_UNIT_TEST(CreateTableWithoutTypes) { - NYql::TAstParseResult res = SqlToYql("USE plato; CREATE TABLE t (a, primary key(a));"); - UNIT_ASSERT(!res.Root); - } - - Y_UNIT_TEST(CreateTableAsSelectWithTypes) { - NYql::TAstParseResult res = SqlToYql("USE plato; CREATE TABLE t (a int32, primary key(a)) AS SELECT * FROM ts;"); - UNIT_ASSERT(!res.Root); - } - - Y_UNIT_TEST(CreateTableAsSelect) { - NYql::TAstParseResult res = SqlToYql("USE plato; CREATE TABLE t (a, b, primary key(a)) AS SELECT * FROM ts;"); - UNIT_ASSERT_C(res.Root, res.Issues.ToString()); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write!") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, - line.find(R"__((let world (Write! world sink (Key '('tablescheme (String '"t"))) values '('('mode 'create) '('columns '('('"a") '('"b"))) '('primarykey '('"a"))))))__")); - } - if (word == "Read!") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, - line.find(R"__((Read! world (DataSource '"yt" '"plato") (MrTableConcat (Key '('table (String '"ts")))))__")); - } - }; - - TWordCountHive elementStat = {{TString("Write!"), 0}, {TString("Read!"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write!"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Read!"]); - } - - Y_UNIT_TEST(CreateTableAsSelectOnlyPrimary) { - NYql::TAstParseResult res = SqlToYql("USE plato; CREATE TABLE t (primary key(a)) AS SELECT * FROM ts;"); - UNIT_ASSERT_C(res.Root, res.Issues.ToString()); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write!") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, - line.find(R"__((let world (Write! world sink (Key '('tablescheme (String '"t"))) values '('('mode 'create) '('columns '()) '('primarykey '('"a"))))))__")); - } - if (word == "Read!") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, - line.find(R"__((Read! world (DataSource '"yt" '"plato") (MrTableConcat (Key '('table (String '"ts")))))__")); - } - }; - - TWordCountHive elementStat = {{TString("Write!"), 0}, {TString("Read!"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write!"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Read!"]); - } - - Y_UNIT_TEST(CreateTableAsValuesFail) { - NYql::TAstParseResult res = SqlToYql("USE plato; CREATE TABLE t (a, primary key(a)) AS VALUES (1), (2);"); - UNIT_ASSERT(!res.Root); - } - - Y_UNIT_TEST(CreateTableDuplicatedPkColumnsFail) { - NYql::TAstParseResult res = SqlToYql("USE plato; CREATE TABLE t (a int32 not null, primary key(a, a));"); - UNIT_ASSERT(!res.Root); - } - - Y_UNIT_TEST(DeleteFromTableByKey) { - NYql::TAstParseResult res = SqlToYql("delete from plato.Input where key = 200;", 10, "kikimr"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("('mode 'delete)")); - } - }; - - TWordCountHive elementStat = {{TString("Write"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(DeleteFromTable) { - NYql::TAstParseResult res = SqlToYql("delete from plato.Input;", 10, "kikimr"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("('mode 'delete)")); - } - }; - - TWordCountHive elementStat = {{TString("Write"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(DeleteFromTableBatch) { - NYql::TAstParseResult res = SqlToYql("batch delete from plato.Input;", 10, "kikimr"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("('mode 'delete)")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'('is_batch 'true)")); - } - }; - - TWordCountHive elementStat = {{TString("Write"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(DeleteFromTableOnValues) { - NYql::TAstParseResult res = SqlToYql("delete from plato.Input on (key) values (1);", - 10, "kikimr"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("('mode 'delete_on)")); - } - }; - - TWordCountHive elementStat = {{TString("Write"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(DeleteFromTableOnSelect) { - NYql::TAstParseResult res = SqlToYql( - "delete from plato.Input on select key from plato.Input where value > 0;", 10, "kikimr"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("('mode 'delete_on)")); - } - }; - - TWordCountHive elementStat = {{TString("Write"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(DeleteFromTableOnBatch) { - NYql::TAstParseResult res = SqlToYql("batch delete from plato.Input on (key) values (1);", - 10, "kikimr"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:6: Error: BATCH DELETE is unsupported with ON\n"); - } - - Y_UNIT_TEST(UpdateByValues) { - NYql::TAstParseResult res = SqlToYql("update plato.Input set key = 777, value = 'cool' where key = 200;", 10, "kikimr"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("('mode 'update)")); - } else if (word == "AsStruct") { - const bool isKey = line.find("key") != TString::npos; - const bool isValue = line.find("value") != TString::npos; - UNIT_ASSERT(isKey || isValue); - if (isKey) { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(Quote("777"))); - } else if (isValue) { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(Quote("cool"))); - } - } - }; - - TWordCountHive elementStat = {{TString("Write"), 0}, {TString("AsStruct"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["AsStruct"]); - } - - Y_UNIT_TEST(UpdateByValuesBatch) { - NYql::TAstParseResult res = SqlToYql("batch update plato.Input set key = 777, value = 'cool' where key = 200;", 10, "kikimr"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("('mode 'update)")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'('is_batch 'true)")); - } - }; - - TWordCountHive elementStat = {{TString("Write"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(UpdateByMultiValues) { - NYql::TAstParseResult res = SqlToYql("update plato.Input set (key, value, subkey) = ('2','ddd',':') where key = 200;", 10, "kikimr"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("('mode 'update)")); - } else if (word == "AsStruct") { - const bool isKey = line.find("key") != TString::npos; - const bool isSubkey = line.find("subkey") != TString::npos; - const bool isValue = line.find("value") != TString::npos; - UNIT_ASSERT(isKey || isSubkey || isValue); - if (isKey && !isSubkey) { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(Quote("2"))); - } else if (isSubkey) { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(Quote(":"))); - } else if (isValue) { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(Quote("ddd"))); - } - } - }; - - TWordCountHive elementStat = {{TString("Write"), 0}, {TString("AsStruct"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["AsStruct"]); - } - - Y_UNIT_TEST(UpdateBySelect) { - NYql::TAstParseResult res = SqlToYql("update plato.Input set (key, value, subkey) = (select key, value, subkey from plato.Input where key = 911) where key = 200;", 10, "kikimr"); - UNIT_ASSERT(res.Root); - - int lineIndex = 0; - int writeLineIndex = -1; - bool found = false; - - TVerifyLineFunc verifyLine = [&lineIndex, &writeLineIndex, &found](const TString& word, const TString& line) { - if (word == "Write") { - writeLineIndex = lineIndex; - found = line.find("('mode 'update)") != TString::npos; - } else if (word == "mode") { - found |= lineIndex == writeLineIndex + 1 && line.find("('mode 'update)") != TString::npos; - UNIT_ASSERT(found); - } - - ++lineIndex; - }; - - TWordCountHive elementStat = {{TString("Write"), 0}, {TString("mode"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(UpdateSelfModifyAll) { - NYql::TAstParseResult res = SqlToYql("update plato.Input set subkey = subkey + 's';", 10, "kikimr"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("('mode 'update)")); - } else if (word == "AsStruct") { - const bool isSubkey = line.find("subkey") != TString::npos; - UNIT_ASSERT(isSubkey); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(Quote("subkey"))); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(Quote("s"))); - } - }; - - TWordCountHive elementStat = {{TString("Write"), 0}, {TString("AsStruct"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["AsStruct"]); - } - - Y_UNIT_TEST(UpdateOnValues) { - NYql::TAstParseResult res = SqlToYql("update plato.Input on (key, value) values (5, 'cool')", 10, "kikimr"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("('mode 'update_on)")); - } - }; - - TWordCountHive elementStat = {{TString("Write"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(UpdateOnSelect) { - NYql::TAstParseResult res = SqlToYql( - "update plato.Input on select key, value + 1 as value from plato.Input", 10, "kikimr"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("('mode 'update_on)")); - } - }; - - TWordCountHive elementStat = {{TString("Write"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(UpdateOnBatch) { - NYql::TAstParseResult res = SqlToYql("batch update plato.Input on (key, value) values (5, 'cool')", 10, "kikimr"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:6: Error: BATCH UPDATE is unsupported with ON\n"); - } - - Y_UNIT_TEST(UnionAllTest) { - NYql::TAstParseResult res = SqlToYql("PRAGMA DisableEmitUnionMerge; SELECT key FROM plato.Input UNION ALL select subkey FROM plato.Input;"); - UNIT_ASSERT(res.Root); - - TWordCountHive elementStat = {{TString("UnionAll"), 0}}; - VerifyProgram(res, elementStat, {}); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["UnionAll"]); - } - - Y_UNIT_TEST(UnionAllMergeTest) { - NYql::TAstParseResult res = SqlToYql("PRAGMA EmitUnionMerge; SELECT key FROM plato.Input UNION ALL select subkey FROM plato.Input;"); - UNIT_ASSERT(res.Root); - - TWordCountHive elementStat = {{TString("UnionMerge"), 0}}; - VerifyProgram(res, elementStat, {}); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["UnionMerge"]); - } - - Y_UNIT_TEST(UnionTest) { - NYql::TAstParseResult res = SqlToYql("SELECT key FROM plato.Input UNION select subkey FROM plato.Input;"); - UNIT_ASSERT(res.Root); - - TWordCountHive elementStat = {{TString("Union"), 0}}; - VerifyProgram(res, elementStat, {}); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Union"]); - } - - Y_UNIT_TEST(UnionAggregationTest) { - NYql::TAstParseResult res = SqlToYql(R"( - PRAGMA DisableEmitUnionMerge; - SELECT 1 - UNION ALL - SELECT 1 UNION ALL SELECT 1 UNION ALL SELECT 1 - UNION - SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 - UNION ALL - SELECT 1 UNION ALL SELECT 1 UNION ALL SELECT 1; - )"); - UNIT_ASSERT(res.Root); - - TWordCountHive elementStat = {{TString("Union"), 0}, {TString("UnionAll"), 0}}; - VerifyProgram(res, elementStat, {}); - UNIT_ASSERT_VALUES_EQUAL(2, elementStat["UnionAll"]); - UNIT_ASSERT_VALUES_EQUAL(3, elementStat["Union"]); - } - - Y_UNIT_TEST(UnionMergeAggregationTest) { - NYql::TAstParseResult res = SqlToYql(R"( - PRAGMA EmitUnionMerge; - SELECT 1 - UNION ALL - SELECT 1 UNION ALL SELECT 1 UNION ALL SELECT 1 - UNION - SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 - UNION ALL - SELECT 1 UNION ALL SELECT 1 UNION ALL SELECT 1; - )"); - UNIT_ASSERT(res.Root); - - TWordCountHive elementStat = {{TString("Union"), 0}, {TString("UnionMerge"), 0}}; - VerifyProgram(res, elementStat, {}); - UNIT_ASSERT_VALUES_EQUAL(2, elementStat["UnionMerge"]); - UNIT_ASSERT_VALUES_EQUAL(3, elementStat["Union"]); - } - - Y_UNIT_TEST(DeclareDecimalParameter) { - NYql::TAstParseResult res = SqlToYql("declare $value as Decimal(22,9); select $value as cnt;"); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(SimpleGroupBy) { - NYql::TAstParseResult res = SqlToYql("select count(1),z from plato.Input group by key as z order by z;"); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(EmptyColumnName0) { - /// Now it's parsed well and error occur on validate step like "4:31:Empty struct member name is not allowed" in "4:31:Function: AddMember" - NYql::TAstParseResult res = SqlToYql("insert into plato.Output (``, list1) values (0, AsList(0, 1, 2));"); - /// Verify that parsed well without crash - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(KikimrRollback) { - NYql::TAstParseResult res = SqlToYql("use plato; select * from Input; rollback;", 10, "kikimr"); - UNIT_ASSERT(res.Root); - - TWordCountHive elementStat = {{TString("rollback"), 0}}; - VerifyProgram(res, elementStat); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["rollback"]); - } - - Y_UNIT_TEST(PragmaFile) { - NYql::TAstParseResult res = SqlToYql(R"(pragma file("HW", "sbr:181041334");)"); - UNIT_ASSERT(res.Root); - - TWordCountHive elementStat = {{TString(R"((let world (Configure! world (DataSource '"config") '"AddFileByUrl" '"HW" '"sbr:181041334")))"), 0}}; - VerifyProgram(res, elementStat); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat.cbegin()->second); - } - - Y_UNIT_TEST(DoNotCrashOnNamedInFilter) { - NYql::TAstParseResult res = SqlToYql("USE plato; $all = ($table_name) -> { return true; }; SELECT * FROM FILTER(Input, $all)"); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(PragmasFileAndUdfOrder) { - NYql::TAstParseResult res = SqlToYql(R"( - PRAGMA file("libvideoplayers_udf.so", "https://proxy.sandbox.yandex-team.ru/235185290"); - PRAGMA udf("libvideoplayers_udf.so"); - )"); - UNIT_ASSERT(res.Root); - - const auto programm = GetPrettyPrint(res); - const auto file = programm.find("AddFileByUrl"); - const auto udfs = programm.find("ImportUdfs"); - UNIT_ASSERT(file < udfs); - } - - Y_UNIT_TEST(ProcessUserType) { - NYql::TAstParseResult res = SqlToYql("process plato.Input using Kikimr::PushData(TableRows());", 1, TString(NYql::KikimrProviderName)); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Kikimr.PushData") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("TupleType")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("TypeOf")); - } - }; - - TWordCountHive elementStat = {{TString("Kikimr.PushData"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Kikimr.PushData"]); - } - - Y_UNIT_TEST(ProcessUserTypeAuth) { - NYql::TAstParseResult res = SqlToYql("process plato.Input using YDB::PushData(TableRows(), AsTuple('oauth', SecureParam('api:oauth')));", 1, TString(NYql::KikimrProviderName)); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "YDB.PushData") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("TupleType")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("TypeOf")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("api:oauth")); - } - }; - - TWordCountHive elementStat = {{TString("YDB.PushData"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["YDB.PushData"]); - } - - Y_UNIT_TEST(SelectStreamRtmr) { - NYql::TAstParseResult res = SqlToYql( - "USE plato; INSERT INTO Output SELECT STREAM key FROM Input;", - 10, TString(NYql::RtmrProviderName)); - UNIT_ASSERT(res.Root); - - res = SqlToYql( - "USE plato; INSERT INTO Output SELECT key FROM Input;", - 10, TString(NYql::RtmrProviderName)); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(SelectStreamRtmrJoinWithYt) { - NYql::TAstParseResult res = SqlToYql( - "USE plato; INSERT INTO Output SELECT STREAM key FROM Input LEFT JOIN hahn.ttt as t ON Input.key = t.Name;", - 10, TString(NYql::RtmrProviderName)); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(SelectStreamNonRtmr) { - NYql::TAstParseResult res = SqlToYql( - "USE plato; INSERT INTO Output SELECT STREAM key FROM Input;", - 10); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:31: Error: SELECT STREAM is unsupported for non-streaming sources\n"); - } - - Y_UNIT_TEST(GroupByHopRtmr) { - NYql::TAstParseResult res = SqlToYql(R"( - USE plato; INSERT INTO Output SELECT key, SUM(value) AS value FROM Input - GROUP BY key, HOP(subkey, "PT10S", "PT30S", "PT20S"); - )", 10, TString(NYql::RtmrProviderName)); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(GroupByHopRtmrSubquery) { - // 'use plato' intentially avoided - NYql::TAstParseResult res = SqlToYql(R"( - SELECT COUNT(*) AS value FROM (SELECT * FROM plato.Input) - GROUP BY HOP(Data, "PT10S", "PT30S", "PT20S") - )", 10, TString(NYql::RtmrProviderName)); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(GroupByHopRtmrSubqueryBinding) { - NYql::TAstParseResult res = SqlToYql(R"( - USE plato; - $q = SELECT * FROM Input; - INSERT INTO Output SELECT STREAM * FROM ( - SELECT COUNT(*) AS value FROM $q - GROUP BY HOP(Data, "PT10S", "PT30S", "PT20S") - ); - )", 10, TString(NYql::RtmrProviderName)); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(GroupByNoHopRtmr) { - NYql::TAstParseResult res = SqlToYql(R"( - USE plato; INSERT INTO Output SELECT STREAM key, SUM(value) AS value FROM Input - GROUP BY key; - )", 10, TString(NYql::RtmrProviderName)); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:22: Error: Streaming group by query must have a hopping window specification.\n"); - } - - Y_UNIT_TEST(KikimrInserts) { - NYql::TAstParseResult res = SqlToYql(R"( - USE plato; - INSERT INTO Output SELECT key, value FROM Input; - INSERT OR ABORT INTO Output SELECT key, value FROM Input; - INSERT OR IGNORE INTO Output SELECT key, value FROM Input; - INSERT OR REVERT INTO Output SELECT key, value FROM Input; - )", 10, TString(NYql::KikimrProviderName)); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(WarnMissingIsBeforeNotNull) { - NYql::TAstParseResult res = SqlToYql("select 1 NOT NULL"); - UNIT_ASSERT(res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Warning: Missing IS keyword before NOT NULL, code: 4507\n"); - } - - Y_UNIT_TEST(Subqueries) { - NYql::TAstParseResult res = SqlToYql(R"( - USE plato; - $sq1 = (SELECT * FROM plato.Input); - - $sq2 = SELECT * FROM plato.Input; - - $squ1 = ( - SELECT * FROM plato.Input - UNION ALL - SELECT * FROM plato.Input - ); - - $squ2 = - SELECT * FROM plato.Input - UNION ALL - SELECT * FROM plato.Input; - - $squ3 = ( - (SELECT * FROM plato.Input) - UNION ALL - (SELECT * FROM plato.Input) - ); - - SELECT * FROM $sq1; - SELECT * FROM $sq2; - SELECT * FROM $squ1; - SELECT * FROM $squ2; - SELECT * FROM $squ3; - )"); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(SubqueriesJoin) { - NYql::TAstParseResult res = SqlToYql(R"( - USE plato; - - $left = SELECT * FROM plato.Input1 WHERE value != "BadValue"; - $right = SELECT * FROM plato.Input2; - - SELECT * FROM $left AS l - JOIN $right AS r - ON l.key == r.key; - )"); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(AnyInBackticksAsTableName) { - NYql::TAstParseResult res = SqlToYql("use plato; select * from `any`;"); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(AnyJoinForTableAndSubQuery) { - NYql::TAstParseResult res = SqlToYql(R"( - USE plato; - - $r = SELECT * FROM plato.Input2; - - SELECT * FROM ANY plato.Input1 AS l - LEFT JOIN ANY $r AS r - USING (key); - )"); - - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "EquiJoin") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'('left 'any)")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'('right 'any)")); - } - }; - - TWordCountHive elementStat = {{TString("left"), 0}, {TString("right"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["left"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["right"]); - } - - Y_UNIT_TEST(AnyJoinForTableAndTableSource) { - NYql::TAstParseResult res = SqlToYql(R"( - USE plato; - - $r = AsList( - AsStruct("aaa" as key, "bbb" as subkey, "ccc" as value) - ); - - SELECT * FROM ANY plato.Input1 AS l - LEFT JOIN ANY AS_TABLE($r) AS r - USING (key); - )"); - - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "EquiJoin") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'('left 'any)")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'('right 'any)")); - } - }; - - TWordCountHive elementStat = {{TString("left"), 0}, {TString("right"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["left"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["right"]); - } - - Y_UNIT_TEST(AnyJoinNested) { - NYql::TAstParseResult res = SqlToYql(R"( - USE plato; - - FROM ANY Input1 as a - JOIN Input2 as b ON a.key = b.key - LEFT JOIN ANY Input3 as c ON a.key = c.key - RIGHT JOIN ANY Input4 as d ON d.key = b.key - CROSS JOIN Input5 - SELECT *; - )"); - - UNIT_ASSERT(res.Root); - - TWordCountHive elementStat = {{TString("left"), 0}, {TString("right"), 0}}; - VerifyProgram(res, elementStat); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["left"]); - UNIT_ASSERT_VALUES_EQUAL(2, elementStat["right"]); - } - - Y_UNIT_TEST(InlineAction) { - NYql::TAstParseResult res = SqlToYql( - "do begin\n" - " select 1\n" - "; end do\n"); - UNIT_ASSERT(res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), ""); - } - - Y_UNIT_TEST(FlattenByCorrelationName) { - UNIT_ASSERT(SqlToYql("select * from plato.Input as t flatten by t.x").IsOk()); - UNIT_ASSERT(SqlToYql("select * from plato.Input as t flatten by t -- same as flatten by t.t").IsOk()); - } - - Y_UNIT_TEST(DiscoveryMode) { - UNIT_ASSERT(SqlToYqlWithMode("insert into plato.Output select * from plato.Input", NSQLTranslation::ESqlMode::DISCOVERY).IsOk()); - UNIT_ASSERT(SqlToYqlWithMode("select * from plato.concat(Input1, Input2)", NSQLTranslation::ESqlMode::DISCOVERY).IsOk()); - UNIT_ASSERT(SqlToYqlWithMode("select * from plato.each(AsList(\"Input1\", \"Input2\"))", NSQLTranslation::ESqlMode::DISCOVERY).IsOk()); - } - - Y_UNIT_TEST(CubeWithAutoGeneratedLikeColumnName) { - UNIT_ASSERT(SqlToYql("select key,subkey,group from plato.Input group by cube(key,subkey,group)").IsOk()); - } - - Y_UNIT_TEST(CubeWithAutoGeneratedLikeAlias) { - UNIT_ASSERT(SqlToYql("select key,subkey,group from plato.Input group by cube(key,subkey,value as group)").IsOk()); - } - - Y_UNIT_TEST(FilterCanBeUsedAsColumnIdOrBind) { - UNIT_ASSERT(SqlToYql("select filter from plato.Input").IsOk()); - UNIT_ASSERT(SqlToYql("select 1 as filter").IsOk()); - UNIT_ASSERT(SqlToYql("$filter = 1; select $filter").IsOk()); - } - - Y_UNIT_TEST(DuplicateSemicolonsAreAllowedBetweenTopLevelStatements) { - UNIT_ASSERT(SqlToYql(";;select 1; ; select 2;/*comment*/;select 3;;--comment\n;select 4;;").IsOk()); - } - - Y_UNIT_TEST(DuplicateAndMissingTrailingSemicolonsAreAllowedBetweenActionStatements) { - TString req = - "define action $action($b,$c) as\n" - " ;;$d = $b + $c;\n" - " select $b;\n" - " select $c;;\n" - " select $d,\n" - "end define;\n" - "\n" - "do $action(1,2);"; - UNIT_ASSERT(SqlToYql(req).IsOk()); - } - - Y_UNIT_TEST(DuplicateAndMissingTrailingSemicolonsAreAllowedBetweenInlineActionStatements) { - TString req = - "do begin\n" - " ;select 1,\n" - "end do;\n" - "evaluate for $i in AsList(1,2,3) do begin\n" - " select $i;;\n" - " select $i + $i;;\n" - "end do;"; - UNIT_ASSERT(SqlToYql(req).IsOk()); - } - - Y_UNIT_TEST(DuplicateSemicolonsAreAllowedBetweenLambdaStatements) { - TString req = - "$x=1;\n" - "$foo = ($a, $b)->{\n" - " ;;$v = $a + $b;\n" - " $bar = ($c) -> {; return $c << $x};;\n" - " return $bar($v);;\n" - "};\n" - "select $foo(1,2);"; - UNIT_ASSERT(SqlToYql(req).IsOk()); - } - - Y_UNIT_TEST(StringLiteralWithEscapedBackslash) { - NYql::TAstParseResult res1 = SqlToYql(R"foo(SELECT 'a\\';)foo"); - NYql::TAstParseResult res2 = SqlToYql(R"foo(SELECT "a\\";)foo"); - UNIT_ASSERT(res1.Root); - UNIT_ASSERT(res2.Root); - - TWordCountHive elementStat = {{TString("a\\"), 0}}; - - VerifyProgram(res1, elementStat); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["a\\"]); - - VerifyProgram(res2, elementStat); - UNIT_ASSERT_VALUES_EQUAL(2, elementStat["a\\"]); - } - - Y_UNIT_TEST(StringMultiLineLiteralWithEscapes) { - UNIT_ASSERT(SqlToYql("SELECT @@@foo@@@@bar@@@").IsOk()); - UNIT_ASSERT(SqlToYql("SELECT @@@@@@@@@").IsOk()); - } - - Y_UNIT_TEST(StringMultiLineLiteralConsequitiveAt) { - UNIT_ASSERT(!SqlToYql("SELECT @").IsOk()); - UNIT_ASSERT(!SqlToYql("SELECT @@").IsOk()); - UNIT_ASSERT(!SqlToYql("SELECT @@@").IsOk()); - UNIT_ASSERT( SqlToYql("SELECT @@@@").IsOk()); - UNIT_ASSERT( SqlToYql("SELECT @@@@@").IsOk()); - - UNIT_ASSERT(!SqlToYql("SELECT @@@@@@").IsOk()); - UNIT_ASSERT(!SqlToYql("SELECT @@@@@@@").IsOk()); - - UNIT_ASSERT( SqlToYql("SELECT @@@@@@@@").IsOk()); - UNIT_ASSERT( SqlToYql("SELECT @@@@@@@@@").IsOk()); - UNIT_ASSERT(!SqlToYql("SELECT @@@@@@@@@@").IsOk()); - } - - Y_UNIT_TEST(ConstnessForListDictSetCreate) { - auto req = "$foo = ($x, $y) -> (\"aaaa\");\n" - "\n" - "select\n" - " $foo(sum(key), ListCreate(String)),\n" - " $foo(sum(key), DictCreate(String, String)),\n" - " $foo(sum(key), SetCreate(String)),\n" - "from (select 1 as key);"; - UNIT_ASSERT(SqlToYql(req).IsOk()); - } - - Y_UNIT_TEST(CanUseEmptyTupleInWindowPartitionBy) { - auto req = "select sum(key) over w\n" - "from plato.Input\n" - "window w as (partition compact by (), (subkey), (), value || value as dvalue);"; - UNIT_ASSERT(SqlToYql(req).IsOk()); - } - - Y_UNIT_TEST(DenyAnsiOrderByLimitLegacyMode) { - auto req = "pragma DisableAnsiOrderByLimitInUnionAll;\n" - "use plato;\n" - "\n" - "select * from Input order by key limit 10\n" - "union all\n" - "select * from Input order by key limit 1;"; - - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Error: DisableAnsiOrderByLimitInUnionAll pragma is deprecated and no longer supported\n"); - } - - Y_UNIT_TEST(ReduceUsingUdfWithShortcutsWorks) { - auto req = "use plato;\n" - "\n" - "$arg = 'foo';\n" - "$func = XXX::YYY($arg);\n" - "\n" - "REDUCE Input ON key using $func(subkey);\n" - "REDUCE Input ON key using $func(UUU::VVV(TableRow()));\n"; - UNIT_ASSERT(SqlToYql(req).IsOk()); - req = "use plato;\n" - "\n" - "$arg = 'foo';\n" - "$func = XXX::YYY($arg);\n" - "\n" - "REDUCE Input ON key using all $func(subkey);\n" - "REDUCE Input ON key using all $func(UUU::VVV(TableRow()));"; - UNIT_ASSERT(SqlToYql(req).IsOk()); - } - - Y_UNIT_TEST(YsonDisableStrict) { - UNIT_ASSERT(SqlToYql("pragma yson.DisableStrict = \"false\";").IsOk()); - UNIT_ASSERT(SqlToYql("pragma yson.DisableStrict;").IsOk()); - } - - Y_UNIT_TEST(YsonStrict) { - UNIT_ASSERT(SqlToYql("pragma yson.Strict = \"false\";").IsOk()); - UNIT_ASSERT(SqlToYql("pragma yson.Strict;").IsOk()); - } - - Y_UNIT_TEST(JoinByTuple) { - auto req = "use plato;\n" - "\n" - "select * from T1 as a\n" - "join T2 as b\n" - "on AsTuple(a.key, a.subkey) = AsTuple(b.key, b.subkey);"; - UNIT_ASSERT(SqlToYql(req).IsOk()); - } - - Y_UNIT_TEST(JoinByStruct) { - auto req = "use plato;\n" - "\n" - "select * from T1 as a\n" - "join T2 as b\n" - "on AsStruct(a.key as k, a.subkey as sk) = AsStruct(b.key as k, b.subkey as sk);"; - UNIT_ASSERT(SqlToYql(req).IsOk()); - } - - Y_UNIT_TEST(JoinByUdf) { - auto req = "use plato;\n" - "\n" - "select a.align\n" - "from T1 as a\n" - "join T2 as b\n" - "on Yson::SerializeJsonEncodeUtf8(a.align)=b.align;"; - UNIT_ASSERT(SqlToYql(req).IsOk()); - } - - Y_UNIT_TEST(EscapedIdentifierAsLambdaArg) { - auto req = "$f = ($`foo bar`, $x) -> { return $`foo bar` + $x; };\n" - "\n" - "select $f(1, 2);"; - auto res = SqlToYql(req); - UNIT_ASSERT(res.Root); - const auto programm = GetPrettyPrint(res); - auto expected = "(lambda '(\"$foo bar\" \"$x\")"; - UNIT_ASSERT(programm.find(expected) != TString::npos); - } - - Y_UNIT_TEST(UdfSyntaxSugarOnlyCallable) { - auto req = "SELECT Udf(DateTime::FromString)('2022-01-01');"; - auto res = SqlToYql(req); - UNIT_ASSERT(res.Root); - const auto programm = GetPrettyPrint(res); - auto expected = "(SqlCall '\"DateTime.FromString\" '((PositionalArgs (String '\"2022-01-01\")) (AsStruct)) (TupleType (TypeOf '((String '\"2022-01-01\"))) (TypeOf (AsStruct)) (TupleType)))"; - UNIT_ASSERT(programm.find(expected) != TString::npos); - } - - Y_UNIT_TEST(UdfSyntaxSugarTypeNoRun) { - auto req = "SELECT Udf(DateTime::FromString, String, Tuple<Int32, Float>, 'foo' as TypeConfig)('2022-01-01');"; - auto res = SqlToYql(req); - UNIT_ASSERT(res.Root); - const auto programm = GetPrettyPrint(res); - auto expected = "(SqlCall '\"DateTime.FromString\" '((PositionalArgs (String '\"2022-01-01\")) (AsStruct)) (TupleType (TypeOf '((String '\"2022-01-01\"))) (TypeOf (AsStruct)) (TupleType (DataType 'String) (TupleType (DataType 'Int32) (DataType 'Float)))) '\"foo\")"; - UNIT_ASSERT(programm.find(expected) != TString::npos); - } - - Y_UNIT_TEST(UdfSyntaxSugarRunNoType) { - auto req = "SELECT Udf(DateTime::FromString, String, Tuple<Int32, Float>, Void() as RunConfig)('2022-01-01');"; - auto res = SqlToYql(req); - UNIT_ASSERT(res.Root); - const auto programm = GetPrettyPrint(res); - auto expected = "(SqlCall '\"DateTime.FromString\" '((PositionalArgs (String '\"2022-01-01\")) (AsStruct)) (TupleType (TypeOf '((String '\"2022-01-01\"))) (TypeOf (AsStruct)) (TupleType (DataType 'String) (TupleType (DataType 'Int32) (DataType 'Float)))) '\"\" (Void))"; - UNIT_ASSERT(programm.find(expected) != TString::npos); - } - - Y_UNIT_TEST(UdfSyntaxSugarFullTest) { - auto req = "SELECT Udf(DateTime::FromString, String, Tuple<Int32, Float>, 'foo' as TypeConfig, Void() As RunConfig)('2022-01-01');"; - auto res = SqlToYql(req); - UNIT_ASSERT(res.Root); - const auto programm = GetPrettyPrint(res); - auto expected = "(SqlCall '\"DateTime.FromString\" '((PositionalArgs (String '\"2022-01-01\")) (AsStruct)) (TupleType (TypeOf '((String '\"2022-01-01\"))) (TypeOf (AsStruct)) (TupleType (DataType 'String) (TupleType (DataType 'Int32) (DataType 'Float)))) '\"foo\" (Void))"; - UNIT_ASSERT(programm.find(expected) != TString::npos); - } - - Y_UNIT_TEST(UdfSyntaxSugarOtherRunConfigs) { - auto req = "SELECT Udf(DateTime::FromString, String, Tuple<Int32, Float>, 'foo' as TypeConfig, '55' As RunConfig)('2022-01-01');"; - auto res = SqlToYql(req); - UNIT_ASSERT(res.Root); - const auto programm = GetPrettyPrint(res); - auto expected = "(SqlCall '\"DateTime.FromString\" '((PositionalArgs (String '\"2022-01-01\")) (AsStruct)) (TupleType (TypeOf '((String '\"2022-01-01\"))) (TypeOf (AsStruct)) (TupleType (DataType 'String) (TupleType (DataType 'Int32) (DataType 'Float)))) '\"foo\" (String '\"55\"))"; - UNIT_ASSERT(programm.find(expected) != TString::npos); - } - - Y_UNIT_TEST(UdfSyntaxSugarOtherRunConfigs2) { - auto req = "SELECT Udf(DateTime::FromString, String, Tuple<Int32, Float>, 'foo' as TypeConfig, AsTuple(32, 'no', AsStruct(1e-9 As SomeFloat)) As RunConfig)('2022-01-01');"; - auto res = SqlToYql(req); - UNIT_ASSERT(res.Root); - const auto programm = GetPrettyPrint(res); - auto expected = "(SqlCall '\"DateTime.FromString\" '((PositionalArgs (String '\"2022-01-01\")) (AsStruct)) (TupleType (TypeOf '((String '\"2022-01-01\"))) (TypeOf (AsStruct)) (TupleType (DataType 'String) (TupleType (DataType 'Int32) (DataType 'Float)))) '\"foo\" '((Int32 '\"32\") (String '\"no\") (AsStruct '('\"SomeFloat\" (Double '\"1e-9\")))))"; - UNIT_ASSERT(programm.find(expected) != TString::npos); - } - - Y_UNIT_TEST(UdfSyntaxSugarOptional) { - auto req = "SELECT Udf(DateTime::FromString, String?, Int32??, Tuple<Int32, Float>, \"foo\" as TypeConfig, Void() As RunConfig)(\"2022-01-01\");"; - auto res = SqlToYql(req); - UNIT_ASSERT(res.Root); - const auto programm = GetPrettyPrint(res); - auto expected = "(SqlCall '\"DateTime.FromString\" '((PositionalArgs (String '\"2022-01-01\")) (AsStruct)) (TupleType (TypeOf '((String '\"2022-01-01\"))) (TypeOf (AsStruct)) (TupleType (OptionalType (DataType 'String)) (OptionalType (OptionalType (DataType 'Int32))) (TupleType (DataType 'Int32) (DataType 'Float)))) '\"foo\" (Void))"; - UNIT_ASSERT(programm.find(expected) != TString::npos); - } - - Y_UNIT_TEST(CompactionPolicyParseCorrect) { - NYql::TAstParseResult res = SqlToYql( - R"( USE plato; - CREATE TABLE tableName (Key Uint32, Value String, PRIMARY KEY (Key)) - WITH ( COMPACTION_POLICY = "SomeCompactionPreset" );)" - ); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("compactionPolicy")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("SomeCompactionPreset")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(AutoPartitioningBySizeParseCorrect) { - NYql::TAstParseResult res = SqlToYql( - R"( USE plato; - CREATE TABLE tableName (Key Uint32, Value String, PRIMARY KEY (Key)) - WITH ( AUTO_PARTITIONING_BY_SIZE = ENABLED );)" - ); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("autoPartitioningBySize")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("ENABLED")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(UniformPartitionsParseCorrect) { - NYql::TAstParseResult res = SqlToYql( - R"( USE plato; - CREATE TABLE tableName (Key Uint32, Value String, PRIMARY KEY (Key)) - WITH ( UNIFORM_PARTITIONS = 16 );)" - ); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("uniformPartitions")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("16")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(DateTimeTtlParseCorrect) { - NYql::TAstParseResult res = SqlToYql( - R"( USE plato; - CREATE TABLE tableName (Key Uint32, CreatedAt Timestamp, PRIMARY KEY (Key)) - WITH (TTL = Interval("P1D") On CreatedAt);)" - ); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("setTtlSettings")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("tiers")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("evictionDelay")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("86400000")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(IntTtlParseCorrect) { - NYql::TAstParseResult res = SqlToYql( - R"( USE plato; - CREATE TABLE tableName (Key Uint32, CreatedAt Uint32, PRIMARY KEY (Key)) - WITH (TTL = Interval("P1D") On CreatedAt AS SECONDS);)" - ); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("setTtlSettings")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("tiers")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("evictionDelay")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("86400000")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("columnUnit")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("seconds")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(TtlTieringParseCorrect) { - NYql::TAstParseResult res = SqlToYql( - R"( USE plato; - CREATE TABLE tableName (Key Uint32, CreatedAt Uint32, PRIMARY KEY (Key)) - WITH (TTL = - Interval("P1D") TO EXTERNAL DATA SOURCE Tier1, - Interval("P2D") TO EXTERNAL DATA SOURCE Tier2, - Interval("P30D") DELETE - ON CreatedAt AS SECONDS);)" - ); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("setTtlSettings")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("tiers")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("evictionDelay")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("storageName")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("Tier1")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("Tier2")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("86400000")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("172800000")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("2592000000")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("columnUnit")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("seconds")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(TtlTieringWithOtherActionsParseCorrect) { - NYql::TAstParseResult res = SqlToYql( - R"( USE plato; - ALTER TABLE tableName - ADD FAMILY cold (DATA = "rot"), - SET TTL - Interval("P1D") TO EXTERNAL DATA SOURCE Tier1, - Interval("P2D") TO EXTERNAL DATA SOURCE Tier2, - Interval("P30D") DELETE - ON CreatedAt, - ALTER COLUMN payload_v2 SET FAMILY cold, - ALTER FAMILY default SET DATA "ssd" - ;)" - ); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("addColumnFamilies")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("cold")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("alterColumnFamilies")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("default")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("setTtlSettings")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("tiers")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("evictionDelay")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("storageName")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("Tier1")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("Tier2")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("86400000")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("172800000")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("2592000000")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(TieringParseCorrect) { - NYql::TAstParseResult res = SqlToYql( - R"( USE plato; - CREATE TABLE tableName (Key Uint32, Value String, PRIMARY KEY (Key)) - WITH ( TIERING = 'my_tiering' );)" - ); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("tiering")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("my_tiering")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(StoreExternalBlobsParseCorrect) { - NYql::TAstParseResult res = SqlToYql( - R"( USE plato; - CREATE TABLE tableName (Key Uint32, Value String, PRIMARY KEY (Key)) - WITH ( STORE_EXTERNAL_BLOBS = ENABLED );)" - ); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("storeExternalBlobs")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("ENABLED")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(DefaultValueColumn2) { - auto res = SqlToYql(R"( use plato; - $lambda = () -> { - RETURN CAST(RandomUuid(2) as String) - }; - - CREATE TABLE tableName ( - Key Uint32 DEFAULT RandomNumber(1), - Value String DEFAULT $lambda, - PRIMARY KEY (Key) - ); - )"); - - UNIT_ASSERT_C(res.Root, Err2Str(res)); - - const auto program = GetPrettyPrint(res); - - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, program.find("RandomNumber")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, program.find("RandomUuid")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, program.find("columnConstrains")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, program.find("columnConstrains")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, program.find("Write")); - -#if 0 - Cerr << program << Endl; -#endif - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(DefaultValueColumn3) { - auto res = SqlToYql(R"( use plato; - - CREATE TABLE tableName ( - database_id Utf8, - cloud_id Utf8, - global_id Utf8 DEFAULT database_id || "=====", - PRIMARY KEY (database_id) - ); - )"); - - UNIT_ASSERT_VALUES_EQUAL(Err2Str(res), "<main>:6:40: Error: Column reference \"database_id\" is not allowed in current scope\n"); - UNIT_ASSERT(!res.Root); - } - - Y_UNIT_TEST(DefaultValueColumn) { - auto res = SqlToYql(R"( use plato; - CREATE TABLE tableName ( - Key Uint32 FAMILY cold DEFAULT 5, - Value String FAMILY default DEFAULT "empty", - PRIMARY KEY (Key), - FAMILY default ( - DATA = "test", - COMPRESSION = "lz4" - ), - FAMILY cold ( - DATA = "test", - COMPRESSION = "off" - ) - ); - )"); - - UNIT_ASSERT_C(res.Root, Err2Str(res)); - -#if 0 - const auto program = GetPrettyPrint(res); - Cerr << program << Endl; -#endif - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("default")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("columnConstrains")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("columnFamilies")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(ChangefeedParseCorrect) { - auto res = SqlToYql(R"( USE plato; - CREATE TABLE tableName ( - Key Uint32, PRIMARY KEY (Key), - CHANGEFEED feedName WITH ( - MODE = 'KEYS_ONLY', - FORMAT = 'json', - INITIAL_SCAN = TRUE, - VIRTUAL_TIMESTAMPS = FALSE, - BARRIERS_INTERVAL = Interval("PT1S"), - RETENTION_PERIOD = Interval("P1D"), - TOPIC_MIN_ACTIVE_PARTITIONS = 10, - AWS_REGION = 'aws:region' - ) - ); - )"); - UNIT_ASSERT_C(res.Root, Err2Str(res)); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("changefeed")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("mode")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("KEYS_ONLY")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("format")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("json")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("initial_scan")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("true")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("virtual_timestamps")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("false")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("barriers_interval")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("retention_period")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("topic_min_active_partitions")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("aws_region")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("aws:region")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(CloneForAsTableWorksWithCube) { - UNIT_ASSERT(SqlToYql("SELECT * FROM AS_TABLE([<|k1:1, k2:1|>]) GROUP BY CUBE(k1, k2);").IsOk()); - } - - Y_UNIT_TEST(WindowPartitionByColumnProperlyEscaped) { - NYql::TAstParseResult res = SqlToYql("SELECT SUM(key) OVER w FROM plato.Input WINDOW w AS (PARTITION BY `column with space`);"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "CalcOverWindow") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("\"column with space\"")); - } - }; - - TWordCountHive elementStat = { {TString("CalcOverWindow"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["CalcOverWindow"]); - } - - Y_UNIT_TEST(WindowPartitionByExpressionWithoutAliasesAreAllowed) { - NYql::TAstParseResult res = SqlToYql("SELECT SUM(key) OVER w FROM plato.Input as i WINDOW w AS (PARTITION BY ii.subkey);"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "AddMember") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("AddMember row 'group_w_0 (SqlAccess 'struct (Member row '\"ii\")")); - } - if (word == "CalcOverWindow") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("CalcOverWindow core '('\"group_w_0\")")); - } - }; - - TWordCountHive elementStat = { {TString("CalcOverWindow"), 0}, {TString("AddMember"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["CalcOverWindow"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["AddMember"]); - } - - Y_UNIT_TEST(PqReadByAfterUse) { - ExpectFailWithError("use plato; pragma PqReadBy='plato2';", - "<main>:1:28: Error: Cluster in PqReadPqBy pragma differs from cluster specified in USE statement: plato2 != plato\n"); - - UNIT_ASSERT(SqlToYql("pragma PqReadBy='plato2';").IsOk()); - UNIT_ASSERT(SqlToYql("pragma PqReadBy='plato2'; use plato;").IsOk()); - UNIT_ASSERT(SqlToYql("$x='plato'; use rtmr:$x; pragma PqReadBy='plato2';").IsOk()); - UNIT_ASSERT(SqlToYql("use plato; pragma PqReadBy='dq';").IsOk()); - } - - Y_UNIT_TEST(MrObject) { - NYql::TAstParseResult res = SqlToYql( - "declare $path as String;\n" - "select * from plato.object($path, `format`, \"comp\" || \"ression\" as compression, 1 as bar) with schema (Int32 as y, String as x)" - ); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "MrObject") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, - line.find(R"__((MrObject (EvaluateAtom "$path") '"format" '('('"compression" (Concat (String '"comp") (String '"ression"))) '('"bar" (Int32 '"1")))))__")); - } else if (word == "userschema") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, - line.find(R"__('('('"userschema" (StructType '('"y" (DataType 'Int32)) '('"x" (DataType 'String))) '('"y" '"x"))))__")); - } - }; - - TWordCountHive elementStat = {{TString("MrObject"), 0}, {TString("userschema"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["MrObject"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["userschema"]); - } - - Y_UNIT_TEST(TableBindings) { - NSQLTranslation::TTranslationSettings settings = GetSettingsWithS3Binding("foo"); - NYql::TAstParseResult res = SqlToYqlWithSettings( - "select * from bindings.foo", - settings - ); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "MrObject") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, - line.find(R"__((MrTableConcat (Key '('table (String '"path")))) (Void) '('('"bar" '"1") '('"compression" '"ccompression") '('"format" '"format") '('"partitionedby" '"key" '"subkey") '('"userschema" (SqlTypeFromYson)__")); - } - }; - - TWordCountHive elementStat = {{TString("MrTableConcat"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["MrTableConcat"]); - - settings.DefaultCluster = "plato"; - settings.BindingsMode = NSQLTranslation::EBindingsMode::DISABLED; - res = SqlToYqlWithSettings( - "select * from bindings.foo", - settings - ); - UNIT_ASSERT_VALUES_EQUAL(Err2Str(res), "<main>:1:15: Error: Please remove 'bindings.' from your query, the support for this syntax has ended, code: 4601\n"); - UNIT_ASSERT(!res.Root); - - settings.BindingsMode = NSQLTranslation::EBindingsMode::DROP; - res = SqlToYqlWithSettings( - "select * from bindings.foo", - settings - ); - - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine2 = [](const TString& word, const TString& line) { - if (word == "MrTableConcat") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, - line.find(R"__((MrTableConcat (Key '('table (String '"foo")))) (Void) '())))__")); - } - }; - - TWordCountHive elementStat2 = {{TString("MrTableConcat"), 0}}; - VerifyProgram(res, elementStat2, verifyLine2); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat2["MrTableConcat"]); - - settings.BindingsMode = NSQLTranslation::EBindingsMode::DROP_WITH_WARNING; - res = SqlToYqlWithSettings( - "select * from bindings.foo", - settings - ); - UNIT_ASSERT_VALUES_EQUAL(Err2Str(res), "<main>:1:15: Warning: Please remove 'bindings.' from your query, the support for this syntax will be dropped soon, code: 4538\n"); - UNIT_ASSERT(res.Root); - - TWordCountHive elementStat3 = {{TString("MrTableConcat"), 0}}; - VerifyProgram(res, elementStat3, verifyLine2); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat3["MrTableConcat"]); - } - - Y_UNIT_TEST(TableBindingsWithInsert) { - NSQLTranslation::TTranslationSettings settings = GetSettingsWithS3Binding("foo"); - NYql::TAstParseResult res = SqlToYqlWithSettings( - "insert into bindings.foo with truncate (x, y) values (1, 2);", - settings - ); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write!") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, - line.find(R"__((Write! world sink (Key '('table (String '"path"))) values '('('"bar" '"1") '('"compression" '"ccompression") '('"format" '"format") '('"partitionedby" '"key" '"subkey") '('"userschema" (SqlTypeFromYson)__")); - } - }; - - TWordCountHive elementStat = {{TString("Write!"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write!"]); - - settings.DefaultCluster = "plato"; - settings.BindingsMode = NSQLTranslation::EBindingsMode::DISABLED; - res = SqlToYqlWithSettings( - "insert into bindings.foo with truncate (x, y) values (1, 2);", - settings - ); - UNIT_ASSERT_VALUES_EQUAL(Err2Str(res), "<main>:1:13: Error: Please remove 'bindings.' from your query, the support for this syntax has ended, code: 4601\n"); - UNIT_ASSERT(!res.Root); - - settings.BindingsMode = NSQLTranslation::EBindingsMode::DROP; - res = SqlToYqlWithSettings( - "insert into bindings.foo with truncate (x, y) values (1, 2);", - settings - ); - UNIT_ASSERT_VALUES_EQUAL(Err2Str(res), ""); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine2 = [](const TString& word, const TString& line) { - if (word == "Write!") { - //UNIT_ASSERT_VALUES_EQUAL(line, ""); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, - line.find(R"__((Write! world sink (Key '('table (String '"foo"))) values '('('mode 'renew)))__")); - } - }; - - TWordCountHive elementStat2 = {{TString("Write!"), 0}}; - VerifyProgram(res, elementStat2, verifyLine2); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat2["Write!"]); - - settings.BindingsMode = NSQLTranslation::EBindingsMode::DROP_WITH_WARNING; - res = SqlToYqlWithSettings( - "insert into bindings.foo with truncate (x, y) values (1, 2);", - settings - ); - UNIT_ASSERT_VALUES_EQUAL(Err2Str(res), "<main>:1:13: Warning: Please remove 'bindings.' from your query, the support for this syntax will be dropped soon, code: 4538\n"); - UNIT_ASSERT(res.Root); - - TWordCountHive elementStat3 = {{TString("Write!"), 0}}; - VerifyProgram(res, elementStat3, verifyLine2); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat3["Write!"]); - } - - Y_UNIT_TEST(TrailingCommaInWithout) { - UNIT_ASSERT(SqlToYql("SELECT * WITHOUT stream, FROM plato.Input").IsOk()); - UNIT_ASSERT(SqlToYql("SELECT a.* WITHOUT a.intersect, FROM plato.Input AS a").IsOk()); - UNIT_ASSERT(SqlToYql("SELECT a.* WITHOUT col1, col2, a.col3, FROM plato.Input AS a").IsOk()); - } - - Y_UNIT_TEST(NoStackOverflowOnBigCaseStatement) { - TStringBuilder req; - req << "select case 1 + 123"; - for (size_t i = 0; i < 20000; ++i) { - req << " when " << i << " then " << i + 1; - } - req << " else 100500 end;"; - UNIT_ASSERT(SqlToYql(req).IsOk()); - } - - Y_UNIT_TEST(CollectPreaggregatedInListLiteral) { - UNIT_ASSERT(SqlToYql("SELECT [COUNT(DISTINCT a+b)] FROM plato.Input").IsOk()); - } - - Y_UNIT_TEST(SmartParenInGroupByClause) { - UNIT_ASSERT(SqlToYql("SELECT * FROM plato.Input GROUP BY (k, v)").IsOk()); - } - - Y_UNIT_TEST(AlterTableRenameToIsCorrect) { - UNIT_ASSERT(SqlToYql("USE plato; ALTER TABLE table RENAME TO moved").IsOk()); - } - - Y_UNIT_TEST(AlterTableAddDropColumnIsCorrect) { - UNIT_ASSERT(SqlToYql("USE plato; ALTER TABLE table ADD COLUMN addc uint64, DROP COLUMN dropc, ADD addagain uint64").IsOk()); - } - - Y_UNIT_TEST(AlterTableSetTTLIsCorrect) { - UNIT_ASSERT(SqlToYql("USE plato; ALTER TABLE table SET (TTL = Interval(\"PT3H\") ON column)").IsOk()); - UNIT_ASSERT(SqlToYql("USE plato; ALTER TABLE table SET (TTL = Interval(\"PT3H\") ON column AS SECONDS)").IsOk()); - } - - Y_UNIT_TEST(AlterTableSetTieringIsCorrect) { - UNIT_ASSERT(SqlToYql("USE plato; ALTER TABLE table SET (TIERING = 'my_tiering')").IsOk()); - } - - Y_UNIT_TEST(AlterTableAddChangefeedIsCorrect) { - UNIT_ASSERT(SqlToYql("USE plato; ALTER TABLE table ADD CHANGEFEED feed WITH (MODE = 'UPDATES', FORMAT = 'json')").IsOk()); - } - - Y_UNIT_TEST(AlterTableAlterChangefeedIsCorrect) { - UNIT_ASSERT(SqlToYql("USE plato; ALTER TABLE table ALTER CHANGEFEED feed DISABLE").IsOk()); - ExpectFailWithError("USE plato; ALTER TABLE table ALTER CHANGEFEED feed SET (FORMAT = 'proto');", - "<main>:1:57: Error: FORMAT alter is not supported\n"); - } - - Y_UNIT_TEST(AlterTableDropChangefeedIsCorrect) { - UNIT_ASSERT(SqlToYql("USE plato; ALTER TABLE table DROP CHANGEFEED feed").IsOk()); - } - - Y_UNIT_TEST(AlterTableSetPartitioningIsCorrect) { - UNIT_ASSERT(SqlToYql("USE plato; ALTER TABLE table SET (AUTO_PARTITIONING_BY_SIZE = DISABLED)").IsOk()); - } - - Y_UNIT_TEST(AlterTableAddIndexWithIsNotSupported) { - ExpectFailWithFuzzyError("USE plato; ALTER TABLE table ADD INDEX idx GLOBAL ON (col) WITH (a=b)", - "<main>:1:40: Error: with: alternative is not implemented yet: \\d+:\\d+: global_index\\n"); - } - - Y_UNIT_TEST(AlterTableAddIndexLocalIsNotSupported) { - ExpectFailWithFuzzyError("USE plato; ALTER TABLE table ADD INDEX idx LOCAL ON (col)", - "<main>:1:40: Error: local: alternative is not implemented yet: \\d+:\\d+: local_index\\n"); - } - - Y_UNIT_TEST(CreateTableAddIndexVector) { - const auto result = SqlToYql(R"(USE plato; - CREATE TABLE table ( - pk INT32 NOT NULL, - col String, - INDEX idx GLOBAL USING vector_kmeans_tree - ON (col) COVER (col) - WITH (distance=cosine, vector_type=float, vector_dimension=1024,), - PRIMARY KEY (pk)) - )"); - UNIT_ASSERT_C(result.IsOk(), result.Issues.ToString()); - } - - Y_UNIT_TEST(AlterTableAddIndexVector) { - const auto result = SqlToYql(R"(USE plato; - ALTER TABLE table ADD INDEX idx - GLOBAL USING vector_kmeans_tree - ON (col) COVER (col) - WITH (distance=cosine, vector_type="float", vector_dimension=1024) - )"); - UNIT_ASSERT_C(result.IsOk(), result.Issues.ToString()); - } - - Y_UNIT_TEST(AlterTableAddIndexUnknownSubtype) { - ExpectFailWithError("USE plato; ALTER TABLE table ADD INDEX idx GLOBAL USING unknown ON (col)", - "<main>:1:57: Error: UNKNOWN index subtype is not supported\n"); - } - - Y_UNIT_TEST(AlterTableAddIndexMissedParameter) { - ExpectFailWithError(R"(USE plato; - ALTER TABLE table ADD INDEX idx - GLOBAL USING vector_kmeans_tree - ON (col) - WITH (distance=cosine, vector_type=float) - )", - "<main>:5:52: Error: vector_dimension should be set\n"); - } - - Y_UNIT_TEST(AlterTableAlterIndexSetPartitioningIsCorrect) { - const auto result = SqlToYql("USE plato; ALTER TABLE table ALTER INDEX index SET AUTO_PARTITIONING_MIN_PARTITIONS_COUNT 10"); - UNIT_ASSERT_C(result.IsOk(), result.Issues.ToString()); - } - - Y_UNIT_TEST(AlterTableAlterIndexSetMultiplePartitioningSettings) { - const auto result = SqlToYql("USE plato; ALTER TABLE table ALTER INDEX index SET " - "(AUTO_PARTITIONING_BY_LOAD = ENABLED, AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = 10)" - ); - UNIT_ASSERT_C(result.IsOk(), result.Issues.ToString()); - } - - Y_UNIT_TEST(AlterTableAlterIndexResetPartitioningIsNotSupported) { - ExpectFailWithError("USE plato; ALTER TABLE table ALTER INDEX index RESET (AUTO_PARTITIONING_MIN_PARTITIONS_COUNT)", - "<main>:1:55: Error: AUTO_PARTITIONING_MIN_PARTITIONS_COUNT reset is not supported\n" - ); - } - - Y_UNIT_TEST(AlterTableAlterColumnDropNotNullAstCorrect) { - auto reqSetNull = SqlToYql(R"( - USE plato; - CREATE TABLE tableName ( - id Uint32, - val Uint32 NOT NULL, - PRIMARY KEY (id) - ); - - COMMIT; - ALTER TABLE tableName ALTER COLUMN val DROP NOT NULL; - )"); - - UNIT_ASSERT(reqSetNull.IsOk()); - UNIT_ASSERT(reqSetNull.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - Y_UNUSED(word); - - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find( - R"(let world (Write! world sink (Key '('tablescheme (String '"tableName"))) (Void) '('('mode 'alter) '('actions '('('alterColumns '('('"val" '('changeColumnConstraints '('('drop_not_null)))))))))))" - )); - }; - - TWordCountHive elementStat({TString("\'mode \'alter")}); - VerifyProgram(reqSetNull, elementStat, verifyLine); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["\'mode \'alter"]); - } - - Y_UNIT_TEST(AlterSequence) { - UNIT_ASSERT(SqlToYql(R"( - USE plato; - ALTER SEQUENCE sequence START WITH 10 INCREMENT 2 RESTART WITH 5; - )").IsOk()); - UNIT_ASSERT(SqlToYql(R"( - USE plato; - ALTER SEQUENCE sequence INCREMENT 2; - )").IsOk()); - UNIT_ASSERT(SqlToYql(R"( - USE plato; - ALTER SEQUENCE sequence INCREMENT 2 START 1000; - )").IsOk()); - UNIT_ASSERT(SqlToYql(R"( - USE plato; - ALTER SEQUENCE sequence RESTART START 1000; - )").IsOk()); - UNIT_ASSERT(SqlToYql(R"( - USE plato; - ALTER SEQUENCE IF EXISTS sequence INCREMENT 1000 START 100 RESTART; - )").IsOk()); - UNIT_ASSERT(SqlToYql(R"( - USE plato; - ALTER SEQUENCE IF EXISTS sequence RESTART 1000 START WITH 100 INCREMENT BY 7; - )").IsOk()); - } - - Y_UNIT_TEST(AlterSequenceIncorrect) { - { - NYql::TAstParseResult res = SqlToYql("USE plato; ALTER SEQUENCE sequence START WITH 10 INCREMENT 2 RESTART WITH 5 RESTART;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:75: Error: Restart value defined more than once\n"); - } - { - NYql::TAstParseResult res = SqlToYql("USE plato; ALTER SEQUENCE sequence START WITH 10 INCREMENT 2 START 100 RESTART WITH 5;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:60: Error: Start value defined more than once\n"); - } - { - NYql::TAstParseResult res = SqlToYql("USE plato; ALTER SEQUENCE sequence INCREMENT BY 7 START WITH 10 INCREMENT 2 RESTART WITH 5 RESTART;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:62: Error: Increment defined more than once\n"); - } - { - NYql::TAstParseResult res = SqlToYql("USE plato; ALTER SEQUENCE sequence RESTART WITH 100 START WITH 10 INCREMENT 2 RESTART WITH 5;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:77: Error: Restart value defined more than once\n"); - } - { - NYql::TAstParseResult res = SqlToYql("USE plato; ALTER SEQUENCE sequence RESTART WITH 1234234543563435151456 START WITH 10 INCREMENT 2;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:49: Error: Failed to parse number from string: 1234234543563435151456, number limit overflow\n"); - } - { - NYql::TAstParseResult res = SqlToYql("USE plato; ALTER SEQUENCE sequence RESTART WITH 1 START WITH 9223372036854775817 INCREMENT 4;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:12: Error: Start value: 9223372036854775817 cannot be greater than max value: 9223372036854775807\n"); - } - { - NYql::TAstParseResult res = SqlToYql("USE plato; ALTER SEQUENCE sequence RESTART WITH 9223372036854775827 START WITH 5 INCREMENT 4;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:12: Error: Restart value: 9223372036854775827 cannot be greater than max value: 9223372036854775807\n"); - } - { - NYql::TAstParseResult res = SqlToYql("USE plato; ALTER SEQUENCE sequence RESTART WITH 1 START WITH 4 INCREMENT 0;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:12: Error: Increment must not be zero\n"); - } - { - NYql::TAstParseResult res = SqlToYql("USE plato; ALTER SEQUENCE sequence RESTART WITH 0 START WITH 4 INCREMENT 1;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:12: Error: Restart value: 0 cannot be less than min value: 1\n"); - } - { - NYql::TAstParseResult res = SqlToYql("USE plato; ALTER SEQUENCE sequence RESTART WITH 1 START WITH 0 INCREMENT 1;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:12: Error: Start value: 0 cannot be less than min value: 1\n"); - } - { - NYql::TAstParseResult res = SqlToYql("USE plato; ALTER SEQUENCE sequence RESTART WITH 1 START WITH 1 INCREMENT 9223372036854775837;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:12: Error: Increment: 9223372036854775837 cannot be greater than max value: 9223372036854775807\n"); - } - } - - Y_UNIT_TEST(AlterSequenceCorrect) { - { - NYql::TAstParseResult res = SqlToYql("USE plato; ALTER SEQUENCE sequence START WITH 10 INCREMENT 2 RESTART WITH 5;"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("sequence")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("alter")); - UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("alter_if_exists")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("start")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("increment")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("restart")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - { - NYql::TAstParseResult res = SqlToYql("USE plato; ALTER SEQUENCE IF EXISTS sequence INCREMENT 2 RESTART;"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("sequence")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("alter_if_exists")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("increment")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("restart")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - { - NYql::TAstParseResult res = SqlToYql("USE plato; ALTER SEQUENCE IF EXISTS sequence START 10 INCREMENT BY 2;"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("sequence")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("alter_if_exists")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("start")); - UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("restart")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("increment")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - } - - Y_UNIT_TEST(ShowCreateTable) { - NYql::TAstParseResult res = SqlToYql(R"( - USE plato; - SHOW CREATE TABLE user; - )"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Read") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("showCreateTable")); - } - }; - - TWordCountHive elementStat = {{TString("Read"), 0}, {TString("showCreateTable"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Read"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["showCreateTable"]); - } - - Y_UNIT_TEST(OptionalAliases) { - UNIT_ASSERT(SqlToYql("USE plato; SELECT foo FROM (SELECT key foo FROM Input);").IsOk()); - UNIT_ASSERT(SqlToYql("USE plato; SELECT a.x FROM Input1 a JOIN Input2 b ON a.key = b.key;").IsOk()); - UNIT_ASSERT(SqlToYql("USE plato; SELECT a.x FROM (VALUES (1,2), (3,4)) a(x,key) JOIN Input b ON a.key = b.key;").IsOk()); - } - - Y_UNIT_TEST(TableNameConstness) { - UNIT_ASSERT(SqlToYql("USE plato; $path = 'foo'; SELECT TableName($path), count(*) FROM Input;").IsOk()); - UNIT_ASSERT(SqlToYql("$path = 'foo'; SELECT TableName($path, 'yt'), count(*) FROM plato.Input;").IsOk()); - ExpectFailWithError("USE plato; SELECT TableName(), count(*) FROM plato.Input;", - "<main>:1:19: Error: Expression has to be an aggregation function or key column, because aggregation is used elsewhere in this subquery\n"); - } - - Y_UNIT_TEST(UseShouldWorkAsColumnName) { - UNIT_ASSERT(SqlToYql("select use from (select 1 as use);").IsOk()); - } - - Y_UNIT_TEST(TrueFalseWorkAfterDollar) { - UNIT_ASSERT(SqlToYql("$ true = false; SELECT $ true or false;").IsOk()); - UNIT_ASSERT(SqlToYql("$False = 0; SELECT $False;").IsOk()); - } - - Y_UNIT_TEST(WithSchemaEquals) { - UNIT_ASSERT(SqlToYql("select * from plato.T with schema Struct<a:Int32, b:String>;").IsOk()); - UNIT_ASSERT(SqlToYql("select * from plato.T with columns = Struct<a:Int32, b:String>;").IsOk()); - } - - Y_UNIT_TEST(WithNonStructSchemaS3) { - NSQLTranslation::TTranslationSettings settings; - settings.ClusterMapping["s3bucket"] = NYql::S3ProviderName; - UNIT_ASSERT(SqlToYqlWithSettings("select * from s3bucket.`foo` with schema (col1 Int32, String as col2, Int64 as col3);", settings).IsOk()); - } - - Y_UNIT_TEST(AllowNestedTuplesInGroupBy) { - NYql::TAstParseResult res = SqlToYql("select count(*) from plato.Input group by 1 + (x, y, z);"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [&](const TString& word, const TString& line) { - Y_UNUSED(word); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("(Aggregate core '('\"group0\")")); - }; - - TWordCountHive elementStat({"Aggregate"}); - VerifyProgram(res, elementStat, verifyLine); - UNIT_ASSERT(elementStat["Aggregate"] == 1); - } - - Y_UNIT_TEST(AllowGroupByWithParens) { - NYql::TAstParseResult res = SqlToYql("select count(*) from plato.Input group by (x, y as alias1, z);"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [&](const TString& word, const TString& line) { - Y_UNUSED(word); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("(Aggregate core '('\"x\" '\"alias1\" '\"z\")")); - }; - - TWordCountHive elementStat({"Aggregate"}); - VerifyProgram(res, elementStat, verifyLine); - UNIT_ASSERT(elementStat["Aggregate"] == 1); - } - - Y_UNIT_TEST(CreateAsyncReplicationParseCorrect) { - auto req = R"( - USE plato; - CREATE ASYNC REPLICATION MyReplication - FOR table1 AS table2, table3 AS table4 - WITH ( - CONNECTION_STRING = "grpc://localhost:2135/?database=/MyDatabase", - ENDPOINT = "localhost:2135", - DATABASE = "/MyDatabase" - ); - )"; - auto res = SqlToYql(req); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("MyReplication")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("create")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("table1")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("table2")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("table3")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("table4")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("connection_string")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("grpc://localhost:2135/?database=/MyDatabase")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("endpoint")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("localhost:2135")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("database")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("/MyDatabase")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(CreateAsyncReplicationUnsupportedSettings) { - auto reqTpl = R"( - USE plato; - CREATE ASYNC REPLICATION MyReplication - FOR table1 AS table2, table3 AS table4 - WITH ( - %s = "%s" - ) - )"; - - auto settings = THashMap<TString, TString>{ - {"STATE", "DONE"}, - {"FAILOVER_MODE", "FORCE"}, - }; - - for (const auto& [k, v] : settings) { - auto req = Sprintf(reqTpl, k.c_str(), v.c_str()); - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), Sprintf("<main>:6:%zu: Error: %s is not supported in CREATE\n", 20 + k.size(), k.c_str())); - } - } - - Y_UNIT_TEST(AsyncReplicationInvalidCommitInterval) { - auto req = R"( - USE plato; - CREATE ASYNC REPLICATION MyReplication - FOR table1 AS table2, table3 AS table4 - WITH ( - COMMIT_INTERVAL = "FOO" - ); - )"; - - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:6:35: Error: Literal of Interval type is expected for COMMIT_INTERVAL\n"); - } - - Y_UNIT_TEST(AlterAsyncReplicationParseCorrect) { - auto req = R"( - USE plato; - ALTER ASYNC REPLICATION MyReplication - SET ( - STATE = "DONE", - FAILOVER_MODE = "FORCE" - ); - )"; - auto res = SqlToYql(req); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("MyReplication")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("alter")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("state")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("DONE")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("failover_mode")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("FORCE")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(AlterAsyncReplicationSettings) { - auto reqTpl = R"( - USE plato; - ALTER ASYNC REPLICATION MyReplication - SET ( - %s = "%s" - ) - )"; - - auto settings = THashMap<TString, TString>{ - {"connection_string", "grpc://localhost:2135/?database=/MyDatabase"}, - {"endpoint", "localhost:2135"}, - {"database", "/MyDatabase"}, - {"token", "foo"}, - {"token_secret_name", "foo_secret_name"}, - {"user", "user"}, - {"password", "bar"}, - {"password_secret_name", "bar_secret_name"}, - }; - - for (const auto& [k, v] : settings) { - auto req = Sprintf(reqTpl, k.c_str(), v.c_str()); - auto res = SqlToYql(req); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [&k, &v](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("MyReplication")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("alter")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(k)); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(v)); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - } - - Y_UNIT_TEST(AlterAsyncReplicationUnsupportedSettings) { - { - auto req = R"( - USE plato; - ALTER ASYNC REPLICATION MyReplication SET (CONSISTENCY_LEVEL = "GLOBAL"); - )"; - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:80: Error: CONSISTENCY_LEVEL is not supported in ALTER\n"); - } - { - auto req = R"( - USE plato; - ALTER ASYNC REPLICATION MyReplication SET (COMMIT_INTERVAL = Interval("PT10S")); - )"; - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:87: Error: COMMIT_INTERVAL is not supported in ALTER\n"); - } - } - - Y_UNIT_TEST(AsyncReplicationInvalidSettings) { - auto req = R"( - USE plato; - ALTER ASYNC REPLICATION MyReplication SET (FOO = "BAR"); - )"; - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:62: Error: Unknown replication setting: FOO\n"); - } - - Y_UNIT_TEST(DropAsyncReplicationParseCorrect) { - auto req = R"( - USE plato; - DROP ASYNC REPLICATION MyReplication; - )"; - auto res = SqlToYql(req); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("MyReplication")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("drop")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(DropAsyncReplicationCascade) { - auto req = R"( - USE plato; - DROP ASYNC REPLICATION MyReplication CASCADE; - )"; - auto res = SqlToYql(req); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("dropCascade")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(PragmaCompactGroupBy) { - auto req = "PRAGMA CompactGroupBy; SELECT key, COUNT(*) FROM plato.Input GROUP BY key;"; - auto res = SqlToYql(req); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Aggregate") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'('compact)")); - } - }; - - TWordCountHive elementStat = { {TString("Aggregate"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Aggregate"]); - } - - Y_UNIT_TEST(PragmaDisableCompactGroupBy) { - auto req = "PRAGMA DisableCompactGroupBy; SELECT key, COUNT(*) FROM plato.Input GROUP /*+ compact() */ BY key;"; - auto res = SqlToYql(req); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Aggregate") { - UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("'('compact)")); - } - }; - - TWordCountHive elementStat = { {TString("Aggregate"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Aggregate"]); - } - - Y_UNIT_TEST(AutoSampleWorksWithNamedSubquery) { - UNIT_ASSERT(SqlToYql("$src = select * from plato.Input; select * from $src sample 0.2").IsOk()); - } - - Y_UNIT_TEST(AutoSampleWorksWithSubquery) { - UNIT_ASSERT(SqlToYql("select * from (select * from plato.Input) sample 0.2").IsOk()); - } - - Y_UNIT_TEST(CreateTableTrailingComma) { - UNIT_ASSERT(SqlToYql("USE plato; CREATE TABLE tableName (Key Uint32, PRIMARY KEY (Key),);").IsOk()); - UNIT_ASSERT(SqlToYql("USE plato; CREATE TABLE tableName (Key Uint32,);").IsOk()); - } - - Y_UNIT_TEST(BetweenSymmetric) { - UNIT_ASSERT(SqlToYql("select 3 between symmetric 5 and 4;").IsOk()); - UNIT_ASSERT(SqlToYql("select 3 between asymmetric 5 and 4;").IsOk()); - UNIT_ASSERT(SqlToYql("use plato; select key between symmetric and and and from Input;").IsOk()); - UNIT_ASSERT(SqlToYql("use plato; select key between and and and from Input;").IsOk()); - } -} - -Y_UNIT_TEST_SUITE(ExternalFunction) { - Y_UNIT_TEST(ValidUseFunctions) { - - UNIT_ASSERT(SqlToYql( - "PROCESS plato.Input" - " USING EXTERNAL FUNCTION('YANDEX-CLOUD', 'foo', <|a: 123, b: a + 641|>)" - " WITH INPUT_TYPE=Struct<a:Int32>, OUTPUT_TYPE=Struct<b:Int32>," - " CONCURRENCY=3, OPTIMIZE_FOR='CALLS'").IsOk()); - - // use CALLS without quotes, as keyword - UNIT_ASSERT(SqlToYql( - "PROCESS plato.Input" - " USING EXTERNAL FUNCTION('YANDEX-CLOUD', 'foo')" - " WITH INPUT_TYPE=Struct<a:Int32>, OUTPUT_TYPE=Struct<b:Int32>," - " OPTIMIZE_FOR=CALLS").IsOk()); - - UNIT_ASSERT(SqlToYql( - "PROCESS plato.Input" - " USING EXTERNAL FUNCTION('YANDEX-CLOUD', 'foo', TableRow())" - " WITH INPUT_TYPE=Struct<a:Int32>, OUTPUT_TYPE=Struct<b:Int32>," - " CONCURRENCY=3").IsOk()); - - UNIT_ASSERT(SqlToYql( - "PROCESS plato.Input" - " USING EXTERNAL FUNCTION('YANDEX-CLOUD', 'foo')" - " WITH INPUT_TYPE=Struct<a:Int32>, OUTPUT_TYPE=Struct<b:Int32>," - " CONCURRENCY=3, BATCH_SIZE=1000000, CONNECTION='yc-folder34fse-con'," - " INIT=[0, 900]").IsOk()); - - UNIT_ASSERT(SqlToYql( - "PROCESS plato.Input" - " USING EXTERNAL FUNCTION('YANDEX-CLOUD', 'bar', TableRow())" - " WITH UNKNOWN_PARAM_1='837747712', UNKNOWN_PARAM_2=Tuple<Uint16, Utf8>," - " INPUT_TYPE=Struct<a:Int32>, OUTPUT_TYPE=Struct<b:Int32>").IsOk()); - } - - - Y_UNIT_TEST(InValidUseFunctions) { - ExpectFailWithError("PROCESS plato.Input USING some::udf(*) WITH INPUT_TYPE=Struct<a:Int32>", - "<main>:1:33: Error: PROCESS without USING EXTERNAL FUNCTION doesn't allow WITH block\n"); - - ExpectFailWithError("PROCESS plato.Input USING EXTERNAL FUNCTION('YANDEX-CLOUD', 'jhhjfh88134d')" - " WITH INPUT_TYPE=Struct<a:Int32>, OUTPUT_TYPE=Struct<b:Int32>" - " ASSUME ORDER BY key", - "<main>:1:129: Error: PROCESS with USING EXTERNAL FUNCTION doesn't allow ASSUME block\n"); - - ExpectFailWithError("PROCESS plato.Input USING EXTERNAL FUNCTION('YANDEX-CLOUD', 'foo', 'bar', 'baz')", - "<main>:1:15: Error: EXTERNAL FUNCTION requires from 2 to 3 arguments, but got: 4\n"); - - ExpectFailWithError("PROCESS plato.Input\n" - " USING EXTERNAL FUNCTION('YANDEX-CLOUD', 'foo', <|field_1: a1, field_b: b1|>)\n" - " WITH INPUT_TYPE=Struct<a:Int32>, OUTPUT_TYPE=Struct<b:Int32>,\n" - " CONCURRENCY=3, BATCH_SIZE=1000000, CONNECTION='yc-folder34fse-con',\n" - " CONCURRENCY=5, INPUT_TYPE=Struct<b:Bool>,\n" - " INIT=[0, 900]\n", - "<main>:5:2: Error: WITH \"CONCURRENCY\" clause should be specified only once\n" - "<main>:5:17: Error: WITH \"INPUT_TYPE\" clause should be specified only once\n"); - } -} - -Y_UNIT_TEST_SUITE(SqlToYQLErrors) { - Y_UNIT_TEST(UdfSyntaxSugarMissingCall) { - auto req = "SELECT Udf(DateTime::FromString, \"foo\" as RunConfig);"; - auto res = SqlToYql(req); - TString a1 = Err2Str(res); - TString a2("<main>:1:8: Error: Abstract Udf Node can't be used as a part of expression.\n"); - UNIT_ASSERT_NO_DIFF(a1, a2); - } - - Y_UNIT_TEST(UdfSyntaxSugarIsNotCallable) { - auto req = "SELECT Udf(123, \"foo\" as RunConfig);"; - auto res = SqlToYql(req); - TString a1 = Err2Str(res); - TString a2("<main>:1:8: Error: Udf: first argument must be a callable, like Foo::Bar\n"); - UNIT_ASSERT_NO_DIFF(a1, a2); - } - - Y_UNIT_TEST(UdfSyntaxSugarNoArgs) { - auto req = "SELECT Udf()();"; - auto res = SqlToYql(req); - TString a1 = Err2Str(res); - TString a2("<main>:1:8: Error: Udf: expected at least one argument\n"); - UNIT_ASSERT_NO_DIFF(a1, a2); - } - - Y_UNIT_TEST(StrayUTF8) { - /// 'c' in plato is russian here - NYql::TAstParseResult res = SqlToYql("select * from сedar.Input"); - UNIT_ASSERT(!res.Root); - - TString a1 = Err2Str(res); - TString a2(R"foo(<main>:1:14: Error: Unexpected character 'с' (Unicode character <1089>) : cannot match to any predicted input... - -<main>:1:15: Error: Unexpected character : cannot match to any predicted input... - -)foo"); - - UNIT_ASSERT_NO_DIFF(a1, a2); - } - - Y_UNIT_TEST(IvalidStringLiteralWithEscapedBackslash) { - NYql::TAstParseResult res1 = SqlToYql(R"foo($bar = 'a\\'b';)foo"); - NYql::TAstParseResult res2 = SqlToYql(R"foo($bar = "a\\"b";)foo"); - UNIT_ASSERT(!res1.Root); - UNIT_ASSERT(!res2.Root); - - UNIT_ASSERT_NO_DIFF(Err2Str(res1), "<main>:1:15: Error: Unexpected character : syntax error...\n\n"); - UNIT_ASSERT_NO_DIFF(Err2Str(res2), "<main>:1:15: Error: Unexpected character : syntax error...\n\n"); - } - - Y_UNIT_TEST(InvalidHexInStringLiteral) { - NYql::TAstParseResult res = SqlToYql("select \"foo\\x1\\xfe\""); - UNIT_ASSERT(!res.Root); - TString a1 = Err2Str(res); - TString a2 = "<main>:1:15: Error: Failed to parse string literal: Invalid hexadecimal value\n"; - - UNIT_ASSERT_NO_DIFF(a1, a2); - } - - Y_UNIT_TEST(InvalidOctalInMultilineStringLiteral) { - NYql::TAstParseResult res = SqlToYql("select \"foo\n" - "bar\n" - "\\01\""); - UNIT_ASSERT(!res.Root); - TString a1 = Err2Str(res); - TString a2 = "<main>:3:4: Error: Failed to parse string literal: Invalid octal value\n"; - - UNIT_ASSERT_NO_DIFF(a1, a2); - } - - Y_UNIT_TEST(InvalidDoubleAtString) { - NYql::TAstParseResult res = SqlToYql("select @@@@@@"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:13: Error: Unexpected character : syntax error...\n\n"); - } - - Y_UNIT_TEST(InvalidDoubleAtStringWhichWasAcceptedEarlier) { - NYql::TAstParseResult res = SqlToYql("SELECT @@foo@@ @ @@bar@@"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:7: Error: Unexpected token '@@foo@@' : cannot match to any predicted input...\n\n"); - } - - Y_UNIT_TEST(InvalidStringFromTable) { - NYql::TAstParseResult res = SqlToYql("select \"FOO\"\"BAR from plato.foo"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:31: Error: Unexpected character : syntax error...\n\n"); - } - - Y_UNIT_TEST(InvalidDoubleAtStringFromTable) { - NYql::TAstParseResult res = SqlToYql("select @@@@@@ from plato.foo"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:28: Error: Unexpected character : syntax error...\n\n"); - } - - Y_UNIT_TEST(SelectInvalidSyntax) { - NYql::TAstParseResult res = SqlToYql("select 1 form Wat"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:14: Error: Unexpected token 'Wat' : cannot match to any predicted input...\n\n"); - } - - Y_UNIT_TEST(SelectNoCluster) { - NYql::TAstParseResult res = SqlToYql("select foo from bar"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:1: Error: No cluster name given and no default cluster is selected\n"); - } - - Y_UNIT_TEST(SelectDuplicateColumns) { - NYql::TAstParseResult res = SqlToYql("select a, a from plato.Input"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:11: Error: Unable to use duplicate column names. Collision in name: a\n"); - } - - Y_UNIT_TEST(SelectDuplicateLabels) { - NYql::TAstParseResult res = SqlToYql("select a as foo, b as foo from plato.Input"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:18: Error: Unable to use duplicate column names. Collision in name: foo\n"); - } - - Y_UNIT_TEST(SelectCaseWithoutThen) { - NYql::TAstParseResult res = SqlToYql("select case when true 1;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), - "<main>:1:22: Error: Unexpected token absence : Missing THEN \n\n" - "<main>:1:23: Error: Unexpected token absence : Missing END \n\n" - ); - } - - Y_UNIT_TEST(SelectComplexCaseWithoutThen) { - NYql::TAstParseResult res = SqlToYql( - "SELECT *\n" - "FROM plato.Input AS a\n" - "WHERE CASE WHEN a.key = \"foo\" a.subkey ELSE a.value END\n" - ); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:30: Error: Unexpected token absence : Missing THEN \n\n"); - } - - Y_UNIT_TEST(SelectCaseWithoutEnd) { - NYql::TAstParseResult res = SqlToYql("select case a when b then c end from plato.Input"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Error: ELSE is required\n"); - } - - Y_UNIT_TEST(SelectWithBadAggregationNoInput) { - NYql::TAstParseResult res = SqlToYql("select a, Min(b), c"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), - "<main>:1:1: Error: Column references are not allowed without FROM\n" - "<main>:1:8: Error: Column reference 'a'\n" - "<main>:1:1: Error: Column references are not allowed without FROM\n" - "<main>:1:15: Error: Column reference 'b'\n" - "<main>:1:1: Error: Column references are not allowed without FROM\n" - "<main>:1:19: Error: Column reference 'c'\n" - ); - } - - Y_UNIT_TEST(SelectWithBadAggregation) { - ExpectFailWithError("select count(*), 1 + key from plato.Input", - "<main>:1:22: Error: Column `key` must either be a key column in GROUP BY or it should be used in aggregation function\n"); - } - - Y_UNIT_TEST(SelectWithBadAggregatedTerms) { - ExpectFailWithError("select key, 2 * subkey from plato.Input group by key", - "<main>:1:17: Error: Column `subkey` must either be a key column in GROUP BY or it should be used in aggregation function\n"); - } - - Y_UNIT_TEST(SelectDistinctWithBadAggregation) { - ExpectFailWithError("select distinct count(*), 1 + key from plato.Input", - "<main>:1:31: Error: Column `key` must either be a key column in GROUP BY or it should be used in aggregation function\n"); - ExpectFailWithError("select distinct key, 2 * subkey from plato.Input group by key", - "<main>:1:26: Error: Column `subkey` must either be a key column in GROUP BY or it should be used in aggregation function\n"); - } - - Y_UNIT_TEST(SelectWithBadAggregationInHaving) { - ExpectFailWithError("select key from plato.Input group by key\n" - "having \"f\" || value == \"foo\"", - "<main>:2:15: Error: Column `value` must either be a key column in GROUP BY or it should be used in aggregation function\n"); - } - - Y_UNIT_TEST(JoinWithNonAggregatedColumnInProjection) { - ExpectFailWithError("select a.key, 1 + b.subkey\n" - "from plato.Input1 as a join plato.Input2 as b using(key)\n" - "group by a.key;", - "<main>:1:19: Error: Column `b.subkey` must either be a key column in GROUP BY or it should be used in aggregation function\n"); - - ExpectFailWithError("select a.key, 1 + b.subkey.x\n" - "from plato.Input1 as a join plato.Input2 as b using(key)\n" - "group by a.key;", - "<main>:1:19: Error: Column must either be a key column in GROUP BY or it should be used in aggregation function\n"); - } - - Y_UNIT_TEST(SelectWithBadAggregatedTermsWithSources) { - ExpectFailWithError("select key, 1 + a.subkey\n" - "from plato.Input1 as a\n" - "group by a.key;", - "<main>:1:17: Error: Column `a.subkey` must either be a key column in GROUP BY or it should be used in aggregation function\n"); - ExpectFailWithError("select key, 1 + a.subkey.x\n" - "from plato.Input1 as a\n" - "group by a.key;", - "<main>:1:17: Error: Column must either be a key column in GROUP BY or it should be used in aggregation function\n"); - } - - Y_UNIT_TEST(WarnForAggregationBySelectAlias) { - NYql::TAstParseResult res = SqlToYql("select c + 1 as c from plato.Input\n" - "group by c"); - UNIT_ASSERT(res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), - "<main>:2:11: Warning: GROUP BY will aggregate by column `c` instead of aggregating by SELECT expression with same alias, code: 4532\n" - "<main>:1:10: Warning: You should probably use alias in GROUP BY instead of using it here. Please consult documentation for more details, code: 4532\n"); - - res = SqlToYql("select c + 1 as c from plato.Input\n" - "group by Math::Floor(c + 2) as c;"); - - UNIT_ASSERT(res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), - "<main>:2:22: Warning: GROUP BY will aggregate by column `c` instead of aggregating by SELECT expression with same alias, code: 4532\n" - "<main>:1:10: Warning: You should probably use alias in GROUP BY instead of using it here. Please consult documentation for more details, code: 4532\n"); - } - - Y_UNIT_TEST(NoWarnForAggregationBySelectAliasWhenAggrFunctionsAreUsedInAlias) { - NYql::TAstParseResult res = SqlToYql("select\n" - " cast(avg(val) as int) as value,\n" - " value as key\n" - "from\n" - " plato.Input\n" - "group by value"); - - UNIT_ASSERT(res.Root); - UNIT_ASSERT(res.Issues.Size() == 0); - - res = SqlToYql("select\n" - " cast(avg(val) over w as int) as value,\n" - " value as key\n" - "from\n" - " plato.Input\n" - "group by value\n" - "window w as ()"); - - UNIT_ASSERT(res.Root); - UNIT_ASSERT(res.Issues.Size() == 0); - } - - Y_UNIT_TEST(NoWarnForAggregationBySelectAliasWhenQualifiedNameIsUsed) { - NYql::TAstParseResult res = SqlToYql("select\n" - " Unwrap(a.key) as key\n" - "from plato.Input as a\n" - "join plato.Input2 as b using(k)\n" - "group by a.key;"); - UNIT_ASSERT(res.Root); - UNIT_ASSERT(res.Issues.Size() == 0); - - res = SqlToYql("select Unwrap(a.key) as key\n" - "from plato.Input as a\n" - "group by a.key;"); - UNIT_ASSERT(res.Root); - UNIT_ASSERT(res.Issues.Size() == 0); - } - - Y_UNIT_TEST(NoWarnForAggregationBySelectAliasWhenTrivialRenamingIsUsed) { - NYql::TAstParseResult res = SqlToYql("select a.key as key\n" - "from plato.Input as a\n" - "group by key;"); - UNIT_ASSERT(res.Root); - UNIT_ASSERT(res.Issues.Size() == 0); - - res = SqlToYql("select key as key\n" - "from plato.Input\n" - "group by key;"); - UNIT_ASSERT(res.Root); - UNIT_ASSERT(res.Issues.Size() == 0); - } - - Y_UNIT_TEST(ErrorByAggregatingByExpressionWithSameExpressionInSelect) { - ExpectFailWithError("select k * 2 from plato.Input group by k * 2", - "<main>:1:8: Error: Column `k` must either be a key column in GROUP BY or it should be used in aggregation function\n"); - } - - Y_UNIT_TEST(ErrorForAggregationBySelectAlias) { - ExpectFailWithError("select key, Math::Floor(1.1 + a.subkey) as foo\n" - "from plato.Input as a\n" - "group by a.key, foo;", - "<main>:3:17: Warning: GROUP BY will aggregate by column `foo` instead of aggregating by SELECT expression with same alias, code: 4532\n" - "<main>:1:19: Warning: You should probably use alias in GROUP BY instead of using it here. Please consult documentation for more details, code: 4532\n" - "<main>:1:31: Error: Column `a.subkey` must either be a key column in GROUP BY or it should be used in aggregation function\n"); - - ExpectFailWithError("select c + 1 as c from plato.Input\n" - "group by Math::Floor(c + 2);", - "<main>:2:22: Warning: GROUP BY will aggregate by column `c` instead of aggregating by SELECT expression with same alias, code: 4532\n" - "<main>:1:10: Warning: You should probably use alias in GROUP BY instead of using it here. Please consult documentation for more details, code: 4532\n" - "<main>:1:8: Error: Column `c` must either be a key column in GROUP BY or it should be used in aggregation function\n"); - } - - Y_UNIT_TEST(ExplainQueryPlan) { - NYql::TAstParseResult res = SqlToYql("EXPLAIN Q U E R Y PLAN SELECT 1;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_STRING_CONTAINS(Err2Str(res), "<main>:1:8: Error: Unexpected token 'Q' : cannot match to any predicted input"); - } - - Y_UNIT_TEST(SelectWithDuplicateGroupingColumns) { - NYql::TAstParseResult res = SqlToYql("select c from plato.Input group by c, c"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Error: Duplicate grouping column: c\n"); - } - - Y_UNIT_TEST(SelectWithBadAggregationInGrouping) { - NYql::TAstParseResult res = SqlToYql("select a, Min(b), c group by c"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:1: Error: Column references are not allowed without FROM\n" - "<main>:1:30: Error: Column reference 'c'\n"); - } - - Y_UNIT_TEST(SelectWithOpOnBadAggregation) { - ExpectFailWithError("select 1 + a + Min(b) from plato.Input", - "<main>:1:12: Error: Column `a` must either be a key column in GROUP BY or it should be used in aggregation function\n"); - } - - Y_UNIT_TEST(SelectOrderByConstantNum) { - NYql::TAstParseResult res = SqlToYql("select a from plato.Input order by 1"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:36: Error: Unable to ORDER BY constant expression\n"); - } - - Y_UNIT_TEST(SelectOrderByConstantExpr) { - NYql::TAstParseResult res = SqlToYql("select a from plato.Input order by 1 * 42"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:38: Error: Unable to ORDER BY constant expression\n"); - } - - Y_UNIT_TEST(SelectOrderByConstantString) { - NYql::TAstParseResult res = SqlToYql("select a from plato.Input order by \"nest\""); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:36: Error: Unable to ORDER BY constant expression\n"); - } - - Y_UNIT_TEST(SelectOrderByAggregated) { - NYql::TAstParseResult res = SqlToYql("select a from plato.Input order by min(a)"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:36: Error: Unable to ORDER BY aggregated values\n"); - } - - Y_UNIT_TEST(ErrorInOrderByExpresison) { - NYql::TAstParseResult res = SqlToYql("select key, value from plato.Input order by (key as zey)"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:45: Error: You should use in ORDER BY column name, qualified field, callable function or expression\n"); - } - - Y_UNIT_TEST(ErrorsInOrderByWhenColumnIsMissingInProjection) { - ExpectFailWithError("select subkey from (select 1 as subkey) order by key", "<main>:1:50: Error: Column key is not in source column set\n"); - ExpectFailWithError("select subkey from plato.Input as a order by x.key", "<main>:1:46: Error: Unknown correlation name: x\n"); - ExpectFailWithError("select distinct a, b from plato.Input order by c", "<main>:1:48: Error: Column c is not in source column set. Did you mean a?\n"); - ExpectFailWithError("select count(*) as a from plato.Input order by c", "<main>:1:48: Error: Column c is not in source column set. Did you mean a?\n"); - ExpectFailWithError("select count(*) as a, b, from plato.Input group by b order by c", "<main>:1:63: Error: Column c is not in source column set. Did you mean a?\n"); - UNIT_ASSERT(SqlToYql("select a, b from plato.Input order by c").IsOk()); - } - - Y_UNIT_TEST(SelectAggregatedWhere) { - NYql::TAstParseResult res = SqlToYql("select * from plato.Input where count(key)"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:33: Error: Can not use aggregated values in filtering\n"); - } - - Y_UNIT_TEST(DoubleFrom) { - NYql::TAstParseResult res = SqlToYql("from plato.Input select * from plato.Input"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:27: Error: Only one FROM clause is allowed\n"); - } - - Y_UNIT_TEST(SelectJoinMissingCorrName) { - NYql::TAstParseResult res = SqlToYql("select * from plato.Input1 as a join plato.Input2 as b on a.key == key"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:65: Error: JOIN: column requires correlation name\n"); - } - - Y_UNIT_TEST(SelectJoinMissingCorrName1) { - NYql::TAstParseResult res = SqlToYql( - "use plato;\n" - "$foo = select * from Input1;\n" - "select * from Input2 join $foo USING(key);\n" - ); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:27: Error: JOIN: missing correlation name for source\n"); - } - - Y_UNIT_TEST(SelectJoinMissingCorrName2) { - NYql::TAstParseResult res = SqlToYql( - "use plato;\n" - "$foo = select * from Input1;\n" - "select * from Input2 cross join $foo;\n" - ); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:33: Error: JOIN: missing correlation name for source\n"); - } - - Y_UNIT_TEST(SelectJoinEmptyCorrNames) { - NYql::TAstParseResult res = SqlToYql( - "$left = (SELECT * FROM plato.Input1 LIMIT 2);\n" - "$right = (SELECT * FROM plato.Input2 LIMIT 2);\n" - "SELECT * FROM $left FULL JOIN $right USING (key);\n" - ); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:45: Error: At least one correlation name is required in join\n"); - } - - Y_UNIT_TEST(SelectJoinSameCorrNames) { - NYql::TAstParseResult res = SqlToYql("SELECT Input.key FROM plato.Input JOIN plato.Input1 ON Input.key == Input.subkey\n"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:66: Error: JOIN: different correlation names are required for joined tables\n"); - } - - Y_UNIT_TEST(SelectJoinConstPredicateArg) { - NYql::TAstParseResult res = SqlToYql("SELECT * FROM plato.Input1 as A JOIN plato.Input2 as B ON A.key == B.key AND A.subkey == \"wtf\"\n"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:87: Error: JOIN: each equality predicate argument must depend on exactly one JOIN input\n"); - } - - Y_UNIT_TEST(SelectJoinNonEqualityPredicate) { - NYql::TAstParseResult res = SqlToYql("SELECT * FROM plato.Input1 as A JOIN plato.Input2 as B ON A.key == B.key AND A.subkey > B.subkey\n"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:87: Error: JOIN ON expression must be a conjunction of equality predicates\n"); - } - - Y_UNIT_TEST(SelectEquiJoinCorrNameOutOfScope) { - NYql::TAstParseResult res = SqlToYql( - "PRAGMA equijoin;\n" - "SELECT * FROM plato.A JOIN plato.B ON A.key == C.key JOIN plato.C ON A.subkey == C.subkey;\n" - ); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:2:45: Error: JOIN: can not use source: C in equality predicate, it is out of current join scope\n"); - } - - Y_UNIT_TEST(SelectEquiJoinNoRightSource) { - NYql::TAstParseResult res = SqlToYql( - "PRAGMA equijoin;\n" - "SELECT * FROM plato.A JOIN plato.B ON A.key == B.key JOIN plato.C ON A.subkey == B.subkey;\n" - ); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:2:79: Error: JOIN ON equality predicate must have one of its arguments from the rightmost source\n"); - } - - Y_UNIT_TEST(SelectEquiJoinOuterWithoutType) { - NYql::TAstParseResult res = SqlToYql( - "SELECT * FROM plato.A Outer JOIN plato.B ON A.key == B.key;\n" - ); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:23: Error: Invalid join type: OUTER JOIN. OUTER keyword is optional and can only be used after LEFT, RIGHT or FULL\n"); - } - - Y_UNIT_TEST(SelectEquiJoinOuterWithWrongType) { - NYql::TAstParseResult res = SqlToYql( - "SELECT * FROM plato.A LEFT semi OUTER JOIN plato.B ON A.key == B.key;\n" - ); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:33: Error: Invalid join type: LEFT SEMI OUTER JOIN. OUTER keyword is optional and can only be used after LEFT, RIGHT or FULL\n"); - } - - Y_UNIT_TEST(InsertNoCluster) { - NYql::TAstParseResult res = SqlToYql("insert into Output (foo) values (1)"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:1: Error: No cluster name given and no default cluster is selected\n"); - } - - Y_UNIT_TEST(InsertValuesNoLabels) { - NYql::TAstParseResult res = SqlToYql("insert into plato.Output values (1)"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:19: Error: INSERT INTO ... VALUES requires specification of table columns\n"); - } - - Y_UNIT_TEST(UpsertValuesNoLabelsKikimr) { - NYql::TAstParseResult res = SqlToYql("upsert into plato.Output values (1)", 10, TString(NYql::KikimrProviderName)); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:19: Error: UPSERT INTO ... VALUES requires specification of table columns\n"); - } - - Y_UNIT_TEST(ReplaceValuesNoLabelsKikimr) { - NYql::TAstParseResult res = SqlToYql("replace into plato.Output values (1)", 10, TString(NYql::KikimrProviderName)); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:20: Error: REPLACE INTO ... VALUES requires specification of table columns\n"); - } - - Y_UNIT_TEST(InsertValuesInvalidLabels) { - NYql::TAstParseResult res = SqlToYql("insert into plato.Output (foo) values (1, 2)"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:27: Error: VALUES have 2 columns, INSERT INTO expects: 1\n"); - } - - Y_UNIT_TEST(BuiltinFileOpNoArgs) { - NYql::TAstParseResult res = SqlToYql("select FilePath()"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Error: FilePath() requires exactly 1 arguments, given: 0\n"); - } - - Y_UNIT_TEST(ProcessWithHaving) { - NYql::TAstParseResult res = SqlToYql("process plato.Input using some::udf(value) having value == 1"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:15: Error: PROCESS does not allow HAVING yet! You may request it on yql@ maillist.\n"); - } - - Y_UNIT_TEST(ReduceNoBy) { - NYql::TAstParseResult res = SqlToYql("reduce plato.Input using some::udf(value)"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:19: Error: Unexpected token absence : Missing ON \n\n<main>:1:25: Error: Unexpected token absence : Missing USING \n\n"); - } - - Y_UNIT_TEST(ReduceDistinct) { - NYql::TAstParseResult res = SqlToYql("reduce plato.Input on key using some::udf(distinct value)"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:43: Error: DISTINCT can not be used in PROCESS/REDUCE\n"); - } - - Y_UNIT_TEST(CreateTableWithView) { - NYql::TAstParseResult res = SqlToYql("CREATE TABLE plato.foo:bar (key INT);"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:22: Error: Unexpected token ':' : syntax error...\n\n"); - } - - Y_UNIT_TEST(AsteriskWithSomethingAfter) { - NYql::TAstParseResult res = SqlToYql("select *, LENGTH(value) from plato.Input;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Error: Unable to use plain '*' with other projection items. Please use qualified asterisk instead: '<table>.*' (<table> can be either table name or table alias).\n"); - } - - Y_UNIT_TEST(AsteriskWithSomethingBefore) { - NYql::TAstParseResult res = SqlToYql("select LENGTH(value), * from plato.Input;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:23: Error: Unable to use plain '*' with other projection items. Please use qualified asterisk instead: '<table>.*' (<table> can be either table name or table alias).\n"); - } - - Y_UNIT_TEST(DuplicatedQualifiedAsterisk) { - NYql::TAstParseResult res = SqlToYql("select in.*, key, in.* from plato.Input as in;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:19: Error: Unable to use twice same quialified asterisk. Invalid source: in\n"); - } - - Y_UNIT_TEST(BrokenLabel) { - NYql::TAstParseResult res = SqlToYql("select in.*, key as `funny.label` from plato.Input as in;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:14: Error: Unable to use '.' in column name. Invalid column name: funny.label\n"); - } - - Y_UNIT_TEST(KeyConflictDetect0) { - NYql::TAstParseResult res = SqlToYql("select key, in.key as key from plato.Input as in;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:13: Error: Unable to use duplicate column names. Collision in name: key\n"); - } - - Y_UNIT_TEST(KeyConflictDetect1) { - NYql::TAstParseResult res = SqlToYql("select length(key) as key, key from plato.Input;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:28: Error: Unable to use duplicate column names. Collision in name: key\n"); - } - - Y_UNIT_TEST(KeyConflictDetect2) { - NYql::TAstParseResult res = SqlToYql("select key, in.key from plato.Input as in;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:1: Error: Duplicate column: key\n"); - } - - Y_UNIT_TEST(AutogenerationAliasWithCollisionConflict1) { - UNIT_ASSERT(SqlToYql("select LENGTH(Value), key as column0 from plato.Input;").IsOk()); - } - - Y_UNIT_TEST(AutogenerationAliasWithCollisionConflict2) { - UNIT_ASSERT(SqlToYql("select key as column1, LENGTH(Value) from plato.Input;").IsOk()); - } - - Y_UNIT_TEST(MissedSourceTableForQualifiedAsteriskOnSimpleSelect) { - NYql::TAstParseResult res = SqlToYql("use plato; select Intop.*, Input.key from plato.Input;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:19: Error: Unknown correlation name: Intop\n"); - } - - Y_UNIT_TEST(MissedSourceTableForQualifiedAsteriskOnJoin) { - NYql::TAstParseResult res = SqlToYql("use plato; select tmissed.*, t2.*, t1.key from plato.Input as t1 join plato.Input as t2 on t1.key==t2.key;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:19: Error: Unknown correlation name for asterisk: tmissed\n"); - } - - Y_UNIT_TEST(UnableToReferenceOnNotExistSubcolumn) { - NYql::TAstParseResult res = SqlToYql("select b.subkey from (select key from plato.Input as a) as b;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Error: Column subkey is not in source column set\n"); - } - - Y_UNIT_TEST(ConflictOnSameNameWithQualify0) { - NYql::TAstParseResult res = SqlToYql("select in.key, in.key as key from plato.Input as in;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:1: Error: Duplicate column: key\n"); - } - - Y_UNIT_TEST(ConflictOnSameNameWithQualify1) { - NYql::TAstParseResult res = SqlToYql("select in.key, length(key) as key from plato.Input as in;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:1: Error: Duplicate column: key\n"); - } - - Y_UNIT_TEST(ConflictOnSameNameWithQualify2) { - NYql::TAstParseResult res = SqlToYql("select key, in.key from plato.Input as in;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:1: Error: Duplicate column: key\n"); - } - - Y_UNIT_TEST(ConflictOnSameNameWithQualify3) { - NYql::TAstParseResult res = SqlToYql("select in.key, subkey as key from plato.Input as in;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:1: Error: Duplicate column: key\n"); - } - - Y_UNIT_TEST(SelectFlattenBySameColumns) { - NYql::TAstParseResult res = SqlToYql("select key from plato.Input flatten by (key, key as kk)"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:46: Error: Duplicate column name found: key in FlattenBy section\n"); - } - - Y_UNIT_TEST(SelectFlattenBySameAliases) { - NYql::TAstParseResult res = SqlToYql("select key from plato.Input flatten by (key as kk, subkey as kk);"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:52: Error: Duplicate alias found: kk in FlattenBy section\n"); - } - - Y_UNIT_TEST(SelectFlattenByExprSameAliases) { - NYql::TAstParseResult res = SqlToYql("select key from plato.Input flatten by (key as kk, ListSkip(subkey,1) as kk);"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:52: Error: Collision between alias and column name: kk in FlattenBy section\n"); - } - - Y_UNIT_TEST(SelectFlattenByConflictNameAndAlias0) { - NYql::TAstParseResult res = SqlToYql("select key from plato.Input flatten by (key, subkey as key);"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:46: Error: Collision between alias and column name: key in FlattenBy section\n"); - } - - Y_UNIT_TEST(SelectFlattenByConflictNameAndAlias1) { - NYql::TAstParseResult res = SqlToYql("select key from plato.Input flatten by (key as kk, subkey as key);"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:52: Error: Collision between alias and column name: key in FlattenBy section\n"); - } - - Y_UNIT_TEST(SelectFlattenByExprConflictNameAndAlias1) { - NYql::TAstParseResult res = SqlToYql("select key from plato.Input flatten by (key as kk, ListSkip(subkey,1) as key);"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:52: Error: Duplicate column name found: key in FlattenBy section\n"); - } - - Y_UNIT_TEST(SelectFlattenByUnnamedExpr) { - NYql::TAstParseResult res = SqlToYql("select key from plato.Input flatten by (key, ListSkip(key, 1))"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:46: Error: Unnamed expression after FLATTEN BY is not allowed\n"); - } - - Y_UNIT_TEST(UseInOnStrings) { - NYql::TAstParseResult res = SqlToYql("select * from plato.Input where \"foo\" in \"foovalue\";"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:42: Error: Unable to use IN predicate with string argument, it won't search substring - " - "expecting tuple, list, dict or single column table source\n"); - } - - Y_UNIT_TEST(UseSubqueryInScalarContextInsideIn) { - NYql::TAstParseResult res = SqlToYql("$q = (select key from plato.Input); select * from plato.Input where subkey in ($q);"); - UNIT_ASSERT(res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:79: Warning: Using subrequest in scalar context after IN, " - "perhaps you should remove parenthesis here, code: 4501\n"); - } - - Y_UNIT_TEST(InHintsWithKeywordClash) { - NYql::TAstParseResult res = SqlToYql("SELECT COMPACT FROM plato.Input WHERE COMPACT IN COMPACT `COMPACT`(1,2,3)"); - UNIT_ASSERT(!res.Root); - // should try to parse last compact as call expression - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:58: Error: Unknown builtin: COMPACT\n"); - } - - Y_UNIT_TEST(ErrorColumnPosition) { - NYql::TAstParseResult res = SqlToYql( - "USE plato;\n" - "SELECT \n" - "value FROM (\n" - "select key from Input\n" - ");\n" - ); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:1: Error: Column value is not in source column set\n"); - } - - Y_UNIT_TEST(PrimaryViewAbortMapReduce) { - NYql::TAstParseResult res = SqlToYql("SELECT key FROM plato.Input VIEW PRIMARY KEY"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:17: Error: primary view is not supported for yt tables\n"); - } - - Y_UNIT_TEST(InsertAbortMapReduce) { - NYql::TAstParseResult res = SqlToYql("INSERT OR ABORT INTO plato.Output SELECT key FROM plato.Input"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:0: Error: INSERT OR ABORT INTO is not supported for yt tables\n"); - } - - Y_UNIT_TEST(ReplaceIntoMapReduce) { - NYql::TAstParseResult res = SqlToYql("REPLACE INTO plato.Output SELECT key FROM plato.Input"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:0: Error: Meaning of REPLACE INTO has been changed, now you should use INSERT INTO <table> WITH TRUNCATE ... for yt\n"); - } - - Y_UNIT_TEST(UpsertIntoMapReduce) { - NYql::TAstParseResult res = SqlToYql("UPSERT INTO plato.Output SELECT key FROM plato.Input"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:0: Error: UPSERT INTO is not supported for yt tables\n"); - } - - Y_UNIT_TEST(UpdateMapReduce) { - NYql::TAstParseResult res = SqlToYql("UPDATE plato.Output SET value = value + 1 WHERE key < 1"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:0: Error: UPDATE is unsupported for yt\n"); - } - - Y_UNIT_TEST(DeleteMapReduce) { - NYql::TAstParseResult res = SqlToYql("DELETE FROM plato.Output WHERE key < 1"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:0: Error: DELETE is unsupported for yt\n"); - } - - Y_UNIT_TEST(ReplaceIntoWithTruncate) { - NYql::TAstParseResult res = SqlToYql("REPLACE INTO plato.Output WITH TRUNCATE SELECT key FROM plato.Input"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:32: Error: Unable REPLACE INTO with truncate mode\n"); - } - - Y_UNIT_TEST(UpsertIntoWithTruncate) { - NYql::TAstParseResult res = SqlToYql("UPSERT INTO plato.Output WITH TRUNCATE SELECT key FROM plato.Input"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:31: Error: Unable UPSERT INTO with truncate mode\n"); - } - - Y_UNIT_TEST(InsertIntoWithTruncateKikimr) { - NYql::TAstParseResult res = SqlToYql("INSERT INTO plato.Output WITH TRUNCATE SELECT key FROM plato.Input", 10, TString(NYql::KikimrProviderName)); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:0: Error: INSERT INTO WITH TRUNCATE is not supported for kikimr tables\n"); - } - - Y_UNIT_TEST(InsertIntoWithWrongArgumentCount) { - NYql::TAstParseResult res = SqlToYql("insert into plato.Output with truncate (key, value, subkey) values (5, '1', '2', '3');"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:53: Error: VALUES have 4 columns, INSERT INTO ... WITH TRUNCATE expects: 3\n"); - } - - Y_UNIT_TEST(UpsertWithWrongArgumentCount) { - NYql::TAstParseResult res = SqlToYql("upsert into plato.Output (key, value, subkey) values (2, '3');", 10, TString(NYql::KikimrProviderName)); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:39: Error: VALUES have 2 columns, UPSERT INTO expects: 3\n"); - } - - Y_UNIT_TEST(GroupingSetByExprWithoutAlias) { - NYql::TAstParseResult res = SqlToYql("SELECT key FROM plato.Input GROUP BY GROUPING SETS (cast(key as uint32), subkey);"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:53: Error: Unnamed expressions are not supported in GROUPING SETS. Please use '<expr> AS <name>'.\n"); - } - - Y_UNIT_TEST(GroupingSetByExprWithoutAlias2) { - NYql::TAstParseResult res = SqlToYql("SELECT key FROM plato.Input GROUP BY subkey || subkey, GROUPING SETS (\n" - "cast(key as uint32), subkey);"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:2:1: Error: Unnamed expressions are not supported in GROUPING SETS. Please use '<expr> AS <name>'.\n"); - } - - Y_UNIT_TEST(CubeByExprWithoutAlias) { - NYql::TAstParseResult res = SqlToYql("SELECT key FROM plato.Input GROUP BY CUBE (key, subkey / key);"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:56: Error: Unnamed expressions are not supported in CUBE. Please use '<expr> AS <name>'.\n"); - } - - Y_UNIT_TEST(RollupByExprWithoutAlias) { - NYql::TAstParseResult res = SqlToYql("SELECT key FROM plato.Input GROUP BY ROLLUP (subkey / key);"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:53: Error: Unnamed expressions are not supported in ROLLUP. Please use '<expr> AS <name>'.\n"); - } - - Y_UNIT_TEST(GroupByHugeCubeDeniedNoPragma) { - NYql::TAstParseResult res = SqlToYql("SELECT key FROM plato.Input GROUP BY CUBE (key, subkey, value, key + subkey as sum, key - subkey as sub, key + val as keyval);"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:119: Error: GROUP BY CUBE is allowed only for 5 columns, but you use 6\n"); - } - - Y_UNIT_TEST(GroupByInvalidPragma) { - NYql::TAstParseResult res = SqlToYql("PRAGMA GroupByCubeLimit = '-4';"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:27: Error: Expected unsigned integer literal as a single argument for: GroupByCubeLimit\n"); - } - - Y_UNIT_TEST(GroupByHugeCubeDeniedPragme) { - NYql::TAstParseResult res = SqlToYql("PRAGMA GroupByCubeLimit = '4'; SELECT key FROM plato.Input GROUP BY CUBE (key, subkey, value, key + subkey as sum, key - subkey as sub);"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:132: Error: GROUP BY CUBE is allowed only for 4 columns, but you use 5\n"); - } - - Y_UNIT_TEST(GroupByFewBigCubes) { - NYql::TAstParseResult res = SqlToYql("SELECT key FROM plato.Input GROUP BY CUBE(key, subkey, key + subkey as sum), CUBE(value, value + key + subkey as total);"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:1: Error: Unable to GROUP BY more than 64 groups, you try use 80 groups\n"); - } - - Y_UNIT_TEST(GroupByFewBigCubesWithPragmaLimit) { - NYql::TAstParseResult res = SqlToYql("PRAGMA GroupByLimit = '16'; SELECT key FROM plato.Input GROUP BY GROUPING SETS(key, subkey, key + subkey as sum), ROLLUP(value, value + key + subkey as total);"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:29: Error: Unable to GROUP BY more than 16 groups, you try use 18 groups\n"); - } - - Y_UNIT_TEST(NoGroupingColumn0) { - NYql::TAstParseResult res = SqlToYql( - "select count(1), key_first, val_first, grouping(key_first, val_first, nomind) as group\n" - "from plato.Input group by grouping sets (cast(key as uint32) /100 as key_first, Substring(value, 1, 1) as val_first);"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:71: Error: Column 'nomind' is not a grouping column\n"); - } - - Y_UNIT_TEST(NoGroupingColumn1) { - NYql::TAstParseResult res = SqlToYql("select count(1), grouping(key, value) as group_duo from plato.Input group by cube (key, subkey);"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:32: Error: Column 'value' is not a grouping column\n"); - } - - Y_UNIT_TEST(EmptyAccess0) { - NYql::TAstParseResult res = SqlToYql("insert into plato.Output (list0, list1) values (AsList(0, 1, 2), AsList(``));"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:73: Error: Column reference \"\" is not allowed in current scope\n"); - } - - Y_UNIT_TEST(EmptyAccess1) { - NYql::TAstParseResult res = SqlToYql("insert into plato.Output (list0, list1) values (AsList(0, 1, 2), ``);"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:66: Error: Column reference \"\" is not allowed in current scope\n"); - } - - Y_UNIT_TEST(UseUnknownColumnInInsert) { - NYql::TAstParseResult res = SqlToYql("insert into plato.Output (list0, list1) values (AsList(0, 1, 2), AsList(`test`));"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:73: Error: Column reference \"test\" is not allowed in current scope\n"); - } - - Y_UNIT_TEST(GroupByEmptyColumn) { - NYql::TAstParseResult res = SqlToYql("select count(1) from plato.Input group by ``;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:43: Error: Column name can not be empty\n"); - } - - Y_UNIT_TEST(ConvertNumberOutOfBase) { - NYql::TAstParseResult res = SqlToYql("select 0o80l;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Error: Failed to parse number from string: 0o80l, char: '8' is out of base: 8\n"); - } - - Y_UNIT_TEST(ConvertNumberOutOfRangeForInt64ButFitsInUint64) { - NYql::TAstParseResult res = SqlToYql("select 0xc000000000000000l;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Error: Failed to parse 13835058055282163712 as integer literal of Int64 type: value out of range for Int64\n"); - } - - Y_UNIT_TEST(ConvertNumberOutOfRangeUint64) { - NYql::TAstParseResult res = SqlToYql("select 0xc0000000000000000l;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Error: Failed to parse number from string: 0xc0000000000000000l, number limit overflow\n"); - - res = SqlToYql("select 1234234543563435151456;\n"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Error: Failed to parse number from string: 1234234543563435151456, number limit overflow\n"); - } - - Y_UNIT_TEST(ConvertNumberNegativeOutOfRange) { - NYql::TAstParseResult res = SqlToYql("select -9223372036854775808;\n" - "select -9223372036854775809;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:2:8: Error: Failed to parse negative integer: -9223372036854775809, number limit overflow\n"); - } - - Y_UNIT_TEST(InvaildUsageReal0) { - NYql::TAstParseResult res = SqlToYql("select .0;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:7: Error: Unexpected token '.' : cannot match to any predicted input...\n\n"); - } - - Y_UNIT_TEST(InvaildUsageReal1) { - NYql::TAstParseResult res = SqlToYql("select .0f;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:7: Error: Unexpected token '.' : cannot match to any predicted input...\n\n"); - } - - Y_UNIT_TEST(InvaildUsageWinFunctionWithoutWindow) { - NYql::TAstParseResult res = SqlToYql("select lead(key, 2) from plato.Input;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Error: Failed to use window function Lead without window specification\n"); - } - - Y_UNIT_TEST(DropTableWithIfExists) { - NYql::TAstParseResult res = SqlToYql("DROP TABLE IF EXISTS plato.foo;"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("drop_if_exists")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(TooManyErrors) { - const char* q = R"( - USE plato; - select A, B, C, D, E, F, G, H, I, J, K, L, M, N from (select b from `abc`); -)"; - - NYql::TAstParseResult res = SqlToYql(q, 10); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), - R"(<main>:3:16: Error: Column A is not in source column set. Did you mean b? -<main>:3:19: Error: Column B is not in source column set. Did you mean b? -<main>:3:22: Error: Column C is not in source column set. Did you mean b? -<main>:3:25: Error: Column D is not in source column set. Did you mean b? -<main>:3:28: Error: Column E is not in source column set. Did you mean b? -<main>:3:31: Error: Column F is not in source column set. Did you mean b? -<main>:3:34: Error: Column G is not in source column set. Did you mean b? -<main>:3:37: Error: Column H is not in source column set. Did you mean b? -<main>:3:40: Error: Column I is not in source column set. Did you mean b? -<main>: Error: Too many issues, code: 1 -)"); - }; - - Y_UNIT_TEST(ShouldCloneBindingForNamedParameter) { - NYql::TAstParseResult res = SqlToYql(R"($f = () -> { - $value_type = TypeOf(1); - $pair_type = StructType( - TypeOf("2") AS key, - $value_type AS value - ); - - RETURN TupleType( - ListType($value_type), - $pair_type); -}; - -select FormatType($f()); -)"); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(BlockedInvalidFrameBounds) { - auto check = [](const TString& frame, const TString& err) { - const TString prefix = "SELECT SUM(x) OVER w FROM plato.Input WINDOW w AS (PARTITION BY key ORDER BY subkey\n"; - NYql::TAstParseResult res = SqlToYql(prefix + frame + ")"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), err); - }; - - check("ROWS UNBOUNDED FOLLOWING", "<main>:2:5: Error: Frame cannot start from UNBOUNDED FOLLOWING\n"); - check("ROWS BETWEEN 5 PRECEDING AND UNBOUNDED PRECEDING", "<main>:2:29: Error: Frame cannot end with UNBOUNDED PRECEDING\n"); - check("ROWS BETWEEN CURRENT ROW AND 5 PRECEDING", "<main>:2:13: Error: Frame cannot start from CURRENT ROW and end with PRECEDING\n"); - check("ROWS BETWEEN 5 FOLLOWING AND CURRENT ROW", "<main>:2:14: Error: Frame cannot start from FOLLOWING and end with CURRENT ROW\n"); - check("ROWS BETWEEN 5 FOLLOWING AND 5 PRECEDING", "<main>:2:14: Error: Frame cannot start from FOLLOWING and end with PRECEDING\n"); - } - - Y_UNIT_TEST(BlockedRangeValueWithoutSingleOrderBy) { - UNIT_ASSERT(SqlToYql("SELECT COUNT(*) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) FROM plato.Input").IsOk()); - UNIT_ASSERT(SqlToYql("SELECT COUNT(*) OVER (RANGE BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING) FROM plato.Input").IsOk()); - - auto res = SqlToYql("SELECT COUNT(*) OVER (RANGE 5 PRECEDING) FROM plato.Input"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:29: Error: RANGE with <offset> PRECEDING/FOLLOWING requires exactly one expression in ORDER BY partition clause\n"); - - res = SqlToYql("SELECT COUNT(*) OVER (ORDER BY key, value RANGE 5 PRECEDING) FROM plato.Input"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:49: Error: RANGE with <offset> PRECEDING/FOLLOWING requires exactly one expression in ORDER BY partition clause\n"); - } - - Y_UNIT_TEST(NoColumnsInFrameBounds) { - NYql::TAstParseResult res = SqlToYql( - "SELECT SUM(x) OVER w FROM plato.Input WINDOW w AS (ROWS BETWEEN\n" - " 1 + key PRECEDING AND 2 + key FOLLOWING);"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:2:6: Error: Column reference \"key\" is not allowed in current scope\n"); - } - - Y_UNIT_TEST(WarnOnEmptyFrameBounds) { - NYql::TAstParseResult res = SqlToYql( - "SELECT SUM(x) OVER w FROM plato.Input WINDOW w AS (PARTITION BY key ORDER BY subkey\n" - "ROWS BETWEEN 10 FOLLOWING AND 5 FOLLOWING)"); - UNIT_ASSERT(res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:2:14: Warning: Used frame specification implies empty window frame, code: 4520\n"); - } - - Y_UNIT_TEST(WarnOnRankWithUnorderedWindow) { - NYql::TAstParseResult res = SqlToYql("SELECT RANK() OVER w FROM plato.Input WINDOW w AS ()"); - UNIT_ASSERT(res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Warning: Rank() is used with unordered window - all rows will be considered equal to each other, code: 4521\n"); - } - - Y_UNIT_TEST(WarnOnRankExprWithUnorderedWindow) { - NYql::TAstParseResult res = SqlToYql("SELECT RANK(key) OVER w FROM plato.Input WINDOW w AS ()"); - UNIT_ASSERT(res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Warning: Rank(<expression>) is used with unordered window - the result is likely to be undefined, code: 4521\n"); - } - - Y_UNIT_TEST(AnyAsTableName) { - NYql::TAstParseResult res = SqlToYql("use plato; select * from any;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:28: Error: Unexpected token ';' : syntax error...\n\n"); - } - - Y_UNIT_TEST(IncorrectOrderOfLambdaOptionalArgs) { - NYql::TAstParseResult res = SqlToYql("$f = ($x?, $y)->($x + $y); select $f(1);"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:12: Error: Non-optional argument can not follow optional one\n"); - } - - Y_UNIT_TEST(IncorrectOrderOfActionOptionalArgs) { - NYql::TAstParseResult res = SqlToYql("define action $f($x?, $y) as select $x,$y; end define; do $f(1);"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:23: Error: Non-optional argument can not follow optional one\n"); - } - - Y_UNIT_TEST(NotAllowedQuestionOnNamedNode) { - NYql::TAstParseResult res = SqlToYql("$f = 1; select $f?;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:18: Error: Unexpected token '?' at the end of expression\n"); - } - - Y_UNIT_TEST(AnyAndCrossJoin) { - NYql::TAstParseResult res = SqlToYql("use plato; select * from any Input1 cross join Input2"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:26: Error: ANY should not be used with Cross JOIN\n"); - - res = SqlToYql("use plato; select * from Input1 cross join any Input2"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:44: Error: ANY should not be used with Cross JOIN\n"); - } - - Y_UNIT_TEST(AnyWithCartesianProduct) { - NYql::TAstParseResult res = SqlToYql("pragma AnsiImplicitCrossJoin; use plato; select * from any Input1, Input2"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:56: Error: ANY should not be used with Cross JOIN\n"); - - res = SqlToYql("pragma AnsiImplicitCrossJoin; use plato; select * from Input1, any Input2"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:64: Error: ANY should not be used with Cross JOIN\n"); - } - - Y_UNIT_TEST(ErrorPlainEndAsInlineActionTerminator) { - NYql::TAstParseResult res = SqlToYql( - "do begin\n" - " select 1\n" - "; end\n"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:4:0: Error: Unexpected token absence : Missing DO \n\n"); - } - - Y_UNIT_TEST(ErrorMultiWayJoinWithUsing) { - NYql::TAstParseResult res = SqlToYql( - "USE plato;\n" - "PRAGMA DisableSimpleColumns;\n" - "SELECT *\n" - "FROM Input1 AS a\n" - "JOIN Input2 AS b USING(key)\n" - "JOIN Input3 AS c ON a.key = c.key;\n" - ); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), - "<main>:5:24: Error: Multi-way JOINs should be connected with ON clause instead of USING clause\n" - ); - } - - Y_UNIT_TEST(RequireLabelInFlattenByWithDot) { - NYql::TAstParseResult res = SqlToYql("select * from plato.Input flatten by x.y"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), - "<main>:1:40: Error: Unnamed expression after FLATTEN BY is not allowed\n" - ); - } - - Y_UNIT_TEST(WarnUnnamedColumns) { - NYql::TAstParseResult res = SqlToYql( - "PRAGMA WarnUnnamedColumns;\n" - "\n" - "SELECT key, subkey, key || subkey FROM plato.Input ORDER BY subkey;\n"); - UNIT_ASSERT(res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:28: Warning: Autogenerated column name column2 will be used for expression, code: 4516\n"); - } - - Y_UNIT_TEST(WarnSourceColumnMismatch) { - NYql::TAstParseResult res = SqlToYql( - "insert into plato.Output (key, subkey, new_value, one_more_value) select key as Key, subkey, value, \"x\" from plato.Input;"); - UNIT_ASSERT(res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:51: Warning: Column names in SELECT don't match column specification in parenthesis. \"key\" doesn't match \"Key\". \"new_value\" doesn't match \"value\", code: 4517\n"); - } - - Y_UNIT_TEST(YtCaseInsensitive) { - NYql::TAstParseResult res = SqlToYql("select * from PlatO.foo;"); - UNIT_ASSERT(res.Root); - - res = SqlToYql("use PlatO; select * from foo;"); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(KikimrCaseSensitive) { - NYql::TAstParseResult res = SqlToYql("select * from PlatO.foo;", 10, "kikimr"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:15: Error: Unknown cluster: PlatO\n"); - - res = SqlToYql("use PlatO; select * from foo;", 10, "kikimr"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:5: Error: Unknown cluster: PlatO\n"); - } - - Y_UNIT_TEST(DiscoveryModeForbidden) { - NYql::TAstParseResult res = SqlToYqlWithMode("insert into plato.Output select * from plato.range(\"\", Input1, Input4)", NSQLTranslation::ESqlMode::DISCOVERY); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:40: Error: range is not allowed in Discovery mode, code: 4600\n"); - - res = SqlToYqlWithMode("insert into plato.Output select * from plato.like(\"\", \"Input%\")", NSQLTranslation::ESqlMode::DISCOVERY); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:40: Error: like is not allowed in Discovery mode, code: 4600\n"); - - res = SqlToYqlWithMode("insert into plato.Output select * from plato.regexp(\"\", \"Input.\")", NSQLTranslation::ESqlMode::DISCOVERY); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:40: Error: regexp is not allowed in Discovery mode, code: 4600\n"); - - res = SqlToYqlWithMode("insert into plato.Output select * from plato.filter(\"\", ($name) -> { return find($name, \"Input\") is not null; })", NSQLTranslation::ESqlMode::DISCOVERY); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:40: Error: filter is not allowed in Discovery mode, code: 4600\n"); - - res = SqlToYqlWithMode("select Path from plato.folder(\"\") where Type == \"table\"", NSQLTranslation::ESqlMode::DISCOVERY); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:18: Error: folder is not allowed in Discovery mode, code: 4600\n"); - } - - Y_UNIT_TEST(YsonFuncWithoutArgs) { - UNIT_ASSERT(SqlToYql("SELECT Yson::SerializeText(Yson::From());").IsOk()); - } - - Y_UNIT_TEST(CanNotUseOrderByInNonLastSelectInUnionAllChain) { - auto req = "pragma AnsiOrderByLimitInUnionAll;\n" - "use plato;\n" - "\n" - "select * from Input order by key\n" - "union all\n" - "select * from Input order by key limit 1;"; - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:4:21: Error: ORDER BY within UNION ALL is only allowed after last subquery\n"); - } - - Y_UNIT_TEST(CanNotUseLimitInNonLastSelectInUnionAllChain) { - auto req = "pragma AnsiOrderByLimitInUnionAll;\n" - "use plato;\n" - "\n" - "select * from Input limit 1\n" - "union all\n" - "select * from Input order by key limit 1;"; - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:4:21: Error: LIMIT within UNION ALL is only allowed after last subquery\n"); - } - - Y_UNIT_TEST(CanNotUseDiscardInNonFirstSelectInUnionAllChain) { - auto req = "pragma AnsiOrderByLimitInUnionAll;\n" - "use plato;\n" - "\n" - "select * from Input\n" - "union all\n" - "discard select * from Input;"; - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:6:1: Error: DISCARD within UNION ALL is only allowed before first subquery\n"); - } - - Y_UNIT_TEST(CanNotUseIntoResultInNonLastSelectInUnionAllChain) { - auto req = "use plato;\n" - "pragma AnsiOrderByLimitInUnionAll;\n" - "\n" - "select * from Input\n" - "union all\n" - "discard select * from Input;"; - - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:6:1: Error: DISCARD within UNION ALL is only allowed before first subquery\n"); - } - - Y_UNIT_TEST(YsonStrictInvalidPragma) { - auto res = SqlToYql("pragma yson.Strict = \"wrong\";"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:22: Error: Expected 'true', 'false' or no parameter for: Strict\n"); - } - - Y_UNIT_TEST(WarnTableNameInSomeContexts) { - UNIT_ASSERT(SqlToYql("use plato; select TableName() from Input;").IsOk()); - UNIT_ASSERT(SqlToYql("use plato; select TableName(\"aaaa\");").IsOk()); - UNIT_ASSERT(SqlToYql("select TableName(\"aaaa\", \"yt\");").IsOk()); - - auto res = SqlToYql("select TableName() from plato.Input;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Error: TableName requires either service name as second argument or current cluster name\n"); - - res = SqlToYql("use plato;\n" - "select TableName() from Input1 as a join Input2 as b using(key);"); - UNIT_ASSERT(res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:2:8: Warning: TableName() may produce empty result when used in ambiguous context (with JOIN), code: 4525\n"); - - res = SqlToYql("use plato;\n" - "select SOME(TableName()), key from Input group by key;"); - UNIT_ASSERT(res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:2:13: Warning: TableName() will produce empty result when used with aggregation.\n" - "Please consult documentation for possible workaround, code: 4525\n"); - } - - Y_UNIT_TEST(WarnOnDistincWithHavingWithoutAggregations) { - auto res = SqlToYql("select distinct key from plato.Input having key != '0';"); - UNIT_ASSERT(res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:49: Warning: The usage of HAVING without aggregations with SELECT DISTINCT is non-standard and will stop working soon. Please use WHERE instead., code: 4526\n"); - } - - Y_UNIT_TEST(FlattenByExprWithNestedNull) { - auto res = SqlToYql("USE plato;\n" - "\n" - "SELECT * FROM (SELECT 1 AS region_id)\n" - "FLATTEN BY (\n" - " CAST($unknown(region_id) AS List<String>) AS region\n" - ")"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:5:10: Error: Unknown name: $unknown\n"); - } - - Y_UNIT_TEST(EmptySymbolNameIsForbidden) { - auto req = " $`` = 1; select $``;"; - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:5: Error: Empty symbol name is not allowed\n"); - } - - Y_UNIT_TEST(WarnOnBinaryOpWithNullArg) { - auto req = "select * from plato.Input where cast(key as Int32) != NULL"; - auto res = SqlToYql(req); - UNIT_ASSERT(res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:52: Warning: Binary operation != will return NULL here, code: 4529\n"); - - req = "select 1 or null"; - res = SqlToYql(req); - UNIT_ASSERT(res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), ""); - } - - Y_UNIT_TEST(ErrorIfTableSampleArgUsesColumns) { - auto req = "SELECT key FROM plato.Input TABLESAMPLE BERNOULLI(MIN_OF(100.0, CAST(subkey as Int32)));"; - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:70: Error: Column reference \"subkey\" is not allowed in current scope\n"); - } - - Y_UNIT_TEST(DerivedColumnListForSelectIsNotSupportedYet) { - auto req = "SELECT a,b,c FROM plato.Input as t(x,y,z);"; - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:35: Error: Derived column list is only supported for VALUES\n"); - } - - Y_UNIT_TEST(ErrorIfValuesHasDifferentCountOfColumns) { - auto req = "VALUES (1,2,3), (4,5);"; - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:17: Error: All VALUES items should have same size: expecting 3, got 2\n"); - } - - Y_UNIT_TEST(ErrorIfDerivedColumnSizeExceedValuesColumnCount) { - auto req = "SELECT * FROM(VALUES (1,2), (3,4)) as t(x,y,z);"; - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:40: Error: Derived column list size exceeds column count in VALUES\n"); - } - - Y_UNIT_TEST(WarnoOnAutogeneratedNamesForValues) { - auto req = "PRAGMA WarnUnnamedColumns;\n" - "SELECT * FROM (VALUES (1,2,3,4), (5,6,7,8)) as t(x,y);"; - auto res = SqlToYql(req); - UNIT_ASSERT(res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:2:16: Warning: Autogenerated column names column2...column3 will be used here, code: 4516\n"); - } - - Y_UNIT_TEST(ErrUnionAllWithOrderByWithoutExplicitLegacyMode) { - auto req = "use plato;\n" - "\n" - "select * from Input order by key\n" - "union all\n" - "select * from Input order by key;"; - - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:21: Error: ORDER BY within UNION ALL is only allowed after last subquery\n"); - } - - Y_UNIT_TEST(ErrUnionAllWithLimitWithoutExplicitLegacyMode) { - auto req = "use plato;\n" - "\n" - "select * from Input limit 10\n" - "union all\n" - "select * from Input limit 1;"; - - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:21: Error: LIMIT within UNION ALL is only allowed after last subquery\n"); - } - - Y_UNIT_TEST(ErrUnionAllWithIntoResultWithoutExplicitLegacyMode) { - auto req = "use plato;\n" - "\n" - "select * from Input into result aaa\n" - "union all\n" - "select * from Input;"; - - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:21: Error: INTO RESULT within UNION ALL is only allowed after last subquery\n"); - } - - Y_UNIT_TEST(ErrUnionAllWithDiscardWithoutExplicitLegacyMode) { - auto req = "use plato;\n" - "\n" - "select * from Input\n" - "union all\n" - "discard select * from Input;"; - - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:5:1: Error: DISCARD within UNION ALL is only allowed before first subquery\n"); - } - - Y_UNIT_TEST(ErrUnionAllKeepsIgnoredOrderByWarning) { - auto req = "use plato;\n" - "\n" - "SELECT * FROM (\n" - " SELECT * FROM Input\n" - " UNION ALL\n" - " SELECT t.* FROM Input AS t ORDER BY t.key\n" - ");"; - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:4:3: Warning: ORDER BY without LIMIT in subquery will be ignored, code: 4504\n" - "<main>:6:39: Error: Unknown correlation name: t\n"); - } - - Y_UNIT_TEST(ErrOrderByIgnoredButCheckedForMissingColumns) { - auto req = "$src = SELECT key FROM (SELECT 1 as key, 2 as subkey) ORDER BY x; SELECT * FROM $src;"; - ExpectFailWithError(req, "<main>:1:8: Warning: ORDER BY without LIMIT in subquery will be ignored, code: 4504\n" - "<main>:1:64: Error: Column x is not in source column set\n"); - - req = "$src = SELECT key FROM plato.Input ORDER BY x; SELECT * FROM $src;"; - auto res = SqlToYql(req); - UNIT_ASSERT(res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Warning: ORDER BY without LIMIT in subquery will be ignored, code: 4504\n"); - } - - Y_UNIT_TEST(InvalidTtlInterval) { - auto req = R"( - USE plato; - CREATE TABLE tableName (Key Uint32, CreatedAt Timestamp, PRIMARY KEY (Key)) - WITH (TTL = 1 On CreatedAt); - )"; - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:4:25: Error: Literal of Interval type is expected for TTL\n" - "<main>:4:25: Error: Invalid TTL settings\n"); - } - - Y_UNIT_TEST(InvalidTtlUnit) { - auto req = R"( - USE plato; - CREATE TABLE tableName (Key Uint32, CreatedAt Uint32, PRIMARY KEY (Key)) - WITH (TTL = Interval("P1D") On CreatedAt AS PICOSECONDS); - )"; - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_STRING_CONTAINS(Err2Str(res), "<main>:4:56: Error: Unexpected token 'PICOSECONDS'"); - } - - Y_UNIT_TEST(InvalidChangefeedSink) { - auto req = R"( - USE plato; - CREATE TABLE tableName ( - Key Uint32, PRIMARY KEY (Key), - CHANGEFEED feedName WITH (SINK_TYPE = "S3", MODE = "KEYS_ONLY", FORMAT = "json") - ); - )"; - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:5:55: Error: Unknown changefeed sink type: S3\n"); - } - - Y_UNIT_TEST(InvalidChangefeedSettings) { - auto req = R"( - USE plato; - CREATE TABLE tableName ( - Key Uint32, PRIMARY KEY (Key), - CHANGEFEED feedName WITH (SINK_TYPE = "local", FOO = "bar") - ); - )"; - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:5:64: Error: Unknown changefeed setting: FOO\n"); - } - - Y_UNIT_TEST(InvalidChangefeedInitialScan) { - auto req = R"( - USE plato; - CREATE TABLE tableName ( - Key Uint32, PRIMARY KEY (Key), - CHANGEFEED feedName WITH (MODE = "KEYS_ONLY", FORMAT = "json", INITIAL_SCAN = "foo") - ); - )"; - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:5:95: Error: Literal of Bool type is expected for INITIAL_SCAN\n"); - } - - Y_UNIT_TEST(InvalidChangefeedVirtualTimestamps) { - auto req = R"( - USE plato; - CREATE TABLE tableName ( - Key Uint32, PRIMARY KEY (Key), - CHANGEFEED feedName WITH (MODE = "KEYS_ONLY", FORMAT = "json", VIRTUAL_TIMESTAMPS = "foo") - ); - )"; - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:5:101: Error: Literal of Bool type is expected for VIRTUAL_TIMESTAMPS\n"); - } - - Y_UNIT_TEST(InvalidChangefeedResolvedTimestamps) { - auto req = R"( - USE plato; - CREATE TABLE tableName ( - Key Uint32, PRIMARY KEY (Key), - CHANGEFEED feedName WITH (MODE = "KEYS_ONLY", FORMAT = "json", BARRIERS_INTERVAL = "foo") - ); - )"; - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:5:100: Error: Literal of Interval type is expected for BARRIERS_INTERVAL\n"); - } - - Y_UNIT_TEST(InvalidChangefeedRetentionPeriod) { - auto req = R"( - USE plato; - CREATE TABLE tableName ( - Key Uint32, PRIMARY KEY (Key), - CHANGEFEED feedName WITH (MODE = "KEYS_ONLY", FORMAT = "json", RETENTION_PERIOD = "foo") - ); - )"; - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:5:99: Error: Literal of Interval type is expected for RETENTION_PERIOD\n"); - } - - Y_UNIT_TEST(InvalidChangefeedTopicPartitions) { - auto req = R"( - USE plato; - CREATE TABLE tableName ( - Key Uint32, PRIMARY KEY (Key), - CHANGEFEED feedName WITH (MODE = "KEYS_ONLY", FORMAT = "json", TOPIC_MIN_ACTIVE_PARTITIONS = "foo") - ); - )"; - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:5:110: Error: Literal of integer type is expected for TOPIC_MIN_ACTIVE_PARTITIONS\n"); - } - - Y_UNIT_TEST(InvalidChangefeedAwsRegion) { - auto req = R"( - USE plato; - CREATE TABLE tableName ( - Key Uint32, PRIMARY KEY (Key), - CHANGEFEED feedName WITH (MODE = "KEYS_ONLY", FORMAT = "json", AWS_REGION = true) - ); - )"; - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:5:93: Error: Literal of String type is expected for AWS_REGION\n"); - } - - Y_UNIT_TEST(ErrJoinWithGroupingSetsWithoutCorrelationName) { - auto req = "USE plato;\n" - "\n" - "SELECT k1, k2, subkey\n" - "FROM T1 AS a JOIN T2 AS b USING (key)\n" - "GROUP BY GROUPING SETS(\n" - " (a.key as k1, b.subkey as k2),\n" - " (k1),\n" - " (subkey)\n" - ");"; - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:8:4: Error: Columns in grouping sets should have correlation name, error in key: subkey\n"); - } - - Y_UNIT_TEST(ErrJoinWithGroupByWithoutCorrelationName) { - auto req = "USE plato;\n" - "\n" - "SELECT k1, k2,\n" - " value\n" - "FROM T1 AS a JOIN T2 AS b USING (key)\n" - "GROUP BY a.key as k1, b.subkey as k2,\n" - " value;"; - ExpectFailWithError(req, - "<main>:7:5: Error: Columns in GROUP BY should have correlation name, error in key: value\n"); - } - - Y_UNIT_TEST(ErrWithMissingFrom) { - auto req = "select 1 as key where 1 > 1;"; - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:25: Error: Filtering is not allowed without FROM\n"); - - req = "select 1 + count(*);"; - res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:12: Error: Aggregation is not allowed without FROM\n"); - - req = "select 1 as key, subkey + value;"; - res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:1: Error: Column references are not allowed without FROM\n" - "<main>:1:18: Error: Column reference 'subkey'\n" - "<main>:1:1: Error: Column references are not allowed without FROM\n" - "<main>:1:27: Error: Column reference 'value'\n"); - - req = "select count(1) group by key;"; - res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:1: Error: Column references are not allowed without FROM\n" - "<main>:1:26: Error: Column reference 'key'\n"); - } - - Y_UNIT_TEST(ErrWithMissingFromForWindow) { - auto req = "$c = () -> (1 + count(1) over w);\n" - "select $c();"; - ExpectFailWithError(req, - "<main>:1:9: Error: Window and aggregation functions are not allowed in this context\n" - "<main>:1:17: Error: Failed to use aggregation function Count without window specification or in wrong place\n"); - - req = "$c = () -> (1 + lead(1) over w);\n" - "select $c();"; - ExpectFailWithError(req, - "<main>:1:17: Error: Window functions are not allowed in this context\n" - "<main>:1:17: Error: Failed to use window function Lead without window specification or in wrong place\n"); - - req = "select 1 + count(1) over w window w as ();"; - ExpectFailWithError(req, - "<main>:1:1: Error: Window and aggregation functions are not allowed without FROM\n" - "<main>:1:12: Error: Failed to use aggregation function Count without window specification or in wrong place\n"); - - req = "select 1 + lead(1) over w window w as ();"; - ExpectFailWithError(req, - "<main>:1:12: Error: Window functions are not allowed without FROM\n" - "<main>:1:12: Error: Failed to use window function Lead without window specification or in wrong place\n"); - } - - Y_UNIT_TEST(ErrWithMissingFromForInplaceWindow) { - auto req = "$c = () -> (1 + count(1) over ());\n" - "select $c();"; - ExpectFailWithError(req, - "<main>:1:26: Error: Window and aggregation functions are not allowed in this context\n"); - - req = "$c = () -> (1 + lead(1) over (rows between unbounded preceding and current row));\n" - "select $c();"; - ExpectFailWithError(req, - "<main>:1:25: Error: Window and aggregation functions are not allowed in this context\n"); - - req = "select 1 + count(1) over ();"; - ExpectFailWithError(req, - "<main>:1:1: Error: Window and aggregation functions are not allowed without FROM\n" - "<main>:1:12: Error: Failed to use aggregation function Count without window specification or in wrong place\n"); - - req = "select 1 + lead(1) over (rows between current row and unbounded following);"; - ExpectFailWithError(req, - "<main>:1:12: Error: Window functions are not allowed without FROM\n" - "<main>:1:12: Error: Failed to use window function Lead without window specification or in wrong place\n"); - } - - Y_UNIT_TEST(ErrDistinctInWrongPlace) { - auto req = "select Some::Udf(distinct key) from plato.Input;"; - ExpectFailWithError(req, - "<main>:1:18: Error: DISTINCT can only be used in aggregation functions\n"); - req = "select sum(key)(distinct foo) from plato.Input;"; - ExpectFailWithError(req, - "<main>:1:17: Error: DISTINCT can only be used in aggregation functions\n"); - - req = "select len(distinct foo) from plato.Input;"; - ExpectFailWithError(req, - "<main>:1:8: Error: DISTINCT can only be used in aggregation functions\n"); - - req = "$foo = ($x) -> ($x); select $foo(distinct key) from plato.Input;"; - ExpectFailWithError(req, - "<main>:1:34: Error: DISTINCT can only be used in aggregation functions\n"); - } - - Y_UNIT_TEST(ErrForNotSingleChildInInlineAST) { - ExpectFailWithError("select YQL::\"\"", - "<main>:1:8: Error: Failed to parse YQL: expecting AST root node with single child, but got 0\n"); - ExpectFailWithError("select YQL::@@ \t@@", - "<main>:1:8: Error: Failed to parse YQL: expecting AST root node with single child, but got 0\n"); - auto req = "$lambda = YQL::@@(lambda '(x)(+ x x)) (lambda '(y)(+ y y))@@;\n" - "select ListMap([1, 2, 3], $lambda);"; - ExpectFailWithError(req, - "<main>:1:11: Error: Failed to parse YQL: expecting AST root node with single child, but got 2\n"); - } - - Y_UNIT_TEST(ErrEmptyColumnName) { - ExpectFailWithError("select * without \"\" from plato.Input", - "<main>:1:18: Error: String literal can not be used here\n"); - - ExpectFailWithError("select * without `` from plato.Input;", - "<main>:1:18: Error: Empty column name is not allowed\n"); - - ExpectFailWithErrorForAnsiLexer("select * without \"\" from plato.Input", - "<main>:1:18: Error: Empty column name is not allowed\n"); - - ExpectFailWithErrorForAnsiLexer("select * without `` from plato.Input;", - "<main>:1:18: Error: Empty column name is not allowed\n"); - } - - Y_UNIT_TEST(ErrOnNonZeroArgumentsForTableRows) { - ExpectFailWithError("$udf=\"\";process plato.Input using $udf(TableRows(k))", - "<main>:1:40: Error: TableRows requires exactly 0 arguments\n"); - } - - Y_UNIT_TEST(ErrGroupByWithAggregationFunctionAndDistinctExpr) { - ExpectFailWithError("select * from plato.Input group by count(distinct key|key)", - "<main>:1:36: Error: Unable to GROUP BY aggregated values\n"); - } - - // FIXME: check if we can get old behaviour -#if 0 - Y_UNIT_TEST(ErrWithSchemaWithColumnsWithoutType) { - ExpectFailWithError("select * from plato.Input with COLUMNs", - "<main>:1:32: Error: Expected type after COLUMNS\n" - "<main>:1:32: Error: Failed to parse table hints\n"); - - ExpectFailWithError("select * from plato.Input with scheMa", - "<main>:1:32: Error: Expected type after SCHEMA\n" - "<main>:1:32: Error: Failed to parse table hints\n"); - } -#endif - - Y_UNIT_TEST(ErrCollectPreaggregatedInListLiteralWithoutFrom) { - ExpectFailWithError("SELECT([VARIANCE(DISTINCT[])])", - "<main>:1:1: Error: Column references are not allowed without FROM\n" - "<main>:1:9: Error: Column reference '_yql_preagg_Variance0'\n"); - } - - Y_UNIT_TEST(ErrGroupBySmartParenAsTuple) { - ExpectFailWithError("SELECT * FROM plato.Input GROUP BY (k, v,)", - "<main>:1:41: Error: Unexpected trailing comma in grouping elements list\n"); - } - - Y_UNIT_TEST(HandleNestedSmartParensInGroupBy) { - ExpectFailWithError("SELECT * FROM plato.Input GROUP BY (+() as k)", - "<main>:1:37: Error: Unable to GROUP BY constant expression\n"); - } - - Y_UNIT_TEST(ErrRenameWithAddColumn) { - ExpectFailWithError("USE plato; ALTER TABLE table RENAME TO moved, ADD COLUMN addc uint64", - "<main>:1:40: Error: RENAME TO can not be used together with another table action\n"); - } - - Y_UNIT_TEST(ErrAddColumnAndRename) { - // FIXME: fix positions in ALTER TABLE - ExpectFailWithError("USE plato; ALTER TABLE table ADD COLUMN addc uint64, RENAME TO moved", - "<main>:1:46: Error: RENAME TO can not be used together with another table action\n"); - } - - Y_UNIT_TEST(InvalidUuidValue) { - ExpectFailWithError("SELECT Uuid('123e4567ae89ba12d3aa456a426614174ab0')", - "<main>:1:8: Error: Invalid value \"123e4567ae89ba12d3aa456a426614174ab0\" for type Uuid\n"); - ExpectFailWithError("SELECT Uuid('123e4567ae89b-12d3-a456-426614174000')", - "<main>:1:8: Error: Invalid value \"123e4567ae89b-12d3-a456-426614174000\" for type Uuid\n"); - } - - Y_UNIT_TEST(WindowFunctionWithoutOver) { - ExpectFailWithError("SELECT LAST_VALUE(foo) FROM plato.Input", - "<main>:1:8: Error: Can't use window function LastValue without window specification (OVER keyword is missing)\n"); - ExpectFailWithError("SELECT LAST_VALUE(foo) FROM plato.Input GROUP BY key", - "<main>:1:8: Error: Can't use window function LastValue without window specification (OVER keyword is missing)\n"); - } - - Y_UNIT_TEST(CreateAlterUserWithLoginNoLogin) { - auto reqCreateUser = SqlToYql(R"( - USE plato; - CREATE USER user1; - )"); - - UNIT_ASSERT(reqCreateUser.IsOk()); - - auto reqAlterUser = SqlToYql(R"( - USE plato; - ALTER USER user1; - )"); - - UNIT_ASSERT(!reqAlterUser.IsOk()); - UNIT_ASSERT_STRING_CONTAINS(reqAlterUser.Issues.ToString(), "Error: Unexpected token ';' : cannot match to any predicted input..."); - - auto reqPasswordAndLogin = SqlToYql(R"( - USE plato; - CREATE USER user1 PASSWORD '123' LOGIN; - )"); - - UNIT_ASSERT(reqPasswordAndLogin.IsOk()); - - auto reqPasswordAndNoLogin = SqlToYql(R"( - USE plato; - CREATE USER user1 PASSWORD '123' NOLOGIN; - )"); - - UNIT_ASSERT(reqPasswordAndNoLogin.IsOk()); - - auto reqLogin = SqlToYql(R"( - USE plato; - CREATE USER user1 LOGIN; - )"); - - UNIT_ASSERT(reqLogin.IsOk()); - - auto reqNoLogin = SqlToYql(R"( - USE plato; - CREATE USER user1 NOLOGIN; - )"); - - UNIT_ASSERT(reqNoLogin.IsOk()); - - auto reqLoginNoLogin = SqlToYql(R"( - USE plato; - CREATE USER user1 LOGIN NOLOGIN; - )"); - - UNIT_ASSERT(!reqLoginNoLogin.IsOk()); - UNIT_ASSERT_STRING_CONTAINS(reqLoginNoLogin.Issues.ToString(), "Error: Conflicting or redundant options"); - - auto reqAlterLoginNoLogin = SqlToYql(R"( - USE plato; - CREATE USER user1 LOGIN; - ALTER USER user1 NOLOGIN; - )"); - - UNIT_ASSERT(reqAlterLoginNoLogin.IsOk()); - - auto reqAlterLoginNoLoginWithPassword = SqlToYql(R"( - USE plato; - CREATE USER user1 LOGIN; - ALTER USER user1 PASSWORD '321' NOLOGIN; - )"); - - UNIT_ASSERT(reqAlterLoginNoLoginWithPassword.IsOk()); - } - - Y_UNIT_TEST(CreateUserWithHash) { - auto reqCreateUser = SqlToYql(R"( - USE plato; - CREATE USER user1 HASH '{ - "hash": "p4ffeMugohqyBwyckYCK1TjJfz3LIHbKiGL+t+oEhzw=", - "salt": "U+tzBtgo06EBQCjlARA6Jg==", - "type": "argon2id" - }'; - )"); - - UNIT_ASSERT(reqCreateUser.IsOk()); - - auto reqCreateUserWithNoLogin = SqlToYql(R"( - USE plato; - CREATE USER user1 HASH '{ - "hash": "p4ffeMugohqyBwyckYCK1TjJfz3LIHbKiGL+t+oEhzw=", - "salt": "U+tzBtgo06EBQCjlARA6Jg==", - "type": "argon2id" - }' - NOLOGIN; - )"); - - UNIT_ASSERT(reqCreateUserWithNoLogin.IsOk()); - - auto reqCreateUserWithPassword = SqlToYql(R"( - USE plato; - CREATE USER user1 HASH '{ - "hash": "p4ffeMugohqyBwyckYCK1TjJfz3LIHbKiGL+t+oEhzw=", - "salt": "U+tzBtgo06EBQCjlARA6Jg==", - "type": "argon2id" - }' - PASSWORD '123'; - )"); - - UNIT_ASSERT(!reqCreateUserWithPassword.IsOk()); - UNIT_ASSERT_STRING_CONTAINS(reqCreateUserWithPassword.Issues.ToString(), "Error: Conflicting or redundant options"); - - auto reqAlterUser = SqlToYql(R"( - USE plato; - CREATE USER user1; - ALTER USER user1 HASH '{ - "hash": "p4ffeMugohqyBwyckYCK1TjJfz3LIHbKiGL+t+oEhzw=", - "salt": "U+tzBtgo06EBQCjlARA6Jg==", - "type": "argon2id" - }'; - )"); - - UNIT_ASSERT(reqAlterUser.IsOk()); - } - - Y_UNIT_TEST(CreateAlterUserWithoutCluster) { - ExpectFailWithError("\n CREATE USER user ENCRYPTED PASSWORD 'foobar';", "<main>:2:2: Error: USE statement is missing - no default cluster is selected\n"); - ExpectFailWithError("ALTER USER CURRENT_USER RENAME TO $foo;", "<main>:1:1: Error: USE statement is missing - no default cluster is selected\n"); - } - - Y_UNIT_TEST(ModifyPermissionsWithoutCluster) { - ExpectFailWithError("\n GRANT CONNECT ON `/Root` TO user;", "<main>:2:2: Error: USE statement is missing - no default cluster is selected\n"); - ExpectFailWithError("\n REVOKE MANAGE ON `/Root` FROM user;", "<main>:2:2: Error: USE statement is missing - no default cluster is selected\n"); - } - - Y_UNIT_TEST(ReservedRoleNames) { - ExpectFailWithError("USE plato; CREATE USER current_User;", "<main>:1:24: Error: System role CURRENT_USER can not be used here\n"); - ExpectFailWithError("USE plato; ALTER USER current_User RENAME TO Current_role", "<main>:1:46: Error: System role CURRENT_ROLE can not be used here\n"); - UNIT_ASSERT(SqlToYql("USE plato; DROP GROUP IF EXISTS a, b, c, current_User;").IsOk()); - } - - Y_UNIT_TEST(DisableClassicDivisionWithError) { - ExpectFailWithError("pragma ClassicDivision = 'false'; select $foo / 30;", "<main>:1:42: Error: Unknown name: $foo\n"); - } - - Y_UNIT_TEST(AggregationOfAgrregatedDistinctExpr) { - ExpectFailWithError("select sum(sum(distinct x + 1)) from plato.Input", "<main>:1:12: Error: Aggregation of aggregated values is forbidden\n"); - } - - Y_UNIT_TEST(WarnForUnusedSqlHint) { - NYql::TAstParseResult res = SqlToYql("select * from plato.Input1 as a join /*+ merge() */ plato.Input2 as b using(key);\n" - "select --+ foo(bar)\n" - " 1;"); - UNIT_ASSERT(res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:2:23: Warning: Hint foo will not be used, code: 4534\n"); - } - - Y_UNIT_TEST(WarnForDeprecatedSchema) { - NSQLTranslation::TTranslationSettings settings; - settings.ClusterMapping["s3bucket"] = NYql::S3ProviderName; - NYql::TAstParseResult res = SqlToYqlWithSettings("select * from s3bucket.`foo` with schema (col1 Int32, String as col2, Int64 as col3);", settings); - UNIT_ASSERT(res.Root); - UNIT_ASSERT_STRING_CONTAINS(res.Issues.ToString(), "Warning: Deprecated syntax for positional schema: please use 'column type' instead of 'type AS column', code: 4535\n"); - } - - Y_UNIT_TEST(ErrorOnColumnNameInMaxByLimit) { - ExpectFailWithError( - "SELECT AGGREGATE_BY(AsTuple(value, key), AggregationFactory(\"MAX_BY\", subkey)) FROM plato.Input;", - "<main>:1:42: Error: Source does not allow column references\n" - "<main>:1:71: Error: Column reference 'subkey'\n"); - } - - Y_UNIT_TEST(ErrorInLibraryWithTopLevelNamedSubquery) { - TString withUnusedSubq = "$unused = select max(key) from plato.Input;\n" - "\n" - "define subquery $foo() as\n" - " $count = select count(*) from plato.Input;\n" - " select * from plato.Input limit $count / 2;\n" - "end define;\n" - "export $foo;\n"; - UNIT_ASSERT(SqlToYqlWithMode(withUnusedSubq, NSQLTranslation::ESqlMode::LIBRARY).IsOk()); - - TString withTopLevelSubq = "$count = select count(*) from plato.Input;\n" - "\n" - "define subquery $foo() as\n" - " select * from plato.Input limit $count / 2;\n" - "end define;\n" - "export $foo;\n"; - auto res = SqlToYqlWithMode(withTopLevelSubq, NSQLTranslation::ESqlMode::LIBRARY); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:17: Error: Named subquery can not be used as a top level statement in libraries\n"); - } - - Y_UNIT_TEST(SessionStartAndSessionStateShouldSurviveSessionWindowArgsError){ - TString query = R"( - $init = ($_row) -> (min(1, 2)); -- error: aggregation func min() can not be used here - $calculate = ($_row, $_state) -> (1); - $update = ($_row, $_state) -> (2); - SELECT - SessionStart() over w as session_start, - SessionState() over w as session_state, - FROM plato.Input as t - WINDOW w AS ( - PARTITION BY user, SessionWindow(ts + 1, $init, $update, $calculate) - ) - )"; - ExpectFailWithError(query, "<main>:2:33: Error: Aggregation function Min requires exactly 1 argument(s), given: 2\n"); - } - - Y_UNIT_TEST(ScalarContextUsage1) { - TString query = R"( - $a = (select 1 as x, 2 as y); - select 1 + $a; - )"; - ExpectFailWithError(query, "<main>:2:39: Error: Source used in expression should contain one concrete column\n" - "<main>:3:24: Error: Source is used here\n"); - } - - Y_UNIT_TEST(ScalarContextUsage2) { - TString query = R"( - use plato; - $a = (select 1 as x, 2 as y); - select * from concat($a); - )"; - ExpectFailWithError(query, "<main>:3:39: Error: Source used in expression should contain one concrete column\n" - "<main>:4:34: Error: Source is used here\n"); - } - - Y_UNIT_TEST(ScalarContextUsage3) { - TString query = R"( - use plato; - $a = (select 1 as x, 2 as y); - select * from range($a); - )"; - ExpectFailWithError(query, "<main>:3:39: Error: Source used in expression should contain one concrete column\n" - "<main>:4:33: Error: Source is used here\n"); - } - - Y_UNIT_TEST(ScalarContextUsage4) { - TString query = R"( - use plato; - $a = (select 1 as x, 2 as y); - insert into $a select 1; - )"; - ExpectFailWithError(query, "<main>:3:39: Error: Source used in expression should contain one concrete column\n" - "<main>:4:25: Error: Source is used here\n"); - } -} - -void CheckUnused(const TString& req, const TString& symbol, unsigned row, unsigned col) { - auto res = SqlToYql(req); - - UNIT_ASSERT(res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), TStringBuilder() << "<main>:" << row << ":" << col << ": Warning: Symbol " << symbol << " is not used, code: 4527\n"); -} - -Y_UNIT_TEST_SUITE(WarnUnused) { - Y_UNIT_TEST(ActionOrSubquery) { - TString req = " $a()\n" - "as select 1;\n" - "end define;\n" - "\n" - "select 1;"; - CheckUnused("define action\n" + req, "$a", 2, 3); - CheckUnused("define subquery\n" + req, "$a", 2, 3); - } - - Y_UNIT_TEST(Import) { - TString req = "import lib1 symbols\n" - " $sqr;\n" - "select 1;"; - CheckUnused(req, "$sqr", 2, 3); - - req = "import lib1 symbols\n" - " $sqr as\n" - " $sq;\n" - "select 1;"; - CheckUnused(req, "$sq", 3, 5); - } - - Y_UNIT_TEST(NamedNodeStatement) { - TString req = " $a, $a = AsTuple(1, 2);\n" - "select $a;"; - CheckUnused(req, "$a", 1, 2); - req = "$a, $b = AsTuple(1, 2);\n" - "select $a;"; - CheckUnused(req, "$b", 1, 6); - CheckUnused(" $a = 1; $a = 2; select $a;", "$a", 1, 2); - } - - Y_UNIT_TEST(Declare) { - CheckUnused("declare $a as String;select 1;", "$a", 1, 9); - } - - Y_UNIT_TEST(ActionParams) { - TString req = "define action $a($x, $y) as\n" - " select $x;\n" - "end define;\n" - "\n" - "do $a(1,2);"; - CheckUnused(req, "$y", 1, 22); - } - - Y_UNIT_TEST(SubqueryParams) { - TString req = "use plato;\n" - "define subquery $q($name, $x) as\n" - " select * from $name;\n" - "end define;\n" - "\n" - "select * from $q(\"Input\", 1);"; - CheckUnused(req, "$x", 2, 27); - } - - Y_UNIT_TEST(For) { - TString req = "define action $a() as\n" - " select 1;\n" - "end define;\n" - "\n" - "for $i in ListFromRange(1, 10)\n" - "do $a();"; - CheckUnused(req, "$i", 5, 5); - } - - Y_UNIT_TEST(LambdaParams) { - TString req = "$lambda = ($x, $y) -> ($x);\n" - "select $lambda(1, 2);"; - CheckUnused(req, "$y", 1, 16); - } - - Y_UNIT_TEST(InsideLambdaBody) { - TString req = "$lambda = () -> {\n" - " $x = 1; return 1;\n" - "};\n" - "select $lambda();"; - CheckUnused(req, "$x", 2, 3); - req = "$lambda = () -> {\n" - " $x = 1; $x = 2; return $x;\n" - "};\n" - "select $lambda();"; - CheckUnused(req, "$x", 2, 3); - } - - Y_UNIT_TEST(InsideAction) { - TString req = "define action $a() as\n" - " $x = 1; select 1;\n" - "end define;\n" - "\n" - "do $a();"; - CheckUnused(req, "$x", 2, 3); - req = "define action $a() as\n" - " $x = 1; $x = 2; select $x;\n" - "end define;\n" - "\n" - "do $a();"; - CheckUnused(req, "$x", 2, 3); - } - - Y_UNIT_TEST(NoWarnOnNestedActions) { - auto req = "pragma warning(\"error\", \"4527\");\n" - "define action $action($b) as\n" - " define action $aaa() as\n" - " select $b;\n" - " end define;\n" - " do $aaa();\n" - "end define;\n" - "\n" - "do $action(1);"; - UNIT_ASSERT(SqlToYql(req).IsOk()); - } - - Y_UNIT_TEST(NoWarnForUsageAfterSubquery) { - auto req = "use plato;\n" - "pragma warning(\"error\", \"4527\");\n" - "\n" - "$a = 1;\n" - "\n" - "define subquery $q($table) as\n" - " select * from $table;\n" - "end define;\n" - "\n" - "select * from $q(\"Input\");\n" - "select $a;"; - UNIT_ASSERT(SqlToYql(req).IsOk()); - } -} - -Y_UNIT_TEST_SUITE(AnonymousNames) { - Y_UNIT_TEST(ReferenceAnonymousVariableIsForbidden) { - auto req = "$_ = 1; select $_;"; - - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:16: Error: Unable to reference anonymous name $_\n"); - - req = "$`_` = 1; select $`_`;"; - res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:18: Error: Unable to reference anonymous name $_\n"); - } - - Y_UNIT_TEST(Declare) { - auto req = "declare $_ as String;"; - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:9: Error: Can not use anonymous name '$_' in DECLARE statement\n"); - } - - Y_UNIT_TEST(ActionSubquery) { - auto req = "define action $_() as select 1; end define;"; - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:15: Error: Can not use anonymous name '$_' as ACTION name\n"); - - req = "define subquery $_() as select 1; end define;"; - res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:17: Error: Can not use anonymous name '$_' as SUBQUERY name\n"); - } - - Y_UNIT_TEST(Import) { - auto req = "import lib symbols $sqr as $_;"; - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:28: Error: Can not import anonymous name $_\n"); - } - - Y_UNIT_TEST(Export) { - auto req = "export $_;"; - auto res = SqlToYqlWithMode(req, NSQLTranslation::ESqlMode::LIBRARY); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Error: Can not export anonymous name $_\n"); - } - - Y_UNIT_TEST(AnonymousInActionArgs) { - auto req = "pragma warning(\"error\", \"4527\");\n" - "define action $a($_, $y, $_) as\n" - " select $y;\n" - "end define;\n" - "\n" - "do $a(1,2,3);"; - UNIT_ASSERT(SqlToYql(req).IsOk()); - } - - Y_UNIT_TEST(AnonymousInSubqueryArgs) { - auto req = "use plato;\n" - "pragma warning(\"error\", \"4527\");\n" - "define subquery $q($_, $y, $_) as\n" - " select * from $y;\n" - "end define;\n" - "\n" - "select * from $q(1,\"Input\",3);"; - UNIT_ASSERT(SqlToYql(req).IsOk()); - } - - Y_UNIT_TEST(AnonymousInLambdaArgs) { - auto req = "pragma warning(\"error\", \"4527\");\n" - "$lambda = ($_, $x, $_) -> ($x);\n" - "select $lambda(1,2,3);"; - UNIT_ASSERT(SqlToYql(req).IsOk()); - } - - Y_UNIT_TEST(AnonymousInFor) { - auto req = "pragma warning(\"error\", \"4527\");\n" - "evaluate for $_ in ListFromRange(1, 10) do begin select 1; end do;"; - UNIT_ASSERT(SqlToYql(req).IsOk()); - } - - Y_UNIT_TEST(Assignment) { - auto req = "pragma warning(\"error\", \"4527\");\n" - "$_ = 1;\n" - "$_, $x, $_ = AsTuple(1,2,3);\n" - "select $x;"; - UNIT_ASSERT(SqlToYql(req).IsOk()); - } } -Y_UNIT_TEST_SUITE(JsonValue) { - Y_UNIT_TEST(JsonValueArgumentCount) { - NYql::TAstParseResult res = SqlToYql("select JSON_VALUE(CAST(@@{\"key\": 1238}@@ as Json));"); - - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:49: Error: Unexpected token ')' : syntax error...\n\n"); - } - - Y_UNIT_TEST(JsonValueJsonPathMustBeLiteralString) { - NYql::TAstParseResult res = SqlToYql("$jsonPath = \"strict $.key\"; select JSON_VALUE(CAST(@@{\"key\": 1238}@@ as Json), $jsonPath);"); - - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:79: Error: Unexpected token absence : Missing STRING_VALUE \n\n"); - } - - Y_UNIT_TEST(JsonValueTranslation) { - NYql::TAstParseResult res = SqlToYql("select JSON_VALUE(CAST(@@{\"key\": 1238}@@ as Json), \"strict $.key\");"); - - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - Y_UNUSED(word); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'\"strict $.key\"")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("SafeCast")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("DataType 'Json")); - }; - - TWordCountHive elementStat({"JsonValue"}); - VerifyProgram(res, elementStat, verifyLine); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["JsonValue"]); - } - - Y_UNIT_TEST(JsonValueReturningSection) { - for (const auto& typeName : {"Bool", "Int64", "Double", "String"}) { - NYql::TAstParseResult res = SqlToYql( - TStringBuilder() << "select JSON_VALUE(CAST(@@{\"key\": 1238}@@ as Json), \"strict $.key\" RETURNING " << typeName << ");" - ); - - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [&](const TString& word, const TString& line) { - Y_UNUSED(word); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'\"strict $.key\"")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("SafeCast")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("DataType 'Json")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(TStringBuilder() << "DataType '" << typeName)); - }; - - TWordCountHive elementStat({typeName}); - VerifyProgram(res, elementStat, verifyLine); - UNIT_ASSERT(elementStat[typeName] > 0); - } - } - - Y_UNIT_TEST(JsonValueInvalidReturningType) { - NYql::TAstParseResult res = SqlToYql("select JSON_VALUE(CAST(@@{'key': 1238}@@ as Json), 'strict $.key' RETURNING invalid);"); - - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:77: Error: Unknown simple type 'invalid'\n"); - } - - Y_UNIT_TEST(JsonValueAndReturningInExpressions) { - NYql::TAstParseResult res = SqlToYql( - "USE plato\n;" - "$json_value = \"some string\";\n" - "SELECT $json_value;\n" - "SELECT 1 as json_value;\n" - "SELECT $json_value as json_value;\n" - "$returning = \"another string\";\n" - "SELECT $returning;\n" - "SELECT 1 as returning;\n" - "SELECT $returning as returning;\n" - ); - - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(JsonValueValidCaseHandlers) { - const TVector<std::pair<TString, TString>> testCases = { - {"", "'DefaultValue (Null)"}, - {"NULL", "'DefaultValue (Null)"}, - {"ERROR", "'Error (Null)"}, - {"DEFAULT 123", "'DefaultValue (Int32 '\"123\")"}, - }; - - for (const auto& onEmpty : testCases) { - for (const auto& onError : testCases) { - TStringBuilder query; - query << "$json = CAST(@@{\"key\": 1238}@@ as Json);\n" - << "SELECT JSON_VALUE($json, \"strict $.key\""; - if (!onEmpty.first.empty()) { - query << " " << onEmpty.first << " ON EMPTY"; - } - if (!onError.first.empty()) { - query << " " << onError.first << " ON ERROR"; - } - query << ");\n"; - - NYql::TAstParseResult res = SqlToYql(query); - - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [&](const TString& word, const TString& line) { - Y_UNUSED(word); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(onEmpty.second + " " + onError.second)); - }; - - TWordCountHive elementStat({"JsonValue"}); - VerifyProgram(res, elementStat, verifyLine); - UNIT_ASSERT(elementStat["JsonValue"] > 0); - } - } - } - - Y_UNIT_TEST(JsonValueTooManyCaseHandlers) { - NYql::TAstParseResult res = SqlToYql( - "select JSON_VALUE(CAST(@@{\"key\": 1238}@@ as Json), \"strict $.key\" NULL ON EMPTY NULL ON ERROR NULL ON EMPTY);\n" - ); - - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF( - Err2Str(res), - "<main>:1:52: Error: Only 1 ON EMPTY and/or 1 ON ERROR clause is expected\n" - ); - } - - Y_UNIT_TEST(JsonValueTooManyOnEmpty) { - NYql::TAstParseResult res = SqlToYql( - "select JSON_VALUE(CAST(@@{\"key\": 1238}@@ as Json), \"strict $.key\" NULL ON EMPTY NULL ON EMPTY);\n" - ); - - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF( - Err2Str(res), - "<main>:1:52: Error: Only 1 ON EMPTY clause is expected\n" - ); - } - - Y_UNIT_TEST(JsonValueTooManyOnError) { - NYql::TAstParseResult res = SqlToYql( - "select JSON_VALUE(CAST(@@{\"key\": 1238}@@ as Json), \"strict $.key\" NULL ON ERROR NULL ON ERROR);\n" - ); - - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF( - Err2Str(res), - "<main>:1:52: Error: Only 1 ON ERROR clause is expected\n" - ); - } - - Y_UNIT_TEST(JsonValueOnEmptyAfterOnError) { - NYql::TAstParseResult res = SqlToYql( - "select JSON_VALUE(CAST(@@{\"key\": 1238}@@ as Json), \"strict $.key\" NULL ON ERROR NULL ON EMPTY);\n" - ); - - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF( - Err2Str(res), - "<main>:1:52: Error: ON EMPTY clause must be before ON ERROR clause\n" - ); - } - - Y_UNIT_TEST(JsonValueNullInput) { - NYql::TAstParseResult res = SqlToYql(R"(SELECT JSON_VALUE(NULL, "strict $.key");)"); - - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [&](const TString& word, const TString& line) { - Y_UNUSED(word); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("(Nothing (OptionalType (DataType 'Json)))")); - }; - - TWordCountHive elementStat({"JsonValue"}); - VerifyProgram(res, elementStat, verifyLine); - UNIT_ASSERT(elementStat["JsonValue"] > 0); - } -} - -Y_UNIT_TEST_SUITE(JsonExists) { - Y_UNIT_TEST(JsonExistsValidHandlers) { - const TVector<std::pair<TString, TString>> testCases = { - {"", "(Just (Bool '\"false\"))"}, - {"TRUE ON ERROR", "(Just (Bool '\"true\"))"}, - {"FALSE ON ERROR", "(Just (Bool '\"false\"))"}, - {"UNKNOWN ON ERROR", "(Nothing (OptionalType (DataType 'Bool)))"}, - // NOTE: in this case we expect arguments of JsonExists callable to end immediately - // after variables. This parenthesis at the end of the expression is left on purpose - {"ERROR ON ERROR", "(Utf8 '\"strict $.key\") (JsonVariables))"}, - }; - - for (const auto& item : testCases) { - NYql::TAstParseResult res = SqlToYql( - TStringBuilder() << R"( - $json = CAST(@@{"key": 1238}@@ as Json); - SELECT JSON_EXISTS($json, "strict $.key" )" << item.first << ");\n" - ); - - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [&](const TString& word, const TString& line) { - Y_UNUSED(word); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(item.second)); - }; - - TWordCountHive elementStat({"JsonExists"}); - VerifyProgram(res, elementStat, verifyLine); - UNIT_ASSERT(elementStat["JsonExists"] > 0); - } - } - - Y_UNIT_TEST(JsonExistsInvalidHandler) { - NYql::TAstParseResult res = SqlToYql(R"( - $json = CAST(@@{"key": 1238}@@ as Json); - $default = false; - SELECT JSON_EXISTS($json, "strict $.key" $default ON ERROR); - )"); - - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:4:53: Error: Unexpected token absence : Missing RPAREN \n\n"); - } - - Y_UNIT_TEST(JsonExistsNullInput) { - NYql::TAstParseResult res = SqlToYql(R"(SELECT JSON_EXISTS(NULL, "strict $.key");)"); - - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [&](const TString& word, const TString& line) { - Y_UNUSED(word); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("(Nothing (OptionalType (DataType 'Json)))")); - }; - - TWordCountHive elementStat({"JsonExists"}); - VerifyProgram(res, elementStat, verifyLine); - UNIT_ASSERT(elementStat["JsonExists"] > 0); - } -} - -Y_UNIT_TEST_SUITE(JsonQuery) { - Y_UNIT_TEST(JsonQueryValidHandlers) { - using TTestSuite = const TVector<std::pair<TString, TString>>; - TTestSuite wrapCases = { - {"", "'NoWrap"}, - {"WITHOUT WRAPPER", "'NoWrap"}, - {"WITHOUT ARRAY WRAPPER", "'NoWrap"}, - {"WITH WRAPPER", "'Wrap"}, - {"WITH ARRAY WRAPPER", "'Wrap"}, - {"WITH UNCONDITIONAL WRAPPER", "'Wrap"}, - {"WITH UNCONDITIONAL ARRAY WRAPPER", "'Wrap"}, - {"WITH CONDITIONAL WRAPPER", "'ConditionalWrap"}, - {"WITH CONDITIONAL ARRAY WRAPPER", "'ConditionalWrap"}, - }; - TTestSuite handlerCases = { - {"", "'Null"}, - {"ERROR", "'Error"}, - {"NULL", "'Null"}, - {"EMPTY ARRAY", "'EmptyArray"}, - {"EMPTY OBJECT", "'EmptyObject"}, - }; - - for (const auto& wrap : wrapCases) { - for (const auto& onError : handlerCases) { - for (const auto& onEmpty : handlerCases) { - TStringBuilder query; - query << R"($json = CAST(@@{"key": [123]}@@ as Json); - SELECT JSON_QUERY($json, "strict $.key" )" << wrap.first; - if (!onEmpty.first.empty()) { - if (wrap.first.StartsWith("WITH ")) { - continue; - } - query << " " << onEmpty.first << " ON EMPTY"; - } - if (!onError.first.empty()) { - query << " " << onError.first << " ON ERROR"; - } - query << ");\n"; - - NYql::TAstParseResult res = SqlToYql(query); - - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [&](const TString& word, const TString& line) { - Y_UNUSED(word); - const TString args = TStringBuilder() << wrap.second << " " << onEmpty.second << " " << onError.second; - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(args)); - }; - - Cout << wrap.first << " " << onEmpty.first << " " << onError.first << Endl; - - TWordCountHive elementStat({"JsonQuery"}); - VerifyProgram(res, elementStat, verifyLine); - UNIT_ASSERT(elementStat["JsonQuery"] > 0); - } - } - } - } - - Y_UNIT_TEST(JsonQueryOnEmptyWithWrapper) { - NYql::TAstParseResult res = SqlToYql(R"( - $json = CAST(@@{"key": 1238}@@ as Json); - SELECT JSON_QUERY($json, "strict $" WITH ARRAY WRAPPER EMPTY ARRAY ON EMPTY); - )"); - - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:38: Error: ON EMPTY is prohibited because WRAPPER clause is specified\n"); - } - - Y_UNIT_TEST(JsonQueryNullInput) { - NYql::TAstParseResult res = SqlToYql(R"(SELECT JSON_QUERY(NULL, "strict $.key");)"); - - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [&](const TString& word, const TString& line) { - Y_UNUSED(word); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("(Nothing (OptionalType (DataType 'Json)))")); - }; - - TWordCountHive elementStat({"JsonQuery"}); - VerifyProgram(res, elementStat, verifyLine); - UNIT_ASSERT(elementStat["JsonQuery"] > 0); - } -} - -Y_UNIT_TEST_SUITE(JsonPassing) { - Y_UNIT_TEST(SupportedVariableTypes) { - const TVector<TString> functions = {"JSON_EXISTS", "JSON_VALUE", "JSON_QUERY"}; - - for (const auto& function : functions) { - const auto query = Sprintf(R"( - pragma CompactNamedExprs; - $json = CAST(@@{"key": 1238}@@ as Json); - SELECT %s( - $json, - "strict $.key" - PASSING - "string" as var1, - 1.234 as var2, - CAST(1 as Int64) as var3, - true as var4, - $json as var5 - ))", - function.data() - ); - NYql::TAstParseResult res = SqlToYql(query); - - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [&](const TString& word, const TString& line) { - Y_UNUSED(word); - UNIT_ASSERT_VALUES_UNEQUAL_C(TString::npos, line.find(R"('('"var1" (String '"string")))"), "Cannot find `var1`"); - UNIT_ASSERT_VALUES_UNEQUAL_C(TString::npos, line.find(R"('('"var2" (Double '"1.234")))"), "Cannot find `var2`"); - UNIT_ASSERT_VALUES_UNEQUAL_C(TString::npos, line.find(R"('('"var3" (SafeCast (Int32 '"1") (DataType 'Int64))))"), "Cannot find `var3`"); - UNIT_ASSERT_VALUES_UNEQUAL_C(TString::npos, line.find(R"('('"var4" (Bool '"true")))"), "Cannot find `var4`"); - UNIT_ASSERT_VALUES_UNEQUAL_C(TString::npos, line.find(R"('('"var5" namedexprnode0))"), "Cannot find `var5`"); - }; - - TWordCountHive elementStat({"JsonVariables"}); - VerifyProgram(res, elementStat, verifyLine); - UNIT_ASSERT(elementStat["JsonVariables"] > 0); - } - } - - Y_UNIT_TEST(ValidVariableNames) { - const TVector<TString> functions = {"JSON_EXISTS", "JSON_VALUE", "JSON_QUERY"}; - - for (const auto& function : functions) { - const auto query = Sprintf(R"( - $json = CAST(@@{"key": 1238}@@ as Json); - SELECT %s( - $json, - "strict $.key" - PASSING - "one" as var1, - "two" as "VaR2", - "three" as `var3`, - "four" as VaR4 - ))", - function.data() - ); - NYql::TAstParseResult res = SqlToYql(query); - - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [&](const TString& word, const TString& line) { - Y_UNUSED(word); - UNIT_ASSERT_VALUES_UNEQUAL_C(TString::npos, line.find(R"('('"var1" (String '"one")))"), "Cannot find `var1`"); - UNIT_ASSERT_VALUES_UNEQUAL_C(TString::npos, line.find(R"('('"VaR2" (String '"two")))"), "Cannot find `VaR2`"); - UNIT_ASSERT_VALUES_UNEQUAL_C(TString::npos, line.find(R"('('"var3" (String '"three")))"), "Cannot find `var3`"); - UNIT_ASSERT_VALUES_UNEQUAL_C(TString::npos, line.find(R"('('"VaR4" (String '"four")))"), "Cannot find `VaR4`"); - }; - - TWordCountHive elementStat({"JsonVariables"}); - VerifyProgram(res, elementStat, verifyLine); - UNIT_ASSERT(elementStat["JsonVariables"] > 0); - } - } -} - -Y_UNIT_TEST_SUITE(MigrationToJsonApi) { - Y_UNIT_TEST(WarningOnDeprecatedJsonUdf) { - NYql::TAstParseResult res = SqlToYql(R"( - $json = CAST(@@{"key": 1234}@@ as Json); - SELECT Json::Parse($json); - )"); - - UNIT_ASSERT(res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:26: Warning: Json UDF is deprecated. Please use JSON API instead, code: 4506\n"); - } -} - -Y_UNIT_TEST_SUITE(AnsiIdentsNegative) { - Y_UNIT_TEST(EnableAnsiLexerFromRequestSpecialComments) { - auto req = "\n" - "\t --!ansi_lexer \n" - "-- Some comment\n" - "-- another comment\n" - "pragma SimpleColumns;\n" - "\n" - "select 1, '''' as empty;"; - - auto res = SqlToYql(req); - UNIT_ASSERT(res.IsOk()); - UNIT_ASSERT(res.Issues.Size() == 0); - } - - Y_UNIT_TEST(AnsiLexerShouldNotBeEnabledHere) { - auto req = "$str = '\n" - "--!ansi_lexer\n" - "--!syntax_v1\n" - "';\n" - "\n" - "select 1, $str, \"\" as empty;"; - auto res = SqlToYql(req); - UNIT_ASSERT(res.IsOk()); - UNIT_ASSERT(res.Issues.Size() == 0); - } - - Y_UNIT_TEST(DoubleQuotesInDictsTuplesOrLists) { - auto req = "$d = { 'a': 1, \"b\": 2, 'c': 3,};"; - - auto res = SqlToYqlWithAnsiLexer(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:16: Error: Column reference \"b\" is not allowed in current scope\n"); - - req = "$t = (1, 2, \"a\");"; - - res = SqlToYqlWithAnsiLexer(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:13: Error: Column reference \"a\" is not allowed in current scope\n"); - - req = "$l = ['a', 'b', \"c\"];"; - - res = SqlToYqlWithAnsiLexer(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:17: Error: Column reference \"c\" is not allowed in current scope\n"); - } - - Y_UNIT_TEST(MultilineComments) { - auto req = "/*/**/ select 1;"; - auto res = SqlToYql(req); - UNIT_ASSERT(res.Root); - res = SqlToYqlWithAnsiLexer(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:16: Error: Unexpected character : syntax error...\n\n"); - - req = "/*\n" - "--/*\n" - "*/ select 1;"; - res = SqlToYql(req); - UNIT_ASSERT(res.Root); - res = SqlToYqlWithAnsiLexer(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:12: Error: Unexpected character : syntax error...\n\n"); - - req = "/*\n" - "/*\n" - "--*/\n" - "*/ select 1;"; - res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:4:0: Error: Unexpected token '*' : cannot match to any predicted input...\n\n"); - res = SqlToYqlWithAnsiLexer(req); - UNIT_ASSERT(res.Root); - } -} - -Y_UNIT_TEST_SUITE(AnsiOptionalAs) { - Y_UNIT_TEST(OptionalAsInProjection) { - UNIT_ASSERT(SqlToYql("PRAGMA AnsiOptionalAs; SELECT a b, c FROM plato.Input;").IsOk()); - ExpectFailWithError("PRAGMA DisableAnsiOptionalAs;\n" - "SELECT a b, c FROM plato.Input;", - "<main>:2:10: Error: Expecting mandatory AS here. Did you miss comma? Please add PRAGMA AnsiOptionalAs; for ANSI compatibility\n"); - } - - Y_UNIT_TEST(OptionalAsWithKeywords) { - UNIT_ASSERT(SqlToYql("PRAGMA AnsiOptionalAs; SELECT a type, b data, c source FROM plato.Input;").IsOk()); - } -} - -Y_UNIT_TEST_SUITE(SessionWindowNegative) { - Y_UNIT_TEST(SessionWindowWithoutSource) { - ExpectFailWithError("SELECT 1 + SessionWindow(ts, 32);", - "<main>:1:12: Error: SessionWindow requires data source\n"); - } - - Y_UNIT_TEST(SessionWindowInProjection) { - ExpectFailWithError("SELECT 1 + SessionWindow(ts, 32) from plato.Input;", - "<main>:1:12: Error: SessionWindow can only be used as a top-level GROUP BY / PARTITION BY expression\n"); - } - - Y_UNIT_TEST(SessionWindowWithNonConstSecondArg) { - ExpectFailWithError( - "SELECT key, session_start FROM plato.Input\n" - "GROUP BY SessionWindow(ts, 32 + subkey) as session_start, key;", - - "<main>:2:10: Error: Source does not allow column references\n" - "<main>:2:33: Error: Column reference 'subkey'\n"); - } - - Y_UNIT_TEST(SessionWindowWithWrongNumberOfArgs) { - ExpectFailWithError("SELECT * FROM plato.Input GROUP BY SessionWindow()", - "<main>:1:36: Error: SessionWindow requires either two or four arguments\n"); - ExpectFailWithError("SELECT * FROM plato.Input GROUP BY SessionWindow(key, subkey, 100)", - "<main>:1:36: Error: SessionWindow requires either two or four arguments\n"); - } - - Y_UNIT_TEST(DuplicateSessionWindow) { - ExpectFailWithError( - "SELECT\n" - " *\n" - "FROM plato.Input\n" - "GROUP BY\n" - " SessionWindow(ts, 10),\n" - " user,\n" - " SessionWindow(ts, 20)\n" - ";", - - "<main>:7:5: Error: Duplicate session window specification:\n" - "<main>:5:5: Error: Previous session window is declared here\n"); - - ExpectFailWithError( - "SELECT\n" - " MIN(key) over w\n" - "FROM plato.Input\n" - "WINDOW w AS (\n" - " PARTITION BY SessionWindow(ts, 10), user,\n" - " SessionWindow(ts, 20)\n" - ");", - - "<main>:6:5: Error: Duplicate session window specification:\n" - "<main>:5:18: Error: Previous session window is declared here\n"); - } - - Y_UNIT_TEST(SessionStartStateWithoutSource) { - ExpectFailWithError("SELECT 1 + SessionStart();", - "<main>:1:12: Error: SessionStart requires data source\n"); - ExpectFailWithError("SELECT 1 + SessionState();", - "<main>:1:12: Error: SessionState requires data source\n"); - } - - Y_UNIT_TEST(SessionStartStateWithoutGroupByOrWindow) { - ExpectFailWithError("SELECT 1 + SessionStart() from plato.Input;", - "<main>:1:12: Error: SessionStart can not be used without aggregation by SessionWindow\n"); - ExpectFailWithError("SELECT 1 + SessionState() from plato.Input;", - "<main>:1:12: Error: SessionState can not be used without aggregation by SessionWindow\n"); - } - - Y_UNIT_TEST(SessionStartStateWithGroupByWithoutSession) { - ExpectFailWithError("SELECT 1 + SessionStart() from plato.Input group by user;", - "<main>:1:12: Error: SessionStart can not be used here: SessionWindow specification is missing in GROUP BY\n"); - ExpectFailWithError("SELECT 1 + SessionState() from plato.Input group by user;", - "<main>:1:12: Error: SessionState can not be used here: SessionWindow specification is missing in GROUP BY\n"); - } - - Y_UNIT_TEST(SessionStartStateWithoutOverWithWindowWithoutSession) { - ExpectFailWithError("SELECT 1 + SessionStart(), MIN(key) over w from plato.Input window w as ()", - "<main>:1:12: Error: SessionStart can not be used without aggregation by SessionWindow. Maybe you forgot to add OVER `window_name`?\n"); - ExpectFailWithError("SELECT 1 + SessionState(), MIN(key) over w from plato.Input window w as ()", - "<main>:1:12: Error: SessionState can not be used without aggregation by SessionWindow. Maybe you forgot to add OVER `window_name`?\n"); - } - - Y_UNIT_TEST(SessionStartStateWithWindowWithoutSession) { - ExpectFailWithError("SELECT 1 + SessionStart() over w, MIN(key) over w from plato.Input window w as ()", - "<main>:1:12: Error: SessionStart can not be used with window w: SessionWindow specification is missing in PARTITION BY\n"); - ExpectFailWithError("SELECT 1 + SessionState() over w, MIN(key) over w from plato.Input window w as ()", - "<main>:1:12: Error: SessionState can not be used with window w: SessionWindow specification is missing in PARTITION BY\n"); - } - - Y_UNIT_TEST(SessionStartStateWithSessionedWindow) { - ExpectFailWithError("SELECT 1 + SessionStart(), MIN(key) over w from plato.Input group by key window w as (partition by SessionWindow(ts, 1)) ", - "<main>:1:12: Error: SessionStart can not be used here: SessionWindow specification is missing in GROUP BY. Maybe you forgot to add OVER `window_name`?\n"); - ExpectFailWithError("SELECT 1 + SessionState(), MIN(key) over w from plato.Input group by key window w as (partition by SessionWindow(ts, 1)) ", - "<main>:1:12: Error: SessionState can not be used here: SessionWindow specification is missing in GROUP BY. Maybe you forgot to add OVER `window_name`?\n"); - } - - Y_UNIT_TEST(AggregationBySessionStateIsNotSupportedYet) { - ExpectFailWithError("SELECT SOME(1 + SessionState()), key from plato.Input group by key, SessionWindow(ts, 1);", - "<main>:1:17: Error: SessionState with GROUP BY is not supported yet\n"); - } - - Y_UNIT_TEST(SessionWindowInRtmr) { - NYql::TAstParseResult res = SqlToYql( - "SELECT * FROM plato.Input GROUP BY SessionWindow(ts, 10);", - 10, TString(NYql::RtmrProviderName)); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:54: Error: Streaming group by query must have a hopping window specification.\n"); - - res = SqlToYql(R"( - SELECT key, SUM(value) AS value FROM plato.Input - GROUP BY key, HOP(subkey, "PT10S", "PT30S", "PT20S"), SessionWindow(ts, 10); - )", 10, TString(NYql::RtmrProviderName)); - - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:2:13: Error: SessionWindow is unsupported for streaming sources\n"); - } -} - -Y_UNIT_TEST_SUITE(LibraSqlSugar) { - auto makeResult = [](TStringBuf settings) { - return SqlToYql( - TStringBuilder() - << settings - << "\n$udf1 = MyLibra::MakeLibraPreprocessor($settings);" - << "\n$udf2 = CustomLibra::MakeLibraPreprocessor($settings);" - << "\nPROCESS plato.Input USING $udf1(TableRow())" - << "\nUNION ALL" - << "\nPROCESS plato.Input USING $udf2(TableRow());" - ); - }; - - Y_UNIT_TEST(EmptySettings) { - auto res = makeResult(R"( - $settings = AsStruct(); - )"); - UNIT_ASSERT(res.IsOk()); - } - - Y_UNIT_TEST(OnlyEntities) { - auto res = makeResult(R"( - $settings = AsStruct( - AsList("A", "B", "C") AS Entities - ); - )"); - UNIT_ASSERT(res.IsOk()); - } - - Y_UNIT_TEST(EntitiesWithStrategy) { - auto res = makeResult(R"( - $settings = AsStruct( - AsList("A", "B", "C") AS Entities, - "blacklist" AS EntitiesStrategy - ); - )"); - UNIT_ASSERT(res.IsOk()); - } - - Y_UNIT_TEST(AllSettings) { - auto res = makeResult(R"( - $settings = AsStruct( - AsList("A", "B", "C") AS Entities, - "whitelist" AS EntitiesStrategy, - "path" AS BlockstatDict, - false AS ParseWithFat, - "map" AS Mode - ); - )"); - UNIT_ASSERT(res.IsOk()); - } - - Y_UNIT_TEST(BadStrategy) { - auto res = makeResult(R"( - $settings = AsStruct("bad" AS EntitiesStrategy); - )"); - UNIT_ASSERT_STRING_CONTAINS( - Err2Str(res), - "Error: MakeLibraPreprocessor got invalid entities strategy: expected 'whitelist' or 'blacklist'" - ); - } - - Y_UNIT_TEST(BadEntities) { - auto res = makeResult(R"( - $settings = AsStruct(AsList("A", 1) AS Entities); - )"); - UNIT_ASSERT_STRING_CONTAINS(Err2Str(res), "Error: MakeLibraPreprocessor entity must be string literal"); - } -} - -Y_UNIT_TEST_SUITE(TrailingQuestionsNegative) { - Y_UNIT_TEST(Basic) { - ExpectFailWithError("SELECT 1?;", "<main>:1:9: Error: Unexpected token '?' at the end of expression\n"); - ExpectFailWithError("SELECT 1? + 1;", "<main>:1:10: Error: Unexpected token '+' : cannot match to any predicted input...\n\n"); - ExpectFailWithError("SELECT 1 + 1??? < 2", "<main>:1:13: Error: Unexpected token '?' at the end of expression\n"); - ExpectFailWithError("SELECT 1? > 2? > 3?", - "<main>:1:11: Error: Unexpected token '?' at the end of expression\n" - "<main>:1:16: Error: Unexpected token '?' at the end of expression\n" - "<main>:1:21: Error: Unexpected token '?' at the end of expression\n"); - } - - Y_UNIT_TEST(SmartParen) { - ExpectFailWithError("$x = 1; SELECT (Int32?, $x?)", "<main>:1:27: Error: Unexpected token '?' at the end of expression\n"); - ExpectFailWithError("SELECT (Int32, foo?)", "<main>:1:19: Error: Unexpected token '?' at the end of expression\n"); - } - - Y_UNIT_TEST(LambdaOptArgs) { - ExpectFailWithError("$l = ($x, $y?, $z??, $t?) -> ($x);", "<main>:1:18: Error: Expecting at most one '?' token here (for optional lambda parameters), but got 2\n"); - } -} - -Y_UNIT_TEST_SUITE(FlexibleTypes) { - Y_UNIT_TEST(AssumeOrderByType) { - UNIT_ASSERT(SqlToYql("PRAGMA FlexibleTypes; SELECT 1 AS int32 ASSUME ORDER BY int32").IsOk()); - } - - Y_UNIT_TEST(GroupingSets) { - UNIT_ASSERT(SqlToYql("PRAGMA FlexibleTypes; SELECT COUNT(*) AS cnt, text, uuid FROM plato.Input GROUP BY GROUPING SETS((uuid), (uuid, text));").IsOk()); - } - - Y_UNIT_TEST(WeakField) { - UNIT_ASSERT(SqlToYql("PRAGMA FlexibleTypes; SELECT WeakField(text, string) as text FROM plato.Input").IsOk()); - } - - Y_UNIT_TEST(Aggregation1) { - TString q = - "PRAGMA FlexibleTypes;\n" - "$foo = ($x, $const, $type) -> ($x || $const || FormatType($type));\n" - "SELECT $foo(SOME(x), 'aaa', String) FROM plato.Input GROUP BY y;"; - UNIT_ASSERT(SqlToYql(q).IsOk()); - } - - Y_UNIT_TEST(Aggregation2) { - TString q = - "PRAGMA FlexibleTypes;\n" - "SELECT 1 + String + MAX(key) FROM plato.Input;"; - UNIT_ASSERT(SqlToYql(q).IsOk()); - } -} - -Y_UNIT_TEST_SUITE(ExternalDeclares) { - Y_UNIT_TEST(BasicUsage) { - NSQLTranslation::TTranslationSettings settings; - settings.DeclaredNamedExprs["foo"] = "String"; - auto res = SqlToYqlWithSettings("select $foo;", settings); - UNIT_ASSERT(res.IsOk()); - UNIT_ASSERT(res.Issues.Size() == 0); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "declare") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(R"__((declare "$foo" (DataType 'String)))__")); - } - }; - - TWordCountHive elementStat = {{TString("declare"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["declare"]); - } - - Y_UNIT_TEST(DeclareOverrides) { - NSQLTranslation::TTranslationSettings settings; - settings.DeclaredNamedExprs["foo"] = "String"; - auto res = SqlToYqlWithSettings("declare $foo as Int32; select $foo;", settings); - UNIT_ASSERT(res.IsOk()); - UNIT_ASSERT(res.Issues.Size() == 0); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "declare") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(R"__((declare "$foo" (DataType 'Int32)))__")); - } - }; - - TWordCountHive elementStat = {{TString("declare"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["declare"]); - } - - Y_UNIT_TEST(UnusedDeclareDoesNotProduceWarning) { - NSQLTranslation::TTranslationSettings settings; - settings.DeclaredNamedExprs["foo"] = "String"; - auto res = SqlToYqlWithSettings("select 1;", settings); - UNIT_ASSERT(res.IsOk()); - UNIT_ASSERT(res.Issues.Size() == 0); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "declare") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(R"__((declare "$foo" (DataType 'String)))__")); - } - }; - - TWordCountHive elementStat = {{TString("declare"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["declare"]); - } - - Y_UNIT_TEST(DeclaresWithInvalidTypesFails) { - NSQLTranslation::TTranslationSettings settings; - settings.DeclaredNamedExprs["foo"] = "List<BadType>"; - auto res = SqlToYqlWithSettings("select 1;", settings); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), - "<main>:0:5: Error: Unknown type: 'BadType'\n" - "<main>: Error: Failed to parse type for externally declared name 'foo'\n"); - } -} - -Y_UNIT_TEST_SUITE(ExternalDataSource) { - Y_UNIT_TEST(CreateExternalDataSourceWithAuthNone) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - CREATE EXTERNAL DATA SOURCE MyDataSource WITH ( - SOURCE_TYPE="ObjectStorage", - LOCATION="my-bucket", - AUTH_METHOD="NONE" - ); - )sql"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_STRING_CONTAINS(line, R"#('('('"auth_method" '"NONE") '('"location" '"my-bucket") '('"source_type" '"ObjectStorage"))#"); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("createObject")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(CreateExternalDataSourceWithAuthServiceAccount) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - CREATE EXTERNAL DATA SOURCE MyDataSource WITH ( - SOURCE_TYPE="ObjectStorage", - LOCATION="my-bucket", - AUTH_METHOD="SERVICE_ACCOUNT", - SERVICE_ACCOUNT_ID="sa", - SERVICE_ACCOUNT_SECRET_NAME="sa_secret_name" - ); - )sql"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_STRING_CONTAINS(line, R"#('('('"auth_method" '"SERVICE_ACCOUNT") '('"location" '"my-bucket") '('"service_account_id" '"sa") '('"service_account_secret_name" '"sa_secret_name") '('"source_type" '"ObjectStorage"))#"); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("createObject")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(CreateExternalDataSourceWithBasic) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - CREATE EXTERNAL DATA SOURCE MyDataSource WITH ( - SOURCE_TYPE="PostgreSQL", - LOCATION="protocol://host:port/", - AUTH_METHOD="BASIC", - LOGIN="admin", - PASSWORD_SECRET_NAME="secret_name" - ); - )sql"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_STRING_CONTAINS(line, R"#('('('"auth_method" '"BASIC") '('"location" '"protocol://host:port/") '('"login" '"admin") '('"password_secret_name" '"secret_name") '('"source_type" '"PostgreSQL"))#"); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("createObject")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(CreateExternalDataSourceWithMdbBasic) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - CREATE EXTERNAL DATA SOURCE MyDataSource WITH ( - SOURCE_TYPE="PostgreSQL", - LOCATION="protocol://host:port/", - AUTH_METHOD="MDB_BASIC", - SERVICE_ACCOUNT_ID="sa", - SERVICE_ACCOUNT_SECRET_NAME="sa_secret_name", - LOGIN="admin", - PASSWORD_SECRET_NAME="secret_name" - ); - )sql"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_STRING_CONTAINS(line, R"#('('('"auth_method" '"MDB_BASIC") '('"location" '"protocol://host:port/") '('"login" '"admin") '('"password_secret_name" '"secret_name") '('"service_account_id" '"sa") '('"service_account_secret_name" '"sa_secret_name") '('"source_type" '"PostgreSQL"))#"); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("createObject")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(CreateExternalDataSourceWithAws) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - CREATE EXTERNAL DATA SOURCE MyDataSource WITH ( - SOURCE_TYPE="PostgreSQL", - LOCATION="protocol://host:port/", - AUTH_METHOD="AWS", - AWS_ACCESS_KEY_ID_SECRET_NAME="secred_id_name", - AWS_SECRET_ACCESS_KEY_SECRET_NAME="secret_key_name", - AWS_REGION="ru-central-1" - ); - )sql"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_STRING_CONTAINS(line, R"#('('('"auth_method" '"AWS") '('"aws_access_key_id_secret_name" '"secred_id_name") '('"aws_region" '"ru-central-1") '('"aws_secret_access_key_secret_name" '"secret_key_name") '('"location" '"protocol://host:port/") '('"source_type" '"PostgreSQL"))#"); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("createObject")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(CreateExternalDataSourceWithToken) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - CREATE EXTERNAL DATA SOURCE MyDataSource WITH ( - SOURCE_TYPE="YT", - LOCATION="protocol://host:port/", - AUTH_METHOD="TOKEN", - TOKEN_SECRET_NAME="token_name" - ); - )sql"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_STRING_CONTAINS(line, R"#('('('"auth_method" '"TOKEN") '('"location" '"protocol://host:port/") '('"source_type" '"YT") '('"token_secret_name" '"token_name"))#"); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("createObject")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(CreateExternalDataSourceWithTablePrefix) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - pragma TablePathPrefix='/aba'; - CREATE EXTERNAL DATA SOURCE MyDataSource WITH ( - SOURCE_TYPE="ObjectStorage", - LOCATION="my-bucket", - AUTH_METHOD="NONE" - ); - )sql"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_STRING_CONTAINS(line, "/aba/MyDataSource"); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("createObject")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(CreateExternalDataSourceIfNotExists) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - CREATE EXTERNAL DATA SOURCE IF NOT EXISTS MyDataSource WITH ( - SOURCE_TYPE="ObjectStorage", - LOCATION="my-bucket", - AUTH_METHOD="NONE" - ); - )sql"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_STRING_CONTAINS(line, R"#('('('"auth_method" '"NONE") '('"location" '"my-bucket") '('"source_type" '"ObjectStorage"))#"); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("createObjectIfNotExists")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(AlterExternalDataSource) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - ALTER EXTERNAL DATA SOURCE MyDataSource - SET (SOURCE_TYPE = "ObjectStorage", Login = "Admin"), - SET Location "bucket", - RESET (Auth_Method, Service_Account_Id, Service_Account_Secret_Name); - )sql"); - UNIT_ASSERT_C(res.Root, res.Issues.ToString()); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_STRING_CONTAINS(line, R"#(('mode 'alterObject))#"); - UNIT_ASSERT_STRING_CONTAINS(line, R"#('('features '('('"location" '"bucket") '('"login" '"Admin") '('"source_type" '"ObjectStorage"))))#"); - UNIT_ASSERT_STRING_CONTAINS(line, R"#('('resetFeatures '('"auth_method" '"service_account_id" '"service_account_secret_name")))#"); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(CreateExternalDataSourceOrReplace) { - NYql::TAstParseResult res = SqlToYql(R"( - USE plato; - CREATE OR REPLACE EXTERNAL DATA SOURCE MyDataSource WITH ( - SOURCE_TYPE="ObjectStorage", - LOCATION="my-bucket", - AUTH_METHOD="NONE" - ); - )"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_STRING_CONTAINS(line, R"#('('('"auth_method" '"NONE") '('"location" '"my-bucket") '('"source_type" '"ObjectStorage"))#"); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("createObjectOrReplace")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(CreateOrReplaceForUnsupportedTableTypesShouldFail) { - ExpectFailWithError(R"sql( - USE plato; - CREATE OR REPLACE TABLE t (a int32 not null, primary key(a, a)); - )sql" , "<main>:3:23: Error: OR REPLACE feature is supported only for EXTERNAL DATA SOURCE and EXTERNAL TABLE\n"); - - ExpectFailWithError(R"sql( - USE plato; - CREATE OR REPLACE TABLE t ( - Key Uint64, - Value1 String, - PRIMARY KEY (Key) - ) - WITH ( - STORE = COLUMN, - AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = 10 - ); - )sql" , "<main>:3:23: Error: OR REPLACE feature is supported only for EXTERNAL DATA SOURCE and EXTERNAL TABLE\n"); - } - - Y_UNIT_TEST(CreateExternalDataSourceWithBadArguments) { - ExpectFailWithError(R"sql( - USE plato; - CREATE EXTERNAL DATA SOURCE MyDataSource; - )sql" , "<main>:3:56: Error: Unexpected token ';' : syntax error...\n\n"); - - ExpectFailWithError(R"sql( - USE plato; - CREATE EXTERNAL DATA SOURCE MyDataSource WITH ( - LOCATION="my-bucket", - AUTH_METHOD="NONE" - ); - )sql" , "<main>:5:33: Error: SOURCE_TYPE requires key\n"); - - ExpectFailWithError(R"sql( - USE plato; - CREATE EXTERNAL DATA SOURCE MyDataSource WITH ( - SOURCE_TYPE="ObjectStorage", - LOCATION="my-bucket" - ); - )sql" , "<main>:5:30: Error: AUTH_METHOD requires key\n"); - - - ExpectFailWithError(R"sql( - USE plato; - CREATE EXTERNAL DATA SOURCE MyDataSource WITH ( - SOURCE_TYPE="ObjectStorage", - LOCATION="my-bucket", - AUTH_METHOD="NONE1" - ); - )sql" , "<main>:6:33: Error: Unknown AUTH_METHOD = NONE1\n"); - - ExpectFailWithError(R"sql( - USE plato; - CREATE EXTERNAL DATA SOURCE MyDataSource WITH ( - SOURCE_TYPE="ObjectStorage", - LOCATION="my-bucket", - AUTH_METHOD="SERVICE_ACCOUNT" - ); - )sql" , "<main>:6:33: Error: SERVICE_ACCOUNT_ID requires key\n"); - - ExpectFailWithError(R"sql( - USE plato; - CREATE EXTERNAL DATA SOURCE MyDataSource WITH ( - SOURCE_TYPE="ObjectStorage", - LOCATION="my-bucket", - AUTH_METHOD="SERVICE_ACCOUNT", - SERVICE_ACCOUNT_ID="s1" - ); - )sql" , "<main>:7:40: Error: SERVICE_ACCOUNT_SECRET_NAME requires key\n"); - - ExpectFailWithError(R"sql( - USE plato; - CREATE EXTERNAL DATA SOURCE MyDataSource WITH ( - SOURCE_TYPE="ObjectStorage", - LOCATION="my-bucket", - AUTH_METHOD="SERVICE_ACCOUNT", - SERVICE_ACCOUNT_SECRET_NAME="s1" - ); - )sql" , "<main>:7:49: Error: SERVICE_ACCOUNT_ID requires key\n"); - - ExpectFailWithError(R"sql( - USE plato; - CREATE EXTERNAL DATA SOURCE MyDataSource WITH ( - SOURCE_TYPE="PostgreSQL", - LOCATION="protocol://host:port/", - AUTH_METHOD="BASIC", - LOGIN="admin" - ); - )sql" , "<main>:7:27: Error: PASSWORD_SECRET_NAME requires key\n"); - - ExpectFailWithError(R"sql( - USE plato; - CREATE EXTERNAL DATA SOURCE MyDataSource WITH ( - SOURCE_TYPE="PostgreSQL", - LOCATION="protocol://host:port/", - AUTH_METHOD="BASIC", - PASSWORD_SECRET_NAME="secret_name" - ); - )sql" , "<main>:7:42: Error: LOGIN requires key\n"); - - ExpectFailWithError(R"sql( - USE plato; - CREATE EXTERNAL DATA SOURCE MyDataSource WITH ( - SOURCE_TYPE="PostgreSQL", - LOCATION="protocol://host:port/", - AUTH_METHOD="MDB_BASIC", - SERVICE_ACCOUNT_SECRET_NAME="sa_secret_name", - LOGIN="admin", - PASSWORD_SECRET_NAME="secret_name" - ); - )sql" , "<main>:9:42: Error: SERVICE_ACCOUNT_ID requires key\n"); - - ExpectFailWithError(R"sql( - USE plato; - CREATE EXTERNAL DATA SOURCE MyDataSource WITH ( - SOURCE_TYPE="PostgreSQL", - LOCATION="protocol://host:port/", - AUTH_METHOD="MDB_BASIC", - SERVICE_ACCOUNT_ID="sa", - LOGIN="admin", - PASSWORD_SECRET_NAME="secret_name" - ); - )sql" , "<main>:9:42: Error: SERVICE_ACCOUNT_SECRET_NAME requires key\n"); - - ExpectFailWithError(R"sql( - USE plato; - CREATE EXTERNAL DATA SOURCE MyDataSource WITH ( - SOURCE_TYPE="PostgreSQL", - LOCATION="protocol://host:port/", - AUTH_METHOD="MDB_BASIC", - SERVICE_ACCOUNT_ID="sa", - SERVICE_ACCOUNT_SECRET_NAME="sa_secret_name", - PASSWORD_SECRET_NAME="secret_name" - ); - )sql" , "<main>:9:42: Error: LOGIN requires key\n"); - - ExpectFailWithError(R"sql( - USE plato; - CREATE EXTERNAL DATA SOURCE MyDataSource WITH ( - SOURCE_TYPE="PostgreSQL", - LOCATION="protocol://host:port/", - AUTH_METHOD="MDB_BASIC", - SERVICE_ACCOUNT_ID="sa", - SERVICE_ACCOUNT_SECRET_NAME="sa_secret_name", - LOGIN="admin" - ); - )sql" , "<main>:9:27: Error: PASSWORD_SECRET_NAME requires key\n"); - - ExpectFailWithError(R"sql( - USE plato; - CREATE EXTERNAL DATA SOURCE MyDataSource WITH ( - SOURCE_TYPE="PostgreSQL", - LOCATION="protocol://host:port/", - AUTH_METHOD="AWS", - AWS_SECRET_ACCESS_KEY_SECRET_NAME="secret_key_name", - AWS_REGION="ru-central-1" - ); - )sql" , "<main>:8:32: Error: AWS_ACCESS_KEY_ID_SECRET_NAME requires key\n"); - - ExpectFailWithError(R"sql( - USE plato; - CREATE EXTERNAL DATA SOURCE MyDataSource WITH ( - SOURCE_TYPE="PostgreSQL", - LOCATION="protocol://host:port/", - AUTH_METHOD="AWS", - AWS_ACCESS_KEY_ID_SECRET_NAME="secred_id_name", - AWS_REGION="ru-central-1" - ); - )sql" , "<main>:8:32: Error: AWS_SECRET_ACCESS_KEY_SECRET_NAME requires key\n"); - - ExpectFailWithError(R"sql( - USE plato; - CREATE EXTERNAL DATA SOURCE MyDataSource WITH ( - SOURCE_TYPE="PostgreSQL", - LOCATION="protocol://host:port/", - AUTH_METHOD="AWS", - AWS_SECRET_ACCESS_KEY_SECRET_NAME="secret_key_name", - AWS_ACCESS_KEY_ID_SECRET_NAME="secred_id_name" - ); - )sql" , "<main>:8:51: Error: AWS_REGION requires key\n"); - } - - Y_UNIT_TEST(DropExternalDataSourceWithTablePrefix) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - DROP EXTERNAL DATA SOURCE MyDataSource; - )sql"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("'features")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("dropObject")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(DropExternalDataSource) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - pragma TablePathPrefix='/aba'; - DROP EXTERNAL DATA SOURCE MyDataSource; - )sql"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_STRING_CONTAINS(line, "/aba/MyDataSource"); - UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("'features")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("dropObject")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(DropExternalDataSourceIfExists) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - DROP EXTERNAL DATA SOURCE IF EXISTS MyDataSource; - )sql"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_STRING_CONTAINS(line, "MyDataSource"); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("dropObjectIfExists")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } -} - -Y_UNIT_TEST_SUITE(ExternalTable) { - Y_UNIT_TEST(CreateExternalTable) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - CREATE EXTERNAL TABLE mytable ( - a int - ) WITH ( - DATA_SOURCE="/Root/mydatasource", - LOCATION="/folder1/*" - ); - )sql"); - UNIT_ASSERT_C(res.Root, res.Issues.ToOneLineString()); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_STRING_CONTAINS(line, R"#('('('data_source_path (String '"/Root/mydatasource")) '('location (String '"/folder1/*")))) '('tableType 'externalTable)))))#"); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("tablescheme")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(CreateExternalTableWithTablePrefix) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - pragma TablePathPrefix='/aba'; - CREATE EXTERNAL TABLE mytable ( - a int - ) WITH ( - DATA_SOURCE="mydatasource", - LOCATION="/folder1/*" - ); - )sql"); - UNIT_ASSERT_C(res.Root, res.Issues.ToOneLineString()); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_STRING_CONTAINS(line, "/aba/mydatasource"); - UNIT_ASSERT_STRING_CONTAINS(line, "/aba/mytable"); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("tablescheme")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(CreateExternalTableObjectStorage) { - auto res = SqlToYql(R"sql( - USE plato; - CREATE EXTERNAL TABLE mytable ( - a int, - year Int - ) WITH ( - DATA_SOURCE="/Root/mydatasource", - LOCATION="/folder1/*", - FORMAT="json_as_string", - `projection.enabled`="true", - `projection.year.type`="integer", - `projection.year.min`="2010", - `projection.year.max`="2022", - `projection.year.interval`="1", - `projection.month.type`="integer", - `projection.month.min`="1", - `projection.month.max`="12", - `projection.month.interval`="1", - `projection.month.digits`="2", - `storage.location.template`="${year}/${month}", - PARTITONED_BY = "[year, month]" - ); - )sql"); - UNIT_ASSERT_C(res.IsOk(), res.Issues.ToString()); - } - - Y_UNIT_TEST(CreateExternalTableIfNotExists) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - CREATE EXTERNAL TABLE IF NOT EXISTS mytable ( - a int - ) WITH ( - DATA_SOURCE="/Root/mydatasource", - LOCATION="/folder1/*" - ); - )sql"); - UNIT_ASSERT_C(res.Root, res.Issues.ToOneLineString()); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_STRING_CONTAINS(line, R"#('('('data_source_path (String '"/Root/mydatasource")) '('location (String '"/folder1/*")))) '('tableType 'externalTable)))))#"); - UNIT_ASSERT_STRING_CONTAINS(line, "create_if_not_exists"); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(CreateExternalTableOrReplace) { - NYql::TAstParseResult res = SqlToYql(R"( - USE plato; - CREATE OR REPLACE EXTERNAL TABLE mytable ( - a int - ) WITH ( - DATA_SOURCE="/Root/mydatasource", - LOCATION="/folder1/*" - ); - )"); - UNIT_ASSERT_C(res.Root, res.Issues.ToOneLineString()); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_STRING_CONTAINS(line, R"#('('('data_source_path (String '"/Root/mydatasource")) '('location (String '"/folder1/*")))) '('tableType 'externalTable)))))#"); - UNIT_ASSERT_STRING_CONTAINS(line, "create_or_replace"); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(AlterExternalTableAddColumn) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - ALTER EXTERNAL TABLE mytable - ADD COLUMN my_column int32, - RESET (LOCATION); - )sql"); - UNIT_ASSERT_C(res.Root, res.Issues.ToOneLineString()); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_STRING_CONTAINS(line, R"#('actions '('('addColumns '('('"my_column" (AsOptionalType (DataType 'Int32))#"); - UNIT_ASSERT_STRING_CONTAINS(line, R"#(('setTableSettings '('('location)))#"); - UNIT_ASSERT_STRING_CONTAINS(line, R"#(('tableType 'externalTable))#"); - UNIT_ASSERT_STRING_CONTAINS(line, R"#(('mode 'alter))#"); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(AlterExternalTableDropColumn) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - ALTER EXTERNAL TABLE mytable - DROP COLUMN my_column, - SET (Location = "abc", Other_Prop = "42"), - SET x 'y'; - )sql"); - UNIT_ASSERT_C(res.Root, res.Issues.ToOneLineString()); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_STRING_CONTAINS(line, R"#('actions '('('dropColumns '('"my_column")#"); - UNIT_ASSERT_STRING_CONTAINS(line, R"#(('setTableSettings '('('location (String '"abc")) '('Other_Prop (String '"42")) '('x (String '"y")))))#"); - UNIT_ASSERT_STRING_CONTAINS(line, R"#(('tableType 'externalTable))#"); - UNIT_ASSERT_STRING_CONTAINS(line, R"#(('mode 'alter))#"); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(CreateExternalTableWithBadArguments) { - ExpectFailWithError(R"sql( - USE plato; - CREATE EXTERNAL TABLE mytable; - )sql" , "<main>:3:45: Error: Unexpected token ';' : syntax error...\n\n"); - - ExpectFailWithError(R"sql( - USE plato; - CREATE EXTERNAL TABLE mytable ( - a int - ); - )sql" , "<main>:4:23: Error: DATA_SOURCE requires key\n"); - - ExpectFailWithError(R"sql( - USE plato; - CREATE EXTERNAL TABLE mytable ( - a int - ) WITH ( - DATA_SOURCE="/Root/mydatasource" - ); - )sql" , "<main>:6:33: Error: LOCATION requires key\n"); - - ExpectFailWithError(R"sql( - USE plato; - CREATE EXTERNAL TABLE mytable ( - a int - ) WITH ( - LOCATION="/folder1/*" - ); - )sql" , "<main>:6:30: Error: DATA_SOURCE requires key\n"); - - ExpectFailWithError(R"sql( - USE plato; - CREATE EXTERNAL TABLE mytable ( - a int, - PRIMARY KEY(a) - ) WITH ( - DATA_SOURCE="/Root/mydatasource", - LOCATION="/folder1/*" - ); - )sql" , "<main>:8:30: Error: PRIMARY KEY is not supported for external table\n"); - } - - Y_UNIT_TEST(DropExternalTable) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - DROP EXTERNAL TABLE MyExternalTable; - )sql"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("tablescheme")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(DropExternalTableWithTablePrefix) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - pragma TablePathPrefix='/aba'; - DROP EXTERNAL TABLE MyExternalTable; - )sql"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_STRING_CONTAINS(line, "/aba/MyExternalTable"); - UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("'tablescheme")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(DropExternalTableIfExists) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - DROP EXTERNAL TABLE IF EXISTS MyExternalTable; - )sql"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("tablescheme")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("drop_if_exists")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } -} - -Y_UNIT_TEST_SUITE(TopicsDDL) { - void TestQuery(const TString& query, bool expectOk = true) { - TStringBuilder finalQuery; - - finalQuery << "use plato;" << Endl << query; - auto res = SqlToYql(finalQuery, 10, "kikimr"); - if (expectOk) { - UNIT_ASSERT_C(res.IsOk(), res.Issues.ToString()); - } else { - UNIT_ASSERT(!res.IsOk()); - } - } - - Y_UNIT_TEST(CreateTopicSimple) { - TestQuery(R"( - CREATE TOPIC topic1; - )"); - TestQuery(R"( - CREATE TOPIC `cluster1.topic1`; - )"); - TestQuery(R"( - CREATE TOPIC topic1 WITH (metering_mode = "str_value", partition_count_limit = 123, retention_period = Interval('PT1H')); - )"); - } - - Y_UNIT_TEST(CreateTopicConsumer) { - TestQuery(R"( - CREATE TOPIC topic1 (CONSUMER cons1); - )"); - TestQuery(R"( - CREATE TOPIC topic1 (CONSUMER cons1, CONSUMER cons2 WITH (important = false)); - )"); - TestQuery(R"( - CREATE TOPIC topic1 (CONSUMER cons1, CONSUMER cons2 WITH (important = false)) WITH (supported_codecs = "1,2,3"); - )"); - } - - Y_UNIT_TEST(AlterTopicSimple) { - TestQuery(R"( - ALTER TOPIC topic1 SET (retention_period = Interval('PT1H')); - )"); - TestQuery(R"( - ALTER TOPIC topic1 SET (retention_storage_mb = 3, partition_count_limit = 50); - )"); - TestQuery(R"( - ALTER TOPIC topic1 RESET (supported_codecs, retention_period); - )"); - TestQuery(R"( - ALTER TOPIC topic1 RESET (partition_write_speed_bytes_per_second), - SET (partition_write_burst_bytes = 11111, min_active_partitions = 1); - )"); - } - Y_UNIT_TEST(AlterTopicConsumer) { - TestQuery(R"( - ALTER TOPIC topic1 ADD CONSUMER consumer1, - ADD CONSUMER consumer2 WITH (important = false, supported_codecs = "RAW"), - ALTER CONSUMER consumer3 SET (important = false, read_from = 1), - ALTER CONSUMER consumer3 RESET (supported_codecs), - DROP CONSUMER consumer4, - SET (partition_count_limit = 11, retention_period = Interval('PT1H')), - RESET(metering_mode) - )"); - } - Y_UNIT_TEST(DropTopic) { - TestQuery(R"( - DROP TOPIC topic1; - )"); - } - - Y_UNIT_TEST(TopicBadRequests) { - TestQuery(R"( - CREATE TOPIC topic1(); - )", false); - TestQuery(R"( - CREATE TOPIC topic1 SET setting1 = value1; - )", false); - TestQuery(R"( - ALTER TOPIC topic1 SET setting1 value1; - )", false); - TestQuery(R"( - ALTER TOPIC topic1 RESET setting1; - )", false); - - TestQuery(R"( - ALTER TOPIC topic1 DROP CONSUMER consumer4 WITH (k1 = v1); - )", false); - - TestQuery(R"( - CREATE TOPIC topic1 WITH (retention_period = 123); - )", false); - TestQuery(R"( - CREATE TOPIC topic1 (CONSUMER cons1, CONSUMER cons1 WITH (important = false)); - )", false); - TestQuery(R"( - CREATE TOPIC topic1 (CONSUMER cons1 WITH (bad_option = false)); - )", false); - TestQuery(R"( - ALTER TOPIC topic1 ADD CONSUMER cons1, ALTER CONSUMER cons1 RESET (important); - )", false); - TestQuery(R"( - ALTER TOPIC topic1 ADD CONSUMER consumer1, - ALTER CONSUMER consumer3 SET (supported_codecs = "RAW", read_from = 1), - ALTER CONSUMER consumer3 RESET (supported_codecs); - )", false); - TestQuery(R"( - ALTER TOPIC topic1 ADD CONSUMER consumer1, - ALTER CONSUMER consumer3 SET (supported_codecs = "RAW", read_from = 1), - ALTER CONSUMER consumer3 SET (read_from = 2); - )", false); - } - - Y_UNIT_TEST(TopicWithPrefix) { - NYql::TAstParseResult res = SqlToYql(R"( - USE plato; - PRAGMA TablePathPrefix = '/database/path/to/tables'; - ALTER TOPIC `my_table/my_feed` ADD CONSUMER `my_consumer`; - )"); - UNIT_ASSERT(res.Root); - - TWordCountHive elementStat = {{TString("/database/path/to/tables/my_table/my_feed"), 0}, {"topic", 0}}; - VerifyProgram(res, elementStat); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["topic"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["/database/path/to/tables/my_table/my_feed"]); - } -} - -Y_UNIT_TEST_SUITE(BlockEnginePragma) { - Y_UNIT_TEST(Basic) { - const TVector<TString> values = {"auto", "force", "disable"}; - for (const auto& value : values) { - const auto query = TStringBuilder() << "pragma Blockengine='" << value << "'; select 1;"; - NYql::TAstParseResult res = SqlToYql(query); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [&](const TString& word, const TString& line) { - Y_UNUSED(word); - UNIT_ASSERT_STRING_CONTAINS(line, TStringBuilder() << R"(Configure! world (DataSource '"config") '"BlockEngine" '")" << value << "\""); - }; - - TWordCountHive elementStat({"BlockEngine"}); - VerifyProgram(res, elementStat, verifyLine); - UNIT_ASSERT(elementStat["BlockEngine"] == ((value == "disable") ? 0 : 1)); - } - } - - Y_UNIT_TEST(UnknownSetting) { - ExpectFailWithError("use plato; pragma BlockEngine='foo';", - "<main>:1:31: Error: Expected `disable|auto|force' argument for: BlockEngine\n"); - } -} - -Y_UNIT_TEST_SUITE(TViewSyntaxTest) { - Y_UNIT_TEST(CreateViewSimple) { - NYql::TAstParseResult res = SqlToYql(R"( - USE plato; - CREATE VIEW TheView WITH (security_invoker = TRUE) AS SELECT 1; - )" - ); - UNIT_ASSERT_C(res.Root, res.Issues.ToString()); - } - - Y_UNIT_TEST(CreateViewIfNotExists) { - constexpr const char* name = "TheView"; - NYql::TAstParseResult res = SqlToYql(std::format(R"( - USE plato; - CREATE VIEW IF NOT EXISTS {} WITH (security_invoker = TRUE) AS SELECT 1; - )", name - )); - UNIT_ASSERT_C(res.Root, res.Issues.ToString()); - - TVerifyLineFunc verifyLine = [&](const TString& word, const TString& line) { - if (word == "Write!") { - UNIT_ASSERT_STRING_CONTAINS(line, name); - UNIT_ASSERT_STRING_CONTAINS(line, "createObjectIfNotExists"); - } - }; - - TWordCountHive elementStat = { {"Write!"} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(elementStat["Write!"], 1); - } - - Y_UNIT_TEST(CreateViewFromTable) { - constexpr const char* path = "/PathPrefix/TheView"; - constexpr const char* query = R"( - SELECT * FROM SomeTable - )"; - - NYql::TAstParseResult res = SqlToYql(std::format(R"( - USE plato; - CREATE VIEW `{}` WITH (security_invoker = TRUE) AS {}; - )", - path, - query - ) - ); - UNIT_ASSERT_C(res.Root, res.Issues.ToString()); - - TVerifyLineFunc verifyLine = [&](const TString& word, const TString& line) { - if (word == "Write!") { - UNIT_ASSERT_STRING_CONTAINS(line, path); - UNIT_ASSERT_STRING_CONTAINS(line, "createObject"); - } - }; - TWordCountHive elementStat = { {"Write!"} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(elementStat["Write!"], 1); - } - - Y_UNIT_TEST(CheckReconstructedQuery) { - constexpr const char* path = "/PathPrefix/TheView"; - constexpr const char* query = R"( - SELECT * FROM FirstTable JOIN SecondTable ON FirstTable.key == SecondTable.key - )"; - - NYql::TAstParseResult res = SqlToYql(std::format(R"( - USE plato; - CREATE VIEW `{}` WITH (security_invoker = TRUE) AS {}; - )", - path, - query - ) - ); - UNIT_ASSERT_C(res.Root, res.Issues.ToString()); - - TString reconstructedQuery = ToString(Tokenize(query)); - TVerifyLineFunc verifyLine = [&](const TString& word, const TString& line) { - if (word == "query_text") { - UNIT_ASSERT_STRING_CONTAINS(line, reconstructedQuery); - } - }; - TWordCountHive elementStat = { {"Write!"} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(elementStat["Write!"], 1); - } - - Y_UNIT_TEST(DropView) { - constexpr const char* path = "/PathPrefix/TheView"; - NYql::TAstParseResult res = SqlToYql(std::format(R"( - USE plato; - DROP VIEW `{}`; - )", - path - ) - ); - UNIT_ASSERT_C(res.Root, res.Issues.ToString()); - - TVerifyLineFunc verifyLine = [&](const TString& word, const TString& line) { - if (word == "Write!") { - UNIT_ASSERT_STRING_CONTAINS(line, path); - UNIT_ASSERT_STRING_CONTAINS(line, "dropObject"); - } - }; - TWordCountHive elementStat = { {"Write!"} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(elementStat["Write!"], 1); - } - - Y_UNIT_TEST(DropViewIfExists) { - constexpr const char* name = "TheView"; - NYql::TAstParseResult res = SqlToYql(std::format(R"( - USE plato; - DROP VIEW IF EXISTS {}; - )", name - )); - UNIT_ASSERT_C(res.Root, res.Issues.ToString()); - - TVerifyLineFunc verifyLine = [&](const TString& word, const TString& line) { - if (word == "Write!") { - UNIT_ASSERT_STRING_CONTAINS(line, name); - UNIT_ASSERT_STRING_CONTAINS(line, "dropObjectIfExists"); - } - }; - - TWordCountHive elementStat = { {"Write!"} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(elementStat["Write!"], 1); - } - - Y_UNIT_TEST(CreateViewWithTablePrefix) { - NYql::TAstParseResult res = SqlToYql(R"( - USE plato; - PRAGMA TablePathPrefix='/PathPrefix'; - CREATE VIEW TheView WITH (security_invoker = TRUE) AS SELECT 1; - )" - ); - UNIT_ASSERT_C(res.Root, res.Issues.ToString()); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write!") { - UNIT_ASSERT_STRING_CONTAINS(line, "/PathPrefix/TheView"); - UNIT_ASSERT_STRING_CONTAINS(line, "createObject"); - } - }; - - TWordCountHive elementStat = { {"Write!"} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(elementStat["Write!"], 1); - } - - Y_UNIT_TEST(DropViewWithTablePrefix) { - NYql::TAstParseResult res = SqlToYql(R"( - USE plato; - PRAGMA TablePathPrefix='/PathPrefix'; - DROP VIEW TheView; - )" - ); - UNIT_ASSERT_C(res.Root, res.Issues.ToString()); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_STRING_CONTAINS(line, "/PathPrefix/TheView"); - UNIT_ASSERT_STRING_CONTAINS(line, "dropObject"); - } - }; - - TWordCountHive elementStat = { {"Write!"} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(elementStat["Write!"], 1); - } - - Y_UNIT_TEST(YtAlternativeSchemaSyntax) { - NYql::TAstParseResult res = SqlToYql(R"( - SELECT * FROM plato.Input WITH schema(y Int32, x String not null); - )"); - UNIT_ASSERT_C(res.Root, res.Issues.ToString()); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "userschema") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, - line.find(R"__('('('"userschema" (StructType '('"y" (AsOptionalType (DataType 'Int32))) '('"x" (DataType 'String))))))__")); - } - }; - - TWordCountHive elementStat = {{TString("userschema"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["userschema"]); - } - - Y_UNIT_TEST(UseViewAndFullColumnId) { - NYql::TAstParseResult res = SqlToYql("USE plato; SELECT Input.x FROM Input VIEW uitzicht;"); - UNIT_ASSERT(res.Root); - - TWordCountHive elementStat = {{TString("SqlAccess"), 0}, {"SqlProjectItem", 0}, {"Read!", 0}}; - VerifyProgram(res, elementStat); - UNIT_ASSERT_VALUES_EQUAL(0, elementStat["SqlAccess"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SqlProjectItem"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Read!"]); - } -} - -Y_UNIT_TEST_SUITE(CompactNamedExprs) { - Y_UNIT_TEST(SourceCallablesInWrongContext) { - TString query = R"( - pragma CompactNamedExprs; - $foo = %s(); - select $foo from plato.Input; - )"; - - THashMap<TString, TString> errs = { - {"TableRow", "<main>:3:20: Error: TableRow requires data source\n"}, - {"JoinTableRow", "<main>:3:20: Error: JoinTableRow requires data source\n"}, - {"TableRecordIndex", "<main>:3:20: Error: Unable to use function: TableRecord without source\n"}, - {"TablePath", "<main>:3:20: Error: Unable to use function: TablePath without source\n"}, - {"SystemMetadata", "<main>:3:20: Error: Unable to use function: SystemMetadata without source\n"}, - }; - - for (TString callable : { "TableRow", "JoinTableRow", "TableRecordIndex", "TablePath", "SystemMetadata"}) { - auto req = Sprintf(query.c_str(), callable.c_str()); - ExpectFailWithError(req, errs[callable]); - } - } - - Y_UNIT_TEST(ValidateUnusedExprs) { - TString query = R"( - pragma warning("disable", "4527"); - pragma CompactNamedExprs; - pragma ValidateUnusedExprs; - - $foo = count(1); - select 1; - )"; - ExpectFailWithError(query, "<main>:6:20: Error: Aggregation is not allowed in this context\n"); - query = R"( - pragma warning("disable", "4527"); - pragma CompactNamedExprs; - pragma ValidateUnusedExprs; - - define subquery $x() as - select count(1, 2); - end define; - select 1; - )"; - ExpectFailWithError(query, "<main>:7:24: Error: Aggregation function Count requires exactly 1 argument(s), given: 2\n"); - } - - Y_UNIT_TEST(DisableValidateUnusedExprs) { - TString query = R"( - pragma warning("disable", "4527"); - pragma CompactNamedExprs; - pragma DisableValidateUnusedExprs; - - $foo = count(1); - select 1; - )"; - SqlToYql(query).IsOk(); - query = R"( - pragma warning("disable", "4527"); - pragma CompactNamedExprs; - pragma DisableValidateUnusedExprs; - - define subquery $x() as - select count(1, 2); - end define; - select 1; - )"; - SqlToYql(query).IsOk(); - } -} - -Y_UNIT_TEST_SUITE(ResourcePool) { - Y_UNIT_TEST(CreateResourcePool) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - CREATE RESOURCE POOL MyResourcePool WITH ( - CONCURRENT_QUERY_LIMIT=20, - QUERY_CANCEL_AFTER_SECONDS=86400, - QUEUE_TYPE="FIFO" - ); - )sql"); - UNIT_ASSERT_C(res.Root, res.Issues.ToString()); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_STRING_CONTAINS(line, R"#('('('"concurrent_query_limit" (Int32 '"20")) '('"query_cancel_after_seconds" (Int32 '"86400")) '('"queue_type" '"FIFO"))#"); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("createObject")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(CreateResourcePoolWithBadArguments) { - ExpectFailWithError(R"sql( - USE plato; - CREATE RESOURCE POOL MyResourcePool; - )sql" , "<main>:3:51: Error: Unexpected token ';' : syntax error...\n\n"); - - ExpectFailWithError(R"sql( - USE plato; - CREATE RESOURCE POOL MyResourcePool WITH ( - DUPLICATE_SETTING="first_value", - DUPLICATE_SETTING="second_value" - ); - )sql" , "<main>:5:21: Error: DUPLICATE_SETTING duplicate keys\n"); - } - - Y_UNIT_TEST(AlterResourcePool) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - ALTER RESOURCE POOL MyResourcePool - SET (CONCURRENT_QUERY_LIMIT = 30, Weight = 5, QUEUE_TYPE = "UNORDERED"), - RESET (Query_Cancel_After_Seconds, Query_Count_Limit); - )sql"); - UNIT_ASSERT_C(res.Root, res.Issues.ToString()); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_STRING_CONTAINS(line, R"#(('mode 'alterObject))#"); - UNIT_ASSERT_STRING_CONTAINS(line, R"#('('features '('('"concurrent_query_limit" (Int32 '"30")) '('"queue_type" '"UNORDERED") '('"weight" (Int32 '"5")))))#"); - UNIT_ASSERT_STRING_CONTAINS(line, R"#('('resetFeatures '('"query_cancel_after_seconds" '"query_count_limit")))#"); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(DropResourcePool) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - DROP RESOURCE POOL MyResourcePool; - )sql"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("'features")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("dropObject")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } -} - -Y_UNIT_TEST_SUITE(BackupCollection) { - Y_UNIT_TEST(CreateBackupCollection) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - CREATE BACKUP COLLECTION TestCollection WITH ( - STORAGE="local", - TAG="test" -- for testing purposes, not a real thing - ); - )sql"); - UNIT_ASSERT_C(res.Root, res.Issues.ToString()); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(R"#('"TestCollection")#")); - UNIT_ASSERT_STRING_CONTAINS(line, R"#(('"storage" (String '"local")))#"); - UNIT_ASSERT_STRING_CONTAINS(line, R"#(('"tag" (String '"test"))))#"); - UNIT_ASSERT_STRING_CONTAINS(line, R"#(('entries '()))#"); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'create")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(CreateBackupCollectionWithDatabase) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - CREATE BACKUP COLLECTION TestCollection DATABASE WITH ( - STORAGE="local", - TAG="test" -- for testing purposes, not a real thing - ); - )sql"); - UNIT_ASSERT_C(res.Root, res.Issues.ToString()); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(R"#('"TestCollection")#")); - UNIT_ASSERT_STRING_CONTAINS(line, R"#(('"storage" (String '"local")))#"); - UNIT_ASSERT_STRING_CONTAINS(line, R"#(('"tag" (String '"test"))))#"); - UNIT_ASSERT_STRING_CONTAINS(line, R"#(('entries '('('('type 'database)))))#"); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'create")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(CreateBackupCollectionWithTables) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - CREATE BACKUP COLLECTION TestCollection ( - TABLE someTable, - TABLE `prefix/anotherTable` - ) WITH ( - STORAGE="local", - TAG="test" -- for testing purposes, not a real thing - ); - )sql"); - UNIT_ASSERT_C(res.Root, res.Issues.ToString()); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(R"#('"TestCollection")#")); - UNIT_ASSERT_STRING_CONTAINS(line, R"#(('"storage" (String '"local")))#"); - UNIT_ASSERT_STRING_CONTAINS(line, R"#(('"tag" (String '"test"))))#"); - UNIT_ASSERT_STRING_CONTAINS(line, R"#(('entries '('('('type 'table) '('path '"someTable")) '('('type 'table) '('path '"prefix/anotherTable")))))#"); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'create")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(CreateBackupCollectionWithBadArguments) { - ExpectFailWithError(R"sql( - USE plato; - CREATE BACKUP COLLECTION TestCollection; - )sql" , "<main>:3:55: Error: Unexpected token ';' : syntax error...\n\n"); - - ExpectFailWithError(R"sql( - USE plato; - CREATE BACKUP COLLECTION TABLE TestCollection; - )sql" , "<main>:3:47: Error: Unexpected token 'TestCollection' : syntax error...\n\n"); - - ExpectFailWithError(R"sql( - USE plato; - CREATE BACKUP COLLECTION DATABASE `test` TestCollection; - )sql" , "<main>:3:50: Error: Unexpected token '`test`' : syntax error...\n\n"); - - ExpectFailWithError(R"sql( - USE plato; - CREATE BACKUP COLLECTION TestCollection WITH ( - DUPLICATE_SETTING="first_value", - DUPLICATE_SETTING="second_value" - ); - )sql" , "<main>:5:21: Error: DUPLICATE_SETTING duplicate keys\n"); - - ExpectFailWithError(R"sql( - USE plato; - CREATE BACKUP COLLECTION TestCollection WITH ( - INT_SETTING=1 - ); - )sql" , "<main>:4:21: Error: INT_SETTING value should be a string literal\n"); - } - - Y_UNIT_TEST(AlterBackupCollection) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - ALTER BACKUP COLLECTION TestCollection - SET (STORAGE="remote"), -- also just for test - SET (TAG1 = "123"), - RESET (TAG2, TAG3); - )sql"); - UNIT_ASSERT_C(res.Root, res.Issues.ToString()); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(R"#('"TestCollection")#")); - UNIT_ASSERT_STRING_CONTAINS(line, R"#(('mode 'alter))#"); - UNIT_ASSERT_STRING_CONTAINS(line, R"#(('"storage" (String '"remote")))#"); - UNIT_ASSERT_STRING_CONTAINS(line, R"#(('"tag1" (String '"123"))))#"); - UNIT_ASSERT_STRING_CONTAINS(line, R"#('('resetSettings '('"tag2" '"tag3")))#"); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(AlterBackupCollectionEntries) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - ALTER BACKUP COLLECTION TestCollection - DROP TABLE `test`, - ADD DATABASE; - )sql"); - UNIT_ASSERT_C(res.Root, res.Issues.ToString()); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(R"#('"TestCollection")#")); - UNIT_ASSERT_STRING_CONTAINS(line, R"#(('mode 'alter))#"); - UNIT_ASSERT_STRING_CONTAINS(line, R"#('alterEntries)#"); - UNIT_ASSERT_STRING_CONTAINS(line, R"#('('('type 'table) '('path '"test") '('action 'drop)))#"); - UNIT_ASSERT_STRING_CONTAINS(line, R"#('('('type 'database) '('action 'add)))#"); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(DropBackupCollection) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - DROP BACKUP COLLECTION TestCollection; - )sql"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(R"#('"TestCollection")#")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("drop")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } -} - -Y_UNIT_TEST_SUITE(ResourcePoolClassifier) { - Y_UNIT_TEST(CreateResourcePoolClassifier) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - CREATE RESOURCE POOL CLASSIFIER MyResourcePoolClassifier WITH ( - RANK=20, - RESOURCE_POOL='wgUserQueries', - MEMBER_NAME='yandex_query@abc' - ); - )sql"); - UNIT_ASSERT_C(res.Root, res.Issues.ToString()); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_STRING_CONTAINS(line, R"#('('('"member_name" '"yandex_query@abc") '('"rank" (Int32 '"20")) '('"resource_pool" '"wgUserQueries"))#"); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("createObject")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(CreateResourcePoolClassifierWithBadArguments) { - ExpectFailWithError(R"sql( - USE plato; - CREATE RESOURCE POOL CLASSIFIER MyResourcePoolClassifier; - )sql" , "<main>:3:72: Error: Unexpected token ';' : syntax error...\n\n"); - - ExpectFailWithError(R"sql( - USE plato; - CREATE RESOURCE POOL CLASSIFIER MyResourcePoolClassifier WITH ( - DUPLICATE_SETTING="first_value", - DUPLICATE_SETTING="second_value" - ); - )sql" , "<main>:5:21: Error: DUPLICATE_SETTING duplicate keys\n"); - } - - Y_UNIT_TEST(AlterResourcePoolClassifier) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - ALTER RESOURCE POOL CLASSIFIER MyResourcePoolClassifier - SET (RANK = 30, Weight = 5, MEMBER_NAME = "test@user"), - RESET (Resource_Pool); - )sql"); - UNIT_ASSERT_C(res.Root, res.Issues.ToString()); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_STRING_CONTAINS(line, R"#(('mode 'alterObject))#"); - UNIT_ASSERT_STRING_CONTAINS(line, R"#('('features '('('"member_name" '"test@user") '('"rank" (Int32 '"30")) '('"weight" (Int32 '"5")))))#"); - UNIT_ASSERT_STRING_CONTAINS(line, R"#('('resetFeatures '('"resource_pool")))#"); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(DropResourcePoolClassifier) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - DROP RESOURCE POOL CLASSIFIER MyResourcePoolClassifier; - )sql"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("'features")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("dropObject")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(BacktickMatching) { - auto req = "select\n" - " 1 as `Schema has \\`RealCost\\``\n" - " -- foo`bar"; - auto res = SqlToYql(req); - UNIT_ASSERT(res.Root); - UNIT_ASSERT(res.IsOk()); - UNIT_ASSERT(res.Issues.Size() == 0); - res = SqlToYqlWithAnsiLexer(req); - UNIT_ASSERT(res.Root); - UNIT_ASSERT(res.IsOk()); - UNIT_ASSERT(res.Issues.Size() == 0); - - req = "select 1 as `a``b`, 2 as ````, 3 as `\\x60a\\x60`, 4 as ```b```, 5 as `\\`c\\``"; - res = SqlToYql(req); - UNIT_ASSERT(res.Root); - UNIT_ASSERT(res.IsOk()); - UNIT_ASSERT(res.Issues.Size() == 0); - res = SqlToYqlWithAnsiLexer(req); - UNIT_ASSERT(res.Root); - UNIT_ASSERT(res.IsOk()); - UNIT_ASSERT(res.Issues.Size() == 0); - } -} - -Y_UNIT_TEST_SUITE(OlapPartitionCount) { - Y_UNIT_TEST(CorrectUsage) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - CREATE TABLE `mytable` (id Uint32, PRIMARY KEY (id)) - PARTITION BY HASH(id) - WITH (STORE = COLUMN, PARTITION_COUNT = 8); - )sql"); - - UNIT_ASSERT_C(res.IsOk(), res.Issues.ToString()); - } - - Y_UNIT_TEST(UseWithoutColumnStore) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - CREATE TABLE `mytable` (id Uint32, PRIMARY KEY (id)) - WITH (PARTITION_COUNT = 8); - )sql"); - - UNIT_ASSERT(!res.IsOk()); - UNIT_ASSERT(res.Issues.Size() == 1); - UNIT_ASSERT_STRING_CONTAINS(res.Issues.ToString(), "PARTITION_COUNT can be used only with STORE=COLUMN"); - } -} - -Y_UNIT_TEST_SUITE(Backup) { - Y_UNIT_TEST(Simple) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - BACKUP TestCollection; - )sql"); - UNIT_ASSERT_C(res.Root, res.Issues.ToString()); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(R"#('"TestCollection")#")); - UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("'Incremental")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'backup")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(Incremental) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - BACKUP TestCollection INCREMENTAL; - )sql"); - UNIT_ASSERT_C(res.Root, res.Issues.ToString()); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(R"#('"TestCollection")#")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'backupIncremental")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } -} - -Y_UNIT_TEST_SUITE(Restore) { - Y_UNIT_TEST(Simple) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - RESTORE TestCollection; - )sql"); - UNIT_ASSERT_C(res.Root, res.Issues.ToString()); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(R"#('"TestCollection")#")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'restore")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(AtPoint) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - RESTORE TestCollection AT '2024-06-16_20-14-02'; - )sql"); - UNIT_ASSERT_C(res.Root, res.Issues.ToString()); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(R"#('"TestCollection")#")); - UNIT_ASSERT_STRING_CONTAINS(line, R"#('at '"2024-06-16_20-14-02")#"); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'restore")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } -} - -Y_UNIT_TEST_SUITE(ColumnFamily) { - Y_UNIT_TEST(CompressionLevelCorrectUsage) { - NYql::TAstParseResult res = SqlToYql(R"( use plato; - CREATE TABLE tableName ( - Key Uint32 FAMILY default, - Value String FAMILY family1, - PRIMARY KEY (Key), - FAMILY default ( - DATA = "test", - COMPRESSION = "lz4", - COMPRESSION_LEVEL = 5 - ), - FAMILY family1 ( - DATA = "test", - COMPRESSION = "lz4", - COMPRESSION_LEVEL = 3 - ) - ); - )"); - UNIT_ASSERT(res.IsOk()); - UNIT_ASSERT(res.Issues.Size() == 0); - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("compression_level")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("5")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("3")); - } - }; - - TWordCountHive elementStat = { { TString("Write"), 0 }, { TString("compression_level"), 0 } }; - VerifyProgram(res, elementStat, verifyLine); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - UNIT_ASSERT_VALUES_EQUAL(2, elementStat["compression_level"]); - } - - Y_UNIT_TEST(FieldDataIsNotString) { - NYql::TAstParseResult res = SqlToYql(R"( use plato; - CREATE TABLE tableName ( - Key Uint32 FAMILY default, - PRIMARY KEY (Key), - FAMILY default ( - DATA = 1, - COMPRESSION = "lz4", - COMPRESSION_LEVEL = 5 - ) - ); - )"); - UNIT_ASSERT(!res.IsOk()); - UNIT_ASSERT(res.Issues.Size() == 1); - UNIT_ASSERT_STRING_CONTAINS(res.Issues.ToString(), "DATA value should be a string literal"); - } - - Y_UNIT_TEST(FieldCompressionIsNotString) { - NYql::TAstParseResult res = SqlToYql(R"( use plato; - CREATE TABLE tableName ( - Key Uint32 FAMILY default, - PRIMARY KEY (Key), - FAMILY default ( - DATA = "test", - COMPRESSION = 2, - COMPRESSION_LEVEL = 5 - ), - ); - )"); - UNIT_ASSERT(!res.IsOk()); - UNIT_ASSERT(res.Issues.Size() == 1); - UNIT_ASSERT_STRING_CONTAINS(res.Issues.ToString(), "COMPRESSION value should be a string literal"); - } - - Y_UNIT_TEST(FieldCompressionLevelIsNotInteger) { - NYql::TAstParseResult res = SqlToYql(R"( use plato; - CREATE TABLE tableName ( - Key Uint32 FAMILY default, - PRIMARY KEY (Key), - FAMILY default ( - DATA = "test", - COMPRESSION = "lz4", - COMPRESSION_LEVEL = "5" - ) - ); - )"); - UNIT_ASSERT(!res.IsOk()); - UNIT_ASSERT(res.Issues.Size() == 1); - UNIT_ASSERT_STRING_CONTAINS(res.Issues.ToString(), "COMPRESSION_LEVEL value should be an integer"); - } - - Y_UNIT_TEST(AlterCompressionCorrectUsage) { - NYql::TAstParseResult res = SqlToYql(R"( use plato; - ALTER TABLE tableName ALTER FAMILY default SET COMPRESSION "lz4"; - )"); - UNIT_ASSERT(res.IsOk()); - UNIT_ASSERT(res.Issues.Size() == 0); - } - - Y_UNIT_TEST(AlterCompressionFieldIsNotString) { - NYql::TAstParseResult res = SqlToYql(R"( use plato; - ALTER TABLE tableName ALTER FAMILY default SET COMPRESSION lz4; - )"); - UNIT_ASSERT(!res.IsOk()); - UNIT_ASSERT(res.Issues.Size() == 1); - UNIT_ASSERT_STRING_CONTAINS(res.Issues.ToString(), "Unexpected token 'lz4' : cannot match to any predicted input"); - } - - Y_UNIT_TEST(AlterCompressionLevelCorrectUsage) { - NYql::TAstParseResult res = SqlToYql(R"( use plato; - ALTER TABLE tableName ALTER FAMILY default SET COMPRESSION_LEVEL 5; - )"); - UNIT_ASSERT(res.IsOk()); - UNIT_ASSERT(res.Issues.Size() == 0); - } - - Y_UNIT_TEST(AlterCompressionLevelFieldIsNotInteger) { - NYql::TAstParseResult res = SqlToYql(R"( use plato; - ALTER TABLE tableName ALTER FAMILY default SET COMPRESSION_LEVEL "5"; - )"); - UNIT_ASSERT(!res.IsOk()); - UNIT_ASSERT(res.Issues.Size() == 1); - UNIT_ASSERT_STRING_CONTAINS(res.Issues.ToString(), "COMPRESSION_LEVEL value should be an integer"); - } -} +#define ANTLR_VER 3 +#include "sql_ut_common.h" Y_UNIT_TEST_SUITE(QuerySplit) { Y_UNIT_TEST(Simple) { @@ -8207,42 +95,3 @@ Y_UNIT_TEST_SUITE(QuerySplit) { };)"); } } - -Y_UNIT_TEST_SUITE(MatchRecognizeMeasuresAggregation) { - Y_UNIT_TEST(InsideSelect) { - ExpectFailWithError(R"sql( - SELECT FIRST(0), LAST(1); - )sql", - "<main>:2:20: Error: Cannot use FIRST and LAST outside the MATCH_RECOGNIZE context\n" - "<main>:2:30: Error: Cannot use FIRST and LAST outside the MATCH_RECOGNIZE context\n" - ); - } - - Y_UNIT_TEST(OutsideSelect) { - ExpectFailWithError(R"sql( - $lambda = ($x) -> (FIRST($x) + LAST($x)); - SELECT $lambda(x) FROM plato.Input; - )sql", - "<main>:2:32: Error: Cannot use FIRST and LAST outside the MATCH_RECOGNIZE context\n" - "<main>:2:44: Error: Cannot use FIRST and LAST outside the MATCH_RECOGNIZE context\n" - ); - } - - Y_UNIT_TEST(AsAggregateFunction) { - ExpectFailWithError(R"sql( - SELECT FIRST(x), LAST(x) FROM plato.Input; - )sql", - "<main>:2:20: Error: Cannot use FIRST and LAST outside the MATCH_RECOGNIZE context\n" - "<main>:2:30: Error: Cannot use FIRST and LAST outside the MATCH_RECOGNIZE context\n" - ); - } - - Y_UNIT_TEST(AsWindowFunction) { - ExpectFailWithError(R"sql( - SELECT FIRST(x) OVER(), LAST(x) OVER() FROM plato.Input; - )sql", - "<main>:2:20: Error: Cannot use FIRST and LAST outside the MATCH_RECOGNIZE context\n" - "<main>:2:37: Error: Cannot use FIRST and LAST outside the MATCH_RECOGNIZE context\n" - ); - } -} diff --git a/yql/essentials/sql/v1/sql_ut_antlr4.cpp b/yql/essentials/sql/v1/sql_ut_antlr4.cpp index 6d54a4f409..18480d048c 100644 --- a/yql/essentials/sql/v1/sql_ut_antlr4.cpp +++ b/yql/essentials/sql/v1/sql_ut_antlr4.cpp @@ -30,8093 +30,10 @@ TParsedTokenList Tokenize(const TString& query) { return tokens; } -TString ToString(const TParsedTokenList& tokens) { - TStringBuilder reconstructedQuery; - for (const auto& token : tokens) { - if (token.Name == "WS" || token.Name == "EOF") { - continue; - } - if (!reconstructedQuery.empty()) { - reconstructedQuery << ' '; - } - reconstructedQuery << token.Content; - } - return reconstructedQuery; -} - -} - -Y_UNIT_TEST_SUITE(AnsiMode) { - Y_UNIT_TEST(PragmaAnsi) { - UNIT_ASSERT(SqlToYql("PRAGMA ANSI 2016;").IsOk()); - } -} - -Y_UNIT_TEST_SUITE(SqlParsingOnly) { - ///This function is used in BACKWARD COMPATIBILITY tests below that LIMIT the sets of token that CAN NOT be used - ///as identifiers in different contexts in a SQL request - ///\return list of tokens that failed this check - TVector<TString> ValidateTokens(const THashSet<TString>& forbidden, const std::function<TString (const TString& )>& makeRequest) { - THashMap<TString, bool> allTokens; - for (const auto& t: NSQLFormat::GetKeywords()) { - allTokens[t] = !forbidden.contains((t)); - } - for (const auto& f: forbidden) { - UNIT_ASSERT(allTokens.contains(f)); //check that forbidden list contains tokens only(argument check) - } - TVector<TString> failed; - for (const auto& [token, allowed]: allTokens) { - if (SqlToYql(makeRequest(token)).IsOk() != allowed) - failed.push_back(token); - } - return failed; - } - - Y_UNIT_TEST(TokensAsColumnName) { //id_expr - auto failed = ValidateTokens({ - "ALL", "ANY", "AS", "ASSUME", "ASYMMETRIC", "AUTOMAP", "BETWEEN", "BITCAST", - "CALLABLE", "CASE", "CAST", "CUBE", "CURRENT_DATE", "CURRENT_TIME", "CURRENT_TIMESTAMP", - "DICT", "DISTINCT", "ENUM", "ERASE", "EXCEPT", "EXISTS", "FLOW", "FROM", "FULL", - "HAVING", "HOP", "INTERSECT", "JSON_EXISTS", "JSON_QUERY", "JSON_VALUE", "LIMIT", "LIST", - "NOT", "OPTIONAL", "PROCESS", "REDUCE", "REPEATABLE", "RESOURCE", "RETURN", "RETURNING", "ROLLUP", - "SELECT", "SET", "STREAM", "STRUCT", "SYMMETRIC", "TAGGED", "TUPLE", "UNBOUNDED", - "UNION", "VARIANT", "WHEN", "WHERE", "WINDOW", "WITHOUT" - }, - [](const TString& token){ - TStringBuilder req; - req << "SELECT " << token << " FROM Plato.Input"; - return req; - } - ); - UNIT_ASSERT_VALUES_EQUAL(failed, TVector<TString>{}); - } - - Y_UNIT_TEST(TokensAsWithoutColumnName) { //id_without - auto failed = ValidateTokens({ - "ALL", "AS", "ASSUME", "ASYMMETRIC", "AUTOMAP", "BETWEEN", "BITCAST", - "CALLABLE", "CASE", "CAST", "CUBE", "CURRENT_DATE", "CURRENT_TIME", "CURRENT_TIMESTAMP", - "DICT", "DISTINCT", "EMPTY_ACTION", "ENUM", "EXCEPT", "EXISTS", "FALSE", "FLOW", "FROM", "FULL", - "HAVING", "HOP", "INTERSECT", "JSON_EXISTS", "JSON_QUERY", "JSON_VALUE", "LIMIT", "LIST", - "NOT", "NULL", "OPTIONAL", "PROCESS", "REDUCE", "REPEATABLE", "RESOURCE", "RETURN", "RETURNING", "ROLLUP", - "SELECT", "SET", "STRUCT", "SYMMETRIC", "TAGGED", "TRUE", "TUPLE", "UNBOUNDED", - "UNION", "VARIANT", "WHEN", "WHERE", "WINDOW", "WITHOUT" - }, - [](const TString& token){ - TStringBuilder req; - req << "SELECT * WITHOUT " << token << " FROM Plato.Input"; - return req; - } - ); - UNIT_ASSERT_VALUES_EQUAL(failed, TVector<TString>{}); - } - - Y_UNIT_TEST(TokensAsColumnNameInAddColumn) { //id_schema - auto failed = ValidateTokens({ - "ANY", "AUTOMAP", "CALLABLE", "COLUMN", "DICT", "ENUM", "ERASE", "FALSE", "FLOW", - "LIST", "OPTIONAL", "REPEATABLE", "RESOURCE", - "SET", "STREAM", "STRUCT", "TAGGED", "TRUE", "TUPLE", "VARIANT" - }, - [](const TString& token){ - TStringBuilder req; - req << "ALTER TABLE Plato.Input ADD COLUMN " << token << " Bool"; - return req; - } - ); - UNIT_ASSERT_VALUES_EQUAL(failed, TVector<TString>{}); - } - - Y_UNIT_TEST(TokensAsColumnAlias) { - auto failed = ValidateTokens({ - "AUTOMAP", "FALSE", - "REPEATABLE", "TRUE" - }, - [](const TString& token){ - TStringBuilder req; - req << "SELECT Col as " << token << " FROM Plato.Input"; - return req; - } - ); - UNIT_ASSERT_VALUES_EQUAL(failed, TVector<TString>{}); - } - - Y_UNIT_TEST(TokensAsTableName) { //id_table_or_type - auto failed = ValidateTokens({ - "ANY", "AUTOMAP", "COLUMN", "ERASE", "FALSE", - "REPEATABLE", "STREAM", "TRUE" - }, - [](const TString& token){ - TStringBuilder req; - req << "SELECT * FROM Plato." << token; - return req; - } - ); - UNIT_ASSERT_VALUES_EQUAL(failed, TVector<TString>{}); - } - - Y_UNIT_TEST(TokensAsTableAlias) { //id_table - auto failed = ValidateTokens({ - "AUTOMAP", "CALLABLE", "DICT", "ENUM","FALSE", "FLOW", - "LIST", "OPTIONAL", "REPEATABLE", "RESOURCE", - "SET", "STRUCT", "TAGGED", "TRUE", "TUPLE", "VARIANT" - }, - [](const TString& token){ - TStringBuilder req; - req << "SELECT * FROM Plato.Input AS " << token; - return req; - } - ); - UNIT_ASSERT_VALUES_EQUAL(failed, TVector<TString>{}); - } - - Y_UNIT_TEST(TokensAsHints) { //id_hint - auto failed = ValidateTokens({ - "AUTOMAP", "CALLABLE", "COLUMNS", "DICT", "ENUM", "FALSE", "FLOW", - "LIST", "OPTIONAL", "REPEATABLE", "RESOURCE", - "SCHEMA", "SET", "STRUCT", "TAGGED", "TRUE", "TUPLE", "VARIANT" - }, - [](const TString& token){ - TStringBuilder req; - req << "SELECT * FROM Plato.Input WITH " << token; - return req; - } - ); - UNIT_ASSERT_VALUES_EQUAL(failed, TVector<TString>{}); - } - - Y_UNIT_TEST(TokensAsWindow) { //id_window - auto failed = ValidateTokens({ - "AUTOMAP", "CALLABLE", "DICT", "ENUM", "FALSE", "FLOW", "GROUPS", "LIST", "OPTIONAL", - "RANGE", "REPEATABLE", "RESOURCE", "ROWS", "SET", "STRUCT", "TAGGED" ,"TRUE", "TUPLE", "VARIANT" - }, - [](const TString& token){ - TStringBuilder req; - req << "SELECT * FROM Plato.Input WINDOW " << token << " AS ()"; - return req; - } - ); - UNIT_ASSERT_VALUES_EQUAL(failed, TVector<TString>{}); - } - - Y_UNIT_TEST(TokensAsIdExprIn) { //id_expr_in - auto failed = ValidateTokens({ - "ALL", "ANY", "AS", "ASSUME", "ASYMMETRIC", "AUTOMAP", "BETWEEN", "BITCAST", - "CALLABLE", "CASE", "CAST", "COMPACT", "CUBE", "CURRENT_DATE", "CURRENT_TIME", "CURRENT_TIMESTAMP", - "DICT", "DISTINCT", "ENUM", "ERASE", "EXCEPT", "EXISTS", "FLOW", "FROM", "FULL", - "HAVING", "HOP", "INTERSECT", "JSON_EXISTS", "JSON_QUERY", "JSON_VALUE", "LIMIT", "LIST", - "NOT", "OPTIONAL", "PROCESS", "REDUCE", "REPEATABLE", "RESOURCE", "RETURN", "RETURNING", "ROLLUP", - "SELECT", "SET", "STREAM", "STRUCT", "SYMMETRIC", "TAGGED", "TUPLE", "UNBOUNDED", - "UNION", "VARIANT", "WHEN", "WHERE", "WINDOW", "WITHOUT" - }, - [](const TString& token){ - TStringBuilder req; - req << "SELECT * FROM Plato.Input WHERE q IN " << token; - return req; - } - ); - UNIT_ASSERT_VALUES_EQUAL(failed, TVector<TString>{}); - } - - Y_UNIT_TEST(TableHints) { - UNIT_ASSERT(SqlToYql("SELECT * FROM plato.Input WITH INFER_SCHEMA").IsOk()); - UNIT_ASSERT(SqlToYql("SELECT * FROM plato.Input WITH (INFER_SCHEMA)").IsOk()); - } - - Y_UNIT_TEST(InNoHints) { - TString query = "SELECT * FROM plato.Input WHERE key IN (1,2,3)"; - - VerifySqlInHints(query, { "'('('warnNoAnsi))" }, {}); - VerifySqlInHints(query, { "'()" }, false); - VerifySqlInHints(query, { "'('('ansi))" }, true); - } - - Y_UNIT_TEST(InHintCompact) { - // should parse COMPACT as hint - TString query = "SELECT * FROM plato.Input WHERE key IN COMPACT(1, 2, 3)"; - - VerifySqlInHints(query, { "'('isCompact)" }); - } - - Y_UNIT_TEST(InHintSubquery) { - // should parse tableSource as hint - TString query = "$subq = (SELECT key FROM plato.Input); SELECT * FROM plato.Input WHERE key IN $subq"; - - VerifySqlInHints(query, { "'('tableSource)" }); - } - - Y_UNIT_TEST(InHintCompactSubquery) { - TString query = "$subq = (SELECT key FROM plato.Input); SELECT * FROM plato.Input WHERE key IN COMPACT $subq"; - - VerifySqlInHints(query, { "'('isCompact)", "'('tableSource)" }); - } - - Y_UNIT_TEST(CompactKeywordNotReservedForNames) { - UNIT_ASSERT(SqlToYql("SELECT COMPACT FROM plato.Input WHERE COMPACT IN COMPACT(1, 2, 3)").IsOk()); - UNIT_ASSERT(SqlToYql("USE plato; SELECT * FROM COMPACT").IsOk()); - } - - Y_UNIT_TEST(FamilyKeywordNotReservedForNames) { - // FIXME: check if we can get old behaviour - //UNIT_ASSERT(SqlToYql("USE plato; CREATE TABLE FAMILY (FAMILY Uint32, PRIMARY KEY (FAMILY));").IsOk()); - //UNIT_ASSERT(SqlToYql("USE plato; SELECT FAMILY FROM FAMILY").IsOk()); - UNIT_ASSERT(SqlToYql("USE plato; SELECT FAMILY FROM Input").IsOk()); - } - - Y_UNIT_TEST(ResetKeywordNotReservedForNames) { - UNIT_ASSERT(SqlToYql("USE plato; CREATE TABLE RESET (RESET Uint32, PRIMARY KEY (RESET));").IsOk()); - UNIT_ASSERT(SqlToYql("USE plato; SELECT RESET FROM RESET").IsOk()); - } - - Y_UNIT_TEST(SyncKeywordNotReservedForNames) { - UNIT_ASSERT(SqlToYql("USE plato; CREATE TABLE SYNC (SYNC Uint32, PRIMARY KEY (SYNC));").IsOk()); - UNIT_ASSERT(SqlToYql("USE plato; SELECT SYNC FROM SYNC").IsOk()); - } - - Y_UNIT_TEST(AsyncKeywordNotReservedForNames) { - UNIT_ASSERT(SqlToYql("USE plato; CREATE TABLE ASYNC (ASYNC Uint32, PRIMARY KEY (ASYNC));").IsOk()); - UNIT_ASSERT(SqlToYql("USE plato; SELECT ASYNC FROM ASYNC").IsOk()); - } - - Y_UNIT_TEST(DisableKeywordNotReservedForNames) { - UNIT_ASSERT(SqlToYql("USE plato; CREATE TABLE DISABLE (DISABLE Uint32, PRIMARY KEY (DISABLE));").IsOk()); - UNIT_ASSERT(SqlToYql("USE plato; SELECT DISABLE FROM DISABLE").IsOk()); - } - - Y_UNIT_TEST(ChangefeedKeywordNotReservedForNames) { - UNIT_ASSERT(SqlToYql("USE plato; CREATE TABLE CHANGEFEED (CHANGEFEED Uint32, PRIMARY KEY (CHANGEFEED));").IsOk()); - UNIT_ASSERT(SqlToYql("USE plato; SELECT CHANGEFEED FROM CHANGEFEED").IsOk()); - } - - Y_UNIT_TEST(ReplicationKeywordNotReservedForNames) { - UNIT_ASSERT(SqlToYql("USE plato; CREATE TABLE REPLICATION (REPLICATION Uint32, PRIMARY KEY (REPLICATION));").IsOk()); - UNIT_ASSERT(SqlToYql("USE plato; SELECT REPLICATION FROM REPLICATION").IsOk()); - } - - Y_UNIT_TEST(TransferKeywordNotReservedForNames) { - UNIT_ASSERT(SqlToYql("USE plato; CREATE TABLE TRANSFER (TRANSFER Uint32, PRIMARY KEY (TRANSFER));").IsOk()); - UNIT_ASSERT(SqlToYql("USE plato; SELECT TRANSFER FROM TRANSFER").IsOk()); - } - - Y_UNIT_TEST(SecondsKeywordNotReservedForNames) { - UNIT_ASSERT(SqlToYql("USE plato; CREATE TABLE SECONDS (SECONDS Uint32, PRIMARY KEY (SECONDS));").IsOk()); - UNIT_ASSERT(SqlToYql("USE plato; SELECT SECONDS FROM SECONDS").IsOk()); - } - - Y_UNIT_TEST(MillisecondsKeywordNotReservedForNames) { - UNIT_ASSERT(SqlToYql("USE plato; CREATE TABLE MILLISECONDS (MILLISECONDS Uint32, PRIMARY KEY (MILLISECONDS));").IsOk()); - UNIT_ASSERT(SqlToYql("USE plato; SELECT MILLISECONDS FROM MILLISECONDS").IsOk()); - } - - Y_UNIT_TEST(MicrosecondsKeywordNotReservedForNames) { - UNIT_ASSERT(SqlToYql("USE plato; CREATE TABLE MICROSECONDS (MICROSECONDS Uint32, PRIMARY KEY (MICROSECONDS));").IsOk()); - UNIT_ASSERT(SqlToYql("USE plato; SELECT MICROSECONDS FROM MICROSECONDS").IsOk()); - } - - Y_UNIT_TEST(NanosecondsKeywordNotReservedForNames) { - UNIT_ASSERT(SqlToYql("USE plato; CREATE TABLE NANOSECONDS (NANOSECONDS Uint32, PRIMARY KEY (NANOSECONDS));").IsOk()); - UNIT_ASSERT(SqlToYql("USE plato; SELECT NANOSECONDS FROM NANOSECONDS").IsOk()); - } - - Y_UNIT_TEST(Jubilee) { - NYql::TAstParseResult res = SqlToYql("USE plato; INSERT INTO Arcadia (r2000000) VALUES (\"2M GET!!!\");"); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(QualifiedAsteriskBefore) { - NYql::TAstParseResult res = SqlToYql( - "PRAGMA DisableSimpleColumns;" - "select interested_table.*, LENGTH(value) AS megahelpful_len from plato.Input as interested_table;" - ); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - static bool seenStar = false; - if (word == "FlattenMembers") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("interested_table.")); - } else if (word == "SqlProjectItem") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(Quote("megahelpful_len"))); - UNIT_ASSERT_VALUES_EQUAL(seenStar, true); - } else if (word == "SqlProjectStarItem") { - seenStar = true; - } - }; - TWordCountHive elementStat = {{TString("FlattenMembers"), 0}, {TString("SqlProjectItem"), 0}, {TString("SqlProjectStarItem"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["FlattenMembers"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SqlProjectItem"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SqlProjectStarItem"]); - } - - Y_UNIT_TEST(QualifiedAsteriskAfter) { - NYql::TAstParseResult res = SqlToYql( - "PRAGMA DisableSimpleColumns;" - "select LENGTH(value) AS megahelpful_len, interested_table.* from plato.Input as interested_table;" - ); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - static bool seenStar = false; - if (word == "FlattenMembers") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("interested_table.")); - } else if (word == "SqlProjectItem") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(Quote("megahelpful_len"))); - UNIT_ASSERT_VALUES_EQUAL(seenStar, false); - } else if (word == "SqlProjectStarItem") { - seenStar = true; - } - }; - TWordCountHive elementStat = {{TString("FlattenMembers"), 0}, {TString("SqlProjectItem"), 0}, {TString("SqlProjectStarItem"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["FlattenMembers"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SqlProjectItem"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SqlProjectStarItem"]); - } - - Y_UNIT_TEST(QualifiedMembers) { - NYql::TAstParseResult res = SqlToYql("select interested_table.key, interested_table.value from plato.Input as interested_table;"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - const bool fieldKey = TString::npos != line.find(Quote("key")); - const bool fieldValue = TString::npos != line.find(Quote("value")); - const bool refOnTable = TString::npos != line.find("interested_table."); - if (word == "SqlProjectItem") { - UNIT_ASSERT(fieldKey || fieldValue); - UNIT_ASSERT(!refOnTable); - } else if (word == "Write!") { - UNIT_ASSERT(fieldKey && fieldValue && !refOnTable); - } - }; - TWordCountHive elementStat = {{TString("SqlProjectStarItem"), 0}, {TString("SqlProjectItem"), 0}, {TString("Write!"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - UNIT_ASSERT_VALUES_EQUAL(0, elementStat["SqlProjectStarItem"]); - UNIT_ASSERT_VALUES_EQUAL(2, elementStat["SqlProjectItem"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write!"]); - } - - Y_UNIT_TEST(ExplainQueryPlan) { - UNIT_ASSERT(SqlToYql("EXPLAIN SELECT 1;").IsOk()); - UNIT_ASSERT(SqlToYql("EXPLAIN QUERY PLAN SELECT 1;").IsOk()); - } - - Y_UNIT_TEST(JoinParseCorrect) { - NYql::TAstParseResult res = SqlToYql( - "PRAGMA DisableSimpleColumns;" - " SELECT table_bb.*, table_aa.key as megakey" - " FROM plato.Input AS table_aa" - " JOIN plato.Input AS table_bb" - " ON table_aa.value == table_bb.value;" - ); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "SelectMembers") { - UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("table_aa.")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("table_bb.")); - } else if (word == "SqlProjectItem") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(Quote("megakey"))); - } else if (word == "SqlColumn") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(Quote("table_aa"))); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(Quote("key"))); - } - }; - TWordCountHive elementStat = {{TString("SqlProjectItem"), 0}, {TString("SqlProjectStarItem"), 0}, {TString("SelectMembers"), 0}, {TString("SqlColumn"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SqlProjectItem"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SqlProjectStarItem"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SelectMembers"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SqlColumn"]); - } - - Y_UNIT_TEST(Join3Table) { - NYql::TAstParseResult res = SqlToYql( - " PRAGMA DisableSimpleColumns;" - " SELECT table_bb.*, table_aa.key as gigakey, table_cc.* " - " FROM plato.Input AS table_aa" - " JOIN plato.Input AS table_bb ON table_aa.key == table_bb.key" - " JOIN plato.Input AS table_cc ON table_aa.subkey == table_cc.subkey;" - ); - Err2Str(res); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "SelectMembers") { - UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("table_aa.")); - UNIT_ASSERT(line.find("table_bb.") != TString::npos || line.find("table_cc.") != TString::npos); - } else if (word == "SqlProjectItem") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(Quote("gigakey"))); - } else if (word == "SqlColumn") { - const auto posTableAA = line.find(Quote("table_aa")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, posTableAA); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(Quote("key"))); - UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("table_aa", posTableAA + 3)); - } - }; - TWordCountHive elementStat = {{TString("SqlProjectItem"), 0}, {TString("SqlProjectStarItem"), 0}, {TString("SelectMembers"), 0}, {TString("SqlColumn"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SqlProjectItem"]); - UNIT_ASSERT_VALUES_EQUAL(2, elementStat["SqlProjectStarItem"]); - UNIT_ASSERT_VALUES_EQUAL(2, elementStat["SelectMembers"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SqlColumn"]); - } - - Y_UNIT_TEST(DisabledJoinCartesianProduct) { - NYql::TAstParseResult res = SqlToYql("pragma DisableAnsiImplicitCrossJoin; use plato; select * from A,B,C"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_STRINGS_EQUAL(res.Issues.ToString(), "<main>:1:67: Error: Cartesian product of tables is disabled. Please use explicit CROSS JOIN or enable it via PRAGMA AnsiImplicitCrossJoin\n"); - } - - Y_UNIT_TEST(JoinCartesianProduct) { - NYql::TAstParseResult res = SqlToYql("pragma AnsiImplicitCrossJoin; use plato; select * from A,B,C"); - UNIT_ASSERT(res.Root); - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "EquiJoin") { - auto pos = line.find("Cross"); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, pos); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("Cross", pos + 1)); - } - }; - TWordCountHive elementStat = {{TString("EquiJoin"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["EquiJoin"]); - } - - Y_UNIT_TEST(CreateAlterUserWithLoginNoLogin) { - auto reqCreateUser = SqlToYql(R"( - USE plato; - CREATE USER user1; - )"); - - UNIT_ASSERT(reqCreateUser.IsOk()); - - auto reqAlterUser = SqlToYql(R"( - USE plato; - ALTER USER user1; - )"); - - UNIT_ASSERT(!reqAlterUser.IsOk()); - UNIT_ASSERT_STRING_CONTAINS(reqAlterUser.Issues.ToString(), "Error: mismatched input ';' expecting {ENCRYPTED, HASH, LOGIN, NOLOGIN, PASSWORD, RENAME, WITH}"); - - auto reqPasswordAndLogin = SqlToYql(R"( - USE plato; - CREATE USER user1 LOgin; - )"); - - UNIT_ASSERT(reqPasswordAndLogin.IsOk()); - - auto reqPasswordAndNoLogin = SqlToYql(R"( - USE plato; - CREATE USER user1 PASSWORD '123' NOLOGIN; - )"); - - UNIT_ASSERT(reqPasswordAndNoLogin.IsOk()); - - auto reqLogin = SqlToYql(R"( - USE plato; - CREATE USER user1 LOGIN; - )"); - - UNIT_ASSERT(reqLogin.IsOk()); - - auto reqNoLogin = SqlToYql(R"( - USE plato; - CREATE USER user1 NOLOGIN; - )"); - - UNIT_ASSERT(reqNoLogin.IsOk()); - - auto reqLoginNoLogin = SqlToYql(R"( - USE plato; - CREATE USER user1 LOGIN NOLOGIN; - )"); - - UNIT_ASSERT(!reqLoginNoLogin.IsOk()); - UNIT_ASSERT_STRING_CONTAINS(reqLoginNoLogin.Issues.ToString(), "Error: Conflicting or redundant options"); - - auto reqAlterLoginNoLogin = SqlToYql(R"( - USE plato; - CREATE USER user1 LOGIN; - ALTER USER user1 NOLOGIN; - )"); - - UNIT_ASSERT(reqAlterLoginNoLogin.IsOk()); - - auto reqAlterLoginNoLoginWithPassword = SqlToYql(R"( - USE plato; - CREATE USER user1 LOGIN; - ALTER USER user1 PASSWORD '321' NOLOGIN; - )"); - - UNIT_ASSERT(reqAlterLoginNoLoginWithPassword.IsOk()); - } - - Y_UNIT_TEST(CreateUserWithHash) { - auto reqCreateUser = SqlToYql(R"( - USE plato; - CREATE USER user1 HASH '{ - "hash": "p4ffeMugohqyBwyckYCK1TjJfz3LIHbKiGL+t+oEhzw=", - "salt": "U+tzBtgo06EBQCjlARA6Jg==", - "type": "argon2id" - }'; - )"); - - UNIT_ASSERT(reqCreateUser.IsOk()); - - auto reqCreateUserWithNoLogin = SqlToYql(R"( - USE plato; - CREATE USER user1 HASH '{ - "hash": "p4ffeMugohqyBwyckYCK1TjJfz3LIHbKiGL+t+oEhzw=", - "salt": "U+tzBtgo06EBQCjlARA6Jg==", - "type": "argon2id" - }' - NOLOGIN; - )"); - - UNIT_ASSERT(reqCreateUserWithNoLogin.IsOk()); - - auto reqCreateUserWithPassword = SqlToYql(R"( - USE plato; - CREATE USER user1 HASH '{ - "hash": "p4ffeMugohqyBwyckYCK1TjJfz3LIHbKiGL+t+oEhzw=", - "salt": "U+tzBtgo06EBQCjlARA6Jg==", - "type": "argon2id" - }' - PASSWORD '123'; - )"); - - UNIT_ASSERT(!reqCreateUserWithPassword.IsOk()); - UNIT_ASSERT_STRING_CONTAINS(reqCreateUserWithPassword.Issues.ToString(), "Error: Conflicting or redundant options"); - - auto reqAlterUser = SqlToYql(R"( - USE plato; - CREATE USER user1; - ALTER USER user1 HASH '{ - "hash": "p4ffeMugohqyBwyckYCK1TjJfz3LIHbKiGL+t+oEhzw=", - "salt": "U+tzBtgo06EBQCjlARA6Jg==", - "type": "argon2id" - }'; - )"); - - UNIT_ASSERT(reqAlterUser.IsOk()); - } - - Y_UNIT_TEST(JoinWithoutConcreteColumns) { - NYql::TAstParseResult res = SqlToYql( - " use plato;" - " SELECT a.v, b.value" - " FROM `Input1` VIEW `ksv` AS a" - " JOIN `Input2` AS b" - " ON a.k == b.key;" - ); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "SqlProjectItem") { - UNIT_ASSERT(line.find(Quote("a.v")) != TString::npos || line.find(Quote("b.value")) != TString::npos); - } else if (word == "SqlColumn") { - const auto posTableA = line.find(Quote("a")); - const auto posTableB = line.find(Quote("b")); - if (posTableA != TString::npos) { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(Quote("v"))); - } else { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, posTableB); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(Quote("value"))); - } - } - }; - TWordCountHive elementStat = {{TString("SqlProjectStarItem"), 0}, {TString("SqlProjectItem"), 0}, {TString("SqlColumn"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - UNIT_ASSERT_VALUES_EQUAL(0, elementStat["SqlProjectStarItem"]); - UNIT_ASSERT_VALUES_EQUAL(2, elementStat["SqlProjectItem"]); - UNIT_ASSERT_VALUES_EQUAL(2, elementStat["SqlColumn"]); - } - - Y_UNIT_TEST(JoinWithSameValues) { - NYql::TAstParseResult res = SqlToYql("SELECT a.value, b.value FROM plato.Input AS a JOIN plato.Input as b ON a.key == b.key;"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "SqlProjectItem") { - const bool isValueFromA = TString::npos != line.find(Quote("a.value")); - const bool isValueFromB = TString::npos != line.find(Quote("b.value")); - UNIT_ASSERT(isValueFromA || isValueFromB); - } if (word == "Write!") { - UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("a.a.")); - UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("b.b.")); - } - }; - TWordCountHive elementStat = {{TString("SqlProjectStarItem"), 0}, {TString("SqlProjectItem"), 0}, {"Write!", 0}}; - VerifyProgram(res, elementStat, verifyLine); - UNIT_ASSERT_VALUES_EQUAL(0, elementStat["SqlProjectStarItem"]); - UNIT_ASSERT_VALUES_EQUAL(2, elementStat["SqlProjectItem"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write!"]); - } - - Y_UNIT_TEST(SameColumnsForDifferentTables) { - NYql::TAstParseResult res = SqlToYql("SELECT a.key, b.key FROM plato.Input as a JOIN plato.Input as b on a.key==b.key;"); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(SameColumnsForDifferentTablesFullJoin) { - NYql::TAstParseResult res = SqlToYql("SELECT a.key, b.key, a.value, b.value FROM plato.Input AS a FULL JOIN plato.Input AS b USING(key);"); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(JoinStreamLookupStrategyHint) { - { - NYql::TAstParseResult res = SqlToYql("SELECT * FROM plato.Input AS a JOIN /*+ StreamLookup() */ plato.Input AS b USING(key);"); - UNIT_ASSERT(res.Root); - } - //case insensitive - { - NYql::TAstParseResult res = SqlToYql("SELECT * FROM plato.Input AS a JOIN /*+ streamlookup() */ plato.Input AS b USING(key);"); - UNIT_ASSERT(res.Root); - } - } - - Y_UNIT_TEST(JoinConflictingStrategyHint) { - { - NYql::TAstParseResult res = SqlToYql("SELECT * FROM plato.Input AS a JOIN /*+ StreamLookup() */ /*+ Merge() */ plato.Input AS b USING(key);"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_STRINGS_EQUAL(res.Issues.ToString(), "<main>:1:91: Error: Conflicting join strategy hints\n"); - } - } - - Y_UNIT_TEST(JoinDuplicatingStrategyHint) { - { - NYql::TAstParseResult res = SqlToYql("SELECT * FROM plato.Input AS a JOIN /*+ StreamLookup() */ /*+ StreamLookup() */ plato.Input AS b USING(key);"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_STRINGS_EQUAL(res.Issues.ToString(), "<main>:1:98: Error: Duplicate join strategy hint\n"); - } - } - - Y_UNIT_TEST(WarnCrossJoinStrategyHint) { - NYql::TAstParseResult res = SqlToYql("SELECT * FROM plato.Input AS a CROSS JOIN /*+ merge() */ plato.Input AS b;"); - UNIT_ASSERT(res.Root); - UNIT_ASSERT_STRINGS_EQUAL(res.Issues.ToString(), "<main>:1:32: Warning: Non-default join strategy will not be used for CROSS JOIN, code: 4534\n"); - } - - Y_UNIT_TEST(WarnCartesianProductStrategyHint) { - NYql::TAstParseResult res = SqlToYql("pragma AnsiImplicitCrossJoin; use plato; SELECT * FROM A, /*+ merge() */ B;"); - UNIT_ASSERT(res.Root); - UNIT_ASSERT_STRINGS_EQUAL(res.Issues.ToString(), "<main>:1:74: Warning: Non-default join strategy will not be used for CROSS JOIN, code: 4534\n"); - } - - Y_UNIT_TEST(WarnUnknownJoinStrategyHint) { - NYql::TAstParseResult res = SqlToYql("SELECT * FROM plato.Input AS a JOIN /*+ xmerge() */ plato.Input AS b USING (key);"); - UNIT_ASSERT(res.Root); - UNIT_ASSERT_STRINGS_EQUAL(res.Issues.ToString(), "<main>:1:41: Warning: Unsupported join hint: xmerge, code: 4534\n"); - } - - Y_UNIT_TEST(ReverseLabels) { - NYql::TAstParseResult res = SqlToYql("select in.key as subkey, subkey as key from plato.Input as in;"); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(AutogenerationAliasWithoutCollisionConflict1) { - NYql::TAstParseResult res = SqlToYql("select LENGTH(Value), key as column1 from plato.Input;"); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(AutogenerationAliasWithoutCollision2Conflict2) { - NYql::TAstParseResult res = SqlToYql("select key as column0, LENGTH(Value) from plato.Input;"); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(InputAliasForQualifiedAsterisk) { - NYql::TAstParseResult res = SqlToYql("use plato; select zyuzya.*, key from plato.Input as zyuzya;"); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(SelectSupportsResultColumnsWithTrailingComma) { - NYql::TAstParseResult res = SqlToYql("select a, b, c, from plato.Input;"); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(SelectOrderByLabeledColumn) { - NYql::TAstParseResult res = SqlToYql("pragma DisableOrderedColumns; select key as goal from plato.Input order by goal"); - UNIT_ASSERT(res.Root); - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "DataSource") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("plato")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("Input")); - - UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("goal")); - } else if (word == "Sort") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("goal")); - - UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("key")); - } - }; - TWordCountHive elementStat = {{TString("DataSource"), 0}, {TString("Sort"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["DataSource"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Sort"]); - } - - Y_UNIT_TEST(SelectOrderBySimpleExpr) { - NYql::TAstParseResult res = SqlToYql("select a from plato.Input order by a + a"); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(SelectAssumeOrderByTableRowAccess) { - NYql::TAstParseResult res = SqlToYql("$key = 'foo';select * from plato.Input assume order by TableRow().$key"); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(SelectOrderByDuplicateLabels) { - NYql::TAstParseResult res = SqlToYql("select a from plato.Input order by a, a"); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(SelectOrderByExpression) { - NYql::TAstParseResult res = SqlToYql("select * from plato.Input as i order by cast(key as uint32) + cast(subkey as uint32)"); - UNIT_ASSERT(res.Root); - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Sort") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("\"+MayWarn\"")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("key")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("subkey")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("(Bool 'true)")); - - UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("i.key")); - UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("i.subkey")); - } - }; - TWordCountHive elementStat = {{TString("Sort"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Sort"]); - } - - Y_UNIT_TEST(SelectOrderByExpressionDesc) { - NYql::TAstParseResult res = SqlToYql("pragma disablesimplecolumns; select i.*, key, subkey from plato.Input as i order by cast(i.key as uint32) - cast(i.subkey as uint32) desc"); - UNIT_ASSERT(res.Root); - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Sort") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("\"-MayWarn\"")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("\"key\"")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("\"subkey\"")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("(Bool 'false)")); - } else if (word == "Write!") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'columns")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("\"key\"")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("\"subkey\"")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("prefix")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("\"i.\"")); - } - }; - TWordCountHive elementStat = {{TString("Sort"), 0}, {TString("Write!"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Sort"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write!"]); - } - - Y_UNIT_TEST(SelectOrderByExpressionAsc) { - NYql::TAstParseResult res = SqlToYql("select i.key, i.subkey from plato.Input as i order by cast(key as uint32) % cast(i.subkey as uint32) asc"); - UNIT_ASSERT(res.Root); - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Sort") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("\"%MayWarn\"")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("\"key\"")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("\"subkey\"")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("(Bool 'true)")); - } else if (word == "Write!") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'columns")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("\"key\"")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("\"subkey\"")); - UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("i.")); - } - }; - TWordCountHive elementStat = {{TString("Sort"), 0}, {TString("Write!"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Sort"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write!"]); - } - - Y_UNIT_TEST(ReferenceToKeyInSubselect) { - NYql::TAstParseResult res = SqlToYql("select b.key from (select a.key from plato.Input as a) as b;"); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(OrderByCastValue) { - NYql::TAstParseResult res = SqlToYql("select i.key, i.subkey from plato.Input as i order by cast(key as uint32) desc;"); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(GroupByCastValue) { - NYql::TAstParseResult res = SqlToYql("select count(1) from plato.Input as i group by cast(key as uint8);"); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(KeywordInSelectColumns) { - NYql::TAstParseResult res = SqlToYql("select in, s.check from (select 1 as in, \"test\" as check) as s;"); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(SelectAllGroupBy) { - NYql::TAstParseResult res = SqlToYql("select * from plato.Input group by subkey;"); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(CreateObjectWithFeatures) { - NYql::TAstParseResult res = SqlToYql("USE plato; CREATE OBJECT secretId (TYPE SECRET) WITH (Key1=Value1, K2=V2);"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'('\"K2\" '\"V2\") '('\"Key1\" '\"Value1\")")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("createObject")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0}, {TString("SECRET"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SECRET"]); - } - - Y_UNIT_TEST(CreateObjectIfNotExists) { - NYql::TAstParseResult res = SqlToYql("USE plato; CREATE OBJECT IF NOT EXISTS secretId (TYPE SECRET) WITH (Key1=Value1, K2=V2);"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("createObjectIfNotExists")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0}, {TString("SECRET"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SECRET"]); - } - - Y_UNIT_TEST(CreateObjectWithFeaturesStrings) { - NYql::TAstParseResult res = SqlToYql("USE plato; CREATE OBJECT secretId (TYPE SECRET) WITH (Key1=\"Value1\", K2='V2', K3=V3, K4='', K5=`aaa`, K6='a\\'aa');"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'('\"K2\" '\"V2\") '('\"K3\" '\"V3\") '('\"K4\" '\"\") '('\"K5\" '\"aaa\") '('\"K6\" '\"a'aa\") '('\"Key1\" '\"Value1\")")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("createObject")); - } - }; - - TWordCountHive elementStat = {{TString("Write"), 0}, {TString("SECRET"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - } - - Y_UNIT_TEST(UpsertObjectWithFeatures) { - NYql::TAstParseResult res = SqlToYql("USE plato; UPSERT OBJECT secretId (TYPE SECRET) WITH (Key1=Value1, K2=V2);"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'('\"K2\" '\"V2\") '('\"Key1\" '\"Value1\")")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("upsertObject")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0}, {TString("SECRET"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SECRET"]); - } - - Y_UNIT_TEST(CreateObjectWithFeaturesAndFlags) { - NYql::TAstParseResult res = SqlToYql("USE plato; CREATE OBJECT secretId (TYPE SECRET) WITH (Key1=Value1, K2=V2, RECURSE);"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'('\"Key1\" '\"Value1\") '('\"RECURSE\")")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("createObject")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0}, {TString("SECRET"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SECRET"]); - } - - Y_UNIT_TEST(Select1Type) { - NYql::TAstParseResult res = SqlToYql("SELECT 1 type;"); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(SelectTableType) { - NYql::TAstParseResult res = SqlToYql("USE plato; SELECT * from T type;"); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(CreateObjectNoFeatures) { - NYql::TAstParseResult res = SqlToYql("USE plato; CREATE OBJECT secretId (TYPE SECRET);"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("'features")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("createObject")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0}, {TString("SECRET"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SECRET"]); - } - - Y_UNIT_TEST(AlterObjectWithFeatures) { - NYql::TAstParseResult res = SqlToYql( - "USE plato;\n" - "declare $path as String;\n" - "ALTER OBJECT secretId (TYPE SECRET) SET (Key1=$path, K2=V2);" - ); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'features")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'\"Key1\" (EvaluateAtom \"$path\"")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'\"K2\" '\"V2\"")); - - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("alterObject")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0}, {TString("SECRET"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SECRET"]); - } - - Y_UNIT_TEST(AlterObjectNoFeatures) { - NYql::TAstParseResult res = SqlToYql("USE plato; ALTER OBJECT secretId (TYPE SECRET);"); - UNIT_ASSERT(!res.Root); - } - - Y_UNIT_TEST(DropObjectNoFeatures) { - NYql::TAstParseResult res = SqlToYql("USE plato; DROP OBJECT secretId (TYPE SECRET);"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("'features")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("dropObject")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0}, {TString("SECRET"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SECRET"]); - } - - Y_UNIT_TEST(DropObjectWithFeatures) { - NYql::TAstParseResult res = SqlToYql("USE plato; DROP OBJECT secretId (TYPE SECRET) WITH (A, B, C);"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'features")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("dropObject")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0}, {TString("SECRET"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SECRET"]); - } - - Y_UNIT_TEST(DropObjectWithOneOption) { - NYql::TAstParseResult res = SqlToYql("USE plato; DROP OBJECT secretId (TYPE SECRET) WITH OVERRIDE;"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'features")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'\"OVERRIDE\"")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("dropObject")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0}, {TString("SECRET"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SECRET"]); - } - - Y_UNIT_TEST(DropObjectIfExists) { - NYql::TAstParseResult res = SqlToYql("USE plato; DROP OBJECT IF EXISTS secretId (TYPE SECRET);"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("dropObjectIfExists")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0}, {TString("SECRET"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SECRET"]); - } - - Y_UNIT_TEST(PrimaryKeyParseCorrect) { - NYql::TAstParseResult res = SqlToYql("USE plato; CREATE TABLE tableName (Key Uint32, Subkey Int64, Value String, PRIMARY KEY (Key, Subkey));"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("\"Key\"")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("\"Subkey\"")); - } - }; - - TWordCountHive elementStat = {{TString("Write"), 0}, {TString("primarykey"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["primarykey"]); - } - - Y_UNIT_TEST(AlterDatabaseAst) { - NYql::TAstParseResult request = SqlToYql("USE plato; ALTER DATABASE `/Root/test` OWNER TO user1;"); - UNIT_ASSERT(request.IsOk()); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - Y_UNUSED(word); - - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find( - R"(let world (Write! world sink (Key '('databasePath (String '"/Root/test"))) (Void) '('('mode 'alterDatabase) '('owner '"user1"))))" - )); - }; - - TWordCountHive elementStat({TString("\'mode \'alterDatabase")}); - VerifyProgram(request, elementStat, verifyLine); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["\'mode \'alterDatabase"]); - } - - Y_UNIT_TEST(CreateTableNonNullableYqlTypeAstCorrect) { - NYql::TAstParseResult res = SqlToYql("USE plato; CREATE TABLE t (a int32 not null);"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write!") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, - line.find(R"__((Write! world sink (Key '('tablescheme (String '"t"))) values '('('mode 'create) '('columns '('('"a" (DataType 'Int32) '('columnConstrains '('('not_null))) '())))))))__")); - } - }; - - TWordCountHive elementStat = {{TString("Write!"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write!"]); - } - - Y_UNIT_TEST(CreateTableNullableYqlTypeAstCorrect) { - NYql::TAstParseResult res = SqlToYql("USE plato; CREATE TABLE t (a int32);"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write!") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, - line.find(R"__((Write! world sink (Key '('tablescheme (String '"t"))) values '('('mode 'create) '('columns '('('"a" (AsOptionalType (DataType 'Int32)) '('columnConstrains '()) '()))))))__")); - } - }; - - TWordCountHive elementStat = {{TString("Write!"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write!"]); - } - - Y_UNIT_TEST(CreateTableNonNullablePgTypeAstCorrect) { - NYql::TAstParseResult res = SqlToYql("USE plato; CREATE TABLE t (a pg_int4 not null);"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write!") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, - line.find(R"__((Write! world sink (Key '('tablescheme (String '"t"))) values '('('mode 'create) '('columns '('('"a" (PgType '_int4) '('columnConstrains '('('not_null))) '())))))))__")); - } - }; - - TWordCountHive elementStat = {{TString("Write!"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write!"]); - } - - Y_UNIT_TEST(CreateTableNullablePgTypeAstCorrect) { - NYql::TAstParseResult res = SqlToYql("USE plato; CREATE TABLE t (a pg_int4);"); - UNIT_ASSERT(res.Root); - - res.Root->PrettyPrintTo(Cout, PRETTY_FLAGS); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write!") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, - line.find(R"__((Write! world sink (Key '('tablescheme (String '"t"))) values '('('mode 'create) '('columns '('('"a" (AsOptionalType (PgType '_int4)) '('columnConstrains '()) '()))))))__")); - } - }; - - TWordCountHive elementStat = {{TString("Write!"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write!"]); - } - - Y_UNIT_TEST(CreateTableNullPkColumnsAreAllowed) { - NYql::TAstParseResult res = SqlToYql("USE plato; CREATE TABLE t (a int32, primary key(a));"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write!") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, - line.find(R"__((Write! world sink (Key '('tablescheme (String '"t"))) values '('('mode 'create) '('columns '('('"a" (AsOptionalType (DataType 'Int32)) '('columnConstrains '()) '()))) '('primarykey '('"a")))))__")); - } - }; - - TWordCountHive elementStat = {{TString("Write!"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write!"]); - } - - Y_UNIT_TEST(CreateTableNotNullPkColumnsAreIdempotentAstCorrect) { - NYql::TAstParseResult res = SqlToYql("USE plato; CREATE TABLE t (a int32 not null, primary key(a));"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write!") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, - line.find(R"__((Write! world sink (Key '('tablescheme (String '"t"))) values '('('mode 'create) '('columns '('('"a" (DataType 'Int32) '('columnConstrains '('('not_null))) '()))) '('primarykey '('"a"))))))__")); - } - }; - - TWordCountHive elementStat = {{TString("Write!"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write!"]); - } - - Y_UNIT_TEST(CreateTableWithIfNotExists) { - NYql::TAstParseResult res = SqlToYql("USE plato; CREATE TABLE IF NOT EXISTS t (a int32, primary key(a));"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write!") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, - line.find(R"__((Write! world sink (Key '('tablescheme (String '"t"))) values '('('mode 'create_if_not_exists) '('columns '('('"a" (AsOptionalType (DataType 'Int32)) '('columnConstrains '()) '()))) '('primarykey '('"a")))))__")); - } - }; - - TWordCountHive elementStat = {{TString("Write!"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write!"]); - } - - Y_UNIT_TEST(CreateTempTable) { - NYql::TAstParseResult res = SqlToYql("USE plato; CREATE TEMP TABLE t (a int32, primary key(a));"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write!") { - UNIT_ASSERT_VALUES_UNEQUAL_C(TString::npos, - line.find(R"__((Write! world sink (Key '('tablescheme (String '"t"))) values '('('mode 'create) '('columns '('('"a" (AsOptionalType (DataType 'Int32)) '('columnConstrains '()) '()))) '('primarykey '('"a")) '('temporary))))__"), line); - } - }; - - TWordCountHive elementStat = {{TString("Write!"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write!"]); - } - - Y_UNIT_TEST(CreateTemporaryTable) { - NYql::TAstParseResult res = SqlToYql("USE plato; CREATE TEMPORARY TABLE t (a int32, primary key(a));"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write!") { - UNIT_ASSERT_VALUES_UNEQUAL_C(TString::npos, - line.find(R"__((Write! world sink (Key '('tablescheme (String '"t"))) values '('('mode 'create) '('columns '('('"a" (AsOptionalType (DataType 'Int32)) '('columnConstrains '()) '()))) '('primarykey '('"a")) '('temporary))))__"), line); - } - }; - - TWordCountHive elementStat = {{TString("Write!"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write!"]); - } - - Y_UNIT_TEST(CreateTableWithoutTypes) { - NYql::TAstParseResult res = SqlToYql("USE plato; CREATE TABLE t (a, primary key(a));"); - UNIT_ASSERT(!res.Root); - } - - Y_UNIT_TEST(CreateTableAsSelectWithTypes) { - NYql::TAstParseResult res = SqlToYql("USE plato; CREATE TABLE t (a int32, primary key(a)) AS SELECT * FROM ts;"); - UNIT_ASSERT(!res.Root); - } - - Y_UNIT_TEST(CreateTableAsSelect) { - NYql::TAstParseResult res = SqlToYql("USE plato; CREATE TABLE t (a, b, primary key(a)) AS SELECT * FROM ts;"); - UNIT_ASSERT_C(res.Root, res.Issues.ToString()); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write!") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, - line.find(R"__((let world (Write! world sink (Key '('tablescheme (String '"t"))) values '('('mode 'create) '('columns '('('"a") '('"b"))) '('primarykey '('"a"))))))__")); - } - if (word == "Read!") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, - line.find(R"__((Read! world (DataSource '"yt" '"plato") (MrTableConcat (Key '('table (String '"ts")))))__")); - } - }; - - TWordCountHive elementStat = {{TString("Write!"), 0}, {TString("Read!"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write!"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Read!"]); - } - - Y_UNIT_TEST(CreateTableAsSelectOnlyPrimary) { - NYql::TAstParseResult res = SqlToYql("USE plato; CREATE TABLE t (primary key(a)) AS SELECT * FROM ts;"); - UNIT_ASSERT_C(res.Root, res.Issues.ToString()); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write!") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, - line.find(R"__((let world (Write! world sink (Key '('tablescheme (String '"t"))) values '('('mode 'create) '('columns '()) '('primarykey '('"a"))))))__")); - } - if (word == "Read!") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, - line.find(R"__((Read! world (DataSource '"yt" '"plato") (MrTableConcat (Key '('table (String '"ts")))))__")); - } - }; - - TWordCountHive elementStat = {{TString("Write!"), 0}, {TString("Read!"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write!"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Read!"]); - } - - Y_UNIT_TEST(CreateTableAsValuesFail) { - NYql::TAstParseResult res = SqlToYql("USE plato; CREATE TABLE t (a, primary key(a)) AS VALUES (1), (2);"); - UNIT_ASSERT(!res.Root); - } - - Y_UNIT_TEST(CreateTableDuplicatedPkColumnsFail) { - NYql::TAstParseResult res = SqlToYql("USE plato; CREATE TABLE t (a int32 not null, primary key(a, a));"); - UNIT_ASSERT(!res.Root); - } - - Y_UNIT_TEST(DeleteFromTableByKey) { - NYql::TAstParseResult res = SqlToYql("delete from plato.Input where key = 200;", 10, "kikimr"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("('mode 'delete)")); - } - }; - - TWordCountHive elementStat = {{TString("Write"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(DeleteFromTable) { - NYql::TAstParseResult res = SqlToYql("delete from plato.Input;", 10, "kikimr"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("('mode 'delete)")); - } - }; - - TWordCountHive elementStat = {{TString("Write"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(DeleteFromTableBatch) { - NYql::TAstParseResult res = SqlToYql("batch delete from plato.Input;", 10, "kikimr"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("('mode 'delete)")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'('is_batch 'true)")); - } - }; - - TWordCountHive elementStat = {{TString("Write"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(DeleteFromTableOnValues) { - NYql::TAstParseResult res = SqlToYql("delete from plato.Input on (key) values (1);", - 10, "kikimr"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("('mode 'delete_on)")); - } - }; - - TWordCountHive elementStat = {{TString("Write"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(DeleteFromTableOnSelect) { - NYql::TAstParseResult res = SqlToYql( - "delete from plato.Input on select key from plato.Input where value > 0;", 10, "kikimr"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("('mode 'delete_on)")); - } - }; - - TWordCountHive elementStat = {{TString("Write"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(DeleteFromTableOnBatch) { - NYql::TAstParseResult res = SqlToYql("batch delete from plato.Input on (key) values (1);", - 10, "kikimr"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:6: Error: BATCH DELETE is unsupported with ON\n"); - } - - Y_UNIT_TEST(UpdateByValues) { - NYql::TAstParseResult res = SqlToYql("update plato.Input set key = 777, value = 'cool' where key = 200;", 10, "kikimr"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("('mode 'update)")); - } else if (word == "AsStruct") { - const bool isKey = line.find("key") != TString::npos; - const bool isValue = line.find("value") != TString::npos; - UNIT_ASSERT(isKey || isValue); - if (isKey) { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(Quote("777"))); - } else if (isValue) { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(Quote("cool"))); - } - } - }; - - TWordCountHive elementStat = {{TString("Write"), 0}, {TString("AsStruct"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["AsStruct"]); - } - - Y_UNIT_TEST(UpdateByValuesBatch) { - NYql::TAstParseResult res = SqlToYql("batch update plato.Input set key = 777, value = 'cool' where key = 200;", 10, "kikimr"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("('mode 'update)")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'('is_batch 'true)")); - } - }; - - TWordCountHive elementStat = {{TString("Write"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(UpdateByMultiValues) { - NYql::TAstParseResult res = SqlToYql("update plato.Input set (key, value, subkey) = ('2','ddd',':') where key = 200;", 10, "kikimr"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("('mode 'update)")); - } else if (word == "AsStruct") { - const bool isKey = line.find("key") != TString::npos; - const bool isSubkey = line.find("subkey") != TString::npos; - const bool isValue = line.find("value") != TString::npos; - UNIT_ASSERT(isKey || isSubkey || isValue); - if (isKey && !isSubkey) { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(Quote("2"))); - } else if (isSubkey) { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(Quote(":"))); - } else if (isValue) { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(Quote("ddd"))); - } - } - }; - - TWordCountHive elementStat = {{TString("Write"), 0}, {TString("AsStruct"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["AsStruct"]); - } - - Y_UNIT_TEST(UpdateBySelect) { - NYql::TAstParseResult res = SqlToYql("update plato.Input set (key, value, subkey) = (select key, value, subkey from plato.Input where key = 911) where key = 200;", 10, "kikimr"); - UNIT_ASSERT(res.Root); - - int lineIndex = 0; - int writeLineIndex = -1; - bool found = false; - - TVerifyLineFunc verifyLine = [&lineIndex, &writeLineIndex, &found](const TString& word, const TString& line) { - if (word == "Write") { - writeLineIndex = lineIndex; - found = line.find("('mode 'update)") != TString::npos; - } else if (word == "mode") { - found |= lineIndex == writeLineIndex + 1 && line.find("('mode 'update)") != TString::npos; - UNIT_ASSERT(found); - } - - ++lineIndex; - }; - - TWordCountHive elementStat = {{TString("Write"), 0}, {TString("mode"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(UpdateSelfModifyAll) { - NYql::TAstParseResult res = SqlToYql("update plato.Input set subkey = subkey + 's';", 10, "kikimr"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("('mode 'update)")); - } else if (word == "AsStruct") { - const bool isSubkey = line.find("subkey") != TString::npos; - UNIT_ASSERT(isSubkey); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(Quote("subkey"))); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(Quote("s"))); - } - }; - - TWordCountHive elementStat = {{TString("Write"), 0}, {TString("AsStruct"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["AsStruct"]); - } - - Y_UNIT_TEST(UpdateOnValues) { - NYql::TAstParseResult res = SqlToYql("update plato.Input on (key, value) values (5, 'cool')", 10, "kikimr"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("('mode 'update_on)")); - } - }; - - TWordCountHive elementStat = {{TString("Write"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(UpdateOnSelect) { - NYql::TAstParseResult res = SqlToYql( - "update plato.Input on select key, value + 1 as value from plato.Input", 10, "kikimr"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("('mode 'update_on)")); - } - }; - - TWordCountHive elementStat = {{TString("Write"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(UpdateOnBatch) { - NYql::TAstParseResult res = SqlToYql("batch update plato.Input on (key, value) values (5, 'cool')", 10, "kikimr"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:6: Error: BATCH UPDATE is unsupported with ON\n"); - } - - Y_UNIT_TEST(UnionAllTest) { - NYql::TAstParseResult res = SqlToYql("PRAGMA DisableEmitUnionMerge; SELECT key FROM plato.Input UNION ALL select subkey FROM plato.Input;"); - UNIT_ASSERT(res.Root); - - TWordCountHive elementStat = {{TString("UnionAll"), 0}}; - VerifyProgram(res, elementStat, {}); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["UnionAll"]); - } - - Y_UNIT_TEST(UnionAllMergeTest) { - NYql::TAstParseResult res = SqlToYql("PRAGMA EmitUnionMerge; SELECT key FROM plato.Input UNION ALL select subkey FROM plato.Input;"); - UNIT_ASSERT(res.Root); - - TWordCountHive elementStat = {{TString("UnionMerge"), 0}}; - VerifyProgram(res, elementStat, {}); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["UnionMerge"]); - } - - Y_UNIT_TEST(UnionTest) { - NYql::TAstParseResult res = SqlToYql("SELECT key FROM plato.Input UNION select subkey FROM plato.Input;"); - UNIT_ASSERT(res.Root); - - TWordCountHive elementStat = {{TString("Union"), 0}}; - VerifyProgram(res, elementStat, {}); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Union"]); - } - - Y_UNIT_TEST(UnionAggregationTest) { - NYql::TAstParseResult res = SqlToYql(R"( - PRAGMA DisableEmitUnionMerge; - SELECT 1 - UNION ALL - SELECT 1 UNION ALL SELECT 1 UNION ALL SELECT 1 - UNION - SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 - UNION ALL - SELECT 1 UNION ALL SELECT 1 UNION ALL SELECT 1; - )"); - UNIT_ASSERT(res.Root); - - TWordCountHive elementStat = {{TString("Union"), 0}, {TString("UnionAll"), 0}}; - VerifyProgram(res, elementStat, {}); - UNIT_ASSERT_VALUES_EQUAL(2, elementStat["UnionAll"]); - UNIT_ASSERT_VALUES_EQUAL(3, elementStat["Union"]); - } - - Y_UNIT_TEST(UnionMergeAggregationTest) { - NYql::TAstParseResult res = SqlToYql(R"( - PRAGMA EmitUnionMerge; - SELECT 1 - UNION ALL - SELECT 1 UNION ALL SELECT 1 UNION ALL SELECT 1 - UNION - SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 - UNION ALL - SELECT 1 UNION ALL SELECT 1 UNION ALL SELECT 1; - )"); - UNIT_ASSERT(res.Root); - - TWordCountHive elementStat = {{TString("Union"), 0}, {TString("UnionMerge"), 0}}; - VerifyProgram(res, elementStat, {}); - UNIT_ASSERT_VALUES_EQUAL(2, elementStat["UnionMerge"]); - UNIT_ASSERT_VALUES_EQUAL(3, elementStat["Union"]); - } - - Y_UNIT_TEST(DeclareDecimalParameter) { - NYql::TAstParseResult res = SqlToYql("declare $value as Decimal(22,9); select $value as cnt;"); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(SimpleGroupBy) { - NYql::TAstParseResult res = SqlToYql("select count(1),z from plato.Input group by key as z order by z;"); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(EmptyColumnName0) { - /// Now it's parsed well and error occur on validate step like "4:31:Empty struct member name is not allowed" in "4:31:Function: AddMember" - NYql::TAstParseResult res = SqlToYql("insert into plato.Output (``, list1) values (0, AsList(0, 1, 2));"); - /// Verify that parsed well without crash - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(KikimrRollback) { - NYql::TAstParseResult res = SqlToYql("use plato; select * from Input; rollback;", 10, "kikimr"); - UNIT_ASSERT(res.Root); - - TWordCountHive elementStat = {{TString("rollback"), 0}}; - VerifyProgram(res, elementStat); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["rollback"]); - } - - Y_UNIT_TEST(PragmaFile) { - NYql::TAstParseResult res = SqlToYql(R"(pragma file("HW", "sbr:181041334");)"); - UNIT_ASSERT(res.Root); - - TWordCountHive elementStat = {{TString(R"((let world (Configure! world (DataSource '"config") '"AddFileByUrl" '"HW" '"sbr:181041334")))"), 0}}; - VerifyProgram(res, elementStat); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat.cbegin()->second); - } - - Y_UNIT_TEST(DoNotCrashOnNamedInFilter) { - NYql::TAstParseResult res = SqlToYql("USE plato; $all = ($table_name) -> { return true; }; SELECT * FROM FILTER(Input, $all)"); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(PragmasFileAndUdfOrder) { - NYql::TAstParseResult res = SqlToYql(R"( - PRAGMA file("libvideoplayers_udf.so", "https://proxy.sandbox.yandex-team.ru/235185290"); - PRAGMA udf("libvideoplayers_udf.so"); - )"); - UNIT_ASSERT(res.Root); - - const auto programm = GetPrettyPrint(res); - const auto file = programm.find("AddFileByUrl"); - const auto udfs = programm.find("ImportUdfs"); - UNIT_ASSERT(file < udfs); - } - - Y_UNIT_TEST(ProcessUserType) { - NYql::TAstParseResult res = SqlToYql("process plato.Input using Kikimr::PushData(TableRows());", 1, TString(NYql::KikimrProviderName)); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Kikimr.PushData") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("TupleType")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("TypeOf")); - } - }; - - TWordCountHive elementStat = {{TString("Kikimr.PushData"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Kikimr.PushData"]); - } - - Y_UNIT_TEST(ProcessUserTypeAuth) { - NYql::TAstParseResult res = SqlToYql("process plato.Input using YDB::PushData(TableRows(), AsTuple('oauth', SecureParam('api:oauth')));", 1, TString(NYql::KikimrProviderName)); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "YDB.PushData") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("TupleType")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("TypeOf")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("api:oauth")); - } - }; - - TWordCountHive elementStat = {{TString("YDB.PushData"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["YDB.PushData"]); - } - - Y_UNIT_TEST(SelectStreamRtmr) { - NYql::TAstParseResult res = SqlToYql( - "USE plato; INSERT INTO Output SELECT STREAM key FROM Input;", - 10, TString(NYql::RtmrProviderName)); - UNIT_ASSERT(res.Root); - - res = SqlToYql( - "USE plato; INSERT INTO Output SELECT key FROM Input;", - 10, TString(NYql::RtmrProviderName)); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(SelectStreamRtmrJoinWithYt) { - NYql::TAstParseResult res = SqlToYql( - "USE plato; INSERT INTO Output SELECT STREAM key FROM Input LEFT JOIN hahn.ttt as t ON Input.key = t.Name;", - 10, TString(NYql::RtmrProviderName)); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(SelectStreamNonRtmr) { - NYql::TAstParseResult res = SqlToYql( - "USE plato; INSERT INTO Output SELECT STREAM key FROM Input;", - 10); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:31: Error: SELECT STREAM is unsupported for non-streaming sources\n"); - } - - Y_UNIT_TEST(GroupByHopRtmr) { - NYql::TAstParseResult res = SqlToYql(R"( - USE plato; INSERT INTO Output SELECT key, SUM(value) AS value FROM Input - GROUP BY key, HOP(subkey, "PT10S", "PT30S", "PT20S"); - )", 10, TString(NYql::RtmrProviderName)); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(GroupByHopRtmrSubquery) { - // 'use plato' intentially avoided - NYql::TAstParseResult res = SqlToYql(R"( - SELECT COUNT(*) AS value FROM (SELECT * FROM plato.Input) - GROUP BY HOP(Data, "PT10S", "PT30S", "PT20S") - )", 10, TString(NYql::RtmrProviderName)); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(GroupByHopRtmrSubqueryBinding) { - NYql::TAstParseResult res = SqlToYql(R"( - USE plato; - $q = SELECT * FROM Input; - INSERT INTO Output SELECT STREAM * FROM ( - SELECT COUNT(*) AS value FROM $q - GROUP BY HOP(Data, "PT10S", "PT30S", "PT20S") - ); - )", 10, TString(NYql::RtmrProviderName)); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(GroupByNoHopRtmr) { - NYql::TAstParseResult res = SqlToYql(R"( - USE plato; INSERT INTO Output SELECT STREAM key, SUM(value) AS value FROM Input - GROUP BY key; - )", 10, TString(NYql::RtmrProviderName)); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:22: Error: Streaming group by query must have a hopping window specification.\n"); - } - - Y_UNIT_TEST(KikimrInserts) { - NYql::TAstParseResult res = SqlToYql(R"( - USE plato; - INSERT INTO Output SELECT key, value FROM Input; - INSERT OR ABORT INTO Output SELECT key, value FROM Input; - INSERT OR IGNORE INTO Output SELECT key, value FROM Input; - INSERT OR REVERT INTO Output SELECT key, value FROM Input; - )", 10, TString(NYql::KikimrProviderName)); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(WarnMissingIsBeforeNotNull) { - NYql::TAstParseResult res = SqlToYql("select 1 NOT NULL"); - UNIT_ASSERT(res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Warning: Missing IS keyword before NOT NULL, code: 4507\n"); - } - - Y_UNIT_TEST(Subqueries) { - NYql::TAstParseResult res = SqlToYql(R"( - USE plato; - $sq1 = (SELECT * FROM plato.Input); - - $sq2 = SELECT * FROM plato.Input; - - $squ1 = ( - SELECT * FROM plato.Input - UNION ALL - SELECT * FROM plato.Input - ); - - $squ2 = - SELECT * FROM plato.Input - UNION ALL - SELECT * FROM plato.Input; - - $squ3 = ( - (SELECT * FROM plato.Input) - UNION ALL - (SELECT * FROM plato.Input) - ); - - SELECT * FROM $sq1; - SELECT * FROM $sq2; - SELECT * FROM $squ1; - SELECT * FROM $squ2; - SELECT * FROM $squ3; - )"); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(SubqueriesJoin) { - NYql::TAstParseResult res = SqlToYql(R"( - USE plato; - - $left = SELECT * FROM plato.Input1 WHERE value != "BadValue"; - $right = SELECT * FROM plato.Input2; - - SELECT * FROM $left AS l - JOIN $right AS r - ON l.key == r.key; - )"); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(AnyInBackticksAsTableName) { - NYql::TAstParseResult res = SqlToYql("use plato; select * from `any`;"); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(AnyJoinForTableAndSubQuery) { - NYql::TAstParseResult res = SqlToYql(R"( - USE plato; - - $r = SELECT * FROM plato.Input2; - - SELECT * FROM ANY plato.Input1 AS l - LEFT JOIN ANY $r AS r - USING (key); - )"); - - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "EquiJoin") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'('left 'any)")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'('right 'any)")); - } - }; - - TWordCountHive elementStat = {{TString("left"), 0}, {TString("right"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["left"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["right"]); - } - - Y_UNIT_TEST(AnyJoinForTableAndTableSource) { - NYql::TAstParseResult res = SqlToYql(R"( - USE plato; - - $r = AsList( - AsStruct("aaa" as key, "bbb" as subkey, "ccc" as value) - ); - - SELECT * FROM ANY plato.Input1 AS l - LEFT JOIN ANY AS_TABLE($r) AS r - USING (key); - )"); - - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "EquiJoin") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'('left 'any)")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'('right 'any)")); - } - }; - - TWordCountHive elementStat = {{TString("left"), 0}, {TString("right"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["left"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["right"]); - } - - Y_UNIT_TEST(AnyJoinNested) { - NYql::TAstParseResult res = SqlToYql(R"( - USE plato; - - FROM ANY Input1 as a - JOIN Input2 as b ON a.key = b.key - LEFT JOIN ANY Input3 as c ON a.key = c.key - RIGHT JOIN ANY Input4 as d ON d.key = b.key - CROSS JOIN Input5 - SELECT *; - )"); - - UNIT_ASSERT(res.Root); - - TWordCountHive elementStat = {{TString("left"), 0}, {TString("right"), 0}}; - VerifyProgram(res, elementStat); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["left"]); - UNIT_ASSERT_VALUES_EQUAL(2, elementStat["right"]); - } - - Y_UNIT_TEST(InlineAction) { - NYql::TAstParseResult res = SqlToYql( - "do begin\n" - " select 1\n" - "; end do\n"); - UNIT_ASSERT(res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), ""); - } - - Y_UNIT_TEST(FlattenByCorrelationName) { - UNIT_ASSERT(SqlToYql("select * from plato.Input as t flatten by t.x").IsOk()); - UNIT_ASSERT(SqlToYql("select * from plato.Input as t flatten by t -- same as flatten by t.t").IsOk()); - } - - Y_UNIT_TEST(DiscoveryMode) { - UNIT_ASSERT(SqlToYqlWithMode("insert into plato.Output select * from plato.Input", NSQLTranslation::ESqlMode::DISCOVERY).IsOk()); - UNIT_ASSERT(SqlToYqlWithMode("select * from plato.concat(Input1, Input2)", NSQLTranslation::ESqlMode::DISCOVERY).IsOk()); - UNIT_ASSERT(SqlToYqlWithMode("select * from plato.each(AsList(\"Input1\", \"Input2\"))", NSQLTranslation::ESqlMode::DISCOVERY).IsOk()); - } - - Y_UNIT_TEST(CubeWithAutoGeneratedLikeColumnName) { - UNIT_ASSERT(SqlToYql("select key,subkey,group from plato.Input group by cube(key,subkey,group)").IsOk()); - } - - Y_UNIT_TEST(CubeWithAutoGeneratedLikeAlias) { - UNIT_ASSERT(SqlToYql("select key,subkey,group from plato.Input group by cube(key,subkey,value as group)").IsOk()); - } - - Y_UNIT_TEST(FilterCanBeUsedAsColumnIdOrBind) { - UNIT_ASSERT(SqlToYql("select filter from plato.Input").IsOk()); - UNIT_ASSERT(SqlToYql("select 1 as filter").IsOk()); - UNIT_ASSERT(SqlToYql("$filter = 1; select $filter").IsOk()); - } - - Y_UNIT_TEST(DuplicateSemicolonsAreAllowedBetweenTopLevelStatements) { - UNIT_ASSERT(SqlToYql(";;select 1; ; select 2;/*comment*/;select 3;;--comment\n;select 4;;").IsOk()); - } - - Y_UNIT_TEST(DuplicateAndMissingTrailingSemicolonsAreAllowedBetweenActionStatements) { - TString req = - "define action $action($b,$c) as\n" - " ;;$d = $b + $c;\n" - " select $b;\n" - " select $c;;\n" - " select $d,\n" - "end define;\n" - "\n" - "do $action(1,2);"; - UNIT_ASSERT(SqlToYql(req).IsOk()); - } - - Y_UNIT_TEST(DuplicateAndMissingTrailingSemicolonsAreAllowedBetweenInlineActionStatements) { - TString req = - "do begin\n" - " ;select 1,\n" - "end do;\n" - "evaluate for $i in AsList(1,2,3) do begin\n" - " select $i;;\n" - " select $i + $i;;\n" - "end do;"; - UNIT_ASSERT(SqlToYql(req).IsOk()); - } - - Y_UNIT_TEST(DuplicateSemicolonsAreAllowedBetweenLambdaStatements) { - TString req = - "$x=1;\n" - "$foo = ($a, $b)->{\n" - " ;;$v = $a + $b;\n" - " $bar = ($c) -> {; return $c << $x};;\n" - " return $bar($v);;\n" - "};\n" - "select $foo(1,2);"; - UNIT_ASSERT(SqlToYql(req).IsOk()); - } - - Y_UNIT_TEST(StringLiteralWithEscapedBackslash) { - NYql::TAstParseResult res1 = SqlToYql(R"foo(SELECT 'a\\';)foo"); - NYql::TAstParseResult res2 = SqlToYql(R"foo(SELECT "a\\";)foo"); - UNIT_ASSERT(res1.Root); - UNIT_ASSERT(res2.Root); - - TWordCountHive elementStat = {{TString("a\\"), 0}}; - - VerifyProgram(res1, elementStat); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["a\\"]); - - VerifyProgram(res2, elementStat); - UNIT_ASSERT_VALUES_EQUAL(2, elementStat["a\\"]); - } - - Y_UNIT_TEST(StringMultiLineLiteralWithEscapes) { - UNIT_ASSERT(SqlToYql("SELECT @@@foo@@@@bar@@@").IsOk()); - UNIT_ASSERT(SqlToYql("SELECT @@@@@@@@@").IsOk()); - } - - Y_UNIT_TEST(StringMultiLineLiteralConsequitiveAt) { - UNIT_ASSERT(!SqlToYql("SELECT @").IsOk()); - UNIT_ASSERT(!SqlToYql("SELECT @@").IsOk()); - UNIT_ASSERT(!SqlToYql("SELECT @@@").IsOk()); - UNIT_ASSERT( SqlToYql("SELECT @@@@").IsOk()); - UNIT_ASSERT( SqlToYql("SELECT @@@@@").IsOk()); - - UNIT_ASSERT(!SqlToYql("SELECT @@@@@@").IsOk()); - UNIT_ASSERT(!SqlToYql("SELECT @@@@@@@").IsOk()); - - UNIT_ASSERT( SqlToYql("SELECT @@@@@@@@").IsOk()); - UNIT_ASSERT( SqlToYql("SELECT @@@@@@@@@").IsOk()); - UNIT_ASSERT(!SqlToYql("SELECT @@@@@@@@@@").IsOk()); - } - - Y_UNIT_TEST(ConstnessForListDictSetCreate) { - auto req = "$foo = ($x, $y) -> (\"aaaa\");\n" - "\n" - "select\n" - " $foo(sum(key), ListCreate(String)),\n" - " $foo(sum(key), DictCreate(String, String)),\n" - " $foo(sum(key), SetCreate(String)),\n" - "from (select 1 as key);"; - UNIT_ASSERT(SqlToYql(req).IsOk()); - } - - Y_UNIT_TEST(CanUseEmptyTupleInWindowPartitionBy) { - auto req = "select sum(key) over w\n" - "from plato.Input\n" - "window w as (partition compact by (), (subkey), (), value || value as dvalue);"; - UNIT_ASSERT(SqlToYql(req).IsOk()); - } - - Y_UNIT_TEST(DenyAnsiOrderByLimitLegacyMode) { - auto req = "pragma DisableAnsiOrderByLimitInUnionAll;\n" - "use plato;\n" - "\n" - "select * from Input order by key limit 10\n" - "union all\n" - "select * from Input order by key limit 1;"; - - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Error: DisableAnsiOrderByLimitInUnionAll pragma is deprecated and no longer supported\n"); - } - - Y_UNIT_TEST(ReduceUsingUdfWithShortcutsWorks) { - auto req = "use plato;\n" - "\n" - "$arg = 'foo';\n" - "$func = XXX::YYY($arg);\n" - "\n" - "REDUCE Input ON key using $func(subkey);\n" - "REDUCE Input ON key using $func(UUU::VVV(TableRow()));\n"; - UNIT_ASSERT(SqlToYql(req).IsOk()); - req = "use plato;\n" - "\n" - "$arg = 'foo';\n" - "$func = XXX::YYY($arg);\n" - "\n" - "REDUCE Input ON key using all $func(subkey);\n" - "REDUCE Input ON key using all $func(UUU::VVV(TableRow()));"; - UNIT_ASSERT(SqlToYql(req).IsOk()); - } - - Y_UNIT_TEST(YsonDisableStrict) { - UNIT_ASSERT(SqlToYql("pragma yson.DisableStrict = \"false\";").IsOk()); - UNIT_ASSERT(SqlToYql("pragma yson.DisableStrict;").IsOk()); - } - - Y_UNIT_TEST(YsonStrict) { - UNIT_ASSERT(SqlToYql("pragma yson.Strict = \"false\";").IsOk()); - UNIT_ASSERT(SqlToYql("pragma yson.Strict;").IsOk()); - } - - Y_UNIT_TEST(JoinByTuple) { - auto req = "use plato;\n" - "\n" - "select * from T1 as a\n" - "join T2 as b\n" - "on AsTuple(a.key, a.subkey) = AsTuple(b.key, b.subkey);"; - UNIT_ASSERT(SqlToYql(req).IsOk()); - } - - Y_UNIT_TEST(JoinByStruct) { - auto req = "use plato;\n" - "\n" - "select * from T1 as a\n" - "join T2 as b\n" - "on AsStruct(a.key as k, a.subkey as sk) = AsStruct(b.key as k, b.subkey as sk);"; - UNIT_ASSERT(SqlToYql(req).IsOk()); - } - - Y_UNIT_TEST(JoinByUdf) { - auto req = "use plato;\n" - "\n" - "select a.align\n" - "from T1 as a\n" - "join T2 as b\n" - "on Yson::SerializeJsonEncodeUtf8(a.align)=b.align;"; - UNIT_ASSERT(SqlToYql(req).IsOk()); - } - - Y_UNIT_TEST(EscapedIdentifierAsLambdaArg) { - auto req = "$f = ($`foo bar`, $x) -> { return $`foo bar` + $x; };\n" - "\n" - "select $f(1, 2);"; - auto res = SqlToYql(req); - UNIT_ASSERT(res.Root); - const auto programm = GetPrettyPrint(res); - auto expected = "(lambda '(\"$foo bar\" \"$x\")"; - UNIT_ASSERT(programm.find(expected) != TString::npos); - } - - Y_UNIT_TEST(UdfSyntaxSugarOnlyCallable) { - auto req = "SELECT Udf(DateTime::FromString)('2022-01-01');"; - auto res = SqlToYql(req); - UNIT_ASSERT(res.Root); - const auto programm = GetPrettyPrint(res); - auto expected = "(SqlCall '\"DateTime.FromString\" '((PositionalArgs (String '\"2022-01-01\")) (AsStruct)) (TupleType (TypeOf '((String '\"2022-01-01\"))) (TypeOf (AsStruct)) (TupleType)))"; - UNIT_ASSERT(programm.find(expected) != TString::npos); - } - - Y_UNIT_TEST(UdfSyntaxSugarTypeNoRun) { - auto req = "SELECT Udf(DateTime::FromString, String, Tuple<Int32, Float>, 'foo' as TypeConfig)('2022-01-01');"; - auto res = SqlToYql(req); - UNIT_ASSERT(res.Root); - const auto programm = GetPrettyPrint(res); - auto expected = "(SqlCall '\"DateTime.FromString\" '((PositionalArgs (String '\"2022-01-01\")) (AsStruct)) (TupleType (TypeOf '((String '\"2022-01-01\"))) (TypeOf (AsStruct)) (TupleType (DataType 'String) (TupleType (DataType 'Int32) (DataType 'Float)))) '\"foo\")"; - UNIT_ASSERT(programm.find(expected) != TString::npos); - } - - Y_UNIT_TEST(UdfSyntaxSugarRunNoType) { - auto req = "SELECT Udf(DateTime::FromString, String, Tuple<Int32, Float>, Void() as RunConfig)('2022-01-01');"; - auto res = SqlToYql(req); - UNIT_ASSERT(res.Root); - const auto programm = GetPrettyPrint(res); - auto expected = "(SqlCall '\"DateTime.FromString\" '((PositionalArgs (String '\"2022-01-01\")) (AsStruct)) (TupleType (TypeOf '((String '\"2022-01-01\"))) (TypeOf (AsStruct)) (TupleType (DataType 'String) (TupleType (DataType 'Int32) (DataType 'Float)))) '\"\" (Void))"; - UNIT_ASSERT(programm.find(expected) != TString::npos); - } - - Y_UNIT_TEST(UdfSyntaxSugarFullTest) { - auto req = "SELECT Udf(DateTime::FromString, String, Tuple<Int32, Float>, 'foo' as TypeConfig, Void() As RunConfig)('2022-01-01');"; - auto res = SqlToYql(req); - UNIT_ASSERT(res.Root); - const auto programm = GetPrettyPrint(res); - auto expected = "(SqlCall '\"DateTime.FromString\" '((PositionalArgs (String '\"2022-01-01\")) (AsStruct)) (TupleType (TypeOf '((String '\"2022-01-01\"))) (TypeOf (AsStruct)) (TupleType (DataType 'String) (TupleType (DataType 'Int32) (DataType 'Float)))) '\"foo\" (Void))"; - UNIT_ASSERT(programm.find(expected) != TString::npos); - } - - Y_UNIT_TEST(UdfSyntaxSugarOtherRunConfigs) { - auto req = "SELECT Udf(DateTime::FromString, String, Tuple<Int32, Float>, 'foo' as TypeConfig, '55' As RunConfig)('2022-01-01');"; - auto res = SqlToYql(req); - UNIT_ASSERT(res.Root); - const auto programm = GetPrettyPrint(res); - auto expected = "(SqlCall '\"DateTime.FromString\" '((PositionalArgs (String '\"2022-01-01\")) (AsStruct)) (TupleType (TypeOf '((String '\"2022-01-01\"))) (TypeOf (AsStruct)) (TupleType (DataType 'String) (TupleType (DataType 'Int32) (DataType 'Float)))) '\"foo\" (String '\"55\"))"; - UNIT_ASSERT(programm.find(expected) != TString::npos); - } - - Y_UNIT_TEST(UdfSyntaxSugarOtherRunConfigs2) { - auto req = "SELECT Udf(DateTime::FromString, String, Tuple<Int32, Float>, 'foo' as TypeConfig, AsTuple(32, 'no', AsStruct(1e-9 As SomeFloat)) As RunConfig)('2022-01-01');"; - auto res = SqlToYql(req); - UNIT_ASSERT(res.Root); - const auto programm = GetPrettyPrint(res); - auto expected = "(SqlCall '\"DateTime.FromString\" '((PositionalArgs (String '\"2022-01-01\")) (AsStruct)) (TupleType (TypeOf '((String '\"2022-01-01\"))) (TypeOf (AsStruct)) (TupleType (DataType 'String) (TupleType (DataType 'Int32) (DataType 'Float)))) '\"foo\" '((Int32 '\"32\") (String '\"no\") (AsStruct '('\"SomeFloat\" (Double '\"1e-9\")))))"; - UNIT_ASSERT(programm.find(expected) != TString::npos); - } - - Y_UNIT_TEST(UdfSyntaxSugarOptional) { - auto req = "SELECT Udf(DateTime::FromString, String?, Int32??, Tuple<Int32, Float>, \"foo\" as TypeConfig, Void() As RunConfig)(\"2022-01-01\");"; - auto res = SqlToYql(req); - UNIT_ASSERT(res.Root); - const auto programm = GetPrettyPrint(res); - auto expected = "(SqlCall '\"DateTime.FromString\" '((PositionalArgs (String '\"2022-01-01\")) (AsStruct)) (TupleType (TypeOf '((String '\"2022-01-01\"))) (TypeOf (AsStruct)) (TupleType (OptionalType (DataType 'String)) (OptionalType (OptionalType (DataType 'Int32))) (TupleType (DataType 'Int32) (DataType 'Float)))) '\"foo\" (Void))"; - UNIT_ASSERT(programm.find(expected) != TString::npos); - } - - Y_UNIT_TEST(CompactionPolicyParseCorrect) { - NYql::TAstParseResult res = SqlToYql( - R"( USE plato; - CREATE TABLE tableName (Key Uint32, Value String, PRIMARY KEY (Key)) - WITH ( COMPACTION_POLICY = "SomeCompactionPreset" );)" - ); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("compactionPolicy")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("SomeCompactionPreset")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(AutoPartitioningBySizeParseCorrect) { - NYql::TAstParseResult res = SqlToYql( - R"( USE plato; - CREATE TABLE tableName (Key Uint32, Value String, PRIMARY KEY (Key)) - WITH ( AUTO_PARTITIONING_BY_SIZE = ENABLED );)" - ); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("autoPartitioningBySize")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("ENABLED")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(UniformPartitionsParseCorrect) { - NYql::TAstParseResult res = SqlToYql( - R"( USE plato; - CREATE TABLE tableName (Key Uint32, Value String, PRIMARY KEY (Key)) - WITH ( UNIFORM_PARTITIONS = 16 );)" - ); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("uniformPartitions")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("16")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(DateTimeTtlParseCorrect) { - NYql::TAstParseResult res = SqlToYql( - R"( USE plato; - CREATE TABLE tableName (Key Uint32, CreatedAt Timestamp, PRIMARY KEY (Key)) - WITH (TTL = Interval("P1D") On CreatedAt);)" - ); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("setTtlSettings")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("tiers")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("evictionDelay")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("86400000")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(IntTtlParseCorrect) { - NYql::TAstParseResult res = SqlToYql( - R"( USE plato; - CREATE TABLE tableName (Key Uint32, CreatedAt Uint32, PRIMARY KEY (Key)) - WITH (TTL = Interval("P1D") On CreatedAt AS SECONDS);)" - ); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("setTtlSettings")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("tiers")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("evictionDelay")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("86400000")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("columnUnit")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("seconds")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(TtlTieringParseCorrect) { - NYql::TAstParseResult res = SqlToYql( - R"( USE plato; - CREATE TABLE tableName (Key Uint32, CreatedAt Uint32, PRIMARY KEY (Key)) - WITH (TTL = - Interval("P1D") TO EXTERNAL DATA SOURCE Tier1, - Interval("P2D") TO EXTERNAL DATA SOURCE Tier2, - Interval("P30D") DELETE - On CreatedAt AS SECONDS);)" - ); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("setTtlSettings")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("tiers")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("evictionDelay")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("storageName")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("Tier1")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("Tier2")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("86400000")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("172800000")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("2592000000")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("columnUnit")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("seconds")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(TtlTieringWithOtherActionsParseCorrect) { - NYql::TAstParseResult res = SqlToYql( - R"( USE plato; - ALTER TABLE tableName - ADD FAMILY cold (DATA = "rot"), - SET TTL - Interval("P1D") TO EXTERNAL DATA SOURCE Tier1, - Interval("P2D") TO EXTERNAL DATA SOURCE Tier2, - Interval("P30D") DELETE - ON CreatedAt, - ALTER COLUMN payload_v2 SET FAMILY cold, - ALTER FAMILY default SET DATA "ssd" - ;)" - ); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("addColumnFamilies")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("cold")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("alterColumnFamilies")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("default")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("setTtlSettings")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("tiers")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("evictionDelay")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("storageName")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("Tier1")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("Tier2")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("86400000")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("172800000")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("2592000000")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(TieringParseCorrect) { - NYql::TAstParseResult res = SqlToYql( - R"( USE plato; - CREATE TABLE tableName (Key Uint32, Value String, PRIMARY KEY (Key)) - WITH ( TIERING = 'my_tiering' );)" - ); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("tiering")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("my_tiering")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(StoreExternalBlobsParseCorrect) { - NYql::TAstParseResult res = SqlToYql( - R"( USE plato; - CREATE TABLE tableName (Key Uint32, Value String, PRIMARY KEY (Key)) - WITH ( STORE_EXTERNAL_BLOBS = ENABLED );)" - ); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("storeExternalBlobs")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("ENABLED")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(DefaultValueColumn2) { - auto res = SqlToYql(R"( use plato; - $lambda = () -> { - RETURN CAST(RandomUuid(2) as String) - }; - - CREATE TABLE tableName ( - Key Uint32 DEFAULT RandomNumber(1), - Value String DEFAULT $lambda, - PRIMARY KEY (Key) - ); - )"); - - UNIT_ASSERT_C(res.Root, Err2Str(res)); - - const auto program = GetPrettyPrint(res); - - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, program.find("RandomNumber")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, program.find("RandomUuid")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, program.find("columnConstrains")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, program.find("columnConstrains")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, program.find("Write")); - -#if 0 - Cerr << program << Endl; -#endif - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(DefaultValueColumn3) { - auto res = SqlToYql(R"( use plato; - - CREATE TABLE tableName ( - database_id Utf8, - cloud_id Utf8, - global_id Utf8 DEFAULT database_id || "=====", - PRIMARY KEY (database_id) - ); - )"); - - UNIT_ASSERT_VALUES_EQUAL(Err2Str(res), "<main>:6:40: Error: Column reference \"database_id\" is not allowed in current scope\n"); - UNIT_ASSERT(!res.Root); - } - - Y_UNIT_TEST(DefaultValueColumn) { - auto res = SqlToYql(R"( use plato; - CREATE TABLE tableName ( - Key Uint32 FAMILY cold DEFAULT 5, - Value String FAMILY default DEFAULT "empty", - PRIMARY KEY (Key), - FAMILY default ( - DATA = "test", - COMPRESSION = "lz4" - ), - FAMILY cold ( - DATA = "test", - COMPRESSION = "off" - ) - ); - )"); - - UNIT_ASSERT_C(res.Root, Err2Str(res)); - -#if 0 - const auto program = GetPrettyPrint(res); - Cerr << program << Endl; -#endif - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("default")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("columnConstrains")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("columnFamilies")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(ChangefeedParseCorrect) { - auto res = SqlToYql(R"( USE plato; - CREATE TABLE tableName ( - Key Uint32, PRIMARY KEY (Key), - CHANGEFEED feedName WITH ( - MODE = 'KEYS_ONLY', - FORMAT = 'json', - INITIAL_SCAN = TRUE, - VIRTUAL_TIMESTAMPS = FALSE, - BARRIERS_INTERVAL = Interval("PT1S"), - RETENTION_PERIOD = Interval("P1D"), - TOPIC_MIN_ACTIVE_PARTITIONS = 10, - AWS_REGION = 'aws:region' - ) - ); - )"); - UNIT_ASSERT_C(res.Root, Err2Str(res)); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("changefeed")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("mode")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("KEYS_ONLY")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("format")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("json")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("initial_scan")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("true")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("virtual_timestamps")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("false")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("barriers_interval")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("retention_period")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("topic_min_active_partitions")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("aws_region")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("aws:region")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(CloneForAsTableWorksWithCube) { - UNIT_ASSERT(SqlToYql("SELECT * FROM AS_TABLE([<|k1:1, k2:1|>]) GROUP BY CUBE(k1, k2);").IsOk()); - } - - Y_UNIT_TEST(WindowPartitionByColumnProperlyEscaped) { - NYql::TAstParseResult res = SqlToYql("SELECT SUM(key) OVER w FROM plato.Input WINDOW w AS (PARTITION BY `column with space`);"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "CalcOverWindow") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("\"column with space\"")); - } - }; - - TWordCountHive elementStat = { {TString("CalcOverWindow"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["CalcOverWindow"]); - } - - Y_UNIT_TEST(WindowPartitionByExpressionWithoutAliasesAreAllowed) { - NYql::TAstParseResult res = SqlToYql("SELECT SUM(key) OVER w FROM plato.Input as i WINDOW w AS (PARTITION BY ii.subkey);"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "AddMember") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("AddMember row 'group_w_0 (SqlAccess 'struct (Member row '\"ii\")")); - } - if (word == "CalcOverWindow") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("CalcOverWindow core '('\"group_w_0\")")); - } - }; - - TWordCountHive elementStat = { {TString("CalcOverWindow"), 0}, {TString("AddMember"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["CalcOverWindow"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["AddMember"]); - } - - Y_UNIT_TEST(PqReadByAfterUse) { - ExpectFailWithError("use plato; pragma PqReadBy='plato2';", - "<main>:1:28: Error: Cluster in PqReadPqBy pragma differs from cluster specified in USE statement: plato2 != plato\n"); - - UNIT_ASSERT(SqlToYql("pragma PqReadBy='plato2';").IsOk()); - UNIT_ASSERT(SqlToYql("pragma PqReadBy='plato2'; use plato;").IsOk()); - UNIT_ASSERT(SqlToYql("$x='plato'; use rtmr:$x; pragma PqReadBy='plato2';").IsOk()); - UNIT_ASSERT(SqlToYql("use plato; pragma PqReadBy='dq';").IsOk()); - } - - Y_UNIT_TEST(MrObject) { - NYql::TAstParseResult res = SqlToYql( - "declare $path as String;\n" - "select * from plato.object($path, `format`, \"comp\" || \"ression\" as compression, 1 as bar) with schema (Int32 as y, String as x)" - ); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "MrObject") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, - line.find(R"__((MrObject (EvaluateAtom "$path") '"format" '('('"compression" (Concat (String '"comp") (String '"ression"))) '('"bar" (Int32 '"1")))))__")); - } else if (word == "userschema") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, - line.find(R"__('('('"userschema" (StructType '('"y" (DataType 'Int32)) '('"x" (DataType 'String))) '('"y" '"x"))))__")); - } - }; - - TWordCountHive elementStat = {{TString("MrObject"), 0}, {TString("userschema"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["MrObject"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["userschema"]); - } - - Y_UNIT_TEST(TableBindings) { - NSQLTranslation::TTranslationSettings settings = GetSettingsWithS3Binding("foo"); - NYql::TAstParseResult res = SqlToYqlWithSettings( - "select * from bindings.foo", - settings - ); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "MrObject") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, - line.find(R"__((MrTableConcat (Key '('table (String '"path")))) (Void) '('('"bar" '"1") '('"compression" '"ccompression") '('"format" '"format") '('"partitionedby" '"key" '"subkey") '('"userschema" (SqlTypeFromYson)__")); - } - }; - - TWordCountHive elementStat = {{TString("MrTableConcat"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["MrTableConcat"]); - - settings.DefaultCluster = "plato"; - settings.BindingsMode = NSQLTranslation::EBindingsMode::DISABLED; - res = SqlToYqlWithSettings( - "select * from bindings.foo", - settings - ); - UNIT_ASSERT_VALUES_EQUAL(Err2Str(res), "<main>:1:15: Error: Please remove 'bindings.' from your query, the support for this syntax has ended, code: 4601\n"); - UNIT_ASSERT(!res.Root); - - settings.BindingsMode = NSQLTranslation::EBindingsMode::DROP; - res = SqlToYqlWithSettings( - "select * from bindings.foo", - settings - ); - - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine2 = [](const TString& word, const TString& line) { - if (word == "MrTableConcat") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, - line.find(R"__((MrTableConcat (Key '('table (String '"foo")))) (Void) '())))__")); - } - }; - - TWordCountHive elementStat2 = {{TString("MrTableConcat"), 0}}; - VerifyProgram(res, elementStat2, verifyLine2); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat2["MrTableConcat"]); - - settings.BindingsMode = NSQLTranslation::EBindingsMode::DROP_WITH_WARNING; - res = SqlToYqlWithSettings( - "select * from bindings.foo", - settings - ); - UNIT_ASSERT_VALUES_EQUAL(Err2Str(res), "<main>:1:15: Warning: Please remove 'bindings.' from your query, the support for this syntax will be dropped soon, code: 4538\n"); - UNIT_ASSERT(res.Root); - - TWordCountHive elementStat3 = {{TString("MrTableConcat"), 0}}; - VerifyProgram(res, elementStat3, verifyLine2); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat3["MrTableConcat"]); - } - - Y_UNIT_TEST(TableBindingsWithInsert) { - NSQLTranslation::TTranslationSettings settings = GetSettingsWithS3Binding("foo"); - NYql::TAstParseResult res = SqlToYqlWithSettings( - "insert into bindings.foo with truncate (x, y) values (1, 2);", - settings - ); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write!") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, - line.find(R"__((Write! world sink (Key '('table (String '"path"))) values '('('"bar" '"1") '('"compression" '"ccompression") '('"format" '"format") '('"partitionedby" '"key" '"subkey") '('"userschema" (SqlTypeFromYson)__")); - } - }; - - TWordCountHive elementStat = {{TString("Write!"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write!"]); - - settings.DefaultCluster = "plato"; - settings.BindingsMode = NSQLTranslation::EBindingsMode::DISABLED; - res = SqlToYqlWithSettings( - "insert into bindings.foo with truncate (x, y) values (1, 2);", - settings - ); - UNIT_ASSERT_VALUES_EQUAL(Err2Str(res), "<main>:1:13: Error: Please remove 'bindings.' from your query, the support for this syntax has ended, code: 4601\n"); - UNIT_ASSERT(!res.Root); - - settings.BindingsMode = NSQLTranslation::EBindingsMode::DROP; - res = SqlToYqlWithSettings( - "insert into bindings.foo with truncate (x, y) values (1, 2);", - settings - ); - UNIT_ASSERT_VALUES_EQUAL(Err2Str(res), ""); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine2 = [](const TString& word, const TString& line) { - if (word == "Write!") { - //UNIT_ASSERT_VALUES_EQUAL(line, ""); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, - line.find(R"__((Write! world sink (Key '('table (String '"foo"))) values '('('mode 'renew)))__")); - } - }; - - TWordCountHive elementStat2 = {{TString("Write!"), 0}}; - VerifyProgram(res, elementStat2, verifyLine2); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat2["Write!"]); - - settings.BindingsMode = NSQLTranslation::EBindingsMode::DROP_WITH_WARNING; - res = SqlToYqlWithSettings( - "insert into bindings.foo with truncate (x, y) values (1, 2);", - settings - ); - UNIT_ASSERT_VALUES_EQUAL(Err2Str(res), "<main>:1:13: Warning: Please remove 'bindings.' from your query, the support for this syntax will be dropped soon, code: 4538\n"); - UNIT_ASSERT(res.Root); - - TWordCountHive elementStat3 = {{TString("Write!"), 0}}; - VerifyProgram(res, elementStat3, verifyLine2); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat3["Write!"]); - } - - Y_UNIT_TEST(TrailingCommaInWithout) { - UNIT_ASSERT(SqlToYql("SELECT * WITHOUT stream, FROM plato.Input").IsOk()); - UNIT_ASSERT(SqlToYql("SELECT a.* WITHOUT a.intersect, FROM plato.Input AS a").IsOk()); - UNIT_ASSERT(SqlToYql("SELECT a.* WITHOUT col1, col2, a.col3, FROM plato.Input AS a").IsOk()); - } - - Y_UNIT_TEST(NoStackOverflowOnBigCaseStatement) { - TStringBuilder req; - req << "select case 1 + 123"; - for (size_t i = 0; i < 20000; ++i) { - req << " when " << i << " then " << i + 1; - } - req << " else 100500 end;"; - UNIT_ASSERT(SqlToYql(req).IsOk()); - } - - Y_UNIT_TEST(CollectPreaggregatedInListLiteral) { - UNIT_ASSERT(SqlToYql("SELECT [COUNT(DISTINCT a+b)] FROM plato.Input").IsOk()); - } - - Y_UNIT_TEST(SmartParenInGroupByClause) { - UNIT_ASSERT(SqlToYql("SELECT * FROM plato.Input GROUP BY (k, v)").IsOk()); - } - - Y_UNIT_TEST(AlterTableRenameToIsCorrect) { - UNIT_ASSERT(SqlToYql("USE plato; ALTER TABLE table RENAME TO moved").IsOk()); - } - - Y_UNIT_TEST(AlterTableAddDropColumnIsCorrect) { - UNIT_ASSERT(SqlToYql("USE plato; ALTER TABLE table ADD COLUMN addc uint64, DROP COLUMN dropc, ADD addagain uint64").IsOk()); - } - - Y_UNIT_TEST(AlterTableSetTTLIsCorrect) { - UNIT_ASSERT(SqlToYql("USE plato; ALTER TABLE table SET (TTL = Interval(\"PT3H\") ON column)").IsOk()); - UNIT_ASSERT(SqlToYql("USE plato; ALTER TABLE table SET (TTL = Interval(\"PT3H\") ON column AS SECONDS)").IsOk()); - } - - Y_UNIT_TEST(AlterTableSetTieringIsCorrect) { - UNIT_ASSERT(SqlToYql("USE plato; ALTER TABLE table SET (TIERING = 'my_tiering')").IsOk()); - } - - Y_UNIT_TEST(AlterTableAddChangefeedIsCorrect) { - UNIT_ASSERT(SqlToYql("USE plato; ALTER TABLE table ADD CHANGEFEED feed WITH (MODE = 'UPDATES', FORMAT = 'json')").IsOk()); - } - - Y_UNIT_TEST(AlterTableAlterChangefeedIsCorrect) { - UNIT_ASSERT(SqlToYql("USE plato; ALTER TABLE table ALTER CHANGEFEED feed DISABLE").IsOk()); - ExpectFailWithError("USE plato; ALTER TABLE table ALTER CHANGEFEED feed SET (FORMAT = 'proto');", - "<main>:1:57: Error: FORMAT alter is not supported\n"); - } - - Y_UNIT_TEST(AlterTableDropChangefeedIsCorrect) { - UNIT_ASSERT(SqlToYql("USE plato; ALTER TABLE table DROP CHANGEFEED feed").IsOk()); - } - - Y_UNIT_TEST(AlterTableSetPartitioningIsCorrect) { - UNIT_ASSERT(SqlToYql("USE plato; ALTER TABLE table SET (AUTO_PARTITIONING_BY_SIZE = DISABLED)").IsOk()); - } - - Y_UNIT_TEST(AlterTableAddIndexWithIsNotSupported) { - ExpectFailWithError("USE plato; ALTER TABLE table ADD INDEX idx GLOBAL ON (col) WITH (a=b)", - "<main>:1:40: Error: with: alternative is not implemented yet: \n"); - } - - Y_UNIT_TEST(AlterTableAddIndexLocalIsNotSupported) { - ExpectFailWithError("USE plato; ALTER TABLE table ADD INDEX idx LOCAL ON (col)", - "<main>:1:40: Error: local: alternative is not implemented yet: \n"); - } - - Y_UNIT_TEST(CreateTableAddIndexVector) { - const auto result = SqlToYql(R"(USE plato; - CREATE TABLE table ( - pk INT32 NOT NULL, - col String, - INDEX idx GLOBAL USING vector_kmeans_tree - ON (col) COVER (col) - WITH (distance=cosine, vector_type=float, vector_dimension=1024,), - PRIMARY KEY (pk)) - )"); - UNIT_ASSERT_C(result.IsOk(), result.Issues.ToString()); - } - - Y_UNIT_TEST(AlterTableAddIndexVector) { - const auto result = SqlToYql(R"(USE plato; - ALTER TABLE table ADD INDEX idx - GLOBAL USING vector_kmeans_tree - ON (col) COVER (col) - WITH (distance=cosine, vector_type="float", vector_dimension=1024) - )"); - UNIT_ASSERT_C(result.IsOk(), result.Issues.ToString()); - } - - Y_UNIT_TEST(AlterTableAddIndexUnknownSubtype) { - ExpectFailWithError("USE plato; ALTER TABLE table ADD INDEX idx GLOBAL USING unknown ON (col)", - "<main>:1:57: Error: UNKNOWN index subtype is not supported\n"); - } - - Y_UNIT_TEST(AlterTableAddIndexMissedParameter) { - ExpectFailWithError(R"(USE plato; - ALTER TABLE table ADD INDEX idx - GLOBAL USING vector_kmeans_tree - ON (col) - WITH (distance=cosine, vector_type=float) - )", - "<main>:5:52: Error: vector_dimension should be set\n"); - } - - Y_UNIT_TEST(AlterTableAlterIndexSetPartitioningIsCorrect) { - const auto result = SqlToYql("USE plato; ALTER TABLE table ALTER INDEX index SET AUTO_PARTITIONING_MIN_PARTITIONS_COUNT 10"); - UNIT_ASSERT_C(result.IsOk(), result.Issues.ToString()); - } - - Y_UNIT_TEST(AlterTableAlterIndexSetMultiplePartitioningSettings) { - const auto result = SqlToYql("USE plato; ALTER TABLE table ALTER INDEX index SET " - "(AUTO_PARTITIONING_BY_LOAD = ENABLED, AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = 10)" - ); - UNIT_ASSERT_C(result.IsOk(), result.Issues.ToString()); - } - - Y_UNIT_TEST(AlterTableAlterIndexResetPartitioningIsNotSupported) { - ExpectFailWithError("USE plato; ALTER TABLE table ALTER INDEX index RESET (AUTO_PARTITIONING_MIN_PARTITIONS_COUNT)", - "<main>:1:55: Error: AUTO_PARTITIONING_MIN_PARTITIONS_COUNT reset is not supported\n" - ); - } - - Y_UNIT_TEST(AlterTableAlterColumnDropNotNullAstCorrect) { - auto reqSetNull = SqlToYql(R"( - USE plato; - CREATE TABLE tableName ( - id Uint32, - val Uint32 NOT NULL, - PRIMARY KEY (id) - ); - - COMMIT; - ALTER TABLE tableName ALTER COLUMN val DROP NOT NULL; - )"); - - UNIT_ASSERT(reqSetNull.IsOk()); - UNIT_ASSERT(reqSetNull.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - Y_UNUSED(word); - - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find( - R"(let world (Write! world sink (Key '('tablescheme (String '"tableName"))) (Void) '('('mode 'alter) '('actions '('('alterColumns '('('"val" '('changeColumnConstraints '('('drop_not_null)))))))))))" - )); - }; - - TWordCountHive elementStat({TString("\'mode \'alter")}); - VerifyProgram(reqSetNull, elementStat, verifyLine); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["\'mode \'alter"]); - } - - Y_UNIT_TEST(AlterSequence) { - UNIT_ASSERT(SqlToYql(R"( - USE plato; - ALTER SEQUENCE sequence START WITH 10 INCREMENT 2 RESTART WITH 5; - )").IsOk()); - UNIT_ASSERT(SqlToYql(R"( - USE plato; - ALTER SEQUENCE sequence INCREMENT 2; - )").IsOk()); - UNIT_ASSERT(SqlToYql(R"( - USE plato; - ALTER SEQUENCE sequence INCREMENT 2 START 1000; - )").IsOk()); - UNIT_ASSERT(SqlToYql(R"( - USE plato; - ALTER SEQUENCE sequence RESTART START 1000; - )").IsOk()); - UNIT_ASSERT(SqlToYql(R"( - USE plato; - ALTER SEQUENCE IF EXISTS sequence INCREMENT 1000 START 100 RESTART; - )").IsOk()); - UNIT_ASSERT(SqlToYql(R"( - USE plato; - ALTER SEQUENCE IF EXISTS sequence RESTART 1000 START WITH 100 INCREMENT BY 7; - )").IsOk()); - } - - Y_UNIT_TEST(AlterSequenceIncorrect) { - { - NYql::TAstParseResult res = SqlToYql("USE plato; ALTER SEQUENCE sequence START WITH 10 INCREMENT 2 RESTART WITH 5 RESTART;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:75: Error: Restart value defined more than once\n"); - } - { - NYql::TAstParseResult res = SqlToYql("USE plato; ALTER SEQUENCE sequence START WITH 10 INCREMENT 2 START 100 RESTART WITH 5;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:60: Error: Start value defined more than once\n"); - } - { - NYql::TAstParseResult res = SqlToYql("USE plato; ALTER SEQUENCE sequence INCREMENT BY 7 START WITH 10 INCREMENT 2 RESTART WITH 5 RESTART;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:62: Error: Increment defined more than once\n"); - } - { - NYql::TAstParseResult res = SqlToYql("USE plato; ALTER SEQUENCE sequence RESTART WITH 100 START WITH 10 INCREMENT 2 RESTART WITH 5;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:77: Error: Restart value defined more than once\n"); - } - { - NYql::TAstParseResult res = SqlToYql("USE plato; ALTER SEQUENCE sequence RESTART WITH 1234234543563435151456 START WITH 10 INCREMENT 2;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:49: Error: Failed to parse number from string: 1234234543563435151456, number limit overflow\n"); - } - { - NYql::TAstParseResult res = SqlToYql("USE plato; ALTER SEQUENCE sequence RESTART WITH 1 START WITH 9223372036854775817 INCREMENT 4;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:12: Error: Start value: 9223372036854775817 cannot be greater than max value: 9223372036854775807\n"); - } - { - NYql::TAstParseResult res = SqlToYql("USE plato; ALTER SEQUENCE sequence RESTART WITH 9223372036854775827 START WITH 5 INCREMENT 4;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:12: Error: Restart value: 9223372036854775827 cannot be greater than max value: 9223372036854775807\n"); - } - { - NYql::TAstParseResult res = SqlToYql("USE plato; ALTER SEQUENCE sequence RESTART WITH 1 START WITH 4 INCREMENT 0;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:12: Error: Increment must not be zero\n"); - } - { - NYql::TAstParseResult res = SqlToYql("USE plato; ALTER SEQUENCE sequence RESTART WITH 0 START WITH 4 INCREMENT 1;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:12: Error: Restart value: 0 cannot be less than min value: 1\n"); - } - { - NYql::TAstParseResult res = SqlToYql("USE plato; ALTER SEQUENCE sequence RESTART WITH 1 START WITH 0 INCREMENT 1;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:12: Error: Start value: 0 cannot be less than min value: 1\n"); - } - { - NYql::TAstParseResult res = SqlToYql("USE plato; ALTER SEQUENCE sequence RESTART WITH 1 START WITH 1 INCREMENT 9223372036854775837;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:12: Error: Increment: 9223372036854775837 cannot be greater than max value: 9223372036854775807\n"); - } - } - - Y_UNIT_TEST(AlterSequenceCorrect) { - { - NYql::TAstParseResult res = SqlToYql("USE plato; ALTER SEQUENCE sequence START WITH 10 INCREMENT 2 RESTART WITH 5;"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("sequence")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("alter")); - UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("alter_if_exists")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("start")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("increment")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("restart")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - { - NYql::TAstParseResult res = SqlToYql("USE plato; ALTER SEQUENCE IF EXISTS sequence INCREMENT 2 RESTART;"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("sequence")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("alter_if_exists")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("increment")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("restart")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - { - NYql::TAstParseResult res = SqlToYql("USE plato; ALTER SEQUENCE IF EXISTS sequence START 10 INCREMENT BY 2;"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("sequence")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("alter_if_exists")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("start")); - UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("restart")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("increment")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - } - - Y_UNIT_TEST(ShowCreateTable) { - NYql::TAstParseResult res = SqlToYql(R"( - USE plato; - SHOW CREATE TABLE user; - )"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Read") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("showCreateTable")); - } - }; - - TWordCountHive elementStat = {{TString("Read"), 0}, {TString("showCreateTable"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Read"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["showCreateTable"]); - } - - Y_UNIT_TEST(OptionalAliases) { - UNIT_ASSERT(SqlToYql("USE plato; SELECT foo FROM (SELECT key foo FROM Input);").IsOk()); - UNIT_ASSERT(SqlToYql("USE plato; SELECT a.x FROM Input1 a JOIN Input2 b ON a.key = b.key;").IsOk()); - UNIT_ASSERT(SqlToYql("USE plato; SELECT a.x FROM (VALUES (1,2), (3,4)) a(x,key) JOIN Input b ON a.key = b.key;").IsOk()); - } - - Y_UNIT_TEST(TableNameConstness) { - UNIT_ASSERT(SqlToYql("USE plato; $path = 'foo'; SELECT TableName($path), count(*) FROM Input;").IsOk()); - UNIT_ASSERT(SqlToYql("$path = 'foo'; SELECT TableName($path, 'yt'), count(*) FROM plato.Input;").IsOk()); - ExpectFailWithError("USE plato; SELECT TableName(), count(*) FROM plato.Input;", - "<main>:1:19: Error: Expression has to be an aggregation function or key column, because aggregation is used elsewhere in this subquery\n"); - } - - Y_UNIT_TEST(UseShouldWorkAsColumnName) { - UNIT_ASSERT(SqlToYql("select use from (select 1 as use);").IsOk()); - } - - Y_UNIT_TEST(TrueFalseWorkAfterDollar) { - UNIT_ASSERT(SqlToYql("$ true = false; SELECT $ true or false;").IsOk()); - UNIT_ASSERT(SqlToYql("$False = 0; SELECT $False;").IsOk()); - } - - Y_UNIT_TEST(WithSchemaEquals) { - UNIT_ASSERT(SqlToYql("select * from plato.T with schema Struct<a:Int32, b:String>;").IsOk()); - UNIT_ASSERT(SqlToYql("select * from plato.T with columns = Struct<a:Int32, b:String>;").IsOk()); - } - - Y_UNIT_TEST(WithNonStructSchemaS3) { - NSQLTranslation::TTranslationSettings settings; - settings.ClusterMapping["s3bucket"] = NYql::S3ProviderName; - UNIT_ASSERT(SqlToYqlWithSettings("select * from s3bucket.`foo` with schema (col1 Int32, String as col2, Int64 as col3);", settings).IsOk()); - } - - Y_UNIT_TEST(AllowNestedTuplesInGroupBy) { - NYql::TAstParseResult res = SqlToYql("select count(*) from plato.Input group by 1 + (x, y, z);"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [&](const TString& word, const TString& line) { - Y_UNUSED(word); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("(Aggregate core '('\"group0\")")); - }; - - TWordCountHive elementStat({"Aggregate"}); - VerifyProgram(res, elementStat, verifyLine); - UNIT_ASSERT(elementStat["Aggregate"] == 1); - } - - Y_UNIT_TEST(AllowGroupByWithParens) { - NYql::TAstParseResult res = SqlToYql("select count(*) from plato.Input group by (x, y as alias1, z);"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [&](const TString& word, const TString& line) { - Y_UNUSED(word); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("(Aggregate core '('\"x\" '\"alias1\" '\"z\")")); - }; - - TWordCountHive elementStat({"Aggregate"}); - VerifyProgram(res, elementStat, verifyLine); - UNIT_ASSERT(elementStat["Aggregate"] == 1); - } - - Y_UNIT_TEST(CreateAsyncReplicationParseCorrect) { - auto req = R"( - USE plato; - CREATE ASYNC REPLICATION MyReplication - FOR table1 AS table2, table3 AS table4 - WITH ( - CONNECTION_STRING = "grpc://localhost:2135/?database=/MyDatabase", - ENDPOINT = "localhost:2135", - DATABASE = "/MyDatabase" - ); - )"; - auto res = SqlToYql(req); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("MyReplication")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("create")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("table1")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("table2")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("table3")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("table4")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("connection_string")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("grpc://localhost:2135/?database=/MyDatabase")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("endpoint")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("localhost:2135")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("database")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("/MyDatabase")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(CreateAsyncReplicationUnsupportedSettings) { - auto reqTpl = R"( - USE plato; - CREATE ASYNC REPLICATION MyReplication - FOR table1 AS table2, table3 AS table4 - WITH ( - %s = "%s" - ) - )"; - - auto settings = THashMap<TString, TString>{ - {"STATE", "DONE"}, - {"FAILOVER_MODE", "FORCE"}, - }; - - for (const auto& [k, v] : settings) { - auto req = Sprintf(reqTpl, k.c_str(), v.c_str()); - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), Sprintf("<main>:6:%zu: Error: %s is not supported in CREATE\n", 20 + k.size(), k.c_str())); - } - } - - Y_UNIT_TEST(AsyncReplicationInvalidCommitInterval) { - auto req = R"( - USE plato; - CREATE ASYNC REPLICATION MyReplication - FOR table1 AS table2, table3 AS table4 - WITH ( - COMMIT_INTERVAL = "FOO" - ); - )"; - - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:6:35: Error: Literal of Interval type is expected for COMMIT_INTERVAL\n"); - } - - Y_UNIT_TEST(AlterAsyncReplicationParseCorrect) { - auto req = R"( - USE plato; - ALTER ASYNC REPLICATION MyReplication - SET ( - STATE = "DONE", - FAILOVER_MODE = "FORCE" - ); - )"; - auto res = SqlToYql(req); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("MyReplication")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("alter")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("state")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("DONE")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("failover_mode")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("FORCE")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(AlterAsyncReplicationSettings) { - auto reqTpl = R"( - USE plato; - ALTER ASYNC REPLICATION MyReplication - SET ( - %s = "%s" - ) - )"; - - auto settings = THashMap<TString, TString>{ - {"connection_string", "grpc://localhost:2135/?database=/MyDatabase"}, - {"endpoint", "localhost:2135"}, - {"database", "/MyDatabase"}, - {"token", "foo"}, - {"token_secret_name", "foo_secret_name"}, - {"user", "user"}, - {"password", "bar"}, - {"password_secret_name", "bar_secret_name"}, - }; - - for (const auto& [k, v] : settings) { - auto req = Sprintf(reqTpl, k.c_str(), v.c_str()); - auto res = SqlToYql(req); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [&k, &v](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("MyReplication")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("alter")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(k)); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(v)); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - } - - Y_UNIT_TEST(AlterAsyncReplicationUnsupportedSettings) { - { - auto req = R"( - USE plato; - ALTER ASYNC REPLICATION MyReplication SET (CONSISTENCY_LEVEL = "GLOBAL"); - )"; - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:80: Error: CONSISTENCY_LEVEL is not supported in ALTER\n"); - } - { - auto req = R"( - USE plato; - ALTER ASYNC REPLICATION MyReplication SET (COMMIT_INTERVAL = Interval("PT10S")); - )"; - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:87: Error: COMMIT_INTERVAL is not supported in ALTER\n"); - } - } - - Y_UNIT_TEST(AsyncReplicationInvalidSettings) { - auto req = R"( - USE plato; - ALTER ASYNC REPLICATION MyReplication SET (FOO = "BAR"); - )"; - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:62: Error: Unknown replication setting: FOO\n"); - } - - Y_UNIT_TEST(DropAsyncReplicationParseCorrect) { - auto req = R"( - USE plato; - DROP ASYNC REPLICATION MyReplication; - )"; - auto res = SqlToYql(req); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("MyReplication")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("drop")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(DropAsyncReplicationCascade) { - auto req = R"( - USE plato; - DROP ASYNC REPLICATION MyReplication CASCADE; - )"; - auto res = SqlToYql(req); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("dropCascade")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(PragmaCompactGroupBy) { - auto req = "PRAGMA CompactGroupBy; SELECT key, COUNT(*) FROM plato.Input GROUP BY key;"; - auto res = SqlToYql(req); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Aggregate") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'('compact)")); - } - }; - - TWordCountHive elementStat = { {TString("Aggregate"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Aggregate"]); - } - - Y_UNIT_TEST(PragmaDisableCompactGroupBy) { - auto req = "PRAGMA DisableCompactGroupBy; SELECT key, COUNT(*) FROM plato.Input GROUP /*+ compact() */ BY key;"; - auto res = SqlToYql(req); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Aggregate") { - UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("'('compact)")); - } - }; - - TWordCountHive elementStat = { {TString("Aggregate"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Aggregate"]); - } - - Y_UNIT_TEST(AutoSampleWorksWithNamedSubquery) { - UNIT_ASSERT(SqlToYql("$src = select * from plato.Input; select * from $src sample 0.2").IsOk()); - } - - Y_UNIT_TEST(AutoSampleWorksWithSubquery) { - UNIT_ASSERT(SqlToYql("select * from (select * from plato.Input) sample 0.2").IsOk()); - } - - Y_UNIT_TEST(CreateTableTrailingComma) { - UNIT_ASSERT(SqlToYql("USE plato; CREATE TABLE tableName (Key Uint32, PRIMARY KEY (Key),);").IsOk()); - UNIT_ASSERT(SqlToYql("USE plato; CREATE TABLE tableName (Key Uint32,);").IsOk()); - } - - Y_UNIT_TEST(BetweenSymmetric) { - UNIT_ASSERT(SqlToYql("select 3 between symmetric 5 and 4;").IsOk()); - UNIT_ASSERT(SqlToYql("select 3 between asymmetric 5 and 4;").IsOk()); - UNIT_ASSERT(SqlToYql("use plato; select key between symmetric and and and from Input;").IsOk()); - UNIT_ASSERT(SqlToYql("use plato; select key between and and and from Input;").IsOk()); - } -} - -Y_UNIT_TEST_SUITE(ExternalFunction) { - Y_UNIT_TEST(ValidUseFunctions) { - - UNIT_ASSERT(SqlToYql( - "PROCESS plato.Input" - " USING EXTERNAL FUNCTION('YANDEX-CLOUD', 'foo', <|a: 123, b: a + 641|>)" - " WITH INPUT_TYPE=Struct<a:Int32>, OUTPUT_TYPE=Struct<b:Int32>," - " CONCURRENCY=3, OPTIMIZE_FOR='CALLS'").IsOk()); - - // use CALLS without quotes, as keyword - UNIT_ASSERT(SqlToYql( - "PROCESS plato.Input" - " USING EXTERNAL FUNCTION('YANDEX-CLOUD', 'foo')" - " WITH INPUT_TYPE=Struct<a:Int32>, OUTPUT_TYPE=Struct<b:Int32>," - " OPTIMIZE_FOR=CALLS").IsOk()); - - UNIT_ASSERT(SqlToYql( - "PROCESS plato.Input" - " USING EXTERNAL FUNCTION('YANDEX-CLOUD', 'foo', TableRow())" - " WITH INPUT_TYPE=Struct<a:Int32>, OUTPUT_TYPE=Struct<b:Int32>," - " CONCURRENCY=3").IsOk()); - - UNIT_ASSERT(SqlToYql( - "PROCESS plato.Input" - " USING EXTERNAL FUNCTION('YANDEX-CLOUD', 'foo')" - " WITH INPUT_TYPE=Struct<a:Int32>, OUTPUT_TYPE=Struct<b:Int32>," - " CONCURRENCY=3, BATCH_SIZE=1000000, CONNECTION='yc-folder34fse-con'," - " INIT=[0, 900]").IsOk()); - - UNIT_ASSERT(SqlToYql( - "PROCESS plato.Input" - " USING EXTERNAL FUNCTION('YANDEX-CLOUD', 'bar', TableRow())" - " WITH UNKNOWN_PARAM_1='837747712', UNKNOWN_PARAM_2=Tuple<Uint16, Utf8>," - " INPUT_TYPE=Struct<a:Int32>, OUTPUT_TYPE=Struct<b:Int32>").IsOk()); - } - - - Y_UNIT_TEST(InValidUseFunctions) { - ExpectFailWithError("PROCESS plato.Input USING some::udf(*) WITH INPUT_TYPE=Struct<a:Int32>", - "<main>:1:33: Error: PROCESS without USING EXTERNAL FUNCTION doesn't allow WITH block\n"); - - ExpectFailWithError("PROCESS plato.Input USING EXTERNAL FUNCTION('YANDEX-CLOUD', 'jhhjfh88134d')" - " WITH INPUT_TYPE=Struct<a:Int32>, OUTPUT_TYPE=Struct<b:Int32>" - " ASSUME ORDER BY key", - "<main>:1:129: Error: PROCESS with USING EXTERNAL FUNCTION doesn't allow ASSUME block\n"); - - ExpectFailWithError("PROCESS plato.Input USING EXTERNAL FUNCTION('YANDEX-CLOUD', 'foo', 'bar', 'baz')", - "<main>:1:15: Error: EXTERNAL FUNCTION requires from 2 to 3 arguments, but got: 4\n"); - - ExpectFailWithError("PROCESS plato.Input\n" - " USING EXTERNAL FUNCTION('YANDEX-CLOUD', 'foo', <|field_1: a1, field_b: b1|>)\n" - " WITH INPUT_TYPE=Struct<a:Int32>, OUTPUT_TYPE=Struct<b:Int32>,\n" - " CONCURRENCY=3, BATCH_SIZE=1000000, CONNECTION='yc-folder34fse-con',\n" - " CONCURRENCY=5, INPUT_TYPE=Struct<b:Bool>,\n" - " INIT=[0, 900]\n", - "<main>:5:2: Error: WITH \"CONCURRENCY\" clause should be specified only once\n" - "<main>:5:17: Error: WITH \"INPUT_TYPE\" clause should be specified only once\n"); - } } -Y_UNIT_TEST_SUITE(SqlToYQLErrors) { - Y_UNIT_TEST(UdfSyntaxSugarMissingCall) { - auto req = "SELECT Udf(DateTime::FromString, \"foo\" as RunConfig);"; - auto res = SqlToYql(req); - TString a1 = Err2Str(res); - TString a2("<main>:1:8: Error: Abstract Udf Node can't be used as a part of expression.\n"); - UNIT_ASSERT_NO_DIFF(a1, a2); - } - - Y_UNIT_TEST(UdfSyntaxSugarIsNotCallable) { - auto req = "SELECT Udf(123, \"foo\" as RunConfig);"; - auto res = SqlToYql(req); - TString a1 = Err2Str(res); - TString a2("<main>:1:8: Error: Udf: first argument must be a callable, like Foo::Bar\n"); - UNIT_ASSERT_NO_DIFF(a1, a2); - } - - Y_UNIT_TEST(UdfSyntaxSugarNoArgs) { - auto req = "SELECT Udf()();"; - auto res = SqlToYql(req); - TString a1 = Err2Str(res); - TString a2("<main>:1:8: Error: Udf: expected at least one argument\n"); - UNIT_ASSERT_NO_DIFF(a1, a2); - } - - Y_UNIT_TEST(StrayUTF8) { - /// 'c' in plato is russian here - NYql::TAstParseResult res = SqlToYql("select * from сedar.Input"); - UNIT_ASSERT(!res.Root); - - TString a1 = Err2Str(res); - TString a2(R"foo(<main>:1:14: Error: token recognition error at: 'с' -)foo"); - - UNIT_ASSERT_NO_DIFF(a1, a2); - } - - Y_UNIT_TEST(IvalidStringLiteralWithEscapedBackslash) { - NYql::TAstParseResult res1 = SqlToYql(R"foo($bar = 'a\\'b';)foo"); - NYql::TAstParseResult res2 = SqlToYql(R"foo($bar = "a\\"b";)foo"); - UNIT_ASSERT(!res1.Root); - UNIT_ASSERT(!res2.Root); - - UNIT_ASSERT_NO_DIFF(Err2Str(res1), "<main>:1:13: Error: token recognition error at: '';'\n"); - UNIT_ASSERT_NO_DIFF(Err2Str(res2), "<main>:1:13: Error: token recognition error at: '\";'\n"); - } - - Y_UNIT_TEST(InvalidHexInStringLiteral) { - NYql::TAstParseResult res = SqlToYql("select \"foo\\x1\\xfe\""); - UNIT_ASSERT(!res.Root); - TString a1 = Err2Str(res); - TString a2 = "<main>:1:15: Error: Failed to parse string literal: Invalid hexadecimal value\n"; - - UNIT_ASSERT_NO_DIFF(a1, a2); - } - - Y_UNIT_TEST(InvalidOctalInMultilineStringLiteral) { - NYql::TAstParseResult res = SqlToYql("select \"foo\n" - "bar\n" - "\\01\""); - UNIT_ASSERT(!res.Root); - TString a1 = Err2Str(res); - TString a2 = "<main>:3:4: Error: Failed to parse string literal: Invalid octal value\n"; - - UNIT_ASSERT_NO_DIFF(a1, a2); - } - - Y_UNIT_TEST(InvalidDoubleAtString) { - NYql::TAstParseResult res = SqlToYql("select @@@@@@"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:12: Error: extraneous input '@' expecting {<EOF>, ';'}\n"); - } - - Y_UNIT_TEST(InvalidDoubleAtStringWhichWasAcceptedEarlier) { - NYql::TAstParseResult res = SqlToYql("SELECT @@foo@@ @ @@bar@@"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:15: Error: mismatched input '@' expecting {<EOF>, ';'}\n"); - } - - Y_UNIT_TEST(InvalidStringFromTable) { - NYql::TAstParseResult res = SqlToYql("select \"FOO\"\"BAR from plato.foo"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:12: Error: token recognition error at: '\"BAR from plato.foo'\n"); - } - - Y_UNIT_TEST(InvalidDoubleAtStringFromTable) { - NYql::TAstParseResult res = SqlToYql("select @@@@@@ from plato.foo"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:12: Error: mismatched input '@' expecting {<EOF>, ';'}\n"); - } - - Y_UNIT_TEST(SelectInvalidSyntax) { - NYql::TAstParseResult res = SqlToYql("select 1 form Wat"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:14: Error: extraneous input 'Wat' expecting {<EOF>, ';'}\n"); - } - - Y_UNIT_TEST(SelectNoCluster) { - NYql::TAstParseResult res = SqlToYql("select foo from bar"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:1: Error: No cluster name given and no default cluster is selected\n"); - } - - Y_UNIT_TEST(SelectDuplicateColumns) { - NYql::TAstParseResult res = SqlToYql("select a, a from plato.Input"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:11: Error: Unable to use duplicate column names. Collision in name: a\n"); - } - - Y_UNIT_TEST(SelectDuplicateLabels) { - NYql::TAstParseResult res = SqlToYql("select a as foo, b as foo from plato.Input"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:18: Error: Unable to use duplicate column names. Collision in name: foo\n"); - } - - Y_UNIT_TEST(SelectCaseWithoutThen) { - NYql::TAstParseResult res = SqlToYql("select case when true 1;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), - "<main>:1:22: Error: missing THEN at \'1\'\n" - "<main>:1:23: Error: extraneous input \';\' expecting {ELSE, END, WHEN}\n" - ); - } - - Y_UNIT_TEST(SelectComplexCaseWithoutThen) { - NYql::TAstParseResult res = SqlToYql( - "SELECT *\n" - "FROM plato.Input AS a\n" - "WHERE CASE WHEN a.key = \"foo\" a.subkey ELSE a.value END\n" - ); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:30: Error: missing THEN at 'a'\n"); - } - - Y_UNIT_TEST(SelectCaseWithoutEnd) { - NYql::TAstParseResult res = SqlToYql("select case a when b then c end from plato.Input"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Error: ELSE is required\n"); - } - - Y_UNIT_TEST(SelectWithBadAggregationNoInput) { - NYql::TAstParseResult res = SqlToYql("select a, Min(b), c"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), - "<main>:1:1: Error: Column references are not allowed without FROM\n" - "<main>:1:8: Error: Column reference 'a'\n" - "<main>:1:1: Error: Column references are not allowed without FROM\n" - "<main>:1:15: Error: Column reference 'b'\n" - "<main>:1:1: Error: Column references are not allowed without FROM\n" - "<main>:1:19: Error: Column reference 'c'\n" - ); - } - - Y_UNIT_TEST(SelectWithBadAggregation) { - ExpectFailWithError("select count(*), 1 + key from plato.Input", - "<main>:1:22: Error: Column `key` must either be a key column in GROUP BY or it should be used in aggregation function\n"); - } - - Y_UNIT_TEST(SelectWithBadAggregatedTerms) { - ExpectFailWithError("select key, 2 * subkey from plato.Input group by key", - "<main>:1:17: Error: Column `subkey` must either be a key column in GROUP BY or it should be used in aggregation function\n"); - } - - Y_UNIT_TEST(SelectDistinctWithBadAggregation) { - ExpectFailWithError("select distinct count(*), 1 + key from plato.Input", - "<main>:1:31: Error: Column `key` must either be a key column in GROUP BY or it should be used in aggregation function\n"); - ExpectFailWithError("select distinct key, 2 * subkey from plato.Input group by key", - "<main>:1:26: Error: Column `subkey` must either be a key column in GROUP BY or it should be used in aggregation function\n"); - } - - Y_UNIT_TEST(SelectWithBadAggregationInHaving) { - ExpectFailWithError("select key from plato.Input group by key\n" - "having \"f\" || value == \"foo\"", - "<main>:2:15: Error: Column `value` must either be a key column in GROUP BY or it should be used in aggregation function\n"); - } - - Y_UNIT_TEST(JoinWithNonAggregatedColumnInProjection) { - ExpectFailWithError("select a.key, 1 + b.subkey\n" - "from plato.Input1 as a join plato.Input2 as b using(key)\n" - "group by a.key;", - "<main>:1:19: Error: Column `b.subkey` must either be a key column in GROUP BY or it should be used in aggregation function\n"); - - ExpectFailWithError("select a.key, 1 + b.subkey.x\n" - "from plato.Input1 as a join plato.Input2 as b using(key)\n" - "group by a.key;", - "<main>:1:19: Error: Column must either be a key column in GROUP BY or it should be used in aggregation function\n"); - } - - Y_UNIT_TEST(SelectWithBadAggregatedTermsWithSources) { - ExpectFailWithError("select key, 1 + a.subkey\n" - "from plato.Input1 as a\n" - "group by a.key;", - "<main>:1:17: Error: Column `a.subkey` must either be a key column in GROUP BY or it should be used in aggregation function\n"); - ExpectFailWithError("select key, 1 + a.subkey.x\n" - "from plato.Input1 as a\n" - "group by a.key;", - "<main>:1:17: Error: Column must either be a key column in GROUP BY or it should be used in aggregation function\n"); - } - - Y_UNIT_TEST(WarnForAggregationBySelectAlias) { - NYql::TAstParseResult res = SqlToYql("select c + 1 as c from plato.Input\n" - "group by c"); - UNIT_ASSERT(res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), - "<main>:2:11: Warning: GROUP BY will aggregate by column `c` instead of aggregating by SELECT expression with same alias, code: 4532\n" - "<main>:1:10: Warning: You should probably use alias in GROUP BY instead of using it here. Please consult documentation for more details, code: 4532\n"); - - res = SqlToYql("select c + 1 as c from plato.Input\n" - "group by Math::Floor(c + 2) as c;"); - - UNIT_ASSERT(res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), - "<main>:2:22: Warning: GROUP BY will aggregate by column `c` instead of aggregating by SELECT expression with same alias, code: 4532\n" - "<main>:1:10: Warning: You should probably use alias in GROUP BY instead of using it here. Please consult documentation for more details, code: 4532\n"); - } - - Y_UNIT_TEST(NoWarnForAggregationBySelectAliasWhenAggrFunctionsAreUsedInAlias) { - NYql::TAstParseResult res = SqlToYql("select\n" - " cast(avg(val) as int) as value,\n" - " value as key\n" - "from\n" - " plato.Input\n" - "group by value"); - - UNIT_ASSERT(res.Root); - UNIT_ASSERT(res.Issues.Size() == 0); - - res = SqlToYql("select\n" - " cast(avg(val) over w as int) as value,\n" - " value as key\n" - "from\n" - " plato.Input\n" - "group by value\n" - "window w as ()"); - - UNIT_ASSERT(res.Root); - UNIT_ASSERT(res.Issues.Size() == 0); - } - - Y_UNIT_TEST(NoWarnForAggregationBySelectAliasWhenQualifiedNameIsUsed) { - NYql::TAstParseResult res = SqlToYql("select\n" - " Unwrap(a.key) as key\n" - "from plato.Input as a\n" - "join plato.Input2 as b using(k)\n" - "group by a.key;"); - UNIT_ASSERT(res.Root); - UNIT_ASSERT(res.Issues.Size() == 0); - - res = SqlToYql("select Unwrap(a.key) as key\n" - "from plato.Input as a\n" - "group by a.key;"); - UNIT_ASSERT(res.Root); - UNIT_ASSERT(res.Issues.Size() == 0); - } - - Y_UNIT_TEST(NoWarnForAggregationBySelectAliasWhenTrivialRenamingIsUsed) { - NYql::TAstParseResult res = SqlToYql("select a.key as key\n" - "from plato.Input as a\n" - "group by key;"); - UNIT_ASSERT(res.Root); - UNIT_ASSERT(res.Issues.Size() == 0); - - res = SqlToYql("select key as key\n" - "from plato.Input\n" - "group by key;"); - UNIT_ASSERT(res.Root); - UNIT_ASSERT(res.Issues.Size() == 0); - } - - Y_UNIT_TEST(ErrorByAggregatingByExpressionWithSameExpressionInSelect) { - ExpectFailWithError("select k * 2 from plato.Input group by k * 2", - "<main>:1:8: Error: Column `k` must either be a key column in GROUP BY or it should be used in aggregation function\n"); - } - - Y_UNIT_TEST(ErrorForAggregationBySelectAlias) { - ExpectFailWithError("select key, Math::Floor(1.1 + a.subkey) as foo\n" - "from plato.Input as a\n" - "group by a.key, foo;", - "<main>:3:17: Warning: GROUP BY will aggregate by column `foo` instead of aggregating by SELECT expression with same alias, code: 4532\n" - "<main>:1:19: Warning: You should probably use alias in GROUP BY instead of using it here. Please consult documentation for more details, code: 4532\n" - "<main>:1:31: Error: Column `a.subkey` must either be a key column in GROUP BY or it should be used in aggregation function\n"); - - ExpectFailWithError("select c + 1 as c from plato.Input\n" - "group by Math::Floor(c + 2);", - "<main>:2:22: Warning: GROUP BY will aggregate by column `c` instead of aggregating by SELECT expression with same alias, code: 4532\n" - "<main>:1:10: Warning: You should probably use alias in GROUP BY instead of using it here. Please consult documentation for more details, code: 4532\n" - "<main>:1:8: Error: Column `c` must either be a key column in GROUP BY or it should be used in aggregation function\n"); - } - - Y_UNIT_TEST(ExplainQueryPlan) { - NYql::TAstParseResult res = SqlToYql("EXPLAIN Q U E R Y PLAN SELECT 1;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_STRING_CONTAINS(Err2Str(res), "<main>:1:8: Error: mismatched input 'Q' expecting {"); - } - - Y_UNIT_TEST(SelectWithDuplicateGroupingColumns) { - NYql::TAstParseResult res = SqlToYql("select c from plato.Input group by c, c"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Error: Duplicate grouping column: c\n"); - } - - Y_UNIT_TEST(SelectWithBadAggregationInGrouping) { - NYql::TAstParseResult res = SqlToYql("select a, Min(b), c group by c"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:1: Error: Column references are not allowed without FROM\n" - "<main>:1:30: Error: Column reference 'c'\n"); - } - - Y_UNIT_TEST(SelectWithOpOnBadAggregation) { - ExpectFailWithError("select 1 + a + Min(b) from plato.Input", - "<main>:1:12: Error: Column `a` must either be a key column in GROUP BY or it should be used in aggregation function\n"); - } - - Y_UNIT_TEST(SelectOrderByConstantNum) { - NYql::TAstParseResult res = SqlToYql("select a from plato.Input order by 1"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:36: Error: Unable to ORDER BY constant expression\n"); - } - - Y_UNIT_TEST(SelectOrderByConstantExpr) { - NYql::TAstParseResult res = SqlToYql("select a from plato.Input order by 1 * 42"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:38: Error: Unable to ORDER BY constant expression\n"); - } - - Y_UNIT_TEST(SelectOrderByConstantString) { - NYql::TAstParseResult res = SqlToYql("select a from plato.Input order by \"nest\""); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:36: Error: Unable to ORDER BY constant expression\n"); - } - - Y_UNIT_TEST(SelectOrderByAggregated) { - NYql::TAstParseResult res = SqlToYql("select a from plato.Input order by min(a)"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:36: Error: Unable to ORDER BY aggregated values\n"); - } - - Y_UNIT_TEST(ErrorInOrderByExpresison) { - NYql::TAstParseResult res = SqlToYql("select key, value from plato.Input order by (key as zey)"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:45: Error: You should use in ORDER BY column name, qualified field, callable function or expression\n"); - } - - Y_UNIT_TEST(ErrorsInOrderByWhenColumnIsMissingInProjection) { - ExpectFailWithError("select subkey from (select 1 as subkey) order by key", "<main>:1:50: Error: Column key is not in source column set\n"); - ExpectFailWithError("select subkey from plato.Input as a order by x.key", "<main>:1:46: Error: Unknown correlation name: x\n"); - ExpectFailWithError("select distinct a, b from plato.Input order by c", "<main>:1:48: Error: Column c is not in source column set. Did you mean a?\n"); - ExpectFailWithError("select count(*) as a from plato.Input order by c", "<main>:1:48: Error: Column c is not in source column set. Did you mean a?\n"); - ExpectFailWithError("select count(*) as a, b, from plato.Input group by b order by c", "<main>:1:63: Error: Column c is not in source column set. Did you mean a?\n"); - UNIT_ASSERT(SqlToYql("select a, b from plato.Input order by c").IsOk()); - } - - Y_UNIT_TEST(SelectAggregatedWhere) { - NYql::TAstParseResult res = SqlToYql("select * from plato.Input where count(key)"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:33: Error: Can not use aggregated values in filtering\n"); - } - - Y_UNIT_TEST(DoubleFrom) { - NYql::TAstParseResult res = SqlToYql("from plato.Input select * from plato.Input"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:27: Error: Only one FROM clause is allowed\n"); - } - - Y_UNIT_TEST(SelectJoinMissingCorrName) { - NYql::TAstParseResult res = SqlToYql("select * from plato.Input1 as a join plato.Input2 as b on a.key == key"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:65: Error: JOIN: column requires correlation name\n"); - } - - Y_UNIT_TEST(SelectJoinMissingCorrName1) { - NYql::TAstParseResult res = SqlToYql( - "use plato;\n" - "$foo = select * from Input1;\n" - "select * from Input2 join $foo USING(key);\n" - ); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:27: Error: JOIN: missing correlation name for source\n"); - } - - Y_UNIT_TEST(SelectJoinMissingCorrName2) { - NYql::TAstParseResult res = SqlToYql( - "use plato;\n" - "$foo = select * from Input1;\n" - "select * from Input2 cross join $foo;\n" - ); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:33: Error: JOIN: missing correlation name for source\n"); - } - - Y_UNIT_TEST(SelectJoinEmptyCorrNames) { - NYql::TAstParseResult res = SqlToYql( - "$left = (SELECT * FROM plato.Input1 LIMIT 2);\n" - "$right = (SELECT * FROM plato.Input2 LIMIT 2);\n" - "SELECT * FROM $left FULL JOIN $right USING (key);\n" - ); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:45: Error: At least one correlation name is required in join\n"); - } - - Y_UNIT_TEST(SelectJoinSameCorrNames) { - NYql::TAstParseResult res = SqlToYql("SELECT Input.key FROM plato.Input JOIN plato.Input1 ON Input.key == Input.subkey\n"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:66: Error: JOIN: different correlation names are required for joined tables\n"); - } - - Y_UNIT_TEST(SelectJoinConstPredicateArg) { - NYql::TAstParseResult res = SqlToYql("SELECT * FROM plato.Input1 as A JOIN plato.Input2 as B ON A.key == B.key AND A.subkey == \"wtf\"\n"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:87: Error: JOIN: each equality predicate argument must depend on exactly one JOIN input\n"); - } - - Y_UNIT_TEST(SelectJoinNonEqualityPredicate) { - NYql::TAstParseResult res = SqlToYql("SELECT * FROM plato.Input1 as A JOIN plato.Input2 as B ON A.key == B.key AND A.subkey > B.subkey\n"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:87: Error: JOIN ON expression must be a conjunction of equality predicates\n"); - } - - Y_UNIT_TEST(SelectEquiJoinCorrNameOutOfScope) { - NYql::TAstParseResult res = SqlToYql( - "PRAGMA equijoin;\n" - "SELECT * FROM plato.A JOIN plato.B ON A.key == C.key JOIN plato.C ON A.subkey == C.subkey;\n" - ); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:2:45: Error: JOIN: can not use source: C in equality predicate, it is out of current join scope\n"); - } - - Y_UNIT_TEST(SelectEquiJoinNoRightSource) { - NYql::TAstParseResult res = SqlToYql( - "PRAGMA equijoin;\n" - "SELECT * FROM plato.A JOIN plato.B ON A.key == B.key JOIN plato.C ON A.subkey == B.subkey;\n" - ); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:2:79: Error: JOIN ON equality predicate must have one of its arguments from the rightmost source\n"); - } - - Y_UNIT_TEST(SelectEquiJoinOuterWithoutType) { - NYql::TAstParseResult res = SqlToYql( - "SELECT * FROM plato.A Outer JOIN plato.B ON A.key == B.key;\n" - ); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:23: Error: Invalid join type: OUTER JOIN. OUTER keyword is optional and can only be used after LEFT, RIGHT or FULL\n"); - } - - Y_UNIT_TEST(SelectEquiJoinOuterWithWrongType) { - NYql::TAstParseResult res = SqlToYql( - "SELECT * FROM plato.A LEFT semi OUTER JOIN plato.B ON A.key == B.key;\n" - ); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:33: Error: Invalid join type: LEFT SEMI OUTER JOIN. OUTER keyword is optional and can only be used after LEFT, RIGHT or FULL\n"); - } - - Y_UNIT_TEST(InsertNoCluster) { - NYql::TAstParseResult res = SqlToYql("insert into Output (foo) values (1)"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:1: Error: No cluster name given and no default cluster is selected\n"); - } - - Y_UNIT_TEST(InsertValuesNoLabels) { - NYql::TAstParseResult res = SqlToYql("insert into plato.Output values (1)"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:19: Error: INSERT INTO ... VALUES requires specification of table columns\n"); - } - - Y_UNIT_TEST(UpsertValuesNoLabelsKikimr) { - NYql::TAstParseResult res = SqlToYql("upsert into plato.Output values (1)", 10, TString(NYql::KikimrProviderName)); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:19: Error: UPSERT INTO ... VALUES requires specification of table columns\n"); - } - - Y_UNIT_TEST(ReplaceValuesNoLabelsKikimr) { - NYql::TAstParseResult res = SqlToYql("replace into plato.Output values (1)", 10, TString(NYql::KikimrProviderName)); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:20: Error: REPLACE INTO ... VALUES requires specification of table columns\n"); - } - - Y_UNIT_TEST(InsertValuesInvalidLabels) { - NYql::TAstParseResult res = SqlToYql("insert into plato.Output (foo) values (1, 2)"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:27: Error: VALUES have 2 columns, INSERT INTO expects: 1\n"); - } - - Y_UNIT_TEST(BuiltinFileOpNoArgs) { - NYql::TAstParseResult res = SqlToYql("select FilePath()"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Error: FilePath() requires exactly 1 arguments, given: 0\n"); - } - - Y_UNIT_TEST(ProcessWithHaving) { - NYql::TAstParseResult res = SqlToYql("process plato.Input using some::udf(value) having value == 1"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:15: Error: PROCESS does not allow HAVING yet! You may request it on yql@ maillist.\n"); - } - - Y_UNIT_TEST(ReduceNoBy) { - NYql::TAstParseResult res = SqlToYql("reduce plato.Input using some::udf(value)"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:19: Error: mismatched input 'using' expecting {',', ON, PRESORT}\n"); - } - - Y_UNIT_TEST(ReduceDistinct) { - NYql::TAstParseResult res = SqlToYql("reduce plato.Input on key using some::udf(distinct value)"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:43: Error: DISTINCT can not be used in PROCESS/REDUCE\n"); - } - - Y_UNIT_TEST(CreateTableWithView) { - NYql::TAstParseResult res = SqlToYql("CREATE TABLE plato.foo:bar (key INT);"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:22: Error: mismatched input ':' expecting '('\n"); - } - - Y_UNIT_TEST(AsteriskWithSomethingAfter) { - NYql::TAstParseResult res = SqlToYql("select *, LENGTH(value) from plato.Input;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Error: Unable to use plain '*' with other projection items. Please use qualified asterisk instead: '<table>.*' (<table> can be either table name or table alias).\n"); - } - - Y_UNIT_TEST(AsteriskWithSomethingBefore) { - NYql::TAstParseResult res = SqlToYql("select LENGTH(value), * from plato.Input;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:23: Error: Unable to use plain '*' with other projection items. Please use qualified asterisk instead: '<table>.*' (<table> can be either table name or table alias).\n"); - } - - Y_UNIT_TEST(DuplicatedQualifiedAsterisk) { - NYql::TAstParseResult res = SqlToYql("select in.*, key, in.* from plato.Input as in;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:19: Error: Unable to use twice same quialified asterisk. Invalid source: in\n"); - } - - Y_UNIT_TEST(BrokenLabel) { - NYql::TAstParseResult res = SqlToYql("select in.*, key as `funny.label` from plato.Input as in;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:14: Error: Unable to use '.' in column name. Invalid column name: funny.label\n"); - } - - Y_UNIT_TEST(KeyConflictDetect0) { - NYql::TAstParseResult res = SqlToYql("select key, in.key as key from plato.Input as in;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:13: Error: Unable to use duplicate column names. Collision in name: key\n"); - } - - Y_UNIT_TEST(KeyConflictDetect1) { - NYql::TAstParseResult res = SqlToYql("select length(key) as key, key from plato.Input;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:28: Error: Unable to use duplicate column names. Collision in name: key\n"); - } - - Y_UNIT_TEST(KeyConflictDetect2) { - NYql::TAstParseResult res = SqlToYql("select key, in.key from plato.Input as in;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:1: Error: Duplicate column: key\n"); - } - - Y_UNIT_TEST(AutogenerationAliasWithCollisionConflict1) { - UNIT_ASSERT(SqlToYql("select LENGTH(Value), key as column0 from plato.Input;").IsOk()); - } - - Y_UNIT_TEST(AutogenerationAliasWithCollisionConflict2) { - UNIT_ASSERT(SqlToYql("select key as column1, LENGTH(Value) from plato.Input;").IsOk()); - } - - Y_UNIT_TEST(MissedSourceTableForQualifiedAsteriskOnSimpleSelect) { - NYql::TAstParseResult res = SqlToYql("use plato; select Intop.*, Input.key from plato.Input;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:19: Error: Unknown correlation name: Intop\n"); - } - - Y_UNIT_TEST(MissedSourceTableForQualifiedAsteriskOnJoin) { - NYql::TAstParseResult res = SqlToYql("use plato; select tmissed.*, t2.*, t1.key from plato.Input as t1 join plato.Input as t2 on t1.key==t2.key;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:19: Error: Unknown correlation name for asterisk: tmissed\n"); - } - - Y_UNIT_TEST(UnableToReferenceOnNotExistSubcolumn) { - NYql::TAstParseResult res = SqlToYql("select b.subkey from (select key from plato.Input as a) as b;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Error: Column subkey is not in source column set\n"); - } - - Y_UNIT_TEST(ConflictOnSameNameWithQualify0) { - NYql::TAstParseResult res = SqlToYql("select in.key, in.key as key from plato.Input as in;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:1: Error: Duplicate column: key\n"); - } - - Y_UNIT_TEST(ConflictOnSameNameWithQualify1) { - NYql::TAstParseResult res = SqlToYql("select in.key, length(key) as key from plato.Input as in;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:1: Error: Duplicate column: key\n"); - } - - Y_UNIT_TEST(ConflictOnSameNameWithQualify2) { - NYql::TAstParseResult res = SqlToYql("select key, in.key from plato.Input as in;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:1: Error: Duplicate column: key\n"); - } - - Y_UNIT_TEST(ConflictOnSameNameWithQualify3) { - NYql::TAstParseResult res = SqlToYql("select in.key, subkey as key from plato.Input as in;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:1: Error: Duplicate column: key\n"); - } - - Y_UNIT_TEST(SelectFlattenBySameColumns) { - NYql::TAstParseResult res = SqlToYql("select key from plato.Input flatten by (key, key as kk)"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:46: Error: Duplicate column name found: key in FlattenBy section\n"); - } - - Y_UNIT_TEST(SelectFlattenBySameAliases) { - NYql::TAstParseResult res = SqlToYql("select key from plato.Input flatten by (key as kk, subkey as kk);"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:52: Error: Duplicate alias found: kk in FlattenBy section\n"); - } - - Y_UNIT_TEST(SelectFlattenByExprSameAliases) { - NYql::TAstParseResult res = SqlToYql("select key from plato.Input flatten by (key as kk, ListSkip(subkey,1) as kk);"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:52: Error: Collision between alias and column name: kk in FlattenBy section\n"); - } - - Y_UNIT_TEST(SelectFlattenByConflictNameAndAlias0) { - NYql::TAstParseResult res = SqlToYql("select key from plato.Input flatten by (key, subkey as key);"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:46: Error: Collision between alias and column name: key in FlattenBy section\n"); - } - - Y_UNIT_TEST(SelectFlattenByConflictNameAndAlias1) { - NYql::TAstParseResult res = SqlToYql("select key from plato.Input flatten by (key as kk, subkey as key);"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:52: Error: Collision between alias and column name: key in FlattenBy section\n"); - } - - Y_UNIT_TEST(SelectFlattenByExprConflictNameAndAlias1) { - NYql::TAstParseResult res = SqlToYql("select key from plato.Input flatten by (key as kk, ListSkip(subkey,1) as key);"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:52: Error: Duplicate column name found: key in FlattenBy section\n"); - } - - Y_UNIT_TEST(SelectFlattenByUnnamedExpr) { - NYql::TAstParseResult res = SqlToYql("select key from plato.Input flatten by (key, ListSkip(key, 1))"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:46: Error: Unnamed expression after FLATTEN BY is not allowed\n"); - } - - Y_UNIT_TEST(UseInOnStrings) { - NYql::TAstParseResult res = SqlToYql("select * from plato.Input where \"foo\" in \"foovalue\";"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:42: Error: Unable to use IN predicate with string argument, it won't search substring - " - "expecting tuple, list, dict or single column table source\n"); - } - - Y_UNIT_TEST(UseSubqueryInScalarContextInsideIn) { - NYql::TAstParseResult res = SqlToYql("$q = (select key from plato.Input); select * from plato.Input where subkey in ($q);"); - UNIT_ASSERT(res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:79: Warning: Using subrequest in scalar context after IN, " - "perhaps you should remove parenthesis here, code: 4501\n"); - } - - Y_UNIT_TEST(InHintsWithKeywordClash) { - NYql::TAstParseResult res = SqlToYql("SELECT COMPACT FROM plato.Input WHERE COMPACT IN COMPACT `COMPACT`(1,2,3)"); - UNIT_ASSERT(!res.Root); - // should try to parse last compact as call expression - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:58: Error: Unknown builtin: COMPACT\n"); - } - - Y_UNIT_TEST(ErrorColumnPosition) { - NYql::TAstParseResult res = SqlToYql( - "USE plato;\n" - "SELECT \n" - "value FROM (\n" - "select key from Input\n" - ");\n" - ); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:1: Error: Column value is not in source column set\n"); - } - - Y_UNIT_TEST(PrimaryViewAbortMapReduce) { - NYql::TAstParseResult res = SqlToYql("SELECT key FROM plato.Input VIEW PRIMARY KEY"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:17: Error: primary view is not supported for yt tables\n"); - } - - Y_UNIT_TEST(InsertAbortMapReduce) { - NYql::TAstParseResult res = SqlToYql("INSERT OR ABORT INTO plato.Output SELECT key FROM plato.Input"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:0: Error: INSERT OR ABORT INTO is not supported for yt tables\n"); - } - - Y_UNIT_TEST(ReplaceIntoMapReduce) { - NYql::TAstParseResult res = SqlToYql("REPLACE INTO plato.Output SELECT key FROM plato.Input"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:0: Error: Meaning of REPLACE INTO has been changed, now you should use INSERT INTO <table> WITH TRUNCATE ... for yt\n"); - } - - Y_UNIT_TEST(UpsertIntoMapReduce) { - NYql::TAstParseResult res = SqlToYql("UPSERT INTO plato.Output SELECT key FROM plato.Input"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:0: Error: UPSERT INTO is not supported for yt tables\n"); - } - - Y_UNIT_TEST(UpdateMapReduce) { - NYql::TAstParseResult res = SqlToYql("UPDATE plato.Output SET value = value + 1 WHERE key < 1"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:0: Error: UPDATE is unsupported for yt\n"); - } - - Y_UNIT_TEST(DeleteMapReduce) { - NYql::TAstParseResult res = SqlToYql("DELETE FROM plato.Output WHERE key < 1"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:0: Error: DELETE is unsupported for yt\n"); - } - - Y_UNIT_TEST(ReplaceIntoWithTruncate) { - NYql::TAstParseResult res = SqlToYql("REPLACE INTO plato.Output WITH TRUNCATE SELECT key FROM plato.Input"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:32: Error: Unable REPLACE INTO with truncate mode\n"); - } - - Y_UNIT_TEST(UpsertIntoWithTruncate) { - NYql::TAstParseResult res = SqlToYql("UPSERT INTO plato.Output WITH TRUNCATE SELECT key FROM plato.Input"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:31: Error: Unable UPSERT INTO with truncate mode\n"); - } - - Y_UNIT_TEST(InsertIntoWithTruncateKikimr) { - NYql::TAstParseResult res = SqlToYql("INSERT INTO plato.Output WITH TRUNCATE SELECT key FROM plato.Input", 10, TString(NYql::KikimrProviderName)); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:0: Error: INSERT INTO WITH TRUNCATE is not supported for kikimr tables\n"); - } - - Y_UNIT_TEST(InsertIntoWithWrongArgumentCount) { - NYql::TAstParseResult res = SqlToYql("insert into plato.Output with truncate (key, value, subkey) values (5, '1', '2', '3');"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:53: Error: VALUES have 4 columns, INSERT INTO ... WITH TRUNCATE expects: 3\n"); - } - - Y_UNIT_TEST(UpsertWithWrongArgumentCount) { - NYql::TAstParseResult res = SqlToYql("upsert into plato.Output (key, value, subkey) values (2, '3');", 10, TString(NYql::KikimrProviderName)); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:39: Error: VALUES have 2 columns, UPSERT INTO expects: 3\n"); - } - - Y_UNIT_TEST(GroupingSetByExprWithoutAlias) { - NYql::TAstParseResult res = SqlToYql("SELECT key FROM plato.Input GROUP BY GROUPING SETS (cast(key as uint32), subkey);"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:53: Error: Unnamed expressions are not supported in GROUPING SETS. Please use '<expr> AS <name>'.\n"); - } - - Y_UNIT_TEST(GroupingSetByExprWithoutAlias2) { - NYql::TAstParseResult res = SqlToYql("SELECT key FROM plato.Input GROUP BY subkey || subkey, GROUPING SETS (\n" - "cast(key as uint32), subkey);"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:2:1: Error: Unnamed expressions are not supported in GROUPING SETS. Please use '<expr> AS <name>'.\n"); - } - - Y_UNIT_TEST(CubeByExprWithoutAlias) { - NYql::TAstParseResult res = SqlToYql("SELECT key FROM plato.Input GROUP BY CUBE (key, subkey / key);"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:56: Error: Unnamed expressions are not supported in CUBE. Please use '<expr> AS <name>'.\n"); - } - - Y_UNIT_TEST(RollupByExprWithoutAlias) { - NYql::TAstParseResult res = SqlToYql("SELECT key FROM plato.Input GROUP BY ROLLUP (subkey / key);"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:53: Error: Unnamed expressions are not supported in ROLLUP. Please use '<expr> AS <name>'.\n"); - } - - Y_UNIT_TEST(GroupByHugeCubeDeniedNoPragma) { - NYql::TAstParseResult res = SqlToYql("SELECT key FROM plato.Input GROUP BY CUBE (key, subkey, value, key + subkey as sum, key - subkey as sub, key + val as keyval);"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:119: Error: GROUP BY CUBE is allowed only for 5 columns, but you use 6\n"); - } - - Y_UNIT_TEST(GroupByInvalidPragma) { - NYql::TAstParseResult res = SqlToYql("PRAGMA GroupByCubeLimit = '-4';"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:27: Error: Expected unsigned integer literal as a single argument for: GroupByCubeLimit\n"); - } - - Y_UNIT_TEST(GroupByHugeCubeDeniedPragme) { - NYql::TAstParseResult res = SqlToYql("PRAGMA GroupByCubeLimit = '4'; SELECT key FROM plato.Input GROUP BY CUBE (key, subkey, value, key + subkey as sum, key - subkey as sub);"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:132: Error: GROUP BY CUBE is allowed only for 4 columns, but you use 5\n"); - } - - Y_UNIT_TEST(GroupByFewBigCubes) { - NYql::TAstParseResult res = SqlToYql("SELECT key FROM plato.Input GROUP BY CUBE(key, subkey, key + subkey as sum), CUBE(value, value + key + subkey as total);"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:1: Error: Unable to GROUP BY more than 64 groups, you try use 80 groups\n"); - } - - Y_UNIT_TEST(GroupByFewBigCubesWithPragmaLimit) { - NYql::TAstParseResult res = SqlToYql("PRAGMA GroupByLimit = '16'; SELECT key FROM plato.Input GROUP BY GROUPING SETS(key, subkey, key + subkey as sum), ROLLUP(value, value + key + subkey as total);"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:29: Error: Unable to GROUP BY more than 16 groups, you try use 18 groups\n"); - } - - Y_UNIT_TEST(NoGroupingColumn0) { - NYql::TAstParseResult res = SqlToYql( - "select count(1), key_first, val_first, grouping(key_first, val_first, nomind) as group\n" - "from plato.Input group by grouping sets (cast(key as uint32) /100 as key_first, Substring(value, 1, 1) as val_first);"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:71: Error: Column 'nomind' is not a grouping column\n"); - } - - Y_UNIT_TEST(NoGroupingColumn1) { - NYql::TAstParseResult res = SqlToYql("select count(1), grouping(key, value) as group_duo from plato.Input group by cube (key, subkey);"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:32: Error: Column 'value' is not a grouping column\n"); - } - - Y_UNIT_TEST(EmptyAccess0) { - NYql::TAstParseResult res = SqlToYql("insert into plato.Output (list0, list1) values (AsList(0, 1, 2), AsList(``));"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:73: Error: Column reference \"\" is not allowed in current scope\n"); - } - - Y_UNIT_TEST(EmptyAccess1) { - NYql::TAstParseResult res = SqlToYql("insert into plato.Output (list0, list1) values (AsList(0, 1, 2), ``);"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:66: Error: Column reference \"\" is not allowed in current scope\n"); - } - - Y_UNIT_TEST(UseUnknownColumnInInsert) { - NYql::TAstParseResult res = SqlToYql("insert into plato.Output (list0, list1) values (AsList(0, 1, 2), AsList(`test`));"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:73: Error: Column reference \"test\" is not allowed in current scope\n"); - } - - Y_UNIT_TEST(GroupByEmptyColumn) { - NYql::TAstParseResult res = SqlToYql("select count(1) from plato.Input group by ``;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:43: Error: Column name can not be empty\n"); - } - - Y_UNIT_TEST(ConvertNumberOutOfBase) { - NYql::TAstParseResult res = SqlToYql("select 0o80l;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Error: Failed to parse number from string: 0o80l, char: '8' is out of base: 8\n"); - } - - Y_UNIT_TEST(ConvertNumberOutOfRangeForInt64ButFitsInUint64) { - NYql::TAstParseResult res = SqlToYql("select 0xc000000000000000l;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Error: Failed to parse 13835058055282163712 as integer literal of Int64 type: value out of range for Int64\n"); - } - - Y_UNIT_TEST(ConvertNumberOutOfRangeUint64) { - NYql::TAstParseResult res = SqlToYql("select 0xc0000000000000000l;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Error: Failed to parse number from string: 0xc0000000000000000l, number limit overflow\n"); - - res = SqlToYql("select 1234234543563435151456;\n"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Error: Failed to parse number from string: 1234234543563435151456, number limit overflow\n"); - } - - Y_UNIT_TEST(ConvertNumberNegativeOutOfRange) { - NYql::TAstParseResult res = SqlToYql("select -9223372036854775808;\n" - "select -9223372036854775809;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:2:8: Error: Failed to parse negative integer: -9223372036854775809, number limit overflow\n"); - } - - Y_UNIT_TEST(InvaildUsageReal0) { - NYql::TAstParseResult res = SqlToYql("select .0;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_STRING_CONTAINS(Err2Str(res), "<main>:1:7: Error: extraneous input '.' expecting {"); - } - - Y_UNIT_TEST(InvaildUsageReal1) { - NYql::TAstParseResult res = SqlToYql("select .0f;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_STRING_CONTAINS(Err2Str(res), "<main>:1:7: Error: extraneous input '.' expecting {"); - } - - Y_UNIT_TEST(InvaildUsageWinFunctionWithoutWindow) { - NYql::TAstParseResult res = SqlToYql("select lead(key, 2) from plato.Input;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Error: Failed to use window function Lead without window specification\n"); - } - - Y_UNIT_TEST(DropTableWithIfExists) { - NYql::TAstParseResult res = SqlToYql("DROP TABLE IF EXISTS plato.foo;"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("drop_if_exists")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(TooManyErrors) { - const char* q = R"( - USE plato; - select A, B, C, D, E, F, G, H, I, J, K, L, M, N from (select b from `abc`); -)"; - - NYql::TAstParseResult res = SqlToYql(q, 10); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), - R"(<main>:3:16: Error: Column A is not in source column set. Did you mean b? -<main>:3:19: Error: Column B is not in source column set. Did you mean b? -<main>:3:22: Error: Column C is not in source column set. Did you mean b? -<main>:3:25: Error: Column D is not in source column set. Did you mean b? -<main>:3:28: Error: Column E is not in source column set. Did you mean b? -<main>:3:31: Error: Column F is not in source column set. Did you mean b? -<main>:3:34: Error: Column G is not in source column set. Did you mean b? -<main>:3:37: Error: Column H is not in source column set. Did you mean b? -<main>:3:40: Error: Column I is not in source column set. Did you mean b? -<main>: Error: Too many issues, code: 1 -)"); - }; - - Y_UNIT_TEST(ShouldCloneBindingForNamedParameter) { - NYql::TAstParseResult res = SqlToYql(R"($f = () -> { - $value_type = TypeOf(1); - $pair_type = StructType( - TypeOf("2") AS key, - $value_type AS value - ); - - RETURN TupleType( - ListType($value_type), - $pair_type); -}; - -select FormatType($f()); -)"); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(BlockedInvalidFrameBounds) { - auto check = [](const TString& frame, const TString& err) { - const TString prefix = "SELECT SUM(x) OVER w FROM plato.Input WINDOW w AS (PARTITION BY key ORDER BY subkey\n"; - NYql::TAstParseResult res = SqlToYql(prefix + frame + ")"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), err); - }; - - check("ROWS UNBOUNDED FOLLOWING", "<main>:2:5: Error: Frame cannot start from UNBOUNDED FOLLOWING\n"); - check("ROWS BETWEEN 5 PRECEDING AND UNBOUNDED PRECEDING", "<main>:2:29: Error: Frame cannot end with UNBOUNDED PRECEDING\n"); - check("ROWS BETWEEN CURRENT ROW AND 5 PRECEDING", "<main>:2:13: Error: Frame cannot start from CURRENT ROW and end with PRECEDING\n"); - check("ROWS BETWEEN 5 FOLLOWING AND CURRENT ROW", "<main>:2:14: Error: Frame cannot start from FOLLOWING and end with CURRENT ROW\n"); - check("ROWS BETWEEN 5 FOLLOWING AND 5 PRECEDING", "<main>:2:14: Error: Frame cannot start from FOLLOWING and end with PRECEDING\n"); - } - - Y_UNIT_TEST(BlockedRangeValueWithoutSingleOrderBy) { - UNIT_ASSERT(SqlToYql("SELECT COUNT(*) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) FROM plato.Input").IsOk()); - UNIT_ASSERT(SqlToYql("SELECT COUNT(*) OVER (RANGE BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING) FROM plato.Input").IsOk()); - - auto res = SqlToYql("SELECT COUNT(*) OVER (RANGE 5 PRECEDING) FROM plato.Input"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:29: Error: RANGE with <offset> PRECEDING/FOLLOWING requires exactly one expression in ORDER BY partition clause\n"); - - res = SqlToYql("SELECT COUNT(*) OVER (ORDER BY key, value RANGE 5 PRECEDING) FROM plato.Input"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:49: Error: RANGE with <offset> PRECEDING/FOLLOWING requires exactly one expression in ORDER BY partition clause\n"); - } - - Y_UNIT_TEST(NoColumnsInFrameBounds) { - NYql::TAstParseResult res = SqlToYql( - "SELECT SUM(x) OVER w FROM plato.Input WINDOW w AS (ROWS BETWEEN\n" - " 1 + key PRECEDING AND 2 + key FOLLOWING);"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:2:6: Error: Column reference \"key\" is not allowed in current scope\n"); - } - - Y_UNIT_TEST(WarnOnEmptyFrameBounds) { - NYql::TAstParseResult res = SqlToYql( - "SELECT SUM(x) OVER w FROM plato.Input WINDOW w AS (PARTITION BY key ORDER BY subkey\n" - "ROWS BETWEEN 10 FOLLOWING AND 5 FOLLOWING)"); - UNIT_ASSERT(res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:2:14: Warning: Used frame specification implies empty window frame, code: 4520\n"); - } - - Y_UNIT_TEST(WarnOnRankWithUnorderedWindow) { - NYql::TAstParseResult res = SqlToYql("SELECT RANK() OVER w FROM plato.Input WINDOW w AS ()"); - UNIT_ASSERT(res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Warning: Rank() is used with unordered window - all rows will be considered equal to each other, code: 4521\n"); - } - - Y_UNIT_TEST(WarnOnRankExprWithUnorderedWindow) { - NYql::TAstParseResult res = SqlToYql("SELECT RANK(key) OVER w FROM plato.Input WINDOW w AS ()"); - UNIT_ASSERT(res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Warning: Rank(<expression>) is used with unordered window - the result is likely to be undefined, code: 4521\n"); - } - - Y_UNIT_TEST(AnyAsTableName) { - NYql::TAstParseResult res = SqlToYql("use plato; select * from any;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:28: Error: no viable alternative at input 'any;'\n"); - } - - Y_UNIT_TEST(IncorrectOrderOfLambdaOptionalArgs) { - NYql::TAstParseResult res = SqlToYql("$f = ($x?, $y)->($x + $y); select $f(1);"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:12: Error: Non-optional argument can not follow optional one\n"); - } - - Y_UNIT_TEST(IncorrectOrderOfActionOptionalArgs) { - NYql::TAstParseResult res = SqlToYql("define action $f($x?, $y) as select $x,$y; end define; do $f(1);"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:23: Error: Non-optional argument can not follow optional one\n"); - } - - Y_UNIT_TEST(NotAllowedQuestionOnNamedNode) { - NYql::TAstParseResult res = SqlToYql("$f = 1; select $f?;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:18: Error: Unexpected token '?' at the end of expression\n"); - } - - Y_UNIT_TEST(AnyAndCrossJoin) { - NYql::TAstParseResult res = SqlToYql("use plato; select * from any Input1 cross join Input2"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:26: Error: ANY should not be used with Cross JOIN\n"); - - res = SqlToYql("use plato; select * from Input1 cross join any Input2"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:44: Error: ANY should not be used with Cross JOIN\n"); - } - - Y_UNIT_TEST(AnyWithCartesianProduct) { - NYql::TAstParseResult res = SqlToYql("pragma AnsiImplicitCrossJoin; use plato; select * from any Input1, Input2"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:56: Error: ANY should not be used with Cross JOIN\n"); - - res = SqlToYql("pragma AnsiImplicitCrossJoin; use plato; select * from Input1, any Input2"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:64: Error: ANY should not be used with Cross JOIN\n"); - } - - Y_UNIT_TEST(ErrorPlainEndAsInlineActionTerminator) { - NYql::TAstParseResult res = SqlToYql( - "do begin\n" - " select 1\n" - "; end\n"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:4:0: Error: missing DO at '<EOF>'\n"); - } - - Y_UNIT_TEST(ErrorMultiWayJoinWithUsing) { - NYql::TAstParseResult res = SqlToYql( - "USE plato;\n" - "PRAGMA DisableSimpleColumns;\n" - "SELECT *\n" - "FROM Input1 AS a\n" - "JOIN Input2 AS b USING(key)\n" - "JOIN Input3 AS c ON a.key = c.key;\n" - ); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), - "<main>:5:24: Error: Multi-way JOINs should be connected with ON clause instead of USING clause\n" - ); - } - - Y_UNIT_TEST(RequireLabelInFlattenByWithDot) { - NYql::TAstParseResult res = SqlToYql("select * from plato.Input flatten by x.y"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), - "<main>:1:40: Error: Unnamed expression after FLATTEN BY is not allowed\n" - ); - } - - Y_UNIT_TEST(WarnUnnamedColumns) { - NYql::TAstParseResult res = SqlToYql( - "PRAGMA WarnUnnamedColumns;\n" - "\n" - "SELECT key, subkey, key || subkey FROM plato.Input ORDER BY subkey;\n"); - UNIT_ASSERT(res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:28: Warning: Autogenerated column name column2 will be used for expression, code: 4516\n"); - } - - Y_UNIT_TEST(WarnSourceColumnMismatch) { - NYql::TAstParseResult res = SqlToYql( - "insert into plato.Output (key, subkey, new_value, one_more_value) select key as Key, subkey, value, \"x\" from plato.Input;"); - UNIT_ASSERT(res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:51: Warning: Column names in SELECT don't match column specification in parenthesis. \"key\" doesn't match \"Key\". \"new_value\" doesn't match \"value\", code: 4517\n"); - } - - Y_UNIT_TEST(YtCaseInsensitive) { - NYql::TAstParseResult res = SqlToYql("select * from PlatO.foo;"); - UNIT_ASSERT(res.Root); - - res = SqlToYql("use PlatO; select * from foo;"); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(KikimrCaseSensitive) { - NYql::TAstParseResult res = SqlToYql("select * from PlatO.foo;", 10, "kikimr"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:15: Error: Unknown cluster: PlatO\n"); - - res = SqlToYql("use PlatO; select * from foo;", 10, "kikimr"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:5: Error: Unknown cluster: PlatO\n"); - } - - Y_UNIT_TEST(DiscoveryModeForbidden) { - NYql::TAstParseResult res = SqlToYqlWithMode("insert into plato.Output select * from plato.range(\"\", Input1, Input4)", NSQLTranslation::ESqlMode::DISCOVERY); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:40: Error: range is not allowed in Discovery mode, code: 4600\n"); - - res = SqlToYqlWithMode("insert into plato.Output select * from plato.like(\"\", \"Input%\")", NSQLTranslation::ESqlMode::DISCOVERY); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:40: Error: like is not allowed in Discovery mode, code: 4600\n"); - - res = SqlToYqlWithMode("insert into plato.Output select * from plato.regexp(\"\", \"Input.\")", NSQLTranslation::ESqlMode::DISCOVERY); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:40: Error: regexp is not allowed in Discovery mode, code: 4600\n"); - - res = SqlToYqlWithMode("insert into plato.Output select * from plato.filter(\"\", ($name) -> { return find($name, \"Input\") is not null; })", NSQLTranslation::ESqlMode::DISCOVERY); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:40: Error: filter is not allowed in Discovery mode, code: 4600\n"); - - res = SqlToYqlWithMode("select Path from plato.folder(\"\") where Type == \"table\"", NSQLTranslation::ESqlMode::DISCOVERY); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:18: Error: folder is not allowed in Discovery mode, code: 4600\n"); - } - - Y_UNIT_TEST(YsonFuncWithoutArgs) { - UNIT_ASSERT(SqlToYql("SELECT Yson::SerializeText(Yson::From());").IsOk()); - } - - Y_UNIT_TEST(CanNotUseOrderByInNonLastSelectInUnionAllChain) { - auto req = "pragma AnsiOrderByLimitInUnionAll;\n" - "use plato;\n" - "\n" - "select * from Input order by key\n" - "union all\n" - "select * from Input order by key limit 1;"; - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:4:21: Error: ORDER BY within UNION ALL is only allowed after last subquery\n"); - } - - Y_UNIT_TEST(CanNotUseLimitInNonLastSelectInUnionAllChain) { - auto req = "pragma AnsiOrderByLimitInUnionAll;\n" - "use plato;\n" - "\n" - "select * from Input limit 1\n" - "union all\n" - "select * from Input order by key limit 1;"; - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:4:21: Error: LIMIT within UNION ALL is only allowed after last subquery\n"); - } - - Y_UNIT_TEST(CanNotUseDiscardInNonFirstSelectInUnionAllChain) { - auto req = "pragma AnsiOrderByLimitInUnionAll;\n" - "use plato;\n" - "\n" - "select * from Input\n" - "union all\n" - "discard select * from Input;"; - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:6:1: Error: DISCARD within UNION ALL is only allowed before first subquery\n"); - } - - Y_UNIT_TEST(CanNotUseIntoResultInNonLastSelectInUnionAllChain) { - auto req = "use plato;\n" - "pragma AnsiOrderByLimitInUnionAll;\n" - "\n" - "select * from Input\n" - "union all\n" - "discard select * from Input;"; - - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:6:1: Error: DISCARD within UNION ALL is only allowed before first subquery\n"); - } - - Y_UNIT_TEST(YsonStrictInvalidPragma) { - auto res = SqlToYql("pragma yson.Strict = \"wrong\";"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:22: Error: Expected 'true', 'false' or no parameter for: Strict\n"); - } - - Y_UNIT_TEST(WarnTableNameInSomeContexts) { - UNIT_ASSERT(SqlToYql("use plato; select TableName() from Input;").IsOk()); - UNIT_ASSERT(SqlToYql("use plato; select TableName(\"aaaa\");").IsOk()); - UNIT_ASSERT(SqlToYql("select TableName(\"aaaa\", \"yt\");").IsOk()); - - auto res = SqlToYql("select TableName() from plato.Input;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Error: TableName requires either service name as second argument or current cluster name\n"); - - res = SqlToYql("use plato;\n" - "select TableName() from Input1 as a join Input2 as b using(key);"); - UNIT_ASSERT(res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:2:8: Warning: TableName() may produce empty result when used in ambiguous context (with JOIN), code: 4525\n"); - - res = SqlToYql("use plato;\n" - "select SOME(TableName()), key from Input group by key;"); - UNIT_ASSERT(res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:2:13: Warning: TableName() will produce empty result when used with aggregation.\n" - "Please consult documentation for possible workaround, code: 4525\n"); - } - - Y_UNIT_TEST(WarnOnDistincWithHavingWithoutAggregations) { - auto res = SqlToYql("select distinct key from plato.Input having key != '0';"); - UNIT_ASSERT(res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:49: Warning: The usage of HAVING without aggregations with SELECT DISTINCT is non-standard and will stop working soon. Please use WHERE instead., code: 4526\n"); - } - - Y_UNIT_TEST(FlattenByExprWithNestedNull) { - auto res = SqlToYql("USE plato;\n" - "\n" - "SELECT * FROM (SELECT 1 AS region_id)\n" - "FLATTEN BY (\n" - " CAST($unknown(region_id) AS List<String>) AS region\n" - ")"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:5:10: Error: Unknown name: $unknown\n"); - } - - Y_UNIT_TEST(EmptySymbolNameIsForbidden) { - auto req = " $`` = 1; select $``;"; - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:5: Error: Empty symbol name is not allowed\n"); - } - - Y_UNIT_TEST(WarnOnBinaryOpWithNullArg) { - auto req = "select * from plato.Input where cast(key as Int32) != NULL"; - auto res = SqlToYql(req); - UNIT_ASSERT(res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:52: Warning: Binary operation != will return NULL here, code: 4529\n"); - - req = "select 1 or null"; - res = SqlToYql(req); - UNIT_ASSERT(res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), ""); - } - - Y_UNIT_TEST(ErrorIfTableSampleArgUsesColumns) { - auto req = "SELECT key FROM plato.Input TABLESAMPLE BERNOULLI(MIN_OF(100.0, CAST(subkey as Int32)));"; - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:70: Error: Column reference \"subkey\" is not allowed in current scope\n"); - } - - Y_UNIT_TEST(DerivedColumnListForSelectIsNotSupportedYet) { - auto req = "SELECT a,b,c FROM plato.Input as t(x,y,z);"; - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:35: Error: Derived column list is only supported for VALUES\n"); - } - - Y_UNIT_TEST(ErrorIfValuesHasDifferentCountOfColumns) { - auto req = "VALUES (1,2,3), (4,5);"; - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:17: Error: All VALUES items should have same size: expecting 3, got 2\n"); - } - - Y_UNIT_TEST(ErrorIfDerivedColumnSizeExceedValuesColumnCount) { - auto req = "SELECT * FROM(VALUES (1,2), (3,4)) as t(x,y,z);"; - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:40: Error: Derived column list size exceeds column count in VALUES\n"); - } - - Y_UNIT_TEST(WarnoOnAutogeneratedNamesForValues) { - auto req = "PRAGMA WarnUnnamedColumns;\n" - "SELECT * FROM (VALUES (1,2,3,4), (5,6,7,8)) as t(x,y);"; - auto res = SqlToYql(req); - UNIT_ASSERT(res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:2:16: Warning: Autogenerated column names column2...column3 will be used here, code: 4516\n"); - } - - Y_UNIT_TEST(ErrUnionAllWithOrderByWithoutExplicitLegacyMode) { - auto req = "use plato;\n" - "\n" - "select * from Input order by key\n" - "union all\n" - "select * from Input order by key;"; - - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:21: Error: ORDER BY within UNION ALL is only allowed after last subquery\n"); - } - - Y_UNIT_TEST(ErrUnionAllWithLimitWithoutExplicitLegacyMode) { - auto req = "use plato;\n" - "\n" - "select * from Input limit 10\n" - "union all\n" - "select * from Input limit 1;"; - - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:21: Error: LIMIT within UNION ALL is only allowed after last subquery\n"); - } - - Y_UNIT_TEST(ErrUnionAllWithIntoResultWithoutExplicitLegacyMode) { - auto req = "use plato;\n" - "\n" - "select * from Input into result aaa\n" - "union all\n" - "select * from Input;"; - - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:21: Error: INTO RESULT within UNION ALL is only allowed after last subquery\n"); - } - - Y_UNIT_TEST(ErrUnionAllWithDiscardWithoutExplicitLegacyMode) { - auto req = "use plato;\n" - "\n" - "select * from Input\n" - "union all\n" - "discard select * from Input;"; - - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:5:1: Error: DISCARD within UNION ALL is only allowed before first subquery\n"); - } - - Y_UNIT_TEST(ErrUnionAllKeepsIgnoredOrderByWarning) { - auto req = "use plato;\n" - "\n" - "SELECT * FROM (\n" - " SELECT * FROM Input\n" - " UNION ALL\n" - " SELECT t.* FROM Input AS t ORDER BY t.key\n" - ");"; - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:4:3: Warning: ORDER BY without LIMIT in subquery will be ignored, code: 4504\n" - "<main>:6:39: Error: Unknown correlation name: t\n"); - } - - Y_UNIT_TEST(ErrOrderByIgnoredButCheckedForMissingColumns) { - auto req = "$src = SELECT key FROM (SELECT 1 as key, 2 as subkey) ORDER BY x; SELECT * FROM $src;"; - ExpectFailWithError(req, "<main>:1:8: Warning: ORDER BY without LIMIT in subquery will be ignored, code: 4504\n" - "<main>:1:64: Error: Column x is not in source column set\n"); - - req = "$src = SELECT key FROM plato.Input ORDER BY x; SELECT * FROM $src;"; - auto res = SqlToYql(req); - UNIT_ASSERT(res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Warning: ORDER BY without LIMIT in subquery will be ignored, code: 4504\n"); - } - - Y_UNIT_TEST(InvalidTtlInterval) { - auto req = R"( - USE plato; - CREATE TABLE tableName (Key Uint32, CreatedAt Timestamp, PRIMARY KEY (Key)) - WITH (TTL = 1 On CreatedAt); - )"; - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:4:25: Error: Literal of Interval type is expected for TTL\n" - "<main>:4:25: Error: Invalid TTL settings\n"); - } - - Y_UNIT_TEST(InvalidTtlUnit) { - auto req = R"( - USE plato; - CREATE TABLE tableName (Key Uint32, CreatedAt Uint32, PRIMARY KEY (Key)) - WITH (TTL = Interval("P1D") On CreatedAt AS PICOSECONDS); - )"; - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_STRING_CONTAINS(Err2Str(res), "mismatched input 'PICOSECONDS' expecting {MICROSECONDS, MILLISECONDS, NANOSECONDS, SECONDS}"); - } - - Y_UNIT_TEST(InvalidChangefeedSink) { - auto req = R"( - USE plato; - CREATE TABLE tableName ( - Key Uint32, PRIMARY KEY (Key), - CHANGEFEED feedName WITH (SINK_TYPE = "S3", MODE = "KEYS_ONLY", FORMAT = "json") - ); - )"; - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:5:55: Error: Unknown changefeed sink type: S3\n"); - } - - Y_UNIT_TEST(InvalidChangefeedSettings) { - auto req = R"( - USE plato; - CREATE TABLE tableName ( - Key Uint32, PRIMARY KEY (Key), - CHANGEFEED feedName WITH (SINK_TYPE = "local", FOO = "bar") - ); - )"; - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:5:64: Error: Unknown changefeed setting: FOO\n"); - } - - Y_UNIT_TEST(InvalidChangefeedInitialScan) { - auto req = R"( - USE plato; - CREATE TABLE tableName ( - Key Uint32, PRIMARY KEY (Key), - CHANGEFEED feedName WITH (MODE = "KEYS_ONLY", FORMAT = "json", INITIAL_SCAN = "foo") - ); - )"; - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:5:95: Error: Literal of Bool type is expected for INITIAL_SCAN\n"); - } - - Y_UNIT_TEST(InvalidChangefeedVirtualTimestamps) { - auto req = R"( - USE plato; - CREATE TABLE tableName ( - Key Uint32, PRIMARY KEY (Key), - CHANGEFEED feedName WITH (MODE = "KEYS_ONLY", FORMAT = "json", VIRTUAL_TIMESTAMPS = "foo") - ); - )"; - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:5:101: Error: Literal of Bool type is expected for VIRTUAL_TIMESTAMPS\n"); - } - - Y_UNIT_TEST(InvalidChangefeedResolvedTimestamps) { - auto req = R"( - USE plato; - CREATE TABLE tableName ( - Key Uint32, PRIMARY KEY (Key), - CHANGEFEED feedName WITH (MODE = "KEYS_ONLY", FORMAT = "json", BARRIERS_INTERVAL = "foo") - ); - )"; - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:5:100: Error: Literal of Interval type is expected for BARRIERS_INTERVAL\n"); - } - - Y_UNIT_TEST(InvalidChangefeedRetentionPeriod) { - auto req = R"( - USE plato; - CREATE TABLE tableName ( - Key Uint32, PRIMARY KEY (Key), - CHANGEFEED feedName WITH (MODE = "KEYS_ONLY", FORMAT = "json", RETENTION_PERIOD = "foo") - ); - )"; - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:5:99: Error: Literal of Interval type is expected for RETENTION_PERIOD\n"); - } - - Y_UNIT_TEST(InvalidChangefeedTopicPartitions) { - auto req = R"( - USE plato; - CREATE TABLE tableName ( - Key Uint32, PRIMARY KEY (Key), - CHANGEFEED feedName WITH (MODE = "KEYS_ONLY", FORMAT = "json", TOPIC_MIN_ACTIVE_PARTITIONS = "foo") - ); - )"; - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:5:110: Error: Literal of integer type is expected for TOPIC_MIN_ACTIVE_PARTITIONS\n"); - } - - Y_UNIT_TEST(InvalidChangefeedAwsRegion) { - auto req = R"( - USE plato; - CREATE TABLE tableName ( - Key Uint32, PRIMARY KEY (Key), - CHANGEFEED feedName WITH (MODE = "KEYS_ONLY", FORMAT = "json", AWS_REGION = true) - ); - )"; - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:5:93: Error: Literal of String type is expected for AWS_REGION\n"); - } - - Y_UNIT_TEST(ErrJoinWithGroupingSetsWithoutCorrelationName) { - auto req = "USE plato;\n" - "\n" - "SELECT k1, k2, subkey\n" - "FROM T1 AS a JOIN T2 AS b USING (key)\n" - "GROUP BY GROUPING SETS(\n" - " (a.key as k1, b.subkey as k2),\n" - " (k1),\n" - " (subkey)\n" - ");"; - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:8:4: Error: Columns in grouping sets should have correlation name, error in key: subkey\n"); - } - - Y_UNIT_TEST(ErrJoinWithGroupByWithoutCorrelationName) { - auto req = "USE plato;\n" - "\n" - "SELECT k1, k2,\n" - " value\n" - "FROM T1 AS a JOIN T2 AS b USING (key)\n" - "GROUP BY a.key as k1, b.subkey as k2,\n" - " value;"; - ExpectFailWithError(req, - "<main>:7:5: Error: Columns in GROUP BY should have correlation name, error in key: value\n"); - } - - Y_UNIT_TEST(ErrWithMissingFrom) { - auto req = "select 1 as key where 1 > 1;"; - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:25: Error: Filtering is not allowed without FROM\n"); - - req = "select 1 + count(*);"; - res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:12: Error: Aggregation is not allowed without FROM\n"); - - req = "select 1 as key, subkey + value;"; - res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:1: Error: Column references are not allowed without FROM\n" - "<main>:1:18: Error: Column reference 'subkey'\n" - "<main>:1:1: Error: Column references are not allowed without FROM\n" - "<main>:1:27: Error: Column reference 'value'\n"); - - req = "select count(1) group by key;"; - res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:1: Error: Column references are not allowed without FROM\n" - "<main>:1:26: Error: Column reference 'key'\n"); - } - - Y_UNIT_TEST(ErrWithMissingFromForWindow) { - auto req = "$c = () -> (1 + count(1) over w);\n" - "select $c();"; - ExpectFailWithError(req, - "<main>:1:9: Error: Window and aggregation functions are not allowed in this context\n" - "<main>:1:17: Error: Failed to use aggregation function Count without window specification or in wrong place\n"); - - req = "$c = () -> (1 + lead(1) over w);\n" - "select $c();"; - ExpectFailWithError(req, - "<main>:1:17: Error: Window functions are not allowed in this context\n" - "<main>:1:17: Error: Failed to use window function Lead without window specification or in wrong place\n"); - - req = "select 1 + count(1) over w window w as ();"; - ExpectFailWithError(req, - "<main>:1:1: Error: Window and aggregation functions are not allowed without FROM\n" - "<main>:1:12: Error: Failed to use aggregation function Count without window specification or in wrong place\n"); - - req = "select 1 + lead(1) over w window w as ();"; - ExpectFailWithError(req, - "<main>:1:12: Error: Window functions are not allowed without FROM\n" - "<main>:1:12: Error: Failed to use window function Lead without window specification or in wrong place\n"); - } - - Y_UNIT_TEST(ErrWithMissingFromForInplaceWindow) { - auto req = "$c = () -> (1 + count(1) over ());\n" - "select $c();"; - ExpectFailWithError(req, - "<main>:1:26: Error: Window and aggregation functions are not allowed in this context\n"); - - req = "$c = () -> (1 + lead(1) over (rows between unbounded preceding and current row));\n" - "select $c();"; - ExpectFailWithError(req, - "<main>:1:25: Error: Window and aggregation functions are not allowed in this context\n"); - - req = "select 1 + count(1) over ();"; - ExpectFailWithError(req, - "<main>:1:1: Error: Window and aggregation functions are not allowed without FROM\n" - "<main>:1:12: Error: Failed to use aggregation function Count without window specification or in wrong place\n"); - - req = "select 1 + lead(1) over (rows between current row and unbounded following);"; - ExpectFailWithError(req, - "<main>:1:12: Error: Window functions are not allowed without FROM\n" - "<main>:1:12: Error: Failed to use window function Lead without window specification or in wrong place\n"); - } - - Y_UNIT_TEST(ErrDistinctInWrongPlace) { - auto req = "select Some::Udf(distinct key) from plato.Input;"; - ExpectFailWithError(req, - "<main>:1:18: Error: DISTINCT can only be used in aggregation functions\n"); - req = "select sum(key)(distinct foo) from plato.Input;"; - ExpectFailWithError(req, - "<main>:1:17: Error: DISTINCT can only be used in aggregation functions\n"); - - req = "select len(distinct foo) from plato.Input;"; - ExpectFailWithError(req, - "<main>:1:8: Error: DISTINCT can only be used in aggregation functions\n"); - - req = "$foo = ($x) -> ($x); select $foo(distinct key) from plato.Input;"; - ExpectFailWithError(req, - "<main>:1:34: Error: DISTINCT can only be used in aggregation functions\n"); - } - - Y_UNIT_TEST(ErrForNotSingleChildInInlineAST) { - ExpectFailWithError("select YQL::\"\"", - "<main>:1:8: Error: Failed to parse YQL: expecting AST root node with single child, but got 0\n"); - ExpectFailWithError("select YQL::@@ \t@@", - "<main>:1:8: Error: Failed to parse YQL: expecting AST root node with single child, but got 0\n"); - auto req = "$lambda = YQL::@@(lambda '(x)(+ x x)) (lambda '(y)(+ y y))@@;\n" - "select ListMap([1, 2, 3], $lambda);"; - ExpectFailWithError(req, - "<main>:1:11: Error: Failed to parse YQL: expecting AST root node with single child, but got 2\n"); - } - - Y_UNIT_TEST(ErrEmptyColumnName) { - ExpectFailWithError("select * without \"\" from plato.Input", - "<main>:1:18: Error: String literal can not be used here\n"); - - ExpectFailWithError("select * without `` from plato.Input;", - "<main>:1:18: Error: Empty column name is not allowed\n"); - - ExpectFailWithErrorForAnsiLexer("select * without \"\" from plato.Input", - "<main>:1:18: Error: Empty column name is not allowed\n"); - - ExpectFailWithErrorForAnsiLexer("select * without `` from plato.Input;", - "<main>:1:18: Error: Empty column name is not allowed\n"); - } - - Y_UNIT_TEST(ErrOnNonZeroArgumentsForTableRows) { - ExpectFailWithError("$udf=\"\";process plato.Input using $udf(TableRows(k))", - "<main>:1:40: Error: TableRows requires exactly 0 arguments\n"); - } - - Y_UNIT_TEST(ErrGroupByWithAggregationFunctionAndDistinctExpr) { - ExpectFailWithError("select * from plato.Input group by count(distinct key|key)", - "<main>:1:36: Error: Unable to GROUP BY aggregated values\n"); - } - - // FIXME: check if we can get old behaviour -#if 0 - Y_UNIT_TEST(ErrWithSchemaWithColumnsWithoutType) { - ExpectFailWithError("select * from plato.Input with COLUMNs", - "<main>:1:32: Error: Expected type after COLUMNS\n" - "<main>:1:32: Error: Failed to parse table hints\n"); - - ExpectFailWithError("select * from plato.Input with scheMa", - "<main>:1:32: Error: Expected type after SCHEMA\n" - "<main>:1:32: Error: Failed to parse table hints\n"); - } -#endif - - Y_UNIT_TEST(ErrCollectPreaggregatedInListLiteralWithoutFrom) { - ExpectFailWithError("SELECT([VARIANCE(DISTINCT[])])", - "<main>:1:1: Error: Column references are not allowed without FROM\n" - "<main>:1:9: Error: Column reference '_yql_preagg_Variance0'\n"); - } - - Y_UNIT_TEST(ErrGroupBySmartParenAsTuple) { - ExpectFailWithError("SELECT * FROM plato.Input GROUP BY (k, v,)", - "<main>:1:41: Error: Unexpected trailing comma in grouping elements list\n"); - } - - Y_UNIT_TEST(HandleNestedSmartParensInGroupBy) { - ExpectFailWithError("SELECT * FROM plato.Input GROUP BY (+() as k)", - "<main>:1:37: Error: Unable to GROUP BY constant expression\n"); - } - - Y_UNIT_TEST(ErrRenameWithAddColumn) { - ExpectFailWithError("USE plato; ALTER TABLE table RENAME TO moved, ADD COLUMN addc uint64", - "<main>:1:40: Error: RENAME TO can not be used together with another table action\n"); - } - - Y_UNIT_TEST(ErrAddColumnAndRename) { - // FIXME: fix positions in ALTER TABLE - ExpectFailWithError("USE plato; ALTER TABLE table ADD COLUMN addc uint64, RENAME TO moved", - "<main>:1:46: Error: RENAME TO can not be used together with another table action\n"); - } - - Y_UNIT_TEST(InvalidUuidValue) { - ExpectFailWithError("SELECT Uuid('123e4567ae89ba12d3aa456a426614174ab0')", - "<main>:1:8: Error: Invalid value \"123e4567ae89ba12d3aa456a426614174ab0\" for type Uuid\n"); - ExpectFailWithError("SELECT Uuid('123e4567ae89b-12d3-a456-426614174000')", - "<main>:1:8: Error: Invalid value \"123e4567ae89b-12d3-a456-426614174000\" for type Uuid\n"); - } - - Y_UNIT_TEST(WindowFunctionWithoutOver) { - ExpectFailWithError("SELECT LAST_VALUE(foo) FROM plato.Input", - "<main>:1:8: Error: Can't use window function LastValue without window specification (OVER keyword is missing)\n"); - ExpectFailWithError("SELECT LAST_VALUE(foo) FROM plato.Input GROUP BY key", - "<main>:1:8: Error: Can't use window function LastValue without window specification (OVER keyword is missing)\n"); - } - - Y_UNIT_TEST(CreateAlterUserWithoutCluster) { - ExpectFailWithError("\n CREATE USER user ENCRYPTED PASSWORD 'foobar';", "<main>:2:2: Error: USE statement is missing - no default cluster is selected\n"); - ExpectFailWithError("ALTER USER CURRENT_USER RENAME TO $foo;", "<main>:1:1: Error: USE statement is missing - no default cluster is selected\n"); - } - - Y_UNIT_TEST(ModifyPermissionsWithoutCluster) { - ExpectFailWithError("\n GRANT CONNECT ON `/Root` TO user;", "<main>:2:2: Error: USE statement is missing - no default cluster is selected\n"); - ExpectFailWithError("\n REVOKE MANAGE ON `/Root` FROM user;", "<main>:2:2: Error: USE statement is missing - no default cluster is selected\n"); - } - - Y_UNIT_TEST(ReservedRoleNames) { - ExpectFailWithError("USE plato; CREATE USER current_User;", "<main>:1:24: Error: System role CURRENT_USER can not be used here\n"); - ExpectFailWithError("USE plato; ALTER USER current_User RENAME TO Current_role", "<main>:1:46: Error: System role CURRENT_ROLE can not be used here\n"); - UNIT_ASSERT(SqlToYql("USE plato; DROP GROUP IF EXISTS a, b, c, current_User;").IsOk()); - } - - Y_UNIT_TEST(DisableClassicDivisionWithError) { - ExpectFailWithError("pragma ClassicDivision = 'false'; select $foo / 30;", "<main>:1:42: Error: Unknown name: $foo\n"); - } - - Y_UNIT_TEST(AggregationOfAgrregatedDistinctExpr) { - ExpectFailWithError("select sum(sum(distinct x + 1)) from plato.Input", "<main>:1:12: Error: Aggregation of aggregated values is forbidden\n"); - } - - Y_UNIT_TEST(WarnForUnusedSqlHint) { - NYql::TAstParseResult res = SqlToYql("select * from plato.Input1 as a join /*+ merge() */ plato.Input2 as b using(key);\n" - "select --+ foo(bar)\n" - " 1;"); - UNIT_ASSERT(res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:2:23: Warning: Hint foo will not be used, code: 4534\n"); - } - - Y_UNIT_TEST(WarnForDeprecatedSchema) { - NSQLTranslation::TTranslationSettings settings; - settings.ClusterMapping["s3bucket"] = NYql::S3ProviderName; - NYql::TAstParseResult res = SqlToYqlWithSettings("select * from s3bucket.`foo` with schema (col1 Int32, String as col2, Int64 as col3);", settings); - UNIT_ASSERT(res.Root); - UNIT_ASSERT_STRING_CONTAINS(res.Issues.ToString(), "Warning: Deprecated syntax for positional schema: please use 'column type' instead of 'type AS column', code: 4535\n"); - } - - Y_UNIT_TEST(ErrorOnColumnNameInMaxByLimit) { - ExpectFailWithError( - "SELECT AGGREGATE_BY(AsTuple(value, key), AggregationFactory(\"MAX_BY\", subkey)) FROM plato.Input;", - "<main>:1:42: Error: Source does not allow column references\n" - "<main>:1:71: Error: Column reference 'subkey'\n"); - } - - Y_UNIT_TEST(ErrorInLibraryWithTopLevelNamedSubquery) { - TString withUnusedSubq = "$unused = select max(key) from plato.Input;\n" - "\n" - "define subquery $foo() as\n" - " $count = select count(*) from plato.Input;\n" - " select * from plato.Input limit $count / 2;\n" - "end define;\n" - "export $foo;\n"; - UNIT_ASSERT(SqlToYqlWithMode(withUnusedSubq, NSQLTranslation::ESqlMode::LIBRARY).IsOk()); - - TString withTopLevelSubq = "$count = select count(*) from plato.Input;\n" - "\n" - "define subquery $foo() as\n" - " select * from plato.Input limit $count / 2;\n" - "end define;\n" - "export $foo;\n"; - auto res = SqlToYqlWithMode(withTopLevelSubq, NSQLTranslation::ESqlMode::LIBRARY); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:17: Error: Named subquery can not be used as a top level statement in libraries\n"); - } - - Y_UNIT_TEST(SessionStartAndSessionStateShouldSurviveSessionWindowArgsError){ - TString query = R"( - $init = ($_row) -> (min(1, 2)); -- error: aggregation func min() can not be used here - $calculate = ($_row, $_state) -> (1); - $update = ($_row, $_state) -> (2); - SELECT - SessionStart() over w as session_start, - SessionState() over w as session_state, - FROM plato.Input as t - WINDOW w AS ( - PARTITION BY user, SessionWindow(ts + 1, $init, $update, $calculate) - ) - )"; - ExpectFailWithError(query, "<main>:2:33: Error: Aggregation function Min requires exactly 1 argument(s), given: 2\n"); - } - - Y_UNIT_TEST(ScalarContextUsage1) { - TString query = R"( - $a = (select 1 as x, 2 as y); - select 1 + $a; - )"; - ExpectFailWithError(query, "<main>:2:39: Error: Source used in expression should contain one concrete column\n" - "<main>:3:24: Error: Source is used here\n"); - } - - Y_UNIT_TEST(ScalarContextUsage2) { - TString query = R"( - use plato; - $a = (select 1 as x, 2 as y); - select * from concat($a); - )"; - ExpectFailWithError(query, "<main>:3:39: Error: Source used in expression should contain one concrete column\n" - "<main>:4:34: Error: Source is used here\n"); - } - - Y_UNIT_TEST(ScalarContextUsage3) { - TString query = R"( - use plato; - $a = (select 1 as x, 2 as y); - select * from range($a); - )"; - ExpectFailWithError(query, "<main>:3:39: Error: Source used in expression should contain one concrete column\n" - "<main>:4:33: Error: Source is used here\n"); - } - - Y_UNIT_TEST(ScalarContextUsage4) { - TString query = R"( - use plato; - $a = (select 1 as x, 2 as y); - insert into $a select 1; - )"; - ExpectFailWithError(query, "<main>:3:39: Error: Source used in expression should contain one concrete column\n" - "<main>:4:25: Error: Source is used here\n"); - } -} - -void CheckUnused(const TString& req, const TString& symbol, unsigned row, unsigned col) { - auto res = SqlToYql(req); - - UNIT_ASSERT(res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), TStringBuilder() << "<main>:" << row << ":" << col << ": Warning: Symbol " << symbol << " is not used, code: 4527\n"); -} - -Y_UNIT_TEST_SUITE(WarnUnused) { - Y_UNIT_TEST(ActionOrSubquery) { - TString req = " $a()\n" - "as select 1;\n" - "end define;\n" - "\n" - "select 1;"; - CheckUnused("define action\n" + req, "$a", 2, 3); - CheckUnused("define subquery\n" + req, "$a", 2, 3); - } - - Y_UNIT_TEST(Import) { - TString req = "import lib1 symbols\n" - " $sqr;\n" - "select 1;"; - CheckUnused(req, "$sqr", 2, 3); - - req = "import lib1 symbols\n" - " $sqr as\n" - " $sq;\n" - "select 1;"; - CheckUnused(req, "$sq", 3, 5); - } - - Y_UNIT_TEST(NamedNodeStatement) { - TString req = " $a, $a = AsTuple(1, 2);\n" - "select $a;"; - CheckUnused(req, "$a", 1, 2); - req = "$a, $b = AsTuple(1, 2);\n" - "select $a;"; - CheckUnused(req, "$b", 1, 6); - CheckUnused(" $a = 1; $a = 2; select $a;", "$a", 1, 2); - } - - Y_UNIT_TEST(Declare) { - CheckUnused("declare $a as String;select 1;", "$a", 1, 9); - } - - Y_UNIT_TEST(ActionParams) { - TString req = "define action $a($x, $y) as\n" - " select $x;\n" - "end define;\n" - "\n" - "do $a(1,2);"; - CheckUnused(req, "$y", 1, 22); - } - - Y_UNIT_TEST(SubqueryParams) { - TString req = "use plato;\n" - "define subquery $q($name, $x) as\n" - " select * from $name;\n" - "end define;\n" - "\n" - "select * from $q(\"Input\", 1);"; - CheckUnused(req, "$x", 2, 27); - } - - Y_UNIT_TEST(For) { - TString req = "define action $a() as\n" - " select 1;\n" - "end define;\n" - "\n" - "for $i in ListFromRange(1, 10)\n" - "do $a();"; - CheckUnused(req, "$i", 5, 5); - } - - Y_UNIT_TEST(LambdaParams) { - TString req = "$lambda = ($x, $y) -> ($x);\n" - "select $lambda(1, 2);"; - CheckUnused(req, "$y", 1, 16); - } - - Y_UNIT_TEST(InsideLambdaBody) { - TString req = "$lambda = () -> {\n" - " $x = 1; return 1;\n" - "};\n" - "select $lambda();"; - CheckUnused(req, "$x", 2, 3); - req = "$lambda = () -> {\n" - " $x = 1; $x = 2; return $x;\n" - "};\n" - "select $lambda();"; - CheckUnused(req, "$x", 2, 3); - } - - Y_UNIT_TEST(InsideAction) { - TString req = "define action $a() as\n" - " $x = 1; select 1;\n" - "end define;\n" - "\n" - "do $a();"; - CheckUnused(req, "$x", 2, 3); - req = "define action $a() as\n" - " $x = 1; $x = 2; select $x;\n" - "end define;\n" - "\n" - "do $a();"; - CheckUnused(req, "$x", 2, 3); - } - - Y_UNIT_TEST(NoWarnOnNestedActions) { - auto req = "pragma warning(\"error\", \"4527\");\n" - "define action $action($b) as\n" - " define action $aaa() as\n" - " select $b;\n" - " end define;\n" - " do $aaa();\n" - "end define;\n" - "\n" - "do $action(1);"; - UNIT_ASSERT(SqlToYql(req).IsOk()); - } - - Y_UNIT_TEST(NoWarnForUsageAfterSubquery) { - auto req = "use plato;\n" - "pragma warning(\"error\", \"4527\");\n" - "\n" - "$a = 1;\n" - "\n" - "define subquery $q($table) as\n" - " select * from $table;\n" - "end define;\n" - "\n" - "select * from $q(\"Input\");\n" - "select $a;"; - UNIT_ASSERT(SqlToYql(req).IsOk()); - } -} - -Y_UNIT_TEST_SUITE(AnonymousNames) { - Y_UNIT_TEST(ReferenceAnonymousVariableIsForbidden) { - auto req = "$_ = 1; select $_;"; - - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:16: Error: Unable to reference anonymous name $_\n"); - - req = "$`_` = 1; select $`_`;"; - res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:18: Error: Unable to reference anonymous name $_\n"); - } - - Y_UNIT_TEST(Declare) { - auto req = "declare $_ as String;"; - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:9: Error: Can not use anonymous name '$_' in DECLARE statement\n"); - } - - Y_UNIT_TEST(ActionSubquery) { - auto req = "define action $_() as select 1; end define;"; - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:15: Error: Can not use anonymous name '$_' as ACTION name\n"); - - req = "define subquery $_() as select 1; end define;"; - res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:17: Error: Can not use anonymous name '$_' as SUBQUERY name\n"); - } - - Y_UNIT_TEST(Import) { - auto req = "import lib symbols $sqr as $_;"; - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:28: Error: Can not import anonymous name $_\n"); - } - - Y_UNIT_TEST(Export) { - auto req = "export $_;"; - auto res = SqlToYqlWithMode(req, NSQLTranslation::ESqlMode::LIBRARY); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Error: Can not export anonymous name $_\n"); - } - - Y_UNIT_TEST(AnonymousInActionArgs) { - auto req = "pragma warning(\"error\", \"4527\");\n" - "define action $a($_, $y, $_) as\n" - " select $y;\n" - "end define;\n" - "\n" - "do $a(1,2,3);"; - UNIT_ASSERT(SqlToYql(req).IsOk()); - } - - Y_UNIT_TEST(AnonymousInSubqueryArgs) { - auto req = "use plato;\n" - "pragma warning(\"error\", \"4527\");\n" - "define subquery $q($_, $y, $_) as\n" - " select * from $y;\n" - "end define;\n" - "\n" - "select * from $q(1,\"Input\",3);"; - UNIT_ASSERT(SqlToYql(req).IsOk()); - } - - Y_UNIT_TEST(AnonymousInLambdaArgs) { - auto req = "pragma warning(\"error\", \"4527\");\n" - "$lambda = ($_, $x, $_) -> ($x);\n" - "select $lambda(1,2,3);"; - UNIT_ASSERT(SqlToYql(req).IsOk()); - } - - Y_UNIT_TEST(AnonymousInFor) { - auto req = "pragma warning(\"error\", \"4527\");\n" - "evaluate for $_ in ListFromRange(1, 10) do begin select 1; end do;"; - UNIT_ASSERT(SqlToYql(req).IsOk()); - } - - Y_UNIT_TEST(Assignment) { - auto req = "pragma warning(\"error\", \"4527\");\n" - "$_ = 1;\n" - "$_, $x, $_ = AsTuple(1,2,3);\n" - "select $x;"; - UNIT_ASSERT(SqlToYql(req).IsOk()); - } -} - -Y_UNIT_TEST_SUITE(JsonValue) { - Y_UNIT_TEST(JsonValueArgumentCount) { - NYql::TAstParseResult res = SqlToYql("select JSON_VALUE(CAST(@@{\"key\": 1238}@@ as Json));"); - - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:49: Error: mismatched input ')' expecting ','\n"); - } - - Y_UNIT_TEST(JsonValueJsonPathMustBeLiteralString) { - NYql::TAstParseResult res = SqlToYql("$jsonPath = \"strict $.key\"; select JSON_VALUE(CAST(@@{\"key\": 1238}@@ as Json), $jsonPath);"); - - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:79: Error: mismatched input '$' expecting STRING_VALUE\n"); - } - - Y_UNIT_TEST(JsonValueTranslation) { - NYql::TAstParseResult res = SqlToYql("select JSON_VALUE(CAST(@@{\"key\": 1238}@@ as Json), \"strict $.key\");"); - - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - Y_UNUSED(word); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'\"strict $.key\"")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("SafeCast")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("DataType 'Json")); - }; - - TWordCountHive elementStat({"JsonValue"}); - VerifyProgram(res, elementStat, verifyLine); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["JsonValue"]); - } - - Y_UNIT_TEST(JsonValueReturningSection) { - for (const auto& typeName : {"Bool", "Int64", "Double", "String"}) { - NYql::TAstParseResult res = SqlToYql( - TStringBuilder() << "select JSON_VALUE(CAST(@@{\"key\": 1238}@@ as Json), \"strict $.key\" RETURNING " << typeName << ");" - ); - - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [&](const TString& word, const TString& line) { - Y_UNUSED(word); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'\"strict $.key\"")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("SafeCast")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("DataType 'Json")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(TStringBuilder() << "DataType '" << typeName)); - }; - - TWordCountHive elementStat({typeName}); - VerifyProgram(res, elementStat, verifyLine); - UNIT_ASSERT(elementStat[typeName] > 0); - } - } - - Y_UNIT_TEST(JsonValueInvalidReturningType) { - NYql::TAstParseResult res = SqlToYql("select JSON_VALUE(CAST(@@{'key': 1238}@@ as Json), 'strict $.key' RETURNING invalid);"); - - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:77: Error: Unknown simple type 'invalid'\n"); - } - - Y_UNIT_TEST(JsonValueAndReturningInExpressions) { - NYql::TAstParseResult res = SqlToYql( - "USE plato\n;" - "$json_value = \"some string\";\n" - "SELECT $json_value;\n" - "SELECT 1 as json_value;\n" - "SELECT $json_value as json_value;\n" - "$returning = \"another string\";\n" - "SELECT $returning;\n" - "SELECT 1 as returning;\n" - "SELECT $returning as returning;\n" - ); - - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(JsonValueValidCaseHandlers) { - const TVector<std::pair<TString, TString>> testCases = { - {"", "'DefaultValue (Null)"}, - {"NULL", "'DefaultValue (Null)"}, - {"ERROR", "'Error (Null)"}, - {"DEFAULT 123", "'DefaultValue (Int32 '\"123\")"}, - }; - - for (const auto& onEmpty : testCases) { - for (const auto& onError : testCases) { - TStringBuilder query; - query << "$json = CAST(@@{\"key\": 1238}@@ as Json);\n" - << "SELECT JSON_VALUE($json, \"strict $.key\""; - if (!onEmpty.first.empty()) { - query << " " << onEmpty.first << " ON EMPTY"; - } - if (!onError.first.empty()) { - query << " " << onError.first << " ON ERROR"; - } - query << ");\n"; - - NYql::TAstParseResult res = SqlToYql(query); - - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [&](const TString& word, const TString& line) { - Y_UNUSED(word); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(onEmpty.second + " " + onError.second)); - }; - - TWordCountHive elementStat({"JsonValue"}); - VerifyProgram(res, elementStat, verifyLine); - UNIT_ASSERT(elementStat["JsonValue"] > 0); - } - } - } - - Y_UNIT_TEST(JsonValueTooManyCaseHandlers) { - NYql::TAstParseResult res = SqlToYql( - "select JSON_VALUE(CAST(@@{\"key\": 1238}@@ as Json), \"strict $.key\" NULL ON EMPTY NULL ON ERROR NULL ON EMPTY);\n" - ); - - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF( - Err2Str(res), - "<main>:1:52: Error: Only 1 ON EMPTY and/or 1 ON ERROR clause is expected\n" - ); - } - - Y_UNIT_TEST(JsonValueTooManyOnEmpty) { - NYql::TAstParseResult res = SqlToYql( - "select JSON_VALUE(CAST(@@{\"key\": 1238}@@ as Json), \"strict $.key\" NULL ON EMPTY NULL ON EMPTY);\n" - ); - - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF( - Err2Str(res), - "<main>:1:52: Error: Only 1 ON EMPTY clause is expected\n" - ); - } - - Y_UNIT_TEST(JsonValueTooManyOnError) { - NYql::TAstParseResult res = SqlToYql( - "select JSON_VALUE(CAST(@@{\"key\": 1238}@@ as Json), \"strict $.key\" NULL ON ERROR NULL ON ERROR);\n" - ); - - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF( - Err2Str(res), - "<main>:1:52: Error: Only 1 ON ERROR clause is expected\n" - ); - } - - Y_UNIT_TEST(JsonValueOnEmptyAfterOnError) { - NYql::TAstParseResult res = SqlToYql( - "select JSON_VALUE(CAST(@@{\"key\": 1238}@@ as Json), \"strict $.key\" NULL ON ERROR NULL ON EMPTY);\n" - ); - - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF( - Err2Str(res), - "<main>:1:52: Error: ON EMPTY clause must be before ON ERROR clause\n" - ); - } - - Y_UNIT_TEST(JsonValueNullInput) { - NYql::TAstParseResult res = SqlToYql(R"(SELECT JSON_VALUE(NULL, "strict $.key");)"); - - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [&](const TString& word, const TString& line) { - Y_UNUSED(word); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("(Nothing (OptionalType (DataType 'Json)))")); - }; - - TWordCountHive elementStat({"JsonValue"}); - VerifyProgram(res, elementStat, verifyLine); - UNIT_ASSERT(elementStat["JsonValue"] > 0); - } -} - -Y_UNIT_TEST_SUITE(JsonExists) { - Y_UNIT_TEST(JsonExistsValidHandlers) { - const TVector<std::pair<TString, TString>> testCases = { - {"", "(Just (Bool '\"false\"))"}, - {"TRUE ON ERROR", "(Just (Bool '\"true\"))"}, - {"FALSE ON ERROR", "(Just (Bool '\"false\"))"}, - {"UNKNOWN ON ERROR", "(Nothing (OptionalType (DataType 'Bool)))"}, - // NOTE: in this case we expect arguments of JsonExists callable to end immediately - // after variables. This parenthesis at the end of the expression is left on purpose - {"ERROR ON ERROR", "(Utf8 '\"strict $.key\") (JsonVariables))"}, - }; - - for (const auto& item : testCases) { - NYql::TAstParseResult res = SqlToYql( - TStringBuilder() << R"( - $json = CAST(@@{"key": 1238}@@ as Json); - SELECT JSON_EXISTS($json, "strict $.key" )" << item.first << ");\n" - ); - - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [&](const TString& word, const TString& line) { - Y_UNUSED(word); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(item.second)); - }; - - TWordCountHive elementStat({"JsonExists"}); - VerifyProgram(res, elementStat, verifyLine); - UNIT_ASSERT(elementStat["JsonExists"] > 0); - } - } - - Y_UNIT_TEST(JsonExistsInvalidHandler) { - NYql::TAstParseResult res = SqlToYql(R"( - $json = CAST(@@{"key": 1238}@@ as Json); - $default = false; - SELECT JSON_EXISTS($json, "strict $.key" $default ON ERROR); - )"); - - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:4:53: Error: mismatched input '$' expecting {')', ERROR, FALSE, TRUE, UNKNOWN}\n"); - } - - Y_UNIT_TEST(JsonExistsNullInput) { - NYql::TAstParseResult res = SqlToYql(R"(SELECT JSON_EXISTS(NULL, "strict $.key");)"); - - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [&](const TString& word, const TString& line) { - Y_UNUSED(word); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("(Nothing (OptionalType (DataType 'Json)))")); - }; - - TWordCountHive elementStat({"JsonExists"}); - VerifyProgram(res, elementStat, verifyLine); - UNIT_ASSERT(elementStat["JsonExists"] > 0); - } -} - -Y_UNIT_TEST_SUITE(JsonQuery) { - Y_UNIT_TEST(JsonQueryValidHandlers) { - using TTestSuite = const TVector<std::pair<TString, TString>>; - TTestSuite wrapCases = { - {"", "'NoWrap"}, - {"WITHOUT WRAPPER", "'NoWrap"}, - {"WITHOUT ARRAY WRAPPER", "'NoWrap"}, - {"WITH WRAPPER", "'Wrap"}, - {"WITH ARRAY WRAPPER", "'Wrap"}, - {"WITH UNCONDITIONAL WRAPPER", "'Wrap"}, - {"WITH UNCONDITIONAL ARRAY WRAPPER", "'Wrap"}, - {"WITH CONDITIONAL WRAPPER", "'ConditionalWrap"}, - {"WITH CONDITIONAL ARRAY WRAPPER", "'ConditionalWrap"}, - }; - TTestSuite handlerCases = { - {"", "'Null"}, - {"ERROR", "'Error"}, - {"NULL", "'Null"}, - {"EMPTY ARRAY", "'EmptyArray"}, - {"EMPTY OBJECT", "'EmptyObject"}, - }; - - for (const auto& wrap : wrapCases) { - for (const auto& onError : handlerCases) { - for (const auto& onEmpty : handlerCases) { - TStringBuilder query; - query << R"($json = CAST(@@{"key": [123]}@@ as Json); - SELECT JSON_QUERY($json, "strict $.key" )" << wrap.first; - if (!onEmpty.first.empty()) { - if (wrap.first.StartsWith("WITH ")) { - continue; - } - query << " " << onEmpty.first << " ON EMPTY"; - } - if (!onError.first.empty()) { - query << " " << onError.first << " ON ERROR"; - } - query << ");\n"; - - NYql::TAstParseResult res = SqlToYql(query); - - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [&](const TString& word, const TString& line) { - Y_UNUSED(word); - const TString args = TStringBuilder() << wrap.second << " " << onEmpty.second << " " << onError.second; - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(args)); - }; - - Cout << wrap.first << " " << onEmpty.first << " " << onError.first << Endl; - - TWordCountHive elementStat({"JsonQuery"}); - VerifyProgram(res, elementStat, verifyLine); - UNIT_ASSERT(elementStat["JsonQuery"] > 0); - } - } - } - } - - Y_UNIT_TEST(JsonQueryOnEmptyWithWrapper) { - NYql::TAstParseResult res = SqlToYql(R"( - $json = CAST(@@{"key": 1238}@@ as Json); - SELECT JSON_QUERY($json, "strict $" WITH ARRAY WRAPPER EMPTY ARRAY ON EMPTY); - )"); - - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:38: Error: ON EMPTY is prohibited because WRAPPER clause is specified\n"); - } - - Y_UNIT_TEST(JsonQueryNullInput) { - NYql::TAstParseResult res = SqlToYql(R"(SELECT JSON_QUERY(NULL, "strict $.key");)"); - - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [&](const TString& word, const TString& line) { - Y_UNUSED(word); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("(Nothing (OptionalType (DataType 'Json)))")); - }; - - TWordCountHive elementStat({"JsonQuery"}); - VerifyProgram(res, elementStat, verifyLine); - UNIT_ASSERT(elementStat["JsonQuery"] > 0); - } -} - -Y_UNIT_TEST_SUITE(JsonPassing) { - Y_UNIT_TEST(SupportedVariableTypes) { - const TVector<TString> functions = {"JSON_EXISTS", "JSON_VALUE", "JSON_QUERY"}; - - for (const auto& function : functions) { - const auto query = Sprintf(R"( - pragma CompactNamedExprs; - $json = CAST(@@{"key": 1238}@@ as Json); - SELECT %s( - $json, - "strict $.key" - PASSING - "string" as var1, - 1.234 as var2, - CAST(1 as Int64) as var3, - true as var4, - $json as var5 - ))", - function.data() - ); - NYql::TAstParseResult res = SqlToYql(query); - - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [&](const TString& word, const TString& line) { - Y_UNUSED(word); - UNIT_ASSERT_VALUES_UNEQUAL_C(TString::npos, line.find(R"('('"var1" (String '"string")))"), "Cannot find `var1`"); - UNIT_ASSERT_VALUES_UNEQUAL_C(TString::npos, line.find(R"('('"var2" (Double '"1.234")))"), "Cannot find `var2`"); - UNIT_ASSERT_VALUES_UNEQUAL_C(TString::npos, line.find(R"('('"var3" (SafeCast (Int32 '"1") (DataType 'Int64))))"), "Cannot find `var3`"); - UNIT_ASSERT_VALUES_UNEQUAL_C(TString::npos, line.find(R"('('"var4" (Bool '"true")))"), "Cannot find `var4`"); - UNIT_ASSERT_VALUES_UNEQUAL_C(TString::npos, line.find(R"('('"var5" namedexprnode0))"), "Cannot find `var5`"); - }; - - TWordCountHive elementStat({"JsonVariables"}); - VerifyProgram(res, elementStat, verifyLine); - UNIT_ASSERT(elementStat["JsonVariables"] > 0); - } - } - - Y_UNIT_TEST(ValidVariableNames) { - const TVector<TString> functions = {"JSON_EXISTS", "JSON_VALUE", "JSON_QUERY"}; - - for (const auto& function : functions) { - const auto query = Sprintf(R"( - $json = CAST(@@{"key": 1238}@@ as Json); - SELECT %s( - $json, - "strict $.key" - PASSING - "one" as var1, - "two" as "VaR2", - "three" as `var3`, - "four" as VaR4 - ))", - function.data() - ); - NYql::TAstParseResult res = SqlToYql(query); - - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [&](const TString& word, const TString& line) { - Y_UNUSED(word); - UNIT_ASSERT_VALUES_UNEQUAL_C(TString::npos, line.find(R"('('"var1" (String '"one")))"), "Cannot find `var1`"); - UNIT_ASSERT_VALUES_UNEQUAL_C(TString::npos, line.find(R"('('"VaR2" (String '"two")))"), "Cannot find `VaR2`"); - UNIT_ASSERT_VALUES_UNEQUAL_C(TString::npos, line.find(R"('('"var3" (String '"three")))"), "Cannot find `var3`"); - UNIT_ASSERT_VALUES_UNEQUAL_C(TString::npos, line.find(R"('('"VaR4" (String '"four")))"), "Cannot find `VaR4`"); - }; - - TWordCountHive elementStat({"JsonVariables"}); - VerifyProgram(res, elementStat, verifyLine); - UNIT_ASSERT(elementStat["JsonVariables"] > 0); - } - } -} - -Y_UNIT_TEST_SUITE(MigrationToJsonApi) { - Y_UNIT_TEST(WarningOnDeprecatedJsonUdf) { - NYql::TAstParseResult res = SqlToYql(R"( - $json = CAST(@@{"key": 1234}@@ as Json); - SELECT Json::Parse($json); - )"); - - UNIT_ASSERT(res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:26: Warning: Json UDF is deprecated. Please use JSON API instead, code: 4506\n"); - } -} - -Y_UNIT_TEST_SUITE(AnsiIdentsNegative) { - Y_UNIT_TEST(EnableAnsiLexerFromRequestSpecialComments) { - auto req = "\n" - "\t --!ansi_lexer \n" - "-- Some comment\n" - "-- another comment\n" - "pragma SimpleColumns;\n" - "\n" - "select 1, '''' as empty;"; - - auto res = SqlToYql(req); - UNIT_ASSERT(res.IsOk()); - UNIT_ASSERT(res.Issues.Size() == 0); - } - - Y_UNIT_TEST(AnsiLexerShouldNotBeEnabledHere) { - auto req = "$str = '\n" - "--!ansi_lexer\n" - "--!syntax_v1\n" - "';\n" - "\n" - "select 1, $str, \"\" as empty;"; - auto res = SqlToYql(req); - UNIT_ASSERT(res.IsOk()); - UNIT_ASSERT(res.Issues.Size() == 0); - } - - Y_UNIT_TEST(DoubleQuotesInDictsTuplesOrLists) { - auto req = "$d = { 'a': 1, \"b\": 2, 'c': 3,};"; - - auto res = SqlToYqlWithAnsiLexer(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:16: Error: Column reference \"b\" is not allowed in current scope\n"); - - req = "$t = (1, 2, \"a\");"; - - res = SqlToYqlWithAnsiLexer(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:13: Error: Column reference \"a\" is not allowed in current scope\n"); - - req = "$l = ['a', 'b', \"c\"];"; - - res = SqlToYqlWithAnsiLexer(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:17: Error: Column reference \"c\" is not allowed in current scope\n"); - } - - Y_UNIT_TEST(MultilineComments) { - auto req = "/*/**/ select 1;"; - auto res = SqlToYql(req); - UNIT_ASSERT(res.Root); - res = SqlToYqlWithAnsiLexer(req); - UNIT_ASSERT(res.IsOk()); - UNIT_ASSERT(res.Issues.Size() == 0); - - req = "/*\n" - "--/*\n" - "*/ select 1;"; - res = SqlToYql(req); - UNIT_ASSERT(res.Root); - res = SqlToYqlWithAnsiLexer(req); - UNIT_ASSERT(res.IsOk()); - UNIT_ASSERT(res.Issues.Size() == 0); - - req = "/*\n" - "/*\n" - "--*/\n" - "*/ select 1;"; - res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:4:0: Error: mismatched input '*' expecting {';', '(', '$', ALTER, ANALYZE, BACKUP, BATCH, COMMIT, CREATE, DECLARE, DEFINE, DELETE, DISCARD, DO, DROP, EVALUATE, EXPLAIN, EXPORT, FOR, FROM, GRANT, IF, IMPORT, INSERT, PARALLEL, PRAGMA, PROCESS, REDUCE, REPLACE, RESTORE, REVOKE, ROLLBACK, SELECT, SHOW, UPDATE, UPSERT, USE, VALUES}\n"); - res = SqlToYqlWithAnsiLexer(req); - UNIT_ASSERT(res.Root); - } -} - -Y_UNIT_TEST_SUITE(AnsiOptionalAs) { - Y_UNIT_TEST(OptionalAsInProjection) { - UNIT_ASSERT(SqlToYql("PRAGMA AnsiOptionalAs; SELECT a b, c FROM plato.Input;").IsOk()); - ExpectFailWithError("PRAGMA DisableAnsiOptionalAs;\n" - "SELECT a b, c FROM plato.Input;", - "<main>:2:10: Error: Expecting mandatory AS here. Did you miss comma? Please add PRAGMA AnsiOptionalAs; for ANSI compatibility\n"); - } - - Y_UNIT_TEST(OptionalAsWithKeywords) { - UNIT_ASSERT(SqlToYql("PRAGMA AnsiOptionalAs; SELECT a type, b data, c source FROM plato.Input;").IsOk()); - } -} - -Y_UNIT_TEST_SUITE(SessionWindowNegative) { - Y_UNIT_TEST(SessionWindowWithoutSource) { - ExpectFailWithError("SELECT 1 + SessionWindow(ts, 32);", - "<main>:1:12: Error: SessionWindow requires data source\n"); - } - - Y_UNIT_TEST(SessionWindowInProjection) { - ExpectFailWithError("SELECT 1 + SessionWindow(ts, 32) from plato.Input;", - "<main>:1:12: Error: SessionWindow can only be used as a top-level GROUP BY / PARTITION BY expression\n"); - } - - Y_UNIT_TEST(SessionWindowWithNonConstSecondArg) { - ExpectFailWithError( - "SELECT key, session_start FROM plato.Input\n" - "GROUP BY SessionWindow(ts, 32 + subkey) as session_start, key;", - - "<main>:2:10: Error: Source does not allow column references\n" - "<main>:2:33: Error: Column reference 'subkey'\n"); - } - - Y_UNIT_TEST(SessionWindowWithWrongNumberOfArgs) { - ExpectFailWithError("SELECT * FROM plato.Input GROUP BY SessionWindow()", - "<main>:1:36: Error: SessionWindow requires either two or four arguments\n"); - ExpectFailWithError("SELECT * FROM plato.Input GROUP BY SessionWindow(key, subkey, 100)", - "<main>:1:36: Error: SessionWindow requires either two or four arguments\n"); - } - - Y_UNIT_TEST(DuplicateSessionWindow) { - ExpectFailWithError( - "SELECT\n" - " *\n" - "FROM plato.Input\n" - "GROUP BY\n" - " SessionWindow(ts, 10),\n" - " user,\n" - " SessionWindow(ts, 20)\n" - ";", - - "<main>:7:5: Error: Duplicate session window specification:\n" - "<main>:5:5: Error: Previous session window is declared here\n"); - - ExpectFailWithError( - "SELECT\n" - " MIN(key) over w\n" - "FROM plato.Input\n" - "WINDOW w AS (\n" - " PARTITION BY SessionWindow(ts, 10), user,\n" - " SessionWindow(ts, 20)\n" - ");", - - "<main>:6:5: Error: Duplicate session window specification:\n" - "<main>:5:18: Error: Previous session window is declared here\n"); - } - - Y_UNIT_TEST(SessionStartStateWithoutSource) { - ExpectFailWithError("SELECT 1 + SessionStart();", - "<main>:1:12: Error: SessionStart requires data source\n"); - ExpectFailWithError("SELECT 1 + SessionState();", - "<main>:1:12: Error: SessionState requires data source\n"); - } - - Y_UNIT_TEST(SessionStartStateWithoutGroupByOrWindow) { - ExpectFailWithError("SELECT 1 + SessionStart() from plato.Input;", - "<main>:1:12: Error: SessionStart can not be used without aggregation by SessionWindow\n"); - ExpectFailWithError("SELECT 1 + SessionState() from plato.Input;", - "<main>:1:12: Error: SessionState can not be used without aggregation by SessionWindow\n"); - } - - Y_UNIT_TEST(SessionStartStateWithGroupByWithoutSession) { - ExpectFailWithError("SELECT 1 + SessionStart() from plato.Input group by user;", - "<main>:1:12: Error: SessionStart can not be used here: SessionWindow specification is missing in GROUP BY\n"); - ExpectFailWithError("SELECT 1 + SessionState() from plato.Input group by user;", - "<main>:1:12: Error: SessionState can not be used here: SessionWindow specification is missing in GROUP BY\n"); - } - - Y_UNIT_TEST(SessionStartStateWithoutOverWithWindowWithoutSession) { - ExpectFailWithError("SELECT 1 + SessionStart(), MIN(key) over w from plato.Input window w as ()", - "<main>:1:12: Error: SessionStart can not be used without aggregation by SessionWindow. Maybe you forgot to add OVER `window_name`?\n"); - ExpectFailWithError("SELECT 1 + SessionState(), MIN(key) over w from plato.Input window w as ()", - "<main>:1:12: Error: SessionState can not be used without aggregation by SessionWindow. Maybe you forgot to add OVER `window_name`?\n"); - } - - Y_UNIT_TEST(SessionStartStateWithWindowWithoutSession) { - ExpectFailWithError("SELECT 1 + SessionStart() over w, MIN(key) over w from plato.Input window w as ()", - "<main>:1:12: Error: SessionStart can not be used with window w: SessionWindow specification is missing in PARTITION BY\n"); - ExpectFailWithError("SELECT 1 + SessionState() over w, MIN(key) over w from plato.Input window w as ()", - "<main>:1:12: Error: SessionState can not be used with window w: SessionWindow specification is missing in PARTITION BY\n"); - } - - Y_UNIT_TEST(SessionStartStateWithSessionedWindow) { - ExpectFailWithError("SELECT 1 + SessionStart(), MIN(key) over w from plato.Input group by key window w as (partition by SessionWindow(ts, 1)) ", - "<main>:1:12: Error: SessionStart can not be used here: SessionWindow specification is missing in GROUP BY. Maybe you forgot to add OVER `window_name`?\n"); - ExpectFailWithError("SELECT 1 + SessionState(), MIN(key) over w from plato.Input group by key window w as (partition by SessionWindow(ts, 1)) ", - "<main>:1:12: Error: SessionState can not be used here: SessionWindow specification is missing in GROUP BY. Maybe you forgot to add OVER `window_name`?\n"); - } - - Y_UNIT_TEST(AggregationBySessionStateIsNotSupportedYet) { - ExpectFailWithError("SELECT SOME(1 + SessionState()), key from plato.Input group by key, SessionWindow(ts, 1);", - "<main>:1:17: Error: SessionState with GROUP BY is not supported yet\n"); - } - - Y_UNIT_TEST(SessionWindowInRtmr) { - NYql::TAstParseResult res = SqlToYql( - "SELECT * FROM plato.Input GROUP BY SessionWindow(ts, 10);", - 10, TString(NYql::RtmrProviderName)); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:54: Error: Streaming group by query must have a hopping window specification.\n"); - - res = SqlToYql(R"( - SELECT key, SUM(value) AS value FROM plato.Input - GROUP BY key, HOP(subkey, "PT10S", "PT30S", "PT20S"), SessionWindow(ts, 10); - )", 10, TString(NYql::RtmrProviderName)); - - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:2:13: Error: SessionWindow is unsupported for streaming sources\n"); - } -} - -Y_UNIT_TEST_SUITE(LibraSqlSugar) { - auto makeResult = [](TStringBuf settings) { - return SqlToYql( - TStringBuilder() - << settings - << "\n$udf1 = MyLibra::MakeLibraPreprocessor($settings);" - << "\n$udf2 = CustomLibra::MakeLibraPreprocessor($settings);" - << "\nPROCESS plato.Input USING $udf1(TableRow())" - << "\nUNION ALL" - << "\nPROCESS plato.Input USING $udf2(TableRow());" - ); - }; - - Y_UNIT_TEST(EmptySettings) { - auto res = makeResult(R"( - $settings = AsStruct(); - )"); - UNIT_ASSERT(res.IsOk()); - } - - Y_UNIT_TEST(OnlyEntities) { - auto res = makeResult(R"( - $settings = AsStruct( - AsList("A", "B", "C") AS Entities - ); - )"); - UNIT_ASSERT(res.IsOk()); - } - - Y_UNIT_TEST(EntitiesWithStrategy) { - auto res = makeResult(R"( - $settings = AsStruct( - AsList("A", "B", "C") AS Entities, - "blacklist" AS EntitiesStrategy - ); - )"); - UNIT_ASSERT(res.IsOk()); - } - - Y_UNIT_TEST(AllSettings) { - auto res = makeResult(R"( - $settings = AsStruct( - AsList("A", "B", "C") AS Entities, - "whitelist" AS EntitiesStrategy, - "path" AS BlockstatDict, - false AS ParseWithFat, - "map" AS Mode - ); - )"); - UNIT_ASSERT(res.IsOk()); - } - - Y_UNIT_TEST(BadStrategy) { - auto res = makeResult(R"( - $settings = AsStruct("bad" AS EntitiesStrategy); - )"); - UNIT_ASSERT_STRING_CONTAINS( - Err2Str(res), - "Error: MakeLibraPreprocessor got invalid entities strategy: expected 'whitelist' or 'blacklist'" - ); - } - - Y_UNIT_TEST(BadEntities) { - auto res = makeResult(R"( - $settings = AsStruct(AsList("A", 1) AS Entities); - )"); - UNIT_ASSERT_STRING_CONTAINS(Err2Str(res), "Error: MakeLibraPreprocessor entity must be string literal"); - } -} - -Y_UNIT_TEST_SUITE(TrailingQuestionsNegative) { - Y_UNIT_TEST(Basic) { - ExpectFailWithError("SELECT 1?;", "<main>:1:9: Error: Unexpected token '?' at the end of expression\n"); - ExpectFailWithError("SELECT 1? + 1;", "<main>:1:10: Error: mismatched input '+' expecting {<EOF>, ';'}\n"); - ExpectFailWithError("SELECT 1 + 1??? < 2", "<main>:1:13: Error: Unexpected token '?' at the end of expression\n"); - ExpectFailWithError("SELECT 1? > 2? > 3?", - "<main>:1:11: Error: Unexpected token '?' at the end of expression\n" - "<main>:1:16: Error: Unexpected token '?' at the end of expression\n" - "<main>:1:21: Error: Unexpected token '?' at the end of expression\n"); - } - - Y_UNIT_TEST(SmartParen) { - ExpectFailWithError("$x = 1; SELECT (Int32?, $x?)", "<main>:1:27: Error: Unexpected token '?' at the end of expression\n"); - ExpectFailWithError("SELECT (Int32, foo?)", "<main>:1:19: Error: Unexpected token '?' at the end of expression\n"); - } - - Y_UNIT_TEST(LambdaOptArgs) { - ExpectFailWithError("$l = ($x, $y?, $z??, $t?) -> ($x);", "<main>:1:18: Error: Expecting at most one '?' token here (for optional lambda parameters), but got 2\n"); - } -} - -Y_UNIT_TEST_SUITE(FlexibleTypes) { - Y_UNIT_TEST(AssumeOrderByType) { - UNIT_ASSERT(SqlToYql("PRAGMA FlexibleTypes; SELECT 1 AS int32 ASSUME ORDER BY int32").IsOk()); - } - - Y_UNIT_TEST(GroupingSets) { - UNIT_ASSERT(SqlToYql("PRAGMA FlexibleTypes; SELECT COUNT(*) AS cnt, text, uuid FROM plato.Input GROUP BY GROUPING SETS((uuid), (uuid, text));").IsOk()); - } - - Y_UNIT_TEST(WeakField) { - UNIT_ASSERT(SqlToYql("PRAGMA FlexibleTypes; SELECT WeakField(text, string) as text FROM plato.Input").IsOk()); - } - - Y_UNIT_TEST(Aggregation1) { - TString q = - "PRAGMA FlexibleTypes;\n" - "$foo = ($x, $const, $type) -> ($x || $const || FormatType($type));\n" - "SELECT $foo(SOME(x), 'aaa', String) FROM plato.Input GROUP BY y;"; - UNIT_ASSERT(SqlToYql(q).IsOk()); - } - - Y_UNIT_TEST(Aggregation2) { - TString q = - "PRAGMA FlexibleTypes;\n" - "SELECT 1 + String + MAX(key) FROM plato.Input;"; - UNIT_ASSERT(SqlToYql(q).IsOk()); - } -} - -Y_UNIT_TEST_SUITE(ExternalDeclares) { - Y_UNIT_TEST(BasicUsage) { - NSQLTranslation::TTranslationSettings settings; - settings.DeclaredNamedExprs["foo"] = "String"; - auto res = SqlToYqlWithSettings("select $foo;", settings); - UNIT_ASSERT(res.IsOk()); - UNIT_ASSERT(res.Issues.Size() == 0); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "declare") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(R"__((declare "$foo" (DataType 'String)))__")); - } - }; - - TWordCountHive elementStat = {{TString("declare"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["declare"]); - } - - Y_UNIT_TEST(DeclareOverrides) { - NSQLTranslation::TTranslationSettings settings; - settings.DeclaredNamedExprs["foo"] = "String"; - auto res = SqlToYqlWithSettings("declare $foo as Int32; select $foo;", settings); - UNIT_ASSERT(res.IsOk()); - UNIT_ASSERT(res.Issues.Size() == 0); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "declare") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(R"__((declare "$foo" (DataType 'Int32)))__")); - } - }; - - TWordCountHive elementStat = {{TString("declare"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["declare"]); - } - - Y_UNIT_TEST(UnusedDeclareDoesNotProduceWarning) { - NSQLTranslation::TTranslationSettings settings; - settings.DeclaredNamedExprs["foo"] = "String"; - auto res = SqlToYqlWithSettings("select 1;", settings); - UNIT_ASSERT(res.IsOk()); - UNIT_ASSERT(res.Issues.Size() == 0); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "declare") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(R"__((declare "$foo" (DataType 'String)))__")); - } - }; - - TWordCountHive elementStat = {{TString("declare"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["declare"]); - } - - Y_UNIT_TEST(DeclaresWithInvalidTypesFails) { - NSQLTranslation::TTranslationSettings settings; - settings.DeclaredNamedExprs["foo"] = "List<BadType>"; - auto res = SqlToYqlWithSettings("select 1;", settings); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), - "<main>:0:5: Error: Unknown type: 'BadType'\n" - "<main>: Error: Failed to parse type for externally declared name 'foo'\n"); - } -} - -Y_UNIT_TEST_SUITE(ExternalDataSource) { - Y_UNIT_TEST(CreateExternalDataSourceWithAuthNone) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - CREATE EXTERNAL DATA SOURCE MyDataSource WITH ( - SOURCE_TYPE="ObjectStorage", - LOCATION="my-bucket", - AUTH_METHOD="NONE" - ); - )sql"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_STRING_CONTAINS(line, R"#('('('"auth_method" '"NONE") '('"location" '"my-bucket") '('"source_type" '"ObjectStorage"))#"); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("createObject")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(CreateExternalDataSourceWithAuthServiceAccount) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - CREATE EXTERNAL DATA SOURCE MyDataSource WITH ( - SOURCE_TYPE="ObjectStorage", - LOCATION="my-bucket", - AUTH_METHOD="SERVICE_ACCOUNT", - SERVICE_ACCOUNT_ID="sa", - SERVICE_ACCOUNT_SECRET_NAME="sa_secret_name" - ); - )sql"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_STRING_CONTAINS(line, R"#('('('"auth_method" '"SERVICE_ACCOUNT") '('"location" '"my-bucket") '('"service_account_id" '"sa") '('"service_account_secret_name" '"sa_secret_name") '('"source_type" '"ObjectStorage"))#"); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("createObject")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(CreateExternalDataSourceWithBasic) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - CREATE EXTERNAL DATA SOURCE MyDataSource WITH ( - SOURCE_TYPE="PostgreSQL", - LOCATION="protocol://host:port/", - AUTH_METHOD="BASIC", - LOGIN="admin", - PASSWORD_SECRET_NAME="secret_name" - ); - )sql"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_STRING_CONTAINS(line, R"#('('('"auth_method" '"BASIC") '('"location" '"protocol://host:port/") '('"login" '"admin") '('"password_secret_name" '"secret_name") '('"source_type" '"PostgreSQL"))#"); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("createObject")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(CreateExternalDataSourceWithMdbBasic) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - CREATE EXTERNAL DATA SOURCE MyDataSource WITH ( - SOURCE_TYPE="PostgreSQL", - LOCATION="protocol://host:port/", - AUTH_METHOD="MDB_BASIC", - SERVICE_ACCOUNT_ID="sa", - SERVICE_ACCOUNT_SECRET_NAME="sa_secret_name", - LOGIN="admin", - PASSWORD_SECRET_NAME="secret_name" - ); - )sql"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_STRING_CONTAINS(line, R"#('('('"auth_method" '"MDB_BASIC") '('"location" '"protocol://host:port/") '('"login" '"admin") '('"password_secret_name" '"secret_name") '('"service_account_id" '"sa") '('"service_account_secret_name" '"sa_secret_name") '('"source_type" '"PostgreSQL"))#"); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("createObject")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(CreateExternalDataSourceWithAws) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - CREATE EXTERNAL DATA SOURCE MyDataSource WITH ( - SOURCE_TYPE="PostgreSQL", - LOCATION="protocol://host:port/", - AUTH_METHOD="AWS", - AWS_ACCESS_KEY_ID_SECRET_NAME="secred_id_name", - AWS_SECRET_ACCESS_KEY_SECRET_NAME="secret_key_name", - AWS_REGION="ru-central-1" - ); - )sql"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_STRING_CONTAINS(line, R"#('('('"auth_method" '"AWS") '('"aws_access_key_id_secret_name" '"secred_id_name") '('"aws_region" '"ru-central-1") '('"aws_secret_access_key_secret_name" '"secret_key_name") '('"location" '"protocol://host:port/") '('"source_type" '"PostgreSQL"))#"); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("createObject")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(CreateExternalDataSourceWithToken) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - CREATE EXTERNAL DATA SOURCE MyDataSource WITH ( - SOURCE_TYPE="YT", - LOCATION="protocol://host:port/", - AUTH_METHOD="TOKEN", - TOKEN_SECRET_NAME="token_name" - ); - )sql"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_STRING_CONTAINS(line, R"#('('('"auth_method" '"TOKEN") '('"location" '"protocol://host:port/") '('"source_type" '"YT") '('"token_secret_name" '"token_name"))#"); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("createObject")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(CreateExternalDataSourceWithTablePrefix) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - pragma TablePathPrefix='/aba'; - CREATE EXTERNAL DATA SOURCE MyDataSource WITH ( - SOURCE_TYPE="ObjectStorage", - LOCATION="my-bucket", - AUTH_METHOD="NONE" - ); - )sql"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_STRING_CONTAINS(line, "/aba/MyDataSource"); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("createObject")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(CreateExternalDataSourceIfNotExists) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - CREATE EXTERNAL DATA SOURCE IF NOT EXISTS MyDataSource WITH ( - SOURCE_TYPE="ObjectStorage", - LOCATION="my-bucket", - AUTH_METHOD="NONE" - ); - )sql"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_STRING_CONTAINS(line, R"#('('('"auth_method" '"NONE") '('"location" '"my-bucket") '('"source_type" '"ObjectStorage"))#"); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("createObjectIfNotExists")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(AlterExternalDataSource) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - ALTER EXTERNAL DATA SOURCE MyDataSource - SET (SOURCE_TYPE = "ObjectStorage", Login = "Admin"), - SET Location "bucket", - RESET (Auth_Method, Service_Account_Id, Service_Account_Secret_Name); - )sql"); - UNIT_ASSERT_C(res.Root, res.Issues.ToString()); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_STRING_CONTAINS(line, R"#(('mode 'alterObject))#"); - UNIT_ASSERT_STRING_CONTAINS(line, R"#('('features '('('"location" '"bucket") '('"login" '"Admin") '('"source_type" '"ObjectStorage"))))#"); - UNIT_ASSERT_STRING_CONTAINS(line, R"#('('resetFeatures '('"auth_method" '"service_account_id" '"service_account_secret_name")))#"); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(CreateExternalDataSourceOrReplace) { - NYql::TAstParseResult res = SqlToYql(R"( - USE plato; - CREATE OR REPLACE EXTERNAL DATA SOURCE MyDataSource WITH ( - SOURCE_TYPE="ObjectStorage", - LOCATION="my-bucket", - AUTH_METHOD="NONE" - ); - )"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_STRING_CONTAINS(line, R"#('('('"auth_method" '"NONE") '('"location" '"my-bucket") '('"source_type" '"ObjectStorage"))#"); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("createObjectOrReplace")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(CreateOrReplaceForUnsupportedTableTypesShouldFail) { - ExpectFailWithError(R"sql( - USE plato; - CREATE OR REPLACE TABLE t (a int32 not null, primary key(a, a)); - )sql" , "<main>:3:23: Error: OR REPLACE feature is supported only for EXTERNAL DATA SOURCE and EXTERNAL TABLE\n"); - - ExpectFailWithError(R"sql( - USE plato; - CREATE OR REPLACE TABLE t ( - Key Uint64, - Value1 String, - PRIMARY KEY (Key) - ) - WITH ( - STORE = COLUMN, - AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = 10 - ); - )sql" , "<main>:3:23: Error: OR REPLACE feature is supported only for EXTERNAL DATA SOURCE and EXTERNAL TABLE\n"); - } - - Y_UNIT_TEST(CreateExternalDataSourceWithBadArguments) { - ExpectFailWithError(R"sql( - USE plato; - CREATE EXTERNAL DATA SOURCE MyDataSource; - )sql" , "<main>:3:56: Error: mismatched input ';' expecting WITH\n"); - - ExpectFailWithError(R"sql( - USE plato; - CREATE EXTERNAL DATA SOURCE MyDataSource WITH ( - LOCATION="my-bucket", - AUTH_METHOD="NONE" - ); - )sql" , "<main>:5:33: Error: SOURCE_TYPE requires key\n"); - - ExpectFailWithError(R"sql( - USE plato; - CREATE EXTERNAL DATA SOURCE MyDataSource WITH ( - SOURCE_TYPE="ObjectStorage", - LOCATION="my-bucket" - ); - )sql" , "<main>:5:30: Error: AUTH_METHOD requires key\n"); - - - ExpectFailWithError(R"sql( - USE plato; - CREATE EXTERNAL DATA SOURCE MyDataSource WITH ( - SOURCE_TYPE="ObjectStorage", - LOCATION="my-bucket", - AUTH_METHOD="NONE1" - ); - )sql" , "<main>:6:33: Error: Unknown AUTH_METHOD = NONE1\n"); - - ExpectFailWithError(R"sql( - USE plato; - CREATE EXTERNAL DATA SOURCE MyDataSource WITH ( - SOURCE_TYPE="ObjectStorage", - LOCATION="my-bucket", - AUTH_METHOD="SERVICE_ACCOUNT" - ); - )sql" , "<main>:6:33: Error: SERVICE_ACCOUNT_ID requires key\n"); - - ExpectFailWithError(R"sql( - USE plato; - CREATE EXTERNAL DATA SOURCE MyDataSource WITH ( - SOURCE_TYPE="ObjectStorage", - LOCATION="my-bucket", - AUTH_METHOD="SERVICE_ACCOUNT", - SERVICE_ACCOUNT_ID="s1" - ); - )sql" , "<main>:7:40: Error: SERVICE_ACCOUNT_SECRET_NAME requires key\n"); - - ExpectFailWithError(R"sql( - USE plato; - CREATE EXTERNAL DATA SOURCE MyDataSource WITH ( - SOURCE_TYPE="ObjectStorage", - LOCATION="my-bucket", - AUTH_METHOD="SERVICE_ACCOUNT", - SERVICE_ACCOUNT_SECRET_NAME="s1" - ); - )sql" , "<main>:7:49: Error: SERVICE_ACCOUNT_ID requires key\n"); - - ExpectFailWithError(R"sql( - USE plato; - CREATE EXTERNAL DATA SOURCE MyDataSource WITH ( - SOURCE_TYPE="PostgreSQL", - LOCATION="protocol://host:port/", - AUTH_METHOD="BASIC", - LOGIN="admin" - ); - )sql" , "<main>:7:27: Error: PASSWORD_SECRET_NAME requires key\n"); - - ExpectFailWithError(R"sql( - USE plato; - CREATE EXTERNAL DATA SOURCE MyDataSource WITH ( - SOURCE_TYPE="PostgreSQL", - LOCATION="protocol://host:port/", - AUTH_METHOD="BASIC", - PASSWORD_SECRET_NAME="secret_name" - ); - )sql" , "<main>:7:42: Error: LOGIN requires key\n"); - - ExpectFailWithError(R"sql( - USE plato; - CREATE EXTERNAL DATA SOURCE MyDataSource WITH ( - SOURCE_TYPE="PostgreSQL", - LOCATION="protocol://host:port/", - AUTH_METHOD="MDB_BASIC", - SERVICE_ACCOUNT_SECRET_NAME="sa_secret_name", - LOGIN="admin", - PASSWORD_SECRET_NAME="secret_name" - ); - )sql" , "<main>:9:42: Error: SERVICE_ACCOUNT_ID requires key\n"); - - ExpectFailWithError(R"sql( - USE plato; - CREATE EXTERNAL DATA SOURCE MyDataSource WITH ( - SOURCE_TYPE="PostgreSQL", - LOCATION="protocol://host:port/", - AUTH_METHOD="MDB_BASIC", - SERVICE_ACCOUNT_ID="sa", - LOGIN="admin", - PASSWORD_SECRET_NAME="secret_name" - ); - )sql" , "<main>:9:42: Error: SERVICE_ACCOUNT_SECRET_NAME requires key\n"); - - ExpectFailWithError(R"sql( - USE plato; - CREATE EXTERNAL DATA SOURCE MyDataSource WITH ( - SOURCE_TYPE="PostgreSQL", - LOCATION="protocol://host:port/", - AUTH_METHOD="MDB_BASIC", - SERVICE_ACCOUNT_ID="sa", - SERVICE_ACCOUNT_SECRET_NAME="sa_secret_name", - PASSWORD_SECRET_NAME="secret_name" - ); - )sql" , "<main>:9:42: Error: LOGIN requires key\n"); - - ExpectFailWithError(R"sql( - USE plato; - CREATE EXTERNAL DATA SOURCE MyDataSource WITH ( - SOURCE_TYPE="PostgreSQL", - LOCATION="protocol://host:port/", - AUTH_METHOD="MDB_BASIC", - SERVICE_ACCOUNT_ID="sa", - SERVICE_ACCOUNT_SECRET_NAME="sa_secret_name", - LOGIN="admin" - ); - )sql" , "<main>:9:27: Error: PASSWORD_SECRET_NAME requires key\n"); - - ExpectFailWithError(R"sql( - USE plato; - CREATE EXTERNAL DATA SOURCE MyDataSource WITH ( - SOURCE_TYPE="PostgreSQL", - LOCATION="protocol://host:port/", - AUTH_METHOD="AWS", - AWS_SECRET_ACCESS_KEY_SECRET_NAME="secret_key_name", - AWS_REGION="ru-central-1" - ); - )sql" , "<main>:8:32: Error: AWS_ACCESS_KEY_ID_SECRET_NAME requires key\n"); - - ExpectFailWithError(R"sql( - USE plato; - CREATE EXTERNAL DATA SOURCE MyDataSource WITH ( - SOURCE_TYPE="PostgreSQL", - LOCATION="protocol://host:port/", - AUTH_METHOD="AWS", - AWS_ACCESS_KEY_ID_SECRET_NAME="secred_id_name", - AWS_REGION="ru-central-1" - ); - )sql" , "<main>:8:32: Error: AWS_SECRET_ACCESS_KEY_SECRET_NAME requires key\n"); - - ExpectFailWithError(R"sql( - USE plato; - CREATE EXTERNAL DATA SOURCE MyDataSource WITH ( - SOURCE_TYPE="PostgreSQL", - LOCATION="protocol://host:port/", - AUTH_METHOD="AWS", - AWS_SECRET_ACCESS_KEY_SECRET_NAME="secret_key_name", - AWS_ACCESS_KEY_ID_SECRET_NAME="secred_id_name" - ); - )sql" , "<main>:8:51: Error: AWS_REGION requires key\n"); - } - - Y_UNIT_TEST(DropExternalDataSourceWithTablePrefix) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - DROP EXTERNAL DATA SOURCE MyDataSource; - )sql"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("'features")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("dropObject")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(DropExternalDataSource) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - pragma TablePathPrefix='/aba'; - DROP EXTERNAL DATA SOURCE MyDataSource; - )sql"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_STRING_CONTAINS(line, "/aba/MyDataSource"); - UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("'features")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("dropObject")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(DropExternalDataSourceIfExists) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - DROP EXTERNAL DATA SOURCE IF EXISTS MyDataSource; - )sql"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_STRING_CONTAINS(line, "MyDataSource"); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("dropObjectIfExists")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } -} - -Y_UNIT_TEST_SUITE(ExternalTable) { - Y_UNIT_TEST(CreateExternalTable) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - CREATE EXTERNAL TABLE mytable ( - a int - ) WITH ( - DATA_SOURCE="/Root/mydatasource", - LOCATION="/folder1/*" - ); - )sql"); - UNIT_ASSERT_C(res.Root, res.Issues.ToOneLineString()); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_STRING_CONTAINS(line, R"#('('('data_source_path (String '"/Root/mydatasource")) '('location (String '"/folder1/*")))) '('tableType 'externalTable)))))#"); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("tablescheme")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(CreateExternalTableWithTablePrefix) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - pragma TablePathPrefix='/aba'; - CREATE EXTERNAL TABLE mytable ( - a int - ) WITH ( - DATA_SOURCE="mydatasource", - LOCATION="/folder1/*" - ); - )sql"); - UNIT_ASSERT_C(res.Root, res.Issues.ToOneLineString()); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_STRING_CONTAINS(line, "/aba/mydatasource"); - UNIT_ASSERT_STRING_CONTAINS(line, "/aba/mytable"); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("tablescheme")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(CreateExternalTableObjectStorage) { - auto res = SqlToYql(R"sql( - USE plato; - CREATE EXTERNAL TABLE mytable ( - a int, - year Int - ) WITH ( - DATA_SOURCE="/Root/mydatasource", - LOCATION="/folder1/*", - FORMAT="json_as_string", - `projection.enabled`="true", - `projection.year.type`="integer", - `projection.year.min`="2010", - `projection.year.max`="2022", - `projection.year.interval`="1", - `projection.month.type`="integer", - `projection.month.min`="1", - `projection.month.max`="12", - `projection.month.interval`="1", - `projection.month.digits`="2", - `storage.location.template`="${year}/${month}", - PARTITONED_BY = "[year, month]" - ); - )sql"); - UNIT_ASSERT_C(res.IsOk(), res.Issues.ToString()); - } - - Y_UNIT_TEST(CreateExternalTableIfNotExists) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - CREATE EXTERNAL TABLE IF NOT EXISTS mytable ( - a int - ) WITH ( - DATA_SOURCE="/Root/mydatasource", - LOCATION="/folder1/*" - ); - )sql"); - UNIT_ASSERT_C(res.Root, res.Issues.ToOneLineString()); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_STRING_CONTAINS(line, R"#('('('data_source_path (String '"/Root/mydatasource")) '('location (String '"/folder1/*")))) '('tableType 'externalTable)))))#"); - UNIT_ASSERT_STRING_CONTAINS(line, "create_if_not_exists"); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(CreateExternalTableOrReplace) { - NYql::TAstParseResult res = SqlToYql(R"( - USE plato; - CREATE OR REPLACE EXTERNAL TABLE mytable ( - a int - ) WITH ( - DATA_SOURCE="/Root/mydatasource", - LOCATION="/folder1/*" - ); - )"); - UNIT_ASSERT_C(res.Root, res.Issues.ToOneLineString()); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_STRING_CONTAINS(line, R"#('('('data_source_path (String '"/Root/mydatasource")) '('location (String '"/folder1/*")))) '('tableType 'externalTable)))))#"); - UNIT_ASSERT_STRING_CONTAINS(line, "create_or_replace"); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(AlterExternalTableAddColumn) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - ALTER EXTERNAL TABLE mytable - ADD COLUMN my_column int32, - RESET (LOCATION); - )sql"); - UNIT_ASSERT_C(res.Root, res.Issues.ToOneLineString()); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_STRING_CONTAINS(line, R"#('actions '('('addColumns '('('"my_column" (AsOptionalType (DataType 'Int32))#"); - UNIT_ASSERT_STRING_CONTAINS(line, R"#(('setTableSettings '('('location)))#"); - UNIT_ASSERT_STRING_CONTAINS(line, R"#(('tableType 'externalTable))#"); - UNIT_ASSERT_STRING_CONTAINS(line, R"#(('mode 'alter))#"); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(AlterExternalTableDropColumn) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - ALTER EXTERNAL TABLE mytable - DROP COLUMN my_column, - SET (Location = "abc", Other_Prop = "42"), - SET x 'y'; - )sql"); - UNIT_ASSERT_C(res.Root, res.Issues.ToOneLineString()); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_STRING_CONTAINS(line, R"#('actions '('('dropColumns '('"my_column")#"); - UNIT_ASSERT_STRING_CONTAINS(line, R"#(('setTableSettings '('('location (String '"abc")) '('Other_Prop (String '"42")) '('x (String '"y")))))#"); - UNIT_ASSERT_STRING_CONTAINS(line, R"#(('tableType 'externalTable))#"); - UNIT_ASSERT_STRING_CONTAINS(line, R"#(('mode 'alter))#"); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(CreateExternalTableWithBadArguments) { - ExpectFailWithError(R"sql( - USE plato; - CREATE EXTERNAL TABLE mytable; - )sql" , "<main>:3:45: Error: mismatched input ';' expecting '('\n"); - - ExpectFailWithError(R"sql( - USE plato; - CREATE EXTERNAL TABLE mytable ( - a int - ); - )sql" , "<main>:4:23: Error: DATA_SOURCE requires key\n"); - - ExpectFailWithError(R"sql( - USE plato; - CREATE EXTERNAL TABLE mytable ( - a int - ) WITH ( - DATA_SOURCE="/Root/mydatasource" - ); - )sql" , "<main>:6:33: Error: LOCATION requires key\n"); - - ExpectFailWithError(R"sql( - USE plato; - CREATE EXTERNAL TABLE mytable ( - a int - ) WITH ( - LOCATION="/folder1/*" - ); - )sql" , "<main>:6:30: Error: DATA_SOURCE requires key\n"); - - ExpectFailWithError(R"sql( - USE plato; - CREATE EXTERNAL TABLE mytable ( - a int, - PRIMARY KEY(a) - ) WITH ( - DATA_SOURCE="/Root/mydatasource", - LOCATION="/folder1/*" - ); - )sql" , "<main>:8:30: Error: PRIMARY KEY is not supported for external table\n"); - } - - Y_UNIT_TEST(DropExternalTable) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - DROP EXTERNAL TABLE MyExternalTable; - )sql"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("tablescheme")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(DropExternalTableWithTablePrefix) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - pragma TablePathPrefix='/aba'; - DROP EXTERNAL TABLE MyExternalTable; - )sql"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_STRING_CONTAINS(line, "/aba/MyExternalTable"); - UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("'tablescheme")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(DropExternalTableIfExists) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - DROP EXTERNAL TABLE IF EXISTS MyExternalTable; - )sql"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("tablescheme")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("drop_if_exists")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } -} - -Y_UNIT_TEST_SUITE(TopicsDDL) { - void TestQuery(const TString& query, bool expectOk = true) { - TStringBuilder finalQuery; - - finalQuery << "use plato;" << Endl << query; - auto res = SqlToYql(finalQuery, 10, "kikimr"); - if (expectOk) { - UNIT_ASSERT_C(res.IsOk(), res.Issues.ToString()); - } else { - UNIT_ASSERT(!res.IsOk()); - } - } - - Y_UNIT_TEST(CreateTopicSimple) { - TestQuery(R"( - CREATE TOPIC topic1; - )"); - TestQuery(R"( - CREATE TOPIC `cluster1.topic1`; - )"); - TestQuery(R"( - CREATE TOPIC topic1 WITH (metering_mode = "str_value", partition_count_limit = 123, retention_period = Interval('PT1H')); - )"); - } - - Y_UNIT_TEST(CreateTopicConsumer) { - TestQuery(R"( - CREATE TOPIC topic1 (CONSUMER cons1); - )"); - TestQuery(R"( - CREATE TOPIC topic1 (CONSUMER cons1, CONSUMER cons2 WITH (important = false)); - )"); - TestQuery(R"( - CREATE TOPIC topic1 (CONSUMER cons1, CONSUMER cons2 WITH (important = false)) WITH (supported_codecs = "1,2,3"); - )"); - } - - Y_UNIT_TEST(AlterTopicSimple) { - TestQuery(R"( - ALTER TOPIC topic1 SET (retention_period = Interval('PT1H')); - )"); - TestQuery(R"( - ALTER TOPIC topic1 SET (retention_storage_mb = 3, partition_count_limit = 50); - )"); - TestQuery(R"( - ALTER TOPIC topic1 RESET (supported_codecs, retention_period); - )"); - TestQuery(R"( - ALTER TOPIC topic1 RESET (partition_write_speed_bytes_per_second), - SET (partition_write_burst_bytes = 11111, min_active_partitions = 1); - )"); - } - Y_UNIT_TEST(AlterTopicConsumer) { - TestQuery(R"( - ALTER TOPIC topic1 ADD CONSUMER consumer1, - ADD CONSUMER consumer2 WITH (important = false, supported_codecs = "RAW"), - ALTER CONSUMER consumer3 SET (important = false, read_from = 1), - ALTER CONSUMER consumer3 RESET (supported_codecs), - DROP CONSUMER consumer4, - SET (partition_count_limit = 11, retention_period = Interval('PT1H')), - RESET(metering_mode) - )"); - } - Y_UNIT_TEST(DropTopic) { - TestQuery(R"( - DROP TOPIC topic1; - )"); - } - - Y_UNIT_TEST(TopicBadRequests) { - TestQuery(R"( - CREATE TOPIC topic1(); - )", false); - TestQuery(R"( - CREATE TOPIC topic1 SET setting1 = value1; - )", false); - TestQuery(R"( - ALTER TOPIC topic1 SET setting1 value1; - )", false); - TestQuery(R"( - ALTER TOPIC topic1 RESET setting1; - )", false); - - TestQuery(R"( - ALTER TOPIC topic1 DROP CONSUMER consumer4 WITH (k1 = v1); - )", false); - - TestQuery(R"( - CREATE TOPIC topic1 WITH (retention_period = 123); - )", false); - TestQuery(R"( - CREATE TOPIC topic1 (CONSUMER cons1, CONSUMER cons1 WITH (important = false)); - )", false); - TestQuery(R"( - CREATE TOPIC topic1 (CONSUMER cons1 WITH (bad_option = false)); - )", false); - TestQuery(R"( - ALTER TOPIC topic1 ADD CONSUMER cons1, ALTER CONSUMER cons1 RESET (important); - )", false); - TestQuery(R"( - ALTER TOPIC topic1 ADD CONSUMER consumer1, - ALTER CONSUMER consumer3 SET (supported_codecs = "RAW", read_from = 1), - ALTER CONSUMER consumer3 RESET (supported_codecs); - )", false); - TestQuery(R"( - ALTER TOPIC topic1 ADD CONSUMER consumer1, - ALTER CONSUMER consumer3 SET (supported_codecs = "RAW", read_from = 1), - ALTER CONSUMER consumer3 SET (read_from = 2); - )", false); - } - - Y_UNIT_TEST(TopicWithPrefix) { - NYql::TAstParseResult res = SqlToYql(R"( - USE plato; - PRAGMA TablePathPrefix = '/database/path/to/tables'; - ALTER TOPIC `my_table/my_feed` ADD CONSUMER `my_consumer`; - )"); - UNIT_ASSERT(res.Root); - - TWordCountHive elementStat = {{TString("/database/path/to/tables/my_table/my_feed"), 0}, {"topic", 0}}; - VerifyProgram(res, elementStat); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["topic"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["/database/path/to/tables/my_table/my_feed"]); - } -} - -Y_UNIT_TEST_SUITE(BlockEnginePragma) { - Y_UNIT_TEST(Basic) { - const TVector<TString> values = {"auto", "force", "disable"}; - for (const auto& value : values) { - const auto query = TStringBuilder() << "pragma Blockengine='" << value << "'; select 1;"; - NYql::TAstParseResult res = SqlToYql(query); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [&](const TString& word, const TString& line) { - Y_UNUSED(word); - UNIT_ASSERT_STRING_CONTAINS(line, TStringBuilder() << R"(Configure! world (DataSource '"config") '"BlockEngine" '")" << value << "\""); - }; - - TWordCountHive elementStat({"BlockEngine"}); - VerifyProgram(res, elementStat, verifyLine); - UNIT_ASSERT(elementStat["BlockEngine"] == ((value == "disable") ? 0 : 1)); - } - } - - Y_UNIT_TEST(UnknownSetting) { - ExpectFailWithError("use plato; pragma BlockEngine='foo';", - "<main>:1:31: Error: Expected `disable|auto|force' argument for: BlockEngine\n"); - } -} - -Y_UNIT_TEST_SUITE(TViewSyntaxTest) { - Y_UNIT_TEST(CreateViewSimple) { - NYql::TAstParseResult res = SqlToYql(R"( - USE plato; - CREATE VIEW TheView WITH (security_invoker = TRUE) AS SELECT 1; - )" - ); - UNIT_ASSERT_C(res.Root, res.Issues.ToString()); - } - - Y_UNIT_TEST(CreateViewIfNotExists) { - constexpr const char* name = "TheView"; - NYql::TAstParseResult res = SqlToYql(std::format(R"( - USE plato; - CREATE VIEW IF NOT EXISTS {} AS SELECT 1; - )", name - )); - UNIT_ASSERT_C(res.Root, res.Issues.ToString()); - - TVerifyLineFunc verifyLine = [&](const TString& word, const TString& line) { - if (word == "Write!") { - UNIT_ASSERT_STRING_CONTAINS(line, name); - UNIT_ASSERT_STRING_CONTAINS(line, "createObjectIfNotExists"); - } - }; - - TWordCountHive elementStat = { {"Write!"} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(elementStat["Write!"], 1); - } - - Y_UNIT_TEST(CreateViewFromTable) { - constexpr const char* path = "/PathPrefix/TheView"; - constexpr const char* query = R"( - SELECT * FROM SomeTable - )"; - - NYql::TAstParseResult res = SqlToYql(std::format(R"( - USE plato; - CREATE VIEW `{}` WITH (security_invoker = TRUE) AS {}; - )", - path, - query - ) - ); - UNIT_ASSERT_C(res.Root, res.Issues.ToString()); - - TVerifyLineFunc verifyLine = [&](const TString& word, const TString& line) { - if (word == "Write!") { - UNIT_ASSERT_STRING_CONTAINS(line, path); - UNIT_ASSERT_STRING_CONTAINS(line, "createObject"); - } - }; - TWordCountHive elementStat = { {"Write!"} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(elementStat["Write!"], 1); - } - - Y_UNIT_TEST(CheckReconstructedQuery) { - constexpr const char* path = "/PathPrefix/TheView"; - constexpr const char* query = R"( - SELECT * FROM FirstTable JOIN SecondTable ON FirstTable.key == SecondTable.key - )"; - - NYql::TAstParseResult res = SqlToYql(std::format(R"( - USE plato; - CREATE VIEW `{}` WITH (security_invoker = TRUE) AS {}; - )", - path, - query - ) - ); - UNIT_ASSERT_C(res.Root, res.Issues.ToString()); - - TString reconstructedQuery = ToString(Tokenize(query)); - TVerifyLineFunc verifyLine = [&](const TString& word, const TString& line) { - if (word == "query_text") { - UNIT_ASSERT_STRING_CONTAINS(line, reconstructedQuery); - } - }; - TWordCountHive elementStat = { {"Write!"} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(elementStat["Write!"], 1); - } - - Y_UNIT_TEST(DropView) { - constexpr const char* path = "/PathPrefix/TheView"; - NYql::TAstParseResult res = SqlToYql(std::format(R"( - USE plato; - DROP VIEW `{}`; - )", - path - ) - ); - UNIT_ASSERT_C(res.Root, res.Issues.ToString()); - - TVerifyLineFunc verifyLine = [&](const TString& word, const TString& line) { - if (word == "Write!") { - UNIT_ASSERT_STRING_CONTAINS(line, path); - UNIT_ASSERT_STRING_CONTAINS(line, "dropObject"); - } - }; - TWordCountHive elementStat = { {"Write!"} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(elementStat["Write!"], 1); - } - - Y_UNIT_TEST(DropViewIfExists) { - constexpr const char* name = "TheView"; - NYql::TAstParseResult res = SqlToYql(std::format(R"( - USE plato; - DROP VIEW IF EXISTS {}; - )", name - )); - UNIT_ASSERT_C(res.Root, res.Issues.ToString()); - - TVerifyLineFunc verifyLine = [&](const TString& word, const TString& line) { - if (word == "Write!") { - UNIT_ASSERT_STRING_CONTAINS(line, name); - UNIT_ASSERT_STRING_CONTAINS(line, "dropObjectIfExists"); - } - }; - - TWordCountHive elementStat = { {"Write!"} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(elementStat["Write!"], 1); - } - - Y_UNIT_TEST(CreateViewWithTablePrefix) { - NYql::TAstParseResult res = SqlToYql(R"( - USE plato; - PRAGMA TablePathPrefix='/PathPrefix'; - CREATE VIEW TheView WITH (security_invoker = TRUE) AS SELECT 1; - )" - ); - UNIT_ASSERT_C(res.Root, res.Issues.ToString()); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write!") { - UNIT_ASSERT_STRING_CONTAINS(line, "/PathPrefix/TheView"); - UNIT_ASSERT_STRING_CONTAINS(line, "createObject"); - } - }; - - TWordCountHive elementStat = { {"Write!"} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(elementStat["Write!"], 1); - } - - Y_UNIT_TEST(DropViewWithTablePrefix) { - NYql::TAstParseResult res = SqlToYql(R"( - USE plato; - PRAGMA TablePathPrefix='/PathPrefix'; - DROP VIEW TheView; - )" - ); - UNIT_ASSERT_C(res.Root, res.Issues.ToString()); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_STRING_CONTAINS(line, "/PathPrefix/TheView"); - UNIT_ASSERT_STRING_CONTAINS(line, "dropObject"); - } - }; - - TWordCountHive elementStat = { {"Write!"} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(elementStat["Write!"], 1); - } - - Y_UNIT_TEST(YtAlternativeSchemaSyntax) { - NYql::TAstParseResult res = SqlToYql(R"( - SELECT * FROM plato.Input WITH schema(y Int32, x String not null); - )"); - UNIT_ASSERT_C(res.Root, res.Issues.ToString()); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "userschema") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, - line.find(R"__('('('"userschema" (StructType '('"y" (AsOptionalType (DataType 'Int32))) '('"x" (DataType 'String))))))__")); - } - }; - - TWordCountHive elementStat = {{TString("userschema"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["userschema"]); - } - - Y_UNIT_TEST(UseViewAndFullColumnId) { - NYql::TAstParseResult res = SqlToYql("USE plato; SELECT Input.x FROM Input VIEW uitzicht;"); - UNIT_ASSERT(res.Root); - - TWordCountHive elementStat = {{TString("SqlAccess"), 0}, {"SqlProjectItem", 0}, {"Read!", 0}}; - VerifyProgram(res, elementStat); - UNIT_ASSERT_VALUES_EQUAL(0, elementStat["SqlAccess"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SqlProjectItem"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Read!"]); - } -} - -Y_UNIT_TEST_SUITE(CompactNamedExprs) { - Y_UNIT_TEST(SourceCallablesInWrongContext) { - TString query = R"( - pragma CompactNamedExprs; - $foo = %s(); - select $foo from plato.Input; - )"; - - THashMap<TString, TString> errs = { - {"TableRow", "<main>:3:20: Error: TableRow requires data source\n"}, - {"JoinTableRow", "<main>:3:20: Error: JoinTableRow requires data source\n"}, - {"TableRecordIndex", "<main>:3:20: Error: Unable to use function: TableRecord without source\n"}, - {"TablePath", "<main>:3:20: Error: Unable to use function: TablePath without source\n"}, - {"SystemMetadata", "<main>:3:20: Error: Unable to use function: SystemMetadata without source\n"}, - }; - - for (TString callable : { "TableRow", "JoinTableRow", "TableRecordIndex", "TablePath", "SystemMetadata"}) { - auto req = Sprintf(query.c_str(), callable.c_str()); - ExpectFailWithError(req, errs[callable]); - } - } - - Y_UNIT_TEST(ValidateUnusedExprs) { - TString query = R"( - pragma warning("disable", "4527"); - pragma CompactNamedExprs; - pragma ValidateUnusedExprs; - - $foo = count(1); - select 1; - )"; - ExpectFailWithError(query, "<main>:6:20: Error: Aggregation is not allowed in this context\n"); - query = R"( - pragma warning("disable", "4527"); - pragma CompactNamedExprs; - pragma ValidateUnusedExprs; - - define subquery $x() as - select count(1, 2); - end define; - select 1; - )"; - ExpectFailWithError(query, "<main>:7:24: Error: Aggregation function Count requires exactly 1 argument(s), given: 2\n"); - } - - Y_UNIT_TEST(DisableValidateUnusedExprs) { - TString query = R"( - pragma warning("disable", "4527"); - pragma CompactNamedExprs; - pragma DisableValidateUnusedExprs; - - $foo = count(1); - select 1; - )"; - SqlToYql(query).IsOk(); - query = R"( - pragma warning("disable", "4527"); - pragma CompactNamedExprs; - pragma DisableValidateUnusedExprs; - - define subquery $x() as - select count(1, 2); - end define; - select 1; - )"; - SqlToYql(query).IsOk(); - } -} - -Y_UNIT_TEST_SUITE(ResourcePool) { - Y_UNIT_TEST(CreateResourcePool) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - CREATE RESOURCE POOL MyResourcePool WITH ( - CONCURRENT_QUERY_LIMIT=20, - QUERY_CANCEL_AFTER_SECONDS=86400, - QUEUE_TYPE="FIFO" - ); - )sql"); - UNIT_ASSERT_C(res.Root, res.Issues.ToString()); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_STRING_CONTAINS(line, R"#('('('"concurrent_query_limit" (Int32 '"20")) '('"query_cancel_after_seconds" (Int32 '"86400")) '('"queue_type" '"FIFO"))#"); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("createObject")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(CreateResourcePoolWithBadArguments) { - ExpectFailWithError(R"sql( - USE plato; - CREATE RESOURCE POOL MyResourcePool; - )sql" , "<main>:3:51: Error: mismatched input ';' expecting WITH\n"); - - ExpectFailWithError(R"sql( - USE plato; - CREATE RESOURCE POOL MyResourcePool WITH ( - DUPLICATE_SETTING="first_value", - DUPLICATE_SETTING="second_value" - ); - )sql" , "<main>:5:21: Error: DUPLICATE_SETTING duplicate keys\n"); - } - - Y_UNIT_TEST(AlterResourcePool) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - ALTER RESOURCE POOL MyResourcePool - SET (CONCURRENT_QUERY_LIMIT = 30, Weight = 5, QUEUE_TYPE = "UNORDERED"), - RESET (Query_Cancel_After_Seconds, Query_Count_Limit); - )sql"); - UNIT_ASSERT_C(res.Root, res.Issues.ToString()); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_STRING_CONTAINS(line, R"#(('mode 'alterObject))#"); - UNIT_ASSERT_STRING_CONTAINS(line, R"#('('features '('('"concurrent_query_limit" (Int32 '"30")) '('"queue_type" '"UNORDERED") '('"weight" (Int32 '"5")))))#"); - UNIT_ASSERT_STRING_CONTAINS(line, R"#('('resetFeatures '('"query_cancel_after_seconds" '"query_count_limit")))#"); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(DropResourcePool) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - DROP RESOURCE POOL MyResourcePool; - )sql"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("'features")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("dropObject")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } -} - -Y_UNIT_TEST_SUITE(BackupCollection) { - Y_UNIT_TEST(CreateBackupCollection) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - CREATE BACKUP COLLECTION TestCollection WITH ( - STORAGE="local", - TAG="test" -- for testing purposes, not a real thing - ); - )sql"); - UNIT_ASSERT_C(res.Root, res.Issues.ToString()); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(R"#('"TestCollection")#")); - UNIT_ASSERT_STRING_CONTAINS(line, R"#(('"storage" (String '"local")))#"); - UNIT_ASSERT_STRING_CONTAINS(line, R"#(('"tag" (String '"test"))))#"); - UNIT_ASSERT_STRING_CONTAINS(line, R"#(('entries '()))#"); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'create")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(CreateBackupCollectionWithDatabase) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - CREATE BACKUP COLLECTION TestCollection DATABASE WITH ( - STORAGE="local", - TAG="test" -- for testing purposes, not a real thing - ); - )sql"); - UNIT_ASSERT_C(res.Root, res.Issues.ToString()); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(R"#('"TestCollection")#")); - UNIT_ASSERT_STRING_CONTAINS(line, R"#(('"storage" (String '"local")))#"); - UNIT_ASSERT_STRING_CONTAINS(line, R"#(('"tag" (String '"test"))))#"); - UNIT_ASSERT_STRING_CONTAINS(line, R"#(('entries '('('('type 'database)))))#"); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'create")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(CreateBackupCollectionWithTables) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - CREATE BACKUP COLLECTION TestCollection ( - TABLE someTable, - TABLE `prefix/anotherTable` - ) WITH ( - STORAGE="local", - TAG="test" -- for testing purposes, not a real thing - ); - )sql"); - UNIT_ASSERT_C(res.Root, res.Issues.ToString()); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(R"#('"TestCollection")#")); - UNIT_ASSERT_STRING_CONTAINS(line, R"#(('"storage" (String '"local")))#"); - UNIT_ASSERT_STRING_CONTAINS(line, R"#(('"tag" (String '"test"))))#"); - UNIT_ASSERT_STRING_CONTAINS(line, R"#(('entries '('('('type 'table) '('path '"someTable")) '('('type 'table) '('path '"prefix/anotherTable")))))#"); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'create")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(CreateBackupCollectionWithBadArguments) { - ExpectFailWithError(R"sql( - USE plato; - CREATE BACKUP COLLECTION TestCollection; - )sql" , "<main>:3:55: Error: mismatched input ';' expecting {'(', DATABASE, WITH}\n"); - - ExpectFailWithError(R"sql( - USE plato; - CREATE BACKUP COLLECTION TABLE TestCollection; - )sql" , "<main>:3:47: Error: mismatched input 'TestCollection' expecting {'(', DATABASE, WITH}\n"); - - ExpectFailWithError(R"sql( - USE plato; - CREATE BACKUP COLLECTION DATABASE `test` TestCollection; - )sql" , "<main>:3:50: Error: mismatched input '`test`' expecting {'(', DATABASE, WITH}\n"); - - ExpectFailWithError(R"sql( - USE plato; - CREATE BACKUP COLLECTION TestCollection WITH ( - DUPLICATE_SETTING="first_value", - DUPLICATE_SETTING="second_value" - ); - )sql" , "<main>:5:21: Error: DUPLICATE_SETTING duplicate keys\n"); - - ExpectFailWithError(R"sql( - USE plato; - CREATE BACKUP COLLECTION TestCollection WITH ( - INT_SETTING=1 - ); - )sql" , "<main>:4:21: Error: INT_SETTING value should be a string literal\n"); - } - - Y_UNIT_TEST(AlterBackupCollection) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - ALTER BACKUP COLLECTION TestCollection - SET (STORAGE="remote"), -- also just for test - SET (TAG1 = "123"), - RESET (TAG2, TAG3); - )sql"); - UNIT_ASSERT_C(res.Root, res.Issues.ToString()); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(R"#('"TestCollection")#")); - UNIT_ASSERT_STRING_CONTAINS(line, R"#(('mode 'alter))#"); - UNIT_ASSERT_STRING_CONTAINS(line, R"#(('"storage" (String '"remote")))#"); - UNIT_ASSERT_STRING_CONTAINS(line, R"#(('"tag1" (String '"123"))))#"); - UNIT_ASSERT_STRING_CONTAINS(line, R"#('('resetSettings '('"tag2" '"tag3")))#"); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(AlterBackupCollectionEntries) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - ALTER BACKUP COLLECTION TestCollection - DROP TABLE `test`, - ADD DATABASE; - )sql"); - UNIT_ASSERT_C(res.Root, res.Issues.ToString()); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(R"#('"TestCollection")#")); - UNIT_ASSERT_STRING_CONTAINS(line, R"#(('mode 'alter))#"); - UNIT_ASSERT_STRING_CONTAINS(line, R"#('alterEntries)#"); - UNIT_ASSERT_STRING_CONTAINS(line, R"#('('('type 'table) '('path '"test") '('action 'drop)))#"); - UNIT_ASSERT_STRING_CONTAINS(line, R"#('('('type 'database) '('action 'add)))#"); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(DropBackupCollection) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - DROP BACKUP COLLECTION TestCollection; - )sql"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(R"#('"TestCollection")#")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("drop")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } -} - -Y_UNIT_TEST_SUITE(ResourcePoolClassifier) { - Y_UNIT_TEST(CreateResourcePoolClassifier) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - CREATE RESOURCE POOL CLASSIFIER MyResourcePoolClassifier WITH ( - RANK=20, - RESOURCE_POOL='wgUserQueries', - MEMBER_NAME='yandex_query@abc' - ); - )sql"); - UNIT_ASSERT_C(res.Root, res.Issues.ToString()); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_STRING_CONTAINS(line, R"#('('('"member_name" '"yandex_query@abc") '('"rank" (Int32 '"20")) '('"resource_pool" '"wgUserQueries"))#"); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("createObject")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(CreateResourcePoolClassifierWithBadArguments) { - ExpectFailWithError(R"sql( - USE plato; - CREATE RESOURCE POOL CLASSIFIER MyResourcePoolClassifier; - )sql" , "<main>:3:72: Error: mismatched input ';' expecting WITH\n"); - - ExpectFailWithError(R"sql( - USE plato; - CREATE RESOURCE POOL CLASSIFIER MyResourcePoolClassifier WITH ( - DUPLICATE_SETTING="first_value", - DUPLICATE_SETTING="second_value" - ); - )sql" , "<main>:5:21: Error: DUPLICATE_SETTING duplicate keys\n"); - } - - Y_UNIT_TEST(AlterResourcePoolClassifier) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - ALTER RESOURCE POOL CLASSIFIER MyResourcePoolClassifier - SET (RANK = 30, Weight = 5, MEMBER_NAME = "test@user"), - RESET (Resource_Pool); - )sql"); - UNIT_ASSERT_C(res.Root, res.Issues.ToString()); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_STRING_CONTAINS(line, R"#(('mode 'alterObject))#"); - UNIT_ASSERT_STRING_CONTAINS(line, R"#('('features '('('"member_name" '"test@user") '('"rank" (Int32 '"30")) '('"weight" (Int32 '"5")))))#"); - UNIT_ASSERT_STRING_CONTAINS(line, R"#('('resetFeatures '('"resource_pool")))#"); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(DropResourcePoolClassifier) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - DROP RESOURCE POOL CLASSIFIER MyResourcePoolClassifier; - )sql"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("'features")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("dropObject")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(BacktickMatching) { - auto req = "select\n" - " 1 as `Schema has \\`RealCost\\``\n" - " -- foo`bar"; - auto res = SqlToYql(req); - UNIT_ASSERT(res.Root); - UNIT_ASSERT(res.IsOk()); - UNIT_ASSERT(res.Issues.Size() == 0); - res = SqlToYqlWithAnsiLexer(req); - UNIT_ASSERT(res.Root); - UNIT_ASSERT(res.IsOk()); - UNIT_ASSERT(res.Issues.Size() == 0); - - req = "select 1 as `a``b`, 2 as ````, 3 as `\\x60a\\x60`, 4 as ```b```, 5 as `\\`c\\``"; - res = SqlToYql(req); - UNIT_ASSERT(res.Root); - UNIT_ASSERT(res.IsOk()); - UNIT_ASSERT(res.Issues.Size() == 0); - res = SqlToYqlWithAnsiLexer(req); - UNIT_ASSERT(res.Root); - UNIT_ASSERT(res.IsOk()); - UNIT_ASSERT(res.Issues.Size() == 0); - } -} - -Y_UNIT_TEST_SUITE(Backup) { - Y_UNIT_TEST(Simple) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - BACKUP TestCollection; - )sql"); - UNIT_ASSERT_C(res.Root, res.Issues.ToString()); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(R"#('"TestCollection")#")); - UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("'Incremental")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'backup")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(Incremental) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - BACKUP TestCollection INCREMENTAL; - )sql"); - UNIT_ASSERT_C(res.Root, res.Issues.ToString()); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(R"#('"TestCollection")#")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'backupIncremental")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } -} - -Y_UNIT_TEST_SUITE(Restore) { - Y_UNIT_TEST(Simple) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - RESTORE TestCollection; - )sql"); - UNIT_ASSERT_C(res.Root, res.Issues.ToString()); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(R"#('"TestCollection")#")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'restore")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(AtPoint) { - NYql::TAstParseResult res = SqlToYql(R"sql( - USE plato; - RESTORE TestCollection AT '2024-06-16_20-14-02'; - )sql"); - UNIT_ASSERT_C(res.Root, res.Issues.ToString()); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(R"#('"TestCollection")#")); - UNIT_ASSERT_STRING_CONTAINS(line, R"#('at '"2024-06-16_20-14-02")#"); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'restore")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } -} - -Y_UNIT_TEST_SUITE(ColumnFamily) { - Y_UNIT_TEST(CompressionLevelCorrectUsage) { - NYql::TAstParseResult res = SqlToYql(R"( use plato; - CREATE TABLE tableName ( - Key Uint32 FAMILY default, - Value String FAMILY family1, - PRIMARY KEY (Key), - FAMILY default ( - DATA = "test", - COMPRESSION = "lz4", - COMPRESSION_LEVEL = 5 - ), - FAMILY family1 ( - DATA = "test", - COMPRESSION = "lz4", - COMPRESSION_LEVEL = 3 - ) - ); - )"); - UNIT_ASSERT(res.IsOk()); - UNIT_ASSERT(res.Issues.Size() == 0); - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("compression_level")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("5")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("3")); - } - }; - - TWordCountHive elementStat = { { TString("Write"), 0 }, { TString("compression_level"), 0 } }; - VerifyProgram(res, elementStat, verifyLine); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - UNIT_ASSERT_VALUES_EQUAL(2, elementStat["compression_level"]); - } - - Y_UNIT_TEST(FieldDataIsNotString) { - NYql::TAstParseResult res = SqlToYql(R"( use plato; - CREATE TABLE tableName ( - Key Uint32 FAMILY default, - PRIMARY KEY (Key), - FAMILY default ( - DATA = 1, - COMPRESSION = "lz4", - COMPRESSION_LEVEL = 5 - ) - ); - )"); - UNIT_ASSERT(!res.IsOk()); - UNIT_ASSERT(res.Issues.Size() == 1); - UNIT_ASSERT_STRING_CONTAINS(res.Issues.ToString(), "DATA value should be a string literal"); - } - - Y_UNIT_TEST(FieldCompressionIsNotString) { - NYql::TAstParseResult res = SqlToYql(R"( use plato; - CREATE TABLE tableName ( - Key Uint32 FAMILY default, - PRIMARY KEY (Key), - FAMILY default ( - DATA = "test", - COMPRESSION = 2, - COMPRESSION_LEVEL = 5 - ), - ); - )"); - UNIT_ASSERT(!res.IsOk()); - UNIT_ASSERT(res.Issues.Size() == 1); - UNIT_ASSERT_STRING_CONTAINS(res.Issues.ToString(), "COMPRESSION value should be a string literal"); - } - - Y_UNIT_TEST(FieldCompressionLevelIsNotInteger) { - NYql::TAstParseResult res = SqlToYql(R"( use plato; - CREATE TABLE tableName ( - Key Uint32 FAMILY default, - PRIMARY KEY (Key), - FAMILY default ( - DATA = "test", - COMPRESSION = "lz4", - COMPRESSION_LEVEL = "5" - ) - ); - )"); - UNIT_ASSERT(!res.IsOk()); - UNIT_ASSERT(res.Issues.Size() == 1); - UNIT_ASSERT_STRING_CONTAINS(res.Issues.ToString(), "COMPRESSION_LEVEL value should be an integer"); - } - - Y_UNIT_TEST(AlterCompressionCorrectUsage) { - NYql::TAstParseResult res = SqlToYql(R"( use plato; - ALTER TABLE tableName ALTER FAMILY default SET COMPRESSION "lz4"; - )"); - UNIT_ASSERT(res.IsOk()); - UNIT_ASSERT(res.Issues.Size() == 0); - } - - Y_UNIT_TEST(AlterCompressionFieldIsNotString) { - NYql::TAstParseResult res = SqlToYql(R"( use plato; - ALTER TABLE tableName ALTER FAMILY default SET COMPRESSION lz4; - )"); - UNIT_ASSERT(!res.IsOk()); - UNIT_ASSERT(res.Issues.Size() == 1); - UNIT_ASSERT_STRING_CONTAINS(res.Issues.ToString(), "mismatched input 'lz4' expecting {STRING_VALUE, DIGITS, INTEGER_VALUE}"); - } - - Y_UNIT_TEST(AlterCompressionLevelCorrectUsage) { - NYql::TAstParseResult res = SqlToYql(R"( use plato; - ALTER TABLE tableName ALTER FAMILY default SET COMPRESSION_LEVEL 5; - )"); - UNIT_ASSERT(res.IsOk()); - UNIT_ASSERT(res.Issues.Size() == 0); - } - - Y_UNIT_TEST(AlterCompressionLevelFieldIsNotInteger) { - NYql::TAstParseResult res = SqlToYql(R"( use plato; - ALTER TABLE tableName ALTER FAMILY default SET COMPRESSION_LEVEL "5"; - )"); - UNIT_ASSERT(!res.IsOk()); - UNIT_ASSERT(res.Issues.Size() == 1); - UNIT_ASSERT_STRING_CONTAINS(res.Issues.ToString(), "COMPRESSION_LEVEL value should be an integer"); - } -} +#define ANTLR_VER 4 +#include "sql_ut_common.h" Y_UNIT_TEST_SUITE(QuerySplit) { Y_UNIT_TEST(Simple) { @@ -8180,86 +97,4 @@ Y_UNIT_TEST_SUITE(QuerySplit) { } } -Y_UNIT_TEST_SUITE(Transfer) { - Y_UNIT_TEST(Lambda) { - NYql::TAstParseResult res = SqlToYql(R"( use plato; - -- Русский коммент, empty statement - ; - - -- befor comment - $a = "А"; - - SELECT * FROM Input; - - $b = ($x) -> { return $a || $x; }; - - CREATE TRANSFER `TransferName` - FROM `TopicName` TO `TableName` - USING ($x) -> { - -- internal comment - return $b($x); - } - WITH ( - CONNECTION_STRING = "grpc://localhost:2135/?database=/Root" - ); - )"); - - UNIT_ASSERT_C(res.IsOk(), res.Issues.ToString()); - UNIT_ASSERT_VALUES_EQUAL_C(res.Issues.Size(), 0, res.Issues.ToString()); - - const auto programm = GetPrettyPrint(res); - - Cerr << ">>>>> Root " << programm << Endl; - auto expected = R"('transformLambda 'use plato; --- befor comment - $a = "А"; -$b = ($x) -> { return $a || $x; }; -$__ydb_transfer_lambda = ($x) -> { - -- internal comment - return $b($x); - }; -))"; - - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, programm.find(expected)); - - } -} - -Y_UNIT_TEST_SUITE(MatchRecognizeMeasuresAggregation) { - Y_UNIT_TEST(InsideSelect) { - ExpectFailWithError(R"sql( - SELECT FIRST(0), LAST(1); - )sql", - "<main>:2:20: Error: Cannot use FIRST and LAST outside the MATCH_RECOGNIZE context\n" - "<main>:2:30: Error: Cannot use FIRST and LAST outside the MATCH_RECOGNIZE context\n" - ); - } - - Y_UNIT_TEST(OutsideSelect) { - ExpectFailWithError(R"sql( - $lambda = ($x) -> (FIRST($x) + LAST($x)); - SELECT $lambda(x) FROM plato.Input; - )sql", - "<main>:2:32: Error: Cannot use FIRST and LAST outside the MATCH_RECOGNIZE context\n" - "<main>:2:44: Error: Cannot use FIRST and LAST outside the MATCH_RECOGNIZE context\n" - ); - } - - Y_UNIT_TEST(AsAggregateFunction) { - ExpectFailWithError(R"sql( - SELECT FIRST(x), LAST(x) FROM plato.Input; - )sql", - "<main>:2:20: Error: Cannot use FIRST and LAST outside the MATCH_RECOGNIZE context\n" - "<main>:2:30: Error: Cannot use FIRST and LAST outside the MATCH_RECOGNIZE context\n" - ); - } - Y_UNIT_TEST(AsWindowFunction) { - ExpectFailWithError(R"sql( - SELECT FIRST(x) OVER(), LAST(x) OVER() FROM plato.Input; - )sql", - "<main>:2:20: Error: Cannot use FIRST and LAST outside the MATCH_RECOGNIZE context\n" - "<main>:2:37: Error: Cannot use FIRST and LAST outside the MATCH_RECOGNIZE context\n" - ); - } -} diff --git a/yql/essentials/sql/v1/sql_ut_antlr4.h b/yql/essentials/sql/v1/sql_ut_antlr4.h index 4f4203a789..03d804707f 100644 --- a/yql/essentials/sql/v1/sql_ut_antlr4.h +++ b/yql/essentials/sql/v1/sql_ut_antlr4.h @@ -10,6 +10,7 @@ #include <library/cpp/testing/unittest/registar.h> +#include <library/cpp/regex/pcre/pcre.h> #include <util/string/split.h> #include <deque> #include <unordered_set> @@ -84,6 +85,13 @@ inline void ExpectFailWithError(const TString& query, const TString& error) { UNIT_ASSERT_NO_DIFF(Err2Str(res), error); } +inline void ExpectFailWithFuzzyError(const TString& query, const TString& errorRegex) { + NYql::TAstParseResult res = SqlToYql(query); + + UNIT_ASSERT(!res.Root); + UNIT_ASSERT(NPcre::TPcre<char>(errorRegex.c_str()).Matches(Err2Str(res))); +} + inline NYql::TAstParseResult SqlToYqlWithAnsiLexer(const TString& query, size_t maxErrors = 10, const TString& provider = {}, EDebugOutput debug = EDebugOutput::None) { bool ansiLexer = true; return SqlToYqlWithMode(query, NSQLTranslation::ESqlMode::QUERY, maxErrors, provider, debug, ansiLexer); diff --git a/yql/essentials/sql/v1/sql_ut_common.h b/yql/essentials/sql/v1/sql_ut_common.h new file mode 100644 index 0000000000..325b439b25 --- /dev/null +++ b/yql/essentials/sql/v1/sql_ut_common.h @@ -0,0 +1,8398 @@ +namespace { + TString ToString(const TParsedTokenList& tokens) { + TStringBuilder reconstructedQuery; + for (const auto& token : tokens) { + if (token.Name == "WS" || token.Name == "EOF") { + continue; + } + if (!reconstructedQuery.empty()) { + reconstructedQuery << ' '; + } + reconstructedQuery << token.Content; + } + return reconstructedQuery; + } +} + +Y_UNIT_TEST_SUITE(AnsiMode) { + Y_UNIT_TEST(PragmaAnsi) { + UNIT_ASSERT(SqlToYql("PRAGMA ANSI 2016;").IsOk()); + } +} + +Y_UNIT_TEST_SUITE(SqlParsingOnly) { + ///This function is used in BACKWARD COMPATIBILITY tests below that LIMIT the sets of token that CAN NOT be used + ///as identifiers in different contexts in a SQL request + ///\return list of tokens that failed this check + TVector<TString> ValidateTokens(const THashSet<TString>& forbidden, const std::function<TString (const TString& )>& makeRequest) { + THashMap<TString, bool> allTokens; + for (const auto& t: NSQLFormat::GetKeywords()) { + allTokens[t] = !forbidden.contains((t)); + } + for (const auto& f: forbidden) { + UNIT_ASSERT(allTokens.contains(f)); //check that forbidden list contains tokens only(argument check) + } + TVector<TString> failed; + for (const auto& [token, allowed]: allTokens) { + if (SqlToYql(makeRequest(token)).IsOk() != allowed) + failed.push_back(token); + } + return failed; + } + + Y_UNIT_TEST(TokensAsColumnName) { //id_expr + auto failed = ValidateTokens({ + "ALL", "ANY", "AS", "ASSUME", "ASYMMETRIC", "AUTOMAP", "BETWEEN", "BITCAST", + "CALLABLE", "CASE", "CAST", "CUBE", "CURRENT_DATE", "CURRENT_TIME", "CURRENT_TIMESTAMP", + "DICT", "DISTINCT", "ENUM", "ERASE", "EXCEPT", "EXISTS", "FLOW", "FROM", "FULL", + "HAVING", "HOP", "INTERSECT", "JSON_EXISTS", "JSON_QUERY", "JSON_VALUE", "LIMIT", "LIST", + "NOT", "OPTIONAL", "PROCESS", "REDUCE", "REPEATABLE", "RESOURCE", "RETURN", "RETURNING", "ROLLUP", + "SELECT", "SET", "STREAM", "STRUCT", "SYMMETRIC", "TAGGED", "TUPLE", "UNBOUNDED", + "UNION", "VARIANT", "WHEN", "WHERE", "WINDOW", "WITHOUT" + }, + [](const TString& token){ + TStringBuilder req; + req << "SELECT " << token << " FROM Plato.Input"; + return req; + } + ); + UNIT_ASSERT_VALUES_EQUAL(failed, TVector<TString>{}); + } + + Y_UNIT_TEST(TokensAsWithoutColumnName) { //id_without + auto failed = ValidateTokens({ + "ALL", "AS", "ASSUME", "ASYMMETRIC", "AUTOMAP", "BETWEEN", "BITCAST", + "CALLABLE", "CASE", "CAST", "CUBE", "CURRENT_DATE", "CURRENT_TIME", "CURRENT_TIMESTAMP", + "DICT", "DISTINCT", "EMPTY_ACTION", "ENUM", "EXCEPT", "EXISTS", "FALSE", "FLOW", "FROM", "FULL", + "HAVING", "HOP", "INTERSECT", "JSON_EXISTS", "JSON_QUERY", "JSON_VALUE", "LIMIT", "LIST", + "NOT", "NULL", "OPTIONAL", "PROCESS", "REDUCE", "REPEATABLE", "RESOURCE", "RETURN", "RETURNING", "ROLLUP", + "SELECT", "SET", "STRUCT", "SYMMETRIC", "TAGGED", "TRUE", "TUPLE", "UNBOUNDED", + "UNION", "VARIANT", "WHEN", "WHERE", "WINDOW", "WITHOUT" + }, + [](const TString& token){ + TStringBuilder req; + req << "SELECT * WITHOUT " << token << " FROM Plato.Input"; + return req; + } + ); + UNIT_ASSERT_VALUES_EQUAL(failed, TVector<TString>{}); + } + + Y_UNIT_TEST(TokensAsColumnNameInAddColumn) { //id_schema + auto failed = ValidateTokens({ + "ANY", "AUTOMAP", "CALLABLE", "COLUMN", "DICT", "ENUM", "ERASE", "FALSE", "FLOW", + "LIST", "OPTIONAL", "REPEATABLE", "RESOURCE", + "SET", "STREAM", "STRUCT", "TAGGED", "TRUE", "TUPLE", "VARIANT" + }, + [](const TString& token){ + TStringBuilder req; + req << "ALTER TABLE Plato.Input ADD COLUMN " << token << " Bool"; + return req; + } + ); + UNIT_ASSERT_VALUES_EQUAL(failed, TVector<TString>{}); + } + + Y_UNIT_TEST(TokensAsColumnAlias) { + auto failed = ValidateTokens({ + "AUTOMAP", "FALSE", + "REPEATABLE", "TRUE" + }, + [](const TString& token){ + TStringBuilder req; + req << "SELECT Col as " << token << " FROM Plato.Input"; + return req; + } + ); + UNIT_ASSERT_VALUES_EQUAL(failed, TVector<TString>{}); + } + + Y_UNIT_TEST(TokensAsTableName) { //id_table_or_type + auto failed = ValidateTokens({ + "ANY", "AUTOMAP", "COLUMN", "ERASE", "FALSE", + "REPEATABLE", "STREAM", "TRUE" + }, + [](const TString& token){ + TStringBuilder req; + req << "SELECT * FROM Plato." << token; + return req; + } + ); + UNIT_ASSERT_VALUES_EQUAL(failed, TVector<TString>{}); + } + + Y_UNIT_TEST(TokensAsTableAlias) { //id_table + auto failed = ValidateTokens({ + "AUTOMAP", "CALLABLE", "DICT", "ENUM","FALSE", "FLOW", + "LIST", "OPTIONAL", "REPEATABLE", "RESOURCE", + "SET", "STRUCT", "TAGGED", "TRUE", "TUPLE", "VARIANT" + }, + [](const TString& token){ + TStringBuilder req; + req << "SELECT * FROM Plato.Input AS " << token; + return req; + } + ); + UNIT_ASSERT_VALUES_EQUAL(failed, TVector<TString>{}); + } + + Y_UNIT_TEST(TokensAsHints) { //id_hint + auto failed = ValidateTokens({ + "AUTOMAP", "CALLABLE", "COLUMNS", "DICT", "ENUM", "FALSE", "FLOW", + "LIST", "OPTIONAL", "REPEATABLE", "RESOURCE", + "SCHEMA", "SET", "STRUCT", "TAGGED", "TRUE", "TUPLE", "VARIANT" + }, + [](const TString& token){ + TStringBuilder req; + req << "SELECT * FROM Plato.Input WITH " << token; + return req; + } + ); + UNIT_ASSERT_VALUES_EQUAL(failed, TVector<TString>{}); + } + + Y_UNIT_TEST(TokensAsWindow) { //id_window + auto failed = ValidateTokens({ + "AUTOMAP", "CALLABLE", "DICT", "ENUM", "FALSE", "FLOW", "GROUPS", "LIST", "OPTIONAL", + "RANGE", "REPEATABLE", "RESOURCE", "ROWS", "SET", "STRUCT", "TAGGED" ,"TRUE", "TUPLE", "VARIANT" + }, + [](const TString& token){ + TStringBuilder req; + req << "SELECT * FROM Plato.Input WINDOW " << token << " AS ()"; + return req; + } + ); + UNIT_ASSERT_VALUES_EQUAL(failed, TVector<TString>{}); + } + + Y_UNIT_TEST(TokensAsIdExprIn) { //id_expr_in + auto failed = ValidateTokens({ + "ALL", "ANY", "AS", "ASSUME", "ASYMMETRIC", "AUTOMAP", "BETWEEN", "BITCAST", + "CALLABLE", "CASE", "CAST", "COMPACT", "CUBE", "CURRENT_DATE", "CURRENT_TIME", "CURRENT_TIMESTAMP", + "DICT", "DISTINCT", "ENUM", "ERASE", "EXCEPT", "EXISTS", "FLOW", "FROM", "FULL", + "HAVING", "HOP", "INTERSECT", "JSON_EXISTS", "JSON_QUERY", "JSON_VALUE", "LIMIT", "LIST", + "NOT", "OPTIONAL", "PROCESS", "REDUCE", "REPEATABLE", "RESOURCE", "RETURN", "RETURNING", "ROLLUP", + "SELECT", "SET", "STREAM", "STRUCT", "SYMMETRIC", "TAGGED", "TUPLE", "UNBOUNDED", + "UNION", "VARIANT", "WHEN", "WHERE", "WINDOW", "WITHOUT" + }, + [](const TString& token){ + TStringBuilder req; + req << "SELECT * FROM Plato.Input WHERE q IN " << token; + return req; + } + ); + UNIT_ASSERT_VALUES_EQUAL(failed, TVector<TString>{}); + } + + Y_UNIT_TEST(TableHints) { + UNIT_ASSERT(SqlToYql("SELECT * FROM plato.Input WITH INFER_SCHEMA").IsOk()); + UNIT_ASSERT(SqlToYql("SELECT * FROM plato.Input WITH (INFER_SCHEMA)").IsOk()); + } + + Y_UNIT_TEST(InNoHints) { + TString query = "SELECT * FROM plato.Input WHERE key IN (1,2,3)"; + + VerifySqlInHints(query, { "'('('warnNoAnsi))" }, {}); + VerifySqlInHints(query, { "'()" }, false); + VerifySqlInHints(query, { "'('('ansi))" }, true); + } + + Y_UNIT_TEST(InHintCompact) { + // should parse COMPACT as hint + TString query = "SELECT * FROM plato.Input WHERE key IN COMPACT(1, 2, 3)"; + + VerifySqlInHints(query, { "'('isCompact)" }); + } + + Y_UNIT_TEST(InHintSubquery) { + // should parse tableSource as hint + TString query = "$subq = (SELECT key FROM plato.Input); SELECT * FROM plato.Input WHERE key IN $subq"; + + VerifySqlInHints(query, { "'('tableSource)" }); + } + + Y_UNIT_TEST(InHintCompactSubquery) { + TString query = "$subq = (SELECT key FROM plato.Input); SELECT * FROM plato.Input WHERE key IN COMPACT $subq"; + + VerifySqlInHints(query, { "'('isCompact)", "'('tableSource)" }); + } + + Y_UNIT_TEST(CompactKeywordNotReservedForNames) { + UNIT_ASSERT(SqlToYql("SELECT COMPACT FROM plato.Input WHERE COMPACT IN COMPACT(1, 2, 3)").IsOk()); + UNIT_ASSERT(SqlToYql("USE plato; SELECT * FROM COMPACT").IsOk()); + } + + Y_UNIT_TEST(FamilyKeywordNotReservedForNames) { + // FIXME: check if we can get old behaviour + //UNIT_ASSERT(SqlToYql("USE plato; CREATE TABLE FAMILY (FAMILY Uint32, PRIMARY KEY (FAMILY));").IsOk()); + //UNIT_ASSERT(SqlToYql("USE plato; SELECT FAMILY FROM FAMILY").IsOk()); + UNIT_ASSERT(SqlToYql("USE plato; SELECT FAMILY FROM Input").IsOk()); + } + + Y_UNIT_TEST(ResetKeywordNotReservedForNames) { + UNIT_ASSERT(SqlToYql("USE plato; CREATE TABLE RESET (RESET Uint32, PRIMARY KEY (RESET));").IsOk()); + UNIT_ASSERT(SqlToYql("USE plato; SELECT RESET FROM RESET").IsOk()); + } + + Y_UNIT_TEST(SyncKeywordNotReservedForNames) { + UNIT_ASSERT(SqlToYql("USE plato; CREATE TABLE SYNC (SYNC Uint32, PRIMARY KEY (SYNC));").IsOk()); + UNIT_ASSERT(SqlToYql("USE plato; SELECT SYNC FROM SYNC").IsOk()); + } + + Y_UNIT_TEST(AsyncKeywordNotReservedForNames) { + UNIT_ASSERT(SqlToYql("USE plato; CREATE TABLE ASYNC (ASYNC Uint32, PRIMARY KEY (ASYNC));").IsOk()); + UNIT_ASSERT(SqlToYql("USE plato; SELECT ASYNC FROM ASYNC").IsOk()); + } + + Y_UNIT_TEST(DisableKeywordNotReservedForNames) { + UNIT_ASSERT(SqlToYql("USE plato; CREATE TABLE DISABLE (DISABLE Uint32, PRIMARY KEY (DISABLE));").IsOk()); + UNIT_ASSERT(SqlToYql("USE plato; SELECT DISABLE FROM DISABLE").IsOk()); + } + + Y_UNIT_TEST(ChangefeedKeywordNotReservedForNames) { + UNIT_ASSERT(SqlToYql("USE plato; CREATE TABLE CHANGEFEED (CHANGEFEED Uint32, PRIMARY KEY (CHANGEFEED));").IsOk()); + UNIT_ASSERT(SqlToYql("USE plato; SELECT CHANGEFEED FROM CHANGEFEED").IsOk()); + } + + Y_UNIT_TEST(ReplicationKeywordNotReservedForNames) { + UNIT_ASSERT(SqlToYql("USE plato; CREATE TABLE REPLICATION (REPLICATION Uint32, PRIMARY KEY (REPLICATION));").IsOk()); + UNIT_ASSERT(SqlToYql("USE plato; SELECT REPLICATION FROM REPLICATION").IsOk()); + } + + Y_UNIT_TEST(TransferKeywordNotReservedForNames) { + UNIT_ASSERT(SqlToYql("USE plato; CREATE TABLE TRANSFER (TRANSFER Uint32, PRIMARY KEY (TRANSFER));").IsOk()); + UNIT_ASSERT(SqlToYql("USE plato; SELECT TRANSFER FROM TRANSFER").IsOk()); + } + + Y_UNIT_TEST(SecondsKeywordNotReservedForNames) { + UNIT_ASSERT(SqlToYql("USE plato; CREATE TABLE SECONDS (SECONDS Uint32, PRIMARY KEY (SECONDS));").IsOk()); + UNIT_ASSERT(SqlToYql("USE plato; SELECT SECONDS FROM SECONDS").IsOk()); + } + + Y_UNIT_TEST(MillisecondsKeywordNotReservedForNames) { + UNIT_ASSERT(SqlToYql("USE plato; CREATE TABLE MILLISECONDS (MILLISECONDS Uint32, PRIMARY KEY (MILLISECONDS));").IsOk()); + UNIT_ASSERT(SqlToYql("USE plato; SELECT MILLISECONDS FROM MILLISECONDS").IsOk()); + } + + Y_UNIT_TEST(MicrosecondsKeywordNotReservedForNames) { + UNIT_ASSERT(SqlToYql("USE plato; CREATE TABLE MICROSECONDS (MICROSECONDS Uint32, PRIMARY KEY (MICROSECONDS));").IsOk()); + UNIT_ASSERT(SqlToYql("USE plato; SELECT MICROSECONDS FROM MICROSECONDS").IsOk()); + } + + Y_UNIT_TEST(NanosecondsKeywordNotReservedForNames) { + UNIT_ASSERT(SqlToYql("USE plato; CREATE TABLE NANOSECONDS (NANOSECONDS Uint32, PRIMARY KEY (NANOSECONDS));").IsOk()); + UNIT_ASSERT(SqlToYql("USE plato; SELECT NANOSECONDS FROM NANOSECONDS").IsOk()); + } + + Y_UNIT_TEST(Jubilee) { + NYql::TAstParseResult res = SqlToYql("USE plato; INSERT INTO Arcadia (r2000000) VALUES (\"2M GET!!!\");"); + UNIT_ASSERT(res.Root); + } + + Y_UNIT_TEST(QualifiedAsteriskBefore) { + NYql::TAstParseResult res = SqlToYql( + "PRAGMA DisableSimpleColumns;" + "select interested_table.*, LENGTH(value) AS megahelpful_len from plato.Input as interested_table;" + ); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + static bool seenStar = false; + if (word == "FlattenMembers") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("interested_table.")); + } else if (word == "SqlProjectItem") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(Quote("megahelpful_len"))); + UNIT_ASSERT_VALUES_EQUAL(seenStar, true); + } else if (word == "SqlProjectStarItem") { + seenStar = true; + } + }; + TWordCountHive elementStat = {{TString("FlattenMembers"), 0}, {TString("SqlProjectItem"), 0}, {TString("SqlProjectStarItem"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["FlattenMembers"]); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SqlProjectItem"]); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SqlProjectStarItem"]); + } + + Y_UNIT_TEST(QualifiedAsteriskAfter) { + NYql::TAstParseResult res = SqlToYql( + "PRAGMA DisableSimpleColumns;" + "select LENGTH(value) AS megahelpful_len, interested_table.* from plato.Input as interested_table;" + ); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + static bool seenStar = false; + if (word == "FlattenMembers") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("interested_table.")); + } else if (word == "SqlProjectItem") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(Quote("megahelpful_len"))); + UNIT_ASSERT_VALUES_EQUAL(seenStar, false); + } else if (word == "SqlProjectStarItem") { + seenStar = true; + } + }; + TWordCountHive elementStat = {{TString("FlattenMembers"), 0}, {TString("SqlProjectItem"), 0}, {TString("SqlProjectStarItem"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["FlattenMembers"]); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SqlProjectItem"]); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SqlProjectStarItem"]); + } + + Y_UNIT_TEST(QualifiedMembers) { + NYql::TAstParseResult res = SqlToYql("select interested_table.key, interested_table.value from plato.Input as interested_table;"); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + const bool fieldKey = TString::npos != line.find(Quote("key")); + const bool fieldValue = TString::npos != line.find(Quote("value")); + const bool refOnTable = TString::npos != line.find("interested_table."); + if (word == "SqlProjectItem") { + UNIT_ASSERT(fieldKey || fieldValue); + UNIT_ASSERT(!refOnTable); + } else if (word == "Write!") { + UNIT_ASSERT(fieldKey && fieldValue && !refOnTable); + } + }; + TWordCountHive elementStat = {{TString("SqlProjectStarItem"), 0}, {TString("SqlProjectItem"), 0}, {TString("Write!"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + UNIT_ASSERT_VALUES_EQUAL(0, elementStat["SqlProjectStarItem"]); + UNIT_ASSERT_VALUES_EQUAL(2, elementStat["SqlProjectItem"]); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write!"]); + } + + Y_UNIT_TEST(ExplainQueryPlan) { + UNIT_ASSERT(SqlToYql("EXPLAIN SELECT 1;").IsOk()); + UNIT_ASSERT(SqlToYql("EXPLAIN QUERY PLAN SELECT 1;").IsOk()); + } + + Y_UNIT_TEST(JoinParseCorrect) { + NYql::TAstParseResult res = SqlToYql( + "PRAGMA DisableSimpleColumns;" + " SELECT table_bb.*, table_aa.key as megakey" + " FROM plato.Input AS table_aa" + " JOIN plato.Input AS table_bb" + " ON table_aa.value == table_bb.value;" + ); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "SelectMembers") { + UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("table_aa.")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("table_bb.")); + } else if (word == "SqlProjectItem") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(Quote("megakey"))); + } else if (word == "SqlColumn") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(Quote("table_aa"))); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(Quote("key"))); + } + }; + TWordCountHive elementStat = {{TString("SqlProjectItem"), 0}, {TString("SqlProjectStarItem"), 0}, {TString("SelectMembers"), 0}, {TString("SqlColumn"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SqlProjectItem"]); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SqlProjectStarItem"]); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SelectMembers"]); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SqlColumn"]); + } + + Y_UNIT_TEST(Join3Table) { + NYql::TAstParseResult res = SqlToYql( + " PRAGMA DisableSimpleColumns;" + " SELECT table_bb.*, table_aa.key as gigakey, table_cc.* " + " FROM plato.Input AS table_aa" + " JOIN plato.Input AS table_bb ON table_aa.key == table_bb.key" + " JOIN plato.Input AS table_cc ON table_aa.subkey == table_cc.subkey;" + ); + Err2Str(res); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "SelectMembers") { + UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("table_aa.")); + UNIT_ASSERT(line.find("table_bb.") != TString::npos || line.find("table_cc.") != TString::npos); + } else if (word == "SqlProjectItem") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(Quote("gigakey"))); + } else if (word == "SqlColumn") { + const auto posTableAA = line.find(Quote("table_aa")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, posTableAA); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(Quote("key"))); + UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("table_aa", posTableAA + 3)); + } + }; + TWordCountHive elementStat = {{TString("SqlProjectItem"), 0}, {TString("SqlProjectStarItem"), 0}, {TString("SelectMembers"), 0}, {TString("SqlColumn"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SqlProjectItem"]); + UNIT_ASSERT_VALUES_EQUAL(2, elementStat["SqlProjectStarItem"]); + UNIT_ASSERT_VALUES_EQUAL(2, elementStat["SelectMembers"]); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SqlColumn"]); + } + + Y_UNIT_TEST(DisabledJoinCartesianProduct) { + NYql::TAstParseResult res = SqlToYql("pragma DisableAnsiImplicitCrossJoin; use plato; select * from A,B,C"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_STRINGS_EQUAL(res.Issues.ToString(), "<main>:1:67: Error: Cartesian product of tables is disabled. Please use explicit CROSS JOIN or enable it via PRAGMA AnsiImplicitCrossJoin\n"); + } + + Y_UNIT_TEST(JoinCartesianProduct) { + NYql::TAstParseResult res = SqlToYql("pragma AnsiImplicitCrossJoin; use plato; select * from A,B,C"); + UNIT_ASSERT(res.Root); + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "EquiJoin") { + auto pos = line.find("Cross"); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, pos); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("Cross", pos + 1)); + } + }; + TWordCountHive elementStat = {{TString("EquiJoin"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["EquiJoin"]); + } + + Y_UNIT_TEST(CreateAlterUserWithLoginNoLogin) { + auto reqCreateUser = SqlToYql(R"( + USE plato; + CREATE USER user1; + )"); + + UNIT_ASSERT(reqCreateUser.IsOk()); + + auto reqAlterUser = SqlToYql(R"( + USE plato; + ALTER USER user1; + )"); + + UNIT_ASSERT(!reqAlterUser.IsOk()); +#if ANTLR_VER == 3 + UNIT_ASSERT_STRING_CONTAINS(reqAlterUser.Issues.ToString(), "Error: Unexpected token ';' : cannot match to any predicted input..."); +#else + UNIT_ASSERT_STRING_CONTAINS(reqAlterUser.Issues.ToString(), "Error: mismatched input ';' expecting {ENCRYPTED, HASH, LOGIN, NOLOGIN, PASSWORD, RENAME, WITH}"); +#endif + + auto reqPasswordAndLogin = SqlToYql(R"( + USE plato; + CREATE USER user1 LOgin; + )"); + + UNIT_ASSERT(reqPasswordAndLogin.IsOk()); + + auto reqPasswordAndNoLogin = SqlToYql(R"( + USE plato; + CREATE USER user1 PASSWORD '123' NOLOGIN; + )"); + + UNIT_ASSERT(reqPasswordAndNoLogin.IsOk()); + + auto reqLogin = SqlToYql(R"( + USE plato; + CREATE USER user1 LOGIN; + )"); + + UNIT_ASSERT(reqLogin.IsOk()); + + auto reqNoLogin = SqlToYql(R"( + USE plato; + CREATE USER user1 NOLOGIN; + )"); + + UNIT_ASSERT(reqNoLogin.IsOk()); + + auto reqLoginNoLogin = SqlToYql(R"( + USE plato; + CREATE USER user1 LOGIN NOLOGIN; + )"); + + UNIT_ASSERT(!reqLoginNoLogin.IsOk()); + UNIT_ASSERT_STRING_CONTAINS(reqLoginNoLogin.Issues.ToString(), "Error: Conflicting or redundant options"); + + auto reqAlterLoginNoLogin = SqlToYql(R"( + USE plato; + CREATE USER user1 LOGIN; + ALTER USER user1 NOLOGIN; + )"); + + UNIT_ASSERT(reqAlterLoginNoLogin.IsOk()); + + auto reqAlterLoginNoLoginWithPassword = SqlToYql(R"( + USE plato; + CREATE USER user1 LOGIN; + ALTER USER user1 PASSWORD '321' NOLOGIN; + )"); + + UNIT_ASSERT(reqAlterLoginNoLoginWithPassword.IsOk()); + } + + Y_UNIT_TEST(CreateUserWithHash) { + auto reqCreateUser = SqlToYql(R"( + USE plato; + CREATE USER user1 HASH '{ + "hash": "p4ffeMugohqyBwyckYCK1TjJfz3LIHbKiGL+t+oEhzw=", + "salt": "U+tzBtgo06EBQCjlARA6Jg==", + "type": "argon2id" + }'; + )"); + + UNIT_ASSERT(reqCreateUser.IsOk()); + + auto reqCreateUserWithNoLogin = SqlToYql(R"( + USE plato; + CREATE USER user1 HASH '{ + "hash": "p4ffeMugohqyBwyckYCK1TjJfz3LIHbKiGL+t+oEhzw=", + "salt": "U+tzBtgo06EBQCjlARA6Jg==", + "type": "argon2id" + }' + NOLOGIN; + )"); + + UNIT_ASSERT(reqCreateUserWithNoLogin.IsOk()); + + auto reqCreateUserWithPassword = SqlToYql(R"( + USE plato; + CREATE USER user1 HASH '{ + "hash": "p4ffeMugohqyBwyckYCK1TjJfz3LIHbKiGL+t+oEhzw=", + "salt": "U+tzBtgo06EBQCjlARA6Jg==", + "type": "argon2id" + }' + PASSWORD '123'; + )"); + + UNIT_ASSERT(!reqCreateUserWithPassword.IsOk()); + UNIT_ASSERT_STRING_CONTAINS(reqCreateUserWithPassword.Issues.ToString(), "Error: Conflicting or redundant options"); + + auto reqAlterUser = SqlToYql(R"( + USE plato; + CREATE USER user1; + ALTER USER user1 HASH '{ + "hash": "p4ffeMugohqyBwyckYCK1TjJfz3LIHbKiGL+t+oEhzw=", + "salt": "U+tzBtgo06EBQCjlARA6Jg==", + "type": "argon2id" + }'; + )"); + + UNIT_ASSERT(reqAlterUser.IsOk()); + } + + Y_UNIT_TEST(CreateUserQoutas) { + { + auto req = SqlToYql(R"( + use plato; + CREATE USER user1 PASSWORD passwd; + )"); + +#if ANTLR_VER == 3 + TString error = "<main>:3:43: Error: Unexpected token 'passwd' : unexpected input : nothing is expected here\n\n"; +#else + TString error = "<main>:3:43: Error: mismatched input 'passwd' expecting {NULL, STRING_VALUE}\n"; +#endif + UNIT_ASSERT_VALUES_EQUAL(Err2Str(req), error); + UNIT_ASSERT(!req.Root); + } + + { + auto req = SqlToYql(R"( + use plato; + CREATE USER user2 PASSWORD NULL; + )"); + + UNIT_ASSERT(req.Root); + } + + { + auto req = SqlToYql(R"( + use plato; + CREATE USER user3 PASSWORD ''; + )"); + + UNIT_ASSERT(req.Root); + } + + + { + auto req = SqlToYql(R"( + use plato; + CREATE USER user1 PASSWORD 'password1'; + CREATE USER user2 PASSWORD 'password2'; + CREATE USER user3; + )"); + + UNIT_ASSERT(req.Root); + } + } + + Y_UNIT_TEST(JoinWithoutConcreteColumns) { + NYql::TAstParseResult res = SqlToYql( + " use plato;" + " SELECT a.v, b.value" + " FROM `Input1` VIEW `ksv` AS a" + " JOIN `Input2` AS b" + " ON a.k == b.key;" + ); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "SqlProjectItem") { + UNIT_ASSERT(line.find(Quote("a.v")) != TString::npos || line.find(Quote("b.value")) != TString::npos); + } else if (word == "SqlColumn") { + const auto posTableA = line.find(Quote("a")); + const auto posTableB = line.find(Quote("b")); + if (posTableA != TString::npos) { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(Quote("v"))); + } else { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, posTableB); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(Quote("value"))); + } + } + }; + TWordCountHive elementStat = {{TString("SqlProjectStarItem"), 0}, {TString("SqlProjectItem"), 0}, {TString("SqlColumn"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + UNIT_ASSERT_VALUES_EQUAL(0, elementStat["SqlProjectStarItem"]); + UNIT_ASSERT_VALUES_EQUAL(2, elementStat["SqlProjectItem"]); + UNIT_ASSERT_VALUES_EQUAL(2, elementStat["SqlColumn"]); + } + + Y_UNIT_TEST(JoinWithSameValues) { + NYql::TAstParseResult res = SqlToYql("SELECT a.value, b.value FROM plato.Input AS a JOIN plato.Input as b ON a.key == b.key;"); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "SqlProjectItem") { + const bool isValueFromA = TString::npos != line.find(Quote("a.value")); + const bool isValueFromB = TString::npos != line.find(Quote("b.value")); + UNIT_ASSERT(isValueFromA || isValueFromB); + } if (word == "Write!") { + UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("a.a.")); + UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("b.b.")); + } + }; + TWordCountHive elementStat = {{TString("SqlProjectStarItem"), 0}, {TString("SqlProjectItem"), 0}, {"Write!", 0}}; + VerifyProgram(res, elementStat, verifyLine); + UNIT_ASSERT_VALUES_EQUAL(0, elementStat["SqlProjectStarItem"]); + UNIT_ASSERT_VALUES_EQUAL(2, elementStat["SqlProjectItem"]); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write!"]); + } + + Y_UNIT_TEST(SameColumnsForDifferentTables) { + NYql::TAstParseResult res = SqlToYql("SELECT a.key, b.key FROM plato.Input as a JOIN plato.Input as b on a.key==b.key;"); + UNIT_ASSERT(res.Root); + } + + Y_UNIT_TEST(SameColumnsForDifferentTablesFullJoin) { + NYql::TAstParseResult res = SqlToYql("SELECT a.key, b.key, a.value, b.value FROM plato.Input AS a FULL JOIN plato.Input AS b USING(key);"); + UNIT_ASSERT(res.Root); + } + + Y_UNIT_TEST(JoinStreamLookupStrategyHint) { + { + NYql::TAstParseResult res = SqlToYql("SELECT * FROM plato.Input AS a JOIN /*+ StreamLookup() */ plato.Input AS b USING(key);"); + UNIT_ASSERT(res.Root); + } + //case insensitive + { + NYql::TAstParseResult res = SqlToYql("SELECT * FROM plato.Input AS a JOIN /*+ streamlookup() */ plato.Input AS b USING(key);"); + UNIT_ASSERT(res.Root); + } + } + + Y_UNIT_TEST(JoinConflictingStrategyHint) { + { + NYql::TAstParseResult res = SqlToYql("SELECT * FROM plato.Input AS a JOIN /*+ StreamLookup() */ /*+ Merge() */ plato.Input AS b USING(key);"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_STRINGS_EQUAL(res.Issues.ToString(), "<main>:1:91: Error: Conflicting join strategy hints\n"); + } + } + + Y_UNIT_TEST(JoinDuplicatingStrategyHint) { + { + NYql::TAstParseResult res = SqlToYql("SELECT * FROM plato.Input AS a JOIN /*+ StreamLookup() */ /*+ StreamLookup() */ plato.Input AS b USING(key);"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_STRINGS_EQUAL(res.Issues.ToString(), "<main>:1:98: Error: Duplicate join strategy hint\n"); + } + } + + Y_UNIT_TEST(WarnCrossJoinStrategyHint) { + NYql::TAstParseResult res = SqlToYql("SELECT * FROM plato.Input AS a CROSS JOIN /*+ merge() */ plato.Input AS b;"); + UNIT_ASSERT(res.Root); + UNIT_ASSERT_STRINGS_EQUAL(res.Issues.ToString(), "<main>:1:32: Warning: Non-default join strategy will not be used for CROSS JOIN, code: 4534\n"); + } + + Y_UNIT_TEST(WarnCartesianProductStrategyHint) { + NYql::TAstParseResult res = SqlToYql("pragma AnsiImplicitCrossJoin; use plato; SELECT * FROM A, /*+ merge() */ B;"); + UNIT_ASSERT(res.Root); + UNIT_ASSERT_STRINGS_EQUAL(res.Issues.ToString(), "<main>:1:74: Warning: Non-default join strategy will not be used for CROSS JOIN, code: 4534\n"); + } + + Y_UNIT_TEST(WarnUnknownJoinStrategyHint) { + NYql::TAstParseResult res = SqlToYql("SELECT * FROM plato.Input AS a JOIN /*+ xmerge() */ plato.Input AS b USING (key);"); + UNIT_ASSERT(res.Root); + UNIT_ASSERT_STRINGS_EQUAL(res.Issues.ToString(), "<main>:1:41: Warning: Unsupported join hint: xmerge, code: 4534\n"); + } + + Y_UNIT_TEST(ReverseLabels) { + NYql::TAstParseResult res = SqlToYql("select in.key as subkey, subkey as key from plato.Input as in;"); + UNIT_ASSERT(res.Root); + } + + Y_UNIT_TEST(AutogenerationAliasWithoutCollisionConflict1) { + NYql::TAstParseResult res = SqlToYql("select LENGTH(Value), key as column1 from plato.Input;"); + UNIT_ASSERT(res.Root); + } + + Y_UNIT_TEST(AutogenerationAliasWithoutCollision2Conflict2) { + NYql::TAstParseResult res = SqlToYql("select key as column0, LENGTH(Value) from plato.Input;"); + UNIT_ASSERT(res.Root); + } + + Y_UNIT_TEST(InputAliasForQualifiedAsterisk) { + NYql::TAstParseResult res = SqlToYql("use plato; select zyuzya.*, key from plato.Input as zyuzya;"); + UNIT_ASSERT(res.Root); + } + + Y_UNIT_TEST(SelectSupportsResultColumnsWithTrailingComma) { + NYql::TAstParseResult res = SqlToYql("select a, b, c, from plato.Input;"); + UNIT_ASSERT(res.Root); + } + + Y_UNIT_TEST(SelectOrderByLabeledColumn) { + NYql::TAstParseResult res = SqlToYql("pragma DisableOrderedColumns; select key as goal from plato.Input order by goal"); + UNIT_ASSERT(res.Root); + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "DataSource") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("plato")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("Input")); + + UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("goal")); + } else if (word == "Sort") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("goal")); + + UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("key")); + } + }; + TWordCountHive elementStat = {{TString("DataSource"), 0}, {TString("Sort"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["DataSource"]); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Sort"]); + } + + Y_UNIT_TEST(SelectOrderBySimpleExpr) { + NYql::TAstParseResult res = SqlToYql("select a from plato.Input order by a + a"); + UNIT_ASSERT(res.Root); + } + + Y_UNIT_TEST(SelectAssumeOrderByTableRowAccess) { + NYql::TAstParseResult res = SqlToYql("$key = 'foo';select * from plato.Input assume order by TableRow().$key"); + UNIT_ASSERT(res.Root); + } + + Y_UNIT_TEST(SelectOrderByDuplicateLabels) { + NYql::TAstParseResult res = SqlToYql("select a from plato.Input order by a, a"); + UNIT_ASSERT(res.Root); + } + + Y_UNIT_TEST(SelectOrderByExpression) { + NYql::TAstParseResult res = SqlToYql("select * from plato.Input as i order by cast(key as uint32) + cast(subkey as uint32)"); + UNIT_ASSERT(res.Root); + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Sort") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("\"+MayWarn\"")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("key")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("subkey")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("(Bool 'true)")); + + UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("i.key")); + UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("i.subkey")); + } + }; + TWordCountHive elementStat = {{TString("Sort"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Sort"]); + } + + Y_UNIT_TEST(SelectOrderByExpressionDesc) { + NYql::TAstParseResult res = SqlToYql("pragma disablesimplecolumns; select i.*, key, subkey from plato.Input as i order by cast(i.key as uint32) - cast(i.subkey as uint32) desc"); + UNIT_ASSERT(res.Root); + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Sort") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("\"-MayWarn\"")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("\"key\"")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("\"subkey\"")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("(Bool 'false)")); + } else if (word == "Write!") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'columns")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("\"key\"")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("\"subkey\"")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("prefix")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("\"i.\"")); + } + }; + TWordCountHive elementStat = {{TString("Sort"), 0}, {TString("Write!"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Sort"]); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write!"]); + } + + Y_UNIT_TEST(SelectOrderByExpressionAsc) { + NYql::TAstParseResult res = SqlToYql("select i.key, i.subkey from plato.Input as i order by cast(key as uint32) % cast(i.subkey as uint32) asc"); + UNIT_ASSERT(res.Root); + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Sort") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("\"%MayWarn\"")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("\"key\"")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("\"subkey\"")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("(Bool 'true)")); + } else if (word == "Write!") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'columns")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("\"key\"")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("\"subkey\"")); + UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("i.")); + } + }; + TWordCountHive elementStat = {{TString("Sort"), 0}, {TString("Write!"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Sort"]); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write!"]); + } + + Y_UNIT_TEST(ReferenceToKeyInSubselect) { + NYql::TAstParseResult res = SqlToYql("select b.key from (select a.key from plato.Input as a) as b;"); + UNIT_ASSERT(res.Root); + } + + Y_UNIT_TEST(OrderByCastValue) { + NYql::TAstParseResult res = SqlToYql("select i.key, i.subkey from plato.Input as i order by cast(key as uint32) desc;"); + UNIT_ASSERT(res.Root); + } + + Y_UNIT_TEST(GroupByCastValue) { + NYql::TAstParseResult res = SqlToYql("select count(1) from plato.Input as i group by cast(key as uint8);"); + UNIT_ASSERT(res.Root); + } + + Y_UNIT_TEST(KeywordInSelectColumns) { + NYql::TAstParseResult res = SqlToYql("select in, s.check from (select 1 as in, \"test\" as check) as s;"); + UNIT_ASSERT(res.Root); + } + + Y_UNIT_TEST(SelectAllGroupBy) { + NYql::TAstParseResult res = SqlToYql("select * from plato.Input group by subkey;"); + UNIT_ASSERT(res.Root); + } + + Y_UNIT_TEST(CreateObjectWithFeatures) { + NYql::TAstParseResult res = SqlToYql("USE plato; CREATE OBJECT secretId (TYPE SECRET) WITH (Key1=Value1, K2=V2);"); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'('\"K2\" '\"V2\") '('\"Key1\" '\"Value1\")")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("createObject")); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0}, {TString("SECRET"), 0} }; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SECRET"]); + } + + Y_UNIT_TEST(CreateObjectIfNotExists) { + NYql::TAstParseResult res = SqlToYql("USE plato; CREATE OBJECT IF NOT EXISTS secretId (TYPE SECRET) WITH (Key1=Value1, K2=V2);"); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("createObjectIfNotExists")); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0}, {TString("SECRET"), 0} }; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SECRET"]); + } + + Y_UNIT_TEST(CreateObjectWithFeaturesStrings) { + NYql::TAstParseResult res = SqlToYql("USE plato; CREATE OBJECT secretId (TYPE SECRET) WITH (Key1=\"Value1\", K2='V2', K3=V3, K4='', K5=`aaa`, K6='a\\'aa');"); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'('\"K2\" '\"V2\") '('\"K3\" '\"V3\") '('\"K4\" '\"\") '('\"K5\" '\"aaa\") '('\"K6\" '\"a'aa\") '('\"Key1\" '\"Value1\")")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("createObject")); + } + }; + + TWordCountHive elementStat = {{TString("Write"), 0}, {TString("SECRET"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + } + + Y_UNIT_TEST(UpsertObjectWithFeatures) { + NYql::TAstParseResult res = SqlToYql("USE plato; UPSERT OBJECT secretId (TYPE SECRET) WITH (Key1=Value1, K2=V2);"); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'('\"K2\" '\"V2\") '('\"Key1\" '\"Value1\")")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("upsertObject")); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0}, {TString("SECRET"), 0} }; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SECRET"]); + } + + Y_UNIT_TEST(CreateObjectWithFeaturesAndFlags) { + NYql::TAstParseResult res = SqlToYql("USE plato; CREATE OBJECT secretId (TYPE SECRET) WITH (Key1=Value1, K2=V2, RECURSE);"); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'('\"Key1\" '\"Value1\") '('\"RECURSE\")")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("createObject")); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0}, {TString("SECRET"), 0} }; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SECRET"]); + } + + Y_UNIT_TEST(Select1Type) { + NYql::TAstParseResult res = SqlToYql("SELECT 1 type;"); + UNIT_ASSERT(res.Root); + } + + Y_UNIT_TEST(SelectTableType) { + NYql::TAstParseResult res = SqlToYql("USE plato; SELECT * from T type;"); + UNIT_ASSERT(res.Root); + } + + Y_UNIT_TEST(CreateObjectNoFeatures) { + NYql::TAstParseResult res = SqlToYql("USE plato; CREATE OBJECT secretId (TYPE SECRET);"); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("'features")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("createObject")); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0}, {TString("SECRET"), 0} }; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SECRET"]); + } + + Y_UNIT_TEST(AlterObjectWithFeatures) { + NYql::TAstParseResult res = SqlToYql( + "USE plato;\n" + "declare $path as String;\n" + "ALTER OBJECT secretId (TYPE SECRET) SET (Key1=$path, K2=V2);" + ); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'features")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'\"Key1\" (EvaluateAtom \"$path\"")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'\"K2\" '\"V2\"")); + + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("alterObject")); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0}, {TString("SECRET"), 0} }; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SECRET"]); + } + + Y_UNIT_TEST(AlterObjectNoFeatures) { + NYql::TAstParseResult res = SqlToYql("USE plato; ALTER OBJECT secretId (TYPE SECRET);"); + UNIT_ASSERT(!res.Root); + } + + Y_UNIT_TEST(DropObjectNoFeatures) { + NYql::TAstParseResult res = SqlToYql("USE plato; DROP OBJECT secretId (TYPE SECRET);"); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("'features")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("dropObject")); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0}, {TString("SECRET"), 0} }; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SECRET"]); + } + + Y_UNIT_TEST(DropObjectWithFeatures) { + NYql::TAstParseResult res = SqlToYql("USE plato; DROP OBJECT secretId (TYPE SECRET) WITH (A, B, C);"); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'features")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("dropObject")); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0}, {TString("SECRET"), 0} }; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SECRET"]); + } + + Y_UNIT_TEST(DropObjectWithOneOption) { + NYql::TAstParseResult res = SqlToYql("USE plato; DROP OBJECT secretId (TYPE SECRET) WITH OVERRIDE;"); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'features")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'\"OVERRIDE\"")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("dropObject")); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0}, {TString("SECRET"), 0} }; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SECRET"]); + } + + Y_UNIT_TEST(DropObjectIfExists) { + NYql::TAstParseResult res = SqlToYql("USE plato; DROP OBJECT IF EXISTS secretId (TYPE SECRET);"); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("dropObjectIfExists")); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0}, {TString("SECRET"), 0} }; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SECRET"]); + } + + Y_UNIT_TEST(PrimaryKeyParseCorrect) { + NYql::TAstParseResult res = SqlToYql("USE plato; CREATE TABLE tableName (Key Uint32, Subkey Int64, Value String, PRIMARY KEY (Key, Subkey));"); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("\"Key\"")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("\"Subkey\"")); + } + }; + + TWordCountHive elementStat = {{TString("Write"), 0}, {TString("primarykey"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["primarykey"]); + } + + Y_UNIT_TEST(AlterDatabaseAst) { + NYql::TAstParseResult request = SqlToYql("USE plato; ALTER DATABASE `/Root/test` OWNER TO user1;"); + UNIT_ASSERT(request.IsOk()); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + Y_UNUSED(word); + + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find( + R"(let world (Write! world sink (Key '('databasePath (String '"/Root/test"))) (Void) '('('mode 'alterDatabase) '('owner '"user1"))))" + )); + }; + + TWordCountHive elementStat({TString("\'mode \'alterDatabase")}); + VerifyProgram(request, elementStat, verifyLine); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["\'mode \'alterDatabase"]); + } + + Y_UNIT_TEST(CreateTableNonNullableYqlTypeAstCorrect) { + NYql::TAstParseResult res = SqlToYql("USE plato; CREATE TABLE t (a int32 not null);"); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write!") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, + line.find(R"__((Write! world sink (Key '('tablescheme (String '"t"))) values '('('mode 'create) '('columns '('('"a" (DataType 'Int32) '('columnConstrains '('('not_null))) '())))))))__")); + } + }; + + TWordCountHive elementStat = {{TString("Write!"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write!"]); + } + + Y_UNIT_TEST(CreateTableNullableYqlTypeAstCorrect) { + NYql::TAstParseResult res = SqlToYql("USE plato; CREATE TABLE t (a int32);"); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write!") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, + line.find(R"__((Write! world sink (Key '('tablescheme (String '"t"))) values '('('mode 'create) '('columns '('('"a" (AsOptionalType (DataType 'Int32)) '('columnConstrains '()) '()))))))__")); + } + }; + + TWordCountHive elementStat = {{TString("Write!"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write!"]); + } + + Y_UNIT_TEST(CreateTableNonNullablePgTypeAstCorrect) { + NYql::TAstParseResult res = SqlToYql("USE plato; CREATE TABLE t (a pg_int4 not null);"); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write!") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, + line.find(R"__((Write! world sink (Key '('tablescheme (String '"t"))) values '('('mode 'create) '('columns '('('"a" (PgType '_int4) '('columnConstrains '('('not_null))) '())))))))__")); + } + }; + + TWordCountHive elementStat = {{TString("Write!"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write!"]); + } + + Y_UNIT_TEST(CreateTableNullablePgTypeAstCorrect) { + NYql::TAstParseResult res = SqlToYql("USE plato; CREATE TABLE t (a pg_int4);"); + UNIT_ASSERT(res.Root); + + res.Root->PrettyPrintTo(Cout, PRETTY_FLAGS); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write!") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, + line.find(R"__((Write! world sink (Key '('tablescheme (String '"t"))) values '('('mode 'create) '('columns '('('"a" (AsOptionalType (PgType '_int4)) '('columnConstrains '()) '()))))))__")); + } + }; + + TWordCountHive elementStat = {{TString("Write!"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write!"]); + } + + Y_UNIT_TEST(CreateTableNullPkColumnsAreAllowed) { + NYql::TAstParseResult res = SqlToYql("USE plato; CREATE TABLE t (a int32, primary key(a));"); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write!") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, + line.find(R"__((Write! world sink (Key '('tablescheme (String '"t"))) values '('('mode 'create) '('columns '('('"a" (AsOptionalType (DataType 'Int32)) '('columnConstrains '()) '()))) '('primarykey '('"a")))))__")); + } + }; + + TWordCountHive elementStat = {{TString("Write!"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write!"]); + } + + Y_UNIT_TEST(CreateTableNotNullPkColumnsAreIdempotentAstCorrect) { + NYql::TAstParseResult res = SqlToYql("USE plato; CREATE TABLE t (a int32 not null, primary key(a));"); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write!") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, + line.find(R"__((Write! world sink (Key '('tablescheme (String '"t"))) values '('('mode 'create) '('columns '('('"a" (DataType 'Int32) '('columnConstrains '('('not_null))) '()))) '('primarykey '('"a"))))))__")); + } + }; + + TWordCountHive elementStat = {{TString("Write!"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write!"]); + } + + Y_UNIT_TEST(CreateTableWithIfNotExists) { + NYql::TAstParseResult res = SqlToYql("USE plato; CREATE TABLE IF NOT EXISTS t (a int32, primary key(a));"); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write!") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, + line.find(R"__((Write! world sink (Key '('tablescheme (String '"t"))) values '('('mode 'create_if_not_exists) '('columns '('('"a" (AsOptionalType (DataType 'Int32)) '('columnConstrains '()) '()))) '('primarykey '('"a")))))__")); + } + }; + + TWordCountHive elementStat = {{TString("Write!"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write!"]); + } + + Y_UNIT_TEST(CreateTempTable) { + NYql::TAstParseResult res = SqlToYql("USE plato; CREATE TEMP TABLE t (a int32, primary key(a));"); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write!") { + UNIT_ASSERT_VALUES_UNEQUAL_C(TString::npos, + line.find(R"__((Write! world sink (Key '('tablescheme (String '"t"))) values '('('mode 'create) '('columns '('('"a" (AsOptionalType (DataType 'Int32)) '('columnConstrains '()) '()))) '('primarykey '('"a")) '('temporary))))__"), line); + } + }; + + TWordCountHive elementStat = {{TString("Write!"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write!"]); + } + + Y_UNIT_TEST(CreateTemporaryTable) { + NYql::TAstParseResult res = SqlToYql("USE plato; CREATE TEMPORARY TABLE t (a int32, primary key(a));"); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write!") { + UNIT_ASSERT_VALUES_UNEQUAL_C(TString::npos, + line.find(R"__((Write! world sink (Key '('tablescheme (String '"t"))) values '('('mode 'create) '('columns '('('"a" (AsOptionalType (DataType 'Int32)) '('columnConstrains '()) '()))) '('primarykey '('"a")) '('temporary))))__"), line); + } + }; + + TWordCountHive elementStat = {{TString("Write!"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write!"]); + } + + Y_UNIT_TEST(CreateTableWithoutTypes) { + NYql::TAstParseResult res = SqlToYql("USE plato; CREATE TABLE t (a, primary key(a));"); + UNIT_ASSERT(!res.Root); + } + + Y_UNIT_TEST(CreateTableAsSelectWithTypes) { + NYql::TAstParseResult res = SqlToYql("USE plato; CREATE TABLE t (a int32, primary key(a)) AS SELECT * FROM ts;"); + UNIT_ASSERT(!res.Root); + } + + Y_UNIT_TEST(CreateTableAsSelect) { + NYql::TAstParseResult res = SqlToYql("USE plato; CREATE TABLE t (a, b, primary key(a)) AS SELECT * FROM ts;"); + UNIT_ASSERT_C(res.Root, res.Issues.ToString()); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write!") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, + line.find(R"__((let world (Write! world sink (Key '('tablescheme (String '"t"))) values '('('mode 'create) '('columns '('('"a") '('"b"))) '('primarykey '('"a"))))))__")); + } + if (word == "Read!") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, + line.find(R"__((Read! world (DataSource '"yt" '"plato") (MrTableConcat (Key '('table (String '"ts")))))__")); + } + }; + + TWordCountHive elementStat = {{TString("Write!"), 0}, {TString("Read!"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write!"]); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Read!"]); + } + + Y_UNIT_TEST(CreateTableAsSelectOnlyPrimary) { + NYql::TAstParseResult res = SqlToYql("USE plato; CREATE TABLE t (primary key(a)) AS SELECT * FROM ts;"); + UNIT_ASSERT_C(res.Root, res.Issues.ToString()); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write!") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, + line.find(R"__((let world (Write! world sink (Key '('tablescheme (String '"t"))) values '('('mode 'create) '('columns '()) '('primarykey '('"a"))))))__")); + } + if (word == "Read!") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, + line.find(R"__((Read! world (DataSource '"yt" '"plato") (MrTableConcat (Key '('table (String '"ts")))))__")); + } + }; + + TWordCountHive elementStat = {{TString("Write!"), 0}, {TString("Read!"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write!"]); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Read!"]); + } + + Y_UNIT_TEST(CreateTableAsValuesFail) { + NYql::TAstParseResult res = SqlToYql("USE plato; CREATE TABLE t (a, primary key(a)) AS VALUES (1), (2);"); + UNIT_ASSERT(!res.Root); + } + + Y_UNIT_TEST(CreateTableDuplicatedPkColumnsFail) { + NYql::TAstParseResult res = SqlToYql("USE plato; CREATE TABLE t (a int32 not null, primary key(a, a));"); + UNIT_ASSERT(!res.Root); + } + + Y_UNIT_TEST(DeleteFromTableByKey) { + NYql::TAstParseResult res = SqlToYql("delete from plato.Input where key = 200;", 10, "kikimr"); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("('mode 'delete)")); + } + }; + + TWordCountHive elementStat = {{TString("Write"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } + + Y_UNIT_TEST(DeleteFromTable) { + NYql::TAstParseResult res = SqlToYql("delete from plato.Input;", 10, "kikimr"); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("('mode 'delete)")); + } + }; + + TWordCountHive elementStat = {{TString("Write"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } + + Y_UNIT_TEST(DeleteFromTableBatch) { + NYql::TAstParseResult res = SqlToYql("batch delete from plato.Input;", 10, "kikimr"); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("('mode 'delete)")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'('is_batch 'true)")); + } + }; + + TWordCountHive elementStat = {{TString("Write"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } + + Y_UNIT_TEST(DeleteFromTableOnValues) { + NYql::TAstParseResult res = SqlToYql("delete from plato.Input on (key) values (1);", + 10, "kikimr"); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("('mode 'delete_on)")); + } + }; + + TWordCountHive elementStat = {{TString("Write"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } + + Y_UNIT_TEST(DeleteFromTableOnSelect) { + NYql::TAstParseResult res = SqlToYql( + "delete from plato.Input on select key from plato.Input where value > 0;", 10, "kikimr"); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("('mode 'delete_on)")); + } + }; + + TWordCountHive elementStat = {{TString("Write"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } + + Y_UNIT_TEST(DeleteFromTableOnBatch) { + NYql::TAstParseResult res = SqlToYql("batch delete from plato.Input on (key) values (1);", + 10, "kikimr"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:6: Error: BATCH DELETE is unsupported with ON\n"); + } + + Y_UNIT_TEST(UpdateByValues) { + NYql::TAstParseResult res = SqlToYql("update plato.Input set key = 777, value = 'cool' where key = 200;", 10, "kikimr"); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("('mode 'update)")); + } else if (word == "AsStruct") { + const bool isKey = line.find("key") != TString::npos; + const bool isValue = line.find("value") != TString::npos; + UNIT_ASSERT(isKey || isValue); + if (isKey) { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(Quote("777"))); + } else if (isValue) { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(Quote("cool"))); + } + } + }; + + TWordCountHive elementStat = {{TString("Write"), 0}, {TString("AsStruct"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["AsStruct"]); + } + + Y_UNIT_TEST(UpdateByValuesBatch) { + NYql::TAstParseResult res = SqlToYql("batch update plato.Input set key = 777, value = 'cool' where key = 200;", 10, "kikimr"); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("('mode 'update)")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'('is_batch 'true)")); + } + }; + + TWordCountHive elementStat = {{TString("Write"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } + + Y_UNIT_TEST(UpdateByMultiValues) { + NYql::TAstParseResult res = SqlToYql("update plato.Input set (key, value, subkey) = ('2','ddd',':') where key = 200;", 10, "kikimr"); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("('mode 'update)")); + } else if (word == "AsStruct") { + const bool isKey = line.find("key") != TString::npos; + const bool isSubkey = line.find("subkey") != TString::npos; + const bool isValue = line.find("value") != TString::npos; + UNIT_ASSERT(isKey || isSubkey || isValue); + if (isKey && !isSubkey) { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(Quote("2"))); + } else if (isSubkey) { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(Quote(":"))); + } else if (isValue) { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(Quote("ddd"))); + } + } + }; + + TWordCountHive elementStat = {{TString("Write"), 0}, {TString("AsStruct"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["AsStruct"]); + } + + Y_UNIT_TEST(UpdateBySelect) { + NYql::TAstParseResult res = SqlToYql("update plato.Input set (key, value, subkey) = (select key, value, subkey from plato.Input where key = 911) where key = 200;", 10, "kikimr"); + UNIT_ASSERT(res.Root); + + int lineIndex = 0; + int writeLineIndex = -1; + bool found = false; + + TVerifyLineFunc verifyLine = [&lineIndex, &writeLineIndex, &found](const TString& word, const TString& line) { + if (word == "Write") { + writeLineIndex = lineIndex; + found = line.find("('mode 'update)") != TString::npos; + } else if (word == "mode") { + found |= lineIndex == writeLineIndex + 1 && line.find("('mode 'update)") != TString::npos; + UNIT_ASSERT(found); + } + + ++lineIndex; + }; + + TWordCountHive elementStat = {{TString("Write"), 0}, {TString("mode"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } + + Y_UNIT_TEST(UpdateSelfModifyAll) { + NYql::TAstParseResult res = SqlToYql("update plato.Input set subkey = subkey + 's';", 10, "kikimr"); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("('mode 'update)")); + } else if (word == "AsStruct") { + const bool isSubkey = line.find("subkey") != TString::npos; + UNIT_ASSERT(isSubkey); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(Quote("subkey"))); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(Quote("s"))); + } + }; + + TWordCountHive elementStat = {{TString("Write"), 0}, {TString("AsStruct"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["AsStruct"]); + } + + Y_UNIT_TEST(UpdateOnValues) { + NYql::TAstParseResult res = SqlToYql("update plato.Input on (key, value) values (5, 'cool')", 10, "kikimr"); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("('mode 'update_on)")); + } + }; + + TWordCountHive elementStat = {{TString("Write"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } + + Y_UNIT_TEST(UpdateOnSelect) { + NYql::TAstParseResult res = SqlToYql( + "update plato.Input on select key, value + 1 as value from plato.Input", 10, "kikimr"); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("('mode 'update_on)")); + } + }; + + TWordCountHive elementStat = {{TString("Write"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } + + Y_UNIT_TEST(UpdateOnBatch) { + NYql::TAstParseResult res = SqlToYql("batch update plato.Input on (key, value) values (5, 'cool')", 10, "kikimr"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:6: Error: BATCH UPDATE is unsupported with ON\n"); + } + + Y_UNIT_TEST(UnionAllTest) { + NYql::TAstParseResult res = SqlToYql("PRAGMA DisableEmitUnionMerge; SELECT key FROM plato.Input UNION ALL select subkey FROM plato.Input;"); + UNIT_ASSERT(res.Root); + + TWordCountHive elementStat = {{TString("UnionAll"), 0}}; + VerifyProgram(res, elementStat, {}); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["UnionAll"]); + } + + Y_UNIT_TEST(UnionAllMergeTest) { + NYql::TAstParseResult res = SqlToYql("PRAGMA EmitUnionMerge; SELECT key FROM plato.Input UNION ALL select subkey FROM plato.Input;"); + UNIT_ASSERT(res.Root); + + TWordCountHive elementStat = {{TString("UnionMerge"), 0}}; + VerifyProgram(res, elementStat, {}); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["UnionMerge"]); + } + + Y_UNIT_TEST(UnionTest) { + NYql::TAstParseResult res = SqlToYql("SELECT key FROM plato.Input UNION select subkey FROM plato.Input;"); + UNIT_ASSERT(res.Root); + + TWordCountHive elementStat = {{TString("Union"), 0}}; + VerifyProgram(res, elementStat, {}); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Union"]); + } + + Y_UNIT_TEST(UnionAggregationTest) { + NYql::TAstParseResult res = SqlToYql(R"( + PRAGMA DisableEmitUnionMerge; + SELECT 1 + UNION ALL + SELECT 1 UNION ALL SELECT 1 UNION ALL SELECT 1 + UNION + SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 + UNION ALL + SELECT 1 UNION ALL SELECT 1 UNION ALL SELECT 1; + )"); + UNIT_ASSERT(res.Root); + + TWordCountHive elementStat = {{TString("Union"), 0}, {TString("UnionAll"), 0}}; + VerifyProgram(res, elementStat, {}); + UNIT_ASSERT_VALUES_EQUAL(2, elementStat["UnionAll"]); + UNIT_ASSERT_VALUES_EQUAL(3, elementStat["Union"]); + } + + Y_UNIT_TEST(UnionMergeAggregationTest) { + NYql::TAstParseResult res = SqlToYql(R"( + PRAGMA EmitUnionMerge; + SELECT 1 + UNION ALL + SELECT 1 UNION ALL SELECT 1 UNION ALL SELECT 1 + UNION + SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION SELECT 1 + UNION ALL + SELECT 1 UNION ALL SELECT 1 UNION ALL SELECT 1; + )"); + UNIT_ASSERT(res.Root); + + TWordCountHive elementStat = {{TString("Union"), 0}, {TString("UnionMerge"), 0}}; + VerifyProgram(res, elementStat, {}); + UNIT_ASSERT_VALUES_EQUAL(2, elementStat["UnionMerge"]); + UNIT_ASSERT_VALUES_EQUAL(3, elementStat["Union"]); + } + + Y_UNIT_TEST(DeclareDecimalParameter) { + NYql::TAstParseResult res = SqlToYql("declare $value as Decimal(22,9); select $value as cnt;"); + UNIT_ASSERT(res.Root); + } + + Y_UNIT_TEST(SimpleGroupBy) { + NYql::TAstParseResult res = SqlToYql("select count(1),z from plato.Input group by key as z order by z;"); + UNIT_ASSERT(res.Root); + } + + Y_UNIT_TEST(EmptyColumnName0) { + /// Now it's parsed well and error occur on validate step like "4:31:Empty struct member name is not allowed" in "4:31:Function: AddMember" + NYql::TAstParseResult res = SqlToYql("insert into plato.Output (``, list1) values (0, AsList(0, 1, 2));"); + /// Verify that parsed well without crash + UNIT_ASSERT(res.Root); + } + + Y_UNIT_TEST(KikimrRollback) { + NYql::TAstParseResult res = SqlToYql("use plato; select * from Input; rollback;", 10, "kikimr"); + UNIT_ASSERT(res.Root); + + TWordCountHive elementStat = {{TString("rollback"), 0}}; + VerifyProgram(res, elementStat); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["rollback"]); + } + + Y_UNIT_TEST(PragmaFile) { + NYql::TAstParseResult res = SqlToYql(R"(pragma file("HW", "sbr:181041334");)"); + UNIT_ASSERT(res.Root); + + TWordCountHive elementStat = {{TString(R"((let world (Configure! world (DataSource '"config") '"AddFileByUrl" '"HW" '"sbr:181041334")))"), 0}}; + VerifyProgram(res, elementStat); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat.cbegin()->second); + } + + Y_UNIT_TEST(DoNotCrashOnNamedInFilter) { + NYql::TAstParseResult res = SqlToYql("USE plato; $all = ($table_name) -> { return true; }; SELECT * FROM FILTER(Input, $all)"); + UNIT_ASSERT(res.Root); + } + + Y_UNIT_TEST(PragmasFileAndUdfOrder) { + NYql::TAstParseResult res = SqlToYql(R"( + PRAGMA file("libvideoplayers_udf.so", "https://proxy.sandbox.yandex-team.ru/235185290"); + PRAGMA udf("libvideoplayers_udf.so"); + )"); + UNIT_ASSERT(res.Root); + + const auto programm = GetPrettyPrint(res); + const auto file = programm.find("AddFileByUrl"); + const auto udfs = programm.find("ImportUdfs"); + UNIT_ASSERT(file < udfs); + } + + Y_UNIT_TEST(ProcessUserType) { + NYql::TAstParseResult res = SqlToYql("process plato.Input using Kikimr::PushData(TableRows());", 1, TString(NYql::KikimrProviderName)); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Kikimr.PushData") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("TupleType")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("TypeOf")); + } + }; + + TWordCountHive elementStat = {{TString("Kikimr.PushData"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Kikimr.PushData"]); + } + + Y_UNIT_TEST(ProcessUserTypeAuth) { + NYql::TAstParseResult res = SqlToYql("process plato.Input using YDB::PushData(TableRows(), AsTuple('oauth', SecureParam('api:oauth')));", 1, TString(NYql::KikimrProviderName)); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "YDB.PushData") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("TupleType")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("TypeOf")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("api:oauth")); + } + }; + + TWordCountHive elementStat = {{TString("YDB.PushData"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["YDB.PushData"]); + } + + Y_UNIT_TEST(SelectStreamRtmr) { + NYql::TAstParseResult res = SqlToYql( + "USE plato; INSERT INTO Output SELECT STREAM key FROM Input;", + 10, TString(NYql::RtmrProviderName)); + UNIT_ASSERT(res.Root); + + res = SqlToYql( + "USE plato; INSERT INTO Output SELECT key FROM Input;", + 10, TString(NYql::RtmrProviderName)); + UNIT_ASSERT(res.Root); + } + + Y_UNIT_TEST(SelectStreamRtmrJoinWithYt) { + NYql::TAstParseResult res = SqlToYql( + "USE plato; INSERT INTO Output SELECT STREAM key FROM Input LEFT JOIN hahn.ttt as t ON Input.key = t.Name;", + 10, TString(NYql::RtmrProviderName)); + UNIT_ASSERT(res.Root); + } + + Y_UNIT_TEST(SelectStreamNonRtmr) { + NYql::TAstParseResult res = SqlToYql( + "USE plato; INSERT INTO Output SELECT STREAM key FROM Input;", + 10); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:31: Error: SELECT STREAM is unsupported for non-streaming sources\n"); + } + + Y_UNIT_TEST(GroupByHopRtmr) { + NYql::TAstParseResult res = SqlToYql(R"( + USE plato; INSERT INTO Output SELECT key, SUM(value) AS value FROM Input + GROUP BY key, HOP(subkey, "PT10S", "PT30S", "PT20S"); + )", 10, TString(NYql::RtmrProviderName)); + UNIT_ASSERT(res.Root); + } + + Y_UNIT_TEST(GroupByHopRtmrSubquery) { + // 'use plato' intentially avoided + NYql::TAstParseResult res = SqlToYql(R"( + SELECT COUNT(*) AS value FROM (SELECT * FROM plato.Input) + GROUP BY HOP(Data, "PT10S", "PT30S", "PT20S") + )", 10, TString(NYql::RtmrProviderName)); + UNIT_ASSERT(res.Root); + } + + Y_UNIT_TEST(GroupByHopRtmrSubqueryBinding) { + NYql::TAstParseResult res = SqlToYql(R"( + USE plato; + $q = SELECT * FROM Input; + INSERT INTO Output SELECT STREAM * FROM ( + SELECT COUNT(*) AS value FROM $q + GROUP BY HOP(Data, "PT10S", "PT30S", "PT20S") + ); + )", 10, TString(NYql::RtmrProviderName)); + UNIT_ASSERT(res.Root); + } + + Y_UNIT_TEST(GroupByNoHopRtmr) { + NYql::TAstParseResult res = SqlToYql(R"( + USE plato; INSERT INTO Output SELECT STREAM key, SUM(value) AS value FROM Input + GROUP BY key; + )", 10, TString(NYql::RtmrProviderName)); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:22: Error: Streaming group by query must have a hopping window specification.\n"); + } + + Y_UNIT_TEST(KikimrInserts) { + NYql::TAstParseResult res = SqlToYql(R"( + USE plato; + INSERT INTO Output SELECT key, value FROM Input; + INSERT OR ABORT INTO Output SELECT key, value FROM Input; + INSERT OR IGNORE INTO Output SELECT key, value FROM Input; + INSERT OR REVERT INTO Output SELECT key, value FROM Input; + )", 10, TString(NYql::KikimrProviderName)); + UNIT_ASSERT(res.Root); + } + + Y_UNIT_TEST(WarnMissingIsBeforeNotNull) { + NYql::TAstParseResult res = SqlToYql("select 1 NOT NULL"); + UNIT_ASSERT(res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Warning: Missing IS keyword before NOT NULL, code: 4507\n"); + } + + Y_UNIT_TEST(Subqueries) { + NYql::TAstParseResult res = SqlToYql(R"( + USE plato; + $sq1 = (SELECT * FROM plato.Input); + + $sq2 = SELECT * FROM plato.Input; + + $squ1 = ( + SELECT * FROM plato.Input + UNION ALL + SELECT * FROM plato.Input + ); + + $squ2 = + SELECT * FROM plato.Input + UNION ALL + SELECT * FROM plato.Input; + + $squ3 = ( + (SELECT * FROM plato.Input) + UNION ALL + (SELECT * FROM plato.Input) + ); + + SELECT * FROM $sq1; + SELECT * FROM $sq2; + SELECT * FROM $squ1; + SELECT * FROM $squ2; + SELECT * FROM $squ3; + )"); + UNIT_ASSERT(res.Root); + } + + Y_UNIT_TEST(SubqueriesJoin) { + NYql::TAstParseResult res = SqlToYql(R"( + USE plato; + + $left = SELECT * FROM plato.Input1 WHERE value != "BadValue"; + $right = SELECT * FROM plato.Input2; + + SELECT * FROM $left AS l + JOIN $right AS r + ON l.key == r.key; + )"); + UNIT_ASSERT(res.Root); + } + + Y_UNIT_TEST(AnyInBackticksAsTableName) { + NYql::TAstParseResult res = SqlToYql("use plato; select * from `any`;"); + UNIT_ASSERT(res.Root); + } + + Y_UNIT_TEST(AnyJoinForTableAndSubQuery) { + NYql::TAstParseResult res = SqlToYql(R"( + USE plato; + + $r = SELECT * FROM plato.Input2; + + SELECT * FROM ANY plato.Input1 AS l + LEFT JOIN ANY $r AS r + USING (key); + )"); + + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "EquiJoin") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'('left 'any)")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'('right 'any)")); + } + }; + + TWordCountHive elementStat = {{TString("left"), 0}, {TString("right"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["left"]); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["right"]); + } + + Y_UNIT_TEST(AnyJoinForTableAndTableSource) { + NYql::TAstParseResult res = SqlToYql(R"( + USE plato; + + $r = AsList( + AsStruct("aaa" as key, "bbb" as subkey, "ccc" as value) + ); + + SELECT * FROM ANY plato.Input1 AS l + LEFT JOIN ANY AS_TABLE($r) AS r + USING (key); + )"); + + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "EquiJoin") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'('left 'any)")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'('right 'any)")); + } + }; + + TWordCountHive elementStat = {{TString("left"), 0}, {TString("right"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["left"]); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["right"]); + } + + Y_UNIT_TEST(AnyJoinNested) { + NYql::TAstParseResult res = SqlToYql(R"( + USE plato; + + FROM ANY Input1 as a + JOIN Input2 as b ON a.key = b.key + LEFT JOIN ANY Input3 as c ON a.key = c.key + RIGHT JOIN ANY Input4 as d ON d.key = b.key + CROSS JOIN Input5 + SELECT *; + )"); + + UNIT_ASSERT(res.Root); + + TWordCountHive elementStat = {{TString("left"), 0}, {TString("right"), 0}}; + VerifyProgram(res, elementStat); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["left"]); + UNIT_ASSERT_VALUES_EQUAL(2, elementStat["right"]); + } + + Y_UNIT_TEST(InlineAction) { + NYql::TAstParseResult res = SqlToYql( + "do begin\n" + " select 1\n" + "; end do\n"); + UNIT_ASSERT(res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), ""); + } + + Y_UNIT_TEST(FlattenByCorrelationName) { + UNIT_ASSERT(SqlToYql("select * from plato.Input as t flatten by t.x").IsOk()); + UNIT_ASSERT(SqlToYql("select * from plato.Input as t flatten by t -- same as flatten by t.t").IsOk()); + } + + Y_UNIT_TEST(DiscoveryMode) { + UNIT_ASSERT(SqlToYqlWithMode("insert into plato.Output select * from plato.Input", NSQLTranslation::ESqlMode::DISCOVERY).IsOk()); + UNIT_ASSERT(SqlToYqlWithMode("select * from plato.concat(Input1, Input2)", NSQLTranslation::ESqlMode::DISCOVERY).IsOk()); + UNIT_ASSERT(SqlToYqlWithMode("select * from plato.each(AsList(\"Input1\", \"Input2\"))", NSQLTranslation::ESqlMode::DISCOVERY).IsOk()); + } + + Y_UNIT_TEST(CubeWithAutoGeneratedLikeColumnName) { + UNIT_ASSERT(SqlToYql("select key,subkey,group from plato.Input group by cube(key,subkey,group)").IsOk()); + } + + Y_UNIT_TEST(CubeWithAutoGeneratedLikeAlias) { + UNIT_ASSERT(SqlToYql("select key,subkey,group from plato.Input group by cube(key,subkey,value as group)").IsOk()); + } + + Y_UNIT_TEST(FilterCanBeUsedAsColumnIdOrBind) { + UNIT_ASSERT(SqlToYql("select filter from plato.Input").IsOk()); + UNIT_ASSERT(SqlToYql("select 1 as filter").IsOk()); + UNIT_ASSERT(SqlToYql("$filter = 1; select $filter").IsOk()); + } + + Y_UNIT_TEST(DuplicateSemicolonsAreAllowedBetweenTopLevelStatements) { + UNIT_ASSERT(SqlToYql(";;select 1; ; select 2;/*comment*/;select 3;;--comment\n;select 4;;").IsOk()); + } + + Y_UNIT_TEST(DuplicateAndMissingTrailingSemicolonsAreAllowedBetweenActionStatements) { + TString req = + "define action $action($b,$c) as\n" + " ;;$d = $b + $c;\n" + " select $b;\n" + " select $c;;\n" + " select $d,\n" + "end define;\n" + "\n" + "do $action(1,2);"; + UNIT_ASSERT(SqlToYql(req).IsOk()); + } + + Y_UNIT_TEST(DuplicateAndMissingTrailingSemicolonsAreAllowedBetweenInlineActionStatements) { + TString req = + "do begin\n" + " ;select 1,\n" + "end do;\n" + "evaluate for $i in AsList(1,2,3) do begin\n" + " select $i;;\n" + " select $i + $i;;\n" + "end do;"; + UNIT_ASSERT(SqlToYql(req).IsOk()); + } + + Y_UNIT_TEST(DuplicateSemicolonsAreAllowedBetweenLambdaStatements) { + TString req = + "$x=1;\n" + "$foo = ($a, $b)->{\n" + " ;;$v = $a + $b;\n" + " $bar = ($c) -> {; return $c << $x};;\n" + " return $bar($v);;\n" + "};\n" + "select $foo(1,2);"; + UNIT_ASSERT(SqlToYql(req).IsOk()); + } + + Y_UNIT_TEST(StringLiteralWithEscapedBackslash) { + NYql::TAstParseResult res1 = SqlToYql(R"foo(SELECT 'a\\';)foo"); + NYql::TAstParseResult res2 = SqlToYql(R"foo(SELECT "a\\";)foo"); + UNIT_ASSERT(res1.Root); + UNIT_ASSERT(res2.Root); + + TWordCountHive elementStat = {{TString("a\\"), 0}}; + + VerifyProgram(res1, elementStat); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["a\\"]); + + VerifyProgram(res2, elementStat); + UNIT_ASSERT_VALUES_EQUAL(2, elementStat["a\\"]); + } + + Y_UNIT_TEST(StringMultiLineLiteralWithEscapes) { + UNIT_ASSERT(SqlToYql("SELECT @@@foo@@@@bar@@@").IsOk()); + UNIT_ASSERT(SqlToYql("SELECT @@@@@@@@@").IsOk()); + } + + Y_UNIT_TEST(StringMultiLineLiteralConsequitiveAt) { + UNIT_ASSERT(!SqlToYql("SELECT @").IsOk()); + UNIT_ASSERT(!SqlToYql("SELECT @@").IsOk()); + UNIT_ASSERT(!SqlToYql("SELECT @@@").IsOk()); + UNIT_ASSERT( SqlToYql("SELECT @@@@").IsOk()); + UNIT_ASSERT( SqlToYql("SELECT @@@@@").IsOk()); + + UNIT_ASSERT(!SqlToYql("SELECT @@@@@@").IsOk()); + UNIT_ASSERT(!SqlToYql("SELECT @@@@@@@").IsOk()); + + UNIT_ASSERT( SqlToYql("SELECT @@@@@@@@").IsOk()); + UNIT_ASSERT( SqlToYql("SELECT @@@@@@@@@").IsOk()); + UNIT_ASSERT(!SqlToYql("SELECT @@@@@@@@@@").IsOk()); + } + + Y_UNIT_TEST(ConstnessForListDictSetCreate) { + auto req = "$foo = ($x, $y) -> (\"aaaa\");\n" + "\n" + "select\n" + " $foo(sum(key), ListCreate(String)),\n" + " $foo(sum(key), DictCreate(String, String)),\n" + " $foo(sum(key), SetCreate(String)),\n" + "from (select 1 as key);"; + UNIT_ASSERT(SqlToYql(req).IsOk()); + } + + Y_UNIT_TEST(CanUseEmptyTupleInWindowPartitionBy) { + auto req = "select sum(key) over w\n" + "from plato.Input\n" + "window w as (partition compact by (), (subkey), (), value || value as dvalue);"; + UNIT_ASSERT(SqlToYql(req).IsOk()); + } + + Y_UNIT_TEST(DenyAnsiOrderByLimitLegacyMode) { + auto req = "pragma DisableAnsiOrderByLimitInUnionAll;\n" + "use plato;\n" + "\n" + "select * from Input order by key limit 10\n" + "union all\n" + "select * from Input order by key limit 1;"; + + auto res = SqlToYql(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Error: DisableAnsiOrderByLimitInUnionAll pragma is deprecated and no longer supported\n"); + } + + Y_UNIT_TEST(ReduceUsingUdfWithShortcutsWorks) { + auto req = "use plato;\n" + "\n" + "$arg = 'foo';\n" + "$func = XXX::YYY($arg);\n" + "\n" + "REDUCE Input ON key using $func(subkey);\n" + "REDUCE Input ON key using $func(UUU::VVV(TableRow()));\n"; + UNIT_ASSERT(SqlToYql(req).IsOk()); + req = "use plato;\n" + "\n" + "$arg = 'foo';\n" + "$func = XXX::YYY($arg);\n" + "\n" + "REDUCE Input ON key using all $func(subkey);\n" + "REDUCE Input ON key using all $func(UUU::VVV(TableRow()));"; + UNIT_ASSERT(SqlToYql(req).IsOk()); + } + + Y_UNIT_TEST(YsonDisableStrict) { + UNIT_ASSERT(SqlToYql("pragma yson.DisableStrict = \"false\";").IsOk()); + UNIT_ASSERT(SqlToYql("pragma yson.DisableStrict;").IsOk()); + } + + Y_UNIT_TEST(YsonStrict) { + UNIT_ASSERT(SqlToYql("pragma yson.Strict = \"false\";").IsOk()); + UNIT_ASSERT(SqlToYql("pragma yson.Strict;").IsOk()); + } + + Y_UNIT_TEST(JoinByTuple) { + auto req = "use plato;\n" + "\n" + "select * from T1 as a\n" + "join T2 as b\n" + "on AsTuple(a.key, a.subkey) = AsTuple(b.key, b.subkey);"; + UNIT_ASSERT(SqlToYql(req).IsOk()); + } + + Y_UNIT_TEST(JoinByStruct) { + auto req = "use plato;\n" + "\n" + "select * from T1 as a\n" + "join T2 as b\n" + "on AsStruct(a.key as k, a.subkey as sk) = AsStruct(b.key as k, b.subkey as sk);"; + UNIT_ASSERT(SqlToYql(req).IsOk()); + } + + Y_UNIT_TEST(JoinByUdf) { + auto req = "use plato;\n" + "\n" + "select a.align\n" + "from T1 as a\n" + "join T2 as b\n" + "on Yson::SerializeJsonEncodeUtf8(a.align)=b.align;"; + UNIT_ASSERT(SqlToYql(req).IsOk()); + } + + Y_UNIT_TEST(EscapedIdentifierAsLambdaArg) { + auto req = "$f = ($`foo bar`, $x) -> { return $`foo bar` + $x; };\n" + "\n" + "select $f(1, 2);"; + auto res = SqlToYql(req); + UNIT_ASSERT(res.Root); + const auto programm = GetPrettyPrint(res); + auto expected = "(lambda '(\"$foo bar\" \"$x\")"; + UNIT_ASSERT(programm.find(expected) != TString::npos); + } + + Y_UNIT_TEST(UdfSyntaxSugarOnlyCallable) { + auto req = "SELECT Udf(DateTime::FromString)('2022-01-01');"; + auto res = SqlToYql(req); + UNIT_ASSERT(res.Root); + const auto programm = GetPrettyPrint(res); + auto expected = "(SqlCall '\"DateTime.FromString\" '((PositionalArgs (String '\"2022-01-01\")) (AsStruct)) (TupleType (TypeOf '((String '\"2022-01-01\"))) (TypeOf (AsStruct)) (TupleType)))"; + UNIT_ASSERT(programm.find(expected) != TString::npos); + } + + Y_UNIT_TEST(UdfSyntaxSugarTypeNoRun) { + auto req = "SELECT Udf(DateTime::FromString, String, Tuple<Int32, Float>, 'foo' as TypeConfig)('2022-01-01');"; + auto res = SqlToYql(req); + UNIT_ASSERT(res.Root); + const auto programm = GetPrettyPrint(res); + auto expected = "(SqlCall '\"DateTime.FromString\" '((PositionalArgs (String '\"2022-01-01\")) (AsStruct)) (TupleType (TypeOf '((String '\"2022-01-01\"))) (TypeOf (AsStruct)) (TupleType (DataType 'String) (TupleType (DataType 'Int32) (DataType 'Float)))) '\"foo\")"; + UNIT_ASSERT(programm.find(expected) != TString::npos); + } + + Y_UNIT_TEST(UdfSyntaxSugarRunNoType) { + auto req = "SELECT Udf(DateTime::FromString, String, Tuple<Int32, Float>, Void() as RunConfig)('2022-01-01');"; + auto res = SqlToYql(req); + UNIT_ASSERT(res.Root); + const auto programm = GetPrettyPrint(res); + auto expected = "(SqlCall '\"DateTime.FromString\" '((PositionalArgs (String '\"2022-01-01\")) (AsStruct)) (TupleType (TypeOf '((String '\"2022-01-01\"))) (TypeOf (AsStruct)) (TupleType (DataType 'String) (TupleType (DataType 'Int32) (DataType 'Float)))) '\"\" (Void))"; + UNIT_ASSERT(programm.find(expected) != TString::npos); + } + + Y_UNIT_TEST(UdfSyntaxSugarFullTest) { + auto req = "SELECT Udf(DateTime::FromString, String, Tuple<Int32, Float>, 'foo' as TypeConfig, Void() As RunConfig)('2022-01-01');"; + auto res = SqlToYql(req); + UNIT_ASSERT(res.Root); + const auto programm = GetPrettyPrint(res); + auto expected = "(SqlCall '\"DateTime.FromString\" '((PositionalArgs (String '\"2022-01-01\")) (AsStruct)) (TupleType (TypeOf '((String '\"2022-01-01\"))) (TypeOf (AsStruct)) (TupleType (DataType 'String) (TupleType (DataType 'Int32) (DataType 'Float)))) '\"foo\" (Void))"; + UNIT_ASSERT(programm.find(expected) != TString::npos); + } + + Y_UNIT_TEST(UdfSyntaxSugarOtherRunConfigs) { + auto req = "SELECT Udf(DateTime::FromString, String, Tuple<Int32, Float>, 'foo' as TypeConfig, '55' As RunConfig)('2022-01-01');"; + auto res = SqlToYql(req); + UNIT_ASSERT(res.Root); + const auto programm = GetPrettyPrint(res); + auto expected = "(SqlCall '\"DateTime.FromString\" '((PositionalArgs (String '\"2022-01-01\")) (AsStruct)) (TupleType (TypeOf '((String '\"2022-01-01\"))) (TypeOf (AsStruct)) (TupleType (DataType 'String) (TupleType (DataType 'Int32) (DataType 'Float)))) '\"foo\" (String '\"55\"))"; + UNIT_ASSERT(programm.find(expected) != TString::npos); + } + + Y_UNIT_TEST(UdfSyntaxSugarOtherRunConfigs2) { + auto req = "SELECT Udf(DateTime::FromString, String, Tuple<Int32, Float>, 'foo' as TypeConfig, AsTuple(32, 'no', AsStruct(1e-9 As SomeFloat)) As RunConfig)('2022-01-01');"; + auto res = SqlToYql(req); + UNIT_ASSERT(res.Root); + const auto programm = GetPrettyPrint(res); + auto expected = "(SqlCall '\"DateTime.FromString\" '((PositionalArgs (String '\"2022-01-01\")) (AsStruct)) (TupleType (TypeOf '((String '\"2022-01-01\"))) (TypeOf (AsStruct)) (TupleType (DataType 'String) (TupleType (DataType 'Int32) (DataType 'Float)))) '\"foo\" '((Int32 '\"32\") (String '\"no\") (AsStruct '('\"SomeFloat\" (Double '\"1e-9\")))))"; + UNIT_ASSERT(programm.find(expected) != TString::npos); + } + + Y_UNIT_TEST(UdfSyntaxSugarOptional) { + auto req = "SELECT Udf(DateTime::FromString, String?, Int32??, Tuple<Int32, Float>, \"foo\" as TypeConfig, Void() As RunConfig)(\"2022-01-01\");"; + auto res = SqlToYql(req); + UNIT_ASSERT(res.Root); + const auto programm = GetPrettyPrint(res); + auto expected = "(SqlCall '\"DateTime.FromString\" '((PositionalArgs (String '\"2022-01-01\")) (AsStruct)) (TupleType (TypeOf '((String '\"2022-01-01\"))) (TypeOf (AsStruct)) (TupleType (OptionalType (DataType 'String)) (OptionalType (OptionalType (DataType 'Int32))) (TupleType (DataType 'Int32) (DataType 'Float)))) '\"foo\" (Void))"; + UNIT_ASSERT(programm.find(expected) != TString::npos); + } + + Y_UNIT_TEST(CompactionPolicyParseCorrect) { + NYql::TAstParseResult res = SqlToYql( + R"( USE plato; + CREATE TABLE tableName (Key Uint32, Value String, PRIMARY KEY (Key)) + WITH ( COMPACTION_POLICY = "SomeCompactionPreset" );)" + ); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("compactionPolicy")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("SomeCompactionPreset")); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0} }; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } + + Y_UNIT_TEST(AutoPartitioningBySizeParseCorrect) { + NYql::TAstParseResult res = SqlToYql( + R"( USE plato; + CREATE TABLE tableName (Key Uint32, Value String, PRIMARY KEY (Key)) + WITH ( AUTO_PARTITIONING_BY_SIZE = ENABLED );)" + ); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("autoPartitioningBySize")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("ENABLED")); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0} }; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } + + Y_UNIT_TEST(UniformPartitionsParseCorrect) { + NYql::TAstParseResult res = SqlToYql( + R"( USE plato; + CREATE TABLE tableName (Key Uint32, Value String, PRIMARY KEY (Key)) + WITH ( UNIFORM_PARTITIONS = 16 );)" + ); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("uniformPartitions")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("16")); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0} }; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } + + Y_UNIT_TEST(DateTimeTtlParseCorrect) { + NYql::TAstParseResult res = SqlToYql( + R"( USE plato; + CREATE TABLE tableName (Key Uint32, CreatedAt Timestamp, PRIMARY KEY (Key)) + WITH (TTL = Interval("P1D") On CreatedAt);)" + ); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("setTtlSettings")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("tiers")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("evictionDelay")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("86400000")); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0} }; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } + + Y_UNIT_TEST(IntTtlParseCorrect) { + NYql::TAstParseResult res = SqlToYql( + R"( USE plato; + CREATE TABLE tableName (Key Uint32, CreatedAt Uint32, PRIMARY KEY (Key)) + WITH (TTL = Interval("P1D") On CreatedAt AS SECONDS);)" + ); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("setTtlSettings")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("tiers")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("evictionDelay")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("86400000")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("columnUnit")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("seconds")); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0} }; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } + + Y_UNIT_TEST(TtlTieringParseCorrect) { + NYql::TAstParseResult res = SqlToYql( + R"( USE plato; + CREATE TABLE tableName (Key Uint32, CreatedAt Uint32, PRIMARY KEY (Key)) + WITH (TTL = + Interval("P1D") TO EXTERNAL DATA SOURCE Tier1, + Interval("P2D") TO EXTERNAL DATA SOURCE Tier2, + Interval("P30D") DELETE + ON CreatedAt AS SECONDS);)" + ); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("setTtlSettings")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("tiers")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("evictionDelay")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("storageName")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("Tier1")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("Tier2")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("86400000")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("172800000")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("2592000000")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("columnUnit")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("seconds")); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0} }; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } + + Y_UNIT_TEST(TtlTieringWithOtherActionsParseCorrect) { + NYql::TAstParseResult res = SqlToYql( + R"( USE plato; + ALTER TABLE tableName + ADD FAMILY cold (DATA = "rot"), + SET TTL + Interval("P1D") TO EXTERNAL DATA SOURCE Tier1, + Interval("P2D") TO EXTERNAL DATA SOURCE Tier2, + Interval("P30D") DELETE + ON CreatedAt, + ALTER COLUMN payload_v2 SET FAMILY cold, + ALTER FAMILY default SET DATA "ssd" + ;)" + ); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("addColumnFamilies")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("cold")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("alterColumnFamilies")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("default")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("setTtlSettings")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("tiers")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("evictionDelay")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("storageName")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("Tier1")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("Tier2")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("86400000")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("172800000")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("2592000000")); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0} }; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } + + Y_UNIT_TEST(TieringParseCorrect) { + NYql::TAstParseResult res = SqlToYql( + R"( USE plato; + CREATE TABLE tableName (Key Uint32, Value String, PRIMARY KEY (Key)) + WITH ( TIERING = 'my_tiering' );)" + ); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("tiering")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("my_tiering")); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0} }; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } + + Y_UNIT_TEST(StoreExternalBlobsParseCorrect) { + NYql::TAstParseResult res = SqlToYql( + R"( USE plato; + CREATE TABLE tableName (Key Uint32, Value String, PRIMARY KEY (Key)) + WITH ( STORE_EXTERNAL_BLOBS = ENABLED );)" + ); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("storeExternalBlobs")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("ENABLED")); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0} }; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } + + Y_UNIT_TEST(DefaultValueColumn2) { + auto res = SqlToYql(R"( use plato; + $lambda = () -> { + RETURN CAST(RandomUuid(2) as String) + }; + + CREATE TABLE tableName ( + Key Uint32 DEFAULT RandomNumber(1), + Value String DEFAULT $lambda, + PRIMARY KEY (Key) + ); + )"); + + UNIT_ASSERT_C(res.Root, Err2Str(res)); + + const auto program = GetPrettyPrint(res); + + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, program.find("RandomNumber")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, program.find("RandomUuid")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, program.find("columnConstrains")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, program.find("columnConstrains")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, program.find("Write")); + +#if 0 + Cerr << program << Endl; +#endif + + TWordCountHive elementStat = { {TString("Write"), 0} }; + VerifyProgram(res, elementStat); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } + + Y_UNIT_TEST(DefaultValueColumn3) { + auto res = SqlToYql(R"( use plato; + + CREATE TABLE tableName ( + database_id Utf8, + cloud_id Utf8, + global_id Utf8 DEFAULT database_id || "=====", + PRIMARY KEY (database_id) + ); + )"); + + UNIT_ASSERT_VALUES_EQUAL(Err2Str(res), "<main>:6:40: Error: Column reference \"database_id\" is not allowed in current scope\n"); + UNIT_ASSERT(!res.Root); + } + + Y_UNIT_TEST(DefaultValueColumn) { + auto res = SqlToYql(R"( use plato; + CREATE TABLE tableName ( + Key Uint32 FAMILY cold DEFAULT 5, + Value String FAMILY default DEFAULT "empty", + PRIMARY KEY (Key), + FAMILY default ( + DATA = "test", + COMPRESSION = "lz4" + ), + FAMILY cold ( + DATA = "test", + COMPRESSION = "off" + ) + ); + )"); + + UNIT_ASSERT_C(res.Root, Err2Str(res)); + +#if 0 + const auto program = GetPrettyPrint(res); + Cerr << program << Endl; +#endif + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("default")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("columnConstrains")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("columnFamilies")); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0} }; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } + + Y_UNIT_TEST(ChangefeedParseCorrect) { + auto res = SqlToYql(R"( USE plato; + CREATE TABLE tableName ( + Key Uint32, PRIMARY KEY (Key), + CHANGEFEED feedName WITH ( + MODE = 'KEYS_ONLY', + FORMAT = 'json', + INITIAL_SCAN = TRUE, + VIRTUAL_TIMESTAMPS = FALSE, + BARRIERS_INTERVAL = Interval("PT1S"), + RETENTION_PERIOD = Interval("P1D"), + TOPIC_MIN_ACTIVE_PARTITIONS = 10, + AWS_REGION = 'aws:region' + ) + ); + )"); + UNIT_ASSERT_C(res.Root, Err2Str(res)); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("changefeed")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("mode")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("KEYS_ONLY")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("format")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("json")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("initial_scan")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("true")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("virtual_timestamps")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("false")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("barriers_interval")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("retention_period")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("topic_min_active_partitions")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("aws_region")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("aws:region")); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0} }; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } + + Y_UNIT_TEST(CloneForAsTableWorksWithCube) { + UNIT_ASSERT(SqlToYql("SELECT * FROM AS_TABLE([<|k1:1, k2:1|>]) GROUP BY CUBE(k1, k2);").IsOk()); + } + + Y_UNIT_TEST(WindowPartitionByColumnProperlyEscaped) { + NYql::TAstParseResult res = SqlToYql("SELECT SUM(key) OVER w FROM plato.Input WINDOW w AS (PARTITION BY `column with space`);"); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "CalcOverWindow") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("\"column with space\"")); + } + }; + + TWordCountHive elementStat = { {TString("CalcOverWindow"), 0} }; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["CalcOverWindow"]); + } + + Y_UNIT_TEST(WindowPartitionByExpressionWithoutAliasesAreAllowed) { + NYql::TAstParseResult res = SqlToYql("SELECT SUM(key) OVER w FROM plato.Input as i WINDOW w AS (PARTITION BY ii.subkey);"); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "AddMember") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("AddMember row 'group_w_0 (SqlAccess 'struct (Member row '\"ii\")")); + } + if (word == "CalcOverWindow") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("CalcOverWindow core '('\"group_w_0\")")); + } + }; + + TWordCountHive elementStat = { {TString("CalcOverWindow"), 0}, {TString("AddMember"), 0} }; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["CalcOverWindow"]); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["AddMember"]); + } + + Y_UNIT_TEST(PqReadByAfterUse) { + ExpectFailWithError("use plato; pragma PqReadBy='plato2';", + "<main>:1:28: Error: Cluster in PqReadPqBy pragma differs from cluster specified in USE statement: plato2 != plato\n"); + + UNIT_ASSERT(SqlToYql("pragma PqReadBy='plato2';").IsOk()); + UNIT_ASSERT(SqlToYql("pragma PqReadBy='plato2'; use plato;").IsOk()); + UNIT_ASSERT(SqlToYql("$x='plato'; use rtmr:$x; pragma PqReadBy='plato2';").IsOk()); + UNIT_ASSERT(SqlToYql("use plato; pragma PqReadBy='dq';").IsOk()); + } + + Y_UNIT_TEST(MrObject) { + NYql::TAstParseResult res = SqlToYql( + "declare $path as String;\n" + "select * from plato.object($path, `format`, \"comp\" || \"ression\" as compression, 1 as bar) with schema (Int32 as y, String as x)" + ); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "MrObject") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, + line.find(R"__((MrObject (EvaluateAtom "$path") '"format" '('('"compression" (Concat (String '"comp") (String '"ression"))) '('"bar" (Int32 '"1")))))__")); + } else if (word == "userschema") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, + line.find(R"__('('('"userschema" (StructType '('"y" (DataType 'Int32)) '('"x" (DataType 'String))) '('"y" '"x"))))__")); + } + }; + + TWordCountHive elementStat = {{TString("MrObject"), 0}, {TString("userschema"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["MrObject"]); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["userschema"]); + } + + Y_UNIT_TEST(TableBindings) { + NSQLTranslation::TTranslationSettings settings = GetSettingsWithS3Binding("foo"); + NYql::TAstParseResult res = SqlToYqlWithSettings( + "select * from bindings.foo", + settings + ); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "MrObject") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, + line.find(R"__((MrTableConcat (Key '('table (String '"path")))) (Void) '('('"bar" '"1") '('"compression" '"ccompression") '('"format" '"format") '('"partitionedby" '"key" '"subkey") '('"userschema" (SqlTypeFromYson)__")); + } + }; + + TWordCountHive elementStat = {{TString("MrTableConcat"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["MrTableConcat"]); + + settings.DefaultCluster = "plato"; + settings.BindingsMode = NSQLTranslation::EBindingsMode::DISABLED; + res = SqlToYqlWithSettings( + "select * from bindings.foo", + settings + ); + UNIT_ASSERT_VALUES_EQUAL(Err2Str(res), "<main>:1:15: Error: Please remove 'bindings.' from your query, the support for this syntax has ended, code: 4601\n"); + UNIT_ASSERT(!res.Root); + + settings.BindingsMode = NSQLTranslation::EBindingsMode::DROP; + res = SqlToYqlWithSettings( + "select * from bindings.foo", + settings + ); + + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine2 = [](const TString& word, const TString& line) { + if (word == "MrTableConcat") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, + line.find(R"__((MrTableConcat (Key '('table (String '"foo")))) (Void) '())))__")); + } + }; + + TWordCountHive elementStat2 = {{TString("MrTableConcat"), 0}}; + VerifyProgram(res, elementStat2, verifyLine2); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat2["MrTableConcat"]); + + settings.BindingsMode = NSQLTranslation::EBindingsMode::DROP_WITH_WARNING; + res = SqlToYqlWithSettings( + "select * from bindings.foo", + settings + ); + UNIT_ASSERT_VALUES_EQUAL(Err2Str(res), "<main>:1:15: Warning: Please remove 'bindings.' from your query, the support for this syntax will be dropped soon, code: 4538\n"); + UNIT_ASSERT(res.Root); + + TWordCountHive elementStat3 = {{TString("MrTableConcat"), 0}}; + VerifyProgram(res, elementStat3, verifyLine2); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat3["MrTableConcat"]); + } + + Y_UNIT_TEST(TableBindingsWithInsert) { + NSQLTranslation::TTranslationSettings settings = GetSettingsWithS3Binding("foo"); + NYql::TAstParseResult res = SqlToYqlWithSettings( + "insert into bindings.foo with truncate (x, y) values (1, 2);", + settings + ); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write!") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, + line.find(R"__((Write! world sink (Key '('table (String '"path"))) values '('('"bar" '"1") '('"compression" '"ccompression") '('"format" '"format") '('"partitionedby" '"key" '"subkey") '('"userschema" (SqlTypeFromYson)__")); + } + }; + + TWordCountHive elementStat = {{TString("Write!"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write!"]); + + settings.DefaultCluster = "plato"; + settings.BindingsMode = NSQLTranslation::EBindingsMode::DISABLED; + res = SqlToYqlWithSettings( + "insert into bindings.foo with truncate (x, y) values (1, 2);", + settings + ); + UNIT_ASSERT_VALUES_EQUAL(Err2Str(res), "<main>:1:13: Error: Please remove 'bindings.' from your query, the support for this syntax has ended, code: 4601\n"); + UNIT_ASSERT(!res.Root); + + settings.BindingsMode = NSQLTranslation::EBindingsMode::DROP; + res = SqlToYqlWithSettings( + "insert into bindings.foo with truncate (x, y) values (1, 2);", + settings + ); + UNIT_ASSERT_VALUES_EQUAL(Err2Str(res), ""); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine2 = [](const TString& word, const TString& line) { + if (word == "Write!") { + //UNIT_ASSERT_VALUES_EQUAL(line, ""); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, + line.find(R"__((Write! world sink (Key '('table (String '"foo"))) values '('('mode 'renew)))__")); + } + }; + + TWordCountHive elementStat2 = {{TString("Write!"), 0}}; + VerifyProgram(res, elementStat2, verifyLine2); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat2["Write!"]); + + settings.BindingsMode = NSQLTranslation::EBindingsMode::DROP_WITH_WARNING; + res = SqlToYqlWithSettings( + "insert into bindings.foo with truncate (x, y) values (1, 2);", + settings + ); + UNIT_ASSERT_VALUES_EQUAL(Err2Str(res), "<main>:1:13: Warning: Please remove 'bindings.' from your query, the support for this syntax will be dropped soon, code: 4538\n"); + UNIT_ASSERT(res.Root); + + TWordCountHive elementStat3 = {{TString("Write!"), 0}}; + VerifyProgram(res, elementStat3, verifyLine2); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat3["Write!"]); + } + + Y_UNIT_TEST(TrailingCommaInWithout) { + UNIT_ASSERT(SqlToYql("SELECT * WITHOUT stream, FROM plato.Input").IsOk()); + UNIT_ASSERT(SqlToYql("SELECT a.* WITHOUT a.intersect, FROM plato.Input AS a").IsOk()); + UNIT_ASSERT(SqlToYql("SELECT a.* WITHOUT col1, col2, a.col3, FROM plato.Input AS a").IsOk()); + } + + Y_UNIT_TEST(NoStackOverflowOnBigCaseStatement) { + TStringBuilder req; + req << "select case 1 + 123"; + for (size_t i = 0; i < 20000; ++i) { + req << " when " << i << " then " << i + 1; + } + req << " else 100500 end;"; + UNIT_ASSERT(SqlToYql(req).IsOk()); + } + + Y_UNIT_TEST(CollectPreaggregatedInListLiteral) { + UNIT_ASSERT(SqlToYql("SELECT [COUNT(DISTINCT a+b)] FROM plato.Input").IsOk()); + } + + Y_UNIT_TEST(SmartParenInGroupByClause) { + UNIT_ASSERT(SqlToYql("SELECT * FROM plato.Input GROUP BY (k, v)").IsOk()); + } + + Y_UNIT_TEST(AlterTableRenameToIsCorrect) { + UNIT_ASSERT(SqlToYql("USE plato; ALTER TABLE table RENAME TO moved").IsOk()); + } + + Y_UNIT_TEST(AlterTableAddDropColumnIsCorrect) { + UNIT_ASSERT(SqlToYql("USE plato; ALTER TABLE table ADD COLUMN addc uint64, DROP COLUMN dropc, ADD addagain uint64").IsOk()); + } + + Y_UNIT_TEST(AlterTableSetTTLIsCorrect) { + UNIT_ASSERT(SqlToYql("USE plato; ALTER TABLE table SET (TTL = Interval(\"PT3H\") ON column)").IsOk()); + UNIT_ASSERT(SqlToYql("USE plato; ALTER TABLE table SET (TTL = Interval(\"PT3H\") ON column AS SECONDS)").IsOk()); + } + + Y_UNIT_TEST(AlterTableSetTieringIsCorrect) { + UNIT_ASSERT(SqlToYql("USE plato; ALTER TABLE table SET (TIERING = 'my_tiering')").IsOk()); + } + + Y_UNIT_TEST(AlterTableAddChangefeedIsCorrect) { + UNIT_ASSERT(SqlToYql("USE plato; ALTER TABLE table ADD CHANGEFEED feed WITH (MODE = 'UPDATES', FORMAT = 'json')").IsOk()); + } + + Y_UNIT_TEST(AlterTableAlterChangefeedIsCorrect) { + UNIT_ASSERT(SqlToYql("USE plato; ALTER TABLE table ALTER CHANGEFEED feed DISABLE").IsOk()); + ExpectFailWithError("USE plato; ALTER TABLE table ALTER CHANGEFEED feed SET (FORMAT = 'proto');", + "<main>:1:57: Error: FORMAT alter is not supported\n"); + } + + Y_UNIT_TEST(AlterTableDropChangefeedIsCorrect) { + UNIT_ASSERT(SqlToYql("USE plato; ALTER TABLE table DROP CHANGEFEED feed").IsOk()); + } + + Y_UNIT_TEST(AlterTableSetPartitioningIsCorrect) { + UNIT_ASSERT(SqlToYql("USE plato; ALTER TABLE table SET (AUTO_PARTITIONING_BY_SIZE = DISABLED)").IsOk()); + } + + Y_UNIT_TEST(AlterTableAddIndexWithIsNotSupported) { +#if ANTLR_VER == 3 + ExpectFailWithFuzzyError("USE plato; ALTER TABLE table ADD INDEX idx GLOBAL ON (col) WITH (a=b)", + "<main>:1:40: Error: with: alternative is not implemented yet: \\d+:\\d+: global_index\\n"); +#else + ExpectFailWithError("USE plato; ALTER TABLE table ADD INDEX idx GLOBAL ON (col) WITH (a=b)", + "<main>:1:40: Error: with: alternative is not implemented yet: \n"); +#endif + } + + Y_UNIT_TEST(AlterTableAddIndexLocalIsNotSupported) { +#if ANTLR_VER == 3 + ExpectFailWithFuzzyError("USE plato; ALTER TABLE table ADD INDEX idx LOCAL ON (col)", + "<main>:1:40: Error: local: alternative is not implemented yet: \\d+:\\d+: local_index\\n"); +#else + ExpectFailWithError("USE plato; ALTER TABLE table ADD INDEX idx LOCAL ON (col)", + "<main>:1:40: Error: local: alternative is not implemented yet: \n"); +#endif + } + + Y_UNIT_TEST(CreateTableAddIndexVector) { + const auto result = SqlToYql(R"(USE plato; + CREATE TABLE table ( + pk INT32 NOT NULL, + col String, + INDEX idx GLOBAL USING vector_kmeans_tree + ON (col) COVER (col) + WITH (distance=cosine, vector_type=float, vector_dimension=1024,), + PRIMARY KEY (pk)) + )"); + UNIT_ASSERT_C(result.IsOk(), result.Issues.ToString()); + } + + Y_UNIT_TEST(AlterTableAddIndexVector) { + const auto result = SqlToYql(R"(USE plato; + ALTER TABLE table ADD INDEX idx + GLOBAL USING vector_kmeans_tree + ON (col) COVER (col) + WITH (distance=cosine, vector_type="float", vector_dimension=1024) + )"); + UNIT_ASSERT_C(result.IsOk(), result.Issues.ToString()); + } + + Y_UNIT_TEST(AlterTableAddIndexUnknownSubtype) { + ExpectFailWithError("USE plato; ALTER TABLE table ADD INDEX idx GLOBAL USING unknown ON (col)", + "<main>:1:57: Error: UNKNOWN index subtype is not supported\n"); + } + + Y_UNIT_TEST(AlterTableAddIndexMissedParameter) { + ExpectFailWithError(R"(USE plato; + ALTER TABLE table ADD INDEX idx + GLOBAL USING vector_kmeans_tree + ON (col) + WITH (distance=cosine, vector_type=float) + )", + "<main>:5:52: Error: vector_dimension should be set\n"); + } + + Y_UNIT_TEST(AlterTableAlterIndexSetPartitioningIsCorrect) { + const auto result = SqlToYql("USE plato; ALTER TABLE table ALTER INDEX index SET AUTO_PARTITIONING_MIN_PARTITIONS_COUNT 10"); + UNIT_ASSERT_C(result.IsOk(), result.Issues.ToString()); + } + + Y_UNIT_TEST(AlterTableAlterIndexSetMultiplePartitioningSettings) { + const auto result = SqlToYql("USE plato; ALTER TABLE table ALTER INDEX index SET " + "(AUTO_PARTITIONING_BY_LOAD = ENABLED, AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = 10)" + ); + UNIT_ASSERT_C(result.IsOk(), result.Issues.ToString()); + } + + Y_UNIT_TEST(AlterTableAlterIndexResetPartitioningIsNotSupported) { + ExpectFailWithError("USE plato; ALTER TABLE table ALTER INDEX index RESET (AUTO_PARTITIONING_MIN_PARTITIONS_COUNT)", + "<main>:1:55: Error: AUTO_PARTITIONING_MIN_PARTITIONS_COUNT reset is not supported\n" + ); + } + + Y_UNIT_TEST(AlterTableAlterColumnDropNotNullAstCorrect) { + auto reqSetNull = SqlToYql(R"( + USE plato; + CREATE TABLE tableName ( + id Uint32, + val Uint32 NOT NULL, + PRIMARY KEY (id) + ); + + COMMIT; + ALTER TABLE tableName ALTER COLUMN val DROP NOT NULL; + )"); + + UNIT_ASSERT(reqSetNull.IsOk()); + UNIT_ASSERT(reqSetNull.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + Y_UNUSED(word); + + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find( + R"(let world (Write! world sink (Key '('tablescheme (String '"tableName"))) (Void) '('('mode 'alter) '('actions '('('alterColumns '('('"val" '('changeColumnConstraints '('('drop_not_null)))))))))))" + )); + }; + + TWordCountHive elementStat({TString("\'mode \'alter")}); + VerifyProgram(reqSetNull, elementStat, verifyLine); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["\'mode \'alter"]); + } + + Y_UNIT_TEST(AlterSequence) { + UNIT_ASSERT(SqlToYql(R"( + USE plato; + ALTER SEQUENCE sequence START WITH 10 INCREMENT 2 RESTART WITH 5; + )").IsOk()); + UNIT_ASSERT(SqlToYql(R"( + USE plato; + ALTER SEQUENCE sequence INCREMENT 2; + )").IsOk()); + UNIT_ASSERT(SqlToYql(R"( + USE plato; + ALTER SEQUENCE sequence INCREMENT 2 START 1000; + )").IsOk()); + UNIT_ASSERT(SqlToYql(R"( + USE plato; + ALTER SEQUENCE sequence RESTART START 1000; + )").IsOk()); + UNIT_ASSERT(SqlToYql(R"( + USE plato; + ALTER SEQUENCE IF EXISTS sequence INCREMENT 1000 START 100 RESTART; + )").IsOk()); + UNIT_ASSERT(SqlToYql(R"( + USE plato; + ALTER SEQUENCE IF EXISTS sequence RESTART 1000 START WITH 100 INCREMENT BY 7; + )").IsOk()); + } + + Y_UNIT_TEST(AlterSequenceIncorrect) { + { + NYql::TAstParseResult res = SqlToYql("USE plato; ALTER SEQUENCE sequence START WITH 10 INCREMENT 2 RESTART WITH 5 RESTART;"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:75: Error: Restart value defined more than once\n"); + } + { + NYql::TAstParseResult res = SqlToYql("USE plato; ALTER SEQUENCE sequence START WITH 10 INCREMENT 2 START 100 RESTART WITH 5;"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:60: Error: Start value defined more than once\n"); + } + { + NYql::TAstParseResult res = SqlToYql("USE plato; ALTER SEQUENCE sequence INCREMENT BY 7 START WITH 10 INCREMENT 2 RESTART WITH 5 RESTART;"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:62: Error: Increment defined more than once\n"); + } + { + NYql::TAstParseResult res = SqlToYql("USE plato; ALTER SEQUENCE sequence RESTART WITH 100 START WITH 10 INCREMENT 2 RESTART WITH 5;"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:77: Error: Restart value defined more than once\n"); + } + { + NYql::TAstParseResult res = SqlToYql("USE plato; ALTER SEQUENCE sequence RESTART WITH 1234234543563435151456 START WITH 10 INCREMENT 2;"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:49: Error: Failed to parse number from string: 1234234543563435151456, number limit overflow\n"); + } + { + NYql::TAstParseResult res = SqlToYql("USE plato; ALTER SEQUENCE sequence RESTART WITH 1 START WITH 9223372036854775817 INCREMENT 4;"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:12: Error: Start value: 9223372036854775817 cannot be greater than max value: 9223372036854775807\n"); + } + { + NYql::TAstParseResult res = SqlToYql("USE plato; ALTER SEQUENCE sequence RESTART WITH 9223372036854775827 START WITH 5 INCREMENT 4;"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:12: Error: Restart value: 9223372036854775827 cannot be greater than max value: 9223372036854775807\n"); + } + { + NYql::TAstParseResult res = SqlToYql("USE plato; ALTER SEQUENCE sequence RESTART WITH 1 START WITH 4 INCREMENT 0;"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:12: Error: Increment must not be zero\n"); + } + { + NYql::TAstParseResult res = SqlToYql("USE plato; ALTER SEQUENCE sequence RESTART WITH 0 START WITH 4 INCREMENT 1;"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:12: Error: Restart value: 0 cannot be less than min value: 1\n"); + } + { + NYql::TAstParseResult res = SqlToYql("USE plato; ALTER SEQUENCE sequence RESTART WITH 1 START WITH 0 INCREMENT 1;"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:12: Error: Start value: 0 cannot be less than min value: 1\n"); + } + { + NYql::TAstParseResult res = SqlToYql("USE plato; ALTER SEQUENCE sequence RESTART WITH 1 START WITH 1 INCREMENT 9223372036854775837;"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:12: Error: Increment: 9223372036854775837 cannot be greater than max value: 9223372036854775807\n"); + } + } + + Y_UNIT_TEST(AlterSequenceCorrect) { + { + NYql::TAstParseResult res = SqlToYql("USE plato; ALTER SEQUENCE sequence START WITH 10 INCREMENT 2 RESTART WITH 5;"); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("sequence")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("alter")); + UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("alter_if_exists")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("start")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("increment")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("restart")); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } + + { + NYql::TAstParseResult res = SqlToYql("USE plato; ALTER SEQUENCE IF EXISTS sequence INCREMENT 2 RESTART;"); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("sequence")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("alter_if_exists")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("increment")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("restart")); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } + + { + NYql::TAstParseResult res = SqlToYql("USE plato; ALTER SEQUENCE IF EXISTS sequence START 10 INCREMENT BY 2;"); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("sequence")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("alter_if_exists")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("start")); + UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("restart")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("increment")); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } + } + + Y_UNIT_TEST(ShowCreateTable) { + NYql::TAstParseResult res = SqlToYql(R"( + USE plato; + SHOW CREATE TABLE user; + )"); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Read") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("showCreateTable")); + } + }; + + TWordCountHive elementStat = {{TString("Read"), 0}, {TString("showCreateTable"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Read"]); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["showCreateTable"]); + } + + Y_UNIT_TEST(OptionalAliases) { + UNIT_ASSERT(SqlToYql("USE plato; SELECT foo FROM (SELECT key foo FROM Input);").IsOk()); + UNIT_ASSERT(SqlToYql("USE plato; SELECT a.x FROM Input1 a JOIN Input2 b ON a.key = b.key;").IsOk()); + UNIT_ASSERT(SqlToYql("USE plato; SELECT a.x FROM (VALUES (1,2), (3,4)) a(x,key) JOIN Input b ON a.key = b.key;").IsOk()); + } + + Y_UNIT_TEST(TableNameConstness) { + UNIT_ASSERT(SqlToYql("USE plato; $path = 'foo'; SELECT TableName($path), count(*) FROM Input;").IsOk()); + UNIT_ASSERT(SqlToYql("$path = 'foo'; SELECT TableName($path, 'yt'), count(*) FROM plato.Input;").IsOk()); + ExpectFailWithError("USE plato; SELECT TableName(), count(*) FROM plato.Input;", + "<main>:1:19: Error: Expression has to be an aggregation function or key column, because aggregation is used elsewhere in this subquery\n"); + } + + Y_UNIT_TEST(UseShouldWorkAsColumnName) { + UNIT_ASSERT(SqlToYql("select use from (select 1 as use);").IsOk()); + } + + Y_UNIT_TEST(TrueFalseWorkAfterDollar) { + UNIT_ASSERT(SqlToYql("$ true = false; SELECT $ true or false;").IsOk()); + UNIT_ASSERT(SqlToYql("$False = 0; SELECT $False;").IsOk()); + } + + Y_UNIT_TEST(WithSchemaEquals) { + UNIT_ASSERT(SqlToYql("select * from plato.T with schema Struct<a:Int32, b:String>;").IsOk()); + UNIT_ASSERT(SqlToYql("select * from plato.T with columns = Struct<a:Int32, b:String>;").IsOk()); + } + + Y_UNIT_TEST(WithNonStructSchemaS3) { + NSQLTranslation::TTranslationSettings settings; + settings.ClusterMapping["s3bucket"] = NYql::S3ProviderName; + UNIT_ASSERT(SqlToYqlWithSettings("select * from s3bucket.`foo` with schema (col1 Int32, String as col2, Int64 as col3);", settings).IsOk()); + } + + Y_UNIT_TEST(AllowNestedTuplesInGroupBy) { + NYql::TAstParseResult res = SqlToYql("select count(*) from plato.Input group by 1 + (x, y, z);"); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [&](const TString& word, const TString& line) { + Y_UNUSED(word); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("(Aggregate core '('\"group0\")")); + }; + + TWordCountHive elementStat({"Aggregate"}); + VerifyProgram(res, elementStat, verifyLine); + UNIT_ASSERT(elementStat["Aggregate"] == 1); + } + + Y_UNIT_TEST(AllowGroupByWithParens) { + NYql::TAstParseResult res = SqlToYql("select count(*) from plato.Input group by (x, y as alias1, z);"); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [&](const TString& word, const TString& line) { + Y_UNUSED(word); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("(Aggregate core '('\"x\" '\"alias1\" '\"z\")")); + }; + + TWordCountHive elementStat({"Aggregate"}); + VerifyProgram(res, elementStat, verifyLine); + UNIT_ASSERT(elementStat["Aggregate"] == 1); + } + + Y_UNIT_TEST(CreateAsyncReplicationParseCorrect) { + auto req = R"( + USE plato; + CREATE ASYNC REPLICATION MyReplication + FOR table1 AS table2, table3 AS table4 + WITH ( + CONNECTION_STRING = "grpc://localhost:2135/?database=/MyDatabase", + ENDPOINT = "localhost:2135", + DATABASE = "/MyDatabase" + ); + )"; + auto res = SqlToYql(req); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("MyReplication")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("create")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("table1")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("table2")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("table3")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("table4")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("connection_string")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("grpc://localhost:2135/?database=/MyDatabase")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("endpoint")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("localhost:2135")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("database")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("/MyDatabase")); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } + + Y_UNIT_TEST(CreateAsyncReplicationUnsupportedSettings) { + auto reqTpl = R"( + USE plato; + CREATE ASYNC REPLICATION MyReplication + FOR table1 AS table2, table3 AS table4 + WITH ( + %s = "%s" + ) + )"; + + auto settings = THashMap<TString, TString>{ + {"STATE", "DONE"}, + {"FAILOVER_MODE", "FORCE"}, + }; + + for (const auto& [k, v] : settings) { + auto req = Sprintf(reqTpl, k.c_str(), v.c_str()); + auto res = SqlToYql(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), Sprintf("<main>:6:%zu: Error: %s is not supported in CREATE\n", 20 + k.size(), k.c_str())); + } + } + + Y_UNIT_TEST(AsyncReplicationInvalidCommitInterval) { + auto req = R"( + USE plato; + CREATE ASYNC REPLICATION MyReplication + FOR table1 AS table2, table3 AS table4 + WITH ( + COMMIT_INTERVAL = "FOO" + ); + )"; + + auto res = SqlToYql(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:6:35: Error: Literal of Interval type is expected for COMMIT_INTERVAL\n"); + } + + Y_UNIT_TEST(AlterAsyncReplicationParseCorrect) { + auto req = R"( + USE plato; + ALTER ASYNC REPLICATION MyReplication + SET ( + STATE = "DONE", + FAILOVER_MODE = "FORCE" + ); + )"; + auto res = SqlToYql(req); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("MyReplication")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("alter")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("state")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("DONE")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("failover_mode")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("FORCE")); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } + + Y_UNIT_TEST(AlterAsyncReplicationSettings) { + auto reqTpl = R"( + USE plato; + ALTER ASYNC REPLICATION MyReplication + SET ( + %s = "%s" + ) + )"; + + auto settings = THashMap<TString, TString>{ + {"connection_string", "grpc://localhost:2135/?database=/MyDatabase"}, + {"endpoint", "localhost:2135"}, + {"database", "/MyDatabase"}, + {"token", "foo"}, + {"token_secret_name", "foo_secret_name"}, + {"user", "user"}, + {"password", "bar"}, + {"password_secret_name", "bar_secret_name"}, + }; + + for (const auto& [k, v] : settings) { + auto req = Sprintf(reqTpl, k.c_str(), v.c_str()); + auto res = SqlToYql(req); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [&k, &v](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("MyReplication")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("alter")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(k)); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(v)); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } + } + + Y_UNIT_TEST(AlterAsyncReplicationUnsupportedSettings) { + { + auto req = R"( + USE plato; + ALTER ASYNC REPLICATION MyReplication SET (CONSISTENCY_LEVEL = "GLOBAL"); + )"; + auto res = SqlToYql(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:80: Error: CONSISTENCY_LEVEL is not supported in ALTER\n"); + } + { + auto req = R"( + USE plato; + ALTER ASYNC REPLICATION MyReplication SET (COMMIT_INTERVAL = Interval("PT10S")); + )"; + auto res = SqlToYql(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:87: Error: COMMIT_INTERVAL is not supported in ALTER\n"); + } + } + + Y_UNIT_TEST(AsyncReplicationInvalidSettings) { + auto req = R"( + USE plato; + ALTER ASYNC REPLICATION MyReplication SET (FOO = "BAR"); + )"; + auto res = SqlToYql(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:62: Error: Unknown replication setting: FOO\n"); + } + + Y_UNIT_TEST(DropAsyncReplicationParseCorrect) { + auto req = R"( + USE plato; + DROP ASYNC REPLICATION MyReplication; + )"; + auto res = SqlToYql(req); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("MyReplication")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("drop")); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } + + Y_UNIT_TEST(DropAsyncReplicationCascade) { + auto req = R"( + USE plato; + DROP ASYNC REPLICATION MyReplication CASCADE; + )"; + auto res = SqlToYql(req); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("dropCascade")); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } + + Y_UNIT_TEST(PragmaCompactGroupBy) { + auto req = "PRAGMA CompactGroupBy; SELECT key, COUNT(*) FROM plato.Input GROUP BY key;"; + auto res = SqlToYql(req); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Aggregate") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'('compact)")); + } + }; + + TWordCountHive elementStat = { {TString("Aggregate"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Aggregate"]); + } + + Y_UNIT_TEST(PragmaDisableCompactGroupBy) { + auto req = "PRAGMA DisableCompactGroupBy; SELECT key, COUNT(*) FROM plato.Input GROUP /*+ compact() */ BY key;"; + auto res = SqlToYql(req); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Aggregate") { + UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("'('compact)")); + } + }; + + TWordCountHive elementStat = { {TString("Aggregate"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Aggregate"]); + } + + Y_UNIT_TEST(AutoSampleWorksWithNamedSubquery) { + UNIT_ASSERT(SqlToYql("$src = select * from plato.Input; select * from $src sample 0.2").IsOk()); + } + + Y_UNIT_TEST(AutoSampleWorksWithSubquery) { + UNIT_ASSERT(SqlToYql("select * from (select * from plato.Input) sample 0.2").IsOk()); + } + + Y_UNIT_TEST(CreateTableTrailingComma) { + UNIT_ASSERT(SqlToYql("USE plato; CREATE TABLE tableName (Key Uint32, PRIMARY KEY (Key),);").IsOk()); + UNIT_ASSERT(SqlToYql("USE plato; CREATE TABLE tableName (Key Uint32,);").IsOk()); + } + + Y_UNIT_TEST(BetweenSymmetric) { + UNIT_ASSERT(SqlToYql("select 3 between symmetric 5 and 4;").IsOk()); + UNIT_ASSERT(SqlToYql("select 3 between asymmetric 5 and 4;").IsOk()); + UNIT_ASSERT(SqlToYql("use plato; select key between symmetric and and and from Input;").IsOk()); + UNIT_ASSERT(SqlToYql("use plato; select key between and and and from Input;").IsOk()); + } +} + +Y_UNIT_TEST_SUITE(ExternalFunction) { + Y_UNIT_TEST(ValidUseFunctions) { + + UNIT_ASSERT(SqlToYql( + "PROCESS plato.Input" + " USING EXTERNAL FUNCTION('YANDEX-CLOUD', 'foo', <|a: 123, b: a + 641|>)" + " WITH INPUT_TYPE=Struct<a:Int32>, OUTPUT_TYPE=Struct<b:Int32>," + " CONCURRENCY=3, OPTIMIZE_FOR='CALLS'").IsOk()); + + // use CALLS without quotes, as keyword + UNIT_ASSERT(SqlToYql( + "PROCESS plato.Input" + " USING EXTERNAL FUNCTION('YANDEX-CLOUD', 'foo')" + " WITH INPUT_TYPE=Struct<a:Int32>, OUTPUT_TYPE=Struct<b:Int32>," + " OPTIMIZE_FOR=CALLS").IsOk()); + + UNIT_ASSERT(SqlToYql( + "PROCESS plato.Input" + " USING EXTERNAL FUNCTION('YANDEX-CLOUD', 'foo', TableRow())" + " WITH INPUT_TYPE=Struct<a:Int32>, OUTPUT_TYPE=Struct<b:Int32>," + " CONCURRENCY=3").IsOk()); + + UNIT_ASSERT(SqlToYql( + "PROCESS plato.Input" + " USING EXTERNAL FUNCTION('YANDEX-CLOUD', 'foo')" + " WITH INPUT_TYPE=Struct<a:Int32>, OUTPUT_TYPE=Struct<b:Int32>," + " CONCURRENCY=3, BATCH_SIZE=1000000, CONNECTION='yc-folder34fse-con'," + " INIT=[0, 900]").IsOk()); + + UNIT_ASSERT(SqlToYql( + "PROCESS plato.Input" + " USING EXTERNAL FUNCTION('YANDEX-CLOUD', 'bar', TableRow())" + " WITH UNKNOWN_PARAM_1='837747712', UNKNOWN_PARAM_2=Tuple<Uint16, Utf8>," + " INPUT_TYPE=Struct<a:Int32>, OUTPUT_TYPE=Struct<b:Int32>").IsOk()); + } + + + Y_UNIT_TEST(InValidUseFunctions) { + ExpectFailWithError("PROCESS plato.Input USING some::udf(*) WITH INPUT_TYPE=Struct<a:Int32>", + "<main>:1:33: Error: PROCESS without USING EXTERNAL FUNCTION doesn't allow WITH block\n"); + + ExpectFailWithError("PROCESS plato.Input USING EXTERNAL FUNCTION('YANDEX-CLOUD', 'jhhjfh88134d')" + " WITH INPUT_TYPE=Struct<a:Int32>, OUTPUT_TYPE=Struct<b:Int32>" + " ASSUME ORDER BY key", + "<main>:1:129: Error: PROCESS with USING EXTERNAL FUNCTION doesn't allow ASSUME block\n"); + + ExpectFailWithError("PROCESS plato.Input USING EXTERNAL FUNCTION('YANDEX-CLOUD', 'foo', 'bar', 'baz')", + "<main>:1:15: Error: EXTERNAL FUNCTION requires from 2 to 3 arguments, but got: 4\n"); + + ExpectFailWithError("PROCESS plato.Input\n" + " USING EXTERNAL FUNCTION('YANDEX-CLOUD', 'foo', <|field_1: a1, field_b: b1|>)\n" + " WITH INPUT_TYPE=Struct<a:Int32>, OUTPUT_TYPE=Struct<b:Int32>,\n" + " CONCURRENCY=3, BATCH_SIZE=1000000, CONNECTION='yc-folder34fse-con',\n" + " CONCURRENCY=5, INPUT_TYPE=Struct<b:Bool>,\n" + " INIT=[0, 900]\n", + "<main>:5:2: Error: WITH \"CONCURRENCY\" clause should be specified only once\n" + "<main>:5:17: Error: WITH \"INPUT_TYPE\" clause should be specified only once\n"); + } +} + +Y_UNIT_TEST_SUITE(SqlToYQLErrors) { + Y_UNIT_TEST(UdfSyntaxSugarMissingCall) { + auto req = "SELECT Udf(DateTime::FromString, \"foo\" as RunConfig);"; + auto res = SqlToYql(req); + TString a1 = Err2Str(res); + TString a2("<main>:1:8: Error: Abstract Udf Node can't be used as a part of expression.\n"); + UNIT_ASSERT_NO_DIFF(a1, a2); + } + + Y_UNIT_TEST(UdfSyntaxSugarIsNotCallable) { + auto req = "SELECT Udf(123, \"foo\" as RunConfig);"; + auto res = SqlToYql(req); + TString a1 = Err2Str(res); + TString a2("<main>:1:8: Error: Udf: first argument must be a callable, like Foo::Bar\n"); + UNIT_ASSERT_NO_DIFF(a1, a2); + } + + Y_UNIT_TEST(UdfSyntaxSugarNoArgs) { + auto req = "SELECT Udf()();"; + auto res = SqlToYql(req); + TString a1 = Err2Str(res); + TString a2("<main>:1:8: Error: Udf: expected at least one argument\n"); + UNIT_ASSERT_NO_DIFF(a1, a2); + } + + Y_UNIT_TEST(StrayUTF8) { + /// 'c' in plato is russian here + NYql::TAstParseResult res = SqlToYql("select * from сedar.Input"); + UNIT_ASSERT(!res.Root); + + TString a1 = Err2Str(res); +#if ANTLR_VER == 3 + TString a2(R"foo(<main>:1:14: Error: Unexpected character 'с' (Unicode character <1089>) : cannot match to any predicted input... + +<main>:1:15: Error: Unexpected character : cannot match to any predicted input... + +)foo"); +#else + TString a2(R"foo(<main>:1:14: Error: token recognition error at: 'с' +)foo"); +#endif + UNIT_ASSERT_NO_DIFF(a1, a2); + } + + Y_UNIT_TEST(IvalidStringLiteralWithEscapedBackslash) { + NYql::TAstParseResult res1 = SqlToYql(R"foo($bar = 'a\\'b';)foo"); + NYql::TAstParseResult res2 = SqlToYql(R"foo($bar = "a\\"b";)foo"); + UNIT_ASSERT(!res1.Root); + UNIT_ASSERT(!res2.Root); + +#if ANTLR_VER == 3 + UNIT_ASSERT_NO_DIFF(Err2Str(res1), "<main>:1:15: Error: Unexpected character : syntax error...\n\n"); + UNIT_ASSERT_NO_DIFF(Err2Str(res2), "<main>:1:15: Error: Unexpected character : syntax error...\n\n"); +#else + UNIT_ASSERT_NO_DIFF(Err2Str(res1), "<main>:1:13: Error: token recognition error at: '';'\n"); + UNIT_ASSERT_NO_DIFF(Err2Str(res2), "<main>:1:13: Error: token recognition error at: '\";'\n"); +#endif + } + + Y_UNIT_TEST(InvalidHexInStringLiteral) { + NYql::TAstParseResult res = SqlToYql("select \"foo\\x1\\xfe\""); + UNIT_ASSERT(!res.Root); + TString a1 = Err2Str(res); + TString a2 = "<main>:1:15: Error: Failed to parse string literal: Invalid hexadecimal value\n"; + + UNIT_ASSERT_NO_DIFF(a1, a2); + } + + Y_UNIT_TEST(InvalidOctalInMultilineStringLiteral) { + NYql::TAstParseResult res = SqlToYql("select \"foo\n" + "bar\n" + "\\01\""); + UNIT_ASSERT(!res.Root); + TString a1 = Err2Str(res); + TString a2 = "<main>:3:4: Error: Failed to parse string literal: Invalid octal value\n"; + + UNIT_ASSERT_NO_DIFF(a1, a2); + } + + Y_UNIT_TEST(InvalidDoubleAtString) { + NYql::TAstParseResult res = SqlToYql("select @@@@@@"); + UNIT_ASSERT(!res.Root); +#if ANTLR_VER == 3 + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:13: Error: Unexpected character : syntax error...\n\n"); +#else + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:12: Error: extraneous input '@' expecting {<EOF>, ';'}\n"); +#endif + } + + Y_UNIT_TEST(InvalidDoubleAtStringWhichWasAcceptedEarlier) { + NYql::TAstParseResult res = SqlToYql("SELECT @@foo@@ @ @@bar@@"); + UNIT_ASSERT(!res.Root); +#if ANTLR_VER == 3 + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:7: Error: Unexpected token '@@foo@@' : cannot match to any predicted input...\n\n"); +#else + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:15: Error: mismatched input '@' expecting {<EOF>, ';'}\n"); +#endif + } + + Y_UNIT_TEST(InvalidStringFromTable) { + NYql::TAstParseResult res = SqlToYql("select \"FOO\"\"BAR from plato.foo"); + UNIT_ASSERT(!res.Root); +#if ANTLR_VER == 3 + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:31: Error: Unexpected character : syntax error...\n\n"); +#else + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:12: Error: token recognition error at: '\"BAR from plato.foo'\n"); +#endif + } + + Y_UNIT_TEST(InvalidDoubleAtStringFromTable) { + NYql::TAstParseResult res = SqlToYql("select @@@@@@ from plato.foo"); + UNIT_ASSERT(!res.Root); +#if ANTLR_VER == 3 + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:28: Error: Unexpected character : syntax error...\n\n"); +#else + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:12: Error: mismatched input '@' expecting {<EOF>, ';'}\n"); +#endif + } + + Y_UNIT_TEST(SelectInvalidSyntax) { + NYql::TAstParseResult res = SqlToYql("select 1 form Wat"); + UNIT_ASSERT(!res.Root); +#if ANTLR_VER == 3 + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:14: Error: Unexpected token 'Wat' : cannot match to any predicted input...\n\n"); +#else + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:14: Error: extraneous input 'Wat' expecting {<EOF>, ';'}\n"); +#endif + } + + Y_UNIT_TEST(SelectNoCluster) { + NYql::TAstParseResult res = SqlToYql("select foo from bar"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:1: Error: No cluster name given and no default cluster is selected\n"); + } + + Y_UNIT_TEST(SelectDuplicateColumns) { + NYql::TAstParseResult res = SqlToYql("select a, a from plato.Input"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:11: Error: Unable to use duplicate column names. Collision in name: a\n"); + } + + Y_UNIT_TEST(SelectDuplicateLabels) { + NYql::TAstParseResult res = SqlToYql("select a as foo, b as foo from plato.Input"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:18: Error: Unable to use duplicate column names. Collision in name: foo\n"); + } + + Y_UNIT_TEST(SelectCaseWithoutThen) { + NYql::TAstParseResult res = SqlToYql("select case when true 1;"); + UNIT_ASSERT(!res.Root); +#if ANTLR_VER == 3 + UNIT_ASSERT_NO_DIFF(Err2Str(res), + "<main>:1:22: Error: Unexpected token absence : Missing THEN \n\n" + "<main>:1:23: Error: Unexpected token absence : Missing END \n\n" + ); +#else + UNIT_ASSERT_NO_DIFF(Err2Str(res), + "<main>:1:22: Error: missing THEN at \'1\'\n" + "<main>:1:23: Error: extraneous input \';\' expecting {ELSE, END, WHEN}\n" + ); +#endif + } + + Y_UNIT_TEST(SelectComplexCaseWithoutThen) { + NYql::TAstParseResult res = SqlToYql( + "SELECT *\n" + "FROM plato.Input AS a\n" + "WHERE CASE WHEN a.key = \"foo\" a.subkey ELSE a.value END\n" + ); +#if ANTLR_VER == 3 + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:30: Error: Unexpected token absence : Missing THEN \n\n"); +#else + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:30: Error: missing THEN at 'a'\n"); +#endif + } + + Y_UNIT_TEST(SelectCaseWithoutEnd) { + NYql::TAstParseResult res = SqlToYql("select case a when b then c end from plato.Input"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Error: ELSE is required\n"); + } + + Y_UNIT_TEST(SelectWithBadAggregationNoInput) { + NYql::TAstParseResult res = SqlToYql("select a, Min(b), c"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), + "<main>:1:1: Error: Column references are not allowed without FROM\n" + "<main>:1:8: Error: Column reference 'a'\n" + "<main>:1:1: Error: Column references are not allowed without FROM\n" + "<main>:1:15: Error: Column reference 'b'\n" + "<main>:1:1: Error: Column references are not allowed without FROM\n" + "<main>:1:19: Error: Column reference 'c'\n" + ); + } + + Y_UNIT_TEST(SelectWithBadAggregation) { + ExpectFailWithError("select count(*), 1 + key from plato.Input", + "<main>:1:22: Error: Column `key` must either be a key column in GROUP BY or it should be used in aggregation function\n"); + } + + Y_UNIT_TEST(SelectWithBadAggregatedTerms) { + ExpectFailWithError("select key, 2 * subkey from plato.Input group by key", + "<main>:1:17: Error: Column `subkey` must either be a key column in GROUP BY or it should be used in aggregation function\n"); + } + + Y_UNIT_TEST(SelectDistinctWithBadAggregation) { + ExpectFailWithError("select distinct count(*), 1 + key from plato.Input", + "<main>:1:31: Error: Column `key` must either be a key column in GROUP BY or it should be used in aggregation function\n"); + ExpectFailWithError("select distinct key, 2 * subkey from plato.Input group by key", + "<main>:1:26: Error: Column `subkey` must either be a key column in GROUP BY or it should be used in aggregation function\n"); + } + + Y_UNIT_TEST(SelectWithBadAggregationInHaving) { + ExpectFailWithError("select key from plato.Input group by key\n" + "having \"f\" || value == \"foo\"", + "<main>:2:15: Error: Column `value` must either be a key column in GROUP BY or it should be used in aggregation function\n"); + } + + Y_UNIT_TEST(JoinWithNonAggregatedColumnInProjection) { + ExpectFailWithError("select a.key, 1 + b.subkey\n" + "from plato.Input1 as a join plato.Input2 as b using(key)\n" + "group by a.key;", + "<main>:1:19: Error: Column `b.subkey` must either be a key column in GROUP BY or it should be used in aggregation function\n"); + + ExpectFailWithError("select a.key, 1 + b.subkey.x\n" + "from plato.Input1 as a join plato.Input2 as b using(key)\n" + "group by a.key;", + "<main>:1:19: Error: Column must either be a key column in GROUP BY or it should be used in aggregation function\n"); + } + + Y_UNIT_TEST(SelectWithBadAggregatedTermsWithSources) { + ExpectFailWithError("select key, 1 + a.subkey\n" + "from plato.Input1 as a\n" + "group by a.key;", + "<main>:1:17: Error: Column `a.subkey` must either be a key column in GROUP BY or it should be used in aggregation function\n"); + ExpectFailWithError("select key, 1 + a.subkey.x\n" + "from plato.Input1 as a\n" + "group by a.key;", + "<main>:1:17: Error: Column must either be a key column in GROUP BY or it should be used in aggregation function\n"); + } + + Y_UNIT_TEST(WarnForAggregationBySelectAlias) { + NYql::TAstParseResult res = SqlToYql("select c + 1 as c from plato.Input\n" + "group by c"); + UNIT_ASSERT(res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), + "<main>:2:11: Warning: GROUP BY will aggregate by column `c` instead of aggregating by SELECT expression with same alias, code: 4532\n" + "<main>:1:10: Warning: You should probably use alias in GROUP BY instead of using it here. Please consult documentation for more details, code: 4532\n"); + + res = SqlToYql("select c + 1 as c from plato.Input\n" + "group by Math::Floor(c + 2) as c;"); + + UNIT_ASSERT(res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), + "<main>:2:22: Warning: GROUP BY will aggregate by column `c` instead of aggregating by SELECT expression with same alias, code: 4532\n" + "<main>:1:10: Warning: You should probably use alias in GROUP BY instead of using it here. Please consult documentation for more details, code: 4532\n"); + } + + Y_UNIT_TEST(NoWarnForAggregationBySelectAliasWhenAggrFunctionsAreUsedInAlias) { + NYql::TAstParseResult res = SqlToYql("select\n" + " cast(avg(val) as int) as value,\n" + " value as key\n" + "from\n" + " plato.Input\n" + "group by value"); + + UNIT_ASSERT(res.Root); + UNIT_ASSERT(res.Issues.Size() == 0); + + res = SqlToYql("select\n" + " cast(avg(val) over w as int) as value,\n" + " value as key\n" + "from\n" + " plato.Input\n" + "group by value\n" + "window w as ()"); + + UNIT_ASSERT(res.Root); + UNIT_ASSERT(res.Issues.Size() == 0); + } + + Y_UNIT_TEST(NoWarnForAggregationBySelectAliasWhenQualifiedNameIsUsed) { + NYql::TAstParseResult res = SqlToYql("select\n" + " Unwrap(a.key) as key\n" + "from plato.Input as a\n" + "join plato.Input2 as b using(k)\n" + "group by a.key;"); + UNIT_ASSERT(res.Root); + UNIT_ASSERT(res.Issues.Size() == 0); + + res = SqlToYql("select Unwrap(a.key) as key\n" + "from plato.Input as a\n" + "group by a.key;"); + UNIT_ASSERT(res.Root); + UNIT_ASSERT(res.Issues.Size() == 0); + } + + Y_UNIT_TEST(NoWarnForAggregationBySelectAliasWhenTrivialRenamingIsUsed) { + NYql::TAstParseResult res = SqlToYql("select a.key as key\n" + "from plato.Input as a\n" + "group by key;"); + UNIT_ASSERT(res.Root); + UNIT_ASSERT(res.Issues.Size() == 0); + + res = SqlToYql("select key as key\n" + "from plato.Input\n" + "group by key;"); + UNIT_ASSERT(res.Root); + UNIT_ASSERT(res.Issues.Size() == 0); + } + + Y_UNIT_TEST(ErrorByAggregatingByExpressionWithSameExpressionInSelect) { + ExpectFailWithError("select k * 2 from plato.Input group by k * 2", + "<main>:1:8: Error: Column `k` must either be a key column in GROUP BY or it should be used in aggregation function\n"); + } + + Y_UNIT_TEST(ErrorForAggregationBySelectAlias) { + ExpectFailWithError("select key, Math::Floor(1.1 + a.subkey) as foo\n" + "from plato.Input as a\n" + "group by a.key, foo;", + "<main>:3:17: Warning: GROUP BY will aggregate by column `foo` instead of aggregating by SELECT expression with same alias, code: 4532\n" + "<main>:1:19: Warning: You should probably use alias in GROUP BY instead of using it here. Please consult documentation for more details, code: 4532\n" + "<main>:1:31: Error: Column `a.subkey` must either be a key column in GROUP BY or it should be used in aggregation function\n"); + + ExpectFailWithError("select c + 1 as c from plato.Input\n" + "group by Math::Floor(c + 2);", + "<main>:2:22: Warning: GROUP BY will aggregate by column `c` instead of aggregating by SELECT expression with same alias, code: 4532\n" + "<main>:1:10: Warning: You should probably use alias in GROUP BY instead of using it here. Please consult documentation for more details, code: 4532\n" + "<main>:1:8: Error: Column `c` must either be a key column in GROUP BY or it should be used in aggregation function\n"); + } + + Y_UNIT_TEST(SelectWithDuplicateGroupingColumns) { + NYql::TAstParseResult res = SqlToYql("select c from plato.Input group by c, c"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Error: Duplicate grouping column: c\n"); + } + + Y_UNIT_TEST(SelectWithBadAggregationInGrouping) { + NYql::TAstParseResult res = SqlToYql("select a, Min(b), c group by c"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:1: Error: Column references are not allowed without FROM\n" + "<main>:1:30: Error: Column reference 'c'\n"); + } + + Y_UNIT_TEST(SelectWithOpOnBadAggregation) { + ExpectFailWithError("select 1 + a + Min(b) from plato.Input", + "<main>:1:12: Error: Column `a` must either be a key column in GROUP BY or it should be used in aggregation function\n"); + } + + Y_UNIT_TEST(SelectOrderByConstantNum) { + NYql::TAstParseResult res = SqlToYql("select a from plato.Input order by 1"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:36: Error: Unable to ORDER BY constant expression\n"); + } + + Y_UNIT_TEST(SelectOrderByConstantExpr) { + NYql::TAstParseResult res = SqlToYql("select a from plato.Input order by 1 * 42"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:38: Error: Unable to ORDER BY constant expression\n"); + } + + Y_UNIT_TEST(SelectOrderByConstantString) { + NYql::TAstParseResult res = SqlToYql("select a from plato.Input order by \"nest\""); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:36: Error: Unable to ORDER BY constant expression\n"); + } + + Y_UNIT_TEST(SelectOrderByAggregated) { + NYql::TAstParseResult res = SqlToYql("select a from plato.Input order by min(a)"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:36: Error: Unable to ORDER BY aggregated values\n"); + } + + Y_UNIT_TEST(ErrorInOrderByExpresison) { + NYql::TAstParseResult res = SqlToYql("select key, value from plato.Input order by (key as zey)"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:45: Error: You should use in ORDER BY column name, qualified field, callable function or expression\n"); + } + + Y_UNIT_TEST(ErrorsInOrderByWhenColumnIsMissingInProjection) { + ExpectFailWithError("select subkey from (select 1 as subkey) order by key", "<main>:1:50: Error: Column key is not in source column set\n"); + ExpectFailWithError("select subkey from plato.Input as a order by x.key", "<main>:1:46: Error: Unknown correlation name: x\n"); + ExpectFailWithError("select distinct a, b from plato.Input order by c", "<main>:1:48: Error: Column c is not in source column set. Did you mean a?\n"); + ExpectFailWithError("select count(*) as a from plato.Input order by c", "<main>:1:48: Error: Column c is not in source column set. Did you mean a?\n"); + ExpectFailWithError("select count(*) as a, b, from plato.Input group by b order by c", "<main>:1:63: Error: Column c is not in source column set. Did you mean a?\n"); + UNIT_ASSERT(SqlToYql("select a, b from plato.Input order by c").IsOk()); + } + + Y_UNIT_TEST(SelectAggregatedWhere) { + NYql::TAstParseResult res = SqlToYql("select * from plato.Input where count(key)"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:33: Error: Can not use aggregated values in filtering\n"); + } + + Y_UNIT_TEST(DoubleFrom) { + NYql::TAstParseResult res = SqlToYql("from plato.Input select * from plato.Input"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:27: Error: Only one FROM clause is allowed\n"); + } + + Y_UNIT_TEST(SelectJoinMissingCorrName) { + NYql::TAstParseResult res = SqlToYql("select * from plato.Input1 as a join plato.Input2 as b on a.key == key"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:65: Error: JOIN: column requires correlation name\n"); + } + + Y_UNIT_TEST(SelectJoinMissingCorrName1) { + NYql::TAstParseResult res = SqlToYql( + "use plato;\n" + "$foo = select * from Input1;\n" + "select * from Input2 join $foo USING(key);\n" + ); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:27: Error: JOIN: missing correlation name for source\n"); + } + + Y_UNIT_TEST(SelectJoinMissingCorrName2) { + NYql::TAstParseResult res = SqlToYql( + "use plato;\n" + "$foo = select * from Input1;\n" + "select * from Input2 cross join $foo;\n" + ); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:33: Error: JOIN: missing correlation name for source\n"); + } + + Y_UNIT_TEST(SelectJoinEmptyCorrNames) { + NYql::TAstParseResult res = SqlToYql( + "$left = (SELECT * FROM plato.Input1 LIMIT 2);\n" + "$right = (SELECT * FROM plato.Input2 LIMIT 2);\n" + "SELECT * FROM $left FULL JOIN $right USING (key);\n" + ); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:45: Error: At least one correlation name is required in join\n"); + } + + Y_UNIT_TEST(SelectJoinSameCorrNames) { + NYql::TAstParseResult res = SqlToYql("SELECT Input.key FROM plato.Input JOIN plato.Input1 ON Input.key == Input.subkey\n"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:66: Error: JOIN: different correlation names are required for joined tables\n"); + } + + Y_UNIT_TEST(SelectJoinConstPredicateArg) { + NYql::TAstParseResult res = SqlToYql("SELECT * FROM plato.Input1 as A JOIN plato.Input2 as B ON A.key == B.key AND A.subkey == \"wtf\"\n"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:87: Error: JOIN: each equality predicate argument must depend on exactly one JOIN input\n"); + } + + Y_UNIT_TEST(SelectJoinNonEqualityPredicate) { + NYql::TAstParseResult res = SqlToYql("SELECT * FROM plato.Input1 as A JOIN plato.Input2 as B ON A.key == B.key AND A.subkey > B.subkey\n"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:87: Error: JOIN ON expression must be a conjunction of equality predicates\n"); + } + + Y_UNIT_TEST(SelectEquiJoinCorrNameOutOfScope) { + NYql::TAstParseResult res = SqlToYql( + "PRAGMA equijoin;\n" + "SELECT * FROM plato.A JOIN plato.B ON A.key == C.key JOIN plato.C ON A.subkey == C.subkey;\n" + ); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:2:45: Error: JOIN: can not use source: C in equality predicate, it is out of current join scope\n"); + } + + Y_UNIT_TEST(SelectEquiJoinNoRightSource) { + NYql::TAstParseResult res = SqlToYql( + "PRAGMA equijoin;\n" + "SELECT * FROM plato.A JOIN plato.B ON A.key == B.key JOIN plato.C ON A.subkey == B.subkey;\n" + ); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:2:79: Error: JOIN ON equality predicate must have one of its arguments from the rightmost source\n"); + } + + Y_UNIT_TEST(SelectEquiJoinOuterWithoutType) { + NYql::TAstParseResult res = SqlToYql( + "SELECT * FROM plato.A Outer JOIN plato.B ON A.key == B.key;\n" + ); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:23: Error: Invalid join type: OUTER JOIN. OUTER keyword is optional and can only be used after LEFT, RIGHT or FULL\n"); + } + + Y_UNIT_TEST(SelectEquiJoinOuterWithWrongType) { + NYql::TAstParseResult res = SqlToYql( + "SELECT * FROM plato.A LEFT semi OUTER JOIN plato.B ON A.key == B.key;\n" + ); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:33: Error: Invalid join type: LEFT SEMI OUTER JOIN. OUTER keyword is optional and can only be used after LEFT, RIGHT or FULL\n"); + } + + Y_UNIT_TEST(InsertNoCluster) { + NYql::TAstParseResult res = SqlToYql("insert into Output (foo) values (1)"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:1: Error: No cluster name given and no default cluster is selected\n"); + } + + Y_UNIT_TEST(InsertValuesNoLabels) { + NYql::TAstParseResult res = SqlToYql("insert into plato.Output values (1)"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:19: Error: INSERT INTO ... VALUES requires specification of table columns\n"); + } + + Y_UNIT_TEST(UpsertValuesNoLabelsKikimr) { + NYql::TAstParseResult res = SqlToYql("upsert into plato.Output values (1)", 10, TString(NYql::KikimrProviderName)); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:19: Error: UPSERT INTO ... VALUES requires specification of table columns\n"); + } + + Y_UNIT_TEST(ReplaceValuesNoLabelsKikimr) { + NYql::TAstParseResult res = SqlToYql("replace into plato.Output values (1)", 10, TString(NYql::KikimrProviderName)); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:20: Error: REPLACE INTO ... VALUES requires specification of table columns\n"); + } + + Y_UNIT_TEST(InsertValuesInvalidLabels) { + NYql::TAstParseResult res = SqlToYql("insert into plato.Output (foo) values (1, 2)"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:27: Error: VALUES have 2 columns, INSERT INTO expects: 1\n"); + } + + Y_UNIT_TEST(BuiltinFileOpNoArgs) { + NYql::TAstParseResult res = SqlToYql("select FilePath()"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Error: FilePath() requires exactly 1 arguments, given: 0\n"); + } + + Y_UNIT_TEST(ProcessWithHaving) { + NYql::TAstParseResult res = SqlToYql("process plato.Input using some::udf(value) having value == 1"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:15: Error: PROCESS does not allow HAVING yet! You may request it on yql@ maillist.\n"); + } + + Y_UNIT_TEST(ReduceNoBy) { + NYql::TAstParseResult res = SqlToYql("reduce plato.Input using some::udf(value)"); + UNIT_ASSERT(!res.Root); +#if ANTLR_VER == 3 + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:19: Error: Unexpected token absence : Missing ON \n\n<main>:1:25: Error: Unexpected token absence : Missing USING \n\n"); +#else + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:19: Error: mismatched input 'using' expecting {',', ON, PRESORT}\n"); +#endif + } + + Y_UNIT_TEST(ReduceDistinct) { + NYql::TAstParseResult res = SqlToYql("reduce plato.Input on key using some::udf(distinct value)"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:43: Error: DISTINCT can not be used in PROCESS/REDUCE\n"); + } + + Y_UNIT_TEST(CreateTableWithView) { + NYql::TAstParseResult res = SqlToYql("CREATE TABLE plato.foo:bar (key INT);"); + UNIT_ASSERT(!res.Root); +#if ANTLR_VER == 3 + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:22: Error: Unexpected token ':' : syntax error...\n\n"); +#else + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:22: Error: mismatched input ':' expecting '('\n"); +#endif + } + + Y_UNIT_TEST(AsteriskWithSomethingAfter) { + NYql::TAstParseResult res = SqlToYql("select *, LENGTH(value) from plato.Input;"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Error: Unable to use plain '*' with other projection items. Please use qualified asterisk instead: '<table>.*' (<table> can be either table name or table alias).\n"); + } + + Y_UNIT_TEST(AsteriskWithSomethingBefore) { + NYql::TAstParseResult res = SqlToYql("select LENGTH(value), * from plato.Input;"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:23: Error: Unable to use plain '*' with other projection items. Please use qualified asterisk instead: '<table>.*' (<table> can be either table name or table alias).\n"); + } + + Y_UNIT_TEST(DuplicatedQualifiedAsterisk) { + NYql::TAstParseResult res = SqlToYql("select in.*, key, in.* from plato.Input as in;"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:19: Error: Unable to use twice same quialified asterisk. Invalid source: in\n"); + } + + Y_UNIT_TEST(BrokenLabel) { + NYql::TAstParseResult res = SqlToYql("select in.*, key as `funny.label` from plato.Input as in;"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:14: Error: Unable to use '.' in column name. Invalid column name: funny.label\n"); + } + + Y_UNIT_TEST(KeyConflictDetect0) { + NYql::TAstParseResult res = SqlToYql("select key, in.key as key from plato.Input as in;"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:13: Error: Unable to use duplicate column names. Collision in name: key\n"); + } + + Y_UNIT_TEST(KeyConflictDetect1) { + NYql::TAstParseResult res = SqlToYql("select length(key) as key, key from plato.Input;"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:28: Error: Unable to use duplicate column names. Collision in name: key\n"); + } + + Y_UNIT_TEST(KeyConflictDetect2) { + NYql::TAstParseResult res = SqlToYql("select key, in.key from plato.Input as in;"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:1: Error: Duplicate column: key\n"); + } + + Y_UNIT_TEST(AutogenerationAliasWithCollisionConflict1) { + UNIT_ASSERT(SqlToYql("select LENGTH(Value), key as column0 from plato.Input;").IsOk()); + } + + Y_UNIT_TEST(AutogenerationAliasWithCollisionConflict2) { + UNIT_ASSERT(SqlToYql("select key as column1, LENGTH(Value) from plato.Input;").IsOk()); + } + + Y_UNIT_TEST(MissedSourceTableForQualifiedAsteriskOnSimpleSelect) { + NYql::TAstParseResult res = SqlToYql("use plato; select Intop.*, Input.key from plato.Input;"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:19: Error: Unknown correlation name: Intop\n"); + } + + Y_UNIT_TEST(MissedSourceTableForQualifiedAsteriskOnJoin) { + NYql::TAstParseResult res = SqlToYql("use plato; select tmissed.*, t2.*, t1.key from plato.Input as t1 join plato.Input as t2 on t1.key==t2.key;"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:19: Error: Unknown correlation name for asterisk: tmissed\n"); + } + + Y_UNIT_TEST(UnableToReferenceOnNotExistSubcolumn) { + NYql::TAstParseResult res = SqlToYql("select b.subkey from (select key from plato.Input as a) as b;"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Error: Column subkey is not in source column set\n"); + } + + Y_UNIT_TEST(ConflictOnSameNameWithQualify0) { + NYql::TAstParseResult res = SqlToYql("select in.key, in.key as key from plato.Input as in;"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:1: Error: Duplicate column: key\n"); + } + + Y_UNIT_TEST(ConflictOnSameNameWithQualify1) { + NYql::TAstParseResult res = SqlToYql("select in.key, length(key) as key from plato.Input as in;"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:1: Error: Duplicate column: key\n"); + } + + Y_UNIT_TEST(ConflictOnSameNameWithQualify2) { + NYql::TAstParseResult res = SqlToYql("select key, in.key from plato.Input as in;"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:1: Error: Duplicate column: key\n"); + } + + Y_UNIT_TEST(ConflictOnSameNameWithQualify3) { + NYql::TAstParseResult res = SqlToYql("select in.key, subkey as key from plato.Input as in;"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:1: Error: Duplicate column: key\n"); + } + + Y_UNIT_TEST(SelectFlattenBySameColumns) { + NYql::TAstParseResult res = SqlToYql("select key from plato.Input flatten by (key, key as kk)"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:46: Error: Duplicate column name found: key in FlattenBy section\n"); + } + + Y_UNIT_TEST(SelectFlattenBySameAliases) { + NYql::TAstParseResult res = SqlToYql("select key from plato.Input flatten by (key as kk, subkey as kk);"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:52: Error: Duplicate alias found: kk in FlattenBy section\n"); + } + + Y_UNIT_TEST(SelectFlattenByExprSameAliases) { + NYql::TAstParseResult res = SqlToYql("select key from plato.Input flatten by (key as kk, ListSkip(subkey,1) as kk);"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:52: Error: Collision between alias and column name: kk in FlattenBy section\n"); + } + + Y_UNIT_TEST(SelectFlattenByConflictNameAndAlias0) { + NYql::TAstParseResult res = SqlToYql("select key from plato.Input flatten by (key, subkey as key);"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:46: Error: Collision between alias and column name: key in FlattenBy section\n"); + } + + Y_UNIT_TEST(SelectFlattenByConflictNameAndAlias1) { + NYql::TAstParseResult res = SqlToYql("select key from plato.Input flatten by (key as kk, subkey as key);"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:52: Error: Collision between alias and column name: key in FlattenBy section\n"); + } + + Y_UNIT_TEST(SelectFlattenByExprConflictNameAndAlias1) { + NYql::TAstParseResult res = SqlToYql("select key from plato.Input flatten by (key as kk, ListSkip(subkey,1) as key);"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:52: Error: Duplicate column name found: key in FlattenBy section\n"); + } + + Y_UNIT_TEST(SelectFlattenByUnnamedExpr) { + NYql::TAstParseResult res = SqlToYql("select key from plato.Input flatten by (key, ListSkip(key, 1))"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:46: Error: Unnamed expression after FLATTEN BY is not allowed\n"); + } + + Y_UNIT_TEST(UseInOnStrings) { + NYql::TAstParseResult res = SqlToYql("select * from plato.Input where \"foo\" in \"foovalue\";"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:42: Error: Unable to use IN predicate with string argument, it won't search substring - " + "expecting tuple, list, dict or single column table source\n"); + } + + Y_UNIT_TEST(UseSubqueryInScalarContextInsideIn) { + NYql::TAstParseResult res = SqlToYql("$q = (select key from plato.Input); select * from plato.Input where subkey in ($q);"); + UNIT_ASSERT(res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:79: Warning: Using subrequest in scalar context after IN, " + "perhaps you should remove parenthesis here, code: 4501\n"); + } + + Y_UNIT_TEST(InHintsWithKeywordClash) { + NYql::TAstParseResult res = SqlToYql("SELECT COMPACT FROM plato.Input WHERE COMPACT IN COMPACT `COMPACT`(1,2,3)"); + UNIT_ASSERT(!res.Root); + // should try to parse last compact as call expression + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:58: Error: Unknown builtin: COMPACT\n"); + } + + Y_UNIT_TEST(ErrorColumnPosition) { + NYql::TAstParseResult res = SqlToYql( + "USE plato;\n" + "SELECT \n" + "value FROM (\n" + "select key from Input\n" + ");\n" + ); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:1: Error: Column value is not in source column set\n"); + } + + Y_UNIT_TEST(PrimaryViewAbortMapReduce) { + NYql::TAstParseResult res = SqlToYql("SELECT key FROM plato.Input VIEW PRIMARY KEY"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:17: Error: primary view is not supported for yt tables\n"); + } + + Y_UNIT_TEST(InsertAbortMapReduce) { + NYql::TAstParseResult res = SqlToYql("INSERT OR ABORT INTO plato.Output SELECT key FROM plato.Input"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:0: Error: INSERT OR ABORT INTO is not supported for yt tables\n"); + } + + Y_UNIT_TEST(ReplaceIntoMapReduce) { + NYql::TAstParseResult res = SqlToYql("REPLACE INTO plato.Output SELECT key FROM plato.Input"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:0: Error: Meaning of REPLACE INTO has been changed, now you should use INSERT INTO <table> WITH TRUNCATE ... for yt\n"); + } + + Y_UNIT_TEST(UpsertIntoMapReduce) { + NYql::TAstParseResult res = SqlToYql("UPSERT INTO plato.Output SELECT key FROM plato.Input"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:0: Error: UPSERT INTO is not supported for yt tables\n"); + } + + Y_UNIT_TEST(UpdateMapReduce) { + NYql::TAstParseResult res = SqlToYql("UPDATE plato.Output SET value = value + 1 WHERE key < 1"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:0: Error: UPDATE is unsupported for yt\n"); + } + + Y_UNIT_TEST(DeleteMapReduce) { + NYql::TAstParseResult res = SqlToYql("DELETE FROM plato.Output WHERE key < 1"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:0: Error: DELETE is unsupported for yt\n"); + } + + Y_UNIT_TEST(ReplaceIntoWithTruncate) { + NYql::TAstParseResult res = SqlToYql("REPLACE INTO plato.Output WITH TRUNCATE SELECT key FROM plato.Input"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:32: Error: Unable REPLACE INTO with truncate mode\n"); + } + + Y_UNIT_TEST(UpsertIntoWithTruncate) { + NYql::TAstParseResult res = SqlToYql("UPSERT INTO plato.Output WITH TRUNCATE SELECT key FROM plato.Input"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:31: Error: Unable UPSERT INTO with truncate mode\n"); + } + + Y_UNIT_TEST(InsertIntoWithTruncateKikimr) { + NYql::TAstParseResult res = SqlToYql("INSERT INTO plato.Output WITH TRUNCATE SELECT key FROM plato.Input", 10, TString(NYql::KikimrProviderName)); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:0: Error: INSERT INTO WITH TRUNCATE is not supported for kikimr tables\n"); + } + + Y_UNIT_TEST(InsertIntoWithWrongArgumentCount) { + NYql::TAstParseResult res = SqlToYql("insert into plato.Output with truncate (key, value, subkey) values (5, '1', '2', '3');"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:53: Error: VALUES have 4 columns, INSERT INTO ... WITH TRUNCATE expects: 3\n"); + } + + Y_UNIT_TEST(UpsertWithWrongArgumentCount) { + NYql::TAstParseResult res = SqlToYql("upsert into plato.Output (key, value, subkey) values (2, '3');", 10, TString(NYql::KikimrProviderName)); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:39: Error: VALUES have 2 columns, UPSERT INTO expects: 3\n"); + } + + Y_UNIT_TEST(GroupingSetByExprWithoutAlias) { + NYql::TAstParseResult res = SqlToYql("SELECT key FROM plato.Input GROUP BY GROUPING SETS (cast(key as uint32), subkey);"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:53: Error: Unnamed expressions are not supported in GROUPING SETS. Please use '<expr> AS <name>'.\n"); + } + + Y_UNIT_TEST(GroupingSetByExprWithoutAlias2) { + NYql::TAstParseResult res = SqlToYql("SELECT key FROM plato.Input GROUP BY subkey || subkey, GROUPING SETS (\n" + "cast(key as uint32), subkey);"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:2:1: Error: Unnamed expressions are not supported in GROUPING SETS. Please use '<expr> AS <name>'.\n"); + } + + Y_UNIT_TEST(CubeByExprWithoutAlias) { + NYql::TAstParseResult res = SqlToYql("SELECT key FROM plato.Input GROUP BY CUBE (key, subkey / key);"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:56: Error: Unnamed expressions are not supported in CUBE. Please use '<expr> AS <name>'.\n"); + } + + Y_UNIT_TEST(RollupByExprWithoutAlias) { + NYql::TAstParseResult res = SqlToYql("SELECT key FROM plato.Input GROUP BY ROLLUP (subkey / key);"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:53: Error: Unnamed expressions are not supported in ROLLUP. Please use '<expr> AS <name>'.\n"); + } + + Y_UNIT_TEST(GroupByHugeCubeDeniedNoPragma) { + NYql::TAstParseResult res = SqlToYql("SELECT key FROM plato.Input GROUP BY CUBE (key, subkey, value, key + subkey as sum, key - subkey as sub, key + val as keyval);"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:119: Error: GROUP BY CUBE is allowed only for 5 columns, but you use 6\n"); + } + + Y_UNIT_TEST(GroupByInvalidPragma) { + NYql::TAstParseResult res = SqlToYql("PRAGMA GroupByCubeLimit = '-4';"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:27: Error: Expected unsigned integer literal as a single argument for: GroupByCubeLimit\n"); + } + + Y_UNIT_TEST(GroupByHugeCubeDeniedPragme) { + NYql::TAstParseResult res = SqlToYql("PRAGMA GroupByCubeLimit = '4'; SELECT key FROM plato.Input GROUP BY CUBE (key, subkey, value, key + subkey as sum, key - subkey as sub);"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:132: Error: GROUP BY CUBE is allowed only for 4 columns, but you use 5\n"); + } + + Y_UNIT_TEST(GroupByFewBigCubes) { + NYql::TAstParseResult res = SqlToYql("SELECT key FROM plato.Input GROUP BY CUBE(key, subkey, key + subkey as sum), CUBE(value, value + key + subkey as total);"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:1: Error: Unable to GROUP BY more than 64 groups, you try use 80 groups\n"); + } + + Y_UNIT_TEST(GroupByFewBigCubesWithPragmaLimit) { + NYql::TAstParseResult res = SqlToYql("PRAGMA GroupByLimit = '16'; SELECT key FROM plato.Input GROUP BY GROUPING SETS(key, subkey, key + subkey as sum), ROLLUP(value, value + key + subkey as total);"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:29: Error: Unable to GROUP BY more than 16 groups, you try use 18 groups\n"); + } + + Y_UNIT_TEST(NoGroupingColumn0) { + NYql::TAstParseResult res = SqlToYql( + "select count(1), key_first, val_first, grouping(key_first, val_first, nomind) as group\n" + "from plato.Input group by grouping sets (cast(key as uint32) /100 as key_first, Substring(value, 1, 1) as val_first);"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:71: Error: Column 'nomind' is not a grouping column\n"); + } + + Y_UNIT_TEST(NoGroupingColumn1) { + NYql::TAstParseResult res = SqlToYql("select count(1), grouping(key, value) as group_duo from plato.Input group by cube (key, subkey);"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:32: Error: Column 'value' is not a grouping column\n"); + } + + Y_UNIT_TEST(EmptyAccess0) { + NYql::TAstParseResult res = SqlToYql("insert into plato.Output (list0, list1) values (AsList(0, 1, 2), AsList(``));"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:73: Error: Column reference \"\" is not allowed in current scope\n"); + } + + Y_UNIT_TEST(EmptyAccess1) { + NYql::TAstParseResult res = SqlToYql("insert into plato.Output (list0, list1) values (AsList(0, 1, 2), ``);"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:66: Error: Column reference \"\" is not allowed in current scope\n"); + } + + Y_UNIT_TEST(UseUnknownColumnInInsert) { + NYql::TAstParseResult res = SqlToYql("insert into plato.Output (list0, list1) values (AsList(0, 1, 2), AsList(`test`));"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:73: Error: Column reference \"test\" is not allowed in current scope\n"); + } + + Y_UNIT_TEST(GroupByEmptyColumn) { + NYql::TAstParseResult res = SqlToYql("select count(1) from plato.Input group by ``;"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:43: Error: Column name can not be empty\n"); + } + + Y_UNIT_TEST(ConvertNumberOutOfBase) { + NYql::TAstParseResult res = SqlToYql("select 0o80l;"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Error: Failed to parse number from string: 0o80l, char: '8' is out of base: 8\n"); + } + + Y_UNIT_TEST(ConvertNumberOutOfRangeForInt64ButFitsInUint64) { + NYql::TAstParseResult res = SqlToYql("select 0xc000000000000000l;"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Error: Failed to parse 13835058055282163712 as integer literal of Int64 type: value out of range for Int64\n"); + } + + Y_UNIT_TEST(ConvertNumberOutOfRangeUint64) { + NYql::TAstParseResult res = SqlToYql("select 0xc0000000000000000l;"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Error: Failed to parse number from string: 0xc0000000000000000l, number limit overflow\n"); + + res = SqlToYql("select 1234234543563435151456;\n"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Error: Failed to parse number from string: 1234234543563435151456, number limit overflow\n"); + } + + Y_UNIT_TEST(ConvertNumberNegativeOutOfRange) { + NYql::TAstParseResult res = SqlToYql("select -9223372036854775808;\n" + "select -9223372036854775809;"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:2:8: Error: Failed to parse negative integer: -9223372036854775809, number limit overflow\n"); + } + + Y_UNIT_TEST(InvaildUsageReal0) { + NYql::TAstParseResult res = SqlToYql("select .0;"); + UNIT_ASSERT(!res.Root); +#if ANTLR_VER == 3 + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:7: Error: Unexpected token '.' : cannot match to any predicted input...\n\n"); +#else + UNIT_ASSERT_STRING_CONTAINS(Err2Str(res), "<main>:1:7: Error: extraneous input '.' expecting {"); +#endif + } + + Y_UNIT_TEST(InvaildUsageReal1) { + NYql::TAstParseResult res = SqlToYql("select .0f;"); + UNIT_ASSERT(!res.Root); +#if ANTLR_VER == 3 + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:7: Error: Unexpected token '.' : cannot match to any predicted input...\n\n"); +#else + UNIT_ASSERT_STRING_CONTAINS(Err2Str(res), "<main>:1:7: Error: extraneous input '.' expecting {"); +#endif + } + + Y_UNIT_TEST(InvaildUsageWinFunctionWithoutWindow) { + NYql::TAstParseResult res = SqlToYql("select lead(key, 2) from plato.Input;"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Error: Failed to use window function Lead without window specification\n"); + } + + Y_UNIT_TEST(DropTableWithIfExists) { + NYql::TAstParseResult res = SqlToYql("DROP TABLE IF EXISTS plato.foo;"); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("drop_if_exists")); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } + + Y_UNIT_TEST(TooManyErrors) { + const char* q = R"( + USE plato; + select A, B, C, D, E, F, G, H, I, J, K, L, M, N from (select b from `abc`); +)"; + + NYql::TAstParseResult res = SqlToYql(q, 10); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), + R"(<main>:3:16: Error: Column A is not in source column set. Did you mean b? +<main>:3:19: Error: Column B is not in source column set. Did you mean b? +<main>:3:22: Error: Column C is not in source column set. Did you mean b? +<main>:3:25: Error: Column D is not in source column set. Did you mean b? +<main>:3:28: Error: Column E is not in source column set. Did you mean b? +<main>:3:31: Error: Column F is not in source column set. Did you mean b? +<main>:3:34: Error: Column G is not in source column set. Did you mean b? +<main>:3:37: Error: Column H is not in source column set. Did you mean b? +<main>:3:40: Error: Column I is not in source column set. Did you mean b? +<main>: Error: Too many issues, code: 1 +)"); + }; + + Y_UNIT_TEST(ShouldCloneBindingForNamedParameter) { + NYql::TAstParseResult res = SqlToYql(R"($f = () -> { + $value_type = TypeOf(1); + $pair_type = StructType( + TypeOf("2") AS key, + $value_type AS value + ); + + RETURN TupleType( + ListType($value_type), + $pair_type); +}; + +select FormatType($f()); +)"); + UNIT_ASSERT(res.Root); + } + + Y_UNIT_TEST(BlockedInvalidFrameBounds) { + auto check = [](const TString& frame, const TString& err) { + const TString prefix = "SELECT SUM(x) OVER w FROM plato.Input WINDOW w AS (PARTITION BY key ORDER BY subkey\n"; + NYql::TAstParseResult res = SqlToYql(prefix + frame + ")"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), err); + }; + + check("ROWS UNBOUNDED FOLLOWING", "<main>:2:5: Error: Frame cannot start from UNBOUNDED FOLLOWING\n"); + check("ROWS BETWEEN 5 PRECEDING AND UNBOUNDED PRECEDING", "<main>:2:29: Error: Frame cannot end with UNBOUNDED PRECEDING\n"); + check("ROWS BETWEEN CURRENT ROW AND 5 PRECEDING", "<main>:2:13: Error: Frame cannot start from CURRENT ROW and end with PRECEDING\n"); + check("ROWS BETWEEN 5 FOLLOWING AND CURRENT ROW", "<main>:2:14: Error: Frame cannot start from FOLLOWING and end with CURRENT ROW\n"); + check("ROWS BETWEEN 5 FOLLOWING AND 5 PRECEDING", "<main>:2:14: Error: Frame cannot start from FOLLOWING and end with PRECEDING\n"); + } + + Y_UNIT_TEST(BlockedRangeValueWithoutSingleOrderBy) { + UNIT_ASSERT(SqlToYql("SELECT COUNT(*) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) FROM plato.Input").IsOk()); + UNIT_ASSERT(SqlToYql("SELECT COUNT(*) OVER (RANGE BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING) FROM plato.Input").IsOk()); + + auto res = SqlToYql("SELECT COUNT(*) OVER (RANGE 5 PRECEDING) FROM plato.Input"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:29: Error: RANGE with <offset> PRECEDING/FOLLOWING requires exactly one expression in ORDER BY partition clause\n"); + + res = SqlToYql("SELECT COUNT(*) OVER (ORDER BY key, value RANGE 5 PRECEDING) FROM plato.Input"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:49: Error: RANGE with <offset> PRECEDING/FOLLOWING requires exactly one expression in ORDER BY partition clause\n"); + } + + Y_UNIT_TEST(NoColumnsInFrameBounds) { + NYql::TAstParseResult res = SqlToYql( + "SELECT SUM(x) OVER w FROM plato.Input WINDOW w AS (ROWS BETWEEN\n" + " 1 + key PRECEDING AND 2 + key FOLLOWING);"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:2:6: Error: Column reference \"key\" is not allowed in current scope\n"); + } + + Y_UNIT_TEST(WarnOnEmptyFrameBounds) { + NYql::TAstParseResult res = SqlToYql( + "SELECT SUM(x) OVER w FROM plato.Input WINDOW w AS (PARTITION BY key ORDER BY subkey\n" + "ROWS BETWEEN 10 FOLLOWING AND 5 FOLLOWING)"); + UNIT_ASSERT(res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:2:14: Warning: Used frame specification implies empty window frame, code: 4520\n"); + } + + Y_UNIT_TEST(WarnOnRankWithUnorderedWindow) { + NYql::TAstParseResult res = SqlToYql("SELECT RANK() OVER w FROM plato.Input WINDOW w AS ()"); + UNIT_ASSERT(res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Warning: Rank() is used with unordered window - all rows will be considered equal to each other, code: 4521\n"); + } + + Y_UNIT_TEST(WarnOnRankExprWithUnorderedWindow) { + NYql::TAstParseResult res = SqlToYql("SELECT RANK(key) OVER w FROM plato.Input WINDOW w AS ()"); + UNIT_ASSERT(res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Warning: Rank(<expression>) is used with unordered window - the result is likely to be undefined, code: 4521\n"); + } + + Y_UNIT_TEST(AnyAsTableName) { + NYql::TAstParseResult res = SqlToYql("use plato; select * from any;"); + UNIT_ASSERT(!res.Root); +#if ANTLR_VER == 3 + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:28: Error: Unexpected token ';' : syntax error...\n\n"); +#else + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:28: Error: no viable alternative at input 'any;'\n"); +#endif + } + + Y_UNIT_TEST(IncorrectOrderOfLambdaOptionalArgs) { + NYql::TAstParseResult res = SqlToYql("$f = ($x?, $y)->($x + $y); select $f(1);"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:12: Error: Non-optional argument can not follow optional one\n"); + } + + Y_UNIT_TEST(IncorrectOrderOfActionOptionalArgs) { + NYql::TAstParseResult res = SqlToYql("define action $f($x?, $y) as select $x,$y; end define; do $f(1);"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:23: Error: Non-optional argument can not follow optional one\n"); + } + + Y_UNIT_TEST(NotAllowedQuestionOnNamedNode) { + NYql::TAstParseResult res = SqlToYql("$f = 1; select $f?;"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:18: Error: Unexpected token '?' at the end of expression\n"); + } + + Y_UNIT_TEST(AnyAndCrossJoin) { + NYql::TAstParseResult res = SqlToYql("use plato; select * from any Input1 cross join Input2"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:26: Error: ANY should not be used with Cross JOIN\n"); + + res = SqlToYql("use plato; select * from Input1 cross join any Input2"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:44: Error: ANY should not be used with Cross JOIN\n"); + } + + Y_UNIT_TEST(AnyWithCartesianProduct) { + NYql::TAstParseResult res = SqlToYql("pragma AnsiImplicitCrossJoin; use plato; select * from any Input1, Input2"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:56: Error: ANY should not be used with Cross JOIN\n"); + + res = SqlToYql("pragma AnsiImplicitCrossJoin; use plato; select * from Input1, any Input2"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:64: Error: ANY should not be used with Cross JOIN\n"); + } + + Y_UNIT_TEST(ErrorPlainEndAsInlineActionTerminator) { + NYql::TAstParseResult res = SqlToYql( + "do begin\n" + " select 1\n" + "; end\n"); + UNIT_ASSERT(!res.Root); +#if ANTLR_VER == 3 + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:4:0: Error: Unexpected token absence : Missing DO \n\n"); +#else + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:4:0: Error: missing DO at '<EOF>'\n"); +#endif + } + + Y_UNIT_TEST(ErrorMultiWayJoinWithUsing) { + NYql::TAstParseResult res = SqlToYql( + "USE plato;\n" + "PRAGMA DisableSimpleColumns;\n" + "SELECT *\n" + "FROM Input1 AS a\n" + "JOIN Input2 AS b USING(key)\n" + "JOIN Input3 AS c ON a.key = c.key;\n" + ); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), + "<main>:5:24: Error: Multi-way JOINs should be connected with ON clause instead of USING clause\n" + ); + } + + Y_UNIT_TEST(RequireLabelInFlattenByWithDot) { + NYql::TAstParseResult res = SqlToYql("select * from plato.Input flatten by x.y"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), + "<main>:1:40: Error: Unnamed expression after FLATTEN BY is not allowed\n" + ); + } + + Y_UNIT_TEST(WarnUnnamedColumns) { + NYql::TAstParseResult res = SqlToYql( + "PRAGMA WarnUnnamedColumns;\n" + "\n" + "SELECT key, subkey, key || subkey FROM plato.Input ORDER BY subkey;\n"); + UNIT_ASSERT(res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:28: Warning: Autogenerated column name column2 will be used for expression, code: 4516\n"); + } + + Y_UNIT_TEST(WarnSourceColumnMismatch) { + NYql::TAstParseResult res = SqlToYql( + "insert into plato.Output (key, subkey, new_value, one_more_value) select key as Key, subkey, value, \"x\" from plato.Input;"); + UNIT_ASSERT(res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:51: Warning: Column names in SELECT don't match column specification in parenthesis. \"key\" doesn't match \"Key\". \"new_value\" doesn't match \"value\", code: 4517\n"); + } + + Y_UNIT_TEST(YtCaseInsensitive) { + NYql::TAstParseResult res = SqlToYql("select * from PlatO.foo;"); + UNIT_ASSERT(res.Root); + + res = SqlToYql("use PlatO; select * from foo;"); + UNIT_ASSERT(res.Root); + } + + Y_UNIT_TEST(KikimrCaseSensitive) { + NYql::TAstParseResult res = SqlToYql("select * from PlatO.foo;", 10, "kikimr"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:15: Error: Unknown cluster: PlatO\n"); + + res = SqlToYql("use PlatO; select * from foo;", 10, "kikimr"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:5: Error: Unknown cluster: PlatO\n"); + } + + Y_UNIT_TEST(DiscoveryModeForbidden) { + NYql::TAstParseResult res = SqlToYqlWithMode("insert into plato.Output select * from plato.range(\"\", Input1, Input4)", NSQLTranslation::ESqlMode::DISCOVERY); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:40: Error: range is not allowed in Discovery mode, code: 4600\n"); + + res = SqlToYqlWithMode("insert into plato.Output select * from plato.like(\"\", \"Input%\")", NSQLTranslation::ESqlMode::DISCOVERY); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:40: Error: like is not allowed in Discovery mode, code: 4600\n"); + + res = SqlToYqlWithMode("insert into plato.Output select * from plato.regexp(\"\", \"Input.\")", NSQLTranslation::ESqlMode::DISCOVERY); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:40: Error: regexp is not allowed in Discovery mode, code: 4600\n"); + + res = SqlToYqlWithMode("insert into plato.Output select * from plato.filter(\"\", ($name) -> { return find($name, \"Input\") is not null; })", NSQLTranslation::ESqlMode::DISCOVERY); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:40: Error: filter is not allowed in Discovery mode, code: 4600\n"); + + res = SqlToYqlWithMode("select Path from plato.folder(\"\") where Type == \"table\"", NSQLTranslation::ESqlMode::DISCOVERY); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:18: Error: folder is not allowed in Discovery mode, code: 4600\n"); + } + + Y_UNIT_TEST(YsonFuncWithoutArgs) { + UNIT_ASSERT(SqlToYql("SELECT Yson::SerializeText(Yson::From());").IsOk()); + } + + Y_UNIT_TEST(CanNotUseOrderByInNonLastSelectInUnionAllChain) { + auto req = "pragma AnsiOrderByLimitInUnionAll;\n" + "use plato;\n" + "\n" + "select * from Input order by key\n" + "union all\n" + "select * from Input order by key limit 1;"; + auto res = SqlToYql(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:4:21: Error: ORDER BY within UNION ALL is only allowed after last subquery\n"); + } + + Y_UNIT_TEST(CanNotUseLimitInNonLastSelectInUnionAllChain) { + auto req = "pragma AnsiOrderByLimitInUnionAll;\n" + "use plato;\n" + "\n" + "select * from Input limit 1\n" + "union all\n" + "select * from Input order by key limit 1;"; + auto res = SqlToYql(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:4:21: Error: LIMIT within UNION ALL is only allowed after last subquery\n"); + } + + Y_UNIT_TEST(CanNotUseDiscardInNonFirstSelectInUnionAllChain) { + auto req = "pragma AnsiOrderByLimitInUnionAll;\n" + "use plato;\n" + "\n" + "select * from Input\n" + "union all\n" + "discard select * from Input;"; + auto res = SqlToYql(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:6:1: Error: DISCARD within UNION ALL is only allowed before first subquery\n"); + } + + Y_UNIT_TEST(CanNotUseIntoResultInNonLastSelectInUnionAllChain) { + auto req = "use plato;\n" + "pragma AnsiOrderByLimitInUnionAll;\n" + "\n" + "select * from Input\n" + "union all\n" + "discard select * from Input;"; + + auto res = SqlToYql(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:6:1: Error: DISCARD within UNION ALL is only allowed before first subquery\n"); + } + + Y_UNIT_TEST(YsonStrictInvalidPragma) { + auto res = SqlToYql("pragma yson.Strict = \"wrong\";"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:22: Error: Expected 'true', 'false' or no parameter for: Strict\n"); + } + + Y_UNIT_TEST(WarnTableNameInSomeContexts) { + UNIT_ASSERT(SqlToYql("use plato; select TableName() from Input;").IsOk()); + UNIT_ASSERT(SqlToYql("use plato; select TableName(\"aaaa\");").IsOk()); + UNIT_ASSERT(SqlToYql("select TableName(\"aaaa\", \"yt\");").IsOk()); + + auto res = SqlToYql("select TableName() from plato.Input;"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Error: TableName requires either service name as second argument or current cluster name\n"); + + res = SqlToYql("use plato;\n" + "select TableName() from Input1 as a join Input2 as b using(key);"); + UNIT_ASSERT(res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:2:8: Warning: TableName() may produce empty result when used in ambiguous context (with JOIN), code: 4525\n"); + + res = SqlToYql("use plato;\n" + "select SOME(TableName()), key from Input group by key;"); + UNIT_ASSERT(res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:2:13: Warning: TableName() will produce empty result when used with aggregation.\n" + "Please consult documentation for possible workaround, code: 4525\n"); + } + + Y_UNIT_TEST(WarnOnDistincWithHavingWithoutAggregations) { + auto res = SqlToYql("select distinct key from plato.Input having key != '0';"); + UNIT_ASSERT(res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:49: Warning: The usage of HAVING without aggregations with SELECT DISTINCT is non-standard and will stop working soon. Please use WHERE instead., code: 4526\n"); + } + + Y_UNIT_TEST(FlattenByExprWithNestedNull) { + auto res = SqlToYql("USE plato;\n" + "\n" + "SELECT * FROM (SELECT 1 AS region_id)\n" + "FLATTEN BY (\n" + " CAST($unknown(region_id) AS List<String>) AS region\n" + ")"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:5:10: Error: Unknown name: $unknown\n"); + } + + Y_UNIT_TEST(EmptySymbolNameIsForbidden) { + auto req = " $`` = 1; select $``;"; + auto res = SqlToYql(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:5: Error: Empty symbol name is not allowed\n"); + } + + Y_UNIT_TEST(WarnOnBinaryOpWithNullArg) { + auto req = "select * from plato.Input where cast(key as Int32) != NULL"; + auto res = SqlToYql(req); + UNIT_ASSERT(res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:52: Warning: Binary operation != will return NULL here, code: 4529\n"); + + req = "select 1 or null"; + res = SqlToYql(req); + UNIT_ASSERT(res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), ""); + } + + Y_UNIT_TEST(ErrorIfTableSampleArgUsesColumns) { + auto req = "SELECT key FROM plato.Input TABLESAMPLE BERNOULLI(MIN_OF(100.0, CAST(subkey as Int32)));"; + auto res = SqlToYql(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:70: Error: Column reference \"subkey\" is not allowed in current scope\n"); + } + + Y_UNIT_TEST(DerivedColumnListForSelectIsNotSupportedYet) { + auto req = "SELECT a,b,c FROM plato.Input as t(x,y,z);"; + auto res = SqlToYql(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:35: Error: Derived column list is only supported for VALUES\n"); + } + + Y_UNIT_TEST(ErrorIfValuesHasDifferentCountOfColumns) { + auto req = "VALUES (1,2,3), (4,5);"; + auto res = SqlToYql(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:17: Error: All VALUES items should have same size: expecting 3, got 2\n"); + } + + Y_UNIT_TEST(ErrorIfDerivedColumnSizeExceedValuesColumnCount) { + auto req = "SELECT * FROM(VALUES (1,2), (3,4)) as t(x,y,z);"; + auto res = SqlToYql(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:40: Error: Derived column list size exceeds column count in VALUES\n"); + } + + Y_UNIT_TEST(WarnoOnAutogeneratedNamesForValues) { + auto req = "PRAGMA WarnUnnamedColumns;\n" + "SELECT * FROM (VALUES (1,2,3,4), (5,6,7,8)) as t(x,y);"; + auto res = SqlToYql(req); + UNIT_ASSERT(res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:2:16: Warning: Autogenerated column names column2...column3 will be used here, code: 4516\n"); + } + + Y_UNIT_TEST(ErrUnionAllWithOrderByWithoutExplicitLegacyMode) { + auto req = "use plato;\n" + "\n" + "select * from Input order by key\n" + "union all\n" + "select * from Input order by key;"; + + auto res = SqlToYql(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:21: Error: ORDER BY within UNION ALL is only allowed after last subquery\n"); + } + + Y_UNIT_TEST(ErrUnionAllWithLimitWithoutExplicitLegacyMode) { + auto req = "use plato;\n" + "\n" + "select * from Input limit 10\n" + "union all\n" + "select * from Input limit 1;"; + + auto res = SqlToYql(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:21: Error: LIMIT within UNION ALL is only allowed after last subquery\n"); + } + + Y_UNIT_TEST(ErrUnionAllWithIntoResultWithoutExplicitLegacyMode) { + auto req = "use plato;\n" + "\n" + "select * from Input into result aaa\n" + "union all\n" + "select * from Input;"; + + auto res = SqlToYql(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:21: Error: INTO RESULT within UNION ALL is only allowed after last subquery\n"); + } + + Y_UNIT_TEST(ErrUnionAllWithDiscardWithoutExplicitLegacyMode) { + auto req = "use plato;\n" + "\n" + "select * from Input\n" + "union all\n" + "discard select * from Input;"; + + auto res = SqlToYql(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:5:1: Error: DISCARD within UNION ALL is only allowed before first subquery\n"); + } + + Y_UNIT_TEST(ErrUnionAllKeepsIgnoredOrderByWarning) { + auto req = "use plato;\n" + "\n" + "SELECT * FROM (\n" + " SELECT * FROM Input\n" + " UNION ALL\n" + " SELECT t.* FROM Input AS t ORDER BY t.key\n" + ");"; + auto res = SqlToYql(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:4:3: Warning: ORDER BY without LIMIT in subquery will be ignored, code: 4504\n" + "<main>:6:39: Error: Unknown correlation name: t\n"); + } + + Y_UNIT_TEST(ErrOrderByIgnoredButCheckedForMissingColumns) { + auto req = "$src = SELECT key FROM (SELECT 1 as key, 2 as subkey) ORDER BY x; SELECT * FROM $src;"; + ExpectFailWithError(req, "<main>:1:8: Warning: ORDER BY without LIMIT in subquery will be ignored, code: 4504\n" + "<main>:1:64: Error: Column x is not in source column set\n"); + + req = "$src = SELECT key FROM plato.Input ORDER BY x; SELECT * FROM $src;"; + auto res = SqlToYql(req); + UNIT_ASSERT(res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Warning: ORDER BY without LIMIT in subquery will be ignored, code: 4504\n"); + } + + Y_UNIT_TEST(InvalidTtlInterval) { + auto req = R"( + USE plato; + CREATE TABLE tableName (Key Uint32, CreatedAt Timestamp, PRIMARY KEY (Key)) + WITH (TTL = 1 On CreatedAt); + )"; + auto res = SqlToYql(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:4:25: Error: Literal of Interval type is expected for TTL\n" + "<main>:4:25: Error: Invalid TTL settings\n"); + } + + Y_UNIT_TEST(InvalidTtlUnit) { + auto req = R"( + USE plato; + CREATE TABLE tableName (Key Uint32, CreatedAt Uint32, PRIMARY KEY (Key)) + WITH (TTL = Interval("P1D") On CreatedAt AS PICOSECONDS); + )"; + auto res = SqlToYql(req); + UNIT_ASSERT(!res.Root); +#if ANTLR_VER == 3 + UNIT_ASSERT_STRING_CONTAINS(Err2Str(res), "<main>:4:56: Error: Unexpected token 'PICOSECONDS'"); +#else + UNIT_ASSERT_STRING_CONTAINS(Err2Str(res), "mismatched input 'PICOSECONDS' expecting {MICROSECONDS, MILLISECONDS, NANOSECONDS, SECONDS}"); +#endif + } + + Y_UNIT_TEST(InvalidChangefeedSink) { + auto req = R"( + USE plato; + CREATE TABLE tableName ( + Key Uint32, PRIMARY KEY (Key), + CHANGEFEED feedName WITH (SINK_TYPE = "S3", MODE = "KEYS_ONLY", FORMAT = "json") + ); + )"; + auto res = SqlToYql(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:5:55: Error: Unknown changefeed sink type: S3\n"); + } + + Y_UNIT_TEST(InvalidChangefeedSettings) { + auto req = R"( + USE plato; + CREATE TABLE tableName ( + Key Uint32, PRIMARY KEY (Key), + CHANGEFEED feedName WITH (SINK_TYPE = "local", FOO = "bar") + ); + )"; + auto res = SqlToYql(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:5:64: Error: Unknown changefeed setting: FOO\n"); + } + + Y_UNIT_TEST(InvalidChangefeedInitialScan) { + auto req = R"( + USE plato; + CREATE TABLE tableName ( + Key Uint32, PRIMARY KEY (Key), + CHANGEFEED feedName WITH (MODE = "KEYS_ONLY", FORMAT = "json", INITIAL_SCAN = "foo") + ); + )"; + auto res = SqlToYql(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:5:95: Error: Literal of Bool type is expected for INITIAL_SCAN\n"); + } + + Y_UNIT_TEST(InvalidChangefeedVirtualTimestamps) { + auto req = R"( + USE plato; + CREATE TABLE tableName ( + Key Uint32, PRIMARY KEY (Key), + CHANGEFEED feedName WITH (MODE = "KEYS_ONLY", FORMAT = "json", VIRTUAL_TIMESTAMPS = "foo") + ); + )"; + auto res = SqlToYql(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:5:101: Error: Literal of Bool type is expected for VIRTUAL_TIMESTAMPS\n"); + } + + Y_UNIT_TEST(InvalidChangefeedResolvedTimestamps) { + auto req = R"( + USE plato; + CREATE TABLE tableName ( + Key Uint32, PRIMARY KEY (Key), + CHANGEFEED feedName WITH (MODE = "KEYS_ONLY", FORMAT = "json", BARRIERS_INTERVAL = "foo") + ); + )"; + auto res = SqlToYql(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:5:100: Error: Literal of Interval type is expected for BARRIERS_INTERVAL\n"); + } + + Y_UNIT_TEST(InvalidChangefeedRetentionPeriod) { + auto req = R"( + USE plato; + CREATE TABLE tableName ( + Key Uint32, PRIMARY KEY (Key), + CHANGEFEED feedName WITH (MODE = "KEYS_ONLY", FORMAT = "json", RETENTION_PERIOD = "foo") + ); + )"; + auto res = SqlToYql(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:5:99: Error: Literal of Interval type is expected for RETENTION_PERIOD\n"); + } + + Y_UNIT_TEST(InvalidChangefeedTopicPartitions) { + auto req = R"( + USE plato; + CREATE TABLE tableName ( + Key Uint32, PRIMARY KEY (Key), + CHANGEFEED feedName WITH (MODE = "KEYS_ONLY", FORMAT = "json", TOPIC_MIN_ACTIVE_PARTITIONS = "foo") + ); + )"; + auto res = SqlToYql(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:5:110: Error: Literal of integer type is expected for TOPIC_MIN_ACTIVE_PARTITIONS\n"); + } + + Y_UNIT_TEST(InvalidChangefeedAwsRegion) { + auto req = R"( + USE plato; + CREATE TABLE tableName ( + Key Uint32, PRIMARY KEY (Key), + CHANGEFEED feedName WITH (MODE = "KEYS_ONLY", FORMAT = "json", AWS_REGION = true) + ); + )"; + auto res = SqlToYql(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:5:93: Error: Literal of String type is expected for AWS_REGION\n"); + } + + Y_UNIT_TEST(ErrJoinWithGroupingSetsWithoutCorrelationName) { + auto req = "USE plato;\n" + "\n" + "SELECT k1, k2, subkey\n" + "FROM T1 AS a JOIN T2 AS b USING (key)\n" + "GROUP BY GROUPING SETS(\n" + " (a.key as k1, b.subkey as k2),\n" + " (k1),\n" + " (subkey)\n" + ");"; + auto res = SqlToYql(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:8:4: Error: Columns in grouping sets should have correlation name, error in key: subkey\n"); + } + + Y_UNIT_TEST(ErrJoinWithGroupByWithoutCorrelationName) { + auto req = "USE plato;\n" + "\n" + "SELECT k1, k2,\n" + " value\n" + "FROM T1 AS a JOIN T2 AS b USING (key)\n" + "GROUP BY a.key as k1, b.subkey as k2,\n" + " value;"; + ExpectFailWithError(req, + "<main>:7:5: Error: Columns in GROUP BY should have correlation name, error in key: value\n"); + } + + Y_UNIT_TEST(ErrWithMissingFrom) { + auto req = "select 1 as key where 1 > 1;"; + auto res = SqlToYql(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:25: Error: Filtering is not allowed without FROM\n"); + + req = "select 1 + count(*);"; + res = SqlToYql(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:12: Error: Aggregation is not allowed without FROM\n"); + + req = "select 1 as key, subkey + value;"; + res = SqlToYql(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:1: Error: Column references are not allowed without FROM\n" + "<main>:1:18: Error: Column reference 'subkey'\n" + "<main>:1:1: Error: Column references are not allowed without FROM\n" + "<main>:1:27: Error: Column reference 'value'\n"); + + req = "select count(1) group by key;"; + res = SqlToYql(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:1: Error: Column references are not allowed without FROM\n" + "<main>:1:26: Error: Column reference 'key'\n"); + } + + Y_UNIT_TEST(ErrWithMissingFromForWindow) { + auto req = "$c = () -> (1 + count(1) over w);\n" + "select $c();"; + ExpectFailWithError(req, + "<main>:1:9: Error: Window and aggregation functions are not allowed in this context\n" + "<main>:1:17: Error: Failed to use aggregation function Count without window specification or in wrong place\n"); + + req = "$c = () -> (1 + lead(1) over w);\n" + "select $c();"; + ExpectFailWithError(req, + "<main>:1:17: Error: Window functions are not allowed in this context\n" + "<main>:1:17: Error: Failed to use window function Lead without window specification or in wrong place\n"); + + req = "select 1 + count(1) over w window w as ();"; + ExpectFailWithError(req, + "<main>:1:1: Error: Window and aggregation functions are not allowed without FROM\n" + "<main>:1:12: Error: Failed to use aggregation function Count without window specification or in wrong place\n"); + + req = "select 1 + lead(1) over w window w as ();"; + ExpectFailWithError(req, + "<main>:1:12: Error: Window functions are not allowed without FROM\n" + "<main>:1:12: Error: Failed to use window function Lead without window specification or in wrong place\n"); + } + + Y_UNIT_TEST(ErrWithMissingFromForInplaceWindow) { + auto req = "$c = () -> (1 + count(1) over ());\n" + "select $c();"; + ExpectFailWithError(req, + "<main>:1:26: Error: Window and aggregation functions are not allowed in this context\n"); + + req = "$c = () -> (1 + lead(1) over (rows between unbounded preceding and current row));\n" + "select $c();"; + ExpectFailWithError(req, + "<main>:1:25: Error: Window and aggregation functions are not allowed in this context\n"); + + req = "select 1 + count(1) over ();"; + ExpectFailWithError(req, + "<main>:1:1: Error: Window and aggregation functions are not allowed without FROM\n" + "<main>:1:12: Error: Failed to use aggregation function Count without window specification or in wrong place\n"); + + req = "select 1 + lead(1) over (rows between current row and unbounded following);"; + ExpectFailWithError(req, + "<main>:1:12: Error: Window functions are not allowed without FROM\n" + "<main>:1:12: Error: Failed to use window function Lead without window specification or in wrong place\n"); + } + + Y_UNIT_TEST(ErrDistinctInWrongPlace) { + auto req = "select Some::Udf(distinct key) from plato.Input;"; + ExpectFailWithError(req, + "<main>:1:18: Error: DISTINCT can only be used in aggregation functions\n"); + req = "select sum(key)(distinct foo) from plato.Input;"; + ExpectFailWithError(req, + "<main>:1:17: Error: DISTINCT can only be used in aggregation functions\n"); + + req = "select len(distinct foo) from plato.Input;"; + ExpectFailWithError(req, + "<main>:1:8: Error: DISTINCT can only be used in aggregation functions\n"); + + req = "$foo = ($x) -> ($x); select $foo(distinct key) from plato.Input;"; + ExpectFailWithError(req, + "<main>:1:34: Error: DISTINCT can only be used in aggregation functions\n"); + } + + Y_UNIT_TEST(ErrForNotSingleChildInInlineAST) { + ExpectFailWithError("select YQL::\"\"", + "<main>:1:8: Error: Failed to parse YQL: expecting AST root node with single child, but got 0\n"); + ExpectFailWithError("select YQL::@@ \t@@", + "<main>:1:8: Error: Failed to parse YQL: expecting AST root node with single child, but got 0\n"); + auto req = "$lambda = YQL::@@(lambda '(x)(+ x x)) (lambda '(y)(+ y y))@@;\n" + "select ListMap([1, 2, 3], $lambda);"; + ExpectFailWithError(req, + "<main>:1:11: Error: Failed to parse YQL: expecting AST root node with single child, but got 2\n"); + } + + Y_UNIT_TEST(ErrEmptyColumnName) { + ExpectFailWithError("select * without \"\" from plato.Input", + "<main>:1:18: Error: String literal can not be used here\n"); + + ExpectFailWithError("select * without `` from plato.Input;", + "<main>:1:18: Error: Empty column name is not allowed\n"); + + ExpectFailWithErrorForAnsiLexer("select * without \"\" from plato.Input", + "<main>:1:18: Error: Empty column name is not allowed\n"); + + ExpectFailWithErrorForAnsiLexer("select * without `` from plato.Input;", + "<main>:1:18: Error: Empty column name is not allowed\n"); + } + + Y_UNIT_TEST(ErrOnNonZeroArgumentsForTableRows) { + ExpectFailWithError("$udf=\"\";process plato.Input using $udf(TableRows(k))", + "<main>:1:40: Error: TableRows requires exactly 0 arguments\n"); + } + + Y_UNIT_TEST(ErrGroupByWithAggregationFunctionAndDistinctExpr) { + ExpectFailWithError("select * from plato.Input group by count(distinct key|key)", + "<main>:1:36: Error: Unable to GROUP BY aggregated values\n"); + } + + // FIXME: check if we can get old behaviour +#if 0 + Y_UNIT_TEST(ErrWithSchemaWithColumnsWithoutType) { + ExpectFailWithError("select * from plato.Input with COLUMNs", + "<main>:1:32: Error: Expected type after COLUMNS\n" + "<main>:1:32: Error: Failed to parse table hints\n"); + + ExpectFailWithError("select * from plato.Input with scheMa", + "<main>:1:32: Error: Expected type after SCHEMA\n" + "<main>:1:32: Error: Failed to parse table hints\n"); + } +#endif + + Y_UNIT_TEST(ErrCollectPreaggregatedInListLiteralWithoutFrom) { + ExpectFailWithError("SELECT([VARIANCE(DISTINCT[])])", + "<main>:1:1: Error: Column references are not allowed without FROM\n" + "<main>:1:9: Error: Column reference '_yql_preagg_Variance0'\n"); + } + + Y_UNIT_TEST(ErrGroupBySmartParenAsTuple) { + ExpectFailWithError("SELECT * FROM plato.Input GROUP BY (k, v,)", + "<main>:1:41: Error: Unexpected trailing comma in grouping elements list\n"); + } + + Y_UNIT_TEST(HandleNestedSmartParensInGroupBy) { + ExpectFailWithError("SELECT * FROM plato.Input GROUP BY (+() as k)", + "<main>:1:37: Error: Unable to GROUP BY constant expression\n"); + } + + Y_UNIT_TEST(ErrRenameWithAddColumn) { + ExpectFailWithError("USE plato; ALTER TABLE table RENAME TO moved, ADD COLUMN addc uint64", + "<main>:1:40: Error: RENAME TO can not be used together with another table action\n"); + } + + Y_UNIT_TEST(ErrAddColumnAndRename) { + // FIXME: fix positions in ALTER TABLE + ExpectFailWithError("USE plato; ALTER TABLE table ADD COLUMN addc uint64, RENAME TO moved", + "<main>:1:46: Error: RENAME TO can not be used together with another table action\n"); + } + + Y_UNIT_TEST(InvalidUuidValue) { + ExpectFailWithError("SELECT Uuid('123e4567ae89ba12d3aa456a426614174ab0')", + "<main>:1:8: Error: Invalid value \"123e4567ae89ba12d3aa456a426614174ab0\" for type Uuid\n"); + ExpectFailWithError("SELECT Uuid('123e4567ae89b-12d3-a456-426614174000')", + "<main>:1:8: Error: Invalid value \"123e4567ae89b-12d3-a456-426614174000\" for type Uuid\n"); + } + + Y_UNIT_TEST(WindowFunctionWithoutOver) { + ExpectFailWithError("SELECT LAST_VALUE(foo) FROM plato.Input", + "<main>:1:8: Error: Can't use window function LastValue without window specification (OVER keyword is missing)\n"); + ExpectFailWithError("SELECT LAST_VALUE(foo) FROM plato.Input GROUP BY key", + "<main>:1:8: Error: Can't use window function LastValue without window specification (OVER keyword is missing)\n"); + } + + Y_UNIT_TEST(CreateAlterUserWithoutCluster) { + ExpectFailWithError("\n CREATE USER user ENCRYPTED PASSWORD 'foobar';", "<main>:2:2: Error: USE statement is missing - no default cluster is selected\n"); + ExpectFailWithError("ALTER USER CURRENT_USER RENAME TO $foo;", "<main>:1:1: Error: USE statement is missing - no default cluster is selected\n"); + } + + Y_UNIT_TEST(ModifyPermissionsWithoutCluster) { + ExpectFailWithError("\n GRANT CONNECT ON `/Root` TO user;", "<main>:2:2: Error: USE statement is missing - no default cluster is selected\n"); + ExpectFailWithError("\n REVOKE MANAGE ON `/Root` FROM user;", "<main>:2:2: Error: USE statement is missing - no default cluster is selected\n"); + } + + Y_UNIT_TEST(ReservedRoleNames) { + ExpectFailWithError("USE plato; CREATE USER current_User;", "<main>:1:24: Error: System role CURRENT_USER can not be used here\n"); + ExpectFailWithError("USE plato; ALTER USER current_User RENAME TO Current_role", "<main>:1:46: Error: System role CURRENT_ROLE can not be used here\n"); + UNIT_ASSERT(SqlToYql("USE plato; DROP GROUP IF EXISTS a, b, c, current_User;").IsOk()); + } + + Y_UNIT_TEST(DisableClassicDivisionWithError) { + ExpectFailWithError("pragma ClassicDivision = 'false'; select $foo / 30;", "<main>:1:42: Error: Unknown name: $foo\n"); + } + + Y_UNIT_TEST(AggregationOfAgrregatedDistinctExpr) { + ExpectFailWithError("select sum(sum(distinct x + 1)) from plato.Input", "<main>:1:12: Error: Aggregation of aggregated values is forbidden\n"); + } + + Y_UNIT_TEST(WarnForUnusedSqlHint) { + NYql::TAstParseResult res = SqlToYql("select * from plato.Input1 as a join /*+ merge() */ plato.Input2 as b using(key);\n" + "select --+ foo(bar)\n" + " 1;"); + UNIT_ASSERT(res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:2:23: Warning: Hint foo will not be used, code: 4534\n"); + } + + Y_UNIT_TEST(WarnForDeprecatedSchema) { + NSQLTranslation::TTranslationSettings settings; + settings.ClusterMapping["s3bucket"] = NYql::S3ProviderName; + NYql::TAstParseResult res = SqlToYqlWithSettings("select * from s3bucket.`foo` with schema (col1 Int32, String as col2, Int64 as col3);", settings); + UNIT_ASSERT(res.Root); + UNIT_ASSERT_STRING_CONTAINS(res.Issues.ToString(), "Warning: Deprecated syntax for positional schema: please use 'column type' instead of 'type AS column', code: 4535\n"); + } + + Y_UNIT_TEST(ErrorOnColumnNameInMaxByLimit) { + ExpectFailWithError( + "SELECT AGGREGATE_BY(AsTuple(value, key), AggregationFactory(\"MAX_BY\", subkey)) FROM plato.Input;", + "<main>:1:42: Error: Source does not allow column references\n" + "<main>:1:71: Error: Column reference 'subkey'\n"); + } + + Y_UNIT_TEST(ErrorInLibraryWithTopLevelNamedSubquery) { + TString withUnusedSubq = "$unused = select max(key) from plato.Input;\n" + "\n" + "define subquery $foo() as\n" + " $count = select count(*) from plato.Input;\n" + " select * from plato.Input limit $count / 2;\n" + "end define;\n" + "export $foo;\n"; + UNIT_ASSERT(SqlToYqlWithMode(withUnusedSubq, NSQLTranslation::ESqlMode::LIBRARY).IsOk()); + + TString withTopLevelSubq = "$count = select count(*) from plato.Input;\n" + "\n" + "define subquery $foo() as\n" + " select * from plato.Input limit $count / 2;\n" + "end define;\n" + "export $foo;\n"; + auto res = SqlToYqlWithMode(withTopLevelSubq, NSQLTranslation::ESqlMode::LIBRARY); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:17: Error: Named subquery can not be used as a top level statement in libraries\n"); + } + + Y_UNIT_TEST(SessionStartAndSessionStateShouldSurviveSessionWindowArgsError){ + TString query = R"( + $init = ($_row) -> (min(1, 2)); -- error: aggregation func min() can not be used here + $calculate = ($_row, $_state) -> (1); + $update = ($_row, $_state) -> (2); + SELECT + SessionStart() over w as session_start, + SessionState() over w as session_state, + FROM plato.Input as t + WINDOW w AS ( + PARTITION BY user, SessionWindow(ts + 1, $init, $update, $calculate) + ) + )"; + ExpectFailWithError(query, "<main>:2:33: Error: Aggregation function Min requires exactly 1 argument(s), given: 2\n"); + } + + Y_UNIT_TEST(ScalarContextUsage1) { + TString query = R"( + $a = (select 1 as x, 2 as y); + select 1 + $a; + )"; + ExpectFailWithError(query, "<main>:2:39: Error: Source used in expression should contain one concrete column\n" + "<main>:3:24: Error: Source is used here\n"); + } + + Y_UNIT_TEST(ScalarContextUsage2) { + TString query = R"( + use plato; + $a = (select 1 as x, 2 as y); + select * from concat($a); + )"; + ExpectFailWithError(query, "<main>:3:39: Error: Source used in expression should contain one concrete column\n" + "<main>:4:34: Error: Source is used here\n"); + } + + Y_UNIT_TEST(ScalarContextUsage3) { + TString query = R"( + use plato; + $a = (select 1 as x, 2 as y); + select * from range($a); + )"; + ExpectFailWithError(query, "<main>:3:39: Error: Source used in expression should contain one concrete column\n" + "<main>:4:33: Error: Source is used here\n"); + } + + Y_UNIT_TEST(ScalarContextUsage4) { + TString query = R"( + use plato; + $a = (select 1 as x, 2 as y); + insert into $a select 1; + )"; + ExpectFailWithError(query, "<main>:3:39: Error: Source used in expression should contain one concrete column\n" + "<main>:4:25: Error: Source is used here\n"); + } +} + +void CheckUnused(const TString& req, const TString& symbol, unsigned row, unsigned col) { + auto res = SqlToYql(req); + + UNIT_ASSERT(res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), TStringBuilder() << "<main>:" << row << ":" << col << ": Warning: Symbol " << symbol << " is not used, code: 4527\n"); +} + +Y_UNIT_TEST_SUITE(WarnUnused) { + Y_UNIT_TEST(ActionOrSubquery) { + TString req = " $a()\n" + "as select 1;\n" + "end define;\n" + "\n" + "select 1;"; + CheckUnused("define action\n" + req, "$a", 2, 3); + CheckUnused("define subquery\n" + req, "$a", 2, 3); + } + + Y_UNIT_TEST(Import) { + TString req = "import lib1 symbols\n" + " $sqr;\n" + "select 1;"; + CheckUnused(req, "$sqr", 2, 3); + + req = "import lib1 symbols\n" + " $sqr as\n" + " $sq;\n" + "select 1;"; + CheckUnused(req, "$sq", 3, 5); + } + + Y_UNIT_TEST(NamedNodeStatement) { + TString req = " $a, $a = AsTuple(1, 2);\n" + "select $a;"; + CheckUnused(req, "$a", 1, 2); + req = "$a, $b = AsTuple(1, 2);\n" + "select $a;"; + CheckUnused(req, "$b", 1, 6); + CheckUnused(" $a = 1; $a = 2; select $a;", "$a", 1, 2); + } + + Y_UNIT_TEST(Declare) { + CheckUnused("declare $a as String;select 1;", "$a", 1, 9); + } + + Y_UNIT_TEST(ActionParams) { + TString req = "define action $a($x, $y) as\n" + " select $x;\n" + "end define;\n" + "\n" + "do $a(1,2);"; + CheckUnused(req, "$y", 1, 22); + } + + Y_UNIT_TEST(SubqueryParams) { + TString req = "use plato;\n" + "define subquery $q($name, $x) as\n" + " select * from $name;\n" + "end define;\n" + "\n" + "select * from $q(\"Input\", 1);"; + CheckUnused(req, "$x", 2, 27); + } + + Y_UNIT_TEST(For) { + TString req = "define action $a() as\n" + " select 1;\n" + "end define;\n" + "\n" + "for $i in ListFromRange(1, 10)\n" + "do $a();"; + CheckUnused(req, "$i", 5, 5); + } + + Y_UNIT_TEST(LambdaParams) { + TString req = "$lambda = ($x, $y) -> ($x);\n" + "select $lambda(1, 2);"; + CheckUnused(req, "$y", 1, 16); + } + + Y_UNIT_TEST(InsideLambdaBody) { + TString req = "$lambda = () -> {\n" + " $x = 1; return 1;\n" + "};\n" + "select $lambda();"; + CheckUnused(req, "$x", 2, 3); + req = "$lambda = () -> {\n" + " $x = 1; $x = 2; return $x;\n" + "};\n" + "select $lambda();"; + CheckUnused(req, "$x", 2, 3); + } + + Y_UNIT_TEST(InsideAction) { + TString req = "define action $a() as\n" + " $x = 1; select 1;\n" + "end define;\n" + "\n" + "do $a();"; + CheckUnused(req, "$x", 2, 3); + req = "define action $a() as\n" + " $x = 1; $x = 2; select $x;\n" + "end define;\n" + "\n" + "do $a();"; + CheckUnused(req, "$x", 2, 3); + } + + Y_UNIT_TEST(NoWarnOnNestedActions) { + auto req = "pragma warning(\"error\", \"4527\");\n" + "define action $action($b) as\n" + " define action $aaa() as\n" + " select $b;\n" + " end define;\n" + " do $aaa();\n" + "end define;\n" + "\n" + "do $action(1);"; + UNIT_ASSERT(SqlToYql(req).IsOk()); + } + + Y_UNIT_TEST(NoWarnForUsageAfterSubquery) { + auto req = "use plato;\n" + "pragma warning(\"error\", \"4527\");\n" + "\n" + "$a = 1;\n" + "\n" + "define subquery $q($table) as\n" + " select * from $table;\n" + "end define;\n" + "\n" + "select * from $q(\"Input\");\n" + "select $a;"; + UNIT_ASSERT(SqlToYql(req).IsOk()); + } +} + +Y_UNIT_TEST_SUITE(AnonymousNames) { + Y_UNIT_TEST(ReferenceAnonymousVariableIsForbidden) { + auto req = "$_ = 1; select $_;"; + + auto res = SqlToYql(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:16: Error: Unable to reference anonymous name $_\n"); + + req = "$`_` = 1; select $`_`;"; + res = SqlToYql(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:18: Error: Unable to reference anonymous name $_\n"); + } + + Y_UNIT_TEST(Declare) { + auto req = "declare $_ as String;"; + auto res = SqlToYql(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:9: Error: Can not use anonymous name '$_' in DECLARE statement\n"); + } + + Y_UNIT_TEST(ActionSubquery) { + auto req = "define action $_() as select 1; end define;"; + auto res = SqlToYql(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:15: Error: Can not use anonymous name '$_' as ACTION name\n"); + + req = "define subquery $_() as select 1; end define;"; + res = SqlToYql(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:17: Error: Can not use anonymous name '$_' as SUBQUERY name\n"); + } + + Y_UNIT_TEST(Import) { + auto req = "import lib symbols $sqr as $_;"; + auto res = SqlToYql(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:28: Error: Can not import anonymous name $_\n"); + } + + Y_UNIT_TEST(Export) { + auto req = "export $_;"; + auto res = SqlToYqlWithMode(req, NSQLTranslation::ESqlMode::LIBRARY); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Error: Can not export anonymous name $_\n"); + } + + Y_UNIT_TEST(AnonymousInActionArgs) { + auto req = "pragma warning(\"error\", \"4527\");\n" + "define action $a($_, $y, $_) as\n" + " select $y;\n" + "end define;\n" + "\n" + "do $a(1,2,3);"; + UNIT_ASSERT(SqlToYql(req).IsOk()); + } + + Y_UNIT_TEST(AnonymousInSubqueryArgs) { + auto req = "use plato;\n" + "pragma warning(\"error\", \"4527\");\n" + "define subquery $q($_, $y, $_) as\n" + " select * from $y;\n" + "end define;\n" + "\n" + "select * from $q(1,\"Input\",3);"; + UNIT_ASSERT(SqlToYql(req).IsOk()); + } + + Y_UNIT_TEST(AnonymousInLambdaArgs) { + auto req = "pragma warning(\"error\", \"4527\");\n" + "$lambda = ($_, $x, $_) -> ($x);\n" + "select $lambda(1,2,3);"; + UNIT_ASSERT(SqlToYql(req).IsOk()); + } + + Y_UNIT_TEST(AnonymousInFor) { + auto req = "pragma warning(\"error\", \"4527\");\n" + "evaluate for $_ in ListFromRange(1, 10) do begin select 1; end do;"; + UNIT_ASSERT(SqlToYql(req).IsOk()); + } + + Y_UNIT_TEST(Assignment) { + auto req = "pragma warning(\"error\", \"4527\");\n" + "$_ = 1;\n" + "$_, $x, $_ = AsTuple(1,2,3);\n" + "select $x;"; + UNIT_ASSERT(SqlToYql(req).IsOk()); + } +} + +Y_UNIT_TEST_SUITE(JsonValue) { + Y_UNIT_TEST(JsonValueArgumentCount) { + NYql::TAstParseResult res = SqlToYql("select JSON_VALUE(CAST(@@{\"key\": 1238}@@ as Json));"); + + UNIT_ASSERT(!res.Root); +#if ANTLR_VER == 3 + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:49: Error: Unexpected token ')' : syntax error...\n\n"); +#else + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:49: Error: mismatched input ')' expecting ','\n"); +#endif + } + + Y_UNIT_TEST(JsonValueJsonPathMustBeLiteralString) { + NYql::TAstParseResult res = SqlToYql("$jsonPath = \"strict $.key\"; select JSON_VALUE(CAST(@@{\"key\": 1238}@@ as Json), $jsonPath);"); + + UNIT_ASSERT(!res.Root); +#if ANTLR_VER == 3 + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:79: Error: Unexpected token absence : Missing STRING_VALUE \n\n"); +#else + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:79: Error: mismatched input '$' expecting STRING_VALUE\n"); +#endif + } + + Y_UNIT_TEST(JsonValueTranslation) { + NYql::TAstParseResult res = SqlToYql("select JSON_VALUE(CAST(@@{\"key\": 1238}@@ as Json), \"strict $.key\");"); + + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + Y_UNUSED(word); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'\"strict $.key\"")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("SafeCast")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("DataType 'Json")); + }; + + TWordCountHive elementStat({"JsonValue"}); + VerifyProgram(res, elementStat, verifyLine); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["JsonValue"]); + } + + Y_UNIT_TEST(JsonValueReturningSection) { + for (const auto& typeName : {"Bool", "Int64", "Double", "String"}) { + NYql::TAstParseResult res = SqlToYql( + TStringBuilder() << "select JSON_VALUE(CAST(@@{\"key\": 1238}@@ as Json), \"strict $.key\" RETURNING " << typeName << ");" + ); + + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [&](const TString& word, const TString& line) { + Y_UNUSED(word); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'\"strict $.key\"")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("SafeCast")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("DataType 'Json")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(TStringBuilder() << "DataType '" << typeName)); + }; + + TWordCountHive elementStat({typeName}); + VerifyProgram(res, elementStat, verifyLine); + UNIT_ASSERT(elementStat[typeName] > 0); + } + } + + Y_UNIT_TEST(JsonValueInvalidReturningType) { + NYql::TAstParseResult res = SqlToYql("select JSON_VALUE(CAST(@@{'key': 1238}@@ as Json), 'strict $.key' RETURNING invalid);"); + + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:77: Error: Unknown simple type 'invalid'\n"); + } + + Y_UNIT_TEST(JsonValueAndReturningInExpressions) { + NYql::TAstParseResult res = SqlToYql( + "USE plato\n;" + "$json_value = \"some string\";\n" + "SELECT $json_value;\n" + "SELECT 1 as json_value;\n" + "SELECT $json_value as json_value;\n" + "$returning = \"another string\";\n" + "SELECT $returning;\n" + "SELECT 1 as returning;\n" + "SELECT $returning as returning;\n" + ); + + UNIT_ASSERT(res.Root); + } + + Y_UNIT_TEST(JsonValueValidCaseHandlers) { + const TVector<std::pair<TString, TString>> testCases = { + {"", "'DefaultValue (Null)"}, + {"NULL", "'DefaultValue (Null)"}, + {"ERROR", "'Error (Null)"}, + {"DEFAULT 123", "'DefaultValue (Int32 '\"123\")"}, + }; + + for (const auto& onEmpty : testCases) { + for (const auto& onError : testCases) { + TStringBuilder query; + query << "$json = CAST(@@{\"key\": 1238}@@ as Json);\n" + << "SELECT JSON_VALUE($json, \"strict $.key\""; + if (!onEmpty.first.empty()) { + query << " " << onEmpty.first << " ON EMPTY"; + } + if (!onError.first.empty()) { + query << " " << onError.first << " ON ERROR"; + } + query << ");\n"; + + NYql::TAstParseResult res = SqlToYql(query); + + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [&](const TString& word, const TString& line) { + Y_UNUSED(word); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(onEmpty.second + " " + onError.second)); + }; + + TWordCountHive elementStat({"JsonValue"}); + VerifyProgram(res, elementStat, verifyLine); + UNIT_ASSERT(elementStat["JsonValue"] > 0); + } + } + } + + Y_UNIT_TEST(JsonValueTooManyCaseHandlers) { + NYql::TAstParseResult res = SqlToYql( + "select JSON_VALUE(CAST(@@{\"key\": 1238}@@ as Json), \"strict $.key\" NULL ON EMPTY NULL ON ERROR NULL ON EMPTY);\n" + ); + + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF( + Err2Str(res), + "<main>:1:52: Error: Only 1 ON EMPTY and/or 1 ON ERROR clause is expected\n" + ); + } + + Y_UNIT_TEST(JsonValueTooManyOnEmpty) { + NYql::TAstParseResult res = SqlToYql( + "select JSON_VALUE(CAST(@@{\"key\": 1238}@@ as Json), \"strict $.key\" NULL ON EMPTY NULL ON EMPTY);\n" + ); + + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF( + Err2Str(res), + "<main>:1:52: Error: Only 1 ON EMPTY clause is expected\n" + ); + } + + Y_UNIT_TEST(JsonValueTooManyOnError) { + NYql::TAstParseResult res = SqlToYql( + "select JSON_VALUE(CAST(@@{\"key\": 1238}@@ as Json), \"strict $.key\" NULL ON ERROR NULL ON ERROR);\n" + ); + + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF( + Err2Str(res), + "<main>:1:52: Error: Only 1 ON ERROR clause is expected\n" + ); + } + + Y_UNIT_TEST(JsonValueOnEmptyAfterOnError) { + NYql::TAstParseResult res = SqlToYql( + "select JSON_VALUE(CAST(@@{\"key\": 1238}@@ as Json), \"strict $.key\" NULL ON ERROR NULL ON EMPTY);\n" + ); + + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF( + Err2Str(res), + "<main>:1:52: Error: ON EMPTY clause must be before ON ERROR clause\n" + ); + } + + Y_UNIT_TEST(JsonValueNullInput) { + NYql::TAstParseResult res = SqlToYql(R"(SELECT JSON_VALUE(NULL, "strict $.key");)"); + + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [&](const TString& word, const TString& line) { + Y_UNUSED(word); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("(Nothing (OptionalType (DataType 'Json)))")); + }; + + TWordCountHive elementStat({"JsonValue"}); + VerifyProgram(res, elementStat, verifyLine); + UNIT_ASSERT(elementStat["JsonValue"] > 0); + } +} + +Y_UNIT_TEST_SUITE(JsonExists) { + Y_UNIT_TEST(JsonExistsValidHandlers) { + const TVector<std::pair<TString, TString>> testCases = { + {"", "(Just (Bool '\"false\"))"}, + {"TRUE ON ERROR", "(Just (Bool '\"true\"))"}, + {"FALSE ON ERROR", "(Just (Bool '\"false\"))"}, + {"UNKNOWN ON ERROR", "(Nothing (OptionalType (DataType 'Bool)))"}, + // NOTE: in this case we expect arguments of JsonExists callable to end immediately + // after variables. This parenthesis at the end of the expression is left on purpose + {"ERROR ON ERROR", "(Utf8 '\"strict $.key\") (JsonVariables))"}, + }; + + for (const auto& item : testCases) { + NYql::TAstParseResult res = SqlToYql( + TStringBuilder() << R"( + $json = CAST(@@{"key": 1238}@@ as Json); + SELECT JSON_EXISTS($json, "strict $.key" )" << item.first << ");\n" + ); + + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [&](const TString& word, const TString& line) { + Y_UNUSED(word); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(item.second)); + }; + + TWordCountHive elementStat({"JsonExists"}); + VerifyProgram(res, elementStat, verifyLine); + UNIT_ASSERT(elementStat["JsonExists"] > 0); + } + } + + Y_UNIT_TEST(JsonExistsInvalidHandler) { + NYql::TAstParseResult res = SqlToYql(R"( + $json = CAST(@@{"key": 1238}@@ as Json); + $default = false; + SELECT JSON_EXISTS($json, "strict $.key" $default ON ERROR); + )"); + + UNIT_ASSERT(!res.Root); +#if ANTLR_VER == 3 + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:4:53: Error: Unexpected token absence : Missing RPAREN \n\n"); +#else + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:4:53: Error: mismatched input '$' expecting {')', ERROR, FALSE, TRUE, UNKNOWN}\n"); +#endif + } + + Y_UNIT_TEST(JsonExistsNullInput) { + NYql::TAstParseResult res = SqlToYql(R"(SELECT JSON_EXISTS(NULL, "strict $.key");)"); + + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [&](const TString& word, const TString& line) { + Y_UNUSED(word); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("(Nothing (OptionalType (DataType 'Json)))")); + }; + + TWordCountHive elementStat({"JsonExists"}); + VerifyProgram(res, elementStat, verifyLine); + UNIT_ASSERT(elementStat["JsonExists"] > 0); + } +} + +Y_UNIT_TEST_SUITE(JsonQuery) { + Y_UNIT_TEST(JsonQueryValidHandlers) { + using TTestSuite = const TVector<std::pair<TString, TString>>; + TTestSuite wrapCases = { + {"", "'NoWrap"}, + {"WITHOUT WRAPPER", "'NoWrap"}, + {"WITHOUT ARRAY WRAPPER", "'NoWrap"}, + {"WITH WRAPPER", "'Wrap"}, + {"WITH ARRAY WRAPPER", "'Wrap"}, + {"WITH UNCONDITIONAL WRAPPER", "'Wrap"}, + {"WITH UNCONDITIONAL ARRAY WRAPPER", "'Wrap"}, + {"WITH CONDITIONAL WRAPPER", "'ConditionalWrap"}, + {"WITH CONDITIONAL ARRAY WRAPPER", "'ConditionalWrap"}, + }; + TTestSuite handlerCases = { + {"", "'Null"}, + {"ERROR", "'Error"}, + {"NULL", "'Null"}, + {"EMPTY ARRAY", "'EmptyArray"}, + {"EMPTY OBJECT", "'EmptyObject"}, + }; + + for (const auto& wrap : wrapCases) { + for (const auto& onError : handlerCases) { + for (const auto& onEmpty : handlerCases) { + TStringBuilder query; + query << R"($json = CAST(@@{"key": [123]}@@ as Json); + SELECT JSON_QUERY($json, "strict $.key" )" << wrap.first; + if (!onEmpty.first.empty()) { + if (wrap.first.StartsWith("WITH ")) { + continue; + } + query << " " << onEmpty.first << " ON EMPTY"; + } + if (!onError.first.empty()) { + query << " " << onError.first << " ON ERROR"; + } + query << ");\n"; + + NYql::TAstParseResult res = SqlToYql(query); + + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [&](const TString& word, const TString& line) { + Y_UNUSED(word); + const TString args = TStringBuilder() << wrap.second << " " << onEmpty.second << " " << onError.second; + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(args)); + }; + + Cout << wrap.first << " " << onEmpty.first << " " << onError.first << Endl; + + TWordCountHive elementStat({"JsonQuery"}); + VerifyProgram(res, elementStat, verifyLine); + UNIT_ASSERT(elementStat["JsonQuery"] > 0); + } + } + } + } + + Y_UNIT_TEST(JsonQueryOnEmptyWithWrapper) { + NYql::TAstParseResult res = SqlToYql(R"( + $json = CAST(@@{"key": 1238}@@ as Json); + SELECT JSON_QUERY($json, "strict $" WITH ARRAY WRAPPER EMPTY ARRAY ON EMPTY); + )"); + + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:38: Error: ON EMPTY is prohibited because WRAPPER clause is specified\n"); + } + + Y_UNIT_TEST(JsonQueryNullInput) { + NYql::TAstParseResult res = SqlToYql(R"(SELECT JSON_QUERY(NULL, "strict $.key");)"); + + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [&](const TString& word, const TString& line) { + Y_UNUSED(word); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("(Nothing (OptionalType (DataType 'Json)))")); + }; + + TWordCountHive elementStat({"JsonQuery"}); + VerifyProgram(res, elementStat, verifyLine); + UNIT_ASSERT(elementStat["JsonQuery"] > 0); + } +} + +Y_UNIT_TEST_SUITE(JsonPassing) { + Y_UNIT_TEST(SupportedVariableTypes) { + const TVector<TString> functions = {"JSON_EXISTS", "JSON_VALUE", "JSON_QUERY"}; + + for (const auto& function : functions) { + const auto query = Sprintf(R"( + pragma CompactNamedExprs; + $json = CAST(@@{"key": 1238}@@ as Json); + SELECT %s( + $json, + "strict $.key" + PASSING + "string" as var1, + 1.234 as var2, + CAST(1 as Int64) as var3, + true as var4, + $json as var5 + ))", + function.data() + ); + NYql::TAstParseResult res = SqlToYql(query); + + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [&](const TString& word, const TString& line) { + Y_UNUSED(word); + UNIT_ASSERT_VALUES_UNEQUAL_C(TString::npos, line.find(R"('('"var1" (String '"string")))"), "Cannot find `var1`"); + UNIT_ASSERT_VALUES_UNEQUAL_C(TString::npos, line.find(R"('('"var2" (Double '"1.234")))"), "Cannot find `var2`"); + UNIT_ASSERT_VALUES_UNEQUAL_C(TString::npos, line.find(R"('('"var3" (SafeCast (Int32 '"1") (DataType 'Int64))))"), "Cannot find `var3`"); + UNIT_ASSERT_VALUES_UNEQUAL_C(TString::npos, line.find(R"('('"var4" (Bool '"true")))"), "Cannot find `var4`"); + UNIT_ASSERT_VALUES_UNEQUAL_C(TString::npos, line.find(R"('('"var5" namedexprnode0))"), "Cannot find `var5`"); + }; + + TWordCountHive elementStat({"JsonVariables"}); + VerifyProgram(res, elementStat, verifyLine); + UNIT_ASSERT(elementStat["JsonVariables"] > 0); + } + } + + Y_UNIT_TEST(ValidVariableNames) { + const TVector<TString> functions = {"JSON_EXISTS", "JSON_VALUE", "JSON_QUERY"}; + + for (const auto& function : functions) { + const auto query = Sprintf(R"( + $json = CAST(@@{"key": 1238}@@ as Json); + SELECT %s( + $json, + "strict $.key" + PASSING + "one" as var1, + "two" as "VaR2", + "three" as `var3`, + "four" as VaR4 + ))", + function.data() + ); + NYql::TAstParseResult res = SqlToYql(query); + + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [&](const TString& word, const TString& line) { + Y_UNUSED(word); + UNIT_ASSERT_VALUES_UNEQUAL_C(TString::npos, line.find(R"('('"var1" (String '"one")))"), "Cannot find `var1`"); + UNIT_ASSERT_VALUES_UNEQUAL_C(TString::npos, line.find(R"('('"VaR2" (String '"two")))"), "Cannot find `VaR2`"); + UNIT_ASSERT_VALUES_UNEQUAL_C(TString::npos, line.find(R"('('"var3" (String '"three")))"), "Cannot find `var3`"); + UNIT_ASSERT_VALUES_UNEQUAL_C(TString::npos, line.find(R"('('"VaR4" (String '"four")))"), "Cannot find `VaR4`"); + }; + + TWordCountHive elementStat({"JsonVariables"}); + VerifyProgram(res, elementStat, verifyLine); + UNIT_ASSERT(elementStat["JsonVariables"] > 0); + } + } +} + +Y_UNIT_TEST_SUITE(MigrationToJsonApi) { + Y_UNIT_TEST(WarningOnDeprecatedJsonUdf) { + NYql::TAstParseResult res = SqlToYql(R"( + $json = CAST(@@{"key": 1234}@@ as Json); + SELECT Json::Parse($json); + )"); + + UNIT_ASSERT(res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:26: Warning: Json UDF is deprecated. Please use JSON API instead, code: 4506\n"); + } +} + +Y_UNIT_TEST_SUITE(AnsiIdentsNegative) { + Y_UNIT_TEST(EnableAnsiLexerFromRequestSpecialComments) { + auto req = "\n" + "\t --!ansi_lexer \n" + "-- Some comment\n" + "-- another comment\n" + "pragma SimpleColumns;\n" + "\n" + "select 1, '''' as empty;"; + + auto res = SqlToYql(req); + UNIT_ASSERT(res.IsOk()); + UNIT_ASSERT(res.Issues.Size() == 0); + } + + Y_UNIT_TEST(AnsiLexerShouldNotBeEnabledHere) { + auto req = "$str = '\n" + "--!ansi_lexer\n" + "--!syntax_v1\n" + "';\n" + "\n" + "select 1, $str, \"\" as empty;"; + auto res = SqlToYql(req); + UNIT_ASSERT(res.IsOk()); + UNIT_ASSERT(res.Issues.Size() == 0); + } + + Y_UNIT_TEST(DoubleQuotesInDictsTuplesOrLists) { + auto req = "$d = { 'a': 1, \"b\": 2, 'c': 3,};"; + + auto res = SqlToYqlWithAnsiLexer(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:16: Error: Column reference \"b\" is not allowed in current scope\n"); + + req = "$t = (1, 2, \"a\");"; + + res = SqlToYqlWithAnsiLexer(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:13: Error: Column reference \"a\" is not allowed in current scope\n"); + + req = "$l = ['a', 'b', \"c\"];"; + + res = SqlToYqlWithAnsiLexer(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:17: Error: Column reference \"c\" is not allowed in current scope\n"); + } + + Y_UNIT_TEST(MultilineComments) { + auto req = "/*/**/ select 1;"; + auto res = SqlToYql(req); + UNIT_ASSERT(res.Root); + res = SqlToYqlWithAnsiLexer(req); +#if ANTLR_VER == 3 + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:16: Error: Unexpected character : syntax error...\n\n"); +#else + UNIT_ASSERT(res.IsOk()); + UNIT_ASSERT(res.Issues.Size() == 0); +#endif + req = "/*\n" + "--/*\n" + "*/ select 1;"; + res = SqlToYql(req); + UNIT_ASSERT(res.Root); + res = SqlToYqlWithAnsiLexer(req); +#if ANTLR_VER == 3 + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:12: Error: Unexpected character : syntax error...\n\n"); +#else + UNIT_ASSERT(res.IsOk()); + UNIT_ASSERT(res.Issues.Size() == 0); +#endif + req = "/*\n" + "/*\n" + "--*/\n" + "*/ select 1;"; + res = SqlToYql(req); + UNIT_ASSERT(!res.Root); +#if ANTLR_VER == 3 + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:4:0: Error: Unexpected token '*' : cannot match to any predicted input...\n\n"); +#else + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:4:0: Error: mismatched input '*' expecting {';', '(', '$', ALTER, ANALYZE, BACKUP, BATCH, COMMIT, CREATE, DECLARE, DEFINE, DELETE, DISCARD, DO, DROP, EVALUATE, EXPLAIN, EXPORT, FOR, FROM, GRANT, IF, IMPORT, INSERT, PARALLEL, PRAGMA, PROCESS, REDUCE, REPLACE, RESTORE, REVOKE, ROLLBACK, SELECT, SHOW, UPDATE, UPSERT, USE, VALUES}\n"); +#endif + res = SqlToYqlWithAnsiLexer(req); + UNIT_ASSERT(res.Root); + } +} + +Y_UNIT_TEST_SUITE(AnsiOptionalAs) { + Y_UNIT_TEST(OptionalAsInProjection) { + UNIT_ASSERT(SqlToYql("PRAGMA AnsiOptionalAs; SELECT a b, c FROM plato.Input;").IsOk()); + ExpectFailWithError("PRAGMA DisableAnsiOptionalAs;\n" + "SELECT a b, c FROM plato.Input;", + "<main>:2:10: Error: Expecting mandatory AS here. Did you miss comma? Please add PRAGMA AnsiOptionalAs; for ANSI compatibility\n"); + } + + Y_UNIT_TEST(OptionalAsWithKeywords) { + UNIT_ASSERT(SqlToYql("PRAGMA AnsiOptionalAs; SELECT a type, b data, c source FROM plato.Input;").IsOk()); + } +} + +Y_UNIT_TEST_SUITE(SessionWindowNegative) { + Y_UNIT_TEST(SessionWindowWithoutSource) { + ExpectFailWithError("SELECT 1 + SessionWindow(ts, 32);", + "<main>:1:12: Error: SessionWindow requires data source\n"); + } + + Y_UNIT_TEST(SessionWindowInProjection) { + ExpectFailWithError("SELECT 1 + SessionWindow(ts, 32) from plato.Input;", + "<main>:1:12: Error: SessionWindow can only be used as a top-level GROUP BY / PARTITION BY expression\n"); + } + + Y_UNIT_TEST(SessionWindowWithNonConstSecondArg) { + ExpectFailWithError( + "SELECT key, session_start FROM plato.Input\n" + "GROUP BY SessionWindow(ts, 32 + subkey) as session_start, key;", + + "<main>:2:10: Error: Source does not allow column references\n" + "<main>:2:33: Error: Column reference 'subkey'\n"); + } + + Y_UNIT_TEST(SessionWindowWithWrongNumberOfArgs) { + ExpectFailWithError("SELECT * FROM plato.Input GROUP BY SessionWindow()", + "<main>:1:36: Error: SessionWindow requires either two or four arguments\n"); + ExpectFailWithError("SELECT * FROM plato.Input GROUP BY SessionWindow(key, subkey, 100)", + "<main>:1:36: Error: SessionWindow requires either two or four arguments\n"); + } + + Y_UNIT_TEST(DuplicateSessionWindow) { + ExpectFailWithError( + "SELECT\n" + " *\n" + "FROM plato.Input\n" + "GROUP BY\n" + " SessionWindow(ts, 10),\n" + " user,\n" + " SessionWindow(ts, 20)\n" + ";", + + "<main>:7:5: Error: Duplicate session window specification:\n" + "<main>:5:5: Error: Previous session window is declared here\n"); + + ExpectFailWithError( + "SELECT\n" + " MIN(key) over w\n" + "FROM plato.Input\n" + "WINDOW w AS (\n" + " PARTITION BY SessionWindow(ts, 10), user,\n" + " SessionWindow(ts, 20)\n" + ");", + + "<main>:6:5: Error: Duplicate session window specification:\n" + "<main>:5:18: Error: Previous session window is declared here\n"); + } + + Y_UNIT_TEST(SessionStartStateWithoutSource) { + ExpectFailWithError("SELECT 1 + SessionStart();", + "<main>:1:12: Error: SessionStart requires data source\n"); + ExpectFailWithError("SELECT 1 + SessionState();", + "<main>:1:12: Error: SessionState requires data source\n"); + } + + Y_UNIT_TEST(SessionStartStateWithoutGroupByOrWindow) { + ExpectFailWithError("SELECT 1 + SessionStart() from plato.Input;", + "<main>:1:12: Error: SessionStart can not be used without aggregation by SessionWindow\n"); + ExpectFailWithError("SELECT 1 + SessionState() from plato.Input;", + "<main>:1:12: Error: SessionState can not be used without aggregation by SessionWindow\n"); + } + + Y_UNIT_TEST(SessionStartStateWithGroupByWithoutSession) { + ExpectFailWithError("SELECT 1 + SessionStart() from plato.Input group by user;", + "<main>:1:12: Error: SessionStart can not be used here: SessionWindow specification is missing in GROUP BY\n"); + ExpectFailWithError("SELECT 1 + SessionState() from plato.Input group by user;", + "<main>:1:12: Error: SessionState can not be used here: SessionWindow specification is missing in GROUP BY\n"); + } + + Y_UNIT_TEST(SessionStartStateWithoutOverWithWindowWithoutSession) { + ExpectFailWithError("SELECT 1 + SessionStart(), MIN(key) over w from plato.Input window w as ()", + "<main>:1:12: Error: SessionStart can not be used without aggregation by SessionWindow. Maybe you forgot to add OVER `window_name`?\n"); + ExpectFailWithError("SELECT 1 + SessionState(), MIN(key) over w from plato.Input window w as ()", + "<main>:1:12: Error: SessionState can not be used without aggregation by SessionWindow. Maybe you forgot to add OVER `window_name`?\n"); + } + + Y_UNIT_TEST(SessionStartStateWithWindowWithoutSession) { + ExpectFailWithError("SELECT 1 + SessionStart() over w, MIN(key) over w from plato.Input window w as ()", + "<main>:1:12: Error: SessionStart can not be used with window w: SessionWindow specification is missing in PARTITION BY\n"); + ExpectFailWithError("SELECT 1 + SessionState() over w, MIN(key) over w from plato.Input window w as ()", + "<main>:1:12: Error: SessionState can not be used with window w: SessionWindow specification is missing in PARTITION BY\n"); + } + + Y_UNIT_TEST(SessionStartStateWithSessionedWindow) { + ExpectFailWithError("SELECT 1 + SessionStart(), MIN(key) over w from plato.Input group by key window w as (partition by SessionWindow(ts, 1)) ", + "<main>:1:12: Error: SessionStart can not be used here: SessionWindow specification is missing in GROUP BY. Maybe you forgot to add OVER `window_name`?\n"); + ExpectFailWithError("SELECT 1 + SessionState(), MIN(key) over w from plato.Input group by key window w as (partition by SessionWindow(ts, 1)) ", + "<main>:1:12: Error: SessionState can not be used here: SessionWindow specification is missing in GROUP BY. Maybe you forgot to add OVER `window_name`?\n"); + } + + Y_UNIT_TEST(AggregationBySessionStateIsNotSupportedYet) { + ExpectFailWithError("SELECT SOME(1 + SessionState()), key from plato.Input group by key, SessionWindow(ts, 1);", + "<main>:1:17: Error: SessionState with GROUP BY is not supported yet\n"); + } + + Y_UNIT_TEST(SessionWindowInRtmr) { + NYql::TAstParseResult res = SqlToYql( + "SELECT * FROM plato.Input GROUP BY SessionWindow(ts, 10);", + 10, TString(NYql::RtmrProviderName)); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:54: Error: Streaming group by query must have a hopping window specification.\n"); + + res = SqlToYql(R"( + SELECT key, SUM(value) AS value FROM plato.Input + GROUP BY key, HOP(subkey, "PT10S", "PT30S", "PT20S"), SessionWindow(ts, 10); + )", 10, TString(NYql::RtmrProviderName)); + + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:2:13: Error: SessionWindow is unsupported for streaming sources\n"); + } +} + +Y_UNIT_TEST_SUITE(LibraSqlSugar) { + auto makeResult = [](TStringBuf settings) { + return SqlToYql( + TStringBuilder() + << settings + << "\n$udf1 = MyLibra::MakeLibraPreprocessor($settings);" + << "\n$udf2 = CustomLibra::MakeLibraPreprocessor($settings);" + << "\nPROCESS plato.Input USING $udf1(TableRow())" + << "\nUNION ALL" + << "\nPROCESS plato.Input USING $udf2(TableRow());" + ); + }; + + Y_UNIT_TEST(EmptySettings) { + auto res = makeResult(R"( + $settings = AsStruct(); + )"); + UNIT_ASSERT(res.IsOk()); + } + + Y_UNIT_TEST(OnlyEntities) { + auto res = makeResult(R"( + $settings = AsStruct( + AsList("A", "B", "C") AS Entities + ); + )"); + UNIT_ASSERT(res.IsOk()); + } + + Y_UNIT_TEST(EntitiesWithStrategy) { + auto res = makeResult(R"( + $settings = AsStruct( + AsList("A", "B", "C") AS Entities, + "blacklist" AS EntitiesStrategy + ); + )"); + UNIT_ASSERT(res.IsOk()); + } + + Y_UNIT_TEST(AllSettings) { + auto res = makeResult(R"( + $settings = AsStruct( + AsList("A", "B", "C") AS Entities, + "whitelist" AS EntitiesStrategy, + "path" AS BlockstatDict, + false AS ParseWithFat, + "map" AS Mode + ); + )"); + UNIT_ASSERT(res.IsOk()); + } + + Y_UNIT_TEST(BadStrategy) { + auto res = makeResult(R"( + $settings = AsStruct("bad" AS EntitiesStrategy); + )"); + UNIT_ASSERT_STRING_CONTAINS( + Err2Str(res), + "Error: MakeLibraPreprocessor got invalid entities strategy: expected 'whitelist' or 'blacklist'" + ); + } + + Y_UNIT_TEST(BadEntities) { + auto res = makeResult(R"( + $settings = AsStruct(AsList("A", 1) AS Entities); + )"); + UNIT_ASSERT_STRING_CONTAINS(Err2Str(res), "Error: MakeLibraPreprocessor entity must be string literal"); + } +} + +Y_UNIT_TEST_SUITE(TrailingQuestionsNegative) { + Y_UNIT_TEST(Basic) { + ExpectFailWithError("SELECT 1?;", "<main>:1:9: Error: Unexpected token '?' at the end of expression\n"); +#if ANTLR_VER == 3 + ExpectFailWithError("SELECT 1? + 1;", "<main>:1:10: Error: Unexpected token '+' : cannot match to any predicted input...\n\n"); +#else + ExpectFailWithError("SELECT 1? + 1;", "<main>:1:10: Error: mismatched input '+' expecting {<EOF>, ';'}\n"); +#endif + ExpectFailWithError("SELECT 1 + 1??? < 2", "<main>:1:13: Error: Unexpected token '?' at the end of expression\n"); + ExpectFailWithError("SELECT 1? > 2? > 3?", + "<main>:1:11: Error: Unexpected token '?' at the end of expression\n" + "<main>:1:16: Error: Unexpected token '?' at the end of expression\n" + "<main>:1:21: Error: Unexpected token '?' at the end of expression\n"); + } + + Y_UNIT_TEST(SmartParen) { + ExpectFailWithError("$x = 1; SELECT (Int32?, $x?)", "<main>:1:27: Error: Unexpected token '?' at the end of expression\n"); + ExpectFailWithError("SELECT (Int32, foo?)", "<main>:1:19: Error: Unexpected token '?' at the end of expression\n"); + } + + Y_UNIT_TEST(LambdaOptArgs) { + ExpectFailWithError("$l = ($x, $y?, $z??, $t?) -> ($x);", "<main>:1:18: Error: Expecting at most one '?' token here (for optional lambda parameters), but got 2\n"); + } +} + +Y_UNIT_TEST_SUITE(FlexibleTypes) { + Y_UNIT_TEST(AssumeOrderByType) { + UNIT_ASSERT(SqlToYql("PRAGMA FlexibleTypes; SELECT 1 AS int32 ASSUME ORDER BY int32").IsOk()); + } + + Y_UNIT_TEST(GroupingSets) { + UNIT_ASSERT(SqlToYql("PRAGMA FlexibleTypes; SELECT COUNT(*) AS cnt, text, uuid FROM plato.Input GROUP BY GROUPING SETS((uuid), (uuid, text));").IsOk()); + } + + Y_UNIT_TEST(WeakField) { + UNIT_ASSERT(SqlToYql("PRAGMA FlexibleTypes; SELECT WeakField(text, string) as text FROM plato.Input").IsOk()); + } + + Y_UNIT_TEST(Aggregation1) { + TString q = + "PRAGMA FlexibleTypes;\n" + "$foo = ($x, $const, $type) -> ($x || $const || FormatType($type));\n" + "SELECT $foo(SOME(x), 'aaa', String) FROM plato.Input GROUP BY y;"; + UNIT_ASSERT(SqlToYql(q).IsOk()); + } + + Y_UNIT_TEST(Aggregation2) { + TString q = + "PRAGMA FlexibleTypes;\n" + "SELECT 1 + String + MAX(key) FROM plato.Input;"; + UNIT_ASSERT(SqlToYql(q).IsOk()); + } +} + +Y_UNIT_TEST_SUITE(ExternalDeclares) { + Y_UNIT_TEST(BasicUsage) { + NSQLTranslation::TTranslationSettings settings; + settings.DeclaredNamedExprs["foo"] = "String"; + auto res = SqlToYqlWithSettings("select $foo;", settings); + UNIT_ASSERT(res.IsOk()); + UNIT_ASSERT(res.Issues.Size() == 0); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "declare") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(R"__((declare "$foo" (DataType 'String)))__")); + } + }; + + TWordCountHive elementStat = {{TString("declare"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["declare"]); + } + + Y_UNIT_TEST(DeclareOverrides) { + NSQLTranslation::TTranslationSettings settings; + settings.DeclaredNamedExprs["foo"] = "String"; + auto res = SqlToYqlWithSettings("declare $foo as Int32; select $foo;", settings); + UNIT_ASSERT(res.IsOk()); + UNIT_ASSERT(res.Issues.Size() == 0); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "declare") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(R"__((declare "$foo" (DataType 'Int32)))__")); + } + }; + + TWordCountHive elementStat = {{TString("declare"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["declare"]); + } + + Y_UNIT_TEST(UnusedDeclareDoesNotProduceWarning) { + NSQLTranslation::TTranslationSettings settings; + settings.DeclaredNamedExprs["foo"] = "String"; + auto res = SqlToYqlWithSettings("select 1;", settings); + UNIT_ASSERT(res.IsOk()); + UNIT_ASSERT(res.Issues.Size() == 0); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "declare") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(R"__((declare "$foo" (DataType 'String)))__")); + } + }; + + TWordCountHive elementStat = {{TString("declare"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["declare"]); + } + + Y_UNIT_TEST(DeclaresWithInvalidTypesFails) { + NSQLTranslation::TTranslationSettings settings; + settings.DeclaredNamedExprs["foo"] = "List<BadType>"; + auto res = SqlToYqlWithSettings("select 1;", settings); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), + "<main>:0:5: Error: Unknown type: 'BadType'\n" + "<main>: Error: Failed to parse type for externally declared name 'foo'\n"); + } +} + +Y_UNIT_TEST_SUITE(ExternalDataSource) { + Y_UNIT_TEST(CreateExternalDataSourceWithAuthNone) { + NYql::TAstParseResult res = SqlToYql(R"sql( + USE plato; + CREATE EXTERNAL DATA SOURCE MyDataSource WITH ( + SOURCE_TYPE="ObjectStorage", + LOCATION="my-bucket", + AUTH_METHOD="NONE" + ); + )sql"); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_STRING_CONTAINS(line, R"#('('('"auth_method" '"NONE") '('"location" '"my-bucket") '('"source_type" '"ObjectStorage"))#"); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("createObject")); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0} }; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } + + Y_UNIT_TEST(CreateExternalDataSourceWithAuthServiceAccount) { + NYql::TAstParseResult res = SqlToYql(R"sql( + USE plato; + CREATE EXTERNAL DATA SOURCE MyDataSource WITH ( + SOURCE_TYPE="ObjectStorage", + LOCATION="my-bucket", + AUTH_METHOD="SERVICE_ACCOUNT", + SERVICE_ACCOUNT_ID="sa", + SERVICE_ACCOUNT_SECRET_NAME="sa_secret_name" + ); + )sql"); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_STRING_CONTAINS(line, R"#('('('"auth_method" '"SERVICE_ACCOUNT") '('"location" '"my-bucket") '('"service_account_id" '"sa") '('"service_account_secret_name" '"sa_secret_name") '('"source_type" '"ObjectStorage"))#"); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("createObject")); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0} }; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } + + Y_UNIT_TEST(CreateExternalDataSourceWithBasic) { + NYql::TAstParseResult res = SqlToYql(R"sql( + USE plato; + CREATE EXTERNAL DATA SOURCE MyDataSource WITH ( + SOURCE_TYPE="PostgreSQL", + LOCATION="protocol://host:port/", + AUTH_METHOD="BASIC", + LOGIN="admin", + PASSWORD_SECRET_NAME="secret_name" + ); + )sql"); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_STRING_CONTAINS(line, R"#('('('"auth_method" '"BASIC") '('"location" '"protocol://host:port/") '('"login" '"admin") '('"password_secret_name" '"secret_name") '('"source_type" '"PostgreSQL"))#"); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("createObject")); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0} }; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } + + Y_UNIT_TEST(CreateExternalDataSourceWithMdbBasic) { + NYql::TAstParseResult res = SqlToYql(R"sql( + USE plato; + CREATE EXTERNAL DATA SOURCE MyDataSource WITH ( + SOURCE_TYPE="PostgreSQL", + LOCATION="protocol://host:port/", + AUTH_METHOD="MDB_BASIC", + SERVICE_ACCOUNT_ID="sa", + SERVICE_ACCOUNT_SECRET_NAME="sa_secret_name", + LOGIN="admin", + PASSWORD_SECRET_NAME="secret_name" + ); + )sql"); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_STRING_CONTAINS(line, R"#('('('"auth_method" '"MDB_BASIC") '('"location" '"protocol://host:port/") '('"login" '"admin") '('"password_secret_name" '"secret_name") '('"service_account_id" '"sa") '('"service_account_secret_name" '"sa_secret_name") '('"source_type" '"PostgreSQL"))#"); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("createObject")); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0} }; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } + + Y_UNIT_TEST(CreateExternalDataSourceWithAws) { + NYql::TAstParseResult res = SqlToYql(R"sql( + USE plato; + CREATE EXTERNAL DATA SOURCE MyDataSource WITH ( + SOURCE_TYPE="PostgreSQL", + LOCATION="protocol://host:port/", + AUTH_METHOD="AWS", + AWS_ACCESS_KEY_ID_SECRET_NAME="secred_id_name", + AWS_SECRET_ACCESS_KEY_SECRET_NAME="secret_key_name", + AWS_REGION="ru-central-1" + ); + )sql"); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_STRING_CONTAINS(line, R"#('('('"auth_method" '"AWS") '('"aws_access_key_id_secret_name" '"secred_id_name") '('"aws_region" '"ru-central-1") '('"aws_secret_access_key_secret_name" '"secret_key_name") '('"location" '"protocol://host:port/") '('"source_type" '"PostgreSQL"))#"); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("createObject")); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0} }; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } + + Y_UNIT_TEST(CreateExternalDataSourceWithToken) { + NYql::TAstParseResult res = SqlToYql(R"sql( + USE plato; + CREATE EXTERNAL DATA SOURCE MyDataSource WITH ( + SOURCE_TYPE="YT", + LOCATION="protocol://host:port/", + AUTH_METHOD="TOKEN", + TOKEN_SECRET_NAME="token_name" + ); + )sql"); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_STRING_CONTAINS(line, R"#('('('"auth_method" '"TOKEN") '('"location" '"protocol://host:port/") '('"source_type" '"YT") '('"token_secret_name" '"token_name"))#"); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("createObject")); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0} }; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } + + Y_UNIT_TEST(CreateExternalDataSourceWithTablePrefix) { + NYql::TAstParseResult res = SqlToYql(R"sql( + USE plato; + pragma TablePathPrefix='/aba'; + CREATE EXTERNAL DATA SOURCE MyDataSource WITH ( + SOURCE_TYPE="ObjectStorage", + LOCATION="my-bucket", + AUTH_METHOD="NONE" + ); + )sql"); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_STRING_CONTAINS(line, "/aba/MyDataSource"); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("createObject")); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0} }; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } + + Y_UNIT_TEST(CreateExternalDataSourceIfNotExists) { + NYql::TAstParseResult res = SqlToYql(R"sql( + USE plato; + CREATE EXTERNAL DATA SOURCE IF NOT EXISTS MyDataSource WITH ( + SOURCE_TYPE="ObjectStorage", + LOCATION="my-bucket", + AUTH_METHOD="NONE" + ); + )sql"); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_STRING_CONTAINS(line, R"#('('('"auth_method" '"NONE") '('"location" '"my-bucket") '('"source_type" '"ObjectStorage"))#"); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("createObjectIfNotExists")); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0} }; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } + + Y_UNIT_TEST(AlterExternalDataSource) { + NYql::TAstParseResult res = SqlToYql(R"sql( + USE plato; + ALTER EXTERNAL DATA SOURCE MyDataSource + SET (SOURCE_TYPE = "ObjectStorage", Login = "Admin"), + SET Location "bucket", + RESET (Auth_Method, Service_Account_Id, Service_Account_Secret_Name); + )sql"); + UNIT_ASSERT_C(res.Root, res.Issues.ToString()); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_STRING_CONTAINS(line, R"#(('mode 'alterObject))#"); + UNIT_ASSERT_STRING_CONTAINS(line, R"#('('features '('('"location" '"bucket") '('"login" '"Admin") '('"source_type" '"ObjectStorage"))))#"); + UNIT_ASSERT_STRING_CONTAINS(line, R"#('('resetFeatures '('"auth_method" '"service_account_id" '"service_account_secret_name")))#"); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0} }; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } + + Y_UNIT_TEST(CreateExternalDataSourceOrReplace) { + NYql::TAstParseResult res = SqlToYql(R"( + USE plato; + CREATE OR REPLACE EXTERNAL DATA SOURCE MyDataSource WITH ( + SOURCE_TYPE="ObjectStorage", + LOCATION="my-bucket", + AUTH_METHOD="NONE" + ); + )"); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_STRING_CONTAINS(line, R"#('('('"auth_method" '"NONE") '('"location" '"my-bucket") '('"source_type" '"ObjectStorage"))#"); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("createObjectOrReplace")); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0} }; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } + + Y_UNIT_TEST(CreateOrReplaceForUnsupportedTableTypesShouldFail) { + ExpectFailWithError(R"sql( + USE plato; + CREATE OR REPLACE TABLE t (a int32 not null, primary key(a, a)); + )sql" , "<main>:3:23: Error: OR REPLACE feature is supported only for EXTERNAL DATA SOURCE and EXTERNAL TABLE\n"); + + ExpectFailWithError(R"sql( + USE plato; + CREATE OR REPLACE TABLE t ( + Key Uint64, + Value1 String, + PRIMARY KEY (Key) + ) + WITH ( + STORE = COLUMN, + AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = 10 + ); + )sql" , "<main>:3:23: Error: OR REPLACE feature is supported only for EXTERNAL DATA SOURCE and EXTERNAL TABLE\n"); + } + + Y_UNIT_TEST(CreateExternalDataSourceWithBadArguments) { +#if ANTLR_VER == 3 + ExpectFailWithError(R"sql( + USE plato; + CREATE EXTERNAL DATA SOURCE MyDataSource; + )sql" , "<main>:3:52: Error: Unexpected token ';' : syntax error...\n\n"); +#else + ExpectFailWithError(R"sql( + USE plato; + CREATE EXTERNAL DATA SOURCE MyDataSource; + )sql" , "<main>:3:52: Error: mismatched input ';' expecting WITH\n"); +#endif + ExpectFailWithError(R"sql( + USE plato; + CREATE EXTERNAL DATA SOURCE MyDataSource WITH ( + LOCATION="my-bucket", + AUTH_METHOD="NONE" + ); + )sql" , "<main>:5:33: Error: SOURCE_TYPE requires key\n"); + + ExpectFailWithError(R"sql( + USE plato; + CREATE EXTERNAL DATA SOURCE MyDataSource WITH ( + SOURCE_TYPE="ObjectStorage", + LOCATION="my-bucket" + ); + )sql" , "<main>:5:30: Error: AUTH_METHOD requires key\n"); + + + ExpectFailWithError(R"sql( + USE plato; + CREATE EXTERNAL DATA SOURCE MyDataSource WITH ( + SOURCE_TYPE="ObjectStorage", + LOCATION="my-bucket", + AUTH_METHOD="NONE1" + ); + )sql" , "<main>:6:33: Error: Unknown AUTH_METHOD = NONE1\n"); + + ExpectFailWithError(R"sql( + USE plato; + CREATE EXTERNAL DATA SOURCE MyDataSource WITH ( + SOURCE_TYPE="ObjectStorage", + LOCATION="my-bucket", + AUTH_METHOD="SERVICE_ACCOUNT" + ); + )sql" , "<main>:6:33: Error: SERVICE_ACCOUNT_ID requires key\n"); + + ExpectFailWithError(R"sql( + USE plato; + CREATE EXTERNAL DATA SOURCE MyDataSource WITH ( + SOURCE_TYPE="ObjectStorage", + LOCATION="my-bucket", + AUTH_METHOD="SERVICE_ACCOUNT", + SERVICE_ACCOUNT_ID="s1" + ); + )sql" , "<main>:7:40: Error: SERVICE_ACCOUNT_SECRET_NAME requires key\n"); + + ExpectFailWithError(R"sql( + USE plato; + CREATE EXTERNAL DATA SOURCE MyDataSource WITH ( + SOURCE_TYPE="ObjectStorage", + LOCATION="my-bucket", + AUTH_METHOD="SERVICE_ACCOUNT", + SERVICE_ACCOUNT_SECRET_NAME="s1" + ); + )sql" , "<main>:7:49: Error: SERVICE_ACCOUNT_ID requires key\n"); + + ExpectFailWithError(R"sql( + USE plato; + CREATE EXTERNAL DATA SOURCE MyDataSource WITH ( + SOURCE_TYPE="PostgreSQL", + LOCATION="protocol://host:port/", + AUTH_METHOD="BASIC", + LOGIN="admin" + ); + )sql" , "<main>:7:27: Error: PASSWORD_SECRET_NAME requires key\n"); + + ExpectFailWithError(R"sql( + USE plato; + CREATE EXTERNAL DATA SOURCE MyDataSource WITH ( + SOURCE_TYPE="PostgreSQL", + LOCATION="protocol://host:port/", + AUTH_METHOD="BASIC", + PASSWORD_SECRET_NAME="secret_name" + ); + )sql" , "<main>:7:42: Error: LOGIN requires key\n"); + + ExpectFailWithError(R"sql( + USE plato; + CREATE EXTERNAL DATA SOURCE MyDataSource WITH ( + SOURCE_TYPE="PostgreSQL", + LOCATION="protocol://host:port/", + AUTH_METHOD="MDB_BASIC", + SERVICE_ACCOUNT_SECRET_NAME="sa_secret_name", + LOGIN="admin", + PASSWORD_SECRET_NAME="secret_name" + ); + )sql" , "<main>:9:42: Error: SERVICE_ACCOUNT_ID requires key\n"); + + ExpectFailWithError(R"sql( + USE plato; + CREATE EXTERNAL DATA SOURCE MyDataSource WITH ( + SOURCE_TYPE="PostgreSQL", + LOCATION="protocol://host:port/", + AUTH_METHOD="MDB_BASIC", + SERVICE_ACCOUNT_ID="sa", + LOGIN="admin", + PASSWORD_SECRET_NAME="secret_name" + ); + )sql" , "<main>:9:42: Error: SERVICE_ACCOUNT_SECRET_NAME requires key\n"); + + ExpectFailWithError(R"sql( + USE plato; + CREATE EXTERNAL DATA SOURCE MyDataSource WITH ( + SOURCE_TYPE="PostgreSQL", + LOCATION="protocol://host:port/", + AUTH_METHOD="MDB_BASIC", + SERVICE_ACCOUNT_ID="sa", + SERVICE_ACCOUNT_SECRET_NAME="sa_secret_name", + PASSWORD_SECRET_NAME="secret_name" + ); + )sql" , "<main>:9:42: Error: LOGIN requires key\n"); + + ExpectFailWithError(R"sql( + USE plato; + CREATE EXTERNAL DATA SOURCE MyDataSource WITH ( + SOURCE_TYPE="PostgreSQL", + LOCATION="protocol://host:port/", + AUTH_METHOD="MDB_BASIC", + SERVICE_ACCOUNT_ID="sa", + SERVICE_ACCOUNT_SECRET_NAME="sa_secret_name", + LOGIN="admin" + ); + )sql" , "<main>:9:27: Error: PASSWORD_SECRET_NAME requires key\n"); + + ExpectFailWithError(R"sql( + USE plato; + CREATE EXTERNAL DATA SOURCE MyDataSource WITH ( + SOURCE_TYPE="PostgreSQL", + LOCATION="protocol://host:port/", + AUTH_METHOD="AWS", + AWS_SECRET_ACCESS_KEY_SECRET_NAME="secret_key_name", + AWS_REGION="ru-central-1" + ); + )sql" , "<main>:8:32: Error: AWS_ACCESS_KEY_ID_SECRET_NAME requires key\n"); + + ExpectFailWithError(R"sql( + USE plato; + CREATE EXTERNAL DATA SOURCE MyDataSource WITH ( + SOURCE_TYPE="PostgreSQL", + LOCATION="protocol://host:port/", + AUTH_METHOD="AWS", + AWS_ACCESS_KEY_ID_SECRET_NAME="secred_id_name", + AWS_REGION="ru-central-1" + ); + )sql" , "<main>:8:32: Error: AWS_SECRET_ACCESS_KEY_SECRET_NAME requires key\n"); + + ExpectFailWithError(R"sql( + USE plato; + CREATE EXTERNAL DATA SOURCE MyDataSource WITH ( + SOURCE_TYPE="PostgreSQL", + LOCATION="protocol://host:port/", + AUTH_METHOD="AWS", + AWS_SECRET_ACCESS_KEY_SECRET_NAME="secret_key_name", + AWS_ACCESS_KEY_ID_SECRET_NAME="secred_id_name" + ); + )sql" , "<main>:8:51: Error: AWS_REGION requires key\n"); + } + + Y_UNIT_TEST(DropExternalDataSourceWithTablePrefix) { + NYql::TAstParseResult res = SqlToYql(R"sql( + USE plato; + DROP EXTERNAL DATA SOURCE MyDataSource; + )sql"); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("'features")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("dropObject")); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } + + Y_UNIT_TEST(DropExternalDataSourceIfExists) { + NYql::TAstParseResult res = SqlToYql(R"sql( + USE plato; + DROP EXTERNAL DATA SOURCE IF EXISTS MyDataSource; + )sql"); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_STRING_CONTAINS(line, "MyDataSource"); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("dropObjectIfExists")); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } + + Y_UNIT_TEST(DropExternalDataSource) { + NYql::TAstParseResult res = SqlToYql(R"sql( + USE plato; + pragma TablePathPrefix='/aba'; + DROP EXTERNAL DATA SOURCE MyDataSource; + )sql"); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_STRING_CONTAINS(line, "/aba/MyDataSource"); + UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("'features")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("dropObject")); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } +} + +Y_UNIT_TEST_SUITE(ExternalTable) { + Y_UNIT_TEST(CreateExternalTable) { + NYql::TAstParseResult res = SqlToYql(R"sql( + USE plato; + CREATE EXTERNAL TABLE mytable ( + a int + ) WITH ( + DATA_SOURCE="/Root/mydatasource", + LOCATION="/folder1/*" + ); + )sql"); + UNIT_ASSERT_C(res.Root, res.Issues.ToOneLineString()); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_STRING_CONTAINS(line, R"#('('('data_source_path (String '"/Root/mydatasource")) '('location (String '"/folder1/*")))) '('tableType 'externalTable)))))#"); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("tablescheme")); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0} }; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } + + Y_UNIT_TEST(CreateExternalTableWithTablePrefix) { + NYql::TAstParseResult res = SqlToYql(R"sql( + USE plato; + pragma TablePathPrefix='/aba'; + CREATE EXTERNAL TABLE mytable ( + a int + ) WITH ( + DATA_SOURCE="mydatasource", + LOCATION="/folder1/*" + ); + )sql"); + UNIT_ASSERT_C(res.Root, res.Issues.ToOneLineString()); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_STRING_CONTAINS(line, "/aba/mydatasource"); + UNIT_ASSERT_STRING_CONTAINS(line, "/aba/mytable"); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("tablescheme")); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0} }; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } + + Y_UNIT_TEST(CreateExternalTableObjectStorage) { + auto res = SqlToYql(R"sql( + USE plato; + CREATE EXTERNAL TABLE mytable ( + a int, + year Int + ) WITH ( + DATA_SOURCE="/Root/mydatasource", + LOCATION="/folder1/*", + FORMAT="json_as_string", + `projection.enabled`="true", + `projection.year.type`="integer", + `projection.year.min`="2010", + `projection.year.max`="2022", + `projection.year.interval`="1", + `projection.month.type`="integer", + `projection.month.min`="1", + `projection.month.max`="12", + `projection.month.interval`="1", + `projection.month.digits`="2", + `storage.location.template`="${year}/${month}", + PARTITONED_BY = "[year, month]" + ); + )sql"); + UNIT_ASSERT_C(res.IsOk(), res.Issues.ToString()); + } + + Y_UNIT_TEST(CreateExternalTableIfNotExists) { + NYql::TAstParseResult res = SqlToYql(R"sql( + USE plato; + CREATE EXTERNAL TABLE IF NOT EXISTS mytable ( + a int + ) WITH ( + DATA_SOURCE="/Root/mydatasource", + LOCATION="/folder1/*" + ); + )sql"); + UNIT_ASSERT_C(res.Root, res.Issues.ToOneLineString()); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_STRING_CONTAINS(line, R"#('('('data_source_path (String '"/Root/mydatasource")) '('location (String '"/folder1/*")))) '('tableType 'externalTable)))))#"); + UNIT_ASSERT_STRING_CONTAINS(line, "create_if_not_exists"); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0} }; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } + + Y_UNIT_TEST(CreateExternalTableOrReplace) { + NYql::TAstParseResult res = SqlToYql(R"( + USE plato; + CREATE OR REPLACE EXTERNAL TABLE mytable ( + a int + ) WITH ( + DATA_SOURCE="/Root/mydatasource", + LOCATION="/folder1/*" + ); + )"); + UNIT_ASSERT_C(res.Root, res.Issues.ToOneLineString()); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_STRING_CONTAINS(line, R"#('('('data_source_path (String '"/Root/mydatasource")) '('location (String '"/folder1/*")))) '('tableType 'externalTable)))))#"); + UNIT_ASSERT_STRING_CONTAINS(line, "create_or_replace"); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0} }; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } + + Y_UNIT_TEST(AlterExternalTableAddColumn) { + NYql::TAstParseResult res = SqlToYql(R"sql( + USE plato; + ALTER EXTERNAL TABLE mytable + ADD COLUMN my_column int32, + RESET (LOCATION); + )sql"); + UNIT_ASSERT_C(res.Root, res.Issues.ToOneLineString()); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_STRING_CONTAINS(line, R"#('actions '('('addColumns '('('"my_column" (AsOptionalType (DataType 'Int32))#"); + UNIT_ASSERT_STRING_CONTAINS(line, R"#(('setTableSettings '('('location)))#"); + UNIT_ASSERT_STRING_CONTAINS(line, R"#(('tableType 'externalTable))#"); + UNIT_ASSERT_STRING_CONTAINS(line, R"#(('mode 'alter))#"); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0} }; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } + + Y_UNIT_TEST(AlterExternalTableDropColumn) { + NYql::TAstParseResult res = SqlToYql(R"sql( + USE plato; + ALTER EXTERNAL TABLE mytable + DROP COLUMN my_column, + SET (Location = "abc", Other_Prop = "42"), + SET x 'y'; + )sql"); + UNIT_ASSERT_C(res.Root, res.Issues.ToOneLineString()); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_STRING_CONTAINS(line, R"#('actions '('('dropColumns '('"my_column")#"); + UNIT_ASSERT_STRING_CONTAINS(line, R"#(('setTableSettings '('('location (String '"abc")) '('Other_Prop (String '"42")) '('x (String '"y")))))#"); + UNIT_ASSERT_STRING_CONTAINS(line, R"#(('tableType 'externalTable))#"); + UNIT_ASSERT_STRING_CONTAINS(line, R"#(('mode 'alter))#"); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0} }; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } + + Y_UNIT_TEST(CreateExternalTableWithBadArguments) { +#if ANTLR_VER == 3 + ExpectFailWithError(R"sql( + USE plato; + CREATE EXTERNAL TABLE mytable; + )sql" , "<main>:3:41: Error: Unexpected token ';' : syntax error...\n\n"); +#else + ExpectFailWithError(R"sql( + USE plato; + CREATE EXTERNAL TABLE mytable; + )sql" , "<main>:3:45: Error: mismatched input ';' expecting '('\n"); +#endif + + ExpectFailWithError(R"sql( + USE plato; + CREATE EXTERNAL TABLE mytable ( + a int + ); + )sql" , "<main>:4:23: Error: DATA_SOURCE requires key\n"); + + ExpectFailWithError(R"sql( + USE plato; + CREATE EXTERNAL TABLE mytable ( + a int + ) WITH ( + DATA_SOURCE="/Root/mydatasource" + ); + )sql" , "<main>:6:33: Error: LOCATION requires key\n"); + + ExpectFailWithError(R"sql( + USE plato; + CREATE EXTERNAL TABLE mytable ( + a int + ) WITH ( + LOCATION="/folder1/*" + ); + )sql" , "<main>:6:30: Error: DATA_SOURCE requires key\n"); + + ExpectFailWithError(R"sql( + USE plato; + CREATE EXTERNAL TABLE mytable ( + a int, + PRIMARY KEY(a) + ) WITH ( + DATA_SOURCE="/Root/mydatasource", + LOCATION="/folder1/*" + ); + )sql" , "<main>:8:30: Error: PRIMARY KEY is not supported for external table\n"); + } + + Y_UNIT_TEST(DropExternalTable) { + NYql::TAstParseResult res = SqlToYql(R"sql( + USE plato; + DROP EXTERNAL TABLE MyExternalTable; + )sql"); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("tablescheme")); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } + + Y_UNIT_TEST(DropExternalTableWithTablePrefix) { + NYql::TAstParseResult res = SqlToYql(R"sql( + USE plato; + pragma TablePathPrefix='/aba'; + DROP EXTERNAL TABLE MyExternalTable; + )sql"); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_STRING_CONTAINS(line, "/aba/MyExternalTable"); + UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("'tablescheme")); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } + + Y_UNIT_TEST(DropExternalTableIfExists) { + NYql::TAstParseResult res = SqlToYql(R"sql( + USE plato; + DROP EXTERNAL TABLE IF EXISTS MyExternalTable; + )sql"); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("tablescheme")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("drop_if_exists")); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } +} + +Y_UNIT_TEST_SUITE(TopicsDDL) { + void TestQuery(const TString& query, bool expectOk = true) { + TStringBuilder finalQuery; + + finalQuery << "use plato;" << Endl << query; + auto res = SqlToYql(finalQuery, 10, "kikimr"); + if (expectOk) { + UNIT_ASSERT_C(res.IsOk(), res.Issues.ToString()); + } else { + UNIT_ASSERT(!res.IsOk()); + } + } + + Y_UNIT_TEST(CreateTopicSimple) { + TestQuery(R"( + CREATE TOPIC topic1; + )"); + TestQuery(R"( + CREATE TOPIC `cluster1.topic1`; + )"); + TestQuery(R"( + CREATE TOPIC topic1 WITH (metering_mode = "str_value", partition_count_limit = 123, retention_period = Interval('PT1H')); + )"); + } + + Y_UNIT_TEST(CreateTopicConsumer) { + TestQuery(R"( + CREATE TOPIC topic1 (CONSUMER cons1); + )"); + TestQuery(R"( + CREATE TOPIC topic1 (CONSUMER cons1, CONSUMER cons2 WITH (important = false)); + )"); + TestQuery(R"( + CREATE TOPIC topic1 (CONSUMER cons1, CONSUMER cons2 WITH (important = false)) WITH (supported_codecs = "1,2,3"); + )"); + } + + Y_UNIT_TEST(AlterTopicSimple) { + TestQuery(R"( + ALTER TOPIC topic1 SET (retention_period = Interval('PT1H')); + )"); + TestQuery(R"( + ALTER TOPIC topic1 SET (retention_storage_mb = 3, partition_count_limit = 50); + )"); + TestQuery(R"( + ALTER TOPIC topic1 RESET (supported_codecs, retention_period); + )"); + TestQuery(R"( + ALTER TOPIC topic1 RESET (partition_write_speed_bytes_per_second), + SET (partition_write_burst_bytes = 11111, min_active_partitions = 1); + )"); + } + Y_UNIT_TEST(AlterTopicConsumer) { + TestQuery(R"( + ALTER TOPIC topic1 ADD CONSUMER consumer1, + ADD CONSUMER consumer2 WITH (important = false, supported_codecs = "RAW"), + ALTER CONSUMER consumer3 SET (important = false, read_from = 1), + ALTER CONSUMER consumer3 RESET (supported_codecs), + DROP CONSUMER consumer4, + SET (partition_count_limit = 11, retention_period = Interval('PT1H')), + RESET(metering_mode) + )"); + } + Y_UNIT_TEST(DropTopic) { + TestQuery(R"( + DROP TOPIC topic1; + )"); + } + + Y_UNIT_TEST(TopicBadRequests) { + TestQuery(R"( + CREATE TOPIC topic1(); + )", false); + TestQuery(R"( + CREATE TOPIC topic1 SET setting1 = value1; + )", false); + TestQuery(R"( + ALTER TOPIC topic1 SET setting1 value1; + )", false); + TestQuery(R"( + ALTER TOPIC topic1 RESET setting1; + )", false); + + TestQuery(R"( + ALTER TOPIC topic1 DROP CONSUMER consumer4 WITH (k1 = v1); + )", false); + + TestQuery(R"( + CREATE TOPIC topic1 WITH (retention_period = 123); + )", false); + TestQuery(R"( + CREATE TOPIC topic1 (CONSUMER cons1, CONSUMER cons1 WITH (important = false)); + )", false); + TestQuery(R"( + CREATE TOPIC topic1 (CONSUMER cons1 WITH (bad_option = false)); + )", false); + TestQuery(R"( + ALTER TOPIC topic1 ADD CONSUMER cons1, ALTER CONSUMER cons1 RESET (important); + )", false); + TestQuery(R"( + ALTER TOPIC topic1 ADD CONSUMER consumer1, + ALTER CONSUMER consumer3 SET (supported_codecs = "RAW", read_from = 1), + ALTER CONSUMER consumer3 RESET (supported_codecs); + )", false); + TestQuery(R"( + ALTER TOPIC topic1 ADD CONSUMER consumer1, + ALTER CONSUMER consumer3 SET (supported_codecs = "RAW", read_from = 1), + ALTER CONSUMER consumer3 SET (read_from = 2); + )", false); + } + + Y_UNIT_TEST(TopicWithPrefix) { + NYql::TAstParseResult res = SqlToYql(R"( + USE plato; + PRAGMA TablePathPrefix = '/database/path/to/tables'; + ALTER TOPIC `my_table/my_feed` ADD CONSUMER `my_consumer`; + )"); + UNIT_ASSERT(res.Root); + + TWordCountHive elementStat = {{TString("/database/path/to/tables/my_table/my_feed"), 0}, {"topic", 0}}; + VerifyProgram(res, elementStat); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["topic"]); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["/database/path/to/tables/my_table/my_feed"]); + } +} + +Y_UNIT_TEST_SUITE(BlockEnginePragma) { + Y_UNIT_TEST(Basic) { + const TVector<TString> values = {"auto", "force", "disable"}; + for (const auto& value : values) { + const auto query = TStringBuilder() << "pragma Blockengine='" << value << "'; select 1;"; + NYql::TAstParseResult res = SqlToYql(query); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [&](const TString& word, const TString& line) { + Y_UNUSED(word); + UNIT_ASSERT_STRING_CONTAINS(line, TStringBuilder() << R"(Configure! world (DataSource '"config") '"BlockEngine" '")" << value << "\""); + }; + + TWordCountHive elementStat({"BlockEngine"}); + VerifyProgram(res, elementStat, verifyLine); + UNIT_ASSERT(elementStat["BlockEngine"] == ((value == "disable") ? 0 : 1)); + } + } + + Y_UNIT_TEST(UnknownSetting) { + ExpectFailWithError("use plato; pragma BlockEngine='foo';", + "<main>:1:31: Error: Expected `disable|auto|force' argument for: BlockEngine\n"); + } +} + +Y_UNIT_TEST_SUITE(TViewSyntaxTest) { + Y_UNIT_TEST(CreateViewSimple) { + NYql::TAstParseResult res = SqlToYql(R"( + USE plato; + CREATE VIEW TheView WITH (security_invoker = TRUE) AS SELECT 1; + )" + ); + UNIT_ASSERT_C(res.Root, res.Issues.ToString()); + } + + Y_UNIT_TEST(CreateViewIfNotExists) { + constexpr const char* name = "TheView"; + NYql::TAstParseResult res = SqlToYql(std::format(R"( + USE plato; + CREATE VIEW IF NOT EXISTS {} AS SELECT 1; + )", name + )); + UNIT_ASSERT_C(res.Root, res.Issues.ToString()); + + TVerifyLineFunc verifyLine = [&](const TString& word, const TString& line) { + if (word == "Write!") { + UNIT_ASSERT_STRING_CONTAINS(line, name); + UNIT_ASSERT_STRING_CONTAINS(line, "createObjectIfNotExists"); + } + }; + + TWordCountHive elementStat = { {"Write!"} }; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(elementStat["Write!"], 1); + } + + Y_UNIT_TEST(CreateViewFromTable) { + constexpr const char* path = "/PathPrefix/TheView"; + constexpr const char* query = R"( + SELECT * FROM SomeTable + )"; + + NYql::TAstParseResult res = SqlToYql(std::format(R"( + USE plato; + CREATE VIEW `{}` WITH (security_invoker = TRUE) AS {}; + )", + path, + query + ) + ); + UNIT_ASSERT_C(res.Root, res.Issues.ToString()); + + TVerifyLineFunc verifyLine = [&](const TString& word, const TString& line) { + if (word == "Write!") { + UNIT_ASSERT_STRING_CONTAINS(line, path); + UNIT_ASSERT_STRING_CONTAINS(line, "createObject"); + } + }; + TWordCountHive elementStat = { {"Write!"} }; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(elementStat["Write!"], 1); + } + + Y_UNIT_TEST(CheckReconstructedQuery) { + constexpr const char* path = "/PathPrefix/TheView"; + constexpr const char* query = R"( + SELECT * FROM FirstTable JOIN SecondTable ON FirstTable.key == SecondTable.key + )"; + + NYql::TAstParseResult res = SqlToYql(std::format(R"( + USE plato; + CREATE VIEW `{}` WITH (security_invoker = TRUE) AS {}; + )", + path, + query + ) + ); + UNIT_ASSERT_C(res.Root, res.Issues.ToString()); + + TString reconstructedQuery = ToString(Tokenize(query)); + TVerifyLineFunc verifyLine = [&](const TString& word, const TString& line) { + if (word == "query_text") { + UNIT_ASSERT_STRING_CONTAINS(line, reconstructedQuery); + } + }; + TWordCountHive elementStat = { {"Write!"} }; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(elementStat["Write!"], 1); + } + + Y_UNIT_TEST(DropView) { + constexpr const char* path = "/PathPrefix/TheView"; + NYql::TAstParseResult res = SqlToYql(std::format(R"( + USE plato; + DROP VIEW `{}`; + )", + path + ) + ); + UNIT_ASSERT_C(res.Root, res.Issues.ToString()); + + TVerifyLineFunc verifyLine = [&](const TString& word, const TString& line) { + if (word == "Write!") { + UNIT_ASSERT_STRING_CONTAINS(line, path); + UNIT_ASSERT_STRING_CONTAINS(line, "dropObject"); + } + }; + TWordCountHive elementStat = { {"Write!"} }; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(elementStat["Write!"], 1); + } + + Y_UNIT_TEST(DropViewIfExists) { + constexpr const char* name = "TheView"; + NYql::TAstParseResult res = SqlToYql(std::format(R"( + USE plato; + DROP VIEW IF EXISTS {}; + )", name + )); + UNIT_ASSERT_C(res.Root, res.Issues.ToString()); + + TVerifyLineFunc verifyLine = [&](const TString& word, const TString& line) { + if (word == "Write!") { + UNIT_ASSERT_STRING_CONTAINS(line, name); + UNIT_ASSERT_STRING_CONTAINS(line, "dropObjectIfExists"); + } + }; + + TWordCountHive elementStat = { {"Write!"} }; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(elementStat["Write!"], 1); + } + + Y_UNIT_TEST(CreateViewWithTablePrefix) { + NYql::TAstParseResult res = SqlToYql(R"( + USE plato; + PRAGMA TablePathPrefix='/PathPrefix'; + CREATE VIEW TheView WITH (security_invoker = TRUE) AS SELECT 1; + )" + ); + UNIT_ASSERT_C(res.Root, res.Issues.ToString()); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write!") { + UNIT_ASSERT_STRING_CONTAINS(line, "/PathPrefix/TheView"); + UNIT_ASSERT_STRING_CONTAINS(line, "createObject"); + } + }; + + TWordCountHive elementStat = { {"Write!"} }; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(elementStat["Write!"], 1); + } + + Y_UNIT_TEST(DropViewWithTablePrefix) { + NYql::TAstParseResult res = SqlToYql(R"( + USE plato; + PRAGMA TablePathPrefix='/PathPrefix'; + DROP VIEW TheView; + )" + ); + UNIT_ASSERT_C(res.Root, res.Issues.ToString()); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_STRING_CONTAINS(line, "/PathPrefix/TheView"); + UNIT_ASSERT_STRING_CONTAINS(line, "dropObject"); + } + }; + + TWordCountHive elementStat = { {"Write!"} }; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(elementStat["Write!"], 1); + } + + Y_UNIT_TEST(YtAlternativeSchemaSyntax) { + NYql::TAstParseResult res = SqlToYql(R"( + SELECT * FROM plato.Input WITH schema(y Int32, x String not null); + )"); + UNIT_ASSERT_C(res.Root, res.Issues.ToString()); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "userschema") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, + line.find(R"__('('('"userschema" (StructType '('"y" (AsOptionalType (DataType 'Int32))) '('"x" (DataType 'String))))))__")); + } + }; + + TWordCountHive elementStat = {{TString("userschema"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["userschema"]); + } + + Y_UNIT_TEST(UseViewAndFullColumnId) { + NYql::TAstParseResult res = SqlToYql("USE plato; SELECT Input.x FROM Input VIEW uitzicht;"); + UNIT_ASSERT(res.Root); + + TWordCountHive elementStat = {{TString("SqlAccess"), 0}, {"SqlProjectItem", 0}, {"Read!", 0}}; + VerifyProgram(res, elementStat); + UNIT_ASSERT_VALUES_EQUAL(0, elementStat["SqlAccess"]); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SqlProjectItem"]); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Read!"]); + } +} + +Y_UNIT_TEST_SUITE(CompactNamedExprs) { + Y_UNIT_TEST(SourceCallablesInWrongContext) { + TString query = R"( + pragma CompactNamedExprs; + $foo = %s(); + select $foo from plato.Input; + )"; + + THashMap<TString, TString> errs = { + {"TableRow", "<main>:3:20: Error: TableRow requires data source\n"}, + {"JoinTableRow", "<main>:3:20: Error: JoinTableRow requires data source\n"}, + {"TableRecordIndex", "<main>:3:20: Error: Unable to use function: TableRecord without source\n"}, + {"TablePath", "<main>:3:20: Error: Unable to use function: TablePath without source\n"}, + {"SystemMetadata", "<main>:3:20: Error: Unable to use function: SystemMetadata without source\n"}, + }; + + for (TString callable : { "TableRow", "JoinTableRow", "TableRecordIndex", "TablePath", "SystemMetadata"}) { + auto req = Sprintf(query.c_str(), callable.c_str()); + ExpectFailWithError(req, errs[callable]); + } + } + + Y_UNIT_TEST(ValidateUnusedExprs) { + TString query = R"( + pragma warning("disable", "4527"); + pragma CompactNamedExprs; + pragma ValidateUnusedExprs; + + $foo = count(1); + select 1; + )"; + ExpectFailWithError(query, "<main>:6:20: Error: Aggregation is not allowed in this context\n"); + query = R"( + pragma warning("disable", "4527"); + pragma CompactNamedExprs; + pragma ValidateUnusedExprs; + + define subquery $x() as + select count(1, 2); + end define; + select 1; + )"; + ExpectFailWithError(query, "<main>:7:24: Error: Aggregation function Count requires exactly 1 argument(s), given: 2\n"); + } + + Y_UNIT_TEST(DisableValidateUnusedExprs) { + TString query = R"( + pragma warning("disable", "4527"); + pragma CompactNamedExprs; + pragma DisableValidateUnusedExprs; + + $foo = count(1); + select 1; + )"; + SqlToYql(query).IsOk(); + query = R"( + pragma warning("disable", "4527"); + pragma CompactNamedExprs; + pragma DisableValidateUnusedExprs; + + define subquery $x() as + select count(1, 2); + end define; + select 1; + )"; + SqlToYql(query).IsOk(); + } +} + +Y_UNIT_TEST_SUITE(ResourcePool) { + Y_UNIT_TEST(CreateResourcePool) { + NYql::TAstParseResult res = SqlToYql(R"sql( + USE plato; + CREATE RESOURCE POOL MyResourcePool WITH ( + CONCURRENT_QUERY_LIMIT=20, + QUERY_CANCEL_AFTER_SECONDS=86400, + QUEUE_TYPE="FIFO" + ); + )sql"); + UNIT_ASSERT_C(res.Root, res.Issues.ToString()); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_STRING_CONTAINS(line, R"#('('('"concurrent_query_limit" (Int32 '"20")) '('"query_cancel_after_seconds" (Int32 '"86400")) '('"queue_type" '"FIFO"))#"); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("createObject")); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0} }; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } + + Y_UNIT_TEST(CreateResourcePoolWithBadArguments) { +#if ANTLR_VER == 3 + ExpectFailWithError(R"sql( + USE plato; + CREATE RESOURCE POOL MyResourcePool; + )sql" , "<main>:3:47: Error: Unexpected token ';' : syntax error...\n\n"); +#else + ExpectFailWithError(R"sql( + USE plato; + CREATE RESOURCE POOL MyResourcePool; + )sql" , "<main>:3:47: Error: mismatched input ';' expecting WITH\n"); +#endif + + ExpectFailWithError(R"sql( + USE plato; + CREATE RESOURCE POOL MyResourcePool WITH ( + DUPLICATE_SETTING="first_value", + DUPLICATE_SETTING="second_value" + ); + )sql" , "<main>:5:21: Error: DUPLICATE_SETTING duplicate keys\n"); + } + + Y_UNIT_TEST(AlterResourcePool) { + NYql::TAstParseResult res = SqlToYql(R"sql( + USE plato; + ALTER RESOURCE POOL MyResourcePool + SET (CONCURRENT_QUERY_LIMIT = 30, Weight = 5, QUEUE_TYPE = "UNORDERED"), + RESET (Query_Cancel_After_Seconds, Query_Count_Limit); + )sql"); + UNIT_ASSERT_C(res.Root, res.Issues.ToString()); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_STRING_CONTAINS(line, R"#(('mode 'alterObject))#"); + UNIT_ASSERT_STRING_CONTAINS(line, R"#('('features '('('"concurrent_query_limit" (Int32 '"30")) '('"queue_type" '"UNORDERED") '('"weight" (Int32 '"5")))))#"); + UNIT_ASSERT_STRING_CONTAINS(line, R"#('('resetFeatures '('"query_cancel_after_seconds" '"query_count_limit")))#"); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0} }; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } + + Y_UNIT_TEST(DropResourcePool) { + NYql::TAstParseResult res = SqlToYql(R"sql( + USE plato; + DROP RESOURCE POOL MyResourcePool; + )sql"); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("'features")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("dropObject")); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } +} + +Y_UNIT_TEST_SUITE(BackupCollection) { + Y_UNIT_TEST(CreateBackupCollection) { + NYql::TAstParseResult res = SqlToYql(R"sql( + USE plato; + CREATE BACKUP COLLECTION TestCollection WITH ( + STORAGE="local", + TAG="test" -- for testing purposes, not a real thing + ); + )sql"); + UNIT_ASSERT_C(res.Root, res.Issues.ToString()); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(R"#('"TestCollection")#")); + UNIT_ASSERT_STRING_CONTAINS(line, R"#(('"storage" (String '"local")))#"); + UNIT_ASSERT_STRING_CONTAINS(line, R"#(('"tag" (String '"test"))))#"); + UNIT_ASSERT_STRING_CONTAINS(line, R"#(('entries '()))#"); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'create")); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0} }; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } + + Y_UNIT_TEST(CreateBackupCollectionWithDatabase) { + NYql::TAstParseResult res = SqlToYql(R"sql( + USE plato; + CREATE BACKUP COLLECTION TestCollection DATABASE WITH ( + STORAGE="local", + TAG="test" -- for testing purposes, not a real thing + ); + )sql"); + UNIT_ASSERT_C(res.Root, res.Issues.ToString()); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(R"#('"TestCollection")#")); + UNIT_ASSERT_STRING_CONTAINS(line, R"#(('"storage" (String '"local")))#"); + UNIT_ASSERT_STRING_CONTAINS(line, R"#(('"tag" (String '"test"))))#"); + UNIT_ASSERT_STRING_CONTAINS(line, R"#(('entries '('('('type 'database)))))#"); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'create")); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0} }; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } + + Y_UNIT_TEST(CreateBackupCollectionWithTables) { + NYql::TAstParseResult res = SqlToYql(R"sql( + USE plato; + CREATE BACKUP COLLECTION TestCollection ( + TABLE someTable, + TABLE `prefix/anotherTable` + ) WITH ( + STORAGE="local", + TAG="test" -- for testing purposes, not a real thing + ); + )sql"); + UNIT_ASSERT_C(res.Root, res.Issues.ToString()); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(R"#('"TestCollection")#")); + UNIT_ASSERT_STRING_CONTAINS(line, R"#(('"storage" (String '"local")))#"); + UNIT_ASSERT_STRING_CONTAINS(line, R"#(('"tag" (String '"test"))))#"); + UNIT_ASSERT_STRING_CONTAINS(line, R"#(('entries '('('('type 'table) '('path '"someTable")) '('('type 'table) '('path '"prefix/anotherTable")))))#"); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'create")); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0} }; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } + + Y_UNIT_TEST(CreateBackupCollectionWithBadArguments) { +#if ANTLR_VER == 3 + ExpectFailWithError(R"sql( + USE plato; + CREATE BACKUP COLLECTION TestCollection; + )sql" , "<main>:3:51: Error: Unexpected token ';' : syntax error...\n\n"); + + ExpectFailWithError(R"sql( + USE plato; + CREATE BACKUP COLLECTION TABLE TestCollection; + )sql" , "<main>:3:43: Error: Unexpected token 'TestCollection' : syntax error...\n\n"); + + ExpectFailWithError(R"sql( + USE plato; + CREATE BACKUP COLLECTION DATABASE `test` TestCollection; + )sql" , "<main>:3:46: Error: Unexpected token '`test`' : syntax error...\n\n"); +#else + ExpectFailWithError(R"sql( + USE plato; + CREATE BACKUP COLLECTION TestCollection; + )sql" , "<main>:3:55: Error: mismatched input ';' expecting {'(', DATABASE, WITH}\n"); + + ExpectFailWithError(R"sql( + USE plato; + CREATE BACKUP COLLECTION TABLE TestCollection; + )sql" , "<main>:3:47: Error: mismatched input 'TestCollection' expecting {'(', DATABASE, WITH}\n"); + + ExpectFailWithError(R"sql( + USE plato; + CREATE BACKUP COLLECTION DATABASE `test` TestCollection; + )sql" , "<main>:3:50: Error: mismatched input '`test`' expecting {'(', DATABASE, WITH}\n"); +#endif + + ExpectFailWithError(R"sql( + USE plato; + CREATE BACKUP COLLECTION TestCollection WITH ( + DUPLICATE_SETTING="first_value", + DUPLICATE_SETTING="second_value" + ); + )sql" , "<main>:5:21: Error: DUPLICATE_SETTING duplicate keys\n"); + + ExpectFailWithError(R"sql( + USE plato; + CREATE BACKUP COLLECTION TestCollection WITH ( + INT_SETTING=1 + ); + )sql" , "<main>:4:21: Error: INT_SETTING value should be a string literal\n"); + } + + Y_UNIT_TEST(AlterBackupCollection) { + NYql::TAstParseResult res = SqlToYql(R"sql( + USE plato; + ALTER BACKUP COLLECTION TestCollection + SET (STORAGE="remote"), -- also just for test + SET (TAG1 = "123"), + RESET (TAG2, TAG3); + )sql"); + UNIT_ASSERT_C(res.Root, res.Issues.ToString()); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(R"#('"TestCollection")#")); + UNIT_ASSERT_STRING_CONTAINS(line, R"#(('mode 'alter))#"); + UNIT_ASSERT_STRING_CONTAINS(line, R"#(('"storage" (String '"remote")))#"); + UNIT_ASSERT_STRING_CONTAINS(line, R"#(('"tag1" (String '"123"))))#"); + UNIT_ASSERT_STRING_CONTAINS(line, R"#('('resetSettings '('"tag2" '"tag3")))#"); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0} }; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } + + Y_UNIT_TEST(AlterBackupCollectionEntries) { + NYql::TAstParseResult res = SqlToYql(R"sql( + USE plato; + ALTER BACKUP COLLECTION TestCollection + DROP TABLE `test`, + ADD DATABASE; + )sql"); + UNIT_ASSERT_C(res.Root, res.Issues.ToString()); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(R"#('"TestCollection")#")); + UNIT_ASSERT_STRING_CONTAINS(line, R"#(('mode 'alter))#"); + UNIT_ASSERT_STRING_CONTAINS(line, R"#('alterEntries)#"); + UNIT_ASSERT_STRING_CONTAINS(line, R"#('('('type 'table) '('path '"test") '('action 'drop)))#"); + UNIT_ASSERT_STRING_CONTAINS(line, R"#('('('type 'database) '('action 'add)))#"); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0} }; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } + + Y_UNIT_TEST(DropBackupCollection) { + NYql::TAstParseResult res = SqlToYql(R"sql( + USE plato; + DROP BACKUP COLLECTION TestCollection; + )sql"); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(R"#('"TestCollection")#")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("drop")); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } +} + +Y_UNIT_TEST_SUITE(ResourcePoolClassifier) { + Y_UNIT_TEST(CreateResourcePoolClassifier) { + NYql::TAstParseResult res = SqlToYql(R"sql( + USE plato; + CREATE RESOURCE POOL CLASSIFIER MyResourcePoolClassifier WITH ( + RANK=20, + RESOURCE_POOL='wgUserQueries', + MEMBER_NAME='yandex_query@abc' + ); + )sql"); + UNIT_ASSERT_C(res.Root, res.Issues.ToString()); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_STRING_CONTAINS(line, R"#('('('"member_name" '"yandex_query@abc") '('"rank" (Int32 '"20")) '('"resource_pool" '"wgUserQueries"))#"); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("createObject")); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0} }; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } + + Y_UNIT_TEST(CreateResourcePoolClassifierWithBadArguments) { +#if ANTLR_VER == 3 + ExpectFailWithError(R"sql( + USE plato; + CREATE RESOURCE POOL CLASSIFIER MyResourcePoolClassifier; + )sql" , "<main>:3:68: Error: Unexpected token ';' : syntax error...\n\n"); +#else + ExpectFailWithError(R"sql( + USE plato; + CREATE RESOURCE POOL CLASSIFIER MyResourcePoolClassifier; + )sql" , "<main>:3:72: Error: mismatched input ';' expecting WITH\n"); +#endif + + ExpectFailWithError(R"sql( + USE plato; + CREATE RESOURCE POOL CLASSIFIER MyResourcePoolClassifier WITH ( + DUPLICATE_SETTING="first_value", + DUPLICATE_SETTING="second_value" + ); + )sql" , "<main>:5:21: Error: DUPLICATE_SETTING duplicate keys\n"); + } + + Y_UNIT_TEST(AlterResourcePoolClassifier) { + NYql::TAstParseResult res = SqlToYql(R"sql( + USE plato; + ALTER RESOURCE POOL CLASSIFIER MyResourcePoolClassifier + SET (RANK = 30, Weight = 5, MEMBER_NAME = "test@user"), + RESET (Resource_Pool); + )sql"); + UNIT_ASSERT_C(res.Root, res.Issues.ToString()); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_STRING_CONTAINS(line, R"#(('mode 'alterObject))#"); + UNIT_ASSERT_STRING_CONTAINS(line, R"#('('features '('('"member_name" '"test@user") '('"rank" (Int32 '"30")) '('"weight" (Int32 '"5")))))#"); + UNIT_ASSERT_STRING_CONTAINS(line, R"#('('resetFeatures '('"resource_pool")))#"); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0} }; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } + + Y_UNIT_TEST(DropResourcePoolClassifier) { + NYql::TAstParseResult res = SqlToYql(R"sql( + USE plato; + DROP RESOURCE POOL CLASSIFIER MyResourcePoolClassifier; + )sql"); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("'features")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("dropObject")); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } + + Y_UNIT_TEST(BacktickMatching) { + auto req = "select\n" + " 1 as `Schema has \\`RealCost\\``\n" + " -- foo`bar"; + auto res = SqlToYql(req); + UNIT_ASSERT(res.Root); + UNIT_ASSERT(res.IsOk()); + UNIT_ASSERT(res.Issues.Size() == 0); + res = SqlToYqlWithAnsiLexer(req); + UNIT_ASSERT(res.Root); + UNIT_ASSERT(res.IsOk()); + UNIT_ASSERT(res.Issues.Size() == 0); + + req = "select 1 as `a``b`, 2 as ````, 3 as `\\x60a\\x60`, 4 as ```b```, 5 as `\\`c\\``"; + res = SqlToYql(req); + UNIT_ASSERT(res.Root); + UNIT_ASSERT(res.IsOk()); + UNIT_ASSERT(res.Issues.Size() == 0); + res = SqlToYqlWithAnsiLexer(req); + UNIT_ASSERT(res.Root); + UNIT_ASSERT(res.IsOk()); + UNIT_ASSERT(res.Issues.Size() == 0); + } +} + +Y_UNIT_TEST_SUITE(Backup) { + Y_UNIT_TEST(Simple) { + NYql::TAstParseResult res = SqlToYql(R"sql( + USE plato; + BACKUP TestCollection; + )sql"); + UNIT_ASSERT_C(res.Root, res.Issues.ToString()); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(R"#('"TestCollection")#")); + UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("'Incremental")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'backup")); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0} }; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } + + Y_UNIT_TEST(Incremental) { + NYql::TAstParseResult res = SqlToYql(R"sql( + USE plato; + BACKUP TestCollection INCREMENTAL; + )sql"); + UNIT_ASSERT_C(res.Root, res.Issues.ToString()); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(R"#('"TestCollection")#")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'backupIncremental")); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0} }; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } +} + +Y_UNIT_TEST_SUITE(Restore) { + Y_UNIT_TEST(Simple) { + NYql::TAstParseResult res = SqlToYql(R"sql( + USE plato; + RESTORE TestCollection; + )sql"); + UNIT_ASSERT_C(res.Root, res.Issues.ToString()); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(R"#('"TestCollection")#")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'restore")); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0} }; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } + + Y_UNIT_TEST(AtPoint) { + NYql::TAstParseResult res = SqlToYql(R"sql( + USE plato; + RESTORE TestCollection AT '2024-06-16_20-14-02'; + )sql"); + UNIT_ASSERT_C(res.Root, res.Issues.ToString()); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(R"#('"TestCollection")#")); + UNIT_ASSERT_STRING_CONTAINS(line, R"#('at '"2024-06-16_20-14-02")#"); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'restore")); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0} }; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } +} + +Y_UNIT_TEST_SUITE(ColumnFamily) { + Y_UNIT_TEST(CompressionLevelCorrectUsage) { + NYql::TAstParseResult res = SqlToYql(R"( use plato; + CREATE TABLE tableName ( + Key Uint32 FAMILY default, + Value String FAMILY family1, + PRIMARY KEY (Key), + FAMILY default ( + DATA = "test", + COMPRESSION = "lz4", + COMPRESSION_LEVEL = 5 + ), + FAMILY family1 ( + DATA = "test", + COMPRESSION = "lz4", + COMPRESSION_LEVEL = 3 + ) + ); + )"); + UNIT_ASSERT(res.IsOk()); + UNIT_ASSERT(res.Issues.Size() == 0); + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("compression_level")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("5")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("3")); + } + }; + + TWordCountHive elementStat = { { TString("Write"), 0 }, { TString("compression_level"), 0 } }; + VerifyProgram(res, elementStat, verifyLine); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + UNIT_ASSERT_VALUES_EQUAL(2, elementStat["compression_level"]); + } + + Y_UNIT_TEST(FieldDataIsNotString) { + NYql::TAstParseResult res = SqlToYql(R"( use plato; + CREATE TABLE tableName ( + Key Uint32 FAMILY default, + PRIMARY KEY (Key), + FAMILY default ( + DATA = 1, + COMPRESSION = "lz4", + COMPRESSION_LEVEL = 5 + ) + ); + )"); + UNIT_ASSERT(!res.IsOk()); + UNIT_ASSERT(res.Issues.Size() == 1); + UNIT_ASSERT_STRING_CONTAINS(res.Issues.ToString(), "DATA value should be a string literal"); + } + + Y_UNIT_TEST(FieldCompressionIsNotString) { + NYql::TAstParseResult res = SqlToYql(R"( use plato; + CREATE TABLE tableName ( + Key Uint32 FAMILY default, + PRIMARY KEY (Key), + FAMILY default ( + DATA = "test", + COMPRESSION = 2, + COMPRESSION_LEVEL = 5 + ), + ); + )"); + UNIT_ASSERT(!res.IsOk()); + UNIT_ASSERT(res.Issues.Size() == 1); + UNIT_ASSERT_STRING_CONTAINS(res.Issues.ToString(), "COMPRESSION value should be a string literal"); + } + + Y_UNIT_TEST(FieldCompressionLevelIsNotInteger) { + NYql::TAstParseResult res = SqlToYql(R"( use plato; + CREATE TABLE tableName ( + Key Uint32 FAMILY default, + PRIMARY KEY (Key), + FAMILY default ( + DATA = "test", + COMPRESSION = "lz4", + COMPRESSION_LEVEL = "5" + ) + ); + )"); + UNIT_ASSERT(!res.IsOk()); + UNIT_ASSERT(res.Issues.Size() == 1); + UNIT_ASSERT_STRING_CONTAINS(res.Issues.ToString(), "COMPRESSION_LEVEL value should be an integer"); + } + + Y_UNIT_TEST(AlterCompressionCorrectUsage) { + NYql::TAstParseResult res = SqlToYql(R"( use plato; + ALTER TABLE tableName ALTER FAMILY default SET COMPRESSION "lz4"; + )"); + UNIT_ASSERT(res.IsOk()); + UNIT_ASSERT(res.Issues.Size() == 0); + } + + Y_UNIT_TEST(AlterCompressionFieldIsNotString) { + NYql::TAstParseResult res = SqlToYql(R"( use plato; + ALTER TABLE tableName ALTER FAMILY default SET COMPRESSION lz4; + )"); + UNIT_ASSERT(!res.IsOk()); + UNIT_ASSERT(res.Issues.Size() == 1); +#if ANTLR_VER == 3 + UNIT_ASSERT_STRING_CONTAINS(res.Issues.ToString(), "Unexpected token 'lz4' : cannot match to any predicted input"); +#else + UNIT_ASSERT_STRING_CONTAINS(res.Issues.ToString(), "mismatched input 'lz4' expecting {STRING_VALUE, DIGITS, INTEGER_VALUE}"); +#endif + } + + Y_UNIT_TEST(AlterCompressionLevelCorrectUsage) { + NYql::TAstParseResult res = SqlToYql(R"( use plato; + ALTER TABLE tableName ALTER FAMILY default SET COMPRESSION_LEVEL 5; + )"); + UNIT_ASSERT(res.IsOk()); + UNIT_ASSERT(res.Issues.Size() == 0); + } + + Y_UNIT_TEST(AlterCompressionLevelFieldIsNotInteger) { + NYql::TAstParseResult res = SqlToYql(R"( use plato; + ALTER TABLE tableName ALTER FAMILY default SET COMPRESSION_LEVEL "5"; + )"); + UNIT_ASSERT(!res.IsOk()); + UNIT_ASSERT(res.Issues.Size() == 1); + UNIT_ASSERT_STRING_CONTAINS(res.Issues.ToString(), "COMPRESSION_LEVEL value should be an integer"); + } +} + +Y_UNIT_TEST_SUITE(Transfer) { + Y_UNIT_TEST(Lambda) { + NYql::TAstParseResult res = SqlToYql(R"( use plato; + -- Русский коммент, empty statement + ; + + -- befor comment + $a = "А"; + + SELECT * FROM Input; + + $b = ($x) -> { return $a || $x; }; + + CREATE TRANSFER `TransferName` + FROM `TopicName` TO `TableName` + USING ($x) -> { + -- internal comment + return $b($x); + } + WITH ( + CONNECTION_STRING = "grpc://localhost:2135/?database=/Root" + ); + )"); + + UNIT_ASSERT_C(res.IsOk(), res.Issues.ToString()); + UNIT_ASSERT_VALUES_EQUAL_C(res.Issues.Size(), 0, res.Issues.ToString()); + + const auto programm = GetPrettyPrint(res); + + Cerr << ">>>>> Root " << programm << Endl; + auto expected = R"('transformLambda 'use plato; +-- befor comment + $a = "А"; +$b = ($x) -> { return $a || $x; }; +$__ydb_transfer_lambda = ($x) -> { + -- internal comment + return $b($x); + }; +))"; + + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, programm.find(expected)); + + } +} + +Y_UNIT_TEST_SUITE(MatchRecognizeMeasuresAggregation) { + Y_UNIT_TEST(InsideSelect) { + ExpectFailWithError(R"sql( + SELECT FIRST(0), LAST(1); + )sql", + "<main>:2:20: Error: Cannot use FIRST and LAST outside the MATCH_RECOGNIZE context\n" + "<main>:2:30: Error: Cannot use FIRST and LAST outside the MATCH_RECOGNIZE context\n" + ); + } + + Y_UNIT_TEST(OutsideSelect) { + ExpectFailWithError(R"sql( + $lambda = ($x) -> (FIRST($x) + LAST($x)); + SELECT $lambda(x) FROM plato.Input; + )sql", + "<main>:2:32: Error: Cannot use FIRST and LAST outside the MATCH_RECOGNIZE context\n" + "<main>:2:44: Error: Cannot use FIRST and LAST outside the MATCH_RECOGNIZE context\n" + ); + } + + Y_UNIT_TEST(AsAggregateFunction) { + ExpectFailWithError(R"sql( + SELECT FIRST(x), LAST(x) FROM plato.Input; + )sql", + "<main>:2:20: Error: Cannot use FIRST and LAST outside the MATCH_RECOGNIZE context\n" + "<main>:2:30: Error: Cannot use FIRST and LAST outside the MATCH_RECOGNIZE context\n" + ); + } + + Y_UNIT_TEST(AsWindowFunction) { + ExpectFailWithError(R"sql( + SELECT FIRST(x) OVER(), LAST(x) OVER() FROM plato.Input; + )sql", + "<main>:2:20: Error: Cannot use FIRST and LAST outside the MATCH_RECOGNIZE context\n" + "<main>:2:37: Error: Cannot use FIRST and LAST outside the MATCH_RECOGNIZE context\n" + ); + } +} + +Y_UNIT_TEST_SUITE(OlapPartitionCount) { + Y_UNIT_TEST(CorrectUsage) { + NYql::TAstParseResult res = SqlToYql(R"sql( + USE plato; + CREATE TABLE `mytable` (id Uint32, PRIMARY KEY (id)) + PARTITION BY HASH(id) + WITH (STORE = COLUMN, PARTITION_COUNT = 8); + )sql"); + + UNIT_ASSERT_C(res.IsOk(), res.Issues.ToString()); + } + + Y_UNIT_TEST(UseWithoutColumnStore) { + NYql::TAstParseResult res = SqlToYql(R"sql( + USE plato; + CREATE TABLE `mytable` (id Uint32, PRIMARY KEY (id)) + WITH (PARTITION_COUNT = 8); + )sql"); + + UNIT_ASSERT(!res.IsOk()); + UNIT_ASSERT(res.Issues.Size() == 1); + UNIT_ASSERT_STRING_CONTAINS(res.Issues.ToString(), "PARTITION_COUNT can be used only with STORE=COLUMN"); + } +} diff --git a/yql/essentials/sql/v1/ya.make b/yql/essentials/sql/v1/ya.make index 08fd499b3f..1d2105f6fa 100644 --- a/yql/essentials/sql/v1/ya.make +++ b/yql/essentials/sql/v1/ya.make @@ -56,6 +56,7 @@ GENERATE_ENUM_SERIALIZATION(sql_call_param.h) END() RECURSE( + complete format lexer perf diff --git a/yql/essentials/tests/common/test_framework/yql_utils.py b/yql/essentials/tests/common/test_framework/yql_utils.py index 8314349358..2136729be9 100644 --- a/yql/essentials/tests/common/test_framework/yql_utils.py +++ b/yql/essentials/tests/common/test_framework/yql_utils.py @@ -52,7 +52,7 @@ def do_custom_query_check(res, sql_query): def do_custom_error_check(res, sql_query): err_string = None - custom_error = re.search(r"/\* custom error:(.*)\*/", sql_query) + custom_error = re.search(r"/\* custom error:(.*?)\*/", sql_query, re.DOTALL) if custom_error: err_string = custom_error.group(1).strip() assert err_string, 'Expected custom error check in test.\nTest error: %s' % res.std_err diff --git a/yql/essentials/tools/ya.make b/yql/essentials/tools/ya.make index becd8907f6..248945e3fb 100644 --- a/yql/essentials/tools/ya.make +++ b/yql/essentials/tools/ya.make @@ -11,6 +11,7 @@ RECURSE( udf_dep_stub udf_probe udf_resolver + yql_complete yql_facade_run yql_linter ) diff --git a/yql/essentials/tools/yql_complete/ya.make b/yql/essentials/tools/yql_complete/ya.make new file mode 100644 index 0000000000..627abb2eeb --- /dev/null +++ b/yql/essentials/tools/yql_complete/ya.make @@ -0,0 +1,12 @@ +PROGRAM() + +PEERDIR( + library/cpp/getopt + yql/essentials/sql/v1/complete +) + +SRCS( + yql_complete.cpp +) + +END() diff --git a/yql/essentials/tools/yql_complete/yql_complete.cpp b/yql/essentials/tools/yql_complete/yql_complete.cpp new file mode 100644 index 0000000000..289573190b --- /dev/null +++ b/yql/essentials/tools/yql_complete/yql_complete.cpp @@ -0,0 +1,53 @@ +#include <yql/essentials/sql/v1/complete/sql_complete.h> + +#include <library/cpp/getopt/last_getopt.h> +#include <util/stream/file.h> + +int Run(int argc, char* argv[]) { + NLastGetopt::TOpts opts = NLastGetopt::TOpts::Default(); + + TString inFileName; + TMaybe<ui64> pos; + opts.AddLongOption('i', "input", "input file").RequiredArgument("input").StoreResult(&inFileName); + opts.AddLongOption('p', "pos", "position").StoreResult(&pos); + opts.SetFreeArgsNum(0); + opts.AddHelpOption(); + + NLastGetopt::TOptsParseResult res(&opts, argc, argv); + + THolder<TUnbufferedFileInput> inFile; + if (!inFileName.empty()) { + inFile.Reset(new TUnbufferedFileInput(inFileName)); + } + IInputStream& in = inFile ? *inFile.Get() : Cin; + + auto queryString = in.ReadAll(); + auto engine = NSQLComplete::MakeSqlCompletionEngine(); + NSQLComplete::TCompletionInput input; + input.Text = queryString; + if (pos) { + input.CursorPosition = *pos; + } else { + input.CursorPosition = queryString.size(); + } + + auto output = engine->Complete(input); + for (const auto& c : output.Candidates) { + Cout << "[" << c.Kind << "] " << c.Content << "\n"; + } + + return 0; +} + +int main(int argc, char* argv[]) { + try { + return Run(argc, argv); + } catch (const yexception& e) { + Cerr << "Caught exception:" << e.what() << Endl; + return 1; + } catch (...) { + Cerr << CurrentExceptionMessage() << Endl; + return 1; + } + return 0; +} diff --git a/yql/essentials/udfs/common/datetime2/datetime_udf.cpp b/yql/essentials/udfs/common/datetime2/datetime_udf.cpp index c0157f2365..c56b22a7fa 100644 --- a/yql/essentials/udfs/common/datetime2/datetime_udf.cpp +++ b/yql/essentials/udfs/common/datetime2/datetime_udf.cpp @@ -13,6 +13,9 @@ using namespace NUdf; using namespace NYql::DateTime; extern const char SplitUDF[] = "Split"; +extern const char ToDaysUDF[] = "ToDays"; +extern const char ToHoursUDF[] = "ToHours"; +extern const char ToMinutesUDF[] = "ToMinutes"; extern const char ToSecondsUDF[] = "ToSeconds"; extern const char ToMillisecondsUDF[] = "ToMilliseconds"; extern const char ToMicrosecondsUDF[] = "ToMicroseconds"; @@ -59,7 +62,7 @@ const auto UsecondsInMinute = 60000000ll; const auto UsecondsInSecond = 1000000ll; const auto UsecondsInMilliseconds = 1000ll; -template <const char* TFuncName, typename TResult, ui32 ScaleAfterSeconds> +template <const char* TFuncName, typename TResult, typename TWResult, ui32 ScaleAfterSeconds> class TToUnits { public: typedef bool TTypeAwareMarker; @@ -69,6 +72,10 @@ public: return value * ui32(86400) * TResult(ScaleAfterSeconds); } + static TWResult Date32Core(i32 value) { + return value * i64(86400) * TWResult(ScaleAfterSeconds); + } + template<typename TTzDate> static TResult TzBlockCore(TBlockItem tzDate); @@ -91,14 +98,26 @@ public: return value * TResult(ScaleAfterSeconds); } + static TWResult Datetime64Core(i64 value) { + return value * TWResult(ScaleAfterSeconds); + } + static TResult TimestampCore(ui64 value) { return TResult(value / (1000000u / ScaleAfterSeconds)); } + static TWResult Timestamp64Core(i64 value) { + return TWResult(value / (1000000u / ScaleAfterSeconds)); + } + static TSignedResult IntervalCore(i64 value) { return TSignedResult(value / (1000000u / ScaleAfterSeconds)); } + static TWResult Interval64Core(i64 value) { + return TWResult(value / (1000000u / ScaleAfterSeconds)); + } + static const TStringRef& Name() { static auto name = TStringRef(TFuncName, std::strlen(TFuncName)); return name; @@ -120,125 +139,166 @@ public: return false; } - try { - auto typeInfoHelper = builder.TypeInfoHelper(); - TTupleTypeInspector tuple(*typeInfoHelper, userType); - Y_ENSURE(tuple); - Y_ENSURE(tuple.GetElementsCount() > 0); - TTupleTypeInspector argsTuple(*typeInfoHelper, tuple.GetElementType(0)); - Y_ENSURE(argsTuple); - if (argsTuple.GetElementsCount() != 1) { - builder.SetError("Expected one argument"); - return true; - } + if (!userType) { + builder.SetError("User type is missing"); + return true; + } + builder.UserType(userType); - auto argType = argsTuple.GetElementType(0); - TVector<const TType*> argBlockTypes; - argBlockTypes.push_back(argType); + const auto typeInfoHelper = builder.TypeInfoHelper(); + TTupleTypeInspector tuple(*typeInfoHelper, userType); + Y_ENSURE(tuple, "Tuple with args and options tuples expected"); + Y_ENSURE(tuple.GetElementsCount() > 0, + "Tuple has to contain positional arguments"); - TBlockTypeInspector block(*typeInfoHelper, argType); - if (block) { - Y_ENSURE(!block.IsScalar()); - argType = block.GetItemType(); - } + TTupleTypeInspector argsTuple(*typeInfoHelper, tuple.GetElementType(0)); + Y_ENSURE(argsTuple, "Tuple with args expected"); + if (argsTuple.GetElementsCount() != 1) { + builder.SetError("Single argument expected"); + return true; + } - bool isOptional = false; - if (auto opt = TOptionalTypeInspector(*typeInfoHelper, argType)) { - argType = opt.GetItemType(); - isOptional = true; - } + auto argType = argsTuple.GetElementType(0); + TVector<const TType*> argBlockTypes; + argBlockTypes.push_back(argType); - TDataTypeInspector data(*typeInfoHelper, argType); - if (!data) { - builder.SetError("Expected data type"); - return true; - } + TBlockTypeInspector block(*typeInfoHelper, argType); + if (block) { + Y_ENSURE(!block.IsScalar()); + argType = block.GetItemType(); + } - auto typeId = data.GetTypeId(); - if (!(typeId == TDataType<TDate>::Id || typeId == TDataType<TTzDate>::Id || - typeId == TDataType<TDatetime>::Id || typeId == TDataType<TTzDatetime>::Id || - typeId == TDataType<TTimestamp>::Id || typeId == TDataType<TTzTimestamp>::Id || - typeId == TDataType<TInterval>::Id)) { - builder.SetError(TStringBuilder() << "Type " << GetDataTypeInfo(GetDataSlot(typeId)).Name << " is not supported"); - } + bool isOptional = false; + if (auto opt = TOptionalTypeInspector(*typeInfoHelper, argType)) { + argType = opt.GetItemType(); + isOptional = true; + } - builder.Args()->Add(argsTuple.GetElementType(0)).Done(); - const TType* retType; - if (typeId != TDataType<TInterval>::Id) { - retType = builder.SimpleType<TResult>(); - } else { - retType = builder.SimpleType<TSignedResult>(); - } - if (isOptional) { - retType = builder.Optional()->Item(retType).Build(); - } + TDataTypeInspector data(*typeInfoHelper, argType); + if (!data) { + builder.SetError("Data type expected"); + return true; + } - auto outputType = retType; - if (block) { - retType = builder.Block(block.IsScalar())->Item(retType).Build(); - } + const auto typeId = data.GetTypeId(); + const auto features = NUdf::GetDataTypeInfo(NUdf::GetDataSlot(typeId)).Features; + + if (!(features & (NUdf::DateType | NUdf::TzDateType | NUdf::TimeIntervalType))) { + builder.SetError(TStringBuilder() + << "Type " + << GetDataTypeInfo(GetDataSlot(typeId)).Name + << " is not supported"); + return true; + } + + builder.Args()->Add(argsTuple.GetElementType(0)); + const TType* retType; + if (features & NUdf::BigDateType) { + retType = builder.SimpleType<TWResult>(); + } else if (features & NUdf::TimeIntervalType) { + retType = builder.SimpleType<TSignedResult>(); + } else { + retType = builder.SimpleType<TResult>(); + } + + if (isOptional) { + retType = builder.Optional()->Item(retType).Build(); + } - builder.Returns(retType); + auto outputType = retType; + if (block) { + retType = builder.Block(block.IsScalar())->Item(retType).Build(); + } + + builder.Returns(retType); + if (!(features & NUdf::BigDateType)) { + // FIXME: Only non-wide overloads support block rewrite now. builder.SupportsBlocks(); - builder.IsStrict(); + } + builder.IsStrict(); - builder.UserType(userType); - if (!typesOnly) { - if (typeId == TDataType<TDate>::Id || typeId == TDataType<TTzDate>::Id) { - if (block) { - const auto exec = (typeId == TDataType<TTzDate>::Id) - ? MakeTzBlockExec<TTzDate, TResult>() - : UnaryPreallocatedExecImpl<ui16, TResult, DateCore>; - - builder.Implementation(new TSimpleArrowUdfImpl(argBlockTypes, outputType, block.IsScalar(), - exec, builder, TString(name), arrow::compute::NullHandling::INTERSECTION)); - } else { - builder.Implementation(new TUnaryOverOptionalImpl<ui16, TResult, DateCore>()); - } + if (!typesOnly) { + if (typeId == TDataType<TDate>::Id || typeId == TDataType<TTzDate>::Id) { + if (block) { + const auto exec = (typeId == TDataType<TTzDate>::Id) + ? MakeTzBlockExec<TTzDate, TResult>() + : UnaryPreallocatedExecImpl<ui16, TResult, DateCore>; + + builder.Implementation(new TSimpleArrowUdfImpl(argBlockTypes, outputType, block.IsScalar(), + exec, builder, TString(name), arrow::compute::NullHandling::INTERSECTION)); + } else { + builder.Implementation(new TUnaryOverOptionalImpl<ui16, TResult, DateCore>()); } + return true; + } - if (typeId == TDataType<TDatetime>::Id || typeId == TDataType<TTzDatetime>::Id) { - if (block) { - const auto exec = (typeId == TDataType<TTzDatetime>::Id) - ? MakeTzBlockExec<TTzDatetime, TResult>() - : UnaryPreallocatedExecImpl<ui32, TResult, DatetimeCore>; + if (typeId == TDataType<TDatetime>::Id || typeId == TDataType<TTzDatetime>::Id) { + if (block) { + const auto exec = (typeId == TDataType<TTzDatetime>::Id) + ? MakeTzBlockExec<TTzDatetime, TResult>() + : UnaryPreallocatedExecImpl<ui32, TResult, DatetimeCore>; - builder.Implementation(new TSimpleArrowUdfImpl(argBlockTypes, outputType, block.IsScalar(), - exec, builder, TString(name), arrow::compute::NullHandling::INTERSECTION)); - } else { - builder.Implementation(new TUnaryOverOptionalImpl<ui32, TResult, DatetimeCore>()); - } + builder.Implementation(new TSimpleArrowUdfImpl(argBlockTypes, outputType, block.IsScalar(), + exec, builder, TString(name), arrow::compute::NullHandling::INTERSECTION)); + } else { + builder.Implementation(new TUnaryOverOptionalImpl<ui32, TResult, DatetimeCore>()); } + return true; + } - if (typeId == TDataType<TTimestamp>::Id || typeId == TDataType<TTzTimestamp>::Id) { - if (block) { - const auto exec = (typeId == TDataType<TTzTimestamp>::Id) - ? MakeTzBlockExec<TTzTimestamp, TResult>() - : UnaryPreallocatedExecImpl<ui64, TResult, TimestampCore>; + if (typeId == TDataType<TTimestamp>::Id || typeId == TDataType<TTzTimestamp>::Id) { + if (block) { + const auto exec = (typeId == TDataType<TTzTimestamp>::Id) + ? MakeTzBlockExec<TTzTimestamp, TResult>() + : UnaryPreallocatedExecImpl<ui64, TResult, TimestampCore>; - builder.Implementation(new TSimpleArrowUdfImpl(argBlockTypes, outputType, block.IsScalar(), - exec, builder, TString(name), arrow::compute::NullHandling::INTERSECTION)); - } else { - builder.Implementation(new TUnaryOverOptionalImpl<ui64, TResult, TimestampCore>()); - } + builder.Implementation(new TSimpleArrowUdfImpl(argBlockTypes, outputType, block.IsScalar(), + exec, builder, TString(name), arrow::compute::NullHandling::INTERSECTION)); + } else { + builder.Implementation(new TUnaryOverOptionalImpl<ui64, TResult, TimestampCore>()); } + return true; + } - if (typeId == TDataType<TInterval>::Id) { - if (block) { - builder.Implementation(new TSimpleArrowUdfImpl(argBlockTypes, outputType, block.IsScalar(), - UnaryPreallocatedExecImpl<i64, TSignedResult, IntervalCore>, builder, TString(name), arrow::compute::NullHandling::INTERSECTION)); - } else { - builder.Implementation(new TUnaryOverOptionalImpl<i64, TSignedResult, IntervalCore>()); - } + if (typeId == TDataType<TInterval>::Id) { + if (block) { + builder.Implementation(new TSimpleArrowUdfImpl(argBlockTypes, outputType, block.IsScalar(), + UnaryPreallocatedExecImpl<i64, TSignedResult, IntervalCore>, builder, TString(name), arrow::compute::NullHandling::INTERSECTION)); + } else { + builder.Implementation(new TUnaryOverOptionalImpl<i64, TSignedResult, IntervalCore>()); } + return true; } - } catch (const std::exception& e) { - builder.SetError(TStringBuf(e.what())); - } + if (typeId == TDataType<TDate32>::Id || typeId == TDataType<TTzDate32>::Id) { + Y_ABORT_IF(block, "Block overload is not supported"); + builder.Implementation(new TUnaryOverOptionalImpl<i32, TWResult, Date32Core>()); + return true; + } + + if (typeId == TDataType<TDatetime64>::Id || typeId == TDataType<TTzDatetime64>::Id) { + Y_ABORT_IF(block, "Block overload is not supported"); + builder.Implementation(new TUnaryOverOptionalImpl<i64, TWResult, Datetime64Core>()); + return true; + } + + if (typeId == TDataType<TTimestamp64>::Id || typeId == TDataType<TTzTimestamp64>::Id) { + Y_ABORT_IF(block, "Block overload is not supported"); + builder.Implementation(new TUnaryOverOptionalImpl<i64, TWResult, Timestamp64Core>()); + return true; + } + + if (typeId == TDataType<TInterval64>::Id) { + Y_ABORT_IF(block, "Block overload is not supported"); + builder.Implementation(new TUnaryOverOptionalImpl<i64, TWResult, Interval64Core>()); + return true; + } + + Y_UNREACHABLE(); + } return true; } }; @@ -1770,29 +1830,106 @@ TUnboxedValue GetTimezoneName(const IValueBuilder* valueBuilder, const TUnboxedV // To* - BEGIN_SIMPLE_STRICT_ARROW_UDF(TToDays, i32(TAutoMap<TInterval>)) { - Y_UNUSED(valueBuilder); - return TUnboxedValuePod(i32(args[0].Get<i64>() / UsecondsInDay)); +template<const char* TUdfName, typename TResult, typename TWResult, i64 ScaleSeconds> +class TToConverter : public TBoxedValue { +public: + typedef bool TTypeAwareMarker; + static const ::NYql::NUdf::TStringRef& Name() { + static auto name = TStringRef(TUdfName, std::strlen(TUdfName)); + return name; } - END_SIMPLE_ARROW_UDF_WITH_NULL_HANDLING(TToDays, - (UnaryPreallocatedExecImpl<i64, i32, [] (i64 arg) { return i32(arg / UsecondsInDay); }>), - arrow::compute::NullHandling::INTERSECTION); - BEGIN_SIMPLE_STRICT_ARROW_UDF(TToHours, i32(TAutoMap<TInterval>)) { - Y_UNUSED(valueBuilder); - return TUnboxedValuePod(i32(args[0].Get<i64>() / UsecondsInHour)); + static bool DeclareSignature( + const TStringRef& name, + TType* userType, + IFunctionTypeInfoBuilder& builder, + bool typesOnly) + { + if (Name() != name) { + return false; + } + + if (!userType) { + builder.SetError("User type is missing"); + return true; + } + + builder.UserType(userType); + + const auto typeInfoHelper = builder.TypeInfoHelper(); + TTupleTypeInspector tuple(*typeInfoHelper, userType); + Y_ENSURE(tuple, "Tuple with args and options tuples expected"); + Y_ENSURE(tuple.GetElementsCount() > 0, + "Tuple has to contain positional arguments"); + + TTupleTypeInspector argsTuple(*typeInfoHelper, tuple.GetElementType(0)); + Y_ENSURE(argsTuple, "Tuple with args expected"); + if (argsTuple.GetElementsCount() != 1) { + builder.SetError("Single argument expected"); + return true; + } + + auto argType = argsTuple.GetElementType(0); + + if (const auto optType = TOptionalTypeInspector(*typeInfoHelper, argType)) { + argType = optType.GetItemType(); + } + + TDataTypeInspector data(*typeInfoHelper, argType); + if (!data) { + SetInvalidTypeError(builder, typeInfoHelper, argType); + return true; + } + + const auto features = NUdf::GetDataTypeInfo(NUdf::GetDataSlot(data.GetTypeId())).Features; + if (features & NUdf::TimeIntervalType) { + if (features & NUdf::BigDateType) { + BuildSignature<TInterval64, TWResult>(builder, typesOnly); + } else { + BuildSignature<TInterval, TResult>(builder, typesOnly); + } + return true; + } + SetInvalidTypeError(builder, typeInfoHelper, argType); + return true; + } +private: + class TImpl : public TBoxedValue { + public: + TUnboxedValue Run(const IValueBuilder* valueBuilder, const TUnboxedValuePod* args) const final { + try { + Y_UNUSED(valueBuilder); + return TUnboxedValuePod(i64(args[0].Get<i64>() / ScaleSeconds)); + } catch (const std::exception& e) { + TStringBuilder sb; + sb << CurrentExceptionMessage(); + sb << Endl << "[" << TStringBuf(Name()) << "]" ; + UdfTerminate(sb.c_str()); + } + } + + }; + + static void SetInvalidTypeError(NUdf::IFunctionTypeInfoBuilder& builder, + ITypeInfoHelper::TPtr typeInfoHelper, const TType* argType) + { + ::TStringBuilder sb; + sb << "Invalid argument type: got "; + TTypePrinter(*typeInfoHelper, argType).Out(sb.Out); + sb << ", but Interval or Interval64 expected"; + builder.SetError(sb); } - END_SIMPLE_ARROW_UDF_WITH_NULL_HANDLING(TToHours, - (UnaryPreallocatedExecImpl<i64, i32, [] (i64 arg) { return i32(arg / UsecondsInHour); }>), - arrow::compute::NullHandling::INTERSECTION); - BEGIN_SIMPLE_STRICT_ARROW_UDF(TToMinutes, i32(TAutoMap<TInterval>)) { - Y_UNUSED(valueBuilder); - return TUnboxedValuePod(i32(args[0].Get<i64>() / UsecondsInMinute)); + template<typename TInput, typename TOutput> + static void BuildSignature(NUdf::IFunctionTypeInfoBuilder& builder, bool typesOnly) { + builder.Returns<TOutput>(); + builder.Args()->Add<TAutoMap<TInput>>(); + builder.IsStrict(); + if (!typesOnly) { + builder.Implementation(new TImpl()); + } } - END_SIMPLE_ARROW_UDF_WITH_NULL_HANDLING(TToMinutes, - (UnaryPreallocatedExecImpl<i64, i32, [] (i64 arg) { return i32(arg / UsecondsInMinute); }>), - arrow::compute::NullHandling::INTERSECTION); +}; // StartOf* @@ -3298,9 +3435,9 @@ private: TInterval64FromMilliseconds, TInterval64FromMicroseconds, - TToDays, - TToHours, - TToMinutes, + TToConverter<ToDaysUDF, i32, i32, UsecondsInDay>, + TToConverter<ToHoursUDF, i32, i64, UsecondsInHour>, + TToConverter<ToMinutesUDF, i32, i64, UsecondsInMinute>, TBoundaryOf<StartOfYearUDF, SimpleDatetimeToDatetimeUdf<TMResourceName, StartOfYear<TTMStorage>>, SimpleDatetimeToDatetimeUdf<TM64ResourceName, StartOfYear<TTM64Storage>>>, @@ -3333,9 +3470,9 @@ private: TBoundaryOfInterval<EndOfUDF, SimpleDatetimeToIntervalUdf<TMResourceName, EndOf<TTMStorage>>, SimpleDatetimeToIntervalUdf<TM64ResourceName, EndOf<TTM64Storage>>>, - TToUnits<ToSecondsUDF, ui32, 1>, - TToUnits<ToMillisecondsUDF, ui64, 1000>, - TToUnits<ToMicrosecondsUDF, ui64, 1000000>, + TToUnits<ToSecondsUDF, ui32, i64, 1>, + TToUnits<ToMillisecondsUDF, ui64, i64, 1000>, + TToUnits<ToMicrosecondsUDF, ui64, i64, 1000000>, TFormat, TParse<ParseUDF, TMResourceName>, diff --git a/yql/essentials/udfs/common/datetime2/test_bigdates/canondata/result.json b/yql/essentials/udfs/common/datetime2/test_bigdates/canondata/result.json index 79e742fc9f..c88d3bdf2a 100644 --- a/yql/essentials/udfs/common/datetime2/test_bigdates/canondata/result.json +++ b/yql/essentials/udfs/common/datetime2/test_bigdates/canondata/result.json @@ -44,6 +44,11 @@ "uri": "file://test.test_StartOf_/results.txt" } ], + "test.test[To]": [ + { + "uri": "file://test.test_To_/results.txt" + } + ], "test.test[UpdateTz]": [ { "uri": "file://test.test_UpdateTz_/results.txt" diff --git a/yql/essentials/udfs/common/datetime2/test_bigdates/canondata/test.test_To_/results.txt b/yql/essentials/udfs/common/datetime2/test_bigdates/canondata/test.test_To_/results.txt new file mode 100644 index 0000000000..2c6a80c75a --- /dev/null +++ b/yql/essentials/udfs/common/datetime2/test_bigdates/canondata/test.test_To_/results.txt @@ -0,0 +1,826 @@ +[ + { + "Write" = [ + { + "Type" = [ + "ListType"; + [ + "StructType"; + [ + [ + "interval64_to_days"; + [ + "OptionalType"; + [ + "DataType"; + "Int32" + ] + ] + ]; + [ + "interval64_to_hours"; + [ + "OptionalType"; + [ + "DataType"; + "Int64" + ] + ] + ]; + [ + "interval64_to_minutes"; + [ + "OptionalType"; + [ + "DataType"; + "Int64" + ] + ] + ]; + [ + "interval64_to_seconds"; + [ + "OptionalType"; + [ + "DataType"; + "Int64" + ] + ] + ]; + [ + "interval64_to_msec"; + [ + "OptionalType"; + [ + "DataType"; + "Int64" + ] + ] + ]; + [ + "interval64_to_usec"; + [ + "OptionalType"; + [ + "DataType"; + "Int64" + ] + ] + ]; + [ + "date32_to_seconds"; + [ + "OptionalType"; + [ + "DataType"; + "Int64" + ] + ] + ]; + [ + "datetime64_to_seconds"; + [ + "OptionalType"; + [ + "DataType"; + "Int64" + ] + ] + ]; + [ + "timestamp64_to_seconds"; + [ + "OptionalType"; + [ + "DataType"; + "Int64" + ] + ] + ]; + [ + "tzdate32_to_seconds"; + [ + "OptionalType"; + [ + "DataType"; + "Int64" + ] + ] + ]; + [ + "tzdatetime64_to_seconds"; + [ + "OptionalType"; + [ + "DataType"; + "Int64" + ] + ] + ]; + [ + "tztimestamp64_to_seconds"; + [ + "OptionalType"; + [ + "DataType"; + "Int64" + ] + ] + ]; + [ + "date32_to_msec"; + [ + "OptionalType"; + [ + "DataType"; + "Int64" + ] + ] + ]; + [ + "datetime64_to_msec"; + [ + "OptionalType"; + [ + "DataType"; + "Int64" + ] + ] + ]; + [ + "timestamp64_to_msec"; + [ + "OptionalType"; + [ + "DataType"; + "Int64" + ] + ] + ]; + [ + "tzdate32_to_msec"; + [ + "OptionalType"; + [ + "DataType"; + "Int64" + ] + ] + ]; + [ + "tzdatetime64_to_msec"; + [ + "OptionalType"; + [ + "DataType"; + "Int64" + ] + ] + ]; + [ + "tztimestamp64_to_msec"; + [ + "OptionalType"; + [ + "DataType"; + "Int64" + ] + ] + ]; + [ + "date32_to_usec"; + [ + "OptionalType"; + [ + "DataType"; + "Int64" + ] + ] + ]; + [ + "datetime64_to_usec"; + [ + "OptionalType"; + [ + "DataType"; + "Int64" + ] + ] + ]; + [ + "timestamp64_to_usec"; + [ + "OptionalType"; + [ + "DataType"; + "Int64" + ] + ] + ]; + [ + "tzdate32_to_usec"; + [ + "OptionalType"; + [ + "DataType"; + "Int64" + ] + ] + ]; + [ + "tzdatetime64_to_usec"; + [ + "OptionalType"; + [ + "DataType"; + "Int64" + ] + ] + ]; + [ + "tztimestamp64_to_usec"; + [ + "OptionalType"; + [ + "DataType"; + "Int64" + ] + ] + ] + ] + ] + ]; + "Data" = [ + [ + [ + "-106751616" + ]; + [ + "-2562038807" + ]; + [ + "-153722328479" + ]; + [ + "-9223339708799" + ]; + [ + "-9223339708799000" + ]; + [ + "-9223339708799000000" + ]; + [ + "-4611669897600" + ]; + [ + "-4611669897600" + ]; + [ + "-4611669897600" + ]; + [ + "-4611669897600" + ]; + [ + "-4611669897600" + ]; + [ + "-4611669897600" + ]; + [ + "-4611669897600000" + ]; + [ + "-4611669897600000" + ]; + [ + "-4611669897600000" + ]; + [ + "-4611669897600000" + ]; + [ + "-4611669897600000" + ]; + [ + "-4611669897600000" + ]; + [ + "-4611669897600000000" + ]; + [ + "-4611669897600000000" + ]; + [ + "-4611669897600000000" + ]; + [ + "-4611669897600000000" + ]; + [ + "-4611669897600000000" + ]; + [ + "-4611669897600000000" + ] + ]; + [ + [ + "-53375809" + ]; + [ + "-1281019416" + ]; + [ + "-76861164960" + ]; + [ + "-4611669897600" + ]; + [ + "-4611669897600000" + ]; + [ + "-4611669897600000000" + ]; + [ + "-62167219200" + ]; + [ + "-62167132801" + ]; + [ + "-62167132800" + ]; + [ + "-62167305600" + ]; + [ + "-62167141818" + ]; + [ + "-62167141817" + ]; + [ + "-62167219200000" + ]; + [ + "-62167132801000" + ]; + [ + "-62167132800000" + ]; + [ + "-62167305600000" + ]; + [ + "-62167141818000" + ]; + [ + "-62167141817000" + ]; + [ + "-62167219200000000" + ]; + [ + "-62167132801000000" + ]; + [ + "-62167132800000001" + ]; + [ + "-62167305600000000" + ]; + [ + "-62167141818000000" + ]; + [ + "-62167141817000001" + ] + ]; + [ + [ + "-1" + ]; + [ + "-24" + ]; + [ + "-1440" + ]; + [ + "-86400" + ]; + [ + "-86400000" + ]; + [ + "-86400000000" + ]; + [ + "-62135596800" + ]; + [ + "-62135596800" + ]; + [ + "-62135596800" + ]; + [ + "-62135683200" + ]; + [ + "-62135605817" + ]; + [ + "-62135605817" + ]; + [ + "-62135596800000" + ]; + [ + "-62135596800000" + ]; + [ + "-62135596800000" + ]; + [ + "-62135683200000" + ]; + [ + "-62135605817000" + ]; + [ + "-62135605817000" + ]; + [ + "-62135596800000000" + ]; + [ + "-62135596800000000" + ]; + [ + "-62135596800000000" + ]; + [ + "-62135683200000000" + ]; + [ + "-62135605817000000" + ]; + [ + "-62135605817000000" + ] + ]; + [ + #; + #; + #; + #; + #; + #; + [ + "-86400" + ]; + [ + "-1" + ]; + [ + "0" + ]; + [ + "-172800" + ]; + [ + "-10801" + ]; + [ + "-10800" + ]; + [ + "-86400000" + ]; + [ + "-1000" + ]; + [ + "0" + ]; + [ + "-172800000" + ]; + [ + "-10801000" + ]; + [ + "-10800000" + ]; + [ + "-86400000000" + ]; + [ + "-1000000" + ]; + [ + "-1" + ]; + [ + "-172800000000" + ]; + [ + "-10801000000" + ]; + [ + "-10800000001" + ] + ]; + [ + [ + "1" + ]; + [ + "24" + ]; + [ + "1440" + ]; + [ + "86400" + ]; + [ + "86400000" + ]; + [ + "86400000000" + ]; + [ + "0" + ]; + [ + "0" + ]; + [ + "0" + ]; + [ + "-86400" + ]; + [ + "-10800" + ]; + [ + "-10800" + ]; + [ + "0" + ]; + [ + "0" + ]; + [ + "0" + ]; + [ + "-86400000" + ]; + [ + "-10800000" + ]; + [ + "-10800000" + ]; + [ + "0" + ]; + [ + "0" + ]; + [ + "0" + ]; + [ + "-86400000000" + ]; + [ + "-10800000000" + ]; + [ + "-10800000000" + ] + ]; + [ + #; + #; + #; + #; + #; + #; + [ + "1735689600" + ]; + [ + "1735693323" + ]; + [ + "1735693323" + ]; + [ + "1735603200" + ]; + [ + "1735693323" + ]; + [ + "1735693323" + ]; + [ + "1735689600000" + ]; + [ + "1735693323000" + ]; + [ + "1735693323456" + ]; + [ + "1735603200000" + ]; + [ + "1735693323000" + ]; + [ + "1735693323456" + ]; + [ + "1735689600000000" + ]; + [ + "1735693323000000" + ]; + [ + "1735693323456789" + ]; + [ + "1735603200000000" + ]; + [ + "1735693323000000" + ]; + [ + "1735693323456789" + ] + ]; + [ + [ + "53375807" + ]; + [ + "1281019391" + ]; + [ + "76861163519" + ]; + [ + "4611669811199" + ]; + [ + "4611669811199000" + ]; + [ + "4611669811199000000" + ]; + [ + "4291747200" + ]; + [ + "4291747200" + ]; + [ + "4291747200" + ]; + [ + "4291660800" + ]; + [ + "4291747200" + ]; + [ + "4291747200" + ]; + [ + "4291747200000" + ]; + [ + "4291747200000" + ]; + [ + "4291747200000" + ]; + [ + "4291660800000" + ]; + [ + "4291747200000" + ]; + [ + "4291747200000" + ]; + [ + "4291747200000000" + ]; + [ + "4291747200000000" + ]; + [ + "4291747200000000" + ]; + [ + "4291660800000000" + ]; + [ + "4291747200000000" + ]; + [ + "4291747200000000" + ] + ]; + [ + [ + "106751616" + ]; + [ + "2562038807" + ]; + [ + "153722328479" + ]; + [ + "9223339708799" + ]; + [ + "9223339708799000" + ]; + [ + "9223339708799000000" + ]; + [ + "4611669724800" + ]; + [ + "4611669811199" + ]; + [ + "4611669811199" + ]; + [ + "4611669638400" + ]; + [ + "4611669811199" + ]; + [ + "4611669811199" + ]; + [ + "4611669724800000" + ]; + [ + "4611669811199000" + ]; + [ + "4611669811199999" + ]; + [ + "4611669638400000" + ]; + [ + "4611669811199000" + ]; + [ + "4611669811199999" + ]; + [ + "4611669724800000000" + ]; + [ + "4611669811199000000" + ]; + [ + "4611669811199999999" + ]; + [ + "4611669638400000000" + ]; + [ + "4611669811199000000" + ]; + [ + "4611669811199999999" + ] + ] + ] + } + ] + } +]
\ No newline at end of file diff --git a/yql/essentials/udfs/common/datetime2/test_bigdates/cases/To.cfg b/yql/essentials/udfs/common/datetime2/test_bigdates/cases/To.cfg new file mode 100644 index 0000000000..31260d619b --- /dev/null +++ b/yql/essentials/udfs/common/datetime2/test_bigdates/cases/To.cfg @@ -0,0 +1 @@ +in plato.Input To.in diff --git a/yql/essentials/udfs/common/datetime2/test_bigdates/cases/To.in b/yql/essentials/udfs/common/datetime2/test_bigdates/cases/To.in new file mode 100644 index 0000000000..1097a51534 --- /dev/null +++ b/yql/essentials/udfs/common/datetime2/test_bigdates/cases/To.in @@ -0,0 +1,72 @@ +{ + "fdate32"="-144169-01-01"; + "fdatetime64"="-144169-01-01T00:00:00Z"; + "ftimestamp64"="-144169-01-01T00:00:00.000000Z"; + "ftzdate32"="-144169-01-02,Europe/Moscow"; + "ftzdatetime64"="-144169-01-01T02:30:17,Europe/Moscow"; + "ftztimestamp64"="-144169-01-01T02:30:17.000000,Europe/Moscow"; + "finterval64"="-PT9223339708799S"; +}; +{ + "fdate32"="-1-01-01"; + "fdatetime64"="-1-01-01T23:59:59Z"; + "ftimestamp64"="-1-01-01T23:59:59.999999Z"; + "ftzdate32"="-1-01-01,Europe/Moscow"; + "ftzdatetime64"="-1-01-01T23:59:59,Europe/Moscow"; + "ftztimestamp64"="-1-01-01T23:59:59.999999,Europe/Moscow"; + "finterval64"="-PT4611669897600S"; +}; +{ + "fdate32"="1-01-01"; + "fdatetime64"="1-01-01T00:00:00Z"; + "ftimestamp64"="1-01-01T00:00:00.000000Z"; + "ftzdate32"="1-01-01,Europe/Moscow"; + "ftzdatetime64"="1-01-01T00:00:00,Europe/Moscow"; + "ftztimestamp64"="1-01-01T00:00:00.000000,Europe/Moscow"; + "finterval64"="-P1D"; +}; +{ + "fdate32"="1969-12-31"; + "fdatetime64"="1969-12-31T23:59:59Z"; + "ftimestamp64"="1969-12-31T23:59:59.999999Z"; + "ftzdate32"="1969-12-31,Europe/Moscow"; + "ftzdatetime64"="1969-12-31T23:59:59,Europe/Moscow"; + "ftztimestamp64"="1969-12-31T23:59:59.999999,Europe/Moscow"; + "finterval64"="P1W1DT1H1M1.000001S"; +}; +{ + "fdate32"="1970-01-01"; + "fdatetime64"="1970-01-01T00:00:00Z"; + "ftimestamp64"="1970-01-01T00:00:00.000000Z"; + "ftzdate32"="1970-01-01,Europe/Moscow"; + "ftzdatetime64"="1970-01-01T00:00:00,Europe/Moscow"; + "ftztimestamp64"="1970-01-01T00:00:00.000000,Europe/Moscow"; + "finterval64"="P1D"; +}; +{ + "fdate32"="2025-01-01"; + "fdatetime64"="2025-01-01T01:02:03Z"; + "ftimestamp64"="2025-01-01T01:02:03.456789Z"; + "ftzdate32"="2025-01-01,Europe/Moscow"; + "ftzdatetime64"="2025-01-01T04:02:03,Europe/Moscow"; + "ftztimestamp64"="2025-01-01T04:02:03.456789,Europe/Moscow"; + "finterval64"="P1W2DT3H4M5.678912S"; +}; +{ + "fdate32"="2106-01-01"; + "fdatetime64"="2106-01-01T00:00:00Z"; + "ftimestamp64"="2106-01-01T00:00:00.000000Z"; + "ftzdate32"="2106-01-01,Europe/Moscow"; + "ftzdatetime64"="2106-01-01T03:00:00,Europe/Moscow"; + "ftztimestamp64"="2106-01-01T03:00:00.000000,Europe/Moscow"; + "finterval64"="PT4611669811199S"; +}; +{ + "fdate32"="148107-12-31"; + "fdatetime64"="148107-12-31T23:59:59Z"; + "ftimestamp64"="148107-12-31T23:59:59.999999Z"; + "ftzdate32"="148107-12-31,Europe/Moscow"; + "ftzdatetime64"="148108-01-01T02:59:59,Europe/Moscow"; + "ftztimestamp64"="148108-01-01T02:59:59.999999,Europe/Moscow"; + "finterval64"="PT9223339708799S"; +}; diff --git a/yql/essentials/udfs/common/datetime2/test_bigdates/cases/To.in.attr b/yql/essentials/udfs/common/datetime2/test_bigdates/cases/To.in.attr new file mode 100644 index 0000000000..53b63c62f6 --- /dev/null +++ b/yql/essentials/udfs/common/datetime2/test_bigdates/cases/To.in.attr @@ -0,0 +1,59 @@ +{ + "_yql_row_spec" = { + "Type" = [ + "StructType"; + [ + [ + "fdate32"; + [ + "DataType"; + "String" + ] + ]; + [ + "fdatetime64"; + [ + "DataType"; + "String" + ] + ]; + [ + "ftimestamp64"; + [ + "DataType"; + "String" + ] + ]; + [ + "ftzdate32"; + [ + "DataType"; + "String" + ] + ]; + [ + "ftzdatetime64"; + [ + "DataType"; + "String" + ] + ]; + [ + "ftztimestamp64"; + [ + "DataType"; + "String" + ] + ]; + [ + "finterval64"; + [ + "DataType"; + "String" + ] + ]; + ] + ] + } +} + diff --git a/yql/essentials/udfs/common/datetime2/test_bigdates/cases/To.sql b/yql/essentials/udfs/common/datetime2/test_bigdates/cases/To.sql new file mode 100644 index 0000000000..07c15913d3 --- /dev/null +++ b/yql/essentials/udfs/common/datetime2/test_bigdates/cases/To.sql @@ -0,0 +1,40 @@ +/* syntax version 1 */ +SELECT + DateTime::ToDays(finterval64) as interval64_to_days, + DateTime::ToHours(finterval64) as interval64_to_hours, + DateTime::ToMinutes(finterval64) as interval64_to_minutes, + DateTime::ToSeconds(finterval64) as interval64_to_seconds, + DateTime::ToMilliseconds(finterval64) as interval64_to_msec, + DateTime::ToMicroseconds(finterval64) as interval64_to_usec, + + DateTime::ToSeconds(fdate32) as date32_to_seconds, + DateTime::ToSeconds(fdatetime64) as datetime64_to_seconds, + DateTime::ToSeconds(ftimestamp64) as timestamp64_to_seconds, + DateTime::ToSeconds(ftzdate32) as tzdate32_to_seconds, + DateTime::ToSeconds(ftzdatetime64) as tzdatetime64_to_seconds, + DateTime::ToSeconds(ftztimestamp64) as tztimestamp64_to_seconds, + + DateTime::ToMilliseconds(fdate32) as date32_to_msec, + DateTime::ToMilliseconds(fdatetime64) as datetime64_to_msec, + DateTime::ToMilliseconds(ftimestamp64) as timestamp64_to_msec, + DateTime::ToMilliseconds(ftzdate32) as tzdate32_to_msec, + DateTime::ToMilliseconds(ftzdatetime64) as tzdatetime64_to_msec, + DateTime::ToMilliseconds(ftztimestamp64) as tztimestamp64_to_msec, + + DateTime::ToMicroseconds(fdate32) as date32_to_usec, + DateTime::ToMicroseconds(fdatetime64) as datetime64_to_usec, + DateTime::ToMicroseconds(ftimestamp64) as timestamp64_to_usec, + DateTime::ToMicroseconds(ftzdate32) as tzdate32_to_usec, + DateTime::ToMicroseconds(ftzdatetime64) as tzdatetime64_to_usec, + DateTime::ToMicroseconds(ftztimestamp64) as tztimestamp64_to_usec, +FROM ( + SELECT + CAST(fdate32 as Date32) as fdate32, + CAST(fdatetime64 as Datetime64) as fdatetime64, + CAST(ftimestamp64 as Timestamp64) as ftimestamp64, + CAST(finterval64 as Interval64) as finterval64, + CAST(ftzdate32 as TzDate32) as ftzdate32, + CAST(ftzdatetime64 as TzDatetime64) as ftzdatetime64, + CAST(ftztimestamp64 as TzTimestamp64) as ftztimestamp64, + from Input +); diff --git a/yql/essentials/udfs/common/python/bindings/py_callable_ut.cpp b/yql/essentials/udfs/common/python/bindings/py_callable_ut.cpp index 1c58d7b371..36cc13a1da 100644 --- a/yql/essentials/udfs/common/python/bindings/py_callable_ut.cpp +++ b/yql/essentials/udfs/common/python/bindings/py_callable_ut.cpp @@ -1,4 +1,4 @@ -#include "ut3/py_test_engine.h" +#include "py_test_engine.h" #include <library/cpp/testing/unittest/registar.h> diff --git a/yql/essentials/udfs/common/python/bindings/py_cast.cpp b/yql/essentials/udfs/common/python/bindings/py_cast.cpp index 078239962e..ac44ac42e5 100644 --- a/yql/essentials/udfs/common/python/bindings/py_cast.cpp +++ b/yql/essentials/udfs/common/python/bindings/py_cast.cpp @@ -22,6 +22,7 @@ #include <library/cpp/containers/stack_vector/stack_vec.h> +#include <util/generic/scope.h> #include <util/string/join.h> #include <util/string/builder.h> @@ -252,7 +253,7 @@ template <> \ bool TryPyCast(PyObject* value, Type& result) { \ if (PyUnicode_Check(value)) { \ - const TPyObjectPtr utf8(PyUnicode_AsUTF8String(value)); \ + const TPyObjectPtr utf8(AsUtf8StringOrThrow(value)); \ char* str = nullptr; \ Py_ssize_t size = 0; \ int rc = PyBytes_AsStringAndSize(utf8.Get(), &str, &size); \ @@ -279,6 +280,22 @@ namespace NPython { using namespace NKikimr; +namespace { + +NPython::TPyObjectPtr AsUtf8StringOrThrow(PyObject* obj) { + auto* utf8String = PyUnicode_AsUTF8String(obj); + if (!utf8String) { + Y_ENSURE(PyErr_Occurred()); + Y_DEFER { + PyErr_Clear(); + }; + throw yexception() << "Failed to convert the string to UTF-8 format. Original message is:\n" << GetLastErrorAsString() << "\n"; + } + return NPython::TPyObjectPtr(utf8String); +} + +} // namespace + inline void ThrowCastTypeException(PyObject* value, TStringBuf toType) { throw yexception() << "Can't cast object '" << Py_TYPE(value)->tp_name << "' to " << toType << "; Object repr: " << PyObjectRepr(value); @@ -548,7 +565,7 @@ NUdf::TUnboxedValue FromPyData( case NUdf::TDataType<NUdf::TUtf8>::Id: case NUdf::TDataType<NUdf::TJson>::Id: if (PyUnicode_Check(value)) { - const TPyObjectPtr uif8(PyUnicode_AsUTF8String(value)); + const TPyObjectPtr uif8(AsUtf8StringOrThrow(value)); return ctx->ValueBuilder->NewString(PyCast<NUdf::TStringRef>(uif8.Get())); } throw yexception() << "Python object " << PyObjectRepr(value) << " has invalid value for unicode"; @@ -557,7 +574,7 @@ NUdf::TUnboxedValue FromPyData( case NUdf::TDataType<NUdf::TJson>::Id: case NUdf::TDataType<NUdf::TUtf8>::Id: { if (PyUnicode_Check(value)) { - const TPyObjectPtr utf8(PyUnicode_AsUTF8String(value)); + const TPyObjectPtr utf8(AsUtf8StringOrThrow(value)); return ctx->ValueBuilder->NewString(PyCast<NUdf::TStringRef>(utf8.Get())); } diff --git a/yql/essentials/udfs/common/python/bindings/py_cast_ut.cpp b/yql/essentials/udfs/common/python/bindings/py_cast_ut.cpp index 47f65ab6fa..d4ee90278c 100644 --- a/yql/essentials/udfs/common/python/bindings/py_cast_ut.cpp +++ b/yql/essentials/udfs/common/python/bindings/py_cast_ut.cpp @@ -1,9 +1,39 @@ -#include "ut3/py_test_engine.h" +#include "py_test_engine.h" #include <library/cpp/testing/unittest/registar.h> +#include <util/string/strip.h> using namespace NPython; +namespace { +template <typename TType> +void TestBadUtf8Encode() { +#if PY_MAJOR_VERSION == 2 + // In Python 2, strings can encode single surrogate pairs, so this issue does not occur there. + return; +#endif // PY_MAJOR_VERSION == 2 + + TPythonTestEngine engine; + + constexpr char programToRun[] = R"( +def Test(): + return "\uDC00" +)"; + constexpr char expectedError[] = R"( +Failed to convert the string to UTF-8 format. Original message is: +UnicodeEncodeError: 'utf-8' codec can't encode character '\udc00' in position 0: surrogates not allowed +)"; + + UNIT_ASSERT_EXCEPTION_CONTAINS( + engine.ToMiniKQL<TType>( + StripString(TString(programToRun)), + [](const NUdf::TUnboxedValuePod& value) { + Y_UNUSED(value); + }), + yexception, StripString(TString(expectedError))); +} +} // namespace + Y_UNIT_TEST_SUITE(TPyCastTest) { Y_UNIT_TEST(FromPyStrToInt) { TPythonTestEngine engine; @@ -87,4 +117,11 @@ Y_UNIT_TEST_SUITE(TPyCastTest) { yexception, "Cast error object " RETVAL " to Long"); } -} + Y_UNIT_TEST(BadFromPythonUtf8) { + TestBadUtf8Encode<NUdf::TUtf8>(); + } + + Y_UNIT_TEST(BadFromPythonJson) { + TestBadUtf8Encode<NUdf::TJson>(); + } +} // Y_UNIT_TEST_SUITE(TPyCastTest) diff --git a/yql/essentials/udfs/common/python/bindings/py_decimal_ut.cpp b/yql/essentials/udfs/common/python/bindings/py_decimal_ut.cpp index 8388c110f3..3f0298013a 100644 --- a/yql/essentials/udfs/common/python/bindings/py_decimal_ut.cpp +++ b/yql/essentials/udfs/common/python/bindings/py_decimal_ut.cpp @@ -1,4 +1,4 @@ -#include "ut3/py_test_engine.h" +#include "py_test_engine.h" #include <library/cpp/testing/unittest/registar.h> diff --git a/yql/essentials/udfs/common/python/bindings/py_dict_ut.cpp b/yql/essentials/udfs/common/python/bindings/py_dict_ut.cpp index 9ac9627ebb..edb3d36e8c 100644 --- a/yql/essentials/udfs/common/python/bindings/py_dict_ut.cpp +++ b/yql/essentials/udfs/common/python/bindings/py_dict_ut.cpp @@ -1,4 +1,4 @@ -#include "ut3/py_test_engine.h" +#include "py_test_engine.h" #include <yql/essentials/public/udf/udf_ut_helpers.h> diff --git a/yql/essentials/udfs/common/python/bindings/py_list_ut.cpp b/yql/essentials/udfs/common/python/bindings/py_list_ut.cpp index f16165fc54..b2e9a640d4 100644 --- a/yql/essentials/udfs/common/python/bindings/py_list_ut.cpp +++ b/yql/essentials/udfs/common/python/bindings/py_list_ut.cpp @@ -1,4 +1,4 @@ -#include "ut3/py_test_engine.h" +#include "py_test_engine.h" #include <yql/essentials/public/udf/udf_ut_helpers.h> diff --git a/yql/essentials/udfs/common/python/bindings/py_number_ut.cpp b/yql/essentials/udfs/common/python/bindings/py_number_ut.cpp index c55e25891d..35c94d5e8e 100644 --- a/yql/essentials/udfs/common/python/bindings/py_number_ut.cpp +++ b/yql/essentials/udfs/common/python/bindings/py_number_ut.cpp @@ -1,4 +1,4 @@ -#include "ut3/py_test_engine.h" +#include "py_test_engine.h" #include <library/cpp/testing/unittest/registar.h> diff --git a/yql/essentials/udfs/common/python/bindings/py_optional_ut.cpp b/yql/essentials/udfs/common/python/bindings/py_optional_ut.cpp index d13ea65da6..4cc45f1184 100644 --- a/yql/essentials/udfs/common/python/bindings/py_optional_ut.cpp +++ b/yql/essentials/udfs/common/python/bindings/py_optional_ut.cpp @@ -1,4 +1,4 @@ -#include "ut3/py_test_engine.h" +#include "py_test_engine.h" #include <library/cpp/testing/unittest/registar.h> diff --git a/yql/essentials/udfs/common/python/bindings/py_resource_ut.cpp b/yql/essentials/udfs/common/python/bindings/py_resource_ut.cpp index aaa9899c4f..25b43cbf6a 100644 --- a/yql/essentials/udfs/common/python/bindings/py_resource_ut.cpp +++ b/yql/essentials/udfs/common/python/bindings/py_resource_ut.cpp @@ -1,4 +1,4 @@ -#include "ut3/py_test_engine.h" +#include "py_test_engine.h" #include <library/cpp/testing/unittest/registar.h> diff --git a/yql/essentials/udfs/common/python/bindings/py_stream_ut.cpp b/yql/essentials/udfs/common/python/bindings/py_stream_ut.cpp index 4a24dd1a13..4a36f7b8f3 100644 --- a/yql/essentials/udfs/common/python/bindings/py_stream_ut.cpp +++ b/yql/essentials/udfs/common/python/bindings/py_stream_ut.cpp @@ -1,4 +1,4 @@ -#include "ut3/py_test_engine.h" +#include "py_test_engine.h" #include <library/cpp/testing/unittest/registar.h> diff --git a/yql/essentials/udfs/common/python/bindings/py_string_ut.cpp b/yql/essentials/udfs/common/python/bindings/py_string_ut.cpp index 444b7b0c5b..b1f5a13786 100644 --- a/yql/essentials/udfs/common/python/bindings/py_string_ut.cpp +++ b/yql/essentials/udfs/common/python/bindings/py_string_ut.cpp @@ -1,4 +1,4 @@ -#include "ut3/py_test_engine.h" +#include "py_test_engine.h" #include <library/cpp/testing/unittest/registar.h> diff --git a/yql/essentials/udfs/common/python/bindings/py_struct_ut.cpp b/yql/essentials/udfs/common/python/bindings/py_struct_ut.cpp index a97507f549..9b232d99a3 100644 --- a/yql/essentials/udfs/common/python/bindings/py_struct_ut.cpp +++ b/yql/essentials/udfs/common/python/bindings/py_struct_ut.cpp @@ -1,4 +1,4 @@ -#include "ut3/py_test_engine.h" +#include "py_test_engine.h" #include <library/cpp/testing/unittest/registar.h> diff --git a/yql/essentials/udfs/common/python/bindings/ut3/py_test_engine.h b/yql/essentials/udfs/common/python/bindings/py_test_engine.h index a36e19fa32..c86febf779 100644 --- a/yql/essentials/udfs/common/python/bindings/ut3/py_test_engine.h +++ b/yql/essentials/udfs/common/python/bindings/py_test_engine.h @@ -27,6 +27,8 @@ struct TPyInitializer { PrepareYqlModule(); Py_Initialize(); InitYqlModule(NYql::NUdf::EPythonFlavor::Arcadia); + const auto rc = PyRun_SimpleString(NYql::NUdf::STANDART_STREAM_PROXY_INJECTION_SCRIPT); + Y_ENSURE(rc >= 0, "Can't setup module"); } ~TPyInitializer() { TermYqlModule(); diff --git a/yql/essentials/udfs/common/python/bindings/py_tuple_ut.cpp b/yql/essentials/udfs/common/python/bindings/py_tuple_ut.cpp index a6b9b6cc3e..f465f0ebb6 100644 --- a/yql/essentials/udfs/common/python/bindings/py_tuple_ut.cpp +++ b/yql/essentials/udfs/common/python/bindings/py_tuple_ut.cpp @@ -1,4 +1,4 @@ -#include "ut3/py_test_engine.h" +#include "py_test_engine.h" #include <library/cpp/testing/unittest/registar.h> diff --git a/yql/essentials/udfs/common/python/bindings/py_tzdate_ut.cpp b/yql/essentials/udfs/common/python/bindings/py_tzdate_ut.cpp index e9f5971c78..08b6b78b16 100644 --- a/yql/essentials/udfs/common/python/bindings/py_tzdate_ut.cpp +++ b/yql/essentials/udfs/common/python/bindings/py_tzdate_ut.cpp @@ -1,5 +1,5 @@ #include "py_variant.h" -#include "ut3/py_test_engine.h" +#include "py_test_engine.h" #include <yql/essentials/minikql/mkql_type_ops.h> #include <library/cpp/testing/unittest/registar.h> diff --git a/yql/essentials/udfs/common/python/bindings/py_variant_ut.cpp b/yql/essentials/udfs/common/python/bindings/py_variant_ut.cpp index 77ab9bc6e8..d792449d82 100644 --- a/yql/essentials/udfs/common/python/bindings/py_variant_ut.cpp +++ b/yql/essentials/udfs/common/python/bindings/py_variant_ut.cpp @@ -1,5 +1,5 @@ #include "py_variant.h" -#include "ut3/py_test_engine.h" +#include "py_test_engine.h" #include <library/cpp/testing/unittest/registar.h> diff --git a/yql/essentials/udfs/common/python/bindings/py_void_ut.cpp b/yql/essentials/udfs/common/python/bindings/py_void_ut.cpp index 7fbeca2043..e6e8a72768 100644 --- a/yql/essentials/udfs/common/python/bindings/py_void_ut.cpp +++ b/yql/essentials/udfs/common/python/bindings/py_void_ut.cpp @@ -1,4 +1,4 @@ -#include "ut3/py_test_engine.h" +#include "py_test_engine.h" #include <library/cpp/testing/unittest/registar.h> diff --git a/yql/essentials/udfs/common/python/bindings/ut2/ya.make b/yql/essentials/udfs/common/python/bindings/ut2/ya.make new file mode 100644 index 0000000000..b289e188bb --- /dev/null +++ b/yql/essentials/udfs/common/python/bindings/ut2/ya.make @@ -0,0 +1,9 @@ +IF (OS_LINUX) + IF (NOT WITH_VALGRIND) + UNITTEST_FOR(yql/essentials/udfs/common/python/bindings) + + INCLUDE(../ya.make.test.inc) + USE_PYTHON2() + END() + ENDIF() +ENDIF() diff --git a/yql/essentials/udfs/common/python/bindings/ut3/ya.make b/yql/essentials/udfs/common/python/bindings/ut3/ya.make index cfce3cb9be..3a3c52dfbf 100644 --- a/yql/essentials/udfs/common/python/bindings/ut3/ya.make +++ b/yql/essentials/udfs/common/python/bindings/ut3/ya.make @@ -2,36 +2,11 @@ IF (OS_LINUX) IF (NOT WITH_VALGRIND) UNITTEST_FOR(yql/essentials/udfs/common/python/bindings) - SRCS( - py_callable_ut.cpp - py_cast_ut.cpp - py_dict_ut.cpp - py_list_ut.cpp - py_decimal_ut.cpp - py_number_ut.cpp - py_optional_ut.cpp - py_resource_ut.cpp - py_stream_ut.cpp - py_string_ut.cpp - py_struct_ut.cpp - py_tuple_ut.cpp - py_tzdate_ut.cpp - py_utils_ut.cpp - py_variant_ut.cpp - py_void_ut.cpp - ) - - USE_PYTHON3() - + INCLUDE(../ya.make.test.inc) PEERDIR( library/python/type_info - yql/essentials/minikql/computation/llvm16 - yql/essentials/public/udf/service/exception_policy - yql/essentials/sql/pg_dummy ) - - YQL_LAST_ABI_VERSION() - + USE_PYTHON3() END() ENDIF() ENDIF() diff --git a/yql/essentials/udfs/common/python/bindings/ya.make b/yql/essentials/udfs/common/python/bindings/ya.make index 83380ce4c4..aea3e54717 100644 --- a/yql/essentials/udfs/common/python/bindings/ya.make +++ b/yql/essentials/udfs/common/python/bindings/ya.make @@ -51,6 +51,7 @@ END() IF (NOT EXPORT_CMAKE) RECURSE_FOR_TESTS( + ut2 ut3 ) ENDIF() diff --git a/yql/essentials/udfs/common/python/bindings/ya.make.test.inc b/yql/essentials/udfs/common/python/bindings/ya.make.test.inc new file mode 100644 index 0000000000..67803ad18f --- /dev/null +++ b/yql/essentials/udfs/common/python/bindings/ya.make.test.inc @@ -0,0 +1,27 @@ +SRCS( + py_callable_ut.cpp + py_cast_ut.cpp + py_dict_ut.cpp + py_list_ut.cpp + py_decimal_ut.cpp + py_number_ut.cpp + py_optional_ut.cpp + py_resource_ut.cpp + py_stream_ut.cpp + py_string_ut.cpp + py_struct_ut.cpp + py_tuple_ut.cpp + py_tzdate_ut.cpp + py_utils_ut.cpp + py_variant_ut.cpp + py_void_ut.cpp +) + +PEERDIR( + yql/essentials/minikql/computation/llvm16 + yql/essentials/public/udf/service/exception_policy + yql/essentials/sql/pg_dummy + yql/essentials/udfs/common/python/python_udf +) + +YQL_LAST_ABI_VERSION() diff --git a/yql/essentials/udfs/common/python/python_udf/python_udf.cpp b/yql/essentials/udfs/common/python/python_udf/python_udf.cpp index b1739a1775..1007c75edc 100644 --- a/yql/essentials/udfs/common/python/python_udf/python_udf.cpp +++ b/yql/essentials/udfs/common/python/python_udf/python_udf.cpp @@ -59,59 +59,7 @@ public: InitYqlModule(pythonFlavor, standalone); - const auto rc = PyRun_SimpleString(R"( -# numpy on import may find installed openblas library and load it, -# which in turn causes it to start CPUCOUNT threads -# with approx. 40Mb memory reserved for each thread; -# -# See more detailed explanation here: https://st.yandex-team.ru/STATLIBS-1715#5bfc68ecbbc039001cec572a -# -# Thus, we reduce negative effects as much as possible -import os -os.environ['OPENBLAS_NUM_THREADS'] = '1' - - -# Following part allows us later to format tracebacks via sys.excepthook -# in thread-safe manner -import sys -import threading -if sys.version_info >= (3, 0): - from io import StringIO, TextIOWrapper as SysStderrType -else: - from cStringIO import StringIO - SysStderrType = file - -class StderrLocal(threading.local): - - def __init__(self): - self.is_real_mode = True - self.buffer = StringIO() - - -class StderrProxy(object): - def __init__(self, stderr): - self._stderr = stderr - self._tls = StderrLocal() - - def _toggle_real_mode(self): - self._tls.is_real_mode = not self._tls.is_real_mode - if not self._tls.is_real_mode: - self._tls.buffer.clear() - - def _get_value(self): - assert not self._tls.is_real_mode - return self._tls.buffer.getvalue() - - def __getattr__(self, attr): - target = self._stderr - if not self._tls.is_real_mode: - target = self._tls.buffer - - return getattr(target, attr) - -if isinstance(sys.stderr, SysStderrType): - sys.stderr = StderrProxy(sys.stderr) -)"); + const auto rc = PyRun_SimpleString(STANDART_STREAM_PROXY_INJECTION_SCRIPT); Y_ABORT_UNLESS(rc >= 0, "Can't setup module"); if (pythonFlavor == EPythonFlavor::Arcadia) { diff --git a/yql/essentials/udfs/common/python/python_udf/python_udf.h b/yql/essentials/udfs/common/python/python_udf/python_udf.h index 16d7da096d..83b3bb86e6 100644 --- a/yql/essentials/udfs/common/python/python_udf/python_udf.h +++ b/yql/essentials/udfs/common/python/python_udf/python_udf.h @@ -5,6 +5,61 @@ namespace NYql { namespace NUdf { +inline constexpr char STANDART_STREAM_PROXY_INJECTION_SCRIPT[] = +R"( +# numpy on import may find installed openblas library and load it, +# which in turn causes it to start CPUCOUNT threads +# with approx. 40Mb memory reserved for each thread; +# +# See more detailed explanation here: https://st.yandex-team.ru/STATLIBS-1715#5bfc68ecbbc039001cec572a +# +# Thus, we reduce negative effects as much as possible +import os +os.environ['OPENBLAS_NUM_THREADS'] = '1' + + +# Following part allows us later to format tracebacks via sys.excepthook +# in thread-safe manner +import sys +import threading +if sys.version_info >= (3, 0): + from io import StringIO, TextIOWrapper as SysStderrType +else: + from cStringIO import StringIO + SysStderrType = file + +class StderrLocal(threading.local): + + def __init__(self): + self.is_real_mode = True + self.buffer = StringIO() + + +class StderrProxy(object): + def __init__(self, stderr): + self._stderr = stderr + self._tls = StderrLocal() + + def _toggle_real_mode(self): + self._tls.is_real_mode = not self._tls.is_real_mode + if not self._tls.is_real_mode: + self._tls.buffer.clear() + + def _get_value(self): + assert not self._tls.is_real_mode + return self._tls.buffer.getvalue() + + def __getattr__(self, attr): + target = self._stderr + if not self._tls.is_real_mode: + target = self._tls.buffer + + return getattr(target, attr) + +if isinstance(sys.stderr, SysStderrType): + sys.stderr = StderrProxy(sys.stderr) +)"; + enum class EPythonFlavor { System, Arcadia, diff --git a/yql/essentials/udfs/common/stat/static/stat_udf.h b/yql/essentials/udfs/common/stat/static/stat_udf.h index f0c11a6812..36a1bad7c4 100644 --- a/yql/essentials/udfs/common/stat/static/stat_udf.h +++ b/yql/essentials/udfs/common/stat/static/stat_udf.h @@ -22,7 +22,7 @@ namespace { UdfTerminate((TStringBuilder() << GetPos() << " Invalid combination of delta/K values").data()); } - return TUnboxedValuePod(new TDigestResource(delta, K, args[0].Get<double>())); + return TUnboxedValuePod(new TDigestResource(delta, K, args[0].Get<double>(), true)); } SIMPLE_STRICT_UDF(TTDigest_AddValue, TResource<DigestResourceName>(TResource<DigestResourceName>, double)) { @@ -46,14 +46,17 @@ namespace { SIMPLE_UDF(TTDigest_Deserialize, TResource<DigestResourceName>(char*)) { Y_UNUSED(valueBuilder); - return TUnboxedValuePod(new TDigestResource(TString(args[0].AsStringRef()))); + return TUnboxedValuePod(new TDigestResource(TString(args[0].AsStringRef()), true)); } SIMPLE_STRICT_UDF(TTDigest_Merge, TResource<DigestResourceName>(TResource<DigestResourceName>, TResource<DigestResourceName>)) { Y_UNUSED(valueBuilder); TDigestResource::Validate(args[0]); TDigestResource::Validate(args[1]); - return TUnboxedValuePod(new TDigestResource(static_cast<TDigestResource*>(args[0].AsBoxed().Get())->Get(), static_cast<TDigestResource*>(args[1].AsBoxed().Get())->Get())); + return TUnboxedValuePod(new TDigestResource( + static_cast<TDigestResource*>(args[0].AsBoxed().Get())->Get(), + static_cast<TDigestResource*>(args[1].AsBoxed().Get())->Get(), + true)); } /* diff --git a/yql/essentials/udfs/common/stat/test/canondata/result.json b/yql/essentials/udfs/common/stat/test/canondata/result.json new file mode 100644 index 0000000000..44314e0309 --- /dev/null +++ b/yql/essentials/udfs/common/stat/test/canondata/result.json @@ -0,0 +1,7 @@ +{ + "test.test[nan]": [ + { + "uri": "file://test.test_nan_/results.txt" + } + ] +} diff --git a/yql/essentials/udfs/common/stat/test/canondata/test.test_nan_/results.txt b/yql/essentials/udfs/common/stat/test/canondata/test.test_nan_/results.txt new file mode 100644 index 0000000000..1ae063d52a --- /dev/null +++ b/yql/essentials/udfs/common/stat/test/canondata/test.test_nan_/results.txt @@ -0,0 +1,46 @@ +[ + { + "Write" = [ + { + "Type" = [ + "ListType"; + [ + "StructType"; + [ + [ + "column0"; + [ + "OptionalType"; + [ + "DataType"; + "Double" + ] + ] + ]; + [ + "column1"; + [ + "OptionalType"; + [ + "DataType"; + "Double" + ] + ] + ] + ] + ] + ]; + "Data" = [ + [ + [ + "1.1" + ]; + [ + "nan" + ] + ] + ] + } + ] + } +]
\ No newline at end of file diff --git a/yql/essentials/udfs/common/stat/test/cases/nan.sql b/yql/essentials/udfs/common/stat/test/cases/nan.sql new file mode 100644 index 0000000000..5ab4027b05 --- /dev/null +++ b/yql/essentials/udfs/common/stat/test/cases/nan.sql @@ -0,0 +1,2 @@ +select percentile(x,0.99),percentile(x,1.0) +from (values (double("nan")),(1.1),(0.5)) as a(x) diff --git a/yql/essentials/udfs/common/stat/test/ya.make b/yql/essentials/udfs/common/stat/test/ya.make new file mode 100644 index 0000000000..4a14f530f1 --- /dev/null +++ b/yql/essentials/udfs/common/stat/test/ya.make @@ -0,0 +1,13 @@ +YQL_UDF_TEST() + +DEPENDS(yql/essentials/udfs/common/stat) + +TIMEOUT(300) + +SIZE(MEDIUM) + +IF (SANITIZER_TYPE == "memory") + TAG(ya:not_autocheck) # YQL-15385 +ENDIF() + +END() diff --git a/yql/essentials/udfs/common/stat/ya.make b/yql/essentials/udfs/common/stat/ya.make index 8a5538b371..d1e622b444 100644 --- a/yql/essentials/udfs/common/stat/ya.make +++ b/yql/essentials/udfs/common/stat/ya.make @@ -18,6 +18,7 @@ YQL_UDF_CONTRIB(stat_udf) IF (NOT EXPORT_CMAKE) RECURSE_FOR_TESTS( + test ut ) ENDIF() diff --git a/yt/cpp/mapreduce/client/client.cpp b/yt/cpp/mapreduce/client/client.cpp index 6e58903308..bf9936a64a 100644 --- a/yt/cpp/mapreduce/client/client.cpp +++ b/yt/cpp/mapreduce/client/client.cpp @@ -1496,8 +1496,7 @@ IClientPtr TClient::GetParentClient(bool ignoreGlobalTx) RawClient_, Context_, TTransactionId(), - ClientRetryPolicy_ - ); + ClientRetryPolicy_); } else { return this; } @@ -1510,15 +1509,10 @@ void TClient::CheckShutdown() const } } -TClientContext CreateClientContext( - const TString& serverName, - const TCreateClientOptions& options) +void SetupClusterContext( + TClientContext& context, + const TString& serverName) { - TClientContext context; - context.Config = options.Config_ ? options.Config_ : TConfig::Get(); - context.TvmOnly = options.TvmOnly_; - context.ProxyAddress = options.ProxyAddress_; - context.ServerName = serverName; ApplyProxyUrlAliasingRules(context.ServerName); @@ -1531,9 +1525,8 @@ TClientContext CreateClientContext( static constexpr char httpUrlSchema[] = "http://"; static constexpr char httpsUrlSchema[] = "https://"; - if (options.UseTLS_) { - context.UseTLS = *options.UseTLS_; - } else { + + if (!context.UseTLS) { context.UseTLS = context.ServerName.StartsWith(httpsUrlSchema); } @@ -1555,9 +1548,25 @@ TClientContext CreateClientContext( if (context.ServerName.find(':') == TString::npos) { context.ServerName = CreateHostNameWithPort(context.ServerName, context); } - if (options.TvmOnly_) { + if (context.TvmOnly) { context.ServerName = Format("tvm.%v", context.ServerName); } +} + +TClientContext CreateClientContext( + const TString& serverName, + const TCreateClientOptions& options) +{ + TClientContext context; + context.Config = options.Config_ ? options.Config_ : TConfig::Get(); + context.TvmOnly = options.TvmOnly_; + context.ProxyAddress = options.ProxyAddress_; + + if (options.UseTLS_) { + context.UseTLS = *options.UseTLS_; + } + + SetupClusterContext(context, serverName); if (options.ProxyRole_) { context.Config->Hosts = "hosts?role=" + *options.ProxyRole_; diff --git a/yt/cpp/mapreduce/client/client.h b/yt/cpp/mapreduce/client/client.h index 9cd650bd2d..ef0741044c 100644 --- a/yt/cpp/mapreduce/client/client.h +++ b/yt/cpp/mapreduce/client/client.h @@ -509,6 +509,10 @@ private: //////////////////////////////////////////////////////////////////////////////// +void SetupClusterContext( + TClientContext& context, + const TString& serverName); + TClientContext CreateClientContext( const TString& serverName, const TCreateClientOptions& options); diff --git a/yt/cpp/mapreduce/client/client_reader.cpp b/yt/cpp/mapreduce/client/client_reader.cpp index 61369859bc..37d39345f1 100644 --- a/yt/cpp/mapreduce/client/client_reader.cpp +++ b/yt/cpp/mapreduce/client/client_reader.cpp @@ -69,7 +69,7 @@ TClientReader::TClientReader( if (useFormatFromTableAttributes) { auto transactionId2 = ReadTransaction_ ? ReadTransaction_->GetId() : ParentTransactionId_; - auto newFormat = GetTableFormat(ClientRetryPolicy_, RawClient_, transactionId2, Path_); + auto newFormat = GetTableFormat(ClientRetryPolicy_, RawClient_, Context_, transactionId2, Path_); if (newFormat) { Format_->Config = *newFormat; } diff --git a/yt/cpp/mapreduce/client/operation.cpp b/yt/cpp/mapreduce/client/operation.cpp index 76a55b436f..4d0cc07bed 100644 --- a/yt/cpp/mapreduce/client/operation.cpp +++ b/yt/cpp/mapreduce/client/operation.cpp @@ -220,11 +220,24 @@ TStructuredJobTableList ApplyProtobufColumnFilters( return tableList; } - auto isDynamic = NRawClient::BatchTransform( + TVector<TRichYPath> tableListPaths; + for (const auto& table: tableList) { + Y_ABORT_UNLESS(table.RichYPath, "Cannot get path to apply column filters"); + tableListPaths.emplace_back(*table.RichYPath); + } + + auto isDynamic = NRawClient::RemoteClustersBatchTransform( preparer.GetClient()->GetRawClient(), - tableList, + preparer.GetContext(), + tableListPaths, [&] (IRawBatchRequestPtr batch, const auto& table) { - return batch->Get(preparer.GetTransactionId(), table.RichYPath->Path_ + "/@dynamic", TGetOptions()); + // In case of external cluster, we can't use the current transaction + // since it is unknown for the external cluster. + // Hence, we should take a global transaction. + if (table.Cluster_ && !table.Cluster_->empty()) { + return batch->Get(TTransactionId(), table.Path_ + "/@dynamic", TGetOptions()); + } + return batch->Get(preparer.GetTransactionId(), table.Path_ + "/@dynamic", TGetOptions()); }); auto newTableList = tableList; diff --git a/yt/cpp/mapreduce/client/skiff.cpp b/yt/cpp/mapreduce/client/skiff.cpp index 537a28f059..e9c5ba2680 100644 --- a/yt/cpp/mapreduce/client/skiff.cpp +++ b/yt/cpp/mapreduce/client/skiff.cpp @@ -279,6 +279,7 @@ TFormat CreateSkiffFormat(const NSkiff::TSkiffSchemaPtr& schema) { NSkiff::TSkiffSchemaPtr CreateSkiffSchemaIfNecessary( const IRawClientPtr& rawClient, + const TClientContext& context, const TTransactionId& transactionId, ENodeReaderFormat nodeReaderFormat, const TVector<TRichYPath>& tablePaths, @@ -301,17 +302,23 @@ NSkiff::TSkiffSchemaPtr CreateSkiffSchemaIfNecessary( } } - auto nodes = NRawClient::BatchTransform( + auto nodes = RemoteClustersBatchTransform( rawClient, - NRawClient::CanonizeYPaths(rawClient, tablePaths), + context, + tablePaths, [&] (IRawBatchRequestPtr batch, const TRichYPath& path) { auto getOptions = TGetOptions() .AttributeFilter( TAttributeFilter() .AddAttribute("schema") .AddAttribute("dynamic") - .AddAttribute("type") - ); + .AddAttribute("type")); + // In case of external cluster, we can't use the current transaction + // since it is unknown for the external cluster. + // Hence, we should take a global transaction. + if (path.Cluster_ && !path.Cluster_->empty()) { + return batch->Get(TTransactionId(), path.Path_, getOptions); + } return batch->Get(transactionId, path.Path_, getOptions); }); diff --git a/yt/cpp/mapreduce/client/skiff.h b/yt/cpp/mapreduce/client/skiff.h index 5f26dc7656..3d9189998d 100644 --- a/yt/cpp/mapreduce/client/skiff.h +++ b/yt/cpp/mapreduce/client/skiff.h @@ -60,6 +60,7 @@ TFormat CreateSkiffFormat(const NSkiff::TSkiffSchemaPtr& schema); NSkiff::TSkiffSchemaPtr CreateSkiffSchemaIfNecessary( const IRawClientPtr& rawClient, + const TClientContext& context, const TTransactionId& transactionId, ENodeReaderFormat nodeReaderFormat, const TVector<TRichYPath>& tablePaths, diff --git a/yt/cpp/mapreduce/client/structured_table_formats.cpp b/yt/cpp/mapreduce/client/structured_table_formats.cpp index 5df2c40873..ea511398cc 100644 --- a/yt/cpp/mapreduce/client/structured_table_formats.cpp +++ b/yt/cpp/mapreduce/client/structured_table_formats.cpp @@ -1,5 +1,6 @@ #include "structured_table_formats.h" +#include "client.h" #include "format_hints.h" #include "skiff.h" @@ -67,15 +68,23 @@ TMaybe<TNode> GetCommonTableFormat( TMaybe<TNode> GetTableFormat( const IClientRetryPolicyPtr& retryPolicy, const IRawClientPtr& rawClient, + const TClientContext& context, const TTransactionId& transactionId, const TRichYPath& path) { + auto newRawClient = rawClient; + if (path.Cluster_ && !path.Cluster_->empty()) { + auto newContext = context; + NDetail::SetupClusterContext(newContext, *path.Cluster_); + newRawClient = rawClient->Clone(newContext); + } + auto formatPath = path.Path_ + "/@_format"; auto exists = NDetail::RequestWithRetry<bool>( retryPolicy->CreatePolicyForGenericRequest(), - [&rawClient, &transactionId, &formatPath] (TMutationId /*mutationId*/) { - return rawClient->Exists(transactionId, formatPath); + [&newRawClient, &transactionId, &formatPath] (TMutationId /*mutationId*/) { + return newRawClient->Exists(transactionId, formatPath); }); if (!exists) { return TMaybe<TNode>(); @@ -83,8 +92,8 @@ TMaybe<TNode> GetTableFormat( auto format = NDetail::RequestWithRetry<TMaybe<TNode>>( retryPolicy->CreatePolicyForGenericRequest(), - [&rawClient, &transactionId, &formatPath] (TMutationId /*mutationId*/) { - return rawClient->Get(transactionId, formatPath); + [&newRawClient, &transactionId, &formatPath] (TMutationId /*mutationId*/) { + return newRawClient->Get(transactionId, formatPath); }); if (format.Get()->AsString() != "yamred_dsv") { return TMaybe<TNode>(); @@ -102,12 +111,13 @@ TMaybe<TNode> GetTableFormat( TMaybe<TNode> GetTableFormats( const IClientRetryPolicyPtr& clientRetryPolicy, const IRawClientPtr& rawClient, + const TClientContext& context, const TTransactionId& transactionId, const TVector<TRichYPath>& inputs) { TVector<TMaybe<TNode>> formats; for (auto& table : inputs) { - formats.push_back(GetTableFormat(clientRetryPolicy, rawClient, transactionId, table)); + formats.push_back(GetTableFormat(clientRetryPolicy, rawClient, context, transactionId, table)); } return GetCommonTableFormat(formats); @@ -121,6 +131,7 @@ namespace NDetail { NSkiff::TSkiffSchemaPtr TryCreateSkiffSchema( const IRawClientPtr& rawClient, + const TClientContext& context, const TTransactionId& transactionId, const TVector<TRichYPath>& tables, const TOperationOptions& options, @@ -134,6 +145,7 @@ NSkiff::TSkiffSchemaPtr TryCreateSkiffSchema( } return CreateSkiffSchemaIfNecessary( rawClient, + context, transactionId, nodeReaderFormat, tables, @@ -387,7 +399,7 @@ std::pair<TFormat, TMaybe<TSmallJobFile>> TFormatBuilder::CreateYamrFormat( Y_ABORT_UNLESS(table.RichYPath, "Cannot use format from table for intermediate table"); tableList.push_back(*table.RichYPath); } - formatFromTableAttributes = GetTableFormats(ClientRetryPolicy_, RawClient_, TransactionId_, tableList); + formatFromTableAttributes = GetTableFormats(ClientRetryPolicy_, RawClient_, Context_, TransactionId_, tableList); } if (formatFromTableAttributes) { return { @@ -432,6 +444,7 @@ std::pair<TFormat, TMaybe<TSmallJobFile>> TFormatBuilder::CreateNodeFormat( } skiffSchema = TryCreateSkiffSchema( RawClient_, + Context_, TransactionId_, tableList, OperationOptions_, diff --git a/yt/cpp/mapreduce/client/structured_table_formats.h b/yt/cpp/mapreduce/client/structured_table_formats.h index 64a44c6f4d..c1ceec4066 100644 --- a/yt/cpp/mapreduce/client/structured_table_formats.h +++ b/yt/cpp/mapreduce/client/structured_table_formats.h @@ -22,12 +22,14 @@ TMaybe<TNode> GetCommonTableFormat( TMaybe<TNode> GetTableFormat( const IClientRetryPolicyPtr& clientRetryPolicy, const IRawClientPtr& rawClient, + const TClientContext& context, const TTransactionId& transactionId, const TRichYPath& path); TMaybe<TNode> GetTableFormats( const IClientRetryPolicyPtr& clientRetryPolicy, const IRawClientPtr& rawClient, + const TClientContext& context, const TTransactionId& transactionId, const TVector<TRichYPath>& paths); diff --git a/yt/cpp/mapreduce/http/context.cpp b/yt/cpp/mapreduce/http/context.cpp index 18d564fe09..531950c39e 100644 --- a/yt/cpp/mapreduce/http/context.cpp +++ b/yt/cpp/mapreduce/http/context.cpp @@ -13,7 +13,8 @@ bool operator==(const TClientContext& lhs, const TClientContext& rhs) lhs.HttpClient == rhs.HttpClient && lhs.UseTLS == rhs.UseTLS && lhs.TvmOnly == rhs.TvmOnly && - lhs.ProxyAddress == rhs.ProxyAddress; + lhs.ProxyAddress == rhs.ProxyAddress && + lhs.ProxyRole == rhs.ProxyRole; } //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/cpp/mapreduce/http/context.h b/yt/cpp/mapreduce/http/context.h index f50c1b9732..4559cfd587 100644 --- a/yt/cpp/mapreduce/http/context.h +++ b/yt/cpp/mapreduce/http/context.h @@ -22,6 +22,7 @@ struct TClientContext bool UseTLS = false; TConfigPtr Config = TConfig::Get(); TMaybe<TString> ProxyAddress; + TMaybe<TString> ProxyRole; }; bool operator==(const TClientContext& lhs, const TClientContext& rhs); diff --git a/yt/cpp/mapreduce/http_client/raw_client.cpp b/yt/cpp/mapreduce/http_client/raw_client.cpp index c86fd2494d..734282c2cf 100644 --- a/yt/cpp/mapreduce/http_client/raw_client.cpp +++ b/yt/cpp/mapreduce/http_client/raw_client.cpp @@ -903,6 +903,11 @@ IRawClientPtr THttpRawClient::Clone() return ::MakeIntrusive<THttpRawClient>(Context_); } +IRawClientPtr THttpRawClient::Clone(const TClientContext& context) +{ + return ::MakeIntrusive<THttpRawClient>(context); +} + //////////////////////////////////////////////////////////////////////////////// } // namespace NYT::NDetail diff --git a/yt/cpp/mapreduce/http_client/raw_client.h b/yt/cpp/mapreduce/http_client/raw_client.h index 6aa670f524..d292688978 100644 --- a/yt/cpp/mapreduce/http_client/raw_client.h +++ b/yt/cpp/mapreduce/http_client/raw_client.h @@ -339,6 +339,8 @@ public: IRawClientPtr Clone() override; + IRawClientPtr Clone(const TClientContext& context) override; + private: const TClientContext Context_; }; diff --git a/yt/cpp/mapreduce/http_client/raw_requests.h b/yt/cpp/mapreduce/http_client/raw_requests.h index 7273b8ad95..a61f303a62 100644 --- a/yt/cpp/mapreduce/http_client/raw_requests.h +++ b/yt/cpp/mapreduce/http_client/raw_requests.h @@ -4,6 +4,8 @@ #include <yt/cpp/mapreduce/common/fwd.h> +#include <yt/cpp/mapreduce/client/client.h> + #include <yt/cpp/mapreduce/http/context.h> #include <yt/cpp/mapreduce/interface/client.h> @@ -91,6 +93,52 @@ auto BatchTransform( return result; } +template <typename TBatchAdder> +auto RemoteClustersBatchTransform( + const IRawClientPtr& rawClient, + const TClientContext& context, + const TVector<TRichYPath>& paths, + TBatchAdder batchAdder, + const TExecuteBatchOptions& options = {}) +{ + // Given inputs from multiple clusters, we need to categorize them based on their cluster names. + // We assume the current cluster name is empty. Within each cluster, + // gather all related inputs to perform a batch request. + std::unordered_map<TString, std::vector<int>> clusterToPathsIndexes; + for (ssize_t index = 0; index < std::ssize(paths); ++index) { + const auto& path = paths[index]; + auto clusterName = path.Cluster_.GetOrElse(""); + clusterToPathsIndexes[clusterName].push_back(index); + } + + std::vector<TNode> result(paths.size()); + for (const auto& [clusterName, pathsIndexes] : clusterToPathsIndexes) { + auto newContext = context; + if (!clusterName.empty()) { + // It is not a current cluster, we switch the cluster context. + SetupClusterContext(newContext, clusterName); + } + auto newRawClient = rawClient->Clone(newContext); + + TVector<TRichYPath> pathsBatch; + pathsBatch.reserve(pathsIndexes.size()); + for (int index : pathsIndexes) { + pathsBatch.push_back(paths[index]); + } + + auto batchResult = NRawClient::BatchTransform( + newRawClient, + NRawClient::CanonizeYPaths(newRawClient, pathsBatch), + batchAdder, + options); + + for (ssize_t index = 0; index < std::ssize(pathsIndexes); ++index) { + result[pathsIndexes[index]] = batchResult[index]; + } + } + return result; +} + //////////////////////////////////////////////////////////////////////////////// } // namespace NDetail::NRawClient diff --git a/yt/cpp/mapreduce/interface/common.cpp b/yt/cpp/mapreduce/interface/common.cpp index 966be8341f..7abbef9127 100644 --- a/yt/cpp/mapreduce/interface/common.cpp +++ b/yt/cpp/mapreduce/interface/common.cpp @@ -552,6 +552,28 @@ TKeyBound::TKeyBound(ERelation relation, TKey key) , Key_(std::move(key)) { } +bool operator==(const TKeyBound& lhs, const TKeyBound& rhs) noexcept +{ + return lhs.Key() == rhs.Key() && lhs.Relation() == rhs.Relation(); +} + +//////////////////////////////////////////////////////////////////////////////// + +bool operator==(const TReadLimit& lhs, const TReadLimit& rhs) noexcept +{ + return lhs.Key_ == rhs.Key_ && lhs.RowIndex_ == rhs.RowIndex_ && + lhs.Offset_ == rhs.Offset_ && lhs.TabletIndex_ == rhs.TabletIndex_ && + lhs.KeyBound_ == rhs.KeyBound_; +} + +//////////////////////////////////////////////////////////////////////////////// + +bool operator==(const TReadRange& lhs, const TReadRange& rhs) noexcept +{ + return lhs.LowerLimit_ == rhs.LowerLimit_ && + lhs.UpperLimit_ == rhs.UpperLimit_ && lhs.Exact_ == rhs.Exact_; +} + //////////////////////////////////////////////////////////////////////////////// TTableSchema CreateTableSchema( diff --git a/yt/cpp/mapreduce/interface/common.h b/yt/cpp/mapreduce/interface/common.h index d595302bbb..9752e15822 100644 --- a/yt/cpp/mapreduce/interface/common.h +++ b/yt/cpp/mapreduce/interface/common.h @@ -880,6 +880,9 @@ struct TKeyBound /// @endcond }; +/// Equality check checks all fields of TKeyBound +bool operator==(const TKeyBound& lhs, const TKeyBound& rhs) noexcept; + /// /// @brief Description of the read limit. /// @@ -923,6 +926,9 @@ struct TReadLimit FLUENT_FIELD_OPTION(i64, TabletIndex); }; +/// Equality check checks all fields of TReadLimit +bool operator==(const TReadLimit& lhs, const TReadLimit& rhs) noexcept; + /// /// @brief Range of a table or a file /// @@ -963,6 +969,9 @@ struct TReadRange } }; +/// Equality check checks all fields of TReadRange +bool operator==(const TReadRange& lhs, const TReadRange& rhs) noexcept; + /// /// @brief Path with additional attributes. /// diff --git a/yt/cpp/mapreduce/interface/raw_client.h b/yt/cpp/mapreduce/interface/raw_client.h index b97c06ce37..24f8de61b6 100644 --- a/yt/cpp/mapreduce/interface/raw_client.h +++ b/yt/cpp/mapreduce/interface/raw_client.h @@ -4,6 +4,8 @@ #include "client_method_options.h" #include "operation.h" +#include <yt/cpp/mapreduce/http/context.h> + namespace NYT { //////////////////////////////////////////////////////////////////////////////// @@ -198,6 +200,7 @@ public: const TGetJobTraceOptions& options = {}) = 0; // Files + virtual std::unique_ptr<IInputStream> ReadFile( const TTransactionId& transactionId, const TRichYPath& path, @@ -332,7 +335,11 @@ public: virtual IRawBatchRequestPtr CreateRawBatchRequest() = 0; + // Other + virtual IRawClientPtr Clone() = 0; + + virtual IRawClientPtr Clone(const TClientContext& context) = 0; }; //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/python/yt/common.py b/yt/python/yt/common.py index 975c6be6b0..26a9599e51 100644 --- a/yt/python/yt/common.py +++ b/yt/python/yt/common.py @@ -223,6 +223,10 @@ class YtError(Exception): """Already exists.""" return self.contains_code(501) + def is_authentication_error(self): + """Authentication error.""" + return self.contains_code(900) + def is_access_denied(self): """Access denied.""" return self.contains_code(901) diff --git a/yt/yql/providers/yt/codec/codegen/llvm14/ya.make b/yt/yql/providers/yt/codec/codegen/llvm14/ya.make deleted file mode 100644 index 19bd72186e..0000000000 --- a/yt/yql/providers/yt/codec/codegen/llvm14/ya.make +++ /dev/null @@ -1,14 +0,0 @@ -LIBRARY() - -SRCDIR(yt/yql/providers/yt/codec/codegen/llvm14) - -PEERDIR( - yql/essentials/minikql/codegen/llvm14 -) - -USE_LLVM_BC14() -SET(LLVM_VER 14) - -INCLUDE(../ya.make.inc) - -END() diff --git a/yt/yql/providers/yt/codec/codegen/ut/llvm14/ya.make b/yt/yql/providers/yt/codec/codegen/ut/llvm14/ya.make deleted file mode 100644 index 64a0664f8b..0000000000 --- a/yt/yql/providers/yt/codec/codegen/ut/llvm14/ya.make +++ /dev/null @@ -1,12 +0,0 @@ -UNITTEST() - -SRCDIR(yt/yql/providers/yt/codec/codegen/ut/llvm14) - -PEERDIR( - yql/essentials/minikql/computation/llvm14 - yt/yql/providers/yt/codec/codegen/llvm14 -) - -INCLUDE(../ya.make.inc) - -END() diff --git a/yt/yql/providers/yt/codec/codegen/ut/ya.make b/yt/yql/providers/yt/codec/codegen/ut/ya.make index 48a2e6f160..35ebc6804c 100644 --- a/yt/yql/providers/yt/codec/codegen/ut/ya.make +++ b/yt/yql/providers/yt/codec/codegen/ut/ya.make @@ -1 +1,3 @@ -RECURSE(llvm14 llvm16) +RECURSE( + llvm16 +) diff --git a/yt/yql/providers/yt/codec/codegen/ya.make b/yt/yql/providers/yt/codec/codegen/ya.make index 4b5f83d5fe..51bc620d5e 100644 --- a/yt/yql/providers/yt/codec/codegen/ya.make +++ b/yt/yql/providers/yt/codec/codegen/ya.make @@ -6,6 +6,9 @@ PEERDIR() END() -RECURSE(llvm14 llvm16 no_llvm) +RECURSE( + llvm16 + no_llvm +) RECURSE_FOR_TESTS(ut) diff --git a/yt/yql/providers/yt/comp_nodes/dq/llvm14/ya.make b/yt/yql/providers/yt/comp_nodes/dq/llvm14/ya.make deleted file mode 100644 index 8d2e1c8ff9..0000000000 --- a/yt/yql/providers/yt/comp_nodes/dq/llvm14/ya.make +++ /dev/null @@ -1,11 +0,0 @@ -LIBRARY() - -PEERDIR( - yt/yql/providers/yt/comp_nodes/dq - yql/essentials/minikql/computation/llvm14 -) - -INCLUDE(../ya.make.inc) - -END() - diff --git a/yt/yql/providers/yt/comp_nodes/dq/ya.make b/yt/yql/providers/yt/comp_nodes/dq/ya.make index 4b86e9e9b2..baf5fc72d9 100644 --- a/yt/yql/providers/yt/comp_nodes/dq/ya.make +++ b/yt/yql/providers/yt/comp_nodes/dq/ya.make @@ -7,4 +7,6 @@ PEERDIR( END() -RECURSE(llvm14 llvm16) +RECURSE( + llvm16 +) diff --git a/yt/yql/providers/yt/comp_nodes/llvm14/ya.make b/yt/yql/providers/yt/comp_nodes/llvm14/ya.make deleted file mode 100644 index e3e64bfe16..0000000000 --- a/yt/yql/providers/yt/comp_nodes/llvm14/ya.make +++ /dev/null @@ -1,13 +0,0 @@ -LIBRARY() - -NO_COMPILER_WARNINGS() - -PEERDIR( - yql/essentials/minikql/codegen/llvm14 - yql/essentials/minikql/invoke_builtins/llvm14 - yt/yql/providers/yt/codec/codegen/llvm14 -) - -INCLUDE(../ya.make.inc) - -END() diff --git a/yt/yql/providers/yt/comp_nodes/ut/llvm14/ya.make b/yt/yql/providers/yt/comp_nodes/ut/llvm14/ya.make deleted file mode 100644 index 0dc094f653..0000000000 --- a/yt/yql/providers/yt/comp_nodes/ut/llvm14/ya.make +++ /dev/null @@ -1,11 +0,0 @@ -UNITTEST() - -PEERDIR( - yt/yql/providers/yt/comp_nodes/llvm14 - yql/essentials/minikql/comp_nodes/llvm14 - yt/yql/providers/yt/comp_nodes/llvm14 -) - -INCLUDE(../ya.make.inc) - -END() diff --git a/yt/yql/providers/yt/comp_nodes/ut/ya.make b/yt/yql/providers/yt/comp_nodes/ut/ya.make index 48a2e6f160..35ebc6804c 100644 --- a/yt/yql/providers/yt/comp_nodes/ut/ya.make +++ b/yt/yql/providers/yt/comp_nodes/ut/ya.make @@ -1 +1,3 @@ -RECURSE(llvm14 llvm16) +RECURSE( + llvm16 +) diff --git a/yt/yql/providers/yt/comp_nodes/ya.make b/yt/yql/providers/yt/comp_nodes/ya.make index 01bbc05e6a..8e6ca98bd0 100644 --- a/yt/yql/providers/yt/comp_nodes/ya.make +++ b/yt/yql/providers/yt/comp_nodes/ya.make @@ -3,7 +3,6 @@ LIBRARY() END() RECURSE( - llvm14 llvm16 no_llvm ) diff --git a/yt/yql/providers/yt/gateway/file/yql_yt_file.cpp b/yt/yql/providers/yt/gateway/file/yql_yt_file.cpp index 3a1e127f3d..f66e356c60 100644 --- a/yt/yql/providers/yt/gateway/file/yql_yt_file.cpp +++ b/yt/yql/providers/yt/gateway/file/yql_yt_file.cpp @@ -924,8 +924,15 @@ public: if (interval || isDuration) { attrs["expiration_timeout"] = isDuration ? duration.MilliSeconds() : interval->MilliSeconds(); } - if (options.Config()->NightlyCompress.Get(cluster).GetOrElse(false)) { - attrs["force_nightly_compress"] = true; + const TMaybe<bool> nightlyCompress = options.Config()->NightlyCompress.Get(cluster); + if (nightlyCompress.Defined()) { + if (*nightlyCompress) { + attrs["force_nightly_compress"] = true; + } else { + NYT::TNode compressSettings = NYT::TNode::CreateMap(); + compressSettings["enabled"] = false; + attrs["nightly_compression_settings"] = compressSettings; + } } } diff --git a/yt/yql/providers/yt/gateway/native/yql_yt_native.cpp b/yt/yql/providers/yt/gateway/native/yql_yt_native.cpp index fb021d149c..a0ab23f0bf 100644 --- a/yt/yql/providers/yt/gateway/native/yql_yt_native.cpp +++ b/yt/yql/providers/yt/gateway/native/yql_yt_native.cpp @@ -67,6 +67,7 @@ #include <util/system/execpath.h> #include <util/system/guard.h> #include <util/system/shellcommand.h> +#include <util/system/mutex.h> #include <util/ysaveload.h> #include <algorithm> @@ -2533,8 +2534,16 @@ private: yqlAttrs["expiration_timeout"] = isDuration ? duration.MilliSeconds() : (*interval).MilliSeconds(); } - if (execCtx->Options_.Config()->NightlyCompress.Get(cluster).GetOrElse(false)) { - yqlAttrs["force_nightly_compress"] = true; + const TMaybe<bool> nightlyCompress = + execCtx->Options_.Config()->NightlyCompress.Get(cluster); + if (nightlyCompress.Defined()) { + if (*nightlyCompress) { + yqlAttrs["force_nightly_compress"] = true; + } else { + NYT::TNode compressSettings = NYT::TNode::CreateMap(); + compressSettings["enabled"] = false; + yqlAttrs["nightly_compression_settings"] = compressSettings; + } } } @@ -2803,76 +2812,73 @@ private: TTableInfoResult& result) { TVector<NYT::TNode> attributes(tables.size()); - TVector<TMaybe<NYT::TNode>> linkAttributes(tables.size()); + NSorted::TSimpleMap<size_t, TString> requestSchemasIdxs; { + TMutex lock; auto batchGet = tx->CreateBatchRequest(); TVector<TFuture<void>> batchRes(Reserve(idxs.size())); for (auto& idx: idxs) { - batchRes.push_back(batchGet->Get(tables[idx.first].Table() + "&/@").Apply( - [&attributes, &linkAttributes, idx] (const TFuture<NYT::TNode>& res) { - try { - NYT::TNode attrs = res.GetValue(); - auto type = GetTypeFromAttributes(attrs, false); - if (type == "link") { - linkAttributes[idx.first] = attrs; - } else { - attributes[idx.first] = attrs; + batchRes.push_back(batchGet->Get(idx.second + "/@").Apply([&attributes, &requestSchemasIdxs, &lock, idx] (const TFuture<NYT::TNode>& res) { + attributes[idx.first] = res.GetValue(); + if (attributes[idx.first].HasKey("schema") && attributes[idx.first]["schema"].IsEntity()) { + with_lock (lock) { + requestSchemasIdxs.push_back(idx); } - } catch (const TErrorResponse& e) { - // Yt returns NoSuchTransaction as inner issue for ResolveError - if (!e.IsResolveError() || e.IsNoSuchTransaction()) { - throw; - } - // Just ignore. Original table path may be deleted at this time } })); } batchGet->ExecuteBatch(); WaitExceptionOrAll(batchRes).GetValue(); } - + if (!requestSchemasIdxs.empty()) { + YQL_CLOG(INFO, ProviderYt) << "Additional request of @schema for " << requestSchemasIdxs.size() << " table(s)"; + auto batchGet = tx->CreateBatchRequest(); + TVector<TFuture<void>> batchRes(Reserve(requestSchemasIdxs.size())); + auto getOpts = TGetOptions() + .AttributeFilter(TAttributeFilter() + .AddAttribute("schema") + ); + for (auto& idx: requestSchemasIdxs) { + batchRes.push_back(batchGet->Get(idx.second + "/@", getOpts).Apply([&attributes, idx] (const TFuture<NYT::TNode>& res) { + attributes[idx.first]["schema"] = res.GetValue().At("schema"); + })); + } + batchGet->ExecuteBatch(); + WaitExceptionOrAll(batchRes).GetValue(); + } { - auto schemaAttrFilter = TAttributeFilter().AddAttribute("schema"); - TVector<NYT::TNode> schemas(tables.size()); - auto batchGet = tx->CreateBatchRequest(); TVector<TFuture<void>> batchRes; - for (auto& idx : idxs) { - batchRes.push_back(batchGet->Get(tables[idx.first].Table() + "/@", TGetOptions().AttributeFilter(schemaAttrFilter)) - .Apply([idx, &schemas] (const TFuture<NYT::TNode>& res) { - try { - schemas[idx.first] = res.GetValue(); - } catch (const TErrorResponse& e) { - // Yt returns NoSuchTransaction as inner issue for ResolveError - if (!e.IsResolveError() || e.IsNoSuchTransaction()) { - throw; - } - // Just ignore. Original table path may be deleted at this time - } - })); - - if (linkAttributes[idx.first]) { - const auto& linkAttr = *linkAttributes[idx.first]; - batchRes.push_back(batchGet->Get(idx.second + "/@").Apply( - [idx, &linkAttr, &attributes](const TFuture<NYT::TNode>& f) { + auto getOpts = TGetOptions() + .AttributeFilter(TAttributeFilter() + .AddAttribute("type") + .AddAttribute(TString{QB2Premapper}) + .AddAttribute(TString{YqlRowSpecAttribute}) + ); + for (auto& idx: idxs) { + batchRes.push_back(batchGet->Get(tables[idx.first].Table() + "&/@", getOpts).Apply([idx, &attributes](const TFuture<NYT::TNode>& f) { + try { NYT::TNode attrs = f.GetValue(); - attributes[idx.first] = attrs; - // override some attributes by the link ones - if (linkAttr.HasKey(QB2Premapper)) { - attributes[idx.first][QB2Premapper] = linkAttr[QB2Premapper]; + if (GetTypeFromAttributes(attrs, false) == "link") { + // override some attributes by the link ones + if (attrs.HasKey(QB2Premapper)) { + attributes[idx.first][QB2Premapper] = attrs[QB2Premapper]; + } + if (attrs.HasKey(YqlRowSpecAttribute)) { + attributes[idx.first][YqlRowSpecAttribute] = attrs[YqlRowSpecAttribute]; + } } - if (linkAttr.HasKey(YqlRowSpecAttribute)) { - attributes[idx.first][YqlRowSpecAttribute] = linkAttr[YqlRowSpecAttribute]; + } catch (const TErrorResponse& e) { + // Yt returns NoSuchTransaction as inner issue for ResolveError + if (!e.IsResolveError() || e.IsNoSuchTransaction()) { + throw; } - })); - } + // Just ignore. Original table path may be deleted at this time + } + })); } batchGet->ExecuteBatch(); WaitExceptionOrAll(batchRes).GetValue(); - - for (auto& idx: idxs) { - attributes[idx.first]["schema"] = schemas[idx.first]["schema"]; - } } auto batchGet = tx->CreateBatchRequest(); @@ -2904,6 +2910,7 @@ private: } statInfo->Id = attrs["id"].AsString(); + YQL_ENSURE(statInfo->Id == TStringBuf(idx.second).Skip(1)); statInfo->TableRevision = attrs["revision"].IntCast<ui64>(); statInfo->Revision = GetContentRevision(attrs); diff --git a/yt/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_sort.cpp b/yt/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_sort.cpp index 9300a67d13..c76709f914 100644 --- a/yt/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_sort.cpp +++ b/yt/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_sort.cpp @@ -667,11 +667,60 @@ TMaybeNode<TExprBase> TYtPhysicalOptProposalTransformer::AssumeConstraints(TExpr const size_t index = FromString(input.Cast<TYtOutput>().OutIndex().Value()); TYtOutTableInfo outTable(op.Output().Item(index)); if (builder) { + YQL_ENSURE(!builder->NeedMap()); builder->FillRowSpecSort(*outTable.RowSpec); } outTable.RowSpec->SetConstraints(assume.Ref().GetConstraintSet()); outTable.SetUnique(assume.Ref().GetConstraint<TDistinctConstraintNode>(), assume.Pos(), ctx); + if (op.Maybe<TYtMap>() || op.Maybe<TYtReduce>()) { + TExprNode::TPtr lambda; + size_t childToReplace = 0; + if (auto map = op.Maybe<TYtMap>()) { + lambda = map.Cast().Mapper().Ptr(); + childToReplace = TYtMap::idx_Mapper; + } else if (auto reduce = op.Maybe<TYtReduce>()) { + lambda = reduce.Cast().Reducer().Ptr(); + childToReplace = TYtReduce::idx_Reducer; + } else { + YQL_ENSURE(false, "unexpected operation"); + } + + auto actualLambdaOutputType = TCoLambda(lambda).Body().Ref().GetTypeAnn(); + auto expectedLambdaOutputType = outTable.RowSpec->GetExtendedType(ctx); + if (!IsSameAnnotation(*actualLambdaOutputType, *expectedLambdaOutputType)) { + // Drop aux columns that are not expected after row spec rebuild + + auto excludeLambda = Build<TCoLambda>(ctx, lambda->Pos()) + .Args({"stream"}) + .Body<TCoOrderedMap>() + .Input("stream") + .Lambda<TCoLambda>() + .Args({"struct"}) + .Body<TCoCastStruct>() + .Struct("struct") + .Type(ExpandType(lambda->Pos(), *expectedLambdaOutputType, ctx)) + .Build() + .Build() + .Build() + .Done(); + + lambda = Build<TCoLambda>(ctx, lambda->Pos()) + .Args({"stream"}) + .Body<TExprApplier>() + .Apply(excludeLambda) + .With<TExprApplier>(0) + .Apply(TCoLambda(lambda)) + .With(0, "stream") + .Build() + .Build() + .Done() + .Ptr(); + + newOp = ctx.ChangeChild(*newOp, childToReplace, std::move(lambda)); + } + } + TVector<TYtOutTable> outputs; for (size_t i = 0; i < op.Output().Size(); ++i) { if (index == i) { diff --git a/yt/yql/tests/sql/suites/order_by/input_sorted_desc.txt b/yt/yql/tests/sql/suites/order_by/input_sorted_desc.txt new file mode 100644 index 0000000000..93fc12aa26 --- /dev/null +++ b/yt/yql/tests/sql/suites/order_by/input_sorted_desc.txt @@ -0,0 +1,3 @@ +{"_yql_column_0"="\xE0\xCC\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFE";"key2"=3;"key1"=3;}; +{"_yql_column_0"="\xE0\xCD\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFE";"key2"=2;"key1"=2;}; +{"_yql_column_0"="\xE0\xCE\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFE";"key2"=1;"key1"=1;}; diff --git a/yt/yql/tests/sql/suites/order_by/input_sorted_desc.txt.attr b/yt/yql/tests/sql/suites/order_by/input_sorted_desc.txt.attr new file mode 100644 index 0000000000..21b8210d8e --- /dev/null +++ b/yt/yql/tests/sql/suites/order_by/input_sorted_desc.txt.attr @@ -0,0 +1,61 @@ +{ + "_yql_row_spec" = { + "Constraints" = { + "Sorted" = [ + [ + [ + "key1"; + ]; + %false; + ]; + [ + [ + "key2"; + ]; + %true; + ]; + ]; + }; + "SortDirections" = [ + 0; + 1; + ]; + "SortMembers" = [ + "key1"; + "key2"; + ]; + "SortedBy" = [ + "_yql_column_0"; + "key2"; + ]; + "SortedByTypes" = [ + [ + "DataType"; + "String"; + ]; + [ + "DataType"; + "Int32"; + ]; + ]; + "Type" = [ + "StructType"; + [ + [ + "key1"; + [ + "DataType"; + "Int32"; + ]; + ]; + [ + "key2"; + [ + "DataType"; + "Int32"; + ]; + ]; + ]; + ]; + }; +} diff --git a/yt/yql/tests/sql/suites/order_by/yql-19598.cfg b/yt/yql/tests/sql/suites/order_by/yql-19598.cfg new file mode 100644 index 0000000000..b7ad0584fe --- /dev/null +++ b/yt/yql/tests/sql/suites/order_by/yql-19598.cfg @@ -0,0 +1 @@ +in Input input_sorted_desc.txt diff --git a/yt/yql/tests/sql/suites/order_by/yql-19598.sql b/yt/yql/tests/sql/suites/order_by/yql-19598.sql new file mode 100644 index 0000000000..cab0eded47 --- /dev/null +++ b/yt/yql/tests/sql/suites/order_by/yql-19598.sql @@ -0,0 +1,4 @@ +/* postgres can not */ +USE plato; + +SELECT * FROM Input WHERE key2 = 2 ASSUME ORDER BY key2; diff --git a/yt/yt/client/api/public.h b/yt/yt/client/api/public.h index d289f6c9b7..20cfb6a553 100644 --- a/yt/yt/client/api/public.h +++ b/yt/yt/client/api/public.h @@ -84,7 +84,7 @@ DEFINE_ENUM(ETransactionCoordinatorPrepareMode, ((Late) (1)) ); -DEFINE_ENUM(EProxyType, +DEFINE_ENUM(EProxyKind, ((Http) (1)) ((Rpc) (2)) ((Grpc) (3)) diff --git a/yt/yt/client/api/queue_transaction.h b/yt/yt/client/api/queue_transaction.h index 1487cb0ca3..42cd2166de 100644 --- a/yt/yt/client/api/queue_transaction.h +++ b/yt/yt/client/api/queue_transaction.h @@ -111,4 +111,3 @@ struct IQueueTransaction //////////////////////////////////////////////////////////////////////////////// } // namespace NYT::NApi - diff --git a/yt/yt/client/api/rpc_proxy/client_base.cpp b/yt/yt/client/api/rpc_proxy/client_base.cpp index 5fd70dd7c3..89aa41b97a 100644 --- a/yt/yt/client/api/rpc_proxy/client_base.cpp +++ b/yt/yt/client/api/rpc_proxy/client_base.cpp @@ -137,9 +137,7 @@ TFuture<ITransactionPtr> TClientBase::StartTransaction( req->set_type(static_cast<NProto::ETransactionType>(type)); req->set_timeout(ToProto(timeout)); - if (options.Deadline) { - req->set_deadline(ToProto(*options.Deadline)); - } + YT_OPTIONAL_SET_PROTO(req, deadline, options.Deadline); if (options.Id) { ToProto(req->mutable_id(), options.Id); } @@ -251,9 +249,7 @@ TFuture<TYsonString> TClientBase::GetNode( req->mutable_legacy_attributes()->set_all(true); } - if (options.MaxSize) { - req->set_max_size(*options.MaxSize); - } + YT_OPTIONAL_SET_PROTO(req, max_size, options.MaxSize); ToProto(req->mutable_complexity_limits(), options.ComplexityLimits); @@ -289,9 +285,7 @@ TFuture<TYsonString> TClientBase::ListNode( req->mutable_legacy_attributes()->set_all(true); } - if (options.MaxSize) { - req->set_max_size(*options.MaxSize); - } + YT_OPTIONAL_SET_PROTO(req, max_size, options.MaxSize); ToProto(req->mutable_complexity_limits(), options.ComplexityLimits); @@ -423,12 +417,8 @@ TFuture<TLockNodeResult> TClientBase::LockNode( req->set_mode(ToProto(mode)); req->set_waitable(options.Waitable); - if (options.ChildKey) { - req->set_child_key(*options.ChildKey); - } - if (options.AttributeKey) { - req->set_attribute_key(*options.AttributeKey); - } + YT_OPTIONAL_TO_PROTO(req, child_key, options.ChildKey); + YT_OPTIONAL_TO_PROTO(req, attribute_key, options.AttributeKey); ToProto(req->mutable_transactional_options(), options); ToProto(req->mutable_prerequisite_options(), options); @@ -642,12 +632,8 @@ TFuture<IFileReaderPtr> TClientBase::CreateFileReader( InitStreamingRequest(*req); req->set_path(path); - if (options.Offset) { - req->set_offset(*options.Offset); - } - if (options.Length) { - req->set_length(*options.Length); - } + YT_OPTIONAL_SET_PROTO(req, offset, options.Offset); + YT_OPTIONAL_SET_PROTO(req, length, options.Length); if (options.Config) { req->set_config(ConvertToYsonString(*options.Config).ToString()); } @@ -691,12 +677,8 @@ IJournalReaderPtr TClientBase::CreateJournalReader( req->set_path(path); - if (options.FirstRowIndex) { - req->set_first_row_index(*options.FirstRowIndex); - } - if (options.RowCount) { - req->set_row_count(*options.RowCount); - } + YT_OPTIONAL_SET_PROTO(req, first_row_index, options.FirstRowIndex); + YT_OPTIONAL_SET_PROTO(req, row_count, options.RowCount); if (options.Config) { req->set_config(ConvertToYsonString(*options.Config).ToString()); } @@ -868,9 +850,7 @@ TFuture<TUnversionedLookupRowsResult> TClientBase::LookupRows( req->set_keep_missing_rows(options.KeepMissingRows); req->set_enable_partial_result(options.EnablePartialResult); req->set_replica_consistency(static_cast<NProto::EReplicaConsistency>(options.ReplicaConsistency)); - if (options.UseLookupCache) { - req->set_use_lookup_cache(*options.UseLookupCache); - } + YT_OPTIONAL_SET_PROTO(req, use_lookup_cache, options.UseLookupCache); req->SetMultiplexingBand(options.MultiplexingBand); req->set_multiplexing_band(static_cast<NProto::EMultiplexingBand>(options.MultiplexingBand)); @@ -920,9 +900,7 @@ TFuture<TVersionedLookupRowsResult> TClientBase::VersionedLookupRows( req->set_keep_missing_rows(options.KeepMissingRows); req->set_enable_partial_result(options.EnablePartialResult); req->set_replica_consistency(static_cast<NProto::EReplicaConsistency>(options.ReplicaConsistency)); - if (options.UseLookupCache) { - req->set_use_lookup_cache(*options.UseLookupCache); - } + YT_OPTIONAL_SET_PROTO(req, use_lookup_cache, options.UseLookupCache); req->SetMultiplexingBand(options.MultiplexingBand); req->set_multiplexing_band(static_cast<NProto::EMultiplexingBand>(options.MultiplexingBand)); @@ -969,9 +947,7 @@ TFuture<std::vector<TUnversionedLookupRowsResult>> TClientBase::MultiLookupRows( } protoSubrequest->set_keep_missing_rows(subrequestOptions.KeepMissingRows); protoSubrequest->set_enable_partial_result(subrequestOptions.EnablePartialResult); - if (subrequestOptions.UseLookupCache) { - protoSubrequest->set_use_lookup_cache(*subrequestOptions.UseLookupCache); - } + YT_OPTIONAL_SET_PROTO(protoSubrequest, use_lookup_cache, subrequestOptions.UseLookupCache); auto rowset = SerializeRowset( subrequest.NameTable, @@ -1079,30 +1055,22 @@ TFuture<TSelectRowsResult> TClientBase::SelectRows( // TODO(lukyan): Move to FillRequestBySelectRowsOptionsBase req->SetTimeout(options.Timeout.value_or(config->DefaultSelectRowsTimeout)); - if (options.InputRowLimit) { - req->set_input_row_limit(*options.InputRowLimit); - } - if (options.OutputRowLimit) { - req->set_output_row_limit(*options.OutputRowLimit); - } + YT_OPTIONAL_SET_PROTO(req, input_row_limit, options.InputRowLimit); + YT_OPTIONAL_SET_PROTO(req, output_row_limit, options.OutputRowLimit); req->set_range_expansion_limit(options.RangeExpansionLimit); req->set_max_subqueries(options.MaxSubqueries); req->set_min_row_count_per_subquery(options.MinRowCountPerSubquery); req->set_allow_full_scan(options.AllowFullScan); req->set_allow_join_without_index(options.AllowJoinWithoutIndex); - if (options.ExecutionPool) { - req->set_execution_pool(*options.ExecutionPool); - } + YT_OPTIONAL_TO_PROTO(req, execution_pool, options.ExecutionPool); if (options.PlaceholderValues) { req->set_placeholder_values(options.PlaceholderValues.ToString()); } req->set_fail_on_incomplete_result(options.FailOnIncompleteResult); req->set_verbose_logging(options.VerboseLogging); req->set_new_range_inference(options.NewRangeInference); - if (options.ExecutionBackend) { - req->set_execution_backend(ToProto(*options.ExecutionBackend)); - } + YT_OPTIONAL_SET_PROTO(req, execution_backend, options.ExecutionBackend); req->set_enable_code_cache(options.EnableCodeCache); req->set_memory_limit_per_node(options.MemoryLimitPerNode); ToProto(req->mutable_suppressable_access_tracking_options(), options); @@ -1110,9 +1078,7 @@ TFuture<TSelectRowsResult> TClientBase::SelectRows( req->set_use_canonical_null_relations(options.UseCanonicalNullRelations); req->set_merge_versioned_rows(options.MergeVersionedRows); ToProto(req->mutable_versioned_read_options(), options.VersionedReadOptions); - if (options.UseLookupCache) { - req->set_use_lookup_cache(*options.UseLookupCache); - } + YT_OPTIONAL_SET_PROTO(req, use_lookup_cache, options.UseLookupCache); return req->Invoke().Apply(BIND([] (const TApiServiceProxy::TRspSelectRowsPtr& rsp) { TSelectRowsResult result; diff --git a/yt/yt/client/api/rpc_proxy/client_impl.cpp b/yt/yt/client/api/rpc_proxy/client_impl.cpp index 0de3dfac4e..33bad4dcfc 100644 --- a/yt/yt/client/api/rpc_proxy/client_impl.cpp +++ b/yt/yt/client/api/rpc_proxy/client_impl.cpp @@ -180,9 +180,7 @@ ITransactionPtr TClient::AttachTransaction( ToProto(req->mutable_transaction_id(), transactionId); // COMPAT(kiselyovp): remove auto_abort from the protocol req->set_auto_abort(false); - if (options.PingPeriod) { - req->set_ping_period(options.PingPeriod->GetValue()); - } + YT_OPTIONAL_SET_PROTO(req, ping_period, options.PingPeriod); req->set_ping(options.Ping); req->set_ping_ancestors(options.PingAncestors); @@ -366,12 +364,8 @@ TFuture<void> TClient::ReshardTable( req->set_path(path); req->set_tablet_count(tabletCount); - if (options.Uniform) { - req->set_uniform(*options.Uniform); - } - if (options.EnableSlicing) { - req->set_enable_slicing(*options.EnableSlicing); - } + YT_OPTIONAL_SET_PROTO(req, uniform, options.Uniform); + YT_OPTIONAL_SET_PROTO(req, enable_slicing, options.EnableSlicing); if (options.SlicingAccuracy) { req->set_slicing_accuracy(*options.SlicingAccuracy); } @@ -438,9 +432,7 @@ TFuture<void> TClient::AlterTable( if (options.SchemaId) { ToProto(req->mutable_schema_id(), *options.SchemaId); } - if (options.Dynamic) { - req->set_dynamic(*options.Dynamic); - } + YT_OPTIONAL_SET_PROTO(req, dynamic, options.Dynamic); if (options.UpstreamReplicaId) { ToProto(req->mutable_upstream_replica_id(), *options.UpstreamReplicaId); } @@ -468,29 +460,20 @@ TFuture<void> TClient::AlterTableReplica( ToProto(req->mutable_replica_id(), replicaId); - if (options.Enabled) { - req->set_enabled(*options.Enabled); - } + YT_OPTIONAL_SET_PROTO(req, enabled, options.Enabled); if (options.Mode) { req->set_mode(static_cast<NProto::ETableReplicaMode>(*options.Mode)); } - if (options.PreserveTimestamps) { - req->set_preserve_timestamps(*options.PreserveTimestamps); - } + YT_OPTIONAL_SET_PROTO(req, preserve_timestamps, options.PreserveTimestamps); if (options.Atomicity) { req->set_atomicity(static_cast<NProto::EAtomicity>(*options.Atomicity)); } - if (options.EnableReplicatedTableTracker) { - req->set_enable_replicated_table_tracker(*options.EnableReplicatedTableTracker); - } - - if (options.ReplicaPath) { - req->set_replica_path(*options.ReplicaPath); - } + YT_OPTIONAL_SET_PROTO(req, enable_replicated_table_tracker, options.EnableReplicatedTableTracker); + YT_OPTIONAL_TO_PROTO(req, replica_path, options.ReplicaPath); ToProto(req->mutable_mutating_options(), options); @@ -566,9 +549,7 @@ TFuture<std::vector<TTableReplicaId>> TClient::GetInSyncReplicas( req->set_timestamp(options.Timestamp); } - if (options.CachedSyncReplicasTimeout) { - req->set_cached_sync_replicas_timeout(NYT::ToProto(*options.CachedSyncReplicasTimeout)); - } + YT_OPTIONAL_SET_PROTO(req, cached_sync_replicas_timeout, options.CachedSyncReplicasTimeout); req->set_path(path); req->Attachments() = SerializeRowset(nameTable, keys, req->mutable_rowset_descriptor()); @@ -592,9 +573,7 @@ TFuture<std::vector<TTableReplicaId>> TClient::GetInSyncReplicas( req->set_timestamp(options.Timestamp); } - if (options.CachedSyncReplicasTimeout) { - req->set_cached_sync_replicas_timeout(NYT::ToProto(*options.CachedSyncReplicasTimeout)); - } + YT_OPTIONAL_SET_PROTO(req, cached_sync_replicas_timeout, options.CachedSyncReplicasTimeout); req->set_path(path); req->RequireServerFeature(ERpcProxyFeature::GetInSyncWithoutKeys); @@ -659,9 +638,7 @@ TFuture<TGetTabletErrorsResult> TClient::GetTabletErrors( SetTimeoutOptions(*req, options); req->set_path(path); - if (options.Limit) { - req->set_limit(*options.Limit); - } + YT_OPTIONAL_SET_PROTO(req, limit, options.Limit); return req->Invoke().Apply(BIND([] (const TErrorOr<TApiServiceProxy::TRspGetTabletErrorsPtr>& rspOrError) { const auto& rsp = rspOrError.ValueOrThrow(); @@ -739,12 +716,8 @@ TFuture<void> TClient::AlterReplicationCard( if (options.ReplicatedTableOptions) { req->set_replicated_table_options(ConvertToYsonString(options.ReplicatedTableOptions).ToString()); } - if (options.EnableReplicatedTableTracker) { - req->set_enable_replicated_table_tracker(*options.EnableReplicatedTableTracker); - } - if (options.ReplicationCardCollocationId) { - ToProto(req->mutable_replication_card_collocation_id(), *options.ReplicationCardCollocationId); - } + YT_OPTIONAL_SET_PROTO(req, enable_replicated_table_tracker, options.EnableReplicatedTableTracker); + YT_OPTIONAL_TO_PROTO(req, replication_card_collocation_id, options.ReplicationCardCollocationId); if (options.CollocationOptions) { req->set_collocation_options(ConvertToYsonString(options.CollocationOptions).ToString()); } @@ -837,9 +810,7 @@ TFuture<IQueueRowsetPtr> TClient::PullQueueConsumer( ToProto(req->mutable_consumer_path(), consumerPath); ToProto(req->mutable_queue_path(), queuePath); - if (offset) { - req->set_offset(*offset); - } + YT_OPTIONAL_SET_PROTO(req, offset, offset); req->set_partition_index(partitionIndex); ToProto(req->mutable_row_batch_read_options(), rowBatchReadOptions); @@ -900,12 +871,8 @@ TFuture<std::vector<TListQueueConsumerRegistrationsResult>> TClient::ListQueueCo auto req = proxy.ListQueueConsumerRegistrations(); SetTimeoutOptions(*req, options); - if (queuePath) { - ToProto(req->mutable_queue_path(), *queuePath); - } - if (consumerPath) { - ToProto(req->mutable_consumer_path(), *consumerPath); - } + YT_OPTIONAL_TO_PROTO(req, queue_path, queuePath); + YT_OPTIONAL_TO_PROTO(req, consumer_path, consumerPath); return req->Invoke().Apply(BIND([] (const TApiServiceProxy::TRspListQueueConsumerRegistrationsPtr& rsp) { std::vector<TListQueueConsumerRegistrationsResult> result; @@ -1030,9 +997,7 @@ TFuture<TCheckPermissionResponse> TClient::CheckPermission( auto* protoColumns = req->mutable_columns(); ToProto(protoColumns->mutable_items(), *options.Columns); } - if (options.Vital) { - req->set_vital(*options.Vital); - } + YT_OPTIONAL_SET_PROTO(req, vital, options.Vital); ToProto(req->mutable_master_read_options(), options); ToProto(req->mutable_transactional_options(), options); @@ -1059,9 +1024,7 @@ TFuture<TCheckPermissionByAclResult> TClient::CheckPermissionByAcl( auto req = proxy.CheckPermissionByAcl(); SetTimeoutOptions(*req, options); - if (user) { - req->set_user(ToProto(*user)); - } + YT_OPTIONAL_SET_PROTO(req, user, user); req->set_permission(ToProto(permission)); req->set_acl(ConvertToYsonString(acl).ToString()); req->set_ignore_missing_subjects(options.IgnoreMissingSubjects); @@ -1148,9 +1111,7 @@ TFuture<void> TClient::AbortOperation( NScheduler::ToProto(req, operationIdOrAlias); - if (options.AbortMessage) { - req->set_abort_message(*options.AbortMessage); - } + YT_OPTIONAL_TO_PROTO(req, abort_message, options.AbortMessage); return req->Invoke().As<void>(); } @@ -1166,9 +1127,7 @@ TFuture<void> TClient::SuspendOperation( NScheduler::ToProto(req, operationIdOrAlias); req->set_abort_running_jobs(options.AbortRunningJobs); - if (options.Reason) { - req->set_reason(*options.Reason); - } + YT_OPTIONAL_TO_PROTO(req, reason, options.Reason); return req->Invoke().As<void>(); } @@ -1349,12 +1308,8 @@ TFuture<TGetJobStderrResponse> TClient::GetJobStderr( NScheduler::ToProto(req, operationIdOrAlias); ToProto(req->mutable_job_id(), jobId); - if (options.Limit) { - req->set_limit(*options.Limit); - } - if (options.Offset) { - req->set_offset(*options.Offset); - } + YT_OPTIONAL_SET_PROTO(req, limit, options.Limit); + YT_OPTIONAL_SET_PROTO(req, offset, options.Offset); return req->Invoke().Apply(BIND([req = req](const TApiServiceProxy::TRspGetJobStderrPtr& rsp) { YT_VERIFY(rsp->Attachments().size() == 1); @@ -1373,24 +1328,12 @@ TFuture<std::vector<TJobTraceEvent>> TClient::GetJobTrace( SetTimeoutOptions(*req, options); NScheduler::ToProto(req, operationIdOrAlias); - if (options.JobId) { - ToProto(req->mutable_job_id(), *options.JobId); - } - if (options.TraceId) { - ToProto(req->mutable_trace_id(), *options.TraceId); - } - if (options.FromTime) { - req->set_from_time(*options.FromTime); - } - if (options.ToTime) { - req->set_to_time(*options.ToTime); - } - if (options.FromEventIndex) { - req->set_from_event_index(*options.FromEventIndex); - } - if (options.ToEventIndex) { - req->set_to_event_index(*options.ToEventIndex); - } + YT_OPTIONAL_TO_PROTO(req, job_id, options.JobId); + YT_OPTIONAL_TO_PROTO(req, trace_id, options.TraceId); + YT_OPTIONAL_SET_PROTO(req, from_time, options.FromTime); + YT_OPTIONAL_SET_PROTO(req, to_time, options.ToTime); + YT_OPTIONAL_SET_PROTO(req, from_event_index, options.FromEventIndex); + YT_OPTIONAL_SET_PROTO(req, to_event_index, options.ToEventIndex); return req->Invoke().Apply(BIND([] (const TApiServiceProxy::TRspGetJobTracePtr& rsp) { return FromProto<std::vector<TJobTraceEvent>>(rsp->events()); @@ -1425,19 +1368,11 @@ TFuture<TListOperationsResult> TClient::ListOperations( auto req = proxy.ListOperations(); SetTimeoutOptions(*req, options); - if (options.FromTime) { - req->set_from_time(NYT::ToProto(*options.FromTime)); - } - if (options.ToTime) { - req->set_to_time(NYT::ToProto(*options.ToTime)); - } - if (options.CursorTime) { - req->set_cursor_time(NYT::ToProto(*options.CursorTime)); - } + YT_OPTIONAL_SET_PROTO(req, from_time, options.FromTime); + YT_OPTIONAL_SET_PROTO(req, to_time, options.ToTime); + YT_OPTIONAL_SET_PROTO(req, cursor_time, options.CursorTime); req->set_cursor_direction(static_cast<NProto::EOperationSortDirection>(options.CursorDirection)); - if (options.UserFilter) { - req->set_user_filter(*options.UserFilter); - } + YT_OPTIONAL_TO_PROTO(req, user_filter, options.UserFilter); if (options.AccessFilter) { req->set_access_filter(ConvertToYsonString(options.AccessFilter).ToString()); @@ -1449,15 +1384,9 @@ TFuture<TListOperationsResult> TClient::ListOperations( if (options.TypeFilter) { req->set_type_filter(NProto::ConvertOperationTypeToProto(*options.TypeFilter)); } - if (options.SubstrFilter) { - req->set_substr_filter(*options.SubstrFilter); - } - if (options.Pool) { - req->set_pool(*options.Pool); - } - if (options.PoolTree) { - req->set_pool_tree(*options.PoolTree); - } + YT_OPTIONAL_TO_PROTO(req, substr_filter, options.SubstrFilter); + YT_OPTIONAL_TO_PROTO(req, pool, options.Pool); + YT_OPTIONAL_TO_PROTO(req, pool_tree, options.PoolTree); if (options.WithFailedJobs) { req->set_with_failed_jobs(*options.WithFailedJobs); } diff --git a/yt/yt/client/api/rpc_proxy/helpers.cpp b/yt/yt/client/api/rpc_proxy/helpers.cpp index c8acdeb3f8..f445a9c17f 100644 --- a/yt/yt/client/api/rpc_proxy/helpers.cpp +++ b/yt/yt/client/api/rpc_proxy/helpers.cpp @@ -96,9 +96,7 @@ void ToProto( proto->set_expire_after_successful_update_time(ToProto(options.ExpireAfterSuccessfulUpdateTime)); proto->set_expire_after_failed_update_time(ToProto(options.ExpireAfterFailedUpdateTime)); proto->set_success_staleness_bound(ToProto(options.SuccessStalenessBound)); - if (options.CacheStickyGroupSize) { - proto->set_cache_sticky_group_size(*options.CacheStickyGroupSize); - } + YT_OPTIONAL_SET_PROTO(proto, cache_sticky_group_size, options.CacheStickyGroupSize); } void ToProto( @@ -122,12 +120,8 @@ void ToProto( NProto::TTabletRangeOptions* proto, const NApi::TTabletRangeOptions& options) { - if (options.FirstTabletIndex) { - proto->set_first_tablet_index(*options.FirstTabletIndex); - } - if (options.LastTabletIndex) { - proto->set_last_tablet_index(*options.LastTabletIndex); - } + YT_OPTIONAL_SET_PROTO(proto, first_tablet_index, options.FirstTabletIndex); + YT_OPTIONAL_SET_PROTO(proto, last_tablet_index, options.LastTabletIndex); } void ToProto( @@ -135,9 +129,7 @@ void ToProto( const NApi::TTabletReadOptionsBase& options) { protoOptions->set_read_from(static_cast<NProto::ETabletReadKind>(options.ReadFrom)); - if (options.CachedSyncReplicasTimeout) { - protoOptions->set_cached_sync_replicas_timeout(ToProto(*options.CachedSyncReplicasTimeout)); - } + YT_OPTIONAL_SET_PROTO(protoOptions, cached_sync_replicas_timeout, options.CachedSyncReplicasTimeout); } //////////////////////////////////////////////////////////////////////////////// @@ -207,14 +199,10 @@ void ToProto( proto->set_action(static_cast<NProto::ESecurityAction>(result.Action)); ToProto(proto->mutable_object_id(), result.ObjectId); - if (result.ObjectName) { - proto->set_object_name(*result.ObjectName); - } + YT_OPTIONAL_TO_PROTO(proto, object_name, result.ObjectName); ToProto(proto->mutable_subject_id(), result.SubjectId); - if (result.SubjectName) { - proto->set_subject_name(*result.SubjectName); - } + YT_OPTIONAL_TO_PROTO(proto, subject_name, result.SubjectName); } void FromProto( @@ -224,18 +212,10 @@ void FromProto( result->Action = static_cast<NSecurityClient::ESecurityAction>(proto.action()); FromProto(&result->ObjectId, proto.object_id()); - if (proto.has_object_name()) { - result->ObjectName = proto.object_name(); - } else { - result->ObjectName.reset(); - } + result->ObjectName = YT_OPTIONAL_FROM_PROTO(proto, object_name); FromProto(&result->SubjectId, proto.subject_id()); - if (proto.has_subject_name()) { - result->SubjectName = proto.subject_name(); - } else { - result->SubjectName.reset(); - } + result->SubjectName = YT_OPTIONAL_FROM_PROTO(proto, subject_name); } void ToProto( @@ -247,9 +227,7 @@ void ToProto( proto->set_action(static_cast<NProto::ESecurityAction>(result.Action)); ToProto(proto->mutable_subject_id(), result.SubjectId); - if (result.SubjectName) { - proto->set_subject_name(*result.SubjectName); - } + YT_OPTIONAL_TO_PROTO(proto, subject_name, result.SubjectName); ToProto(proto->mutable_missing_subjects(), result.MissingSubjects); } @@ -261,11 +239,7 @@ void FromProto( result->Action = static_cast<NSecurityClient::ESecurityAction>(proto.action()); FromProto(&result->SubjectId, proto.subject_id()); - if (proto.has_subject_name()) { - result->SubjectName = proto.subject_name(); - } else { - result->SubjectName.reset(); - } + result->SubjectName = YT_OPTIONAL_FROM_PROTO(proto, subject_name); FromProto(&result->MissingSubjects, proto.missing_subjects()); } @@ -317,9 +291,7 @@ void ToProto( } } - if (result.FailedJobsCount) { - proto->set_failed_jobs_count(*result.FailedJobsCount); - } + YT_OPTIONAL_SET_PROTO(proto, failed_jobs_count, result.FailedJobsCount); proto->set_incomplete(result.Incomplete); } @@ -400,15 +372,9 @@ void ToProto( proto->Clear(); ToProto(proto->mutable_jobs(), result.Jobs); - if (result.CypressJobCount) { - proto->set_cypress_job_count(*result.CypressJobCount); - } - if (result.ControllerAgentJobCount) { - proto->set_controller_agent_job_count(*result.ControllerAgentJobCount); - } - if (result.ArchiveJobCount) { - proto->set_archive_job_count(*result.ArchiveJobCount); - } + YT_OPTIONAL_SET_PROTO(proto, cypress_job_count, result.CypressJobCount); + YT_OPTIONAL_SET_PROTO(proto, controller_agent_job_count, result.ControllerAgentJobCount); + YT_OPTIONAL_SET_PROTO(proto, archive_job_count, result.ArchiveJobCount); if (result.ContinuationToken) { proto->set_continuation_token(*result.ContinuationToken); } @@ -423,26 +389,10 @@ void FromProto( { FromProto(&result->Jobs, proto.jobs()); - if (proto.has_cypress_job_count()) { - result->CypressJobCount = proto.cypress_job_count(); - } else { - result->CypressJobCount.reset(); - } - if (proto.has_controller_agent_job_count()) { - result->ControllerAgentJobCount = proto.controller_agent_job_count(); - } else { - result->ControllerAgentJobCount.reset(); - } - if (proto.has_archive_job_count()) { - result->ArchiveJobCount = proto.archive_job_count(); - } else { - result->ArchiveJobCount.reset(); - } - if (proto.has_continuation_token()) { - result->ContinuationToken = proto.continuation_token(); - } else { - result->ContinuationToken.reset(); - } + result->CypressJobCount = YT_OPTIONAL_FROM_PROTO(proto, cypress_job_count); + result->ControllerAgentJobCount = YT_OPTIONAL_FROM_PROTO(proto, controller_agent_job_count); + result->ArchiveJobCount = YT_OPTIONAL_FROM_PROTO(proto, archive_job_count); + result->ContinuationToken = YT_OPTIONAL_FROM_PROTO(proto, continuation_token); FromProto(&result->Statistics, proto.statistics()); FromProto(&result->Errors, proto.errors()); @@ -483,46 +433,18 @@ void ToProto(NProto::TColumnSchema* protoSchema, const NTableClient::TColumnSche protoSchema->set_type(ToProto(GetPhysicalType(schema.CastToV1Type()))); auto typeV3Yson = ConvertToYsonString(TTypeV3LogicalTypeWrapper{schema.LogicalType()}); protoSchema->set_type_v3(typeV3Yson.ToString()); - if (schema.Lock()) { - protoSchema->set_lock(ToProto(*schema.Lock())); - } else { - protoSchema->clear_lock(); - } - if (schema.Expression()) { - protoSchema->set_expression(*schema.Expression()); - } else { - protoSchema->clear_expression(); - } - if (schema.Materialized()) { - protoSchema->set_materialized(*schema.Materialized()); - } else { - protoSchema->clear_materialized(); - } - if (schema.Aggregate()) { - protoSchema->set_aggregate(ToProto(*schema.Aggregate())); - } else { - protoSchema->clear_aggregate(); - } - if (schema.SortOrder()) { - protoSchema->set_sort_order(ToProto(*schema.SortOrder())); - } else { - protoSchema->clear_sort_order(); - } - if (schema.Group()) { - protoSchema->set_group(ToProto(*schema.Group())); - } else { - protoSchema->clear_group(); - } + YT_OPTIONAL_TO_PROTO(protoSchema, lock, schema.Lock()); + YT_OPTIONAL_TO_PROTO(protoSchema, expression, schema.Expression()); + YT_OPTIONAL_SET_PROTO(protoSchema, materialized, schema.Materialized()); + YT_OPTIONAL_TO_PROTO(protoSchema, aggregate, schema.Aggregate()); + YT_OPTIONAL_SET_PROTO(protoSchema, sort_order, schema.SortOrder()); + YT_OPTIONAL_TO_PROTO(protoSchema, group, schema.Group()); if (schema.Required()) { protoSchema->set_required(schema.Required()); } else { protoSchema->clear_required(); } - if (schema.MaxInlineHunkSize()) { - protoSchema->set_max_inline_hunk_size(*schema.MaxInlineHunkSize()); - } else { - protoSchema->clear_max_inline_hunk_size(); - } + YT_OPTIONAL_SET_PROTO(protoSchema, max_inline_hunk_size, schema.MaxInlineHunkSize()); } void FromProto(NTableClient::TColumnSchema* schema, const NProto::TColumnSchema& protoSchema) @@ -571,13 +493,13 @@ void FromProto(NTableClient::TColumnSchema* schema, const NProto::TColumnSchema& } schema->SetLogicalType(std::move(columnType)); - schema->SetLock(YT_PROTO_OPTIONAL(protoSchema, lock)); - schema->SetExpression(YT_PROTO_OPTIONAL(protoSchema, expression)); - schema->SetMaterialized(YT_PROTO_OPTIONAL(protoSchema, materialized)); - schema->SetAggregate(YT_PROTO_OPTIONAL(protoSchema, aggregate)); + schema->SetLock(YT_OPTIONAL_FROM_PROTO(protoSchema, lock)); + schema->SetExpression(YT_OPTIONAL_FROM_PROTO(protoSchema, expression)); + schema->SetMaterialized(YT_OPTIONAL_FROM_PROTO(protoSchema, materialized)); + schema->SetAggregate(YT_OPTIONAL_FROM_PROTO(protoSchema, aggregate)); schema->SetSortOrder(YT_APPLY_PROTO_OPTIONAL(protoSchema, sort_order, FromProto<ESortOrder>)); - schema->SetGroup(YT_PROTO_OPTIONAL(protoSchema, group)); - schema->SetMaxInlineHunkSize(YT_PROTO_OPTIONAL(protoSchema, max_inline_hunk_size)); + schema->SetGroup(YT_OPTIONAL_FROM_PROTO(protoSchema, group)); + schema->SetMaxInlineHunkSize(YT_OPTIONAL_FROM_PROTO(protoSchema, max_inline_hunk_size)); } void ToProto(NProto::TTableSchema* protoSchema, const NTableClient::TTableSchema& schema) @@ -773,38 +695,13 @@ void ToProto(NProto::TOperation* protoOperation, const NApi::TOperation& operati void FromProto(NApi::TOperation* operation, const NProto::TOperation& protoOperation) { - if (protoOperation.has_id()) { - operation->Id = FromProto<NScheduler::TOperationId>(protoOperation.id()); - } else { - operation->Id.reset(); - } - if (protoOperation.has_type()) { - operation->Type = ConvertOperationTypeFromProto(protoOperation.type()); - } else { - operation->Type.reset(); - } - if (protoOperation.has_state()) { - operation->State = ConvertOperationStateFromProto(protoOperation.state()); - } else { - operation->State.reset(); - } + operation->Id = YT_APPLY_PROTO_OPTIONAL(protoOperation, id, FromProto<NScheduler::TOperationId>); + operation->Type = YT_APPLY_PROTO_OPTIONAL(protoOperation, type, ConvertOperationTypeFromProto); + operation->State = YT_APPLY_PROTO_OPTIONAL(protoOperation, state, ConvertOperationStateFromProto); - if (protoOperation.has_start_time()) { - operation->StartTime = TInstant::FromValue(protoOperation.start_time()); - } else { - operation->StartTime.reset(); - } - if (protoOperation.has_finish_time()) { - operation->FinishTime = TInstant::FromValue(protoOperation.finish_time()); - } else { - operation->FinishTime.reset(); - } - - if (protoOperation.has_authenticated_user()) { - operation->AuthenticatedUser = protoOperation.authenticated_user(); - } else { - operation->AuthenticatedUser.reset(); - } + operation->StartTime = YT_OPTIONAL_FROM_PROTO(protoOperation, start_time, TInstant); + operation->FinishTime = YT_OPTIONAL_FROM_PROTO(protoOperation, finish_time, TInstant); + operation->AuthenticatedUser = YT_OPTIONAL_FROM_PROTO(protoOperation, authenticated_user); if (protoOperation.has_brief_spec()) { operation->BriefSpec = TYsonString(protoOperation.brief_spec()); @@ -860,11 +757,7 @@ void FromProto(NApi::TOperation* operation, const NProto::TOperation& protoOpera operation->RuntimeParameters = TYsonString(); } - if (protoOperation.has_suspended()) { - operation->Suspended = protoOperation.suspended(); - } else { - operation->Suspended.reset(); - } + operation->Suspended = YT_OPTIONAL_FROM_PROTO(protoOperation, suspended); if (protoOperation.has_events()) { operation->Events = TYsonString(protoOperation.events()); @@ -943,28 +836,16 @@ void ToProto(NProto::TJob* protoJob, const NApi::TJob& job) protoJob->set_archive_state(ConvertJobStateToProto(*job.ArchiveState)); } - if (job.StartTime) { - protoJob->set_start_time(ToProto(*job.StartTime)); - } - if (job.FinishTime) { - protoJob->set_finish_time(ToProto(*job.FinishTime)); - } + YT_OPTIONAL_SET_PROTO(protoJob, start_time, job.StartTime); + YT_OPTIONAL_SET_PROTO(protoJob, finish_time, job.FinishTime); - if (job.Address) { - protoJob->set_address(*job.Address); - } + YT_OPTIONAL_TO_PROTO(protoJob, address, job.Address); if (job.Progress) { protoJob->set_progress(*job.Progress); } - if (job.StderrSize) { - protoJob->set_stderr_size(*job.StderrSize); - } - if (job.FailContextSize) { - protoJob->set_fail_context_size(*job.FailContextSize); - } - if (job.HasSpec) { - protoJob->set_has_spec(*job.HasSpec); - } + YT_OPTIONAL_SET_PROTO(protoJob, stderr_size, job.StderrSize); + YT_OPTIONAL_SET_PROTO(protoJob, fail_context_size, job.FailContextSize); + YT_OPTIONAL_SET_PROTO(protoJob, has_spec, job.HasSpec); if (job.Error) { protoJob->set_error(job.Error.ToString()); @@ -988,39 +869,21 @@ void ToProto(NProto::TJob* protoJob, const NApi::TJob& job) if (job.ProbingJobCompetitionId) { ToProto(protoJob->mutable_probing_job_competition_id(), job.ProbingJobCompetitionId); } - if (job.HasCompetitors) { - protoJob->set_has_competitors(*job.HasCompetitors); - } - if (job.HasProbingCompetitors) { - protoJob->set_has_probing_competitors(*job.HasProbingCompetitors); - } - if (job.IsStale) { - protoJob->set_is_stale(*job.IsStale); - } + YT_OPTIONAL_SET_PROTO(protoJob, has_competitors, job.HasCompetitors); + YT_OPTIONAL_SET_PROTO(protoJob, has_probing_competitors, job.HasProbingCompetitors); + YT_OPTIONAL_SET_PROTO(protoJob, is_stale, job.IsStale); if (job.ExecAttributes) { protoJob->set_exec_attributes(job.ExecAttributes.ToString()); } - if (job.TaskName) { - protoJob->set_task_name(*job.TaskName); - } - if (job.PoolTree) { - protoJob->set_pool_tree(*job.PoolTree); - } - if (job.Pool) { - protoJob->set_pool(*job.Pool); - } - if (job.JobCookie) { - protoJob->set_job_cookie(*job.JobCookie); - } + YT_OPTIONAL_TO_PROTO(protoJob, task_name, job.TaskName); + YT_OPTIONAL_TO_PROTO(protoJob, pool_tree, job.PoolTree); + YT_OPTIONAL_TO_PROTO(protoJob, pool, job.Pool); + YT_OPTIONAL_SET_PROTO(protoJob, job_cookie, job.JobCookie); if (job.ArchiveFeatures) { protoJob->set_archive_features(job.ArchiveFeatures.ToString()); } - if (job.MonitoringDescriptor) { - protoJob->set_monitoring_descriptor(*job.MonitoringDescriptor); - } - if (job.OperationIncarnation) { - protoJob->set_operation_incarnation(*job.OperationIncarnation); - } + YT_OPTIONAL_TO_PROTO(protoJob, monitoring_descriptor, job.MonitoringDescriptor); + YT_OPTIONAL_SET_PROTO(protoJob, operation_incarnation, job.OperationIncarnation); } void FromProto(NApi::TJob* job, const NProto::TJob& protoJob) @@ -1035,51 +898,15 @@ void FromProto(NApi::TJob* job, const NProto::TJob& protoJob) } else { job->OperationId = {}; } - if (protoJob.has_type()) { - job->Type = ConvertJobTypeFromProto(protoJob.type()); - } else { - job->Type.reset(); - } - if (protoJob.has_controller_state()) { - job->ControllerState = ConvertJobStateFromProto(protoJob.controller_state()); - } else { - job->ControllerState.reset(); - } - if (protoJob.has_archive_state()) { - job->ArchiveState = ConvertJobStateFromProto(protoJob.archive_state()); - } else { - job->ArchiveState.reset(); - } - if (protoJob.has_start_time()) { - job->StartTime = TInstant::FromValue(protoJob.start_time()); - } else { - job->StartTime.reset(); - } - if (protoJob.has_finish_time()) { - job->FinishTime = TInstant::FromValue(protoJob.finish_time()); - } else { - job->FinishTime.reset(); - } - if (protoJob.has_address()) { - job->Address = protoJob.address(); - } else { - job->Address.reset(); - } - if (protoJob.has_progress()) { - job->Progress = protoJob.progress(); - } else { - job->Progress.reset(); - } - if (protoJob.has_stderr_size()) { - job->StderrSize = protoJob.stderr_size(); - } else { - job->StderrSize.reset(); - } - if (protoJob.has_fail_context_size()) { - job->FailContextSize = protoJob.fail_context_size(); - } else { - job->FailContextSize.reset(); - } + job->Type = YT_APPLY_PROTO_OPTIONAL(protoJob, type, ConvertJobTypeFromProto); + job->ControllerState = YT_APPLY_PROTO_OPTIONAL(protoJob, controller_state, ConvertJobStateFromProto); + job->ArchiveState = YT_APPLY_PROTO_OPTIONAL(protoJob, archive_state, ConvertJobStateFromProto); + job->StartTime = YT_OPTIONAL_FROM_PROTO(protoJob, start_time, TInstant); + job->FinishTime = YT_OPTIONAL_FROM_PROTO(protoJob, finish_time, TInstant); + job->Address = YT_OPTIONAL_FROM_PROTO(protoJob, address); + job->Progress = YT_OPTIONAL_FROM_PROTO(protoJob, progress); + job->StderrSize = YT_OPTIONAL_FROM_PROTO(protoJob, stderr_size); + job->FailContextSize = YT_OPTIONAL_FROM_PROTO(protoJob, fail_context_size); if (protoJob.has_has_spec()) { job->HasSpec = protoJob.has_spec(); } else { @@ -1125,56 +952,24 @@ void FromProto(NApi::TJob* job, const NProto::TJob& protoJob) } else { job->HasCompetitors = false; } - if (protoJob.has_has_probing_competitors()) { - job->HasProbingCompetitors = protoJob.has_probing_competitors(); - } else { - job->HasProbingCompetitors = false; - } - if (protoJob.has_is_stale()) { - job->IsStale = protoJob.is_stale(); - } else { - job->IsStale.reset(); - } + job->HasProbingCompetitors = YT_OPTIONAL_FROM_PROTO(protoJob, has_probing_competitors); + job->IsStale = YT_OPTIONAL_FROM_PROTO(protoJob, is_stale); if (protoJob.has_exec_attributes()) { job->ExecAttributes = TYsonString(protoJob.exec_attributes()); } else { job->ExecAttributes = TYsonString(); } - if (protoJob.has_task_name()) { - job->TaskName = protoJob.task_name(); - } else { - job->TaskName.reset(); - } - if (protoJob.has_pool_tree()) { - job->PoolTree = protoJob.pool_tree(); - } else { - job->PoolTree.reset(); - } - if (protoJob.has_pool()) { - job->Pool = protoJob.pool(); - } else { - job->Pool.reset(); - } - if (protoJob.has_job_cookie()) { - job->JobCookie = protoJob.job_cookie(); - } else { - job->JobCookie.reset(); - } + job->TaskName = YT_OPTIONAL_FROM_PROTO(protoJob, task_name); + job->PoolTree = YT_OPTIONAL_FROM_PROTO(protoJob, pool_tree); + job->Pool = YT_OPTIONAL_FROM_PROTO(protoJob, pool); + job->JobCookie = YT_OPTIONAL_FROM_PROTO(protoJob, job_cookie); if (protoJob.has_archive_features()) { job->ArchiveFeatures = TYsonString(protoJob.archive_features()); } else { job->ArchiveFeatures = TYsonString(); } - if (protoJob.has_monitoring_descriptor()) { - job->MonitoringDescriptor = protoJob.monitoring_descriptor(); - } else { - job->MonitoringDescriptor.reset(); - } - if (protoJob.has_operation_incarnation()) { - job->OperationIncarnation = protoJob.operation_incarnation(); - } else { - job->OperationIncarnation.reset(); - } + job->MonitoringDescriptor = YT_OPTIONAL_FROM_PROTO(protoJob, monitoring_descriptor); + job->OperationIncarnation = YT_OPTIONAL_FROM_PROTO(protoJob, operation_incarnation); } void ToProto( @@ -1265,21 +1060,15 @@ void ToProto( protoStatistics->Clear(); ToProto(protoStatistics->mutable_column_data_weights(), statistics.ColumnDataWeights); - if (statistics.TimestampTotalWeight) { - protoStatistics->set_timestamp_total_weight(*statistics.TimestampTotalWeight); - } + YT_OPTIONAL_SET_PROTO(protoStatistics, timestamp_total_weight, statistics.TimestampTotalWeight); protoStatistics->set_legacy_chunk_data_weight(statistics.LegacyChunkDataWeight); NYT::NTableClient::ToProto(protoStatistics->mutable_column_min_values(), statistics.ColumnMinValues); NYT::NTableClient::ToProto(protoStatistics->mutable_column_max_values(), statistics.ColumnMaxValues); ToProto(protoStatistics->mutable_column_non_null_value_counts(), statistics.ColumnNonNullValueCounts); - if (statistics.ChunkRowCount) { - protoStatistics->set_chunk_row_count(*statistics.ChunkRowCount); - } - if (statistics.LegacyChunkRowCount) { - protoStatistics->set_legacy_chunk_row_count(*statistics.LegacyChunkRowCount); - } + YT_OPTIONAL_SET_PROTO(protoStatistics, chunk_row_count, statistics.ChunkRowCount); + YT_OPTIONAL_SET_PROTO(protoStatistics, legacy_chunk_row_count, statistics.LegacyChunkRowCount); ToProto(protoStatistics->mutable_column_hyperloglog_digests(), statistics.LargeStatistics.ColumnHyperLogLogDigests); } @@ -1289,27 +1078,15 @@ void FromProto( const NProto::TColumnarStatistics& protoStatistics) { FromProto(&statistics->ColumnDataWeights, protoStatistics.column_data_weights()); - if (protoStatistics.has_timestamp_total_weight()) { - statistics->TimestampTotalWeight = protoStatistics.timestamp_total_weight(); - } else { - statistics->TimestampTotalWeight.reset(); - } + statistics->TimestampTotalWeight = YT_OPTIONAL_FROM_PROTO(protoStatistics, timestamp_total_weight); statistics->LegacyChunkDataWeight = protoStatistics.legacy_chunk_data_weight(); NYT::NTableClient::FromProto(&statistics->ColumnMinValues, protoStatistics.column_min_values()); NYT::NTableClient::FromProto(&statistics->ColumnMaxValues, protoStatistics.column_max_values()); FromProto(&statistics->ColumnNonNullValueCounts, protoStatistics.column_non_null_value_counts()); - if (protoStatistics.has_chunk_row_count()) { - statistics->ChunkRowCount = protoStatistics.chunk_row_count(); - } else { - statistics->ChunkRowCount.reset(); - } - if (protoStatistics.has_legacy_chunk_row_count()) { - statistics->LegacyChunkRowCount = protoStatistics.legacy_chunk_row_count(); - } else { - statistics->LegacyChunkRowCount.reset(); - } + statistics->ChunkRowCount = YT_OPTIONAL_FROM_PROTO(protoStatistics, chunk_row_count); + statistics->LegacyChunkRowCount = YT_OPTIONAL_FROM_PROTO(protoStatistics, legacy_chunk_row_count); FromProto(&statistics->LargeStatistics.ColumnHyperLogLogDigests, protoStatistics.column_hyperloglog_digests()); } @@ -1361,9 +1138,7 @@ void ToProto( { proto->set_max_row_count(result.MaxRowCount); proto->set_max_data_weight(result.MaxDataWeight); - if (result.DataWeightPerRowHint) { - proto->set_data_weight_per_row_hint(*result.DataWeightPerRowHint); - } + YT_OPTIONAL_SET_PROTO(proto, data_weight_per_row_hint, result.DataWeightPerRowHint); } void FromProto( @@ -1372,9 +1147,7 @@ void FromProto( { result->MaxRowCount = proto.max_row_count(); result->MaxDataWeight = proto.max_data_weight(); - if (proto.has_data_weight_per_row_hint()) { - result->DataWeightPerRowHint = proto.data_weight_per_row_hint(); - } + result->DataWeightPerRowHint = YT_OPTIONAL_FROM_PROTO(proto, data_weight_per_row_hint); } void ToProto( @@ -1438,24 +1211,16 @@ void ToProto( if (query.Engine) { protoQuery->set_engine(ConvertQueryEngineToProto(*query.Engine)); } - if (query.Query) { - protoQuery->set_query(*query.Query); - } + YT_OPTIONAL_TO_PROTO(protoQuery, query, query.Query); if (query.Files) { protoQuery->set_files(query.Files->ToString()); } - if (query.StartTime) { - protoQuery->set_start_time(NYT::ToProto(*query.StartTime)); - } - if (query.FinishTime) { - protoQuery->set_finish_time(NYT::ToProto(*query.FinishTime)); - } + YT_OPTIONAL_SET_PROTO(protoQuery, start_time, query.StartTime); + YT_OPTIONAL_SET_PROTO(protoQuery, finish_time, query.FinishTime); if (query.Settings) { protoQuery->set_settings(query.Settings.ToString()); } - if (query.User) { - protoQuery->set_user(*query.User); - } + YT_OPTIONAL_TO_PROTO(protoQuery, user, query.User); if (query.AccessControlObject) { protoQuery->set_access_control_object(*query.AccessControlObject); } @@ -1464,15 +1229,11 @@ void ToProto( if (query.State) { protoQuery->set_state(ConvertQueryStateToProto(*query.State)); } - if (query.ResultCount) { - protoQuery->set_result_count(*query.ResultCount); - } + YT_OPTIONAL_SET_PROTO(protoQuery, result_count, query.ResultCount); if (query.Progress) { protoQuery->set_progress(query.Progress.ToString()); } - if (query.Error) { - ToProto(protoQuery->mutable_error(), *query.Error); - } + YT_OPTIONAL_TO_PROTO(protoQuery, error, query.Error); if (query.Annotations) { protoQuery->set_annotations(query.Annotations.ToString()); } @@ -1487,71 +1248,27 @@ void FromProto( { FromProto(&query->Id, protoQuery.id()); - if (protoQuery.has_engine()) { - query->Engine = ConvertQueryEngineFromProto(protoQuery.engine()); - } else { - query->Engine.reset(); - } - if (protoQuery.has_query()) { - query->Query = protoQuery.query(); - } else { - query->Query.reset(); - } - if (protoQuery.has_files()) { - query->Files = TYsonString(protoQuery.files()); - } else { - query->Files.reset(); - } - if (protoQuery.has_start_time()) { - query->StartTime = TInstant::FromValue(protoQuery.start_time()); - } else { - query->StartTime.reset(); - } - if (protoQuery.has_finish_time()) { - query->FinishTime = TInstant::FromValue(protoQuery.finish_time()); - } else { - query->FinishTime.reset(); - } + query->Engine = YT_APPLY_PROTO_OPTIONAL(protoQuery, engine, ConvertQueryEngineFromProto); + query->Query = YT_OPTIONAL_FROM_PROTO(protoQuery, query); + query->Files = YT_APPLY_PROTO_OPTIONAL(protoQuery, files, TYsonString); + query->StartTime = YT_OPTIONAL_FROM_PROTO(protoQuery, start_time, TInstant); + query->FinishTime = YT_OPTIONAL_FROM_PROTO(protoQuery, finish_time, TInstant); if (protoQuery.has_settings()) { query->Settings = TYsonString(protoQuery.settings()); } else { query->Settings = TYsonString{}; } - if (protoQuery.has_user()) { - query->User = protoQuery.user(); - } else { - query->User.reset(); - } - if (protoQuery.has_access_control_object()) { - query->AccessControlObject = protoQuery.access_control_object(); - } else { - query->AccessControlObject.reset(); - } - if (protoQuery.has_access_control_objects()) { - query->AccessControlObjects = TYsonString(protoQuery.access_control_objects()); - } else { - query->AccessControlObjects.reset(); - } - if (protoQuery.has_state()) { - query->State = ConvertQueryStateFromProto(protoQuery.state()); - } else { - query->State.reset(); - } - if (protoQuery.has_result_count()) { - query->ResultCount = protoQuery.result_count(); - } else { - query->ResultCount.reset(); - } + query->User = YT_OPTIONAL_FROM_PROTO(protoQuery, user); + query->AccessControlObject = YT_OPTIONAL_FROM_PROTO(protoQuery, access_control_object); + query->AccessControlObjects = YT_APPLY_PROTO_OPTIONAL(protoQuery, access_control_objects, TYsonString); + query->State = YT_APPLY_PROTO_OPTIONAL(protoQuery, state, ConvertQueryStateFromProto); + query->ResultCount = YT_OPTIONAL_FROM_PROTO(protoQuery, result_count); if (protoQuery.has_progress()) { query->Progress = TYsonString(protoQuery.progress()); } else { query->Progress = TYsonString{}; } - if (protoQuery.has_error()) { - query->Error = FromProto<TError>(protoQuery.error()); - } else { - query->Error.reset(); - } + query->Error = YT_APPLY_PROTO_OPTIONAL(protoQuery, error, FromProto<TError>); if (protoQuery.has_annotations()) { query->Annotations = TYsonString(protoQuery.annotations()); } else { diff --git a/yt/yt/client/api/rpc_proxy/table_mount_cache.cpp b/yt/yt/client/api/rpc_proxy/table_mount_cache.cpp index d0a11cb943..8aaa7aeed3 100644 --- a/yt/yt/client/api/rpc_proxy/table_mount_cache.cpp +++ b/yt/yt/client/api/rpc_proxy/table_mount_cache.cpp @@ -103,8 +103,8 @@ private: auto indexInfo = TIndexInfo{ .TableId = FromProto<NObjectClient::TObjectId>(protoIndexInfo.index_table_id()), .Kind = FromProto<ESecondaryIndexKind>(protoIndexInfo.index_kind()), - .Predicate = YT_PROTO_OPTIONAL(protoIndexInfo, predicate), - .UnfoldedColumn = YT_PROTO_OPTIONAL(protoIndexInfo, unfolded_column), + .Predicate = YT_OPTIONAL_FROM_PROTO(protoIndexInfo, predicate), + .UnfoldedColumn = YT_OPTIONAL_FROM_PROTO(protoIndexInfo, unfolded_column), .Correspondence = protoIndexInfo.has_index_correspondence() ? FromProto<ETableToIndexCorrespondence>(protoIndexInfo.index_correspondence()) : ETableToIndexCorrespondence::Unknown, diff --git a/yt/yt/client/driver/etc_commands.cpp b/yt/yt/client/driver/etc_commands.cpp index c1267fdcdb..debe2a3ffc 100644 --- a/yt/yt/client/driver/etc_commands.cpp +++ b/yt/yt/client/driver/etc_commands.cpp @@ -408,8 +408,9 @@ void TExecuteBatchCommand::DoExecute(ICommandContextPtr context) void TDiscoverProxiesCommand::Register(TRegistrar registrar) { - registrar.Parameter("type", &TThis::Type) - .Default(EProxyType::Rpc); + registrar.Parameter("kind", &TThis::Kind) + .Alias("type") + .Default(EProxyKind::Rpc); registrar.Parameter("role", &TThis::Role) .Default(DefaultRpcProxyRole); registrar.Parameter("address_type", &TThis::AddressType) @@ -423,11 +424,11 @@ void TDiscoverProxiesCommand::Register(TRegistrar registrar) void TDiscoverProxiesCommand::DoExecute(ICommandContextPtr context) { TProxyDiscoveryRequest request{ - .Type = Type, + .Kind = Kind, .Role = Role, .AddressType = AddressType, .NetworkName = NetworkName, - .IgnoreBalancers = IgnoreBalancers + .IgnoreBalancers = IgnoreBalancers, }; const auto& proxyDiscoveryCache = context->GetDriver()->GetProxyDiscoveryCache(); diff --git a/yt/yt/client/driver/etc_commands.h b/yt/yt/client/driver/etc_commands.h index 789f8690d4..f51670a95e 100644 --- a/yt/yt/client/driver/etc_commands.h +++ b/yt/yt/client/driver/etc_commands.h @@ -230,7 +230,7 @@ public: static void Register(TRegistrar registrar); private: - NApi::EProxyType Type; + NApi::EProxyKind Kind; std::string Role; NApi::NRpcProxy::EAddressType AddressType; std::string NetworkName; diff --git a/yt/yt/client/driver/proxy_discovery_cache.cpp b/yt/yt/client/driver/proxy_discovery_cache.cpp index a9612bb359..e025f59105 100644 --- a/yt/yt/client/driver/proxy_discovery_cache.cpp +++ b/yt/yt/client/driver/proxy_discovery_cache.cpp @@ -26,7 +26,7 @@ using namespace NApi::NRpcProxy; TProxyDiscoveryRequest::operator size_t() const { return MultiHash( - Type, + Kind, Role, AddressType, NetworkName, @@ -37,8 +37,8 @@ TProxyDiscoveryRequest::operator size_t() const void FormatValue(TStringBuilderBase* builder, const TProxyDiscoveryRequest& request, TStringBuf /*spec*/) { - builder->AppendFormat("{Type: %v, Role: %v, AddressType: %v, NetworkName: %v, IgnoreBalancers: %v}", - request.Type, + builder->AppendFormat("{Kind: %v, Role: %v, AddressType: %v, NetworkName: %v, IgnoreBalancers: %v}", + request.Kind, request.Role, request.AddressType, request.NetworkName, @@ -97,7 +97,7 @@ private: TYPath path; try { - path = GetProxyRegistryPath(request.Type) + "/@"; + path = GetProxyRegistryPath(request.Kind) + "/@"; } catch (const std::exception& ex) { YT_LOG_ERROR(ex, "Failed to get proxy registry path"); return MakeFuture<std::optional<TProxyDiscoveryResponse>>(ex); @@ -130,7 +130,7 @@ private: TYPath path; try { - path = GetProxyRegistryPath(request.Type); + path = GetProxyRegistryPath(request.Kind); } catch (const std::exception& ex) { YT_LOG_ERROR(ex, "Failed to get proxy registry path"); return MakeFuture<TProxyDiscoveryResponse>(ex); @@ -168,12 +168,12 @@ private: } - static TYPath GetProxyRegistryPath(EProxyType type) + static TYPath GetProxyRegistryPath(EProxyKind type) { switch (type) { - case EProxyType::Rpc: + case EProxyKind::Rpc: return RpcProxiesPath; - case EProxyType::Grpc: + case EProxyKind::Grpc: return GrpcProxiesPath; default: THROW_ERROR_EXCEPTION("Proxy type %Qlv is not supported", diff --git a/yt/yt/client/driver/proxy_discovery_cache.h b/yt/yt/client/driver/proxy_discovery_cache.h index 9564dbdf67..814e440bd4 100644 --- a/yt/yt/client/driver/proxy_discovery_cache.h +++ b/yt/yt/client/driver/proxy_discovery_cache.h @@ -14,7 +14,7 @@ namespace NYT::NDriver { struct TProxyDiscoveryRequest { - NApi::EProxyType Type; + NApi::EProxyKind Kind = NApi::EProxyKind::Rpc; std::string Role = NApi::DefaultRpcProxyRole; NApi::NRpcProxy::EAddressType AddressType = NApi::NRpcProxy::DefaultAddressType; std::string NetworkName = NApi::NRpcProxy::DefaultNetworkName; diff --git a/yt/yt/client/node_tracker_client/node_directory.cpp b/yt/yt/client/node_tracker_client/node_directory.cpp index d3e08b7d84..c9bf77353d 100644 --- a/yt/yt/client/node_tracker_client/node_directory.cpp +++ b/yt/yt/client/node_tracker_client/node_directory.cpp @@ -344,31 +344,13 @@ void ToProto(NNodeTrackerClient::NProto::TNodeDescriptor* protoDescriptor, const ToProto(protoDescriptor->mutable_addresses(), descriptor.Addresses()); - if (auto host = descriptor.GetHost()) { - protoDescriptor->set_host(ToProto(*host)); - } else { - protoDescriptor->clear_host(); - } - - if (auto rack = descriptor.GetRack()) { - protoDescriptor->set_rack(ToProto(*rack)); - } else { - protoDescriptor->clear_rack(); - } - - if (auto dataCenter = descriptor.GetDataCenter()) { - protoDescriptor->set_data_center(ToProto(*dataCenter)); - } else { - protoDescriptor->clear_data_center(); - } + YT_OPTIONAL_SET_PROTO(protoDescriptor, host, descriptor.GetHost()); + YT_OPTIONAL_SET_PROTO(protoDescriptor, rack, descriptor.GetRack()); + YT_OPTIONAL_SET_PROTO(protoDescriptor, data_center, descriptor.GetDataCenter()); ToProto(protoDescriptor->mutable_tags(), descriptor.GetTags()); - if (auto lastHeartbeatTime = descriptor.GetLastSeenTime()) { - protoDescriptor->set_last_seen_time(ToProto(*lastHeartbeatTime)); - } else { - protoDescriptor->clear_last_seen_time(); - } + YT_OPTIONAL_SET_PROTO(protoDescriptor, last_seen_time, descriptor.GetLastSeenTime()); } void FromProto(NNodeTrackerClient::TNodeDescriptor* descriptor, const NNodeTrackerClient::NProto::TNodeDescriptor& protoDescriptor) diff --git a/yt/yt/client/object_client/helpers.cpp b/yt/yt/client/object_client/helpers.cpp index 0fbc5a8c9b..e7ba1b93ce 100644 --- a/yt/yt/client/object_client/helpers.cpp +++ b/yt/yt/client/object_client/helpers.cpp @@ -134,7 +134,8 @@ bool IsVersionedType(EObjectType type) type == EObjectType::SequoiaMapNode || type == EObjectType::Pipeline || type == EObjectType::QueueConsumer || - type == EObjectType::QueueProducer; + type == EObjectType::QueueProducer || + type == EObjectType::CypressProxyMap; } bool IsUserType(EObjectType type) diff --git a/yt/yt/client/object_client/public.h b/yt/yt/client/object_client/public.h index f534c40eac..5a7d177543 100644 --- a/yt/yt/client/object_client/public.h +++ b/yt/yt/client/object_client/public.h @@ -342,8 +342,10 @@ DEFINE_ENUM(EObjectType, ((ReplicationCardCollocation) (1207)) ((VirtualChaosCellMap) (1208)) - // Maintenance tracker stuff + // Other cluster components stuff ((ClusterProxyNode) (1500)) + ((CypressProxyObject) (1501)) + ((CypressProxyMap) (465)) // Zookeeper stuff // COMPAT(babenko): drop completely diff --git a/yt/yt/client/security_client/public.h b/yt/yt/client/security_client/public.h index 2bb6eaeffb..420aaab50b 100644 --- a/yt/yt/client/security_client/public.h +++ b/yt/yt/client/security_client/public.h @@ -76,12 +76,6 @@ YT_DEFINE_ERROR_ENUM( ((IrreversibleAclModification) (909)) ); -// NB: Changing this list requires reign promotion. -DEFINE_ENUM(EProxyKind, - ((Http) (1)) - ((Rpc) (2)) -); - DEFINE_ENUM(EAccessControlObjectNamespace, (AdminCommands) ); diff --git a/yt/yt/client/sequoia_client/public.h b/yt/yt/client/sequoia_client/public.h index 5c213ebd9f..5411c0c770 100644 --- a/yt/yt/client/sequoia_client/public.h +++ b/yt/yt/client/sequoia_client/public.h @@ -7,9 +7,10 @@ namespace NYT::NSequoiaClient { //////////////////////////////////////////////////////////////////////////////// YT_DEFINE_ERROR_ENUM( - ((SequoiaClientNotReady) (6000)) - ((SequoiaTableCorrupted) (6001)) - ((SequoiaRetriableError) (6002)) + ((SequoiaClientNotReady) (6000)) + ((SequoiaTableCorrupted) (6001)) + ((SequoiaRetriableError) (6002)) + ((InvalidSequoiaReign) (6003)) ); //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/client/table_client/config.cpp b/yt/yt/client/table_client/config.cpp index ec590507dc..b8bb96c164 100644 --- a/yt/yt/client/table_client/config.cpp +++ b/yt/yt/client/table_client/config.cpp @@ -289,6 +289,10 @@ void TDictionaryCompressionConfig::Register(TRegistrar registrar) .Default(0.7) .InRange(0, 1); + registrar.Parameter("elect_random_policy", &TThis::ElectRandomPolicy) + .Default(false) + .DontSerializeDefault(); + registrar.Postprocessor([] (TThis* config) { if (config->DesiredSampleCount > config->MaxProcessedSampleCount) { THROW_ERROR_EXCEPTION("\"desired_sample_count\" cannot be greater than \"max_processed_sample_count\""); diff --git a/yt/yt/client/table_client/config.h b/yt/yt/client/table_client/config.h index 44eeba3b6b..fa096afa53 100644 --- a/yt/yt/client/table_client/config.h +++ b/yt/yt/client/table_client/config.h @@ -251,6 +251,9 @@ struct TDictionaryCompressionConfig //! Upper limit on acceptable compression ratio. No chunk compression is performed if this limit is exceeded. double MaxAcceptableCompressionRatio; + //! For testing purposes only. + bool ElectRandomPolicy; + REGISTER_YSON_STRUCT(TDictionaryCompressionConfig); static void Register(TRegistrar registrar); diff --git a/yt/yt/client/table_client/helpers.cpp b/yt/yt/client/table_client/helpers.cpp index 680f5f995b..857893c458 100644 --- a/yt/yt/client/table_client/helpers.cpp +++ b/yt/yt/client/table_client/helpers.cpp @@ -164,7 +164,7 @@ void YTreeNodeToUnversionedValue( } // namespace TUnversionedOwningRow YsonToSchemafulRow( - const TString& yson, + TStringBuf yson, const TTableSchema& tableSchema, bool treatMissingAsNull, NYson::EYsonType ysonType, @@ -172,8 +172,7 @@ TUnversionedOwningRow YsonToSchemafulRow( { auto nameTable = TNameTable::FromSchema(tableSchema); - auto rowParts = ConvertTo<THashMap<TString, INodePtr>>( - TYsonString(yson, ysonType)); + auto rowParts = ConvertTo<THashMap<TString, INodePtr>>(TYsonString(yson, ysonType)); TUnversionedOwningRowBuilder rowBuilder; auto validateAndAddValue = [&rowBuilder, &validateValues] (const TUnversionedValue& value, const TColumnSchema& column) { @@ -266,7 +265,7 @@ TUnversionedOwningRow YsonToSchemafulRow( return rowBuilder.FinishRow(); } -TUnversionedOwningRow YsonToSchemalessRow(const TString& valueYson) +TUnversionedOwningRow YsonToSchemalessRow(TStringBuf valueYson) { TUnversionedOwningRowBuilder builder; @@ -285,8 +284,8 @@ TUnversionedOwningRow YsonToSchemalessRow(const TString& valueYson) TVersionedRow YsonToVersionedRow( const TRowBufferPtr& rowBuffer, - const TString& keyYson, - const TString& valueYson, + TStringBuf keyYson, + TStringBuf valueYson, const std::vector<TTimestamp>& deleteTimestamps, const std::vector<TTimestamp>& extraWriteTimestamps) { @@ -349,8 +348,8 @@ TVersionedRow YsonToVersionedRow( } TVersionedOwningRow YsonToVersionedRow( - const TString& keyYson, - const TString& valueYson, + TStringBuf keyYson, + TStringBuf valueYson, const std::vector<TTimestamp>& deleteTimestamps, const std::vector<TTimestamp>& extraWriteTimestamps) { @@ -360,7 +359,7 @@ TVersionedOwningRow YsonToVersionedRow( return TVersionedOwningRow(row); } -TUnversionedOwningRow YsonToKey(const TString& yson) +TUnversionedOwningRow YsonToKey(TStringBuf yson) { TUnversionedOwningRowBuilder keyBuilder; auto keyParts = ConvertTo<std::vector<INodePtr>>( diff --git a/yt/yt/client/table_client/helpers.h b/yt/yt/client/table_client/helpers.h index d1c5d17325..36bbff3498 100644 --- a/yt/yt/client/table_client/helpers.h +++ b/yt/yt/client/table_client/helpers.h @@ -34,25 +34,24 @@ NChunkClient::EChunkFormat DefaultFormatFromOptimizeFor( // Mostly used in unittests and for debugging purposes. // Quite inefficient. TUnversionedOwningRow YsonToSchemafulRow( - const TString& yson, + TStringBuf yson, const TTableSchema& tableSchema, bool treatMissingAsNull, NYson::EYsonType ysonType = NYson::EYsonType::MapFragment, bool validateValues = false); -TUnversionedOwningRow YsonToSchemalessRow( - const TString& yson); +TUnversionedOwningRow YsonToSchemalessRow(TStringBuf yson); TVersionedRow YsonToVersionedRow( const TRowBufferPtr& rowBuffer, - const TString& keyYson, - const TString& valueYson, + TStringBuf keyYson, + TStringBuf valueYson, const std::vector<TTimestamp>& deleteTimestamps = {}, const std::vector<TTimestamp>& extraWriteTimestamps = {}); TVersionedOwningRow YsonToVersionedRow( - const TString& keyYson, - const TString& valueYson, + TStringBuf keyYson, + TStringBuf valueYson, const std::vector<TTimestamp>& deleteTimestamps = {}, const std::vector<TTimestamp>& extraWriteTimestamps = {}); -TUnversionedOwningRow YsonToKey(const TString& yson); +TUnversionedOwningRow YsonToKey(TStringBuf yson); TString KeyToYson(TUnversionedRow row); //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/client/table_client/schema.cpp b/yt/yt/client/table_client/schema.cpp index 7e28fd9830..a7715d96db 100644 --- a/yt/yt/client/table_client/schema.cpp +++ b/yt/yt/client/table_client/schema.cpp @@ -429,41 +429,13 @@ void ToProto(NProto::TColumnSchema* protoSchema, const TColumnSchema& schema) protoSchema->clear_required(); } ToProto(protoSchema->mutable_logical_type(), schema.LogicalType()); - if (schema.Lock()) { - protoSchema->set_lock(ToProto(*schema.Lock())); - } else { - protoSchema->clear_lock(); - } - if (schema.Expression()) { - protoSchema->set_expression(*schema.Expression()); - } else { - protoSchema->clear_expression(); - } - if (schema.Materialized()) { - protoSchema->set_materialized(*schema.Materialized()); - } else { - protoSchema->clear_materialized(); - } - if (schema.Aggregate()) { - protoSchema->set_aggregate(ToProto(*schema.Aggregate())); - } else { - protoSchema->clear_aggregate(); - } - if (schema.SortOrder()) { - protoSchema->set_sort_order(ToProto(*schema.SortOrder())); - } else { - protoSchema->clear_sort_order(); - } - if (schema.Group()) { - protoSchema->set_group(ToProto(*schema.Group())); - } else { - protoSchema->clear_group(); - } - if (schema.MaxInlineHunkSize()) { - protoSchema->set_max_inline_hunk_size(*schema.MaxInlineHunkSize()); - } else { - protoSchema->clear_max_inline_hunk_size(); - } + YT_OPTIONAL_SET_PROTO(protoSchema, lock, schema.Lock()); + YT_OPTIONAL_TO_PROTO(protoSchema, expression, schema.Expression()); + YT_OPTIONAL_SET_PROTO(protoSchema, materialized, schema.Materialized()); + YT_OPTIONAL_SET_PROTO(protoSchema, aggregate, schema.Aggregate()); + YT_OPTIONAL_SET_PROTO(protoSchema, sort_order, schema.SortOrder()); + YT_OPTIONAL_SET_PROTO(protoSchema, group, schema.Group()); + YT_OPTIONAL_SET_PROTO(protoSchema, max_inline_hunk_size, schema.MaxInlineHunkSize()); } void ToProto(NProto::TDeletedColumn* protoSchema, const TDeletedColumn& schema) @@ -492,13 +464,13 @@ void FromProto(TColumnSchema* schema, const NProto::TColumnSchema& protoSchema) schema->SetLogicalType(MakeLogicalType(GetLogicalType(physicalType), protoSchema.required())); } - schema->SetLock(YT_PROTO_OPTIONAL(protoSchema, lock)); - schema->SetExpression(YT_PROTO_OPTIONAL(protoSchema, expression)); - schema->SetMaterialized(YT_PROTO_OPTIONAL(protoSchema, materialized)); - schema->SetAggregate(YT_PROTO_OPTIONAL(protoSchema, aggregate)); + schema->SetLock(YT_OPTIONAL_FROM_PROTO(protoSchema, lock)); + schema->SetExpression(YT_OPTIONAL_FROM_PROTO(protoSchema, expression)); + schema->SetMaterialized(YT_OPTIONAL_FROM_PROTO(protoSchema, materialized)); + schema->SetAggregate(YT_OPTIONAL_FROM_PROTO(protoSchema, aggregate)); schema->SetSortOrder(YT_APPLY_PROTO_OPTIONAL(protoSchema, sort_order, FromProto<ESortOrder>)); - schema->SetGroup(YT_PROTO_OPTIONAL(protoSchema, group)); - schema->SetMaxInlineHunkSize(YT_PROTO_OPTIONAL(protoSchema, max_inline_hunk_size)); + schema->SetGroup(YT_OPTIONAL_FROM_PROTO(protoSchema, group)); + schema->SetMaxInlineHunkSize(YT_OPTIONAL_FROM_PROTO(protoSchema, max_inline_hunk_size)); } void FromProto(TDeletedColumn* schema, const NProto::TDeletedColumn& protoSchema) diff --git a/yt/yt/core/concurrency/fls-inl.h b/yt/yt/core/concurrency/fls-inl.h index 6267f7f054..a192a1dcb0 100644 --- a/yt/yt/core/concurrency/fls-inl.h +++ b/yt/yt/core/concurrency/fls-inl.h @@ -93,6 +93,12 @@ Y_FORCE_INLINE T* TFlsSlot<T>::GetOrCreate() const } template <class T> +Y_FORCE_INLINE T* TFlsSlot<T>::MaybeGet() const +{ + return static_cast<T*>(GetCurrentFls()->Get(Index_)); +} + +template <class T> T* TFlsSlot<T>::Create() const { auto cookie = new T(); diff --git a/yt/yt/core/concurrency/fls.h b/yt/yt/core/concurrency/fls.h index 616fd7d590..9306ffeeb8 100644 --- a/yt/yt/core/concurrency/fls.h +++ b/yt/yt/core/concurrency/fls.h @@ -42,6 +42,7 @@ public: T* operator->(); T* GetOrCreate() const; + T* MaybeGet() const; const T* Get(const TFls& fls) const; bool IsInitialized() const; diff --git a/yt/yt/core/misc/error.cpp b/yt/yt/core/misc/error.cpp index fb0b4d193f..99d8ed9161 100644 --- a/yt/yt/core/misc/error.cpp +++ b/yt/yt/core/misc/error.cpp @@ -1,10 +1,11 @@ #include "error.h" #include "serialize.h" -#include <yt/yt/core/concurrency/public.h> +#include <yt/yt/core/concurrency/fls.h> #include <yt/yt/core/net/local_address.h> +#include <yt/yt/core/misc/collection_helpers.h> #include <yt/yt/core/misc/protobuf_helpers.h> #include <yt/yt/core/tracing/trace_context.h> @@ -18,6 +19,8 @@ #include <library/cpp/yt/global/variable.h> +#include <library/cpp/yt/misc/global.h> + namespace NYT { using namespace NTracing; @@ -31,6 +34,8 @@ using NYT::ToProto; constexpr TStringBuf OriginalErrorDepthAttribute = "original_error_depth"; +bool TErrorCodicils::Initialized_ = false; + //////////////////////////////////////////////////////////////////////////////// namespace NDetail { @@ -656,4 +661,96 @@ void TErrorSerializer::Load(TStreamLoadContext& context, TError& error) //////////////////////////////////////////////////////////////////////////////// +static YT_DEFINE_GLOBAL(NConcurrency::TFlsSlot<TErrorCodicils>, ErrorCodicilsSlot); + +TErrorCodicils::TGuard::~TGuard() +{ + TErrorCodicils::GetOrCreate().Set(std::move(Key_), std::move(OldGetter_)); +} + +TErrorCodicils::TGuard::TGuard( + std::string key, + TGetter oldGetter) + : Key_(std::move(key)) + , OldGetter_(std::move(oldGetter)) +{ } + +void TErrorCodicils::Initialize() +{ + if (Initialized_) { + // Multiple calls are OK. + return; + } + Initialized_ = true; + + ErrorCodicilsSlot(); // Warm up the slot. + TError::RegisterEnricher([] (TError& error) { + if (auto* codicils = TErrorCodicils::MaybeGet()) { + codicils->Apply(error); + } + }); +} + +TErrorCodicils& TErrorCodicils::GetOrCreate() +{ + return *ErrorCodicilsSlot().GetOrCreate(); +} + +TErrorCodicils* TErrorCodicils::MaybeGet() +{ + return ErrorCodicilsSlot().MaybeGet(); +} + +std::optional<std::string> TErrorCodicils::MaybeEvaluate(const std::string& key) +{ + auto* instance = MaybeGet(); + if (!instance) { + return {}; + } + + auto getter = instance->Get(key); + if (!getter) { + return {}; + } + + return getter(); +} + +auto TErrorCodicils::Guard(std::string key, TGetter getter) -> TGuard +{ + auto& instance = GetOrCreate(); + auto [it, added] = instance.Getters_.try_emplace(key, getter); + TGetter oldGetter; + if (!added) { + oldGetter = std::move(it->second); + it->second = std::move(getter); + } + return TGuard(std::move(key), std::move(oldGetter)); +} + +void TErrorCodicils::Apply(TError& error) const +{ + for (const auto& [key, getter] : Getters_) { + error <<= TErrorAttribute(key, getter()); + } +} + +void TErrorCodicils::Set(std::string key, TGetter getter) +{ + // We could enforce Initialized_, but that could make an error condition worse at runtime. + // Instead, let's keep enrichment optional. + if (getter) { + Getters_.insert_or_assign(std::move(key), std::move(getter)); + } else { + Getters_.erase(std::move(key)); + } +} + +auto TErrorCodicils::Get(const std::string& key) const -> TGetter +{ + return GetOrDefault(Getters_, std::move(key)); +} + +//////////////////////////////////////////////////////////////////////////////// + } // namespace NYT diff --git a/yt/yt/core/misc/error.h b/yt/yt/core/misc/error.h index fd47419ee9..b53da2cf1d 100644 --- a/yt/yt/core/misc/error.h +++ b/yt/yt/core/misc/error.h @@ -13,6 +13,8 @@ #include <library/cpp/yt/error/error.h> #include <library/cpp/yt/error/origin_attributes.h> +#include <util/generic/noncopyable.h> + namespace NYT { //////////////////////////////////////////////////////////////////////////////// @@ -90,6 +92,56 @@ struct TSerializerTraits< //////////////////////////////////////////////////////////////////////////////// +// A fiber-local set of attributes to enrich all errors with. +class TErrorCodicils +{ +public: + using TGetter = std::function<std::string()>; + + class TGuard + : public TNonCopyable + { + public: + ~TGuard(); + + private: + friend class TErrorCodicils; + TGuard(std::string key, TGetter oldGetter); + std::string Key_; + TGetter OldGetter_; + }; + + // Call from single-threaded bootstrapping code. Errors will not be enriched otherwise. + static void Initialize(); + + // Gets or creates an instance for this fiber. + static TErrorCodicils& GetOrCreate(); + + // Gets the instance for this fiber if one was created previously. + static TErrorCodicils* MaybeGet(); + + // Evaluates the codicil for the key if one was set. + static std::optional<std::string> MaybeEvaluate(const std::string& key); + + // Sets the getter and returns an RAII object to restore the previous value on desctruction. + static TGuard Guard(std::string key, TGetter getter); + + // Adds error attributes. + void Apply(TError& error) const; + + // Sets the getter (or erases if the getter is empty). + void Set(std::string key, TGetter getter); + + // Gets the getter. + TGetter Get(const std::string& key) const; + +private: + THashMap<std::string, TGetter> Getters_; + static bool Initialized_; +}; + +//////////////////////////////////////////////////////////////////////////////// + } // namespace NYT #define ERROR_INL_H_ diff --git a/yt/yt/core/misc/protobuf_helpers-inl.h b/yt/yt/core/misc/protobuf_helpers-inl.h index 4ad0fb2272..ed4548cfe4 100644 --- a/yt/yt/core/misc/protobuf_helpers-inl.h +++ b/yt/yt/core/misc/protobuf_helpers-inl.h @@ -47,7 +47,7 @@ DEFINE_TRIVIAL_PROTO_CONVERSIONS(bool) //////////////////////////////////////////////////////////////////////////////// -#define YT_PROTO_OPTIONAL_CONVERT(...) __VA_OPT__(::NYT::FromProto<__VA_ARGS__>) +#define YT_OPTIONAL_FROM_PROTO_CONVERT(...) __VA_OPT__(::NYT::FromProto<__VA_ARGS__>) //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/core/misc/protobuf_helpers.h b/yt/yt/core/misc/protobuf_helpers.h index 050774bff7..c25d7c2902 100644 --- a/yt/yt/core/misc/protobuf_helpers.h +++ b/yt/yt/core/misc/protobuf_helpers.h @@ -464,14 +464,34 @@ google::protobuf::Timestamp GetProtoNow(); //! field. Macro accepts desired target type as optional third parameter. //! Usage: //! // Get as is. -//! int instantInt = YT_PROTO_OPTIONAL(message, instant); +//! int instantInt = YT_OPTIONAL_FROM_PROTO(message, instant); //! // Get with conversion. -//! TInstant instant = YT_PROTO_OPTIONAL(message, instant, TInstant); -#define YT_PROTO_OPTIONAL(message, field, ...) \ +//! TInstant instant = YT_OPTIONAL_FROM_PROTO(message, instant, TInstant); +#define YT_OPTIONAL_FROM_PROTO(message, field, ...) \ (((message).has_##field()) \ - ? std::optional(YT_PROTO_OPTIONAL_CONVERT(__VA_ARGS__)((message).field())) \ + ? std::optional(YT_OPTIONAL_FROM_PROTO_CONVERT(__VA_ARGS__)((message).field())) \ : std::nullopt) +#define YT_OPTIONAL_TO_PROTO(message, field, original) \ + do {\ + if (original.has_value()) {\ + ToProto((message)->mutable_##field(), *original);\ + } else {\ + (message)->clear_##field();\ + }\ + } while (false) + +#define YT_OPTIONAL_SET_PROTO(message, field, original) \ + do {\ + /* Avoid unnecessary computation if <original> is a return value of a call*/\ + const auto& originalRef = (original);\ + if (originalRef.has_value()) {\ + (message)->set_##field(ToProto(*originalRef));\ + } else {\ + (message)->clear_##field();\ + }\ + } while (false) + // TODO(cherepashka): to remove after std::optional::and_then is here. //! This macro may be used to extract std::optional<T> from protobuf message field of type T and to apply some function to value if it is present. #define YT_APPLY_PROTO_OPTIONAL(message, field, function) (((message).has_##field()) ? std::make_optional(function((message).field())) : std::nullopt) diff --git a/yt/yt/core/misc/unittests/error_ut.cpp b/yt/yt/core/misc/unittests/error_ut.cpp index 0c6d32d08c..6f6c3c2efe 100644 --- a/yt/yt/core/misc/unittests/error_ut.cpp +++ b/yt/yt/core/misc/unittests/error_ut.cpp @@ -228,6 +228,20 @@ TEST(TErrorTest, NativeFiberId) .ThrowOnError(); } +TEST(TErrorTest, ErrorCodicils) +{ + EXPECT_FALSE(TError("ErrorCodicils").Attributes().Contains("test_attribute")); + { + auto guard = TErrorCodicils::Guard("test_attribute", [] () -> std::string { + return "test_value"; + }); + EXPECT_EQ("test_value", + TError("ErrorCodicils").Attributes().Get<std::string>("test_attribute")); + EXPECT_FALSE(TError().Attributes().Contains("test_attribute")); + } + EXPECT_FALSE(TError("ErrorCodicils").Attributes().Contains("test_attribute")); +} + //////////////////////////////////////////////////////////////////////////////// } // namespace diff --git a/yt/yt/core/test_framework/framework.cpp b/yt/yt/core/test_framework/framework.cpp index 15df3840cb..a5c6f709ad 100644 --- a/yt/yt/core/test_framework/framework.cpp +++ b/yt/yt/core/test_framework/framework.cpp @@ -156,6 +156,8 @@ Y_TEST_HOOK_BEFORE_RUN(GTEST_YT_SETUP) NYT::TSignalRegistry::Get()->PushDefaultSignalHandler(NYT::AllCrashSignals); #endif + NYT::TErrorCodicils::Initialize(); + auto config = NYT::NLogging::TLogManagerConfig::CreateYTServer("unittester", GetOutputPath().GetPath()); NYT::NLogging::TLogManager::Get()->Configure(config); NYT::NLogging::TLogManager::Get()->EnableReopenOnSighup(); diff --git a/yt/yt/core/ytree/node_detail.cpp b/yt/yt/core/ytree/node_detail.cpp index dd7ea9f3ea..8f7d770f27 100644 --- a/yt/yt/core/ytree/node_detail.cpp +++ b/yt/yt/core/ytree/node_detail.cpp @@ -314,7 +314,7 @@ void TMapNodeMixin::ListSelf( ? FromProto<TAttributeFilter>(request->attributes()) : TAttributeFilter(); - auto limit = YT_PROTO_OPTIONAL(*request, limit); + auto limit = YT_OPTIONAL_FROM_PROTO(*request, limit); context->SetRequestInfo("Limit: %v, AttributeFilter: %v", limit, diff --git a/yt/yt/core/ytree/request_complexity_limits.cpp b/yt/yt/core/ytree/request_complexity_limits.cpp index faea8bdc32..bb3834123a 100644 --- a/yt/yt/core/ytree/request_complexity_limits.cpp +++ b/yt/yt/core/ytree/request_complexity_limits.cpp @@ -41,8 +41,8 @@ void FromProto( TReadRequestComplexityOverrides* original, const NProto::TReadRequestComplexityLimits& serialized) { - original->NodeCount = YT_PROTO_OPTIONAL(serialized, node_count); - original->ResultSize = YT_PROTO_OPTIONAL(serialized, result_size); + original->NodeCount = YT_OPTIONAL_FROM_PROTO(serialized, node_count); + original->ResultSize = YT_OPTIONAL_FROM_PROTO(serialized, result_size); } void ToProto( diff --git a/yt/yt/core/ytree/unittests/yson_struct_update_ut.cpp b/yt/yt/core/ytree/unittests/yson_struct_update_ut.cpp index 0fb160fdcd..fd82265dd7 100644 --- a/yt/yt/core/ytree/unittests/yson_struct_update_ut.cpp +++ b/yt/yt/core/ytree/unittests/yson_struct_update_ut.cpp @@ -176,5 +176,36 @@ TEST(TUpdateYsonStructTest, Nested) EXPECT_EQ(updatedCommand, "sort"); } +TEST(TUpdateYsonStructTest, Validate) +{ + auto oldSpec = ConvertTo<TSpecWithPoolPtr>(TYsonString(TString("{pool=pool;}"))); + auto longPoolSpec = ConvertTo<TSpecWithPoolPtr>(TYsonString(TString("{pool=new_pool;}"))); + auto shortPoolSpec = ConvertTo<TSpecWithPoolPtr>(TYsonString(TString("{pool=p;}"))); + + std::string updatedPool; + + auto configurator = TConfigurator<TSpecWithPool>(); + configurator.Field("pool", &TSpecWithPool::Pool) + .Validator(BIND([&] (const std::string& newPool) { + THROW_ERROR_EXCEPTION_IF( + newPool.size() > 4, + "Pool name too long"); + })) + .Updater(BIND([&] (const std::string& newPool) { + updatedPool = newPool; + })); + + auto sealed = std::move(configurator).Seal(); + + EXPECT_THROW_WITH_SUBSTRING( + sealed.Validate(oldSpec, longPoolSpec), + "Pool name too long"); + + sealed.Validate(oldSpec, shortPoolSpec); + sealed.Update(oldSpec, shortPoolSpec); + + EXPECT_EQ(updatedPool, "p"); +} + } // namespace } // namespace NYT::NYTree diff --git a/yt/yt/core/ytree/yson_struct_update-inl.h b/yt/yt/core/ytree/yson_struct_update-inl.h index 3ba88ec853..c6ae5f1ead 100644 --- a/yt/yt/core/ytree/yson_struct_update-inl.h +++ b/yt/yt/core/ytree/yson_struct_update-inl.h @@ -49,9 +49,27 @@ struct TUnwrapYsonStructIntrusivePtr<TIntrusivePtr<T>> //////////////////////////////////////////////////////////////////////////////// template <class TValue> +TFieldRegistrar<TValue>& TFieldRegistrar<TValue>::Validator(TCallback<void(const TValue&, const TValue&)> validator) +{ + VerifyEmptyValidator(); + Validator_ = validator; + return *this; +} + +template <class TValue> +TFieldRegistrar<TValue>& TFieldRegistrar<TValue>::Validator(TCallback<void(const TValue&)> validator) +{ + VerifyEmptyValidator(); + Validator_ = BIND_NO_PROPAGATE([validator = std::move(validator)] (const TValue& /*oldValue*/, const TValue& newValue) { + validator(std::move(newValue)); + }); + return *this; +} + +template <class TValue> TFieldRegistrar<TValue>& TFieldRegistrar<TValue>::Updater(TCallback<void(const TValue&, const TValue&)> updater) { - VerifyEmpty(); + VerifyEmptyUpdater(); Updater_ = updater; return *this; } @@ -59,7 +77,7 @@ TFieldRegistrar<TValue>& TFieldRegistrar<TValue>::Updater(TCallback<void(const T template <class TValue> TFieldRegistrar<TValue>& TFieldRegistrar<TValue>::Updater(TCallback<void(const TValue&)> updater) { - VerifyEmpty(); + VerifyEmptyUpdater(); Updater_ = BIND_NO_PROPAGATE([updater = std::move(updater)] (const TValue& /*oldValue*/, const TValue& newValue) { updater(std::move(newValue)); }); @@ -90,7 +108,7 @@ TFieldRegistrar<TValue>& TFieldRegistrar<TValue>::NestedUpdater( TUnwrappedValue, typename NDetail::TUnwrapYsonStructIntrusivePtr<TValue>::TStruct>); - VerifyEmpty(); + VerifyEmptyUpdater(); auto configurator = configureCallback(); Updater_ = BIND_NO_PROPAGATE([configurator = std::move(configurator)] (const TValue& oldValue, const TValue& newValue) { configurator.Update(oldValue, newValue); @@ -99,6 +117,23 @@ TFieldRegistrar<TValue>& TFieldRegistrar<TValue>::NestedUpdater( } template <class TValue> +void TFieldRegistrar<TValue>::DoValidate( + IYsonStructParameterPtr parameter, + TYsonStructBase* oldStruct, + TYsonStructBase* newStruct) const +{ + if (!Validator_) { + return; + } + + auto typedParameter = DynamicPointerCast<TYsonStructParameter<TValue>>(parameter); + YT_VERIFY(typedParameter); + Validator_( + typedParameter->GetValue(oldStruct), + typedParameter->GetValue(newStruct)); +} + +template <class TValue> void TFieldRegistrar<TValue>::DoUpdate( IYsonStructParameterPtr parameter, TYsonStructBase* oldStruct, @@ -116,7 +151,13 @@ void TFieldRegistrar<TValue>::DoUpdate( } template <class TValue> -void TFieldRegistrar<TValue>::VerifyEmpty() const +void TFieldRegistrar<TValue>::VerifyEmptyValidator() const +{ + YT_VERIFY(!Validator_); +} + +template <class TValue> +void TFieldRegistrar<TValue>::VerifyEmptyUpdater() const { YT_VERIFY(!Updater_); } @@ -169,6 +210,7 @@ TSealedConfigurator<TStruct> TConfigurator<TStruct>::Seal() && { return std::move(*this); } + //////////////////////////////////////////////////////////////////////////////// template <CYsonStructDerived TStruct> @@ -177,10 +219,27 @@ TSealedConfigurator<TStruct>::TSealedConfigurator(TConfigurator<TStruct> configu { } template <CYsonStructDerived TStruct> +void TSealedConfigurator<TStruct>::Validate( + TIntrusivePtr<TStruct> oldStruct, + TIntrusivePtr<TStruct> newStruct) const +{ + Do(oldStruct, newStruct, &NDetail::IFieldRegistrar::DoValidate); +} + +template <CYsonStructDerived TStruct> void TSealedConfigurator<TStruct>::Update( TIntrusivePtr<TStruct> oldStruct, TIntrusivePtr<TStruct> newStruct) const { + Do(oldStruct, newStruct, &NDetail::IFieldRegistrar::DoUpdate); +} + +template <CYsonStructDerived TStruct> +void TSealedConfigurator<TStruct>::Do( + TIntrusivePtr<TStruct> oldStruct, + TIntrusivePtr<TStruct> newStruct, + TFieldRegistrarMethod fieldMethod) const +{ const auto* meta = oldStruct->GetMeta(); YT_VERIFY(meta == newStruct->GetMeta()); const auto& parameterToFieldRegistrar = RegisteredFields_->ParameterToFieldRegistrar; @@ -194,7 +253,7 @@ void TSealedConfigurator<TStruct>::Update( if (fieldDescIter == parameterToFieldRegistrar.end()) { THROW_ERROR_EXCEPTION("Field %Qv is not marked as updatable, but was changed", name); } else { - fieldDescIter->second->DoUpdate(parameter, oldStruct.Get(), newStruct.Get()); + (*(fieldDescIter->second).*fieldMethod)(parameter, oldStruct.Get(), newStruct.Get()); } } } diff --git a/yt/yt/core/ytree/yson_struct_update.h b/yt/yt/core/ytree/yson_struct_update.h index 4b472021d1..7a517d6f57 100644 --- a/yt/yt/core/ytree/yson_struct_update.h +++ b/yt/yt/core/ytree/yson_struct_update.h @@ -26,6 +26,11 @@ DECLARE_REFCOUNTED_STRUCT(TRegisteredFieldDirectory); struct IFieldRegistrar : public TRefCounted { + virtual void DoValidate( + IYsonStructParameterPtr parameter, + TYsonStructBase* oldStruct, + TYsonStructBase* newStruct) const = 0; + virtual void DoUpdate( IYsonStructParameterPtr parameter, TYsonStructBase* oldStruct, @@ -42,6 +47,12 @@ class TFieldRegistrar : public IFieldRegistrar { public: + // Registers validator that accepts old and new values as arguments. + TFieldRegistrar& Validator(TCallback<void(const TValue&, const TValue&)> validator); + + // Registers validator that accepts only new value as an argument. + TFieldRegistrar& Validator(TCallback<void(const TValue&)> validator); + // Registers updater that accepts old and new values as arguments. TFieldRegistrar& Updater(TCallback<void(const TValue&, const TValue&)> updater); @@ -58,10 +69,17 @@ public: TYsonStructBase* oldStruct, TYsonStructBase* newStruct) const override; + void DoValidate( + IYsonStructParameterPtr parameter, + TYsonStructBase* oldStruct, + TYsonStructBase* newStruct) const override; + private: - void VerifyEmpty() const; + void VerifyEmptyUpdater() const; + void VerifyEmptyValidator() const; TCallback<void(const TValue&, const TValue&)> Updater_; + TCallback<void(const TValue&, const TValue&)> Validator_; }; //////////////////////////////////////////////////////////////////////////////// @@ -114,11 +132,25 @@ class TSealedConfigurator public: TSealedConfigurator(TConfigurator<TStruct> configurator); + void Validate( + TIntrusivePtr<TStruct> oldStruct, + TIntrusivePtr<TStruct> newStruct) const; + void Update( TIntrusivePtr<TStruct> oldStruct, TIntrusivePtr<TStruct> newStruct) const; private: + using TFieldRegistrarMethod = void(NDetail::IFieldRegistrar::*)( + IYsonStructParameterPtr parameter, + TYsonStructBase* oldStruct, + TYsonStructBase* newStruct) const; + + void Do( + TIntrusivePtr<TStruct> oldStruct, + TIntrusivePtr<TStruct> newStruct, + TFieldRegistrarMethod fieldMethod) const; + NDetail::TRegisteredFieldDirectoryPtr RegisteredFields_; }; |