diff options
| author | Alexander Smirnov <[email protected]> | 2025-06-18 14:38:34 +0000 |
|---|---|---|
| committer | Alexander Smirnov <[email protected]> | 2025-06-18 14:38:34 +0000 |
| commit | f0aab26e5fffb68cd8ae3be804d246bb8337a784 (patch) | |
| tree | ce6f8a1847719c9d8a7acc5c654e951ac8a4bf87 | |
| parent | 68fd8a82ce8a9b7dd18072f3941171d8548bb0e1 (diff) | |
| parent | 826c4b4f2be8243bde08a51459dabfb2f271b289 (diff) | |
Merge pull request #19782 from ydb-platform/merge-libs-250618-0050
154 files changed, 2258 insertions, 637 deletions
diff --git a/build/conf/java.conf b/build/conf/java.conf index 552e16fed89..9135fa8020a 100644 --- a/build/conf/java.conf +++ b/build/conf/java.conf @@ -81,6 +81,10 @@ multimodule JAVA_PROGRAM { module JAR_COMPILATION: _JAR_PROGRAM_COMPILATION { .SEM=_BUILD_JAR_SEM SET(MODULE_TYPE JAVA_PROGRAM) + + when ($EXPORT_GRADLE == "yes") { + CONSUME_NON_MANAGEABLE_PEERS=yes + } } } diff --git a/build/export_generators/ide-gradle/javac_flags.jinja b/build/export_generators/ide-gradle/javac_flags.jinja index 642fd68a617..70c60db0a3b 100644 --- a/build/export_generators/ide-gradle/javac_flags.jinja +++ b/build/export_generators/ide-gradle/javac_flags.jinja @@ -25,9 +25,10 @@ tasks.{{ task }}.configure { {%- if compiler_jvm_flags|length %} options.isFork = true options.forkOptions.jvmArgs = listOf( -{%- for compiler_jvm_flag in compiler_jvm_flags -%} -"{{ compiler_jvm_flag|replace("-J", "") }}"{% if not loop.last %}, {% endif -%} -{%- endfor -%}) +{%- for compiler_jvm_flag in compiler_jvm_flags %} + "{{ compiler_jvm_flag|replace("-J", "") }}", +{%- endfor %} + ) {% endif -%} {%- if has_errorprone -%} diff --git a/build/export_generators/ide-gradle/jvm_args.jinja b/build/export_generators/ide-gradle/jvm_args.jinja index a37a9193710..7fe19a82ba1 100644 --- a/build/export_generators/ide-gradle/jvm_args.jinja +++ b/build/export_generators/ide-gradle/jvm_args.jinja @@ -1,20 +1,24 @@ {%- macro JvmArgs(jvm_args) -%} -{%- if jvm_args|length %} - jvmArgs = mutableListOf( -{%- set jnis_args = jvm_args|select('startsWith', 'jnis=') %} -{%- if jnis_args|length -%} -{#- Increase heap, else happened java.lang.OutOfMemoryError #} - "-Xmx2048m", +{%- if jvm_args|length -%} +{%- set jnis_args = jvm_args|select('startsWith', 'jnis=') -%} +{%- set other_jvm_args = jvm_args|reject('startsWith', 'jnis=') -%} +{%- if jnis_args|length %} + val library_path = listOf( {%- for jnis_arg in jnis_args -%} {%- set jnis = split(jnis_arg|replace('jnis=', ''), ',') -%} -{#- Put paths to JNI libraries direct to JVM args -#} {%- for jni in jnis -%} {%- set path_and_name = rsplit(jni, "/", 2) %} - "-Djava.library.path=$arcadia_root/{{ path_and_name[0] }}", + "$arcadia_root/{{ path_and_name[0] }}", {%- endfor -%} -{%- endfor -%} +{%- endfor %} + ).joinToString(":") +{% endif %} + jvmArgs = mutableListOf( +{%- if jnis_args|length -%} +{#- Increase heap, else happened java.lang.OutOfMemoryError #} + "-Xmx2048m", + "-Djava.library.path=$library_path", {%- endif -%} -{%- set other_jvm_args = jvm_args|reject('startsWith', 'jnis=') %} {%- if other_jvm_args|length -%} {%- for jvm_arg in other_jvm_args %} "{{ jvm_arg }}", diff --git a/build/export_generators/ide-gradle/source_sets.jinja b/build/export_generators/ide-gradle/source_sets.jinja index ba2965d4c20..60793eb917e 100644 --- a/build/export_generators/ide-gradle/source_sets.jinja +++ b/build/export_generators/ide-gradle/source_sets.jinja @@ -14,9 +14,8 @@ #} {{ module }}.srcDir({{ PatchRoots(dir) }}) {%- endif -%} -{%- endmacro -%} +{%- endmacro %} -{#- empty string #} sourceSets { {%- set target_jar_source_set = target.jar_source_set|unique -%} {%- set target_jar_resource_set = target.jar_resource_set|unique -%} diff --git a/build/external_resources/gdb/resources.json b/build/external_resources/gdb/resources.json index f7b21801fad..53472db6d38 100644 --- a/build/external_resources/gdb/resources.json +++ b/build/external_resources/gdb/resources.json @@ -7,10 +7,10 @@ "uri": "sbr:3833498694" }, "linux-aarch64": { - "uri": "sbr:8803622849" + "uri": "sbr:8989381149" }, "linux-x86_64": { - "uri": "sbr:8803613328" + "uri": "sbr:8989701781" } } } diff --git a/build/external_resources/ymake/public.resources.json b/build/external_resources/ymake/public.resources.json index 1870648be65..d2a25322b5b 100644 --- a/build/external_resources/ymake/public.resources.json +++ b/build/external_resources/ymake/public.resources.json @@ -1,19 +1,19 @@ { "by_platform": { "darwin": { - "uri": "sbr:8964594623" + "uri": "sbr:8972534424" }, "darwin-arm64": { - "uri": "sbr:8964594063" + "uri": "sbr:8972533338" }, "linux": { - "uri": "sbr:8964595237" + "uri": "sbr:8972536518" }, "linux-aarch64": { - "uri": "sbr:8964593480" + "uri": "sbr:8972533033" }, "win32": { - "uri": "sbr:8964594970" + "uri": "sbr:8972535583" } } } diff --git a/build/external_resources/ymake/resources.json b/build/external_resources/ymake/resources.json index b63ec6ecb8e..bf5530609cb 100644 --- a/build/external_resources/ymake/resources.json +++ b/build/external_resources/ymake/resources.json @@ -1,19 +1,19 @@ { "by_platform": { "darwin": { - "uri": "sbr:8964590476" + "uri": "sbr:8972532408" }, "darwin-arm64": { - "uri": "sbr:8964589998" + "uri": "sbr:8972531932" }, "linux": { - "uri": "sbr:8964591657" + "uri": "sbr:8972532988" }, "linux-aarch64": { - "uri": "sbr:8964589384" + "uri": "sbr:8972531727" }, "win32": { - "uri": "sbr:8964591278" + "uri": "sbr:8972532710" } } } diff --git a/build/mapping.conf.json b/build/mapping.conf.json index 433c3be1e3c..feb56f83db2 100644 --- a/build/mapping.conf.json +++ b/build/mapping.conf.json @@ -561,6 +561,7 @@ "8865992733": "{registry_endpoint}/8865992733", "8895767556": "{registry_endpoint}/8895767556", "8944928850": "{registry_endpoint}/8944928850", + "8980584272": "{registry_endpoint}/8980584272", "5486731632": "{registry_endpoint}/5486731632", "5514350352": "{registry_endpoint}/5514350352", "5514360398": "{registry_endpoint}/5514360398", @@ -797,6 +798,7 @@ "8859746341": "{registry_endpoint}/8859746341", "8926191672": "{registry_endpoint}/8926191672", "8964594623": "{registry_endpoint}/8964594623", + "8972534424": "{registry_endpoint}/8972534424", "5766171800": "{registry_endpoint}/5766171800", "5805430761": "{registry_endpoint}/5805430761", "5829025456": "{registry_endpoint}/5829025456", @@ -884,6 +886,7 @@ "8859745406": "{registry_endpoint}/8859745406", "8926191000": "{registry_endpoint}/8926191000", "8964594063": "{registry_endpoint}/8964594063", + "8972533338": "{registry_endpoint}/8972533338", "5766173070": "{registry_endpoint}/5766173070", "5805432830": "{registry_endpoint}/5805432830", "5829031598": "{registry_endpoint}/5829031598", @@ -971,6 +974,7 @@ "8859748874": "{registry_endpoint}/8859748874", "8926193753": "{registry_endpoint}/8926193753", "8964595237": "{registry_endpoint}/8964595237", + "8972536518": "{registry_endpoint}/8972536518", "5766171341": "{registry_endpoint}/5766171341", "5805430188": "{registry_endpoint}/5805430188", "5829023352": "{registry_endpoint}/5829023352", @@ -1058,6 +1062,7 @@ "8859744568": "{registry_endpoint}/8859744568", "8926190366": "{registry_endpoint}/8926190366", "8964593480": "{registry_endpoint}/8964593480", + "8972533033": "{registry_endpoint}/8972533033", "8270821739": "{registry_endpoint}/8270821739", "8295446553": "{registry_endpoint}/8295446553", "8326170338": "{registry_endpoint}/8326170338", @@ -1075,6 +1080,7 @@ "8859747443": "{registry_endpoint}/8859747443", "8926192834": "{registry_endpoint}/8926192834", "8964594970": "{registry_endpoint}/8964594970", + "8972535583": "{registry_endpoint}/8972535583", "5766172695": "{registry_endpoint}/5766172695", "5805432230": "{registry_endpoint}/5805432230", "5829029743": "{registry_endpoint}/5829029743", @@ -1156,6 +1162,7 @@ "8784096409": "{registry_endpoint}/8784096409", "6406540582": "{registry_endpoint}/6406540582", "6447362348": "{registry_endpoint}/6447362348", + "8989381149": "{registry_endpoint}/8989381149", "6133337898": "{registry_endpoint}/6133337898", "8509776935": "{registry_endpoint}/8509776935", "8803622849": "{registry_endpoint}/8803622849", @@ -1165,6 +1172,7 @@ "8784094118": "{registry_endpoint}/8784094118", "6406437536": "{registry_endpoint}/6406437536", "6447316775": "{registry_endpoint}/6447316775", + "8989701781": "{registry_endpoint}/8989701781", "6133419349": "{registry_endpoint}/6133419349", "8509757921": "{registry_endpoint}/8509757921", "8803613328": "{registry_endpoint}/8803613328", @@ -2040,6 +2048,7 @@ "8865992733": "devtools/ya/test/programs/test_tool/bin/test_tool for linux", "8895767556": "devtools/ya/test/programs/test_tool/bin/test_tool for linux", "8944928850": "devtools/ya/test/programs/test_tool/bin/test_tool for linux", + "8980584272": "devtools/ya/test/programs/test_tool/bin/test_tool for linux", "5486731632": "devtools/ya/test/programs/test_tool/bin3/test_tool3 for linux", "5514350352": "devtools/ya/test/programs/test_tool/bin3/test_tool3 for linux", "5514360398": "devtools/ya/test/programs/test_tool/bin3/test_tool3 for linux", @@ -2276,6 +2285,7 @@ "8859746341": "devtools/ymake/bin/ymake for darwin", "8926191672": "devtools/ymake/bin/ymake for darwin", "8964594623": "devtools/ymake/bin/ymake for darwin", + "8972534424": "devtools/ymake/bin/ymake for darwin", "5766171800": "devtools/ymake/bin/ymake for darwin-arm64", "5805430761": "devtools/ymake/bin/ymake for darwin-arm64", "5829025456": "devtools/ymake/bin/ymake for darwin-arm64", @@ -2363,6 +2373,7 @@ "8859745406": "devtools/ymake/bin/ymake for darwin-arm64", "8926191000": "devtools/ymake/bin/ymake for darwin-arm64", "8964594063": "devtools/ymake/bin/ymake for darwin-arm64", + "8972533338": "devtools/ymake/bin/ymake for darwin-arm64", "5766173070": "devtools/ymake/bin/ymake for linux", "5805432830": "devtools/ymake/bin/ymake for linux", "5829031598": "devtools/ymake/bin/ymake for linux", @@ -2450,6 +2461,7 @@ "8859748874": "devtools/ymake/bin/ymake for linux", "8926193753": "devtools/ymake/bin/ymake for linux", "8964595237": "devtools/ymake/bin/ymake for linux", + "8972536518": "devtools/ymake/bin/ymake for linux", "5766171341": "devtools/ymake/bin/ymake for linux-aarch64", "5805430188": "devtools/ymake/bin/ymake for linux-aarch64", "5829023352": "devtools/ymake/bin/ymake for linux-aarch64", @@ -2537,6 +2549,7 @@ "8859744568": "devtools/ymake/bin/ymake for linux-aarch64", "8926190366": "devtools/ymake/bin/ymake for linux-aarch64", "8964593480": "devtools/ymake/bin/ymake for linux-aarch64", + "8972533033": "devtools/ymake/bin/ymake for linux-aarch64", "8270821739": "devtools/ymake/bin/ymake for win32", "8295446553": "devtools/ymake/bin/ymake for win32", "8326170338": "devtools/ymake/bin/ymake for win32", @@ -2554,6 +2567,7 @@ "8859747443": "devtools/ymake/bin/ymake for win32", "8926192834": "devtools/ymake/bin/ymake for win32", "8964594970": "devtools/ymake/bin/ymake for win32", + "8972535583": "devtools/ymake/bin/ymake for win32", "5766172695": "devtools/ymake/bin/ymake for win32-clang-cl", "5805432230": "devtools/ymake/bin/ymake for win32-clang-cl", "5829029743": "devtools/ymake/bin/ymake for win32-clang-cl", @@ -2635,6 +2649,7 @@ "8784096409": "gdb-14-linux-aarch64-6ea280bfa165555cc850c4ee3d49f07745e6f21b", "6406540582": "gdb-14-linux-aarch64-9d91f66a1caff272af94e2fc97e690a18d910204", "6447362348": "gdb-14-linux-aarch64-9db71d8b25a56ee316036c53fd941934a95831a3", + "8989381149": "gdb-14-linux-aarch64-a3d92401fcd70660cddbe51a1df09c7611122080", "6133337898": "gdb-14-linux-aarch64-b1fa9be28bbf4ee845d6a39a049c7b60018a3695", "8509776935": "gdb-14-linux-aarch64-b96bb13a4532bc0f19859cd0fd8590f36e4ac9f4", "8803622849": "gdb-14-linux-aarch64-e086c237e91cadf80f94fc0b2c3268624f9e1058", @@ -2644,6 +2659,7 @@ "8784094118": "gdb-14-linux-x86_64-6ea280bfa165555cc850c4ee3d49f07745e6f21b", "6406437536": "gdb-14-linux-x86_64-9d91f66a1caff272af94e2fc97e690a18d910204", "6447316775": "gdb-14-linux-x86_64-9db71d8b25a56ee316036c53fd941934a95831a3", + "8989701781": "gdb-14-linux-x86_64-a3d92401fcd70660cddbe51a1df09c7611122080", "6133419349": "gdb-14-linux-x86_64-b1fa9be28bbf4ee845d6a39a049c7b60018a3695", "8509757921": "gdb-14-linux-x86_64-b96bb13a4532bc0f19859cd0fd8590f36e4ac9f4", "8803613328": "gdb-14-linux-x86_64-e086c237e91cadf80f94fc0b2c3268624f9e1058", diff --git a/build/platform/test_tool/host.ya.make.inc b/build/platform/test_tool/host.ya.make.inc index e0e7def8560..f617eedd3fa 100644 --- a/build/platform/test_tool/host.ya.make.inc +++ b/build/platform/test_tool/host.ya.make.inc @@ -1,12 +1,12 @@ IF (HOST_OS_DARWIN AND HOST_ARCH_X86_64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8944950171) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8980614277) ELSEIF (HOST_OS_DARWIN AND HOST_ARCH_ARM64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8944944224) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8980612098) ELSEIF (HOST_OS_LINUX AND HOST_ARCH_X86_64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8944959540) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8980618559) ELSEIF (HOST_OS_LINUX AND HOST_ARCH_AARCH64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8944940103) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8980610092) ELSEIF (HOST_OS_WINDOWS AND HOST_ARCH_X86_64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8944955524) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8980616449) ENDIF() diff --git a/build/platform/test_tool/host_os.ya.make.inc b/build/platform/test_tool/host_os.ya.make.inc index 1744f9a6e44..b88a913e39e 100644 --- a/build/platform/test_tool/host_os.ya.make.inc +++ b/build/platform/test_tool/host_os.ya.make.inc @@ -1,12 +1,12 @@ IF (HOST_OS_DARWIN AND HOST_ARCH_X86_64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8944921953) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8980580464) ELSEIF (HOST_OS_DARWIN AND HOST_ARCH_ARM64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8944917987) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8980578191) ELSEIF (HOST_OS_LINUX AND HOST_ARCH_X86_64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8944928850) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8980584272) ELSEIF (HOST_OS_LINUX AND HOST_ARCH_AARCH64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8944914253) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8980575843) ELSEIF (HOST_OS_WINDOWS AND HOST_ARCH_X86_64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8944925436) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8980582056) ENDIF() diff --git a/build/ymake.core.conf b/build/ymake.core.conf index 8195ad0f667..d2b89fc87e5 100644 --- a/build/ymake.core.conf +++ b/build/ymake.core.conf @@ -2617,22 +2617,23 @@ macro _COPY_FILE_IMPL(TEXT[], AUTO_DST="", NOAUTO_DST="", OUTPUT_INCLUDES[], IND .SEM=target_commands-ITEM && target_commands-macro copy_file && target_commands-args ${input:FILE} ${output:AUTO_DST} ${noauto;output:NOAUTO_DST} } -### @usage: COPY_FILE(File Destination [AUTO] [OUTPUT_INCLUDES Deps...]) +### @usage: COPY_FILE(File Destination [AUTO] [OUTPUT_INCLUDES...] [INDUCED_DEPS...] [TEXT]) ### ### Copy file to build root. It is possible to change both location and the name. ### ### Parameters: ### - File - Source file name. -### - Destination - Output file name. +### - Dest - Output file name. ### - AUTO - Consider copied file for further processing automatically. ### - OUTPUT_INCLUDES output_includes... - Output file dependencies. ### - INDUCED_DEPS $VARs... - Dependencies for generated files. Unlike `OUTPUT_INCLUDES` these may target files further in processing chain. ### In order to do so VAR should be filled by PREPARE_INDUCED_DEPS macro, stating target files (by type) ### and set of dependencies +### - TEXT - deprecated ### ### The file will be just copied if AUTO boolean parameter is not specified. You should explicitly ### mention it in SRCS under new name (or specify AUTO boolean parameter) for further processing. -macro COPY_FILE(File, Destination, AUTO?"true":"false", OUTPUT_INCLUDES[], INDUCED_DEPS[], TEXT?"true":"false") { +macro COPY_FILE(File, Dest, AUTO?"true":"false", OUTPUT_INCLUDES[], INDUCED_DEPS[], TEXT?"true":"false") { # FIXME: the "TEXT" argument is effectively unused since _COPY_FILE_CONTEXT defaulting to TEXT .CMD=$_COPY_FILE_CMD .STRUCT_CMD=yes @@ -2641,27 +2642,29 @@ macro COPY_FILE(File, Destination, AUTO?"true":"false", OUTPUT_INCLUDES[], INDUC } _COPY_FILE_CMD=$IF( \ ${parse_bool:AUTO} \ - THEN $_COPY_FILE_IMPL(TEXT $File AUTO_DST $Destination OUTPUT_INCLUDES $OUTPUT_INCLUDES INDUCED_DEPS $INDUCED_DEPS) \ - ELSE $_COPY_FILE_IMPL(TEXT $File NOAUTO_DST $Destination OUTPUT_INCLUDES $OUTPUT_INCLUDES INDUCED_DEPS $INDUCED_DEPS) \ + THEN $_COPY_FILE_IMPL(TEXT $File AUTO_DST $Dest OUTPUT_INCLUDES $OUTPUT_INCLUDES INDUCED_DEPS $INDUCED_DEPS) \ + ELSE $_COPY_FILE_IMPL(TEXT $File NOAUTO_DST $Dest OUTPUT_INCLUDES $OUTPUT_INCLUDES INDUCED_DEPS $INDUCED_DEPS) \ ) _COPY_FILE_SEM=$IF( \ ${parse_bool:AUTO} \ - THEN $_COPY_FILE_IMPL($File AUTO_DST $Destination) \ - ELSE $_COPY_FILE_IMPL($File NOAUTO_DST $Destination) \ + THEN $_COPY_FILE_IMPL($File AUTO_DST $Dest) \ + ELSE $_COPY_FILE_IMPL($File NOAUTO_DST $Dest) \ ) -### @usage: COPY_FILE_WITH_CONTEXT(FILE DEST [AUTO] [OUTPUT_INCLUDES DEPS...]) +### @usage: COPY_FILE_WITH_CONTEXT(File Dest [AUTO] [OUTPUT_INCLUDES...] [INDUCED_DEPS...]) ### ### Copy file to build root the same way as it is done for COPY_FILE, but also ### propagates the context of the source file. -macro COPY_FILE_WITH_CONTEXT(FILE, DEST, AUTO?"true":"false", OUTPUT_INCLUDES[], INDUCED_DEPS[]) { +macro COPY_FILE_WITH_CONTEXT(File, Dest, AUTO?"true":"false", OUTPUT_INCLUDES[], INDUCED_DEPS[]) { .CMD=$_COPY_FILE_WITH_CONTEXT_CMD .STRUCT_CMD=yes + .SEM=$_COPY_FILE_SEM + .STRUCT_SEM=yes } _COPY_FILE_WITH_CONTEXT_CMD=$IF( \ ${parse_bool:AUTO} \ - THEN $_COPY_FILE_IMPL($FILE AUTO_DST $DEST OUTPUT_INCLUDES_INP $FILE OUTPUT_INCLUDES $OUTPUT_INCLUDES INDUCED_DEPS $INDUCED_DEPS) \ - ELSE $_COPY_FILE_IMPL($FILE NOAUTO_DST $DEST OUTPUT_INCLUDES_INP $FILE OUTPUT_INCLUDES $OUTPUT_INCLUDES INDUCED_DEPS $INDUCED_DEPS) \ + THEN $_COPY_FILE_IMPL($File AUTO_DST $Dest OUTPUT_INCLUDES_INP $File OUTPUT_INCLUDES $OUTPUT_INCLUDES INDUCED_DEPS $INDUCED_DEPS) \ + ELSE $_COPY_FILE_IMPL($File NOAUTO_DST $Dest OUTPUT_INCLUDES_INP $File OUTPUT_INCLUDES $OUTPUT_INCLUDES INDUCED_DEPS $INDUCED_DEPS) \ ) ### This is to join $ALL_RES_ and $EXT diff --git a/contrib/libs/ftxui/.yandex_meta/__init__.py b/contrib/libs/ftxui/.yandex_meta/__init__.py index 77b5f726f43..1ac64d6668f 100644 --- a/contrib/libs/ftxui/.yandex_meta/__init__.py +++ b/contrib/libs/ftxui/.yandex_meta/__init__.py @@ -1,5 +1,15 @@ +from devtools.yamaker.modules import Linkable, Switch from devtools.yamaker.project import CMakeNinjaNixProject + +def post_install(self): + with self.yamakes["."] as ftxui: + ftxui.after( + "CFLAGS", + Switch({"OS_WINDOWS": Linkable(CFLAGS=["-DUNICODE", "-D_UNICODE"])}), + ) + + ftxui = CMakeNinjaNixProject( owners=["segoon", "g:taxi-common"], nixattr="ftxui", @@ -9,4 +19,5 @@ ftxui = CMakeNinjaNixProject( disable_includes=["emscripten.h"], put_with={"ftxui-component": ["ftxui-dom", "ftxui-screen"]}, arcdir="contrib/libs/ftxui", + post_install=post_install, ) diff --git a/contrib/libs/ftxui/.yandex_meta/override.nix b/contrib/libs/ftxui/.yandex_meta/override.nix index c1bffb907ac..4fe68fcd8b5 100644 --- a/contrib/libs/ftxui/.yandex_meta/override.nix +++ b/contrib/libs/ftxui/.yandex_meta/override.nix @@ -14,5 +14,8 @@ pkgs: attrs: with pkgs; with attrs; rec { "-DFTXUI_BUILD_EXAMPLES=OFF" "-DFTXUI_BUILD_DOCS=OFF" "-DFTXUI_BUILD_TESTS=OFF" + ] ++ lib.optionals (!stdenv.isLinux && !stdenv.isDarwin) [ + "-DUNICODE=ON" + "-D_UNICODE=ON" ]; } diff --git a/contrib/libs/ftxui/ya.make b/contrib/libs/ftxui/ya.make index 19175672755..842e979a92d 100644 --- a/contrib/libs/ftxui/ya.make +++ b/contrib/libs/ftxui/ya.make @@ -22,6 +22,13 @@ NO_COMPILER_WARNINGS() NO_UTIL() +IF (OS_WINDOWS) + CFLAGS( + -DUNICODE + -D_UNICODE + ) +ENDIF() + SRCS( src/ftxui/component/animation.cpp src/ftxui/component/button.cpp diff --git a/library/cpp/protobuf/json/json2proto.cpp b/library/cpp/protobuf/json/json2proto.cpp index 823c471b967..a1d7e862ef0 100644 --- a/library/cpp/protobuf/json/json2proto.cpp +++ b/library/cpp/protobuf/json/json2proto.cpp @@ -7,6 +7,8 @@ #include <google/protobuf/util/time_util.h> #include <google/protobuf/message.h> #include <google/protobuf/descriptor.h> +#include <google/protobuf/struct.pb.h> + #include <util/generic/hash.h> #include <util/generic/maybe.h> @@ -517,6 +519,57 @@ namespace NProtobufJson { const google::protobuf::Descriptor* descriptor = proto.GetDescriptor(); Y_ASSERT(!!descriptor); + + if (descriptor->well_known_type() == google::protobuf::Descriptor::WELLKNOWNTYPE_STRUCT) { + Y_ENSURE(json.IsMap(), "Failed to merge json to proto for message: " << descriptor->full_name() << ", expected json map."); + google::protobuf::Struct msg; + for (const auto& [key, value] : json.GetMap()) { + google::protobuf::Value valueMsg; + MergeJson2Proto(value, valueMsg, config); + (*msg.mutable_fields())[key] = std::move(valueMsg); + } + proto.GetReflection()->Swap(&proto, &msg); + return; + } else if (descriptor->well_known_type() == google::protobuf::Descriptor::WELLKNOWNTYPE_VALUE) { + google::protobuf::Value msg; + switch (json.GetType()) { + case NJson::JSON_UNDEFINED: + break; + case NJson::JSON_NULL: + msg.set_null_value({}); + break; + case NJson::JSON_BOOLEAN: + msg.set_bool_value(json.GetBoolean()); + break; + case NJson::JSON_INTEGER: + case NJson::JSON_DOUBLE: + case NJson::JSON_UINTEGER: + msg.set_number_value(json.GetDouble()); + break; + case NJson::JSON_STRING: + msg.set_string_value(json.GetString()); + break; + case NJson::JSON_MAP: + { + auto* structValue = msg.mutable_struct_value(); + MergeJson2Proto(json, *structValue, config); + break; + } + case NJson::JSON_ARRAY: + { + auto* arrayValue = msg.mutable_list_value(); + const auto& jsonArray = json.GetArray(); + arrayValue->mutable_values()->Reserve(jsonArray.size()); + for (const auto& item : jsonArray) { + MergeJson2Proto(item, *arrayValue->add_values(), config); + } + break; + } + } + + proto.GetReflection()->Swap(&proto, &msg); + return; + } Y_ENSURE(json.IsMap(), "Failed to merge json to proto for message: " << descriptor->full_name() << ", expected json map."); for (int f = 0, endF = descriptor->field_count(); f < endF; ++f) { diff --git a/library/cpp/protobuf/json/ut/json2proto_ut.cpp b/library/cpp/protobuf/json/ut/json2proto_ut.cpp index 7b3e829868e..1172b2b8db6 100644 --- a/library/cpp/protobuf/json/ut/json2proto_ut.cpp +++ b/library/cpp/protobuf/json/ut/json2proto_ut.cpp @@ -2,6 +2,8 @@ #include "proto.h" #include "proto2json.h" +#include <google/protobuf/struct.pb.h> + #include <library/cpp/protobuf/json/ut/test.pb.h> #include <library/cpp/json/json_value.h> @@ -1186,4 +1188,81 @@ Y_UNIT_TEST(TestSimplifiedTimestamp) { UNIT_ASSERT_EQUAL(NProtoInterop::CastFromProto(simpleTimestamp.GetTimestamp()), TInstant::ParseIso8601("2014-08-26T15:52:15Z")); } // TestSimplifiedTimestamp +Y_UNIT_TEST(TestValue) { + { + NJson::TJsonValue json = 100.0; + google::protobuf::Value value; + NProtobufJson::Json2Proto(json, value); + UNIT_ASSERT(value.has_number_value()); + UNIT_ASSERT_EQUAL(value.number_value(), 100.0); + } + { + NJson::TJsonValue json = 100LL; + google::protobuf::Value value; + NProtobufJson::Json2Proto(json, value); + UNIT_ASSERT(value.has_number_value()); + UNIT_ASSERT_EQUAL(value.number_value(), 100.0); + } + { + NJson::TJsonValue json = 100ULL; + google::protobuf::Value value; + NProtobufJson::Json2Proto(json, value); + UNIT_ASSERT(value.has_number_value()); + UNIT_ASSERT_EQUAL(value.number_value(), 100.0); + } + { + NJson::TJsonValue json = "TestString"; + google::protobuf::Value value; + NProtobufJson::Json2Proto(json, value); + UNIT_ASSERT(value.has_string_value()); + UNIT_ASSERT_EQUAL(value.string_value(), "TestString"); + } + { + NJson::TJsonValue json; + json.AppendValue("TestString"); + json.AppendValue(2); + google::protobuf::Value value; + NProtobufJson::Json2Proto(json, value); + UNIT_ASSERT(value.has_list_value()); + const auto& list = value.list_value().values(); + UNIT_ASSERT_EQUAL(list.size(), 2); + UNIT_ASSERT(list.Get(0).has_string_value()); + UNIT_ASSERT_EQUAL(list.Get(0).string_value(), "TestString"); + UNIT_ASSERT(list.Get(1).has_number_value()); + UNIT_ASSERT_EQUAL(list.Get(1).number_value(), 2.0); + } + { + NJson::TJsonValue json; + json["str"] = "TestString"; + json["int"] = 2; + google::protobuf::Value value; + NProtobufJson::Json2Proto(json, value); + UNIT_ASSERT(value.has_struct_value()); + const auto& fields = value.struct_value().fields(); + UNIT_ASSERT_EQUAL(fields.size(), 2); + + UNIT_ASSERT(fields.at("str").has_string_value()); + UNIT_ASSERT_EQUAL(fields.at("str").string_value(), "TestString"); + + UNIT_ASSERT(fields.at("int").has_number_value()); + UNIT_ASSERT_EQUAL(fields.at("int").number_value(), 2.0); + } +} // TestValue + +Y_UNIT_TEST(TestStruct) { + NJson::TJsonValue json; + json["str"] = "TestString"; + json["int"] = 2; + google::protobuf::Struct structProto; + NProtobufJson::Json2Proto(json, structProto); + + const auto& fields = structProto.fields(); + UNIT_ASSERT_EQUAL(fields.size(), 2); + UNIT_ASSERT(fields.at("str").has_string_value()); + UNIT_ASSERT_EQUAL(fields.at("str").string_value(), "TestString"); + + UNIT_ASSERT(fields.at("int").has_number_value()); + UNIT_ASSERT_EQUAL(fields.at("int").number_value(), 2.0); +} // TestStruct + } // TJson2ProtoTest @@ -39,33 +39,33 @@ REGISTRY_ENDPOINT = os.environ.get("YA_REGISTRY_ENDPOINT", "https://devtools-reg PLATFORM_MAP = { "data": { "win32": { - "md5": "6930b764b10d4ea8d8e1fdcc86fe81af", + "md5": "9534abf2236bf986a73695c6a6020d9b", "urls": [ - f"{REGISTRY_ENDPOINT}/8945021042" + f"{REGISTRY_ENDPOINT}/8980613788" ] }, "darwin": { - "md5": "f36a717cf732b6f3be1ea5e98c96c08a", + "md5": "a303402147dc93d600c4c23e54ac2313", "urls": [ - f"{REGISTRY_ENDPOINT}/8945018014" + f"{REGISTRY_ENDPOINT}/8980611775" ] }, "darwin-arm64": { - "md5": "20710b2a45d8ac8bcc78a9310dd08511", + "md5": "bf51ba63f8ded4b9b19b841869f24ea7", "urls": [ - f"{REGISTRY_ENDPOINT}/8945012760" + f"{REGISTRY_ENDPOINT}/8980609753" ] }, "linux-aarch64": { - "md5": "a7baedd28b02efb64a1c1101b249f553", + "md5": "74e1dd7dbb2639d9a4590f91da3cbac7", "urls": [ - f"{REGISTRY_ENDPOINT}/8945008840" + f"{REGISTRY_ENDPOINT}/8980607719" ] }, "linux": { - "md5": "3ef5d52fee076763d64a978df3e2f54d", + "md5": "2bb10d9f76f496f5982601a38c61856e", "urls": [ - f"{REGISTRY_ENDPOINT}/8945024867" + f"{REGISTRY_ENDPOINT}/8980615911" ] } } diff --git a/ydb/ci/rightlib.txt b/ydb/ci/rightlib.txt index 852d3419e50..d7ce019752e 100644 --- a/ydb/ci/rightlib.txt +++ b/ydb/ci/rightlib.txt @@ -1 +1 @@ -1496d6ee1dfd8d449a3f9de4703f86dd5a1243f5 +eb5cfac23dd911355748ad88375812f00b50958e diff --git a/yql/essentials/cfg/tests/gateways.conf b/yql/essentials/cfg/tests/gateways.conf index 49af78d8793..920e10d5e1f 100644 --- a/yql/essentials/cfg/tests/gateways.conf +++ b/yql/essentials/cfg/tests/gateways.conf @@ -82,6 +82,11 @@ Yt { Value: "10M" } + DefaultSettings { + Name: "UseIntermediateStreams" + Value: "true" + } + RemoteFilePatterns { Pattern: "yt://([a-zA-Z0-9\\-_]+)/([^&@?]+)$" Cluster: "$1" diff --git a/yql/essentials/core/facade/yql_facade.cpp b/yql/essentials/core/facade/yql_facade.cpp index 2543dcf2d62..e06135256fe 100644 --- a/yql/essentials/core/facade/yql_facade.cpp +++ b/yql/essentials/core/facade/yql_facade.cpp @@ -1989,6 +1989,7 @@ TTypeAnnotationContextPtr TProgram::BuildTypeAnnotationContext(const TString& us auto typeAnnotationContext = MakeIntrusive<TTypeAnnotationContext>(); typeAnnotationContext->LangVer = LangVer_; + typeAnnotationContext->UseTypeDiffForConvertToError = true; typeAnnotationContext->UserDataStorage = UserDataStorage_; typeAnnotationContext->Credentials = Credentials_; typeAnnotationContext->Modules = Modules_; diff --git a/yql/essentials/core/type_ann/type_ann_blocks.cpp b/yql/essentials/core/type_ann/type_ann_blocks.cpp index c988284b2bc..6352feca919 100644 --- a/yql/essentials/core/type_ann/type_ann_blocks.cpp +++ b/yql/essentials/core/type_ann/type_ann_blocks.cpp @@ -968,7 +968,7 @@ IGraphTransformer::TStatus ListFromBlocksWrapper(const TExprNode::TPtr& input, T return IGraphTransformer::TStatus::Ok; } -IGraphTransformer::TStatus WideSkipTakeBlocksWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { +IGraphTransformer::TStatus WideSkipTakeBlocksWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) { if (!EnsureArgsCount(*input, 2U, ctx.Expr)) { return IGraphTransformer::TStatus::Error; } @@ -980,7 +980,7 @@ IGraphTransformer::TStatus WideSkipTakeBlocksWrapper(const TExprNode::TPtr& inpu output = input; const TTypeAnnotationNode* expectedType = ctx.Expr.MakeType<TDataExprType>(EDataSlot::Uint64); - auto convertStatus = TryConvertTo(input->ChildRef(1), *expectedType, ctx.Expr); + auto convertStatus = TryConvertTo(input->ChildRef(1), *expectedType, ctx.Expr, {}, ctx.Types.UseTypeDiffForConvertToError); if (convertStatus.Level == IGraphTransformer::TStatus::Error) { ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), "Can not convert argument to Uint64")); return IGraphTransformer::TStatus::Error; @@ -994,7 +994,7 @@ IGraphTransformer::TStatus WideSkipTakeBlocksWrapper(const TExprNode::TPtr& inpu return IGraphTransformer::TStatus::Ok; } -IGraphTransformer::TStatus WideTopBlocksWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { +IGraphTransformer::TStatus WideTopBlocksWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) { if (!EnsureArgsCount(*input, 3U, ctx.Expr)) { return IGraphTransformer::TStatus::Error; } @@ -1006,7 +1006,7 @@ IGraphTransformer::TStatus WideTopBlocksWrapper(const TExprNode::TPtr& input, TE output = input; const TTypeAnnotationNode* expectedType = ctx.Expr.MakeType<TDataExprType>(EDataSlot::Uint64); - auto convertStatus = TryConvertTo(input->ChildRef(1), *expectedType, ctx.Expr); + auto convertStatus = TryConvertTo(input->ChildRef(1), *expectedType, ctx.Expr, {}, ctx.Types.UseTypeDiffForConvertToError); if (convertStatus.Level == IGraphTransformer::TStatus::Error) { ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), "Can not convert argument to Uint64")); return IGraphTransformer::TStatus::Error; diff --git a/yql/essentials/core/type_ann/type_ann_blocks.h b/yql/essentials/core/type_ann/type_ann_blocks.h index 223758ef7fc..4d417ad4ed4 100644 --- a/yql/essentials/core/type_ann/type_ann_blocks.h +++ b/yql/essentials/core/type_ann/type_ann_blocks.h @@ -31,8 +31,8 @@ namespace NTypeAnnImpl { IGraphTransformer::TStatus ListToBlocksWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx); IGraphTransformer::TStatus WideFromBlocksWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus ListFromBlocksWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); - IGraphTransformer::TStatus WideSkipTakeBlocksWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); - IGraphTransformer::TStatus WideTopBlocksWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); + IGraphTransformer::TStatus WideSkipTakeBlocksWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx); + IGraphTransformer::TStatus WideTopBlocksWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx); IGraphTransformer::TStatus WideSortBlocksWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus BlockPgOpWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus BlockPgCallWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); diff --git a/yql/essentials/core/type_ann/type_ann_core.cpp b/yql/essentials/core/type_ann/type_ann_core.cpp index 166307cdf68..75043a88ede 100644 --- a/yql/essentials/core/type_ann/type_ann_core.cpp +++ b/yql/essentials/core/type_ann/type_ann_core.cpp @@ -2161,7 +2161,7 @@ namespace NTypeAnnImpl { return IGraphTransformer::TStatus::Ok; } - IGraphTransformer::TStatus MinMaxWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + IGraphTransformer::TStatus MinMaxWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) { if (!EnsureMinArgsCount(*input, 1, ctx.Expr)) { return IGraphTransformer::TStatus::Error; } @@ -2184,7 +2184,8 @@ namespace NTypeAnnImpl { return IGraphTransformer::TStatus::Error; } - if (const auto status = ConvertChildrenToType(input, commonItemType, ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + if (const auto status = ConvertChildrenToType(input, commonItemType, ctx.Expr, ctx.Types.UseTypeDiffForConvertToError); + status != IGraphTransformer::TStatus::Ok) { return status; } @@ -3050,7 +3051,7 @@ namespace NTypeAnnImpl { return IGraphTransformer::TStatus::Ok; } - IGraphTransformer::TStatus ShiftWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + IGraphTransformer::TStatus ShiftWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) { if (!EnsureArgsCount(*input, 2, ctx.Expr)) { return IGraphTransformer::TStatus::Error; } @@ -3074,7 +3075,7 @@ namespace NTypeAnnImpl { } const auto expectedType = ctx.Expr.MakeType<TDataExprType>(EDataSlot::Uint8); - const auto convertStatus = TryConvertTo(input->TailRef(), *expectedType, ctx.Expr); + const auto convertStatus = TryConvertTo(input->TailRef(), *expectedType, ctx.Expr, {}, ctx.Types.UseTypeDiffForConvertToError); if (convertStatus.Level == IGraphTransformer::TStatus::Error) { ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Tail().Pos()), "Shift count must be Uint8")); return IGraphTransformer::TStatus::Error; @@ -3214,7 +3215,7 @@ namespace NTypeAnnImpl { return IGraphTransformer::TStatus::Ok; } - IGraphTransformer::TStatus SubstringWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + IGraphTransformer::TStatus SubstringWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) { if (!EnsureArgsCount(*input, 3, ctx.Expr)) { return IGraphTransformer::TStatus::Error; } @@ -3249,7 +3250,7 @@ namespace NTypeAnnImpl { if (kindOne == ETypeAnnotationKind::Optional || kindOne == ETypeAnnotationKind::Null) { expectedTypeOne = ctx.Expr.MakeType<TOptionalExprType>(expectedTypeOne); } - const auto convertStatus1 = TryConvertTo(input->ChildRef(1), *expectedTypeOne, ctx.Expr); + const auto convertStatus1 = TryConvertTo(input->ChildRef(1), *expectedTypeOne, ctx.Expr, {}, ctx.Types.UseTypeDiffForConvertToError); if (convertStatus1.Level == IGraphTransformer::TStatus::Error) { ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), "Mismatch argument types")); return IGraphTransformer::TStatus::Error; @@ -3260,7 +3261,7 @@ namespace NTypeAnnImpl { if (kindTwo == ETypeAnnotationKind::Optional || kindTwo == ETypeAnnotationKind::Null) { expectedTypeTwo = ctx.Expr.MakeType<TOptionalExprType>(expectedTypeTwo); } - const auto convertStatus2 = TryConvertTo(input->ChildRef(2), *expectedTypeTwo, ctx.Expr); + const auto convertStatus2 = TryConvertTo(input->ChildRef(2), *expectedTypeTwo, ctx.Expr, {}, ctx.Types.UseTypeDiffForConvertToError); if (convertStatus2.Level == IGraphTransformer::TStatus::Error) { ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(2)->Pos()), "Mismatch argument types")); return IGraphTransformer::TStatus::Error; @@ -3275,7 +3276,7 @@ namespace NTypeAnnImpl { return IGraphTransformer::TStatus::Ok; } - IGraphTransformer::TStatus FindWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + IGraphTransformer::TStatus FindWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) { if (!EnsureArgsCount(*input, 3, ctx.Expr)) { return IGraphTransformer::TStatus::Error; } @@ -3301,7 +3302,8 @@ namespace NTypeAnnImpl { return IGraphTransformer::TStatus::Error; } - if (const auto convertStatus = TryConvertTo(input->ChildRef(1), *type, ctx.Expr); convertStatus.Level != IGraphTransformer::TStatus::Ok) { + if (const auto convertStatus = TryConvertTo(input->ChildRef(1), *type, ctx.Expr, {}, ctx.Types.UseTypeDiffForConvertToError); + convertStatus.Level != IGraphTransformer::TStatus::Ok) { return convertStatus; } @@ -3310,7 +3312,8 @@ namespace NTypeAnnImpl { expectedType = ctx.Expr.MakeType<TOptionalExprType>(expectedType); } - if (const auto convertStatus = TryConvertTo(input->ChildRef(2), *expectedType, ctx.Expr); convertStatus.Level != IGraphTransformer::TStatus::Ok) { + if (const auto convertStatus = TryConvertTo(input->ChildRef(2), *expectedType, ctx.Expr, {}, ctx.Types.UseTypeDiffForConvertToError); + convertStatus.Level != IGraphTransformer::TStatus::Ok) { return convertStatus; } @@ -3364,7 +3367,7 @@ namespace NTypeAnnImpl { return IGraphTransformer::TStatus::Ok; } - IGraphTransformer::TStatus ByteAtWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + IGraphTransformer::TStatus ByteAtWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) { if (!EnsureArgsCount(*input, 2, ctx.Expr)) { return IGraphTransformer::TStatus::Error; } @@ -3399,7 +3402,7 @@ namespace NTypeAnnImpl { } auto expectedType = ctx.Expr.MakeType<TDataExprType>(EDataSlot::Uint32); - auto convertStatus = TryConvertTo(input->ChildRef(1), *expectedType, ctx.Expr); + auto convertStatus = TryConvertTo(input->ChildRef(1), *expectedType, ctx.Expr, {}, ctx.Types.UseTypeDiffForConvertToError); if (convertStatus.Level == IGraphTransformer::TStatus::Error) { ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), "Mismatch argument types")); return IGraphTransformer::TStatus::Error; @@ -3432,7 +3435,7 @@ namespace NTypeAnnImpl { } template <bool IsStrict> - IGraphTransformer::TStatus AsListWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + IGraphTransformer::TStatus AsListWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) { if (input->ChildrenSize() == 0) { output = ctx.Expr.NewCallable(input->Pos(), "EmptyList", {}); return IGraphTransformer::TStatus::Repeat; @@ -3456,7 +3459,8 @@ namespace NTypeAnnImpl { output = ctx.Expr.RenameNode(*input, "AsList"); return IGraphTransformer::TStatus::Repeat; } else if (const auto commonItemType = CommonTypeForChildren(*input, ctx.Expr, warn)) { - if (const auto status = ConvertChildrenToType(input, commonItemType, ctx.Expr); status != IGraphTransformer::TStatus::Ok) + if (const auto status = ConvertChildrenToType(input, commonItemType, ctx.Expr, ctx.Types.UseTypeDiffForConvertToError); + status != IGraphTransformer::TStatus::Ok) return status; } else { return IGraphTransformer::TStatus::Error; @@ -5394,7 +5398,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> return IGraphTransformer::TStatus::Ok; } - IGraphTransformer::TStatus VariantWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + IGraphTransformer::TStatus VariantWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) { Y_UNUSED(output); if (!EnsureArgsCount(*input, 3, ctx.Expr)) { return IGraphTransformer::TStatus::Error; @@ -5442,7 +5446,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> itemType = tupleType->GetItems()[index]; } - auto convertStatus = TryConvertTo(input->ChildRef(0), *itemType, ctx.Expr); + auto convertStatus = TryConvertTo(input->ChildRef(0), *itemType, ctx.Expr, {}, ctx.Types.UseTypeDiffForConvertToError); if (convertStatus.Level == IGraphTransformer::TStatus::Error) { return IGraphTransformer::TStatus::Error; } else if (convertStatus.Level != IGraphTransformer::TStatus::Ok) { @@ -5915,7 +5919,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> } template <bool IsStrict, bool IsSet> - IGraphTransformer::TStatus AsDictWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + IGraphTransformer::TStatus AsDictWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) { if (input->ChildrenSize() == 0) { output = ctx.Expr.NewCallable(input->Pos(), "EmptyDict", {}); return IGraphTransformer::TStatus::Repeat; @@ -5951,7 +5955,8 @@ template <NKikimr::NUdf::EDataSlot DataSlot> output = ctx.Expr.RenameNode(*input, IsSet ? "AsSet" : "AsDict"); return IGraphTransformer::TStatus::Repeat; } else if (const auto commonType = CommonTypeForChildren(*input, ctx.Expr, warn)) { - if (const auto status = ConvertChildrenToType(input, commonType, ctx.Expr); status != IGraphTransformer::TStatus::Ok) + if (const auto status = ConvertChildrenToType(input, commonType, ctx.Expr, ctx.Types.UseTypeDiffForConvertToError); + status != IGraphTransformer::TStatus::Ok) return status; if (warn) { @@ -5973,7 +5978,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> return IGraphTransformer::TStatus::Ok; } - IGraphTransformer::TStatus DictFromKeysWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + IGraphTransformer::TStatus DictFromKeysWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) { Y_UNUSED(output); if (!EnsureArgsCount(*input, 2, ctx.Expr)) { return IGraphTransformer::TStatus::Error; @@ -5998,7 +6003,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> for (size_t i = 0; i < keysListNode->ChildrenSize(); ++i) { newChildren.push_back(keysListNode->ChildPtr(i)); auto& childRef = newChildren.back(); - auto status = TryConvertTo(childRef, *keyType, ctx.Expr); + auto status = TryConvertTo(childRef, *keyType, ctx.Expr, {}, ctx.Types.UseTypeDiffForConvertToError); totalStatus = totalStatus.Combine(status); if (status.Level == IGraphTransformer::TStatus::Error) { return status; @@ -6020,7 +6025,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> } template <bool IsStrict> - IGraphTransformer::TStatus IfWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + IGraphTransformer::TStatus IfWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) { if (!EnsureArgsCount(*input, 3, ctx.Expr)) { return IGraphTransformer::TStatus::Error; } @@ -6051,7 +6056,9 @@ template <NKikimr::NUdf::EDataSlot DataSlot> return IGraphTransformer::TStatus::Error; } } else if (const auto commonType = CommonType<false>(input->Pos(), thenType, elseType, ctx.Expr)) { - if (const auto status = TryConvertTo(input->ChildRef(1), *commonType, ctx.Expr).Combine(TryConvertTo(input->TailRef(), *commonType, ctx.Expr)); status != IGraphTransformer::TStatus::Ok) + if (const auto status = TryConvertTo(input->ChildRef(1), *commonType, ctx.Expr, {}, ctx.Types.UseTypeDiffForConvertToError) + .Combine(TryConvertTo(input->TailRef(), *commonType, ctx.Expr, {}, ctx.Types.UseTypeDiffForConvertToError)); + status != IGraphTransformer::TStatus::Ok) return status; input->SetTypeAnn(commonType); @@ -7809,7 +7816,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> return IGraphTransformer::TStatus::Repeat; } - auto status = TryConvertTo(input->ChildRef(1), *cachedRunConfigType, ctx.Expr); + auto status = TryConvertTo(input->ChildRef(1), *cachedRunConfigType, ctx.Expr, {}, ctx.Types.UseTypeDiffForConvertToError); if (status.Level != IGraphTransformer::TStatus::Ok) { if (status.Level == IGraphTransformer::TStatus::Error) { ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), TStringBuilder() << "Mismatch type of run config in UDF function " @@ -8580,14 +8587,14 @@ template <NKikimr::NUdf::EDataSlot DataSlot> return IGraphTransformer::TStatus::Ok; } - IGraphTransformer::TStatus NewMTRandWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + IGraphTransformer::TStatus NewMTRandWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) { Y_UNUSED(output); if (!EnsureArgsCount(*input, 1, ctx.Expr)) { return IGraphTransformer::TStatus::Error; } auto expectedType = ctx.Expr.MakeType<TDataExprType>(EDataSlot::Uint64); - auto convertStatus = TryConvertTo(input->ChildRef(0), *expectedType, ctx.Expr); + auto convertStatus = TryConvertTo(input->ChildRef(0), *expectedType, ctx.Expr, {}, ctx.Types.UseTypeDiffForConvertToError); if (convertStatus.Level == IGraphTransformer::TStatus::Error) { ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Head().Pos()), "Mismatch argument types")); return IGraphTransformer::TStatus::Error; @@ -8818,7 +8825,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> return IGraphTransformer::TStatus::Ok; } - IGraphTransformer::TStatus DynamicVariantWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + IGraphTransformer::TStatus DynamicVariantWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) { Y_UNUSED(output); if (!EnsureArgsCount(*input, 3, ctx.Expr)) { return IGraphTransformer::TStatus::Error; @@ -8845,7 +8852,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> EDataSlot::Uint32 : EDataSlot::Utf8)); - auto convertStatus = TryConvertTo(input->ChildRef(1), *expectedType, ctx.Expr); + auto convertStatus = TryConvertTo(input->ChildRef(1), *expectedType, ctx.Expr, {}, ctx.Types.UseTypeDiffForConvertToError); if (convertStatus.Level == IGraphTransformer::TStatus::Error) { ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), "Mismatch argument types")); return IGraphTransformer::TStatus::Error; @@ -8879,7 +8886,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> } } - convertStatus = TryConvertTo(input->ChildRef(0), *firstType, ctx.Expr); + convertStatus = TryConvertTo(input->ChildRef(0), *firstType, ctx.Expr, {}, ctx.Types.UseTypeDiffForConvertToError); if (convertStatus.Level == IGraphTransformer::TStatus::Error) { ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(0)->Pos()), TStringBuilder() << "Mismatch item type, expected: " << *firstType << ", got: " << *input->Child(0)->GetTypeAnn())); @@ -10541,7 +10548,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> if (!EnsureQueueResource(resourceArg, expectedValueType, ctx)) { return IGraphTransformer::TStatus::Error; } - auto convertStatus = TryConvertTo(valueArg, *expectedValueType, ctx.Expr); + auto convertStatus = TryConvertTo(valueArg, *expectedValueType, ctx.Expr, {}, ctx.Types.UseTypeDiffForConvertToError); if (convertStatus.Level != IGraphTransformer::TStatus::Ok) { return convertStatus; } @@ -10577,7 +10584,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> return IGraphTransformer::TStatus::Error; } auto expectedIndexType = ctx.Expr.MakeType<TDataExprType>(EDataSlot::Uint64); - auto convertStatus = TryConvertTo(indexArg, *expectedIndexType, ctx.Expr); + auto convertStatus = TryConvertTo(indexArg, *expectedIndexType, ctx.Expr, {}, ctx.Types.UseTypeDiffForConvertToError); if (convertStatus.Level != IGraphTransformer::TStatus::Ok) { return convertStatus; } @@ -10602,8 +10609,8 @@ template <NKikimr::NUdf::EDataSlot DataSlot> } auto expectedIndexType = ctx.Expr.MakeType<TDataExprType>(EDataSlot::Uint64); - auto convertStatus = TryConvertTo(beginArg, *expectedIndexType, ctx.Expr); - convertStatus = convertStatus.Combine(TryConvertTo(endArg, *expectedIndexType, ctx.Expr)); + auto convertStatus = TryConvertTo(beginArg, *expectedIndexType, ctx.Expr, {}, ctx.Types.UseTypeDiffForConvertToError); + convertStatus = convertStatus.Combine(TryConvertTo(endArg, *expectedIndexType, ctx.Expr, {}, ctx.Types.UseTypeDiffForConvertToError)); if (convertStatus.Level != IGraphTransformer::TStatus::Ok) { return convertStatus; } @@ -10737,7 +10744,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> } } - IGraphTransformer::TStatus WeakFieldWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + IGraphTransformer::TStatus WeakFieldWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) { if (!EnsureMinArgsCount(*input, 3, ctx.Expr) || !EnsureMaxArgsCount(*input, 4, ctx.Expr)) { return IGraphTransformer::TStatus::Error; } @@ -10765,7 +10772,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> const bool isDefault = input->ChildrenSize() == 4; if (isDefault) { - auto convertStatus = TryConvertTo(input->ChildRef(3), *targetTypeExpr, ctx.Expr); + auto convertStatus = TryConvertTo(input->ChildRef(3), *targetTypeExpr, ctx.Expr, {}, ctx.Types.UseTypeDiffForConvertToError); if (convertStatus.Level == IGraphTransformer::TStatus::Error) { ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), TStringBuilder() << "Default value not correspond weak field type: " << targetType)); @@ -10954,7 +10961,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> return IGraphTransformer::TStatus::Repeat; } - IGraphTransformer::TStatus TryWeakMemberFromDictWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + IGraphTransformer::TStatus TryWeakMemberFromDictWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) { Y_UNUSED(output); if (!EnsureArgsCount(*input, 4, ctx.Expr)) { return IGraphTransformer::TStatus::Error; @@ -10966,7 +10973,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> const EDataSlot payloadStr = isYsonPayload ? EDataSlot::Yson : EDataSlot::String; auto expectedType = ctx.Expr.MakeType<TOptionalExprType>(ctx.Expr.MakeType<TDictExprType>( ctx.Expr.MakeType<TDataExprType>(EDataSlot::String), ctx.Expr.MakeType<TDataExprType>(payloadStr))); - auto convertStatus = TryConvertTo(dictNodeRef, *expectedType, ctx.Expr); + auto convertStatus = TryConvertTo(dictNodeRef, *expectedType, ctx.Expr, {}, ctx.Types.UseTypeDiffForConvertToError); if (convertStatus.Level != IGraphTransformer::TStatus::Ok) { return convertStatus; } @@ -11395,7 +11402,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> return IGraphTransformer::TStatus::Ok; } - IGraphTransformer::TStatus TimezoneIdWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + IGraphTransformer::TStatus TimezoneIdWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) { if (!EnsureArgsCount(*input, 1, ctx.Expr)) { return IGraphTransformer::TStatus::Error; } @@ -11412,7 +11419,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> // name } else { const TTypeAnnotationNode* expectedType = isOptional ? optUi16Type : ui16Type; - auto convertStatus = TryConvertTo(input->ChildRef(0), *expectedType, ctx.Expr); + auto convertStatus = TryConvertTo(input->ChildRef(0), *expectedType, ctx.Expr, {}, ctx.Types.UseTypeDiffForConvertToError); if (convertStatus.Level == IGraphTransformer::TStatus::Error) { ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Head().Pos()), "Mismatch argument types")); return IGraphTransformer::TStatus::Error; @@ -11430,7 +11437,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> return IGraphTransformer::TStatus::Ok; } - IGraphTransformer::TStatus TimezoneNameWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + IGraphTransformer::TStatus TimezoneNameWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) { Y_UNUSED(output); if (!EnsureArgsCount(*input, 1, ctx.Expr)) { return IGraphTransformer::TStatus::Error; @@ -11447,7 +11454,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> expectedType = ctx.Expr.MakeType<TOptionalExprType>(expectedType); } - auto convertStatus = TryConvertTo(input->ChildRef(0), *expectedType, ctx.Expr); + auto convertStatus = TryConvertTo(input->ChildRef(0), *expectedType, ctx.Expr, {}, ctx.Types.UseTypeDiffForConvertToError); if (convertStatus.Level == IGraphTransformer::TStatus::Error) { ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Head().Pos()), "Mismatch argument types")); return IGraphTransformer::TStatus::Error; @@ -11461,7 +11468,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> return IGraphTransformer::TStatus::Ok; } - IGraphTransformer::TStatus AddTimezoneWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + IGraphTransformer::TStatus AddTimezoneWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) { Y_UNUSED(output); if (!EnsureArgsCount(*input, 2, ctx.Expr)) { return IGraphTransformer::TStatus::Error; @@ -11487,7 +11494,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> const TTypeAnnotationNode* ui16Type = ctx.Expr.MakeType<TDataExprType>(EDataSlot::Uint16); const TTypeAnnotationNode* optUi16Type = ctx.Expr.MakeType<TOptionalExprType>(ui16Type); const TTypeAnnotationNode* expectedType = isOptional2 ? optUi16Type : ui16Type; - auto convertStatus = TryConvertTo(input->ChildRef(1), *expectedType, ctx.Expr); + auto convertStatus = TryConvertTo(input->ChildRef(1), *expectedType, ctx.Expr, {}, ctx.Types.UseTypeDiffForConvertToError); if (convertStatus.Level == IGraphTransformer::TStatus::Error) { ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), "Mismatch argument types")); return IGraphTransformer::TStatus::Error; @@ -12489,10 +12496,10 @@ template <NKikimr::NUdf::EDataSlot DataSlot> Functions[RightName] = &RightWrapper; Functions[ConsName] = &ConsWrapper; Functions["DataSink"] = &DataSinkWrapper; - Functions["Filter"] = &FilterWrapper; - Functions["OrderedFilter"] = &FilterWrapper; - Functions["TakeWhile"] = &FilterWrapper; - Functions["SkipWhile"] = &FilterWrapper; + ExtFunctions["Filter"] = &FilterWrapper; + ExtFunctions["OrderedFilter"] = &FilterWrapper; + ExtFunctions["TakeWhile"] = &FilterWrapper; + ExtFunctions["SkipWhile"] = &FilterWrapper; Functions["TakeWhileInclusive"] = &InclusiveFilterWrapper<false>; Functions["SkipWhileInclusive"] = &InclusiveFilterWrapper<true>; Functions["Member"] = &MemberWrapper; @@ -12561,8 +12568,8 @@ template <NKikimr::NUdf::EDataSlot DataSlot> Functions["BitAnd"] = &BitOpsWrapper<2>; Functions["BitOr"] = &BitOpsWrapper<2>; Functions["BitXor"] = &BitOpsWrapper<2>; - Functions["Min"] = &MinMaxWrapper; - Functions["Max"] = &MinMaxWrapper; + ExtFunctions["Min"] = &MinMaxWrapper; + ExtFunctions["Max"] = &MinMaxWrapper; Functions["AggrEquals"] = &AggrCompareWrapper<true, false>; Functions["AggrNotEquals"] = &AggrCompareWrapper<false, false>; Functions["AggrLess"] = &AggrCompareWrapper<false, true>; @@ -12574,28 +12581,28 @@ template <NKikimr::NUdf::EDataSlot DataSlot> Functions["IsNotDistinctFrom"] = &DistinctFromWrapper; Functions["IsDistinctFrom"] = &DistinctFromWrapper;; Functions["Abs"] = &AbsWrapper; - Functions["ShiftLeft"] = &ShiftWrapper; - Functions["RotLeft"] = &ShiftWrapper; - Functions["ShiftRight"] = &ShiftWrapper; - Functions["RotRight"] = &ShiftWrapper; + ExtFunctions["ShiftLeft"] = &ShiftWrapper; + ExtFunctions["RotLeft"] = &ShiftWrapper; + ExtFunctions["ShiftRight"] = &ShiftWrapper; + ExtFunctions["RotRight"] = &ShiftWrapper; Functions[SyncName] = &SyncWrapper; Functions["WithWorld"] = &WithWorldWrapper; Functions["Concat"] = &ConcatWrapper; Functions["AggrConcat"] = &AggrConcatWrapper; - Functions["Substring"] = &SubstringWrapper; - Functions["Find"] = &FindWrapper; - Functions["RFind"] = &FindWrapper; + ExtFunctions["Substring"] = &SubstringWrapper; + ExtFunctions["Find"] = &FindWrapper; + ExtFunctions["RFind"] = &FindWrapper; Functions["StartsWith"] = &WithWrapper; Functions["StartsWithIgnoreCase"] = &WithWrapper; Functions["EndsWith"] = &WithWrapper; Functions["EndsWithIgnoreCase"] = &WithWrapper; Functions["StringContains"] = &WithWrapper; Functions["StringContainsIgnoreCase"] = &WithWrapper; - Functions["ByteAt"] = &ByteAtWrapper; + ExtFunctions["ByteAt"] = &ByteAtWrapper; Functions["ListIf"] = &ListIfWrapper; - Functions["AsList"] = &AsListWrapper<false>; - Functions["AsListMayWarn"] = &AsListWrapper<false>; - Functions["AsListStrict"] = &AsListWrapper<true>; + ExtFunctions["AsList"] = &AsListWrapper<false>; + ExtFunctions["AsListMayWarn"] = &AsListWrapper<false>; + ExtFunctions["AsListStrict"] = &AsListWrapper<true>; Functions["ToList"] = &ToListWrapper; Functions["ToOptional"] = &ToOptionalWrapper; Functions["Iterable"] = &IterableWrapper; @@ -12643,14 +12650,14 @@ template <NKikimr::NUdf::EDataSlot DataSlot> Functions["ToSequence"] = &ToSequenceWrapper; Functions["Collect"] = &CollectWrapper; Functions["LazyList"] = &LazyListWrapper; - Functions["ListFromRange"] = &ListFromRangeWrapper; - Functions["Replicate"] = &ReplicateWrapper; + ExtFunctions["ListFromRange"] = &ListFromRangeWrapper; + ExtFunctions["Replicate"] = &ReplicateWrapper; Functions["Switch"] = &SwitchWrapper; Functions["Chopper"] = &ChopperWrapper; Functions["HasItems"] = &HasItemsWrapper; - Functions["Append"] = &AppendWrapper; - Functions["Insert"] = &AppendWrapper; - Functions["Prepend"] = &PrependWrapper; + ExtFunctions["Append"] = &AppendWrapper; + ExtFunctions["Insert"] = &AppendWrapper; + ExtFunctions["Prepend"] = &PrependWrapper; Functions["Extend"] = &ExtendWrapper; Functions["OrderedExtend"] = &ExtendWrapper; Functions["Merge"] = &ExtendWrapper; @@ -12659,8 +12666,8 @@ template <NKikimr::NUdf::EDataSlot DataSlot> Functions["UnionAll"] = &UnionAllWrapper; Functions["UnionMerge"] = &UnionAllWrapper; Functions["Union"] = &UnionAllWrapper; - Functions["ListExtend"] = &ListExtendWrapper<false>; - Functions["ListExtendStrict"] = &ListExtendWrapper<true>; + ExtFunctions["ListExtend"] = &ListExtendWrapper<false>; + ExtFunctions["ListExtendStrict"] = &ListExtendWrapper<true>; Functions["ListUnionAll"] = &ListUnionAllWrapper; Functions["ListZip"] = &ListZipWrapper; Functions["ListZipAll"] = &ListZipAllWrapper; @@ -12676,9 +12683,9 @@ template <NKikimr::NUdf::EDataSlot DataSlot> Functions["AssumeStrict"] = &AssumeStrictWrapper; Functions["AssumeNonStrict"] = &AssumeStrictWrapper; Functions["EnsureStrict"] = &EnsureStrictWrapper; - Functions["Top"] = &TopWrapper; - Functions["TopSort"] = &TopWrapper; - Functions["KeepTop"] = &KeepTopWrapper; + ExtFunctions["Top"] = &TopWrapper; + ExtFunctions["TopSort"] = &TopWrapper; + ExtFunctions["KeepTop"] = &KeepTopWrapper; Functions["Unordered"] = &UnorderedWrapper; Functions["UnorderedSubquery"] = &UnorderedWrapper; Functions["SortTraits"] = &SortTraitsWrapper; @@ -12711,9 +12718,9 @@ template <NKikimr::NUdf::EDataSlot DataSlot> Functions["PartitionByKey"] = &PartitionByKeyWrapper; Functions["PartitionsByKeys"] = &PartitionsByKeysWrapper; Functions["Reverse"] = &ReverseWrapper; - Functions["Skip"] = &TakeWrapper; - Functions["Take"] = &TakeWrapper; - Functions["Limit"] = &TakeWrapper; + ExtFunctions["Skip"] = &TakeWrapperEx; + ExtFunctions["Take"] = &TakeWrapperEx; + ExtFunctions["Limit"] = &TakeWrapperEx; Functions["Fold"] = &FoldWrapper; Functions["Fold1"] = &Fold1Wrapper; Functions["Condense"] = &CondenseWrapper; @@ -12723,7 +12730,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> Functions["Discard"] = &DiscardWrapper; Functions["Zip"] = &ZipWrapper; Functions["ZipAll"] = &ZipAllWrapper; - Functions["Enumerate"] = &EnumerateWrapper; + ExtFunctions["Enumerate"] = &EnumerateWrapper; Functions["GenericType"] = &TypeWrapper<ETypeAnnotationKind::Generic>; Functions["ResourceType"] = &TypeWrapper<ETypeAnnotationKind::Resource>; Functions["ErrorType"] = &TypeWrapper<ETypeAnnotationKind::Error>; @@ -12744,9 +12751,9 @@ template <NKikimr::NUdf::EDataSlot DataSlot> Functions["List"] = &ListWrapper; Functions["DictType"] = &TypeWrapper<ETypeAnnotationKind::Dict>; Functions["Dict"] = &DictWrapper; - Functions["Variant"] = &VariantWrapper; + ExtFunctions["Variant"] = &VariantWrapper; Functions["Enum"] = &EnumWrapper; - Functions["DynamicVariant"] = &DynamicVariantWrapper; + ExtFunctions["DynamicVariant"] = &DynamicVariantWrapper; Functions["AsVariant"] = &AsVariantWrapper; Functions["AsEnum"] = &AsEnumWrapper; Functions["Contains"] = &ContainsLookupWrapper<true>; @@ -12757,17 +12764,17 @@ template <NKikimr::NUdf::EDataSlot DataSlot> Functions["DictPayloads"] = &DictItemsWrapper<EDictItems::Payloads>; Functions["AsStruct"] = &AsStructWrapper; Functions["AsStructUnordered"] = &AsStructWrapper; - Functions["AsDict"] = &AsDictWrapper<false, false>; - Functions["AsDictMayWarn"] = &AsDictWrapper<false, false>; - Functions["AsDictStrict"] = &AsDictWrapper<true, false>; - Functions["AsSet"] = &AsDictWrapper<false, true>; - Functions["AsSetMayWarn"] = &AsDictWrapper<false, true>; - Functions["AsSetStrict"] = &AsDictWrapper<true, true>; - Functions["DictFromKeys"] = &DictFromKeysWrapper; + ExtFunctions["AsDict"] = &AsDictWrapper<false, false>; + ExtFunctions["AsDictMayWarn"] = &AsDictWrapper<false, false>; + ExtFunctions["AsDictStrict"] = &AsDictWrapper<true, false>; + ExtFunctions["AsSet"] = &AsDictWrapper<false, true>; + ExtFunctions["AsSetMayWarn"] = &AsDictWrapper<false, true>; + ExtFunctions["AsSetStrict"] = &AsDictWrapper<true, true>; + ExtFunctions["DictFromKeys"] = &DictFromKeysWrapper; Functions["Uniq"] = &UniqWrapper; Functions["UniqStable"] = &UniqWrapper; - Functions["If"] = &IfWrapper<false>; - Functions["IfStrict"] = &IfWrapper<true>; + ExtFunctions["If"] = &IfWrapper<false>; + ExtFunctions["IfStrict"] = &IfWrapper<true>; Functions[IfName] = &IfWorldWrapper; Functions[ForName] = &ForWorldWrapper; Functions["IfPresent"] = &IfPresentWrapper; @@ -12836,7 +12843,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> Functions["CallableArgumentType"] = &TypeArgWrapper<ETypeArgument::CallableArgument>; Functions["CombineByKey"] = &CombineByKeyWrapper; Functions["FinalizeByKey"] = &CombineByKeyWrapper; - Functions["NewMTRand"] = &NewMTRandWrapper; + ExtFunctions["NewMTRand"] = &NewMTRandWrapper; Functions["NextMTRand"] = &NextMTRandWrapper; Functions["FormatType"] = &FormatTypeWrapper; Functions["FormatTypeDiff"] = &FormatTypeDiffWrapper; @@ -12859,14 +12866,14 @@ template <NKikimr::NUdf::EDataSlot DataSlot> Functions["CalcOverWindow"] = &CalcOverWindowWrapper; Functions["CalcOverSessionWindow"] = &CalcOverWindowWrapper; Functions["CalcOverWindowGroup"] = &CalcOverWindowGroupWrapper; - Functions["Lag"] = &WinLeadLagWrapper; - Functions["Lead"] = &WinLeadLagWrapper; + ExtFunctions["Lag"] = &WinLeadLagWrapper; + ExtFunctions["Lead"] = &WinLeadLagWrapper; Functions["RowNumber"] = &WinRowNumberWrapper; Functions["Rank"] = &WinRankWrapper; Functions["DenseRank"] = &WinRankWrapper; Functions["PercentRank"] = &WinRankWrapper; Functions["CumeDist"] = &WinCumeDistWrapper; - Functions["NTile"] = &WinNTileWrapper; + ExtFunctions["NTile"] = &WinNTileWrapper; Functions["Ascending"] = &PresortWrapper; Functions["Descending"] = &PresortWrapper; Functions["IsKeySwitch"] = &IsKeySwitchWrapper; @@ -12953,56 +12960,56 @@ template <NKikimr::NUdf::EDataSlot DataSlot> Functions["DependsOn"] = &DependsOnWrapper; Functions["Seq"] = &SeqWrapper; Functions["Parameter"] = &ParameterWrapper; - Functions["WeakField"] = &WeakFieldWrapper; - Functions["TryWeakMemberFromDict"] = &TryWeakMemberFromDictWrapper; + ExtFunctions["WeakField"] = &WeakFieldWrapper; + ExtFunctions["TryWeakMemberFromDict"] = &TryWeakMemberFromDictWrapper; Functions["ByteString"] = &ByteStringWrapper; Functions["Utf8String"] = &Utf8StringWrapper; Functions["FromYsonSimpleType"] = &FromYsonSimpleType; Functions["Mux"] = &MuxWrapper; Functions["Demux"] = &DemuxWrapper; - Functions["TimezoneId"] = &TimezoneIdWrapper; - Functions["TimezoneName"] = &TimezoneNameWrapper; - Functions["AddTimezone"] = &AddTimezoneWrapper; + ExtFunctions["TimezoneId"] = &TimezoneIdWrapper; + ExtFunctions["TimezoneName"] = &TimezoneNameWrapper; + ExtFunctions["AddTimezone"] = &AddTimezoneWrapper; Functions["RemoveTimezone"] = &RemoveTimezoneWrapper; Functions["TypeHandle"] = &TypeHandleWrapper; Functions["SerializeTypeHandle"] = &SerializeTypeHandleWrapper; Functions["ParseTypeHandle"] = &ParseTypeHandleWrapper; Functions["TypeKind"] = &TypeKindWrapper; Functions["DataTypeComponents"] = &SplitTypeHandleWrapper<ETypeAnnotationKind::Data>; - Functions["DataTypeHandle"] = &MakeTypeHandleWrapper<ETypeAnnotationKind::Data>; - Functions["OptionalTypeHandle"] = &MakeTypeHandleWrapper<ETypeAnnotationKind::Optional>; - Functions["ListTypeHandle"] = &MakeTypeHandleWrapper<ETypeAnnotationKind::List>; - Functions["StreamTypeHandle"] = &MakeTypeHandleWrapper<ETypeAnnotationKind::Stream>; + ExtFunctions["DataTypeHandle"] = &MakeTypeHandleWrapper<ETypeAnnotationKind::Data>; + ExtFunctions["OptionalTypeHandle"] = &MakeTypeHandleWrapper<ETypeAnnotationKind::Optional>; + ExtFunctions["ListTypeHandle"] = &MakeTypeHandleWrapper<ETypeAnnotationKind::List>; + ExtFunctions["StreamTypeHandle"] = &MakeTypeHandleWrapper<ETypeAnnotationKind::Stream>; Functions["TupleTypeComponents"] = &SplitTypeHandleWrapper<ETypeAnnotationKind::Tuple>; - Functions["TupleTypeHandle"] = &MakeTypeHandleWrapper<ETypeAnnotationKind::Tuple>; + ExtFunctions["TupleTypeHandle"] = &MakeTypeHandleWrapper<ETypeAnnotationKind::Tuple>; Functions["StructTypeComponents"] = &SplitTypeHandleWrapper<ETypeAnnotationKind::Struct>; - Functions["StructTypeHandle"] = &MakeTypeHandleWrapper<ETypeAnnotationKind::Struct>; + ExtFunctions["StructTypeHandle"] = &MakeTypeHandleWrapper<ETypeAnnotationKind::Struct>; Functions["DictTypeComponents"] = &SplitTypeHandleWrapper<ETypeAnnotationKind::Dict>; - Functions["DictTypeHandle"] = &MakeTypeHandleWrapper<ETypeAnnotationKind::Dict>; + ExtFunctions["DictTypeHandle"] = &MakeTypeHandleWrapper<ETypeAnnotationKind::Dict>; Functions["ResourceTypeTag"] = &SplitTypeHandleWrapper<ETypeAnnotationKind::Resource>; - Functions["ResourceTypeHandle"] = &MakeTypeHandleWrapper<ETypeAnnotationKind::Resource>; + ExtFunctions["ResourceTypeHandle"] = &MakeTypeHandleWrapper<ETypeAnnotationKind::Resource>; Functions["TaggedTypeComponents"] = &SplitTypeHandleWrapper<ETypeAnnotationKind::Tagged>; - Functions["TaggedTypeHandle"] = &MakeTypeHandleWrapper<ETypeAnnotationKind::Tagged>; - Functions["VariantTypeHandle"] = &MakeTypeHandleWrapper<ETypeAnnotationKind::Variant>; - Functions["VoidTypeHandle"] = &MakeTypeHandleWrapper<ETypeAnnotationKind::Void>; - Functions["NullTypeHandle"] = &MakeTypeHandleWrapper<ETypeAnnotationKind::Null>; - Functions["EmptyListTypeHandle"] = &MakeTypeHandleWrapper<ETypeAnnotationKind::EmptyList>; - Functions["EmptyDictTypeHandle"] = &MakeTypeHandleWrapper<ETypeAnnotationKind::EmptyDict>; - Functions["CallableTypeComponents"] = &SplitTypeHandleWrapper<ETypeAnnotationKind::Callable>; - Functions["CallableArgument"] = &CallableArgumentWrapper; - Functions["CallableTypeHandle"] = &MakeTypeHandleWrapper<ETypeAnnotationKind::Callable>; - Functions["PgTypeHandle"] = &MakeTypeHandleWrapper<ETypeAnnotationKind::Pg>; + ExtFunctions["TaggedTypeHandle"] = &MakeTypeHandleWrapper<ETypeAnnotationKind::Tagged>; + ExtFunctions["VariantTypeHandle"] = &MakeTypeHandleWrapper<ETypeAnnotationKind::Variant>; + ExtFunctions["VoidTypeHandle"] = &MakeTypeHandleWrapper<ETypeAnnotationKind::Void>; + ExtFunctions["NullTypeHandle"] = &MakeTypeHandleWrapper<ETypeAnnotationKind::Null>; + ExtFunctions["EmptyListTypeHandle"] = &MakeTypeHandleWrapper<ETypeAnnotationKind::EmptyList>; + ExtFunctions["EmptyDictTypeHandle"] = &MakeTypeHandleWrapper<ETypeAnnotationKind::EmptyDict>; + ExtFunctions["CallableTypeComponents"] = &SplitTypeHandleWrapper<ETypeAnnotationKind::Callable>; + ExtFunctions["CallableArgument"] = &CallableArgumentWrapper; + ExtFunctions["CallableTypeHandle"] = &MakeTypeHandleWrapper<ETypeAnnotationKind::Callable>; + ExtFunctions["PgTypeHandle"] = &MakeTypeHandleWrapper<ETypeAnnotationKind::Pg>; Functions["PgTypeName"] = &SplitTypeHandleWrapper<ETypeAnnotationKind::Pg>; Functions["LambdaArgumentsCount"] = LambdaArgumentsCountWrapper; Functions["LambdaOptionalArgumentsCount"] = LambdaOptionalArgumentsCountWrapper; Functions["FormatCode"] = &FormatCodeWrapper; Functions["FormatCodeWithPositions"] = &FormatCodeWrapper; Functions["SerializeCode"] = &FormatCodeWrapper; - Functions["WorldCode"] = &MakeCodeWrapper<TExprNode::World>; - Functions["AtomCode"] = &MakeCodeWrapper<TExprNode::Atom>; - Functions["ListCode"] = &MakeCodeWrapper<TExprNode::List>; - Functions["FuncCode"] = &MakeCodeWrapper<TExprNode::Callable>; - Functions["LambdaCode"] = &MakeCodeWrapper<TExprNode::Lambda>; + ExtFunctions["WorldCode"] = &MakeCodeWrapper<TExprNode::World>; + ExtFunctions["AtomCode"] = &MakeCodeWrapper<TExprNode::Atom>; + ExtFunctions["ListCode"] = &MakeCodeWrapper<TExprNode::List>; + ExtFunctions["FuncCode"] = &MakeCodeWrapper<TExprNode::Callable>; + ExtFunctions["LambdaCode"] = &MakeCodeWrapper<TExprNode::Lambda>; Functions["ReprCode"] = &ReprCodeWrapper; Functions["EvaluateAtom"] = &RestartEvaluationWrapper; Functions["EvaluateExpr"] = &RestartEvaluationWrapper; @@ -13066,7 +13073,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> Functions["ExpandMap"] = &ExpandMapWrapper; Functions["WideMap"] = &WideMapWrapper; - Functions["WideFilter"] = &WideFilterWrapper; + ExtFunctions["WideFilter"] = &WideFilterWrapper; Functions["WideTakeWhile"] = &WideWhileWrapper; Functions["WideSkipWhile"] = &WideWhileWrapper; Functions["WideTakeWhileInclusive"] = &WideWhileWrapper; @@ -13084,11 +13091,11 @@ template <NKikimr::NUdf::EDataSlot DataSlot> Functions["WideFromBlocks"] = &WideFromBlocksWrapper; Functions["ListFromBlocks"] = &ListFromBlocksWrapper; - Functions["WideSkipBlocks"] = &WideSkipTakeBlocksWrapper; - Functions["WideTakeBlocks"] = &WideSkipTakeBlocksWrapper; + ExtFunctions["WideSkipBlocks"] = &WideSkipTakeBlocksWrapper; + ExtFunctions["WideTakeBlocks"] = &WideSkipTakeBlocksWrapper; Functions["BlockCompress"] = &BlockCompressWrapper; - Functions["WideTopBlocks"] = &WideTopBlocksWrapper; - Functions["WideTopSortBlocks"] = &WideTopBlocksWrapper; + ExtFunctions["WideTopBlocks"] = &WideTopBlocksWrapper; + ExtFunctions["WideTopSortBlocks"] = &WideTopBlocksWrapper; Functions["WideSortBlocks"] = &WideSortBlocksWrapper; Functions["BlockExtend"] = &BlockExtendWrapper; Functions["BlockOrderedExtend"] = &BlockExtendWrapper; diff --git a/yql/essentials/core/type_ann/type_ann_list.cpp b/yql/essentials/core/type_ann/type_ann_list.cpp index 508dad05067..a215fdef950 100644 --- a/yql/essentials/core/type_ann/type_ann_list.cpp +++ b/yql/essentials/core/type_ann/type_ann_list.cpp @@ -693,7 +693,7 @@ namespace { } } // namespace - IGraphTransformer::TStatus FilterWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + IGraphTransformer::TStatus FilterWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) { if (!EnsureMinMaxArgsCount(*input, 2U, 3U, ctx.Expr)) { return IGraphTransformer::TStatus::Error; } @@ -710,7 +710,7 @@ namespace { if (input->ChildrenSize() > 2U) { const auto expectedType = ctx.Expr.MakeType<TDataExprType>(EDataSlot::Uint64); - const auto convertStatus = TryConvertTo(input->TailRef(), *expectedType, ctx.Expr); + const auto convertStatus = TryConvertTo(input->TailRef(), *expectedType, ctx.Expr, {}, ctx.Types.UseTypeDiffForConvertToError); if (convertStatus.Level == IGraphTransformer::TStatus::Error) { ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Tail().Pos()), TStringBuilder() << "Mismatch 'limit' type. Expected Uint64, got: " << *input->Tail().GetTypeAnn())); @@ -1963,7 +1963,7 @@ namespace { template IGraphTransformer::TStatus ListAllAnyWrapper<false>(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); - IGraphTransformer::TStatus PrependWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + IGraphTransformer::TStatus PrependWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) { if (!EnsureArgsCount(*input, 2, ctx.Expr)) { return IGraphTransformer::TStatus::Error; } @@ -1978,7 +1978,7 @@ namespace { } auto expectedType = input->Tail().GetTypeAnn()->Cast<TListExprType>()->GetItemType(); - auto convertStatus = TryConvertTo(input->HeadRef(), *expectedType, ctx.Expr); + auto convertStatus = TryConvertTo(input->HeadRef(), *expectedType, ctx.Expr, {}, ctx.Types.UseTypeDiffForConvertToError); if (convertStatus.Level == IGraphTransformer::TStatus::Error) { ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Head().Pos()), "Mismatch type of item being prepended and list")); return IGraphTransformer::TStatus::Error; @@ -1990,7 +1990,7 @@ namespace { return IGraphTransformer::TStatus::Ok; } - IGraphTransformer::TStatus AppendWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + IGraphTransformer::TStatus AppendWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) { if (!EnsureArgsCount(*input, 2, ctx.Expr)) { return IGraphTransformer::TStatus::Error; } @@ -2005,7 +2005,7 @@ namespace { } auto expectedType = input->Head().GetTypeAnn()->Cast<TListExprType>()->GetItemType(); - auto convertStatus = TryConvertTo(input->TailRef(), *expectedType, ctx.Expr); + auto convertStatus = TryConvertTo(input->TailRef(), *expectedType, ctx.Expr, {}, ctx.Types.UseTypeDiffForConvertToError); if (convertStatus.Level == IGraphTransformer::TStatus::Error) { ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Tail().Pos()), "Mismatch type of item being appended and list")); return IGraphTransformer::TStatus::Error; @@ -2255,7 +2255,7 @@ namespace { return IGraphTransformer::TStatus::Ok; } - IGraphTransformer::TStatus ListFromRangeWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + IGraphTransformer::TStatus ListFromRangeWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) { if (!EnsureMinMaxArgsCount(*input, 2U, 3U, ctx.Expr)) { return IGraphTransformer::TStatus::Error; } @@ -2367,7 +2367,9 @@ namespace { if (!commonType) return IGraphTransformer::TStatus::Error; - if (const auto status = TryConvertTo(input->ChildRef(0U), *commonType, ctx.Expr).Combine(TryConvertTo(input->ChildRef(1U), *commonType, ctx.Expr)); status != IGraphTransformer::TStatus::Ok) + if (const auto status = TryConvertTo(input->ChildRef(0U), *commonType, ctx.Expr, {}, ctx.Types.UseTypeDiffForConvertToError) + .Combine(TryConvertTo(input->ChildRef(1U), *commonType, ctx.Expr, {}, ctx.Types.UseTypeDiffForConvertToError)); + status != IGraphTransformer::TStatus::Ok) return status; if (stepIsOpt && ETypeAnnotationKind::Optional != commonType->GetKind()) { @@ -2489,7 +2491,7 @@ namespace { return IGraphTransformer::TStatus::Ok; } - IGraphTransformer::TStatus ReplicateWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + IGraphTransformer::TStatus ReplicateWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) { Y_UNUSED(output); if (!EnsureArgsCount(*input, 2, ctx.Expr)) { return IGraphTransformer::TStatus::Error; @@ -2500,7 +2502,7 @@ namespace { } const TTypeAnnotationNode* expectedType = ctx.Expr.MakeType<TDataExprType>(EDataSlot::Uint64); - auto convertStatus = TryConvertTo(input->TailRef(), *expectedType, ctx.Expr); + auto convertStatus = TryConvertTo(input->TailRef(), *expectedType, ctx.Expr, {}, ctx.Types.UseTypeDiffForConvertToError); if (convertStatus.Level == IGraphTransformer::TStatus::Error) { ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), "Mismatch argument types")); return IGraphTransformer::TStatus::Error; @@ -3038,7 +3040,7 @@ namespace { } template<bool IsStrict> - IGraphTransformer::TStatus ListExtendWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + IGraphTransformer::TStatus ListExtendWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) { if (!input->ChildrenSize()) { output = ctx.Expr.NewCallable(input->Pos(), "EmptyList", {}); return IGraphTransformer::TStatus::Repeat; @@ -3070,7 +3072,7 @@ namespace { if constexpr (!IsStrict) { if (const auto commonType = CommonTypeForChildren(*input, ctx.Expr)) { - if (const auto status = ConvertChildrenToType(input, commonType, ctx.Expr); status != IGraphTransformer::TStatus::Ok) + if (const auto status = ConvertChildrenToType(input, commonType, ctx.Expr, ctx.Types.UseTypeDiffForConvertToError); status != IGraphTransformer::TStatus::Ok) return status; } else return IGraphTransformer::TStatus::Error; @@ -3079,8 +3081,8 @@ namespace { return ListAutomapArgs(input, output, ctx, "OrderedExtend"); } - template IGraphTransformer::TStatus ListExtendWrapper<true>(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); - template IGraphTransformer::TStatus ListExtendWrapper<false>(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); + template IGraphTransformer::TStatus ListExtendWrapper<true>(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx); + template IGraphTransformer::TStatus ListExtendWrapper<false>(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx); IGraphTransformer::TStatus ListUnionAllWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { return ListAutomapArgs(input, output, ctx, "UnionAll"); @@ -3239,7 +3241,7 @@ namespace { return IGraphTransformer::TStatus::Ok; } - IGraphTransformer::TStatus TopWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + IGraphTransformer::TStatus TopWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) { if (!EnsureArgsCount(*input, 4, ctx.Expr)) { return IGraphTransformer::TStatus::Error; } @@ -3255,7 +3257,7 @@ namespace { } const TTypeAnnotationNode* expectedType = ctx.Expr.MakeType<TDataExprType>(EDataSlot::Uint64); - const auto convertStatus = TryConvertTo(input->ChildRef(1), *expectedType, ctx.Expr); + const auto convertStatus = TryConvertTo(input->ChildRef(1), *expectedType, ctx.Expr, {}, ctx.Types.UseTypeDiffForConvertToError); if (convertStatus.Level == IGraphTransformer::TStatus::Error) { ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), "Mismatch argument types")); return IGraphTransformer::TStatus::Error; @@ -3270,7 +3272,7 @@ namespace { return IGraphTransformer::TStatus::Ok; } - IGraphTransformer::TStatus KeepTopWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + IGraphTransformer::TStatus KeepTopWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) { if (!EnsureArgsCount(*input, 5, ctx.Expr)) { return IGraphTransformer::TStatus::Error; } @@ -3285,7 +3287,8 @@ namespace { } const TTypeAnnotationNode* expectedType = ctx.Expr.MakeType<TDataExprType>(EDataSlot::Uint64); - if (const auto convertStatus = TryConvertTo(input->ChildRef(0), *expectedType, ctx.Expr); convertStatus.Level == IGraphTransformer::TStatus::Error) { + if (const auto convertStatus = TryConvertTo(input->ChildRef(0), *expectedType, ctx.Expr, {}, ctx.Types.UseTypeDiffForConvertToError); + convertStatus.Level == IGraphTransformer::TStatus::Error) { ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Head().Pos()), "Mismatch argument types")); return IGraphTransformer::TStatus::Error; } @@ -3776,7 +3779,8 @@ namespace { return IGraphTransformer::TStatus::Ok; } - IGraphTransformer::TStatus TakeWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + IGraphTransformer::TStatus TakeWrapperImpl(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx, + bool useTypeDiffForConvertToError) { if (!EnsureArgsCount(*input, 2, ctx.Expr)) { return IGraphTransformer::TStatus::Error; } @@ -3791,7 +3795,7 @@ namespace { } const TTypeAnnotationNode* expectedType = ctx.Expr.MakeType<TDataExprType>(EDataSlot::Uint64); - auto convertStatus = TryConvertTo(input->ChildRef(1), *expectedType, ctx.Expr); + auto convertStatus = TryConvertTo(input->ChildRef(1), *expectedType, ctx.Expr, {}, useTypeDiffForConvertToError); if (convertStatus.Level == IGraphTransformer::TStatus::Error) { ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), "Mismatch argument types")); return IGraphTransformer::TStatus::Error; @@ -3805,6 +3809,14 @@ namespace { return IGraphTransformer::TStatus::Ok; } + IGraphTransformer::TStatus TakeWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + return TakeWrapperImpl(input, output, ctx, false); + } + + IGraphTransformer::TStatus TakeWrapperEx(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) { + return TakeWrapperImpl(input, output, ctx, ctx.Types.UseTypeDiffForConvertToError); + } + IGraphTransformer::TStatus FoldWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { if (!EnsureArgsCount(*input, 3, ctx.Expr)) { return IGraphTransformer::TStatus::Error; @@ -4298,7 +4310,7 @@ namespace { return IGraphTransformer::TStatus::Ok; } - IGraphTransformer::TStatus EnumerateWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + IGraphTransformer::TStatus EnumerateWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) { if (!EnsureMinArgsCount(*input, 1, ctx.Expr)) { return IGraphTransformer::TStatus::Error; } @@ -4318,7 +4330,7 @@ namespace { const TTypeAnnotationNode* expectedType = ctx.Expr.MakeType<TDataExprType>(EDataSlot::Uint64); if (input->ChildrenSize() > 1) { - auto convertStatus = TryConvertTo(input->ChildRef(1), *expectedType, ctx.Expr); + auto convertStatus = TryConvertTo(input->ChildRef(1), *expectedType, ctx.Expr, {}, ctx.Types.UseTypeDiffForConvertToError); if (convertStatus.Level == IGraphTransformer::TStatus::Error) { ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), "Mismatch argument types")); return IGraphTransformer::TStatus::Error; @@ -4330,7 +4342,7 @@ namespace { } if (input->ChildrenSize() > 2) { - auto convertStatus = TryConvertTo(input->ChildRef(2), *expectedType, ctx.Expr); + auto convertStatus = TryConvertTo(input->ChildRef(2), *expectedType, ctx.Expr, {}, ctx.Types.UseTypeDiffForConvertToError); if (convertStatus.Level == IGraphTransformer::TStatus::Error) { ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(2)->Pos()), "Mismatch argument types")); return IGraphTransformer::TStatus::Error; @@ -6559,7 +6571,7 @@ namespace { return IGraphTransformer::TStatus::Ok; } - IGraphTransformer::TStatus WinLeadLagWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + IGraphTransformer::TStatus WinLeadLagWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) { if (!EnsureMinArgsCount(*input, 2, ctx.Expr) || !EnsureMaxArgsCount(*input, 3, ctx.Expr)) { return IGraphTransformer::TStatus::Error; } @@ -6586,7 +6598,7 @@ namespace { auto status = ConvertToLambda(lambdaValue, ctx.Expr, 1); if (argCount > 2) { - status = status.Combine(TryConvertTo(input->ChildRef(2), *winOffsetType, ctx.Expr)); + status = status.Combine(TryConvertTo(input->ChildRef(2), *winOffsetType, ctx.Expr, {}, ctx.Types.UseTypeDiffForConvertToError)); } if (status.Level != IGraphTransformer::TStatus::Ok) { return status; @@ -6652,7 +6664,7 @@ namespace { return IGraphTransformer::TStatus::Ok; } - IGraphTransformer::TStatus WinNTileWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + IGraphTransformer::TStatus WinNTileWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) { Y_UNUSED(output); if (!EnsureArgsCount(*input, 2, ctx.Expr)) { return IGraphTransformer::TStatus::Error; @@ -6663,7 +6675,7 @@ namespace { } auto expectedType = ctx.Expr.MakeType<TDataExprType>(EDataSlot::Int64); - auto status = TryConvertTo(input->ChildRef(1), *expectedType, ctx.Expr); + auto status = TryConvertTo(input->ChildRef(1), *expectedType, ctx.Expr, {}, ctx.Types.UseTypeDiffForConvertToError); if (status.Level != IGraphTransformer::TStatus::Ok) { return status; } diff --git a/yql/essentials/core/type_ann/type_ann_list.h b/yql/essentials/core/type_ann/type_ann_list.h index abcc3b5ae1b..3e0daa98c90 100644 --- a/yql/essentials/core/type_ann/type_ann_list.h +++ b/yql/essentials/core/type_ann/type_ann_list.h @@ -12,7 +12,7 @@ namespace NTypeAnnImpl { TExprNode::TPtr ExpandToWindowTraits(const TExprNode& input, TExprContext& ctx); bool ValidateAggManyStreams(const TExprNode& value, ui32 aggCount, TExprContext& ctx); - IGraphTransformer::TStatus FilterWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); + IGraphTransformer::TStatus FilterWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx); template <bool InverseCondition> IGraphTransformer::TStatus InclusiveFilterWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus MapWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); @@ -47,8 +47,8 @@ namespace NTypeAnnImpl { IGraphTransformer::TStatus FoldMapWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus Fold1MapWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus Chain1MapWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); - IGraphTransformer::TStatus PrependWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); - IGraphTransformer::TStatus AppendWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); + IGraphTransformer::TStatus PrependWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx); + IGraphTransformer::TStatus AppendWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx); IGraphTransformer::TStatus LengthWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus IteratorWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus EmptyIteratorWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); @@ -57,8 +57,8 @@ namespace NTypeAnnImpl { IGraphTransformer::TStatus ToSequenceWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus LazyListWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus CollectWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); - IGraphTransformer::TStatus ListFromRangeWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); - IGraphTransformer::TStatus ReplicateWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); + IGraphTransformer::TStatus ListFromRangeWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx); + IGraphTransformer::TStatus ReplicateWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx); IGraphTransformer::TStatus SwitchWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus ChopperWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus HasItemsWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); @@ -66,14 +66,14 @@ namespace NTypeAnnImpl { IGraphTransformer::TStatus UnionAllWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus UnionAllPositionalWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx); template <bool IsStrict> - IGraphTransformer::TStatus ListExtendWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); + IGraphTransformer::TStatus ListExtendWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx); IGraphTransformer::TStatus ListUnionAllWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus ListZipWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus ListZipAllWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus PruneKeysWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus SortWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); - IGraphTransformer::TStatus TopWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); - IGraphTransformer::TStatus KeepTopWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); + IGraphTransformer::TStatus TopWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx); + IGraphTransformer::TStatus KeepTopWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx); IGraphTransformer::TStatus UnorderedWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus SortTraitsWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus SessionWindowTraitsWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); @@ -81,6 +81,7 @@ namespace NTypeAnnImpl { IGraphTransformer::TStatus PartitionByKeyWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus PartitionsByKeysWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus ReverseWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); + IGraphTransformer::TStatus TakeWrapperEx(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx); IGraphTransformer::TStatus TakeWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus FoldWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus Fold1Wrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); @@ -91,7 +92,7 @@ namespace NTypeAnnImpl { IGraphTransformer::TStatus DiscardWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus ZipWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus ZipAllWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); - IGraphTransformer::TStatus EnumerateWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); + IGraphTransformer::TStatus EnumerateWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx); IGraphTransformer::TStatus ListWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus CombineByKeyWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus ExtractWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); @@ -117,11 +118,11 @@ namespace NTypeAnnImpl { IGraphTransformer::TStatus ToWindowTraitsWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus CalcOverWindowWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus CalcOverWindowGroupWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); - IGraphTransformer::TStatus WinLeadLagWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); + IGraphTransformer::TStatus WinLeadLagWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx); IGraphTransformer::TStatus WinRowNumberWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus WinCumeDistWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus WinRankWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); - IGraphTransformer::TStatus WinNTileWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); + IGraphTransformer::TStatus WinNTileWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx); IGraphTransformer::TStatus HoppingCoreWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus MultiHoppingCoreWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus HoppingTraitsWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); diff --git a/yql/essentials/core/type_ann/type_ann_pg.cpp b/yql/essentials/core/type_ann/type_ann_pg.cpp index 61e84e700b9..2b5f7625ca1 100644 --- a/yql/essentials/core/type_ann/type_ann_pg.cpp +++ b/yql/essentials/core/type_ann/type_ann_pg.cpp @@ -5187,7 +5187,7 @@ IGraphTransformer::TStatus PgSelectWrapper(const TExprNode::TPtr& input, TExprNo } else { const TTypeAnnotationNode* expectedType = ctx.Expr.MakeType<TOptionalExprType>( ctx.Expr.MakeType<TDataExprType>(EDataSlot::Int64)); - auto convertStatus = TryConvertTo(data, *expectedType, ctx.Expr); + auto convertStatus = TryConvertTo(data, *expectedType, ctx.Expr, {}, ctx.Types.UseTypeDiffForConvertToError); if (convertStatus.Level == IGraphTransformer::TStatus::Error) { ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(data->Pos()), "Mismatch argument types")); return IGraphTransformer::TStatus::Error; diff --git a/yql/essentials/core/type_ann/type_ann_types.cpp b/yql/essentials/core/type_ann/type_ann_types.cpp index aa9e2616bf5..30f93bc3d31 100644 --- a/yql/essentials/core/type_ann/type_ann_types.cpp +++ b/yql/essentials/core/type_ann/type_ann_types.cpp @@ -1270,7 +1270,7 @@ namespace NTypeAnnImpl { } template <> - IGraphTransformer::TStatus MakeTypeHandleWrapper<ETypeAnnotationKind::Data>(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + IGraphTransformer::TStatus MakeTypeHandleWrapper<ETypeAnnotationKind::Data>(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) { Y_UNUSED(output); if (!EnsureArgsCount(*input, 1, ctx.Expr)) { return IGraphTransformer::TStatus::Error; @@ -1290,7 +1290,7 @@ namespace NTypeAnnImpl { } template <> - IGraphTransformer::TStatus MakeTypeHandleWrapper<ETypeAnnotationKind::Optional>(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + IGraphTransformer::TStatus MakeTypeHandleWrapper<ETypeAnnotationKind::Optional>(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) { Y_UNUSED(output); if (!EnsureArgsCount(*input, 1, ctx.Expr)) { return IGraphTransformer::TStatus::Error; @@ -1305,7 +1305,7 @@ namespace NTypeAnnImpl { } template <> - IGraphTransformer::TStatus MakeTypeHandleWrapper<ETypeAnnotationKind::List>(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + IGraphTransformer::TStatus MakeTypeHandleWrapper<ETypeAnnotationKind::List>(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) { Y_UNUSED(output); if (!EnsureArgsCount(*input, 1, ctx.Expr)) { return IGraphTransformer::TStatus::Error; @@ -1320,7 +1320,7 @@ namespace NTypeAnnImpl { } template <> - IGraphTransformer::TStatus MakeTypeHandleWrapper<ETypeAnnotationKind::Stream>(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + IGraphTransformer::TStatus MakeTypeHandleWrapper<ETypeAnnotationKind::Stream>(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) { Y_UNUSED(output); if (!EnsureArgsCount(*input, 1, ctx.Expr)) { return IGraphTransformer::TStatus::Error; @@ -1350,7 +1350,7 @@ namespace NTypeAnnImpl { } template <> - IGraphTransformer::TStatus MakeTypeHandleWrapper<ETypeAnnotationKind::Tuple>(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + IGraphTransformer::TStatus MakeTypeHandleWrapper<ETypeAnnotationKind::Tuple>(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) { Y_UNUSED(output); if (!EnsureArgsCount(*input, 1, ctx.Expr)) { return IGraphTransformer::TStatus::Error; @@ -1358,7 +1358,7 @@ namespace NTypeAnnImpl { auto resType = MakeTypeHandleResourceType(ctx.Expr); auto listType = ctx.Expr.MakeType<TListExprType>(resType); - auto status = TryConvertTo(input->ChildRef(0), *listType, ctx.Expr); + auto status = TryConvertTo(input->ChildRef(0), *listType, ctx.Expr, {}, ctx.Types.UseTypeDiffForConvertToError); if (status != IGraphTransformer::TStatus::Ok) { return status; } @@ -1383,7 +1383,7 @@ namespace NTypeAnnImpl { } template <> - IGraphTransformer::TStatus MakeTypeHandleWrapper<ETypeAnnotationKind::Struct>(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + IGraphTransformer::TStatus MakeTypeHandleWrapper<ETypeAnnotationKind::Struct>(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) { Y_UNUSED(output); if (!EnsureArgsCount(*input, 1, ctx.Expr)) { return IGraphTransformer::TStatus::Error; @@ -1391,7 +1391,7 @@ namespace NTypeAnnImpl { auto itemType = MakeItemDescriptorType(ctx.Expr); auto listType = ctx.Expr.MakeType<TListExprType>(itemType); - auto status = TryConvertTo(input->ChildRef(0), *listType, ctx.Expr); + auto status = TryConvertTo(input->ChildRef(0), *listType, ctx.Expr, {}, ctx.Types.UseTypeDiffForConvertToError); if (status != IGraphTransformer::TStatus::Ok) { return status; } @@ -1416,7 +1416,7 @@ namespace NTypeAnnImpl { } template <> - IGraphTransformer::TStatus MakeTypeHandleWrapper<ETypeAnnotationKind::Dict>(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + IGraphTransformer::TStatus MakeTypeHandleWrapper<ETypeAnnotationKind::Dict>(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) { Y_UNUSED(output); if (!EnsureArgsCount(*input, 2, ctx.Expr)) { return IGraphTransformer::TStatus::Error; @@ -1450,7 +1450,7 @@ namespace NTypeAnnImpl { } template <> - IGraphTransformer::TStatus MakeTypeHandleWrapper<ETypeAnnotationKind::Resource>(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + IGraphTransformer::TStatus MakeTypeHandleWrapper<ETypeAnnotationKind::Resource>(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) { Y_UNUSED(output); if (!EnsureArgsCount(*input, 1, ctx.Expr)) { return IGraphTransformer::TStatus::Error; @@ -1480,7 +1480,7 @@ namespace NTypeAnnImpl { } template <> - IGraphTransformer::TStatus MakeTypeHandleWrapper<ETypeAnnotationKind::Tagged>(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + IGraphTransformer::TStatus MakeTypeHandleWrapper<ETypeAnnotationKind::Tagged>(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) { Y_UNUSED(output); if (!EnsureArgsCount(*input, 2, ctx.Expr)) { return IGraphTransformer::TStatus::Error; @@ -1499,7 +1499,7 @@ namespace NTypeAnnImpl { } template <> - IGraphTransformer::TStatus MakeTypeHandleWrapper<ETypeAnnotationKind::Variant>(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + IGraphTransformer::TStatus MakeTypeHandleWrapper<ETypeAnnotationKind::Variant>(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) { Y_UNUSED(output); if (!EnsureArgsCount(*input, 1, ctx.Expr)) { return IGraphTransformer::TStatus::Error; @@ -1514,7 +1514,7 @@ namespace NTypeAnnImpl { } template <> - IGraphTransformer::TStatus MakeTypeHandleWrapper<ETypeAnnotationKind::Void>(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + IGraphTransformer::TStatus MakeTypeHandleWrapper<ETypeAnnotationKind::Void>(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) { Y_UNUSED(output); if (!EnsureArgsCount(*input, 0, ctx.Expr)) { return IGraphTransformer::TStatus::Error; @@ -1525,7 +1525,7 @@ namespace NTypeAnnImpl { } template <> - IGraphTransformer::TStatus MakeTypeHandleWrapper<ETypeAnnotationKind::Null>(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + IGraphTransformer::TStatus MakeTypeHandleWrapper<ETypeAnnotationKind::Null>(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) { Y_UNUSED(output); if (!EnsureArgsCount(*input, 0, ctx.Expr)) { return IGraphTransformer::TStatus::Error; @@ -1536,7 +1536,7 @@ namespace NTypeAnnImpl { } template <> - IGraphTransformer::TStatus MakeTypeHandleWrapper<ETypeAnnotationKind::EmptyList>(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + IGraphTransformer::TStatus MakeTypeHandleWrapper<ETypeAnnotationKind::EmptyList>(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) { Y_UNUSED(output); if (!EnsureArgsCount(*input, 0, ctx.Expr)) { return IGraphTransformer::TStatus::Error; @@ -1547,7 +1547,7 @@ namespace NTypeAnnImpl { } template <> - IGraphTransformer::TStatus MakeTypeHandleWrapper<ETypeAnnotationKind::EmptyDict>(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + IGraphTransformer::TStatus MakeTypeHandleWrapper<ETypeAnnotationKind::EmptyDict>(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) { Y_UNUSED(output); if (!EnsureArgsCount(*input, 0, ctx.Expr)) { return IGraphTransformer::TStatus::Error; @@ -1558,7 +1558,7 @@ namespace NTypeAnnImpl { } template <> - IGraphTransformer::TStatus MakeTypeHandleWrapper<ETypeAnnotationKind::Pg>(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + IGraphTransformer::TStatus MakeTypeHandleWrapper<ETypeAnnotationKind::Pg>(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) { Y_UNUSED(output); if (!EnsureArgsCount(*input, 1, ctx.Expr)) { return IGraphTransformer::TStatus::Error; @@ -1587,7 +1587,7 @@ namespace NTypeAnnImpl { return IGraphTransformer::TStatus::Ok; } - IGraphTransformer::TStatus CallableArgumentWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + IGraphTransformer::TStatus CallableArgumentWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) { if (!EnsureMinArgsCount(*input, 1, ctx.Expr)) { return IGraphTransformer::TStatus::Error; } @@ -1602,7 +1602,7 @@ namespace NTypeAnnImpl { if (input->ChildrenSize() > 1) { auto optStrType = ctx.Expr.MakeType<TOptionalExprType>(ctx.Expr.MakeType<TDataExprType>(EDataSlot::String)); - auto status = TryConvertTo(input->ChildRef(1), *optStrType, ctx.Expr); + auto status = TryConvertTo(input->ChildRef(1), *optStrType, ctx.Expr, {}, ctx.Types.UseTypeDiffForConvertToError); if (status == IGraphTransformer::TStatus::Error) { return status; } @@ -1610,7 +1610,7 @@ namespace NTypeAnnImpl { if (input->ChildrenSize() > 2) { auto optListStrType = ctx.Expr.MakeType<TOptionalExprType>(ctx.Expr.MakeType<TListExprType>(ctx.Expr.MakeType<TDataExprType>(EDataSlot::String))); - auto status = TryConvertTo(input->ChildRef(2), *optListStrType, ctx.Expr); + auto status = TryConvertTo(input->ChildRef(2), *optListStrType, ctx.Expr, {}, ctx.Types.UseTypeDiffForConvertToError); if (status == IGraphTransformer::TStatus::Error) { return status; } @@ -1666,7 +1666,7 @@ namespace NTypeAnnImpl { } template <> - IGraphTransformer::TStatus MakeTypeHandleWrapper<ETypeAnnotationKind::Callable>(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + IGraphTransformer::TStatus MakeTypeHandleWrapper<ETypeAnnotationKind::Callable>(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) { Y_UNUSED(output); if (!EnsureMinArgsCount(*input, 2, ctx.Expr)) { return IGraphTransformer::TStatus::Error; @@ -1681,14 +1681,14 @@ namespace NTypeAnnImpl { } auto listType = ctx.Expr.MakeType<TListExprType>(MakeArgumentDescriptorType(ctx.Expr)); - auto status = TryConvertTo(input->ChildRef(1), *listType, ctx.Expr); + auto status = TryConvertTo(input->ChildRef(1), *listType, ctx.Expr, {}, ctx.Types.UseTypeDiffForConvertToError); if (status != IGraphTransformer::TStatus::Ok) { return status; } if (input->ChildrenSize() > 2) { auto optUi32Type = ctx.Expr.MakeType<TOptionalExprType>(ctx.Expr.MakeType<TDataExprType>(EDataSlot::Uint32)); - status = TryConvertTo(input->ChildRef(2), *optUi32Type, ctx.Expr); + status = TryConvertTo(input->ChildRef(2), *optUi32Type, ctx.Expr, {}, ctx.Types.UseTypeDiffForConvertToError); if (status != IGraphTransformer::TStatus::Ok) { return status; } @@ -1696,7 +1696,7 @@ namespace NTypeAnnImpl { if (input->ChildrenSize() > 3) { auto optStrType = ctx.Expr.MakeType<TOptionalExprType>(ctx.Expr.MakeType<TDataExprType>(EDataSlot::String)); - status = TryConvertTo(input->ChildRef(3), *optStrType, ctx.Expr); + status = TryConvertTo(input->ChildRef(3), *optStrType, ctx.Expr, {}, ctx.Types.UseTypeDiffForConvertToError); if (status != IGraphTransformer::TStatus::Ok) { return status; } @@ -1756,7 +1756,7 @@ namespace NTypeAnnImpl { } template <> - IGraphTransformer::TStatus MakeCodeWrapper<TExprNode::World>(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + IGraphTransformer::TStatus MakeCodeWrapper<TExprNode::World>(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) { Y_UNUSED(output); if (!EnsureArgsCount(*input, 0, ctx.Expr)) { return IGraphTransformer::TStatus::Error; @@ -1767,14 +1767,14 @@ namespace NTypeAnnImpl { } template <> - IGraphTransformer::TStatus MakeCodeWrapper<TExprNode::Atom>(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + IGraphTransformer::TStatus MakeCodeWrapper<TExprNode::Atom>(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) { Y_UNUSED(output); if (!EnsureArgsCount(*input, 1, ctx.Expr)) { return IGraphTransformer::TStatus::Error; } auto expectedType = ctx.Expr.MakeType<TDataExprType>(EDataSlot::String); - auto status = TryConvertTo(input->ChildRef(0), *expectedType, ctx.Expr); + auto status = TryConvertTo(input->ChildRef(0), *expectedType, ctx.Expr, {}, ctx.Types.UseTypeDiffForConvertToError); if (status != IGraphTransformer::TStatus::Ok) { return status; } @@ -1784,7 +1784,7 @@ namespace NTypeAnnImpl { } template <> - IGraphTransformer::TStatus MakeCodeWrapper<TExprNode::List>(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + IGraphTransformer::TStatus MakeCodeWrapper<TExprNode::List>(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) { Y_UNUSED(output); IGraphTransformer::TStatus status = IGraphTransformer::TStatus::Ok; for (ui32 i = 0; i < input->ChildrenSize(); ++i) { @@ -1801,14 +1801,14 @@ namespace NTypeAnnImpl { } template <> - IGraphTransformer::TStatus MakeCodeWrapper<TExprNode::Callable>(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + IGraphTransformer::TStatus MakeCodeWrapper<TExprNode::Callable>(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) { Y_UNUSED(output); if (!EnsureMinArgsCount(*input, 1, ctx.Expr)) { return IGraphTransformer::TStatus::Error; } auto expectedType = ctx.Expr.MakeType<TDataExprType>(EDataSlot::String); - auto status = TryConvertTo(input->ChildRef(0), *expectedType, ctx.Expr); + auto status = TryConvertTo(input->ChildRef(0), *expectedType, ctx.Expr, {}, ctx.Types.UseTypeDiffForConvertToError); if (status != IGraphTransformer::TStatus::Ok) { return status; } @@ -1827,7 +1827,7 @@ namespace NTypeAnnImpl { } template <> - IGraphTransformer::TStatus MakeCodeWrapper<TExprNode::Lambda>(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + IGraphTransformer::TStatus MakeCodeWrapper<TExprNode::Lambda>(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) { Y_UNUSED(output); if (!EnsureMinArgsCount(*input, 1, ctx.Expr)) { return IGraphTransformer::TStatus::Error; @@ -1844,7 +1844,7 @@ namespace NTypeAnnImpl { } } else { auto ui32Type = ctx.Expr.MakeType<TDataExprType>(EDataSlot::Uint32); - auto status = TryConvertTo(input->ChildRef(0), *ui32Type, ctx.Expr); + auto status = TryConvertTo(input->ChildRef(0), *ui32Type, ctx.Expr, {}, ctx.Types.UseTypeDiffForConvertToError); if (status.Level != IGraphTransformer::TStatus::Ok) { return status; } diff --git a/yql/essentials/core/type_ann/type_ann_types.h b/yql/essentials/core/type_ann/type_ann_types.h index ed47a249790..1223b0877e7 100644 --- a/yql/essentials/core/type_ann/type_ann_types.h +++ b/yql/essentials/core/type_ann/type_ann_types.h @@ -40,13 +40,13 @@ namespace NTypeAnnImpl { IGraphTransformer::TStatus SerializeTypeHandleWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus ParseTypeHandleWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus TypeKindWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); - IGraphTransformer::TStatus CallableArgumentWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); + IGraphTransformer::TStatus CallableArgumentWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx); template <ETypeAnnotationKind> IGraphTransformer::TStatus SplitTypeHandleWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); template <ETypeAnnotationKind> - IGraphTransformer::TStatus MakeTypeHandleWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); + IGraphTransformer::TStatus MakeTypeHandleWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx); IGraphTransformer::TStatus FormatCodeWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus ReprCodeWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); @@ -54,6 +54,6 @@ namespace NTypeAnnImpl { IGraphTransformer::TStatus EvaluateExprIfPureWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); template <TExprNode::EType> - IGraphTransformer::TStatus MakeCodeWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); + IGraphTransformer::TStatus MakeCodeWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx); } // namespace NTypeAnnImpl } // namespace NYql diff --git a/yql/essentials/core/type_ann/type_ann_wide.cpp b/yql/essentials/core/type_ann/type_ann_wide.cpp index 85cc9fba309..4905e85aeef 100644 --- a/yql/essentials/core/type_ann/type_ann_wide.cpp +++ b/yql/essentials/core/type_ann/type_ann_wide.cpp @@ -143,14 +143,14 @@ IGraphTransformer::TStatus WideChain1MapWrapper(const TExprNode::TPtr& input, TE return IGraphTransformer::TStatus::Ok; } -IGraphTransformer::TStatus WideFilterWrapper(const TExprNode::TPtr& input, TExprNode::TPtr&, TContext& ctx) { +IGraphTransformer::TStatus WideFilterWrapper(const TExprNode::TPtr& input, TExprNode::TPtr&, TExtContext& ctx) { if (!EnsureMinMaxArgsCount(*input, 2U, 3U, ctx.Expr)) { return IGraphTransformer::TStatus::Error; } if (input->ChildrenSize() > 2U) { const auto expectedType = ctx.Expr.MakeType<TDataExprType>(EDataSlot::Uint64); - const auto convertStatus = TryConvertTo(input->TailRef(), *expectedType, ctx.Expr); + const auto convertStatus = TryConvertTo(input->TailRef(), *expectedType, ctx.Expr, {}, ctx.Types.UseTypeDiffForConvertToError); if (convertStatus.Level == IGraphTransformer::TStatus::Error) { ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Tail().Pos()), TStringBuilder() << "Mismatch 'limit' type. Expected Uint64, got: " << *input->Tail().GetTypeAnn())); diff --git a/yql/essentials/core/type_ann/type_ann_wide.h b/yql/essentials/core/type_ann/type_ann_wide.h index e5ddb9ff4a4..7b8054b1ffa 100644 --- a/yql/essentials/core/type_ann/type_ann_wide.h +++ b/yql/essentials/core/type_ann/type_ann_wide.h @@ -11,7 +11,7 @@ namespace NTypeAnnImpl { bool ValidateWideTopKeys(TExprNode& keys, const TTypeAnnotationNode::TListType& types, TExprContext& ctx); IGraphTransformer::TStatus WideMapWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); - IGraphTransformer::TStatus WideFilterWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); + IGraphTransformer::TStatus WideFilterWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx); IGraphTransformer::TStatus WideWhileWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus WideCombinerWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus WideCondense1Wrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); diff --git a/yql/essentials/core/yql_expr_type_annotation.cpp b/yql/essentials/core/yql_expr_type_annotation.cpp index 16e29a60c8e..b3a905d72aa 100644 --- a/yql/essentials/core/yql_expr_type_annotation.cpp +++ b/yql/essentials/core/yql_expr_type_annotation.cpp @@ -4246,7 +4246,7 @@ const TTypeAnnotationNode* MakeSequenceType(ETypeAnnotationKind sequenceKind, co } IGraphTransformer::TStatus TryConvertTo(TExprNode::TPtr& node, const TTypeAnnotationNode& expectedType, - TExprContext& ctx, TConvertFlags flags) { + TExprContext& ctx, TConvertFlags flags, bool useTypeDiff) { if (HasError(node->GetTypeAnn(), ctx)) { return IGraphTransformer::TStatus::Error; } @@ -4270,18 +4270,23 @@ IGraphTransformer::TStatus TryConvertTo(TExprNode::TPtr& node, const TTypeAnnota return IGraphTransformer::TStatus::Error; } - return TryConvertTo(node, *node->GetTypeAnn(), expectedType, ctx, flags); + return TryConvertTo(node, *node->GetTypeAnn(), expectedType, ctx, flags, useTypeDiff); } IGraphTransformer::TStatus TryConvertTo(TExprNode::TPtr& node, const TTypeAnnotationNode& sourceType, - const TTypeAnnotationNode& expectedType, TExprContext& ctx, TConvertFlags flags) { + const TTypeAnnotationNode& expectedType, TExprContext& ctx, TConvertFlags flags, bool useTypeDiff) { if (HasError(node->GetTypeAnn(), ctx)) { return IGraphTransformer::TStatus::Error; } TIssueScopeGuard guard(ctx.IssueManager, [&] { - return MakeIntrusive<TIssue>(ctx.GetPosition(node->Pos()), - TStringBuilder() << "Failed to convert type: " << sourceType << " to " << expectedType); + if (useTypeDiff) { + return MakeIntrusive<TIssue>(ctx.GetPosition(node->Pos()), + TStringBuilder() << "Failed to convert, type diff: " << GetTypeDiff(sourceType, expectedType)); + } else { + return MakeIntrusive<TIssue>(ctx.GetPosition(node->Pos()), + TStringBuilder() << "Failed to convert type: " << sourceType << " to " << expectedType); + } }); auto status = TryConvertToImpl(ctx, node, sourceType, expectedType, flags, /* raiseIssues */ true); if (status.Level == IGraphTransformer::TStatus::Error) { @@ -5008,7 +5013,7 @@ IGraphTransformer::TStatus SilentInferCommonType(TExprNode::TPtr& node1, const T return IGraphTransformer::TStatus::Error; } -IGraphTransformer::TStatus ConvertChildrenToType(const TExprNode::TPtr& input, const TTypeAnnotationNode* targetType, TExprContext& ctx) { +IGraphTransformer::TStatus ConvertChildrenToType(const TExprNode::TPtr& input, const TTypeAnnotationNode* targetType, TExprContext& ctx, bool useTypeDiff) { if (!input->ChildrenSize()) { return IGraphTransformer::TStatus::Ok; } @@ -5020,7 +5025,7 @@ IGraphTransformer::TStatus ConvertChildrenToType(const TExprNode::TPtr& input, c return IGraphTransformer::TStatus::Error; } - status = status.Combine(TryConvertTo(input->ChildRef(i), *targetType, ctx)); + status = status.Combine(TryConvertTo(input->ChildRef(i), *targetType, ctx, {}, useTypeDiff)); if (status == IGraphTransformer::TStatus::Error) break; } diff --git a/yql/essentials/core/yql_expr_type_annotation.h b/yql/essentials/core/yql_expr_type_annotation.h index 9bad092f76c..456d57a7b8e 100644 --- a/yql/essentials/core/yql_expr_type_annotation.h +++ b/yql/essentials/core/yql_expr_type_annotation.h @@ -244,9 +244,9 @@ using TConvertFlags = TEnumBitSet<EFlags, DisableTruncation, Last>; using NConvertFlags::TConvertFlags; IGraphTransformer::TStatus TryConvertTo(TExprNode::TPtr& node, const TTypeAnnotationNode& sourceType, - const TTypeAnnotationNode& expectedType, TExprContext& ctx, TConvertFlags flags = {}); + const TTypeAnnotationNode& expectedType, TExprContext& ctx, TConvertFlags flags = {}, bool useTypeDiff = false); IGraphTransformer::TStatus TryConvertTo(TExprNode::TPtr& node, const TTypeAnnotationNode& expectedType, - TExprContext& ctx, TConvertFlags flags = {}); + TExprContext& ctx, TConvertFlags flags = {}, bool useTypeDiff = false); IGraphTransformer::TStatus TrySilentConvertTo(TExprNode::TPtr& node, const TTypeAnnotationNode& expectedType, TExprContext& ctx, TConvertFlags flags = {}); IGraphTransformer::TStatus TrySilentConvertTo(TExprNode::TPtr& node, const TTypeAnnotationNode& sourceType, @@ -257,7 +257,8 @@ IGraphTransformer::TStatus SilentInferCommonType(TExprNode::TPtr& node1, TExprNo IGraphTransformer::TStatus SilentInferCommonType(TExprNode::TPtr& node1, const TTypeAnnotationNode& type1, TExprNode::TPtr& node2, const TTypeAnnotationNode& type2, TExprContext& ctx, const TTypeAnnotationNode*& commonType, TConvertFlags flags = {}); -IGraphTransformer::TStatus ConvertChildrenToType(const TExprNode::TPtr& input,const TTypeAnnotationNode* targetType, TExprContext& ctx); +IGraphTransformer::TStatus ConvertChildrenToType(const TExprNode::TPtr& input,const TTypeAnnotationNode* targetType, TExprContext& ctx, + bool useTypeDiff = false); bool IsSqlInCollectionItemsNullable(const NNodes::TCoSqlIn& node); diff --git a/yql/essentials/core/yql_type_annotation.h b/yql/essentials/core/yql_type_annotation.h index 07908f3bcb5..64ad1207cad 100644 --- a/yql/essentials/core/yql_type_annotation.h +++ b/yql/essentials/core/yql_type_annotation.h @@ -387,6 +387,7 @@ struct TTypeAnnotationContext: public TThrRefBase { TSimpleSharedPtr<NDq::TOrderingsStateMachine> OrderingsFSM; TLangVersion LangVer = MinLangVersion; EBackportCompatibleFeaturesMode BackportMode = EBackportCompatibleFeaturesMode::None; + bool UseTypeDiffForConvertToError = false; THashMap<TString, TIntrusivePtr<TOptimizerStatistics::TColumnStatMap>> ColumnStatisticsByTableName; THashMap<ui64, std::shared_ptr<TOptimizerStatistics>> StatisticsMap; TIntrusivePtr<ITimeProvider> TimeProvider; diff --git a/yql/essentials/docs/en/udf/list/string.md b/yql/essentials/docs/en/udf/list/string.md index 6b8e000c765..3d77857b2e0 100644 --- a/yql/essentials/docs/en/udf/list/string.md +++ b/yql/essentials/docs/en/udf/list/string.md @@ -42,7 +42,7 @@ Functions for ASCII strings: * `String::ReverseFind(String{Flags:AutoMap}, String, [Uint64?]) -> Int64`: Returns the last position found or -1. The optional argument is the offset from the beginning of the string. -* `String::AsciiStartsWithIgnoreCase(String?, String) -> Bool` Added in the version [2025.02](../../changelog/2025.02.md#string-module) +* `String::AsciiStartsWithIgnoreCase(String?, String) -> Bool` * `String::HasPrefix(String?, String) -> Bool` @@ -52,7 +52,7 @@ Functions for ASCII strings: * `String::StartsWithIgnoreCase(String?, String) -> Bool` Removed in the version [2025.02](../../changelog/2025.02.md#string-module) -* `String::AsciiEndsWithIgnoreCase(String?, String) -> Bool` Added in the version [2025.02](../../changelog/2025.02.md#string-module) +* `String::AsciiEndsWithIgnoreCase(String?, String) -> Bool` * `String::HasSuffix(String?, String) -> Bool` diff --git a/yql/essentials/docs/ru/udf/list/string.md b/yql/essentials/docs/ru/udf/list/string.md index c2bf6ec544c..baaf7ef84f1 100644 --- a/yql/essentials/docs/ru/udf/list/string.md +++ b/yql/essentials/docs/ru/udf/list/string.md @@ -62,8 +62,8 @@ SELECT String::Strip("YQL "); -- "YQL" Устаревшие функции, к использованию не рекомендуются. -* `String::AsciiStartsWithIgnoreCase(string:String?, prefix:String) -> Bool` - добавлена в версии [2025.02](../../changelog/2025.02.md#string-module) -* `String::AsciiEndsWithIgnoreCase(string:String?, suffix:String) -> Bool` - добавлена в версии [2025.02](../../changelog/2025.02.md#string-module) +* `String::AsciiStartsWithIgnoreCase(string:String?, prefix:String) -> Bool` +* `String::AsciiEndsWithIgnoreCase(string:String?, suffix:String) -> Bool` * `String::HasPrefixIgnoreCase(string:String?, prefix:String) -> Bool` - удалена в версии [2025.02](../../changelog/2025.02.md#string-module) * `String::StartsWithIgnoreCase(string:String?, prefix:String) -> Bool` - удалена в версии [2025.02](../../changelog/2025.02.md#string-module) * `String::HasSuffixIgnoreCase(string:String?, suffix:String) -> Bool` - удалена в версии [2025.02](../../changelog/2025.02.md#string-module) diff --git a/yql/essentials/minikql/comp_nodes/mkql_block_coalesce.cpp b/yql/essentials/minikql/comp_nodes/mkql_block_coalesce.cpp index 3d7384147f8..6185b7688b1 100644 --- a/yql/essentials/minikql/comp_nodes/mkql_block_coalesce.cpp +++ b/yql/essentials/minikql/comp_nodes/mkql_block_coalesce.cpp @@ -6,6 +6,7 @@ #include <yql/essentials/minikql/computation/mkql_computation_node_holders.h> #include <yql/essentials/minikql/mkql_node_builder.h> #include <yql/essentials/minikql/mkql_node_cast.h> +#include <yql/essentials/minikql/mkql_type_helper.h> #include <yql/essentials/public/udf/arrow/block_builder.h> #include <yql/essentials/public/udf/arrow/block_reader.h> #include <yql/essentials/public/udf/arrow/util.h> @@ -118,18 +119,37 @@ bool DispatchBlendingCoalesce(const arrow::Datum& left, const arrow::Datum& righ } } +std::shared_ptr<arrow::Scalar> UnwrapScalar(std::shared_ptr<arrow::Scalar> scalar, bool firstScalarIsExternalOptional) { + if (firstScalarIsExternalOptional) { + return dynamic_cast<arrow::StructScalar&>(*scalar).value.at(0); + } + return scalar; +} + class TCoalesceBlockExec { public: TCoalesceBlockExec(const std::shared_ptr<arrow::DataType>& returnArrowType, TType* firstItemType, TType* secondItemType, bool needUnwrapFirst) : ReturnArrowType_(returnArrowType) , FirstItemType_(firstItemType) , SecondItemType_(secondItemType) - , NeedUnwrapFirst_(needUnwrapFirst) { + , NeedUnwrapFirst_(needUnwrapFirst) + , FirstScalarIsExternalOptional_(NeedWrapWithExternalOptional(FirstItemType_)) + { } arrow::Status Exec(arrow::compute::KernelContext* ctx, const arrow::compute::ExecBatch& batch, arrow::Datum* res) const { const auto& first = batch.values[0]; const auto& second = batch.values[1]; + + if (first.is_scalar() && second.is_scalar()) { + if (first.scalar()->is_valid) { + *res = NeedUnwrapFirst_ ? UnwrapScalar(first.scalar(), FirstScalarIsExternalOptional_) : first.scalar(); + } else { + *res = second.scalar(); + } + return arrow::Status::OK(); + } + MKQL_ENSURE(!first.is_scalar() || !second.is_scalar(), "Expected at least one array"); size_t length = Max(first.length(), second.length()); auto firstReader = NYql::NUdf::MakeBlockReader(TTypeInfoHelper(), FirstItemType_); @@ -137,7 +157,7 @@ public: if (first.is_scalar()) { auto firstValue = firstReader->GetScalarItem(*first.scalar()); if (firstValue) { - auto builder = NYql::NUdf::MakeArrayBuilder(TTypeInfoHelper(), SecondItemType_, *ctx->memory_pool(), length, nullptr); + auto builder = NYql::NUdf::MakeArrayBuilder(TTypeInfoHelper(), SecondItemType_, *ctx->memory_pool(), length, /*pgBuilder=*/nullptr); builder->Add(NeedUnwrapFirst_ ? firstValue.GetOptionalValue() : firstValue, length); *res = builder->Build(true); } else { @@ -205,6 +225,7 @@ private: TType* const FirstItemType_; TType* const SecondItemType_; const bool NeedUnwrapFirst_; + const bool FirstScalarIsExternalOptional_; }; std::shared_ptr<arrow::compute::ScalarKernel> MakeBlockCoalesceKernel(const TVector<TType*>& argTypes, TType* resultType, bool needUnwrapFirst) { diff --git a/yql/essentials/minikql/comp_nodes/mkql_block_exists.cpp b/yql/essentials/minikql/comp_nodes/mkql_block_exists.cpp index 7df219329e1..f49d3f6b56e 100644 --- a/yql/essentials/minikql/comp_nodes/mkql_block_exists.cpp +++ b/yql/essentials/minikql/comp_nodes/mkql_block_exists.cpp @@ -12,6 +12,10 @@ class TBlockExistsExec { public: arrow::Status Exec(arrow::compute::KernelContext* ctx, const arrow::compute::ExecBatch& batch, arrow::Datum* res) const { const auto& input = batch.values[0]; + if (input.is_scalar()) { + *res = arrow::Datum(static_cast<ui8>(input.scalar()->is_valid)); + return arrow::Status::OK(); + } MKQL_ENSURE(input.is_array(), "Expected array"); const auto& arr = *input.array(); diff --git a/yql/essentials/minikql/comp_nodes/ut/mkql_block_coalesce_ut.cpp b/yql/essentials/minikql/comp_nodes/ut/mkql_block_coalesce_ut.cpp index 4f00f15633c..57ad9137cdf 100644 --- a/yql/essentials/minikql/comp_nodes/ut/mkql_block_coalesce_ut.cpp +++ b/yql/essentials/minikql/comp_nodes/ut/mkql_block_coalesce_ut.cpp @@ -1,16 +1,17 @@ +#include <yql/essentials/ast/yql_expr.h> #include <yql/essentials/minikql/comp_nodes/mkql_block_coalesce.h> -#include <yql/essentials/core/arrow_kernels/request/request.h> #include <yql/essentials/minikql/comp_nodes/mkql_block_coalesce_blending_helper.h> #include <yql/essentials/minikql/comp_nodes/ut/mkql_computation_node_ut.h> +#include <yql/essentials/minikql/computation/mkql_block_impl.h> #include <yql/essentials/minikql/computation/mkql_computation_node_holders.h> #include <yql/essentials/minikql/computation/mkql_block_builder.h> #include <yql/essentials/ast/yql_expr_builder.h> #include <yql/essentials/public/udf/arrow/memory_pool.h> +#include <yql/essentials/minikql/computation/mkql_block_impl.h> #include <yql/essentials/minikql/mkql_node_cast.h> #include <yql/essentials/minikql/arrow/arrow_util.h> -#include <yql/essentials/core/arrow_kernels/request/request.h> -#include <yql/essentials/core/arrow_kernels/registry/registry.h> +#include <yql/essentials/minikql/comp_nodes/ut/mkql_block_helper.h> #include <arrow/compute/exec_internal.h> @@ -105,6 +106,15 @@ arrow::Datum GenerateArray(TTypeInfoHelper& typeInfoHelper, TType* type, std::ve return resultArray; } +template <typename T, typename U, typename V> +void TestScalarCoalesceKernel(T left, U right, V expected) { + TSetup<false> setup; + TestScalarKernel(left, right, expected, setup, + [&](TRuntimeNode left, TRuntimeNode right) { + return setup.PgmBuilder->BlockCoalesce(left, right); + }); +} + enum class ERightOperandType { SCALAR, ARRAY, @@ -116,7 +126,20 @@ template <typename T> using InputOptionalVector = std::vector<TMaybe<typename NUdf::TDataType<T>::TLayout>>; -template <typename T, ERightOperandType rightType = ERightOperandType::ARRAY> +std::unique_ptr<IArrowKernelComputationNode> GetArrowKernel(IComputationGraph* graph) { + std::vector<std::unique_ptr<IArrowKernelComputationNode>> allKernels; + for (auto node : graph->GetNodes()) { + auto kernelNode = node->PrepareArrowKernelComputationNode(graph->GetContext()); + if (!kernelNode) { + continue; + } + allKernels.push_back(std::move(kernelNode)); + } + UNIT_ASSERT_EQUAL(allKernels.size(), 1u); + return std::move(allKernels[0]); +} + +template <typename T, ERightOperandType rightTypeShape = ERightOperandType::ARRAY> void TestBlockCoalesceForVector(InputOptionalVector<T> left, InputOptionalVector<T> right, InputOptionalVector<T> expected, @@ -125,29 +148,22 @@ void TestBlockCoalesceForVector(InputOptionalVector<T> left, using TLayout = typename NUdf::TDataType<T>::TLayout; TSetup<false> setup; NYql::TExprContext exprCtx; - auto* type = setup.PgmBuilder->NewDataType(NUdf::TDataType<T>::Id); - auto* typeNode = exprCtx.template MakeType<NYql::TBlockExprType>( - exprCtx.template MakeType<NYql::TDataExprType>(NUdf::TDataType<T>::Slot)); - - auto* optType = setup.PgmBuilder->NewOptionalType(type); - auto* optTypeNode = exprCtx.template MakeType<NYql::TBlockExprType>( - exprCtx.template MakeType<NYql::TOptionalExprType>( - exprCtx.template MakeType<NYql::TDataExprType>(NUdf::TDataType<T>::Slot))); - if (rightType == ERightOperandType::OPTIONAL_ARRAY || rightType == ERightOperandType::OPTIONAL_SCALAR) { + auto* rightType = setup.PgmBuilder->NewDataType(NUdf::TDataType<T>::Id); + auto* leftType = setup.PgmBuilder->NewOptionalType(rightType); + if (rightTypeShape == ERightOperandType::OPTIONAL_ARRAY || rightTypeShape == ERightOperandType::OPTIONAL_SCALAR) { // Make both operands optional. - type = optType; - typeNode = optTypeNode; + rightType = leftType; } TTypeInfoHelper typeInfoHelper; - arrow::Datum leftOperand = GenerateArray(typeInfoHelper, optType, left, leftOffset); + arrow::Datum leftOperand = GenerateArray(typeInfoHelper, leftType, left, leftOffset); arrow::compute::ExecContext execCtx; arrow::compute::KernelContext ctx(&execCtx); arrow::Datum rightOperand; - if constexpr (rightType == ERightOperandType::SCALAR) { + if constexpr (rightTypeShape == ERightOperandType::SCALAR) { rightOperand = MakeScalarDatum<TLayout>(right[0].GetRef()); - } else if constexpr (rightType == ERightOperandType::OPTIONAL_SCALAR) { + } else if constexpr (rightTypeShape == ERightOperandType::OPTIONAL_SCALAR) { if (right[0]) { rightOperand = MakeScalarDatum<TLayout>(right[0].GetRef()); } else { @@ -155,25 +171,21 @@ void TestBlockCoalesceForVector(InputOptionalVector<T> left, rightOperand.scalar()->is_valid = false; } } else { - rightOperand = GenerateArray(typeInfoHelper, type, right, rightOffset); + rightOperand = GenerateArray(typeInfoHelper, rightType, right, rightOffset); } auto bi = arrow::compute::detail::ExecBatchIterator::Make({leftOperand, rightOperand}, 1000).ValueOrDie(); arrow::compute::ExecBatch batch; UNIT_ASSERT(bi->Next(&batch)); - std::shared_ptr<arrow::DataType> arrowType; - UNIT_ASSERT(ConvertArrowType(type, arrowType, [](TType*) {})); arrow::Datum out; - auto registry = CreateFunctionRegistry(CreateBuiltinRegistry()); - NYql::TKernelRequestBuilder b(*registry); - - b.AddBinaryOp(NYql::TKernelRequestBuilder::EBinaryOp::Coalesce, optTypeNode, typeNode, optTypeNode); - auto serializedNode = b.Serialize(); - auto nodeFactory = GetBuiltinFactory(); - auto kernel = NYql::LoadKernels(serializedNode, *registry, nodeFactory); - Y_ENSURE(kernel.size() == 1); - Y_ENSURE(kernel[0]->exec(&ctx, batch, &out).ok()); - - arrow::Datum expectedArrowArray = GenerateArray(typeInfoHelper, type, expected, 0); + // This graph will never be executed. We need it only to extrace coalesce arrow kernel. + auto graph = setup.BuildGraph( + setup.PgmBuilder->BlockCoalesce( + setup.PgmBuilder->Arg(setup.PgmBuilder->NewBlockType(leftType, TBlockType::EShape::Many)), + setup.PgmBuilder->Arg(setup.PgmBuilder->NewBlockType(rightType, TBlockType::EShape::Many)))); + auto kernel = GetArrowKernel(graph.Get()); + // kernel is exectly coalesce kernel. + Y_ENSURE(kernel->GetArrowKernel().exec(&ctx, batch, &out).ok()); + arrow::Datum expectedArrowArray = GenerateArray(typeInfoHelper, rightType, expected, 0); UNIT_ASSERT_EQUAL_C(out, expectedArrowArray, "Expected : " << expectedArrowArray.make_array()->ToString() << "\n but got : " << out.make_array()->ToString()); } @@ -337,6 +349,23 @@ UNIT_TEST_WITH_INTEGER(KernelRightIsOptionalValidScalar) { {77, 2, 3, 77, 5, 6, 7, max, 9, 77, 11, 12, 13, 77, 77, 77, min, 77, 19, 20}); } -} // Y_UNIT_TEST_SUITE(TMiniKQLBlockCoalesceTest) +Y_UNIT_TEST(OptionalScalar) { + TestScalarCoalesceKernel(TMaybe<i32>{16}, 5, 16); + TestScalarCoalesceKernel(TMaybe<i32>(), 4, 4); + TestScalarCoalesceKernel(TMaybe<i32>(18), TMaybe<i32>(3), TMaybe<i32>(18)); + TestScalarCoalesceKernel(TMaybe<i32>(), TMaybe<i32>(2), TMaybe<i32>(2)); +} +Y_UNIT_TEST(ExternalOptionalScalar) { + using TDoubleMaybe = TMaybe<TMaybe<i32>>; + using TSingleMaybe = TMaybe<i32>; + + TestScalarCoalesceKernel(TDoubleMaybe{TSingleMaybe{25}}, TSingleMaybe{1}, TSingleMaybe{25}); + TestScalarCoalesceKernel(TDoubleMaybe(TSingleMaybe()), TSingleMaybe(9), TSingleMaybe()); + TestScalarCoalesceKernel(TDoubleMaybe(), TSingleMaybe(8), TSingleMaybe(8)); + TestScalarCoalesceKernel(TDoubleMaybe(TSingleMaybe(33)), TDoubleMaybe(TSingleMaybe(7)), TDoubleMaybe(TSingleMaybe(33))); + TestScalarCoalesceKernel(TDoubleMaybe(), TDoubleMaybe(TSingleMaybe(6)), TDoubleMaybe(TSingleMaybe(6))); +} + +} // Y_UNIT_TEST_SUITE(TMiniKQLBlockCoalesceTest) } // namespace NKikimr::NMiniKQL diff --git a/yql/essentials/minikql/comp_nodes/ut/mkql_block_exists_ut.cpp b/yql/essentials/minikql/comp_nodes/ut/mkql_block_exists_ut.cpp index 3b9d41e2b9b..6d0b9915171 100644 --- a/yql/essentials/minikql/comp_nodes/ut/mkql_block_exists_ut.cpp +++ b/yql/essentials/minikql/comp_nodes/ut/mkql_block_exists_ut.cpp @@ -2,12 +2,22 @@ #include <yql/essentials/minikql/computation/mkql_computation_node_holders.h> #include <yql/essentials/minikql/computation/mkql_block_builder.h> +#include <yql/essentials/minikql/comp_nodes/ut/mkql_block_helper.h> namespace NKikimr { namespace NMiniKQL { namespace { +template <typename T, typename U> +void TestScalarExistsKernel(T operand, U expected) { + TSetup<false> setup; + TestScalarKernel(operand, expected, setup, + [&](TRuntimeNode node) { + return setup.PgmBuilder->BlockExists(node); + }); +} + void DoBlockExistsOffset(size_t length, size_t offset) { TSetup<false> setup; TProgramBuilder& pb = *setup.PgmBuilder; @@ -109,6 +119,11 @@ Y_UNIT_TEST(ExistsWithOffset) { } } +Y_UNIT_TEST(ScalarExists) { + TestScalarExistsKernel(TMaybe<ui32>(), false); + TestScalarExistsKernel(TMaybe<ui32>(6), true); +} + } // Y_UNIT_TEST_SUITE } // namespace NMiniKQL diff --git a/yql/essentials/minikql/comp_nodes/ut/mkql_block_helper.h b/yql/essentials/minikql/comp_nodes/ut/mkql_block_helper.h new file mode 100644 index 00000000000..a74aca399ea --- /dev/null +++ b/yql/essentials/minikql/comp_nodes/ut/mkql_block_helper.h @@ -0,0 +1,111 @@ +#include <yql/essentials/ast/yql_expr.h> +#include <yql/essentials/minikql/comp_nodes/ut/mkql_computation_node_ut.h> +#include <yql/essentials/minikql/computation/mkql_computation_node_holders.h> + +namespace NKikimr::NMiniKQL { + +// Template to count TMaybe nesting levels +template <typename T> +struct TMaybeTraits { + static constexpr ui32 value = 0; + using ResultType = T; +}; + +template <typename T> +struct TMaybeTraits<TMaybe<T>> { + static constexpr ui32 value = 1 + TMaybeTraits<T>::value; + using ResultType = TMaybeTraits<T>::ResultType; +}; + +// Template to get the value of the deepest set TMaybe or return 0 +template <typename T> +TMaybeTraits<T>::ResultType GetInnerValue(const T& value) { + if constexpr (TMaybeTraits<T>::value == 0) { + // Base case: not a TMaybe type, return the value itself + return value; + } else { + // TMaybe type + if (value.Defined()) { + return GetInnerValue(value.GetRef()); + } else { + return {}; // Return 0 if not set + } + } +} + +// Template to get the level of the set TMaybe (returns 0 if not set) +template <typename T> +ui32 GetSettedLevel(const T& value) { + if constexpr (TMaybeTraits<T>::value == 0) { + // Base case: not a TMaybe type, always set at level 0 + return 0; + } else { + // TMaybe type + if (value.Defined()) { + return 1 + GetSettedLevel(value.GetRef()); + } else { + return 0; // Return 0 if not set + } + } +} + +template<typename T> +struct TUnpackedMaybe { + ui32 SettedLevel; + ui32 MaybeLevel; + T Value; +}; + +template <typename T> +TRuntimeNode ConvertMaybeToNode(TUnpackedMaybe<T> maybe, TSetup<false>& setup) { + auto data = setup.PgmBuilder->NewDataLiteral<T>(maybe.Value); + + for (ui32 i = maybe.SettedLevel; i < maybe.MaybeLevel; i++) { + data = setup.PgmBuilder->NewEmptyOptional(setup.PgmBuilder->NewOptionalType(data.GetStaticType())); + } + + for (ui32 i = 0; i < maybe.SettedLevel; i++) { + data = setup.PgmBuilder->NewOptional(data); + } + + data = setup.PgmBuilder->AsScalar(data); + return data; +} + +template <typename T, typename U, typename V> +void TestScalarKernel(T left, U right, V expected, TSetup<false>& setup, std::function<TRuntimeNode(TRuntimeNode, TRuntimeNode)> binaryOp) { + NYql::TExprContext exprCtx; + TUnpackedMaybe leftUnpacked{.SettedLevel = GetSettedLevel(left), .MaybeLevel = TMaybeTraits<T>::value, .Value = GetInnerValue(left)}; + TUnpackedMaybe rightUnpacked{.SettedLevel = GetSettedLevel(right), .MaybeLevel = TMaybeTraits<U>::value, .Value = GetInnerValue(right)}; + TUnpackedMaybe expectedUnpacked{.SettedLevel = GetSettedLevel(expected), .MaybeLevel = TMaybeTraits<V>::value, .Value = GetInnerValue(expected)}; + auto leftNode = ConvertMaybeToNode(leftUnpacked, setup); + auto rightNode = ConvertMaybeToNode(rightUnpacked, setup); + auto expectedNode = ConvertMaybeToNode(expectedUnpacked, setup); + + auto resultValue = setup.BuildGraph(binaryOp(leftNode, rightNode))->GetValue(); + auto expectedValue = setup.BuildGraph(expectedNode)->GetValue(); + + auto outDatum = TArrowBlock::From(resultValue).GetDatum(); + auto expectedDatum = TArrowBlock::From(expectedValue).GetDatum(); + + UNIT_ASSERT_EQUAL_C(outDatum, expectedDatum, "Expected : " << outDatum.scalar()->ToString() << "\n but got : " << expectedDatum.scalar()->ToString()); +} + +template <typename T, typename V> +void TestScalarKernel(T left, V expected, TSetup<false>& setup, std::function<TRuntimeNode(TRuntimeNode)> unaryOp) { + NYql::TExprContext exprCtx; + TUnpackedMaybe unpacked{.SettedLevel = GetSettedLevel(left), .MaybeLevel = TMaybeTraits<T>::value, .Value = GetInnerValue(left)}; + TUnpackedMaybe expectedUnpacked{.SettedLevel = GetSettedLevel(expected), .MaybeLevel = TMaybeTraits<V>::value, .Value = GetInnerValue(expected)}; + auto node = ConvertMaybeToNode(unpacked, setup); + auto expectedNode = ConvertMaybeToNode(expectedUnpacked, setup); + + auto resultValue = setup.BuildGraph(unaryOp(node))->GetValue(); + auto expectedValue = setup.BuildGraph(expectedNode)->GetValue(); + + auto outDatum = TArrowBlock::From(resultValue).GetDatum(); + auto expectedDatum = TArrowBlock::From(expectedValue).GetDatum(); + + UNIT_ASSERT_EQUAL_C(outDatum, expectedDatum, "Expected : " << outDatum.scalar()->ToString() << "\n but got : " << expectedDatum.scalar()->ToString()); +} + +} // namespace NKikimr::NMiniKQL diff --git a/yql/essentials/minikql/comp_nodes/ut/ya.make.inc b/yql/essentials/minikql/comp_nodes/ut/ya.make.inc index 4ce865905c9..6f7138c5edc 100644 --- a/yql/essentials/minikql/comp_nodes/ut/ya.make.inc +++ b/yql/essentials/minikql/comp_nodes/ut/ya.make.inc @@ -84,8 +84,6 @@ SET(ORIG_SOURCES ) PEERDIR( - yql/essentials/core/arrow_kernels/request - yql/essentials/core/arrow_kernels/registry yql/essentials/public/udf yql/essentials/public/udf/arrow yql/essentials/public/udf/service/exception_policy diff --git a/yql/essentials/minikql/computation/mkql_block_impl.cpp b/yql/essentials/minikql/computation/mkql_block_impl.cpp index e960e8562e3..0f16206c112 100644 --- a/yql/essentials/minikql/computation/mkql_block_impl.cpp +++ b/yql/essentials/minikql/computation/mkql_block_impl.cpp @@ -7,6 +7,7 @@ #include <yql/essentials/minikql/mkql_node_cast.h> #include <yql/essentials/minikql/arrow/arrow_util.h> #include <yql/essentials/minikql/arrow/mkql_bit_utils.h> +#include <yql/essentials/minikql/mkql_type_helper.h> #include <yql/essentials/public/udf/arrow/args_dechunker.h> #include <yql/essentials/parser/pg_wrapper/interface/arrow.h> @@ -33,6 +34,7 @@ namespace NKikimr::NMiniKQL { namespace { +// TODO(YQL): This must be rewrited via traits dispatcher. template<typename T> arrow::Datum DoConvertScalar(TType* type, const T& value, arrow::MemoryPool& pool) { std::shared_ptr<arrow::DataType> arrowType; @@ -40,15 +42,13 @@ arrow::Datum DoConvertScalar(TType* type, const T& value, arrow::MemoryPool& poo if (!value) { return arrow::MakeNullScalar(arrowType); } + auto needWrapWithExternalOptional = NeedWrapWithExternalOptional(type); - bool isOptional = false; if (type->IsOptional()) { type = AS_TYPE(TOptionalType, type)->GetItemType(); - isOptional = true; } - if (type->IsOptional() || (isOptional && type->IsPg())) { - // nested optionals + if (needWrapWithExternalOptional) { std::vector<std::shared_ptr<arrow::Scalar>> arrowValue; arrowValue.emplace_back(DoConvertScalar(type, value.GetOptionalValue(), pool).scalar()); return arrow::Datum(std::make_shared<arrow::StructScalar>(arrowValue, arrowType)); diff --git a/yql/essentials/minikql/mkql_runtime_version.h b/yql/essentials/minikql/mkql_runtime_version.h index be71a1646f3..eb64e14b02b 100644 --- a/yql/essentials/minikql/mkql_runtime_version.h +++ b/yql/essentials/minikql/mkql_runtime_version.h @@ -24,7 +24,7 @@ namespace NMiniKQL { // 1. Bump this version every time incompatible runtime nodes are introduced. // 2. Make sure you provide runtime node generation for previous runtime versions. #ifndef MKQL_RUNTIME_VERSION -#define MKQL_RUNTIME_VERSION 62U +#define MKQL_RUNTIME_VERSION 63U #endif // History: diff --git a/yql/essentials/minikql/mkql_type_builder.cpp b/yql/essentials/minikql/mkql_type_builder.cpp index f6e529a4a37..cc70d26e5ee 100644 --- a/yql/essentials/minikql/mkql_type_builder.cpp +++ b/yql/essentials/minikql/mkql_type_builder.cpp @@ -3,6 +3,7 @@ #include "mkql_node_builder.h" #include "mkql_alloc.h" +#include <yql/essentials/minikql/mkql_type_helper.h> #include <yql/essentials/public/udf/udf_type_ops.h> #include <yql/essentials/public/udf/arrow/block_item_comparator.h> #include <yql/essentials/public/udf/arrow/block_item_hasher.h> @@ -1555,17 +1556,7 @@ bool ConvertArrowTypeImpl(NUdf::EDataSlot slot, std::shared_ptr<arrow::DataType> } } -inline bool IsSingularType(const TType* type) { - return type->IsNull() || - type->IsVoid() || - type->IsEmptyDict() || - type->IsEmptyList(); -} - -inline bool NeedWrapWithExternalOptional(const TType* type) { - return type->IsPg() || IsSingularType(type); -} - +// TODO(YQL): This must be rewrited via traits dispatcher. bool ConvertArrowTypeImpl(TType* itemType, std::shared_ptr<arrow::DataType>& type, const TArrowConvertFailedCallback& onFail, bool output) { bool isOptional; auto unpacked = UnpackOptional(itemType, isOptional); @@ -1578,7 +1569,7 @@ bool ConvertArrowTypeImpl(TType* itemType, std::shared_ptr<arrow::DataType>& typ return false; } - if (unpacked->IsOptional() || isOptional && NeedWrapWithExternalOptional(unpacked)) { + if (NeedWrapWithExternalOptional(itemType)) { ui32 nestLevel = 0; auto currentType = itemType; auto previousType = itemType; @@ -1588,7 +1579,7 @@ bool ConvertArrowTypeImpl(TType* itemType, std::shared_ptr<arrow::DataType>& typ currentType = AS_TYPE(TOptionalType, currentType)->GetItemType(); } while (currentType->IsOptional()); - if (NeedWrapWithExternalOptional(currentType)) { + if (NeedWrapWithExternalOptional(previousType)) { previousType = currentType; ++nestLevel; } diff --git a/yql/essentials/minikql/mkql_type_helper.h b/yql/essentials/minikql/mkql_type_helper.h new file mode 100644 index 00000000000..87832b6e09f --- /dev/null +++ b/yql/essentials/minikql/mkql_type_helper.h @@ -0,0 +1,27 @@ +#pragma once + +#include <yql/essentials/minikql/mkql_node_builder.h> + +namespace NKikimr::NMiniKQL { + +inline bool IsSingularType(const TType* type) { + return type->IsNull() || + type->IsVoid() || + type->IsEmptyDict() || + type->IsEmptyList(); +} + +inline bool NeedWrapWithExternalOptional(TType* type) { + bool isOptional; + auto unpacked = UnpackOptional(type, isOptional); + if (!isOptional) { + return false; + } else if (unpacked->IsOptional()) { + return true; + } else if (unpacked->IsPg() || IsSingularType(unpacked)) { + return true; + } + return false; +} + +} // namespace NKikimr::NMiniKQL diff --git a/yql/essentials/providers/common/proto/gateways_config.proto b/yql/essentials/providers/common/proto/gateways_config.proto index 86fd7f82544..52210fd4d31 100644 --- a/yql/essentials/providers/common/proto/gateways_config.proto +++ b/yql/essentials/providers/common/proto/gateways_config.proto @@ -657,7 +657,7 @@ message TMongoDbDataSourceOptions { // Returns a key-value pair of ObjectID primary key and a JSON serialized document JSON = 3; }; - + // Ways of dealing with values read in ReadSplit whose types are inconsistent with types that were deduced in DescribeTable enum EUnexpectedTypeDisplayMode { UNEXPECTED_UNSPECIFIED = 0; @@ -671,7 +671,7 @@ message TMongoDbDataSourceOptions { UNSUPPORTED_OMIT = 1; UNSUPPORTED_AS_STRING = 2; }; - + optional EReadingMode reading_mode = 1; optional EUnexpectedTypeDisplayMode unexpected_type_display_mode = 2; optional EUnsupportedTypeDisplayMode unsupported_type_display_mode = 3; @@ -851,7 +851,7 @@ message TGenericGatewayConfig { // // The list of Connectors. - // + // // When a request is served, firstly this list is checked. // If the list has a connector for a request's EGenericDataSourceKind it will // be applied to that request; otherwise, the "Default Connector" is used. @@ -869,6 +869,9 @@ message TGenericGatewayConfig { // [http|https]://host:port/ydbc/cloud-prod/ optional string YdbMvpEndpoint = 7; + // Defines timeout for calling the Connector API method `DescribeTable` + optional uint64 DescribeTableTimeoutSeconds = 9; + repeated TAttr DefaultSettings = 6; reserved 1, 2; diff --git a/yql/essentials/public/purecalc/common/type_from_schema.cpp b/yql/essentials/public/purecalc/common/type_from_schema.cpp index dafb4b29768..d4f12043a13 100644 --- a/yql/essentials/public/purecalc/common/type_from_schema.cpp +++ b/yql/essentials/public/purecalc/common/type_from_schema.cpp @@ -21,10 +21,8 @@ namespace { const auto* expectedItem = expected->GetItems()[*expectedIndex]->GetItemType(); auto arg = ctx.NewArgument(TPositionHandle(), "arg"); - auto fieldConversionStatus = TrySilentConvertTo(arg, *gotItem, *expectedItem, ctx); + auto fieldConversionStatus = TryConvertTo(arg, *gotItem, *expectedItem, ctx, {}, true); if (fieldConversionStatus.Level == IGraphTransformer::TStatus::Error) { - REPORT("Item " << TString{gotNamedItem->GetName()}.Quote() << " expected to be " << - *expectedItem << ", but got " << *gotItem); status = false; } } else { diff --git a/yql/essentials/public/purecalc/common/worker_factory.cpp b/yql/essentials/public/purecalc/common/worker_factory.cpp index 9cd1ce129bd..ea1a4ee5187 100644 --- a/yql/essentials/public/purecalc/common/worker_factory.cpp +++ b/yql/essentials/public/purecalc/common/worker_factory.cpp @@ -167,6 +167,7 @@ TExprNode::TPtr TWorkerFactory<TBase>::Compile( IModuleResolver::TPtr moduleResolver = factoryModuleResolver ? factoryModuleResolver->CreateMutableChild() : nullptr; typeContext = MakeIntrusive<TTypeAnnotationContext>(); typeContext->LangVer = langver; + typeContext->UseTypeDiffForConvertToError = true; typeContext->RandomProvider = CreateDefaultRandomProvider(); typeContext->TimeProvider = DeterministicTimeProviderSeed_ ? CreateDeterministicTimeProvider(*DeterministicTimeProviderSeed_) : diff --git a/yql/essentials/public/purecalc/ut/fake_spec.cpp b/yql/essentials/public/purecalc/ut/fake_spec.cpp index b56f7cfdfd5..8acdbcfb347 100644 --- a/yql/essentials/public/purecalc/ut/fake_spec.cpp +++ b/yql/essentials/public/purecalc/ut/fake_spec.cpp @@ -32,5 +32,51 @@ namespace NYql { spec.Schema = MakeFakeSchema(pg); return spec; } + + NYT::TNode CreateTypeNode(const TString& fieldType) { + return NYT::TNode::CreateList() + .Add("DataType") + .Add(fieldType); + } + + NYT::TNode CreateOptionalTypeNode(const TString& fieldType) { + return NYT::TNode::CreateList() + .Add("OptionalType") + .Add(CreateTypeNode(fieldType)); + } + + void AddField(NYT::TNode& node, const TString& fieldName, const TString& fieldType) { + node.Add( + NYT::TNode::CreateList() + .Add(fieldName) + .Add(CreateOptionalTypeNode(fieldType)) + ); + } + + NYT::TNode MakeFakeStructSchema() { + auto structMembers = NYT::TNode::CreateList(); + AddField(structMembers, "Id", "Uint32"); + AddField(structMembers, "Name", "Utf8"); + AddField(structMembers, "Body", "String"); + + auto rootMembers = NYT::TNode::CreateList(); + rootMembers.Add( + NYT::TNode::CreateList() + .Add("_r") + .Add(NYT::TNode::CreateList() + .Add("StructType") + .Add(std::move(structMembers))) + ); + + return NYT::TNode::CreateList() + .Add("StructType") + .Add(std::move(rootMembers)); + } + + TFakeOutputSpec FakeStructOS() { + auto spec = TFakeOutputSpec(); + spec.Schema = MakeFakeStructSchema(); + return spec; + } } } diff --git a/yql/essentials/public/purecalc/ut/fake_spec.h b/yql/essentials/public/purecalc/ut/fake_spec.h index 3cb1457f01d..5f2ea310821 100644 --- a/yql/essentials/public/purecalc/ut/fake_spec.h +++ b/yql/essentials/public/purecalc/ut/fake_spec.h @@ -50,5 +50,6 @@ namespace NYql { NYT::TNode MakeFakeSchema(bool pg = false); TFakeInputSpec FakeIS(ui32 inputsNumber = 1, bool pg = false); TFakeOutputSpec FakeOS(bool pg = false); + TFakeOutputSpec FakeStructOS(); } } diff --git a/yql/essentials/public/purecalc/ut/test_sql.cpp b/yql/essentials/public/purecalc/ut/test_sql.cpp index 2975f784b73..1b6b69279a8 100644 --- a/yql/essentials/public/purecalc/ut/test_sql.cpp +++ b/yql/essentials/public/purecalc/ut/test_sql.cpp @@ -33,6 +33,27 @@ Y_UNIT_TEST_SUITE(TestSql) { UNIT_ASSERT_VALUES_EQUAL(expectedIssues, program->GetIssues().ToString()); } + Y_UNIT_TEST(TestStructCastMessage) { + auto factory = MakeProgramFactory(); + + auto sql = TString(R"( + $l = ($x) -> { + return <| Id: $x.Name, Name: CAST($x.Name AS String), Body: Just('foo') |>; + }; + + SELECT $l(TableRow()) AS _r FROM Input + )"); + + try { + factory->MakePullListProgram(FakeIS(), FakeStructOS(), sql, ETranslationMode::SQL); + UNIT_ASSERT_C(false, "Unreachable"); + } catch (const NYql::NPureCalc::TCompileError& error) { + auto issue = error.GetIssues(); + UNIT_ASSERT_C(issue.Contains("Failed to convert 'Id': Int32 to Optional<Uint32>"), issue); + UNIT_ASSERT_C(!issue.Contains("Body"), issue); + } + } + Y_UNIT_TEST(TestSqlCompileSingleUnnamedInput) { auto factory = MakeProgramFactory(); diff --git a/yql/essentials/public/udf/arrow/dispatch_traits.h b/yql/essentials/public/udf/arrow/dispatch_traits.h index 93d7084296e..f49b2861630 100644 --- a/yql/essentials/public/udf/arrow/dispatch_traits.h +++ b/yql/essentials/public/udf/arrow/dispatch_traits.h @@ -86,8 +86,7 @@ std::unique_ptr<typename TTraits::TResult> DispatchByArrowTraits(const ITypeInfo unpacked = SkipTaggedType(typeInfoHelper, unpacked); - TOptionalTypeInspector unpackedOpt(typeInfoHelper, unpacked); - if (unpackedOpt || (typeOpt && NeedWrapWithExternalOptional(typeInfoHelper, unpacked))) { + if (NeedWrapWithExternalOptional(typeInfoHelper, type)) { ui32 nestLevel = 0; auto currentType = type; auto previousType = type; @@ -107,7 +106,7 @@ std::unique_ptr<typename TTraits::TResult> DispatchByArrowTraits(const ITypeInfo } } - if (NeedWrapWithExternalOptional(typeInfoHelper, currentType)) { + if (NeedWrapWithExternalOptional(typeInfoHelper, previousType)) { previousType = currentType; ++nestLevel; } diff --git a/yql/essentials/public/udf/arrow/util.h b/yql/essentials/public/udf/arrow/util.h index a4e430aa4ad..a7d5018b558 100644 --- a/yql/essentials/public/udf/arrow/util.h +++ b/yql/essentials/public/udf/arrow/util.h @@ -251,9 +251,18 @@ inline bool IsSingularType(const ITypeInfoHelper& typeInfoHelper, const TType* t const TType* SkipTaggedType(const ITypeInfoHelper& typeInfoHelper, const TType* type); inline bool NeedWrapWithExternalOptional(const ITypeInfoHelper& typeInfoHelper, const TType* type) { - type = SkipTaggedType(typeInfoHelper, type); - - return TPgTypeInspector(typeInfoHelper, type) || IsSingularType(typeInfoHelper, type); + TOptionalTypeInspector typeOpt(typeInfoHelper, type); + if (!typeOpt) { + return false; + } + type = SkipTaggedType(typeInfoHelper, typeOpt.GetItemType()); + TOptionalTypeInspector typeOptOpt(typeInfoHelper, type); + if (typeOptOpt) { + return true; + } else if (TPgTypeInspector(typeInfoHelper, type) || IsSingularType(typeInfoHelper, type)) { + return true; + } + return false; } } // namespace NUdf diff --git a/yql/essentials/sql/v1/complete/analysis/global/column.cpp b/yql/essentials/sql/v1/complete/analysis/global/column.cpp new file mode 100644 index 00000000000..7e185697d3f --- /dev/null +++ b/yql/essentials/sql/v1/complete/analysis/global/column.cpp @@ -0,0 +1,116 @@ +#include "column.h" + +#include "narrowing_visitor.h" + +#include <yql/essentials/sql/v1/complete/syntax/format.h> + +namespace NSQLComplete { + + namespace { + + // TODO: Extract it to `identifier.cpp` and reuse it also at `use.cpp` + // and replace `GetId` at `parse_tree.cpp`. + class TIdentifierVisitor: public SQLv1Antlr4BaseVisitor { + public: + std::any visitCluster_expr(SQLv1::Cluster_exprContext* ctx) override { + if (auto* x = ctx->pure_column_or_named()) { + return visit(x); + } + return {}; + } + + std::any visitTable_key(SQLv1::Table_keyContext* ctx) override { + if (auto* x = ctx->id_table_or_type()) { + return visit(x); + } + return {}; + } + + std::any visitTerminal(antlr4::tree::TerminalNode* node) override { + TString text = GetText(node); + switch (node->getSymbol()->getType()) { + case SQLv1::TOKEN_ID_QUOTED: { + text = Unquoted(std::move(text)); + } break; + } + return text; + } + + private: + TString GetText(antlr4::tree::ParseTree* tree) const { + return TString(tree->getText()); + } + }; + + TMaybe<TString> GetId(antlr4::ParserRuleContext* ctx) { + if (ctx == nullptr) { + return Nothing(); + } + + std::any result = TIdentifierVisitor().visit(ctx); + if (!result.has_value()) { + return Nothing(); + } + return std::any_cast<TString>(result); + } + + class TInferenceVisitor: public SQLv1Antlr4BaseVisitor { + public: + std::any visitTable_ref(SQLv1::Table_refContext* ctx) override { + TString cluster = GetId(ctx->cluster_expr()).GetOrElse(""); + + TMaybe<TString> path = GetId(ctx->table_key()); + if (path.Empty()) { + return {}; + } + + return TColumnContext{ + .Tables = { + {.Cluster = std::move(cluster), .Path = std::move(*path)}, + }, + }; + } + }; + + class TVisitor: public TSQLv1NarrowingVisitor { + public: + TVisitor(antlr4::TokenStream* tokens, size_t cursorPosition) + : TSQLv1NarrowingVisitor(tokens, cursorPosition) + { + } + + std::any visitSql_stmt_core(SQLv1::Sql_stmt_coreContext* ctx) override { + if (IsEnclosing(ctx)) { + return visitChildren(ctx); + } + return {}; + } + + std::any visitSelect_core(SQLv1::Select_coreContext* ctx) override { + SQLv1::Join_sourceContext* source = ctx->join_source(0); + if (source == nullptr) { + source = ctx->join_source(1); + } + if (source == nullptr) { + return {}; + } + + return TInferenceVisitor().visit(ctx); + } + }; + + } // namespace + + TMaybe<TColumnContext> InferColumnContext( + SQLv1::Sql_queryContext* ctx, + antlr4::TokenStream* tokens, + size_t cursorPosition) { + // TODO: add utility `auto ToMaybe<T>(std::any any) -> TMaybe<T>` + std::any result = TVisitor(tokens, cursorPosition).visit(ctx); + if (!result.has_value()) { + return Nothing(); + } + return std::any_cast<TColumnContext>(result); + } + +} // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/analysis/global/column.h b/yql/essentials/sql/v1/complete/analysis/global/column.h new file mode 100644 index 00000000000..790dbee1d15 --- /dev/null +++ b/yql/essentials/sql/v1/complete/analysis/global/column.h @@ -0,0 +1,13 @@ +#pragma once + +#include "global.h" +#include "parse_tree.h" + +namespace NSQLComplete { + + TMaybe<TColumnContext> InferColumnContext( + SQLv1::Sql_queryContext* ctx, + antlr4::TokenStream* tokens, + size_t cursorPosition); + +} // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/analysis/global/global.cpp b/yql/essentials/sql/v1/complete/analysis/global/global.cpp index 8f5df3f6e76..7574b6bdfc0 100644 --- a/yql/essentials/sql/v1/complete/analysis/global/global.cpp +++ b/yql/essentials/sql/v1/complete/analysis/global/global.cpp @@ -1,5 +1,6 @@ #include "global.h" +#include "column.h" #include "function.h" #include "named_node.h" #include "parse_tree.h" @@ -8,6 +9,10 @@ #include <yql/essentials/sql/v1/complete/antlr4/pipeline.h> #include <yql/essentials/sql/v1/complete/syntax/ansi.h> +#include <library/cpp/iterator/functools.h> + +#include <util/string/join.h> + namespace NSQLComplete { class TErrorStrategy: public antlr4::DefaultErrorStrategy { @@ -54,6 +59,11 @@ namespace NSQLComplete { ctx.Use = FindUseStatement(sqlQuery, &Tokens_, input.CursorPosition, env); ctx.Names = CollectNamedNodes(sqlQuery, &Tokens_, input.CursorPosition); ctx.EnclosingFunction = EnclosingFunction(sqlQuery, &Tokens_, input.CursorPosition); + ctx.Column = InferColumnContext(sqlQuery, &Tokens_, input.CursorPosition); + + if (ctx.Use && ctx.Column) { + EnrichTableClusters(*ctx.Column, *ctx.Use); + } return ctx; } @@ -67,6 +77,14 @@ namespace NSQLComplete { return Parser_.sql_query(); } + void EnrichTableClusters(TColumnContext& column, const TUseContext& use) { + for (auto& table : column.Tables) { + if (table.Cluster.empty()) { + table.Cluster = use.Cluster; + } + } + } + antlr4::ANTLRInputStream Chars_; G::TLexer Lexer_; antlr4::CommonTokenStream Tokens_; @@ -97,3 +115,10 @@ namespace NSQLComplete { } } // namespace NSQLComplete + +template <> +void Out<NSQLComplete::TColumnContext>(IOutputStream& out, const NSQLComplete::TColumnContext& value) { + out << "TColumnContext { "; + out << "Tables: " << JoinSeq(", ", value.Tables); + out << " }"; +} diff --git a/yql/essentials/sql/v1/complete/analysis/global/global.h b/yql/essentials/sql/v1/complete/analysis/global/global.h index 3999aeb4a56..1ef1344e3c9 100644 --- a/yql/essentials/sql/v1/complete/analysis/global/global.h +++ b/yql/essentials/sql/v1/complete/analysis/global/global.h @@ -1,7 +1,8 @@ #pragma once -#include <yql/essentials/sql/v1/complete/core/input.h> #include <yql/essentials/sql/v1/complete/core/environment.h> +#include <yql/essentials/sql/v1/complete/core/input.h> +#include <yql/essentials/sql/v1/complete/core/name.h> #include <util/generic/ptr.h> #include <util/generic/maybe.h> @@ -15,12 +16,20 @@ namespace NSQLComplete { TString Cluster; }; + struct TColumnContext { + TVector<TTableId> Tables; + + friend bool operator==(const TColumnContext& lhs, const TColumnContext& rhs) = default; + }; + struct TGlobalContext { TMaybe<TUseContext> Use; TVector<TString> Names; TMaybe<TString> EnclosingFunction; + TMaybe<TColumnContext> Column; }; + // TODO(YQL-19747): Make it thread-safe to make ISqlCompletionEngine thread-safe. class IGlobalAnalysis { public: using TPtr = THolder<IGlobalAnalysis>; diff --git a/yql/essentials/sql/v1/complete/analysis/global/global_ut.cpp b/yql/essentials/sql/v1/complete/analysis/global/global_ut.cpp index 58aea33b379..c8222580cce 100644 --- a/yql/essentials/sql/v1/complete/analysis/global/global_ut.cpp +++ b/yql/essentials/sql/v1/complete/analysis/global/global_ut.cpp @@ -98,4 +98,24 @@ Y_UNIT_TEST_SUITE(GlobalAnalysisTests) { } } + Y_UNIT_TEST(SimpleSelectFrom) { + IGlobalAnalysis::TPtr global = MakeGlobalAnalysis(); + { + TString query = "SELECT # FROM plato.Input"; + + TGlobalContext ctx = global->Analyze(SharpedInput(query), {}); + + TColumnContext expected = {.Tables = {{"plato", "Input"}}}; + UNIT_ASSERT_VALUES_EQUAL(ctx.Column, expected); + } + { + TString query = "SELECT # FROM plato.`//home/input`"; + + TGlobalContext ctx = global->Analyze(SharpedInput(query), {}); + + TColumnContext expected = {.Tables = {{"plato", "//home/input"}}}; + UNIT_ASSERT_VALUES_EQUAL(ctx.Column, expected); + } + } + } // Y_UNIT_TEST_SUITE(GlobalAnalysisTests) diff --git a/yql/essentials/sql/v1/complete/analysis/global/ya.make b/yql/essentials/sql/v1/complete/analysis/global/ya.make index b37fa3faa22..1f1670a9675 100644 --- a/yql/essentials/sql/v1/complete/analysis/global/ya.make +++ b/yql/essentials/sql/v1/complete/analysis/global/ya.make @@ -1,6 +1,7 @@ LIBRARY() SRCS( + column.cpp evaluate.cpp function.cpp global.cpp diff --git a/yql/essentials/sql/v1/complete/analysis/local/local.cpp b/yql/essentials/sql/v1/complete/analysis/local/local.cpp index b7e97b51e59..13e2fce69ad 100644 --- a/yql/essentials/sql/v1/complete/analysis/local/local.cpp +++ b/yql/essentials/sql/v1/complete/analysis/local/local.cpp @@ -51,10 +51,13 @@ namespace NSQLComplete { public: TSpecializedLocalSyntaxAnalysis( - TLexerSupplier lexer, const THashSet<TString>& IgnoredRules) + TLexerSupplier lexer, + const THashSet<TString>& ignoredRules, + const THashMap<TString, THashSet<TString>>& disabledPreviousByToken, + const THashMap<TString, THashSet<TString>>& forcedPreviousByToken) : Grammar_(&GetSqlGrammar()) , Lexer_(lexer(/* ansi = */ IsAnsiLexer)) - , C3_(ComputeC3Config(IgnoredRules)) + , C3_(ComputeC3Config(ignoredRules, disabledPreviousByToken, forcedPreviousByToken)) { } @@ -104,17 +107,23 @@ namespace NSQLComplete { result.Hint = HintMatch(candidates); result.Object = ObjectMatch(context, candidates); result.Cluster = ClusterMatch(context, candidates); + result.Column = ColumnMatch(candidates); result.Binding = BindingMatch(candidates); return result; } private: - IC3Engine::TConfig ComputeC3Config(const THashSet<TString>& IgnoredRules) const { + IC3Engine::TConfig ComputeC3Config( + const THashSet<TString>& ignoredRules, + const THashMap<TString, THashSet<TString>>& disabledPreviousByToken, + const THashMap<TString, THashSet<TString>>& forcedPreviousByToken) const { return { .IgnoredTokens = ComputeIgnoredTokens(), .PreferredRules = ComputePreferredRules(), - .IgnoredRules = ComputeIgnoredRules(IgnoredRules), + .IgnoredRules = ComputeIgnoredRules(ignoredRules), + .DisabledPreviousByToken = Resolved(disabledPreviousByToken), + .ForcedPreviousByToken = Resolved(forcedPreviousByToken), }; } @@ -142,6 +151,23 @@ namespace NSQLComplete { return ignored; } + std::unordered_map<TTokenId, std::unordered_set<TTokenId>> + Resolved(const THashMap<TString, THashSet<TString>>& tokens) const { + std::unordered_map<TTokenId, std::unordered_set<TTokenId>> resolved; + for (const auto& [name, set] : tokens) { + resolved[Grammar_->GetTokenId(name)] = Resolved(set); + } + return resolved; + } + + std::unordered_set<TTokenId> Resolved(const THashSet<TString>& tokens) const { + std::unordered_set<TTokenId> resolved; + for (const TString& name : tokens) { + resolved.emplace(Grammar_->GetTokenId(name)); + } + return resolved; + } + TC3Candidates C3Complete(TCompletionInput statement, const TCursorTokenContext& context) { auto enclosing = context.Enclosing(); @@ -296,6 +322,10 @@ namespace NSQLComplete { return cluster; } + bool ColumnMatch(const TC3Candidates& candidates) const { + return AnyOf(candidates.Rules, RuleAdapted(IsLikelyColumnStack)); + } + bool BindingMatch(const TC3Candidates& candidates) const { return AnyOf(candidates.Rules, RuleAdapted(IsLikelyBindingStack)); } @@ -331,9 +361,12 @@ namespace NSQLComplete { class TLocalSyntaxAnalysis: public ILocalSyntaxAnalysis { public: TLocalSyntaxAnalysis( - TLexerSupplier lexer, const THashSet<TString>& IgnoredRules) - : DefaultEngine_(lexer, IgnoredRules) - , AnsiEngine_(lexer, IgnoredRules) + TLexerSupplier lexer, + const THashSet<TString>& ignoredRules, + const THashMap<TString, THashSet<TString>>& disabledPreviousByToken, + const THashMap<TString, THashSet<TString>>& forcedPreviousByToken) + : DefaultEngine_(lexer, ignoredRules, disabledPreviousByToken, forcedPreviousByToken) + , AnsiEngine_(lexer, ignoredRules, disabledPreviousByToken, forcedPreviousByToken) { } @@ -356,8 +389,11 @@ namespace NSQLComplete { }; ILocalSyntaxAnalysis::TPtr MakeLocalSyntaxAnalysis( - TLexerSupplier lexer, const THashSet<TString>& IgnoredRules) { - return MakeHolder<TLocalSyntaxAnalysis>(lexer, IgnoredRules); + TLexerSupplier lexer, + const THashSet<TString>& ignoredRules, + const THashMap<TString, THashSet<TString>>& disabledPreviousByToken, + const THashMap<TString, THashSet<TString>>& forcedPreviousByToken) { + return MakeHolder<TLocalSyntaxAnalysis>(lexer, ignoredRules, disabledPreviousByToken, forcedPreviousByToken); } } // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/analysis/local/local.h b/yql/essentials/sql/v1/complete/analysis/local/local.h index 5557d6b6dc0..cca182aacdb 100644 --- a/yql/essentials/sql/v1/complete/analysis/local/local.h +++ b/yql/essentials/sql/v1/complete/analysis/local/local.h @@ -55,10 +55,12 @@ namespace NSQLComplete { TMaybe<THint> Hint; TMaybe<TObject> Object; TMaybe<TCluster> Cluster; + bool Column = false; bool Binding = false; TEditRange EditRange; }; + // TODO(YQL-19747): Make it thread-safe to make ISqlCompletionEngine thread-safe. class ILocalSyntaxAnalysis { public: using TPtr = THolder<ILocalSyntaxAnalysis>; @@ -68,6 +70,9 @@ namespace NSQLComplete { }; ILocalSyntaxAnalysis::TPtr MakeLocalSyntaxAnalysis( - TLexerSupplier lexer, const THashSet<TString>& IgnoredRules); + TLexerSupplier lexer, + const THashSet<TString>& ignoredRules, + const THashMap<TString, THashSet<TString>>& disabledPreviousByToken, + const THashMap<TString, THashSet<TString>>& forcedPreviousByToken); } // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/analysis/local/parser_call_stack.cpp b/yql/essentials/sql/v1/complete/analysis/local/parser_call_stack.cpp index 293cb333d5c..8ac4ff17999 100644 --- a/yql/essentials/sql/v1/complete/analysis/local/parser_call_stack.cpp +++ b/yql/essentials/sql/v1/complete/analysis/local/parser_call_stack.cpp @@ -33,6 +33,9 @@ namespace NSQLComplete { RULE(Id_hint), RULE(Opt_id_prefix_or_type), RULE(Type_name_simple), + RULE(Type_name_composite), + RULE(Type_name_decimal), + RULE(Value_constructor), }; TVector<std::string> Symbolized(const TParserCallStack& stack) { @@ -69,6 +72,8 @@ namespace NSQLComplete { bool IsLikelyTypeStack(const TParserCallStack& stack) { return EndsWith({RULE(Type_name_simple)}, stack) || + EndsWith({RULE(Type_name_composite)}, stack) || + EndsWith({RULE(Type_name_decimal)}, stack) || (Contains({RULE(Invoke_expr), RULE(Named_expr_list), RULE(Named_expr), @@ -89,7 +94,8 @@ namespace NSQLComplete { RULE(Atom_expr), RULE(Bind_parameter), RULE(An_id_or_type)}, stack) || - EndsWith({RULE(Atom_expr), RULE(Id_or_type)}, stack); + EndsWith({RULE(Atom_expr), RULE(Id_or_type)}, stack) || + EndsWith({RULE(Value_constructor)}, stack); } bool IsLikelyHintStack(const TParserCallStack& stack) { @@ -117,6 +123,10 @@ namespace NSQLComplete { return Contains({RULE(Cluster_expr)}, stack); } + bool IsLikelyColumnStack(const TParserCallStack& stack) { + return Contains({RULE(Result_column)}, stack); + } + bool IsLikelyBindingStack(const TParserCallStack& stack) { return EndsWith({RULE(Bind_parameter), RULE(An_id_or_type)}, stack); } diff --git a/yql/essentials/sql/v1/complete/analysis/local/parser_call_stack.h b/yql/essentials/sql/v1/complete/analysis/local/parser_call_stack.h index c8daf9114fe..a0c479a8be0 100644 --- a/yql/essentials/sql/v1/complete/analysis/local/parser_call_stack.h +++ b/yql/essentials/sql/v1/complete/analysis/local/parser_call_stack.h @@ -21,6 +21,8 @@ namespace NSQLComplete { bool IsLikelyClusterStack(const TParserCallStack& stack); + bool IsLikelyColumnStack(const TParserCallStack& stack); + bool IsLikelyBindingStack(const TParserCallStack& stack); TMaybe<EStatementKind> StatementKindOf(const TParserCallStack& stack); diff --git a/yql/essentials/sql/v1/complete/antlr4/c3i.h b/yql/essentials/sql/v1/complete/antlr4/c3i.h index 52920882ee0..b3fcaab7ae0 100644 --- a/yql/essentials/sql/v1/complete/antlr4/c3i.h +++ b/yql/essentials/sql/v1/complete/antlr4/c3i.h @@ -37,6 +37,8 @@ namespace NSQLComplete { std::unordered_set<TTokenId> IgnoredTokens; std::unordered_set<TRuleId> PreferredRules; std::unordered_set<TRuleId> IgnoredRules; + std::unordered_map<TTokenId, std::unordered_set<TTokenId>> DisabledPreviousByToken; + std::unordered_map<TTokenId, std::unordered_set<TTokenId>> ForcedPreviousByToken; }; virtual ~IC3Engine() = default; diff --git a/yql/essentials/sql/v1/complete/antlr4/c3t.h b/yql/essentials/sql/v1/complete/antlr4/c3t.h index 2b49068ac8f..d1c80867a59 100644 --- a/yql/essentials/sql/v1/complete/antlr4/c3t.h +++ b/yql/essentials/sql/v1/complete/antlr4/c3t.h @@ -13,6 +13,7 @@ #include <util/generic/fwd.h> #include <util/generic/string.h> #include <util/generic/vector.h> +#include <util/generic/yexception.h> namespace NSQLComplete { @@ -26,6 +27,8 @@ namespace NSQLComplete { , Parser_(&Tokens_) , CompletionCore_(&Parser_) , IgnoredRules_(std::move(config.IgnoredRules)) + , DisabledPreviousByToken_(std::move(config.DisabledPreviousByToken)) + , ForcedPreviousByToken_(std::move(config.ForcedPreviousByToken)) { Lexer_.removeErrorListeners(); Parser_.removeErrorListeners(); @@ -36,15 +39,29 @@ namespace NSQLComplete { for (TRuleId rule : IgnoredRules_) { CompletionCore_.preferredRules.emplace(rule); } + + PurifyForcedTokens(); } TC3Candidates Complete(TStringBuf text, size_t caretTokenIndex) override { Assign(text); auto candidates = CompletionCore_.collectCandidates(caretTokenIndex); - return Converted(std::move(candidates)); + return Converted(std::move(candidates), caretTokenIndex); } private: + void PurifyForcedTokens() { + for (auto it = ForcedPreviousByToken_.begin(); it != ForcedPreviousByToken_.end();) { + const auto& [token, previous] = *it; + if (previous.empty()) { + CompletionCore_.ignoredTokens.emplace(token); + it = ForcedPreviousByToken_.erase(it); + } else { + it = std::next(it); + } + } + } + void Assign(TStringBuf prefix) { Chars_.load(prefix.Data(), prefix.Size(), /* lenient = */ false); Lexer_.reset(); @@ -52,10 +69,14 @@ namespace NSQLComplete { Tokens_.fill(); } - TC3Candidates Converted(c3::CandidatesCollection candidates) const { + TC3Candidates Converted(c3::CandidatesCollection candidates, size_t caretTokenIndex) { TC3Candidates converted; for (auto& [token, following] : candidates.tokens) { + if (IsIgnored(token, caretTokenIndex)) { + continue; + } + converted.Tokens.emplace_back(token, std::move(following)); } @@ -71,17 +92,43 @@ namespace NSQLComplete { return converted; } + bool IsIgnored(TTokenId token, size_t caretTokenIndex) { + auto previous = PreviousToken(caretTokenIndex); + + auto disabled = DisabledPreviousByToken_.find(token); + auto forced = ForcedPreviousByToken_.find(token); + + return (disabled != DisabledPreviousByToken_.end() && disabled->second.contains(previous)) || + (forced != ForcedPreviousByToken_.end() && !forced->second.contains(previous)); + } + bool IsIgnored(TRuleId head, const std::vector<TRuleId> tail) const { return IgnoredRules_.contains(head) || AnyOf(tail, [this](TRuleId r) { return IgnoredRules_.contains(r); }); } + TTokenId PreviousToken(size_t caretTokenIndex) { + ssize_t index = static_cast<ssize_t>(caretTokenIndex) - 1; + while (0 <= index && Tokens_.get(index)->getChannel() == antlr4::Token::HIDDEN_CHANNEL) { + --index; + } + + if (index < 0) { + return antlr4::Token::INVALID_TYPE; + } + + return Tokens_.get(index)->getType(); + } + antlr4::ANTLRInputStream Chars_; G::TLexer Lexer_; antlr4::BufferedTokenStream Tokens_; G::TParser Parser_; c3::CodeCompletionCore CompletionCore_; + std::unordered_set<TRuleId> IgnoredRules_; + std::unordered_map<TTokenId, std::unordered_set<TTokenId>> DisabledPreviousByToken_; + std::unordered_map<TTokenId, std::unordered_set<TTokenId>> ForcedPreviousByToken_; }; } // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/core/name.cpp b/yql/essentials/sql/v1/complete/core/name.cpp new file mode 100644 index 00000000000..65c28e51176 --- /dev/null +++ b/yql/essentials/sql/v1/complete/core/name.cpp @@ -0,0 +1,8 @@ +#include "name.h" + +#include <util/stream/output.h> + +template <> +void Out<NSQLComplete::TTableId>(IOutputStream& out, const NSQLComplete::TTableId& value) { + out << value.Cluster << ".`" << value.Path << "`"; +} diff --git a/yql/essentials/sql/v1/complete/core/name.h b/yql/essentials/sql/v1/complete/core/name.h index 02524766de1..3eac47f3871 100644 --- a/yql/essentials/sql/v1/complete/core/name.h +++ b/yql/essentials/sql/v1/complete/core/name.h @@ -1,5 +1,7 @@ #pragma once +#include <util/generic/string.h> + namespace NSQLComplete { enum class EObjectKind { @@ -7,4 +9,11 @@ namespace NSQLComplete { Table, }; + struct TTableId { + TString Cluster; + TString Path; + + friend bool operator==(const TTableId& lhs, const TTableId& rhs) = default; + }; + } // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/core/statement.cpp b/yql/essentials/sql/v1/complete/core/statement.cpp new file mode 100644 index 00000000000..c93e5a7f803 --- /dev/null +++ b/yql/essentials/sql/v1/complete/core/statement.cpp @@ -0,0 +1 @@ +#include "statement.h" diff --git a/yql/essentials/sql/v1/complete/core/ya.make b/yql/essentials/sql/v1/complete/core/ya.make index 599810301ca..0cd0776b11e 100644 --- a/yql/essentials/sql/v1/complete/core/ya.make +++ b/yql/essentials/sql/v1/complete/core/ya.make @@ -3,6 +3,8 @@ LIBRARY() SRCS( environment.cpp input.cpp + name.cpp + statement.cpp ) PEERDIR( diff --git a/yql/essentials/sql/v1/complete/name/object/schema.h b/yql/essentials/sql/v1/complete/name/object/schema.h index b9aa720d36f..d779b293674 100644 --- a/yql/essentials/sql/v1/complete/name/object/schema.h +++ b/yql/essentials/sql/v1/complete/name/object/schema.h @@ -41,12 +41,26 @@ namespace NSQLComplete { TVector<TFolderEntry> Entries; }; + struct TDescribeTableRequest { + TString TableCluster; + TString TablePath; + TString ColumnPrefix; + size_t ColumnsLimit = 128; // TODO: introduce default limit constant + }; + + struct TDescribeTableResponse { + bool IsExisting = false; + TVector<TString> Columns; + }; + class ISchema: public TThrRefBase { public: using TPtr = TIntrusivePtr<ISchema>; - ~ISchema() override = default; virtual NThreading::TFuture<TListResponse> List(const TListRequest& request) const = 0; + + virtual NThreading::TFuture<TDescribeTableResponse> + Describe(const TDescribeTableRequest& request) const = 0; }; } // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/name/object/simple/schema.cpp b/yql/essentials/sql/v1/complete/name/object/simple/schema.cpp index a9dd20ada99..043464bd73e 100644 --- a/yql/essentials/sql/v1/complete/name/object/simple/schema.cpp +++ b/yql/essentials/sql/v1/complete/name/object/simple/schema.cpp @@ -8,7 +8,7 @@ namespace NSQLComplete { class TSimpleSchema: public ISchema { private: - static auto FilterByName(TString name) { + static auto FilterEntriesByName(TString name) { return [name = std::move(name)](auto f) { TVector<TFolderEntry> entries = f.ExtractValue(); EraseIf(entries, [prefix = ToLowerUTF8(name)](const TFolderEntry& entry) { @@ -18,7 +18,7 @@ namespace NSQLComplete { }; } - static auto FilterByTypes(TMaybe<THashSet<TString>> types) { + static auto FilterEntriesByTypes(TMaybe<THashSet<TString>> types) { return [types = std::move(types)](auto f) mutable { TVector<TFolderEntry> entries = f.ExtractValue(); EraseIf(entries, [types = std::move(types)](const TFolderEntry& entry) { @@ -28,7 +28,7 @@ namespace NSQLComplete { }; } - static auto Crop(size_t limit) { + static auto CropEntries(size_t limit) { return [limit](auto f) { TVector<TFolderEntry> entries = f.ExtractValue(); entries.crop(limit); @@ -36,7 +36,7 @@ namespace NSQLComplete { }; } - static auto ToResponse(TStringBuf name) { + static auto ToListResponse(TStringBuf name) { const auto length = name.length(); return [length](auto f) { return TListResponse{ @@ -46,6 +46,38 @@ namespace NSQLComplete { }; } + static auto FilterColumnsByName(TString name) { + return [name = std::move(name)](auto f) { + return f.ExtractValue().Transform([&](auto&& table) { + EraseIf(table.Columns, [prefix = ToLowerUTF8(name)](const TString& name) { + return !name.StartsWith(prefix); + }); + return table; + }); + }; + } + + static auto CropColumns(size_t limit) { + return [limit](auto f) { + return f.ExtractValue().Transform([&](auto&& table) { + table.Columns.crop(limit); + return table; + }); + }; + } + + static auto ToTableDescribeResponse() { + return [](auto f) { + TMaybe<TTableDetails> table = f.ExtractValue(); + return TDescribeTableResponse{ + .IsExisting = table.Defined(), + .Columns = table + .Transform([](auto&& table) { return table.Columns; }) + .GetOrElse({}), + }; + }; + } + public: explicit TSimpleSchema(ISimpleSchema::TPtr simple) : Simple_(std::move(simple)) @@ -55,10 +87,19 @@ namespace NSQLComplete { NThreading::TFuture<TListResponse> List(const TListRequest& request) const override { auto [path, name] = Simple_->Split(request.Path); return Simple_->List(request.Cluster, TString(path)) - .Apply(FilterByName(TString(name))) - .Apply(FilterByTypes(request.Filter.Types)) - .Apply(Crop(request.Limit)) - .Apply(ToResponse(name)); + .Apply(FilterEntriesByName(TString(name))) + .Apply(FilterEntriesByTypes(request.Filter.Types)) + .Apply(CropEntries(request.Limit)) + .Apply(ToListResponse(name)); + } + + NThreading::TFuture<TDescribeTableResponse> + Describe(const TDescribeTableRequest& request) const override { + return Simple_ + ->DescribeTable(request.TableCluster, request.TablePath) + .Apply(FilterColumnsByName(TString(request.ColumnPrefix))) + .Apply(CropColumns(request.ColumnsLimit)) + .Apply(ToTableDescribeResponse()); } private: @@ -77,6 +118,12 @@ namespace NSQLComplete { return List(std::move(folder)); } + NThreading::TFuture<TMaybe<TTableDetails>> + ISimpleSchema::DescribeTable(const TString& cluster, const TString& path) const { + Y_UNUSED(cluster, path); + return NThreading::MakeFuture<TMaybe<TTableDetails>>(Nothing()); + } + ISchema::TPtr MakeSimpleSchema(ISimpleSchema::TPtr simple) { return ISchema::TPtr(new TSimpleSchema(std::move(simple))); } diff --git a/yql/essentials/sql/v1/complete/name/object/simple/schema.h b/yql/essentials/sql/v1/complete/name/object/simple/schema.h index e3b5cb4cb1b..2bdb428b191 100644 --- a/yql/essentials/sql/v1/complete/name/object/simple/schema.h +++ b/yql/essentials/sql/v1/complete/name/object/simple/schema.h @@ -9,6 +9,10 @@ namespace NSQLComplete { TStringBuf NameHint; }; + struct TTableDetails { + TVector<TString> Columns; + }; + class ISimpleSchema: public TThrRefBase { public: using TPtr = TIntrusivePtr<ISimpleSchema>; @@ -22,6 +26,9 @@ namespace NSQLComplete { virtual NThreading::TFuture<TVector<TFolderEntry>> List(TString cluster, TString folder) const; + + virtual NThreading::TFuture<TMaybe<TTableDetails>> + DescribeTable(const TString& cluster, const TString& path) const; }; ISchema::TPtr MakeSimpleSchema(ISimpleSchema::TPtr simple); diff --git a/yql/essentials/sql/v1/complete/name/object/simple/static/schema.cpp b/yql/essentials/sql/v1/complete/name/object/simple/static/schema.cpp index 3482da8a332..7707d8dc4ba 100644 --- a/yql/essentials/sql/v1/complete/name/object/simple/static/schema.cpp +++ b/yql/essentials/sql/v1/complete/name/object/simple/static/schema.cpp @@ -1,20 +1,32 @@ #include "schema.h" +#include <library/cpp/iterator/concatenate.h> + namespace NSQLComplete { namespace { class TSimpleSchema: public ISimpleSchema { public: - explicit TSimpleSchema(THashMap<TString, THashMap<TString, TVector<TFolderEntry>>> data) - : Data_(std::move(data)) + explicit TSimpleSchema( + THashMap<TString, THashMap<TString, TVector<TFolderEntry>>> folders, + THashMap<TString, THashMap<TString, TTableDetails>> tables) + : Folders_(std::move(folders)) + , Tables_(std::move(tables)) { - for (const auto& [_, tables] : Data_) { - for (const auto& [k, _] : tables) { + for (const auto& [_, paths] : Folders_) { + for (const auto& [k, _] : paths) { Y_ENSURE(k.StartsWith("/"), k << " must start with the '/'"); Y_ENSURE(k.EndsWith("/"), k << " must end with the '/'"); } } + + for (const auto& [_, paths] : Tables_) { + for (const auto& [k, _] : paths) { + Y_ENSURE(k.StartsWith("/"), k << " must start with the '/'"); + Y_ENSURE(!k.EndsWith("/"), k << " must not end with the '/'"); + } + } } TSplittedPath Split(TStringBuf path) const override { @@ -37,7 +49,7 @@ namespace NSQLComplete { const THashMap<TString, TVector<TFolderEntry>>* tables = nullptr; const TVector<TFolderEntry>* items = nullptr; - if ((tables = Data_.FindPtr(cluster)) && + if ((tables = Folders_.FindPtr(cluster)) && (items = tables->FindPtr(folder))) { entries = *items; } @@ -45,15 +57,32 @@ namespace NSQLComplete { return NThreading::MakeFuture(std::move(entries)); } + NThreading::TFuture<TMaybe<TTableDetails>> + DescribeTable(const TString& cluster, const TString& path) const override { + auto* tables = Tables_.FindPtr(cluster); + if (tables == nullptr) { + return NThreading::MakeFuture<TMaybe<TTableDetails>>(Nothing()); + } + + auto* details = tables->FindPtr(path); + if (details == nullptr) { + return NThreading::MakeFuture<TMaybe<TTableDetails>>(Nothing()); + } + + return NThreading::MakeFuture<TMaybe<TTableDetails>>(*details); + } + private: - THashMap<TString, THashMap<TString, TVector<TFolderEntry>>> Data_; + THashMap<TString, THashMap<TString, TVector<TFolderEntry>>> Folders_; + THashMap<TString, THashMap<TString, TTableDetails>> Tables_; }; } // namespace ISimpleSchema::TPtr MakeStaticSimpleSchema( - THashMap<TString, THashMap<TString, TVector<TFolderEntry>>> fs) { - return new TSimpleSchema(std::move(fs)); + THashMap<TString, THashMap<TString, TVector<TFolderEntry>>> folders, + THashMap<TString, THashMap<TString, TTableDetails>> tables) { + return new TSimpleSchema(std::move(folders), std::move(tables)); } } // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/name/object/simple/static/schema.h b/yql/essentials/sql/v1/complete/name/object/simple/static/schema.h index 009b433ee4a..57fb322d6e2 100644 --- a/yql/essentials/sql/v1/complete/name/object/simple/static/schema.h +++ b/yql/essentials/sql/v1/complete/name/object/simple/static/schema.h @@ -5,6 +5,7 @@ namespace NSQLComplete { ISimpleSchema::TPtr MakeStaticSimpleSchema( - THashMap<TString, THashMap<TString, TVector<TFolderEntry>>> fs); + THashMap<TString, THashMap<TString, TVector<TFolderEntry>>> folders, + THashMap<TString, THashMap<TString, TTableDetails>> tables = {}); } // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/name/service/name_service.h b/yql/essentials/sql/v1/complete/name/service/name_service.h index 2095cb35335..5139b27dfa3 100644 --- a/yql/essentials/sql/v1/complete/name/service/name_service.h +++ b/yql/essentials/sql/v1/complete/name/service/name_service.h @@ -30,6 +30,14 @@ namespace NSQLComplete { struct TTypeName: TIndentifier { struct TConstraints {}; + + enum class EKind { + Simple, + Container, + Parameterized, + }; + + EKind Kind = EKind::Simple; }; struct TFunctionName: TIndentifier { @@ -58,6 +66,12 @@ namespace NSQLComplete { struct TConstraints: TNamespaced {}; }; + struct TColumnName: TIndentifier { + struct TConstraints { + TVector<TTableId> Tables; + }; + }; + struct TBindingName: TIndentifier { }; @@ -75,6 +89,7 @@ namespace NSQLComplete { TFolderName, TTableName, TClusterName, + TColumnName, TBindingName, TUnkownName>; @@ -85,6 +100,7 @@ namespace NSQLComplete { TMaybe<THintName::TConstraints> Hint; TMaybe<TObjectNameConstraints> Object; TMaybe<TClusterName::TConstraints> Cluster; + TMaybe<TColumnName::TConstraints> Column; bool IsEmpty() const { return !Pragma && @@ -92,7 +108,8 @@ namespace NSQLComplete { !Function && !Hint && !Object && - !Cluster; + !Cluster && + !Column; } TGenericName Qualified(TGenericName unqualified) const; diff --git a/yql/essentials/sql/v1/complete/name/service/ranking/ranking.cpp b/yql/essentials/sql/v1/complete/name/service/ranking/ranking.cpp index 9f30ead37d5..d8e0e0ff1bb 100644 --- a/yql/essentials/sql/v1/complete/name/service/ranking/ranking.cpp +++ b/yql/essentials/sql/v1/complete/name/service/ranking/ranking.cpp @@ -92,7 +92,8 @@ namespace NSQLComplete { } if constexpr (std::is_same_v<T, TFolderName> || - std::is_same_v<T, TTableName>) { + std::is_same_v<T, TTableName> || + std::is_same_v<T, TColumnName>) { return std::numeric_limits<size_t>::max(); } diff --git a/yql/essentials/sql/v1/complete/name/service/schema/name_service.cpp b/yql/essentials/sql/v1/complete/name/service/schema/name_service.cpp index b21ad828317..2970f06d7cd 100644 --- a/yql/essentials/sql/v1/complete/name/service/schema/name_service.cpp +++ b/yql/essentials/sql/v1/complete/name/service/schema/name_service.cpp @@ -12,13 +12,26 @@ namespace NSQLComplete { } NThreading::TFuture<TNameResponse> Lookup(TNameRequest request) const override { - if (!request.Constraints.Object) { - return NThreading::MakeFuture<TNameResponse>({}); + if (request.Constraints.Object) { + return Schema_ + ->List(ToListRequest(std::move(request))) + .Apply(ToListNameResponse); } - return Schema_ - ->List(ToListRequest(std::move(request))) - .Apply(ToNameResponse); + if (request.Constraints.Column) { + Y_ENSURE(request.Constraints.Column->Tables.size() == 1, "Not Implemented"); + TTableId table = request.Constraints.Column->Tables[0]; + return Schema_ + ->Describe({ + .TableCluster = table.Cluster, + .TablePath = table.Path, + .ColumnPrefix = request.Prefix, + .ColumnsLimit = request.Limit, + }) + .Apply(ToDescribeNameResponse); + } + + return NThreading::MakeFuture<TNameResponse>({}); } private: @@ -53,7 +66,7 @@ namespace NSQLComplete { } } - static TNameResponse ToNameResponse(NThreading::TFuture<TListResponse> f) { + static TNameResponse ToListNameResponse(NThreading::TFuture<TListResponse> f) { TListResponse list = f.ExtractValue(); TNameResponse response; @@ -83,6 +96,18 @@ namespace NSQLComplete { return name; } + static TNameResponse ToDescribeNameResponse(NThreading::TFuture<TDescribeTableResponse> f) { + TDescribeTableResponse table = f.ExtractValue(); + + TNameResponse response; + for (TString& column : table.Columns) { + TColumnName name; + name.Indentifier = std::move(column); + response.RankedNames.emplace_back(std::move(name)); + } + return response; + } + ISchema::TPtr Schema_; }; diff --git a/yql/essentials/sql/v1/complete/name/service/static/name_service.cpp b/yql/essentials/sql/v1/complete/name/service/static/name_service.cpp index b294953720f..ca3a66c2b0c 100644 --- a/yql/essentials/sql/v1/complete/name/service/static/name_service.cpp +++ b/yql/essentials/sql/v1/complete/name/service/static/name_service.cpp @@ -20,7 +20,18 @@ namespace NSQLComplete { template <class T, class S = TStringBuf> void AppendAs(TVector<TGenericName>& target, const TVector<S>& source) { for (const auto& element : source) { - target.emplace_back(T{TString(element)}); + T name; + + TString* content = nullptr; + if constexpr (std::is_same_v<TKeyword, T>) { + content = &name.Content; + } else { + content = &name.Indentifier; + } + + *content = element; + + target.emplace_back(std::move(name)); } } @@ -112,24 +123,54 @@ namespace NSQLComplete { public: TTypeNameService(IRanking::TPtr ranking, TVector<TString> types) : IRankingNameService(std::move(ranking)) - , Types_(BuildNameIndex(std::move(types), NormalizeName)) + , SimpleTypes_(BuildNameIndex(std::move(types), NormalizeName)) + , ContainerTypes_(BuildNameIndex( + { + "Optional", + "Tuple", + "Struct", + "Variant", + "List", + "Stream", + "Flow", + "Dict", + "Set", + "Enum", + "Resource", + "Tagged", + "Callable", + }, NormalizeName)) + , ParameterizedTypes_(BuildNameIndex( + { + "Decimal", + }, NormalizeName)) { } NThreading::TFuture<TNameResponse> LookupAllUnranked(TNameRequest request) const override { TNameResponse response; if (request.Constraints.Type) { - NameIndexScan<TTypeName>( - Types_, - request.Prefix, - request.Constraints, - response.RankedNames); + NameIndexScan<TTypeName>(SimpleTypes_, request.Prefix, request.Constraints, response.RankedNames); + + size_t previousSize = response.RankedNames.size(); + NameIndexScan<TTypeName>(ContainerTypes_, request.Prefix, request.Constraints, response.RankedNames); + for (size_t i = previousSize; i < response.RankedNames.size(); ++i) { + std::get<TTypeName>(response.RankedNames[i]).Kind = TTypeName::EKind::Container; + } + + previousSize = response.RankedNames.size(); + NameIndexScan<TTypeName>(ParameterizedTypes_, request.Prefix, request.Constraints, response.RankedNames); + for (size_t i = previousSize; i < response.RankedNames.size(); ++i) { + std::get<TTypeName>(response.RankedNames[i]).Kind = TTypeName::EKind::Parameterized; + } } return NThreading::MakeFuture<TNameResponse>(std::move(response)); } private: - TNameIndex Types_; + TNameIndex SimpleTypes_; + TNameIndex ContainerTypes_; + TNameIndex ParameterizedTypes_; }; class TFunctionNameService: public IRankingNameService { diff --git a/yql/essentials/sql/v1/complete/sql_complete.cpp b/yql/essentials/sql/v1/complete/sql_complete.cpp index 786155912c0..2ed153f3ecc 100644 --- a/yql/essentials/sql/v1/complete/sql_complete.cpp +++ b/yql/essentials/sql/v1/complete/sql_complete.cpp @@ -22,7 +22,11 @@ namespace NSQLComplete { INameService::TPtr names, ISqlCompletionEngine::TConfiguration configuration) : Configuration_(std::move(configuration)) - , SyntaxAnalysis_(MakeLocalSyntaxAnalysis(lexer, Configuration_.IgnoredRules)) + , SyntaxAnalysis_(MakeLocalSyntaxAnalysis( + lexer, + Configuration_.IgnoredRules, + Configuration_.DisabledPreviousByToken, + Configuration_.ForcedPreviousByToken)) , GlobalAnalysis_(MakeGlobalAnalysis()) , Names_(std::move(names)) { @@ -142,6 +146,11 @@ namespace NSQLComplete { object->Kinds.emplace(EObjectKind::Table); } + if (context.Column && global.Column) { + request.Constraints.Column = TColumnName::TConstraints(); + request.Constraints.Column->Tables = std::move(global.Column->Tables); + } + return request; } @@ -190,6 +199,16 @@ namespace NSQLComplete { } if constexpr (std::is_base_of_v<TTypeName, T>) { + switch (name.Kind) { + case TTypeName::EKind::Simple: { + } break; + case TTypeName::EKind::Container: { + name.Indentifier += "<"; + } break; + case TTypeName::EKind::Parameterized: { + name.Indentifier += "("; + } break; + } return {ECandidateKind::TypeName, std::move(name.Indentifier)}; } @@ -221,6 +240,10 @@ namespace NSQLComplete { return {ECandidateKind::ClusterName, std::move(name.Indentifier)}; } + if constexpr (std::is_base_of_v<TColumnName, T>) { + return {ECandidateKind::ColumnName, std::move(name.Indentifier)}; + } + if constexpr (std::is_base_of_v<TBindingName, T>) { if (!context.Binding) { name.Indentifier.prepend('$'); @@ -253,17 +276,17 @@ namespace NSQLComplete { } ISqlCompletionEngine::TConfiguration MakeYDBConfiguration() { - return { - .IgnoredRules = { - "use_stmt", - "import_stmt", - "export_stmt", - }, + ISqlCompletionEngine::TConfiguration config; + config.IgnoredRules = { + "use_stmt", + "import_stmt", + "export_stmt", }; + return config; } ISqlCompletionEngine::TConfiguration MakeYQLConfiguration() { - return MakeConfiguration(/* allowedStmts = */ { + auto config = MakeConfiguration(/* allowedStmts = */ { "lambda_stmt", "pragma_stmt", "select_stmt", @@ -281,6 +304,18 @@ namespace NSQLComplete { "for_stmt", "values_stmt", }); + + config.DisabledPreviousByToken = {}; + + config.ForcedPreviousByToken = { + {"PARALLEL", {}}, + {"TABLESTORE", {}}, + {"FOR", {"EVALUATE"}}, + {"IF", {"EVALUATE"}}, + {"EXTERNAL", {"USING"}}, + }; + + return config; } ISqlCompletionEngine::TPtr MakeSqlCompletionEngine( @@ -323,6 +358,9 @@ void Out<NSQLComplete::ECandidateKind>(IOutputStream& out, NSQLComplete::ECandid case NSQLComplete::ECandidateKind::BindingName: out << "BindingName"; break; + case NSQLComplete::ECandidateKind::ColumnName: + out << "ColumnName"; + break; case NSQLComplete::ECandidateKind::UnknownName: out << "UnknownName"; break; diff --git a/yql/essentials/sql/v1/complete/sql_complete.h b/yql/essentials/sql/v1/complete/sql_complete.h index df5159aff3c..90edceacbc4 100644 --- a/yql/essentials/sql/v1/complete/sql_complete.h +++ b/yql/essentials/sql/v1/complete/sql_complete.h @@ -9,13 +9,14 @@ #include <util/generic/string.h> #include <util/generic/vector.h> +#include <util/generic/hash.h> #include <util/generic/hash_set.h> namespace NSQLComplete { struct TCompletedToken { TStringBuf Content; - size_t SourcePosition; + size_t SourcePosition = 0; }; enum class ECandidateKind { @@ -27,6 +28,7 @@ namespace NSQLComplete { FolderName, TableName, ClusterName, + ColumnName, BindingName, UnknownName, }; @@ -44,18 +46,31 @@ namespace NSQLComplete { TVector<TCandidate> Candidates; }; + // TODO(YQL-19747): Make it thread-safe. class ISqlCompletionEngine { public: using TPtr = THolder<ISqlCompletionEngine>; struct TConfiguration { + friend class TSqlCompletionEngine; + friend ISqlCompletionEngine::TConfiguration MakeYDBConfiguration(); + friend ISqlCompletionEngine::TConfiguration MakeYQLConfiguration(); + friend ISqlCompletionEngine::TConfiguration MakeConfiguration(THashSet<TString> allowedStmts); + + public: size_t Limit = 256; + + private: THashSet<TString> IgnoredRules; + THashMap<TString, THashSet<TString>> DisabledPreviousByToken; + THashMap<TString, THashSet<TString>> ForcedPreviousByToken; }; virtual ~ISqlCompletionEngine() = default; + virtual TCompletion Complete(TCompletionInput input, TEnvironment env = {}) = 0; + virtual NThreading::TFuture<TCompletion> // TODO(YQL-19747): Migrate YDB CLI to `Complete` method CompleteAsync(TCompletionInput input, TEnvironment env = {}) = 0; }; diff --git a/yql/essentials/sql/v1/complete/sql_complete_ut.cpp b/yql/essentials/sql/v1/complete/sql_complete_ut.cpp index 4ecc12e526f..1f29751d017 100644 --- a/yql/essentials/sql/v1/complete/sql_complete_ut.cpp +++ b/yql/essentials/sql/v1/complete/sql_complete_ut.cpp @@ -43,6 +43,7 @@ public: Y_UNIT_TEST_SUITE(SqlCompleteTests) { using ECandidateKind::BindingName; using ECandidateKind::ClusterName; + using ECandidateKind::ColumnName; using ECandidateKind::FolderName; using ECandidateKind::FunctionName; using ECandidateKind::HintName; @@ -83,7 +84,7 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) { }, }; - THashMap<TString, THashMap<TString, TVector<TFolderEntry>>> fs = { + THashMap<TString, THashMap<TString, TVector<TFolderEntry>>> folders = { {"", {{"/", {{"Folder", "local"}, {"Folder", "test"}, {"Folder", "prod"}, @@ -103,15 +104,18 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) { {{"/", {{"Table", "maxim"}}}}}, }; + THashMap<TString, THashMap<TString, TTableDetails>> tables = { + {"example", {{"/people", {{"name", "age"}}}}}}; + auto clustersIt = NFuncTools::Filter( - [](const auto& x) { return !x.empty(); }, IterateKeys(fs)); + [](const auto& x) { return !x.empty(); }, IterateKeys(folders)); TVector<TString> clusters(begin(clustersIt), end(clustersIt)); TFrequencyData frequency; TVector<INameService::TPtr> children = { MakeStaticNameService(std::move(names), frequency), - MakeSchemaNameService(MakeSimpleSchema(MakeStaticSimpleSchema(std::move(fs)))), + MakeSchemaNameService(MakeSimpleSchema(MakeStaticSimpleSchema(std::move(folders), std::move(tables)))), MakeClusterNameService(MakeStaticClusterDiscovery(std::move(clusters))), }; INameService::TPtr service = MakeUnionNameService( @@ -475,37 +479,40 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) { TVector<TCandidate> expected = { {Keyword, "ALL"}, {Keyword, "BITCAST("}, - {Keyword, "CALLABLE"}, {Keyword, "CASE"}, {Keyword, "CAST("}, {Keyword, "CURRENT_DATE"}, {Keyword, "CURRENT_TIME"}, {Keyword, "CURRENT_TIMESTAMP"}, - {Keyword, "Dict<"}, + {TypeName, "Callable<"}, {Keyword, "DISTINCT"}, {FunctionName, "DateTime::Split("}, + {TypeName, "Decimal("}, + {TypeName, "Dict<"}, {Keyword, "EMPTY_ACTION"}, - {Keyword, "ENUM"}, {Keyword, "EXISTS("}, + {TypeName, "Enum<"}, {Keyword, "FALSE"}, - {Keyword, "Flow<"}, + {TypeName, "Flow<"}, {Keyword, "JSON_EXISTS("}, {Keyword, "JSON_QUERY("}, {Keyword, "JSON_VALUE("}, - {Keyword, "List<"}, + {TypeName, "List<"}, {Keyword, "NOT"}, {Keyword, "NULL"}, - {Keyword, "Optional<"}, + {TypeName, "Optional<"}, {FunctionName, "Python::__private("}, - {Keyword, "Resource<"}, - {Keyword, "Set<"}, + {TypeName, "Resource<"}, {Keyword, "STREAM"}, - {Keyword, "STRUCT"}, + {TypeName, "Set<"}, {FunctionName, "StartsWith("}, - {Keyword, "Tagged<"}, + {TypeName, "Stream<"}, + {TypeName, "Struct<"}, {Keyword, "TRUE"}, - {Keyword, "TUPLE"}, - {Keyword, "VARIANT"}, + {TypeName, "Tagged<"}, + {TypeName, "Tuple<"}, + {TypeName, "Uint64"}, + {TypeName, "Variant<"}, }; auto engine = MakeSqlCompletionEngineUT(); @@ -689,8 +696,8 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) { {Keyword, "ANY"}, }; UNIT_ASSERT_VALUES_EQUAL(Complete(engine, R"( - USE example; - DEFINE ACTION $hello() AS + USE example; + DEFINE ACTION $hello() AS USE yt:saurus; SELECT * FROM #; END DEFINE; @@ -721,36 +728,38 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) { Y_UNIT_TEST(SelectWhere) { TVector<TCandidate> expected = { {Keyword, "BITCAST("}, - {Keyword, "CALLABLE"}, {Keyword, "CASE"}, {Keyword, "CAST("}, {Keyword, "CURRENT_DATE"}, {Keyword, "CURRENT_TIME"}, {Keyword, "CURRENT_TIMESTAMP"}, - {Keyword, "Dict<"}, + {TypeName, "Callable<"}, {FunctionName, "DateTime::Split("}, + {TypeName, "Decimal("}, + {TypeName, "Dict<"}, {Keyword, "EMPTY_ACTION"}, - {Keyword, "ENUM"}, {Keyword, "EXISTS("}, + {TypeName, "Enum<"}, {Keyword, "FALSE"}, - {Keyword, "Flow<"}, + {TypeName, "Flow<"}, {Keyword, "JSON_EXISTS("}, {Keyword, "JSON_QUERY("}, {Keyword, "JSON_VALUE("}, - {Keyword, "List<"}, + {TypeName, "List<"}, {Keyword, "NOT"}, {Keyword, "NULL"}, - {Keyword, "Optional<"}, + {TypeName, "Optional<"}, {FunctionName, "Python::__private("}, - {Keyword, "Resource<"}, - {Keyword, "Set<"}, - {Keyword, "Stream<"}, - {Keyword, "STRUCT"}, + {TypeName, "Resource<"}, + {TypeName, "Set<"}, {FunctionName, "StartsWith("}, - {Keyword, "Tagged<"}, + {TypeName, "Stream<"}, + {TypeName, "Struct<"}, {Keyword, "TRUE"}, - {Keyword, "TUPLE"}, - {Keyword, "VARIANT"}, + {TypeName, "Tagged<"}, + {TypeName, "Tuple<"}, + {TypeName, "Uint64"}, + {TypeName, "Variant<"}, }; auto engine = MakeSqlCompletionEngineUT(); @@ -791,21 +800,21 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) { Y_UNIT_TEST(TypeName) { TVector<TCandidate> expected = { - {Keyword, "Callable<("}, - {Keyword, "Decimal("}, - {Keyword, "Dict<"}, - {Keyword, "Enum<"}, - {Keyword, "Flow<"}, - {Keyword, "List<"}, - {Keyword, "Optional<"}, - {Keyword, "Resource<"}, - {Keyword, "Set<"}, - {Keyword, "Stream<"}, - {Keyword, "STRUCT"}, - {Keyword, "Tagged<"}, - {Keyword, "TUPLE"}, + {TypeName, "Callable<"}, + {TypeName, "Decimal("}, + {TypeName, "Dict<"}, + {TypeName, "Enum<"}, + {TypeName, "Flow<"}, + {TypeName, "List<"}, + {TypeName, "Optional<"}, + {TypeName, "Resource<"}, + {TypeName, "Set<"}, + {TypeName, "Stream<"}, + {TypeName, "Struct<"}, + {TypeName, "Tagged<"}, + {TypeName, "Tuple<"}, {TypeName, "Uint64"}, - {Keyword, "Variant<"}, + {TypeName, "Variant<"}, }; auto engine = MakeSqlCompletionEngineUT(); @@ -825,7 +834,7 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) { } { TVector<TCandidate> expected = { - {Keyword, "Optional<"}, + {TypeName, "Optional<"}, }; UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "SELECT Nothing(Option"), expected); } @@ -995,7 +1004,7 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) { $=0; $a=0; $abac=0; - SELECT + SELECT )"; { @@ -1040,6 +1049,31 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) { CompleteTop(1, engine, "SELECT Max(#)").at(0).Kind, FolderName); } + Y_UNIT_TEST(ColumnsAtSimpleSelect) { + auto engine = MakeSqlCompletionEngineUT(); + { + TVector<TCandidate> expected = { + {ColumnName, "age"}, + {ColumnName, "name"}, + }; + UNIT_ASSERT_VALUES_EQUAL(CompleteTop(2, engine, "SELECT # FROM example.`/people`"), expected); + } + { + TVector<TCandidate> expected = { + {ColumnName, "age"}, + {Keyword, "ALL"}, + }; + UNIT_ASSERT_VALUES_EQUAL(CompleteTop(2, engine, "SELECT a# FROM example.`/people`"), expected); + } + { + TVector<TCandidate> expected = { + {ColumnName, "age"}, + {ColumnName, "name"}, + }; + UNIT_ASSERT_VALUES_EQUAL(CompleteTop(2, engine, "USE example; SELECT # FROM `/people`"), expected); + } + } + Y_UNIT_TEST(Typing) { const auto queryUtf16 = TUtf16String::FromUtf8( "SELECT \n" @@ -1065,7 +1099,7 @@ SELECT 123467, \"Hello, {name}! 编码\"}, (1 + (5 * 1 / 0)), MIN(identifier), Bool(field), Math::Sin(var) -FROM `local/test/space/table` +FROM `local/test/space/table` JOIN yt:$cluster_name.test; )"; @@ -1285,6 +1319,24 @@ JOIN yt:$cluster_name.test; UNIT_ASSERT_UNEQUAL(Complete(engine, {"SELE"}).size(), 0); } + Y_UNIT_TEST(IgnoredTokens) { + auto lexer = MakePureLexerSupplier(); + + TNameSet names; + TFrequencyData frequency; + auto service = MakeStaticNameService(names, MakeDefaultRanking(frequency)); + + auto config = MakeYQLConfiguration(); + auto engine = MakeSqlCompletionEngine(lexer, std::move(service), config); + + UNIT_ASSERT(!FindPtr(Complete(engine, {""}), TCandidate{Keyword, "FOR"})); + UNIT_ASSERT(!FindPtr(Complete(engine, {""}), TCandidate{Keyword, "PARALLEL"})); + + UNIT_ASSERT(FindPtr(Complete(engine, {"EVALUATE "}), TCandidate{Keyword, "FOR"})); + UNIT_ASSERT(FindPtr(Complete(engine, {"EVALUATE "}), TCandidate{Keyword, "FOR"})); + UNIT_ASSERT(FindPtr(Complete(engine, {"EVALUATE /**/"}), TCandidate{Keyword, "FOR"})); + } + Y_UNIT_TEST(CachedSchema) { TLexerSupplier lexer = MakePureLexerSupplier(); diff --git a/yql/essentials/sql/v1/complete/syntax/format.cpp b/yql/essentials/sql/v1/complete/syntax/format.cpp index 855c152bed4..3037faced5c 100644 --- a/yql/essentials/sql/v1/complete/syntax/format.cpp +++ b/yql/essentials/sql/v1/complete/syntax/format.cpp @@ -22,23 +22,6 @@ namespace NSQLComplete { return keywords; }(); - const THashSet<TString> TypeConstructors = { - "DECIMAL", - "OPTIONAL", - "TUPLE", - "STRUCT", - "VARIANT", - "LIST", - "STREAM", - "FLOW", - "DICT", - "SET", - "ENUM", - "RESOURCE", - "TAGGED", - "CALLABLE", - }; - } // namespace TString FormatKeywords(const TVector<TString>& seq) { @@ -46,25 +29,14 @@ namespace NSQLComplete { return ""; } - size_t i = 0; - TString text = seq[i++]; - - if (2 <= seq.size() && - TypeConstructors.contains(text) && - (seq[i] == "<" || seq[i] == "(")) { - text.to_title(); - text.append(seq[i]); - i += 1; - } - - for (; i < seq.size(); ++i) { + TString text = seq[0]; + for (size_t i = 1; i < seq.size(); ++i) { const auto& token = seq[i]; if (Keywords.contains(token)) { text += " "; } text += token; } - return text; } diff --git a/yql/essentials/sql/v1/complete/syntax/grammar.cpp b/yql/essentials/sql/v1/complete/syntax/grammar.cpp index 790f272db86..3c0dd579e4f 100644 --- a/yql/essentials/sql/v1/complete/syntax/grammar.cpp +++ b/yql/essentials/sql/v1/complete/syntax/grammar.cpp @@ -34,6 +34,12 @@ namespace NSQLComplete { return Parser_->getRuleNames().at(rule); } + TTokenId GetTokenId(std::string_view symbolized) const override { + TTokenId type = Parser_->getTokenType(symbolized); + Y_ENSURE(type != antlr4::Token::INVALID_TYPE, "Not found " << symbolized); + return type; + } + TRuleId GetRuleId(std::string_view symbolized) const override { TRuleId index = Parser_->getRuleIndex(std::string(symbolized)); if (index == INVALID_INDEX) { diff --git a/yql/essentials/sql/v1/complete/syntax/grammar.h b/yql/essentials/sql/v1/complete/syntax/grammar.h index 29b81a78984..7e6bb398439 100644 --- a/yql/essentials/sql/v1/complete/syntax/grammar.h +++ b/yql/essentials/sql/v1/complete/syntax/grammar.h @@ -22,6 +22,7 @@ namespace NSQLComplete { virtual ~ISqlGrammar() = default; virtual const antlr4::dfa::Vocabulary& GetVocabulary() const = 0; virtual const std::string& SymbolizedRule(TRuleId rule) const = 0; + virtual TTokenId GetTokenId(std::string_view symbolized) const = 0; virtual TRuleId GetRuleId(std::string_view symbolized) const = 0; virtual const std::vector<std::string>& GetAllRules() const = 0; virtual const std::unordered_set<TTokenId>& GetAllTokens() const = 0; diff --git a/yql/essentials/sql/v1/complete/syntax/ya.make b/yql/essentials/sql/v1/complete/syntax/ya.make index 6a35a80375e..f60a05f7b59 100644 --- a/yql/essentials/sql/v1/complete/syntax/ya.make +++ b/yql/essentials/sql/v1/complete/syntax/ya.make @@ -14,6 +14,7 @@ PEERDIR( yql/essentials/sql/settings yql/essentials/sql/v1/lexer yql/essentials/sql/v1/reflect + yql/essentials/sql/v1/complete/antlr4 yql/essentials/sql/v1/complete/core yql/essentials/sql/v1/complete/text ) diff --git a/yql/essentials/tests/common/test_framework/yql_utils.py b/yql/essentials/tests/common/test_framework/yql_utils.py index 685a6ab7278..81a7fa47bae 100644 --- a/yql/essentials/tests/common/test_framework/yql_utils.py +++ b/yql/essentials/tests/common/test_framework/yql_utils.py @@ -57,7 +57,7 @@ def do_custom_error_check(res, sql_query): err_string = custom_error.group(1).strip() assert err_string, 'Expected custom error check in test.\nTest error: %s' % res.std_err log('Custom error: ' + err_string) - assert err_string in res.std_err, '"' + err_string + '" is not found' + assert err_string in res.std_err, '"' + err_string + '" is not found in "' + res.std_err + "'" def get_gateway_cfg_suffix(): diff --git a/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantLiteralOverStructExtraItemsFail.yqls b/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantLiteralOverStructExtraItemsFail.yqls index 5fac5c72de8..31944700c1e 100644 --- a/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantLiteralOverStructExtraItemsFail.yqls +++ b/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantLiteralOverStructExtraItemsFail.yqls @@ -1,4 +1,4 @@ -# /* custom error: Failed to convert type: Variant<'A':Int32,'B':Int32,'C':Int32,'D':Int32> to Variant<'A':Int32,'B':Int32,'C':Int32>*/ +# /* custom error: type diff: Variant<-D:Int32> */ ( (let config (DataSource 'config)) (let res_sink (DataSink 'result)) diff --git a/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantLiteralOverStructNameMismatchFail.yqls b/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantLiteralOverStructNameMismatchFail.yqls index 213e112427d..402063a8387 100644 --- a/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantLiteralOverStructNameMismatchFail.yqls +++ b/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantLiteralOverStructNameMismatchFail.yqls @@ -1,4 +1,4 @@ -# /* custom error:Failed to convert type: Variant<'A':Int32,'B':Int32,'X':Int32> to Variant<'A':Int32,'B':Int32,'C':Int32>*/ +# /* custom error: type diff: Variant<-X:Int32,+C:Int32> */ ( (let config (DataSource 'config)) (let res_sink (DataSink 'result)) diff --git a/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantLiteralOverStructTargetNameMismatchFail.yqls b/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantLiteralOverStructTargetNameMismatchFail.yqls index c84a22312a6..3747824d740 100644 --- a/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantLiteralOverStructTargetNameMismatchFail.yqls +++ b/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantLiteralOverStructTargetNameMismatchFail.yqls @@ -1,4 +1,4 @@ -# /* custom error:Failed to convert type: Variant<'A':Int32,'C':Int32,'X':Int32> to Variant<'A':Int32,'B':Int32,'C':Int32>*/ +# /* custom error: type diff: Variant<-X:Int32,+B:Int32> */ ( (let config (DataSource 'config)) (let res_sink (DataSink 'result)) diff --git a/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantLiteralOverStructTargetTypeMismatchFail.yqls b/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantLiteralOverStructTargetTypeMismatchFail.yqls index cc374209082..b468cd2f4ee 100644 --- a/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantLiteralOverStructTargetTypeMismatchFail.yqls +++ b/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantLiteralOverStructTargetTypeMismatchFail.yqls @@ -1,4 +1,4 @@ -# /* custom error: Failed to convert type: Variant<'A':Int32,'B':Float,'C':Int32> to Variant<'A':Int32,'B':Int32,'C':Int32>*/ +# /* custom error: type diff: Variant<B:Float!=Int32> */ ( (let config (DataSource 'config)) (let res_sink (DataSink 'result)) diff --git a/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantLiteralOverStructTypeMismatchFail.yqls b/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantLiteralOverStructTypeMismatchFail.yqls index 4a3e6a91d3b..8a89730c778 100644 --- a/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantLiteralOverStructTypeMismatchFail.yqls +++ b/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantLiteralOverStructTypeMismatchFail.yqls @@ -1,4 +1,4 @@ -# /* custom error:Failed to convert type: Variant<'A':Int32,'B':Int32,'C':Float> to Variant<'A':Int32,'B':Int32,'C':Int32>*/ +# /* custom error: type diff: Variant<C:Float!=Int32> */ ( (let config (DataSource 'config)) (let res_sink (DataSink 'result)) diff --git a/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantLiteralOverTupleExtraItemsFail.yqls b/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantLiteralOverTupleExtraItemsFail.yqls index e36ad06513a..bcf1aba80e4 100644 --- a/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantLiteralOverTupleExtraItemsFail.yqls +++ b/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantLiteralOverTupleExtraItemsFail.yqls @@ -1,4 +1,4 @@ -# /* custom error: Failed to convert type: Variant<Int32,Int32,Int32,Int32> to Variant<Int32,Int32,Int32>*/ +# /* custom error: type diff: Variant<-Int32> */ ( (let config (DataSource 'config)) (let res_sink (DataSink 'result)) diff --git a/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantLiteralOverTupleNotEnoughItemsFail.yqls b/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantLiteralOverTupleNotEnoughItemsFail.yqls index a44c8483bc2..f8979971dae 100644 --- a/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantLiteralOverTupleNotEnoughItemsFail.yqls +++ b/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantLiteralOverTupleNotEnoughItemsFail.yqls @@ -1,4 +1,4 @@ -# /* custom error: Failed to convert type: Variant<Int32,Int32,Int32> to Variant<Int32,Int32,Int32,Int32>*/ +# /* custom error: type diff: Variant<+Int32> */ ( (let config (DataSource 'config)) (let res_sink (DataSink 'result)) diff --git a/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantLiteralOverTupleTargetTypeMismatchFail.yqls b/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantLiteralOverTupleTargetTypeMismatchFail.yqls index 82b30b4d4bf..e5fb6288e41 100644 --- a/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantLiteralOverTupleTargetTypeMismatchFail.yqls +++ b/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantLiteralOverTupleTargetTypeMismatchFail.yqls @@ -1,4 +1,4 @@ -# /* custom error:Failed to convert type: Variant<Int32,Float,Int32> to Variant<Int32,Int32,Int32>*/ +# /* custom error: type diff: Variant<1:Float!=Int32> */ ( (let config (DataSource 'config)) (let res_sink (DataSink 'result)) diff --git a/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantLiteralOverTupleTypeMismatchFail.yqls b/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantLiteralOverTupleTypeMismatchFail.yqls index f69cfaaab33..282cb998893 100644 --- a/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantLiteralOverTupleTypeMismatchFail.yqls +++ b/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantLiteralOverTupleTypeMismatchFail.yqls @@ -1,4 +1,4 @@ -# /* custom error: Failed to convert type: Variant<Int32,Int32,Float> to Variant<Int32,Int32,Int32>*/ +# /* custom error: type diff: Variant<2:Float!=Int32> */ ( (let config (DataSource 'config)) (let res_sink (DataSink 'result)) diff --git a/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantLiteralUnderlyingTypeMismatchFail.yqls b/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantLiteralUnderlyingTypeMismatchFail.yqls index 2a0ca17bf6f..962c25bffa9 100644 --- a/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantLiteralUnderlyingTypeMismatchFail.yqls +++ b/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantLiteralUnderlyingTypeMismatchFail.yqls @@ -1,4 +1,4 @@ -# /* custom error:Failed to convert type: Variant<Int32,Int32,Int32> to Variant<'A':Int32,'B':Int32,'C':Int32>*/ +# /* custom error: type diff: Variant<Tuple!=Struct> */ ( (let config (DataSource 'config)) (let res_sink (DataSink 'result)) diff --git a/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantOverStructExtraItemsFail.yqls b/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantOverStructExtraItemsFail.yqls index c081180cc73..7182cda623c 100644 --- a/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantOverStructExtraItemsFail.yqls +++ b/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantOverStructExtraItemsFail.yqls @@ -1,4 +1,4 @@ -# /* custom error:Failed to convert type: Variant<'A':Int32,'B':Int32,'C':Int32,'D':Int32> to Variant<'A':Int32,'B':Int32,'C':Int32>*/ +# /* custom error: type diff: Variant<-D:Int32> */ ( (let config (DataSource 'config)) (let res_sink (DataSink 'result)) diff --git a/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantOverStructNameMismatchFail.yqls b/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantOverStructNameMismatchFail.yqls index 6dd91fa2146..1d2a97e4872 100644 --- a/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantOverStructNameMismatchFail.yqls +++ b/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantOverStructNameMismatchFail.yqls @@ -1,4 +1,4 @@ -# /* custom error:Failed to convert type: Variant<'A':Int32,'B':Int32,'X':Int32> to Variant<'A':Int32,'B':Int32,'C':Int32>*/ +# /* custom error: type diff: Variant<-X:Int32,+C:Int32> */ ( (let config (DataSource 'config)) (let res_sink (DataSink 'result)) diff --git a/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantOverStructTargetNameMismatchFail.yqls b/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantOverStructTargetNameMismatchFail.yqls index 254361eaf77..5b7f16b62be 100644 --- a/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantOverStructTargetNameMismatchFail.yqls +++ b/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantOverStructTargetNameMismatchFail.yqls @@ -1,4 +1,4 @@ -# /* custom error:Failed to convert type: Variant<'A':Int32,'C':Int32,'X':Int32> to Variant<'A':Int32,'B':Int32,'C':Int32>*/ +# /* custom error: type diff: Variant<-X:Int32,+B:Int32> */ ( (let config (DataSource 'config)) (let res_sink (DataSink 'result)) diff --git a/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantOverStructTargetTypeMismatchFail.yqls b/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantOverStructTargetTypeMismatchFail.yqls index 29df2cf7804..d25c3a1a732 100644 --- a/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantOverStructTargetTypeMismatchFail.yqls +++ b/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantOverStructTargetTypeMismatchFail.yqls @@ -1,4 +1,4 @@ -# /* custom error:Failed to convert type: Variant<'A':Int32,'B':Float,'C':Int32> to Variant<'A':Int32,'B':Int32,'C':Int32>*/ +# /* custom error: type diff: Variant<B:Float!=Int32> */ ( (let config (DataSource 'config)) (let res_sink (DataSink 'result)) diff --git a/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantOverStructTypeMismatchFail.yqls b/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantOverStructTypeMismatchFail.yqls index c8b6d4d8559..6de4238813f 100644 --- a/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantOverStructTypeMismatchFail.yqls +++ b/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantOverStructTypeMismatchFail.yqls @@ -1,4 +1,4 @@ -# /* custom error: Failed to convert type: Variant<'A':Int32,'B':Int32,'C':Float> to Variant<'A':Int32,'B':Int32,'C':Int32>*/ +# /* custom error: type diff: Variant<C:Float!=Int32> */ ( (let config (DataSource 'config)) (let res_sink (DataSink 'result)) diff --git a/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantOverTupleNotEnoughItemsFail.yqls b/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantOverTupleNotEnoughItemsFail.yqls index dfbefedea26..4ee670f0dc0 100644 --- a/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantOverTupleNotEnoughItemsFail.yqls +++ b/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantOverTupleNotEnoughItemsFail.yqls @@ -1,4 +1,4 @@ -# /* custom error:Failed to convert type: Variant<Int32,Int32,Int32> to Variant<Int32,Int32,Int32,Int32>*/ +# /* custom error: type diff: Variant<+Int32> */ ( (let config (DataSource 'config)) (let res_sink (DataSink 'result)) diff --git a/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantOverTupleTargetTypeMismatchFail.yqls b/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantOverTupleTargetTypeMismatchFail.yqls index 072e25c9729..ffb47e2676e 100644 --- a/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantOverTupleTargetTypeMismatchFail.yqls +++ b/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantOverTupleTargetTypeMismatchFail.yqls @@ -1,4 +1,4 @@ -# /* custom error:Failed to convert type: Variant<Int32,Float,Int32> to Variant<Int32,Int32,Int32>*/ +# /* custom error: type diff: Variant<1:Float!=Int32> */ ( (let config (DataSource 'config)) (let res_sink (DataSink 'result)) diff --git a/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantOverTupleTypeMismatchFail.yqls b/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantOverTupleTypeMismatchFail.yqls index 9818059f1b2..cb32b8ca7b6 100644 --- a/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantOverTupleTypeMismatchFail.yqls +++ b/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantOverTupleTypeMismatchFail.yqls @@ -1,4 +1,4 @@ -# /* custom error:Failed to convert type: Variant<Int32,Int32,Float> to Variant<Int32,Int32,Int32>*/ +# /* custom error: type diff: Variant<2:Float!=Int32> */ ( (let config (DataSource 'config)) (let res_sink (DataSink 'result)) diff --git a/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantUnderlyingTypeMismatchFail.yqls b/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantUnderlyingTypeMismatchFail.yqls index c2834203dca..bf7a6c76194 100644 --- a/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantUnderlyingTypeMismatchFail.yqls +++ b/yql/essentials/tests/s-expressions/suites/ImplictCasts/VariantUnderlyingTypeMismatchFail.yqls @@ -1,4 +1,4 @@ -# /* custom error:Failed to convert type: Variant<Int32,Int32,Int32> to Variant<'A':Int32,'B':Int32,'C':Int32>*/ +# /* custom error: type diff: Variant<Tuple!=Struct> */ ( (let config (DataSource 'config)) (let res_sink (DataSink 'result)) diff --git a/yql/essentials/tests/sql/minirun/part0/canondata/test.test_expr-dict_common_type--Results_/extracted b/yql/essentials/tests/sql/minirun/part0/canondata/test.test_expr-dict_common_type--Results_/extracted index 1dd6413dbea..a1843926b5c 100644 --- a/yql/essentials/tests/sql/minirun/part0/canondata/test.test_expr-dict_common_type--Results_/extracted +++ b/yql/essentials/tests/sql/minirun/part0/canondata/test.test_expr-dict_common_type--Results_/extracted @@ -6,7 +6,7 @@ <tmp_path>/program.sql:<main>:14:8: Warning: At function: AsList select AsList({100u:100}, $d1, $d2, $d3, $d4); ^ - <tmp_path>/program.sql:<main>:6:7: Warning: Failed to convert type: Dict<Int32,Uint32> to Dict<Int32,Int32> + <tmp_path>/program.sql:<main>:6:7: Warning: Failed to convert, type diff: Dict<payload:Uint32!=Int32> $d1 = ToDict([(1, 1u)]); ^ <tmp_path>/program.sql:<main>:6:7: Warning: Consider using explicit CAST or BITCAST to convert from Uint32 to Int32 @@ -18,13 +18,13 @@ <tmp_path>/program.sql:<main>:15:8: Warning: At function: AsList select AsList({100}, $s1, $s2); ^ - <tmp_path>/program.sql:/lib/yql/core.yqls:xxx:yyy: Warning: Failed to convert type: Set<Uint32> to Set<Int32> + <tmp_path>/program.sql:/lib/yql/core.yqls:xxx:yyy: Warning: Failed to convert, type diff: Dict<key:Uint32!=Int32> ^ <tmp_path>/program.sql:/lib/yql/core.yqls:xxx:yyy: Warning: Consider using explicit CAST or BITCAST to convert from Uint32 to Int32 ^ - <tmp_path>/program.sql:<main>:12:7: Warning: Failed to convert type: Set<Uint32> to Set<Int32> + <tmp_path>/program.sql:<main>:12:7: Warning: Failed to convert, type diff: Dict<key:Uint32!=Int32> $s2 = AsSet(2u); ^ <tmp_path>/program.sql:<main>:12:7: Warning: Consider using explicit CAST or BITCAST to convert from Uint32 to Int32 diff --git a/yql/essentials/tools/yql_complete/yql_complete.cpp b/yql/essentials/tools/yql_complete/yql_complete.cpp index 7d9eaded2e6..2a940c4247e 100644 --- a/yql/essentials/tools/yql_complete/yql_complete.cpp +++ b/yql/essentials/tools/yql_complete/yql_complete.cpp @@ -76,28 +76,19 @@ int Run(int argc, char* argv[]) { auto engine = NSQLComplete::MakeSqlCompletionEngine( MakePureLexerSupplier(), NSQLComplete::MakeStaticNameService( - NSQLComplete::LoadDefaultNameSet(), + NSQLComplete::LoadDefaultNameSet(), std::move(frequency))); - NSQLComplete::TCompletionInput input; - - input.Text = queryString; - if (!NYql::IsUtf8(input.Text)) { + if (!NYql::IsUtf8(queryString)) { ythrow yexception() << "provided input is not UTF encoded"; } - if (pos) { - input.CursorPosition = UTF8PositionToBytes(input.Text, *pos); - } else if (Count(input.Text, '#') == 1) { - Cerr << "Note: found an only '#', setting the cursor position\n"; - input.CursorPosition = input.Text.find('#'); - } else if (Count(input.Text, '#') >= 2) { - Cerr << "Note: found multiple '#', defaulting the cursor position\n"; - input.CursorPosition = queryString.size(); - } else { - input.CursorPosition = queryString.size(); + if (auto count = Count(queryString, '#'); 1 < count) { + ythrow yexception() << "provided input contains " << count << " '#', expected 0 or 1"; } + NSQLComplete::TCompletionInput input = NSQLComplete::SharpedInput(queryString); + auto output = engine->CompleteAsync(input).ExtractValueSync(); for (const auto& c : output.Candidates) { Cout << "[" << c.Kind << "] " << c.Content << "\n"; diff --git a/yql/essentials/udfs/common/string/string_udf.cpp b/yql/essentials/udfs/common/string/string_udf.cpp index 5c24485129d..f42fcfd2479 100644 --- a/yql/essentials/udfs/common/string/string_udf.cpp +++ b/yql/essentials/udfs/common/string/string_udf.cpp @@ -185,7 +185,7 @@ SIMPLE_STRICT_UDF_OPTIONS(TReverse, TOptional<char*>(TOptional<char*>), } \ } -#define STRING_ASCII_CMP_IGNORE_CASE_UDF(udfName, function) \ +#define STRING_ASCII_CMP_IGNORE_CASE_UDF(udfName, function, minVersion) \ TUnboxedValuePod udfName##Impl(const TUnboxedValuePod* args) { \ if (args[0]) { \ const TString haystack(args[0].AsStringRef()); \ @@ -215,7 +215,7 @@ SIMPLE_STRICT_UDF_OPTIONS(TReverse, TOptional<char*>(TOptional<char*>), \ BEGIN_SIMPLE_STRICT_ARROW_UDF_OPTIONS(T##udfName, \ bool(TOptional<char*>, char*), \ - builder.SetMinLangVer(NYql::MakeLangVersion(2025, 2))) \ + builder.SetMinLangVer(minVersion)) \ { \ Y_UNUSED(valueBuilder); \ return udfName##Impl(args); \ @@ -439,10 +439,10 @@ SIMPLE_STRICT_UDF_OPTIONS(TReverse, TOptional<char*>(TOptional<char*>), XX(HasPrefixIgnoreCase, AsciiHasPrefixIgnoreCase) \ XX(HasSuffixIgnoreCase, AsciiHasSuffixIgnoreCase) -#define STRING_ASCII_CMP_IGNORE_CASE_UDF_MAP(XX) \ - XX(AsciiStartsWithIgnoreCase, AsciiHasPrefixIgnoreCase) \ - XX(AsciiEndsWithIgnoreCase, AsciiHasSuffixIgnoreCase) \ - XX(AsciiEqualsIgnoreCase, AsciiEqualsIgnoreCase) +#define STRING_ASCII_CMP_IGNORE_CASE_UDF_MAP(XX) \ + XX(AsciiStartsWithIgnoreCase, AsciiHasPrefixIgnoreCase, NYql::MakeLangVersion(2025, 1)) \ + XX(AsciiEndsWithIgnoreCase, AsciiHasSuffixIgnoreCase, NYql::MakeLangVersion(2025, 1)) \ + XX(AsciiEqualsIgnoreCase, AsciiEqualsIgnoreCase, NYql::MakeLangVersion(2025, 2)) // NOTE: The functions below are marked as deprecated, so block implementation // is not required for them. Hence, STROKA_UDF provides only the scalar one at diff --git a/yql/essentials/udfs/common/string/test/canondata/result.json b/yql/essentials/udfs/common/string/test/canondata/result.json index 0feffac589c..a99f95f7db6 100644 --- a/yql/essentials/udfs/common/string/test/canondata/result.json +++ b/yql/essentials/udfs/common/string/test/canondata/result.json @@ -14,6 +14,11 @@ "uri": "file://test.test_AsciiCmpIgnoreCase_2025_02_/extracted" } ], + "test.test[AsciiCmpIgnoreCase_AllAlternatives_2025_01]": [ + { + "uri": "file://test.test_AsciiCmpIgnoreCase_AllAlternatives_2025_01_/results.txt" + } + ], "test.test[AsciiContainsIgnoreCase_2025_01]": [ { "uri": "file://test.test_AsciiContainsIgnoreCase_2025_01_/extracted" diff --git a/yql/essentials/udfs/common/string/test/canondata/test.test_AsciiCmpIgnoreCase_AllAlternatives_2025_01_/results.txt b/yql/essentials/udfs/common/string/test/canondata/test.test_AsciiCmpIgnoreCase_AllAlternatives_2025_01_/results.txt new file mode 100644 index 00000000000..b467eae86b1 --- /dev/null +++ b/yql/essentials/udfs/common/string/test/canondata/test.test_AsciiCmpIgnoreCase_AllAlternatives_2025_01_/results.txt @@ -0,0 +1,121 @@ +[ + { + "Write" = [ + { + "Type" = [ + "ListType"; + [ + "StructType"; + [ + [ + "value"; + [ + "DataType"; + "String" + ] + ]; + [ + "icprefix"; + [ + "DataType"; + "Bool" + ] + ]; + [ + "icstarts"; + [ + "DataType"; + "Bool" + ] + ]; + [ + "aicstarts"; + [ + "DataType"; + "Bool" + ] + ]; + [ + "icsuffix"; + [ + "DataType"; + "Bool" + ] + ]; + [ + "icends"; + [ + "DataType"; + "Bool" + ] + ]; + [ + "aicends"; + [ + "DataType"; + "Bool" + ] + ] + ] + ] + ]; + "Data" = [ + [ + "fdsa"; + %false; + %false; + %false; + %false; + %false; + %false + ]; + [ + "aswedfg"; + %true; + %true; + %true; + %false; + %false; + %false + ]; + [ + "asdadsaasd"; + %true; + %true; + %true; + %false; + %false; + %false + ]; + [ + "gdsfsassas"; + %false; + %false; + %false; + %true; + %true; + %true + ]; + [ + ""; + %false; + %false; + %false; + %false; + %false; + %false + ]; + [ + "`\xD0\x9F\xD1\x80\xD0\xB8\xD0\xB2\xD0\xB5\xD1\x82, \xD0\xBC\xD0\xB8\xD1\x80!`"; + %false; + %false; + %false; + %false; + %false; + %false + ] + ] + } + ] + } +]
\ No newline at end of file diff --git a/yql/essentials/udfs/common/string/test/canondata/test.test_AsciiContainsIgnoreCase_2025_01_/extracted b/yql/essentials/udfs/common/string/test/canondata/test.test_AsciiContainsIgnoreCase_2025_01_/extracted index 6c66d69cd7f..e663ef8331c 100644 --- a/yql/essentials/udfs/common/string/test/canondata/test.test_AsciiContainsIgnoreCase_2025_01_/extracted +++ b/yql/essentials/udfs/common/string/test/canondata/test.test_AsciiContainsIgnoreCase_2025_01_/extracted @@ -25,26 +25,8 @@ SELECT ^ <tmp_path>/program.sql:<main>:10:13: Error: At function: Apply, At function: Udf - String::AsciiStartsWithIgnoreCase(value, "AS") AS icstarts, - ^ - <tmp_path>/program.sql:<main>:10:13: Error: UDF 'String.AsciiStartsWithIgnoreCase' is not available before version 2025.02 - String::AsciiStartsWithIgnoreCase(value, "AS") AS icstarts, - ^ - <tmp_path>/program.sql:<main>:2:1: Error: At function: SqlProjectItem, At lambda - SELECT - ^ - <tmp_path>/program.sql:<main>:12:13: Error: At function: Apply, At function: Udf - String::AsciiEndsWithIgnoreCase(value, "AS") AS icends, - ^ - <tmp_path>/program.sql:<main>:12:13: Error: UDF 'String.AsciiEndsWithIgnoreCase' is not available before version 2025.02 - String::AsciiEndsWithIgnoreCase(value, "AS") AS icends, - ^ - <tmp_path>/program.sql:<main>:2:1: Error: At function: SqlProjectItem, At lambda - SELECT - ^ - <tmp_path>/program.sql:<main>:14:13: Error: At function: Apply, At function: Udf String::AsciiEqualsIgnoreCase(value, "FDSA") AS icequals, ^ - <tmp_path>/program.sql:<main>:14:13: Error: UDF 'String.AsciiEqualsIgnoreCase' is not available before version 2025.02 + <tmp_path>/program.sql:<main>:10:13: Error: UDF 'String.AsciiEqualsIgnoreCase' is not available before version 2025.02 String::AsciiEqualsIgnoreCase(value, "FDSA") AS icequals, ^
\ No newline at end of file diff --git a/yql/essentials/udfs/common/string/test/canondata/test.test_AsciiContainsIgnoreCase_2025_01_scan_/extracted b/yql/essentials/udfs/common/string/test/canondata/test.test_AsciiContainsIgnoreCase_2025_01_scan_/extracted index 6c66d69cd7f..e663ef8331c 100644 --- a/yql/essentials/udfs/common/string/test/canondata/test.test_AsciiContainsIgnoreCase_2025_01_scan_/extracted +++ b/yql/essentials/udfs/common/string/test/canondata/test.test_AsciiContainsIgnoreCase_2025_01_scan_/extracted @@ -25,26 +25,8 @@ SELECT ^ <tmp_path>/program.sql:<main>:10:13: Error: At function: Apply, At function: Udf - String::AsciiStartsWithIgnoreCase(value, "AS") AS icstarts, - ^ - <tmp_path>/program.sql:<main>:10:13: Error: UDF 'String.AsciiStartsWithIgnoreCase' is not available before version 2025.02 - String::AsciiStartsWithIgnoreCase(value, "AS") AS icstarts, - ^ - <tmp_path>/program.sql:<main>:2:1: Error: At function: SqlProjectItem, At lambda - SELECT - ^ - <tmp_path>/program.sql:<main>:12:13: Error: At function: Apply, At function: Udf - String::AsciiEndsWithIgnoreCase(value, "AS") AS icends, - ^ - <tmp_path>/program.sql:<main>:12:13: Error: UDF 'String.AsciiEndsWithIgnoreCase' is not available before version 2025.02 - String::AsciiEndsWithIgnoreCase(value, "AS") AS icends, - ^ - <tmp_path>/program.sql:<main>:2:1: Error: At function: SqlProjectItem, At lambda - SELECT - ^ - <tmp_path>/program.sql:<main>:14:13: Error: At function: Apply, At function: Udf String::AsciiEqualsIgnoreCase(value, "FDSA") AS icequals, ^ - <tmp_path>/program.sql:<main>:14:13: Error: UDF 'String.AsciiEqualsIgnoreCase' is not available before version 2025.02 + <tmp_path>/program.sql:<main>:10:13: Error: UDF 'String.AsciiEqualsIgnoreCase' is not available before version 2025.02 String::AsciiEqualsIgnoreCase(value, "FDSA") AS icequals, ^
\ No newline at end of file diff --git a/yql/essentials/udfs/common/string/test/cases/AsciiCmpIgnoreCase_AllAlternatives_2025_01.cfg b/yql/essentials/udfs/common/string/test/cases/AsciiCmpIgnoreCase_AllAlternatives_2025_01.cfg new file mode 100644 index 00000000000..02e37255758 --- /dev/null +++ b/yql/essentials/udfs/common/string/test/cases/AsciiCmpIgnoreCase_AllAlternatives_2025_01.cfg @@ -0,0 +1,2 @@ +langver 2025.01 +in plato.Input default.in diff --git a/yql/essentials/udfs/common/string/test/cases/AsciiCmpIgnoreCase_AllAlternatives_2025_01.sql b/yql/essentials/udfs/common/string/test/cases/AsciiCmpIgnoreCase_AllAlternatives_2025_01.sql new file mode 100644 index 00000000000..854e414640b --- /dev/null +++ b/yql/essentials/udfs/common/string/test/cases/AsciiCmpIgnoreCase_AllAlternatives_2025_01.sql @@ -0,0 +1,10 @@ +/* syntax version 1 */ +SELECT + value, + String::HasPrefixIgnoreCase(value, "AS") AS icprefix, + String::StartsWithIgnoreCase(value, "AS") AS icstarts, + String::AsciiStartsWithIgnoreCase(value, "AS") AS aicstarts, + String::HasSuffixIgnoreCase(value, "AS") AS icsuffix, + String::EndsWithIgnoreCase(value, "AS") AS icends, + String::AsciiEndsWithIgnoreCase(value, "AS") AS aicends, +FROM Input; diff --git a/yql/essentials/udfs/common/string/test/cases/AsciiContainsIgnoreCase_2025_01.sql b/yql/essentials/udfs/common/string/test/cases/AsciiContainsIgnoreCase_2025_01.sql index db2ebad20e0..f41abd94ec4 100644 --- a/yql/essentials/udfs/common/string/test/cases/AsciiContainsIgnoreCase_2025_01.sql +++ b/yql/essentials/udfs/common/string/test/cases/AsciiContainsIgnoreCase_2025_01.sql @@ -2,8 +2,6 @@ SELECT value, String::AsciiContainsIgnoreCase(value, "AS") AS iccontains, String::AsciiContainsIgnoreCase(value, "") AS icempty, - String::AsciiStartsWithIgnoreCase(value, "AS") AS icstarts, - String::AsciiEndsWithIgnoreCase(value, "AS") AS icends, String::AsciiEqualsIgnoreCase(value, "FDSA") AS icequals, FROM Input; diff --git a/yql/essentials/udfs/common/string/test/cases/AsciiContainsIgnoreCase_2025_01_scan.sql b/yql/essentials/udfs/common/string/test/cases/AsciiContainsIgnoreCase_2025_01_scan.sql index db2ebad20e0..f41abd94ec4 100644 --- a/yql/essentials/udfs/common/string/test/cases/AsciiContainsIgnoreCase_2025_01_scan.sql +++ b/yql/essentials/udfs/common/string/test/cases/AsciiContainsIgnoreCase_2025_01_scan.sql @@ -2,8 +2,6 @@ SELECT value, String::AsciiContainsIgnoreCase(value, "AS") AS iccontains, String::AsciiContainsIgnoreCase(value, "") AS icempty, - String::AsciiStartsWithIgnoreCase(value, "AS") AS icstarts, - String::AsciiEndsWithIgnoreCase(value, "AS") AS icends, String::AsciiEqualsIgnoreCase(value, "FDSA") AS icequals, FROM Input; diff --git a/yql/essentials/utils/backtrace/backtrace_lib.cpp b/yql/essentials/utils/backtrace/backtrace_lib.cpp index c0c7b2c667a..dd2d295dc30 100644 --- a/yql/essentials/utils/backtrace/backtrace_lib.cpp +++ b/yql/essentials/utils/backtrace/backtrace_lib.cpp @@ -33,7 +33,7 @@ namespace { return 0; } #endif - bool comp(const TDllInfo& a, const TDllInfo& b) { + bool Comp(const TDllInfo& a, const TDllInfo& b) { return strcmp(a.Path, b.Path) < 0; } @@ -49,7 +49,7 @@ namespace NYql { memset(&dlInfo, 0, sizeof(dlInfo)); auto ret = dladdr(reinterpret_cast<void*>(addr), &dlInfo); if (ret) { - auto it = std::lower_bound(DLLs, DLLs + DLLCount, std::remove_reference_t<decltype(DLLs[0])> {dlInfo.dli_fname, {}}, comp); + auto it = std::lower_bound(DLLs, DLLs + DLLCount, std::remove_reference_t<decltype(DLLs[0])> {dlInfo.dli_fname, {}}, Comp); if (it != DLLs + DLLCount && !strcmp(it->Path, dlInfo.dli_fname)) { File = it->Path; Address -= it->BaseAddress; @@ -63,7 +63,7 @@ namespace NYql { DLLCount = 0; dl_iterate_phdr(DlIterCallback, &DLLs); #endif - std::stable_sort(DLLs, DLLs + DLLCount, comp); + std::stable_sort(DLLs, DLLs + DLLCount, Comp); size_t cnt = CollectBacktrace(Stack, Limit, data); return CollectFrames(frames, Stack, cnt); } @@ -75,4 +75,5 @@ namespace NYql { return cnt; } } -}
\ No newline at end of file +} + diff --git a/yql/essentials/utils/failure_injector/failure_injector.cpp b/yql/essentials/utils/failure_injector/failure_injector.cpp index 4f4f9c1eb69..dfaff74503a 100644 --- a/yql/essentials/utils/failure_injector/failure_injector.cpp +++ b/yql/essentials/utils/failure_injector/failure_injector.cpp @@ -30,8 +30,8 @@ THashMap<TString, TFailureInjector::TFailureSpec> TFailureInjector::GetCurrentSt THashMap<TString, TFailureInjector::TFailureSpec> TFailureInjector::GetCurrentStateImpl() { THashMap<TString, TFailureInjector::TFailureSpec> copy; - with_lock(Lock) { - copy = FailureSpecs; + with_lock(Lock_) { + copy = FailureSpecs_; } return copy; } @@ -40,8 +40,8 @@ void TFailureInjector::ReachImpl(std::string_view name, std::function<void()> ac if (!Enabled_.load()) { return; } - with_lock(Lock) { - if (auto failureSpec = FailureSpecs.FindPtr(name)) { + with_lock(Lock_) { + if (auto failureSpec = FailureSpecs_.FindPtr(name)) { YQL_LOG(DEBUG) << "TFailureInjector::Reach: " << name << ", Skip=" << failureSpec->Skip << ", Fails=" << failureSpec->CountOfFails; if (failureSpec->Skip > 0) { --failureSpec->Skip; @@ -55,9 +55,9 @@ void TFailureInjector::ReachImpl(std::string_view name, std::function<void()> ac } void TFailureInjector::SetImpl(std::string_view name, ui64 skip, ui64 countOfFails) { - with_lock(Lock) { + with_lock(Lock_) { YQL_ENSURE(countOfFails > 0, "failure " << name << ", 'countOfFails' must be positive"); - FailureSpecs[TString{name}] = TFailureSpec{skip, countOfFails}; + FailureSpecs_[TString{name}] = TFailureSpec{skip, countOfFails}; } } diff --git a/yql/essentials/utils/failure_injector/failure_injector.h b/yql/essentials/utils/failure_injector/failure_injector.h index 6db79fc8c0e..227eb1518f1 100644 --- a/yql/essentials/utils/failure_injector/failure_injector.h +++ b/yql/essentials/utils/failure_injector/failure_injector.h @@ -29,8 +29,8 @@ private: THashMap<TString, TFailureSpec> GetCurrentStateImpl(); std::atomic<bool> Enabled_ = false; - THashMap<TString, TFailureSpec> FailureSpecs; - TMutex Lock; + THashMap<TString, TFailureSpec> FailureSpecs_; + TMutex Lock_; }; } // NYql diff --git a/yql/essentials/utils/fetch/fetch.cpp b/yql/essentials/utils/fetch/fetch.cpp index d35baeb0974..0797e81b2a7 100644 --- a/yql/essentials/utils/fetch/fetch.cpp +++ b/yql/essentials/utils/fetch/fetch.cpp @@ -64,33 +64,33 @@ public: rqs << "\r\n"; } - Socket.Reset(new TSocket(TNetworkAddress(host, port), timeout)); - SocketInput.Reset(new TSocketInput(*Socket)); - SocketOutput.Reset(new TSocketOutput(*Socket)); + Socket_.Reset(new TSocket(TNetworkAddress(host, port), timeout)); + SocketInput_.Reset(new TSocketInput(*Socket_)); + SocketOutput_.Reset(new TSocketOutput(*Socket_)); - Socket->SetSocketTimeout(timeout.Seconds(), timeout.MilliSeconds() % 1000); + Socket_->SetSocketTimeout(timeout.Seconds(), timeout.MilliSeconds() % 1000); if (https) { - Ssl.Reset(new TOpenSslClientIO(SocketInput.Get(), SocketOutput.Get())); + Ssl_.Reset(new TOpenSslClientIO(SocketInput_.Get(), SocketOutput_.Get())); } { - THttpOutput ho(Ssl ? (IOutputStream*)Ssl.Get() : (IOutputStream*)SocketOutput.Get()); + THttpOutput ho(Ssl_ ? (IOutputStream*)Ssl_.Get() : (IOutputStream*)SocketOutput_.Get()); (ho << req).Finish(); } - HttpInput.Reset(new THttpInput(Ssl ? (IInputStream*)Ssl.Get() : (IInputStream*)SocketInput.Get())); + HttpInput_.Reset(new THttpInput(Ssl_ ? (IInputStream*)Ssl_.Get() : (IInputStream*)SocketInput_.Get())); } THttpInput& GetStream() override { - return *HttpInput; + return *HttpInput_; } unsigned GetRetCode() override { - return ParseHttpRetCode(HttpInput->FirstLine()); + return ParseHttpRetCode(HttpInput_->FirstLine()); } THttpURL GetRedirectURL(const THttpURL& baseUrl) override { - for (auto i = HttpInput->Headers().Begin(); i != HttpInput->Headers().End(); ++i) { + for (auto i = HttpInput_->Headers().Begin(); i != HttpInput_->Headers().End(); ++i) { if (0 == TCiString::compare(i->Name(), TStringBuf("location"))) { THttpURL target = ParseURL(i->Value(), THttpURL::FeaturesAll | NUri::TFeature::FeatureConvertHostIDN); if (!target.IsValidAbs()) { @@ -107,11 +107,11 @@ public: } private: - THolder<TSocket> Socket; - THolder<TSocketInput> SocketInput; - THolder<TSocketOutput> SocketOutput; - THolder<TOpenSslClientIO> Ssl; - THolder<THttpInput> HttpInput; + THolder<TSocket> Socket_; + THolder<TSocketInput> SocketInput_; + THolder<TSocketOutput> SocketOutput_; + THolder<TOpenSslClientIO> Ssl_; + THolder<THttpInput> HttpInput_; }; inline bool IsRedirectCode(unsigned code) { diff --git a/yql/essentials/utils/log/log.cpp b/yql/essentials/utils/log/log.cpp index fc63913c750..519b9aabe9e 100644 --- a/yql/essentials/utils/log/log.cpp +++ b/yql/essentials/utils/log/log.cpp @@ -26,9 +26,9 @@ namespace { class TLimitedLogBackend final : public TLogBackend { public: TLimitedLogBackend(TAutoPtr<TLogBackend> b, TAtomic& flag, ui64 limit) noexcept - : Backend(b) - , Flag(flag) - , Limit(limit) + : Backend_(b) + , Flag_(flag) + , Limit_(limit) { } @@ -36,24 +36,24 @@ public: } void ReopenLog() final { - Backend->ReopenLog(); + Backend_->ReopenLog(); } void WriteData(const TLogRecord& rec) final { - const auto remaining = AtomicGet(Limit); - const bool final = remaining > 0 && AtomicSub(Limit, rec.Len) <= 0; + const auto remaining = AtomicGet(Limit_); + const bool final = remaining > 0 && AtomicSub(Limit_, rec.Len) <= 0; if (remaining > 0 || rec.Priority <= TLOG_WARNING) { - Backend->WriteData(rec); + Backend_->WriteData(rec); } if (final) { - AtomicSet(Flag, 1); + AtomicSet(Flag_, 1); } } private: - THolder<TLogBackend> Backend; - TAtomic& Flag; - TAtomic Limit; + THolder<TLogBackend> Backend_; + TAtomic& Flag_; + TAtomic Limit_; }; class TEmergencyLogOutput: public IOutputStream { diff --git a/yql/essentials/utils/multi_resource_lock.h b/yql/essentials/utils/multi_resource_lock.h index 61804fa9a49..c7f4ea49e27 100644 --- a/yql/essentials/utils/multi_resource_lock.h +++ b/yql/essentials/utils/multi_resource_lock.h @@ -18,7 +18,7 @@ private: return RefCount() == 1; } - TMutex Mutex_; + TMutex Mutex; }; public: @@ -29,7 +29,7 @@ public: , ResourceId_(std::move(resourceId)) { Y_ENSURE(Lock_); - Lock_->Mutex_.Acquire(); + Lock_->Mutex.Acquire(); } TResourceLock(TResourceLock&& other) @@ -47,7 +47,7 @@ public: return; } - Lock_->Mutex_.Release(); + Lock_->Mutex.Release(); // decrement ref count before TryCleanup Lock_ = nullptr; Owner_.TryCleanup(ResourceId_); diff --git a/yql/essentials/utils/oom_helper/inject.cpp b/yql/essentials/utils/oom_helper/inject.cpp index 089d1a3b24f..e0b6a2e0c8e 100644 --- a/yql/essentials/utils/oom_helper/inject.cpp +++ b/yql/essentials/utils/oom_helper/inject.cpp @@ -11,7 +11,7 @@ #define UNIT SYSCALL_MMAP2_UNIT #define OFF_MASK ((-0x2000ULL << (8*sizeof(long)-1)) | (UNIT-1)) -void *__mmap(void *start, size_t len, int prot, int flags, int fd, off_t off) +void *Mmap(void *start, size_t len, int prot, int flags, int fd, off_t off) { void* ret = (void*)-1; if (off & OFF_MASK) { @@ -39,7 +39,7 @@ void *__mmap(void *start, size_t len, int prot, int flags, int fd, off_t off) void *mmap(void *start, size_t len, int prot, int flags, int fd, off_t off) { - auto res = __mmap(start, len, prot, flags, fd, off); + auto res = Mmap(start, len, prot, flags, fd, off); if (res == (void*) -1 && errno == ENOMEM) { fprintf(stderr, "mmap failed with ENOMEM\n"); _exit(2); diff --git a/yql/essentials/utils/signals/signals.cpp b/yql/essentials/utils/signals/signals.cpp index 45e199e95f4..bb22e266240 100644 --- a/yql/essentials/utils/signals/signals.cpp +++ b/yql/essentials/utils/signals/signals.cpp @@ -128,8 +128,8 @@ int SetSignalHandler(int signo, void (*handler)(int)) struct TSignalHandlerDesc { - int signo; - void (*handler)(int); + int Signo; + void (*Handler)(int); }; void SetSignalHandlers(const TSignalHandlerDesc* handlerDescs) @@ -137,11 +137,11 @@ void SetSignalHandlers(const TSignalHandlerDesc* handlerDescs) sigset_t interestedSignals; SigEmptySet(&interestedSignals); - for (int i = 0; handlerDescs[i].signo != -1; i++) { - int signo = handlerDescs[i].signo; + for (int i = 0; handlerDescs[i].Signo != -1; i++) { + int signo = handlerDescs[i].Signo; SigAddSet(&interestedSignals, signo); - if (SetSignalHandler(signo, handlerDescs[i].handler) == -1) { + if (SetSignalHandler(signo, handlerDescs[i].Handler) == -1) { ythrow TSystemError() << "Cannot set handler for signal " << strsignal(signo); } diff --git a/yql/essentials/utils/sys/linux_version.h b/yql/essentials/utils/sys/linux_version.h index c8c32da125c..7ae893d257d 100644 --- a/yql/essentials/utils/sys/linux_version.h +++ b/yql/essentials/utils/sys/linux_version.h @@ -9,5 +9,5 @@ namespace NYql { // returns version, patch level std::pair<int, int> DetectLinuxKernelVersion2(); - bool IsLinuxKernelBelow4_3(); + bool IsLinuxKernelBelow4_3(); // NOLINT(readability-identifier-naming) } diff --git a/yql/essentials/utils/url_builder.cpp b/yql/essentials/utils/url_builder.cpp index 31344fbb221..e8a55561e84 100644 --- a/yql/essentials/utils/url_builder.cpp +++ b/yql/essentials/utils/url_builder.cpp @@ -5,12 +5,12 @@ namespace NYql { TUrlBuilder::TUrlBuilder(const TString& uri) - : MainUri(uri) + : MainUri_(uri) { } TUrlBuilder& TUrlBuilder::AddUrlParam(const TString& name, const TString& value) { - Params.emplace_back(TParam {name, value}); + Params_.emplace_back(TParam {name, value}); return *this; } @@ -19,25 +19,25 @@ TUrlBuilder& TUrlBuilder::AddPathComponent(const TString& value) { throw yexception() << "Empty path component is not allowed"; } TStringBuilder res; - res << MainUri; - if (!MainUri.EndsWith('/')) { + res << MainUri_; + if (!MainUri_.EndsWith('/')) { res << '/'; } res << UrlEscapeRet(value, true); - MainUri = std::move(res); + MainUri_ = std::move(res); return *this; } TString TUrlBuilder::Build() const { - if (Params.empty()) { - return MainUri; + if (Params_.empty()) { + return MainUri_; } TStringBuilder res; - res << MainUri << "?"; + res << MainUri_ << "?"; TStringBuf separator = ""sv; - for (const auto& p : Params) { + for (const auto& p : Params_) { res << separator << p.Name; if (p.Value) { res << "=" << CGIEscapeRet(p.Value); diff --git a/yql/essentials/utils/url_builder.h b/yql/essentials/utils/url_builder.h index 774bc3bcd86..c4d4cfb7854 100644 --- a/yql/essentials/utils/url_builder.h +++ b/yql/essentials/utils/url_builder.h @@ -19,8 +19,8 @@ public: TString Build() const; private: - std::vector<TParam> Params; - TString MainUri; + std::vector<TParam> Params_; + TString MainUri_; }; } // NYql diff --git a/yt/yt/client/api/chaos_client.h b/yt/yt/client/api/chaos_client.h new file mode 100644 index 00000000000..c0caafd1fe0 --- /dev/null +++ b/yt/yt/client/api/chaos_client.h @@ -0,0 +1,43 @@ +#pragma once + +#include "prerequisite.h" +#include "prerequisite_client.h" + +#include <yt/yt/client/chaos_client/public.h> + +namespace NYT::NApi { + +//////////////////////////////////////////////////////////////////////////////// + +struct TChaosLeaseStartOptions + : public TPrerequisiteAttachOptions +{ + std::optional<TDuration> LeaseTimeout; + + NChaosClient::TChaosLeaseId ParentId; + + NYTree::IAttributeDictionaryPtr Attributes; +}; + +//////////////////////////////////////////////////////////////////////////////// + +struct TChaosLeaseAttachOptions + : public TPrerequisiteAttachOptions +{ }; + +//////////////////////////////////////////////////////////////////////////////// + +struct IChaosClient +{ + virtual ~IChaosClient() = default; + + virtual TFuture<IPrerequisitePtr> StartChaosLease(const TChaosLeaseStartOptions& options = {}) = 0; + + virtual TFuture<IPrerequisitePtr> AttachChaosLease( + NChaosClient::TChaosLeaseId chaosLeaseId, + const TChaosLeaseAttachOptions& options = {}) = 0; +}; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NApi diff --git a/yt/yt/client/api/chaos_lease_base.cpp b/yt/yt/client/api/chaos_lease_base.cpp new file mode 100644 index 00000000000..8a9db89fc7d --- /dev/null +++ b/yt/yt/client/api/chaos_lease_base.cpp @@ -0,0 +1,136 @@ +#include "chaos_lease_base.h" + +#include "connection.h" +#include "client.h" + +#include <yt/yt/core/rpc/public.h> + +namespace NYT::NApi { + +//////////////////////////////////////////////////////////////////////////////// + +TChaosLeaseBase::TChaosLeaseBase( + IClientPtr client, + NRpc::IChannelPtr channel, + NChaosClient::TChaosLeaseId id, + TDuration timeout, + bool pingAncestors, + std::optional<TDuration> pingPeriod, + const NLogging::TLogger& logger) + : Client_(std::move(client)) + , Channel_(std::move(channel)) + , Id_(id) + , Timeout_(timeout) + , PingAncestors_(pingAncestors) + , PingPeriod_(pingPeriod) + , Logger(logger.WithTag("ChaosLease: %v, %v", + Id_, + Client_->GetConnection()->GetLoggingTag())) +{ } + +NApi::IClientPtr TChaosLeaseBase::GetClient() const +{ + return Client_; +} + +NPrerequisiteClient::TPrerequisiteId TChaosLeaseBase::GetId() const +{ + return Id_; +} + +TDuration TChaosLeaseBase::GetTimeout() const +{ + return Timeout_; +} + +TFuture<void> TChaosLeaseBase::Ping(const TPrerequisitePingOptions& options) +{ + return DoPing(options).Apply( + BIND([=, this, this_ = MakeStrong(this)] (const TErrorOr<void>& resultOrError) { + if (resultOrError.IsOK()) { + YT_LOG_DEBUG("Chaos lease pinged"); + } else if (resultOrError.FindMatching(NChaosClient::EErrorCode::ChaosLeaseNotKnown)) { + // Hard error. + YT_LOG_DEBUG("Chaos lease has expired or was aborted"); + + { + auto guard = Guard(SpinLock_); + if (!Aborted_.IsFired()) { + Aborted_.Fire(resultOrError); + } + } + + THROW_ERROR(resultOrError); + } else { + // Soft error. + YT_LOG_DEBUG(resultOrError, "Error pinging chaos lease"); + + THROW_ERROR_EXCEPTION("Error pinging chaos lease %v", + GetId()) + << resultOrError; + } + })); +} + +TFuture<void> TChaosLeaseBase::Abort(const TPrerequisiteAbortOptions& options) +{ + { + auto guard = Guard(SpinLock_); + if (AbortPromise_) { + return AbortPromise_.ToFuture(); + } + + AbortPromise_ = NewPromise<void>(); + } + + auto chaosLeasePath = Format("#%v", GetId()); + auto removeOptions = TRemoveNodeOptions{ + .Force = options.Force, + }; + return Client_->RemoveNode(chaosLeasePath, removeOptions) + .Apply(BIND([=, this, this_ = MakeStrong(this)] (const TErrorOr<void>& rspOrError) { + { + auto guard = Guard(SpinLock_); + + if (!AbortPromise_) { + YT_LOG_DEBUG(rspOrError, "Chaos lease is no longer aborting, abort response ignored"); + return; + } + + TError abortError; + if (rspOrError.IsOK()) { + YT_LOG_DEBUG("Chaos lease aborted"); + } else { + YT_LOG_DEBUG(rspOrError, "Error aborting chaos lease"); + + abortError = TError("Error aborting chaos lease %v", + GetId()) + << rspOrError; + } + + auto abortPromise = std::exchange(AbortPromise_, TPromise<void>()); + + if (abortError.IsOK() && !Aborted_.IsFired()) { + Aborted_.Fire(TError("Chaos lease aborted by user request")); + } + + guard.Release(); + + abortPromise.Set(std::move(abortError)); + } + })); +} + +void TChaosLeaseBase::SubscribeAborted(const TAbortedHandler& handler) +{ + Aborted_.Subscribe(handler); +} + +void TChaosLeaseBase::UnsubscribeAborted(const TAbortedHandler& handler) +{ + Aborted_.Unsubscribe(handler); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NApi diff --git a/yt/yt/client/api/chaos_lease_base.h b/yt/yt/client/api/chaos_lease_base.h new file mode 100644 index 00000000000..4029a98851b --- /dev/null +++ b/yt/yt/client/api/chaos_lease_base.h @@ -0,0 +1,58 @@ +#pragma once + +#include "private.h" + +#include "prerequisite.h" + +#include <yt/yt/core/rpc/public.h> + +#include <yt/yt/client/api/public.h> + +namespace NYT::NApi { + +//////////////////////////////////////////////////////////////////////////////// + +class TChaosLeaseBase + : public virtual IPrerequisite +{ +public: + TChaosLeaseBase( + IClientPtr client, + NRpc::IChannelPtr channel, + NChaosClient::TChaosLeaseId id, + TDuration timeout, + bool pingAncestors, + std::optional<TDuration> pingPeriod, + const NLogging::TLogger& logger); + + NApi::IClientPtr GetClient() const override; + NPrerequisiteClient::TPrerequisiteId GetId() const override; + TDuration GetTimeout() const override; + + TFuture<void> Ping(const TPrerequisitePingOptions& options = {}) override; + TFuture<void> Abort(const TPrerequisiteAbortOptions& options = {}) override; + + void SubscribeAborted(const TAbortedHandler& handler) override; + void UnsubscribeAborted(const TAbortedHandler& handler) override; + +protected: + const IClientPtr Client_; + const NRpc::IChannelPtr Channel_; + const NChaosClient::TChaosLeaseId Id_; + const TDuration Timeout_; + const bool PingAncestors_; + const std::optional<TDuration> PingPeriod_; + + const NLogging::TLogger Logger; + + YT_DECLARE_SPIN_LOCK(NThreading::TSpinLock, SpinLock_); + TPromise<void> AbortPromise_; + + TSingleShotCallbackList<TAbortedHandlerSignature> Aborted_; + + virtual TFuture<void> DoPing(const TPrerequisitePingOptions& options) = 0; +}; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NApi::NRpcProxy diff --git a/yt/yt/client/api/client.h b/yt/yt/client/api/client.h index 713c524a1d0..97b6c95df3b 100644 --- a/yt/yt/client/api/client.h +++ b/yt/yt/client/api/client.h @@ -3,6 +3,7 @@ #include "accounting_client.h" #include "admin_client.h" #include "connection.h" +#include "chaos_client.h" #include "cypress_client.h" #include "distributed_table_client.h" #include "etc_client.h" @@ -61,6 +62,7 @@ DEFINE_REFCOUNTED_TYPE(IClientBase) */ struct IClient : public virtual IClientBase + , public IChaosClient , public IPrerequisiteClient , public ITransactionClient , public ITableClient diff --git a/yt/yt/client/api/delegating_client.h b/yt/yt/client/api/delegating_client.h index 4aec618c1c7..8ce31e05185 100644 --- a/yt/yt/client/api/delegating_client.h +++ b/yt/yt/client/api/delegating_client.h @@ -331,6 +331,15 @@ public: const TAlterReplicationCardOptions& options), (replicationCardId, options)) + DELEGATE_METHOD(TFuture<IPrerequisitePtr>, StartChaosLease, ( + const TChaosLeaseStartOptions& options), + (options)); + + DELEGATE_METHOD(TFuture<IPrerequisitePtr>, AttachChaosLease, ( + NChaosClient::TChaosLeaseId chaosLeaseId, + const TChaosLeaseAttachOptions& options), + (chaosLeaseId, options)); + DELEGATE_METHOD(TFuture<NYson::TYsonString>, GetTablePivotKeys, ( const NYPath::TYPath& path, const TGetTablePivotKeysOptions& options), diff --git a/yt/yt/client/api/prerequisite.h b/yt/yt/client/api/prerequisite.h index 229cca00f82..684e004081f 100644 --- a/yt/yt/client/api/prerequisite.h +++ b/yt/yt/client/api/prerequisite.h @@ -35,8 +35,6 @@ struct IPrerequisite virtual TFuture<void> Ping(const TPrerequisitePingOptions& options = {}) = 0; virtual TFuture<void> Abort(const TPrerequisiteAbortOptions& options = {}) = 0; - virtual void Detach() = 0; - virtual NPrerequisiteClient::TPrerequisiteId GetId() const = 0; //! Verified dynamic casts to a more specific interface. diff --git a/yt/yt/client/api/prerequisite_client.h b/yt/yt/client/api/prerequisite_client.h index dfe5be31e4c..d190f50fb7d 100644 --- a/yt/yt/client/api/prerequisite_client.h +++ b/yt/yt/client/api/prerequisite_client.h @@ -7,6 +7,7 @@ namespace NYT::NApi { //////////////////////////////////////////////////////////////////////////////// struct TPrerequisiteAttachOptions + : public TTimeoutOptions { bool AutoAbort = false; std::optional<TDuration> PingPeriod; diff --git a/yt/yt/client/api/public.h b/yt/yt/client/api/public.h index 19d31ab9c38..624bf2fcd33 100644 --- a/yt/yt/client/api/public.h +++ b/yt/yt/client/api/public.h @@ -1,5 +1,7 @@ #pragma once +#include <yt/yt/client/chaos_client/public.h> + #include <yt/yt/client/object_client/public.h> #include <yt/yt/client/table_client/public.h> diff --git a/yt/yt/client/api/rpc_proxy/api_service_proxy.h b/yt/yt/client/api/rpc_proxy/api_service_proxy.h index 9a64989204a..611ca495104 100644 --- a/yt/yt/client/api/rpc_proxy/api_service_proxy.h +++ b/yt/yt/client/api/rpc_proxy/api_service_proxy.h @@ -83,6 +83,7 @@ public: // Chaos DEFINE_RPC_PROXY_METHOD(NRpcProxy::NProto, AlterReplicationCard); + DEFINE_RPC_PROXY_METHOD(NRpcProxy::NProto, PingChaosLease); // Queues // COMPAT(nadya73): For compatability with old versions of clients. diff --git a/yt/yt/client/api/rpc_proxy/chaos_lease.cpp b/yt/yt/client/api/rpc_proxy/chaos_lease.cpp new file mode 100644 index 00000000000..46906be8367 --- /dev/null +++ b/yt/yt/client/api/rpc_proxy/chaos_lease.cpp @@ -0,0 +1,75 @@ +#include "chaos_lease.h" + +#include "client_impl.h" +#include "private.h" + +#include <yt/yt/client/api/chaos_lease_base.h> + +#include <yt/yt/core/rpc/public.h> + +namespace NYT::NApi::NRpcProxy { + +//////////////////////////////////////////////////////////////////////////////// + +class TChaosLease + : public TChaosLeaseBase +{ +public: + TChaosLease( + IClientPtr client, + NRpc::IChannelPtr channel, + NChaosClient::TChaosLeaseId chaosLeaseId, + TDuration timeout, + bool pingAncestors, + std::optional<TDuration> pingPeriod) + : TChaosLeaseBase( + std::move(client), + std::move(channel), + chaosLeaseId, + timeout, + pingAncestors, + pingPeriod, + RpcProxyClientLogger()) + , Proxy_(Channel_) + { } + + TFuture<void> DoPing(const TPrerequisitePingOptions& /*options*/) override + { + auto req = Proxy_.PingChaosLease(); + // TODO(gryzlov-ad): Put correct timeout here. + req->SetTimeout(NRpc::DefaultRpcRequestTimeout); + + ToProto(req->mutable_chaos_lease_id(), GetId()); + req->set_ping_ancestors(PingAncestors_); + + return req->Invoke().AsVoid(); + } + +private: + TApiServiceProxy Proxy_; +}; + +//////////////////////////////////////////////////////////////////////////////// + +IPrerequisitePtr CreateChaosLease( + IClientPtr client, + NRpc::IChannelPtr channel, + NChaosClient::TChaosLeaseId chaosLeaseId, + TDuration timeout, + bool pingAncestors, + std::optional<TDuration> pingPeriod) +{ + auto chaosLease = New<TChaosLease>( + std::move(client), + std::move(channel), + chaosLeaseId, + timeout, + pingAncestors, + pingPeriod); + + return chaosLease; +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NApi::NRpcProxy diff --git a/yt/yt/client/api/rpc_proxy/chaos_lease.h b/yt/yt/client/api/rpc_proxy/chaos_lease.h new file mode 100644 index 00000000000..f2d3e43ff69 --- /dev/null +++ b/yt/yt/client/api/rpc_proxy/chaos_lease.h @@ -0,0 +1,19 @@ +#pragma once + +#include <yt/yt/client/api/public.h> + +namespace NYT::NApi::NRpcProxy { + +//////////////////////////////////////////////////////////////////////////////// + +NApi::IPrerequisitePtr CreateChaosLease( + IClientPtr client, + NRpc::IChannelPtr channel, + NChaosClient::TChaosLeaseId id, + TDuration timeout, + bool pingAncestors, + std::optional<TDuration> pingPeriod); + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NApi::NRpcProxy diff --git a/yt/yt/client/api/rpc_proxy/client_impl.cpp b/yt/yt/client/api/rpc_proxy/client_impl.cpp index 1efb024acbe..f1861a953a3 100644 --- a/yt/yt/client/api/rpc_proxy/client_impl.cpp +++ b/yt/yt/client/api/rpc_proxy/client_impl.cpp @@ -1,6 +1,7 @@ #include "client_impl.h" #include "config.h" +#include "chaos_lease.h" #include "helpers.h" #include "private.h" #include "row_batch_reader.h" @@ -27,6 +28,8 @@ #include <yt/yt/client/table_client/unversioned_row.h> #include <yt/yt/client/table_client/wire_protocol.h> +#include <yt/yt/client/object_client/helpers.h> + #include <yt/yt/client/api/distributed_table_session.h> #include <yt/yt/client/ypath/rich.h> @@ -237,6 +240,64 @@ ITransactionPtr TClient::AttachTransaction( "Transaction attached"); } +TFuture<IPrerequisitePtr> TClient::AttachChaosLease( + TChaosLeaseId chaosLeaseId, + const TChaosLeaseAttachOptions& options) +{ + auto connection = GetRpcProxyConnection(); + auto client = GetRpcProxyClient(); + auto channel = GetRetryingChannel(); + + auto chaosLeasePath = Format("#%v/@", chaosLeaseId); + + return client->GetNode(chaosLeasePath, {}).Apply(BIND([=](const TYsonString& value) { + auto attributes = ConvertToAttributes(value); + auto timeoutValue = attributes->Get<i64>("timeout"); + auto timeout = TDuration::MilliSeconds(timeoutValue); + + auto chaosLease = CreateChaosLease( + std::move(client), + std::move(channel), + chaosLeaseId, + timeout, + options.PingAncestors, + options.PingPeriod); + + if (options.Ping) { + return chaosLease->Ping({}).Apply(BIND([=] { + return chaosLease; + })); + } + + return MakeFuture<IPrerequisitePtr>(chaosLease); + })); +} + +TFuture<IPrerequisitePtr> TClient::StartChaosLease(const TChaosLeaseStartOptions& options) +{ + auto connection = GetRpcProxyConnection(); + auto client = GetRpcProxyClient(); + auto channel = GetRetryingChannel(); + + auto createOptions = TCreateNodeOptions{}; + auto timeout = options.LeaseTimeout.value_or(connection->GetConfig()->DefaultChaosLeaseTimeout); + createOptions.Attributes = ConvertToAttributes(BuildYsonStringFluently() + .BeginMap() + .Item("timeout").Value(timeout) + .OptionalItem("parent_id", options.ParentId) + .EndMap()); + + return client->CreateObject(EObjectType::ChaosLease, {}).Apply(BIND([=](const TChaosLeaseId& chaosLeaseId) { + return CreateChaosLease( + std::move(client), + std::move(channel), + chaosLeaseId, + timeout, + options.PingAncestors, + options.PingPeriod); + })); +} + IPrerequisitePtr TClient::AttachPrerequisite( NPrerequisiteClient::TPrerequisiteId prerequisiteId, const TPrerequisiteAttachOptions& options) diff --git a/yt/yt/client/api/rpc_proxy/client_impl.h b/yt/yt/client/api/rpc_proxy/client_impl.h index 27a5df9a2f5..ce13d5cc284 100644 --- a/yt/yt/client/api/rpc_proxy/client_impl.h +++ b/yt/yt/client/api/rpc_proxy/client_impl.h @@ -125,6 +125,7 @@ public: const std::vector<NYPath::TYPath>& movableTables, const NApi::TBalanceTabletCellsOptions& options) override; + // Chaos. TFuture<NChaosClient::TReplicationCardPtr> GetReplicationCard( NChaosClient::TReplicationCardId replicationCardId, const TGetReplicationCardOptions& options = {}) override; @@ -137,6 +138,13 @@ public: NChaosClient::TReplicationCardId replicationCardId, const TAlterReplicationCardOptions& options = {}) override; + TFuture<NApi::IPrerequisitePtr> AttachChaosLease( + NChaosClient::TChaosLeaseId chaosLeaseId, + const TChaosLeaseAttachOptions& options = {}) override; + + TFuture<NApi::IPrerequisitePtr> StartChaosLease( + const TChaosLeaseStartOptions& options = {}) override; + // Distributed table client TFuture<ITableFragmentWriterPtr> CreateTableFragmentWriter( const TSignedWriteFragmentCookiePtr& cookie, diff --git a/yt/yt/client/api/rpc_proxy/config.cpp b/yt/yt/client/api/rpc_proxy/config.cpp index 5d39bec379b..478201122e0 100644 --- a/yt/yt/client/api/rpc_proxy/config.cpp +++ b/yt/yt/client/api/rpc_proxy/config.cpp @@ -84,6 +84,8 @@ void TConnectionConfig::Register(TRegistrar registrar) .Default(TDuration::Minutes(15)); registrar.Parameter("default_streaming_stall_timeout", &TThis::DefaultStreamingStallTimeout) .Default(TDuration::Minutes(1)); + registrar.Parameter("default_chaos_lease_timeout", &TThis::DefaultChaosLeaseTimeout) + .Default(TDuration::Seconds(30)); registrar.Parameter("default_ping_period", &TThis::DefaultPingPeriod) .Default(TDuration::Seconds(5)); diff --git a/yt/yt/client/api/rpc_proxy/config.h b/yt/yt/client/api/rpc_proxy/config.h index bbac7614512..187fbf83231 100644 --- a/yt/yt/client/api/rpc_proxy/config.h +++ b/yt/yt/client/api/rpc_proxy/config.h @@ -58,6 +58,7 @@ struct TConnectionConfig TDuration DefaultTotalStreamingTimeout; TDuration DefaultStreamingStallTimeout; TDuration DefaultPingPeriod; + TDuration DefaultChaosLeaseTimeout; NBus::TBusConfigPtr BusClient; TDuration IdleChannelTtl; diff --git a/yt/yt/client/api/transaction.h b/yt/yt/client/api/transaction.h index 5a20f09c240..debca7ce828 100644 --- a/yt/yt/client/api/transaction.h +++ b/yt/yt/client/api/transaction.h @@ -121,6 +121,8 @@ struct ITransaction virtual TFuture<TTransactionCommitResult> Commit(const TTransactionCommitOptions& options = {}) = 0; + virtual void Detach() = 0; + virtual TFuture<void> Abort(const TPrerequisiteAbortOptions& options = {}) override; virtual TFuture<void> Abort(const TTransactionAbortOptions& options) = 0; diff --git a/yt/yt/client/chaos_client/public.h b/yt/yt/client/chaos_client/public.h index 5c50d3a806e..6af7ebcd06d 100644 --- a/yt/yt/client/chaos_client/public.h +++ b/yt/yt/client/chaos_client/public.h @@ -42,6 +42,7 @@ YT_DEFINE_ERROR_ENUM( ((ChaosCellSuspended) (3202)) ((ReplicationCollocationNotKnown) (3203)) ((ReplicationCollocationIsMigrating) (3204)) + ((ChaosLeaseNotKnown) (3205)) ); //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/client/driver/chaos_commands.cpp b/yt/yt/client/driver/chaos_commands.cpp index 72b67dc19db..a2f41ce37cf 100644 --- a/yt/yt/client/driver/chaos_commands.cpp +++ b/yt/yt/client/driver/chaos_commands.cpp @@ -85,4 +85,24 @@ void TAlterReplicationCardCommand::DoExecute(ICommandContextPtr context) //////////////////////////////////////////////////////////////////////////////// +void TPingChaosLeaseCommand::Register(TRegistrar registrar) +{ + registrar.Parameter("chaos_lease_id", &TThis::ChaosLeaseId); +} + +void TPingChaosLeaseCommand::DoExecute(ICommandContextPtr context) +{ + auto options = TChaosLeaseAttachOptions{}; + options.Ping = true; + options.PingAncestors = true; + + auto future = context->GetClient()->AttachChaosLease(ChaosLeaseId, options); + auto chaosLease = WaitFor(future) + .ValueOrThrow(); + + ProduceEmptyOutput(context); +} + +//////////////////////////////////////////////////////////////////////////////// + } // namespace NYT::NDriver diff --git a/yt/yt/client/driver/chaos_commands.h b/yt/yt/client/driver/chaos_commands.h index ac50bcf643d..dd95ef4658e 100644 --- a/yt/yt/client/driver/chaos_commands.h +++ b/yt/yt/client/driver/chaos_commands.h @@ -42,4 +42,20 @@ private: //////////////////////////////////////////////////////////////////////////////// +class TPingChaosLeaseCommand + : public TTypedCommand<NApi::TPrerequisitePingOptions> +{ +public: + REGISTER_YSON_STRUCT_LITE(TPingChaosLeaseCommand); + + static void Register(TRegistrar registrar); + +private: + NChaosClient::TChaosLeaseId ChaosLeaseId; + + void DoExecute(ICommandContextPtr context) override; +}; + +//////////////////////////////////////////////////////////////////////////////// + } // namespace NYT::NDriver diff --git a/yt/yt/client/driver/driver.cpp b/yt/yt/client/driver/driver.cpp index 56d7d099aac..3a31bf69cd1 100644 --- a/yt/yt/client/driver/driver.cpp +++ b/yt/yt/client/driver/driver.cpp @@ -256,6 +256,7 @@ public: REGISTER (TUpdateChaosTableReplicaProgressCommand, "update_replication_progress", Null, Structured, false, false, ApiVersion4); REGISTER (TAlterReplicationCardCommand, "alter_replication_card", Null, Structured, false, false, ApiVersion4); + REGISTER (TPingChaosLeaseCommand, "ping_chaos_lease", Null, Structured, true, false, ApiVersion4); REGISTER (TMergeCommand, "merge", Null, Structured, true, false, ApiVersion3); REGISTER (TEraseCommand, "erase", Null, Structured, true, false, ApiVersion3); diff --git a/yt/yt/client/federated/client.cpp b/yt/yt/client/federated/client.cpp index 361e1992a4b..da3cb8f0bea 100644 --- a/yt/yt/client/federated/client.cpp +++ b/yt/yt/client/federated/client.cpp @@ -396,6 +396,8 @@ public: UNIMPLEMENTED_METHOD(TFuture<void>, AlterTable, (const NYPath::TYPath&, const TAlterTableOptions&)); UNIMPLEMENTED_METHOD(TFuture<void>, AlterTableReplica, (NTabletClient::TTableReplicaId, const TAlterTableReplicaOptions&)); UNIMPLEMENTED_METHOD(TFuture<void>, AlterReplicationCard, (NChaosClient::TReplicationCardId, const TAlterReplicationCardOptions&)); + UNIMPLEMENTED_METHOD(TFuture<IPrerequisitePtr>, StartChaosLease, (const TChaosLeaseStartOptions&)); + UNIMPLEMENTED_METHOD(TFuture<IPrerequisitePtr>, AttachChaosLease, (NChaosClient::TChaosLeaseId, const TChaosLeaseAttachOptions&)); UNIMPLEMENTED_METHOD(TFuture<std::vector<NTabletClient::TTableReplicaId>>, GetInSyncReplicas, (const NYPath::TYPath&, const NTableClient::TNameTablePtr&, const TSharedRange<NTableClient::TUnversionedRow>&, const TGetInSyncReplicasOptions&)); UNIMPLEMENTED_METHOD(TFuture<std::vector<NTabletClient::TTableReplicaId>>, GetInSyncReplicas, (const NYPath::TYPath&, const TGetInSyncReplicasOptions&)); UNIMPLEMENTED_METHOD(TFuture<TGetTabletErrorsResult>, GetTabletErrors, (const NYPath::TYPath&, const TGetTabletErrorsOptions&)); diff --git a/yt/yt/client/hedging/hedging.cpp b/yt/yt/client/hedging/hedging.cpp index 0dfd4b1f7ae..420326ad4c4 100644 --- a/yt/yt/client/hedging/hedging.cpp +++ b/yt/yt/client/hedging/hedging.cpp @@ -134,6 +134,8 @@ public: UNSUPPORTED_METHOD(TFuture<void>, AlterTable, (const TYPath&, const TAlterTableOptions&)); UNSUPPORTED_METHOD(TFuture<void>, AlterTableReplica, (NTabletClient::TTableReplicaId, const TAlterTableReplicaOptions&)); UNSUPPORTED_METHOD(TFuture<void>, AlterReplicationCard, (NChaosClient::TReplicationCardId, const TAlterReplicationCardOptions&)); + UNSUPPORTED_METHOD(TFuture<IPrerequisitePtr>, StartChaosLease, (const TChaosLeaseStartOptions&)); + UNSUPPORTED_METHOD(TFuture<IPrerequisitePtr>, AttachChaosLease, (NChaosClient::TChaosLeaseId, const TChaosLeaseAttachOptions&)); UNSUPPORTED_METHOD(TFuture<std::vector<NTabletClient::TTableReplicaId>>, GetInSyncReplicas, (const TYPath&, const NTableClient::TNameTablePtr&, const TSharedRange<NTableClient::TUnversionedRow>&, const TGetInSyncReplicasOptions&)); UNSUPPORTED_METHOD(TFuture<std::vector<NTabletClient::TTableReplicaId>>, GetInSyncReplicas, (const TYPath&, const TGetInSyncReplicasOptions&)); UNSUPPORTED_METHOD(TFuture<std::vector<TTabletInfo>>, GetTabletInfos, (const TYPath&, const std::vector<int>&, const TGetTabletInfosOptions&)); diff --git a/yt/yt/client/unittests/mock/client.h b/yt/yt/client/unittests/mock/client.h index 171c9dcba5c..c8009d65816 100644 --- a/yt/yt/client/unittests/mock/client.h +++ b/yt/yt/client/unittests/mock/client.h @@ -913,6 +913,15 @@ public: const TShuffleWriterOptions& options), (override)); + MOCK_METHOD(TFuture<IPrerequisitePtr>, StartChaosLease, ( + const TChaosLeaseStartOptions& options), + (override)); + + MOCK_METHOD(TFuture<IPrerequisitePtr>, AttachChaosLease, ( + NChaosClient::TChaosLeaseId chaosLeaseId, + const TChaosLeaseAttachOptions& options), + (override)); + private: NTabletClient::ITableMountCachePtr TableMountCache_; NTransactionClient::ITimestampProviderPtr TimestampProvider_; diff --git a/yt/yt/client/ya.make b/yt/yt/client/ya.make index 224cc1ba304..9a00da36c67 100644 --- a/yt/yt/client/ya.make +++ b/yt/yt/client/ya.make @@ -8,6 +8,7 @@ PROTO_NAMESPACE(yt) SRCS( api/config.cpp + api/chaos_lease_base.cpp api/client.cpp api/client_common.cpp api/client_cache.cpp @@ -40,6 +41,7 @@ SRCS( api/rpc_proxy/public.cpp api/rpc_proxy/config.cpp api/rpc_proxy/helpers.cpp + api/rpc_proxy/chaos_lease.cpp api/rpc_proxy/client_impl.cpp api/rpc_proxy/client_base.cpp api/rpc_proxy/connection.cpp diff --git a/yt/yt_proto/yt/client/api/rpc_proxy/proto/api_service.proto b/yt/yt_proto/yt/client/api/rpc_proxy/proto/api_service.proto index 9014cbb0963..8f2c755fa10 100644 --- a/yt/yt_proto/yt/client/api/rpc_proxy/proto/api_service.proto +++ b/yt/yt_proto/yt/client/api/rpc_proxy/proto/api_service.proto @@ -2051,6 +2051,17 @@ message TRspAlterReplicationCard //////////////////////////////////////////////////////////////////////////////// +message TReqPingChaosLease +{ + required NYT.NProto.TGuid chaos_lease_id = 1; // NChaosClient::TChaosLeaseId + optional bool ping_ancestors = 2 [default = true]; +} + +message TRspPingChaosLease +{ } + +//////////////////////////////////////////////////////////////////////////////// + message TFetchChunkSpecConfig { optional int32 max_chunk_per_fetch = 1; diff --git a/yt/yt_proto/yt/client/chaos_client/proto/chaos_lease.proto b/yt/yt_proto/yt/client/chaos_client/proto/chaos_lease.proto deleted file mode 100644 index 970c8e7447b..00000000000 --- a/yt/yt_proto/yt/client/chaos_client/proto/chaos_lease.proto +++ /dev/null @@ -1,10 +0,0 @@ -package NYT.NChaosClient.NProto; - -import "yt_proto/yt/core/misc/proto/guid.proto"; - -//////////////////////////////////////////////////////////////////////////////// - -message TChaosLease -{ } - -//////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt_proto/yt/client/ya.make b/yt/yt_proto/yt/client/ya.make index e799449115f..dbf44fcbc89 100644 --- a/yt/yt_proto/yt/client/ya.make +++ b/yt/yt_proto/yt/client/ya.make @@ -19,7 +19,6 @@ SRCS( cell_master/proto/cell_directory.proto chaos_client/proto/replication_card.proto - chaos_client/proto/chaos_lease.proto chunk_client/proto/data_statistics.proto chunk_client/proto/chunk_meta.proto |