aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorAlexander Smirnov <alex@ydb.tech>2025-02-28 00:51:39 +0000
committerAlexander Smirnov <alex@ydb.tech>2025-02-28 00:51:39 +0000
commitb94957647698326b1b92c652d307acc5352e00ca (patch)
treed7ab62877bf300acdf7aeda2dedaffaf3897b86e
parentce94a364f46bc962c4ff261a631ccea8f717ce4d (diff)
parent56146bfad396d685118eaaec9cfff0c0675eae75 (diff)
downloadydb-b94957647698326b1b92c652d307acc5352e00ca.tar.gz
Merge branch 'rightlib' into merge-libs-250228-0050
-rw-r--r--build/conf/a.yaml2
-rw-r--r--build/conf/compilers/gnu_compiler.conf1
-rw-r--r--build/conf/go.conf4
-rw-r--r--build/conf/settings.conf10
-rw-r--r--build/config/tests/cpp_style/configs_validation_rules.json1
-rw-r--r--build/config/tests/py_style/configs_validation_rules.json1
-rw-r--r--build/plugins/a.yaml2
-rw-r--r--build/plugins/lib/test_const/__init__.py11
-rw-r--r--build/scripts/link_o.py2
-rw-r--r--build/sysincl/misc.yml4
-rw-r--r--contrib/libs/antlr4-c3/ya.make2
-rw-r--r--contrib/libs/cxxsupp/libcxxrt/.yandex_meta/__init__.py39
-rw-r--r--contrib/libs/cxxsupp/libcxxrt/.yandex_meta/override.nix13
-rw-r--r--contrib/libs/cxxsupp/libcxxrt/exception.cc24
-rw-r--r--contrib/libs/cxxsupp/libcxxrt/patches/backtrace_recording.patch144
-rw-r--r--contrib/libs/cxxsupp/libcxxrt/patches/cxa_eh_globals_completion_var.patch12
-rw-r--r--contrib/libs/cxxsupp/libcxxrt/patches/cxa_throw_hook.patch26
-rw-r--r--contrib/libs/cxxsupp/libcxxrt/patches/fix_msan_exception_handling.patch26
-rw-r--r--contrib/libs/cxxsupp/libcxxrt/patches/pr41.1-move-trace.patch74
-rw-r--r--contrib/libs/cxxsupp/libcxxrt/patches/pr41.2-fix-trace-format.patch48
-rw-r--r--contrib/libs/cxxsupp/libcxxrt/patches/preallocated_thread_info.patch107
-rw-r--r--contrib/libs/cxxsupp/libcxxrt/patches/remove_unused_code.patch90
-rw-r--r--contrib/libs/cxxsupp/libcxxrt/patches/throw_specs.patch20
-rw-r--r--contrib/libs/cxxsupp/libcxxrt/patches/typeinfo.patch60
-rw-r--r--contrib/python/ydb/py3/ydb/draft/dynamic_config.py4
-rw-r--r--library/cpp/http/simple/http_client.cpp34
-rw-r--r--library/cpp/http/simple/http_client.h37
-rw-r--r--library/cpp/http/simple/ya.make1
-rw-r--r--library/cpp/threading/cancellation/README.md112
-rw-r--r--library/cpp/threading/cancellation/cancellation_token.cpp1
-rw-r--r--library/cpp/threading/cancellation/cancellation_token.h133
-rw-r--r--library/cpp/threading/cancellation/ya.make11
-rw-r--r--util/datetime/systime.cpp33
-rw-r--r--yql/essentials/cfg/tests/gateways.conf5
-rw-r--r--yql/essentials/core/common_opt/yql_flatmap_over_join.cpp396
-rw-r--r--yql/essentials/core/ut/yql_expr_constraint_ut.cpp46
-rw-r--r--yql/essentials/core/yql_expr_constraint.cpp10
-rw-r--r--yql/essentials/minikql/mkql_alloc.cpp2
-rw-r--r--yql/essentials/minikql/mkql_type_ops.cpp26
-rwxr-xr-xyql/essentials/mount/lib/yql/aggregate.yqls4
-rwxr-xr-xyql/essentials/mount/lib/yql/window.yqls4
-rw-r--r--yql/essentials/public/fastcheck/format.cpp12
-rw-r--r--yql/essentials/public/fastcheck/linter_ut.cpp14
-rw-r--r--yql/essentials/public/fastcheck/parser.cpp1
-rw-r--r--yql/essentials/sql/v0/ya.make2
-rw-r--r--yql/essentials/sql/v1/antlr_token.h16
-rw-r--r--yql/essentials/sql/v1/context.cpp7
-rw-r--r--yql/essentials/sql/v1/context.h14
-rw-r--r--yql/essentials/sql/v1/format/sql_format.cpp4
-rw-r--r--yql/essentials/sql/v1/format/sql_format.h3
-rw-r--r--yql/essentials/sql/v1/lexer/lexer.cpp13
-rw-r--r--yql/essentials/sql/v1/lexer/lexer.h6
-rw-r--r--yql/essentials/sql/v1/lexer/ya.make4
-rw-r--r--yql/essentials/sql/v1/proto_parser/proto_parser.cpp21
-rw-r--r--yql/essentials/sql/v1/proto_parser/proto_parser.h10
-rw-r--r--yql/essentials/sql/v1/proto_parser/ya.make7
-rw-r--r--yql/essentials/sql/v1/sql.cpp25
-rw-r--r--yql/essentials/sql/v1/sql.h11
-rw-r--r--yql/essentials/sql/v1/sql_call_expr.cpp2
-rw-r--r--yql/essentials/sql/v1/sql_expression.cpp49
-rw-r--r--yql/essentials/sql/v1/sql_into_tables.cpp2
-rw-r--r--yql/essentials/sql/v1/sql_into_tables.h1
-rw-r--r--yql/essentials/sql/v1/sql_query.cpp63
-rw-r--r--yql/essentials/sql/v1/sql_translation.cpp19
-rw-r--r--yql/essentials/sql/v1/ya.make9
-rw-r--r--yql/essentials/tests/s-expressions/minirun/part9/canondata/result.json14
-rw-r--r--yql/essentials/tests/s-expressions/suites/Optimizers/EmptyOverListWithComplexUniq.yqls33
-rw-r--r--yql/essentials/tests/sql/minirun/part0/canondata/result.json14
-rw-r--r--yql/essentials/tests/sql/minirun/part7/canondata/result.json14
-rw-r--r--yql/essentials/tests/sql/minirun/part8/canondata/result.json12
-rw-r--r--yql/essentials/tests/sql/minirun/part9/canondata/result.json26
-rw-r--r--yql/essentials/tests/sql/sql2yql/canondata/result.json48
-rw-r--r--yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_bigdate-input_interval64_/formatted.sql30
-rw-r--r--yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_datetime-date_in_/formatted.sql30
-rw-r--r--yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_join-eq_over_join_bad_rotate_/formatted.sql94
-rw-r--r--yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_join-eq_over_join_basic_/formatted.sql69
-rw-r--r--yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_join-eq_over_join_same_keys_/formatted.sql35
-rw-r--r--yql/essentials/tests/sql/suites/bigdate/input_interval64.sql7
-rw-r--r--yql/essentials/tests/sql/suites/datetime/date_in.sql7
-rw-r--r--yql/essentials/tests/sql/suites/join/eq_over_join_bad_rotate.sql50
-rw-r--r--yql/essentials/tests/sql/suites/join/eq_over_join_basic.sql30
-rw-r--r--yql/essentials/tests/sql/suites/join/eq_over_join_same_keys.sql18
-rw-r--r--yql/essentials/udfs/common/string/string_udf.cpp1
-rw-r--r--yql/essentials/udfs/common/string/ya.make1
-rw-r--r--yql/essentials/udfs/common/url_base/lib/ya.make2
-rw-r--r--yql/essentials/utils/fetch/ya.make2
-rw-r--r--yql/tools/yqlrun/http/ya.make2
-rw-r--r--yt/cpp/mapreduce/http_client/raw_requests.cpp2
-rw-r--r--yt/cpp/mapreduce/interface/serialize.h13
-rw-r--r--yt/yql/providers/yt/fmr/coordinator/impl/ut/yql_yt_coordinator_ut.cpp14
-rw-r--r--yt/yql/providers/yt/fmr/coordinator/impl/yql_yt_coordinator_impl.cpp2
-rw-r--r--yt/yql/providers/yt/fmr/coordinator/interface/proto_helpers/yql_yt_coordinator_proto_helpers.cpp3
-rw-r--r--yt/yql/providers/yt/fmr/coordinator/interface/yql_yt_coordinator.h1
-rw-r--r--yt/yql/providers/yt/fmr/job/impl/ut/ya.make16
-rw-r--r--yt/yql/providers/yt/fmr/job/impl/ut/yql_yt_job_ut.cpp318
-rw-r--r--yt/yql/providers/yt/fmr/job/impl/ya.make19
-rw-r--r--yt/yql/providers/yt/fmr/job/impl/yql_yt_job_impl.cpp166
-rw-r--r--yt/yql/providers/yt/fmr/job/impl/yql_yt_job_impl.h12
-rw-r--r--yt/yql/providers/yt/fmr/job/interface/ya.make9
-rw-r--r--yt/yql/providers/yt/fmr/job/interface/yql_yt_job.cpp1
-rw-r--r--yt/yql/providers/yt/fmr/job/interface/yql_yt_job.h20
-rw-r--r--yt/yql/providers/yt/fmr/proto/coordinator.proto1
-rw-r--r--yt/yql/providers/yt/fmr/proto/request_options.proto8
-rw-r--r--yt/yql/providers/yt/fmr/request_options/proto_helpers/yql_yt_request_proto_helpers.cpp25
-rw-r--r--yt/yql/providers/yt/fmr/request_options/proto_helpers/yql_yt_request_proto_helpers.h4
-rw-r--r--yt/yql/providers/yt/fmr/request_options/yql_yt_request_options.cpp4
-rw-r--r--yt/yql/providers/yt/fmr/request_options/yql_yt_request_options.h16
-rw-r--r--yt/yql/providers/yt/fmr/table_data_service/interface/table_data_service.cpp1
-rw-r--r--yt/yql/providers/yt/fmr/table_data_service/interface/table_data_service.h20
-rw-r--r--yt/yql/providers/yt/fmr/table_data_service/interface/ya.make13
-rw-r--r--yt/yql/providers/yt/fmr/table_data_service/local/table_data_service.cpp55
-rw-r--r--yt/yql/providers/yt/fmr/table_data_service/local/table_data_service.h12
-rw-r--r--yt/yql/providers/yt/fmr/table_data_service/local/ut/test_table_service.cpp38
-rw-r--r--yt/yql/providers/yt/fmr/table_data_service/local/ut/ya.make14
-rw-r--r--yt/yql/providers/yt/fmr/table_data_service/local/ya.make18
-rw-r--r--yt/yql/providers/yt/fmr/worker/impl/ut/yql_yt_worker_ut.cpp9
-rw-r--r--yt/yql/providers/yt/fmr/yt_service/impl/ya.make16
-rw-r--r--yt/yql/providers/yt/fmr/yt_service/impl/yql_yt_yt_service_impl.cpp186
-rw-r--r--yt/yql/providers/yt/fmr/yt_service/impl/yql_yt_yt_service_impl.h10
-rw-r--r--yt/yql/providers/yt/fmr/yt_service/interface/ya.make13
-rw-r--r--yt/yql/providers/yt/fmr/yt_service/interface/yql_yt_yt_service.cpp1
-rw-r--r--yt/yql/providers/yt/fmr/yt_service/interface/yql_yt_yt_service.h19
-rw-r--r--yt/yql/providers/yt/fmr/yt_service/mock/ya.make17
-rw-r--r--yt/yql/providers/yt/fmr/yt_service/mock/yql_yt_yt_service_mock.h84
-rw-r--r--yt/yql/providers/yt/gateway/file/yql_yt_file.cpp4
-rw-r--r--yt/yql/providers/yt/gateway/fmr/ya.make4
-rw-r--r--yt/yql/providers/yt/gateway/fmr/yql_yt_fmr.cpp94
-rw-r--r--yt/yql/providers/yt/gateway/fmr/yql_yt_fmr.h9
-rw-r--r--yt/yql/providers/yt/gateway/native/yql_yt_native.cpp124
-rw-r--r--yt/yql/providers/yt/gateway/qplayer/yql_yt_qplayer_gateway.cpp4
-rw-r--r--yt/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_merge.cpp20
-rw-r--r--yt/yql/providers/yt/provider/yql_yt_datasink_type_ann.cpp2
-rw-r--r--yt/yql/providers/yt/provider/yql_yt_gateway.h20
-rw-r--r--yt/yql/providers/yt/provider/yql_yt_physical_finalizing.cpp7
-rw-r--r--yt/yql/tests/sql/suites/ql_filter/integer_bounds.sql4
-rw-r--r--yt/yql/tests/sql/suites/ql_filter/integer_escaping.sql4
-rw-r--r--yt/yql/tests/sql/suites/ql_filter/integer_many_left.sql4
-rw-r--r--yt/yql/tests/sql/suites/ql_filter/integer_many_noskiff.cfg1
-rw-r--r--yt/yql/tests/sql/suites/ql_filter/integer_many_noskiff.sql13
-rw-r--r--yt/yql/tests/sql/suites/ql_filter/integer_many_right.sql4
-rw-r--r--yt/yql/tests/sql/suites/ql_filter/integer_select_other.sql4
-rw-r--r--yt/yql/tools/ytrun/lib/ya.make3
-rw-r--r--yt/yql/tools/ytrun/lib/ytrun_lib.cpp16
-rw-r--r--yt/yt/client/api/rpc_proxy/row_batch_reader.cpp2
-rw-r--r--yt/yt/client/api/rpc_proxy/wire_row_stream.cpp15
-rw-r--r--yt/yt/client/api/rpc_proxy/wire_row_stream.h5
-rw-r--r--yt/yt/client/driver/scheduler_commands.cpp1
-rw-r--r--yt/yt/client/object_client/public.h1
-rw-r--r--yt/yt/client/table_client/wire_protocol.cpp11
-rw-r--r--yt/yt/client/table_client/wire_protocol.h2
150 files changed, 3541 insertions, 453 deletions
diff --git a/build/conf/a.yaml b/build/conf/a.yaml
index cde8d7dc68..fff0bfe189 100644
--- a/build/conf/a.yaml
+++ b/build/conf/a.yaml
@@ -7,7 +7,7 @@ arcanum:
requirements:
- system: arcanum
type: merge_time_fits
- alias: Merge is forbidden from 11:00 to 20:00 MSK (YMAKE-1194)
+ alias: build/conf commits invalidate autocheck dependency caches for ALL and thus are forbidden from 11:00 to 20:00 MSK
data:
merge_intervals_utc:
# 20:00..11:00 MSK
diff --git a/build/conf/compilers/gnu_compiler.conf b/build/conf/compilers/gnu_compiler.conf
index a333438668..1c0b9df04d 100644
--- a/build/conf/compilers/gnu_compiler.conf
+++ b/build/conf/compilers/gnu_compiler.conf
@@ -99,6 +99,7 @@ when ($ARCH_XTENSA == "yes") {
when ($OS_EMSCRIPTEN == "yes") {
FSTACK=-fno-stack-protector
CFLAGS+=-D__EMSCRIPTEN__
+ CFLAGS+=-DSTANDALONE_WASM=1
}
CFLAGS+=$_C_FLAGS $DEBUG_INFO_FLAGS $_C_FOPTIONS $C_WARNING_OPTS $GCC_PREPROCESSOR_OPTS $USER_CFLAGS $USER_CFLAGS_GLOBAL
diff --git a/build/conf/go.conf b/build/conf/go.conf
index 1b1449b631..83b03e753c 100644
--- a/build/conf/go.conf
+++ b/build/conf/go.conf
@@ -213,12 +213,12 @@ macro _GO_COMPILE_SYMABIS(FLAGS[], ASM_FILES...) {
# tag:go-specific
macro _GO_COMPILE_CGO1(NAME, FLAGS[], FILES...) {
- .CMD=${hide:_CGO_FAKEID} ${cwd:ARCADIA_ROOT} $YMAKE_PYTHON ${input:"build/scripts/cgo1_wrapper.py"} $_GO_CGO1_WRAPPER_FLAGS --build-root ${ARCADIA_BUILD_ROOT} --source-root ${ARCADIA_ROOT} --cgo1-files ${output;noext:FILES.cgo1.go} --cgo2-files ${noauto;output;noext:FILES.cgo2.c} -- ${GO_TOOLS_ROOT}/pkg/tool/$_GO_TC_PATH/cgo -objdir $BINDIR -importpath $NAME $GO_CGO1_FLAGS_VALUE $FLAGS -- $C_FLAGS_PLATFORM ${pre=-I:_C__INCLUDE} ${CGO_CFLAGS_VALUE} ${input:FILES} ${hide;output:"_cgo_export.h"} ${hide;output:"_cgo_export.c"} ${hide;output:"_cgo_gotypes.go"} ${hide;noauto;output:"_cgo_main.c"} $GO_TOOLCHAIN_ENV ${hide;kv:"p go"} ${hide;kv:"pc light-blue"} ${hide;kv:"show_out"}
+ .CMD=${hide:_CGO_FAKEID} ${cwd:ARCADIA_ROOT} $YMAKE_PYTHON3 ${input:"build/scripts/cgo1_wrapper.py"} $_GO_CGO1_WRAPPER_FLAGS --build-root ${ARCADIA_BUILD_ROOT} --source-root ${ARCADIA_ROOT} --cgo1-files ${output;noext:FILES.cgo1.go} --cgo2-files ${noauto;output;noext:FILES.cgo2.c} -- ${GO_TOOLS_ROOT}/pkg/tool/$_GO_TC_PATH/cgo -objdir $BINDIR -importpath $NAME $GO_CGO1_FLAGS_VALUE $FLAGS -- $C_FLAGS_PLATFORM ${pre=-I:_C__INCLUDE} ${CGO_CFLAGS_VALUE} ${input:FILES} ${hide;output:"_cgo_export.h"} ${hide;output:"_cgo_export.c"} ${hide;output:"_cgo_gotypes.go"} ${hide;noauto;output:"_cgo_main.c"} $GO_TOOLCHAIN_ENV ${hide;kv:"p go"} ${hide;kv:"pc light-blue"} ${hide;kv:"show_out"}
}
# tag:go-specific
macro _GO_COMPILE_CGO2(NAME, C_FILES[], S_FILES[], OBJ_FILES[], FILES...) {
- .CMD=${hide:_CGO_FAKEID} $C_COMPILER $C_FLAGS_PLATFORM ${pre=-I:_C__INCLUDE} $CGO_CFLAGS_VALUE ${input;tobindir:"_cgo_main.c"} -c -o ${tmp;noauto;suf=${OBJECT_SUF}:"_cgo_main.c"} && $YMAKE_PYTHON ${input:"build/scripts/link_o.py"} $C_COMPILER $C_FLAGS_PLATFORM ${pre=-I:_C__INCLUDE} -o ${tmp;noauto;suf=${OBJECT_SUF}:"_cgo_"} $LDFLAGS $LDFLAGS_GLOBAL $CGO2_LDFLAGS_VALUE ${hide;input:"_cgo_export.h"} ${tmp;noauto;suf=${OBJECT_SUF}:"_cgo_main.c"} ${input;suf=${OBJECT_SUF}:"_cgo_export.c"} ${input;nopath;noext;suf=.cgo2.c${OBJECT_SUF}:FILES} ${input;suf=${OBJECT_SUF}:C_FILES} ${input;suf=.o:S_FILES} ${input:OBJ_FILES} $CGO_LDFLAGS_VALUE && ${GO_TOOLS_ROOT}/pkg/tool/$_GO_TC_PATH/cgo -dynpackage $NAME -dynimport ${tmp;noauto;suf=${OBJECT_SUF}:"_cgo_"} -dynout ${output:"_cgo_import.go"} -dynlinker $GO_CGO2_FLAGS_VALUE $GO_TOOLCHAIN_ENV ${hide;kv:"p go"} ${hide;kv:"pc light-blue"} ${hide;kv:"show_out"}
+ .CMD=${hide:_CGO_FAKEID} $C_COMPILER $C_FLAGS_PLATFORM ${pre=-I:_C__INCLUDE} $CGO_CFLAGS_VALUE ${input;tobindir:"_cgo_main.c"} -c -o ${tmp;noauto;suf=${OBJECT_SUF}:"_cgo_main.c"} && $YMAKE_PYTHON3 ${input:"build/scripts/link_o.py"} $C_COMPILER $C_FLAGS_PLATFORM ${pre=-I:_C__INCLUDE} -o ${tmp;noauto;suf=${OBJECT_SUF}:"_cgo_"} $LDFLAGS $LDFLAGS_GLOBAL $CGO2_LDFLAGS_VALUE ${hide;input:"_cgo_export.h"} ${tmp;noauto;suf=${OBJECT_SUF}:"_cgo_main.c"} ${input;suf=${OBJECT_SUF}:"_cgo_export.c"} ${input;nopath;noext;suf=.cgo2.c${OBJECT_SUF}:FILES} ${input;suf=${OBJECT_SUF}:C_FILES} ${input;suf=.o:S_FILES} ${input:OBJ_FILES} $CGO_LDFLAGS_VALUE && ${GO_TOOLS_ROOT}/pkg/tool/$_GO_TC_PATH/cgo -dynpackage $NAME -dynimport ${tmp;noauto;suf=${OBJECT_SUF}:"_cgo_"} -dynout ${output:"_cgo_import.go"} -dynlinker $GO_CGO2_FLAGS_VALUE $GO_TOOLCHAIN_ENV ${hide;kv:"p go"} ${hide;kv:"pc light-blue"} ${hide;kv:"show_out"}
_USE_LINKER()
}
diff --git a/build/conf/settings.conf b/build/conf/settings.conf
index 23186f3477..3f0ba1ef70 100644
--- a/build/conf/settings.conf
+++ b/build/conf/settings.conf
@@ -1,13 +1,8 @@
-GLOBAL_SRCS_IN_RESULTS=yes
JSON_CACHE_IS_ATTACHED=yes
-USE_ADDINCL_WITHOUT_FOR_AS_ADDINCL_FOR_C=yes
LANGS_REQUIRE_BUILD_AND_SRC_ROOTS=c asm cython proto flatc swig ydl nlg
CHECK_GO_INCORRECT_DEPS=yes
-REPORT_RECURSE_NO_YAMAKE=yes
USE_PREBUILT_TOOLS=yes
-RESOLVE_OUTPUT_INCLUDES_AS_INCLUDES=yes
RESOLVE_FORCE_LISTDIR=no
-CHKPEERS_GLOBAL_SRCS=yes
ENABLE_NODE_SELF_UID=yes
REPORT_CONFIGURE_PROGRESS=yes
FORCE_RESOLVE_MACRO_INCLS=yes
@@ -15,17 +10,12 @@ ENABLE_RERESOLVE_FOR_GENERATED_FILES=yes
REPORT_ALL_DUPSRC=yes
DEPS_CACHE_CONTROL_UIDS_CACHE=yes
USE_GLOBAL_CMD=yes
-YMAKE_USE_NEW_UIDS=yes
PIC_NO_PIE=no
FAIL_PY2=no
MAIN_OUTPUT_AS_EXTRA=yes
USE_REACHABILITY_TO_REPORT_CONF_ERRORS = yes
DISABLE_ATD=yes
-when ($YMAKE_USE_OLD_UIDS == "yes") {
- YMAKE_USE_NEW_UIDS=no
-}
-
when ($OS_WINDOWS == "yes") {
USE_GLOBAL_CMD=no
}
diff --git a/build/config/tests/cpp_style/configs_validation_rules.json b/build/config/tests/cpp_style/configs_validation_rules.json
new file mode 100644
index 0000000000..0967ef424b
--- /dev/null
+++ b/build/config/tests/cpp_style/configs_validation_rules.json
@@ -0,0 +1 @@
+{}
diff --git a/build/config/tests/py_style/configs_validation_rules.json b/build/config/tests/py_style/configs_validation_rules.json
new file mode 100644
index 0000000000..0967ef424b
--- /dev/null
+++ b/build/config/tests/py_style/configs_validation_rules.json
@@ -0,0 +1 @@
+{}
diff --git a/build/plugins/a.yaml b/build/plugins/a.yaml
index 48447c2fc3..7c70d72c9a 100644
--- a/build/plugins/a.yaml
+++ b/build/plugins/a.yaml
@@ -7,7 +7,7 @@ arcanum:
requirements:
- system: arcanum
type: merge_time_fits
- alias: Merge is forbidden from 11:00 to 20:00 MSK (YMAKE-1194)
+ alias: build/plugins commits invalidate autocheck dependency caches for ALL and thus are forbidden from 11:00 to 20:00 MSK
data:
merge_intervals_utc:
# 20:00..11:00 MSK
diff --git a/build/plugins/lib/test_const/__init__.py b/build/plugins/lib/test_const/__init__.py
index 575ec04388..cd7534926c 100644
--- a/build/plugins/lib/test_const/__init__.py
+++ b/build/plugins/lib/test_const/__init__.py
@@ -456,13 +456,10 @@ class DefaultLinterConfig(Enum):
Python = "build/config/tests/py_style/default_configs.json"
-# TODO Fill up like
-"""
-{
- PythonLinterName.Ruff: "build/config/tests/ruff/rules.yaml",
-}
-"""
-LINTER_TO_VALIDATION_CONFIG = {}
+class LinterConfigsValidationRules(Enum):
+ Cpp = "build/config/tests/cpp_style/configs_validation_rules.json"
+ Python = "build/config/tests/py_style/configs_validation_rules.json"
+
LINTER_CONFIG_TYPES = {
CppLinterName.ClangFormat: (".clang-format",),
diff --git a/build/scripts/link_o.py b/build/scripts/link_o.py
index 298f7ee16c..c6f1302440 100644
--- a/build/scripts/link_o.py
+++ b/build/scripts/link_o.py
@@ -22,6 +22,8 @@ def flt(args):
if '-apple-macos' in str(sys.argv):
cmd = sys.argv[1:]
+elif '-apple-darwin' in str(sys.argv):
+ cmd = sys.argv[1:]
else:
cmd = [sys.argv[1]] + list(flt(sys.argv[2:]))
diff --git a/build/sysincl/misc.yml b/build/sysincl/misc.yml
index 050cece3da..a27b6e3de7 100644
--- a/build/sysincl/misc.yml
+++ b/build/sysincl/misc.yml
@@ -462,7 +462,3 @@
- source_filter: "^contrib/libs/svt-av1"
includes:
- filter.h: contrib/libs/svt-av1/Source/Lib/Codec/filter.h
-
-- source_filter: "^sdg/sdc"
- includes:
- - uuid/uuid.h: sdg/sdc/contrib/uuid/uuid/uuid.h
diff --git a/contrib/libs/antlr4-c3/ya.make b/contrib/libs/antlr4-c3/ya.make
index abffa08b29..06e0f3144d 100644
--- a/contrib/libs/antlr4-c3/ya.make
+++ b/contrib/libs/antlr4-c3/ya.make
@@ -12,8 +12,6 @@ ORIGINAL_SOURCE(https://github.com/mike-lischke/antlr4-c3/archive/refs/tags/cpp-
NO_COMPILER_WARNINGS()
-SUBSCRIBER(g:cpp-contrib)
-
PEERDIR(
contrib/libs/antlr4_cpp_runtime
)
diff --git a/contrib/libs/cxxsupp/libcxxrt/.yandex_meta/__init__.py b/contrib/libs/cxxsupp/libcxxrt/.yandex_meta/__init__.py
new file mode 100644
index 0000000000..52e24af9ef
--- /dev/null
+++ b/contrib/libs/cxxsupp/libcxxrt/.yandex_meta/__init__.py
@@ -0,0 +1,39 @@
+import os
+
+from devtools.yamaker.modules import Switch, Linkable
+from devtools.yamaker.project import CMakeNinjaNixProject
+
+
+def post_install(self):
+ os.unlink(self.dstdir + "/unwind-itanium.h")
+ os.unlink(self.dstdir + "/libelftc_dem_gnu3.c")
+ with self.yamakes["."] as libcxxrt:
+ # Do not create peerdir loop from libcxx and libcxxrt
+ libcxxrt.NO_RUNTIME = True
+ libcxxrt.SRCS.remove("libelftc_dem_gnu3.c")
+ libcxxrt.before(
+ "SRCS",
+ Switch({"SANITIZER_TYPE == undefined OR FUZZING": Linkable(NO_SANITIZE=True, NO_SANITIZE_COVERAGE=True)}),
+ )
+ libcxxrt.CXXFLAGS = ["-nostdinc++"]
+ libcxxrt.PEERDIR |= {
+ "contrib/libs/libunwind",
+ "library/cpp/sanitizer/include",
+ }
+
+
+libcxxrt = CMakeNinjaNixProject(
+ arcdir="contrib/libs/cxxsupp/libcxxrt",
+ nixattr="libcxxrt",
+ owners=["g:cpp-committee", "g:cpp-contrib"],
+ install_subdir="src",
+ inclink={"include": {"cxxabi.h"}},
+ keep_paths=[
+ "unwind.h",
+ ],
+ post_install=post_install,
+)
+
+libcxxrt.copy_top_sources_except |= {
+ "INTERFACE_LINK_LIBRARIES.txt",
+}
diff --git a/contrib/libs/cxxsupp/libcxxrt/.yandex_meta/override.nix b/contrib/libs/cxxsupp/libcxxrt/.yandex_meta/override.nix
new file mode 100644
index 0000000000..d80f39e895
--- /dev/null
+++ b/contrib/libs/cxxsupp/libcxxrt/.yandex_meta/override.nix
@@ -0,0 +1,13 @@
+pkgs: attrs: with pkgs; rec {
+ version = "2024-10-14";
+ revision = "76435c4451aeb5e04e9500b090293347a38cef8d";
+
+ src = fetchFromGitHub {
+ owner = "libcxxrt";
+ repo = "libcxxrt";
+ rev = "${revision}";
+ hash = "sha256-U7mq79/0xbyRr2+KUMKgEvyd2lfr3Q5GrByt/8J9sC8=";
+ };
+
+ nativeBuildInputs = [ cmake ];
+}
diff --git a/contrib/libs/cxxsupp/libcxxrt/exception.cc b/contrib/libs/cxxsupp/libcxxrt/exception.cc
index dc8d507655..088e62e449 100644
--- a/contrib/libs/cxxsupp/libcxxrt/exception.cc
+++ b/contrib/libs/cxxsupp/libcxxrt/exception.cc
@@ -312,19 +312,21 @@ static _Unwind_Reason_Code trace(struct _Unwind_Context *context, void *c)
return _URC_CONTINUE_UNWIND;
}
-static void bt_terminate_handler() {
- __cxa_eh_globals* globals = __cxa_get_globals();
- __cxa_exception* thrown_exception = globals->caughtExceptions;
+static void terminate_with_diagnostics() {
+ __cxa_eh_globals *globals = __cxa_get_globals();
+ __cxa_exception *ex = globals->caughtExceptions;
- if (thrown_exception) {
- fprintf(stderr, "uncaught exception:\n address -> %p\n", (void*)thrown_exception);
- thrown_exception = realExceptionFromException(thrown_exception);
+ if (ex != nullptr) {
+ fprintf(stderr, "uncaught exception:\n address -> %p\n", (void*)ex);
+ ex = realExceptionFromException(ex);
- const __class_type_info *e_ti = static_cast<const __class_type_info*>(&typeid(std::exception));
- const __class_type_info *throw_ti = dynamic_cast<const __class_type_info*>(thrown_exception->exceptionType);
+ const __class_type_info *e_ti =
+ static_cast<const __class_type_info*>(&typeid(std::exception));
+ const __class_type_info *throw_ti =
+ dynamic_cast<const __class_type_info*>(ex->exceptionType);
if (throw_ti) {
- void* ptr = thrown_exception + 1;
+ void* ptr = ex + 1;
if (throw_ti->__do_upcast(e_ti, &ptr)) {
std::exception* e = static_cast<std::exception*>(ptr);
@@ -337,7 +339,7 @@ static void bt_terminate_handler() {
size_t bufferSize = 128;
char *demangled = static_cast<char*>(malloc(bufferSize));
- const char *mangled = thrown_exception->exceptionType->name();
+ const char *mangled = ex->exceptionType->name();
int status;
demangled = __cxa_demangle(mangled, demangled, &bufferSize, &status);
fprintf(stderr, " type -> %s\n", status == 0 ? demangled : mangled);
@@ -347,7 +349,7 @@ static void bt_terminate_handler() {
}
/** The global termination handler. */
-static atomic<terminate_handler> terminateHandler = bt_terminate_handler;
+static atomic<terminate_handler> terminateHandler = terminate_with_diagnostics;
/** The global unexpected exception handler. */
static atomic<unexpected_handler> unexpectedHandler = std::terminate;
diff --git a/contrib/libs/cxxsupp/libcxxrt/patches/backtrace_recording.patch b/contrib/libs/cxxsupp/libcxxrt/patches/backtrace_recording.patch
new file mode 100644
index 0000000000..81baccd6fc
--- /dev/null
+++ b/contrib/libs/cxxsupp/libcxxrt/patches/backtrace_recording.patch
@@ -0,0 +1,144 @@
+diff --git a/cxxabi.h b/cxxabi.h
+index e021f85..acf9974 100644
+--- a/cxxabi.h
++++ b/cxxabi.h
+@@ -249,6 +249,10 @@ char* __cxa_demangle(const char* mangled_name,
+ char* buf,
+ size_t* n,
+ int* status);
++
++#ifdef _YNDX_LIBUNWIND_ENABLE_EXCEPTION_BACKTRACE
++size_t __cxa_collect_current_exception_backtrace(void** dest, size_t size);
++#endif
+ #ifdef __cplusplus
+ } // extern "C"
+ } // namespace
+diff --git a/exception.cc b/exception.cc
+index 15f93ae..c4b15ae 100644
+--- a/exception.cc
++++ b/exception.cc
+@@ -255,12 +255,20 @@ namespace std
+ * various checks may test for equality of the class, which is incorrect.
+ */
+ static const uint64_t exception_class =
++#ifdef _YNDX_LIBUNWIND_ENABLE_EXCEPTION_BACKTRACE
++ _YNDX_LIBUNWIND_EXCEPTION_BACKTRACE_PRIMARY_CLASS;
++#else
+ EXCEPTION_CLASS('G', 'N', 'U', 'C', 'C', '+', '+', '\0');
++#endif
+ /**
+ * Class used for dependent exceptions.
+ */
+ static const uint64_t dependent_exception_class =
++#ifdef _YNDX_LIBUNWIND_ENABLE_EXCEPTION_BACKTRACE
++ _YNDX_LIBUNWIND_EXCEPTION_BACKTRACE_DEPENDENT_CLASS;
++#else
+ EXCEPTION_CLASS('G', 'N', 'U', 'C', 'C', '+', '+', '\x01');
++#endif
+ /**
+ * The low four bytes of the exception class, indicating that we conform to the
+ * Itanium C++ ABI. This is currently unused, but should be used in the future
+@@ -605,6 +613,27 @@ static void free_exception(char *e)
+ }
+ #endif
+
++static constexpr size_t align_to(size_t size, size_t alignment) noexcept {
++ return (size + alignment - 1) / alignment * alignment;
++}
++
++static_assert(align_to(15, 16) == 16);
++static_assert(align_to(16, 16) == 16);
++static_assert(align_to(17, 16) == 32);
++
++static constexpr size_t exception_size = align_to(sizeof(__cxa_exception), 16);
++static constexpr size_t dependent_exception_size = align_to(sizeof(__cxa_dependent_exception), 16);
++#ifdef _YNDX_LIBUNWIND_ENABLE_EXCEPTION_BACKTRACE
++static constexpr size_t backtrace_buffer_size = align_to(sizeof(_Unwind_Backtrace_Buffer), 16);
++
++static_assert(
++ _YNDX_LIBUNWIND_EXCEPTION_BACKTRACE_MAGIC_OFFSET ==
++ offsetof(__cxa_exception, unwindHeader) + backtrace_buffer_size - sizeof(_Unwind_Backtrace_Buffer));
++#else
++static constexpr size_t backtrace_buffer_size = 0;
++#endif
++
++
+ /**
+ * Allocates an exception structure. Returns a pointer to the space that can
+ * be used to store an object of thrown_size bytes. This function will use an
+@@ -613,16 +642,19 @@ static void free_exception(char *e)
+ */
+ extern "C" void *__cxa_allocate_exception(size_t thrown_size) _LIBCXXRT_NOEXCEPT
+ {
+- size_t size = thrown_size + sizeof(__cxa_exception);
++ size_t size = thrown_size + exception_size + backtrace_buffer_size;
+ char *buffer = alloc_or_die(size);
+- return buffer+sizeof(__cxa_exception);
++#ifdef _YNDX_LIBUNWIND_ENABLE_EXCEPTION_BACKTRACE
++ ((_Unwind_Backtrace_Buffer *)buffer)->size = 0;
++#endif
++ return buffer + exception_size + backtrace_buffer_size;
+ }
+
+ extern "C" void *__cxa_allocate_dependent_exception(void)
+ {
+- size_t size = sizeof(__cxa_dependent_exception);
++ size_t size = dependent_exception_size + backtrace_buffer_size;
+ char *buffer = alloc_or_die(size);
+- return buffer+sizeof(__cxa_dependent_exception);
++ return buffer + dependent_exception_size + backtrace_buffer_size;
+ }
+
+ /**
+@@ -650,7 +682,8 @@ extern "C" void __cxa_free_exception(void *thrown_exception) _LIBCXXRT_NOEXCEPT
+ }
+ }
+
+- free_exception(reinterpret_cast<char*>(ex));
++ free_exception(
++ reinterpret_cast<char*>(thrown_exception) - exception_size - backtrace_buffer_size);
+ }
+
+ static void releaseException(__cxa_exception *exception)
+@@ -677,7 +710,7 @@ void __cxa_free_dependent_exception(void *thrown_exception)
+ {
+ releaseException(realExceptionFromException(reinterpret_cast<__cxa_exception*>(ex)));
+ }
+- free_exception(reinterpret_cast<char*>(ex));
++ free_exception(reinterpret_cast<char*>(thrown_exception) - dependent_exception_size - backtrace_buffer_size);
+ }
+
+ /**
+@@ -864,6 +897,32 @@ extern "C" void __cxa_decrement_exception_refcount(void* thrown_exception)
+ releaseException(ex);
+ }
+
++#ifdef _YNDX_LIBUNWIND_ENABLE_EXCEPTION_BACKTRACE
++static size_t __cxa_collect_backtrace(__cxa_exception* ex, void** dest, size_t size) {
++ if (!ex) {
++ return 0;
++ }
++ if (!isCXXException(ex->unwindHeader.exception_class)) {
++ return 0;
++ }
++ size_t i = 0;
++ if (isDependentException(ex->unwindHeader.exception_class)) {
++ i = __cxa_collect_backtrace(
++ (__cxa_exception *)((__cxa_dependent_exception *)ex)->primaryException - 1, dest, size);
++ }
++ _Unwind_Backtrace_Buffer* backtraceBuffer = (_Unwind_Backtrace_Buffer*)(
++ (char *)(ex + 1) - exception_size - backtrace_buffer_size);
++ for (size_t j = 0; i != size && j != backtraceBuffer->size; ++i, ++j) {
++ dest[i] = backtraceBuffer->backtrace[j];
++ }
++ return i;
++}
++
++extern "C" size_t __cxa_collect_current_exception_backtrace(void** dest, size_t size) {
++ return __cxa_collect_backtrace(__cxa_get_globals()->caughtExceptions, dest, size);
++}
++#endif
++
+ /**
+ * ABI function. Rethrows the current exception. Does not remove the
+ * exception from the stack or decrement its handler count - the compiler is
diff --git a/contrib/libs/cxxsupp/libcxxrt/patches/cxa_eh_globals_completion_var.patch b/contrib/libs/cxxsupp/libcxxrt/patches/cxa_eh_globals_completion_var.patch
new file mode 100644
index 0000000000..3a1f27243e
--- /dev/null
+++ b/contrib/libs/cxxsupp/libcxxrt/patches/cxa_eh_globals_completion_var.patch
@@ -0,0 +1,12 @@
+--- /cxxabi.h (index)
++++ /cxxabi.h (working tree)
+@@ -183,6 +183,9 @@ struct __cxa_eh_globals
+ */
+ unsigned int uncaughtExceptions;
+ };
++
++#define Y_CXA_EH_GLOBALS_COMPLETE
++
+ /**
+ * ABI function returning the __cxa_eh_globals structure.
+ */
diff --git a/contrib/libs/cxxsupp/libcxxrt/patches/cxa_throw_hook.patch b/contrib/libs/cxxsupp/libcxxrt/patches/cxa_throw_hook.patch
new file mode 100644
index 0000000000..f2333ccba5
--- /dev/null
+++ b/contrib/libs/cxxsupp/libcxxrt/patches/cxa_throw_hook.patch
@@ -0,0 +1,26 @@
+diff --git a/exception.cc b/exception.cc
+index 15f93ae..2f1947f 100644
+--- a/exception.cc
++++ b/exception.cc
+@@ -806,6 +806,9 @@ extern "C" __cxa_exception *__cxa_init_primary_exception(
+ return ex;
+ }
+
++typedef void (*cxa_throw_hook_t)(void*, std::type_info*, void(*)(void*)) noexcept;
++
++__attribute__((weak)) cxa_throw_hook_t cxa_throw_hook = nullptr;
+
+ /**
+ * ABI function for throwing an exception. Takes the object to be thrown (the
+@@ -816,6 +819,11 @@ extern "C" void __cxa_throw(void *thrown_exception,
+ std::type_info *tinfo,
+ void(*dest)(void*))
+ {
++ if (cxa_throw_hook)
++ {
++ cxa_throw_hook(thrown_exception, tinfo, dest);
++ }
++
+ __cxa_exception *ex = __cxa_init_primary_exception(thrown_exception, tinfo, dest);
+ ex->referenceCount = 1;
+
diff --git a/contrib/libs/cxxsupp/libcxxrt/patches/fix_msan_exception_handling.patch b/contrib/libs/cxxsupp/libcxxrt/patches/fix_msan_exception_handling.patch
new file mode 100644
index 0000000000..38aaa99006
--- /dev/null
+++ b/contrib/libs/cxxsupp/libcxxrt/patches/fix_msan_exception_handling.patch
@@ -0,0 +1,26 @@
+diff --git a/exception.cc b/exception.cc
+index bad7530..007b29c 100644
+--- a/exception.cc
++++ b/exception.cc
+@@ -34,6 +34,7 @@
+ #include "dwarf_eh.h"
+ #include "atomic.h"
+ #include "cxxabi.h"
++#include <sanitizer/msan_interface.h>
+
+ #pragma weak pthread_key_create
+ #pragma weak pthread_setspecific
+@@ -1062,6 +1063,13 @@ static void pushCleanupException(_Unwind_Exception *exceptionObject,
+ */
+ extern "C"
+ BEGIN_PERSONALITY_FUNCTION(__gxx_personality_v0)
++#if defined(__SANITIZE_MEMORY__)
++ __msan_unpoison(&version, sizeof(version));
++ __msan_unpoison(&actions, sizeof(actions));
++ __msan_unpoison(&exceptionClass, sizeof(exceptionClass));
++ __msan_unpoison(&exceptionObject, sizeof(exceptionObject));
++ __msan_unpoison(&context, sizeof(context));
++#endif
+ // This personality function is for version 1 of the ABI. If you use it
+ // with a future version of the ABI, it won't know what to do, so it
+ // reports a fatal error and give up before it breaks anything.
diff --git a/contrib/libs/cxxsupp/libcxxrt/patches/pr41.1-move-trace.patch b/contrib/libs/cxxsupp/libcxxrt/patches/pr41.1-move-trace.patch
new file mode 100644
index 0000000000..b1331c9445
--- /dev/null
+++ b/contrib/libs/cxxsupp/libcxxrt/patches/pr41.1-move-trace.patch
@@ -0,0 +1,74 @@
+commit 690915810d43430667f8d7b84fc7f88784f68fdf (HEAD -> update-libcxxrt)
+author: thegeorg
+date: 2025-02-26T13:56:00+03:00
+
+ Revert "Simplify libcxxrt patches"
+
+ This reverts commit 4358151e7723e37a39cf6f5478bfa336aa687fd4, reversing
+ changes made to ad7618a2219d22bcd89a67fe082f290b4a9656ef.
+
+--- contrib/libs/cxxsupp/libcxxrt/exception.cc (27ffe99a31d7fb0dd8a933c936f70853dac6041a)
++++ contrib/libs/cxxsupp/libcxxrt/exception.cc (690915810d43430667f8d7b84fc7f88784f68fdf)
+@@ -287,6 +287,30 @@ namespace std
+
+ using namespace ABI_NAMESPACE;
+
++/**
++ * Callback function used with _Unwind_Backtrace().
++ *
++ * Prints a stack trace. Used only for debugging help.
++ *
++ * Note: As of FreeBSD 8.1, dladd() still doesn't work properly, so this only
++ * correctly prints function names from public, relocatable, symbols.
++ */
++static _Unwind_Reason_Code trace(struct _Unwind_Context *context, void *c)
++{
++ Dl_info myinfo;
++ int mylookup =
++ dladdr(reinterpret_cast<void *>(__cxa_current_exception_type), &myinfo);
++ void *ip = reinterpret_cast<void*>(_Unwind_GetIP(context));
++ Dl_info info;
++ if (dladdr(ip, &info) != 0)
++ {
++ if (mylookup == 0 || strcmp(info.dli_fname, myinfo.dli_fname) != 0)
++ {
++ printf("%p:%s() in %s\n", ip, info.dli_sname, info.dli_fname);
++ }
++ }
++ return _URC_CONTINUE_UNWIND;
++}
+
+
+ /** The global termination handler. */
+@@ -760,31 +785,6 @@ void __cxa_free_dependent_exception(void *thrown_exception)
+ }
+
+ /**
+- * Callback function used with _Unwind_Backtrace().
+- *
+- * Prints a stack trace. Used only for debugging help.
+- *
+- * Note: As of FreeBSD 8.1, dladd() still doesn't work properly, so this only
+- * correctly prints function names from public, relocatable, symbols.
+- */
+-static _Unwind_Reason_Code trace(struct _Unwind_Context *context, void *c)
+-{
+- Dl_info myinfo;
+- int mylookup =
+- dladdr(reinterpret_cast<void *>(__cxa_current_exception_type), &myinfo);
+- void *ip = reinterpret_cast<void*>(_Unwind_GetIP(context));
+- Dl_info info;
+- if (dladdr(ip, &info) != 0)
+- {
+- if (mylookup == 0 || strcmp(info.dli_fname, myinfo.dli_fname) != 0)
+- {
+- printf("%p:%s() in %s\n", ip, info.dli_sname, info.dli_fname);
+- }
+- }
+- return _URC_CONTINUE_UNWIND;
+-}
+-
+-/**
+ * Report a failure that occurred when attempting to throw an exception.
+ *
+ * If the failure happened by falling off the end of the stack without finding
diff --git a/contrib/libs/cxxsupp/libcxxrt/patches/pr41.2-fix-trace-format.patch b/contrib/libs/cxxsupp/libcxxrt/patches/pr41.2-fix-trace-format.patch
new file mode 100644
index 0000000000..a70c234b4f
--- /dev/null
+++ b/contrib/libs/cxxsupp/libcxxrt/patches/pr41.2-fix-trace-format.patch
@@ -0,0 +1,48 @@
+--- contrib/libs/cxxsupp/libcxxrt/exception.cc (index)
++++ contrib/libs/cxxsupp/libcxxrt/exception.cc (working tree)
+@@ -312,9 +312,44 @@ static _Unwind_Reason_Code trace(struct _Unwind_Context *context, void *c)
+ return _URC_CONTINUE_UNWIND;
+ }
+
++static void terminate_with_diagnostics() {
++ __cxa_eh_globals *globals = __cxa_get_globals();
++ __cxa_exception *ex = globals->caughtExceptions;
++
++ if (ex != nullptr) {
++ fprintf(stderr, "uncaught exception:\n address -> %p\n", (void*)ex);
++ ex = realExceptionFromException(ex);
++
++ const __class_type_info *e_ti =
++ static_cast<const __class_type_info*>(&typeid(std::exception));
++ const __class_type_info *throw_ti =
++ dynamic_cast<const __class_type_info*>(ex->exceptionType);
++
++ if (throw_ti) {
++ void* ptr = ex + 1;
++
++ if (throw_ti->__do_upcast(e_ti, &ptr)) {
++ std::exception* e = static_cast<std::exception*>(ptr);
++
++ if (e) {
++ fprintf(stderr, " what() -> \"%s\"\n", e->what());
++ }
++ }
++ }
++
++ size_t bufferSize = 128;
++ char *demangled = static_cast<char*>(malloc(bufferSize));
++ const char *mangled = ex->exceptionType->name();
++ int status;
++ demangled = __cxa_demangle(mangled, demangled, &bufferSize, &status);
++ fprintf(stderr, " type -> %s\n", status == 0 ? demangled : mangled);
++ if (status == 0) { free(demangled); }
++ }
++ abort();
++}
+
+ /** The global termination handler. */
+-static atomic<terminate_handler> terminateHandler = abort;
++static atomic<terminate_handler> terminateHandler = terminate_with_diagnostics;
+ /** The global unexpected exception handler. */
+ static atomic<unexpected_handler> unexpectedHandler = std::terminate;
+
diff --git a/contrib/libs/cxxsupp/libcxxrt/patches/preallocated_thread_info.patch b/contrib/libs/cxxsupp/libcxxrt/patches/preallocated_thread_info.patch
new file mode 100644
index 0000000000..208413ecb2
--- /dev/null
+++ b/contrib/libs/cxxsupp/libcxxrt/patches/preallocated_thread_info.patch
@@ -0,0 +1,107 @@
+--- /exception.cc (index)
++++ /exception.cc (working tree)
+@@ -331,6 +373,44 @@ static void free_exception_list(__cxa_exception *ex)
+ __cxa_free_exception(ex+1);
+ }
+
++#define fast_ti_size 100
++
++static long fast_ti_index;
++static __cxa_thread_info fast_ti[fast_ti_size];
++
++static inline __cxa_thread_info* alloc_thread_info() {
++ {
++ long cur_index;
++
++ __atomic_load(&fast_ti_index, &cur_index, __ATOMIC_SEQ_CST);
++
++ // exausted long time ago
++ if (cur_index >= fast_ti_size) {
++ return static_cast<__cxa_thread_info*>(calloc(1, sizeof(__cxa_thread_info)));
++ }
++ }
++
++ auto my_index = __sync_fetch_and_add(&fast_ti_index, 1);
++
++ // exausted
++ if (my_index >= fast_ti_size) {
++ return static_cast<__cxa_thread_info*>(calloc(1, sizeof(__cxa_thread_info)));
++ }
++
++ // fast path
++ auto& ret = fast_ti[my_index];
++
++ memset(&ret, 0, sizeof(ret));
++
++ return &ret;
++}
++
++static inline void free_thread_info(__cxa_thread_info* ti) {
++ if ((ti < fast_ti) || (ti >= (fast_ti + fast_ti_size))) {
++ free(ti);
++ }
++}
++
+ /**
+ * Cleanup function called when a thread exists to make certain that all of the
+ * per-thread data is deleted.
+@@ -352,10 +352,9 @@ static void thread_cleanup(void* thread_info)
+ free_exception_list(info->globals.caughtExceptions);
+ }
+ }
+- free(thread_info);
++ free_thread_info(info);
+ }
+
+-
+ /**
+ * Once control used to protect the key creation.
+ */
+@@ -389,11 +388,16 @@ static void init_key(void)
+ pthread_setspecific(eh_key, 0);
+ }
+
++static __thread __cxa_thread_info* THR_INFO = nullptr;
++
+ /**
+ * Returns the thread info structure, creating it if it is not already created.
+ */
+ static __cxa_thread_info *thread_info()
+ {
++ if (THR_INFO) {
++ return THR_INFO;
++ }
+ if ((0 == pthread_once) || pthread_once(&once_control, init_key))
+ {
+ fakeTLS = true;
+@@ -402,17 +406,29 @@ static __cxa_thread_info *thread_info()
+ __cxa_thread_info *info = static_cast<__cxa_thread_info*>(pthread_getspecific(eh_key));
+ if (0 == info)
+ {
+- info = static_cast<__cxa_thread_info*>(calloc(1, sizeof(__cxa_thread_info)));
++ info = alloc_thread_info();
+ pthread_setspecific(eh_key, info);
+ }
++ THR_INFO = info;
+ return info;
+ }
++
++// ensure main thread will allocate preallocated tls
++static struct InitMainTls {
++ inline InitMainTls() {
++ thread_info();
++ }
++} init_main_tls;
++
+ /**
+ * Fast version of thread_info(). May fail if thread_info() is not called on
+ * this thread at least once already.
+ */
+ static __cxa_thread_info *thread_info_fast()
+ {
++ if (THR_INFO) {
++ return THR_INFO;
++ }
+ if (fakeTLS) { return &singleThreadInfo; }
+ return static_cast<__cxa_thread_info*>(pthread_getspecific(eh_key));
+ }
diff --git a/contrib/libs/cxxsupp/libcxxrt/patches/remove_unused_code.patch b/contrib/libs/cxxsupp/libcxxrt/patches/remove_unused_code.patch
new file mode 100644
index 0000000000..9a2002511a
--- /dev/null
+++ b/contrib/libs/cxxsupp/libcxxrt/patches/remove_unused_code.patch
@@ -0,0 +1,90 @@
+diff --git a/exception.cc b/exception.cc
+index 1a6a487..2e84ce8 100644
+--- /exception.cc
++++ /exception.cc
+@@ -36,29 +36,6 @@
+ #include "cxxabi.h"
+ #include <sanitizer/msan_interface.h>
+
+-#pragma weak pthread_key_create
+-#pragma weak pthread_setspecific
+-#pragma weak pthread_getspecific
+-#pragma weak pthread_once
+-#ifdef LIBCXXRT_WEAK_LOCKS
+-#pragma weak pthread_mutex_lock
+-#define pthread_mutex_lock(mtx) do {\
+- if (pthread_mutex_lock) pthread_mutex_lock(mtx);\
+- } while(0)
+-#pragma weak pthread_mutex_unlock
+-#define pthread_mutex_unlock(mtx) do {\
+- if (pthread_mutex_unlock) pthread_mutex_unlock(mtx);\
+- } while(0)
+-#pragma weak pthread_cond_signal
+-#define pthread_cond_signal(cv) do {\
+- if (pthread_cond_signal) pthread_cond_signal(cv);\
+- } while(0)
+-#pragma weak pthread_cond_wait
+-#define pthread_cond_wait(cv, mtx) do {\
+- if (pthread_cond_wait) pthread_cond_wait(cv, mtx);\
+- } while(0)
+-#endif
+-
+ using namespace ABI_NAMESPACE;
+
+ /**
+@@ -467,31 +444,16 @@ static void thread_cleanup(void* thread_info)
+ */
+ static pthread_once_t once_control = PTHREAD_ONCE_INIT;
+
+-/**
+- * We may not be linked against a full pthread implementation. If we're not,
+- * then we need to fake the thread-local storage by storing 'thread-local'
+- * things in a global.
+- */
+-static bool fakeTLS;
+-/**
+- * Thread-local storage for a single-threaded program.
+- */
+-static __cxa_thread_info singleThreadInfo;
+ /**
+ * Initialise eh_key.
+ */
+ static void init_key(void)
+ {
+- if ((0 == pthread_key_create) ||
+- (0 == pthread_setspecific) ||
+- (0 == pthread_getspecific))
+- {
+- fakeTLS = true;
+- return;
+- }
+ pthread_key_create(&eh_key, thread_cleanup);
+ pthread_setspecific(eh_key, reinterpret_cast<void *>(0x42));
+- fakeTLS = (pthread_getspecific(eh_key) != reinterpret_cast<void *>(0x42));
++ if (pthread_getspecific(eh_key) != reinterpret_cast<void *>(0x42)) {
++ abort();
++ }
+ pthread_setspecific(eh_key, 0);
+ }
+
+@@ -505,11 +467,7 @@ static __cxa_thread_info *thread_info()
+ if (THR_INFO) {
+ return THR_INFO;
+ }
+- if ((0 == pthread_once) || pthread_once(&once_control, init_key))
+- {
+- fakeTLS = true;
+- }
+- if (fakeTLS) { return &singleThreadInfo; }
++ pthread_once(&once_control, init_key);
+ __cxa_thread_info *info = static_cast<__cxa_thread_info*>(pthread_getspecific(eh_key));
+ if (0 == info)
+ {
+@@ -536,7 +494,6 @@ static __cxa_thread_info *thread_info_fast()
+ if (THR_INFO) {
+ return THR_INFO;
+ }
+- if (fakeTLS) { return &singleThreadInfo; }
+ return static_cast<__cxa_thread_info*>(pthread_getspecific(eh_key));
+ }
+ /**
diff --git a/contrib/libs/cxxsupp/libcxxrt/patches/throw_specs.patch b/contrib/libs/cxxsupp/libcxxrt/patches/throw_specs.patch
new file mode 100644
index 0000000000..a1c0e5abe9
--- /dev/null
+++ b/contrib/libs/cxxsupp/libcxxrt/patches/throw_specs.patch
@@ -0,0 +1,20 @@
+diff --git a/memory.cc b/memory.cc
+index 7beb048..bb6cc74 100644
+--- a/memory.cc
++++ b/memory.cc
+@@ -59,13 +59,13 @@ namespace std
+ * Sets a function to be called when there is a failure in new.
+ */
+ __attribute__((weak))
+- new_handler set_new_handler(new_handler handler)
++ new_handler set_new_handler(new_handler handler) noexcept
+ {
+ return new_handl.exchange(handler);
+ }
+
+ __attribute__((weak))
+- new_handler get_new_handler(void)
++ new_handler get_new_handler(void) noexcept
+ {
+ return new_handl.load();
+ }
diff --git a/contrib/libs/cxxsupp/libcxxrt/patches/typeinfo.patch b/contrib/libs/cxxsupp/libcxxrt/patches/typeinfo.patch
new file mode 100644
index 0000000000..413938e3e0
--- /dev/null
+++ b/contrib/libs/cxxsupp/libcxxrt/patches/typeinfo.patch
@@ -0,0 +1,60 @@
+--- /typeinfo.cc
++++ /typeinfo.cc
+@@ -68,57 +68,3 @@ ABI_NAMESPACE::__vmi_class_type_info::~__vmi_class_type_info() {}
+ ABI_NAMESPACE::__pbase_type_info::~__pbase_type_info() {}
+ ABI_NAMESPACE::__pointer_type_info::~__pointer_type_info() {}
+ ABI_NAMESPACE::__pointer_to_member_type_info::~__pointer_to_member_type_info() {}
+-
+-// From libelftc
+-extern "C" char *__cxa_demangle_gnu3(const char *);
+-
+-extern "C" char* __cxa_demangle(const char* mangled_name,
+- char* buf,
+- size_t* n,
+- int* status)
+-{
+- // TODO: We should probably just be linking against libelf-tc, rather than
+- // copying their code. This requires them to do an actual release,
+- // however, and for our changes to be pushed upstream. We also need to
+- // call a different demangling function here depending on the ABI (e.g.
+- // ARM).
+- char *demangled = __cxa_demangle_gnu3(mangled_name);
+- if (NULL != demangled)
+- {
+- size_t len = strlen(demangled);
+- if (!buf || (*n < len+1))
+- {
+- buf = static_cast<char*>(realloc(buf, len+1));
+- }
+- if (0 != buf)
+- {
+- memcpy(buf, demangled, len);
+- buf[len] = 0;
+- if (n)
+- {
+- *n = len;
+- }
+- if (status)
+- {
+- *status = 0;
+- }
+- }
+- else
+- {
+- if (status)
+- {
+- *status = -1;
+- }
+- }
+- free(demangled);
+- }
+- else
+- {
+- if (status)
+- {
+- *status = -2;
+- }
+- return NULL;
+- }
+- return buf;
+-}
diff --git a/contrib/python/ydb/py3/ydb/draft/dynamic_config.py b/contrib/python/ydb/py3/ydb/draft/dynamic_config.py
index 4648b29a77..2089fcb88a 100644
--- a/contrib/python/ydb/py3/ydb/draft/dynamic_config.py
+++ b/contrib/python/ydb/py3/ydb/draft/dynamic_config.py
@@ -71,7 +71,9 @@ def _get_node_labels_request_factory(node_id):
def _wrap_dynamic_config(config_pb, dynamic_config_cls=None, *args, **kwargs):
dynamic_config_cls = DynamicConfig if dynamic_config_cls is None else dynamic_config_cls
- return dynamic_config_cls(config_pb.identity.version, config_pb.identity.cluster, config_pb.config, *args, **kwargs)
+ return dynamic_config_cls(
+ config_pb.identity[0].version, config_pb.identity[0].cluster, config_pb.config[0], *args, **kwargs
+ )
def _wrap_get_config_response(rpc_state, response):
diff --git a/library/cpp/http/simple/http_client.cpp b/library/cpp/http/simple/http_client.cpp
index bac6bdc39e..2be5a14582 100644
--- a/library/cpp/http/simple/http_client.cpp
+++ b/library/cpp/http/simple/http_client.cpp
@@ -25,26 +25,30 @@ TKeepAliveHttpClient::TKeepAliveHttpClient(const TString& host,
TKeepAliveHttpClient::THttpCode TKeepAliveHttpClient::DoGet(const TStringBuf relativeUrl,
IOutputStream* output,
const THeaders& headers,
- THttpHeaders* outHeaders) {
+ THttpHeaders* outHeaders,
+ NThreading::TCancellationToken cancellation) {
return DoRequest(TStringBuf("GET"),
relativeUrl,
{},
output,
headers,
- outHeaders);
+ outHeaders,
+ std::move(cancellation));
}
TKeepAliveHttpClient::THttpCode TKeepAliveHttpClient::DoPost(const TStringBuf relativeUrl,
const TStringBuf body,
IOutputStream* output,
const THeaders& headers,
- THttpHeaders* outHeaders) {
+ THttpHeaders* outHeaders,
+ NThreading::TCancellationToken cancellation) {
return DoRequest(TStringBuf("POST"),
relativeUrl,
body,
output,
headers,
- outHeaders);
+ outHeaders,
+ std::move(cancellation));
}
TKeepAliveHttpClient::THttpCode TKeepAliveHttpClient::DoRequest(const TStringBuf method,
@@ -52,15 +56,17 @@ TKeepAliveHttpClient::THttpCode TKeepAliveHttpClient::DoRequest(const TStringBuf
const TStringBuf body,
IOutputStream* output,
const THeaders& inHeaders,
- THttpHeaders* outHeaders) {
+ THttpHeaders* outHeaders,
+ NThreading::TCancellationToken cancellation) {
const TString contentLength = IntToString<10, size_t>(body.size());
- return DoRequestReliable(FormRequest(method, relativeUrl, body, inHeaders, contentLength), output, outHeaders);
+ return DoRequestReliable(FormRequest(method, relativeUrl, body, inHeaders, contentLength), output, outHeaders, std::move(cancellation));
}
TKeepAliveHttpClient::THttpCode TKeepAliveHttpClient::DoRequestRaw(const TStringBuf raw,
IOutputStream* output,
- THttpHeaders* outHeaders) {
- return DoRequestReliable(raw, output, outHeaders);
+ THttpHeaders* outHeaders,
+ NThreading::TCancellationToken cancellation) {
+ return DoRequestReliable(raw, output, outHeaders, std::move(cancellation));
}
void TKeepAliveHttpClient::DisableVerificationForHttps() {
@@ -189,28 +195,28 @@ void TSimpleHttpClient::EnableVerificationForHttps() {
HttpsVerification = true;
}
-void TSimpleHttpClient::DoGet(const TStringBuf relativeUrl, IOutputStream* output, const THeaders& headers) const {
+void TSimpleHttpClient::DoGet(const TStringBuf relativeUrl, IOutputStream* output, const THeaders& headers, NThreading::TCancellationToken cancellation) const {
TKeepAliveHttpClient cl = CreateClient();
- TKeepAliveHttpClient::THttpCode code = cl.DoGet(relativeUrl, output, headers);
+ TKeepAliveHttpClient::THttpCode code = cl.DoGet(relativeUrl, output, headers, nullptr, std::move(cancellation));
Y_ENSURE(cl.GetHttpInput());
ProcessResponse(relativeUrl, *cl.GetHttpInput(), output, code);
}
-void TSimpleHttpClient::DoPost(const TStringBuf relativeUrl, TStringBuf body, IOutputStream* output, const THashMap<TString, TString>& headers) const {
+void TSimpleHttpClient::DoPost(const TStringBuf relativeUrl, TStringBuf body, IOutputStream* output, const THashMap<TString, TString>& headers, NThreading::TCancellationToken cancellation) const {
TKeepAliveHttpClient cl = CreateClient();
- TKeepAliveHttpClient::THttpCode code = cl.DoPost(relativeUrl, body, output, headers);
+ TKeepAliveHttpClient::THttpCode code = cl.DoPost(relativeUrl, body, output, headers, nullptr, std::move(cancellation));
Y_ENSURE(cl.GetHttpInput());
ProcessResponse(relativeUrl, *cl.GetHttpInput(), output, code);
}
-void TSimpleHttpClient::DoPostRaw(const TStringBuf relativeUrl, const TStringBuf rawRequest, IOutputStream* output) const {
+void TSimpleHttpClient::DoPostRaw(const TStringBuf relativeUrl, const TStringBuf rawRequest, IOutputStream* output, NThreading::TCancellationToken cancellation) const {
TKeepAliveHttpClient cl = CreateClient();
- TKeepAliveHttpClient::THttpCode code = cl.DoRequestRaw(rawRequest, output);
+ TKeepAliveHttpClient::THttpCode code = cl.DoRequestRaw(rawRequest, output, nullptr, std::move(cancellation));
Y_ENSURE(cl.GetHttpInput());
ProcessResponse(relativeUrl, *cl.GetHttpInput(), output, code);
diff --git a/library/cpp/http/simple/http_client.h b/library/cpp/http/simple/http_client.h
index eab92d42da..224be58a24 100644
--- a/library/cpp/http/simple/http_client.h
+++ b/library/cpp/http/simple/http_client.h
@@ -12,6 +12,7 @@
#include <library/cpp/http/io/stream.h>
#include <library/cpp/http/misc/httpcodes.h>
#include <library/cpp/openssl/io/stream.h>
+#include <library/cpp/threading/cancellation/cancellation_token.h>
class TNetworkAddress;
class IOutputStream;
@@ -54,14 +55,16 @@ public:
THttpCode DoGet(const TStringBuf relativeUrl,
IOutputStream* output = nullptr,
const THeaders& headers = THeaders(),
- THttpHeaders* outHeaders = nullptr);
+ THttpHeaders* outHeaders = nullptr,
+ NThreading::TCancellationToken cancellation = NThreading::TCancellationToken::Default());
// builds post request from headers and body
THttpCode DoPost(const TStringBuf relativeUrl,
const TStringBuf body,
IOutputStream* output = nullptr,
const THeaders& headers = THeaders(),
- THttpHeaders* outHeaders = nullptr);
+ THttpHeaders* outHeaders = nullptr,
+ NThreading::TCancellationToken cancellation = NThreading::TCancellationToken::Default());
// builds request with any HTTP method from headers and body
THttpCode DoRequest(const TStringBuf method,
@@ -69,12 +72,14 @@ public:
const TStringBuf body,
IOutputStream* output = nullptr,
const THeaders& inHeaders = THeaders(),
- THttpHeaders* outHeaders = nullptr);
+ THttpHeaders* outHeaders = nullptr,
+ NThreading::TCancellationToken cancellation = NThreading::TCancellationToken::Default());
// requires already well-formed request
THttpCode DoRequestRaw(const TStringBuf raw,
IOutputStream* output = nullptr,
- THttpHeaders* outHeaders = nullptr);
+ THttpHeaders* outHeaders = nullptr,
+ NThreading::TCancellationToken cancellation = NThreading::TCancellationToken::Default());
void DisableVerificationForHttps();
void SetClientCertificate(const TOpenSslClientIO::TOptions::TClientCert& options);
@@ -93,7 +98,8 @@ private:
template <class T>
THttpCode DoRequestReliable(const T& raw,
IOutputStream* output,
- THttpHeaders* outHeaders);
+ THttpHeaders* outHeaders,
+ NThreading::TCancellationToken cancellation);
TVector<IOutputStream::TPart> FormRequest(TStringBuf method, const TStringBuf relativeUrl,
TStringBuf body,
@@ -166,13 +172,13 @@ public:
void EnableVerificationForHttps();
- void DoGet(const TStringBuf relativeUrl, IOutputStream* output, const THeaders& headers = THeaders()) const;
+ void DoGet(const TStringBuf relativeUrl, IOutputStream* output, const THeaders& headers = THeaders(), NThreading::TCancellationToken cancellation = NThreading::TCancellationToken::Default()) const;
// builds post request from headers and body
- void DoPost(const TStringBuf relativeUrl, TStringBuf body, IOutputStream* output, const THeaders& headers = THeaders()) const;
+ void DoPost(const TStringBuf relativeUrl, TStringBuf body, IOutputStream* output, const THeaders& headers = THeaders(), NThreading::TCancellationToken cancellation = NThreading::TCancellationToken::Default()) const;
// requires already well-formed post request
- void DoPostRaw(const TStringBuf relativeUrl, TStringBuf rawRequest, IOutputStream* output) const;
+ void DoPostRaw(const TStringBuf relativeUrl, TStringBuf rawRequest, IOutputStream* output, NThreading::TCancellationToken cancellation = NThreading::TCancellationToken::Default()) const;
virtual ~TSimpleHttpClient();
@@ -227,6 +233,10 @@ namespace NPrivate {
return HttpIn.Get();
}
+ void Shutdown() {
+ Socket.ShutDown(SHUT_RDWR);
+ }
+
private:
static TNetworkAddress Resolve(const TString& host, ui32 port);
@@ -250,12 +260,18 @@ namespace NPrivate {
template <class T>
TKeepAliveHttpClient::THttpCode TKeepAliveHttpClient::DoRequestReliable(const T& raw,
IOutputStream* output,
- THttpHeaders* outHeaders) {
+ THttpHeaders* outHeaders,
+ NThreading::TCancellationToken cancellation) {
+
for (int i = 0; i < 2; ++i) {
const bool haveNewConnection = CreateNewConnectionIfNeeded();
const bool couldRetry = !haveNewConnection && i == 0; // Actually old connection could be already closed by server,
// so we should try one more time in this case.
try {
+ cancellation.Future().Subscribe([&](auto&) {
+ Connection->Shutdown();
+ });
+
Connection->Write(raw);
THttpCode code = ReadAndTransferHttp(*Connection->GetHttpInput(), output, outHeaders);
@@ -265,16 +281,19 @@ TKeepAliveHttpClient::THttpCode TKeepAliveHttpClient::DoRequestReliable(const T&
return code;
} catch (const TSystemError& e) {
Connection.Reset();
+ cancellation.ThrowIfCancellationRequested();
if (!couldRetry || e.Status() != EPIPE) {
throw;
}
} catch (const THttpReadException&) { // Actually old connection is already closed by server
Connection.Reset();
+ cancellation.ThrowIfCancellationRequested();
if (!couldRetry) {
throw;
}
} catch (const std::exception&) {
Connection.Reset();
+ cancellation.ThrowIfCancellationRequested();
throw;
}
}
diff --git a/library/cpp/http/simple/ya.make b/library/cpp/http/simple/ya.make
index 40744675e8..6a4e5775a4 100644
--- a/library/cpp/http/simple/ya.make
+++ b/library/cpp/http/simple/ya.make
@@ -4,6 +4,7 @@ PEERDIR(
library/cpp/http/io
library/cpp/openssl/io
library/cpp/string_utils/url
+ library/cpp/threading/cancellation
)
SRCS(
diff --git a/library/cpp/threading/cancellation/README.md b/library/cpp/threading/cancellation/README.md
new file mode 100644
index 0000000000..8fb8ea616c
--- /dev/null
+++ b/library/cpp/threading/cancellation/README.md
@@ -0,0 +1,112 @@
+The Cancellation library
+========================
+
+Intro
+-----
+
+This small library provides primitives for implementation of a cooperative cancellation of long running or asynchronous operations.
+The design has been copied from the well-known CancellationTokenSource/CancellationToken classes of the .NET Framework
+
+To use the library include `cancellation_token.h`.
+
+Examples
+--------
+
+1. Simple check for cancellation
+
+ ```c++
+ void LongRunningOperation(TCancellationToken token) {
+ ...
+ if (token.IsCancellationRequested()) {
+ return;
+ }
+ ...
+ }
+
+ TCancellationTokenSource source;
+ TThread thread([token = source.Token()]() { LongRunningOperation(std::move(token)); });
+ thread.Start();
+ ...
+ source.Cancel();
+ thread.Join();
+ ```
+
+2. Exit via an exception
+
+ ```c++
+ void LongRunningOperation(TCancellationToken token) {
+ try {
+ for (;;) {
+ ...
+ token.ThrowIfCancellationRequested();
+ ...
+ }
+ } catch (TOperationCancelledException const&) {
+ return;
+ } catch (...) {
+ Y_ABORT("Never should be there")
+ }
+ }
+
+ TCancellationTokenSource source;
+ TThread thread([token = source.Token()]() { LongRunningOperation(std::move(token)); });
+ thread.Start();
+ ...
+ source.Cancel();
+ thread.Join();
+ ```
+
+3. Periodic poll with cancellation
+
+ ```c++
+ void LongRunningOperation(TCancellationToken token) {
+ while (!token.Wait(PollInterval)) {
+ ...
+ }
+ }
+
+ TCancellationTokenSource source;
+ TThread thread([token = source.Token()]() { LongRunningOperation(std::move(token)); });
+ thread.Start();
+ ...
+ source.Cancel();
+ thread.Join();
+ ```
+
+4. Waiting on the future
+
+ ```c++
+ TFuture<void> InnerOperation();
+ TFuture<void> OuterOperation(TCancellationToken token) {
+ return WaitAny(FirstOperation(), token.Future())
+ .Apply([token = std::move(token)](auto&&) {
+ token.ThrowIfCancellationRequested();
+ });
+ }
+
+ TCancellationTokenSource source;
+ auto future = OuterOperation();
+ ...
+ source.Cancel()
+ ...
+ try {
+ auto value = future.ExtractValueSync();
+ } catch (TOperationCancelledException const&) {
+ // cancelled
+ }
+ ```
+
+5. Using default token when no cancellation needed
+
+ ```c++
+ void LongRunningOperation(TCancellationToken token) {
+ ...
+ if (token.IsCancellationRequested()) {
+ return;
+ }
+ ...
+ }
+
+ // We do not want to cancel the operation. So, there is no need to create a cancellation token source
+ LongRunningOperation(TCancellationToken::Default);
+ ```
diff --git a/library/cpp/threading/cancellation/cancellation_token.cpp b/library/cpp/threading/cancellation/cancellation_token.cpp
new file mode 100644
index 0000000000..1a0a19f690
--- /dev/null
+++ b/library/cpp/threading/cancellation/cancellation_token.cpp
@@ -0,0 +1 @@
+#include "cancellation_token.h"
diff --git a/library/cpp/threading/cancellation/cancellation_token.h b/library/cpp/threading/cancellation/cancellation_token.h
new file mode 100644
index 0000000000..7615965b14
--- /dev/null
+++ b/library/cpp/threading/cancellation/cancellation_token.h
@@ -0,0 +1,133 @@
+#pragma once
+
+#include "operation_cancelled_exception.h"
+
+#include <library/cpp/threading/future/future.h>
+
+#include <util/generic/ptr.h>
+#include <util/generic/singleton.h>
+
+namespace NThreading {
+
+class TCancellationTokenSource;
+
+//! A cancellation token could be passed to an async or long running operation to perform a cooperative operation cancel
+class TCancellationToken {
+private:
+ TFuture<void> Future_;
+ TInstant Deadline_ = TInstant::Max();
+
+public:
+ TCancellationToken() = delete;
+ TCancellationToken(const TCancellationToken&) noexcept = default;
+ TCancellationToken(TCancellationToken&&) noexcept = default;
+ TCancellationToken& operator = (const TCancellationToken&) noexcept = default;
+ TCancellationToken& operator = (TCancellationToken&&) noexcept = default;
+
+ //! Shows whether a cancellation has been requested
+ bool IsCancellationRequested() const {
+ return Future_.HasValue();
+ }
+
+ //! Shows whether a cancellation has been requested
+ bool IsDeadlineReached() const {
+ return TInstant::Now() > Deadline_;
+ }
+
+ //! Throws the TOperationCancelledException if a cancellation has been requested
+ void ThrowIfCancellationRequested() const {
+ if (IsCancellationRequested()) {
+ ythrow TOperationCancelledException();
+ }
+ }
+
+ //! Throws the TOperationCancelledException if the Deadline_ has been reached
+ void ThrowIfDeadlineReached() const {
+ if (IsDeadlineReached()) {
+ ythrow TOperationCancelledException();
+ }
+ }
+
+ //! Throws the TOperationCancelledException if a cancellation has been requested
+ // or reached the Deadline_
+ void ThrowIfTokenCancelled() const {
+ ThrowIfCancellationRequested();
+ ThrowIfDeadlineReached();
+ }
+
+ //! Waits for a cancellation
+ bool Wait(TDuration duration) const {
+ return Future_.Wait(duration);
+ }
+
+ bool Wait(TInstant deadline) const {
+ return Future_.Wait(deadline);
+ }
+
+ void Wait() const {
+ return Future_.Wait();
+ }
+
+ //! Returns a future that could be used for waiting for a cancellation
+ TFuture<void> const& Future() const noexcept {
+ return Future_;
+ }
+
+ //! The default cancellation token that cannot be cancelled
+ static TCancellationToken const& Default() {
+ return *SingletonWithPriority<TCancellationToken, 0>(NewPromise());
+ }
+
+ void SetDeadline(TInstant deadline) {
+ Deadline_ = deadline;
+ }
+
+ TInstant GetDeadline() const {
+ return Deadline_;
+ }
+
+private:
+ TCancellationToken(TFuture<void> future)
+ : Future_(std::move(future))
+ {
+ }
+
+private:
+ friend class TCancellationTokenSource;
+
+ Y_DECLARE_SINGLETON_FRIEND();
+};
+
+//! A cancellation token source produces cancellation tokens to be passed to cancellable operations
+class TCancellationTokenSource {
+private:
+ TPromise<void> Promise;
+
+public:
+ TCancellationTokenSource()
+ : Promise(NewPromise())
+ {
+ }
+
+ TCancellationTokenSource(TCancellationTokenSource const&) = delete;
+ TCancellationTokenSource(TCancellationTokenSource&&) = delete;
+ TCancellationTokenSource& operator=(TCancellationTokenSource const&) = delete;
+ TCancellationTokenSource& operator=(TCancellationTokenSource&&) = delete;
+
+ //! Shows whether a cancellation has been requested
+ bool IsCancellationRequested() const noexcept {
+ return Promise.HasValue();
+ }
+
+ //! Produces a cancellation token
+ TCancellationToken Token() const {
+ return TCancellationToken(Promise.GetFuture());
+ }
+
+ //! Propagates a cancel request to all produced tokens
+ void Cancel() noexcept {
+ Promise.TrySetValue();
+ }
+};
+
+}
diff --git a/library/cpp/threading/cancellation/ya.make b/library/cpp/threading/cancellation/ya.make
new file mode 100644
index 0000000000..e9988f0f08
--- /dev/null
+++ b/library/cpp/threading/cancellation/ya.make
@@ -0,0 +1,11 @@
+LIBRARY()
+
+PEERDIR(
+ library/cpp/threading/future
+)
+
+SRCS(
+ cancellation_token.cpp
+)
+
+END()
diff --git a/util/datetime/systime.cpp b/util/datetime/systime.cpp
index 04972b6e48..8bd20c35ff 100644
--- a/util/datetime/systime.cpp
+++ b/util/datetime/systime.cpp
@@ -124,36 +124,13 @@ namespace {
constexpr ui16 MONTH_TO_DAYS_LEAP[12] = {
0, 31, 60, 91, 121, 152, 182, 213, 244, 274, 305, 335};
- struct TMonth32LUT {
- ui8 LastMonthDay32[12];
- ui8 FirstMonthDay32[12];
- };
-
- constexpr TMonth32LUT COMMON_YEAR = {
- .LastMonthDay32 = {31, 27, 26, 24, 23, 21, 20, 19, 17, 16, 14, 13},
- .FirstMonthDay32 = {0, 1, 5, 6, 8, 9, 11, 12, 13, 15, 16, 18},
- };
-
constexpr int DayOfYearToMonth(ui32& yearDay, const bool leapYear) {
- Y_ASSERT(yearDay < DAYS_IN_YEAR + leapYear);
- if (leapYear) {
- if (yearDay > 59) {
- --yearDay;
- } else if (yearDay == 59) {
- // February, 29th
- yearDay = 28;
- return 1;
- }
+ if (yearDay >= 31 + 28 + leapYear) {
+ yearDay += 2 - leapYear;
}
- const int approxMonth = static_cast<int>(yearDay / 32);
- const int approxMDay = static_cast<int>(yearDay % 32);
- const int dayThreshold = COMMON_YEAR.LastMonthDay32[approxMonth];
- const int currentMonthMDayOffset = COMMON_YEAR.FirstMonthDay32[approxMonth];
- const bool nextMonth = (approxMDay >= dayThreshold);
- const int dayCorrection = nextMonth ? -dayThreshold : currentMonthMDayOffset;
- yearDay = approxMDay + dayCorrection;
- const int month = approxMonth + nextMonth;
- return month;
+ const ui32 month = (yearDay * 67 + 35) >> 11;
+ yearDay -= (month * 489 + 8) >> 4;
+ return static_cast<int>(month);
}
class TDayNoToYearLookupTable {
diff --git a/yql/essentials/cfg/tests/gateways.conf b/yql/essentials/cfg/tests/gateways.conf
index ad271cd26e..a2e09f89fb 100644
--- a/yql/essentials/cfg/tests/gateways.conf
+++ b/yql/essentials/cfg/tests/gateways.conf
@@ -72,6 +72,11 @@ Yt {
Value: "true"
}
+ DefaultSettings {
+ Name: "ColumnGroupMode"
+ Value: "perusage"
+ }
+
RemoteFilePatterns {
Pattern: "yt://([a-zA-Z0-9\\-_]+)/([^&@?]+)$"
Cluster: "$1"
diff --git a/yql/essentials/core/common_opt/yql_flatmap_over_join.cpp b/yql/essentials/core/common_opt/yql_flatmap_over_join.cpp
index efc69c5cb8..c2b3da4867 100644
--- a/yql/essentials/core/common_opt/yql_flatmap_over_join.cpp
+++ b/yql/essentials/core/common_opt/yql_flatmap_over_join.cpp
@@ -1,5 +1,4 @@
#include "yql_flatmap_over_join.h"
-#include "yql_co.h"
#include <yql/essentials/core/yql_expr_optimize.h>
#include <yql/essentials/core/yql_expr_type_annotation.h>
@@ -8,6 +7,8 @@
#include <yql/essentials/utils/log/log.h>
+#include <library/cpp/disjoint_sets/disjoint_sets.h>
+
namespace NYql {
using namespace NNodes;
@@ -922,6 +923,373 @@ bool NeedEmitSkipNullMembers(const TTypeAnnotationContext* types) {
return true;
}
+bool IsEqualityFilterOverJoinEnabled(const TTypeAnnotationContext* types) {
+ YQL_ENSURE(types);
+ static const char flag[] = "EqualityFilterOverJoin";
+ return IsOptimizerEnabled<flag>(*types) && !IsOptimizerDisabled<flag>(*types);
+}
+
+struct TExtraInputPredicates {
+ TExprNode::TPtr Row;
+ TExprNodeList Preds;
+ TString MainColumn;
+};
+
+void AppendEquality(TPositionHandle pos, TExtraInputPredicates& dst, const TString& left, const TString& right, const TJoinLabel& label, TExprContext& ctx) {
+ if (!dst.Row) {
+ YQL_ENSURE(dst.Preds.empty());
+ dst.Row = ctx.NewArgument(pos, "row");
+ }
+
+ TStringBuf lTable = label.TableName(left);
+ TStringBuf rTable = label.TableName(right);
+ TStringBuf lColumn = label.ColumnName(left);
+ TStringBuf rColumn = label.ColumnName(right);
+
+ dst.Preds.push_back(ctx.Builder(pos)
+ .Callable("Coalesce")
+ .Callable(0, "==")
+ .Callable(0, "Member")
+ .Add(0, dst.Row)
+ .Atom(1, label.MemberName(lTable, lColumn))
+ .Seal()
+ .Callable(1, "Member")
+ .Add(0, dst.Row)
+ .Atom(1, label.MemberName(rTable, rColumn))
+ .Seal()
+ .Seal()
+ .Add(1, MakeBool<false>(pos, ctx))
+ .Seal()
+ .Build()
+ );
+}
+
+struct TJoinEqRebuildResult {
+ TExprNode::TPtr JoinTree;
+ TSet<ui32> InputsInScope;
+};
+
+struct TEqColumn {
+ TString Name;
+ size_t UseCount = 0;
+};
+
+template<typename T, typename U>
+bool HasIntersection(const T& a, const U& b) {
+ return AnyOf(a, [&b](const auto& item) { return b.contains(item); });
+}
+
+TVector<ui32> FilterByScope(const TMap<ui32, TEqColumn>& input, const TSet<ui32>& scope) {
+ TVector<ui32> result;
+ for (auto [i, _] : input) {
+ if (scope.contains(i)) {
+ result.push_back(i);
+ }
+ }
+ return result;
+}
+
+TJoinEqRebuildResult RebuildJoinTreeForEquality(TVector<TMap<ui32, TEqColumn>>& equalitySetsByInput, const THashSet<ui32>& notNullInputs, const TJoinLabels& labels, TCoEquiJoinTuple joinTree, TExprContext& ctx) {
+ const TStringBuf joinType = joinTree.Type().Value();
+
+ TJoinEqRebuildResult left;
+ if (joinType != "RightOnly" && joinType != "RightSemi") {
+ if (auto maybeAtom = joinTree.LeftScope().Maybe<TCoAtom>()) {
+ left.JoinTree = joinTree.LeftScope().Ptr();
+ auto inputIdx = labels.FindInputIndex(maybeAtom.Cast().Value());
+ YQL_ENSURE(inputIdx);
+ left.InputsInScope.insert(*inputIdx);
+ } else {
+ left = RebuildJoinTreeForEquality(equalitySetsByInput, notNullInputs, labels, joinTree.LeftScope().Cast<TCoEquiJoinTuple>(), ctx);
+ }
+ } else {
+ left.JoinTree = joinTree.LeftScope().Ptr();
+ }
+
+ TJoinEqRebuildResult right;
+ if (joinType != "LeftOnly" && joinType != "LeftSemi") {
+ if (auto maybeAtom = joinTree.RightScope().Maybe<TCoAtom>()) {
+ right.JoinTree = joinTree.RightScope().Ptr();
+ auto inputIdx = labels.FindInputIndex(maybeAtom.Cast().Value());
+ YQL_ENSURE(inputIdx);
+ right.InputsInScope.insert(*inputIdx);
+ } else {
+ right = RebuildJoinTreeForEquality(equalitySetsByInput, notNullInputs, labels, joinTree.RightScope().Cast<TCoEquiJoinTuple>(), ctx);
+ }
+ } else {
+ right.JoinTree = joinTree.RightScope().Ptr();
+ }
+
+ YQL_ENSURE(!HasIntersection(left.InputsInScope, right.InputsInScope));
+
+ if (joinType == "Exclusion" || !left.JoinTree || !right.JoinTree) {
+ // TODO: support equality over exclustion join
+ return {};
+ }
+
+ const bool leftNotNull = HasIntersection(left.InputsInScope, notNullInputs);
+ const bool rightNotNull = HasIntersection(right.InputsInScope, notNullInputs);
+
+ TStringBuf newJoinType = joinType;
+ if (joinType == "Full") {
+ if (leftNotNull && rightNotNull) {
+ newJoinType = "Inner";
+ } else if (leftNotNull) {
+ newJoinType = "Left";
+ } else if (rightNotNull) {
+ newJoinType = "Right";
+ }
+ } else if (joinType == "Left" && rightNotNull || joinType == "Right" && leftNotNull) {
+ newJoinType = "Inner";
+ }
+
+ TExprNodeList leftKeys = joinTree.LeftKeys().Ref().ChildrenList();
+ TExprNodeList rightKeys = joinTree.RightKeys().Ref().ChildrenList();
+ for (auto& es : equalitySetsByInput) {
+ auto leftInputs = FilterByScope(es, left.InputsInScope);
+ auto rightInputs = FilterByScope(es, right.InputsInScope);
+
+ const size_t sz = std::min(leftInputs.size(), rightInputs.size());
+ for (size_t i = 0; i < sz; ++i) {
+ auto lIdx = leftInputs[i];
+ auto rIdx = rightInputs[i];
+ if (es[lIdx].UseCount && es[rIdx].UseCount) {
+ continue;
+ }
+ es[lIdx].UseCount++;
+ es[rIdx].UseCount++;
+
+ TStringBuf table;
+ TStringBuf column;
+
+ SplitTableName(es[lIdx].Name, table, column);
+ leftKeys.emplace_back(ctx.NewAtom(joinTree.LeftKeys().Pos(), table));
+ leftKeys.emplace_back(ctx.NewAtom(joinTree.LeftKeys().Pos(), column));
+
+ SplitTableName(es[rIdx].Name, table, column);
+ rightKeys.emplace_back(ctx.NewAtom(joinTree.RightKeys().Pos(), table));
+ rightKeys.emplace_back(ctx.NewAtom(joinTree.RightKeys().Pos(), column));
+
+ if (newJoinType == "Cross") {
+ newJoinType = "Inner";
+ }
+ }
+ }
+
+ TJoinEqRebuildResult result;
+ result.JoinTree = Build<TCoEquiJoinTuple>(ctx, joinTree.Pos())
+ .Type().Build(newJoinType)
+ .LeftScope(left.JoinTree)
+ .RightScope(right.JoinTree)
+ .LeftKeys(ctx.NewList(joinTree.LeftKeys().Pos(), std::move(leftKeys)))
+ .RightKeys(ctx.NewList(joinTree.RightKeys().Pos(), std::move(rightKeys)))
+ .Options(joinTree.Options())
+ .Done().Ptr();
+ if (newJoinType != "RightSemi" && newJoinType != "RightOnly") {
+ result.InputsInScope.insert(left.InputsInScope.begin(), left.InputsInScope.end());
+ }
+ if (newJoinType != "LeftSemit" && newJoinType != "LeftOnly") {
+ result.InputsInScope.insert(right.InputsInScope.begin(), right.InputsInScope.end());
+ }
+ return result;
+}
+
+
+TExprBase HandleEqualityFilterOverJoin(const TCoFlatMapBase& node, const TJoinLabels& labels,
+ const THashMap<TString, TString>& backRenameMap, TExprContext& ctx)
+{
+ const auto& row = node.Lambda().Args().Arg(0).Ref();
+ auto predicate = node.Lambda().Body().Ref().ChildPtr(0);
+
+ TExprNodeList andComponents;
+ if (predicate->IsCallable("And")) {
+ andComponents = predicate->ChildrenList();
+ } else {
+ andComponents.push_back(predicate);
+ }
+
+ TExprNodeList rest;
+
+ TVector<TString> columns;
+ TVector<TString> uniqColumns;
+ THashMap<TString, size_t> column2id;
+ THashSet<ui32> makeNoNullInputs;
+ for (auto pred : andComponents) {
+ TExprNode::TPtr left, right;
+ if (!IsEquality(pred, left, right) ||
+ !left->IsCallable("Member") || left->Child(0) != &row ||
+ !right->IsCallable("Member") || right->Child(0) != &row)
+ {
+ rest.push_back(pred);
+ continue;
+ }
+
+ TString leftCol{left->Child(1)->Content()};
+ if (auto it = backRenameMap.find(leftCol); it != backRenameMap.end()) {
+ leftCol = it->second;
+ }
+
+ TString rightCol{right->Child(1)->Content()};
+ if (auto it = backRenameMap.find(rightCol); it != backRenameMap.end()) {
+ rightCol = it->second;
+ }
+
+ if (leftCol == rightCol) {
+ // TODO: add optimizer for "==" over same arguments with optional types
+ rest.push_back(pred);
+ continue;
+ }
+
+ TStringBuf leftTable, rightTable;
+ TStringBuf column;
+ SplitTableName(leftCol, leftTable, column);
+ SplitTableName(rightCol, rightTable, column);
+
+ const auto leftInput = labels.FindInputIndex(leftTable);
+ YQL_ENSURE(leftInput);
+ const auto rightInput = labels.FindInputIndex(rightTable);
+ YQL_ENSURE(rightInput);
+
+ makeNoNullInputs.insert(*leftInput);
+ makeNoNullInputs.insert(*rightInput);
+
+ auto processColumn = [&](const TString& col) {
+ columns.push_back(col);
+ if (column2id.insert({ col, uniqColumns.size() }).second) {
+ uniqColumns.push_back(col);
+ }
+ };
+
+ processColumn(leftCol);
+ processColumn(rightCol);
+ }
+
+ if (columns.empty()) {
+ return node;
+ }
+
+ YQL_ENSURE(columns.size() % 2 == 0);
+
+ TDisjointSets ds(uniqColumns.size());
+ for (size_t i = 0; i < columns.size(); i += 2) {
+ ds.UnionSets(column2id[columns[i]], column2id[columns[i + 1]]);
+ }
+
+ TVector<TSet<TString>> equalitySets(uniqColumns.size());
+ for (const auto& col : uniqColumns) {
+ equalitySets[ds.CanonicSetElement(column2id[col])].insert(col);
+ }
+
+ EraseIf(equalitySets, [](const auto& s) { return s.empty(); });
+ YQL_ENSURE(!equalitySets.empty());
+
+ const TCoEquiJoin equiJoin = node.Input().Cast<TCoEquiJoin>();
+ const size_t inputsCount = equiJoin.ArgCount() - 2;
+ YQL_ENSURE(labels.Inputs.size() == inputsCount);
+
+ TVector<TMap<ui32, TEqColumn>> equalitySetsByInput; // single column for each input (other instances are pushed directly to input)
+ TVector<TExtraInputPredicates> extraInputPreds(inputsCount);
+
+ for (const TSet<TString>& eqSet : equalitySets) {
+ TMap<ui32, TEqColumn>& eqSetByInput = equalitySetsByInput.emplace_back();
+ for (const auto& col : eqSet) {
+ YQL_ENSURE(!col.empty());
+
+ TStringBuf table;
+ TStringBuf column;
+ SplitTableName(col, table, column);
+ auto idx = labels.FindInputIndex(table);
+ YQL_ENSURE(idx && *idx < inputsCount);
+
+ auto it = eqSetByInput.find(*idx);
+ if (it != eqSetByInput.end()) {
+ YQL_ENSURE(col != it->second.Name);
+ const auto& label = labels.Inputs[*idx];
+ extraInputPreds[*idx].MainColumn = it->second.Name;
+ AppendEquality(predicate->Pos(), extraInputPreds[*idx], it->second.Name, col, label, ctx);
+ } else {
+ eqSetByInput.insert({*idx, {col, 0}});
+ }
+ }
+ }
+
+ auto res = RebuildJoinTreeForEquality(equalitySetsByInput, makeNoNullInputs, labels, equiJoin.Arg(inputsCount).Cast<TCoEquiJoinTuple>(), ctx);
+ if (!res.JoinTree) {
+ return node;
+ }
+
+ for (const TMap<ui32, TEqColumn>& es : equalitySetsByInput) {
+ for (auto& [_, eqCol] : es) {
+ YQL_ENSURE(eqCol.UseCount || AnyOf(extraInputPreds, [&](const TExtraInputPredicates& item) { return item.MainColumn == eqCol.Name; } ));
+ }
+ }
+
+ YQL_CLOG(DEBUG, Core) << "Equality filter over EquiJoin: processed " << (columns.size() / 2) << " predicates";
+
+ TExprNodeList equiJoinArgs = equiJoin.Ref().ChildrenList();
+ equiJoinArgs[inputsCount] = res.JoinTree;
+ for (size_t i = 0; i < inputsCount; ++i) {
+ auto& toPush = extraInputPreds[i];
+ if (toPush.Preds.empty()) {
+ continue;
+ }
+ YQL_ENSURE(toPush.Row);
+
+ const auto pos = toPush.Row->Pos();
+ TExprNode::TPtr pred = ctx.NewCallable(pos, "And", std::move(toPush.Preds));
+
+ TExprNode::TPtr& inputTuple = equiJoinArgs[i];
+ TExprNode::TPtr oldInput = inputTuple->ChildPtr(TCoEquiJoinInput::idx_List);
+ auto newInput = ctx.Builder(oldInput->Pos())
+ .Callable("OrderedFilter")
+ .Add(0, oldInput)
+ .Add(1, ctx.NewLambda(pos, ctx.NewArguments(pos, { toPush.Row }), std::move(pred)))
+ .Seal()
+ .Build();
+
+ inputTuple = ctx.ChangeChild(*inputTuple, TCoEquiJoinInput::idx_List, std::move(newInput));
+ }
+
+ auto origJoinItemTypeNode = ExpandType(equiJoin.Pos(), *GetSeqItemType(*node.Input().Ref().GetTypeAnn()).Cast<TStructExprType>(), ctx);
+ auto newEquiJoin = ctx.Builder(equiJoin.Pos())
+ .Callable(node.CallableName() == "OrderedFlatMap" ? "OrderedMap" : "Map")
+ .Add(0, ctx.NewCallable(equiJoin.Pos(), "EquiJoin", std::move(equiJoinArgs)))
+ .Lambda(1)
+ .Param("row")
+ .Callable("EnsureType")
+ .Callable(0, "SafeCast")
+ .Arg(0, "row")
+ .Add(1, origJoinItemTypeNode)
+ .Seal()
+ .Add(1, origJoinItemTypeNode)
+ .Atom(2, "Mismatch type while performing Equality over EquiJoin optimizer")
+ .Seal()
+ .Seal()
+ .Seal()
+ .Build();
+
+ if (rest.empty()) {
+ rest.push_back(MakeBool<true>(predicate->Pos(), ctx));
+ }
+
+ YQL_ENSURE(TCoConditionalValueBase::Match(node.Lambda().Body().Raw()));
+ auto newPred = ctx.NewCallable(predicate->Pos(), "And", std::move(rest));
+ auto newCond = ctx.ChangeChild(node.Lambda().Body().Ref(), TCoConditionalValueBase::idx_Predicate, std::move(newPred));
+ auto newLambda = ctx.ChangeChild(node.Lambda().Ref(), TCoLambda::idx_Body, std::move(newCond));
+
+ return TExprBase(ctx.Builder(node.Pos())
+ .Callable(node.CallableName())
+ .Add(0, newEquiJoin)
+ .Lambda(1)
+ .Param("row")
+ .Apply(newLambda)
+ .With(0, "row")
+ .Seal()
+ .Seal()
+ .Seal()
+ .Build());
+}
+
} // namespace
TExprBase FlatMapOverEquiJoin(
@@ -1046,13 +1414,8 @@ TExprBase FlatMapOverEquiJoin(
}
}
- TExprNode::TListType andTerms;
- bool isPg;
- GatherAndTerms(predicate, andTerms, isPg, ctx);
- TExprNode::TPtr ret;
- TExprNode::TPtr extraPredicate;
- auto joinSettings = equiJoin.Ref().Child(equiJoin.Ref().ChildrenSize() - 1);
- auto renameMap = LoadJoinRenameMap(*joinSettings);
+ const auto joinSettings = equiJoin.Ref().Child(equiJoin.Ref().ChildrenSize() - 1);
+ const auto renameMap = LoadJoinRenameMap(*joinSettings);
THashMap<TString, TString> backRenameMap;
for (auto& x : renameMap) {
if (!x.second.empty()) {
@@ -1062,10 +1425,23 @@ TExprBase FlatMapOverEquiJoin(
}
}
+ if (IsEqualityFilterOverJoinEnabled(types)) {
+ auto newNode = HandleEqualityFilterOverJoin(node, labels, backRenameMap, ctx);
+ if (newNode.Raw() != node.Raw()) {
+ return newNode;
+ }
+ }
+
+ TExprNode::TListType andTerms;
+ bool isPg;
+ GatherAndTerms(predicate, andTerms, isPg, ctx);
+ TExprNode::TPtr ret;
+ TExprNode::TPtr extraPredicate;
+
const bool ordered = node.Maybe<TCoOrderedFlatMap>().IsValid();
const bool skipNulls = NeedEmitSkipNullMembers(types);
- for (auto& andTerm : andTerms) {
+ for (const auto& andTerm : andTerms) {
if (andTerm->IsCallable("Likely")) {
continue;
}
@@ -1101,7 +1477,7 @@ TExprBase FlatMapOverEquiJoin(
}
}
- if (inputs.size() == 2) {
+ if (!IsEqualityFilterOverJoinEnabled(types) && inputs.size() == 2) {
auto newJoin = DecayCrossJoinIntoInner(equiJoin.Ptr(), andTerm,
labels, *inputs.begin(), *(++inputs.begin()), row, backRenameMap, parentsMap, ctx, types->RotateJoinTree);
if (newJoin != equiJoin.Ptr()) {
diff --git a/yql/essentials/core/ut/yql_expr_constraint_ut.cpp b/yql/essentials/core/ut/yql_expr_constraint_ut.cpp
index e9b3efd69f..45d9f6285d 100644
--- a/yql/essentials/core/ut/yql_expr_constraint_ut.cpp
+++ b/yql/essentials/core/ut/yql_expr_constraint_ut.cpp
@@ -3316,6 +3316,52 @@ Y_UNIT_TEST_SUITE(TYqlExprConstraints) {
CheckConstraint<TDistinctConstraintNode>(exprRoot, "PartitionsByKeys", "Distinct((data,group0))");
CheckConstraint<TUniqueConstraintNode>(exprRoot, "PartitionsByKeys", "Unique((data,group0))");
}
+
+ Y_UNIT_TEST(StablePickleOfComplexUnique) {
+ const TStringBuf s = R"(
+(
+ (let config (DataSource 'config))
+ (let res_sink (DataSink 'result))
+
+ (let list (AsList
+ (AsStruct '('key (Uint32 '1)) '('value (Uint32 '2)))
+ (AsStruct '('key (Uint32 '2)) '('value (Uint32 '3)))
+ ))
+
+ (let res (Aggregate list '('key 'value) '() '()))
+ (let res (Map res (lambda '(item)
+ (AsStruct
+ '('composite (AsStruct
+ '('k (Member item 'key))
+ '('v (Member item 'value))
+ ))
+ '('key (Member item 'key))
+ '('value (Member item 'value))
+ )
+ )))
+ (let res (FlatMap res (lambda '(item)
+ (Just (AsStruct
+ '('packed (StablePickle (Member item 'composite)))
+ '('composite (Member item 'composite))
+ '('key (Member item 'key))
+ '('value (Member item 'value))
+ ))
+ )))
+ (let world (Write! world res_sink (Key) res '('('type))))
+ (let world (Commit! world res_sink))
+ (return world)
+)
+ )";
+
+ TExprContext exprCtx;
+ const auto exprRoot = ParseAndAnnotate(s, exprCtx);
+ CheckConstraint<TDistinctConstraintNode>(exprRoot, "StablePickle", "");
+ CheckConstraint<TUniqueConstraintNode>(exprRoot, "StablePickle", "");
+ CheckConstraint<TDistinctConstraintNode>(exprRoot, "Map", "Distinct(({composite/k,key},{composite/v,value}))");
+ CheckConstraint<TUniqueConstraintNode>(exprRoot, "Map", "Unique(({composite/k,key},{composite/v,value}))");
+ CheckConstraint<TDistinctConstraintNode>(exprRoot, "FlatMap", "Distinct(({composite/k,key},{composite/v,value}))");
+ CheckConstraint<TUniqueConstraintNode>(exprRoot, "FlatMap", "Unique(({composite/k,key},{composite/v,value}))");
+ }
}
} // namespace NYql
diff --git a/yql/essentials/core/yql_expr_constraint.cpp b/yql/essentials/core/yql_expr_constraint.cpp
index 49997179f5..7921e31f97 100644
--- a/yql/essentials/core/yql_expr_constraint.cpp
+++ b/yql/essentials/core/yql_expr_constraint.cpp
@@ -248,7 +248,7 @@ public:
Functions["BlockMergeFinalizeHashed"] = &TCallableConstraintTransformer::AggregateWrap<true>;
Functions["BlockMergeManyFinalizeHashed"] = &TCallableConstraintTransformer::AggregateWrap<true>;
Functions["MultiHoppingCore"] = &TCallableConstraintTransformer::MultiHoppingCoreWrap;
- Functions["StablePickle"] = &TCallableConstraintTransformer::FromFirst<TUniqueConstraintNode, TPartOfUniqueConstraintNode, TDistinctConstraintNode, TPartOfDistinctConstraintNode, TPartOfChoppedConstraintNode, TVarIndexConstraintNode>;
+ Functions["StablePickle"] = &TCallableConstraintTransformer::PickleWrap;
Functions["Unpickle"] = &TCallableConstraintTransformer::FromSecond<TUniqueConstraintNode, TPartOfUniqueConstraintNode, TDistinctConstraintNode, TPartOfDistinctConstraintNode, TPartOfChoppedConstraintNode, TVarIndexConstraintNode>;
}
@@ -404,6 +404,14 @@ private:
return FromFirst<TEmptyConstraintNode, TUniqueConstraintNode, TDistinctConstraintNode, TVarIndexConstraintNode>(input, output, ctx);
}
+ TStatus PickleWrap(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) const {
+ if (IsDataOrOptionalOfDataOrPg(input->Head().GetTypeAnn())) {
+ TApplyConstraintFromInput<0, TPartOfChoppedConstraintNode, TPartOfUniqueConstraintNode, TPartOfDistinctConstraintNode>::Do(input);
+ }
+
+ return FromFirst<TUniqueConstraintNode, TDistinctConstraintNode, TVarIndexConstraintNode>(input, output, ctx);
+ }
+
TStatus AssumeConstraintsWrap(const TExprNode::TPtr& input, TExprNode::TPtr& /*output*/, TExprContext& ctx) const {
TConstraintSet set;
try {
diff --git a/yql/essentials/minikql/mkql_alloc.cpp b/yql/essentials/minikql/mkql_alloc.cpp
index 539a3349ed..963f46a67e 100644
--- a/yql/essentials/minikql/mkql_alloc.cpp
+++ b/yql/essentials/minikql/mkql_alloc.cpp
@@ -309,7 +309,7 @@ void MKQLArrowFree(const void* mem, ui64 size) {
Y_ENSURE(size == header->Size);
-#if defined(ALLOW_MEMORY_ALLOCATOR)
+#if defined(ALLOW_DEFAULT_ALLOCATOR)
if (Y_UNLIKELY(TAllocState::IsDefaultAllocatorUsed())) {
free(header);
return;
diff --git a/yql/essentials/minikql/mkql_type_ops.cpp b/yql/essentials/minikql/mkql_type_ops.cpp
index 8fba53831a..8900eba74b 100644
--- a/yql/essentials/minikql/mkql_type_ops.cpp
+++ b/yql/essentials/minikql/mkql_type_ops.cpp
@@ -2223,8 +2223,8 @@ NUdf::TUnboxedValuePod ParseTzTimestamp(NUdf::TStringRef str) {
}
}
-template <bool DecimalPart = false, i8 MaxDigits = 6>
-bool ParseNumber(std::string_view::const_iterator& pos, const std::string_view& buf, ui32& value) {
+template <bool DecimalPart, ui8 MaxDigits>
+bool ParseNumber(std::string_view::const_iterator& pos, const std::string_view& buf, ui64& value) {
value = 0U;
if (buf.cend() == pos || !std::isdigit(*pos)) {
@@ -2263,10 +2263,16 @@ NUdf::TUnboxedValuePod ParseInterval(const std::string_view& buf) {
return NUdf::TUnboxedValuePod();
}
- std::optional<ui32> days, hours, minutes, seconds, microseconds;
- ui32 num;
+ std::optional<ui64> days, hours, minutes, seconds, microseconds;
+ ui64 num;
if (*pos != 'T') {
+ // Estimated upper bound for number of digits in the
+ // numeric representation of days (weeks need less).
+ // * Interval: MAX_DATE (49673) = 5 digits.
+ // * Interval64: MAX_DATE32 - MIN_DATE32 (106751616) = 9 digits.
+ // So, 9 digits is maximum for any interval component with
+ // granularity more than a day.
if (!ParseNumber<false, 9>(pos, buf, num)) {
return NUdf::TUnboxedValuePod();
}
@@ -2289,7 +2295,14 @@ NUdf::TUnboxedValuePod ParseInterval(const std::string_view& buf) {
if (buf.cend() != pos) // TODO: Remove this line later.
do {
- if (!ParseNumber(pos, buf, num)) {
+ // Estimated upper bound for number of digits in the
+ // numeric representation of seconds (hours, minutes,
+ // microseconds need less).
+ // * Interval: MAX_DATETIME (4291747200) = 10 digits.
+ // * Interval64: MAX_DATETIME64 - MIN_DATETIME64 (9223339708799) = 13 digits.
+ // So, 13 digits is maximum for any interval component with
+ // granularity less than a day.
+ if (!ParseNumber<false, 13>(pos, buf, num)) {
return NUdf::TUnboxedValuePod();
}
@@ -2317,7 +2330,8 @@ NUdf::TUnboxedValuePod ParseInterval(const std::string_view& buf) {
return NUdf::TUnboxedValuePod();
}
seconds = num;
- if (!ParseNumber<true>(pos, buf, num) || *pos++ != 'S') {
+ // 6 digits is maximum for microseconds representation.
+ if (!ParseNumber<true, 6>(pos, buf, num) || *pos++ != 'S') {
return NUdf::TUnboxedValuePod();
}
microseconds = num;
diff --git a/yql/essentials/mount/lib/yql/aggregate.yqls b/yql/essentials/mount/lib/yql/aggregate.yqls
index e687adbf80..3518d0af8d 100755
--- a/yql/essentials/mount/lib/yql/aggregate.yqls
+++ b/yql/essentials/mount/lib/yql/aggregate.yqls
@@ -382,8 +382,8 @@
# list_type:type
# doesn't support optional values
(let hyperloglog_traits_factory_raw (lambda '(list_type precision) (block '(
- (let init (lambda '(value parent) (NamedApply (Udf 'HyperLogLog.Create) '((Apply (Udf 'Digest.MurMurHash) (Pickle value)) precision) (AsStruct) (DependsOn parent))))
- (let update (lambda '(value state parent) (NamedApply (Udf 'HyperLogLog.AddValue) '(state (Apply (Udf 'Digest.MurMurHash) (Pickle value))) (AsStruct) (DependsOn parent))))
+ (let init (lambda '(value parent) (NamedApply (Udf 'HyperLogLog.Create) '((Apply (Udf 'Digest.MurMurHash) (StablePickle value)) precision) (AsStruct) (DependsOn parent))))
+ (let update (lambda '(value state parent) (NamedApply (Udf 'HyperLogLog.AddValue) '(state (Apply (Udf 'Digest.MurMurHash) (StablePickle value))) (AsStruct) (DependsOn parent))))
(let save (lambda '(state) (Apply (Udf 'HyperLogLog.Serialize) state)))
(let load (lambda '(state) (Apply (Udf 'HyperLogLog.Deserialize) state)))
(let merge (lambda '(one two) (Apply (Udf 'HyperLogLog.Merge) one two)))
diff --git a/yql/essentials/mount/lib/yql/window.yqls b/yql/essentials/mount/lib/yql/window.yqls
index dd1f537bc9..32c47b2dfd 100755
--- a/yql/essentials/mount/lib/yql/window.yqls
+++ b/yql/essentials/mount/lib/yql/window.yqls
@@ -368,8 +368,8 @@
# list_type:type
# doesn't support optional values
(let hyperloglog_traits_factory_raw (lambda '(list_type n) (block '(
- (let init (lambda '(value parent) (NamedApply (Udf 'HyperLogLog.Create) '((Apply (Udf 'Digest.MurMurHash) (Pickle value)) n) (AsStruct) (DependsOn parent))))
- (let update (lambda '(value state parent) (NamedApply (Udf 'HyperLogLog.AddValue) '(state (Apply (Udf 'Digest.MurMurHash) (Pickle value))) (AsStruct) (DependsOn parent))))
+ (let init (lambda '(value parent) (NamedApply (Udf 'HyperLogLog.Create) '((Apply (Udf 'Digest.MurMurHash) (StablePickle value)) n) (AsStruct) (DependsOn parent))))
+ (let update (lambda '(value state parent) (NamedApply (Udf 'HyperLogLog.AddValue) '(state (Apply (Udf 'Digest.MurMurHash) (StablePickle value))) (AsStruct) (DependsOn parent))))
(let shift (lambda '(value state) (Void)))
(let current (lambda '(state) (Apply (Udf 'HyperLogLog.GetResult) state)))
(return (WindowTraits (ListItemType list_type) init update shift current (Null)))
diff --git a/yql/essentials/public/fastcheck/format.cpp b/yql/essentials/public/fastcheck/format.cpp
index 570073f539..26a1bce2ca 100644
--- a/yql/essentials/public/fastcheck/format.cpp
+++ b/yql/essentials/public/fastcheck/format.cpp
@@ -13,6 +13,16 @@ namespace {
constexpr size_t FormatContextLimit = 100;
+TString NormalizeEOL(TStringBuf input) {
+ TStringBuilder res;
+ TStringBuf tok;
+ while (input.ReadLine(tok)) {
+ res << tok << '\n';
+ }
+
+ return res;
+}
+
class TFormatRunner : public ICheckRunner {
public:
TString GetCheckName() const final {
@@ -66,7 +76,7 @@ private:
auto formatter = NSQLFormat::MakeSqlFormatter(lexers, parsers, settings);
TString formattedQuery;
res.Success = formatter->Format(request.Program, formattedQuery, res.Issues);
- if (res.Success && formattedQuery != request.Program) {
+ if (res.Success && formattedQuery != NormalizeEOL(request.Program)) {
res.Success = false;
TPosition origPos(0, 1, request.File);
TTextWalker origWalker(origPos, true);
diff --git a/yql/essentials/public/fastcheck/linter_ut.cpp b/yql/essentials/public/fastcheck/linter_ut.cpp
index cb635e8b54..be44f88c50 100644
--- a/yql/essentials/public/fastcheck/linter_ut.cpp
+++ b/yql/essentials/public/fastcheck/linter_ut.cpp
@@ -103,6 +103,20 @@ Y_UNIT_TEST_SUITE(TLinterTests) {
UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(), 0);
}
+ Y_UNIT_TEST(GoodFormatYqlWithWinEOL) {
+ TChecksRequest request;
+ request.Program = "SELECT\r\n 1\r\n;\r\n";
+ request.Syntax = ESyntax::YQL;
+ request.Filters.ConstructInPlace();
+ request.Filters->push_back(TCheckFilter{.CheckNameGlob = "format"});
+ auto res = RunChecks(request);
+ UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "format");
+ UNIT_ASSERT(res.Checks[0].Success);
+ Cerr << res.Checks[0].Issues.ToString();
+ UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(), 0);
+ }
+
Y_UNIT_TEST(UnparsedFormatYql) {
TChecksRequest request;
request.Program = "select1\n";
diff --git a/yql/essentials/public/fastcheck/parser.cpp b/yql/essentials/public/fastcheck/parser.cpp
index 17fc18ba57..01f24a63b0 100644
--- a/yql/essentials/public/fastcheck/parser.cpp
+++ b/yql/essentials/public/fastcheck/parser.cpp
@@ -1,4 +1,5 @@
#include "check_runner.h"
+#include <yql/essentials/ast/yql_ast.h>
#include <yql/essentials/sql/v1/lexer/lexer.h>
#include <yql/essentials/sql/v1/lexer/antlr4/lexer.h>
#include <yql/essentials/sql/v1/lexer/antlr4_ansi/lexer.h>
diff --git a/yql/essentials/sql/v0/ya.make b/yql/essentials/sql/v0/ya.make
index e0f6936e94..381f3fa77b 100644
--- a/yql/essentials/sql/v0/ya.make
+++ b/yql/essentials/sql/v0/ya.make
@@ -1,7 +1,7 @@
LIBRARY()
PEERDIR(
- library/cpp/charset
+ library/cpp/charset/lite
library/cpp/enumbitset
yql/essentials/minikql
yql/essentials/public/udf
diff --git a/yql/essentials/sql/v1/antlr_token.h b/yql/essentials/sql/v1/antlr_token.h
new file mode 100644
index 0000000000..329c21ea79
--- /dev/null
+++ b/yql/essentials/sql/v1/antlr_token.h
@@ -0,0 +1,16 @@
+#pragma once
+
+#include <yql/essentials/parser/proto_ast/gen/v1/SQLv1Lexer.h>
+#include <yql/essentials/parser/proto_ast/gen/v1_antlr4/SQLv1Antlr4Lexer.h>
+
+#define ANTLR3_TOKEN(NAME) (SQLv1LexerTokens::TOKEN_##NAME << 16)
+#define ANTLR4_TOKEN(NAME) ((SQLv1Antlr4Lexer::TOKEN_##NAME << 16) + 1)
+#define IS_TOKEN(USE_ANTLR4, ID, NAME) (UnifiedToken(USE_ANTLR4, ID) == ANTLR3_TOKEN(NAME) || UnifiedToken(USE_ANTLR4, ID) == ANTLR4_TOKEN(NAME))
+
+namespace NSQLTranslationV1 {
+
+inline ui32 UnifiedToken(bool useAntlr4, ui32 id) {
+ return useAntlr4 + (id << 16);
+}
+
+} // namespace NSQLTranslationV1
diff --git a/yql/essentials/sql/v1/context.cpp b/yql/essentials/sql/v1/context.cpp
index 9aac807dbc..9790e931dc 100644
--- a/yql/essentials/sql/v1/context.cpp
+++ b/yql/essentials/sql/v1/context.cpp
@@ -81,13 +81,6 @@ THashMap<TStringBuf, TPragmaMaybeField> CTX_PRAGMA_MAYBE_FIELDS = {
} // namespace
-TContext::TContext(const NSQLTranslation::TTranslationSettings& settings,
- const NSQLTranslation::TSQLHints& hints,
- NYql::TIssues& issues,
- const TString& query)
- : TContext(MakeAllLexers(), MakeAllParsers(), settings, hints, issues, query)
-{}
-
TContext::TContext(const TLexers& lexers, const TParsers& parsers,
const NSQLTranslation::TTranslationSettings& settings,
const NSQLTranslation::TSQLHints& hints,
diff --git a/yql/essentials/sql/v1/context.h b/yql/essentials/sql/v1/context.h
index 7c49529a78..7697f78d63 100644
--- a/yql/essentials/sql/v1/context.h
+++ b/yql/essentials/sql/v1/context.h
@@ -18,10 +18,6 @@
#include <util/generic/deque.h>
#include <util/generic/vector.h>
-#define ANTLR3_TOKEN(NAME) SQLv1LexerTokens::TOKEN_##NAME << 16
-#define ANTLR4_TOKEN(NAME) (SQLv1Antlr4Lexer::TOKEN_##NAME << 16) + 1
-#define IS_TOKEN(ID, NAME) (UnifiedToken(ID) == ANTLR3_TOKEN(NAME) || UnifiedToken(ID) == ANTLR4_TOKEN(NAME))
-
namespace NSQLTranslationV1 {
inline bool IsAnonymousName(const TString& name) {
return name == "$_";
@@ -92,12 +88,6 @@ namespace NSQLTranslationV1 {
class TContext {
public:
- //FIXME remove
- TContext(const NSQLTranslation::TTranslationSettings& settings,
- const NSQLTranslation::TSQLHints& hints,
- NYql::TIssues& issues,
- const TString& query = {});
-
TContext(const TLexers& lexers,
const TParsers& parsers,
const NSQLTranslation::TTranslationSettings& settings,
@@ -446,10 +436,6 @@ namespace NSQLTranslationV1 {
return Ctx.Token(token);
}
- ui32 UnifiedToken(ui32 id) const {
- return Ctx.Settings.Antlr4Parser + (id << 16);
- }
-
TString Identifier(const NSQLv1Generated::TToken& token) {
return IdContent(Ctx, Token(token));
}
diff --git a/yql/essentials/sql/v1/format/sql_format.cpp b/yql/essentials/sql/v1/format/sql_format.cpp
index cfb6fcde71..9bdcde0f49 100644
--- a/yql/essentials/sql/v1/format/sql_format.cpp
+++ b/yql/essentials/sql/v1/format/sql_format.cpp
@@ -3223,10 +3223,6 @@ ISqlFormatter::TPtr MakeSqlFormatter(const NSQLTranslationV1::TLexers& lexers,
return ISqlFormatter::TPtr(new TSqlFormatter(lexers, parsers, settings));
}
-ISqlFormatter::TPtr MakeSqlFormatter(const NSQLTranslation::TTranslationSettings& settings) {
- return MakeSqlFormatter(NSQLTranslationV1::MakeAllLexers(), NSQLTranslationV1::MakeAllParsers(), settings);
-}
-
TString MutateQuery(const NSQLTranslationV1::TLexers& lexers,
const TString& query, const NSQLTranslation::TTranslationSettings& settings) {
auto parsedSettings = settings;
diff --git a/yql/essentials/sql/v1/format/sql_format.h b/yql/essentials/sql/v1/format/sql_format.h
index 3233f2031b..55c34a462d 100644
--- a/yql/essentials/sql/v1/format/sql_format.h
+++ b/yql/essentials/sql/v1/format/sql_format.h
@@ -25,9 +25,6 @@ public:
virtual ~ISqlFormatter() = default;
};
-//FIXME remove
-ISqlFormatter::TPtr MakeSqlFormatter(const NSQLTranslation::TTranslationSettings& settings = {});
-
ISqlFormatter::TPtr MakeSqlFormatter(const NSQLTranslationV1::TLexers& lexers,
const NSQLTranslationV1::TParsers& parsers,
const NSQLTranslation::TTranslationSettings& settings = {});
diff --git a/yql/essentials/sql/v1/lexer/lexer.cpp b/yql/essentials/sql/v1/lexer/lexer.cpp
index 9a45971dbc..8c94fff7e1 100644
--- a/yql/essentials/sql/v1/lexer/lexer.cpp
+++ b/yql/essentials/sql/v1/lexer/lexer.cpp
@@ -18,15 +18,6 @@
namespace NSQLTranslationV1 {
-TLexers MakeAllLexers() {
- return TLexers {
- .Antlr3 = MakeAntlr3LexerFactory(),
- .Antlr3Ansi = MakeAntlr3AnsiLexerFactory(),
- .Antlr4 = MakeAntlr4LexerFactory(),
- .Antlr4Ansi = MakeAntlr4AnsiLexerFactory()
- };
-}
-
namespace {
#if defined(_tsan_enabled_)
@@ -85,10 +76,6 @@ private:
} // namespace
-NSQLTranslation::ILexer::TPtr MakeLexer(bool ansi, bool antlr4) {
- return NSQLTranslation::ILexer::TPtr(new TV1Lexer(MakeAllLexers(), ansi, antlr4));
-}
-
NSQLTranslation::ILexer::TPtr MakeLexer(const TLexers& lexers, bool ansi, bool antlr4) {
return NSQLTranslation::ILexer::TPtr(new TV1Lexer(lexers, ansi, antlr4));
}
diff --git a/yql/essentials/sql/v1/lexer/lexer.h b/yql/essentials/sql/v1/lexer/lexer.h
index 0e94cb10bf..2a3af96055 100644
--- a/yql/essentials/sql/v1/lexer/lexer.h
+++ b/yql/essentials/sql/v1/lexer/lexer.h
@@ -11,12 +11,6 @@ struct TLexers {
NSQLTranslation::TLexerFactoryPtr Antlr4Ansi;
};
-//FIXME remove
-TLexers MakeAllLexers();
-
-//FIXME remove
-NSQLTranslation::ILexer::TPtr MakeLexer(bool ansi, bool antlr4);
-
NSQLTranslation::ILexer::TPtr MakeLexer(const TLexers& lexers, bool ansi, bool antlr4);
// "Probably" because YQL keyword can be an identifier
diff --git a/yql/essentials/sql/v1/lexer/ya.make b/yql/essentials/sql/v1/lexer/ya.make
index f32e79a472..8a3f00d36e 100644
--- a/yql/essentials/sql/v1/lexer/ya.make
+++ b/yql/essentials/sql/v1/lexer/ya.make
@@ -3,10 +3,6 @@ LIBRARY()
PEERDIR(
yql/essentials/core/issue/protos
yql/essentials/sql/settings
- yql/essentials/sql/v1/lexer/antlr3
- yql/essentials/sql/v1/lexer/antlr3_ansi
- yql/essentials/sql/v1/lexer/antlr4
- yql/essentials/sql/v1/lexer/antlr4_ansi
)
SRCS(
diff --git a/yql/essentials/sql/v1/proto_parser/proto_parser.cpp b/yql/essentials/sql/v1/proto_parser/proto_parser.cpp
index 7b22ae11e5..5651345215 100644
--- a/yql/essentials/sql/v1/proto_parser/proto_parser.cpp
+++ b/yql/essentials/sql/v1/proto_parser/proto_parser.cpp
@@ -100,25 +100,4 @@ google::protobuf::Message* SqlAST(const TParsers& parsers, const TString& query,
}
}
-google::protobuf::Message* SqlAST(const TString& query, const TString& queryName,
- NYql::TIssues& err, size_t maxErrors, bool ansiLexer, bool antlr4Parser, bool testAntlr4, google::protobuf::Arena* arena) {
- Y_UNUSED(testAntlr4);
- return SqlAST(MakeAllParsers(), query, queryName, err, maxErrors, ansiLexer, antlr4Parser, arena);
-}
-
-google::protobuf::Message* SqlAST(const TString& query, const TString& queryName,
- NProtoAST::IErrorCollector& err, bool ansiLexer, bool antlr4Parser, bool testAntlr4, google::protobuf::Arena* arena) {
- Y_UNUSED(testAntlr4);
- return SqlAST(MakeAllParsers(), query, queryName, err, ansiLexer, antlr4Parser, arena);
-}
-
-TParsers MakeAllParsers() {
- return TParsers {
- .Antlr3 = MakeAntlr3ParserFactory(),
- .Antlr3Ansi = MakeAntlr3AnsiParserFactory(),
- .Antlr4 = MakeAntlr4ParserFactory(),
- .Antlr4Ansi = MakeAntlr4AnsiParserFactory()
- };
-}
-
} // namespace NSQLTranslationV1
diff --git a/yql/essentials/sql/v1/proto_parser/proto_parser.h b/yql/essentials/sql/v1/proto_parser/proto_parser.h
index 6e9ba07cac..b2002d875b 100644
--- a/yql/essentials/sql/v1/proto_parser/proto_parser.h
+++ b/yql/essentials/sql/v1/proto_parser/proto_parser.h
@@ -1,6 +1,5 @@
#pragma once
-#include <yql/essentials/ast/yql_ast.h>
#include <yql/essentials/parser/proto_ast/common.h>
#include <yql/essentials/public/issue/yql_warning.h>
#include <yql/essentials/public/issue/yql_issue_manager.h>
@@ -21,15 +20,6 @@ namespace NSQLTranslationV1 {
NSQLTranslation::TParserFactoryPtr Antlr4Ansi;
};
- //FIXME remove
- TParsers MakeAllParsers();
-
- //FIXME remove
- google::protobuf::Message* SqlAST(const TString& query, const TString& queryName,
- NYql::TIssues& err, size_t maxErrors, bool ansiLexer, bool antlr4Parser, bool testAntlr4, google::protobuf::Arena* arena);
- google::protobuf::Message* SqlAST(const TString& query, const TString& queryName,
- NProtoAST::IErrorCollector& err, bool ansiLexer, bool antlr4Parser, bool testAntlr4, google::protobuf::Arena* arena);
-
google::protobuf::Message* SqlAST(const TParsers& parsers, const TString& query, const TString& queryName,
NYql::TIssues& err, size_t maxErrors, bool ansiLexer, bool antlr4Parser, google::protobuf::Arena* arena);
google::protobuf::Message* SqlAST(const TParsers& parsers, const TString& query, const TString& queryName,
diff --git a/yql/essentials/sql/v1/proto_parser/ya.make b/yql/essentials/sql/v1/proto_parser/ya.make
index 1ef29d39ca..d87741f8a9 100644
--- a/yql/essentials/sql/v1/proto_parser/ya.make
+++ b/yql/essentials/sql/v1/proto_parser/ya.make
@@ -2,14 +2,7 @@ LIBRARY()
PEERDIR(
yql/essentials/utils
- yql/essentials/ast
-
yql/essentials/parser/proto_ast/collect_issues
-
- yql/essentials/sql/v1/proto_parser/antlr3
- yql/essentials/sql/v1/proto_parser/antlr3_ansi
- yql/essentials/sql/v1/proto_parser/antlr4
- yql/essentials/sql/v1/proto_parser/antlr4_ansi
)
SRCS(
diff --git a/yql/essentials/sql/v1/sql.cpp b/yql/essentials/sql/v1/sql.cpp
index e4c82a0744..7201c4b069 100644
--- a/yql/essentials/sql/v1/sql.cpp
+++ b/yql/essentials/sql/v1/sql.cpp
@@ -89,14 +89,6 @@ NYql::TAstParseResult SqlASTToYql(const TLexers& lexers, const TParsers& parsers
return res;
}
-NYql::TAstParseResult SqlASTToYql(const TString& query,
- const google::protobuf::Message& protoAst,
- const NSQLTranslation::TSQLHints& hints,
- const NSQLTranslation::TTranslationSettings& settings)
-{
- return SqlASTToYql(MakeAllLexers(), MakeAllParsers(), query, protoAst, hints, settings);
-}
-
NYql::TAstParseResult SqlToYql(const TLexers& lexers, const TParsers& parsers, const TString& query, const NSQLTranslation::TTranslationSettings& settings, NYql::TWarningRules* warningRules)
{
TAstParseResult res;
@@ -126,10 +118,6 @@ NYql::TAstParseResult SqlToYql(const TLexers& lexers, const TParsers& parsers, c
return res;
}
-NYql::TAstParseResult SqlToYql(const TString& query, const NSQLTranslation::TTranslationSettings& settings, NYql::TWarningRules* warningRules) {
- return SqlToYql(MakeAllLexers(), MakeAllParsers(), query, settings, warningRules);
-}
-
bool NeedUseForAllStatements(const TRule_sql_stmt_core::AltCase& subquery) {
switch (subquery) {
case TRule_sql_stmt_core::kAltSqlStmtCore1: // pragma
@@ -264,10 +252,6 @@ TVector<NYql::TAstParseResult> SqlToAstStatements(const TLexers& lexers, const T
return result;
}
-TVector<NYql::TAstParseResult> SqlToAstStatements(const TString& query, const NSQLTranslation::TTranslationSettings& settings, NYql::TWarningRules* warningRules, TVector<NYql::TStmtParseInfo>* stmtParseInfo) {
- return SqlToAstStatements(MakeAllLexers(), MakeAllParsers(), query, settings, warningRules, stmtParseInfo);
-}
-
bool SplitQueryToStatements(const TLexers& lexers, const TParsers& parsers, const TString& query, TVector<TString>& statements, NYql::TIssues& issues,
const NSQLTranslation::TTranslationSettings& settings) {
auto lexer = NSQLTranslationV1::MakeLexer(lexers, settings.AnsiLexer, settings.Antlr4Parser);
@@ -292,11 +276,6 @@ bool SplitQueryToStatements(const TLexers& lexers, const TParsers& parsers, cons
return true;
}
-bool SplitQueryToStatements(const TString& query, TVector<TString>& statements, NYql::TIssues& issues,
- const NSQLTranslation::TTranslationSettings& settings) {
- return SplitQueryToStatements(MakeAllLexers(), MakeAllParsers(), query, statements, issues, settings);
-}
-
class TTranslator : public NSQLTranslation::ITranslator {
public:
TTranslator(const TLexers& lexers, const TParsers& parsers)
@@ -335,10 +314,6 @@ private:
const TParsers Parsers_;
};
-NSQLTranslation::TTranslatorPtr MakeTranslator() {
- return MakeIntrusive<TTranslator>(MakeAllLexers(), MakeAllParsers());
-}
-
NSQLTranslation::TTranslatorPtr MakeTranslator(const TLexers& lexers, const TParsers& parsers) {
return MakeIntrusive<TTranslator>(lexers, parsers);
}
diff --git a/yql/essentials/sql/v1/sql.h b/yql/essentials/sql/v1/sql.h
index 5ded2321f3..7693d75915 100644
--- a/yql/essentials/sql/v1/sql.h
+++ b/yql/essentials/sql/v1/sql.h
@@ -20,27 +20,16 @@ namespace NSQLTranslation {
namespace NSQLTranslationV1 {
- //FIXME remove
- NYql::TAstParseResult SqlToYql(const TString& query, const NSQLTranslation::TTranslationSettings& settings, NYql::TWarningRules* warningRules = nullptr);
NYql::TAstParseResult SqlToYql(const TLexers& lexers, const TParsers& parsers, const TString& query, const NSQLTranslation::TTranslationSettings& settings, NYql::TWarningRules* warningRules = nullptr);
- //FIXME remove
- NYql::TAstParseResult SqlASTToYql(const TString& query, const google::protobuf::Message& protoAst, const NSQLTranslation::TSQLHints& hints, const NSQLTranslation::TTranslationSettings& settings);
NYql::TAstParseResult SqlASTToYql(const TLexers& lexers, const TParsers& parsers, const TString& query, const google::protobuf::Message& protoAst, const NSQLTranslation::TSQLHints& hints, const NSQLTranslation::TTranslationSettings& settings);
- //FIXME remove
- TVector<NYql::TAstParseResult> SqlToAstStatements(const TString& query, const NSQLTranslation::TTranslationSettings& settings, NYql::TWarningRules* warningRules, TVector<NYql::TStmtParseInfo>* stmtParseInfo = nullptr);
TVector<NYql::TAstParseResult> SqlToAstStatements(const TLexers& lexers, const TParsers& parsers, const TString& query, const NSQLTranslation::TTranslationSettings& settings, NYql::TWarningRules* warningRules, TVector<NYql::TStmtParseInfo>* stmtParseInfo = nullptr);
bool NeedUseForAllStatements(const NSQLv1Generated::TRule_sql_stmt_core::AltCase& subquery);
- //FIXME remove
- bool SplitQueryToStatements(const TString& query, TVector<TString>& statements, NYql::TIssues& issues,
- const NSQLTranslation::TTranslationSettings& settings);
bool SplitQueryToStatements(const TLexers& lexers, const TParsers& parsers, const TString& query, TVector<TString>& statements, NYql::TIssues& issues,
const NSQLTranslation::TTranslationSettings& settings);
- NSQLTranslation::TTranslatorPtr MakeTranslator();
-
NSQLTranslation::TTranslatorPtr MakeTranslator(const TLexers& lexers, const TParsers& parsers);
} // namespace NSQLTranslationV1
diff --git a/yql/essentials/sql/v1/sql_call_expr.cpp b/yql/essentials/sql/v1/sql_call_expr.cpp
index f279e7db77..1eb4edd10a 100644
--- a/yql/essentials/sql/v1/sql_call_expr.cpp
+++ b/yql/essentials/sql/v1/sql_call_expr.cpp
@@ -1,8 +1,6 @@
#include "sql_call_expr.h"
#include "sql_expression.h"
-#include <yql/essentials/parser/proto_ast/gen/v1/SQLv1Lexer.h>
-
#include <yql/essentials/minikql/mkql_program_builder.h>
namespace NSQLTranslationV1 {
diff --git a/yql/essentials/sql/v1/sql_expression.cpp b/yql/essentials/sql/v1/sql_expression.cpp
index 9a92a7abc5..3eb1904cc5 100644
--- a/yql/essentials/sql/v1/sql_expression.cpp
+++ b/yql/essentials/sql/v1/sql_expression.cpp
@@ -2,12 +2,11 @@
#include "sql_call_expr.h"
#include "sql_select.h"
#include "sql_values.h"
-#include <yql/essentials/parser/proto_ast/gen/v1/SQLv1Lexer.h>
-#include <yql/essentials/parser/proto_ast/gen/v1_antlr4/SQLv1Antlr4Lexer.h>
#include <yql/essentials/utils/utf8.h>
#include <util/charset/wide.h>
#include <util/string/ascii.h>
#include <util/string/hex.h>
+#include "antlr_token.h"
namespace NSQLTranslationV1 {
@@ -1661,13 +1660,13 @@ TNodePtr TSqlExpression::SubExpr(const TRule_con_subexpr& node, const TTrailingQ
Token(token);
TPosition pos(Ctx.Pos());
auto tokenId = token.GetId();
- if (IS_TOKEN(tokenId, NOT)) {
+ if (IS_TOKEN(Ctx.Settings.Antlr4Parser, tokenId, NOT)) {
opName = "Not";
- } else if (IS_TOKEN(tokenId, PLUS)) {
+ } else if (IS_TOKEN(Ctx.Settings.Antlr4Parser, tokenId, PLUS)) {
opName = "Plus";
- } else if (IS_TOKEN(tokenId, MINUS)) {
+ } else if (IS_TOKEN(Ctx.Settings.Antlr4Parser, tokenId, MINUS)) {
opName = Ctx.Scoped->PragmaCheckedOps ? "CheckedMinus" : "Minus";
- } else if (IS_TOKEN(tokenId, TILDA)) {
+ } else if (IS_TOKEN(Ctx.Settings.Antlr4Parser, tokenId, TILDA)) {
opName = "BitNot";
} else {
Ctx.IncrementMonCounter("sql_errors", "UnsupportedUnaryOperation");
@@ -1939,7 +1938,7 @@ TNodePtr TSqlExpression::SubExpr(const TRule_xor_subexpr& node, const TTrailingQ
}
case TRule_cond_expr::kAltCondExpr4: {
auto alt = cond.GetAlt_cond_expr4();
- const bool symmetric = alt.HasBlock3() && IS_TOKEN(alt.GetBlock3().GetToken1().GetId(), SYMMETRIC);
+ const bool symmetric = alt.HasBlock3() && IS_TOKEN(Ctx.Settings.Antlr4Parser, alt.GetBlock3().GetToken1().GetId(), SYMMETRIC);
const bool negation = alt.HasBlock1();
TNodePtr left = SubExpr(alt.GetRule_eq_subexpr4(), {});
TNodePtr right = SubExpr(alt.GetRule_eq_subexpr6(), tail);
@@ -2065,28 +2064,28 @@ TNodePtr TSqlExpression::BinOpList(const TNode& node, TGetNode getNode, TIter be
TPosition pos(Ctx.Pos());
TString opName;
auto tokenId = begin->GetToken1().GetId();
- if (IS_TOKEN(tokenId, LESS)) {
+ if (IS_TOKEN(Ctx.Settings.Antlr4Parser, tokenId, LESS)) {
opName = "<";
Ctx.IncrementMonCounter("sql_binary_operations", "Less");
- } else if (IS_TOKEN(tokenId, LESS_OR_EQ)) {
+ } else if (IS_TOKEN(Ctx.Settings.Antlr4Parser, tokenId, LESS_OR_EQ)) {
opName = "<=";
Ctx.IncrementMonCounter("sql_binary_operations", "LessOrEq");
- } else if (IS_TOKEN(tokenId, GREATER)) {
+ } else if (IS_TOKEN(Ctx.Settings.Antlr4Parser, tokenId, GREATER)) {
opName = ">";
Ctx.IncrementMonCounter("sql_binary_operations", "Greater");
- } else if (IS_TOKEN(tokenId, GREATER_OR_EQ)) {
+ } else if (IS_TOKEN(Ctx.Settings.Antlr4Parser, tokenId, GREATER_OR_EQ)) {
opName = ">=";
Ctx.IncrementMonCounter("sql_binary_operations", "GreaterOrEq");
- } else if (IS_TOKEN(tokenId, PLUS)) {
+ } else if (IS_TOKEN(Ctx.Settings.Antlr4Parser, tokenId, PLUS)) {
opName = Ctx.Scoped->PragmaCheckedOps ? "CheckedAdd" : "+MayWarn";
Ctx.IncrementMonCounter("sql_binary_operations", "Plus");
- } else if (IS_TOKEN(tokenId, MINUS)) {
+ } else if (IS_TOKEN(Ctx.Settings.Antlr4Parser, tokenId, MINUS)) {
opName = Ctx.Scoped->PragmaCheckedOps ? "CheckedSub" : "-MayWarn";
Ctx.IncrementMonCounter("sql_binary_operations", "Minus");
- } else if (IS_TOKEN(tokenId, ASTERISK)) {
+ } else if (IS_TOKEN(Ctx.Settings.Antlr4Parser, tokenId, ASTERISK)) {
opName = Ctx.Scoped->PragmaCheckedOps ? "CheckedMul" : "*MayWarn";
Ctx.IncrementMonCounter("sql_binary_operations", "Multiply");
- } else if (IS_TOKEN(tokenId, SLASH)) {
+ } else if (IS_TOKEN(Ctx.Settings.Antlr4Parser, tokenId, SLASH)) {
opName = "/MayWarn";
Ctx.IncrementMonCounter("sql_binary_operations", "Divide");
if (!Ctx.Scoped->PragmaClassicDivision && partialResult) {
@@ -2094,7 +2093,7 @@ TNodePtr TSqlExpression::BinOpList(const TNode& node, TGetNode getNode, TIter be
} else if (Ctx.Scoped->PragmaCheckedOps) {
opName = "CheckedDiv";
}
- } else if (IS_TOKEN(tokenId, PERCENT)) {
+ } else if (IS_TOKEN(Ctx.Settings.Antlr4Parser, tokenId, PERCENT)) {
opName = Ctx.Scoped->PragmaCheckedOps ? "CheckedMod" : "%MayWarn";
Ctx.IncrementMonCounter("sql_binary_operations", "Mod");
} else {
@@ -2121,7 +2120,7 @@ TNodePtr TSqlExpression::BinOpList(const TRule_bit_subexpr& node, TGetNode getNo
case TRule_neq_subexpr_TBlock2_TBlock1::kAlt1: {
Token(begin->GetBlock1().GetAlt1().GetToken1());
auto tokenId = begin->GetBlock1().GetAlt1().GetToken1().GetId();
- if (!IS_TOKEN(tokenId, SHIFT_LEFT)) {
+ if (!IS_TOKEN(Ctx.Settings.Antlr4Parser, tokenId, SHIFT_LEFT)) {
Error() << "Unsupported binary operation token: " << tokenId;
return {};
}
@@ -2137,7 +2136,7 @@ TNodePtr TSqlExpression::BinOpList(const TRule_bit_subexpr& node, TGetNode getNo
case TRule_neq_subexpr_TBlock2_TBlock1::kAlt3: {
Token(begin->GetBlock1().GetAlt3().GetToken1());
auto tokenId = begin->GetBlock1().GetAlt3().GetToken1().GetId();
- if (!IS_TOKEN(tokenId, ROT_LEFT)) {
+ if (!IS_TOKEN(Ctx.Settings.Antlr4Parser, tokenId, ROT_LEFT)) {
Error() << "Unsupported binary operation token: " << tokenId;
return {};
}
@@ -2153,7 +2152,7 @@ TNodePtr TSqlExpression::BinOpList(const TRule_bit_subexpr& node, TGetNode getNo
case TRule_neq_subexpr_TBlock2_TBlock1::kAlt5: {
Token(begin->GetBlock1().GetAlt5().GetToken1());
auto tokenId = begin->GetBlock1().GetAlt5().GetToken1().GetId();
- if (!IS_TOKEN(tokenId, AMPERSAND)) {
+ if (!IS_TOKEN(Ctx.Settings.Antlr4Parser, tokenId, AMPERSAND)) {
Error() << "Unsupported binary operation token: " << tokenId;
return {};
}
@@ -2164,7 +2163,7 @@ TNodePtr TSqlExpression::BinOpList(const TRule_bit_subexpr& node, TGetNode getNo
case TRule_neq_subexpr_TBlock2_TBlock1::kAlt6: {
Token(begin->GetBlock1().GetAlt6().GetToken1());
auto tokenId = begin->GetBlock1().GetAlt6().GetToken1().GetId();
- if (!IS_TOKEN(tokenId, PIPE)) {
+ if (!IS_TOKEN(Ctx.Settings.Antlr4Parser, tokenId, PIPE)) {
Error() << "Unsupported binary operation token: " << tokenId;
return {};
}
@@ -2175,7 +2174,7 @@ TNodePtr TSqlExpression::BinOpList(const TRule_bit_subexpr& node, TGetNode getNo
case TRule_neq_subexpr_TBlock2_TBlock1::kAlt7: {
Token(begin->GetBlock1().GetAlt7().GetToken1());
auto tokenId = begin->GetBlock1().GetAlt7().GetToken1().GetId();
- if (!IS_TOKEN(tokenId, CARET)) {
+ if (!IS_TOKEN(Ctx.Settings.Antlr4Parser, tokenId, CARET)) {
Error() << "Unsupported binary operation token: " << tokenId;
return {};
}
@@ -2205,7 +2204,7 @@ TNodePtr TSqlExpression::BinOpList(const TRule_eq_subexpr& node, TGetNode getNod
case TRule_cond_expr::TAlt5::TBlock1::TBlock1::kAlt1: {
Token(begin->GetBlock1().GetAlt1().GetToken1());
auto tokenId = begin->GetBlock1().GetAlt1().GetToken1().GetId();
- if (!IS_TOKEN(tokenId, EQUALS)) {
+ if (!IS_TOKEN(Ctx.Settings.Antlr4Parser, tokenId, EQUALS)) {
Error() << "Unsupported binary operation token: " << tokenId;
return {};
}
@@ -2216,7 +2215,7 @@ TNodePtr TSqlExpression::BinOpList(const TRule_eq_subexpr& node, TGetNode getNod
case TRule_cond_expr::TAlt5::TBlock1::TBlock1::kAlt2: {
Token(begin->GetBlock1().GetAlt2().GetToken1());
auto tokenId = begin->GetBlock1().GetAlt2().GetToken1().GetId();
- if (!IS_TOKEN(tokenId, EQUALS2)) {
+ if (!IS_TOKEN(Ctx.Settings.Antlr4Parser, tokenId, EQUALS2)) {
Error() << "Unsupported binary operation token: " << tokenId;
return {};
}
@@ -2227,7 +2226,7 @@ TNodePtr TSqlExpression::BinOpList(const TRule_eq_subexpr& node, TGetNode getNod
case TRule_cond_expr::TAlt5::TBlock1::TBlock1::kAlt3: {
Token(begin->GetBlock1().GetAlt3().GetToken1());
auto tokenId = begin->GetBlock1().GetAlt3().GetToken1().GetId();
- if (!IS_TOKEN(tokenId, NOT_EQUALS)) {
+ if (!IS_TOKEN(Ctx.Settings.Antlr4Parser, tokenId, NOT_EQUALS)) {
Error() << "Unsupported binary operation token: " << tokenId;
return {};
}
@@ -2238,7 +2237,7 @@ TNodePtr TSqlExpression::BinOpList(const TRule_eq_subexpr& node, TGetNode getNod
case TRule_cond_expr::TAlt5::TBlock1::TBlock1::kAlt4: {
Token(begin->GetBlock1().GetAlt4().GetToken1());
auto tokenId = begin->GetBlock1().GetAlt4().GetToken1().GetId();
- if (!IS_TOKEN(tokenId, NOT_EQUALS2)) {
+ if (!IS_TOKEN(Ctx.Settings.Antlr4Parser, tokenId, NOT_EQUALS2)) {
Error() << "Unsupported binary operation token: " << tokenId;
return {};
}
diff --git a/yql/essentials/sql/v1/sql_into_tables.cpp b/yql/essentials/sql/v1/sql_into_tables.cpp
index ac9ce53061..77498af54a 100644
--- a/yql/essentials/sql/v1/sql_into_tables.cpp
+++ b/yql/essentials/sql/v1/sql_into_tables.cpp
@@ -7,8 +7,6 @@ using namespace NYql;
namespace NSQLTranslationV1 {
-using NALPDefault::SQLv1LexerTokens;
-
using namespace NSQLv1Generated;
TNodePtr TSqlIntoTable::Build(const TRule_into_table_stmt& node) {
diff --git a/yql/essentials/sql/v1/sql_into_tables.h b/yql/essentials/sql/v1/sql_into_tables.h
index 0e40f5669b..c66bcfab95 100644
--- a/yql/essentials/sql/v1/sql_into_tables.h
+++ b/yql/essentials/sql/v1/sql_into_tables.h
@@ -1,7 +1,6 @@
#pragma once
#include "sql_translation.h"
-#include <yql/essentials/parser/proto_ast/gen/v1/SQLv1Lexer.h>
namespace NSQLTranslationV1 {
diff --git a/yql/essentials/sql/v1/sql_query.cpp b/yql/essentials/sql/v1/sql_query.cpp
index 34f137016a..6dc510bf57 100644
--- a/yql/essentials/sql/v1/sql_query.cpp
+++ b/yql/essentials/sql/v1/sql_query.cpp
@@ -4,8 +4,7 @@
#include "sql_into_tables.h"
#include "sql_values.h"
#include "node.h"
-#include <yql/essentials/parser/proto_ast/gen/v1/SQLv1Lexer.h>
-#include <yql/essentials/parser/proto_ast/gen/v1_antlr4/SQLv1Antlr4Lexer.h>
+#include "antlr_token.h"
#include <yql/essentials/sql/v1/object_processing.h>
#include <yql/essentials/utils/yql_paths.h>
#include <util/generic/scope.h>
@@ -314,8 +313,8 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core&
if (rule.HasBlock2()) { // OR REPLACE
replaceIfExists = true;
Y_DEBUG_ABORT_UNLESS(
- (IS_TOKEN(rule.GetBlock2().GetToken1().GetId(), OR) &&
- IS_TOKEN(rule.GetBlock2().GetToken2().GetId(), REPLACE))
+ (IS_TOKEN(Ctx.Settings.Antlr4Parser, rule.GetBlock2().GetToken1().GetId(), OR) &&
+ IS_TOKEN(Ctx.Settings.Antlr4Parser, rule.GetBlock2().GetToken2().GetId(), REPLACE))
);
}
@@ -324,7 +323,7 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core&
ETableType tableType = ETableType::Table;
bool temporary = false;
if (block.HasAlt2() &&
- IS_TOKEN(block.GetAlt2().GetToken1().GetId(), TABLESTORE)
+ IS_TOKEN(Ctx.Settings.Antlr4Parser, block.GetAlt2().GetToken1().GetId(), TABLESTORE)
) {
tableType = ETableType::TableStore;
if (isCreateTableAs) {
@@ -333,7 +332,7 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core&
return false;
}
} else if (block.HasAlt3() &&
- IS_TOKEN(block.GetAlt3().GetToken1().GetId(), EXTERNAL)
+ IS_TOKEN(Ctx.Settings.Antlr4Parser, block.GetAlt3().GetToken1().GetId(), EXTERNAL)
) {
tableType = ETableType::ExternalTable;
if (isCreateTableAs) {
@@ -341,8 +340,8 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core&
<< "CREATE TABLE AS is not supported for EXTERNAL TABLE";
return false;
}
- } else if (block.HasAlt4() && IS_TOKEN(block.GetAlt4().GetToken1().GetId(), TEMP) ||
- block.HasAlt5() && IS_TOKEN(block.GetAlt5().GetToken1().GetId(), TEMPORARY)) {
+ } else if (block.HasAlt4() && IS_TOKEN(Ctx.Settings.Antlr4Parser, block.GetAlt4().GetToken1().GetId(), TEMP) ||
+ block.HasAlt5() && IS_TOKEN(Ctx.Settings.Antlr4Parser, block.GetAlt5().GetToken1().GetId(), TEMPORARY)) {
temporary = true;
}
@@ -350,9 +349,9 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core&
if (rule.HasBlock4()) { // IF NOT EXISTS
existingOk = true;
Y_DEBUG_ABORT_UNLESS(
- IS_TOKEN(rule.GetBlock4().GetToken1().GetId(), IF) &&
- IS_TOKEN(rule.GetBlock4().GetToken2().GetId(), NOT) &&
- IS_TOKEN(rule.GetBlock4().GetToken3().GetId(), EXISTS)
+ IS_TOKEN(Ctx.Settings.Antlr4Parser, rule.GetBlock4().GetToken1().GetId(), IF) &&
+ IS_TOKEN(Ctx.Settings.Antlr4Parser, rule.GetBlock4().GetToken2().GetId(), NOT) &&
+ IS_TOKEN(Ctx.Settings.Antlr4Parser, rule.GetBlock4().GetToken3().GetId(), EXISTS)
);
}
@@ -439,8 +438,8 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core&
if (rule.HasBlock3()) { // IF EXISTS
missingOk = true;
Y_DEBUG_ABORT_UNLESS(
- IS_TOKEN(rule.GetBlock3().GetToken1().GetId(), IF) &&
- IS_TOKEN(rule.GetBlock3().GetToken2().GetId(), EXISTS)
+ IS_TOKEN(Ctx.Settings.Antlr4Parser, rule.GetBlock3().GetToken1().GetId(), IF) &&
+ IS_TOKEN(Ctx.Settings.Antlr4Parser, rule.GetBlock3().GetToken2().GetId(), EXISTS)
);
}
@@ -531,7 +530,7 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core&
case TRule_sql_stmt_core::kAltSqlStmtCore15: {
Ctx.BodyPart();
const auto& rule = core.GetAlt_sql_stmt_core15().GetRule_alter_table_stmt1();
- const bool isTablestore = IS_TOKEN(rule.GetToken2().GetId(), TABLESTORE);
+ const bool isTablestore = IS_TOKEN(Ctx.Settings.Antlr4Parser, rule.GetToken2().GetId(), TABLESTORE);
TTableRef tr;
if (!SimpleTableRefImpl(rule.GetRule_simple_table_ref3(), tr)) {
return false;
@@ -813,7 +812,7 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core&
switch (node.GetBlock4().Alt_case()) {
case TRule_alter_group_stmt_TBlock4::kAlt1: {
auto& addDropNode = node.GetBlock4().GetAlt1();
- const bool isDrop = IS_TOKEN(addDropNode.GetToken1().GetId(), DROP);
+ const bool isDrop = IS_TOKEN(Ctx.Settings.Antlr4Parser, addDropNode.GetToken1().GetId(), DROP);
TVector<TDeferredAtom> roles;
bool allowSystemRoles = false;
roles.emplace_back();
@@ -862,13 +861,13 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core&
return false;
}
- const bool isUser = IS_TOKEN(node.GetToken2().GetId(), USER);
+ const bool isUser = IS_TOKEN(Ctx.Settings.Antlr4Parser, node.GetToken2().GetId(), USER);
bool missingOk = false;
if (node.HasBlock3()) { // IF EXISTS
missingOk = true;
Y_DEBUG_ABORT_UNLESS(
- IS_TOKEN(node.GetBlock3().GetToken1().GetId(), IF) &&
- IS_TOKEN(node.GetBlock3().GetToken2().GetId(), EXISTS)
+ IS_TOKEN(Ctx.Settings.Antlr4Parser, node.GetBlock3().GetToken1().GetId(), IF) &&
+ IS_TOKEN(Ctx.Settings.Antlr4Parser, node.GetBlock3().GetToken2().GetId(), EXISTS)
);
}
@@ -904,9 +903,9 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core&
if (node.HasBlock3()) { // IF NOT EXISTS
existingOk = true;
Y_DEBUG_ABORT_UNLESS(
- IS_TOKEN(node.GetBlock3().GetToken1().GetId(), IF) &&
- IS_TOKEN(node.GetBlock3().GetToken2().GetId(), NOT) &&
- IS_TOKEN(node.GetBlock3().GetToken3().GetId(), EXISTS)
+ IS_TOKEN(Ctx.Settings.Antlr4Parser, node.GetBlock3().GetToken1().GetId(), IF) &&
+ IS_TOKEN(Ctx.Settings.Antlr4Parser, node.GetBlock3().GetToken2().GetId(), NOT) &&
+ IS_TOKEN(Ctx.Settings.Antlr4Parser, node.GetBlock3().GetToken3().GetId(), EXISTS)
);
}
@@ -958,8 +957,8 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core&
if (node.HasBlock3()) { // IF EXISTS
missingOk = true;
Y_DEBUG_ABORT_UNLESS(
- IS_TOKEN(node.GetBlock3().GetToken1().GetId(), IF) &&
- IS_TOKEN(node.GetBlock3().GetToken2().GetId(), EXISTS)
+ IS_TOKEN(Ctx.Settings.Antlr4Parser, node.GetBlock3().GetToken1().GetId(), IF) &&
+ IS_TOKEN(Ctx.Settings.Antlr4Parser, node.GetBlock3().GetToken2().GetId(), EXISTS)
);
}
@@ -990,8 +989,8 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core&
if (node.HasBlock2()) { // OR REPLACE
replaceIfExists = true;
Y_DEBUG_ABORT_UNLESS(
- IS_TOKEN(node.GetBlock2().GetToken1().GetId(), OR) &&
- IS_TOKEN(node.GetBlock2().GetToken2().GetId(), REPLACE)
+ IS_TOKEN(Ctx.Settings.Antlr4Parser, node.GetBlock2().GetToken1().GetId(), OR) &&
+ IS_TOKEN(Ctx.Settings.Antlr4Parser, node.GetBlock2().GetToken2().GetId(), REPLACE)
);
}
@@ -999,9 +998,9 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core&
if (node.HasBlock6()) { // IF NOT EXISTS
existingOk = true;
Y_DEBUG_ABORT_UNLESS(
- IS_TOKEN(node.GetBlock6().GetToken1().GetId(), IF) &&
- IS_TOKEN(node.GetBlock6().GetToken2().GetId(), NOT) &&
- IS_TOKEN(node.GetBlock6().GetToken3().GetId(), EXISTS)
+ IS_TOKEN(Ctx.Settings.Antlr4Parser, node.GetBlock6().GetToken1().GetId(), IF) &&
+ IS_TOKEN(Ctx.Settings.Antlr4Parser, node.GetBlock6().GetToken2().GetId(), NOT) &&
+ IS_TOKEN(Ctx.Settings.Antlr4Parser, node.GetBlock6().GetToken3().GetId(), EXISTS)
);
}
@@ -1057,8 +1056,8 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core&
if (node.HasBlock5()) { // IF EXISTS
missingOk = true;
Y_DEBUG_ABORT_UNLESS(
- IS_TOKEN(node.GetBlock5().GetToken1().GetId(), IF) &&
- IS_TOKEN(node.GetBlock5().GetToken2().GetId(), EXISTS)
+ IS_TOKEN(Ctx.Settings.Antlr4Parser, node.GetBlock5().GetToken1().GetId(), IF) &&
+ IS_TOKEN(Ctx.Settings.Antlr4Parser, node.GetBlock5().GetToken2().GetId(), EXISTS)
);
}
@@ -1814,8 +1813,8 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core&
if (node.HasBlock3()) { // IF EXISTS
params.MissingOk = true;
Y_DEBUG_ABORT_UNLESS(
- IS_TOKEN(node.GetBlock3().GetToken1().GetId(), IF) &&
- IS_TOKEN(node.GetBlock3().GetToken2().GetId(), EXISTS)
+ IS_TOKEN(Ctx.Settings.Antlr4Parser, node.GetBlock3().GetToken1().GetId(), IF) &&
+ IS_TOKEN(Ctx.Settings.Antlr4Parser, node.GetBlock3().GetToken2().GetId(), EXISTS)
);
}
diff --git a/yql/essentials/sql/v1/sql_translation.cpp b/yql/essentials/sql/v1/sql_translation.cpp
index 3e6f76d1ff..71dfafb3c4 100644
--- a/yql/essentials/sql/v1/sql_translation.cpp
+++ b/yql/essentials/sql/v1/sql_translation.cpp
@@ -5,9 +5,8 @@
#include "sql_values.h"
#include "sql_select.h"
#include "source.h"
+#include "antlr_token.h"
-#include <yql/essentials/parser/proto_ast/gen/v1/SQLv1Lexer.h>
-#include <yql/essentials/parser/proto_ast/gen/v1_antlr4/SQLv1Antlr4Lexer.h>
#include <yql/essentials/sql/settings/partitioning.h>
#include <yql/essentials/sql/v1/proto_parser/proto_parser.h>
@@ -1710,9 +1709,9 @@ bool TSqlTranslation::CreateTableEntry(const TRule_create_table_entry& node, TCr
auto& token = spec.GetBlock2().GetToken1();
auto tokenId = token.GetId();
- if (IS_TOKEN(tokenId, ASC)) {
+ if (IS_TOKEN(Ctx.Settings.Antlr4Parser, tokenId, ASC)) {
return true;
- } else if (IS_TOKEN(tokenId, DESC)) {
+ } else if (IS_TOKEN(Ctx.Settings.Antlr4Parser, tokenId, DESC)) {
desc = true;
return true;
} else {
@@ -3745,9 +3744,9 @@ bool TSqlTranslation::SortSpecification(const TRule_sort_specification& node, TV
const auto& token = node.GetBlock2().GetToken1();
Token(token);
auto tokenId = token.GetId();
- if (IS_TOKEN(tokenId, ASC)) {
+ if (IS_TOKEN(Ctx.Settings.Antlr4Parser, tokenId, ASC)) {
Ctx.IncrementMonCounter("sql_features", "OrderByAsc");
- } else if (IS_TOKEN(tokenId, DESC)) {
+ } else if (IS_TOKEN(Ctx.Settings.Antlr4Parser, tokenId, DESC)) {
asc = false;
Ctx.IncrementMonCounter("sql_features", "OrderByDesc");
} else {
@@ -3777,11 +3776,11 @@ bool TSqlTranslation::SortSpecificationList(const TRule_sort_specification_list&
bool TSqlTranslation::IsDistinctOptSet(const TRule_opt_set_quantifier& node) const {
TPosition pos;
- return node.HasBlock1() && IS_TOKEN(node.GetBlock1().GetToken1().GetId(), DISTINCT);
+ return node.HasBlock1() && IS_TOKEN(Ctx.Settings.Antlr4Parser, node.GetBlock1().GetToken1().GetId(), DISTINCT);
}
bool TSqlTranslation::IsDistinctOptSet(const TRule_opt_set_quantifier& node, TPosition& distinctPos) const {
- if (node.HasBlock1() && IS_TOKEN(node.GetBlock1().GetToken1().GetId(), DISTINCT)) {
+ if (node.HasBlock1() && IS_TOKEN(Ctx.Settings.Antlr4Parser, node.GetBlock1().GetToken1().GetId(), DISTINCT)) {
distinctPos = Ctx.TokenPosition(node.GetBlock1().GetToken1());
return true;
}
@@ -3857,9 +3856,9 @@ void TSqlTranslation::LoginParameter(const TRule_login_option& loginOption, std:
// login_option: LOGIN | NOLOGIN;
auto token = loginOption.GetToken1().GetId();
- if (IS_TOKEN(token, LOGIN)) {
+ if (IS_TOKEN(Ctx.Settings.Antlr4Parser, token, LOGIN)) {
canLogin = true;
- } else if (IS_TOKEN(token, NOLOGIN)) {
+ } else if (IS_TOKEN(Ctx.Settings.Antlr4Parser, token, NOLOGIN)) {
canLogin = false;
} else {
Y_ABORT("You should change implementation according to grammar changes");
diff --git a/yql/essentials/sql/v1/ya.make b/yql/essentials/sql/v1/ya.make
index 3da2293946..08fd499b3f 100644
--- a/yql/essentials/sql/v1/ya.make
+++ b/yql/essentials/sql/v1/ya.make
@@ -1,7 +1,7 @@
LIBRARY()
PEERDIR(
- library/cpp/charset
+ library/cpp/charset/lite
library/cpp/enumbitset
library/cpp/json
library/cpp/yson/node
@@ -13,14 +13,13 @@ PEERDIR(
yql/essentials/core/sql_types
yql/essentials/parser/lexer_common
yql/essentials/parser/proto_ast/collect_issues
- yql/essentials/parser/proto_ast/gen/v1
- yql/essentials/parser/proto_ast/gen/v1_ansi
yql/essentials/parser/proto_ast/gen/v1_proto_split
- yql/essentials/parser/proto_ast/gen/v1_antlr4
- yql/essentials/parser/proto_ast/gen/v1_ansi_antlr4
yql/essentials/parser/pg_catalog
yql/essentials/sql/v1/lexer
yql/essentials/sql/v1/proto_parser
+ # for lexer tokens
+ yql/essentials/parser/proto_ast/gen/v1
+ yql/essentials/parser/proto_ast/gen/v1_antlr4
)
SRCS(
diff --git a/yql/essentials/tests/s-expressions/minirun/part9/canondata/result.json b/yql/essentials/tests/s-expressions/minirun/part9/canondata/result.json
index 352f3fa6c6..f839ab81fd 100644
--- a/yql/essentials/tests/s-expressions/minirun/part9/canondata/result.json
+++ b/yql/essentials/tests/s-expressions/minirun/part9/canondata/result.json
@@ -321,6 +321,20 @@
"uri": "https://{canondata_backend}/1937367/e77ba21bdee3073d5ac816e83de0c7af4a7daf05/resource.tar.gz#test.test_Lib-Agg_List_Compare_By_Opts-default.txt-Results_/results.txt"
}
],
+ "test.test[Optimizers-EmptyOverListWithComplexUniq-default.txt-Debug]": [
+ {
+ "checksum": "fc513f9ab77aa25c63bb1a43467fe29c",
+ "size": 396,
+ "uri": "https://{canondata_backend}/995452/07ee5aca68bd02fba1bfa8356eaea86f0e5a7870/resource.tar.gz#test.test_Optimizers-EmptyOverListWithComplexUniq-default.txt-Debug_/opt.yql"
+ }
+ ],
+ "test.test[Optimizers-EmptyOverListWithComplexUniq-default.txt-Results]": [
+ {
+ "checksum": "8a557f375bf97e6adf4e8c1e40cd18c0",
+ "size": 2123,
+ "uri": "https://{canondata_backend}/995452/07ee5aca68bd02fba1bfa8356eaea86f0e5a7870/resource.tar.gz#test.test_Optimizers-EmptyOverListWithComplexUniq-default.txt-Results_/results.txt"
+ }
+ ],
"test.test[Optimizers-Fold1OneItemTuple-default.txt-Debug]": [
{
"checksum": "d92be86870a01fdf575d08165de8f8a0",
diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/EmptyOverListWithComplexUniq.yqls b/yql/essentials/tests/s-expressions/suites/Optimizers/EmptyOverListWithComplexUniq.yqls
new file mode 100644
index 0000000000..601fc0d548
--- /dev/null
+++ b/yql/essentials/tests/s-expressions/suites/Optimizers/EmptyOverListWithComplexUniq.yqls
@@ -0,0 +1,33 @@
+(
+(let config (DataSource 'config))
+(let res_sink (DataSink 'result))
+
+(let list (AsList
+ (AsStruct '('key (Uint32 '1)) '('value (Uint32 '2)))
+ (AsStruct '('key (Uint32 '2)) '('value (Uint32 '3)))
+))
+
+(let res (Aggregate list '('key 'value) '() '()))
+(let res (Map res (lambda '(item)
+ (AsStruct
+ '('composite (AsStruct
+ '('k (Member item 'key))
+ '('v (Member item 'value))
+ ))
+ '('key (Member item 'key))
+ '('value (Member item 'value))
+ )
+)))
+(let res (Take res (Uint64 '0)))
+(let res (FlatMap res (lambda '(item)
+ (Just (AsStruct
+ '('packed (StablePickle (Member item 'composite)))
+ '('composite (Member item 'composite))
+ '('key (Member item 'key))
+ '('value (Member item 'value))
+ ))
+)))
+(let world (Write! world res_sink (Key) res '('('type))))
+(let world (Commit! world res_sink))
+(return world)
+)
diff --git a/yql/essentials/tests/sql/minirun/part0/canondata/result.json b/yql/essentials/tests/sql/minirun/part0/canondata/result.json
index bdad61722c..3c1aa86fec 100644
--- a/yql/essentials/tests/sql/minirun/part0/canondata/result.json
+++ b/yql/essentials/tests/sql/minirun/part0/canondata/result.json
@@ -698,6 +698,20 @@
"uri": "https://{canondata_backend}/995452/c22e7d9867fa56e2ee0c270fded39566aaa63a48/resource.tar.gz#test.test_flexible_types-group_by2-default.txt-Results_/results.txt"
}
],
+ "test.test[join-eq_over_join_bad_rotate-default.txt-Debug]": [
+ {
+ "checksum": "75bc473eee49f48848cfac6902f99607",
+ "size": 1784,
+ "uri": "https://{canondata_backend}/1946324/fbb1b986a8af3f4a5932aa11a3d24263ba935543/resource.tar.gz#test.test_join-eq_over_join_bad_rotate-default.txt-Debug_/opt.yql"
+ }
+ ],
+ "test.test[join-eq_over_join_bad_rotate-default.txt-Results]": [
+ {
+ "checksum": "e8ae895d664f93e239570274b7b66d8d",
+ "size": 7624,
+ "uri": "https://{canondata_backend}/1946324/fbb1b986a8af3f4a5932aa11a3d24263ba935543/resource.tar.gz#test.test_join-eq_over_join_bad_rotate-default.txt-Results_/results.txt"
+ }
+ ],
"test.test[json-json_query/common_syntax-default.txt-Debug]": [
{
"checksum": "f9ce44fdf704adf735b1895d221411f0",
diff --git a/yql/essentials/tests/sql/minirun/part7/canondata/result.json b/yql/essentials/tests/sql/minirun/part7/canondata/result.json
index 6094e83958..399f6d226b 100644
--- a/yql/essentials/tests/sql/minirun/part7/canondata/result.json
+++ b/yql/essentials/tests/sql/minirun/part7/canondata/result.json
@@ -555,6 +555,20 @@
"uri": "https://{canondata_backend}/1937429/ab4dd66771a60c21f698c24d93aafbe26098e494/resource.tar.gz#test.test_in-in_with_nulls_and_optionals_extra_ansi-default.txt-Results_/results.txt"
}
],
+ "test.test[join-eq_over_join_basic-default.txt-Debug]": [
+ {
+ "checksum": "da1603d7d463a99c9b81d07ab562f3c7",
+ "size": 1625,
+ "uri": "https://{canondata_backend}/937458/0493b86f10f9e96f10e8955e2d365f91a1c2a439/resource.tar.gz#test.test_join-eq_over_join_basic-default.txt-Debug_/opt.yql"
+ }
+ ],
+ "test.test[join-eq_over_join_basic-default.txt-Results]": [
+ {
+ "checksum": "ce622e48816b3c6eb5f8255906683831",
+ "size": 5679,
+ "uri": "https://{canondata_backend}/937458/0493b86f10f9e96f10e8955e2d365f91a1c2a439/resource.tar.gz#test.test_join-eq_over_join_basic-default.txt-Results_/results.txt"
+ }
+ ],
"test.test[join-inmem_with_set_key-default.txt-Debug]": [
{
"checksum": "847a008013add2841f187dbf17a8dc0f",
diff --git a/yql/essentials/tests/sql/minirun/part8/canondata/result.json b/yql/essentials/tests/sql/minirun/part8/canondata/result.json
index 53cd1141ef..6d3cbc281d 100644
--- a/yql/essentials/tests/sql/minirun/part8/canondata/result.json
+++ b/yql/essentials/tests/sql/minirun/part8/canondata/result.json
@@ -298,16 +298,16 @@
],
"test.test[bigdate-input_interval64-default.txt-Debug]": [
{
- "checksum": "632ffcd579b1f6bbf48e367ec38e0f27",
- "size": 6397,
- "uri": "https://{canondata_backend}/1031349/5d9a68084ecc9d27b7e5271088a725bb2d294cc6/resource.tar.gz#test.test_bigdate-input_interval64-default.txt-Debug_/opt.yql"
+ "checksum": "ed3ffc93b25ff8a71b6e6659ce05acd7",
+ "size": 7477,
+ "uri": "https://{canondata_backend}/1031349/aa434f3964bbcd36b9d29e1bd64ac6be96249ceb/resource.tar.gz#test.test_bigdate-input_interval64-default.txt-Debug_/opt.yql"
}
],
"test.test[bigdate-input_interval64-default.txt-Results]": [
{
- "checksum": "c78332dfea021a025164772dffe9ba99",
- "size": 48883,
- "uri": "https://{canondata_backend}/1031349/5d9a68084ecc9d27b7e5271088a725bb2d294cc6/resource.tar.gz#test.test_bigdate-input_interval64-default.txt-Results_/results.txt"
+ "checksum": "5f8fd7865d7554bf3849cf7408f7b752",
+ "size": 57256,
+ "uri": "https://{canondata_backend}/1031349/aa434f3964bbcd36b9d29e1bd64ac6be96249ceb/resource.tar.gz#test.test_bigdate-input_interval64-default.txt-Results_/results.txt"
}
],
"test.test[bigdate-output_timestamp64-default.txt-Debug]": [
diff --git a/yql/essentials/tests/sql/minirun/part9/canondata/result.json b/yql/essentials/tests/sql/minirun/part9/canondata/result.json
index 7be476f491..26102a2ad1 100644
--- a/yql/essentials/tests/sql/minirun/part9/canondata/result.json
+++ b/yql/essentials/tests/sql/minirun/part9/canondata/result.json
@@ -358,16 +358,16 @@
],
"test.test[datetime-date_in-default.txt-Debug]": [
{
- "checksum": "d0c478d299d9317c9796a5a5689ea5ea",
- "size": 5307,
- "uri": "https://{canondata_backend}/1946324/ea335e4b9e4a11417f24b451ef1085a63908db10/resource.tar.gz#test.test_datetime-date_in-default.txt-Debug_/opt.yql"
+ "checksum": "3e6ba7312997f15d807aa25613e84db6",
+ "size": 6390,
+ "uri": "https://{canondata_backend}/1689644/ad5d4ca8c2d76550a459d4741259163b48e36d05/resource.tar.gz#test.test_datetime-date_in-default.txt-Debug_/opt.yql"
}
],
"test.test[datetime-date_in-default.txt-Results]": [
{
- "checksum": "ec449bd5df1be1823e71d727d04cc6a0",
- "size": 38190,
- "uri": "https://{canondata_backend}/1946324/ea335e4b9e4a11417f24b451ef1085a63908db10/resource.tar.gz#test.test_datetime-date_in-default.txt-Results_/results.txt"
+ "checksum": "2a6bd274cc5f3511fd4c04b0b023a275",
+ "size": 46190,
+ "uri": "https://{canondata_backend}/1689644/ad5d4ca8c2d76550a459d4741259163b48e36d05/resource.tar.gz#test.test_datetime-date_in-default.txt-Results_/results.txt"
}
],
"test.test[datetime-date_tz_bounds_scale-default.txt-Debug]": [
@@ -751,6 +751,20 @@
"uri": "https://{canondata_backend}/1942173/f37f53f2bdf2e81f0ffdb8cf146faecaff60e7af/resource.tar.gz#test.test_in-large_in_YQL-19183--Results_/results.txt"
}
],
+ "test.test[join-eq_over_join_same_keys-default.txt-Debug]": [
+ {
+ "checksum": "d333803a811e7a49abcfacfde8ce6a45",
+ "size": 1023,
+ "uri": "https://{canondata_backend}/1937424/b341ff273ddff0681036174e754c5aaffb3b2e18/resource.tar.gz#test.test_join-eq_over_join_same_keys-default.txt-Debug_/opt.yql"
+ }
+ ],
+ "test.test[join-eq_over_join_same_keys-default.txt-Results]": [
+ {
+ "checksum": "206c052c946dbbcaa9a7fb357139610a",
+ "size": 2220,
+ "uri": "https://{canondata_backend}/1942415/88908f3f1c2c172f76ce2e0c2ad414ba2d95573e/resource.tar.gz#test.test_join-eq_over_join_same_keys-default.txt-Results_/results.txt"
+ }
+ ],
"test.test[join-left_join_with_self_aggr-default.txt-Debug]": [
{
"checksum": "270003dd9cfab29ddd670dfc824c2915",
diff --git a/yql/essentials/tests/sql/sql2yql/canondata/result.json b/yql/essentials/tests/sql/sql2yql/canondata/result.json
index 5e3d3921af..f345b0e389 100644
--- a/yql/essentials/tests/sql/sql2yql/canondata/result.json
+++ b/yql/essentials/tests/sql/sql2yql/canondata/result.json
@@ -1135,9 +1135,9 @@
],
"test_sql2yql.test[bigdate-input_interval64]": [
{
- "checksum": "46d987122b7fe9750e1b1f0ce5e4fc84",
- "size": 37353,
- "uri": "https://{canondata_backend}/1942173/99e88108149e222741552e7e6cddef041d6a2846/resource.tar.gz#test_sql2yql.test_bigdate-input_interval64_/sql.yql"
+ "checksum": "b4321aecb81505bfab6638a0d276e6a0",
+ "size": 43605,
+ "uri": "https://{canondata_backend}/1689644/d756eb6544cfd8c72af37ed4481b21fa67255f71/resource.tar.gz#test_sql2yql.test_bigdate-input_interval64_/sql.yql"
}
],
"test_sql2yql.test[bigdate-input_timestamp64]": [
@@ -2094,9 +2094,9 @@
],
"test_sql2yql.test[datetime-date_in]": [
{
- "checksum": "68c9e3557926f3a6b12d602e682e9558",
- "size": 29101,
- "uri": "https://{canondata_backend}/1942173/99e88108149e222741552e7e6cddef041d6a2846/resource.tar.gz#test_sql2yql.test_datetime-date_in_/sql.yql"
+ "checksum": "f042635815e0ece3c9aa69a5cc35c44f",
+ "size": 35323,
+ "uri": "https://{canondata_backend}/1809005/d23a86a41976b5deadc93862c9d4b33c732d638d/resource.tar.gz#test_sql2yql.test_datetime-date_in_/sql.yql"
}
],
"test_sql2yql.test[datetime-date_out]": [
@@ -3793,6 +3793,27 @@
"uri": "https://{canondata_backend}/1942173/99e88108149e222741552e7e6cddef041d6a2846/resource.tar.gz#test_sql2yql.test_join-cross_join_with_lazy_list_/sql.yql"
}
],
+ "test_sql2yql.test[join-eq_over_join_bad_rotate]": [
+ {
+ "checksum": "47d27e638402a2dc26e78efb0e097629",
+ "size": 7441,
+ "uri": "https://{canondata_backend}/1946324/891082d4c661d16090f3d65c259aac9e885ad06c/resource.tar.gz#test_sql2yql.test_join-eq_over_join_bad_rotate_/sql.yql"
+ }
+ ],
+ "test_sql2yql.test[join-eq_over_join_basic]": [
+ {
+ "checksum": "91b591c1976c48ea0a66269257975354",
+ "size": 5943,
+ "uri": "https://{canondata_backend}/937458/9c1b5511bc814fa8de5eebef1b58eb00227de013/resource.tar.gz#test_sql2yql.test_join-eq_over_join_basic_/sql.yql"
+ }
+ ],
+ "test_sql2yql.test[join-eq_over_join_same_keys]": [
+ {
+ "checksum": "19517c88b9123deeb88f95937f20b46d",
+ "size": 3745,
+ "uri": "https://{canondata_backend}/1130705/cce783534f6c1bc1ef0dac74b138f4dd17bb6df8/resource.tar.gz#test_sql2yql.test_join-eq_over_join_same_keys_/sql.yql"
+ }
+ ],
"test_sql2yql.test[join-inmem_by_uncomparable_structs]": [
{
"checksum": "800f3ffe362c85dc001eb5237220bfd7",
@@ -10003,6 +10024,21 @@
"uri": "file://test_sql_format.test_join-cross_join_with_lazy_list_/formatted.sql"
}
],
+ "test_sql_format.test[join-eq_over_join_bad_rotate]": [
+ {
+ "uri": "file://test_sql_format.test_join-eq_over_join_bad_rotate_/formatted.sql"
+ }
+ ],
+ "test_sql_format.test[join-eq_over_join_basic]": [
+ {
+ "uri": "file://test_sql_format.test_join-eq_over_join_basic_/formatted.sql"
+ }
+ ],
+ "test_sql_format.test[join-eq_over_join_same_keys]": [
+ {
+ "uri": "file://test_sql_format.test_join-eq_over_join_same_keys_/formatted.sql"
+ }
+ ],
"test_sql_format.test[join-inmem_by_uncomparable_structs]": [
{
"uri": "file://test_sql_format.test_join-inmem_by_uncomparable_structs_/formatted.sql"
diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_bigdate-input_interval64_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_bigdate-input_interval64_/formatted.sql
index fd9bb1ad40..8afe7e3265 100644
--- a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_bigdate-input_interval64_/formatted.sql
+++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_bigdate-input_interval64_/formatted.sql
@@ -89,11 +89,21 @@ SELECT
;
SELECT
+ CAST('-P106751616D' AS interval64),
+ CAST(CAST('-P106751616D' AS interval64) AS string)
+;
+
+SELECT
CAST('P106751617D' AS interval64),
CAST(CAST('P106751617D' AS interval64) AS string)
;
SELECT
+ CAST('-P106751617D' AS interval64),
+ CAST(CAST('-P106751617D' AS interval64) AS string)
+;
+
+SELECT
CAST('P15250230W' AS interval64),
CAST(CAST('P15250230W' AS interval64) AS string)
;
@@ -104,6 +114,26 @@ SELECT
;
SELECT
+ CAST('PT9223339708799S' AS interval64),
+ CAST(CAST('PT9223339708799S' AS interval64) AS string)
+;
+
+SELECT
+ CAST('-PT9223339708799S' AS interval64),
+ CAST(CAST('-PT9223339708799S' AS interval64) AS string)
+;
+
+SELECT
+ CAST('PT9223339708800S' AS interval64),
+ CAST(CAST('PT9223339708800S' AS interval64) AS string)
+;
+
+SELECT
+ CAST('-PT9223339708800S' AS interval64),
+ CAST(CAST('-PT9223339708800S' AS interval64) AS string)
+;
+
+SELECT
CAST('PT0000000S' AS interval64),
CAST(CAST('PT0000000S' AS interval64) AS string)
;
diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_datetime-date_in_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_datetime-date_in_/formatted.sql
index ee14d8206b..e9923597fe 100644
--- a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_datetime-date_in_/formatted.sql
+++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_datetime-date_in_/formatted.sql
@@ -135,6 +135,36 @@ SELECT
;
SELECT
+ CAST('-P49672DT23H59M59.999999S' AS interval),
+ CAST(CAST('-P49672DT23H59M59.999999S' AS interval) AS string)
+;
+
+SELECT
CAST('P49673D' AS interval),
CAST(CAST('P49673D' AS interval) AS string)
;
+
+SELECT
+ CAST('-P49673D' AS interval),
+ CAST(CAST('-P49673D' AS interval) AS string)
+;
+
+SELECT
+ CAST('PT4291747199S' AS interval),
+ CAST(CAST('PT4291747199S' AS interval) AS string)
+;
+
+SELECT
+ CAST('-PT4291747199S' AS interval),
+ CAST(CAST('-PT4291747199S' AS interval) AS string)
+;
+
+SELECT
+ CAST('PT4291747200S' AS interval),
+ CAST(CAST('PT4291747200S' AS interval) AS string)
+;
+
+SELECT
+ CAST('-PT4291747200S' AS interval),
+ CAST(CAST('-PT4291747200S' AS interval) AS string)
+;
diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_join-eq_over_join_bad_rotate_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_join-eq_over_join_bad_rotate_/formatted.sql
new file mode 100644
index 0000000000..7dd88528df
--- /dev/null
+++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_join-eq_over_join_bad_rotate_/formatted.sql
@@ -0,0 +1,94 @@
+PRAGMA config.flags('OptimizerFlags', 'EqualityFilterOverJoin');
+PRAGMA AnsiOptionalAs;
+
+-- part of tpcds-6
+$item = (
+ SELECT
+ *
+ FROM
+ as_table([
+ <|i_current_price: Just(1.0f), i_category: Just('aaa'), i_item_sk: Just(125l)|>,
+ <|i_current_price: Just(2.0f), i_category: Just('bbb'), i_item_sk: Just(999l)|>,
+ ])
+);
+
+$sub2 = (
+ SELECT
+ i_current_price,
+ i_category
+ FROM
+ $item
+);
+
+$customer_address = (
+ SELECT
+ *
+ FROM
+ as_table([
+ <|ca_address_sk: Just(120l)|>,
+ <|ca_address_sk: Just(150l)|>,
+ ])
+);
+
+$customer = (
+ SELECT
+ *
+ FROM
+ as_table([
+ <|c_current_addr_sk: Just(150l), c_customer_sk: Just(4l)|>,
+ <|c_current_addr_sk: Just(120l), c_customer_sk: Just(2l)|>,
+ ])
+);
+
+$store_sales = (
+ SELECT
+ *
+ FROM
+ as_table([
+ <|ss_sold_date_sk: Just(1l), ss_customer_sk: Just(2l), ss_item_sk: Just(3l)|>,
+ <|ss_sold_date_sk: Just(3l), ss_customer_sk: Just(4l), ss_item_sk: Just(5l)|>,
+ ])
+);
+
+$date_dim = (
+ SELECT
+ *
+ FROM
+ as_table([
+ <|d_date_sk: Just(1l)|>,
+ <|d_date_sk: Just(2l)|>,
+ ])
+);
+
+$item = (
+ SELECT
+ *
+ FROM
+ as_table([
+ <|i_category: Just('aaa'), i_item_sk: Just(3l)|>,
+ <|i_category: Just('bbb'), i_item_sk: Just(5l)|>,
+ ])
+);
+
+SELECT
+ JoinTableRow() cnt
+FROM
+ $customer_address a
+CROSS JOIN
+ $customer c
+CROSS JOIN
+ $store_sales s
+CROSS JOIN
+ $date_dim d
+CROSS JOIN
+ $item i
+LEFT JOIN
+ $sub2 AS j
+ON
+ i.i_category == j.i_category
+WHERE
+ s.ss_sold_date_sk == d.d_date_sk
+ AND a.ca_address_sk == c.c_current_addr_sk
+ AND c.c_customer_sk == s.ss_customer_sk
+ AND s.ss_item_sk == i.i_item_sk
+;
diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_join-eq_over_join_basic_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_join-eq_over_join_basic_/formatted.sql
new file mode 100644
index 0000000000..d3850b024f
--- /dev/null
+++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_join-eq_over_join_basic_/formatted.sql
@@ -0,0 +1,69 @@
+PRAGMA config.flags('OptimizerFlags', 'EqualityFilterOverJoin');
+
+$a = (
+ SELECT
+ *
+ FROM
+ as_table([
+ <|x: Just(1), t: 1, u: 1, extra: 1|>,
+ <|x: 1, t: 1, u: 5, extra: 2|>,
+ ])
+);
+
+$b = (
+ SELECT
+ *
+ FROM
+ as_table([
+ <|y: 1|>,
+ <|y: 1|>,
+ ])
+);
+
+$c = (
+ SELECT
+ *
+ FROM
+ as_table([
+ <|z: 1|>,
+ <|z: 1|>,
+ ])
+);
+
+$d = (
+ SELECT
+ *
+ FROM
+ as_table([
+ <|c: 2, d: 3|>,
+ <|c: 3, d: 3|>,
+ ])
+);
+
+SELECT
+ *
+FROM (
+ SELECT
+ c.z AS cz,
+ b.y AS by,
+ a.u AS au,
+ a.t AS at,
+ a.x AS ax,
+ d.c AS dc,
+ d.d AS dd
+ FROM
+ $a AS a
+ RIGHT JOIN
+ $b AS b
+ ON
+ a.x == b.y
+ CROSS JOIN
+ $d AS d
+ FULL JOIN
+ $c AS c
+ ON
+ b.y == c.z
+)
+WHERE
+ cz == at AND by == au AND ax == by AND dc == dd
+;
diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_join-eq_over_join_same_keys_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_join-eq_over_join_same_keys_/formatted.sql
new file mode 100644
index 0000000000..657baf30f5
--- /dev/null
+++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_join-eq_over_join_same_keys_/formatted.sql
@@ -0,0 +1,35 @@
+PRAGMA config.flags('OptimizerFlags', 'EqualityFilterOverJoin');
+
+$p = 1;
+
+$simpleKey = (
+ SELECT
+ *
+ FROM
+ as_table([<|Key: Just(1), Value: 'qqq'|>, <|Key: Just(2), Value: 'aaa'|>])
+);
+
+$complexKey = (
+ SELECT
+ *
+ FROM
+ as_table([<|Key: Just(2), Fk: 2, Value: 'zzz'|>, <|Key: Just(2), Fk: 3, Value: 'ttt'|>])
+);
+
+SELECT
+ l.Key,
+ l.Fk,
+ l.Value,
+ r.Key,
+ r.Value
+FROM
+ $simpleKey AS r
+INNER JOIN
+ $complexKey AS l
+ON
+ l.Fk == r.Key
+WHERE
+ l.Key == 1 + $p AND l.Key == l.Key
+ORDER BY
+ r.Value
+;
diff --git a/yql/essentials/tests/sql/suites/bigdate/input_interval64.sql b/yql/essentials/tests/sql/suites/bigdate/input_interval64.sql
index 47627b6003..052bf2a126 100644
--- a/yql/essentials/tests/sql/suites/bigdate/input_interval64.sql
+++ b/yql/essentials/tests/sql/suites/bigdate/input_interval64.sql
@@ -21,10 +21,17 @@ select cast("PT999999M" as interval64), cast(cast("PT999999M" as interval64) as
select cast("PT999999H" as interval64), cast(cast("PT999999H" as interval64) as string);
select cast("P106751616D" as interval64), cast(cast("P106751616D" as interval64) as string);
+select cast("-P106751616D" as interval64), cast(cast("-P106751616D" as interval64) as string);
select cast("P106751617D" as interval64), cast(cast("P106751617D" as interval64) as string);
+select cast("-P106751617D" as interval64), cast(cast("-P106751617D" as interval64) as string);
select cast("P15250230W" as interval64), cast(cast("P15250230W" as interval64) as string);
select cast("P15250231W" as interval64), cast(cast("P15250231W" as interval64) as string);
+select cast("PT9223339708799S" as interval64), cast(cast("PT9223339708799S" as interval64) as string);
+select cast("-PT9223339708799S" as interval64), cast(cast("-PT9223339708799S" as interval64) as string);
+select cast("PT9223339708800S" as interval64), cast(cast("PT9223339708800S" as interval64) as string);
+select cast("-PT9223339708800S" as interval64), cast(cast("-PT9223339708800S" as interval64) as string);
+
select cast("PT0000000S" as interval64), cast(cast("PT0000000S" as interval64) as string);
select cast("PT0000000M" as interval64), cast(cast("PT0000000M" as interval64) as string);
select cast("PT0000000H" as interval64), cast(cast("PT0000000H" as interval64) as string);
diff --git a/yql/essentials/tests/sql/suites/datetime/date_in.sql b/yql/essentials/tests/sql/suites/datetime/date_in.sql
index 158c08c964..45a0d0d63f 100644
--- a/yql/essentials/tests/sql/suites/datetime/date_in.sql
+++ b/yql/essentials/tests/sql/suites/datetime/date_in.sql
@@ -32,4 +32,11 @@ select cast("2105-12-31T23:59:59.999999Z" as timestamp),cast(cast("2105-12-31T23
select cast("2106-01-01T00:00:00.000000Z" as timestamp),cast(cast("2106-01-01T00:00:00.000000Z" as timestamp) as string);
select cast("P49672DT23H59M59.999999S" as interval),cast(cast("P49672DT23H59M59.999999S" as interval) as string);
+select cast("-P49672DT23H59M59.999999S" as interval),cast(cast("-P49672DT23H59M59.999999S" as interval) as string);
select cast("P49673D" as interval),cast(cast("P49673D" as interval) as string);
+select cast("-P49673D" as interval),cast(cast("-P49673D" as interval) as string);
+
+select cast("PT4291747199S" as interval),cast(cast("PT4291747199S" as interval) as string);
+select cast("-PT4291747199S" as interval),cast(cast("-PT4291747199S" as interval) as string);
+select cast("PT4291747200S" as interval),cast(cast("PT4291747200S" as interval) as string);
+select cast("-PT4291747200S" as interval),cast(cast("-PT4291747200S" as interval) as string);
diff --git a/yql/essentials/tests/sql/suites/join/eq_over_join_bad_rotate.sql b/yql/essentials/tests/sql/suites/join/eq_over_join_bad_rotate.sql
new file mode 100644
index 0000000000..c3d151ccf7
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/join/eq_over_join_bad_rotate.sql
@@ -0,0 +1,50 @@
+pragma config.flags("OptimizerFlags", "EqualityFilterOverJoin");
+
+pragma AnsiOptionalAs;
+
+-- part of tpcds-6
+
+$item = select * from as_table([
+ <|i_current_price:Just(1.0f), i_category:Just("aaa"), i_item_sk:Just(125l)|>,
+ <|i_current_price:Just(2.0f), i_category:Just("bbb"), i_item_sk:Just(999l)|>,
+ ]);
+
+$sub2 = (select i_current_price, i_category from $item);
+
+$customer_address = select * from as_table([
+ <|ca_address_sk:Just(120l)|>,
+ <|ca_address_sk:Just(150l)|>,
+ ]);
+
+$customer = select * from as_table([
+ <|c_current_addr_sk:Just(150l), c_customer_sk:Just(4l)|>,
+ <|c_current_addr_sk:Just(120l), c_customer_sk:Just(2l)|>,
+ ]);
+
+$store_sales = select * from as_table([
+ <|ss_sold_date_sk:Just(1l), ss_customer_sk:Just(2l), ss_item_sk:Just(3l)|>,
+ <|ss_sold_date_sk:Just(3l), ss_customer_sk:Just(4l), ss_item_sk:Just(5l)|>,
+ ]);
+
+$date_dim = select * from as_table([
+ <|d_date_sk:Just(1l)|>,
+ <|d_date_sk:Just(2l)|>,
+ ]);
+
+$item = select * from as_table([
+ <|i_category:Just("aaa"), i_item_sk:Just(3l)|>,
+ <|i_category:Just("bbb"), i_item_sk:Just(5l)|>,
+ ]);
+
+select JoinTableRow() cnt
+from $customer_address a
+ cross join $customer c
+ cross join $store_sales s
+ cross join $date_dim d
+ cross join $item i
+ left join $sub2 as j on i.i_category = j.i_category
+ where
+ s.ss_sold_date_sk = d.d_date_sk
+ and a.ca_address_sk = c.c_current_addr_sk
+ and c.c_customer_sk = s.ss_customer_sk
+ and s.ss_item_sk = i.i_item_sk
diff --git a/yql/essentials/tests/sql/suites/join/eq_over_join_basic.sql b/yql/essentials/tests/sql/suites/join/eq_over_join_basic.sql
new file mode 100644
index 0000000000..83f706fd6a
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/join/eq_over_join_basic.sql
@@ -0,0 +1,30 @@
+pragma config.flags("OptimizerFlags", "EqualityFilterOverJoin");
+
+$a = select * from as_table([
+ <|x:Just(1), t:1, u:1, extra:1|>,
+ <|x:1, t:1, u:5, extra:2|>,
+ ]);
+
+$b = select * from as_table([
+ <|y:1|>,
+ <|y:1|>,
+ ]);
+
+$c = select * from as_table([
+ <|z:1|>,
+ <|z:1|>,
+ ]);
+
+$d = select * from as_table([
+ <|c:2, d:3|>,
+ <|c:3, d:3|>,
+ ]);
+
+
+select * from (
+ select c.z as cz, b.y as by, a.u as au, a.t as at, a.x as ax, d.c as dc, d.d as dd from
+ $a as a right join $b as b on a.x=b.y
+ cross join $d as d
+ full join $c as c on b.y = c.z
+)
+where cz = at and by = au and ax = by and dc = dd;
diff --git a/yql/essentials/tests/sql/suites/join/eq_over_join_same_keys.sql b/yql/essentials/tests/sql/suites/join/eq_over_join_same_keys.sql
new file mode 100644
index 0000000000..cdd242269d
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/join/eq_over_join_same_keys.sql
@@ -0,0 +1,18 @@
+pragma config.flags("OptimizerFlags", "EqualityFilterOverJoin");
+
+$p = 1;
+
+
+$simpleKey =
+select * from as_table([<|Key:Just(1), Value:"qqq"|>, <|Key:Just(2), Value:"aaa"|>]);
+
+$complexKey =
+select * from as_table([<|Key:Just(2), Fk:2, Value:"zzz"|>, <|Key:Just(2), Fk:3, Value:"ttt"|>]);
+
+
+
+SELECT l.Key, l.Fk, l.Value, r.Key, r.Value FROM $simpleKey AS r
+INNER JOIN $complexKey AS l
+ ON l.Fk = r.Key
+WHERE l.Key = 1 + $p and l.Key = l.Key
+ORDER BY r.Value
diff --git a/yql/essentials/udfs/common/string/string_udf.cpp b/yql/essentials/udfs/common/string/string_udf.cpp
index d621e92582..92a1fe6b5a 100644
--- a/yql/essentials/udfs/common/string/string_udf.cpp
+++ b/yql/essentials/udfs/common/string/string_udf.cpp
@@ -2,7 +2,6 @@
#include <yql/essentials/public/udf/udf_helpers.h>
#include <yql/essentials/public/udf/udf_value_builder.h>
-#include <library/cpp/charset/codepage.h>
#include <library/cpp/deprecated/split/split_iterator.h>
#include <library/cpp/html/pcdata/pcdata.h>
#include <library/cpp/string_utils/base32/base32.h>
diff --git a/yql/essentials/udfs/common/string/ya.make b/yql/essentials/udfs/common/string/ya.make
index 80d2270b32..280686fb46 100644
--- a/yql/essentials/udfs/common/string/ya.make
+++ b/yql/essentials/udfs/common/string/ya.make
@@ -12,7 +12,6 @@ YQL_UDF_CONTRIB(string_udf)
PEERDIR(
yql/essentials/public/udf/arrow
- library/cpp/charset
library/cpp/deprecated/split
library/cpp/html/pcdata
library/cpp/string_utils/base32
diff --git a/yql/essentials/udfs/common/url_base/lib/ya.make b/yql/essentials/udfs/common/url_base/lib/ya.make
index 1d9cfa12d0..9887842303 100644
--- a/yql/essentials/udfs/common/url_base/lib/ya.make
+++ b/yql/essentials/udfs/common/url_base/lib/ya.make
@@ -13,7 +13,7 @@ SRCS(
)
PEERDIR(
- library/cpp/charset
+ library/cpp/charset/lite
library/cpp/string_utils/quote
library/cpp/string_utils/url
library/cpp/tld
diff --git a/yql/essentials/utils/fetch/ya.make b/yql/essentials/utils/fetch/ya.make
index e95b4fde60..07faa82932 100644
--- a/yql/essentials/utils/fetch/ya.make
+++ b/yql/essentials/utils/fetch/ya.make
@@ -5,7 +5,7 @@ SRCS(
)
PEERDIR(
- library/cpp/charset
+ library/cpp/charset/lite
library/cpp/http/io
library/cpp/http/misc
library/cpp/openssl/io
diff --git a/yql/tools/yqlrun/http/ya.make b/yql/tools/yqlrun/http/ya.make
index d9e85ed88e..3b4a4f5fb6 100644
--- a/yql/tools/yqlrun/http/ya.make
+++ b/yql/tools/yqlrun/http/ya.make
@@ -49,7 +49,7 @@ PEERDIR(
library/cpp/logger
library/cpp/yson/node
library/cpp/openssl/io
- library/cpp/charset
+ library/cpp/charset/lite
library/cpp/yson
library/cpp/json
library/cpp/string_utils/quote
diff --git a/yt/cpp/mapreduce/http_client/raw_requests.cpp b/yt/cpp/mapreduce/http_client/raw_requests.cpp
index 130bea7eb8..75514b8682 100644
--- a/yt/cpp/mapreduce/http_client/raw_requests.cpp
+++ b/yt/cpp/mapreduce/http_client/raw_requests.cpp
@@ -43,7 +43,7 @@ TOperationAttributes ParseOperationAttributes(const TNode& node)
if (auto typeNode = mapNode.FindPtr("type")) {
result.Type = FromString<EOperationType>(typeNode->AsString());
} else if (auto operationTypeNode = mapNode.FindPtr("operation_type")) {
- // COMPAT(levysotsky): "operation_type" is a deprecated synonym for "type".
+ // COMPAT(levysotsky): (YT-24340) "operation_type" is a deprecated synonym for "type".
// This branch should be removed when all clusters are updated.
result.Type = FromString<EOperationType>(operationTypeNode->AsString());
}
diff --git a/yt/cpp/mapreduce/interface/serialize.h b/yt/cpp/mapreduce/interface/serialize.h
index 223dd446ba..6127b808c7 100644
--- a/yt/cpp/mapreduce/interface/serialize.h
+++ b/yt/cpp/mapreduce/interface/serialize.h
@@ -7,6 +7,8 @@
#include "common.h"
+#include <library/cpp/yson/writer.h>
+
#include <library/cpp/type_info/fwd.h>
namespace NYT::NYson {
@@ -85,6 +87,17 @@ void Deserialize(TGUID& value, const TNode& node);
void Serialize(const NTi::TTypePtr& type, NYT::NYson::IYsonConsumer* consumer);
void Deserialize(NTi::TTypePtr& type, const TNode& node);
+template <typename T>
+TString ToYsonText(const T& value)
+{
+ TStringStream out;
+ ::NYson::TYsonWriter writer(&out, ::NYson::EYsonFormat::Text);
+
+ Serialize(value, &writer);
+
+ return out.Str();
+}
+
////////////////////////////////////////////////////////////////////////////////
} // namespace NYT
diff --git a/yt/yql/providers/yt/fmr/coordinator/impl/ut/yql_yt_coordinator_ut.cpp b/yt/yql/providers/yt/fmr/coordinator/impl/ut/yql_yt_coordinator_ut.cpp
index 36d7020adc..495057d550 100644
--- a/yt/yql/providers/yt/fmr/coordinator/impl/ut/yql_yt_coordinator_ut.cpp
+++ b/yt/yql/providers/yt/fmr/coordinator/impl/ut/yql_yt_coordinator_ut.cpp
@@ -45,12 +45,17 @@ private:
TDownloadTaskParams downloadTaskParams{
- .Input = TYtTableRef{"Path","Cluster","TransactionId"},
+ .Input = TYtTableRef{"Path","Cluster"},
.Output = TFmrTableRef{"TableId"}
};
TStartOperationRequest CreateOperationRequest(ETaskType taskType = ETaskType::Download, TTaskParams taskParams = downloadTaskParams) {
- return TStartOperationRequest{.TaskType = taskType, .TaskParams = taskParams, .SessionId = "SessionId", .IdempotencyKey = "IdempotencyKey"};
+ return TStartOperationRequest{
+ .TaskType = taskType,
+ .TaskParams = taskParams,
+ .IdempotencyKey = "IdempotencyKey",
+ .ClusterConnection = TClusterConnection{.TransactionId = "transaction_id", .YtServerName = "hahn.yt.yandex.net", .Token = "token"}
+ };
}
std::vector<TStartOperationRequest> CreateSeveralOperationRequests(
@@ -59,7 +64,10 @@ std::vector<TStartOperationRequest> CreateSeveralOperationRequests(
std::vector<TStartOperationRequest> startOperationRequests(numRequests);
for (int i = 0; i < numRequests; ++i) {
startOperationRequests[i] = TStartOperationRequest{
- .TaskType = taskType, .TaskParams = taskParams, .IdempotencyKey = "IdempotencyKey_" + ToString(i)
+ .TaskType = taskType,
+ .TaskParams = taskParams,
+ .IdempotencyKey = "IdempotencyKey_" + ToString(i),
+ .ClusterConnection = TClusterConnection{.TransactionId = "transaction_id", .YtServerName = "hahn", .Token = "token"}
};
}
return startOperationRequests;
diff --git a/yt/yql/providers/yt/fmr/coordinator/impl/yql_yt_coordinator_impl.cpp b/yt/yql/providers/yt/fmr/coordinator/impl/yql_yt_coordinator_impl.cpp
index bec951c180..bd73813a43 100644
--- a/yt/yql/providers/yt/fmr/coordinator/impl/yql_yt_coordinator_impl.cpp
+++ b/yt/yql/providers/yt/fmr/coordinator/impl/yql_yt_coordinator_impl.cpp
@@ -57,7 +57,7 @@ public:
}
TString taskId = GenerateId();
- TTask::TPtr createdTask = MakeTask(request.TaskType, taskId, request.TaskParams, request.SessionId);
+ TTask::TPtr createdTask = MakeTask(request.TaskType, taskId, request.TaskParams, request.SessionId, request.ClusterConnection);
Tasks_[taskId] = TCoordinatorTaskInfo{.Task = createdTask, .TaskStatus = ETaskStatus::Accepted, .OperationId = operationId};
diff --git a/yt/yql/providers/yt/fmr/coordinator/interface/proto_helpers/yql_yt_coordinator_proto_helpers.cpp b/yt/yql/providers/yt/fmr/coordinator/interface/proto_helpers/yql_yt_coordinator_proto_helpers.cpp
index 85f7d5834a..69908b3ac0 100644
--- a/yt/yql/providers/yt/fmr/coordinator/interface/proto_helpers/yql_yt_coordinator_proto_helpers.cpp
+++ b/yt/yql/providers/yt/fmr/coordinator/interface/proto_helpers/yql_yt_coordinator_proto_helpers.cpp
@@ -68,6 +68,8 @@ NProto::TStartOperationRequest StartOperationRequestToProto(const TStartOperatio
protoStartOperationRequest.SetIdempotencyKey(*startOperationRequest.IdempotencyKey);
}
protoStartOperationRequest.SetNumRetries(startOperationRequest.NumRetries);
+ auto protoClusterConnection = ClusterConnectionToProto(startOperationRequest.ClusterConnection);
+ protoStartOperationRequest.MutableClusterConnection()->Swap(&protoClusterConnection);
return protoStartOperationRequest;
}
@@ -80,6 +82,7 @@ TStartOperationRequest StartOperationRequestFromProto(const NProto::TStartOperat
startOperationRequest.IdempotencyKey = protoStartOperationRequest.GetIdempotencyKey();
}
startOperationRequest.NumRetries = protoStartOperationRequest.GetNumRetries();
+ startOperationRequest.ClusterConnection = ClusterConnectionFromProto(protoStartOperationRequest.GetClusterConnection());
return startOperationRequest;
}
diff --git a/yt/yql/providers/yt/fmr/coordinator/interface/yql_yt_coordinator.h b/yt/yql/providers/yt/fmr/coordinator/interface/yql_yt_coordinator.h
index 8be5d757c2..ba9592de9a 100644
--- a/yt/yql/providers/yt/fmr/coordinator/interface/yql_yt_coordinator.h
+++ b/yt/yql/providers/yt/fmr/coordinator/interface/yql_yt_coordinator.h
@@ -26,6 +26,7 @@ struct TStartOperationRequest {
TString SessionId;
TMaybe<TString> IdempotencyKey = Nothing();
ui32 NumRetries = 1; // Not supported yet
+ TClusterConnection ClusterConnection = {};
};
struct TStartOperationResponse {
diff --git a/yt/yql/providers/yt/fmr/job/impl/ut/ya.make b/yt/yql/providers/yt/fmr/job/impl/ut/ya.make
new file mode 100644
index 0000000000..b2fb4a8853
--- /dev/null
+++ b/yt/yql/providers/yt/fmr/job/impl/ut/ya.make
@@ -0,0 +1,16 @@
+UNITTEST()
+
+SRCS(
+ yql_yt_job_ut.cpp
+)
+
+PEERDIR(
+ yt/yql/providers/yt/fmr/job/impl
+ yt/yql/providers/yt/fmr/yt_service/mock
+ yt/yql/providers/yt/fmr/table_data_service/local
+ yql/essentials/utils/log
+)
+
+YQL_LAST_ABI_VERSION()
+
+END()
diff --git a/yt/yql/providers/yt/fmr/job/impl/ut/yql_yt_job_ut.cpp b/yt/yql/providers/yt/fmr/job/impl/ut/yql_yt_job_ut.cpp
new file mode 100644
index 0000000000..053a106ea5
--- /dev/null
+++ b/yt/yql/providers/yt/fmr/job/impl/ut/yql_yt_job_ut.cpp
@@ -0,0 +1,318 @@
+#include <library/cpp/testing/unittest/registar.h>
+#include <yt/yql/providers/yt/fmr/job/impl/yql_yt_job_impl.h>
+#include <yt/yql/providers/yt/fmr/table_data_service/local/table_data_service.h>
+#include <yt/yql/providers/yt/fmr/yt_service/mock/yql_yt_yt_service_mock.h>
+
+namespace NYql::NFmr {
+
+Y_UNIT_TEST_SUITE(FmrJobTests) {
+ Y_UNIT_TEST(DownloadTable) {
+ TString tableContent =
+ "{\"key\"=\"075\";\"subkey\"=\"1\";\"value\"=\"abc\"};"
+ "{\"key\"=\"800\";\"subkey\"=\"2\";\"value\"=\"ddd\"};"
+ "{\"key\"=\"020\";\"subkey\"=\"3\";\"value\"=\"q\"};"
+ "{\"key\"=\"150\";\"subkey\"=\"4\";\"value\"=\"qzz\"};";
+
+ ITableDataService::TPtr tableDataServicePtr = MakeLocalTableDataService(TLocalTableDataServiceSettings(1));
+ TYtUploadedTablesMock::TPtr ytUploadedTablesMock = MakeYtUploadedTablesMock();
+ NYql::NFmr::IYtService::TPtr ytService = MakeYtServiceMock(ytUploadedTablesMock);
+ std::shared_ptr<std::atomic<bool>> cancelFlag = std::make_shared<std::atomic<bool>>(false);
+ IFmrJob::TPtr job = MakeFmrJob(tableDataServicePtr, ytService, cancelFlag);
+
+ TYtTableRef input = TYtTableRef("test_cluster", "test_path");
+ TFmrTableRef output = TFmrTableRef("test_table_id");
+ TDownloadTaskParams params = TDownloadTaskParams(input, output);
+
+ ytUploadedTablesMock->AddTable(input, tableContent);
+
+ auto err = job->Download(params);
+
+ UNIT_ASSERT_C(!err,err.GetRef());
+ UNIT_ASSERT_NO_DIFF(tableDataServicePtr->Get("test_table_id").GetValueSync().GetRef(), tableContent);
+ }
+
+ Y_UNIT_TEST(UploadTable) {
+ TString ytTableContent =
+ "{\"key\"=\"075\";\"subkey\"=\"1\";\"value\"=\"abc\"};"
+ "{\"key\"=\"800\";\"subkey\"=\"2\";\"value\"=\"ddd\"};"
+ "{\"key\"=\"020\";\"subkey\"=\"3\";\"value\"=\"q\"};"
+ "{\"key\"=\"150\";\"subkey\"=\"4\";\"value\"=\"qzz\"};";
+
+ ITableDataService::TPtr tableDataServicePtr = MakeLocalTableDataService(TLocalTableDataServiceSettings(1));
+ TYtUploadedTablesMock::TPtr ytUploadedTablesMock = MakeYtUploadedTablesMock();
+ NYql::NFmr::IYtService::TPtr ytService = MakeYtServiceMock(ytUploadedTablesMock);
+ std::shared_ptr<std::atomic<bool>> cancelFlag = std::make_shared<std::atomic<bool>>(false);
+ IFmrJob::TPtr job = MakeFmrJob(tableDataServicePtr, ytService, cancelFlag);
+
+ TYtTableRef output = TYtTableRef("test_cluster", "test_path");
+ TFmrTableRef input = TFmrTableRef("test_table_id");
+ TUploadTaskParams params = TUploadTaskParams(input, output);
+
+ tableDataServicePtr->Put(input.TableId, ytTableContent);
+
+ auto err = job->Upload(params);
+
+ UNIT_ASSERT_C(!err,err.GetRef());
+ UNIT_ASSERT_NO_DIFF(ytUploadedTablesMock->GetTableContent(output), ytTableContent);
+ }
+
+ Y_UNIT_TEST(MergeFmrTables) {
+ TString TableContent_1 =
+ "{\"key\"=\"1\";\"subkey\"=\"1\";\"value\"=\"abc\"};"
+ "{\"key\"=\"2\";\"subkey\"=\"2\";\"value\"=\"ddd\"};"
+ "{\"key\"=\"3\";\"subkey\"=\"3\";\"value\"=\"q\"};"
+ "{\"key\"=\"4\";\"subkey\"=\"4\";\"value\"=\"qzz\"};)";
+ TString TableContent_2 =
+ "{\"key\"=\"5\";\"subkey\"=\"1\";\"value\"=\"abc\"};"
+ "{\"key\"=\"6\";\"subkey\"=\"2\";\"value\"=\"ddd\"};"
+ "{\"key\"=\"7\";\"subkey\"=\"3\";\"value\"=\"q\"};"
+ "{\"key\"=\"8\";\"subkey\"=\"4\";\"value\"=\"qzz\"};";
+ TString TableContent_3 =
+ "{\"key\"=\"9\";\"subkey\"=\"1\";\"value\"=\"abc\"};"
+ "{\"key\"=\"10\";\"subkey\"=\"2\";\"value\"=\"ddd\"};"
+ "{\"key\"=\"11\";\"subkey\"=\"3\";\"value\"=\"q\"};"
+ "{\"key\"=\"12\";\"subkey\"=\"4\";\"value\"=\"qzz\"};";
+
+ ITableDataService::TPtr tableDataServicePtr = MakeLocalTableDataService(TLocalTableDataServiceSettings(1));
+ TYtUploadedTablesMock::TPtr ytUploadedTablesMock = MakeYtUploadedTablesMock();
+ NYql::NFmr::IYtService::TPtr ytService = MakeYtServiceMock(ytUploadedTablesMock);
+ std::shared_ptr<std::atomic<bool>> cancelFlag = std::make_shared<std::atomic<bool>>(false);
+ IFmrJob::TPtr job = MakeFmrJob(tableDataServicePtr, ytService, cancelFlag);
+
+ TFmrTableRef input_1 = TFmrTableRef("test_table_id_1");
+ TFmrTableRef input_2 = TFmrTableRef("test_table_id_2");
+ TFmrTableRef input_3 = TFmrTableRef("test_table_id_3");
+ TTableRef input_table_ref_1 = {input_1};
+ TTableRef input_table_ref_2 = {input_2};
+ TTableRef input_table_ref_3 = {input_3};
+ TFmrTableRef output = TFmrTableRef("test_table_id_output");
+ std::vector<TTableRef> inputs = {input_table_ref_1, input_table_ref_2, input_table_ref_3};
+ TMergeTaskParams params = TMergeTaskParams(inputs, output);
+
+ tableDataServicePtr->Put(input_1.TableId, TableContent_1);
+ tableDataServicePtr->Put(input_2.TableId, TableContent_2);
+ tableDataServicePtr->Put(input_3.TableId, TableContent_3);
+
+ auto err = job->Merge(params);
+
+ UNIT_ASSERT_C(!err,err.GetRef());
+ UNIT_ASSERT_NO_DIFF(tableDataServicePtr->Get(output.TableId).GetValueSync().GetRef(), TableContent_1 + TableContent_2 + TableContent_3);
+ }
+
+ Y_UNIT_TEST(MergeMixedTables) {
+ TString TableContent_1 =
+ "{\"key\"=\"1\";\"subkey\"=\"1\";\"value\"=\"abc\"};"
+ "{\"key\"=\"2\";\"subkey\"=\"2\";\"value\"=\"ddd\"};"
+ "{\"key\"=\"3\";\"subkey\"=\"3\";\"value\"=\"q\"};"
+ "{\"key\"=\"4\";\"subkey\"=\"4\";\"value\"=\"qzz\"};)";
+ TString TableContent_2 =
+ "{\"key\"=\"5\";\"subkey\"=\"1\";\"value\"=\"abc\"};"
+ "{\"key\"=\"6\";\"subkey\"=\"2\";\"value\"=\"ddd\"};"
+ "{\"key\"=\"7\";\"subkey\"=\"3\";\"value\"=\"q\"};"
+ "{\"key\"=\"8\";\"subkey\"=\"4\";\"value\"=\"qzz\"};";
+ TString TableContent_3 =
+ "{\"key\"=\"9\";\"subkey\"=\"1\";\"value\"=\"abc\"};"
+ "{\"key\"=\"10\";\"subkey\"=\"2\";\"value\"=\"ddd\"};"
+ "{\"key\"=\"11\";\"subkey\"=\"3\";\"value\"=\"q\"};"
+ "{\"key\"=\"12\";\"subkey\"=\"4\";\"value\"=\"qzz\"};";
+
+ ITableDataService::TPtr tableDataServicePtr = MakeLocalTableDataService(TLocalTableDataServiceSettings(1));
+ TYtUploadedTablesMock::TPtr ytUploadedTablesMock = MakeYtUploadedTablesMock();
+ NYql::NFmr::IYtService::TPtr ytService = MakeYtServiceMock(ytUploadedTablesMock);
+ std::shared_ptr<std::atomic<bool>> cancelFlag = std::make_shared<std::atomic<bool>>(false);
+ IFmrJob::TPtr job = MakeFmrJob(tableDataServicePtr, ytService, cancelFlag);
+
+ TFmrTableRef input_1 = TFmrTableRef("test_table_id_1");
+ TYtTableRef input_2 = TYtTableRef("test_path", "test_cluster");
+ TFmrTableRef input_3 = TFmrTableRef("test_table_id_3");
+ TTableRef input_table_ref_1 = {input_1};
+ TTableRef input_table_ref_2 = {input_2};
+ TTableRef input_table_ref_3 = {input_3};
+ TFmrTableRef output = TFmrTableRef("test_table_id_output");
+ std::vector<TTableRef> inputs = {input_table_ref_1, input_table_ref_2, input_table_ref_3};
+ TMergeTaskParams params = TMergeTaskParams(inputs, output);
+
+ tableDataServicePtr->Put(input_1.TableId, TableContent_1);
+ ytUploadedTablesMock->AddTable(input_2, TableContent_2);
+ tableDataServicePtr->Put(input_3.TableId, TableContent_3);
+
+ auto err = job->Merge(params);
+
+ UNIT_ASSERT_C(!err,err.GetRef());
+ UNIT_ASSERT_NO_DIFF(tableDataServicePtr->Get(output.TableId).GetValueSync().GetRef(), TableContent_1 + TableContent_2 + TableContent_3);
+ }
+}
+
+Y_UNIT_TEST_SUITE(TaskRunTests) {
+ Y_UNIT_TEST(RunDownloadTask) {
+ TString ytTableContent =
+ "{\"key\"=\"075\";\"subkey\"=\"1\";\"value\"=\"abc\"};"
+ "{\"key\"=\"800\";\"subkey\"=\"2\";\"value\"=\"ddd\"};"
+ "{\"key\"=\"020\";\"subkey\"=\"3\";\"value\"=\"q\"};"
+ "{\"key\"=\"150\";\"subkey\"=\"4\";\"value\"=\"qzz\"}";
+
+ ITableDataService::TPtr tableDataServicePtr = MakeLocalTableDataService(TLocalTableDataServiceSettings(1));
+ TYtUploadedTablesMock::TPtr ytUploadedTablesMock = MakeYtUploadedTablesMock();
+ NYql::NFmr::IYtService::TPtr ytService = MakeYtServiceMock(ytUploadedTablesMock);
+ std::shared_ptr<std::atomic<bool>> cancelFlag = std::make_shared<std::atomic<bool>>(false);
+
+ TYtTableRef input = TYtTableRef("test_cluster", "test_path");
+ TFmrTableRef output = TFmrTableRef("test_table_id");
+
+ ytUploadedTablesMock->AddTable(input, ytTableContent);
+ TDownloadTaskParams params = TDownloadTaskParams(input, output);
+ TTask::TPtr task = MakeTask(ETaskType::Download, "test_task_id", params, "test_session_id");
+
+
+ ETaskStatus status = RunJob(task, tableDataServicePtr, ytService, cancelFlag);
+
+ UNIT_ASSERT_EQUAL(status, ETaskStatus::Completed);
+ UNIT_ASSERT_NO_DIFF(tableDataServicePtr->Get("test_table_id").GetValueSync().GetRef(), ytTableContent);
+ }
+
+ Y_UNIT_TEST(RunUploadTask) {
+ TString ytTableContent =
+ "{\"key\"=\"075\";\"subkey\"=\"1\";\"value\"=\"abc\"};"
+ "{\"key\"=\"800\";\"subkey\"=\"2\";\"value\"=\"ddd\"};"
+ "{\"key\"=\"020\";\"subkey\"=\"3\";\"value\"=\"q\"};"
+ "{\"key\"=\"150\";\"subkey\"=\"4\";\"value\"=\"qzz\"}";
+
+ ITableDataService::TPtr tableDataServicePtr = MakeLocalTableDataService(TLocalTableDataServiceSettings(1));
+ TYtUploadedTablesMock::TPtr ytUploadedTablesMock = MakeYtUploadedTablesMock();
+ NYql::NFmr::IYtService::TPtr ytService = MakeYtServiceMock(ytUploadedTablesMock);
+ std::shared_ptr<std::atomic<bool>> cancelFlag = std::make_shared<std::atomic<bool>>(false);
+
+ TFmrTableRef input = TFmrTableRef("test_table_id");
+ TYtTableRef output = TYtTableRef("test_cluster", "test_path");
+
+ TUploadTaskParams params = TUploadTaskParams(input, output);
+ TTask::TPtr task = MakeTask(ETaskType::Upload, "test_task_id", params, "test_session_id");
+
+ tableDataServicePtr->Put(input.TableId, ytTableContent);
+
+ ETaskStatus status = RunJob(task, tableDataServicePtr, ytService, cancelFlag);
+
+ UNIT_ASSERT_EQUAL(status, ETaskStatus::Completed);
+ UNIT_ASSERT_NO_DIFF(ytUploadedTablesMock->GetTableContent(output), ytTableContent);
+ }
+
+ Y_UNIT_TEST(RunUploadTaskWithNoTable) {
+ TString ytTableContent =
+ "{\"key\"=\"075\";\"subkey\"=\"1\";\"value\"=\"abc\"};"
+ "{\"key\"=\"800\";\"subkey\"=\"2\";\"value\"=\"ddd\"};"
+ "{\"key\"=\"020\";\"subkey\"=\"3\";\"value\"=\"q\"};"
+ "{\"key\"=\"150\";\"subkey\"=\"4\";\"value\"=\"qzz\"}";
+
+ ITableDataService::TPtr tableDataServicePtr = MakeLocalTableDataService(TLocalTableDataServiceSettings(1));
+ TYtUploadedTablesMock::TPtr ytUploadedTablesMock = MakeYtUploadedTablesMock();
+ NYql::NFmr::IYtService::TPtr ytService = MakeYtServiceMock(ytUploadedTablesMock);
+ std::shared_ptr<std::atomic<bool>> cancelFlag = std::make_shared<std::atomic<bool>>(false);
+
+ TFmrTableRef input = TFmrTableRef("test_table_id");
+ TYtTableRef output = TYtTableRef("test_cluster", "test_path");
+
+ TUploadTaskParams params = TUploadTaskParams(input, output);
+ TTask::TPtr task = MakeTask(ETaskType::Upload, "test_task_id", params, "test_session_id");
+
+ // No table in tableDataServicePtr
+ // tableDataServicePtr->Put(input.TableId, ytTableContent);
+
+ ETaskStatus status = RunJob(task, tableDataServicePtr, ytService, cancelFlag);
+
+ UNIT_ASSERT_EQUAL(status, ETaskStatus::Failed);
+ UNIT_ASSERT(ytUploadedTablesMock->IsEmpty());
+ }
+
+ Y_UNIT_TEST(RunMergeTask) {
+ TString TableContent_1 =
+ "{\"key\"=\"1\";\"subkey\"=\"1\";\"value\"=\"abc\"};"
+ "{\"key\"=\"2\";\"subkey\"=\"2\";\"value\"=\"ddd\"};"
+ "{\"key\"=\"3\";\"subkey\"=\"3\";\"value\"=\"q\"};"
+ "{\"key\"=\"4\";\"subkey\"=\"4\";\"value\"=\"qzz\"};)";
+ TString TableContent_2 =
+ "{\"key\"=\"5\";\"subkey\"=\"1\";\"value\"=\"abc\"};"
+ "{\"key\"=\"6\";\"subkey\"=\"2\";\"value\"=\"ddd\"};"
+ "{\"key\"=\"7\";\"subkey\"=\"3\";\"value\"=\"q\"};"
+ "{\"key\"=\"8\";\"subkey\"=\"4\";\"value\"=\"qzz\"};";
+ TString TableContent_3 =
+ "{\"key\"=\"9\";\"subkey\"=\"1\";\"value\"=\"abc\"};"
+ "{\"key\"=\"10\";\"subkey\"=\"2\";\"value\"=\"ddd\"};"
+ "{\"key\"=\"11\";\"subkey\"=\"3\";\"value\"=\"q\"};"
+ "{\"key\"=\"12\";\"subkey\"=\"4\";\"value\"=\"qzz\"};";
+
+ ITableDataService::TPtr tableDataServicePtr = MakeLocalTableDataService(TLocalTableDataServiceSettings(1));
+ TYtUploadedTablesMock::TPtr ytUploadedTablesMock = MakeYtUploadedTablesMock();
+ NYql::NFmr::IYtService::TPtr ytService = MakeYtServiceMock(ytUploadedTablesMock);
+ std::shared_ptr<std::atomic<bool>> cancelFlag = std::make_shared<std::atomic<bool>>(false);
+
+ TFmrTableRef input_1 = TFmrTableRef("test_table_id_1");
+ TYtTableRef input_2 = TYtTableRef("test_path", "test_cluster");
+ TFmrTableRef input_3 = TFmrTableRef("test_table_id_3");
+ TTableRef input_table_ref_1 = {input_1};
+ TTableRef input_table_ref_2 = {input_2};
+ TTableRef input_table_ref_3 = {input_3};
+ TFmrTableRef output = TFmrTableRef("test_table_id_output");
+ std::vector<TTableRef> inputs = {input_table_ref_1, input_table_ref_2, input_table_ref_3};
+ TMergeTaskParams params = TMergeTaskParams(inputs, output);
+
+ TTask::TPtr task = MakeTask(ETaskType::Upload, "test_task_id", params, "test_session_id");
+
+ tableDataServicePtr->Put(input_1.TableId, TableContent_1);
+ ytUploadedTablesMock->AddTable(input_2, TableContent_2);
+ tableDataServicePtr->Put(input_3.TableId, TableContent_3);
+
+ ETaskStatus status = RunJob(task, tableDataServicePtr, ytService, cancelFlag);
+
+ UNIT_ASSERT_EQUAL(status, ETaskStatus::Completed);
+ UNIT_ASSERT_NO_DIFF(tableDataServicePtr->Get(
+ output.TableId).GetValueSync().GetRef(),
+ TableContent_1 + TableContent_2 + TableContent_3
+ );
+ }
+
+ Y_UNIT_TEST(RunMergeTaskWithNoTable) {
+ TString TableContent_1 =
+ "{\"key\"=\"1\";\"subkey\"=\"1\";\"value\"=\"abc\"};"
+ "{\"key\"=\"2\";\"subkey\"=\"2\";\"value\"=\"ddd\"};"
+ "{\"key\"=\"3\";\"subkey\"=\"3\";\"value\"=\"q\"};"
+ "{\"key\"=\"4\";\"subkey\"=\"4\";\"value\"=\"qzz\"};)";
+ TString TableContent_2 =
+ "{\"key\"=\"5\";\"subkey\"=\"1\";\"value\"=\"abc\"};"
+ "{\"key\"=\"6\";\"subkey\"=\"2\";\"value\"=\"ddd\"};"
+ "{\"key\"=\"7\";\"subkey\"=\"3\";\"value\"=\"q\"};"
+ "{\"key\"=\"8\";\"subkey\"=\"4\";\"value\"=\"qzz\"};";
+ TString TableContent_3 =
+ "{\"key\"=\"9\";\"subkey\"=\"1\";\"value\"=\"abc\"};"
+ "{\"key\"=\"10\";\"subkey\"=\"2\";\"value\"=\"ddd\"};"
+ "{\"key\"=\"11\";\"subkey\"=\"3\";\"value\"=\"q\"};"
+ "{\"key\"=\"12\";\"subkey\"=\"4\";\"value\"=\"qzz\"};";
+
+ ITableDataService::TPtr tableDataServicePtr = MakeLocalTableDataService(TLocalTableDataServiceSettings(1));
+ TYtUploadedTablesMock::TPtr ytUploadedTablesMock = MakeYtUploadedTablesMock();
+ NYql::NFmr::IYtService::TPtr ytService = MakeYtServiceMock(ytUploadedTablesMock);
+ std::shared_ptr<std::atomic<bool>> cancelFlag = std::make_shared<std::atomic<bool>>(false);
+
+ TFmrTableRef input_1 = TFmrTableRef("test_table_id_1");
+ TYtTableRef input_2 = TYtTableRef("test_path", "test_cluster");
+ TFmrTableRef input_3 = TFmrTableRef("test_table_id_3");
+ TTableRef input_table_ref_1 = {input_1};
+ TTableRef input_table_ref_2 = {input_2};
+ TTableRef input_table_ref_3 = {input_3};
+ TFmrTableRef output = TFmrTableRef("test_table_id_output");
+ std::vector<TTableRef> inputs = {input_table_ref_1, input_table_ref_2, input_table_ref_3};
+ TMergeTaskParams params = TMergeTaskParams(inputs, output);
+
+ TTask::TPtr task = MakeTask(ETaskType::Upload, "test_task_id", params, "test_session_id");
+
+ tableDataServicePtr->Put(input_1.TableId, TableContent_1);
+ // No table in Yt
+ // ytUploadedTablesMock->AddTable(input_2, TableContent_2);
+ tableDataServicePtr->Put(input_3.TableId, TableContent_3);
+
+ ETaskStatus status = RunJob(task, tableDataServicePtr, ytService, cancelFlag);
+ UNIT_ASSERT_EQUAL(status, ETaskStatus::Failed);
+ UNIT_ASSERT(!tableDataServicePtr->Get(output.TableId).GetValueSync());
+ }
+}
+
+} // namespace NYql
diff --git a/yt/yql/providers/yt/fmr/job/impl/ya.make b/yt/yql/providers/yt/fmr/job/impl/ya.make
new file mode 100644
index 0000000000..4a647bd58a
--- /dev/null
+++ b/yt/yql/providers/yt/fmr/job/impl/ya.make
@@ -0,0 +1,19 @@
+LIBRARY()
+
+SRCS(
+ yql_yt_job_impl.cpp
+)
+
+PEERDIR(
+ library/cpp/threading/future
+ yt/yql/providers/yt/fmr/job/interface
+ yt/yql/providers/yt/fmr/request_options
+)
+
+YQL_LAST_ABI_VERSION()
+
+END()
+
+RECURSE_FOR_TESTS(
+ ut
+)
diff --git a/yt/yql/providers/yt/fmr/job/impl/yql_yt_job_impl.cpp b/yt/yql/providers/yt/fmr/job/impl/yql_yt_job_impl.cpp
new file mode 100644
index 0000000000..c672d6d6b9
--- /dev/null
+++ b/yt/yql/providers/yt/fmr/job/impl/yql_yt_job_impl.cpp
@@ -0,0 +1,166 @@
+#include <library/cpp/threading/future/core/future.h>
+
+#include <util/stream/file.h>
+
+#include <yt/yql/providers/yt/fmr/job/impl/yql_yt_job_impl.h>
+#include <yt/yql/providers/yt/fmr/request_options/yql_yt_request_options.h>
+#include <yt/yql/providers/yt/fmr/table_data_service/interface/table_data_service.h>
+#include <yt/yql/providers/yt/fmr/yt_service/interface/yql_yt_yt_service.h>
+
+#include <yql/essentials/utils/log/log.h>
+
+namespace NYql::NFmr {
+
+class TFmrJob: public IFmrJob {
+public:
+
+ TFmrJob(ITableDataService::TPtr tableDataService, IYtService::TPtr ytService, std::shared_ptr<std::atomic<bool>> cancelFlag)
+ : TableDataService_(tableDataService), YtService_(ytService), CancelFlag_(cancelFlag)
+ {
+ }
+
+ virtual TMaybe<TString> Download(const TDownloadTaskParams& params, const TClusterConnection& clusterConnection) override { // вынести в приватный метод для переиспользования
+ try {
+ const auto ytTable = params.Input;
+ const auto cluster = params.Input.Cluster;
+ const auto path = params.Input.Path;
+ const auto tableId = params.Output.TableId;
+
+ YQL_CLOG(DEBUG, FastMapReduce) << "Downloading " << cluster << '.' << path;
+
+ auto res = GetYtTableContent(ytTable, clusterConnection);
+ auto err = std::get_if<TError>(&res);
+ if (err) {
+ return err->ErrorMessage;
+ }
+ auto tableContent = std::get<TString>(res);
+ TableDataService_->Put(tableId, tableContent).Wait();
+ } catch (...) {
+ return CurrentExceptionMessage();
+ }
+
+ return Nothing();
+ }
+
+ virtual TMaybe<TString> Upload(const TUploadTaskParams& params, const TClusterConnection& clusterConnection) override {
+ const auto ytTable = params.Output;
+ const auto cluster = params.Output.Cluster;
+ const auto path = params.Output.Path;
+ const auto tableId = params.Input.TableId;
+
+ YQL_CLOG(DEBUG, FastMapReduce) << "Uploading " << cluster << '.' << path;
+
+ TMaybe<TString> getResult = TableDataService_->Get(tableId).GetValueSync();
+
+ if (!getResult) {
+ YQL_CLOG(ERROR, FastMapReduce) << "Table " << tableId << " not found";
+ return "Table not found";
+ }
+
+ TString tableContent = getResult.GetRef();
+ TStringInput inputStream(tableContent);
+
+ YtService_->Upload(ytTable, inputStream, clusterConnection);
+
+ return Nothing();
+ }
+
+ virtual TMaybe<TString> Merge(const TMergeTaskParams& params, const TClusterConnection& clusterConnection) override {
+ const auto inputs = params.Input;
+ const auto output = params.Output;
+
+ YQL_CLOG(DEBUG, FastMapReduce) << "Merging " << inputs.size() << " inputs";
+
+ TString mergedTableContent = "";
+
+ for (const auto& inputTableRef : inputs) {
+ if (CancelFlag_->load()) {
+ return "Canceled";
+ }
+ auto res = GetTableContent(inputTableRef, clusterConnection);
+
+ auto err = std::get_if<TError>(&res);
+ if (err) {
+ return err->ErrorMessage;
+ }
+ TString tableContent = std::get<TString>(res);
+
+ mergedTableContent += tableContent;
+ }
+
+ TableDataService_->Put(output.TableId, mergedTableContent).Wait();
+
+ return Nothing();
+ }
+private:
+ std::variant<TString, TError> GetTableContent(const TTableRef& tableRef, const TClusterConnection& clusterConnection) {
+ auto ytTable = std::get_if<TYtTableRef>(&tableRef);
+ auto fmrTable = std::get_if<TFmrTableRef>(&tableRef);
+ if (ytTable) {
+ return GetYtTableContent(*ytTable, clusterConnection);
+ } else if (fmrTable) {
+ return GetFmrTableContent(*fmrTable);
+ } else {
+ ythrow yexception() << "Unsupported table type";
+ }
+ }
+
+ std::variant<TString, TError> GetYtTableContent(const TYtTableRef& ytTable, const TClusterConnection& clusterConnection) {
+ auto res = YtService_->Download(ytTable, clusterConnection);
+ auto* err = std::get_if<TError>(&res);
+ if (err) {
+ return *err;
+ }
+ auto tableFile = std::get_if<THolder<TTempFileHandle>>(&res);
+ TFileInput inputStream(tableFile->Get()->Name());
+ TString tableContent = inputStream.ReadAll();
+ return tableContent;
+ }
+
+ std::variant<TString, TError> GetFmrTableContent(const TFmrTableRef& fmrTable) {
+ auto res = TableDataService_->Get(fmrTable.TableId);
+ return res.GetValueSync().GetRef();
+ }
+private:
+ ITableDataService::TPtr TableDataService_;
+ IYtService::TPtr YtService_;
+ std::shared_ptr<std::atomic<bool>> CancelFlag_;
+};
+
+IFmrJob::TPtr MakeFmrJob(ITableDataService::TPtr tableDataService, IYtService::TPtr ytService, std::shared_ptr<std::atomic<bool>> cancelFlag) {
+ return MakeIntrusive<TFmrJob>(tableDataService, ytService, cancelFlag);
+}
+
+ETaskStatus RunJob(
+ TTask::TPtr task,
+ ITableDataService::TPtr tableDataService,
+ IYtService::TPtr ytService,
+ std::shared_ptr<std::atomic<bool>> cancelFlag
+) {
+ IFmrJob::TPtr job = MakeFmrJob(tableDataService, ytService, cancelFlag);
+
+ auto processTask = [job, task] (auto&& taskParams) {
+ using T = std::decay_t<decltype(taskParams)>;
+
+ if constexpr (std::is_same_v<T, TUploadTaskParams>) {
+ return job->Upload(taskParams, task->ClusterConnection);
+ } else if constexpr (std::is_same_v<T, TDownloadTaskParams>) {
+ return job->Download(taskParams, task->ClusterConnection);
+ } else if constexpr (std::is_same_v<T, TMergeTaskParams>) {
+ return job->Merge(taskParams, task->ClusterConnection);
+ } else {
+ throw std::runtime_error{"Unsupported task type"};
+ }
+ };
+
+ TMaybe<TString> taskResult = std::visit(processTask, task->TaskParams);
+
+ if (taskResult.Defined()) {
+ YQL_CLOG(ERROR, FastMapReduce) << "Task failed: " << taskResult.GetRef();
+ return ETaskStatus::Failed;
+ }
+
+ return ETaskStatus::Completed;
+};
+
+} // namespace NYql
diff --git a/yt/yql/providers/yt/fmr/job/impl/yql_yt_job_impl.h b/yt/yql/providers/yt/fmr/job/impl/yql_yt_job_impl.h
new file mode 100644
index 0000000000..28280ff0ce
--- /dev/null
+++ b/yt/yql/providers/yt/fmr/job/impl/yql_yt_job_impl.h
@@ -0,0 +1,12 @@
+#include <yt/cpp/mapreduce/interface/fwd.h>
+#include <yt/yql/providers/yt/fmr/job/interface/yql_yt_job.h>
+#include <yt/yql/providers/yt/fmr/table_data_service/interface/table_data_service.h>
+#include <yt/yql/providers/yt/fmr/yt_service/interface/yql_yt_yt_service.h>
+
+namespace NYql::NFmr {
+
+IFmrJob::TPtr MakeFmrJob(ITableDataService::TPtr tableDataService, IYtService::TPtr ytService, std::shared_ptr<std::atomic<bool>> cancelFlag);
+
+ETaskStatus RunJob(TTask::TPtr task, ITableDataService::TPtr tableDataService, IYtService::TPtr ytService, std::shared_ptr<std::atomic<bool>> cancelFlag);
+
+} // namespace NYql
diff --git a/yt/yql/providers/yt/fmr/job/interface/ya.make b/yt/yql/providers/yt/fmr/job/interface/ya.make
new file mode 100644
index 0000000000..7d256622f7
--- /dev/null
+++ b/yt/yql/providers/yt/fmr/job/interface/ya.make
@@ -0,0 +1,9 @@
+LIBRARY()
+
+SRCS(
+ yql_yt_job.cpp
+)
+
+YQL_LAST_ABI_VERSION()
+
+END()
diff --git a/yt/yql/providers/yt/fmr/job/interface/yql_yt_job.cpp b/yt/yql/providers/yt/fmr/job/interface/yql_yt_job.cpp
new file mode 100644
index 0000000000..a80a31a1a9
--- /dev/null
+++ b/yt/yql/providers/yt/fmr/job/interface/yql_yt_job.cpp
@@ -0,0 +1 @@
+#include "yql_yt_job.h"
diff --git a/yt/yql/providers/yt/fmr/job/interface/yql_yt_job.h b/yt/yql/providers/yt/fmr/job/interface/yql_yt_job.h
new file mode 100644
index 0000000000..187ed21d3c
--- /dev/null
+++ b/yt/yql/providers/yt/fmr/job/interface/yql_yt_job.h
@@ -0,0 +1,20 @@
+#pragma once
+
+#include <yt/yql/providers/yt/fmr/request_options/yql_yt_request_options.h>
+
+namespace NYql::NFmr {
+
+class IFmrJob: public TThrRefBase {
+public:
+ using TPtr = TIntrusivePtr<IFmrJob>;
+
+ virtual ~IFmrJob() = default;
+
+ virtual TMaybe<TString> Download(const TDownloadTaskParams& params, const TClusterConnection& clusterConnection = TClusterConnection()) = 0;
+
+ virtual TMaybe<TString> Upload(const TUploadTaskParams& params, const TClusterConnection& clusterConnection = TClusterConnection()) = 0;
+
+ virtual TMaybe<TString> Merge(const TMergeTaskParams& params, const TClusterConnection& clusterConnection = TClusterConnection()) = 0;
+};
+
+} // namespace NYql
diff --git a/yt/yql/providers/yt/fmr/proto/coordinator.proto b/yt/yql/providers/yt/fmr/proto/coordinator.proto
index 33990caa28..c76bb50fc3 100644
--- a/yt/yql/providers/yt/fmr/proto/coordinator.proto
+++ b/yt/yql/providers/yt/fmr/proto/coordinator.proto
@@ -22,6 +22,7 @@ message TStartOperationRequest {
string SessionId = 3;
optional string IdempotencyKey = 4;
uint32 NumRetries = 5;
+ TClusterConnection ClusterConnection = 6;
}
message TStartOperationResponse {
diff --git a/yt/yql/providers/yt/fmr/proto/request_options.proto b/yt/yql/providers/yt/fmr/proto/request_options.proto
index e37583a0d2..7cbd1eecf4 100644
--- a/yt/yql/providers/yt/fmr/proto/request_options.proto
+++ b/yt/yql/providers/yt/fmr/proto/request_options.proto
@@ -48,7 +48,6 @@ message TStatistics {}
message TYtTableRef {
string Path = 1;
string Cluster = 2;
- string TransactionId = 3;
}
message TFmrTableRef {
@@ -85,12 +84,19 @@ message TTaskParams {
}
}
+message TClusterConnection {
+ string TransactionId = 1;
+ string YtServerName = 2;
+ optional string Token = 3;
+}
+
message TTask {
ETaskType TaskType = 1;
string TaskId = 2;
TTaskParams TaskParams = 3;
string SessionId = 4;
optional uint32 NumRetries = 5;
+ TClusterConnection ClusterConnection = 6;
}
message TTaskState {
diff --git a/yt/yql/providers/yt/fmr/request_options/proto_helpers/yql_yt_request_proto_helpers.cpp b/yt/yql/providers/yt/fmr/request_options/proto_helpers/yql_yt_request_proto_helpers.cpp
index 52cc0f3ad6..7a931909ac 100644
--- a/yt/yql/providers/yt/fmr/request_options/proto_helpers/yql_yt_request_proto_helpers.cpp
+++ b/yt/yql/providers/yt/fmr/request_options/proto_helpers/yql_yt_request_proto_helpers.cpp
@@ -38,7 +38,6 @@ NProto::TYtTableRef YtTableRefToProto(const TYtTableRef& ytTableRef) {
NProto::TYtTableRef protoYtTableRef;
protoYtTableRef.SetPath(ytTableRef.Path);
protoYtTableRef.SetCluster(ytTableRef.Cluster);
- protoYtTableRef.SetTransactionId(ytTableRef.TransactionId);
return protoYtTableRef;
}
@@ -46,7 +45,6 @@ TYtTableRef YtTableRefFromProto(const NProto::TYtTableRef protoYtTableRef) {
TYtTableRef ytTableRef;
ytTableRef.Path = protoYtTableRef.GetPath();
ytTableRef.Cluster = protoYtTableRef.GetCluster();
- ytTableRef.TransactionId = protoYtTableRef.GetTransactionId();
return ytTableRef;
}
@@ -170,6 +168,26 @@ TTaskParams TaskParamsFromProto(const NProto::TTaskParams& protoTaskParams) {
return taskParams;
}
+NProto::TClusterConnection ClusterConnectionToProto(const TClusterConnection& clusterConnection) {
+ NProto::TClusterConnection protoClusterConnection;
+ protoClusterConnection.SetTransactionId(clusterConnection.TransactionId);
+ protoClusterConnection.SetYtServerName(clusterConnection.YtServerName);
+ if (clusterConnection.Token) {
+ protoClusterConnection.SetToken(*clusterConnection.Token);
+ }
+ return protoClusterConnection;
+}
+
+TClusterConnection ClusterConnectionFromProto(const NProto::TClusterConnection& protoClusterConnection) {
+ TClusterConnection clusterConnection{};
+ clusterConnection.TransactionId = protoClusterConnection.GetTransactionId();
+ clusterConnection.YtServerName = protoClusterConnection.GetYtServerName();
+ if (protoClusterConnection.HasToken()) {
+ clusterConnection.Token = protoClusterConnection.GetToken();
+ }
+ return clusterConnection;
+}
+
NProto::TTask TaskToProto(const TTask& task) {
NProto::TTask protoTask;
protoTask.SetTaskType(static_cast<NProto::ETaskType>(task.TaskType));
@@ -178,6 +196,8 @@ NProto::TTask TaskToProto(const TTask& task) {
protoTask.MutableTaskParams()->Swap(&taskParams);
protoTask.SetSessionId(task.SessionId);
protoTask.SetNumRetries(task.NumRetries);
+ auto clusterConnection = ClusterConnectionToProto(task.ClusterConnection);
+ protoTask.MutableClusterConnection()->Swap(&clusterConnection);
return protoTask;
}
@@ -188,6 +208,7 @@ TTask TaskFromProto(const NProto::TTask& protoTask) {
task.TaskParams = TaskParamsFromProto(protoTask.GetTaskParams());
task.SessionId = protoTask.GetSessionId();
task.NumRetries = protoTask.GetNumRetries();
+ task.ClusterConnection = ClusterConnectionFromProto(protoTask.GetClusterConnection());
return task;
}
diff --git a/yt/yql/providers/yt/fmr/request_options/proto_helpers/yql_yt_request_proto_helpers.h b/yt/yql/providers/yt/fmr/request_options/proto_helpers/yql_yt_request_proto_helpers.h
index e886e91901..5108bbb0f3 100644
--- a/yt/yql/providers/yt/fmr/request_options/proto_helpers/yql_yt_request_proto_helpers.h
+++ b/yt/yql/providers/yt/fmr/request_options/proto_helpers/yql_yt_request_proto_helpers.h
@@ -37,6 +37,10 @@ NProto::TTaskParams TaskParamsToProto(const TTaskParams& taskParams);
TTaskParams TaskParamsFromProto(const NProto::TTaskParams& protoTaskParams);
+NProto::TClusterConnection ClusterConnectionToProto(const TClusterConnection& clusterConnection);
+
+TClusterConnection ClusterConnectionFromProto(const NProto::TClusterConnection& protoClusterConnection);
+
NProto::TTask TaskToProto(const TTask& task);
TTask TaskFromProto(const NProto::TTask& protoTask);
diff --git a/yt/yql/providers/yt/fmr/request_options/yql_yt_request_options.cpp b/yt/yql/providers/yt/fmr/request_options/yql_yt_request_options.cpp
index c73791bee4..f6c31167c3 100644
--- a/yt/yql/providers/yt/fmr/request_options/yql_yt_request_options.cpp
+++ b/yt/yql/providers/yt/fmr/request_options/yql_yt_request_options.cpp
@@ -2,8 +2,8 @@
namespace NYql::NFmr {
-TTask::TPtr MakeTask(ETaskType taskType, const TString& taskId, const TTaskParams& taskParams, const TString& sessionId) {
- return MakeIntrusive<TTask>(taskType, taskId, taskParams, sessionId);
+TTask::TPtr MakeTask(ETaskType taskType, const TString& taskId, const TTaskParams& taskParams, const TString& sessionId, const TClusterConnection& clusterConnection) {
+ return MakeIntrusive<TTask>(taskType, taskId, taskParams, sessionId, clusterConnection);
}
TTaskState::TPtr MakeTaskState(ETaskStatus taskStatus, const TString& taskId, const TMaybe<TFmrError>& taskErrorMessage) {
diff --git a/yt/yql/providers/yt/fmr/request_options/yql_yt_request_options.h b/yt/yql/providers/yt/fmr/request_options/yql_yt_request_options.h
index b423021a71..de7c4eb435 100644
--- a/yt/yql/providers/yt/fmr/request_options/yql_yt_request_options.h
+++ b/yt/yql/providers/yt/fmr/request_options/yql_yt_request_options.h
@@ -54,7 +54,6 @@ struct TError {
struct TYtTableRef {
TString Path;
TString Cluster;
- TString TransactionId;
};
struct TFmrTableRef {
@@ -160,11 +159,19 @@ using TTaskParams = std::variant<TUploadTaskParams, TDownloadTaskParams, TMergeT
using TTaskParamsNew = std::variant<TUploadTaskParamsNew, TDownloadTaskParamsNew, TMergeTaskParamsNew>;
+struct TClusterConnection {
+ TString TransactionId;
+ TString YtServerName;
+ TMaybe<TString> Token;
+};
+
+using TTaskParams = std::variant<TUploadTaskParams, TDownloadTaskParams, TMergeTaskParams>;
+
struct TTask: public TThrRefBase {
TTask() = default;
- TTask(ETaskType taskType, const TString& taskId, const TTaskParams& taskParams, const TString& sessionId, ui32 numRetries = 1)
- : TaskType(taskType), TaskId(taskId), TaskParams(taskParams), SessionId(sessionId), NumRetries(numRetries)
+ TTask(ETaskType taskType, const TString& taskId, const TTaskParams& taskParams, const TString& sessionId, const TClusterConnection& clusterConnection, ui32 numRetries = 1)
+ : TaskType(taskType), TaskId(taskId), TaskParams(taskParams), SessionId(sessionId), ClusterConnection(clusterConnection), NumRetries(numRetries)
{
}
@@ -172,6 +179,7 @@ struct TTask: public TThrRefBase {
TString TaskId;
TTaskParams TaskParams = {};
TString SessionId;
+ TClusterConnection ClusterConnection = {};
ui32 NumRetries; // Not supported yet
using TPtr = TIntrusivePtr<TTask>;
@@ -205,7 +213,7 @@ struct TTaskResult: public TThrRefBase {
using TPtr = TIntrusivePtr<TTaskResult>;
};
-TTask::TPtr MakeTask(ETaskType taskType, const TString& taskId, const TTaskParams& taskParams, const TString& sessionId);
+TTask::TPtr MakeTask(ETaskType taskType, const TString& taskId, const TTaskParams& taskParams, const TString& sessionId, const TClusterConnection& clusterConnection = TClusterConnection{});
TTaskState::TPtr MakeTaskState(ETaskStatus taskStatus, const TString& taskId, const TMaybe<TFmrError>& taskErrorMessage = Nothing());
diff --git a/yt/yql/providers/yt/fmr/table_data_service/interface/table_data_service.cpp b/yt/yql/providers/yt/fmr/table_data_service/interface/table_data_service.cpp
new file mode 100644
index 0000000000..7f3e3c5135
--- /dev/null
+++ b/yt/yql/providers/yt/fmr/table_data_service/interface/table_data_service.cpp
@@ -0,0 +1 @@
+#include "table_data_service.h"
diff --git a/yt/yql/providers/yt/fmr/table_data_service/interface/table_data_service.h b/yt/yql/providers/yt/fmr/table_data_service/interface/table_data_service.h
new file mode 100644
index 0000000000..98a2ec9ddc
--- /dev/null
+++ b/yt/yql/providers/yt/fmr/table_data_service/interface/table_data_service.h
@@ -0,0 +1,20 @@
+#pragma once
+
+#include <library/cpp/threading/future/core/future.h>
+
+namespace NYql::NFmr {
+
+class ITableDataService: public TThrRefBase {
+public:
+ using TPtr = TIntrusivePtr<ITableDataService>;
+
+ virtual ~ITableDataService() = default;
+
+ virtual NThreading::TFuture<void> Put(const TString& id, const TString& data) = 0;
+
+ virtual NThreading::TFuture<TMaybe<TString>> Get(const TString& id) = 0;
+
+ virtual NThreading::TFuture<void> Delete(const TString& id) = 0;
+};
+
+} // namespace NYql::NFmr
diff --git a/yt/yql/providers/yt/fmr/table_data_service/interface/ya.make b/yt/yql/providers/yt/fmr/table_data_service/interface/ya.make
new file mode 100644
index 0000000000..63abe6d78f
--- /dev/null
+++ b/yt/yql/providers/yt/fmr/table_data_service/interface/ya.make
@@ -0,0 +1,13 @@
+LIBRARY()
+
+SRCS(
+ table_data_service.cpp
+)
+
+PEERDIR(
+ library/cpp/threading/future
+)
+
+YQL_LAST_ABI_VERSION()
+
+END()
diff --git a/yt/yql/providers/yt/fmr/table_data_service/local/table_data_service.cpp b/yt/yql/providers/yt/fmr/table_data_service/local/table_data_service.cpp
new file mode 100644
index 0000000000..445263ff5e
--- /dev/null
+++ b/yt/yql/providers/yt/fmr/table_data_service/local/table_data_service.cpp
@@ -0,0 +1,55 @@
+#include "table_data_service.h"
+
+namespace NYql::NFmr {
+
+namespace {
+
+class TLocalTableDataService: public ITableDataService {
+public:
+ TLocalTableDataService(const TLocalTableDataServiceSettings& settings): NumParts_(settings.NumParts) {
+ Data_.resize(NumParts_);
+ }
+
+ NThreading::TFuture<void> Put(const TString& key, const TString& value) {
+ auto& map = Data_[std::hash<TString>()(key) % NumParts_];
+ auto it = map.find(key);
+ if (it != map.end()) {
+ return NThreading::MakeFuture();
+ }
+ map.insert({key, value});
+ return NThreading::MakeFuture();
+ }
+
+ NThreading::TFuture<TMaybe<TString>> Get(const TString& key) {
+ TMaybe<TString> value = Nothing();
+ auto& map = Data_[std::hash<TString>()(key) % NumParts_];
+ auto it = map.find(key);
+ if (it != map.end()) {
+ value = it->second;
+ }
+ return NThreading::MakeFuture(value);
+ }
+
+ NThreading::TFuture<void> Delete(const TString& key) {
+ auto& map = Data_[std::hash<TString>()(key) % NumParts_];
+ auto it = map.find(key);
+ if (it == map.end()) {
+ return NThreading::MakeFuture();
+ }
+ map.erase(key);
+ return NThreading::MakeFuture();
+ }
+
+private:
+ std::vector<std::unordered_map<TString, TString>> Data_;
+ const ui32 NumParts_;
+};
+
+} // namespace
+
+ITableDataService::TPtr MakeLocalTableDataService(const TLocalTableDataServiceSettings& settings) {
+ return MakeIntrusive<TLocalTableDataService>(settings);
+}
+
+
+} // namespace NYql::NFmr
diff --git a/yt/yql/providers/yt/fmr/table_data_service/local/table_data_service.h b/yt/yql/providers/yt/fmr/table_data_service/local/table_data_service.h
new file mode 100644
index 0000000000..eb4d0cb2f6
--- /dev/null
+++ b/yt/yql/providers/yt/fmr/table_data_service/local/table_data_service.h
@@ -0,0 +1,12 @@
+#include <library/cpp/threading/future/future.h>
+#include <yt/yql/providers/yt/fmr/table_data_service/interface/table_data_service.h>
+
+namespace NYql::NFmr {
+
+struct TLocalTableDataServiceSettings {
+ ui32 NumParts;
+};
+
+ITableDataService::TPtr MakeLocalTableDataService(const TLocalTableDataServiceSettings& settings);
+
+} // namespace NYql::NFmr
diff --git a/yt/yql/providers/yt/fmr/table_data_service/local/ut/test_table_service.cpp b/yt/yql/providers/yt/fmr/table_data_service/local/ut/test_table_service.cpp
new file mode 100644
index 0000000000..a48df5f3df
--- /dev/null
+++ b/yt/yql/providers/yt/fmr/table_data_service/local/ut/test_table_service.cpp
@@ -0,0 +1,38 @@
+#include <library/cpp/testing/unittest/registar.h>
+#include <yt/yql/providers/yt/fmr/table_data_service/local/table_data_service.h>
+
+using namespace NYql::NFmr;
+
+Y_UNIT_TEST_SUITE(TLocalTableServiceTest)
+{
+ Y_UNIT_TEST(GetNonexistentKey) {
+ ITableDataService::TPtr tableDataService = MakeLocalTableDataService({3});
+ auto getFuture = tableDataService->Get("key");
+ UNIT_ASSERT_VALUES_EQUAL(getFuture.GetValueSync(), Nothing());
+ }
+ Y_UNIT_TEST(GetExistingKey) {
+ ITableDataService::TPtr tableDataService = MakeLocalTableDataService({3});
+ auto putFuture = tableDataService->Put("key", "1");
+ putFuture.GetValueSync();
+ auto getFuture = tableDataService->Get("key");
+ UNIT_ASSERT_VALUES_EQUAL(getFuture.GetValueSync(), "1");
+ }
+ Y_UNIT_TEST(DeleteNonexistentKey) {
+ ITableDataService::TPtr tableDataService = MakeLocalTableDataService({3});
+ auto putFuture = tableDataService->Put("key", "1");
+ putFuture.GetValueSync();
+ auto deleteFuture = tableDataService->Delete("other_key");
+ deleteFuture.GetValueSync();
+ auto getFuture = tableDataService->Get("key");
+ UNIT_ASSERT_VALUES_EQUAL(getFuture.GetValueSync(), "1");
+ }
+ Y_UNIT_TEST(DeleteExistingKey) {
+ ITableDataService::TPtr tableDataService = MakeLocalTableDataService({3});
+ auto putFuture = tableDataService->Put("key", "1");
+ putFuture.GetValueSync();
+ auto deleteFuture = tableDataService->Delete("key");
+ deleteFuture.GetValueSync();
+ auto getFuture = tableDataService->Get("key");
+ UNIT_ASSERT_VALUES_EQUAL(getFuture.GetValueSync(), Nothing());
+ }
+}
diff --git a/yt/yql/providers/yt/fmr/table_data_service/local/ut/ya.make b/yt/yql/providers/yt/fmr/table_data_service/local/ut/ya.make
new file mode 100644
index 0000000000..9b57c059db
--- /dev/null
+++ b/yt/yql/providers/yt/fmr/table_data_service/local/ut/ya.make
@@ -0,0 +1,14 @@
+UNITTEST()
+
+SRCS(
+ test_table_service.cpp
+)
+
+PEERDIR(
+ yt/yql/providers/yt/fmr/table_data_service/local
+)
+
+
+YQL_LAST_ABI_VERSION()
+
+END()
diff --git a/yt/yql/providers/yt/fmr/table_data_service/local/ya.make b/yt/yql/providers/yt/fmr/table_data_service/local/ya.make
new file mode 100644
index 0000000000..78290fcd2e
--- /dev/null
+++ b/yt/yql/providers/yt/fmr/table_data_service/local/ya.make
@@ -0,0 +1,18 @@
+LIBRARY()
+
+SRCS(
+ table_data_service.cpp
+)
+
+PEERDIR(
+ library/cpp/threading/future
+ yt/yql/providers/yt/fmr/table_data_service/interface
+)
+
+YQL_LAST_ABI_VERSION()
+
+END()
+
+RECURSE_FOR_TESTS(
+ ut
+)
diff --git a/yt/yql/providers/yt/fmr/worker/impl/ut/yql_yt_worker_ut.cpp b/yt/yql/providers/yt/fmr/worker/impl/ut/yql_yt_worker_ut.cpp
index 4996c932c4..e16e20c911 100644
--- a/yt/yql/providers/yt/fmr/worker/impl/ut/yql_yt_worker_ut.cpp
+++ b/yt/yql/providers/yt/fmr/worker/impl/ut/yql_yt_worker_ut.cpp
@@ -11,12 +11,17 @@
namespace NYql::NFmr {
TDownloadTaskParams downloadTaskParams{
- .Input = TYtTableRef{"Path","Cluster","TransactionId"},
+ .Input = TYtTableRef{"Path","Cluster"},
.Output = TFmrTableRef{"TableId"}
};
TStartOperationRequest CreateOperationRequest(ETaskType taskType = ETaskType::Download, TTaskParams taskParams = downloadTaskParams) {
- return TStartOperationRequest{.TaskType = taskType, .TaskParams = taskParams, .IdempotencyKey = "IdempotencyKey"};
+ return TStartOperationRequest{
+ .TaskType = taskType,
+ .TaskParams = taskParams,
+ .IdempotencyKey = "IdempotencyKey",
+ .ClusterConnection = TClusterConnection{.TransactionId = "transaction_id", .YtServerName = "hahn.yt.yandex.net", .Token = "token"}
+ };
}
Y_UNIT_TEST_SUITE(FmrWorkerTests) {
diff --git a/yt/yql/providers/yt/fmr/yt_service/impl/ya.make b/yt/yql/providers/yt/fmr/yt_service/impl/ya.make
new file mode 100644
index 0000000000..2da71e6b7d
--- /dev/null
+++ b/yt/yql/providers/yt/fmr/yt_service/impl/ya.make
@@ -0,0 +1,16 @@
+LIBRARY()
+
+SRCS(
+ yql_yt_yt_service_impl.cpp
+)
+
+PEERDIR(
+ library/cpp/yson
+ library/cpp/yson/node
+ yt/cpp/mapreduce/client
+ yt/yql/providers/yt/fmr/yt_service/interface
+)
+
+YQL_LAST_ABI_VERSION()
+
+END()
diff --git a/yt/yql/providers/yt/fmr/yt_service/impl/yql_yt_yt_service_impl.cpp b/yt/yql/providers/yt/fmr/yt_service/impl/yql_yt_yt_service_impl.cpp
new file mode 100644
index 0000000000..1da2ff6bd3
--- /dev/null
+++ b/yt/yql/providers/yt/fmr/yt_service/impl/yql_yt_yt_service_impl.cpp
@@ -0,0 +1,186 @@
+#include <library/cpp/yson/node/node_io.h>
+#include <library/cpp/yson/node/node_visitor.h>
+#include <library/cpp/yson/parser.h>
+#include <library/cpp/yson/writer.h>
+#include <yt/cpp/mapreduce/interface/client.h>
+
+#include "yql_yt_yt_service_impl.h"
+
+namespace NYql::NFmr {
+
+namespace {
+
+class TYtFmrRowConsumer: public NYson::TYsonConsumerBase {
+public:
+ TYtFmrRowConsumer(NYT::TTableWriterPtr<NYT::TNode> writer): Writer_(writer)
+ {
+ CurNode_ = NYT::TNode();
+ Builder_ = MakeHolder<NYT::TNodeBuilder>(&CurNode_);
+ }
+
+ void OnStringScalar(TStringBuf value) override {
+ Builder_->OnStringScalar(value);
+ }
+
+ void OnInt64Scalar(i64 value) override {
+ Builder_->OnInt64Scalar(value);
+ }
+
+ void OnUint64Scalar(ui64 value) override {
+ Builder_->OnUint64Scalar(value);
+ }
+
+ void OnDoubleScalar(double value) override {
+ Builder_->OnDoubleScalar(value);
+ }
+
+ void OnBooleanScalar(bool value) override {
+ Builder_->OnBooleanScalar(value);
+ }
+
+ void OnEntity() override {
+ Builder_->OnEntity();
+ }
+
+ void OnBeginList() override {
+ ++Level_;
+ Builder_->OnBeginList();
+ }
+
+ void OnBeginList(ui64 reserveSize) {
+ Builder_->OnBeginList(reserveSize);
+ }
+
+ void OnListItem() override {
+ if (Level_ == 0) {
+ Flush();
+ } else {
+ Builder_->OnListItem();
+ }
+ }
+
+ void OnEndList() override {
+ --Level_;
+ Builder_->OnEndList();
+ }
+
+ void OnBeginMap() override {
+ ++Level_;
+ Builder_->OnBeginMap();
+ }
+
+ void OnBeginMap(ui64 reserveSize) {
+ Builder_->OnBeginMap(reserveSize);
+ }
+
+ void OnKeyedItem(TStringBuf key) override {
+ Builder_->OnKeyedItem(key);
+ }
+
+ void OnEndMap() override {
+ --Level_;
+ Builder_->OnEndMap();
+ }
+
+ void OnBeginAttributes() override {
+ ++Level_;
+ Builder_->OnBeginAttributes();
+ }
+
+ void OnEndAttributes() override {
+ --Level_;
+ Builder_->OnEndAttributes();
+ }
+
+ void OnNode(NYT::TNode node) {
+ Builder_->OnNode(node);
+ }
+
+ ~TYtFmrRowConsumer() {
+ Y_ABORT_UNLESS(!CurNode_.HasValue());
+ }
+
+ void Flush() {
+ if (CurNode_.HasValue()) {
+ Writer_->AddRow(CurNode_);
+ }
+ CurNode_ = NYT::TNode();
+ Builder_ = MakeHolder<NYT::TNodeBuilder>(&CurNode_);
+ }
+private:
+ NYT::TNode CurNode_;
+ NYT::TTableWriterPtr<NYT::TNode> Writer_;
+ THolder<NYT::TNodeBuilder> Builder_;
+ ui32 Level_ = 0;
+};
+
+class TFmrYtService: public NYql::NFmr::IYtService {
+public:
+
+ std::variant<THolder<TTempFileHandle>, TError> Download(const TYtTableRef& ytTable, const TClusterConnection& clusterConnection) override {
+ try {
+ if (!ClusterConnections_.contains(ytTable.Cluster)) {
+ ClusterConnections_[ytTable.Cluster] = clusterConnection;
+ }
+ auto client = CreateClient(ClusterConnections_[ytTable.Cluster]);
+ auto tmpFile = MakeHolder<TTempFileHandle>();
+ auto transaction = client->AttachTransaction(GetGuid(clusterConnection.TransactionId));
+ TFileOutput downloadStream(tmpFile->Name());
+
+ NYson::TYsonWriter writer {&downloadStream, NYT::NYson::EYsonFormat::Text, NYT::NYson::EYsonType::ListFragment};
+ NYT::TNodeVisitor visitor{&writer};
+ auto reader = transaction->CreateTableReader<NYT::TNode>("//" + ytTable.Path);
+ for (; reader->IsValid(); reader->Next()) {
+ auto& row = reader->GetRow();
+ writer.OnListItem();
+ visitor.Visit(row);
+ }
+ downloadStream.Flush();
+ return tmpFile;
+ } catch (...) {
+ return TError{CurrentExceptionMessage()};
+ }
+ }
+
+ TMaybe<TError> Upload(const TYtTableRef& ytTable, IInputStream& tableContent, const TClusterConnection& clusterConnection) override {
+ try {
+ if (!ClusterConnections_.contains(ytTable.Cluster)) {
+ ClusterConnections_[ytTable.Cluster] = clusterConnection;
+ }
+ auto client = CreateClient(ClusterConnections_[ytTable.Cluster]);
+ auto transaction = client->AttachTransaction(GetGuid(clusterConnection.TransactionId));
+ auto options = NYT::TCreateOptions().Recursive(true).IgnoreExisting(true);
+ transaction->Create("//" + ytTable.Path, NYT::NT_TABLE, options);
+ auto path = NYT::TRichYPath("//" + ytTable.Path).Append(true);
+ auto ytWriter = transaction->CreateTableWriter<NYT::TNode>(path);
+
+ TYtFmrRowConsumer builder(ytWriter);
+ NYson::TYsonParser parser(&builder, &tableContent, NYT::NYson::EYsonType::ListFragment);
+ parser.Parse();
+ builder.Flush();
+ return Nothing();
+ } catch (...) {
+ return TError{CurrentExceptionMessage()};
+ }
+ }
+
+private:
+ std::unordered_map<TString, TClusterConnection> ClusterConnections_;
+
+ NYT::IClientPtr CreateClient(const TClusterConnection& clusterConnection) {
+ NYT::TCreateClientOptions createOpts;
+ auto token = clusterConnection.Token;
+ if (token) {
+ createOpts.Token(*token);
+ }
+ return NYT::CreateClient(clusterConnection.YtServerName, createOpts);
+ }
+};
+
+} // namespace
+
+IYtService::TPtr MakeFmrYtSerivce() {
+ return MakeIntrusive<TFmrYtService>();
+}
+
+} // namespace NYql::NFmr
diff --git a/yt/yql/providers/yt/fmr/yt_service/impl/yql_yt_yt_service_impl.h b/yt/yql/providers/yt/fmr/yt_service/impl/yql_yt_yt_service_impl.h
new file mode 100644
index 0000000000..90d2a69430
--- /dev/null
+++ b/yt/yql/providers/yt/fmr/yt_service/impl/yql_yt_yt_service_impl.h
@@ -0,0 +1,10 @@
+#include <yt/yql/providers/yt/fmr/yt_service/interface/yql_yt_yt_service.h>
+
+namespace NYql::NFmr {
+
+struct TFmrYtSerivceSettings {
+};
+
+IYtService::TPtr MakeFmrYtSerivce();
+
+} // namespace NYql::NFmr
diff --git a/yt/yql/providers/yt/fmr/yt_service/interface/ya.make b/yt/yql/providers/yt/fmr/yt_service/interface/ya.make
new file mode 100644
index 0000000000..6ea8e94c4f
--- /dev/null
+++ b/yt/yql/providers/yt/fmr/yt_service/interface/ya.make
@@ -0,0 +1,13 @@
+LIBRARY()
+
+SRCS(
+ yql_yt_yt_service.cpp
+)
+
+PEERDIR(
+ yt/yql/providers/yt/fmr/request_options
+)
+
+YQL_LAST_ABI_VERSION()
+
+END()
diff --git a/yt/yql/providers/yt/fmr/yt_service/interface/yql_yt_yt_service.cpp b/yt/yql/providers/yt/fmr/yt_service/interface/yql_yt_yt_service.cpp
new file mode 100644
index 0000000000..fbcafbc3a2
--- /dev/null
+++ b/yt/yql/providers/yt/fmr/yt_service/interface/yql_yt_yt_service.cpp
@@ -0,0 +1 @@
+#include "yql_yt_yt_service.h"
diff --git a/yt/yql/providers/yt/fmr/yt_service/interface/yql_yt_yt_service.h b/yt/yql/providers/yt/fmr/yt_service/interface/yql_yt_yt_service.h
new file mode 100644
index 0000000000..7d3e59b523
--- /dev/null
+++ b/yt/yql/providers/yt/fmr/yt_service/interface/yql_yt_yt_service.h
@@ -0,0 +1,19 @@
+#pragma once
+
+#include <util/system/tempfile.h>
+#include <yt/yql/providers/yt/fmr/request_options/yql_yt_request_options.h>
+
+namespace NYql::NFmr {
+
+class IYtService: public TThrRefBase {
+public:
+ virtual ~IYtService() = default;
+
+ using TPtr = TIntrusivePtr<IYtService>;
+
+ virtual std::variant<THolder<TTempFileHandle>, TError> Download(const TYtTableRef& ytTable, const TClusterConnection& clusterConnection = TClusterConnection()) = 0;
+
+ virtual TMaybe<TError> Upload(const TYtTableRef& ytTable, IInputStream& tableContent, const TClusterConnection& clusterConnection = TClusterConnection()) = 0;
+};
+
+} // namespace NYql::NFmr
diff --git a/yt/yql/providers/yt/fmr/yt_service/mock/ya.make b/yt/yql/providers/yt/fmr/yt_service/mock/ya.make
new file mode 100644
index 0000000000..707812d60c
--- /dev/null
+++ b/yt/yql/providers/yt/fmr/yt_service/mock/ya.make
@@ -0,0 +1,17 @@
+LIBRARY()
+
+SRCS(
+ yql_yt_yt_service_mock.h
+)
+
+PEERDIR(
+ yt/yql/providers/yt/fmr/yt_service/interface
+)
+
+YQL_LAST_ABI_VERSION()
+
+END()
+
+RECURSE_FOR_TESTS(
+ ut
+)
diff --git a/yt/yql/providers/yt/fmr/yt_service/mock/yql_yt_yt_service_mock.h b/yt/yql/providers/yt/fmr/yt_service/mock/yql_yt_yt_service_mock.h
new file mode 100644
index 0000000000..7d5c4d1076
--- /dev/null
+++ b/yt/yql/providers/yt/fmr/yt_service/mock/yql_yt_yt_service_mock.h
@@ -0,0 +1,84 @@
+#pragma once
+
+#include <yt/yql/providers/yt/fmr/yt_service/interface/yql_yt_yt_service.h>
+
+namespace NYql::NFmr {
+
+namespace {
+
+class TYtUploadedTablesMock: public TThrRefBase {
+public:
+ using TPtr = TIntrusivePtr<TYtUploadedTablesMock>;
+
+ ~TYtUploadedTablesMock() = default;
+
+ void AddTable(const TYtTableRef& ytTable, const TString& tableContent) {
+ TString key = GetTableKey(ytTable);
+ UploadedTables_[key] = tableContent;
+ }
+ bool Contains(const TYtTableRef& ytTable) const {
+ TString key = GetTableKey(ytTable);
+ return UploadedTables_.find(key) != UploadedTables_.end();
+ }
+ const TString& GetTableContent(const TYtTableRef& ytTable) const {
+ TString key = GetTableKey(ytTable);
+ return UploadedTables_.at(key);
+ }
+ void Clear() {
+ UploadedTables_.clear();
+ }
+ bool IsEmpty() const {
+ return UploadedTables_.empty();
+ }
+ int UploadedTablesNum() const {
+ return UploadedTables_.size();
+ }
+
+private:
+ TString GetTableKey(const TYtTableRef& ytTable) const {
+ return ytTable.Cluster + ":" + ytTable.Path;
+ }
+ std::unordered_map<TString, TString> UploadedTables_;
+};
+
+
+class TYtServiceMock: public NYql::NFmr::IYtService {
+public:
+
+ TYtServiceMock(TYtUploadedTablesMock::TPtr ytUploadedTablesMock)
+ : YtUploadedTablesMock_(ytUploadedTablesMock)
+ {
+ }
+
+ std::variant<THolder<TTempFileHandle>, TError> Download(const TYtTableRef& ytTable, const TClusterConnection& /*clusterConnection*/) override {
+ if (!YtUploadedTablesMock_->Contains(ytTable)) {
+ return TError("Table not found");
+ }
+ auto tmpFile = MakeHolder<TTempFileHandle>();
+ TString tableContent = YtUploadedTablesMock_->GetTableContent(ytTable);
+ tmpFile->Write(tableContent.data(), tableContent.size());
+ return tmpFile;
+ }
+
+ TMaybe<TError> Upload(const TYtTableRef& ytTable, IInputStream& tableContent, const TClusterConnection& /*clusterConnection*/) override {
+ YtUploadedTablesMock_->AddTable(ytTable, tableContent.ReadAll());
+ return Nothing();
+ }
+
+private:
+ TYtUploadedTablesMock::TPtr YtUploadedTablesMock_;
+};
+
+} // namespace
+
+IYtService::TPtr MakeYtServiceMock(TYtUploadedTablesMock::TPtr ytUploadedTablesMock) {
+ return MakeIntrusive<TYtServiceMock>(ytUploadedTablesMock);
+}
+
+TYtUploadedTablesMock::TPtr MakeYtUploadedTablesMock() {
+ return MakeIntrusive<TYtUploadedTablesMock>();
+}
+
+} // namespace NYql::NFmr
+
+// TODO - move this to .cpp file
diff --git a/yt/yql/providers/yt/gateway/file/yql_yt_file.cpp b/yt/yql/providers/yt/gateway/file/yql_yt_file.cpp
index b0e53f5cb7..3a1e127f3d 100644
--- a/yt/yql/providers/yt/gateway/file/yql_yt_file.cpp
+++ b/yt/yql/providers/yt/gateway/file/yql_yt_file.cpp
@@ -1583,6 +1583,10 @@ private:
return res;
}
+ TClusterConnectionResult GetClusterConnection(const TClusterConnectionOptions&& /*options*/) override {
+ ythrow yexception() << "GetClusterConnection should not be called for file gateway";
+ }
+
private:
TYtFileServices::TPtr Services_;
diff --git a/yt/yql/providers/yt/gateway/fmr/ya.make b/yt/yql/providers/yt/gateway/fmr/ya.make
index bfc91df2f5..c424b95d0b 100644
--- a/yt/yql/providers/yt/gateway/fmr/ya.make
+++ b/yt/yql/providers/yt/gateway/fmr/ya.make
@@ -6,8 +6,12 @@ SRCS(
PEERDIR(
yql/essentials/utils/log
+ yt/cpp/mapreduce/client
+ yt/yql/providers/yt/gateway/lib
+ yt/yql/providers/yt/gateway/native
yt/yql/providers/yt/expr_nodes
yt/yql/providers/yt/fmr/coordinator/interface
+ yt/yql/providers/yt/lib/config_clusters
yt/yql/providers/yt/provider
)
diff --git a/yt/yql/providers/yt/gateway/fmr/yql_yt_fmr.cpp b/yt/yql/providers/yt/gateway/fmr/yql_yt_fmr.cpp
index db00f2ff59..76a907a5c6 100644
--- a/yt/yql/providers/yt/gateway/fmr/yql_yt_fmr.cpp
+++ b/yt/yql/providers/yt/gateway/fmr/yql_yt_fmr.cpp
@@ -2,7 +2,10 @@
#include <thread>
+#include <yt/cpp/mapreduce/interface/client.h>
#include <yt/yql/providers/yt/expr_nodes/yql_yt_expr_nodes.h>
+#include <yt/yql/providers/yt/gateway/lib/yt_helpers.h>
+#include <yt/yql/providers/yt/gateway/native/yql_yt_native.h>
#include <yt/yql/providers/yt/provider/yql_yt_helpers.h>
#include <yql/essentials/utils/log/log.h>
@@ -40,7 +43,7 @@ public:
with_lock(SessionStates_->Mutex) {
auto checkOperationStatuses = [&] <typename T> (std::unordered_map<TString, TPromise<T>>& operationStatuses, const TString& sessionId) {
for (auto& [operationId, promise]: operationStatuses) {
- YQL_CLOG(DEBUG, FastMapReduce) << "Sending get operation request to coordinator with operationId: " << operationId;
+ YQL_CLOG(TRACE, FastMapReduce) << "Sending get operation request to coordinator with operationId: " << operationId;
auto getOperationFuture = Coordinator_->GetOperation({operationId});
getOperationFuture.Subscribe([&, operationId, sessionId] (const auto& getFuture) {
@@ -96,10 +99,22 @@ public:
return Slave_->Publish(node, ctx, std::move(options));
}
auto publish = TYtPublish(node);
+ TString sessionId = options.SessionId();
auto cluster = publish.DataSink().Cluster().StringValue();
+ auto token = options.Config()->Auth.Get();
+ TString transformedInputPath;
+ TString userName = GetUsername(sessionId);
+ for (auto out: publish.Input()) {
+ auto outTable = GetOutTable(out).Cast<TYtOutTable>();
+ TStringBuf inputPath = outTable.Name().Value();
+ transformedInputPath = NYql::TransformPath(GetTablesTmpFolder(*options.Config()), inputPath, true, userName);
+ break;
+ }
+
+ // TODO - handle several inputs in Publish, use ColumnGroups, Run Merge
+
auto outputPath = publish.Publish().Name().StringValue();
- auto transactionId = GenerateId();
auto idempotencyKey = GenerateId();
auto fmrTableId = cluster + "." + outputPath;
@@ -107,15 +122,14 @@ public:
TFuture<TDownloadTableToFmrResult> downloadToFmrFuture;
TFuture<void> downloadedSuccessfully;
- TString sessionId = options.SessionId();
-
with_lock(SessionStates_->Mutex) {
auto& tablePresenceStatuses = SessionStates_->Sessions[sessionId].TablePresenceStatuses;
if (!tablePresenceStatuses.contains(fmrTableId)) {
- TYtTableRef ytTable{.Path = outputPath, .Cluster = cluster, .TransactionId = transactionId};
+ TYtTableRef ytTable{.Path = transformedInputPath, .Cluster = cluster};
+ TFmrTableRef fmrTable{.TableId = fmrTableId};
tablePresenceStatuses[fmrTableId] = ETablePresenceStatus::Both;
- downloadToFmrFuture = DownloadToFmrTableDataSerivce(ytTable, sessionId);
+ downloadToFmrFuture = DownloadToFmrTableDataSerivce(ytTable, fmrTable, sessionId, options.Config());
downloadedSuccessfully = downloadToFmrFuture.Apply([downloadedSuccessfully] (auto& downloadFuture) {
auto downloadResult = downloadFuture.GetValueSync();
});
@@ -125,15 +139,27 @@ public:
}
downloadedSuccessfully.Wait(); // blocking until download to fmr finishes
- YQL_CLOG(INFO, FastMapReduce) << "Uploading table with cluster " << cluster << " and path " << outputPath << " from fmr to yt";
-
TUploadTaskParams uploadTaskParams{
.Input = TFmrTableRef{fmrTableId},
- .Output = TYtTableRef{outputPath, cluster, transactionId}
+ .Output = TYtTableRef{outputPath, cluster}
};
+ auto clusterConnectionOptions = TClusterConnectionOptions(options.SessionId())
+ .Cluster(cluster).Config(options.Config());
+ auto clusterConnection = GetClusterConnection(std::move(clusterConnectionOptions));
+ YQL_ENSURE(clusterConnection.Success());
+
TStartOperationRequest uploadRequest{
- .TaskType = ETaskType::Upload, .TaskParams = uploadTaskParams, .SessionId = sessionId, .IdempotencyKey=idempotencyKey, .NumRetries=1
+ .TaskType = ETaskType::Upload,
+ .TaskParams = uploadTaskParams,
+ .SessionId = sessionId,
+ .IdempotencyKey=idempotencyKey,
+ .NumRetries=1,
+ .ClusterConnection = TClusterConnection{
+ .TransactionId = clusterConnection.TransactionId,
+ .YtServerName = clusterConnection.YtServerName,
+ .Token = clusterConnection.Token
+ }
};
auto promise = NewPromise<TPublishResult>();
@@ -155,19 +181,34 @@ public:
return future;
}
- TFuture<TDownloadTableToFmrResult> DownloadToFmrTableDataSerivce(const TYtTableRef& ytTableRef, const TString& sessionId) {
+ TFuture<TDownloadTableToFmrResult> DownloadToFmrTableDataSerivce(
+ const TYtTableRef& ytTableRef, const TFmrTableRef& fmrTableRef, const TString& sessionId, TYtSettings::TConstPtr& config)
+ {
YQL_LOG_CTX_SCOPE(TStringBuf("Gateway"), __FUNCTION__);
- TString fmrTableId = ytTableRef.Cluster + "." + ytTableRef.Path;
+ TString fmrTableId = fmrTableRef.TableId;
TDownloadTaskParams downloadTaskParams{
.Input = ytTableRef,
.Output = {fmrTableId}
};
auto idempotencyKey = GenerateId();
+ auto clusterConnectionOptions = TClusterConnectionOptions(sessionId)
+ .Cluster(ytTableRef.Cluster).Config(config);
+ auto clusterConnection = GetClusterConnection(std::move(clusterConnectionOptions));
+ YQL_ENSURE(clusterConnection.Success());
TStartOperationRequest downloadRequest{
- .TaskType = ETaskType::Download, .TaskParams = downloadTaskParams, .SessionId = sessionId, .IdempotencyKey=idempotencyKey, .NumRetries=1
+ .TaskType = ETaskType::Download,
+ .TaskParams = downloadTaskParams,
+ .SessionId = sessionId,
+ .IdempotencyKey = idempotencyKey,
+ .NumRetries=1,
+ .ClusterConnection = TClusterConnection{
+ .TransactionId = clusterConnection.TransactionId,
+ .YtServerName = clusterConnection.YtServerName,
+ .Token = clusterConnection.Token
+ }
};
- YQL_CLOG(DEBUG, FastMapReduce) << "Starting download to from yt table: " << fmrTableId;
+ YQL_CLOG(DEBUG, FastMapReduce) << "Starting download from yt table: " << fmrTableId;
auto promise = NewPromise<TDownloadTableToFmrResult>();
auto future = promise.GetFuture();
@@ -186,22 +227,24 @@ public:
return future;
}
- void OpenSession(TOpenSessionOptions&& options) final {
- Slave_->OpenSession(std::move(options));
+ TClusterConnectionResult GetClusterConnection(const TClusterConnectionOptions&& options) override {
+ return Slave_->GetClusterConnection(std::move(options));
+ }
+ void OpenSession(TOpenSessionOptions&& options) final {
TString sessionId = options.SessionId();
YQL_LOG_CTX_SCOPE(TStringBuf("Gateway"), __FUNCTION__);
with_lock(SessionStates_->Mutex) {
- auto sessions = SessionStates_->Sessions;
+ auto& sessions = SessionStates_->Sessions;
if (sessions.contains(sessionId)) {
YQL_LOG_CTX_THROW yexception() << "Session already exists: " << sessionId;
}
- sessions[sessionId] = TSessionInfo();
+ sessions[sessionId] = TSessionInfo{.UserName = options.UserName()};
}
+ Slave_->OpenSession(std::move(options));
}
TFuture<void> CloseSession(TCloseSessionOptions&& options) final {
- Slave_->CloseSession(std::move(options)).Wait();
YQL_LOG_CTX_SCOPE(TStringBuf("Gateway"), __FUNCTION__);
with_lock(SessionStates_->Mutex) {
@@ -211,11 +254,11 @@ public:
sessions.erase(it);
}
}
+ Slave_->CloseSession(std::move(options)).Wait();
return MakeFuture();
}
TFuture<void> CleanupSession(TCleanupSessionOptions&& options) final {
- Slave_->CleanupSession(std::move(options)).Wait();
YQL_LOG_CTX_SCOPE(TStringBuf("Gateway"), __FUNCTION__);
TString sessionId = options.SessionId();
@@ -239,7 +282,7 @@ public:
cancelOperationsFunc(operationStates.DownloadOperationStatuses);
cancelOperationsFunc(operationStates.UploadOperationStatuses);
}
-
+ Slave_->CleanupSession(std::move(options)).Wait();
return MakeFuture();
}
@@ -252,6 +295,7 @@ private:
struct TSessionInfo {
TFmrGatewayOperationsState OperationStates;
std::unordered_map<TString, ETablePresenceStatus> TablePresenceStatuses; // yt cluster and path -> is it In Yt, Fmr TableDataService
+ TString UserName;
};
struct TSession {
@@ -282,6 +326,14 @@ private:
}
promise.SetValue(commonOperationResult);
}
+
+ TString GetUsername(const TString& sessionId) {
+ with_lock(SessionStates_->Mutex) {
+ YQL_ENSURE(SessionStates_->Sessions.contains(sessionId));
+ auto& session = SessionStates_->Sessions[sessionId];
+ return session.UserName;
+ }
+ }
};
} // namespace
diff --git a/yt/yql/providers/yt/gateway/fmr/yql_yt_fmr.h b/yt/yql/providers/yt/gateway/fmr/yql_yt_fmr.h
index 769bb24fe5..6d102e57ca 100644
--- a/yt/yql/providers/yt/gateway/fmr/yql_yt_fmr.h
+++ b/yt/yql/providers/yt/gateway/fmr/yql_yt_fmr.h
@@ -6,17 +6,14 @@
namespace NYql::NFmr {
struct TFmrYtGatewaySettings {
- TIntrusivePtr<IRandomProvider> RandomProvider;
- TDuration TimeToSleepBetweenGetOperationRequests;
+ TIntrusivePtr<IRandomProvider> RandomProvider = CreateDefaultRandomProvider();
+ TDuration TimeToSleepBetweenGetOperationRequests = TDuration::Seconds(1);
};
IYtGateway::TPtr CreateYtFmrGateway(
IYtGateway::TPtr slave,
IFmrCoordinator::TPtr coordinator = nullptr,
- const TFmrYtGatewaySettings& settings = TFmrYtGatewaySettings{
- .RandomProvider = CreateDefaultRandomProvider(),
- .TimeToSleepBetweenGetOperationRequests = TDuration::Seconds(1)
- }
+ const TFmrYtGatewaySettings& settings = TFmrYtGatewaySettings{}
);
} // namespace NYql::NFmr
diff --git a/yt/yql/providers/yt/gateway/native/yql_yt_native.cpp b/yt/yql/providers/yt/gateway/native/yql_yt_native.cpp
index 40df6077ef..fb021d149c 100644
--- a/yt/yql/providers/yt/gateway/native/yql_yt_native.cpp
+++ b/yt/yql/providers/yt/gateway/native/yql_yt_native.cpp
@@ -361,69 +361,79 @@ void GenerateInputQueryWhereExpression(const TExprNode::TPtr& node, TStringBuild
}
}
-TString GenerateInputQuery(const TExprNode& settings, const TVector<TInputInfo>& inputs) {
+TString GenerateInputQueryWhereExpression(const TExprNode& settings) {
auto qlFilterNode = NYql::GetSetting(settings, EYtSettingType::QLFilter);
if (!qlFilterNode) {
return "";
}
- // TODO: how to handle multiple inputs?
- YQL_ENSURE(inputs.size() == 1, "YtQLFilter: multiple inputs are not supported");
-
qlFilterNode = qlFilterNode->ChildPtr(1);
YQL_ENSURE(qlFilterNode && qlFilterNode->IsCallable("YtQLFilter"));
const TYtQLFilter qlFilter(qlFilterNode);
const auto predicate = qlFilter.Predicate().Body();
TStringBuilder result;
- bool foundSomeColumns = false;
- for (const auto& table: inputs) {
- YQL_ENSURE(table.Path.Columns_ && table.Path.Columns_->Parts_, "YtQLFilter: TRichYPath should have columns");
- for (const auto& column: table.Path.Columns_->Parts_) {
- if (foundSomeColumns) {
+ GenerateInputQueryWhereExpression(predicate.Ptr(), result);
+ YQL_CLOG(INFO, ProviderYt) << __FUNCTION__ << ": " << result;
+ return result;
+}
+
+TString GenerateInputQuery(const TRichYPath& path, const TString& whereExpression, bool useSystemColumns) {
+ YQL_ENSURE(whereExpression);
+ TStringBuilder result;
+ if (!path.Columns_ || !path.Columns_->Parts_) {
+ result << "*";
+ } else {
+ bool first = true;
+ for (const auto& column: path.Columns_->Parts_) {
+ if (!first) {
result << ", ";
}
QuoteColumnForQL(column, result);
- foundSomeColumns = true;
+ first = false;
+ }
+ if (useSystemColumns) {
+ result << ", `$row_index`, `$range_index`";
}
}
- if (!foundSomeColumns) {
- result << "*";
- }
- result << " WHERE ";
- GenerateInputQueryWhereExpression(predicate.Ptr(), result);
+ result << " WHERE " << whereExpression;
YQL_CLOG(INFO, ProviderYt) << __FUNCTION__ << ": " << result;
return result;
}
-void SetInputQuerySpec(NYT::TNode& spec, const TString& inputQuery) {
+void SetInputQuerySpec(NYT::TNode& spec, const TString& inputQuery, bool useSystemColumns) {
spec["input_query"] = inputQuery;
spec["input_query_filter_options"]["enable_chunk_filter"] = true;
spec["input_query_filter_options"]["enable_row_filter"] = true;
+ if (useSystemColumns) {
+ spec["input_query_options"]["use_system_columns"] = true;
+ }
}
-void PrepareInputQueryForMerge(NYT::TNode& spec, TVector<TRichYPath>& paths, const TString& inputQuery) {
+void PrepareInputQueryForMerge(NYT::TNode& spec, TVector<TRichYPath>& paths, const TString& whereExpression) {
// YQL-19382
- if (inputQuery) {
- for (auto& path : paths) {
- path.Columns_.Clear();
- }
- SetInputQuerySpec(spec, inputQuery);
+ if (whereExpression) {
+ YQL_ENSURE(paths.size() == 1, "YtQLFilter: multiple inputs are not supported");
+ auto& path = paths[0];
+ const TString inputQuery = GenerateInputQuery(path, whereExpression, /*useSystemColumns*/ false);
+ path.Columns_.Clear();
+ SetInputQuerySpec(spec, inputQuery, /*useSystemColumns*/ false);
}
}
template <typename T>
-void PrepareInputQueryForMap(NYT::TNode& spec, T& specWithPaths, const TString& inputQuery, const bool useSkiff) {
+void PrepareInputQueryForMap(NYT::TNode& spec, T& specWithPaths, const TString& whereExpression, bool useSystemColumns) {
// YQL-19382
- if (inputQuery) {
- YQL_ENSURE(!useSkiff, "QLFilter can't work with skiff on map/mapreduce right now, try with PRAGMA yt.UseSkiff='false'");
+ if (whereExpression) {
const auto& inputs = specWithPaths.GetInputs();
- for (size_t i = 0; i < inputs.size(); ++i) {
- auto path = inputs[i];
+ YQL_ENSURE(inputs.size() == 1, "YtQLFilter: multiple inputs are not supported");
+ auto path = inputs[0];
+ const TString inputQuery = GenerateInputQuery(path, whereExpression, useSystemColumns);
+ if (path.Columns_) {
path.Columns_.Clear();
- specWithPaths.SetInput(i, path);
+ specWithPaths.SetInput(0, path);
}
- SetInputQuerySpec(spec, inputQuery);
+ SetInputQuerySpec(spec, inputQuery, useSystemColumns);
}
}
@@ -3594,10 +3604,10 @@ private:
bool forceTransform = NYql::HasAnySetting(merge.Settings().Ref(), EYtSettingType::ForceTransform | EYtSettingType::SoftTransform);
bool combineChunks = NYql::HasSetting(merge.Settings().Ref(), EYtSettingType::CombineChunks);
TMaybe<ui64> limit = GetLimit(merge.Settings().Ref());
- const TString inputQuery = GenerateInputQuery(merge.Settings().Ref(), execCtx->InputTables_);
+ const TString inputQueryExpr = GenerateInputQueryWhereExpression(merge.Settings().Ref());
- return execCtx->Session_->Queue_->Async([forceTransform, combineChunks, limit, inputQuery, execCtx]() {
- return execCtx->LookupQueryCacheAsync().Apply([forceTransform, combineChunks, limit, inputQuery, execCtx] (const auto& f) {
+ return execCtx->Session_->Queue_->Async([forceTransform, combineChunks, limit, inputQueryExpr, execCtx]() {
+ return execCtx->LookupQueryCacheAsync().Apply([forceTransform, combineChunks, limit, inputQueryExpr, execCtx] (const auto& f) {
YQL_LOG_CTX_ROOT_SESSION_SCOPE(execCtx->LogCtx_);
auto entry = execCtx->GetEntry();
bool cacheHit = f.GetValue();
@@ -3643,7 +3653,7 @@ private:
spec["schema_inference_mode"] = "from_output"; // YTADMINREQ-17692
}
- PrepareInputQueryForMerge(spec, mergeOpSpec.Inputs_, inputQuery);
+ PrepareInputQueryForMerge(spec, mergeOpSpec.Inputs_, inputQueryExpr);
return execCtx->RunOperation([entry, mergeOpSpec = std::move(mergeOpSpec), spec = std::move(spec)](){
return entry->Tx->Merge(mergeOpSpec, TOperationOptions().StartOperationMode(TOperationOptions::EStartOperationMode::AsyncPrepare).Spec(spec));
@@ -3662,13 +3672,13 @@ private:
TString mapLambda,
const TString& inputType,
const TExpressionResorceUsage& extraUsage,
- const TString& inputQuery,
+ const TString& inputQueryExpr,
const TExecContext<TRunOptions>::TPtr& execCtx
) {
const bool testRun = execCtx->Config_->GetLocalChainTest();
TFuture<bool> ret = testRun ? MakeFuture<bool>(false) : execCtx->LookupQueryCacheAsync();
return ret.Apply([ordered, blockInput, blockOutput, jobCount, limit, sortLimitBy, mapLambda,
- inputType, extraUsage, inputQuery, execCtx, testRun] (const auto& f) mutable
+ inputType, extraUsage, inputQueryExpr, execCtx, testRun] (const auto& f) mutable
{
YQL_LOG_CTX_ROOT_SESSION_SCOPE(execCtx->LogCtx_);
TTransactionCache::TEntry::TPtr entry;
@@ -3831,7 +3841,7 @@ private:
spec["job_count"] = static_cast<i64>(*jobCount);
}
- PrepareInputQueryForMap(spec, mapOpSpec, inputQuery, useSkiff);
+ PrepareInputQueryForMap(spec, mapOpSpec, inputQueryExpr, /*useSystemColumns*/ useSkiff);
TOperationOptions opOpts;
FillOperationOptions(opOpts, execCtx, entry);
@@ -3870,12 +3880,12 @@ private:
}
auto extraUsage = execCtx->ScanExtraResourceUsage(map.Mapper().Body().Ref(), true);
TString inputType = NCommon::WriteTypeToYson(GetSequenceItemType(map.Input().Size() == 1U ? TExprBase(map.Input().Item(0)) : TExprBase(map.Mapper().Args().Arg(0)), true));
- const TString inputQuery = GenerateInputQuery(map.Settings().Ref(), execCtx->InputTables_);
+ const TString inputQueryExpr = GenerateInputQueryWhereExpression(map.Settings().Ref());
- return execCtx->Session_->Queue_->Async([ordered, blockInput, blockOutput, jobCount, limit, sortLimitBy, mapLambda, inputType, extraUsage, inputQuery, execCtx]() {
+ return execCtx->Session_->Queue_->Async([ordered, blockInput, blockOutput, jobCount, limit, sortLimitBy, mapLambda, inputType, extraUsage, inputQueryExpr, execCtx]() {
YQL_LOG_CTX_ROOT_SESSION_SCOPE(execCtx->LogCtx_);
execCtx->MakeUserFiles();
- return ExecMap(ordered, blockInput, blockOutput, jobCount, limit, sortLimitBy, mapLambda, inputType, extraUsage, inputQuery, execCtx);
+ return ExecMap(ordered, blockInput, blockOutput, jobCount, limit, sortLimitBy, mapLambda, inputType, extraUsage, inputQueryExpr, execCtx);
});
}
@@ -4103,14 +4113,14 @@ private:
NYT::TNode intermediateMeta,
const NYT::TNode& intermediateSchema,
const NYT::TNode& intermediateStreams,
- const TString& inputQuery,
+ const TString& inputQueryExpr,
const TExecContext<TRunOptions>::TPtr& execCtx
) {
const bool testRun = execCtx->Config_->GetLocalChainTest();
TFuture<bool> ret = testRun ? MakeFuture<bool>(false) : execCtx->LookupQueryCacheAsync();
return ret.Apply([reduceBy, sortBy, limit, sortLimitBy, mapLambda, mapInputType, mapDirectOutputs,
mapExtraUsage, reduceLambda, reduceInputType, reduceExtraUsage,
- intermediateMeta, intermediateSchema, intermediateStreams, inputQuery, execCtx, testRun]
+ intermediateMeta, intermediateSchema, intermediateStreams, inputQueryExpr, execCtx, testRun]
(const auto& f) mutable
{
YQL_LOG_CTX_ROOT_SESSION_SCOPE(execCtx->LogCtx_);
@@ -4328,7 +4338,7 @@ private:
spec["mapper"]["output_streams"] = intermediateStreams;
}
- PrepareInputQueryForMap(spec, mapReduceOpSpec, inputQuery, useSkiff);
+ PrepareInputQueryForMap(spec, mapReduceOpSpec, inputQueryExpr, /*useSystemColumns*/ useSkiff);
TOperationOptions opOpts;
FillOperationOptions(opOpts, execCtx, entry);
@@ -4351,13 +4361,13 @@ private:
const TExpressionResorceUsage& reduceExtraUsage,
const NYT::TNode& intermediateSchema,
bool useIntermediateStreams,
- const TString& inputQuery,
+ const TString& inputQueryExpr,
const TExecContext<TRunOptions>::TPtr& execCtx
) {
const bool testRun = execCtx->Config_->GetLocalChainTest();
TFuture<bool> ret = testRun ? MakeFuture<bool>(false) : execCtx->LookupQueryCacheAsync();
return ret.Apply([reduceBy, sortBy, limit, sortLimitBy, reduceLambda, reduceInputType,
- reduceExtraUsage, intermediateSchema, useIntermediateStreams, inputQuery, execCtx, testRun]
+ reduceExtraUsage, intermediateSchema, useIntermediateStreams, inputQueryExpr, execCtx, testRun]
(const auto& f) mutable
{
YQL_LOG_CTX_ROOT_SESSION_SCOPE(execCtx->LogCtx_);
@@ -4475,7 +4485,7 @@ private:
spec["reducer"]["enable_input_table_index"] = true;
}
- PrepareInputQueryForMap(spec, mapReduceOpSpec, inputQuery, useSkiff);
+ PrepareInputQueryForMap(spec, mapReduceOpSpec, inputQueryExpr, /*useSystemColumns*/ useSkiff);
TOperationOptions opOpts;
FillOperationOptions(opOpts, execCtx, entry);
@@ -4598,19 +4608,19 @@ private:
limit.Clear();
}
- const TString inputQuery = GenerateInputQuery(mapReduce.Settings().Ref(), execCtx->InputTables_);
+ const TString inputQueryExpr = GenerateInputQueryWhereExpression(mapReduce.Settings().Ref());
return execCtx->Session_->Queue_->Async([reduceBy, sortBy, limit, sortLimitBy, mapLambda, mapInputType, mapDirectOutputs, mapExtraUsage,
- reduceLambda, reduceInputType, reduceExtraUsage, intermediateMeta, intermediateSchema, intermediateStreams, useIntermediateStreams, inputQuery, execCtx]()
+ reduceLambda, reduceInputType, reduceExtraUsage, intermediateMeta, intermediateSchema, intermediateStreams, useIntermediateStreams, inputQueryExpr, execCtx]()
{
YQL_LOG_CTX_ROOT_SESSION_SCOPE(execCtx->LogCtx_);
execCtx->MakeUserFiles();
if (mapLambda) {
return ExecMapReduce(reduceBy, sortBy, limit, sortLimitBy, mapLambda, mapInputType, mapDirectOutputs, mapExtraUsage,
- reduceLambda, reduceInputType, reduceExtraUsage, intermediateMeta, intermediateSchema, intermediateStreams, inputQuery, execCtx);
+ reduceLambda, reduceInputType, reduceExtraUsage, intermediateMeta, intermediateSchema, intermediateStreams, inputQueryExpr, execCtx);
} else {
return ExecMapReduce(reduceBy, sortBy, limit, sortLimitBy, reduceLambda, reduceInputType, reduceExtraUsage, intermediateSchema,
- useIntermediateStreams, inputQuery, execCtx);
+ useIntermediateStreams, inputQueryExpr, execCtx);
}
});
}
@@ -5698,6 +5708,22 @@ private:
return ctx;
}
+ TClusterConnectionResult GetClusterConnection(const TClusterConnectionOptions&& options) override {
+ try {
+ auto session = GetSession(options.SessionId(), true);
+ auto ytServer = Clusters_->GetServer(options.Cluster());
+ auto entry = session->TxCache_.GetEntry(ytServer);
+ TClusterConnectionResult clusterConnectionResult{};
+ clusterConnectionResult.TransactionId = GetGuidAsString(entry->Tx->GetId());
+ clusterConnectionResult.YtServerName = ytServer;
+ clusterConnectionResult.Token = options.Config()->Auth.Get();
+ clusterConnectionResult.SetSuccess();
+ return clusterConnectionResult;
+ } catch (...) {
+ return ResultFromCurrentException<TClusterConnectionResult>({}, true);
+ }
+ }
+
private:
const TYtNativeServices Services_;
const TConfigClusters::TPtr Clusters_;
diff --git a/yt/yql/providers/yt/gateway/qplayer/yql_yt_qplayer_gateway.cpp b/yt/yql/providers/yt/gateway/qplayer/yql_yt_qplayer_gateway.cpp
index 5ad80977e9..3a9bd3272b 100644
--- a/yt/yql/providers/yt/gateway/qplayer/yql_yt_qplayer_gateway.cpp
+++ b/yt/yql/providers/yt/gateway/qplayer/yql_yt_qplayer_gateway.cpp
@@ -961,6 +961,10 @@ public:
return Inner_->AddCluster(cluster);
}
+ TClusterConnectionResult GetClusterConnection(const TClusterConnectionOptions&& options) override {
+ return Inner_->GetClusterConnection(std::move(options));
+ }
+
private:
const IYtGateway::TPtr Inner_;
const TQContext QContext_;
diff --git a/yt/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_merge.cpp b/yt/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_merge.cpp
index 1caead064f..3ca868b3a1 100644
--- a/yt/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_merge.cpp
+++ b/yt/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_merge.cpp
@@ -446,11 +446,29 @@ TMaybeNode<TExprBase> TYtPhysicalOptProposalTransformer::MergeToCopy(TExprBase n
return node;
}
}
- TYtOutTableInfo outTableInfo(merge.Output().Item(0));
+
+ const auto outTable = merge.Output().Item(0);
+ TYtOutTableInfo outTableInfo(outTable);
if (!tableInfo->RowSpec->CompareSortness(*outTableInfo.RowSpec)) {
return node;
}
+ TStringBuf outColGroup;
+ if (auto setting = NYql::GetSetting(outTable.Settings().Ref(), EYtSettingType::ColumnGroups)) {
+ outColGroup = setting->Tail().Content();
+ }
+
+ YQL_ENSURE(path.Table().Maybe<TYtOutput>());
+ TStringBuf inputColGroup;
+ const auto out = path.Table().Cast<TYtOutput>();
+ if (auto setting = NYql::GetSetting(GetOutputOp(out).Output().Item(FromString<ui32>(out.OutIndex().Value())).Settings().Ref(), EYtSettingType::ColumnGroups)) {
+ inputColGroup = setting->Tail().Content();
+ }
+
+ if (outColGroup != inputColGroup) {
+ return node;
+ }
+
return Build<TYtCopy>(ctx, node.Pos())
.World(merge.World())
.DataSink(merge.DataSink())
diff --git a/yt/yql/providers/yt/provider/yql_yt_datasink_type_ann.cpp b/yt/yql/providers/yt/provider/yql_yt_datasink_type_ann.cpp
index 92265f2843..7acfeedcae 100644
--- a/yt/yql/providers/yt/provider/yql_yt_datasink_type_ann.cpp
+++ b/yt/yql/providers/yt/provider/yql_yt_datasink_type_ann.cpp
@@ -994,7 +994,7 @@ private:
if (outGroup != inputColGroupSpec) {
ctx.AddError(TIssue(ctx.GetPosition(copy.Output().Item(0).Settings().Pos()), TStringBuilder() << TYtCopy::CallableName()
- << "has input/output tables with different " << EYtSettingType::ColumnGroups << " values"));
+ << " has input/output tables with different " << EYtSettingType::ColumnGroups << " values"));
return TStatus::Error;
}
diff --git a/yt/yql/providers/yt/provider/yql_yt_gateway.h b/yt/yql/providers/yt/provider/yql_yt_gateway.h
index b331f2142c..a98cd1d877 100644
--- a/yt/yql/providers/yt/provider/yql_yt_gateway.h
+++ b/yt/yql/providers/yt/provider/yql_yt_gateway.h
@@ -613,6 +613,24 @@ public:
struct TUploadTableResult: public NCommon::TOperationResult {
};
+ struct TClusterConnectionOptions: public TCommonOptions {
+ using TSelf = TClusterConnectionOptions;
+
+ TClusterConnectionOptions(const TString& sessionId)
+ : TCommonOptions(sessionId)
+ {
+ }
+
+ OPTION_FIELD(TString, Cluster)
+ OPTION_FIELD(TYtSettings::TConstPtr, Config)
+ };
+
+ struct TClusterConnectionResult: public NCommon::TOperationResult {
+ TString TransactionId;
+ TString YtServerName;
+ TMaybe<TString> Token;
+ };
+
public:
virtual ~IYtGateway() = default;
@@ -673,6 +691,8 @@ public:
virtual TGetTablePartitionsResult GetTablePartitions(TGetTablePartitionsOptions&& options) = 0;
virtual void AddCluster(const TYtClusterConfig& cluster) = 0;
+
+ virtual TClusterConnectionResult GetClusterConnection(const TClusterConnectionOptions&& options) = 0;
};
}
diff --git a/yt/yql/providers/yt/provider/yql_yt_physical_finalizing.cpp b/yt/yql/providers/yt/provider/yql_yt_physical_finalizing.cpp
index ac7a3cb79b..b44fd4918a 100644
--- a/yt/yql/providers/yt/provider/yql_yt_physical_finalizing.cpp
+++ b/yt/yql/providers/yt/provider/yql_yt_physical_finalizing.cpp
@@ -2862,7 +2862,12 @@ private:
// Check all counsumers are known
auto& processed = ProcessedCalculateColumnGroups[writer];
- if (processed.size() == readers.size() && AllOf(readers, [&processed](const auto& item) { return processed.contains(std::get<0>(item)->UniqueId()); })) {
+ if (processed.size() == readers.size() &&
+ AllOf(readers, [&processed](const auto& item) {
+ // Always reprocess ops with merge/copy readers
+ return !TYtCopy::Match(std::get<0>(item)) && !TYtMerge::Match(std::get<0>(item)) && processed.contains(std::get<0>(item)->UniqueId());
+ })
+ ) {
continue;
}
processed.clear();
diff --git a/yt/yql/tests/sql/suites/ql_filter/integer_bounds.sql b/yt/yql/tests/sql/suites/ql_filter/integer_bounds.sql
index 1f763a6b41..0c2df74c4d 100644
--- a/yt/yql/tests/sql/suites/ql_filter/integer_bounds.sql
+++ b/yt/yql/tests/sql/suites/ql_filter/integer_bounds.sql
@@ -1,5 +1,7 @@
+/* yt can not */
+/* waiting for update YT-24048 */
+
pragma yt.UseQLFilter;
-PRAGMA yt.UseSkiff='false';
select c
from plato.Input
diff --git a/yt/yql/tests/sql/suites/ql_filter/integer_escaping.sql b/yt/yql/tests/sql/suites/ql_filter/integer_escaping.sql
index 3e074e24e6..a96d74390f 100644
--- a/yt/yql/tests/sql/suites/ql_filter/integer_escaping.sql
+++ b/yt/yql/tests/sql/suites/ql_filter/integer_escaping.sql
@@ -1,5 +1,7 @@
+/* yt can not */
+/* waiting for update YT-24048 */
+
pragma yt.UseQLFilter;
-PRAGMA yt.UseSkiff='false';
select
`escaping []\``,
diff --git a/yt/yql/tests/sql/suites/ql_filter/integer_many_left.sql b/yt/yql/tests/sql/suites/ql_filter/integer_many_left.sql
index fa2d584077..27eda8b58c 100644
--- a/yt/yql/tests/sql/suites/ql_filter/integer_many_left.sql
+++ b/yt/yql/tests/sql/suites/ql_filter/integer_many_left.sql
@@ -1,5 +1,7 @@
+/* yt can not */
+/* waiting for update YT-24048 */
+
pragma yt.UseQLFilter;
-pragma yt.UseSkiff='false'; -- temporary disable skiff https://st.yandex-team.ru/YT-14644
select a, c, d, e
from plato.Input
diff --git a/yt/yql/tests/sql/suites/ql_filter/integer_many_noskiff.cfg b/yt/yql/tests/sql/suites/ql_filter/integer_many_noskiff.cfg
new file mode 100644
index 0000000000..d0ce4581d7
--- /dev/null
+++ b/yt/yql/tests/sql/suites/ql_filter/integer_many_noskiff.cfg
@@ -0,0 +1 @@
+in Input integer.txt
diff --git a/yt/yql/tests/sql/suites/ql_filter/integer_many_noskiff.sql b/yt/yql/tests/sql/suites/ql_filter/integer_many_noskiff.sql
new file mode 100644
index 0000000000..e721fdbca2
--- /dev/null
+++ b/yt/yql/tests/sql/suites/ql_filter/integer_many_noskiff.sql
@@ -0,0 +1,13 @@
+pragma yt.UseQLFilter;
+pragma yt.UseSkiff='false';
+
+select a, c, d, e
+from plato.Input
+where
+ a > 5
+ and
+ c > 5
+ and
+ d > 5
+ and
+ e > 5; \ No newline at end of file
diff --git a/yt/yql/tests/sql/suites/ql_filter/integer_many_right.sql b/yt/yql/tests/sql/suites/ql_filter/integer_many_right.sql
index 56051ba52b..2b95cbe376 100644
--- a/yt/yql/tests/sql/suites/ql_filter/integer_many_right.sql
+++ b/yt/yql/tests/sql/suites/ql_filter/integer_many_right.sql
@@ -1,5 +1,7 @@
+/* yt can not */
+/* waiting for update YT-24048 */
+
pragma yt.UseQLFilter;
-pragma yt.UseSkiff='false'; -- temporary disable skiff https://st.yandex-team.ru/YT-14644
select a, c, d, e
from plato.Input
diff --git a/yt/yql/tests/sql/suites/ql_filter/integer_select_other.sql b/yt/yql/tests/sql/suites/ql_filter/integer_select_other.sql
index bdf2ebbcd2..88d93eba98 100644
--- a/yt/yql/tests/sql/suites/ql_filter/integer_select_other.sql
+++ b/yt/yql/tests/sql/suites/ql_filter/integer_select_other.sql
@@ -1,5 +1,7 @@
+/* yt can not */
+/* waiting for update YT-24048 */
+
pragma yt.UseQLFilter;
-pragma yt.UseSkiff='false'; -- temporary disable skiff https://st.yandex-team.ru/YT-14644
select b
from plato.Input
diff --git a/yt/yql/tools/ytrun/lib/ya.make b/yt/yql/tools/ytrun/lib/ya.make
index 007272ae8e..8f04604931 100644
--- a/yt/yql/tools/ytrun/lib/ya.make
+++ b/yt/yql/tools/ytrun/lib/ya.make
@@ -8,8 +8,11 @@ PEERDIR(
yt/yql/providers/yt/provider
yt/yql/providers/yt/fmr/coordinator/client
yt/yql/providers/yt/fmr/coordinator/impl
+ yt/yql/providers/yt/fmr/job/impl
yt/yql/providers/yt/fmr/job_factory/impl
+ yt/yql/providers/yt/fmr/table_data_service/local
yt/yql/providers/yt/fmr/worker/impl
+ yt/yql/providers/yt/fmr/yt_service/impl
yt/yql/providers/yt/gateway/native
yt/yql/providers/yt/gateway/fmr
yt/yql/providers/yt/lib/config_clusters
diff --git a/yt/yql/tools/ytrun/lib/ytrun_lib.cpp b/yt/yql/tools/ytrun/lib/ytrun_lib.cpp
index dcd374a2c3..724a8d72e1 100644
--- a/yt/yql/tools/ytrun/lib/ytrun_lib.cpp
+++ b/yt/yql/tools/ytrun/lib/ytrun_lib.cpp
@@ -10,7 +10,10 @@
#include <yt/yql/providers/yt/gateway/fmr/yql_yt_fmr.h>
#include <yt/yql/providers/yt/fmr/coordinator/client/yql_yt_coordinator_client.h>
#include <yt/yql/providers/yt/fmr/coordinator/impl/yql_yt_coordinator_impl.h>
+#include <yt/yql/providers/yt/fmr/job/impl/yql_yt_job_impl.h>
#include <yt/yql/providers/yt/fmr/job_factory/impl/yql_yt_job_factory_impl.h>
+#include <yt/yql/providers/yt/fmr/table_data_service/local/table_data_service.h>
+#include <yt/yql/providers/yt/fmr/yt_service/impl/yql_yt_yt_service_impl.h>
#include <yql/essentials/providers/common/provider/yql_provider_names.h>
#include <yql/essentials/core/peephole_opt/yql_opt_peephole_physical.h>
#include <yql/essentials/core/services/yql_transform_pipeline.h>
@@ -197,13 +200,12 @@ IYtGateway::TPtr TYtRunTool::CreateYtGateway() {
}
if (!DisableLocalFmrWorker_) {
- auto func = [&] (NFmr::TTask::TPtr /*task*/, std::shared_ptr<std::atomic<bool>> cancelFlag) {
- while (!cancelFlag->load()) {
- Sleep(TDuration::Seconds(3));
- return NFmr::ETaskStatus::Completed;
- }
- return NFmr::ETaskStatus::Failed;
- }; // TODO - use function which actually calls Downloader/Uploader based on task params
+ auto tableDataService = MakeLocalTableDataService(NFmr::TLocalTableDataServiceSettings(3));
+ auto fmrYtSerivce = NFmr::MakeFmrYtSerivce();
+
+ auto func = [tableDataService, fmrYtSerivce] (NFmr::TTask::TPtr task, std::shared_ptr<std::atomic<bool>> cancelFlag) mutable {
+ return NFmr::RunJob(task, tableDataService, fmrYtSerivce, cancelFlag);
+ };
NFmr::TFmrJobFactorySettings settings{.Function=func};
auto jobFactory = MakeFmrJobFactory(settings);
diff --git a/yt/yt/client/api/rpc_proxy/row_batch_reader.cpp b/yt/yt/client/api/rpc_proxy/row_batch_reader.cpp
index b74ebb5ab2..a3aaf98bec 100644
--- a/yt/yt/client/api/rpc_proxy/row_batch_reader.cpp
+++ b/yt/yt/client/api/rpc_proxy/row_batch_reader.cpp
@@ -18,7 +18,7 @@ TRowBatchReader::TRowBatchReader(
IAsyncZeroCopyInputStreamPtr underlying,
bool isStreamWithStatistics)
: Underlying_(std::move(underlying))
- , Decoder_(CreateWireRowStreamDecoder(NameTable_))
+ , Decoder_(CreateWireRowStreamDecoder(NameTable_, CreateUnlimitedWireProtocolOptions()))
, IsStreamWithStatistics_(isStreamWithStatistics)
{
YT_VERIFY(Underlying_);
diff --git a/yt/yt/client/api/rpc_proxy/wire_row_stream.cpp b/yt/yt/client/api/rpc_proxy/wire_row_stream.cpp
index 12b1dcf2d9..d25eef2c88 100644
--- a/yt/yt/client/api/rpc_proxy/wire_row_stream.cpp
+++ b/yt/yt/client/api/rpc_proxy/wire_row_stream.cpp
@@ -74,8 +74,11 @@ class TWireRowStreamDecoder
: public IRowStreamDecoder
{
public:
- explicit TWireRowStreamDecoder(TNameTablePtr nameTable)
+ explicit TWireRowStreamDecoder(
+ TNameTablePtr nameTable,
+ TWireProtocolOptions wireProtocolOptions = {})
: NameTable_(std::move(nameTable))
+ , WireProtocolOptions_(std::move(wireProtocolOptions))
{
Descriptor_.set_wire_format_version(NApi::NRpcProxy::CurrentWireFormatVersion);
Descriptor_.set_rowset_kind(NApi::NRpcProxy::NProto::RK_UNVERSIONED);
@@ -86,7 +89,7 @@ public:
const NProto::TRowsetDescriptor& descriptorDelta) override
{
struct TWireRowStreamDecoderTag { };
- auto reader = CreateWireProtocolReader(payloadRef, New<TRowBuffer>(TWireRowStreamDecoderTag()));
+ auto reader = CreateWireProtocolReader(payloadRef, New<TRowBuffer>(TWireRowStreamDecoderTag()), WireProtocolOptions_);
auto rows = reader->ReadUnversionedRowset(true);
auto oldNameTableSize = Descriptor_.name_table_entries_size();
@@ -125,18 +128,20 @@ public:
private:
const TNameTablePtr NameTable_;
+ const TWireProtocolOptions WireProtocolOptions_;
NApi::NRpcProxy::NProto::TRowsetDescriptor Descriptor_;
TNameTableToSchemaIdMapping IdMapping_;
bool HasNontrivialIdMapping_ = false;
};
-IRowStreamDecoderPtr CreateWireRowStreamDecoder(TNameTablePtr nameTable)
+IRowStreamDecoderPtr CreateWireRowStreamDecoder(
+ TNameTablePtr nameTable,
+ TWireProtocolOptions wireProtocolOptions)
{
- return New<TWireRowStreamDecoder>(std::move(nameTable));
+ return New<TWireRowStreamDecoder>(std::move(nameTable), std::move(wireProtocolOptions));
}
////////////////////////////////////////////////////////////////////////////////
} // namespace NYT::NApi::NRpcProxy
-
diff --git a/yt/yt/client/api/rpc_proxy/wire_row_stream.h b/yt/yt/client/api/rpc_proxy/wire_row_stream.h
index 528a15fdeb..aff0ff3097 100644
--- a/yt/yt/client/api/rpc_proxy/wire_row_stream.h
+++ b/yt/yt/client/api/rpc_proxy/wire_row_stream.h
@@ -3,13 +3,16 @@
#include "public.h"
#include <yt/yt/client/table_client/public.h>
+#include <yt/yt/client/table_client/wire_protocol.h>
namespace NYT::NApi::NRpcProxy {
////////////////////////////////////////////////////////////////////////////////
IRowStreamEncoderPtr CreateWireRowStreamEncoder(NTableClient::TNameTablePtr nameTable);
-IRowStreamDecoderPtr CreateWireRowStreamDecoder(NTableClient::TNameTablePtr nameTable);
+IRowStreamDecoderPtr CreateWireRowStreamDecoder(
+ NTableClient::TNameTablePtr nameTable,
+ NTableClient::TWireProtocolOptions wireProtocolOptions = {});
////////////////////////////////////////////////////////////////////////////////
diff --git a/yt/yt/client/driver/scheduler_commands.cpp b/yt/yt/client/driver/scheduler_commands.cpp
index 0f0b4081f0..5694b13e46 100644
--- a/yt/yt/client/driver/scheduler_commands.cpp
+++ b/yt/yt/client/driver/scheduler_commands.cpp
@@ -944,6 +944,7 @@ void TGetOperationCommand::Register(TRegistrar registrar)
[] (TThis* command) -> auto& {
return command->Options.IncludeRuntime;
})
+ // COMPAT(ignat): remove this alias.
.Alias("include_scheduler")
.Optional(/*init*/ false);
diff --git a/yt/yt/client/object_client/public.h b/yt/yt/client/object_client/public.h
index 4830121acc..f534c40eac 100644
--- a/yt/yt/client/object_client/public.h
+++ b/yt/yt/client/object_client/public.h
@@ -35,6 +35,7 @@ YT_DEFINE_ERROR_ENUM(
((RequestInvolvesSequoia) (1007))
((RequestInvolvesCypress) (1008))
((BeginCopyDeprecated) (1009))
+ ((PrerequisitePathDifferFromExecutionPaths) (1010))
);
////////////////////////////////////////////////////////////////////////////////
diff --git a/yt/yt/client/table_client/wire_protocol.cpp b/yt/yt/client/table_client/wire_protocol.cpp
index b80de65602..e7b744c91e 100644
--- a/yt/yt/client/table_client/wire_protocol.cpp
+++ b/yt/yt/client/table_client/wire_protocol.cpp
@@ -1055,6 +1055,17 @@ auto IWireProtocolReader::GetSchemaData(const TTableSchema& schema) -> TSchemaDa
////////////////////////////////////////////////////////////////////////////////
+TWireProtocolOptions CreateUnlimitedWireProtocolOptions()
+{
+ return {
+ .MaxStringValueLength = std::numeric_limits<i64>::max(),
+ .MaxAnyValueLength = std::numeric_limits<i64>::max(),
+ .MaxCompositeValueLength = std::numeric_limits<i64>::max(),
+ };
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
std::unique_ptr<IWireProtocolReader> CreateWireProtocolReader(TSharedRef data, TRowBufferPtr rowBuffer, TWireProtocolOptions options)
{
return std::make_unique<TWireProtocolReader>(std::move(data), std::move(rowBuffer), std::move(options));
diff --git a/yt/yt/client/table_client/wire_protocol.h b/yt/yt/client/table_client/wire_protocol.h
index f701aaa49e..b4eb802f51 100644
--- a/yt/yt/client/table_client/wire_protocol.h
+++ b/yt/yt/client/table_client/wire_protocol.h
@@ -288,6 +288,8 @@ struct TWireProtocolOptions
i64 MaxVersionedRowDataWeight = NTableClient::MaxServerVersionedRowDataWeight;
};
+TWireProtocolOptions CreateUnlimitedWireProtocolOptions();
+
////////////////////////////////////////////////////////////////////////////////
//! Creates wire protocol reader.