diff options
| author | Alexander Smirnov <alex@ydb.tech> | 2025-03-21 00:51:44 +0000 |
|---|---|---|
| committer | Alexander Smirnov <alex@ydb.tech> | 2025-03-21 00:51:44 +0000 |
| commit | f3bd49d0fec44e1330640066b138d65a64cb10fc (patch) | |
| tree | 1e31f490331c7f5d7f75bb75cd2dfe4562241fa5 | |
| parent | d7289771993b7b05333228533aced64bbda25301 (diff) | |
| parent | 28c1921440871833b2fcba9e890319d36e5edd96 (diff) | |
| download | ydb-f3bd49d0fec44e1330640066b138d65a64cb10fc.tar.gz | |
Merge branch 'rightlib' into merge-libs-250321-0050
68 files changed, 1648 insertions, 248 deletions
diff --git a/build/conf/docs.conf b/build/conf/docs.conf index 41fca4a6e7..b190556dc7 100644 --- a/build/conf/docs.conf +++ b/build/conf/docs.conf @@ -33,8 +33,8 @@ _DOCS_SOURCES_VALUE= _DOCS_VARS_FLAG= _DOCS_YFM_OUTPUT_FORMAT= -_DOCS_YFM_BOOK_OUTPUT_FORMAT=--output-format html --allowHTML -_DOCS_YFM_LIB_OUTPUT_FORMAT=--output-format md --add-map-file --allow-custom-resources +_DOCS_YFM_BOOK_OUTPUT_FORMAT=-q -f html --allowHTML --no-template +_DOCS_YFM_LIB_OUTPUT_FORMAT=-q -f md --add-map-file --allow-custom-resources -e "@diplodoc/unarchive-includer-extension" _DOCS_YFM_CMDLINE=\ ${cwd:ARCADIA_BUILD_ROOT} $YMAKE_PYTHON3 ${input:"build/scripts/extract_docs.py"} ${hide;input:"build/scripts/process_command_files.py"} --skip-prefix $ARCADIA_BUILD_ROOT --dest-dir $BINDIR/__s ${rootrel:PEERS} \ diff --git a/build/conf/java.conf b/build/conf/java.conf index 04bf4406cd..1e2081fca2 100644 --- a/build/conf/java.conf +++ b/build/conf/java.conf @@ -311,12 +311,12 @@ multimodule JAVA_CONTRIB_PROGRAM { # tag:java-specific JAVA_VCS_MF_ARG= -COMPILE_JAVA_MF=${hide:JAVA_FAKEID} $COMPILE_JAVA $MAVEN_EXPORT_CMD +COMPILE_JAVA_MF=${hide:JAVA_FAKEID} $COMPILE_JAVA && $MAVEN_EXPORT_CMD # tag:java-specific when ($EXT_JAVA_VCS_INFO == "yes") { JAVA_VCS_MF_ARG=--vcs-mf $VCS_JAVA - COMPILE_JAVA_MF=${hide:JAVA_FAKEID} $GENERATE_VCS_JAVA_INFO_NODEP && $COMPILE_JAVA $MAVEN_EXPORT_CMD + COMPILE_JAVA_MF=${hide:JAVA_FAKEID} $GENERATE_VCS_JAVA_INFO_NODEP && $COMPILE_JAVA && $MAVEN_EXPORT_CMD } # tag:java-specific @@ -339,7 +339,8 @@ macro EMBED_JAVA_VCS_INFO() { module EXTERNAL_JAVA_LIBRARY: _BASE_UNIT { .EXTS=.jsrc .java .jar .mf .NODE_TYPE=Bundle - .CMD=COMPILE_JAVA_MF + .CMD=$COMPILE_JAVA_MF + .STRUCT_CMD=yes # TODO(YMAKE-27) all contribs involved into opensource export must be JAVA_CONTRIB modules # and this module must not be intended for export. Such hange will break JBUILD and thus is # blocked on YMAKE-27 @@ -363,10 +364,10 @@ module EXTERNAL_JAVA_LIBRARY: _BASE_UNIT { when ($MAVEN_EXPORT == "yes") { when ($MAVEN_DEPLOY == "yes") { PEERDIR+=build/external_resources/maven - MAVEN_EXPORT_CMD= && $MAVEN_EXPORT_GEN_DEPLST && $MAVEN_EXPORT_GEN_POM && $MAVEN_DEPLOY_CMD + MAVEN_EXPORT_CMD= $MAVEN_EXPORT_GEN_DEPLST && $MAVEN_EXPORT_GEN_POM && $MAVEN_DEPLOY_CMD } otherwise { - MAVEN_EXPORT_CMD= && $MAVEN_EXPORT_GEN_DEPLST && $MAVEN_EXPORT_GEN_POM + MAVEN_EXPORT_CMD= $MAVEN_EXPORT_GEN_DEPLST && $MAVEN_EXPORT_GEN_POM } MAVEN_EXPORT_COORDS_GLOBAL=$MAVEN_EXPORT_GROUP_ID:${MODULE_PREFIX}${REALPRJNAME}:${MAVEN_EXPORT_VERSION}: } diff --git a/build/conf/proto.conf b/build/conf/proto.conf index 9b28aa7e34..e268dc4236 100644 --- a/build/conf/proto.conf +++ b/build/conf/proto.conf @@ -327,6 +327,7 @@ macro USE_COMMON_GOOGLE_APIS(APIS...) { } # tag:go-specific tag:proto +_GO_PROTO_ENV= GO_PROTO_GEN_PLUGINS= GO_PROTO_OPTS= GO_PROTO_OUTS= @@ -361,6 +362,11 @@ GO_PROTOBUF_WELLKNOWN_TYPES=\ vendor/google.golang.org/protobuf/types/known/wrapperspb # tag:go-specific tag:proto +macro _UPDATE_GO_PROTO_ENV(NAME, ENV[], DUMMY...) { + SET_APPEND(_GO_PROTO_ENV $ENV) +} + +# tag:go-specific tag:proto ### @usage: GO_PROTO_PLUGIN(Name Ext Tool [DEPS dependencies...]) ### ### Define protoc plugin for GO with given Name that emits extra output with provided extension @@ -375,15 +381,15 @@ macro GO_PROTO_PLUGIN(NAME, EXT, TOOL, DEPS[]) { GO_PROTO_CMDLINE=${cwd;rootdir;input:File} $YMAKE_PYTHON3 ${input:"build/scripts/go_proto_wrapper.py"} --arcadia-prefix $GO_ARCADIA_PROJECT_PREFIX --contrib-prefix $GO_CONTRIB_PROJECT_PREFIX --namespace ./$PROTO_NAMESPACE $_GO_PROTO_CHECK_OUTPUT --proto ${input;rootrel:File} -- $PROTOC -I=./$PROTO_NAMESPACE -I=$ARCADIA_ROOT/$PROTO_NAMESPACE ${pre=-I=:_PROTO__INCLUDE} -I=$ARCADIA_BUILD_ROOT -I=$PROTOBUF_INCLUDE_PATH $_PROTOC_FLAGS ${hide:PROTO_FAKEID} # tag:go-specific tag:proto -macro _GO_PROTO_CMD_IMPL(File, OPTS...) { - .CMD=$GO_PROTO_CMDLINE $OPTS ${hide;kv:"p PB"} ${hide;kv:"pc yellow"} +macro _GO_PROTO_CMD_IMPL(File, ENV[], OPTS...) { + .CMD=$GO_PROTO_CMDLINE $OPTS ${env:ENV} ${hide;kv:"p PB"} ${hide;kv:"pc yellow"} .PEERDIR=${GO_PROTOBUF_IMPORTS} ${GO_PROTOBUF_WELLKNOWN_TYPES} .ADDINCL=FOR proto ${PROTOBUF_INCLUDE_PATH} } # tag:go-specific tag:proto macro _GO_PROTO_CMD(File) { - .CMD=$_GO_PROTO_CMD_IMPL($File $GO_PROTO_OPTS $GO_PROTO_OUTS) + .CMD=$_GO_PROTO_CMD_IMPL($File $GO_PROTO_OPTS $GO_PROTO_OUTS ENV $_GO_PROTO_ENV) } # tag:proto tag:docs-specific diff --git a/build/external_resources/ymake/public.resources.json b/build/external_resources/ymake/public.resources.json index cc34f839c7..2bc32b5a2d 100644 --- a/build/external_resources/ymake/public.resources.json +++ b/build/external_resources/ymake/public.resources.json @@ -1,19 +1,19 @@ { "by_platform": { "darwin": { - "uri": "sbr:8270821170" + "uri": "sbr:8295444951" }, "darwin-arm64": { - "uri": "sbr:8270820457" + "uri": "sbr:8295443254" }, "linux": { - "uri": "sbr:8270822128" + "uri": "sbr:8295448171" }, "linux-aarch64": { - "uri": "sbr:8270819892" + "uri": "sbr:8295441779" }, "win32": { - "uri": "sbr:8270821739" + "uri": "sbr:8295446553" } } } diff --git a/build/external_resources/ymake/resources.json b/build/external_resources/ymake/resources.json index 62f615a5ed..a880878468 100644 --- a/build/external_resources/ymake/resources.json +++ b/build/external_resources/ymake/resources.json @@ -1,19 +1,19 @@ { "by_platform": { "darwin": { - "uri": "sbr:8270820098" + "uri": "sbr:8295393914" }, "darwin-arm64": { - "uri": "sbr:8270819506" + "uri": "sbr:8295392618" }, "linux": { - "uri": "sbr:8270821295" + "uri": "sbr:8295396316" }, "linux-aarch64": { - "uri": "sbr:8270818963" + "uri": "sbr:8295391136" }, "win32": { - "uri": "sbr:8270820615" + "uri": "sbr:8295395024" } } } diff --git a/build/mapping.conf.json b/build/mapping.conf.json index df078f8c1a..f4277985dd 100644 --- a/build/mapping.conf.json +++ b/build/mapping.conf.json @@ -521,6 +521,7 @@ "8225046385": "{registry_endpoint}/8225046385", "8249001226": "{registry_endpoint}/8249001226", "8273785013": "{registry_endpoint}/8273785013", + "8307046461": "{registry_endpoint}/8307046461", "5486731632": "{registry_endpoint}/5486731632", "5514350352": "{registry_endpoint}/5514350352", "5514360398": "{registry_endpoint}/5514360398", @@ -720,6 +721,7 @@ "8213344477": "{registry_endpoint}/8213344477", "8220716961": "{registry_endpoint}/8220716961", "8270821170": "{registry_endpoint}/8270821170", + "8295444951": "{registry_endpoint}/8295444951", "5766171800": "{registry_endpoint}/5766171800", "5805430761": "{registry_endpoint}/5805430761", "5829025456": "{registry_endpoint}/5829025456", @@ -791,6 +793,7 @@ "8213343657": "{registry_endpoint}/8213343657", "8220716202": "{registry_endpoint}/8220716202", "8270820457": "{registry_endpoint}/8270820457", + "8295443254": "{registry_endpoint}/8295443254", "5766173070": "{registry_endpoint}/5766173070", "5805432830": "{registry_endpoint}/5805432830", "5829031598": "{registry_endpoint}/5829031598", @@ -862,6 +865,7 @@ "8213346286": "{registry_endpoint}/8213346286", "8220717495": "{registry_endpoint}/8220717495", "8270822128": "{registry_endpoint}/8270822128", + "8295448171": "{registry_endpoint}/8295448171", "5766171341": "{registry_endpoint}/5766171341", "5805430188": "{registry_endpoint}/5805430188", "5829023352": "{registry_endpoint}/5829023352", @@ -933,7 +937,9 @@ "8213342652": "{registry_endpoint}/8213342652", "8220715922": "{registry_endpoint}/8220715922", "8270819892": "{registry_endpoint}/8270819892", + "8295441779": "{registry_endpoint}/8295441779", "8270821739": "{registry_endpoint}/8270821739", + "8295446553": "{registry_endpoint}/8295446553", "5766172695": "{registry_endpoint}/5766172695", "5805432230": "{registry_endpoint}/5805432230", "5829029743": "{registry_endpoint}/5829029743", @@ -1811,6 +1817,7 @@ "8225046385": "devtools/ya/test/programs/test_tool/bin/test_tool for linux", "8249001226": "devtools/ya/test/programs/test_tool/bin/test_tool for linux", "8273785013": "devtools/ya/test/programs/test_tool/bin/test_tool for linux", + "8307046461": "devtools/ya/test/programs/test_tool/bin/test_tool for linux", "5486731632": "devtools/ya/test/programs/test_tool/bin3/test_tool3 for linux", "5514350352": "devtools/ya/test/programs/test_tool/bin3/test_tool3 for linux", "5514360398": "devtools/ya/test/programs/test_tool/bin3/test_tool3 for linux", @@ -2010,6 +2017,7 @@ "8213344477": "devtools/ymake/bin/ymake for darwin", "8220716961": "devtools/ymake/bin/ymake for darwin", "8270821170": "devtools/ymake/bin/ymake for darwin", + "8295444951": "devtools/ymake/bin/ymake for darwin", "5766171800": "devtools/ymake/bin/ymake for darwin-arm64", "5805430761": "devtools/ymake/bin/ymake for darwin-arm64", "5829025456": "devtools/ymake/bin/ymake for darwin-arm64", @@ -2081,6 +2089,7 @@ "8213343657": "devtools/ymake/bin/ymake for darwin-arm64", "8220716202": "devtools/ymake/bin/ymake for darwin-arm64", "8270820457": "devtools/ymake/bin/ymake for darwin-arm64", + "8295443254": "devtools/ymake/bin/ymake for darwin-arm64", "5766173070": "devtools/ymake/bin/ymake for linux", "5805432830": "devtools/ymake/bin/ymake for linux", "5829031598": "devtools/ymake/bin/ymake for linux", @@ -2152,6 +2161,7 @@ "8213346286": "devtools/ymake/bin/ymake for linux", "8220717495": "devtools/ymake/bin/ymake for linux", "8270822128": "devtools/ymake/bin/ymake for linux", + "8295448171": "devtools/ymake/bin/ymake for linux", "5766171341": "devtools/ymake/bin/ymake for linux-aarch64", "5805430188": "devtools/ymake/bin/ymake for linux-aarch64", "5829023352": "devtools/ymake/bin/ymake for linux-aarch64", @@ -2223,7 +2233,9 @@ "8213342652": "devtools/ymake/bin/ymake for linux-aarch64", "8220715922": "devtools/ymake/bin/ymake for linux-aarch64", "8270819892": "devtools/ymake/bin/ymake for linux-aarch64", + "8295441779": "devtools/ymake/bin/ymake for linux-aarch64", "8270821739": "devtools/ymake/bin/ymake for win32", + "8295446553": "devtools/ymake/bin/ymake for win32", "5766172695": "devtools/ymake/bin/ymake for win32-clang-cl", "5805432230": "devtools/ymake/bin/ymake for win32-clang-cl", "5829029743": "devtools/ymake/bin/ymake for win32-clang-cl", diff --git a/build/platform/test_tool/host.ya.make.inc b/build/platform/test_tool/host.ya.make.inc index 773bd59e2b..5af9746a41 100644 --- a/build/platform/test_tool/host.ya.make.inc +++ b/build/platform/test_tool/host.ya.make.inc @@ -1,12 +1,12 @@ IF (HOST_OS_DARWIN AND HOST_ARCH_X86_64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8273827378) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8307070450) ELSEIF (HOST_OS_DARWIN AND HOST_ARCH_ARM64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8273825827) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8307068606) ELSEIF (HOST_OS_LINUX AND HOST_ARCH_X86_64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8273829924) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8307073076) ELSEIF (HOST_OS_LINUX AND HOST_ARCH_AARCH64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8273823828) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8307067350) ELSEIF (HOST_OS_WINDOWS AND HOST_ARCH_X86_64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8273828829) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8307071994) ENDIF() diff --git a/build/platform/test_tool/host_os.ya.make.inc b/build/platform/test_tool/host_os.ya.make.inc index c520359f6f..3b960b3a0d 100644 --- a/build/platform/test_tool/host_os.ya.make.inc +++ b/build/platform/test_tool/host_os.ya.make.inc @@ -1,12 +1,12 @@ IF (HOST_OS_DARWIN AND HOST_ARCH_X86_64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8273783446) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8307042769) ELSEIF (HOST_OS_DARWIN AND HOST_ARCH_ARM64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8273782060) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8307040830) ELSEIF (HOST_OS_LINUX AND HOST_ARCH_X86_64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8273785013) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8307046461) ELSEIF (HOST_OS_LINUX AND HOST_ARCH_AARCH64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8273781403) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8307039261) ELSEIF (HOST_OS_WINDOWS AND HOST_ARCH_X86_64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8273784187) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8307044717) ENDIF() diff --git a/build/platform/yfm/resources.json b/build/platform/yfm/resources.json index 15d7d1fe49..afeed809ea 100644 --- a/build/platform/yfm/resources.json +++ b/build/platform/yfm/resources.json @@ -1,16 +1,16 @@ { "by_platform": { "win32-x86_64": { - "uri": "sbr:8283475316" + "uri": "sbr:8307976502" }, "darwin-x86_64": { - "uri": "sbr:8283075371" + "uri": "sbr:8307976295" }, "linux-x86_64": { - "uri": "sbr:8283087538" + "uri": "sbr:8307976001" }, "darwin-arm64": { - "uri": "sbr:8283075371" + "uri": "sbr:8307976295" } } } diff --git a/build/plugins/_dart_fields.py b/build/plugins/_dart_fields.py index 649b3c81b3..6990007804 100644 --- a/build/plugins/_dart_fields.py +++ b/build/plugins/_dart_fields.py @@ -1153,6 +1153,7 @@ class TestFiles: 'grut/libs/bigrt/queue_info_config', 'grut/libs/bigrt/resharder/compute_shard_number', 'grut/libs/bigrt/server', + 'grut/libs/bigrt/testlib', 'grut/libs/bigrt/transaction', 'grut/libs/shooter', ) diff --git a/util/system/spinlock.h b/util/system/spinlock.h index 9e7f603f49..e296d7a034 100644 --- a/util/system/spinlock.h +++ b/util/system/spinlock.h @@ -32,10 +32,18 @@ public: return Val_.compare_exchange_strong(zero, 1); } + inline void Release() noexcept { + Val_.store(0, std::memory_order_release); + } + inline bool try_lock() noexcept { return TryAcquire(); } + inline void unlock() noexcept { + Release(); + } + protected: std::atomic<intptr_t> Val_{0}; }; @@ -58,10 +66,6 @@ class TSpinLock: public TSpinLockBase { public: using TSpinLockBase::TSpinLockBase; - inline void Release() noexcept { - Val_.store(0, std::memory_order_release); - } - inline void Acquire() noexcept { intptr_t zero = 0; if (Val_.compare_exchange_strong(zero, 1)) { @@ -74,10 +78,6 @@ public: !Val_.compare_exchange_strong(zero, 1)); } - inline void unlock() noexcept { - Release(); - } - inline void lock() noexcept { Acquire(); } @@ -95,10 +95,6 @@ class TAdaptiveLock: public TSpinLockBase { public: using TSpinLockBase::TSpinLockBase; - void Release() noexcept { - Val_.store(0, std::memory_order_release); - } - void Acquire() noexcept { intptr_t zero = 0; if (Val_.compare_exchange_strong(zero, 1)) { @@ -117,10 +113,6 @@ public: } } - inline void unlock() noexcept { - Release(); - } - inline void lock() noexcept { Acquire(); } @@ -39,33 +39,33 @@ REGISTRY_ENDPOINT = os.environ.get("YA_REGISTRY_ENDPOINT", "https://devtools-reg PLATFORM_MAP = { "data": { "win32": { - "md5": "73776ce643504ac4700ed8c7675427d1", + "md5": "873a2f0b74705af5bc1bf41217ce49ce", "urls": [ - f"{REGISTRY_ENDPOINT}/8273799916" + f"{REGISTRY_ENDPOINT}/8307066792" ] }, "darwin": { - "md5": "12d621c56756191c34658428ea4dc053", + "md5": "31492187efa40748dd75653a87c40cc1", "urls": [ - f"{REGISTRY_ENDPOINT}/8273798290" + f"{REGISTRY_ENDPOINT}/8307065239" ] }, "darwin-arm64": { - "md5": "2bd8645df61f9e044c6b6ab8b28239c8", + "md5": "07ac8ac75b798223cd71f5c31b10110b", "urls": [ - f"{REGISTRY_ENDPOINT}/8273797061" + f"{REGISTRY_ENDPOINT}/8307063799" ] }, "linux-aarch64": { - "md5": "de531b8fb1c4ab236cdb75a706e7bfc6", + "md5": "0315db8cf713e6b77de53c738bcb3ea7", "urls": [ - f"{REGISTRY_ENDPOINT}/8273795575" + f"{REGISTRY_ENDPOINT}/8307062757" ] }, "linux": { - "md5": "e3e5c44a172c6fb087e744ffcf6fccc3", + "md5": "1287df93b85d867c3fb93e7f9f972b94", "urls": [ - f"{REGISTRY_ENDPOINT}/8273800987" + f"{REGISTRY_ENDPOINT}/8307068169" ] } } diff --git a/yql/essentials/core/yql_expr_type_annotation.cpp b/yql/essentials/core/yql_expr_type_annotation.cpp index dcca732ead..c968b6fea1 100644 --- a/yql/essentials/core/yql_expr_type_annotation.cpp +++ b/yql/essentials/core/yql_expr_type_annotation.cpp @@ -88,6 +88,75 @@ TExprNode::TPtr RebuildVariant(const TExprNode::TPtr& node, return ret; } +bool IsDatetimeToDatetimeCastAllowed(EDataSlot from, EDataSlot to) { + if (from == EDataSlot::Date && (to == EDataSlot::TzDate || + to == EDataSlot::Date32 || + to == EDataSlot::TzDate32 || + to == EDataSlot::Datetime || + to == EDataSlot::TzDatetime || + to == EDataSlot::Datetime64 || + to == EDataSlot::TzDatetime64 || + to == EDataSlot::Timestamp || + to == EDataSlot::TzTimestamp || + to == EDataSlot::Timestamp64 || + to == EDataSlot::TzTimestamp64) + ) { + return true; + } else if (from == EDataSlot::TzDate && (to == EDataSlot::TzDate32 || + to == EDataSlot::TzDatetime || + to == EDataSlot::TzDatetime64 || + to == EDataSlot::TzTimestamp || + to == EDataSlot::TzTimestamp64) + ) { + return true; + } else if (from == EDataSlot::Date32 && (to == EDataSlot::TzDate32 || + to == EDataSlot::Datetime64 || + to == EDataSlot::TzDatetime64 || + to == EDataSlot::Timestamp64 || + to == EDataSlot::TzTimestamp64) + ) { + return true; + } else if (from == EDataSlot::TzDate32 && (to == EDataSlot::TzDatetime64 || + to == EDataSlot::TzTimestamp64) + ) { + return true; + } else if (from == EDataSlot::Datetime && (to == EDataSlot::TzDatetime || + to == EDataSlot::Datetime64 || + to == EDataSlot::TzDatetime64 || + to == EDataSlot::Timestamp || + to == EDataSlot::TzTimestamp || + to == EDataSlot::Timestamp64 || + to == EDataSlot::TzTimestamp64) + ) { + return true; + } else if (from == EDataSlot::TzDatetime && (to == EDataSlot::TzDatetime64 || + to == EDataSlot::TzTimestamp || + to == EDataSlot::TzTimestamp64) + ) { + return true; + } else if (from == EDataSlot::Datetime64 && (to == EDataSlot::TzDatetime64 || + to == EDataSlot::Timestamp64 || + to == EDataSlot::TzTimestamp64) + ) { + return true; + } else if (from == EDataSlot::TzDatetime64 && to == EDataSlot::TzTimestamp64) { + return true; + } else if (from == EDataSlot::Timestamp && (to == EDataSlot::TzTimestamp || + to == EDataSlot::Timestamp64 || + to == EDataSlot::TzTimestamp64) + ) { + return true; + } else if (from == EDataSlot::TzTimestamp && to == EDataSlot::TzTimestamp64) { + return true; + } else if (from == EDataSlot::Timestamp64 && to == EDataSlot::TzTimestamp64) { + return true; + } else if (from == EDataSlot::Interval && (to == EDataSlot::Interval64)) { + return true; + } else { + return false; + } +} + IGraphTransformer::TStatus TryConvertToImpl(TExprContext& ctx, TExprNode::TPtr& node, const TTypeAnnotationNode& sourceType, const TTypeAnnotationNode& expectedType, TConvertFlags flags, bool raiseIssues = false) { @@ -332,56 +401,12 @@ IGraphTransformer::TStatus TryConvertToImpl(TExprContext& ctx, TExprNode::TPtr& if (IsDataTypeNumeric(from) && IsDataTypeNumeric(to)) { allow = GetNumericDataTypeLevel(to) >= GetNumericDataTypeLevel(from); isSafe = false; - } else if (from == EDataSlot::Date && ( - to == EDataSlot::Date32 || - to == EDataSlot::TzDate || - to == EDataSlot::TzDate32 || - to == EDataSlot::Datetime || - to == EDataSlot::Timestamp || - to == EDataSlot::TzDatetime || - to == EDataSlot::TzTimestamp || - to == EDataSlot::Datetime64 || - to == EDataSlot::Timestamp64 || - to == EDataSlot::TzDatetime64 || - to == EDataSlot::TzTimestamp64)) - { - allow = true; - useCast = true; - } else if (from == EDataSlot::Datetime && ( - to == EDataSlot::Datetime64 || - to == EDataSlot::TzDatetime || - to == EDataSlot::TzDatetime64 || - to == EDataSlot::Timestamp || - to == EDataSlot::TzTimestamp || - to == EDataSlot::Timestamp64 || - to == EDataSlot::TzTimestamp64)) + } else if (IsDataTypeDateOrTzDateOrInterval(from) && + IsDataTypeDateOrTzDateOrInterval(to) && + IsDatetimeToDatetimeCastAllowed(from, to)) { allow = true; useCast = true; - } else if (from == EDataSlot::TzDate && (to == EDataSlot::TzDatetime || to == EDataSlot::TzTimestamp)) { - allow = true; - useCast = true; - } else if (from == EDataSlot::TzDatetime && to == EDataSlot::TzTimestamp) { - allow = true; - useCast = true; - } else if (from == EDataSlot::Timestamp && (to == EDataSlot::TzTimestamp || to == EDataSlot::Timestamp64)) { - allow = true; - useCast = true; - } else if (from == EDataSlot::Date32 && (to == EDataSlot::Datetime64 || to == EDataSlot::Timestamp64)) { - allow = true; - useCast = true; - } else if (from == EDataSlot::TzDate32 && (to == EDataSlot::TzDatetime64 || to == EDataSlot::TzTimestamp64)) { - allow = true; - useCast = true; - } else if (from == EDataSlot::Datetime64 && (to == EDataSlot::Timestamp64)) { - allow = true; - useCast = true; - } else if (from == EDataSlot::TzDatetime64 && to == EDataSlot::TzTimestamp64) { - allow = true; - useCast = true; - } else if (from == EDataSlot::Interval && (to == EDataSlot::Interval64)) { - allow = true; - useCast = true; } else if (from == EDataSlot::Json && to == EDataSlot::Utf8) { allow = true; useCast = true; diff --git a/yql/essentials/minikql/comp_nodes/mkql_grace_join_imp.cpp b/yql/essentials/minikql/comp_nodes/mkql_grace_join_imp.cpp index 475e9a012e..5163d77942 100644 --- a/yql/essentials/minikql/comp_nodes/mkql_grace_join_imp.cpp +++ b/yql/essentials/minikql/comp_nodes/mkql_grace_join_imp.cpp @@ -359,10 +359,15 @@ bool TTable::TryToPreallocateMemoryForJoin(TTable & t1, TTable & t2, EJoinKind / tableForPreallocation.NumberOfKeyStringColumns != 0, tableForPreallocation.NumberOfKeyIColumns != 0); const auto slotSize = ComputeNumberOfSlots(tableForPreallocation.TableBucketsStats[bucket].TuplesNum); + bool wasException = false; try { bucketForPreallocation.JoinSlots.reserve(nSlots*slotSize); bucketForPreallocationStats.BloomFilter.Reserve(bucketForPreallocationStats.TuplesNum); } catch (TMemoryLimitExceededException) { + wasException = true; + } + + if (wasException || TlsAllocState->IsMemoryYellowZoneEnabled()) { for (ui64 i = 0; i < bucket; ++i) { auto& b1 = t1.TableBuckets[i]; b1.JoinSlots.resize(0); diff --git a/yql/essentials/public/udf/arrow/block_reader.h b/yql/essentials/public/udf/arrow/block_reader.h index 6652df2ac6..dc3e7be87b 100644 --- a/yql/essentials/public/udf/arrow/block_reader.h +++ b/yql/essentials/public/udf/arrow/block_reader.h @@ -20,6 +20,7 @@ public: virtual TBlockItem GetScalarItem(const arrow::Scalar& scalar) = 0; virtual ui64 GetDataWeight(const arrow::ArrayData& data) const = 0; + virtual ui64 GetSliceDataWeight(const arrow::ArrayData& data, int64_t offset, int64_t length) const = 0; virtual ui64 GetDataWeight(TBlockItem item) const = 0; virtual ui64 GetDefaultValueWeight() const = 0; @@ -33,8 +34,28 @@ struct TBlockItemSerializeProps { bool IsFixed = true; // true if each block item takes fixed size }; +class TBlockReaderBase : public IBlockReader { +public: + ui64 GetSliceDataWeight(const arrow::ArrayData& data, int64_t offset, int64_t length) const final { + Y_ENSURE(0 <= offset && offset < data.length); + Y_ENSURE(offset + length >= offset); + Y_ENSURE(offset + length <= data.length); + return DoGetSliceDataWeight(data, offset, length); + } + +protected: + virtual ui64 DoGetSliceDataWeight(const arrow::ArrayData& data, int64_t offset, int64_t length) const = 0; + + static ui64 GetBitmaskDataWeight(int64_t dataLength) { + if (dataLength <= 0) { + return 0; + } + return (dataLength - 1) / 8 + 1; + } +}; + template<typename T, bool Nullable, typename TDerived> -class TFixedSizeBlockReaderBase : public IBlockReader { +class TFixedSizeBlockReaderBase : public TBlockReaderBase { public: TBlockItem GetItem(const arrow::ArrayData& data, size_t index) final { if constexpr (Nullable) { @@ -66,10 +87,12 @@ public: } ui64 GetDataWeight(const arrow::ArrayData& data) const final { - if constexpr (Nullable) { - return (1 + sizeof(T)) * data.length; - } - return sizeof(T) * data.length; + return GetDataWeightImpl(data.length); + } + + ui64 DoGetSliceDataWeight(const arrow::ArrayData& data, int64_t offset, int64_t length) const final { + Y_UNUSED(data, offset); + return GetDataWeightImpl(length); } ui64 GetDataWeight(TBlockItem item) const final { @@ -111,6 +134,15 @@ public: out.PushNumber(*static_cast<const T*>(arrow::internal::checked_cast<const arrow::internal::PrimitiveScalarBase&>(scalar).data())); } } + +private: + ui64 GetDataWeightImpl(int64_t dataLength) const { + ui64 size = sizeof(T) * dataLength; + if constexpr (Nullable) { + size += GetBitmaskDataWeight(dataLength); + } + return size; + } }; template<typename T, bool Nullable> @@ -132,7 +164,7 @@ public: }; template<typename TStringType, bool Nullable, NKikimr::NUdf::EDataSlot TOriginal = NKikimr::NUdf::EDataSlot::String> -class TStringBlockReader final : public IBlockReader { +class TStringBlockReader final : public TBlockReaderBase { public: using TOffset = typename TStringType::offset_type; @@ -164,12 +196,11 @@ public: } ui64 GetDataWeight(const arrow::ArrayData& data) const final { - ui64 size = 0; - if constexpr (Nullable) { - size += data.length; - } - size += data.buffers[2] ? data.buffers[2]->size() : 0; - return size; + return GetDataWeightImpl(data.length, data.GetValues<TOffset>(1)); + } + + ui64 DoGetSliceDataWeight(const arrow::ArrayData& data, int64_t offset, int64_t length) const final { + return GetDataWeightImpl(length, data.GetValues<TOffset>(1, offset)); } ui64 GetDataWeight(TBlockItem item) const final { @@ -214,10 +245,21 @@ public: std::string_view str(reinterpret_cast<const char*>(buffer->data()), buffer->size()); out.PushString(str); } + +private: + ui64 GetDataWeightImpl(int64_t dataLength, const TOffset* offsets) const { + ui64 size = 0; + if constexpr (Nullable) { + size += GetBitmaskDataWeight(dataLength); + } + size += offsets[dataLength] - offsets[0]; + size += sizeof(TOffset) * dataLength; + return size; + } }; template<bool Nullable, typename TDerived> -class TTupleBlockReaderBase : public IBlockReader { +class TTupleBlockReaderBase : public TBlockReaderBase { public: TBlockItem GetItem(const arrow::ArrayData& data, size_t index) final { if constexpr (Nullable) { @@ -242,13 +284,21 @@ public: ui64 GetDataWeight(const arrow::ArrayData& data) const final { ui64 size = 0; if constexpr (Nullable) { - size += data.length; + size += GetBitmaskDataWeight(data.length); } - size += static_cast<const TDerived*>(this)->GetChildrenDataWeight(data); return size; } + ui64 DoGetSliceDataWeight(const arrow::ArrayData& data, int64_t offset, int64_t length) const final { + ui64 size = 0; + if constexpr (Nullable) { + size += GetBitmaskDataWeight(length); + } + size += static_cast<const TDerived*>(this)->GetChildrenDataWeight(data, offset, length); + return size; + } + ui64 GetDataWeight(TBlockItem item) const final { return static_cast<const TDerived*>(this)->GetDataWeightImpl(item); } @@ -340,6 +390,14 @@ public: return size; } + size_t GetChildrenDataWeight(const arrow::ArrayData& data, int64_t offset, int64_t length) const { + size_t size = 0; + for (ui32 i = 0; i < Children.size(); ++i) { + size += Children[i]->GetSliceDataWeight(*data.child_data[i], offset, length); + } + return size; + } + size_t GetChildrenDefaultDataWeight() const { size_t size = 0; for (ui32 i = 0; i < Children.size(); ++i) { @@ -393,6 +451,15 @@ public: return size; } + size_t GetChildrenDataWeight(const arrow::ArrayData& data, int64_t offset, int64_t length) const { + Y_DEBUG_ABORT_UNLESS(data.child_data.size() == 2); + + size_t size = 0; + size += DateReader_.GetSliceDataWeight(*data.child_data[0], offset, length); + size += TimezoneReader_.GetSliceDataWeight(*data.child_data[1], offset, length); + return size; + } + size_t GetDataWeightImpl(const TBlockItem& item) const { Y_UNUSED(item); return GetChildrenDefaultDataWeight(); @@ -427,7 +494,7 @@ private: // NOTE: For any singular type we use arrow::null() data type. // This data type DOES NOT support bit mask so for optional type // we have to use |TExternalOptional| wrapper. -class TSingularTypeBlockReader: public IBlockReader { +class TSingularTypeBlockReader: public TBlockReaderBase { public: TSingularTypeBlockReader() = default; @@ -448,6 +515,11 @@ public: return 0; } + ui64 DoGetSliceDataWeight(const arrow::ArrayData& data, int64_t offset, int64_t length) const final { + Y_UNUSED(data, offset, length); + return 0; + } + ui64 GetDataWeight(TBlockItem item) const override { Y_UNUSED(item); return 0; @@ -466,7 +538,7 @@ public: } }; -class TExternalOptionalBlockReader final : public IBlockReader { +class TExternalOptionalBlockReader final : public TBlockReaderBase { public: TExternalOptionalBlockReader(std::unique_ptr<IBlockReader>&& inner) : Inner(std::move(inner)) @@ -490,7 +562,11 @@ public: } ui64 GetDataWeight(const arrow::ArrayData& data) const final { - return data.length + Inner->GetDataWeight(*data.child_data.front()); + return GetBitmaskDataWeight(data.length) + Inner->GetDataWeight(*data.child_data.front()); + } + + ui64 DoGetSliceDataWeight(const arrow::ArrayData& data, int64_t offset, int64_t length) const final { + return GetBitmaskDataWeight(length) + Inner->GetSliceDataWeight(*data.child_data.front(), offset, length); } ui64 GetDataWeight(TBlockItem item) const final { diff --git a/yql/essentials/public/udf/arrow/ut/block_reader_ut.cpp b/yql/essentials/public/udf/arrow/ut/block_reader_ut.cpp new file mode 100644 index 0000000000..1ac4c88d65 --- /dev/null +++ b/yql/essentials/public/udf/arrow/ut/block_reader_ut.cpp @@ -0,0 +1,133 @@ +#include <library/cpp/testing/unittest/registar.h> + +#include <yql/essentials/minikql/invoke_builtins/mkql_builtins.h> +#include <yql/essentials/minikql/mkql_function_registry.h> +#include <yql/essentials/minikql/mkql_program_builder.h> +#include <yql/essentials/public/udf/arrow/block_builder.h> +#include <yql/essentials/public/udf/arrow/block_reader.h> +#include <yql/essentials/public/udf/arrow/memory_pool.h> + +namespace NYql::NUdf { + +namespace { + +using namespace NKikimr; + +class TBlockReaderFixture : public NUnitTest::TBaseFixture { + class TArrayHelpers : public TThrRefBase { + public: + using TPtr = TIntrusivePtr<TArrayHelpers>; + + explicit TArrayHelpers(const NMiniKQL::TType* type, arrow::MemoryPool* const arrowPool) + : Builder(MakeArrayBuilder(NMiniKQL::TTypeInfoHelper(), type, *arrowPool, NMiniKQL::CalcBlockLen(CalcMaxBlockItemSize(type)), nullptr)) + , Reader(MakeBlockReader(NMiniKQL::TTypeInfoHelper(), type)) + {} + + public: + const std::unique_ptr<IArrayBuilder> Builder; + const std::unique_ptr<IBlockReader> Reader; + }; + +public: + TBlockReaderFixture() + : FunctionRegistry(CreateFunctionRegistry(NMiniKQL::CreateBuiltinRegistry())) + , Alloc(__LOCATION__) + , Env(Alloc) + , PgmBuilder(Env, *FunctionRegistry) + , ArrowPool(GetYqlMemoryPool()) + {} + + NMiniKQL::TType* OptionaType(NMiniKQL::TType* type) const { + return PgmBuilder.NewOptionalType(type); + } + + template <typename T> + NMiniKQL::TType* DataType() const { + return PgmBuilder.NewDataType(NUdf::TDataType<T>::Id); + } + + NMiniKQL::TType* DataType(NUdf::EDataSlot dataSlot) const { + return PgmBuilder.NewDataType(dataSlot); + } + + template <typename... TArgs> + NMiniKQL::TType* TupleType(TArgs&&... args) const { + return PgmBuilder.NewTupleType({std::forward<TArgs>(args)...}); + } + + TArrayHelpers::TPtr GetArrayHelpers(const NMiniKQL::TType* type) const { + return MakeIntrusive<TArrayHelpers>(type, ArrowPool); + } + +public: + TIntrusivePtr<NMiniKQL::IFunctionRegistry> FunctionRegistry; + NMiniKQL::TScopedAlloc Alloc; + NMiniKQL::TTypeEnvironment Env; + NMiniKQL::TProgramBuilder PgmBuilder; + arrow::MemoryPool* const ArrowPool; +}; + +} // anonymous namespace + +Y_UNIT_TEST_SUITE(BlockReaderTest) { + Y_UNIT_TEST_F(TestLogicalDataSize, TBlockReaderFixture) { + const std::vector arrayHelpers = { + GetArrayHelpers(DataType<ui32>()), + GetArrayHelpers(OptionaType(DataType<char*>())), + GetArrayHelpers(OptionaType(TupleType(OptionaType(DataType<ui32>()), DataType<char*>()))), + GetArrayHelpers(DataType(NUdf::EDataSlot::TzDate)), + GetArrayHelpers(PgmBuilder.NewNullType()) + }; + + constexpr ui32 size = 1000; + constexpr ui32 stringSize = 37; + for (ui32 i = 0; i < size; ++i) { + arrayHelpers[0]->Builder->Add(TBlockItem(i)); + + const auto str = NUnitTest::RandomString(stringSize, i); + arrayHelpers[1]->Builder->Add((i % 2) ? TBlockItem(str) : TBlockItem()); + + TBlockItem tuple[] = { ((i / 2) % 2) ? TBlockItem(i) : TBlockItem(), TBlockItem(str) }; + arrayHelpers[2]->Builder->Add((i % 2) ? TBlockItem(tuple) : TBlockItem()); + + TBlockItem tzDate(i); + tzDate.SetTimezoneId(i % 100); + arrayHelpers[3]->Builder->Add(tzDate); + arrayHelpers[4]->Builder->Add(TBlockItem::Zero()); + } + + std::vector<std::shared_ptr<arrow::ArrayData>> arrays; + arrays.reserve(arrayHelpers.size()); + for (const auto& helper : arrayHelpers) { + arrays.emplace_back(helper->Builder->Build(true).array()); + } + + constexpr ui32 offset = 133; + constexpr ui32 len = 533; + static_assert(offset + len < size); + + constexpr ui64 offsetSize = sizeof(arrow::BinaryType::offset_type) * len; + constexpr ui64 bitmaskSize = (len - 1) / 8 + 1; + constexpr ui64 nonEmptyStrings = (len - offset % 2) / 2 + offset % 2; + const std::vector<ui64> expectedLogicalSize = { + sizeof(ui32) * len, + bitmaskSize + offsetSize + stringSize * nonEmptyStrings, + 2 * bitmaskSize + offsetSize + sizeof(ui32) * len + stringSize * nonEmptyStrings, + (sizeof(ui16) + sizeof(ui16)) * len, + 0 + }; + + // Test GetDataWeight with offset and length + for (ui32 i = 0; i < arrayHelpers.size(); ++i) { + UNIT_ASSERT_VALUES_EQUAL_C(arrayHelpers[i]->Reader->GetSliceDataWeight(*arrays[i], offset, len), expectedLogicalSize[i], "array: " << i); + } + + // Test GetDataWeight after slize + for (ui32 i = 0; i < arrayHelpers.size(); ++i) { + const auto slice = DeepSlice(arrays[i], offset, len); + UNIT_ASSERT_VALUES_EQUAL_C(arrayHelpers[i]->Reader->GetDataWeight(*slice), expectedLogicalSize[i], "sliced array: " << i); + } + } +} + +} // namespace NYql::NUdf diff --git a/yql/essentials/public/udf/arrow/ut/ya.make b/yql/essentials/public/udf/arrow/ut/ya.make index cf4e1a802a..037deb20fc 100644 --- a/yql/essentials/public/udf/arrow/ut/ya.make +++ b/yql/essentials/public/udf/arrow/ut/ya.make @@ -3,6 +3,7 @@ UNITTEST() SRCS( array_builder_ut.cpp bit_util_ut.cpp + block_reader_ut.cpp ) PEERDIR( diff --git a/yql/essentials/public/udf/udf_data_type.cpp b/yql/essentials/public/udf/udf_data_type.cpp index 25eae58a0a..67f31354ce 100644 --- a/yql/essentials/public/udf/udf_data_type.cpp +++ b/yql/essentials/public/udf/udf_data_type.cpp @@ -49,29 +49,29 @@ static const std::array<std::array<std::optional<TCastResultOptions>, DataSlotCo {{ NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, OK, OK, NO, NO, OK, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO }}, // Uuid - {{ NO, MF, MF, MF, OK, OK, OK, OK, OK, OK, OK, OK, OK, NO, NO, NO, OK, OK, OK, NO, OK, OK, OK, NO, NO, NO, OK, OK, OK, NO, NO, NO, NO }}, // Date - {{ NO, MF, MF, MF, MF, MF, OK, OK, OK, OK, LD, OK, OK, NO, NO, NO, LD, OK, OK, NO, LD, OK, OK, NO, NO, NO, LD, OK, OK, NO, NO, NO, NO }}, // Datetime - {{ NO, MF, MF, MF, MF, MF, MF, OK, OK, LD, LD, OK, OK, NO, NO, NO, LD, LD, OK, NO, LD, LD, OK, NO, NO, NO, LD, LD, OK, NO, NO, NO, NO }}, // Timestamp + {{ NO, MF, MF, MF, OK, OK, OK, OK, OK, OK, OK, OK, OK, NO, NO, NO, OK, OK, OK, NO, OK, OK, OK, NO, NO, NO, OK, OK, OK, NO, OK, OK, OK }}, // Date + {{ NO, MF, MF, MF, MF, MF, OK, OK, OK, OK, LD, OK, OK, NO, NO, NO, LD, OK, OK, NO, LD, OK, OK, NO, NO, NO, LD, OK, OK, NO, LD, OK, OK }}, // Datetime + {{ NO, MF, MF, MF, MF, MF, MF, OK, OK, LD, LD, OK, OK, NO, NO, NO, LD, LD, OK, NO, LD, LD, OK, NO, NO, NO, LD, LD, OK, NO, LD, LD, OK }}, // Timestamp {{ NO, MF, MF, MF, MF, MF, MF, OK, MF, LD, LD, OK, OK, NO, NO, NO, NO, NO, NO, OK, NO, NO, NO, NO, NO, NO, NO, NO, NO, OK, NO, NO, NO }}, // Interval - {{ NO, MF, MF, MF, OK, OK, OK, OK, OK, OK, OK, OK, OK, NO, NO, NO, OK, OK, OK, NO, OK, OK, OK, NO, NO, NO, NO, NO, NO, NO, OK, OK, OK }}, // TzDate - {{ NO, MF, MF, MF, MF, MF, OK, OK, OK, OK, LD, OK, OK, NO, NO, NO, LD, OK, OK, NO, LD, OK, OK, NO, NO, NO, NO, NO, NO, NO, LD, OK, OK }}, // TzDatetime - {{ NO, MF, MF, MF, MF, MF, MF, OK, OK, LD, LD, OK, OK, NO, NO, NO, LD, LD, OK, NO, LD, LD, OK, NO, NO, NO, NO, NO, NO, NO, LD, LD, OK }}, // TzTimestamp + {{ NO, MF, MF, MF, OK, OK, OK, OK, OK, OK, OK, OK, OK, NO, NO, NO, LD, LD, LD, NO, OK, OK, OK, NO, NO, NO, LD, LD, LD, NO, OK, OK, OK }}, // TzDate + {{ NO, MF, MF, MF, MF, MF, OK, OK, OK, OK, LD, OK, OK, NO, NO, NO, LD, LD, LD, NO, LD, OK, OK, NO, NO, NO, LD, LD, LD, NO, LD, OK, OK }}, // TzDatetime + {{ NO, MF, MF, MF, MF, MF, MF, OK, OK, LD, LD, OK, OK, NO, NO, NO, LD, LD, LD, NO, LD, LD, OK, NO, NO, NO, LD, LD, LD, NO, LD, LD, OK }}, // TzTimestamp {{ NO, UN, UN, UN, UN, UN, UN, UN, UN, LD, LD, OK, OK, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, UN, NO, NO, NO, NO, NO, NO, NO, NO, NO }}, // Decimal {{ NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, OK, OK, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, OK, NO, NO, NO, NO, NO, NO, NO, NO }}, // DyNumber {{ NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, OK, OK, NO, OK, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, OK, NO, NO, NO, NO, NO, NO, NO }}, // JsonDocument - {{ NO, MF, MF, MF, MF, OK, MF, OK, MF, LD, OK, OK, OK, NO, NO, NO, MF, MF, MF, NO, NO, NO, NO, NO, NO, NO, OK, OK, OK, NO, NO, NO, NO }}, // Date32 - {{ NO, MF, MF, MF, MF, MF, MF, OK, MF, LD, LD, OK, OK, NO, NO, NO, FL, MF, MF, NO, NO, NO, NO, NO, NO, NO, LD, OK, OK, NO, NO, NO, NO }}, // Datetime64 - {{ NO, MF, MF, MF, MF, MF, MF, OK, MF, LD, LD, OK, OK, NO, NO, NO, FL, FL, MF, NO, NO, NO, NO, NO, NO, NO, LD, LD, OK, NO, NO, NO, NO }}, // Timestamp64 + {{ NO, MF, MF, MF, MF, OK, MF, OK, MF, LD, OK, OK, OK, NO, NO, NO, MF, MF, MF, NO, MF, MF, MF, NO, NO, NO, OK, OK, OK, NO, OK, OK, OK }}, // Date32 + {{ NO, MF, MF, MF, MF, MF, MF, OK, MF, LD, LD, OK, OK, NO, NO, NO, FL, MF, MF, NO, FL, MF, MF, NO, NO, NO, LD, OK, OK, NO, LD, OK, OK }}, // Datetime64 + {{ NO, MF, MF, MF, MF, MF, MF, OK, MF, LD, LD, OK, OK, NO, NO, NO, FL, FL, MF, NO, FL, FL, MF, NO, NO, NO, LD, LD, OK, NO, LD, LD, OK }}, // Timestamp64 {{ NO, MF, MF, MF, MF, MF, MF, OK, MF, LD, LD, OK, OK, NO, NO, NO, NO, NO, NO, MF, NO, NO, NO, NO, NO, NO, NO, NO, NO, OK, NO, NO, NO }}, // Interval64 - {{ NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, OK, OK, NO, NO, NO, MF, MF, MF, NO, MF, MF, MF, NO, NO, NO, OK, OK, OK, NO, OK, OK, OK }}, // TzDate32 - {{ NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, OK, OK, NO, NO, NO, MF, MF, MF, NO, MF, MF, MF, NO, NO, NO, LD, OK, OK, NO, LD, OK, OK }}, // TzDatetime64 - {{ NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, OK, OK, NO, NO, NO, MF, MF, MF, NO, MF, MF, MF, NO, NO, NO, LD, LD, OK, NO, LD, LD, OK }}, // TzTimestamp64 + {{ NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, OK, OK, NO, NO, NO, FL, FL, FL, NO, MF, MF, MF, NO, NO, NO, LD, LD, LD, NO, OK, OK, OK }}, // TzDate32 + {{ NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, OK, OK, NO, NO, NO, FL, FL, FL, NO, FL, MF, MF, NO, NO, NO, LD, LD, LD, NO, LD, OK, OK }}, // TzDatetime64 + {{ NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, OK, OK, NO, NO, NO, FL, FL, FL, NO, FL, FL, MF, NO, NO, NO, LD, LD, LD, NO, LD, LD, OK }}, // TzTimestamp64 }}; } diff --git a/yql/essentials/sql/v1/select.cpp b/yql/essentials/sql/v1/select.cpp index 98029daa51..832495bc3c 100644 --- a/yql/essentials/sql/v1/select.cpp +++ b/yql/essentials/sql/v1/select.cpp @@ -132,7 +132,7 @@ public: return false; } } - src->AddDependentSource(Source.Get()); + src->AddDependentSource(Source); } if (Node && WithTables) { TTableList tableList; @@ -497,7 +497,7 @@ public: } if (src) { - src->AddDependentSource(source.Get()); + src->AddDependentSource(source); } if (!source->Init(ctx, src)) { return false; @@ -1274,7 +1274,7 @@ public: } if (src) { - src->AddDependentSource(Source.Get()); + src->AddDependentSource(Source); } if (!Source->Init(ctx, src)) { return false; diff --git a/yql/essentials/sql/v1/source.cpp b/yql/essentials/sql/v1/source.cpp index c2fb13b103..28afe17d9c 100644 --- a/yql/essentials/sql/v1/source.cpp +++ b/yql/essentials/sql/v1/source.cpp @@ -649,7 +649,7 @@ TMaybe<TString> ISource::FindColumnMistype(const TString& name) const { return result ? result : FindMistypeIn(ExprAliases, name); } -void ISource::AddDependentSource(ISource* usedSource) { +void ISource::AddDependentSource(TSourcePtr usedSource) { UsedSources.push_back(usedSource); } diff --git a/yql/essentials/sql/v1/source.h b/yql/essentials/sql/v1/source.h index f89be5dd98..6eb040f2e4 100644 --- a/yql/essentials/sql/v1/source.h +++ b/yql/essentials/sql/v1/source.h @@ -105,7 +105,7 @@ namespace NSQLTranslationV1 { virtual TMaybe<TString> FindColumnMistype(const TString& name) const; virtual bool InitFilters(TContext& ctx); - void AddDependentSource(ISource* usedSource); + void AddDependentSource(TSourcePtr usedSource); bool IsAlias(EExprSeat exprSeat, const TString& label) const; bool IsExprAlias(const TString& label) const; bool IsExprSeat(EExprSeat exprSeat, EExprType type = EExprType::WithExpression) const; @@ -144,7 +144,7 @@ namespace NSQLTranslationV1 { TLegacyHoppingWindowSpecPtr LegacyHoppingWindowSpec; TNodePtr SessionWindow; TNodePtr HoppingWindow; - TVector<ISource*> UsedSources; + TVector<TSourcePtr> UsedSources; TString FlattenMode; bool FlattenColumns = false; THashMap<TString, ui32> GenIndexes; diff --git a/yql/essentials/sql/v1/sql_ut_common.h b/yql/essentials/sql/v1/sql_ut_common.h index 734d19b8a3..36fe641ba6 100644 --- a/yql/essentials/sql/v1/sql_ut_common.h +++ b/yql/essentials/sql/v1/sql_ut_common.h @@ -8485,3 +8485,14 @@ Y_UNIT_TEST_SUITE(OlapPartitionCount) { UNIT_ASSERT_STRING_CONTAINS(res.Issues.ToString(), "PARTITION_COUNT can be used only with STORE=COLUMN"); } } + +Y_UNIT_TEST_SUITE(Crashes) { + Y_UNIT_TEST(IncorrectCorrQuery) { + NYql::TAstParseResult res = SqlToYql(R"sql( + use plato; + SELECT COUNT(DISTINCT EXISTS (SELECT 1 FROM t1 AS t2)) FROM Input AS t1 + )sql"); + + UNIT_ASSERT_C(res.IsOk(), res.Issues.ToString()); + } +} diff --git a/yql/essentials/tests/sql/minirun/part1/canondata/result.json b/yql/essentials/tests/sql/minirun/part1/canondata/result.json index 164e13258b..baec752656 100644 --- a/yql/essentials/tests/sql/minirun/part1/canondata/result.json +++ b/yql/essentials/tests/sql/minirun/part1/canondata/result.json @@ -181,6 +181,20 @@ "uri": "https://{canondata_backend}/1920236/fd3dfd41a8e63d7fa9cb2e8c3d8e6d8d5f51f2cb/resource.tar.gz#test.test_bigdate-common_type-default.txt-Results_/results.txt" } ], + "test.test[bigdate-date_implicit_casts-default.txt-Debug]": [ + { + "checksum": "789f10569635819d64aa27c444319745", + "size": 2504, + "uri": "https://{canondata_backend}/1784117/f78d2455106ce42ce9bf47234459b8ee215bd2a8/resource.tar.gz#test.test_bigdate-date_implicit_casts-default.txt-Debug_/opt.yql" + } + ], + "test.test[bigdate-date_implicit_casts-default.txt-Results]": [ + { + "checksum": "ccd1abdc388d2fb9efc4ad32e92e494b", + "size": 8256, + "uri": "https://{canondata_backend}/1881367/3d3cb0f3b636c2874422b42dbe0654ce256c590a/resource.tar.gz#test.test_bigdate-date_implicit_casts-default.txt-Results_/results.txt" + } + ], "test.test[bigdate-int_literals-default.txt-Debug]": [ { "checksum": "7e213ec4e7df7188baa32a632462cf2c", diff --git a/yql/essentials/tests/sql/minirun/part3/canondata/result.json b/yql/essentials/tests/sql/minirun/part3/canondata/result.json index 9dc8932e18..0771058e6e 100644 --- a/yql/essentials/tests/sql/minirun/part3/canondata/result.json +++ b/yql/essentials/tests/sql/minirun/part3/canondata/result.json @@ -195,6 +195,20 @@ "uri": "https://{canondata_backend}/1881367/948e93b5cacfa4a732bc9d920d6546ab3f25aba9/resource.tar.gz#test.test_bigdate-const_interval64-default.txt-Results_/results.txt" } ], + "test.test[bigdate-date_tz_implicit_casts-default.txt-Debug]": [ + { + "checksum": "af8f65aa083f5cde0a4bb4f078e10bc9", + "size": 1306, + "uri": "https://{canondata_backend}/1936947/d9fc1c1289e163d90287579127b30a378edfb34d/resource.tar.gz#test.test_bigdate-date_tz_implicit_casts-default.txt-Debug_/opt.yql" + } + ], + "test.test[bigdate-date_tz_implicit_casts-default.txt-Results]": [ + { + "checksum": "619f9a0210499a1f34d5209cd7d1d77a", + "size": 3137, + "uri": "https://{canondata_backend}/1942278/584acca05dd7d7e8eec1b57cf050b7123a65c1ca/resource.tar.gz#test.test_bigdate-date_tz_implicit_casts-default.txt-Results_/results.txt" + } + ], "test.test[bigdate-output_interval64-default.txt-Debug]": [ { "checksum": "298089b9382a0a77c9e268c9f0e95b3a", @@ -418,6 +432,20 @@ "uri": "https://{canondata_backend}/1946324/15d61515f724d2d343da00780ff2c6ac72278fb3/resource.tar.gz#test.test_datetime-date_diff_sub-default.txt-Results_/results.txt" } ], + "test.test[datetime-date_tz_implicit_casts-default.txt-Debug]": [ + { + "checksum": "a12a2924c35fe2b445a4dcd5754c3a3e", + "size": 2810, + "uri": "https://{canondata_backend}/1936947/d9fc1c1289e163d90287579127b30a378edfb34d/resource.tar.gz#test.test_datetime-date_tz_implicit_casts-default.txt-Debug_/opt.yql" + } + ], + "test.test[datetime-date_tz_implicit_casts-default.txt-Results]": [ + { + "checksum": "69877e6b31258394712a217a32aa3104", + "size": 9381, + "uri": "https://{canondata_backend}/1942278/584acca05dd7d7e8eec1b57cf050b7123a65c1ca/resource.tar.gz#test.test_datetime-date_tz_implicit_casts-default.txt-Results_/results.txt" + } + ], "test.test[expr-between-default.txt-Debug]": [ { "checksum": "c82b19a5180d2a2ed8230c2cf45ab2cc", diff --git a/yql/essentials/tests/sql/minirun/part7/canondata/result.json b/yql/essentials/tests/sql/minirun/part7/canondata/result.json index 1ed6dbeb71..af489ad7c1 100644 --- a/yql/essentials/tests/sql/minirun/part7/canondata/result.json +++ b/yql/essentials/tests/sql/minirun/part7/canondata/result.json @@ -300,6 +300,20 @@ "uri": "https://{canondata_backend}/1942525/7665133ac15a3f92aef975c99a4dfe3023179a69/resource.tar.gz#test.test_compute_range-huge_in-default.txt-Results_/results.txt" } ], + "test.test[datetime-date_implicit_casts-default.txt-Debug]": [ + { + "checksum": "2cdfd9b9555629a03ccbe417b073bd0d", + "size": 5074, + "uri": "https://{canondata_backend}/1784117/d599636e2d967126fd75f502ca3a5d02b64bf51b/resource.tar.gz#test.test_datetime-date_implicit_casts-default.txt-Debug_/opt.yql" + } + ], + "test.test[datetime-date_implicit_casts-default.txt-Results]": [ + { + "checksum": "43ef32eed9cca4f5bbae8c64716bd496", + "size": 19531, + "uri": "https://{canondata_backend}/1942278/99edd2df721ae9dc3716d646ed769087f6a44079/resource.tar.gz#test.test_datetime-date_implicit_casts-default.txt-Results_/results.txt" + } + ], "test.test[datetime-date_out-default.txt-Debug]": [ { "checksum": "6e830b11f1faaaa8fba8fca98ab9337c", diff --git a/yql/essentials/tests/sql/sql2yql/canondata/result.json b/yql/essentials/tests/sql/sql2yql/canondata/result.json index d4a9e5b804..6580d9eca7 100644 --- a/yql/essentials/tests/sql/sql2yql/canondata/result.json +++ b/yql/essentials/tests/sql/sql2yql/canondata/result.json @@ -1098,6 +1098,13 @@ "uri": "https://{canondata_backend}/1942173/99e88108149e222741552e7e6cddef041d6a2846/resource.tar.gz#test_sql2yql.test_bigdate-const_timestamp64_/sql.yql" } ], + "test_sql2yql.test[bigdate-date_implicit_casts]": [ + { + "checksum": "a83a400d2d37cac5440715cb909f67e6", + "size": 3333, + "uri": "https://{canondata_backend}/1871102/c2e9767e39c8f9e50c0016d8377ff12660471aaf/resource.tar.gz#test_sql2yql.test_bigdate-date_implicit_casts_/sql.yql" + } + ], "test_sql2yql.test[bigdate-date_tz_bounds]": [ { "checksum": "7e6319138c451a9af0fab81161b22c28", @@ -1112,6 +1119,13 @@ "uri": "https://{canondata_backend}/1880306/7ab74c806ebfa825c90d696b230a4fba9939dddd/resource.tar.gz#test_sql2yql.test_bigdate-date_tz_bounds_scale_/sql.yql" } ], + "test_sql2yql.test[bigdate-date_tz_implicit_casts]": [ + { + "checksum": "38f28381b9bdd5cc1e856fafcba7c1ea", + "size": 3243, + "uri": "https://{canondata_backend}/1871102/c2e9767e39c8f9e50c0016d8377ff12660471aaf/resource.tar.gz#test_sql2yql.test_bigdate-date_tz_implicit_casts_/sql.yql" + } + ], "test_sql2yql.test[bigdate-date_tz_impossible_cast]": [ { "checksum": "96261bbea3e8e2d39aa2f6a88862e07a", @@ -2127,6 +2141,13 @@ "uri": "https://{canondata_backend}/1942173/99e88108149e222741552e7e6cddef041d6a2846/resource.tar.gz#test_sql2yql.test_datetime-date_diff_sub_/sql.yql" } ], + "test_sql2yql.test[datetime-date_implicit_casts]": [ + { + "checksum": "de20b1127905d96a5d384d1547419733", + "size": 3589, + "uri": "https://{canondata_backend}/1871102/c2e9767e39c8f9e50c0016d8377ff12660471aaf/resource.tar.gz#test_sql2yql.test_datetime-date_implicit_casts_/sql.yql" + } + ], "test_sql2yql.test[datetime-date_in]": [ { "checksum": "f042635815e0ece3c9aa69a5cc35c44f", @@ -2232,6 +2253,13 @@ "uri": "https://{canondata_backend}/1942173/99e88108149e222741552e7e6cddef041d6a2846/resource.tar.gz#test_sql2yql.test_datetime-date_tz_expand_gmt_/sql.yql" } ], + "test_sql2yql.test[datetime-date_tz_implicit_casts]": [ + { + "checksum": "c56d5f5cf37c97f0ef2a7ca3000a84bc", + "size": 3373, + "uri": "https://{canondata_backend}/1871102/c2e9767e39c8f9e50c0016d8377ff12660471aaf/resource.tar.gz#test_sql2yql.test_datetime-date_tz_implicit_casts_/sql.yql" + } + ], "test_sql2yql.test[datetime-date_tz_impossible_cast]": [ { "checksum": "5199dc2297404c9343dc321e299c0fb4", @@ -8162,6 +8190,11 @@ "uri": "file://test_sql_format.test_bigdate-const_timestamp64_/formatted.sql" } ], + "test_sql_format.test[bigdate-date_implicit_casts]": [ + { + "uri": "file://test_sql_format.test_bigdate-date_implicit_casts_/formatted.sql" + } + ], "test_sql_format.test[bigdate-date_tz_bounds]": [ { "uri": "file://test_sql_format.test_bigdate-date_tz_bounds_/formatted.sql" @@ -8172,6 +8205,11 @@ "uri": "file://test_sql_format.test_bigdate-date_tz_bounds_scale_/formatted.sql" } ], + "test_sql_format.test[bigdate-date_tz_implicit_casts]": [ + { + "uri": "file://test_sql_format.test_bigdate-date_tz_implicit_casts_/formatted.sql" + } + ], "test_sql_format.test[bigdate-date_tz_impossible_cast]": [ { "uri": "file://test_sql_format.test_bigdate-date_tz_impossible_cast_/formatted.sql" @@ -8897,6 +8935,11 @@ "uri": "file://test_sql_format.test_datetime-date_diff_sub_/formatted.sql" } ], + "test_sql_format.test[datetime-date_implicit_casts]": [ + { + "uri": "file://test_sql_format.test_datetime-date_implicit_casts_/formatted.sql" + } + ], "test_sql_format.test[datetime-date_in]": [ { "uri": "file://test_sql_format.test_datetime-date_in_/formatted.sql" @@ -8972,6 +9015,11 @@ "uri": "file://test_sql_format.test_datetime-date_tz_expand_gmt_/formatted.sql" } ], + "test_sql_format.test[datetime-date_tz_implicit_casts]": [ + { + "uri": "file://test_sql_format.test_datetime-date_tz_implicit_casts_/formatted.sql" + } + ], "test_sql_format.test[datetime-date_tz_impossible_cast]": [ { "uri": "file://test_sql_format.test_datetime-date_tz_impossible_cast_/formatted.sql" diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_bigdate-date_implicit_casts_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_bigdate-date_implicit_casts_/formatted.sql new file mode 100644 index 0000000000..e9cd73c0ff --- /dev/null +++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_bigdate-date_implicit_casts_/formatted.sql @@ -0,0 +1,50 @@ +PRAGMA warning('disable', '4528'); + +$fromTypes = { + 'Date32': '2025-03-19', + 'Datetime64': '2025-03-19T01:02:03Z', + 'Timestamp64': '2025-03-19T01:02:03.456789Z', +}; + +$toTypes = [ + 'Date', 'Datetime', 'Timestamp', + 'TzDate', 'TzDatetime', 'TzTimestamp', + 'Date32', 'Datetime64', 'Timestamp64', + 'TzDate32', 'TzDatetime64', 'TzTimestamp64', +]; + +$allowed = { + 'Date32': { + /* - */ /* - */ /* - */ + /* - */ /* - */ /* - */ + /* - */'DateTime64', 'Timestamp64', + 'TzDate32', 'TzDatetime64', 'TzTimestamp64', + }, + 'Datetime64': { + /* - */ /* - */ /* - */ + /* - */ /* - */ /* - */ + /* - */ /* - */'Timestamp64', + + /* - */ 'TzDatetime64', 'TzTimestamp64', + }, + 'Timestamp64': { + /* - */ /* - */ /* - */ + /* - */ /* - */ /* - */ + /* - */ /* - */ /* - */ + /* - */ /* - */'TzTimestamp64', + }, +}; + +EVALUATE FOR $from IN DictItems($fromTypes) DO BEGIN + EVALUATE FOR $to IN $toTypes DO BEGIN + EVALUATE IF DictContains($allowed[$from.0], $to) DO BEGIN + $callable = Callable(CallableType(0, String, DataType($to)), ($x) -> (CAST($x AS String))); + $srcType = DataType($from.0); + + SELECT + $from.0 || ' => ' || $to, + $callable(Unwrap(CAST($from.1 AS $srcType))) + ; + END DO; + END DO; +END DO; diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_bigdate-date_tz_implicit_casts_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_bigdate-date_tz_implicit_casts_/formatted.sql new file mode 100644 index 0000000000..d21f4bbc7d --- /dev/null +++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_bigdate-date_tz_implicit_casts_/formatted.sql @@ -0,0 +1,49 @@ +PRAGMA warning('disable', '4528'); + +$fromTypes = { + 'TzDate32': '2025-03-19,Europe/Moscow', + 'TzDatetime64': '2025-03-19T01:02:03,Europe/Moscow', + 'TzTimestamp64': '2025-03-19T01:02:03.456789,Europe/Moscow', +}; + +$toTypes = [ + 'Date', 'Datetime', 'Timestamp', + 'TzDate', 'TzDatetime', 'TzTimestamp', + 'Date32', 'Datetime64', 'Timestamp64', + 'TzDate32', 'TzDatetime64', 'TzTimestamp64', +]; + +$allowed = { + 'TzDate32': { + /* - */ /* - */ /* - */ + /* - */ /* - */ /* - */ + /* - */ /* - */ /* - */ + /* - */'TzDatetime64', 'TzTimestamp64', + }, + 'TzDatetime64': { + /* - */ /* - */ /* - */ + /* - */ /* - */ /* - */ + /* - */ /* - */ /* - */ + /* - */ /* - */'TzTimestamp64', + }, + 'TzTimestamp64': { + /* - */ /* - */ /* - */ + /* - */ /* - */ /* - */ + /* - */ /* - */ /* - */ + /* - */ /* - */ /* - */ + }, +}; + +EVALUATE FOR $from IN DictItems($fromTypes) DO BEGIN + EVALUATE FOR $to IN $toTypes DO BEGIN + EVALUATE IF DictContains($allowed[$from.0], $to) DO BEGIN + $callable = Callable(CallableType(0, String, DataType($to)), ($x) -> (CAST($x AS String))); + $srcType = DataType($from.0); + + SELECT + $from.0 || ' => ' || $to, + $callable(Unwrap(CAST($from.1 AS $srcType))) + ; + END DO; + END DO; +END DO; diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_datetime-date_implicit_casts_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_datetime-date_implicit_casts_/formatted.sql new file mode 100644 index 0000000000..685eb7ad13 --- /dev/null +++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_datetime-date_implicit_casts_/formatted.sql @@ -0,0 +1,54 @@ +PRAGMA warning('disable', '4528'); + +$fromTypes = { + 'Date': '2025-03-19', + 'Datetime': '2025-03-19T01:02:03Z', + 'Timestamp': '2025-03-19T01:02:03.456789Z', +}; + +$toTypes = [ + 'Date', 'Datetime', 'Timestamp', + 'TzDate', 'TzDatetime', 'TzTimestamp', + 'Date32', 'Datetime64', 'Timestamp64', + 'TzDate32', 'TzDatetime64', 'TzTimestamp64', +]; + +$allowed = { + 'Date': { + /* - */'Datetime', 'Timestamp', + 'TzDate', 'TzDatetime', 'TzTimestamp', + 'Date32', 'DateTime64', 'Timestamp64', + 'TzDate32', 'TzDatetime64', 'TzTimestamp64', + }, + 'Datetime': { + /* - */ /* - */'Timestamp', + + /* - */ 'TzDatetime', 'TzTimestamp', + + /* - */ 'DateTime64', 'Timestamp64', + + /* - */ 'TzDatetime64', 'TzTimestamp64', + }, + 'Timestamp': { + /* - */ /* - */ /* - */ + /* - */ /* - */'TzTimestamp', + + /* - */ /* - */ 'Timestamp64', + + /* - */ /* - */ 'TzTimestamp64', + }, +}; + +EVALUATE FOR $from IN DictItems($fromTypes) DO BEGIN + EVALUATE FOR $to IN $toTypes DO BEGIN + EVALUATE IF DictContains($allowed[$from.0], $to) DO BEGIN + $callable = Callable(CallableType(0, String, DataType($to)), ($x) -> (CAST($x AS String))); + $srcType = DataType($from.0); + + SELECT + $from.0 || ' => ' || $to, + $callable(Unwrap(CAST($from.1 AS $srcType))) + ; + END DO; + END DO; +END DO; diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_datetime-date_tz_implicit_casts_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_datetime-date_tz_implicit_casts_/formatted.sql new file mode 100644 index 0000000000..ea6079a723 --- /dev/null +++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_datetime-date_tz_implicit_casts_/formatted.sql @@ -0,0 +1,51 @@ +PRAGMA warning('disable', '4528'); + +$fromTypes = { + 'TzDate': '2025-03-19,Europe/Moscow', + 'TzDatetime': '2025-03-19T01:02:03,Europe/Moscow', + 'TzTimestamp': '2025-03-19T01:02:03.456789,Europe/Moscow', +}; + +$toTypes = [ + 'Date', 'Datetime', 'Timestamp', + 'TzDate', 'TzDatetime', 'TzTimestamp', + 'Date32', 'Datetime64', 'Timestamp64', + 'TzDate32', 'TzDatetime64', 'TzTimestamp64', +]; + +$allowed = { + 'TzDate': { + /* - */ /* - */ /* - */ + /* - */'TzDatetime', 'TzTimestamp', + + /* - */ /* - */ /* - */ + 'TzDate32', 'TzDatetime64', 'TzTimestamp64', + }, + 'TzDatetime': { + /* - */ /* - */ /* - */ + /* - */ /* - */'TzTimestamp', + + /* - */ /* - */ /* - */ + /* - */ 'TzDatetime64', 'TzTimestamp64', + }, + 'TzTimestamp': { + /* - */ /* - */ /* - */ + /* - */ /* - */ /* - */ + /* - */ /* - */ /* - */ + /* - */ /* - */'TzTimestamp64', + }, +}; + +EVALUATE FOR $from IN DictItems($fromTypes) DO BEGIN + EVALUATE FOR $to IN $toTypes DO BEGIN + EVALUATE IF DictContains($allowed[$from.0], $to) DO BEGIN + $callable = Callable(CallableType(0, String, DataType($to)), ($x) -> (CAST($x AS String))); + $srcType = DataType($from.0); + + SELECT + $from.0 || ' => ' || $to, + $callable(Unwrap(CAST($from.1 AS $srcType))) + ; + END DO; + END DO; +END DO; diff --git a/yql/essentials/tests/sql/suites/bigdate/date_implicit_casts.sql b/yql/essentials/tests/sql/suites/bigdate/date_implicit_casts.sql new file mode 100644 index 0000000000..735893aedd --- /dev/null +++ b/yql/essentials/tests/sql/suites/bigdate/date_implicit_casts.sql @@ -0,0 +1,47 @@ +PRAGMA warning('disable', '4528'); + +$fromTypes = { + "Date32": "2025-03-19", + "Datetime64": "2025-03-19T01:02:03Z", + "Timestamp64": "2025-03-19T01:02:03.456789Z", +}; + +$toTypes = [ + "Date", "Datetime", "Timestamp", + "TzDate", "TzDatetime", "TzTimestamp", + "Date32", "Datetime64", "Timestamp64", + "TzDate32", "TzDatetime64", "TzTimestamp64", +]; + +$allowed = { + "Date32": { + /* - */ /* - */ /* - */ + /* - */ /* - */ /* - */ + /* - */ "DateTime64", "Timestamp64", + "TzDate32", "TzDatetime64", "TzTimestamp64", + }, + "Datetime64": { + /* - */ /* - */ /* - */ + /* - */ /* - */ /* - */ + /* - */ /* - */ "Timestamp64", + /* - */ "TzDatetime64", "TzTimestamp64", + }, + "Timestamp64": { + /* - */ /* - */ /* - */ + /* - */ /* - */ /* - */ + /* - */ /* - */ /* - */ + /* - */ /* - */ "TzTimestamp64", + }, +}; + +EVALUATE FOR $from IN DictItems($fromTypes) DO BEGIN + EVALUATE FOR $to IN $toTypes DO BEGIN + EVALUATE IF DictContains($allowed[$from.0], $to) DO BEGIN + $callable = Callable(CallableType(0, String, DataType($to)), ($x)->(cast($x as String))); + $srcType = DataType($from.0); + SELECT $from.0 || " => " || $to, + $callable(Unwrap(CAST($from.1 as $srcType))) + END DO; + END DO; +END DO; + diff --git a/yql/essentials/tests/sql/suites/bigdate/date_tz_implicit_casts.sql b/yql/essentials/tests/sql/suites/bigdate/date_tz_implicit_casts.sql new file mode 100644 index 0000000000..d7ff05196b --- /dev/null +++ b/yql/essentials/tests/sql/suites/bigdate/date_tz_implicit_casts.sql @@ -0,0 +1,46 @@ +PRAGMA warning('disable', '4528'); + +$fromTypes = { + "TzDate32": "2025-03-19,Europe/Moscow", + "TzDatetime64": "2025-03-19T01:02:03,Europe/Moscow", + "TzTimestamp64": "2025-03-19T01:02:03.456789,Europe/Moscow", +}; + +$toTypes = [ + "Date", "Datetime", "Timestamp", + "TzDate", "TzDatetime", "TzTimestamp", + "Date32", "Datetime64", "Timestamp64", + "TzDate32", "TzDatetime64", "TzTimestamp64", +]; + +$allowed = { + "TzDate32": { + /* - */ /* - */ /* - */ + /* - */ /* - */ /* - */ + /* - */ /* - */ /* - */ + /* - */ "TzDatetime64", "TzTimestamp64", + }, + "TzDatetime64": { + /* - */ /* - */ /* - */ + /* - */ /* - */ /* - */ + /* - */ /* - */ /* - */ + /* - */ /* - */ "TzTimestamp64", + }, + "TzTimestamp64": { + /* - */ /* - */ /* - */ + /* - */ /* - */ /* - */ + /* - */ /* - */ /* - */ + /* - */ /* - */ /* - */ + }, +}; + +EVALUATE FOR $from IN DictItems($fromTypes) DO BEGIN + EVALUATE FOR $to IN $toTypes DO BEGIN + EVALUATE IF DictContains($allowed[$from.0], $to) DO BEGIN + $callable = Callable(CallableType(0, String, DataType($to)), ($x)->(cast($x as String))); + $srcType = DataType($from.0); + SELECT $from.0 || " => " || $to, + $callable(Unwrap(CAST($from.1 as $srcType))) + END DO; + END DO; +END DO; diff --git a/yql/essentials/tests/sql/suites/datetime/date_implicit_casts.sql b/yql/essentials/tests/sql/suites/datetime/date_implicit_casts.sql new file mode 100644 index 0000000000..0030b0811d --- /dev/null +++ b/yql/essentials/tests/sql/suites/datetime/date_implicit_casts.sql @@ -0,0 +1,46 @@ +PRAGMA warning('disable', '4528'); + +$fromTypes = { + "Date": "2025-03-19", + "Datetime": "2025-03-19T01:02:03Z", + "Timestamp": "2025-03-19T01:02:03.456789Z", +}; + +$toTypes = [ + "Date", "Datetime", "Timestamp", + "TzDate", "TzDatetime", "TzTimestamp", + "Date32", "Datetime64", "Timestamp64", + "TzDate32", "TzDatetime64", "TzTimestamp64", +]; + +$allowed = { + "Date": { + /* - */ "Datetime", "Timestamp", + "TzDate", "TzDatetime", "TzTimestamp", + "Date32", "DateTime64", "Timestamp64", + "TzDate32", "TzDatetime64", "TzTimestamp64", + }, + "Datetime": { + /* - */ /* - */ "Timestamp", + /* - */ "TzDatetime", "TzTimestamp", + /* - */ "DateTime64", "Timestamp64", + /* - */ "TzDatetime64", "TzTimestamp64", + }, + "Timestamp": { + /* - */ /* - */ /* - */ + /* - */ /* - */ "TzTimestamp", + /* - */ /* - */ "Timestamp64", + /* - */ /* - */ "TzTimestamp64", + }, +}; + +EVALUATE FOR $from IN DictItems($fromTypes) DO BEGIN + EVALUATE FOR $to IN $toTypes DO BEGIN + EVALUATE IF DictContains($allowed[$from.0], $to) DO BEGIN + $callable = Callable(CallableType(0, String, DataType($to)), ($x)->(cast($x as String))); + $srcType = DataType($from.0); + SELECT $from.0 || " => " || $to, + $callable(Unwrap(CAST($from.1 as $srcType))) + END DO; + END DO; +END DO; diff --git a/yql/essentials/tests/sql/suites/datetime/date_tz_implicit_casts.sql b/yql/essentials/tests/sql/suites/datetime/date_tz_implicit_casts.sql new file mode 100644 index 0000000000..07c646c654 --- /dev/null +++ b/yql/essentials/tests/sql/suites/datetime/date_tz_implicit_casts.sql @@ -0,0 +1,46 @@ +PRAGMA warning('disable', '4528'); + +$fromTypes = { + "TzDate": "2025-03-19,Europe/Moscow", + "TzDatetime": "2025-03-19T01:02:03,Europe/Moscow", + "TzTimestamp": "2025-03-19T01:02:03.456789,Europe/Moscow", +}; + +$toTypes = [ + "Date", "Datetime", "Timestamp", + "TzDate", "TzDatetime", "TzTimestamp", + "Date32", "Datetime64", "Timestamp64", + "TzDate32", "TzDatetime64", "TzTimestamp64", +]; + +$allowed = { + "TzDate": { + /* - */ /* - */ /* - */ + /* - */ "TzDatetime", "TzTimestamp", + /* - */ /* - */ /* - */ + "TzDate32", "TzDatetime64", "TzTimestamp64", + }, + "TzDatetime": { + /* - */ /* - */ /* - */ + /* - */ /* - */ "TzTimestamp", + /* - */ /* - */ /* - */ + /* - */ "TzDatetime64", "TzTimestamp64", + }, + "TzTimestamp": { + /* - */ /* - */ /* - */ + /* - */ /* - */ /* - */ + /* - */ /* - */ /* - */ + /* - */ /* - */ "TzTimestamp64", + }, +}; + +EVALUATE FOR $from IN DictItems($fromTypes) DO BEGIN + EVALUATE FOR $to IN $toTypes DO BEGIN + EVALUATE IF DictContains($allowed[$from.0], $to) DO BEGIN + $callable = Callable(CallableType(0, String, DataType($to)), ($x)->(cast($x as String))); + $srcType = DataType($from.0); + SELECT $from.0 || " => " || $to, + $callable(Unwrap(CAST($from.1 as $srcType))) + END DO; + END DO; +END DO; diff --git a/yql/tools/yqlrun/lib/yqlrun_lib.cpp b/yql/tools/yqlrun/lib/yqlrun_lib.cpp index 3dc35b4f13..5006bd5b40 100644 --- a/yql/tools/yqlrun/lib/yqlrun_lib.cpp +++ b/yql/tools/yqlrun/lib/yqlrun_lib.cpp @@ -5,9 +5,9 @@ #include <yt/yql/providers/yt/gateway/file/yql_yt_file_services.h> #include <yt/yql/providers/yt/gateway/file/yql_yt_file.h> #include <yql/essentials/providers/common/provider/yql_provider_names.h> +#include <yql/essentials/core/cbo/simple/cbo_simple.h> #include <yql/essentials/core/peephole_opt/yql_opt_peephole_physical.h> #include <yql/essentials/core/services/yql_transform_pipeline.h> -#include <yql/essentials/core/cbo/simple/cbo_simple.h> #include <util/generic/yexception.h> #include <util/folder/iterator.h> @@ -110,12 +110,16 @@ TYqlRunTool::TYqlRunTool() AddProviderFactory([this]() -> NYql::TDataProviderInitializer { auto yqlNativeServices = NFile::TYtFileServices::Make(GetFuncRegistry().Get(), TablesMapping_, GetFileStorage(), TmpDir_, KeepTemp_, TablesDirMapping_); auto ytNativeGateway = CreateYtFileGateway(yqlNativeServices); - return GetYtNativeDataProviderInitializer(ytNativeGateway, MakeSimpleCBOOptimizerFactory(), {}); + auto optimizerFactory = CreateCboFactory(); + return GetYtNativeDataProviderInitializer(ytNativeGateway, optimizerFactory, {}); }); SetPeepholePipelineConfigurator(&PEEPHOLE_CONFIG_INSTANCE); } +IOptimizerFactory::TPtr TYqlRunTool::CreateCboFactory() { + return MakeSimpleCBOOptimizerFactory(); +} } // NYql diff --git a/yql/tools/yqlrun/lib/yqlrun_lib.h b/yql/tools/yqlrun/lib/yqlrun_lib.h index bb0e360ade..be33617c62 100644 --- a/yql/tools/yqlrun/lib/yqlrun_lib.h +++ b/yql/tools/yqlrun/lib/yqlrun_lib.h @@ -1,5 +1,6 @@ #pragma once +#include <yql/essentials/core/cbo/cbo_optimizer_new.h> #include <yql/essentials/tools/yql_facade_run/yql_facade_run.h> #include <util/generic/string.h> @@ -11,6 +12,9 @@ class TYqlRunTool: public TFacadeRunner { public: TYqlRunTool(); +protected: + virtual IOptimizerFactory::TPtr CreateCboFactory(); + private: THashMap<TString, TString> TablesMapping_; THashMap<TString, TString> TablesDirMapping_; diff --git a/yql/tools/yqlrun/ya.make b/yql/tools/yqlrun/ya.make index d885e43c44..389865ca64 100644 --- a/yql/tools/yqlrun/ya.make +++ b/yql/tools/yqlrun/ya.make @@ -1,56 +1,10 @@ PROGRAM(yqlrun) -ALLOCATOR(J) - SRCS( yqlrun.cpp ) -IF (OS_LINUX) - # prevent external python extensions to lookup protobuf symbols (and maybe - # other common stuff) in main binary - EXPORTS_SCRIPT(${ARCADIA_ROOT}/yql/essentials/tools/exports.symlist) -ENDIF() - -PEERDIR( - yql/tools/yqlrun/http - yql/tools/yqlrun/lib - - yt/yql/providers/yt/comp_nodes/llvm16 - yt/yql/providers/yt/codec/codegen - - yql/essentials/providers/common/provider - yql/essentials/providers/common/udf_resolve - yql/essentials/minikql/invoke_builtins - yql/essentials/minikql/invoke_builtins/llvm16 - yql/essentials/minikql/comp_nodes/llvm16 - yql/essentials/parser/pg_wrapper - yql/essentials/parser/pg_catalog - yql/essentials/core/services/mounts - yql/essentials/core/facade - yql/essentials/core/pg_ext - yql/essentials/core/file_storage - yql/essentials/core/file_storage/proto - yql/essentials/core - yql/essentials/public/udf/service/exception_policy - yql/essentials/utils/backtrace - yql/essentials/utils/log - yql/essentials/minikql - yql/essentials/protos - yql/essentials/ast - yql/essentials/sql - yql/essentials/sql/pg - yql/essentials/sql/v1 - yql/essentials/sql/v1/lexer/antlr4 - yql/essentials/sql/v1/lexer/antlr4_ansi - yql/essentials/sql/v1/proto_parser/antlr4 - yql/essentials/sql/v1/proto_parser/antlr4_ansi - - library/cpp/getopt - library/cpp/logger - - contrib/libs/protobuf -) +INCLUDE(ya.make.inc) YQL_LAST_ABI_VERSION() diff --git a/yql/tools/yqlrun/ya.make.inc b/yql/tools/yqlrun/ya.make.inc new file mode 100644 index 0000000000..3661f868ee --- /dev/null +++ b/yql/tools/yqlrun/ya.make.inc @@ -0,0 +1,47 @@ +ALLOCATOR(J) + +IF (OS_LINUX) + # prevent external python extensions to lookup protobuf symbols (and maybe + # other common stuff) in main binary + EXPORTS_SCRIPT(${ARCADIA_ROOT}/yql/essentials/tools/exports.symlist) +ENDIF() + +PEERDIR( + yql/tools/yqlrun/http + yql/tools/yqlrun/lib + + yt/yql/providers/yt/comp_nodes/llvm16 + yt/yql/providers/yt/codec/codegen + + yql/essentials/providers/common/provider + yql/essentials/providers/common/udf_resolve + yql/essentials/minikql/invoke_builtins + yql/essentials/minikql/invoke_builtins/llvm16 + yql/essentials/minikql/comp_nodes/llvm16 + yql/essentials/parser/pg_wrapper + yql/essentials/parser/pg_catalog + yql/essentials/core/services/mounts + yql/essentials/core/facade + yql/essentials/core/pg_ext + yql/essentials/core/file_storage + yql/essentials/core/file_storage/proto + yql/essentials/core + yql/essentials/public/udf/service/exception_policy + yql/essentials/utils/backtrace + yql/essentials/utils/log + yql/essentials/minikql + yql/essentials/protos + yql/essentials/ast + yql/essentials/sql + yql/essentials/sql/pg + yql/essentials/sql/v1 + yql/essentials/sql/v1/lexer/antlr4 + yql/essentials/sql/v1/lexer/antlr4_ansi + yql/essentials/sql/v1/proto_parser/antlr4 + yql/essentials/sql/v1/proto_parser/antlr4_ansi + + library/cpp/getopt + library/cpp/logger + + contrib/libs/protobuf +) diff --git a/yt/yql/providers/yt/gateway/file/yql_yt_file_mkql_compiler.cpp b/yt/yql/providers/yt/gateway/file/yql_yt_file_mkql_compiler.cpp index 89525fcbc5..a3dea494c8 100644 --- a/yt/yql/providers/yt/gateway/file/yql_yt_file_mkql_compiler.cpp +++ b/yt/yql/providers/yt/gateway/file/yql_yt_file_mkql_compiler.cpp @@ -916,8 +916,16 @@ void RegisterYtFileMkqlCompilers(NCommon::TMkqlCallableCompilerBase& compiler) { values = arg->GetTypeAnn()->GetKind() == ETypeAnnotationKind::Flow ? ctx.ProgramBuilder.ToFlow(values) : ctx.ProgramBuilder.Iterator(values, {}); - if (ETypeAnnotationKind::Multi == GetSeqItemType(*ytMapReduce.Mapper().Cast<TCoLambda>().Args().Arg(0).Ref().GetTypeAnn()).GetKind()) + auto& lambdaInputType = GetSeqItemType(*ytMapReduce.Mapper().Cast<TCoLambda>().Args().Arg(0).Ref().GetTypeAnn()); + if (lambdaInputType.GetKind() == ETypeAnnotationKind::Multi) { values = ExpandFlow(values, ctx); + } + + if (IsWideBlockType(lambdaInputType)) { + values = ctx.ProgramBuilder.ToFlow( + ctx.ProgramBuilder.WideToBlocks( + ctx.ProgramBuilder.FromFlow(values))); + } NCommon::TMkqlBuildContext innerCtx(ctx, {{arg, values}}, ytMapReduce.Mapper().Ref().UniqueId()); diff --git a/yt/yql/providers/yt/gateway/native/yql_yt_native.cpp b/yt/yql/providers/yt/gateway/native/yql_yt_native.cpp index d0d38b994e..be8ce859a7 100644 --- a/yt/yql/providers/yt/gateway/native/yql_yt_native.cpp +++ b/yt/yql/providers/yt/gateway/native/yql_yt_native.cpp @@ -4131,6 +4131,7 @@ private: const TString& mapInputType, size_t mapDirectOutputs, const TExpressionResorceUsage& mapExtraUsage, + bool mapBlockInput, TString reduceLambda, const TString& reduceInputType, const TExpressionResorceUsage& reduceExtraUsage, @@ -4143,7 +4144,7 @@ private: const bool testRun = execCtx->Config_->GetLocalChainTest(); TFuture<bool> ret = testRun ? MakeFuture<bool>(false) : execCtx->LookupQueryCacheAsync(); return ret.Apply([reduceBy, sortBy, limit, sortLimitBy, mapLambda, mapInputType, mapDirectOutputs, - mapExtraUsage, reduceLambda, reduceInputType, reduceExtraUsage, + mapExtraUsage, mapBlockInput, reduceLambda, reduceInputType, reduceExtraUsage, intermediateMeta, intermediateSchema, intermediateStreams, inputQueryExpr, execCtx, testRun] (const auto& f) mutable { @@ -4250,6 +4251,7 @@ private: mapJob->SetTableNames(tables); mapJob->SetRowOffsets(rowOffsets); mapJob->SetUseSkiff(useSkiff, TMkqlIOSpecs::ESystemField::RowIndex); + mapJob->SetUseBlockInput(mapBlockInput); mapJob->SetYamrInput(execCtx->YamrInput); auto reduceJob = MakeIntrusive<TYqlUserJob>(); @@ -4532,6 +4534,7 @@ private: const bool useNativeTypes = execCtx->Options_.Config()->UseNativeYtTypes.Get().GetOrElse(DEFAULT_USE_NATIVE_YT_TYPES); const auto nativeTypeCompat = execCtx->Options_.Config()->NativeYtTypeCompatibility.Get(execCtx->Cluster_).GetOrElse(NTCF_LEGACY); const auto useIntermediateStreams = execCtx->Options_.Config()->UseIntermediateStreams.Get().GetOrElse(DEFAULT_USE_INTERMEDIATE_STREAMS); + const bool mapBlockInput = NYql::HasSetting(mapReduce.Settings().Ref(), EYtSettingType::BlockInputApplied); NYT::TNode intermediateMeta; NYT::TNode intermediateSchema; @@ -4637,13 +4640,13 @@ private: const TString inputQueryExpr = GenerateInputQueryWhereExpression(mapReduce.Settings().Ref()); - return execCtx->Session_->Queue_->Async([reduceBy, sortBy, limit, sortLimitBy, mapLambda, mapInputType, mapDirectOutputs, mapExtraUsage, + return execCtx->Session_->Queue_->Async([reduceBy, sortBy, limit, sortLimitBy, mapLambda, mapInputType, mapDirectOutputs, mapExtraUsage, mapBlockInput, reduceLambda, reduceInputType, reduceExtraUsage, intermediateMeta, intermediateSchema, intermediateStreams, useIntermediateStreams, inputQueryExpr, execCtx]() { YQL_LOG_CTX_ROOT_SESSION_SCOPE(execCtx->LogCtx_); execCtx->MakeUserFiles(); if (mapLambda) { - return ExecMapReduce(reduceBy, sortBy, limit, sortLimitBy, mapLambda, mapInputType, mapDirectOutputs, mapExtraUsage, + return ExecMapReduce(reduceBy, sortBy, limit, sortLimitBy, mapLambda, mapInputType, mapDirectOutputs, mapExtraUsage, mapBlockInput, reduceLambda, reduceInputType, reduceExtraUsage, intermediateMeta, intermediateSchema, intermediateStreams, inputQueryExpr, execCtx); } else { return ExecMapReduce(reduceBy, sortBy, limit, sortLimitBy, reduceLambda, reduceInputType, reduceExtraUsage, intermediateSchema, diff --git a/yt/yql/providers/yt/provider/yql_yt_block_input.cpp b/yt/yql/providers/yt/provider/yql_yt_block_input.cpp index bda09645b4..5f0f3e0739 100644 --- a/yt/yql/providers/yt/provider/yql_yt_block_input.cpp +++ b/yt/yql/providers/yt/provider/yql_yt_block_input.cpp @@ -2,8 +2,10 @@ #include <yql/essentials/core/yql_opt_utils.h> #include <yql/essentials/providers/common/transform/yql_optimize.h> -#include <yt/yql/providers/yt/common/yql_names.h> #include <yql/essentials/utils/log/log.h> +#include <yt/yql/providers/yt/common/yql_names.h> +#include <yt/yql/providers/yt/provider/yql_yt_block_io_utils.h> +#include <yt/yql/providers/yt/provider/yql_yt_helpers.h> namespace NYql { @@ -22,54 +24,48 @@ public: , State_(std::move(state)) { #define HNDL(name) "YtBlockInput-"#name, Hndl(&TYtBlockInputTransformer::name) - AddHandler(0, &TYtMap::Match, HNDL(TryTransformMap)); + AddHandler(0, &TYtMap::Match, HNDL(TryTransformMap<TYtMap>)); + AddHandler(0, &TYtMapReduce::Match, HNDL(TryTransformMap<TYtMapReduce>)); AddHandler(0, &TYtTableContent::Match, HNDL(TryTransformTableContent)); #undef HNDL } private: + template<typename TYtOpWithMap> TMaybeNode<TExprBase> TryTransformMap(TExprBase node, TExprContext& ctx) const { - auto map = node.Cast<TYtMap>(); + auto op = node.Cast<TYtOpWithMap>(); if ( - NYql::HasSetting(map.Settings().Ref(), EYtSettingType::BlockInputApplied) - || !NYql::HasSetting(map.Settings().Ref(), EYtSettingType::BlockInputReady) - || !CanRewriteMap(map, ctx) + NYql::HasSetting(op.Settings().Ref(), EYtSettingType::BlockInputApplied) + || !NYql::HasSetting(op.Settings().Ref(), EYtSettingType::BlockInputReady) + || !CanRewriteMap(op, ctx) ) { - return map; + return op; } - YQL_CLOG(INFO, ProviderYt) << "Rewrite YtMap with block input"; + YQL_CLOG(INFO, ProviderYt) << "Rewrite " << TYtOpWithMap::CallableName() << " with block input"; - auto settings = RemoveSetting(map.Settings().Ref(), EYtSettingType::BlockInputReady, ctx); + auto settings = RemoveSetting(op.Settings().Ref(), EYtSettingType::BlockInputReady, ctx); settings = AddSetting(*settings, EYtSettingType::BlockInputApplied, TExprNode::TPtr(), ctx); - auto mapperLambda = Build<TCoLambda>(ctx, map.Mapper().Pos()) - .Args({"flow"}) - .Body<TExprApplier>() - .Apply(map.Mapper()) - .With<TCoToFlow>(0) - .Input<TCoWideFromBlocks>() - .Input<TCoFromFlow>() - .Input("flow") - .Build() - .Build() - .Build() - .Build() - .Done() - .Ptr(); - return Build<TYtMap>(ctx, node.Pos()) - .InitFrom(map) + auto mapperLambda = WrapLambdaWithBlockInput(op.Mapper().template Cast<TCoLambda>(), ctx); + return Build<TYtOpWithMap>(ctx, node.Pos()) + .InitFrom(op) .Settings(settings) .Mapper(mapperLambda) .Done(); } - bool CanRewriteMap(const TYtMap& map, TExprContext& ctx) const { - if (auto flowSetting = NYql::GetSetting(map.Settings().Ref(), EYtSettingType::Flow); !flowSetting || flowSetting->ChildrenSize() < 2) { + bool CanRewriteMap(const TYtWithUserJobsOpBase& op, TExprContext& ctx) const { + auto mapLambda = GetMapLambda(op); + if (!mapLambda) { + return false; + } + + if (auto flowSetting = NYql::GetSetting(op.Settings().Ref(), EYtSettingType::Flow); !flowSetting || flowSetting->ChildrenSize() < 2) { return false; } - return EnsureWideFlowType(map.Mapper().Args().Arg(0).Ref(), ctx); + return EnsureWideFlowType(mapLambda.Cast().Args().Arg(0).Ref(), ctx); } TMaybeNode<TExprBase> TryTransformTableContent(TExprBase node, TExprContext& ctx, const TGetParents& getParents) const { diff --git a/yt/yql/providers/yt/provider/yql_yt_block_io_filter.cpp b/yt/yql/providers/yt/provider/yql_yt_block_io_filter.cpp index 3a1e05d9aa..6d1cc92b4d 100644 --- a/yt/yql/providers/yt/provider/yql_yt_block_io_filter.cpp +++ b/yt/yql/providers/yt/provider/yql_yt_block_io_filter.cpp @@ -21,8 +21,9 @@ public: , Finalizer_(std::move(finalizer)) { #define HNDL(name) "YtBlockIOFilter-"#name, Hndl(&YtBlockIOFilterTransformer::name) - AddHandler(0, &TYtMap::Match, HNDL(HandleMapInput)); + AddHandler(0, &TYtMap::Match, HNDL(HandleMapInput<TYtMap>)); AddHandler(0, &TYtMap::Match, HNDL(HandleMapOutput)); + AddHandler(0, &TYtMapReduce::Match, HNDL(HandleMapInput<TYtMapReduce>)); AddHandler(0, &TYtTableContent::Match, HNDL(HandleTableContent)); #undef HNDL } @@ -40,28 +41,29 @@ private: TOptimizeTransformerBase::Rewind(); } + template<typename TYtOpWithMap> TMaybeNode<TExprBase> HandleMapInput(TExprBase node, TExprContext& ctx) const { - auto map = node.Cast<TYtMap>(); - if (NYql::HasSetting(map.Settings().Ref(), EYtSettingType::BlockInputApplied)) { - return map; + auto op = node.Cast<TYtOpWithMap>(); + if (NYql::HasSetting(op.Settings().Ref(), EYtSettingType::BlockInputApplied)) { + return op; } if (!State_->Configuration->JobBlockInput.Get().GetOrElse(Types->UseBlocks)) { - return map; + return op; } - auto settings = map.Settings().Ptr(); - bool canUseBlockInput = CanUseBlockInputForMap(map); + auto settings = op.Settings().Ptr(); + bool canUseBlockInput = CanUseBlockInputForMap(op); bool hasSetting = HasSetting(*settings, EYtSettingType::BlockInputReady); if (canUseBlockInput && !hasSetting) { settings = AddSetting(*settings, EYtSettingType::BlockInputReady, TExprNode::TPtr(), ctx); } else if (!canUseBlockInput && hasSetting) { settings = RemoveSetting(*settings, EYtSettingType::BlockInputReady, ctx); } else { - return map; + return op; } - return Build<TYtMap>(ctx, node.Pos()) - .InitFrom(map) + return Build<TYtOpWithMap>(ctx, node.Pos()) + .InitFrom(op) .Settings(settings) .Done(); } @@ -98,16 +100,21 @@ private: .Done(); } - bool CanUseBlockInputForMap(const TYtMap& map) const { - if (!NYql::HasSetting(map.Settings().Ref(), EYtSettingType::Flow)) { + bool CanUseBlockInputForMap(const TYtWithUserJobsOpBase& op) const { + auto mapLambda = GetMapLambda(op); + if (!mapLambda) { + return false; + } + + if (!NYql::HasSetting(op.Settings().Ref(), EYtSettingType::Flow)) { return false; } - if (map.Input().Size() > 1) { + if (op.Input().Size() > 1) { return false; } - for (auto path : map.Input().Item(0).Paths()) { + for (auto path : op.Input().Item(0).Paths()) { if (!IsYtTableSuitableForArrowInput(path.Table(), [](const TString&) {})) { return false; } @@ -117,7 +124,7 @@ private: auto supportedTypes = State_->Configuration->JobBlockInputSupportedTypes.Get().GetOrElse(DEFAULT_BLOCK_INPUT_SUPPORTED_TYPES); auto supportedDataTypes = State_->Configuration->JobBlockInputSupportedDataTypes.Get().GetOrElse(DEFAULT_BLOCK_INPUT_SUPPORTED_DATA_TYPES); - auto lambdaInputType = map.Mapper().Args().Arg(0).Ref().GetTypeAnn(); + auto lambdaInputType = mapLambda.Cast().Args().Arg(0).Ref().GetTypeAnn(); if (!CheckBlockIOSupportedTypes(*lambdaInputType, supportedTypes, supportedDataTypes, [](const TString&) {}, wideFlowLimit)) { return false; } diff --git a/yt/yql/providers/yt/provider/yql_yt_block_io_utils.cpp b/yt/yql/providers/yt/provider/yql_yt_block_io_utils.cpp index 639fa14ca0..17fbabe3f6 100644 --- a/yt/yql/providers/yt/provider/yql_yt_block_io_utils.cpp +++ b/yt/yql/providers/yt/provider/yql_yt_block_io_utils.cpp @@ -5,6 +5,8 @@ namespace NYql { +using namespace NNodes; + bool CheckBlockIOSupportedTypes( const TTypeAnnotationNode& type, const TSet<TString>& supportedTypes, @@ -42,4 +44,20 @@ bool CheckBlockIOSupportedTypes( return true; } +TCoLambda WrapLambdaWithBlockInput(TCoLambda lambda, TExprContext& ctx) { + return Build<TCoLambda>(ctx, lambda.Pos()) + .Args({"flow"}) + .Body<TExprApplier>() + .Apply(lambda) + .With<TCoToFlow>(0) + .Input<TCoWideFromBlocks>() + .Input<TCoFromFlow>() + .Input("flow") + .Build() + .Build() + .Build() + .Build() + .Done(); +} + } diff --git a/yt/yql/providers/yt/provider/yql_yt_block_io_utils.h b/yt/yql/providers/yt/provider/yql_yt_block_io_utils.h index dc72518fe6..0d55ffbfde 100644 --- a/yt/yql/providers/yt/provider/yql_yt_block_io_utils.h +++ b/yt/yql/providers/yt/provider/yql_yt_block_io_utils.h @@ -1,6 +1,7 @@ #pragma once #include <yql/essentials/ast/yql_expr.h> +#include <yql/essentials/core/expr_nodes/yql_expr_nodes.h> namespace NYql { @@ -13,4 +14,6 @@ bool CheckBlockIOSupportedTypes( bool allowNestedOptionals = true ); +NNodes::TCoLambda WrapLambdaWithBlockInput(NNodes::TCoLambda lambda, TExprContext& ctx); + } diff --git a/yt/yql/providers/yt/provider/yql_yt_datasink_type_ann.cpp b/yt/yql/providers/yt/provider/yql_yt_datasink_type_ann.cpp index 8f899ae110..09f54d5588 100644 --- a/yt/yql/providers/yt/provider/yql_yt_datasink_type_ann.cpp +++ b/yt/yql/providers/yt/provider/yql_yt_datasink_type_ann.cpp @@ -1294,7 +1294,7 @@ private: auto mapReduce = TYtMapReduce(input); - const auto acceptedSettings = EYtSettingType::ReduceBy + auto acceptedSettings = EYtSettingType::ReduceBy | EYtSettingType::ReduceFilterBy | EYtSettingType::SortBy | EYtSettingType::Limit @@ -1306,6 +1306,10 @@ private: | EYtSettingType::ReduceInputType | EYtSettingType::NoDq | EYtSettingType::QLFilter; + + if (hasMapLambda) { + acceptedSettings |= EYtSettingType::BlockInputReady | EYtSettingType::BlockInputApplied; + } if (!ValidateSettings(mapReduce.Settings().Ref(), acceptedSettings, ctx)) { return TStatus::Error; } @@ -1343,11 +1347,12 @@ private: auto itemType = GetInputItemType(mapReduce.Input(), ctx); const auto useFlow = NYql::GetSetting(mapReduce.Settings().Ref(), EYtSettingType::Flow); + const auto blockInputApplied = NYql::GetSetting(mapReduce.Settings().Ref(), EYtSettingType::BlockInputApplied); auto& mapLambda = input->ChildRef(TYtMapReduce::idx_Mapper); TTypeAnnotationNode::TListType mapDirectOutputTypes; if (hasMapLambda) { - const auto mapLambdaInputType = MakeInputType(itemType, useFlow, TExprNode::TPtr(), ctx); + const auto mapLambdaInputType = MakeInputType(itemType, useFlow, blockInputApplied, ctx); if (!UpdateLambdaAllArgumentsTypes(mapLambda, {mapLambdaInputType}, ctx)) { return TStatus::Error; diff --git a/yt/yql/providers/yt/provider/yql_yt_helpers.cpp b/yt/yql/providers/yt/provider/yql_yt_helpers.cpp index 67f6dd127c..4af9b954c1 100644 --- a/yt/yql/providers/yt/provider/yql_yt_helpers.cpp +++ b/yt/yql/providers/yt/provider/yql_yt_helpers.cpp @@ -2328,4 +2328,14 @@ bool IsYtTableSuitableForArrowInput(NNodes::TExprBase tableNode, std::function<v return true; } +TMaybeNode<TCoLambda> GetMapLambda(const TYtWithUserJobsOpBase& op) { + if (auto map = op.Maybe<TYtMap>()) { + return map.Cast().Mapper(); + } else if (auto maybeLambda = op.Maybe<TYtMapReduce>().Mapper().Maybe<TCoLambda>()) { + return maybeLambda.Cast(); + } + + return {}; +} + } // NYql diff --git a/yt/yql/providers/yt/provider/yql_yt_helpers.h b/yt/yql/providers/yt/provider/yql_yt_helpers.h index 3a821b16d9..53e9a071d5 100644 --- a/yt/yql/providers/yt/provider/yql_yt_helpers.h +++ b/yt/yql/providers/yt/provider/yql_yt_helpers.h @@ -143,4 +143,6 @@ bool HasYtRowNumber(const TExprNode& node); bool IsYtTableSuitableForArrowInput(NNodes::TExprBase table, std::function<void(const TString&)> unsupportedHandler); +NNodes::TMaybeNode<NNodes::TCoLambda> GetMapLambda(const NNodes::TYtWithUserJobsOpBase& op); + } diff --git a/yt/yql/tests/sql/suites/blocks/block_input_mapreduce.cfg b/yt/yql/tests/sql/suites/blocks/block_input_mapreduce.cfg new file mode 100644 index 0000000000..426917836b --- /dev/null +++ b/yt/yql/tests/sql/suites/blocks/block_input_mapreduce.cfg @@ -0,0 +1,2 @@ +in Input input_strings.txt +providers yt diff --git a/yt/yql/tests/sql/suites/blocks/block_input_mapreduce.sql b/yt/yql/tests/sql/suites/blocks/block_input_mapreduce.sql new file mode 100644 index 0000000000..d938dce6fc --- /dev/null +++ b/yt/yql/tests/sql/suites/blocks/block_input_mapreduce.sql @@ -0,0 +1,10 @@ +USE plato; + +PRAGMA yt.EnableFuseMapToMapReduce; +PRAGMA yt.JobBlockInput; + +SELECT + key +FROM Input +WHERE key < "100" +GROUP BY key; diff --git a/yt/yql/tests/sql/suites/join/cbo_4tables.sql b/yt/yql/tests/sql/suites/join/cbo_4tables.sql index 1629efa144..256301bf5d 100644 --- a/yt/yql/tests/sql/suites/join/cbo_4tables.sql +++ b/yt/yql/tests/sql/suites/join/cbo_4tables.sql @@ -1,4 +1,3 @@ -/* ytfile can not */ USE plato; pragma warning("disable", "8001"); -- CBO_MISSING_TABLE_STATS diff --git a/yt/yql/tests/sql/suites/join/cbo_7tables.sql b/yt/yql/tests/sql/suites/join/cbo_7tables.sql index 724cb44869..31bf3bdbfa 100644 --- a/yt/yql/tests/sql/suites/join/cbo_7tables.sql +++ b/yt/yql/tests/sql/suites/join/cbo_7tables.sql @@ -1,4 +1,3 @@ -/* ytfile can not */ USE plato; pragma CostBasedOptimizer="native"; diff --git a/yt/yt/client/api/operation_client.cpp b/yt/yt/client/api/operation_client.cpp index 625b55fe61..e4fe36a8bf 100644 --- a/yt/yt/client/api/operation_client.cpp +++ b/yt/yt/client/api/operation_client.cpp @@ -312,6 +312,7 @@ void Serialize(const TJob& job, NYson::IYsonConsumer* consumer, TStringBuf idKey .OptionalItem("job_cookie", job.JobCookie) .OptionalItem("archive_features", job.ArchiveFeatures) .OptionalItem("operation_incarnation", job.OperationIncarnation) + .OptionalItem("allocation_id", job.AllocationId) .EndMap(); } diff --git a/yt/yt/client/api/operation_client.h b/yt/yt/client/api/operation_client.h index 65154c26d9..9653db4336 100644 --- a/yt/yt/client/api/operation_client.h +++ b/yt/yt/client/api/operation_client.h @@ -391,6 +391,7 @@ struct TJob std::optional<ui64> JobCookie; NYson::TYsonString ArchiveFeatures; std::optional<std::string> OperationIncarnation; + std::optional<NScheduler::TAllocationId> AllocationId; std::optional<bool> IsStale; diff --git a/yt/yt/client/api/rpc_proxy/helpers.cpp b/yt/yt/client/api/rpc_proxy/helpers.cpp index 7afdee7fdc..67b0e789de 100644 --- a/yt/yt/client/api/rpc_proxy/helpers.cpp +++ b/yt/yt/client/api/rpc_proxy/helpers.cpp @@ -884,6 +884,7 @@ void ToProto(NProto::TJob* protoJob, const NApi::TJob& job) } YT_OPTIONAL_TO_PROTO(protoJob, monitoring_descriptor, job.MonitoringDescriptor); YT_OPTIONAL_SET_PROTO(protoJob, operation_incarnation, job.OperationIncarnation); + YT_OPTIONAL_TO_PROTO(protoJob, allocation_id, job.AllocationId); } void FromProto(NApi::TJob* job, const NProto::TJob& protoJob) @@ -970,6 +971,11 @@ void FromProto(NApi::TJob* job, const NProto::TJob& protoJob) } job->MonitoringDescriptor = YT_OPTIONAL_FROM_PROTO(protoJob, monitoring_descriptor); job->OperationIncarnation = YT_OPTIONAL_FROM_PROTO(protoJob, operation_incarnation); + if (protoJob.has_allocation_id()) { + job->AllocationId = NScheduler::TAllocationId(FromProto<TGuid>(protoJob.allocation_id())); + } else { + job->AllocationId = {}; + } } void ToProto( diff --git a/yt/yt/client/scheduler/public.h b/yt/yt/client/scheduler/public.h index 3ac72409ef..74a89e058c 100644 --- a/yt/yt/client/scheduler/public.h +++ b/yt/yt/client/scheduler/public.h @@ -22,6 +22,12 @@ extern const TJobTraceId NullJobTraceId; //////////////////////////////////////////////////////////////////////////////// +YT_DEFINE_STRONG_TYPEDEF(TAllocationId, TGuid); + +constexpr TAllocationId NullAllocationId{}; + +//////////////////////////////////////////////////////////////////////////////// + using NJobTrackerClient::TJobId; using NJobTrackerClient::TOperationId; diff --git a/yt/yt/core/misc/slab_allocator.cpp b/yt/yt/core/misc/slab_allocator.cpp index 96cbb89595..3b7d3e043c 100644 --- a/yt/yt/core/misc/slab_allocator.cpp +++ b/yt/yt/core/misc/slab_allocator.cpp @@ -7,6 +7,7 @@ #include <library/cpp/yt/malloc/malloc.h> #include <library/cpp/yt/memory/memory_usage_tracker.h> +#include <library/cpp/yt/memory/poison.h> namespace NYT { @@ -114,6 +115,7 @@ public: { auto* obj = FreeList_.Extract(); if (Y_LIKELY(obj)) { + RecycleFreedMemory(TMutableRef(&obj[1], ObjectSize_ - sizeof(TFreeListItem))); AllocatedItems.Increment(); AliveItems.Update(GetRefCounter(this)->GetRefCount() + 1); // Fast path. @@ -127,7 +129,10 @@ public: { FreedItems.Increment(); AliveItems.Update(GetRefCounter(this)->GetRefCount() - 1); - FreeList_.Put(static_cast<TFreeListItem*>(obj)); + auto* typedPtr = static_cast<TFreeListItem*>(obj); + // Poison all memory except the header used for FreeList_. + PoisonFreedMemory(TMutableRef(&typedPtr[1], ObjectSize_ - sizeof(TFreeListItem))); + FreeList_.Put(typedPtr); Unref(this); } @@ -193,6 +198,7 @@ private: // Build chain of chunks. auto objectCount = ObjectCount_; auto objectSize = ObjectSize_; + auto poisonedSize = objectSize - sizeof(TFreeListItem); YT_VERIFY(objectCount > 0); YT_VERIFY(objectSize > 0); @@ -202,6 +208,7 @@ private: auto* current = reinterpret_cast<TFreeListItem*>(ptr); ptr += objectSize; + PoisonFreedMemory(TMutableRef(¤t[1], poisonedSize)); current->Next.store(reinterpret_cast<TFreeListItem*>(ptr), std::memory_order::release); } @@ -209,6 +216,7 @@ private: auto* current = reinterpret_cast<TFreeListItem*>(ptr); current->Next.store(nullptr, std::memory_order::release); + PoisonFreedMemory(TMutableRef(¤t[1], poisonedSize)); return {head, current}; } @@ -254,6 +262,7 @@ private: // Extract one element. auto* next = head->Next.load(); FreeList_.Put(next, tail); + RecycleFreedMemory(TMutableRef(head, ObjectSize_)); return head; } }; @@ -281,6 +290,7 @@ public: auto itemCount = ++RefCount_; auto ptr = malloc(allocatedSize); + PoisonUninitializedMemory(TMutableRef(ptr, allocatedSize)); auto header = reinterpret_cast<TSizeHeader*>(ptr); header->Size = allocatedSize; @@ -296,6 +306,7 @@ public: ptr = reinterpret_cast<void*>(reinterpret_cast<char*>(ptr) - sizeof(TSizeHeader)); auto allocatedSize = reinterpret_cast<TSizeHeader*>(ptr)->Size; + PoisonFreedMemory(TMutableRef(ptr, allocatedSize)); ReleaseMemory(allocatedSize); free(ptr); FreedItems.Increment(); diff --git a/yt/yt/core/ya.make b/yt/yt/core/ya.make index 81e4e3fe30..a87958125d 100644 --- a/yt/yt/core/ya.make +++ b/yt/yt/core/ya.make @@ -288,6 +288,7 @@ SRCS( ytree/request_complexity_limiter.cpp ytree/request_complexity_limits.cpp ytree/serialize.cpp + ytree/size.cpp ytree/static_service_dispatcher.cpp ytree/system_attribute_provider.cpp ytree/tree_builder.cpp diff --git a/yt/yt/core/ytree/public.h b/yt/yt/core/ytree/public.h index c4bf9e3014..368840528c 100644 --- a/yt/yt/core/ytree/public.h +++ b/yt/yt/core/ytree/public.h @@ -132,4 +132,8 @@ DECLARE_REFCOUNTED_CLASS(TReadRequestComplexityLimiter) //////////////////////////////////////////////////////////////////////////////// +class TSize; + +//////////////////////////////////////////////////////////////////////////////// + } // namespace NYT::NYTree diff --git a/yt/yt/core/ytree/serialize.cpp b/yt/yt/core/ytree/serialize.cpp index b0ebda3871..b3949aef33 100644 --- a/yt/yt/core/ytree/serialize.cpp +++ b/yt/yt/core/ytree/serialize.cpp @@ -1,5 +1,6 @@ #include "serialize.h" +#include "size.h" #include "tree_visitor.h" #include <yt/yt/core/misc/blob.h> @@ -173,6 +174,13 @@ void Serialize(IInputStream& input, IYsonConsumer* consumer) Serialize(TYsonInput(&input), consumer); } +// TSize +void Serialize(const TSize& value, NYson::IYsonConsumer* consumer) +{ + Serialize(value.Underlying(), consumer); +} + + // TStatisticPath. void Serialize(const NStatisticPath::TStatisticPath& path, IYsonConsumer* consumer) { @@ -360,6 +368,21 @@ void Deserialize(TGuid& value, INodePtr node) value = TGuid::FromString(node->AsString()->GetValue()); } +// TSize +void Deserialize(TSize& value, INodePtr node) +{ + if (node->GetType() == ENodeType::Int64) { + value = TSize(node->AsInt64()->GetValue()); + } else if (node->GetType() == ENodeType::Uint64) { + value = TSize(CheckedIntegralCast<i64>(node->AsUint64()->GetValue())); + } else if (node->GetType() == ENodeType::String) { + value = TSize::FromString(node->AsString()->GetValue()); + } else { + THROW_ERROR_EXCEPTION("Cannot parse TSize value from %Qlv", + node->GetType()); + } +} + // TStatisticPath. void Deserialize(NStatisticPath::TStatisticPath& value, INodePtr node) { diff --git a/yt/yt/core/ytree/serialize.h b/yt/yt/core/ytree/serialize.h index 98622249f0..138c77e1d6 100644 --- a/yt/yt/core/ytree/serialize.h +++ b/yt/yt/core/ytree/serialize.h @@ -178,6 +178,8 @@ void Serialize( template <class T, class TTag> void Serialize(const TStrongTypedef<T, TTag>& value, NYson::IYsonConsumer* consumer); +void Serialize(const TSize& value, NYson::IYsonConsumer* consumer); + void Serialize(const NStatisticPath::TStatisticPath& path, NYson::IYsonConsumer* consumer); //////////////////////////////////////////////////////////////////////////////// @@ -285,6 +287,8 @@ void Deserialize( template <class T, class TTag> void Deserialize(TStrongTypedef<T, TTag>& value, INodePtr node); +void Deserialize(TSize& value, INodePtr node); + void Deserialize(NStatisticPath::TStatisticPath& path, INodePtr node); //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/core/ytree/size-inl.h b/yt/yt/core/ytree/size-inl.h new file mode 100644 index 0000000000..6cd360967f --- /dev/null +++ b/yt/yt/core/ytree/size-inl.h @@ -0,0 +1,121 @@ +#ifndef YTREE_SIZE_INL_H_ +#error "Direct inclusion of this file is not allowed, include size.h" +// For the sake of sane code completion. +#include "size.h" +#endif + +namespace NYT::NYTree { + +//////////////////////////////////////////////////////////////////////////////// + +constexpr TSize::TSize() + : Underlying_(0) +{ } + +constexpr TSize::TSize(TUnderlying underlying) + : Underlying_(underlying) +{ } + +constexpr TSize::operator const TUnderlying&() const +{ + return Underlying_; +} + +constexpr TSize::operator TUnderlying&() +{ + return Underlying_; +} + +constexpr TSize::TUnderlying& TSize::Underlying() & +{ + return Underlying_; +} + +constexpr const TSize::TUnderlying& TSize::Underlying() const & +{ + return Underlying_; +} + +constexpr TSize::TUnderlying TSize::Underlying() && +{ + return Underlying_; +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NYTree + +//////////////////////////////////////////////////////////////////////////////// + +namespace std { + +//////////////////////////////////////////////////////////////////////////////// + +template <> +struct hash<NYT::NYTree::TSize> +{ + size_t operator()(const NYT::NYTree::TSize& value) const + { + return std::hash<NYT::NYTree::TSize::TUnderlying>()(value.Underlying()); + } +}; + +//////////////////////////////////////////////////////////////////////////////// + +template <> +class numeric_limits<NYT::NYTree::TSize> +{ +public: + #define XX(name) \ + static constexpr decltype(numeric_limits<NYT::NYTree::TSize::TUnderlying>::name) name = numeric_limits<NYT::NYTree::TSize::TUnderlying>::name; + + XX(is_specialized) + XX(is_signed) + XX(digits) + XX(digits10) + XX(max_digits10) + XX(is_integer) + XX(is_exact) + XX(radix) + XX(min_exponent) + XX(min_exponent10) + XX(max_exponent) + XX(max_exponent10) + XX(has_infinity) + XX(has_quiet_NaN) + XX(has_signaling_NaN) + XX(has_denorm) + XX(has_denorm_loss) + XX(is_iec559) + XX(is_bounded) + XX(is_modulo) + XX(traps) + XX(tinyness_before) + XX(round_style) + + #undef XX + + #define XX(name) \ + static constexpr NYT::NYTree::TSize name() noexcept \ + { \ + return NYT::NYTree::TSize(numeric_limits<NYT::NYTree::TSize::TUnderlying>::name()); \ + } + + XX(min) + XX(max) + XX(lowest) + XX(epsilon) + XX(round_error) + XX(infinity) + XX(quiet_NaN) + XX(signaling_NaN) + XX(denorm_min) + + #undef XX +}; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace std + +//////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/core/ytree/size.cpp b/yt/yt/core/ytree/size.cpp new file mode 100644 index 0000000000..ec29ebb9a4 --- /dev/null +++ b/yt/yt/core/ytree/size.cpp @@ -0,0 +1,102 @@ +#include "size.h" + +#include <yt/yt/core/misc/error.h> + +#include <util/string/cast.h> + +namespace NYT { + +namespace NYTree { + +//////////////////////////////////////////////////////////////////////////////// + +namespace { + +//////////////////////////////////////////////////////////////////////////////// + +// Use suffixes for 1000, ..., 1000 ** 6 and 1024, ..., 1024 ** 6. +// 1000 ** 6 < 1024 ** 6 = 2 ** 60 < std::numeric_limits<TSize::TUnderlying>::max(). +constexpr int MaxMultiplierOrder = 6; +using TMultipliers = std::array<std::array<TSize::TUnderlying, MaxMultiplierOrder + 1>, 2>; + +constexpr TMultipliers Multipliers = std::invoke([] { + TMultipliers result; + result[0][0] = 1; + result[1][0] = 1; + for (int i = 1; i <= MaxMultiplierOrder; ++i) { + result[0][i] = result[0][i - 1] * 1000; + result[1][i] = result[1][i - 1] * 1024; + } + return result; +}); + +TSize::TUnderlying DeserializeSizeWithSuffixesImpl(TStringBuf originalValue) +{ + TStringBuf value = originalValue; + + bool basedOnPowerOf2 = value.ChopSuffix("i"); + int order = + value.ChopSuffix("K") ? 1 : + value.ChopSuffix("M") ? 2 : + value.ChopSuffix("G") ? 3 : + value.ChopSuffix("T") ? 4 : + value.ChopSuffix("P") ? 5 : + value.ChopSuffix("E") ? 6 : 0; + + TSize::TUnderlying multiplier = Multipliers[static_cast<int>(basedOnPowerOf2)][order]; + TSize::TUnderlying result = FromString<TSize::TUnderlying>(value); + + bool tooLargeValue = result < 0 + ? result < std::numeric_limits<TSize::TUnderlying>::lowest() / multiplier + : result > std::numeric_limits<TSize::TUnderlying>::max() / multiplier; + THROW_ERROR_EXCEPTION_IF(tooLargeValue, "Cannot parse too large value %Qlv as 64-bit integral type", originalValue); + + return result * multiplier; +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace + +//////////////////////////////////////////////////////////////////////////////// + +TSize TSize::FromString(TStringBuf serializedValue) +{ + return TSize(DeserializeSizeWithSuffixesImpl(serializedValue)); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NYTree + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT + +//////////////////////////////////////////////////////////////////////////////// + +template <> +NYT::NYTree::TSize FromStringImpl<NYT::NYTree::TSize, char>(const char* data, size_t size) +{ + return NYT::NYTree::TSize::FromString(TStringBuf(data, size)); +} + +template<> +bool TryFromStringImpl<NYT::NYTree::TSize, char>(const char* data, size_t size, NYT::NYTree::TSize& value) +{ + try { + value = NYT::NYTree::TSize::FromString(TStringBuf(data, size)); + return true; + } catch (const std::exception&) { + return false; + } +} + +//////////////////////////////////////////////////////////////////////////////// + +template <> +void Out<NYT::NYTree::TSize>(IOutputStream& out, const NYT::NYTree::TSize& value) { + out << value.Underlying(); +} + +//////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/core/ytree/size.h b/yt/yt/core/ytree/size.h new file mode 100644 index 0000000000..1f15c9c68b --- /dev/null +++ b/yt/yt/core/ytree/size.h @@ -0,0 +1,48 @@ +#pragma once + +#include "public.h" + +namespace NYT::NYTree { + +//////////////////////////////////////////////////////////////////////////////// + +//! Semi-strong typedef for integral type with advanced parsing from string. +//! Constructor/FromString<TSize> parses "2M" as 2'000'000 and "1Ki" as 1024. +//! Supported suffixes: K, Ki, M, Mi, G, Gi, T, Ti, P, Pi, E, Ei. +class TSize +{ +public: + using TUnderlying = i64; + + constexpr TSize(); + + constexpr explicit TSize(TUnderlying value); + + TSize(const TSize&) = default; + TSize(TSize&&) = default; + + static TSize FromString(TStringBuf serializedValue); + + TSize& operator=(const TSize&) = default; + TSize& operator=(TSize&&) = default; + + constexpr operator const TUnderlying&() const; + constexpr operator TUnderlying&(); + + constexpr auto operator<=>(const TSize& rhs) const = default; + + constexpr TUnderlying& Underlying() &; + constexpr const TUnderlying& Underlying() const &; + constexpr TUnderlying Underlying() &&; + +private: + TUnderlying Underlying_; +}; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NYTree + +#define YTREE_SIZE_INL_H_ +#include "size-inl.h" +#undef YTREE_SIZE_INL_H_ diff --git a/yt/yt/core/ytree/unittests/size_ut.cpp b/yt/yt/core/ytree/unittests/size_ut.cpp new file mode 100644 index 0000000000..77a407b947 --- /dev/null +++ b/yt/yt/core/ytree/unittests/size_ut.cpp @@ -0,0 +1,213 @@ +#include <yt/yt/core/test_framework/framework.h> + +#include <yt/yt/core/ytree/size.h> + +namespace NYT::NYTree { +namespace { + +//////////////////////////////////////////////////////////////////////////////// + +#define XX(name) \ + static_assert(std::numeric_limits<TSize>::name == std::numeric_limits<i64>::name); + +XX(is_signed) +XX(digits) +XX(digits10) +XX(max_digits10) +XX(is_integer) +XX(is_exact) +XX(radix) +XX(min_exponent) +XX(min_exponent10) +XX(max_exponent) +XX(max_exponent10) +XX(has_infinity) +XX(has_quiet_NaN) +XX(has_signaling_NaN) +XX(has_denorm) +XX(has_denorm_loss) +XX(is_iec559) +XX(is_bounded) +XX(is_modulo) +XX(traps) +XX(tinyness_before) +XX(round_style) + +#undef XX + +#define XX(name) \ + static_assert(std::numeric_limits<TSize>::name() == TSize(std::numeric_limits<i64>::name())); + +XX(min) +XX(max) +XX(lowest) +XX(epsilon) +XX(round_error) +XX(infinity) +XX(quiet_NaN) +XX(signaling_NaN) +XX(denorm_min) + +#undef XX + +//////////////////////////////////////////////////////////////////////////////// + +TEST(TYTreeSizeTest, Simple) +{ + EXPECT_EQ(TSize(1), TSize(1)); + EXPECT_NE(TSize(1), TSize(2)); + + EXPECT_EQ(TSize(1), 1); + EXPECT_LE(0, TSize(1)); + EXPECT_GE(TSize(1), 0); + EXPECT_EQ(1u, TSize(1)); + + EXPECT_TRUE(TSize(1)); + EXPECT_FALSE(TSize(0)); + + EXPECT_EQ(TSize(10) + 100, 110); + EXPECT_EQ(100 + TSize(10), 110); + EXPECT_EQ(10 * TSize(10), 100); + EXPECT_EQ(TSize(10) * 10, 100); + EXPECT_EQ(TSize(10) / 10, 1); + EXPECT_EQ(10 / TSize(10), 1); + EXPECT_EQ(10 - TSize(10), 0); + EXPECT_EQ(TSize(10) - 10, 0); + + { + EXPECT_EQ(TSize(10).Underlying(), 10); + + TSize a{56}; + a.Underlying() = 55; + EXPECT_EQ(a.Underlying(), 55); + + const TSize b{58}; + static_assert(std::is_const_v<std::remove_reference_t<decltype(b.Underlying())>>); + EXPECT_EQ(b.Underlying(), 58); + EXPECT_EQ(b, 58); + } +} + +TEST(TYTreeSizeTest, Serialize) +{ + EXPECT_EQ(ToString(TSize(1)), TString{"1"}); + EXPECT_EQ(Format("|%v|", TSize(42)), TString{"|42|"}); + + { + TSize v; + EXPECT_TRUE(::TryFromString("21", 2, v)); + EXPECT_EQ(TSize(21), v); + EXPECT_FALSE(::TryFromString("", 0, v)); + } + + EXPECT_EQ(TSize(46), FromString<TSize>("46")); + + { + TStringBuilder builder; + Format(&builder, "%v", TSize(42)); + EXPECT_EQ(builder.Flush(), "42"); + } + + { + TStringStream stream; + stream << TSize(42); + EXPECT_EQ(stream.Str(), "42"); + } +} + +TEST(TYTreeSizeTest, Constexpr) +{ + static_assert(TSize(1) == TSize(1)); + static_assert(TSize(1000).Underlying() == 1000); +} + +TEST(TYTreeSizeTest, Hash) +{ + for (i64 i = -1000; i < 1000; ++i) { + EXPECT_EQ(THash<TSize>()(TSize(i)), THash<i64>()(i)); + EXPECT_EQ(std::hash<TSize>()(TSize(i)), std::hash<i64>()(i)); + } +} + +i64 DeserializeString(TStringBuf value) +{ + return TSize::FromString(value).Underlying(); +} + +TEST(TYTreeSizeTest, FromString) +{ + EXPECT_EQ(0, DeserializeString("0")); + EXPECT_EQ(127, DeserializeString("127")); + EXPECT_EQ(-2, DeserializeString("-2")); + + EXPECT_EQ(1000, DeserializeString("1K")); + EXPECT_EQ(3000000, DeserializeString("3M")); + EXPECT_EQ(-1024, DeserializeString("-1Ki")); + EXPECT_EQ(2000000000, DeserializeString("2G")); + EXPECT_THROW(DeserializeString("2000000000000P"), std::exception); + + EXPECT_EQ(0, DeserializeString("0E")); + EXPECT_EQ(1000, DeserializeString("1K")); + EXPECT_EQ(1000'000, DeserializeString("1M")); + EXPECT_EQ(1000'000'000, DeserializeString("1G")); + EXPECT_EQ(1000'000'000'000, DeserializeString("1T")); + EXPECT_EQ(1000'000'000'000'000, DeserializeString("1P")); + EXPECT_EQ(1000'000'000'000'000'000, DeserializeString("1E")); + EXPECT_EQ(0LL, DeserializeString("0Ei")); + EXPECT_EQ(1024LL, DeserializeString("1Ki")); + EXPECT_EQ(1024LL * 1024, DeserializeString("1Mi")); + EXPECT_EQ(1024LL * 1024 * 1024, DeserializeString("1Gi")); + EXPECT_EQ(1024LL * 1024 * 1024 * 1024, DeserializeString("1Ti")); + EXPECT_EQ(1024LL * 1024 * 1024 * 1024 * 1024, DeserializeString("1Pi")); + EXPECT_EQ(1024LL * 1024 * 1024 * 1024 * 1024 * 1024, DeserializeString("1Ei")); + + EXPECT_THROW(DeserializeString("G"), std::exception); + EXPECT_THROW(DeserializeString("Gi"), std::exception); + EXPECT_THROW(DeserializeString("1GG"), std::exception); + EXPECT_THROW(DeserializeString("1KG"), std::exception); + EXPECT_THROW(DeserializeString("1KiG"), std::exception); + EXPECT_THROW(DeserializeString("1Ki2"), std::exception); +} + +TEST(TYTreeSizeTest, FromStringBoundaryCases) +{ + EXPECT_EQ(std::numeric_limits<i64>::max(), DeserializeString("9223372036854775807")); + EXPECT_EQ(std::numeric_limits<i64>::lowest(), DeserializeString("-9223372036854775808")); + EXPECT_THROW(DeserializeString("9223372036854775808"), std::exception); + EXPECT_THROW(DeserializeString("-9223372036854775809"), std::exception); + + EXPECT_EQ(9'223'372'036'854'775'000LL, DeserializeString("9223372036854775K")); + EXPECT_EQ(-9'223'372'036'854'775'000LL, DeserializeString("-9223372036854775K")); + EXPECT_THROW(DeserializeString("9223372036854776K"), std::exception); + EXPECT_THROW(DeserializeString("-9223372036854776K"), std::exception); + + EXPECT_EQ(9'223'372'036'854'000'000LL, DeserializeString("9223372036854M")); + EXPECT_EQ(-9'223'372'036'854'000'000LL, DeserializeString("-9223372036854M")); + EXPECT_THROW(DeserializeString("9223372036855M"), std::exception); + EXPECT_THROW(DeserializeString("-9223372036855M"), std::exception); + + EXPECT_EQ(9'223'372'036'000'000'000LL, DeserializeString("9223372036G")); + EXPECT_EQ(-9'223'372'036'000'000'000LL, DeserializeString("-9223372036G")); + EXPECT_THROW(DeserializeString("9223372037G"), std::exception); + EXPECT_THROW(DeserializeString("-9223372037G"), std::exception); + + EXPECT_EQ(9'223'372'000'000'000'000LL, DeserializeString("9223372T")); + EXPECT_EQ(-9'223'372'000'000'000'000LL, DeserializeString("-9223372T")); + EXPECT_THROW(DeserializeString("9223373T"), std::exception); + EXPECT_THROW(DeserializeString("-9223373T"), std::exception); + + EXPECT_EQ(9'223'000'000'000'000'000LL, DeserializeString("9223P")); + EXPECT_EQ(-9'223'000'000'000'000'000LL, DeserializeString("-9223P")); + EXPECT_THROW(DeserializeString("9224P"), std::exception); + EXPECT_THROW(DeserializeString("-9224P"), std::exception); + + EXPECT_EQ(9'000'000'000'000'000'000LL, DeserializeString("9E")); + EXPECT_EQ(-9'000'000'000'000'000'000LL, DeserializeString("-9E")); + EXPECT_THROW(DeserializeString("10E"), std::exception); + EXPECT_THROW(DeserializeString("-10E"), std::exception); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace +} // namespace NYT::NYTree diff --git a/yt/yt/core/ytree/unittests/ya.make b/yt/yt/core/ytree/unittests/ya.make index 172b770294..256b8c014b 100644 --- a/yt/yt/core/ytree/unittests/ya.make +++ b/yt/yt/core/ytree/unittests/ya.make @@ -10,6 +10,7 @@ SRCS( resolver_ut.cpp serialize_ut.cpp service_combiner_ut.cpp + size_ut.cpp text_yson_convert_ut.cpp tree_builder_ut.cpp lazy_ypath_service_ut.cpp diff --git a/yt/yt/core/ytree/unittests/yson_struct_ut.cpp b/yt/yt/core/ytree/unittests/yson_struct_ut.cpp index f38c233f44..1708ccbec8 100644 --- a/yt/yt/core/ytree/unittests/yson_struct_ut.cpp +++ b/yt/yt/core/ytree/unittests/yson_struct_ut.cpp @@ -5,6 +5,7 @@ #include <yt/yt/core/ytree/ephemeral_node_factory.h> #include <yt/yt/core/ytree/fluent.h> #include <yt/yt/core/ytree/polymorphic_yson_struct.h> +#include <yt/yt/core/ytree/size.h> #include <yt/yt/core/ytree/tree_builder.h> #include <yt/yt/core/ytree/tree_visitor.h> #include <yt/yt/core/ytree/ypath_client.h> @@ -42,6 +43,8 @@ struct TTestSubconfig bool MyBool; std::vector<TString> MyStringList; ETestEnum MyEnum; + TDuration MyDuration; + TSize MySize; REGISTER_YSON_STRUCT(TTestSubconfig); @@ -59,6 +62,10 @@ struct TTestSubconfig .Default(); registrar.Parameter("my_enum", &TThis::MyEnum) .Default(ETestEnum::Value1); + registrar.Parameter("my_duration", &TThis::MyDuration) + .Default(TDuration::Seconds(1)); + registrar.Parameter("my_size", &TThis::MySize) + .Default(TSize::FromString("8K")); } }; @@ -135,6 +142,8 @@ auto GetCompleteConfigNode(int offset = 0) .Item().Value("ListItem1") .Item().Value("ListItem2") .EndList() + .Item("my_duration").Value("2h") + .Item("my_size").Value("2M") .EndMap() .Item("sub_list").BeginList() .Item().BeginMap() @@ -147,6 +156,8 @@ auto GetCompleteConfigNode(int offset = 0) .Item().Value("ListItem1") .Item().Value("ListItem2") .EndList() + .Item("my_duration").Value("2h") + .Item("my_size").Value(2'000'000) .EndMap() .Item().BeginMap() .Item("my_int").Value(99 + offset) @@ -158,6 +169,8 @@ auto GetCompleteConfigNode(int offset = 0) .Item().Value("ListItem1") .Item().Value("ListItem2") .EndList() + .Item("my_duration").Value("2h") + .Item("my_size").Value("2000K") .EndMap() .EndList() .Item("sub_map").BeginMap() @@ -171,6 +184,8 @@ auto GetCompleteConfigNode(int offset = 0) .Item().Value("ListItem1") .Item().Value("ListItem2") .EndList() + .Item("my_duration").Value("2h") + .Item("my_size").Value(2'000'000) .EndMap() .Item("sub2").BeginMap() .Item("my_int").Value(99 + offset) @@ -182,6 +197,8 @@ auto GetCompleteConfigNode(int offset = 0) .Item().Value("ListItem1") .Item().Value("ListItem2") .EndList() + .Item("my_duration").Value(2 * 60 * 60 * 1000) + .Item("my_size").Value(2'000'000) .EndMap() .EndMap() .EndMap(); @@ -189,7 +206,7 @@ auto GetCompleteConfigNode(int offset = 0) void TestCompleteSubconfig(TTestSubconfig* subconfig, int offset = 0) { - for (auto field : {"my_int", "my_uint", "my_bool", "my_enum", "my_string_list"}) { + for (auto field : {"my_int", "my_uint", "my_bool", "my_enum", "my_string_list", "my_duration", "my_size"}) { EXPECT_TRUE(subconfig->IsSet(field)); } @@ -201,6 +218,8 @@ void TestCompleteSubconfig(TTestSubconfig* subconfig, int offset = 0) EXPECT_EQ("ListItem1", subconfig->MyStringList[1]); EXPECT_EQ("ListItem2", subconfig->MyStringList[2]); EXPECT_EQ(ETestEnum::Value2, subconfig->MyEnum); + EXPECT_EQ(TDuration::Hours(2), subconfig->MyDuration); + EXPECT_EQ(2'000'000, subconfig->MySize); } void TestCompleteConfig(TIntrusivePtr<TTestConfig> config, int offset = 0) @@ -695,14 +714,18 @@ TEST(TYsonStructTest, Save) "\"my_enum\"=\"value1\";" "\"my_int\"=200;" "\"my_uint\"=50u;" - "\"my_string_list\"=[]}"; + "\"my_string_list\"=[];" + "\"my_duration\"=1000;" + "\"my_size\"=8000}"; TString subconfigYsonOrigin = "{\"my_bool\"=%false;" "\"my_enum\"=\"value1\";" "\"my_int\"=100;" "\"my_uint\"=50u;" - "\"my_string_list\"=[]}"; + "\"my_string_list\"=[];" + "\"my_duration\"=1000;" + "\"my_size\"=8000}"; TString expectedYson; expectedYson += "{\"my_string\"=\"hello!\";"; @@ -713,7 +736,9 @@ TEST(TYsonStructTest, Save) EXPECT_TRUE(AreNodesEqual( ConvertToNode(TYsonString(expectedYson)), - ConvertToNode(TYsonString(output.AsStringBuf())))); + ConvertToNode(TYsonString(output.AsStringBuf())))) + << "Expected: " << expectedYson + << ", got: " << output.AsStringBuf(); } TEST(TYsonStructTest, TestConfigUpdate) diff --git a/yt/yt/core/ytree/yson_struct_update-inl.h b/yt/yt/core/ytree/yson_struct_update-inl.h index 25ebc3652d..f010ccc92d 100644 --- a/yt/yt/core/ytree/yson_struct_update-inl.h +++ b/yt/yt/core/ytree/yson_struct_update-inl.h @@ -49,6 +49,11 @@ struct TUnwrapYsonStructIntrusivePtr<TIntrusivePtr<T>> //////////////////////////////////////////////////////////////////////////////// template <class TValue, class... Args> +requires + requires (const TValue& val) + { [] (Args...) {} (val); } || + requires (const TValue& val) + { [] (Args...) {} (val, val); } TCallback<void(const TValue&, const TValue&)> WrapUserCallback(TCallback<void(Args...)> callback) { return BIND_NO_PROPAGATE([callback = std::move(callback)] (const TValue& oldValue, const TValue& newValue) { diff --git a/yt/yt_proto/yt/client/api/rpc_proxy/proto/api_service.proto b/yt/yt_proto/yt/client/api/rpc_proxy/proto/api_service.proto index 8bb3739bd5..ba3879c8c5 100644 --- a/yt/yt_proto/yt/client/api/rpc_proxy/proto/api_service.proto +++ b/yt/yt_proto/yt/client/api/rpc_proxy/proto/api_service.proto @@ -3180,6 +3180,7 @@ message TJob optional bytes archive_features = 29; // YSON optional string monitoring_descriptor = 30; optional string operation_incarnation = 31; + optional NYT.NProto.TGuid allocation_id = 32; } message TListJobsStatistics |