diff options
author | Alexander Smirnov <alex@ydb.tech> | 2025-01-22 00:22:18 +0000 |
---|---|---|
committer | Alexander Smirnov <alex@ydb.tech> | 2025-01-22 00:22:18 +0000 |
commit | 47763d31b221c4f827cfb2d3dec8c0a23489fccc (patch) | |
tree | ead46596168eae5e9d38b6e8b770456acaf69ec9 | |
parent | f64d8ea3633ca6e8062dd5cd795e72f107353854 (diff) | |
parent | fbaf25eeb4ece3eafbc5f111ff230d1124b86cab (diff) | |
download | ydb-47763d31b221c4f827cfb2d3dec8c0a23489fccc.tar.gz |
Merge branch 'rightlib' into merge-libs-250122-0020
156 files changed, 3626 insertions, 1686 deletions
diff --git a/build/conf/java.conf b/build/conf/java.conf index 0b78fbab23e..541b4bc447a 100644 --- a/build/conf/java.conf +++ b/build/conf/java.conf @@ -752,10 +752,10 @@ when ($BUILD_TYPE == "DEBUG" && $NO_DEBUGINFO != "yes") { } _PACK_SRC_JAR_IMPL=\ -&& $FS_TOOLS md ${BINDIR}/src \ -&& $JAR_GEN_JSRCS \ -&& $LINK_JAR_JSOURCES \ -&& ${cwd;suf=/src:BINDIR} $JDK_RESOURCE/bin/jar cfvm ${output;pre=${BINDIR}/${MODULE_PREFIX};suf=-sources.jar:REALPRJNAME} $VCS_JAVA . + && $FS_TOOLS md ${BINDIR}/src \ + && $JAR_GEN_JSRCS \ + && $LINK_JAR_JSOURCES \ + && ${cwd;suf=/src:BINDIR} $JDK_RESOURCE/bin/jar cfvm ${output;pre=${BINDIR}/${MODULE_PREFIX};suf=-sources.jar:REALPRJNAME} $VCS_JAVA . _PACK_SRC_JAR= # tag:java-specific @@ -835,9 +835,9 @@ MAVEN_EXPORT_CMD= # tag:java-specific JAVA_COVERAGE_SRCLIST_FLAG=--coverage ${output;pre=${MODULE_PREFIX};suf=.cpsf:REALPRJNAME} --source-root ${ARCADIA_ROOT} JAVA_COVERAGE_SRCLIST= -PREPARE_JAVA_BUILD_DIRS=$FS_TOOLS md ${BINDIR}/cls && $FS_TOOLS md ${BINDIR}/misc -COLLECT_JAVA_SRCLIST=${YMAKE_PYTHON} ${input:"build/scripts/make_java_srclists.py"} \ - --moddir ${CURDIR} --java ${BINDIR}/all-java.srclst ${KT_SRSCLIST} ${JAVA_COVERAGE_SRCLIST} \ +PREPARE_JAR_BUILD=${cwd:BINDIR} ${YMAKE_PYTHON3} ${input:"build/scripts/prepare_jar_build.py"} \ + --bindir ${BINDIR} --moddir ${CURDIR} \ + --java ${BINDIR}/all-java.srclst ${KT_SRSCLIST} ${JAVA_COVERAGE_SRCLIST} \ --ya-start-command-file ${ext=.gentar:AUTO_INPUT} ${ALL_JAR_SOURCES} ${ext=.java:AUTO_INPUT} ${ext=.kt:AUTO_INPUT} --ya-end-command-file \ ${hide;input:"build/scripts/process_command_files.py"} \ ${hide;input:"build/scripts/java_pack_to_file.py"} \ @@ -847,8 +847,7 @@ LINK_CLASSPATH=${YMAKE_PYTHON} ${input:"build/scripts/make_manifest_from_bf.py"} COMPILE_JAVA_SRCLIST=$_JAVAC_RUN_HELPER($JAVAC_CMD) PACK_JAR=$_PACK_JAR_HELPER($TARGET) LINK_JAR=${hide:JAVA_FAKEID} ${hide;kv:"p JV"} ${hide;kv:"pc light-blue"} ${hide;kv:"show_out"} ${hide;requirements:"cpu:2"} \ - && $PREPARE_JAVA_BUILD_DIRS \ - && $COLLECT_JAVA_SRCLIST \ + && $PREPARE_JAR_BUILD \ && $JAR_GEN_SRCS \ && $ALL_KT_COMMANDS \ && $COLLECT_CLASSPATH \ @@ -1325,9 +1324,6 @@ module _JAR_TEST: _COMPILABLE_JAR_BASE { } # tag:java-specific -JAVA_SWIG_DELIM=JAVA_SWIG_DELIM - -# tag:java-specific JDK_REAL_VERSION= JDK_VERSION= diff --git a/build/conf/linkers/ld.conf b/build/conf/linkers/ld.conf index b189bf1e179..28d14f24c0f 100644 --- a/build/conf/linkers/ld.conf +++ b/build/conf/linkers/ld.conf @@ -334,12 +334,6 @@ LINK_EXEC_DYN_LIB=\ $DWARF_COMMAND && \ $LINK_ADDITIONAL_SECTIONS_COMMAND -SWIG_DLL_JAR_CMD=\ - $GENERATE_MF && \ - $GENERATE_VCS_C_INFO_NODEP && \ - $REAL_SWIG_DLL_JAR_CMD && \ - $DWARF_COMMAND - _LD_LINK_LIB_EXTRA_INPUT= _LD_TAIL_LINK_LIB=$AUTO_INPUT $_LD_LINK_LIB_EXTRA_INPUT ${hide;kv:"p AR"} $TOOLCHAIN_ENV ${hide;kv:"pc light-red"} ${hide;kv:"show_out"} _LD_ARCHIVER=$YMAKE_PYTHON ${input:"build/scripts/link_lib.py"} ${quo:AR_TOOL} $AR_TYPE $_LD_LLVM_AR_FORMAT $ARCADIA_BUILD_ROOT $_LD_AR_PLUGIN diff --git a/build/conf/linkers/msvc_linker.conf b/build/conf/linkers/msvc_linker.conf index c94cd67c9bc..df638dc2d0b 100644 --- a/build/conf/linkers/msvc_linker.conf +++ b/build/conf/linkers/msvc_linker.conf @@ -174,8 +174,6 @@ REAL_LINK_DYN_LIB_CMDLINE=\ REAL_LINK_DYN_LIB=$REAL_LINK_DYN_LIB_IMPL($_WHOLE_ARCHIVE_PEERS_VALUE) -SWIG_DLL_JAR_CMD=$GENERATE_MF && $GENERATE_VCS_C_INFO_NODEP && $REAL_SWIG_DLL_JAR_CMD - _MSVC_HEAD_LINK_LIB=${TOOLCHAIN_ENV} ${cwd:ARCADIA_BUILD_ROOT} ${LIB_WRAPPER} ${LINK_LIB_CMD} _MSVC_TAIL_LINK_LIB=\ --ya-start-command-file \ diff --git a/build/conf/swig.conf b/build/conf/swig.conf index 2dcb707af44..6c3061e3081 100644 --- a/build/conf/swig.conf +++ b/build/conf/swig.conf @@ -17,8 +17,6 @@ _SWIG_JNI_JAVA_CMD=$YMAKE_PYTHON3 ${input:"build/scripts/jni_swig.py"} --swig ${ _SWIG_JNI_CMD=$YMAKE_PYTHON3 ${input:"build/scripts/jni_swig.py"} --swig ${_SWIG_TOOL} --default-module ${nopath;noext:SRC} --src ${input:SRC} --out-header ${output;main;noext;suf=_wrap.swg.h:SRC} --package-by-file ru/yandex/${input;rootrel:SRC} --jsrc ${output;suf=.jsrc:SRC} -- ${pre=-I:_SWIG__INCLUDE} -o ${output;noext;suf=_wrap.swg.cpp:SRC} ${hide;kv:"p SW"} ${hide;kv:"pc yellow"} _SWIG_JNI_PEERDIR=contrib/libs/jdk -REAL_SWIG_DLL_JAR_CMD=$GENERATE_VCS_JAVA_INFO_NODEP && ${cwd:ARCADIA_BUILD_ROOT} $YMAKE_PYTHON ${input:"build/scripts/build_dll_and_java.py"} $JAVA_SWIG_DELIM $REAL_LINK_DYN_LIB $JAVA_SWIG_DELIM $COMPILE_JAVA $JAVA_SWIG_DELIM $AUTO_INPUT $JAVA_SWIG_DELIM $TARGET $JAVA_SWIG_DELIM ${output;suf=.jar:REALPRJNAME} $JAVA_SWIG_DELIM ${output;suf=-sources.jar:REALPRJNAME} $JAVA_SWIG_DELIM $ARCADIA_BUILD_ROOT $ARCADIA_ROOT - when ($USE_SYSTEM_JDK == "yes" || $OS_ANDROID == "yes") { _SWIG_JNI_PEERDIR= } diff --git a/build/config/tests/cpp_style/config.clang-format b/build/config/tests/cpp_style/config.clang-format index de8822cde53..b2d89e2e756 100644 --- a/build/config/tests/cpp_style/config.clang-format +++ b/build/config/tests/cpp_style/config.clang-format @@ -91,6 +91,7 @@ NamespaceMacros: [Y_UNIT_TEST_SUITE] AttributeMacros: [Y_PRINTF_FORMAT, Y_NO_SANITIZE, Y_FORCE_INLINE, Y_NO_INLINE, Y_WARN_UNUSED_RESULT, Y_HIDDEN, Y_PUBLIC, Y_PURE_FUNCTION] IndentExternBlock: Indent TypenameMacros: [Y_THREAD, Y_STATIC_THREAD, Y_POD_THREAD, Y_POD_STATIC_THREAD] +StatementAttributeLikeMacros: [ythrow] SpacesInLineCommentPrefix: Minimum: 1 Maximum: 1 diff --git a/build/mapping.conf.json b/build/mapping.conf.json index 1d323b752d1..a3d6e3ba0a1 100644 --- a/build/mapping.conf.json +++ b/build/mapping.conf.json @@ -959,6 +959,7 @@ "7552080809": "https://devtools-registry.s3.yandex.net/7552080809", "7830390213": "https://devtools-registry.s3.yandex.net/7830390213", "7833244075": "https://devtools-registry.s3.yandex.net/7833244075", + "7832768362": "https://devtools-registry.s3.yandex.net/7832768362", "5776379446": "https://devtools-registry.s3.yandex.net/5776379446", "5777100597": "https://devtools-registry.s3.yandex.net/5777100597", "5909067709": "https://devtools-registry.s3.yandex.net/5909067709", @@ -973,6 +974,7 @@ "7552077729": "https://devtools-registry.s3.yandex.net/7552077729", "7830388201": "https://devtools-registry.s3.yandex.net/7830388201", "7833241952": "https://devtools-registry.s3.yandex.net/7833241952", + "7832766692": "https://devtools-registry.s3.yandex.net/7832766692", "5776375952": "https://devtools-registry.s3.yandex.net/5776375952", "5777098178": "https://devtools-registry.s3.yandex.net/5777098178", "5909065014": "https://devtools-registry.s3.yandex.net/5909065014", @@ -987,6 +989,7 @@ "7552072726": "https://devtools-registry.s3.yandex.net/7552072726", "7830384362": "https://devtools-registry.s3.yandex.net/7830384362", "7833238161": "https://devtools-registry.s3.yandex.net/7833238161", + "7832762570": "https://devtools-registry.s3.yandex.net/7832762570", "5776374505": "https://devtools-registry.s3.yandex.net/5776374505", "5777096988": "https://devtools-registry.s3.yandex.net/5777096988", "5909063641": "https://devtools-registry.s3.yandex.net/5909063641", @@ -1001,6 +1004,7 @@ "7552070388": "https://devtools-registry.s3.yandex.net/7552070388", "7830382341": "https://devtools-registry.s3.yandex.net/7830382341", "7833236145": "https://devtools-registry.s3.yandex.net/7833236145", + "7832760150": "https://devtools-registry.s3.yandex.net/7832760150", "5776377955": "https://devtools-registry.s3.yandex.net/5776377955", "5777099502": "https://devtools-registry.s3.yandex.net/5777099502", "5909066324": "https://devtools-registry.s3.yandex.net/5909066324", @@ -1015,6 +1019,7 @@ "7552075245": "https://devtools-registry.s3.yandex.net/7552075245", "7830386085": "https://devtools-registry.s3.yandex.net/7830386085", "7833240023": "https://devtools-registry.s3.yandex.net/7833240023", + "7832764927": "https://devtools-registry.s3.yandex.net/7832764927", "6391354461": "https://devtools-registry.s3.yandex.net/6391354461", "7805449010": "https://devtools-registry.s3.yandex.net/7805449010", "6990868751": "https://devtools-registry.s3.yandex.net/6990868751", @@ -2148,6 +2153,7 @@ "7552080809": "none-none-none-result_resources/jdk-darwin-aarch64.yandex.tgz", "7830390213": "none-none-none-result_resources/jdk-darwin-aarch64.yandex.tgz", "7833244075": "none-none-none-result_resources/jdk-darwin-aarch64.yandex.tgz", + "7832768362": "none-none-none-result_resources/jdk-darwin-aarch64.yandex.tgz", "5776379446": "none-none-none-result_resources/jdk-darwin-x86_64.yandex.tgz", "5777100597": "none-none-none-result_resources/jdk-darwin-x86_64.yandex.tgz", "5909067709": "none-none-none-result_resources/jdk-darwin-x86_64.yandex.tgz", @@ -2162,6 +2168,7 @@ "7552077729": "none-none-none-result_resources/jdk-darwin-x86_64.yandex.tgz", "7830388201": "none-none-none-result_resources/jdk-darwin-x86_64.yandex.tgz", "7833241952": "none-none-none-result_resources/jdk-darwin-x86_64.yandex.tgz", + "7832766692": "none-none-none-result_resources/jdk-darwin-x86_64.yandex.tgz", "5776375952": "none-none-none-result_resources/jdk-linux-aarch64.yandex.tgz", "5777098178": "none-none-none-result_resources/jdk-linux-aarch64.yandex.tgz", "5909065014": "none-none-none-result_resources/jdk-linux-aarch64.yandex.tgz", @@ -2176,6 +2183,7 @@ "7552072726": "none-none-none-result_resources/jdk-linux-aarch64.yandex.tgz", "7830384362": "none-none-none-result_resources/jdk-linux-aarch64.yandex.tgz", "7833238161": "none-none-none-result_resources/jdk-linux-aarch64.yandex.tgz", + "7832762570": "none-none-none-result_resources/jdk-linux-aarch64.yandex.tgz", "5776374505": "none-none-none-result_resources/jdk-linux-x86_64.yandex.tgz", "5777096988": "none-none-none-result_resources/jdk-linux-x86_64.yandex.tgz", "5909063641": "none-none-none-result_resources/jdk-linux-x86_64.yandex.tgz", @@ -2190,6 +2198,7 @@ "7552070388": "none-none-none-result_resources/jdk-linux-x86_64.yandex.tgz", "7830382341": "none-none-none-result_resources/jdk-linux-x86_64.yandex.tgz", "7833236145": "none-none-none-result_resources/jdk-linux-x86_64.yandex.tgz", + "7832760150": "none-none-none-result_resources/jdk-linux-x86_64.yandex.tgz", "5776377955": "none-none-none-result_resources/jdk-windows-amd64.yandex.tgz", "5777099502": "none-none-none-result_resources/jdk-windows-amd64.yandex.tgz", "5909066324": "none-none-none-result_resources/jdk-windows-amd64.yandex.tgz", @@ -2204,6 +2213,7 @@ "7552075245": "none-none-none-result_resources/jdk-windows-amd64.yandex.tgz", "7830386085": "none-none-none-result_resources/jdk-windows-amd64.yandex.tgz", "7833240023": "none-none-none-result_resources/jdk-windows-amd64.yandex.tgz", + "7832764927": "none-none-none-result_resources/jdk-windows-amd64.yandex.tgz", "6391354461": "none-none-none-result_resources/protoc-linux-x86_64.tgz", "7805449010": "none-none-none-result_resources/protoc-linux-x86_64.tgz", "6990868751": "none-none-none-sandbox/backup/3527d100-e2d0-4b0e-bb7a-905010853d98/yfm-docs.tar", diff --git a/build/platform/java/jdk/jdk17/jdk.json b/build/platform/java/jdk/jdk17/jdk.json index 9127d41ce1a..dd4db71a476 100644 --- a/build/platform/java/jdk/jdk17/jdk.json +++ b/build/platform/java/jdk/jdk17/jdk.json @@ -1,19 +1,19 @@ { "by_platform": { "darwin-arm64": { - "uri": "sbr:6941864137" + "uri": "sbr:7832768362" }, "darwin-x86_64": { - "uri": "sbr:6941861330" + "uri": "sbr:7832766692" }, "linux-x86_64": { - "uri": "sbr:6941855347" + "uri": "sbr:7832760150" }, "linux-aarch64": { - "uri": "sbr:6941857195" + "uri": "sbr:7832762570" }, "win32-x86_64": { - "uri": "sbr:6941859166" + "uri": "sbr:7832764927" } } } diff --git a/build/scripts/build_dll_and_java.py b/build/scripts/build_dll_and_java.py deleted file mode 100644 index b9d8aff4df7..00000000000 --- a/build/scripts/build_dll_and_java.py +++ /dev/null @@ -1,47 +0,0 @@ -import os -import subprocess -import sys - - -def just_do_it(argv): - delim = argv[0] - args = [] - for item in argv: - if item == delim: - args.append([]) - else: - args[-1].append(item) - dll_cmd, java_cmd, inputs, dll_out, java_out, jsrs_out, roots = args - dll_out, java_out, jsrs_out, build_root, source_root = dll_out[0], java_out[0], jsrs_out[0], roots[0], roots[1] - for inp in inputs: - origin_inp = inp - if os.path.isabs(inp): - if os.path.relpath(inp, build_root).startswith('..'): - inp = os.path.relpath(inp, source_root) - else: - inp = os.path.relpath(inp, build_root) - ext = os.path.splitext(inp)[1] - if ext in ('.o', '.obj'): - if os.path.join(build_root, inp) in java_cmd: - inp = os.path.join(build_root, inp) - if sys.platform == 'win32': - inp = inp.replace('\\', '/') - if inp not in java_cmd: - inp = build_root + '/' + inp - java_cmd.remove(inp) - if ext in ('.java', '.jsrc'): - if origin_inp in dll_cmd: - inp = origin_inp - elif os.path.join(build_root, inp) in dll_cmd: - inp = os.path.join(build_root, inp) - if sys.platform == 'win32': - inp = inp.replace('\\', '/') - dll_cmd.remove(inp) - java_cmd.insert(java_cmd.index(dll_out), java_out) - java_cmd.remove(dll_out) - subprocess.check_call(java_cmd) - subprocess.check_call(dll_cmd) - - -if __name__ == '__main__': - just_do_it(sys.argv[1:]) diff --git a/build/scripts/make_java_srclists.py b/build/scripts/prepare_jar_build.py index a6e275d39c4..a326ca98842 100644 --- a/build/scripts/make_java_srclists.py +++ b/build/scripts/prepare_jar_build.py @@ -112,16 +112,25 @@ class SourcesConsumer: self.coverage.append(rel) +def prepare_build_dirs(bindir): + for dir in [os.path.join(bindir, dirname) for dirname in ['cls', 'misc']]: + if not os.path.exists(dir): + os.makedirs(dir) + + def main(): args = pcf.get_args(sys.argv[1:]) parser = argparse.ArgumentParser() parser.add_argument('--moddir') + parser.add_argument('--bindir') parser.add_argument('--java') parser.add_argument('--kotlin') parser.add_argument('--coverage') parser.add_argument('--source-root') args, remaining_args = parser.parse_known_args(args) + prepare_build_dirs(args.bindir) + src_sorter = SourcesSorter(args.moddir) src_consumer = SourcesConsumer( source_root=args.source_root, diff --git a/build/scripts/ya.make b/build/scripts/ya.make index 44752603d05..24998211c3a 100644 --- a/build/scripts/ya.make +++ b/build/scripts/ya.make @@ -41,7 +41,6 @@ ELSEIF (PY3) TEST_SRCS( append_file.py autotar_gendirs.py - build_dll_and_java.py build_info_gen.py build_java_codenav_index.py build_java_with_error_prone2.py @@ -100,7 +99,7 @@ ELSEIF (PY3) make_container.py make_container_layer.py make_java_classpath_file.py - make_java_srclists.py + prepare_jar_build.py make_manifest_from_bf.py merge_coverage_data.py merge_files.py diff --git a/build/sysincl/intrinsic.yml b/build/sysincl/intrinsic.yml index aa8308593aa..6140d33a915 100644 --- a/build/sysincl/intrinsic.yml +++ b/build/sysincl/intrinsic.yml @@ -1,3 +1,4 @@ +# This set of sysincls corresponds to clang ${CLANG_RESOURCE_GLOBAL}/lib/clang/Headers - includes: - Intrin.h - altivec.h @@ -22,6 +23,7 @@ - bmi2intrin.h - bmiintrin.h - cmnintrin.h + - cuda_wrappers/new - e2kbuiltin.h - emmintrin.h - f16cintrin.h diff --git a/contrib/libs/c-ares/.yandex_meta/__init__.py b/contrib/libs/c-ares/.yandex_meta/__init__.py index abe01d05f1c..fe0c9cf785f 100644 --- a/contrib/libs/c-ares/.yandex_meta/__init__.py +++ b/contrib/libs/c-ares/.yandex_meta/__init__.py @@ -49,7 +49,7 @@ def post_install(self): c_ares = CMakeNinjaNixProject( - owners=["max42", "g:cpp-contrib"], + owners=["g:cpp-contrib"], arcdir="contrib/libs/c-ares", nixattr="c-ares", copy_sources=[ diff --git a/contrib/libs/cxxsupp/builtins/.yandex_meta/build.ym b/contrib/libs/cxxsupp/builtins/.yandex_meta/build.ym index 4d8814ff185..ff0d9d04010 100644 --- a/contrib/libs/cxxsupp/builtins/.yandex_meta/build.ym +++ b/contrib/libs/cxxsupp/builtins/.yandex_meta/build.ym @@ -20,8 +20,6 @@ sed -e 's|.*#include.*zircon/features.h.*||' -i cpu_model/aarch64/lse_atomics/fu {% block ya_make %} SUBSCRIBER( - pg - somov g:contrib g:cpp-contrib ) diff --git a/contrib/libs/cxxsupp/libcxx/.yandex_meta/build.ym b/contrib/libs/cxxsupp/libcxx/.yandex_meta/build.ym index 2c5840b9eb7..09901da2ed7 100644 --- a/contrib/libs/cxxsupp/libcxx/.yandex_meta/build.ym +++ b/contrib/libs/cxxsupp/libcxx/.yandex_meta/build.ym @@ -91,8 +91,6 @@ VERSION({{self.date().strip()}}) ORIGINAL_SOURCE(https://github.com/llvm/llvm-project/archive/{{self.version().strip()}}.tar.gz) SUBSCRIBER( - halyavin - somov g:cpp-committee g:cpp-contrib ) diff --git a/contrib/libs/cxxsupp/libcxx/include/__exception/exception_ptr.h b/contrib/libs/cxxsupp/libcxx/include/__exception/exception_ptr.h index 53df28de7fe..08e34967fae 100644 --- a/contrib/libs/cxxsupp/libcxx/include/__exception/exception_ptr.h +++ b/contrib/libs/cxxsupp/libcxx/include/__exception/exception_ptr.h @@ -36,11 +36,14 @@ struct __cxa_exception; _LIBCPP_OVERRIDABLE_FUNC_VIS __cxa_exception* __cxa_init_primary_exception( void*, std::type_info*, - void( # if defined(_WIN32) - __thiscall + void(__thiscall*)(void*)) throw(); +# elif defined(__wasm__) + // In Wasm, a destructor returns its argument + void* (*)(void*)) throw(); +# else + void (*)(void*)) throw(); # endif - *)(void*)) throw(); } } // namespace __cxxabiv1 @@ -94,8 +97,16 @@ _LIBCPP_HIDE_FROM_ABI exception_ptr make_exception_ptr(_Ep __e) _NOEXCEPT { using _Ep2 = __decay_t<_Ep>; void* __ex = __cxxabiv1::__cxa_allocate_exception(sizeof(_Ep)); +# ifdef __wasm__ + // In Wasm, a destructor returns its argument + (void)__cxxabiv1::__cxa_init_primary_exception(__ex, const_cast<std::type_info*>(&typeid(_Ep)), [](void* __p) -> void* { +# else (void)__cxxabiv1::__cxa_init_primary_exception(__ex, const_cast<std::type_info*>(&typeid(_Ep)), [](void* __p) { +# endif std::__destroy_at(static_cast<_Ep2*>(__p)); +# ifdef __wasm__ + return __p; +# endif }); try { diff --git a/contrib/libs/cxxsupp/libcxx/include/__locale b/contrib/libs/cxxsupp/libcxx/include/__locale index 42b31f52b79..de7c2c61f52 100644 --- a/contrib/libs/cxxsupp/libcxx/include/__locale +++ b/contrib/libs/cxxsupp/libcxx/include/__locale @@ -344,12 +344,12 @@ public: static const mask __regex_word = 0x4000; // 0x8000 and 0x0100 and 0x00ff are used # define _LIBCPP_CTYPE_MASK_IS_COMPOSITE_PRINT # define _LIBCPP_CTYPE_MASK_IS_COMPOSITE_ALPHA -#elif defined(__APPLE__) || defined(__FreeBSD__) || defined(__EMSCRIPTEN__) || defined(__NetBSD__) +#elif defined(__APPLE__) || defined(__FreeBSD__) || defined(__NetBSD__) # ifdef __APPLE__ typedef __uint32_t mask; # elif defined(__FreeBSD__) typedef unsigned long mask; -# elif defined(__EMSCRIPTEN__) || defined(__NetBSD__) +# elif defined(__NetBSD__) typedef unsigned short mask; # endif static const mask space = _CTYPE_S; diff --git a/contrib/libs/cxxsupp/libcxx/patches/00-future-2024-05-22-wasm-eptr.patch b/contrib/libs/cxxsupp/libcxx/patches/00-future-2024-05-22-wasm-eptr.patch new file mode 100644 index 00000000000..f426cdd5f28 --- /dev/null +++ b/contrib/libs/cxxsupp/libcxx/patches/00-future-2024-05-22-wasm-eptr.patch @@ -0,0 +1,58 @@ +diff --git a/include/__exception/exception_ptr.h b/include/__exception/exception_ptr.h +index 53e2f71..73834b6 100644 +--- a/include/__exception/exception_ptr.h ++++ b/include/__exception/exception_ptr.h +@@ -36,11 +36,14 @@ struct __cxa_exception; + _LIBCPP_OVERRIDABLE_FUNC_VIS __cxa_exception* __cxa_init_primary_exception( + void*, + std::type_info*, +- void( + # if defined(_WIN32) +- __thiscall ++ void(__thiscall*)(void*)) throw(); ++# elif defined(__wasm__) ++ // In Wasm, a destructor returns its argument ++ void* (*)(void*)) throw(); ++# else ++ void (*)(void*)) throw(); + # endif +- *)(void*)) throw(); + } + + } // namespace __cxxabiv1 +@@ -88,8 +91,16 @@ _LIBCPP_HIDE_FROM_ABI exception_ptr make_exception_ptr(_Ep __e) _NOEXCEPT { + using _Ep2 = __decay_t<_Ep>; + + void* __ex = __cxxabiv1::__cxa_allocate_exception(sizeof(_Ep)); ++# ifdef __wasm__ ++ // In Wasm, a destructor returns its argument ++ (void)__cxxabiv1::__cxa_init_primary_exception(__ex, const_cast<std::type_info*>(&typeid(_Ep)), [](void* __p) -> void* { ++# else + (void)__cxxabiv1::__cxa_init_primary_exception(__ex, const_cast<std::type_info*>(&typeid(_Ep)), [](void* __p) { ++# endif + std::__destroy_at(static_cast<_Ep2*>(__p)); ++# ifdef __wasm__ ++ return __p; ++# endif + }); + + try { +diff --git a/include/__locale b/include/__locale +index 2186db8..fa987cb 100644 +--- a/include/__locale ++++ b/include/__locale +@@ -344,12 +344,12 @@ public: + static const mask __regex_word = 0x4000; // 0x8000 and 0x0100 and 0x00ff are used + # define _LIBCPP_CTYPE_MASK_IS_COMPOSITE_PRINT + # define _LIBCPP_CTYPE_MASK_IS_COMPOSITE_ALPHA +-#elif defined(__APPLE__) || defined(__FreeBSD__) || defined(__EMSCRIPTEN__) || defined(__NetBSD__) ++#elif defined(__APPLE__) || defined(__FreeBSD__) || defined(__NetBSD__) + # ifdef __APPLE__ + typedef __uint32_t mask; + # elif defined(__FreeBSD__) + typedef unsigned long mask; +-# elif defined(__EMSCRIPTEN__) || defined(__NetBSD__) ++# elif defined(__NetBSD__) + typedef unsigned short mask; + # endif + static const mask space = _CTYPE_S; diff --git a/contrib/libs/cxxsupp/libcxx/patches/23__locale.patch b/contrib/libs/cxxsupp/libcxx/patches/23__locale.patch index 5769dc30150..6739ec5e8d0 100644 --- a/contrib/libs/cxxsupp/libcxx/patches/23__locale.patch +++ b/contrib/libs/cxxsupp/libcxx/patches/23__locale.patch @@ -1,5 +1,5 @@ diff --git a/include/__locale b/include/__locale -index 2186db8..42b31f5 100644 +index fa987cb..de7c2c6 100644 --- a/include/__locale +++ b/include/__locale @@ -17,6 +17,7 @@ diff --git a/contrib/libs/cxxsupp/libcxx/patches/41-exception.patch b/contrib/libs/cxxsupp/libcxx/patches/41-exception.patch index 172c0877383..adb9debf11e 100644 --- a/contrib/libs/cxxsupp/libcxx/patches/41-exception.patch +++ b/contrib/libs/cxxsupp/libcxx/patches/41-exception.patch @@ -1,8 +1,8 @@ diff --git a/include/__exception/exception_ptr.h b/include/__exception/exception_ptr.h -index 53e2f71..b340ac0 100644 +index 73834b6..e641928 100644 --- a/include/__exception/exception_ptr.h +++ b/include/__exception/exception_ptr.h -@@ -49,8 +49,6 @@ _LIBCPP_OVERRIDABLE_FUNC_VIS __cxa_exception* __cxa_init_primary_exception( +@@ -52,8 +52,6 @@ _LIBCPP_OVERRIDABLE_FUNC_VIS __cxa_exception* __cxa_init_primary_exception( namespace std { // purposefully not using versioning namespace @@ -11,7 +11,7 @@ index 53e2f71..b340ac0 100644 class _LIBCPP_EXPORTED_FROM_ABI exception_ptr { void* __ptr_; -@@ -77,10 +75,18 @@ public: +@@ -80,10 +78,18 @@ public: return !(__x == __y); } @@ -30,7 +30,7 @@ index 53e2f71..b340ac0 100644 template <class _Ep> _LIBCPP_HIDE_FROM_ABI exception_ptr make_exception_ptr(_Ep __e) _NOEXCEPT { # ifndef _LIBCPP_HAS_NO_EXCEPTIONS -@@ -114,34 +120,7 @@ _LIBCPP_HIDE_FROM_ABI exception_ptr make_exception_ptr(_Ep __e) _NOEXCEPT { +@@ -125,34 +131,7 @@ _LIBCPP_HIDE_FROM_ABI exception_ptr make_exception_ptr(_Ep __e) _NOEXCEPT { #else // _LIBCPP_ABI_MICROSOFT @@ -66,7 +66,7 @@ index 53e2f71..b340ac0 100644 // This is a built-in template function which automagically extracts the required // information. -@@ -149,7 +128,7 @@ template <class _E> +@@ -160,7 +139,7 @@ template <class _E> void* __GetExceptionInfo(_E); template <class _Ep> diff --git a/contrib/libs/cxxsupp/libcxx/patches/72-abi-has-no-init_primary_exception.patch b/contrib/libs/cxxsupp/libcxx/patches/72-abi-has-no-init_primary_exception.patch index e48292db7af..9564e46456a 100644 --- a/contrib/libs/cxxsupp/libcxx/patches/72-abi-has-no-init_primary_exception.patch +++ b/contrib/libs/cxxsupp/libcxx/patches/72-abi-has-no-init_primary_exception.patch @@ -1,8 +1,8 @@ diff --git a/include/__exception/exception_ptr.h b/include/__exception/exception_ptr.h -index b340ac0..53df28d 100644 +index e641928..08e3496 100644 --- a/include/__exception/exception_ptr.h +++ b/include/__exception/exception_ptr.h -@@ -90,7 +90,7 @@ public: +@@ -93,7 +93,7 @@ public: template <class _Ep> _LIBCPP_HIDE_FROM_ABI exception_ptr make_exception_ptr(_Ep __e) _NOEXCEPT { # ifndef _LIBCPP_HAS_NO_EXCEPTIONS diff --git a/contrib/libs/cxxsupp/libcxxabi-parts/ya.make b/contrib/libs/cxxsupp/libcxxabi-parts/ya.make index ec57a006df3..5425e9f654f 100644 --- a/contrib/libs/cxxsupp/libcxxabi-parts/ya.make +++ b/contrib/libs/cxxsupp/libcxxabi-parts/ya.make @@ -6,9 +6,9 @@ LICENSE( Apache-2.0 WITH LLVM-exception ) -VERSION(17.0.6) +VERSION(18.1.8) -ORIGINAL_SOURCE(https://github.com/llvm/llvm-project/archive/llvmorg-17.0.6.tar.gz) +ORIGINAL_SOURCE(https://github.com/llvm/llvm-project/archive/llvmorg-18.1.8.tar.gz) ADDINCL( contrib/libs/cxxsupp/libcxxabi/include diff --git a/contrib/libs/cxxsupp/libcxxabi/.yandex_meta/__init__.py b/contrib/libs/cxxsupp/libcxxabi/.yandex_meta/__init__.py index c77e96a5b3e..0e45bb48bf1 100644 --- a/contrib/libs/cxxsupp/libcxxabi/.yandex_meta/__init__.py +++ b/contrib/libs/cxxsupp/libcxxabi/.yandex_meta/__init__.py @@ -69,6 +69,7 @@ def post_install(self): CFLAGS( -D_LIBCPP_SAFE_STATIC= -D_LIBCXXABI_DTOR_FUNC= + -D__USING_WASM_EXCEPTIONS__ ) ENDIF() """, @@ -96,6 +97,7 @@ libcxxabi = NixSourceProject( ], disable_includes=[ "aix_state_tab_eh.inc", + "sys/futex.h", ], post_install=post_install, ) diff --git a/contrib/libs/cxxsupp/libcxxabi/.yandex_meta/override.nix b/contrib/libs/cxxsupp/libcxxabi/.yandex_meta/override.nix index 46d08274e30..0ff1b8277f0 100644 --- a/contrib/libs/cxxsupp/libcxxabi/.yandex_meta/override.nix +++ b/contrib/libs/cxxsupp/libcxxabi/.yandex_meta/override.nix @@ -1,11 +1,11 @@ pkgs: attrs: with pkgs; with attrs; rec { - version = "17.0.6"; + version = "18.1.8"; src = fetchFromGitHub { owner = "llvm"; repo = "llvm-project"; rev = "llvmorg-${version}"; - hash = "sha256-8MEDLLhocshmxoEBRSKlJ/GzJ8nfuzQ8qn0X/vLA+ag="; + hash = "sha256-iiZKMRo/WxJaBXct9GdAcAT3cz9d9pnAcO1mmR6oPNE="; }; patches = []; diff --git a/contrib/libs/cxxsupp/libcxxabi/include/cxxabi.h b/contrib/libs/cxxsupp/libcxxabi/include/cxxabi.h index 292bac3b327..211d4a5021b 100644 --- a/contrib/libs/cxxsupp/libcxxabi/include/cxxabi.h +++ b/contrib/libs/cxxsupp/libcxxabi/include/cxxabi.h @@ -36,6 +36,9 @@ class type_info; // forward declaration // runtime routines use C calling conventions, but are in __cxxabiv1 namespace namespace __cxxabiv1 { + +struct __cxa_exception; + extern "C" { // 2.4.2 Allocating the Exception Object @@ -43,15 +46,22 @@ extern _LIBCXXABI_FUNC_VIS void * __cxa_allocate_exception(size_t thrown_size) throw(); extern _LIBCXXABI_FUNC_VIS void __cxa_free_exception(void *thrown_exception) throw(); +// This function is an LLVM extension, which mirrors the same extension in libsupc++ and libcxxrt +extern _LIBCXXABI_FUNC_VIS __cxa_exception* +#ifdef __wasm__ +// In Wasm, a destructor returns its argument +__cxa_init_primary_exception(void* object, std::type_info* tinfo, void*(_LIBCXXABI_DTOR_FUNC* dest)(void*)) throw(); +#else +__cxa_init_primary_exception(void* object, std::type_info* tinfo, void(_LIBCXXABI_DTOR_FUNC* dest)(void*)) throw(); +#endif // 2.4.3 Throwing the Exception Object -#ifdef __USING_WASM_EXCEPTIONS__ extern _LIBCXXABI_FUNC_VIS _LIBCXXABI_NORETURN void __cxa_throw(void *thrown_exception, std::type_info *tinfo, +#ifdef __USING_WASM_EXCEPTIONS__ + // In Wasm, a destructor returns its argument void *(_LIBCXXABI_DTOR_FUNC *dest)(void *)); #else -extern _LIBCXXABI_FUNC_VIS _LIBCXXABI_NORETURN void -__cxa_throw(void *thrown_exception, std::type_info *tinfo, void (_LIBCXXABI_DTOR_FUNC *dest)(void *)); #endif diff --git a/contrib/libs/cxxsupp/libcxxabi/patches/01-commit-e0e82fc-initial.patch b/contrib/libs/cxxsupp/libcxxabi/patches/01-commit-e0e82fc-initial.patch deleted file mode 100644 index aa2fe9e33df..00000000000 --- a/contrib/libs/cxxsupp/libcxxabi/patches/01-commit-e0e82fc-initial.patch +++ /dev/null @@ -1,351 +0,0 @@ -From e0e82fcf1ab23805ce90709320ae688e1ea8be57 Mon Sep 17 00:00:00 2001 -From: Sam Clegg <sbc@chromium.org> -Date: Sat, 22 May 2021 06:39:33 -0700 -Subject: [PATCH] Rebase of changed from emscripten-libs-13.0.0 onto - llvmorg-14.0.0 - -diff --git a/include/cxxabi.h b/include/cxxabi.h ---- a/include/cxxabi.h -+++ b/include/cxxabi.h -@@ -39,3 +39,9 @@ namespace __cxxabiv1 { -+#ifdef __USING_WASM_EXCEPTIONS__ -+extern _LIBCXXABI_FUNC_VIS _LIBCXXABI_NORETURN void -+__cxa_throw(void *thrown_exception, std::type_info *tinfo, -+ void *(_LIBCXXABI_DTOR_FUNC *dest)(void *)); -+#else - extern _LIBCXXABI_FUNC_VIS _LIBCXXABI_NORETURN void - __cxa_throw(void *thrown_exception, std::type_info *tinfo, - void (_LIBCXXABI_DTOR_FUNC *dest)(void *)); -+#endif -diff --git a/src/abort_message.cpp b/src/abort_message.cpp ---- a/src/abort_message.cpp -+++ b/src/abort_message.cpp -@@ -33,12 +33,21 @@ void abort_message(const char* format, ...) - // formatting into the variable-sized buffer fails. - #if !defined(NDEBUG) || !defined(LIBCXXABI_BAREMETAL) - { -+#if defined(__EMSCRIPTEN__) && defined(NDEBUG) -+ // Just trap in a non-debug build. These internal libcxxabi assertions are -+ // very rare, and it's not worth linking in vfprintf stdio support or -+ // even minimal logging for them, as we'll have a proper call stack, which -+ // will show a call into "abort_message", and can help debugging. (In a -+ // debug build that won't be inlined.) -+ abort(); -+#else - fprintf(stderr, "libc++abi: "); - va_list list; - va_start(list, format); - vfprintf(stderr, format, list); - va_end(list); - fprintf(stderr, "\n"); -+#endif - } - #endif - -diff --git a/src/cxa_exception.cpp b/src/cxa_exception.cpp ---- a/src/cxa_exception.cpp -+++ b/src/cxa_exception.cpp -@@ -254,7 +264,13 @@ will call terminate, assuming that there was no handler for the - exception. - */ - void --__cxa_throw(void *thrown_object, std::type_info *tinfo, void (_LIBCXXABI_DTOR_FUNC *dest)(void *)) { -+#ifdef __USING_WASM_EXCEPTIONS__ -+// In wasm, destructors return their argument -+__cxa_throw(void *thrown_object, std::type_info *tinfo, void *(_LIBCXXABI_DTOR_FUNC *dest)(void *)) -+#else -+__cxa_throw(void *thrown_object, std::type_info *tinfo, void (_LIBCXXABI_DTOR_FUNC *dest)(void *)) -+#endif -+{ - __cxa_eh_globals *globals = __cxa_get_globals(); - __cxa_exception* exception_header = cxa_exception_from_thrown_object(thrown_object); - -diff --git a/src/cxa_exception.h b/src/cxa_exception.h ---- a/src/cxa_exception.h -+++ b/src/cxa_exception.h -@@ -19,6 +19,25 @@ - - namespace __cxxabiv1 { - -+#ifdef __USING_EMSCRIPTEN_EXCEPTIONS__ -+ -+struct _LIBCXXABI_HIDDEN __cxa_exception { -+ size_t referenceCount; -+ std::type_info *exceptionType; -+ void (*exceptionDestructor)(void *); -+ uint8_t caught; -+ uint8_t rethrown; -+ void *adjustedPtr; -+ // Add padding to ensure that the size of __cxa_exception is a multiple of -+ // the maximum useful alignment for the target machine. This ensures that -+ // the thrown object that follows has that correct alignment. -+ void *padding; -+}; -+ -+static_assert(sizeof(__cxa_exception) % alignof(max_align_t) == 0, "__cxa_exception must have a size that is multipl of max alignment"); -+ -+#else -+ - static const uint64_t kOurExceptionClass = 0x434C4E47432B2B00; // CLNGC++\0 - static const uint64_t kOurDependentExceptionClass = 0x434C4E47432B2B01; // CLNGC++\1 - static const uint64_t get_vendor_and_language = 0xFFFFFFFFFFFFFF00; // mask for CLNGC++ -@@ -43,7 +62,12 @@ struct _LIBCXXABI_HIDDEN __cxa_exception { - - // Manage the exception object itself. - std::type_info *exceptionType; -+#ifdef __USING_WASM_EXCEPTIONS__ -+ // In wasm, destructors return their argument -+ void *(_LIBCXXABI_DTOR_FUNC *exceptionDestructor)(void *); -+#else - void (_LIBCXXABI_DTOR_FUNC *exceptionDestructor)(void *); -+#endif - std::unexpected_handler unexpectedHandler; - std::terminate_handler terminateHandler; - -@@ -159,6 +183,8 @@ extern "C" _LIBCXXABI_FUNC_VIS __cxa_eh_globals * __cxa_get_globals_fast (); - extern "C" _LIBCXXABI_FUNC_VIS void * __cxa_allocate_dependent_exception (); - extern "C" _LIBCXXABI_FUNC_VIS void __cxa_free_dependent_exception (void * dependent_exception); - -+#endif // !__USING_EMSCRIPTEN_EXCEPTIONS__ -+ - } // namespace __cxxabiv1 - - #endif // _CXA_EXCEPTION_H -diff --git a/src/cxa_handlers.cpp b/src/cxa_handlers.cpp ---- a/src/cxa_handlers.cpp -+++ b/src/cxa_handlers.cpp -@@ -73,7 +73,7 @@ __attribute__((noreturn)) - void - terminate() noexcept - { --#ifndef _LIBCXXABI_NO_EXCEPTIONS -+#if !defined(_LIBCXXABI_NO_EXCEPTIONS) && !defined(__USING_EMSCRIPTEN_EXCEPTIONS__) - // If there might be an uncaught exception - using namespace __cxxabiv1; - __cxa_eh_globals* globals = __cxa_get_globals_fast(); -diff --git a/src/cxa_personality.cpp b/src/cxa_personality.cpp ---- a/src/cxa_personality.cpp -+++ b/src/cxa_personality.cpp -@@ -41,6 +41,8 @@ extern "C" EXCEPTION_DISPOSITION _GCC_specific_handler(PEXCEPTION_RECORD, - _Unwind_Personality_Fn); - #endif - -+#define __USING_SJLJ_OR_WASM_EXCEPTIONS__ (__USING_SJLJ_EXCEPTIONS__ || __USING_WASM_EXCEPTIONS__) -+ - /* - Exception Header Layout: - -@@ -70,7 +72,7 @@ extern "C" EXCEPTION_DISPOSITION _GCC_specific_handler(PEXCEPTION_RECORD, - +------------------+--+-----+-----+------------------------+--------------------------+ - | callSiteTableLength | (ULEB128) | Call Site Table length, used to find Action table | - +---------------------+-----------+---------------------------------------------------+ --#ifndef __USING_SJLJ_EXCEPTIONS__ -+#ifndef __USING_SJLJ_OR_WASM_EXCEPTIONS__ - +---------------------+-----------+------------------------------------------------+ - | Beginning of Call Site Table The current ip lies within the | - | ... (start, length) range of one of these | -@@ -84,7 +86,7 @@ extern "C" EXCEPTION_DISPOSITION _GCC_specific_handler(PEXCEPTION_RECORD, - | +-------------+---------------------------------+------------------------------+ | - | ... | - +----------------------------------------------------------------------------------+ --#else // __USING_SJLJ_EXCEPTIONS__ -+#else // !__USING_SJLJ_OR_WASM_EXCEPTIONS__ - +---------------------+-----------+------------------------------------------------+ - | Beginning of Call Site Table The current ip is a 1-based index into | - | ... this table. Or it is -1 meaning no | -@@ -97,7 +99,7 @@ extern "C" EXCEPTION_DISPOSITION _GCC_specific_handler(PEXCEPTION_RECORD, - | +-------------+---------------------------------+------------------------------+ | - | ... | - +----------------------------------------------------------------------------------+ --#endif // __USING_SJLJ_EXCEPTIONS__ -+#endif // __USING_SJLJ_OR_WASM_EXCEPTIONS__ - +---------------------------------------------------------------------+ - | Beginning of Action Table ttypeIndex == 0 : cleanup | - | ... ttypeIndex > 0 : catch | -@@ -547,7 +549,7 @@ void - set_registers(_Unwind_Exception* unwind_exception, _Unwind_Context* context, - const scan_results& results) - { --#if defined(__USING_SJLJ_EXCEPTIONS__) -+#ifdef __USING_SJLJ_OR_WASM_EXCEPTIONS__ - #define __builtin_eh_return_data_regno(regno) regno - #elif defined(__ibmxl__) - // IBM xlclang++ compiler does not support __builtin_eh_return_data_regno. -@@ -642,7 +644,7 @@ static void scan_eh_tab(scan_results &results, _Unwind_Action actions, - // Get beginning current frame's code (as defined by the - // emitted dwarf code) - uintptr_t funcStart = _Unwind_GetRegionStart(context); --#ifdef __USING_SJLJ_EXCEPTIONS__ -+#ifdef __USING_SJLJ_OR_WASM_EXCEPTIONS__ - if (ip == uintptr_t(-1)) - { - // no action -@@ -652,9 +654,9 @@ static void scan_eh_tab(scan_results &results, _Unwind_Action actions, - else if (ip == 0) - call_terminate(native_exception, unwind_exception); - // ip is 1-based index into call site table --#else // !__USING_SJLJ_EXCEPTIONS__ -+#else // !__USING_SJLJ_OR_WASM_EXCEPTIONS__ - uintptr_t ipOffset = ip - funcStart; --#endif // !defined(_USING_SLJL_EXCEPTIONS__) -+#endif // !defined(__USING_SJLJ_OR_WASM_EXCEPTIONS__) - const uint8_t* classInfo = NULL; - // Note: See JITDwarfEmitter::EmitExceptionTable(...) for corresponding - // dwarf emission -@@ -676,7 +678,7 @@ static void scan_eh_tab(scan_results &results, _Unwind_Action actions, - // Walk call-site table looking for range that - // includes current PC. - uint8_t callSiteEncoding = *lsda++; --#ifdef __USING_SJLJ_EXCEPTIONS__ -+#ifdef __USING_SJLJ_OR_WASM_EXCEPTIONS__ - (void)callSiteEncoding; // When using SjLj exceptions, callSiteEncoding is never used - #endif - uint32_t callSiteTableLength = static_cast<uint32_t>(readULEB128(&lsda)); -@@ -687,7 +689,7 @@ static void scan_eh_tab(scan_results &results, _Unwind_Action actions, - while (callSitePtr < callSiteTableEnd) - { - // There is one entry per call site. --#ifndef __USING_SJLJ_EXCEPTIONS__ -+#ifndef __USING_SJLJ_OR_WASM_EXCEPTIONS__ - // The call sites are non-overlapping in [start, start+length) - // The call sites are ordered in increasing value of start - uintptr_t start = readEncodedPointer(&callSitePtr, callSiteEncoding); -@@ -695,15 +697,15 @@ static void scan_eh_tab(scan_results &results, _Unwind_Action actions, - uintptr_t landingPad = readEncodedPointer(&callSitePtr, callSiteEncoding); - uintptr_t actionEntry = readULEB128(&callSitePtr); - if ((start <= ipOffset) && (ipOffset < (start + length))) --#else // __USING_SJLJ_EXCEPTIONS__ -+#else // __USING_SJLJ_OR_WASM_EXCEPTIONS__ - // ip is 1-based index into this table - uintptr_t landingPad = readULEB128(&callSitePtr); - uintptr_t actionEntry = readULEB128(&callSitePtr); - if (--ip == 0) --#endif // __USING_SJLJ_EXCEPTIONS__ -+#endif // __USING_SJLJ_OR_WASM_EXCEPTIONS__ - { - // Found the call site containing ip. --#ifndef __USING_SJLJ_EXCEPTIONS__ -+#ifndef __USING_SJLJ_OR_WASM_EXCEPTIONS__ - if (landingPad == 0) - { - // No handler here -@@ -711,9 +713,9 @@ static void scan_eh_tab(scan_results &results, _Unwind_Action actions, - return; - } - landingPad = (uintptr_t)lpStart + landingPad; --#else // __USING_SJLJ_EXCEPTIONS__ -+#else // __USING_SJLJ_OR_WASM_EXCEPTIONS__ - ++landingPad; --#endif // __USING_SJLJ_EXCEPTIONS__ -+#endif // __USING_SJLJ_OR_WASM_EXCEPTIONS__ - results.landingPad = landingPad; - if (actionEntry == 0) - { -@@ -841,7 +843,7 @@ static void scan_eh_tab(scan_results &results, _Unwind_Action actions, - action += actionOffset; - } // there is no break out of this loop, only return - } --#ifndef __USING_SJLJ_EXCEPTIONS__ -+#ifndef __USING_SJLJ_OR_WASM_EXCEPTIONS__ - else if (ipOffset < start) - { - // There is no call site for this ip -@@ -849,7 +851,7 @@ static void scan_eh_tab(scan_results &results, _Unwind_Action actions, - // Possible stack corruption. - call_terminate(native_exception, unwind_exception); - } --#endif // !__USING_SJLJ_EXCEPTIONS__ -+#endif // !__USING_SJLJ_OR_WASM_EXCEPTIONS__ - } // there might be some tricky cases which break out of this loop - - // It is possible that no eh table entry specify how to handle -@@ -906,7 +908,9 @@ _UA_CLEANUP_PHASE - */ - - #if !defined(_LIBCXXABI_ARM_EHABI) --#if defined(__SEH__) && !defined(__USING_SJLJ_EXCEPTIONS__) -+#ifdef __USING_WASM_EXCEPTIONS__ -+_Unwind_Reason_Code __gxx_personality_wasm0 -+#elif defined(__SEH__) && !defined(__USING_SJLJ_EXCEPTIONS__) - static _Unwind_Reason_Code __gxx_personality_imp - #else - _LIBCXXABI_FUNC_VIS _Unwind_Reason_Code -diff --git a/src/cxa_thread_atexit.cpp b/src/cxa_thread_atexit.cpp ---- a/src/cxa_thread_atexit.cpp -+++ b/src/cxa_thread_atexit.cpp -@@ -112,9 +112,14 @@ extern "C" { - #ifdef HAVE___CXA_THREAD_ATEXIT_IMPL - return __cxa_thread_atexit_impl(dtor, obj, dso_symbol); - #else -+#ifndef __EMSCRIPTEN__ -+ // Emscripten doesn't fully support weak undefined symbols yet -+ // https://github.com/emscripten-core/emscripten/issues/12819 - if (__cxa_thread_atexit_impl) { - return __cxa_thread_atexit_impl(dtor, obj, dso_symbol); -- } else { -+ } else -+#endif -+ { - // Initialize the dtors std::__libcpp_tls_key (uses __cxa_guard_*() for - // one-time initialization and __cxa_atexit() for destruction) - static DtorsManager manager; -diff --git a/src/private_typeinfo.cpp b/src/private_typeinfo.cpp ---- a/src/private_typeinfo.cpp -+++ b/src/private_typeinfo.cpp -@@ -1323,4 +1323,35 @@ __base_class_type_info::search_below_dst(__dynamic_cast_info* info, - use_strcmp); - } - -+// XXX EMSCRIPTEN -+ -+#ifndef __USING_WASM_EXCEPTIONS__ -+ -+// These functions are used by the emscripten-style exception handling -+// mechanism. -+// Note that they need to be included even in the `-noexcept` build of -+// libc++abi to support the case where some parts of a project are built -+// with exception catching enabled, but at link time exception catching -+// is disabled. In this case dependencies to these functions (and the JS -+// functions which call them) will still exist in the final build. -+extern "C" { -+ -+int __cxa_can_catch(__shim_type_info* catchType, __shim_type_info* excpType, void **thrown) { -+ //std::type_info *t1 = static_cast<std::type_info*>(catchType); -+ //std::type_info *t2 = static_cast<std::type_info*>(excpType); -+ //printf("can %s catch %s (%p)?\n", t1->name(), t2->name(), thrown); -+ -+ void *temp = *thrown; -+ int ret = catchType->can_catch(excpType, temp); -+ if (ret) *thrown = temp; // apply changes only if we are catching -+ return ret; -+} -+ -+int __cxa_is_pointer_type(__shim_type_info* type) { -+ return !!dynamic_cast<__pointer_type_info*>(type); -+} -+ -+} -+#endif // __USING_EMSCRIPTEN_EXCEPTIONS__ -+ - } // __cxxabiv1 -diff --git a/src/stdlib_new_delete.cpp b/src/stdlib_new_delete.cpp ---- a/src/stdlib_new_delete.cpp -+++ b/src/stdlib_new_delete.cpp -@@ -37,9 +37,17 @@ operator new(std::size_t size) _THROW_BAD_ALLOC - else - #ifndef _LIBCPP_HAS_NO_EXCEPTIONS - throw std::bad_alloc(); -+#ifdef __EMSCRIPTEN__ -+ // Abort here so that when exceptions are disabled, we do not just -+ // return 0 when malloc returns 0. -+ // We could also do this with set_new_handler, but that adds a -+ // global constructor and a table entry, overhead that we can avoid -+ // by doing it this way. -+ abort(); - #else - break; - #endif -+#endif - } - return p; - } diff --git a/contrib/libs/cxxsupp/libcxxabi/patches/01-emscripten.patch b/contrib/libs/cxxsupp/libcxxabi/patches/01-emscripten.patch new file mode 100644 index 00000000000..11ab3d48ad4 --- /dev/null +++ b/contrib/libs/cxxsupp/libcxxabi/patches/01-emscripten.patch @@ -0,0 +1,226 @@ +--- contrib/libs/cxxsupp/libcxxabi/include/cxxabi.h (index) ++++ contrib/libs/cxxsupp/libcxxabi/include/cxxabi.h (working tree) +@@ -48,7 +48,12 @@ extern _LIBCXXABI_FUNC_VIS void + __cxa_free_exception(void *thrown_exception) throw(); + // This function is an LLVM extension, which mirrors the same extension in libsupc++ and libcxxrt + extern _LIBCXXABI_FUNC_VIS __cxa_exception* ++#ifdef __wasm__ ++// In Wasm, a destructor returns its argument ++__cxa_init_primary_exception(void* object, std::type_info* tinfo, void*(_LIBCXXABI_DTOR_FUNC* dest)(void*)) throw(); ++#else + __cxa_init_primary_exception(void* object, std::type_info* tinfo, void(_LIBCXXABI_DTOR_FUNC* dest)(void*)) throw(); ++#endif + + // 2.4.3 Throwing the Exception Object + extern _LIBCXXABI_FUNC_VIS _LIBCXXABI_NORETURN void +--- contrib/libs/cxxsupp/libcxxabi/src/abort_message.cpp (index) ++++ contrib/libs/cxxsupp/libcxxabi/src/abort_message.cpp (working tree) +@@ -33,12 +33,21 @@ void abort_message(const char* format, ...) + // formatting into the variable-sized buffer fails. + #if !defined(NDEBUG) || !defined(LIBCXXABI_BAREMETAL) + { ++#if defined(__EMSCRIPTEN__) && defined(NDEBUG) ++ // Just trap in a non-debug build. These internal libcxxabi assertions are ++ // very rare, and it's not worth linking in vfprintf stdio support or ++ // even minimal logging for them, as we'll have a proper call stack, which ++ // will show a call into "abort_message", and can help debugging. (In a ++ // debug build that won't be inlined.) ++ abort(); ++#else + fprintf(stderr, "libc++abi: "); + va_list list; + va_start(list, format); + vfprintf(stderr, format, list); + va_end(list); + fprintf(stderr, "\n"); ++#endif + } + #endif + +--- contrib/libs/cxxsupp/libcxxabi/src/cxa_exception.cpp (index) ++++ contrib/libs/cxxsupp/libcxxabi/src/cxa_exception.cpp (working tree) +@@ -207,7 +207,12 @@ void __cxa_free_exception(void *thrown_object) throw() { + } + + __cxa_exception* __cxa_init_primary_exception(void* object, std::type_info* tinfo, ++#ifdef __USING_WASM_EXCEPTIONS__ ++// In Wasm, a destructor returns its argument ++ void *(_LIBCXXABI_DTOR_FUNC* dest)(void*)) throw() { ++#else + void(_LIBCXXABI_DTOR_FUNC* dest)(void*)) throw() { ++#endif + __cxa_exception* exception_header = cxa_exception_from_thrown_object(object); + exception_header->referenceCount = 0; + exception_header->unexpectedHandler = std::get_unexpected(); +@@ -266,6 +271,13 @@ handler, _Unwind_RaiseException may return. In that case, __cxa_throw + will call terminate, assuming that there was no handler for the + exception. + */ ++ ++#if defined(__EMSCRIPTEN__) && defined(__USING_WASM_EXCEPTIONS__) && !defined(NDEBUG) ++extern "C" { ++void __throw_exception_with_stack_trace(_Unwind_Exception*); ++} // extern "C" ++#endif ++ + void + #ifdef __USING_WASM_EXCEPTIONS__ + // In Wasm, a destructor returns its argument +@@ -286,6 +298,10 @@ __cxa_throw(void *thrown_object, std::type_info *tinfo, void (_LIBCXXABI_DTOR_FU + + #ifdef __USING_SJLJ_EXCEPTIONS__ + _Unwind_SjLj_RaiseException(&exception_header->unwindHeader); ++#elif defined(__EMSCRIPTEN__) && defined(__USING_WASM_EXCEPTIONS__) && !defined(NDEBUG) ++ // In debug mode, call a JS library function to use WebAssembly.Exception JS ++ // API, which enables us to include stack traces ++ __throw_exception_with_stack_trace(&exception_header->unwindHeader); + #else + _Unwind_RaiseException(&exception_header->unwindHeader); + #endif +@@ -635,6 +651,10 @@ void __cxa_rethrow() { + } + #ifdef __USING_SJLJ_EXCEPTIONS__ + _Unwind_SjLj_RaiseException(&exception_header->unwindHeader); ++#elif defined(__EMSCRIPTEN__) && defined(__USING_WASM_EXCEPTIONS__) && !defined(NDEBUG) ++ // In debug mode, call a JS library function to use WebAssembly.Exception JS ++ // API, which enables us to include stack traces ++ __throw_exception_with_stack_trace(&exception_header->unwindHeader); + #else + _Unwind_RaiseException(&exception_header->unwindHeader); + #endif +@@ -760,6 +780,11 @@ __cxa_rethrow_primary_exception(void* thrown_object) + dep_exception_header->unwindHeader.exception_cleanup = dependent_exception_cleanup; + #ifdef __USING_SJLJ_EXCEPTIONS__ + _Unwind_SjLj_RaiseException(&dep_exception_header->unwindHeader); ++#elif defined(__EMSCRIPTEN__) && defined(__USING_WASM_EXCEPTIONS__) && !defined(NDEBUG) ++ // In debug mode, call a JS library function to use ++ // WebAssembly.Exception JS API, which enables us to include stack ++ // traces ++ __throw_exception_with_stack_trace(&exception_header->unwindHeader); + #else + _Unwind_RaiseException(&dep_exception_header->unwindHeader); + #endif +--- contrib/libs/cxxsupp/libcxxabi/src/cxa_exception.h (index) ++++ contrib/libs/cxxsupp/libcxxabi/src/cxa_exception.h (working tree) +@@ -19,6 +19,26 @@ + + namespace __cxxabiv1 { + ++#ifdef __USING_EMSCRIPTEN_EXCEPTIONS__ ++ ++struct _LIBCXXABI_HIDDEN __cxa_exception { ++ size_t referenceCount; ++ std::type_info *exceptionType; ++ // In wasm, destructors return 'this' as in ARM ++ void* (*exceptionDestructor)(void *); ++ uint8_t caught; ++ uint8_t rethrown; ++ void *adjustedPtr; ++ // Add padding to ensure that the size of __cxa_exception is a multiple of ++ // the maximum useful alignment for the target machine. This ensures that ++ // the thrown object that follows has that correct alignment. ++ void *padding; ++}; ++ ++static_assert(sizeof(__cxa_exception) % alignof(max_align_t) == 0, "__cxa_exception must have a size that is multipl of max alignment"); ++ ++#else ++ + static const uint64_t kOurExceptionClass = 0x434C4E47432B2B00; // CLNGC++\0 + static const uint64_t kOurDependentExceptionClass = 0x434C4E47432B2B01; // CLNGC++\1 + static const uint64_t get_vendor_and_language = 0xFFFFFFFFFFFFFF00; // mask for CLNGC++ +@@ -164,6 +184,8 @@ extern "C" _LIBCXXABI_FUNC_VIS __cxa_eh_globals * __cxa_get_globals_fast (); + extern "C" _LIBCXXABI_FUNC_VIS void * __cxa_allocate_dependent_exception (); + extern "C" _LIBCXXABI_FUNC_VIS void __cxa_free_dependent_exception (void * dependent_exception); + ++#endif // !__USING_EMSCRIPTEN_EXCEPTIONS__ ++ + } // namespace __cxxabiv1 + + #endif // _CXA_EXCEPTION_H +--- contrib/libs/cxxsupp/libcxxabi/src/cxa_handlers.cpp (index) ++++ contrib/libs/cxxsupp/libcxxabi/src/cxa_handlers.cpp (working tree) +@@ -73,7 +73,7 @@ __attribute__((noreturn)) + void + terminate() noexcept + { +-#ifndef _LIBCXXABI_NO_EXCEPTIONS ++#if !defined(_LIBCXXABI_NO_EXCEPTIONS) && !defined(__USING_EMSCRIPTEN_EXCEPTIONS__) + // If there might be an uncaught exception + using namespace __cxxabiv1; + __cxa_eh_globals* globals = __cxa_get_globals_fast(); +--- contrib/libs/cxxsupp/libcxxabi/src/cxa_personality.cpp (index) ++++ contrib/libs/cxxsupp/libcxxabi/src/cxa_personality.cpp (working tree) +@@ -663,6 +663,7 @@ static void scan_eh_tab(scan_results &results, _Unwind_Action actions, + const uint8_t* lpStart = lpStartEncoding == DW_EH_PE_omit + ? (const uint8_t*)funcStart + : (const uint8_t*)readEncodedPointer(&lsda, lpStartEncoding, base); ++ (void)(lpStart); // Unused when using SjLj/Wasm exceptions + uint8_t ttypeEncoding = *lsda++; + if (ttypeEncoding != DW_EH_PE_omit) + { +@@ -676,7 +677,7 @@ static void scan_eh_tab(scan_results &results, _Unwind_Action actions, + // includes current PC. + uint8_t callSiteEncoding = *lsda++; + #if defined(__USING_SJLJ_EXCEPTIONS__) || defined(__USING_WASM_EXCEPTIONS__) +- (void)callSiteEncoding; // When using SjLj/Wasm exceptions, callSiteEncoding is never used ++ (void)callSiteEncoding; // Unused when using SjLj/Wasm exceptions + #endif + uint32_t callSiteTableLength = static_cast<uint32_t>(readULEB128(&lsda)); + const uint8_t* callSiteTableStart = lsda; +--- contrib/libs/cxxsupp/libcxxabi/src/cxa_thread_atexit.cpp (index) ++++ contrib/libs/cxxsupp/libcxxabi/src/cxa_thread_atexit.cpp (working tree) +@@ -112,9 +112,14 @@ extern "C" { + #ifdef HAVE___CXA_THREAD_ATEXIT_IMPL + return __cxa_thread_atexit_impl(dtor, obj, dso_symbol); + #else ++#ifndef __EMSCRIPTEN__ ++ // Emscripten doesn't implement __cxa_thread_atexit_impl, so we can simply ++ // avoid this check. + if (__cxa_thread_atexit_impl) { + return __cxa_thread_atexit_impl(dtor, obj, dso_symbol); +- } else { ++ } else ++#endif ++ { + // Initialize the dtors std::__libcpp_tls_key (uses __cxa_guard_*() for + // one-time initialization and __cxa_atexit() for destruction) + static DtorsManager manager; +--- contrib/libs/cxxsupp/libcxxabi/src/private_typeinfo.cpp (index) ++++ contrib/libs/cxxsupp/libcxxabi/src/private_typeinfo.cpp (working tree) +@@ -1531,4 +1531,35 @@ __base_class_type_info::search_below_dst(__dynamic_cast_info* info, + use_strcmp); + } + ++// XXX EMSCRIPTEN ++ ++#ifndef __USING_WASM_EXCEPTIONS__ ++ ++// These functions are used by the emscripten-style exception handling ++// mechanism. ++// Note that they need to be included even in the `-noexcept` build of ++// libc++abi to support the case where some parts of a project are built ++// with exception catching enabled, but at link time exception catching ++// is disabled. In this case dependencies to these functions (and the JS ++// functions which call them) will still exist in the final build. ++extern "C" { ++ ++int __cxa_can_catch(__shim_type_info* catchType, __shim_type_info* excpType, void **thrown) { ++ //std::type_info *t1 = static_cast<std::type_info*>(catchType); ++ //std::type_info *t2 = static_cast<std::type_info*>(excpType); ++ //printf("can %s catch %s (%p)?\n", t1->name(), t2->name(), thrown); ++ ++ void *temp = *thrown; ++ int ret = catchType->can_catch(excpType, temp); ++ if (ret) *thrown = temp; // apply changes only if we are catching ++ return ret; ++} ++ ++int __cxa_is_pointer_type(__shim_type_info* type) { ++ return !!dynamic_cast<__pointer_type_info*>(type); ++} ++ ++} ++#endif // __USING_EMSCRIPTEN_EXCEPTIONS__ ++ + } // __cxxabiv1 diff --git a/contrib/libs/cxxsupp/libcxxabi/patches/02-commit-fd92733-3.1.27.patch b/contrib/libs/cxxsupp/libcxxabi/patches/02-commit-fd92733-3.1.27.patch deleted file mode 100644 index f4c7b96bf50..00000000000 --- a/contrib/libs/cxxsupp/libcxxabi/patches/02-commit-fd92733-3.1.27.patch +++ /dev/null @@ -1,65 +0,0 @@ -From fd92733b4753ad0efdb916a5aca3742b555c9de0 Mon Sep 17 00:00:00 2001 -From: Sam Clegg <sbc@chromium.org> -Date: Tue, 6 Dec 2022 17:01:34 -0800 -Subject: [PATCH] Patches from emscripten 3.1.27 - -diff --git a/src/cxa_exception.cpp b/src/cxa_exception.cpp ---- a/src/cxa_exception.cpp -+++ b/src/cxa_exception.cpp -@@ -263,6 +263,13 @@ handler, _Unwind_RaiseException may return. In that case, __cxa_throw - will call terminate, assuming that there was no handler for the - exception. - */ -+ -+#if defined(__USING_WASM_EXCEPTIONS__) && !defined(NDEBUG) -+extern "C" { -+void __throw_exception_with_stack_trace(_Unwind_Exception*); -+} // extern "C" -+#endif -+ - void - #ifdef __USING_WASM_EXCEPTIONS__ - // In wasm, destructors return their argument -@@ -289,11 +296,27 @@ __cxa_throw(void *thrown_object, std::type_info *tinfo, void (_LIBCXXABI_DTOR_FU - __asan_handle_no_return(); - #endif - -+#ifdef __EMSCRIPTEN__ - #ifdef __USING_SJLJ_EXCEPTIONS__ - _Unwind_SjLj_RaiseException(&exception_header->unwindHeader); -+#elif __USING_WASM_EXCEPTIONS__ -+#ifdef NDEBUG -+ _Unwind_RaiseException(&exception_header->unwindHeader); -+#else -+ // In debug mode, call a JS library function to use WebAssembly.Exception JS -+ // API, which enables us to include stack traces -+ __throw_exception_with_stack_trace(&exception_header->unwindHeader); -+#endif - #else - _Unwind_RaiseException(&exception_header->unwindHeader); - #endif -+#else // !__EMSCRIPTEN__ -+#ifdef __USING_SJLJ_EXCEPTIONS__ -+ _Unwind_SjLj_RaiseException(&exception_header->unwindHeader); -+#else -+ _Unwind_RaiseException(&exception_header->unwindHeader); -+#endif -+#endif - // This only happens when there is no handler, or some unexpected unwinding - // error happens. - failed_throw(exception_header); -diff --git a/src/cxa_personality.cpp b/src/cxa_personality.cpp ---- a/src/cxa_personality.cpp -+++ b/src/cxa_personality.cpp -@@ -977,6 +977,11 @@ __gxx_personality_v0 - exc->languageSpecificData = results.languageSpecificData; - exc->catchTemp = reinterpret_cast<void*>(results.landingPad); - exc->adjustedPtr = results.adjustedPtr; -+#ifdef __USING_WASM_EXCEPTIONS__ -+ // Wasm only uses a single phase (_UA_SEARCH_PHASE), so save the -+ // results here. -+ set_registers(unwind_exception, context, results); -+#endif - } - return _URC_HANDLER_FOUND; - } diff --git a/contrib/libs/cxxsupp/libcxxabi/patches/03-commit-fbea9fc-3.1.31.patch b/contrib/libs/cxxsupp/libcxxabi/patches/03-commit-fbea9fc-3.1.31.patch deleted file mode 100644 index 1a896e52e58..00000000000 --- a/contrib/libs/cxxsupp/libcxxabi/patches/03-commit-fbea9fc-3.1.31.patch +++ /dev/null @@ -1,19 +0,0 @@ -From fbea9fc86eba9a35958b0154b1b08dbc4744fa09 Mon Sep 17 00:00:00 2001 -From: Sam Clegg <sbc@chromium.org> -Date: Fri, 27 Jan 2023 09:02:49 -0800 -Subject: [PATCH] Patches from emscripten 3.1.31 - -diff --git a/src/cxa_thread_atexit.cpp b/src/cxa_thread_atexit.cpp ---- a/src/cxa_thread_atexit.cpp -+++ b/src/cxa_thread_atexit.cpp -@@ -113,8 +113,8 @@ extern "C" { - return __cxa_thread_atexit_impl(dtor, obj, dso_symbol); - #else - #ifndef __EMSCRIPTEN__ -- // Emscripten doesn't fully support weak undefined symbols yet -- // https://github.com/emscripten-core/emscripten/issues/12819 -+ // Emscripten doesn't implement __cxa_thread_atexit_impl, so we can simply -+ // avoid this check. - if (__cxa_thread_atexit_impl) { - return __cxa_thread_atexit_impl(dtor, obj, dso_symbol); - } else diff --git a/contrib/libs/cxxsupp/libcxxabi/src/abort_message.h b/contrib/libs/cxxsupp/libcxxabi/src/abort_message.h index f1d5c12e252..97641777801 100644 --- a/contrib/libs/cxxsupp/libcxxabi/src/abort_message.h +++ b/contrib/libs/cxxsupp/libcxxabi/src/abort_message.h @@ -14,4 +14,15 @@ extern "C" _LIBCXXABI_HIDDEN _LIBCXXABI_NORETURN void abort_message(const char *format, ...) __attribute__((format(printf, 1, 2))); +#ifndef _LIBCXXABI_ASSERT +# define _LIBCXXABI_ASSERT(expr, msg) \ + do { \ + if (!(expr)) { \ + char const* __msg = (msg); \ + ::abort_message("%s:%d: %s", __FILE__, __LINE__, __msg); \ + } \ + } while (false) + #endif + +#endif // __ABORT_MESSAGE_H_ diff --git a/contrib/libs/cxxsupp/libcxxabi/src/cxa_demangle.cpp b/contrib/libs/cxxsupp/libcxxabi/src/cxa_demangle.cpp index 03085cb5903..bece33a007f 100644 --- a/contrib/libs/cxxsupp/libcxxabi/src/cxa_demangle.cpp +++ b/contrib/libs/cxxsupp/libcxxabi/src/cxa_demangle.cpp @@ -10,14 +10,17 @@ // file does not yet support: // - C++ modules TS +#include "abort_message.h" +#define DEMANGLE_ASSERT(expr, msg) _LIBCXXABI_ASSERT(expr, msg) + #include "demangle/DemangleConfig.h" #include "demangle/ItaniumDemangle.h" #include "__cxxabi_config.h" -#include <cassert> #include <cctype> #include <cstdio> #include <cstdlib> #include <cstring> +#include <exception> #include <functional> #include <numeric> #include <string_view> @@ -394,7 +397,7 @@ __cxa_demangle(const char *MangledName, char *Buf, size_t *N, int *Status) { InternalStatus = demangle_invalid_mangled_name; else { OutputBuffer O(Buf, N); - assert(Parser.ForwardTemplateRefs.empty()); + DEMANGLE_ASSERT(Parser.ForwardTemplateRefs.empty(), ""); AST->print(O); O += '\0'; if (N != nullptr) diff --git a/contrib/libs/cxxsupp/libcxxabi/src/cxa_exception.cpp b/contrib/libs/cxxsupp/libcxxabi/src/cxa_exception.cpp index 19c333d9c0d..4a792e08510 100644 --- a/contrib/libs/cxxsupp/libcxxabi/src/cxa_exception.cpp +++ b/contrib/libs/cxxsupp/libcxxabi/src/cxa_exception.cpp @@ -206,6 +206,24 @@ void __cxa_free_exception(void *thrown_object) throw() { __aligned_free_with_fallback((void *)raw_buffer); } +__cxa_exception* __cxa_init_primary_exception(void* object, std::type_info* tinfo, +#ifdef __USING_WASM_EXCEPTIONS__ +// In Wasm, a destructor returns its argument + void *(_LIBCXXABI_DTOR_FUNC* dest)(void*)) throw() { +#else + void(_LIBCXXABI_DTOR_FUNC* dest)(void*)) throw() { +#endif + __cxa_exception* exception_header = cxa_exception_from_thrown_object(object); + exception_header->referenceCount = 0; + exception_header->unexpectedHandler = std::get_unexpected(); + exception_header->terminateHandler = std::get_terminate(); + exception_header->exceptionType = tinfo; + exception_header->exceptionDestructor = dest; + setOurExceptionClass(&exception_header->unwindHeader); + exception_header->unwindHeader.exception_cleanup = exception_cleanup_func; + + return exception_header; +} // This function shall allocate a __cxa_dependent_exception and // return a pointer to it. (Really to the object, not past its' end). @@ -254,7 +272,7 @@ will call terminate, assuming that there was no handler for the exception. */ -#if defined(__USING_WASM_EXCEPTIONS__) && !defined(NDEBUG) +#if defined(__EMSCRIPTEN__) && defined(__USING_WASM_EXCEPTIONS__) && !defined(NDEBUG) extern "C" { void __throw_exception_with_stack_trace(_Unwind_Exception*); } // extern "C" @@ -262,51 +280,31 @@ void __throw_exception_with_stack_trace(_Unwind_Exception*); void #ifdef __USING_WASM_EXCEPTIONS__ -// In wasm, destructors return their argument -__cxa_throw(void *thrown_object, std::type_info *tinfo, void *(_LIBCXXABI_DTOR_FUNC *dest)(void *)) +// In Wasm, a destructor returns its argument +__cxa_throw(void *thrown_object, std::type_info *tinfo, void *(_LIBCXXABI_DTOR_FUNC *dest)(void *)) { #else -__cxa_throw(void *thrown_object, std::type_info *tinfo, void (_LIBCXXABI_DTOR_FUNC *dest)(void *)) +__cxa_throw(void *thrown_object, std::type_info *tinfo, void (_LIBCXXABI_DTOR_FUNC *dest)(void *)) { #endif -{ - __cxa_eh_globals *globals = __cxa_get_globals(); - __cxa_exception* exception_header = cxa_exception_from_thrown_object(thrown_object); - - exception_header->unexpectedHandler = std::get_unexpected(); - exception_header->terminateHandler = std::get_terminate(); - exception_header->exceptionType = tinfo; - exception_header->exceptionDestructor = dest; - setOurExceptionClass(&exception_header->unwindHeader); - exception_header->referenceCount = 1; // This is a newly allocated exception, no need for thread safety. - globals->uncaughtExceptions += 1; // Not atomically, since globals are thread-local + __cxa_eh_globals* globals = __cxa_get_globals(); + globals->uncaughtExceptions += 1; // Not atomically, since globals are thread-local - exception_header->unwindHeader.exception_cleanup = exception_cleanup_func; + __cxa_exception* exception_header = __cxa_init_primary_exception(thrown_object, tinfo, dest); + exception_header->referenceCount = 1; // This is a newly allocated exception, no need for thread safety. #if __has_feature(address_sanitizer) - // Inform the ASan runtime that now might be a good time to clean stuff up. - __asan_handle_no_return(); + // Inform the ASan runtime that now might be a good time to clean stuff up. + __asan_handle_no_return(); #endif -#ifdef __EMSCRIPTEN__ #ifdef __USING_SJLJ_EXCEPTIONS__ _Unwind_SjLj_RaiseException(&exception_header->unwindHeader); -#elif __USING_WASM_EXCEPTIONS__ -#ifdef NDEBUG - _Unwind_RaiseException(&exception_header->unwindHeader); -#else +#elif defined(__EMSCRIPTEN__) && defined(__USING_WASM_EXCEPTIONS__) && !defined(NDEBUG) // In debug mode, call a JS library function to use WebAssembly.Exception JS // API, which enables us to include stack traces __throw_exception_with_stack_trace(&exception_header->unwindHeader); -#endif #else _Unwind_RaiseException(&exception_header->unwindHeader); #endif -#else // !__EMSCRIPTEN__ -#ifdef __USING_SJLJ_EXCEPTIONS__ - _Unwind_SjLj_RaiseException(&exception_header->unwindHeader); -#else - _Unwind_RaiseException(&exception_header->unwindHeader); -#endif -#endif // This only happens when there is no handler, or some unexpected unwinding // error happens. failed_throw(exception_header); @@ -653,6 +651,10 @@ void __cxa_rethrow() { } #ifdef __USING_SJLJ_EXCEPTIONS__ _Unwind_SjLj_RaiseException(&exception_header->unwindHeader); +#elif defined(__EMSCRIPTEN__) && defined(__USING_WASM_EXCEPTIONS__) && !defined(NDEBUG) + // In debug mode, call a JS library function to use WebAssembly.Exception JS + // API, which enables us to include stack traces + __throw_exception_with_stack_trace(&exception_header->unwindHeader); #else _Unwind_RaiseException(&exception_header->unwindHeader); #endif @@ -778,6 +780,11 @@ __cxa_rethrow_primary_exception(void* thrown_object) dep_exception_header->unwindHeader.exception_cleanup = dependent_exception_cleanup; #ifdef __USING_SJLJ_EXCEPTIONS__ _Unwind_SjLj_RaiseException(&dep_exception_header->unwindHeader); +#elif defined(__EMSCRIPTEN__) && defined(__USING_WASM_EXCEPTIONS__) && !defined(NDEBUG) + // In debug mode, call a JS library function to use + // WebAssembly.Exception JS API, which enables us to include stack + // traces + __throw_exception_with_stack_trace(&exception_header->unwindHeader); #else _Unwind_RaiseException(&dep_exception_header->unwindHeader); #endif @@ -800,6 +807,6 @@ __cxa_uncaught_exceptions() throw() return globals->uncaughtExceptions; } -} // extern "C" +} // extern "C" } // abi diff --git a/contrib/libs/cxxsupp/libcxxabi/src/cxa_exception.h b/contrib/libs/cxxsupp/libcxxabi/src/cxa_exception.h index dd20ba1e889..098e0bf95ae 100644 --- a/contrib/libs/cxxsupp/libcxxabi/src/cxa_exception.h +++ b/contrib/libs/cxxsupp/libcxxabi/src/cxa_exception.h @@ -24,7 +24,8 @@ namespace __cxxabiv1 { struct _LIBCXXABI_HIDDEN __cxa_exception { size_t referenceCount; std::type_info *exceptionType; - void (*exceptionDestructor)(void *); + // In wasm, destructors return 'this' as in ARM + void* (*exceptionDestructor)(void *); uint8_t caught; uint8_t rethrown; void *adjustedPtr; @@ -63,7 +64,7 @@ struct _LIBCXXABI_HIDDEN __cxa_exception { // Manage the exception object itself. std::type_info *exceptionType; #ifdef __USING_WASM_EXCEPTIONS__ - // In wasm, destructors return their argument + // In Wasm, a destructor returns its argument void *(_LIBCXXABI_DTOR_FUNC *exceptionDestructor)(void *); #else void (_LIBCXXABI_DTOR_FUNC *exceptionDestructor)(void *); diff --git a/contrib/libs/cxxsupp/libcxxabi/src/cxa_guard.cpp b/contrib/libs/cxxsupp/libcxxabi/src/cxa_guard.cpp index fc1fa905119..514fa962e3d 100644 --- a/contrib/libs/cxxsupp/libcxxabi/src/cxa_guard.cpp +++ b/contrib/libs/cxxsupp/libcxxabi/src/cxa_guard.cpp @@ -48,6 +48,6 @@ _LIBCXXABI_FUNC_VIS void __cxa_guard_abort(guard_type *raw_guard_object) { SelectedImplementation imp(raw_guard_object); imp.cxa_guard_abort(); } -} // extern "C" +} // extern "C" } // __cxxabiv1 diff --git a/contrib/libs/cxxsupp/libcxxabi/src/cxa_guard_impl.h b/contrib/libs/cxxsupp/libcxxabi/src/cxa_guard_impl.h index 7b140d3c360..fb3e9eed8f0 100644 --- a/contrib/libs/cxxsupp/libcxxabi/src/cxa_guard_impl.h +++ b/contrib/libs/cxxsupp/libcxxabi/src/cxa_guard_impl.h @@ -47,6 +47,9 @@ #include "__cxxabi_config.h" #include "include/atomic_support.h" // from libc++ #if defined(__has_include) +# if __has_include(<sys/futex.h>) +# error #include <sys/futex.h> +# endif # if __has_include(<sys/syscall.h>) # include <sys/syscall.h> # endif @@ -411,7 +414,18 @@ private: // Futex Implementation //===----------------------------------------------------------------------===// -#if defined(SYS_futex) +#if defined(__OpenBSD__) +void PlatformFutexWait(int* addr, int expect) { + constexpr int WAIT = 0; + futex(reinterpret_cast<volatile uint32_t*>(addr), WAIT, expect, NULL, NULL); + __tsan_acquire(addr); +} +void PlatformFutexWake(int* addr) { + constexpr int WAKE = 1; + __tsan_release(addr); + futex(reinterpret_cast<volatile uint32_t*>(addr), WAKE, INT_MAX, NULL, NULL); +} +#elif defined(SYS_futex) void PlatformFutexWait(int* addr, int expect) { constexpr int WAIT = 0; syscall(SYS_futex, addr, WAIT, expect, 0); diff --git a/contrib/libs/cxxsupp/libcxxabi/src/cxa_personality.cpp b/contrib/libs/cxxsupp/libcxxabi/src/cxa_personality.cpp index 1a969265f02..501bc75a8c0 100644 --- a/contrib/libs/cxxsupp/libcxxabi/src/cxa_personality.cpp +++ b/contrib/libs/cxxsupp/libcxxabi/src/cxa_personality.cpp @@ -41,8 +41,6 @@ extern "C" EXCEPTION_DISPOSITION _GCC_specific_handler(PEXCEPTION_RECORD, _Unwind_Personality_Fn); #endif -#define __USING_SJLJ_OR_WASM_EXCEPTIONS__ (__USING_SJLJ_EXCEPTIONS__ || __USING_WASM_EXCEPTIONS__) - /* Exception Header Layout: @@ -72,7 +70,7 @@ extern "C" EXCEPTION_DISPOSITION _GCC_specific_handler(PEXCEPTION_RECORD, +------------------+--+-----+-----+------------------------+--------------------------+ | callSiteTableLength | (ULEB128) | Call Site Table length, used to find Action table | +---------------------+-----------+---------------------------------------------------+ -#ifndef __USING_SJLJ_OR_WASM_EXCEPTIONS__ +#if !defined(__USING_SJLJ_EXCEPTIONS__) && !defined(__USING_WASM_EXCEPTIONS__) +---------------------+-----------+------------------------------------------------+ | Beginning of Call Site Table The current ip lies within the | | ... (start, length) range of one of these | @@ -86,7 +84,7 @@ extern "C" EXCEPTION_DISPOSITION _GCC_specific_handler(PEXCEPTION_RECORD, | +-------------+---------------------------------+------------------------------+ | | ... | +----------------------------------------------------------------------------------+ -#else // !__USING_SJLJ_OR_WASM_EXCEPTIONS__ +#else // __USING_SJLJ_EXCEPTIONS__ || __USING_WASM_EXCEPTIONS__ +---------------------+-----------+------------------------------------------------+ | Beginning of Call Site Table The current ip is a 1-based index into | | ... this table. Or it is -1 meaning no | @@ -99,7 +97,7 @@ extern "C" EXCEPTION_DISPOSITION _GCC_specific_handler(PEXCEPTION_RECORD, | +-------------+---------------------------------+------------------------------+ | | ... | +----------------------------------------------------------------------------------+ -#endif // __USING_SJLJ_OR_WASM_EXCEPTIONS__ +#endif // __USING_SJLJ_EXCEPTIONS__ || __USING_WASM_EXCEPTIONS__ +---------------------------------------------------------------------+ | Beginning of Action Table ttypeIndex == 0 : cleanup | | ... ttypeIndex > 0 : catch | @@ -549,7 +547,7 @@ void set_registers(_Unwind_Exception* unwind_exception, _Unwind_Context* context, const scan_results& results) { -#ifdef __USING_SJLJ_OR_WASM_EXCEPTIONS__ +#if defined(__USING_SJLJ_EXCEPTIONS__) || defined(__USING_WASM_EXCEPTIONS__) #define __builtin_eh_return_data_regno(regno) regno #elif defined(__ibmxl__) // IBM xlclang++ compiler does not support __builtin_eh_return_data_regno. @@ -644,7 +642,7 @@ static void scan_eh_tab(scan_results &results, _Unwind_Action actions, // Get beginning current frame's code (as defined by the // emitted dwarf code) uintptr_t funcStart = _Unwind_GetRegionStart(context); -#ifdef __USING_SJLJ_OR_WASM_EXCEPTIONS__ +#if defined(__USING_SJLJ_EXCEPTIONS__) || defined(__USING_WASM_EXCEPTIONS__) if (ip == uintptr_t(-1)) { // no action @@ -654,18 +652,18 @@ static void scan_eh_tab(scan_results &results, _Unwind_Action actions, else if (ip == 0) call_terminate(native_exception, unwind_exception); // ip is 1-based index into call site table -#else // !__USING_SJLJ_OR_WASM_EXCEPTIONS__ +#else // !__USING_SJLJ_EXCEPTIONS__ && !__USING_WASM_EXCEPTIONS__ uintptr_t ipOffset = ip - funcStart; -#endif // !defined(__USING_SJLJ_OR_WASM_EXCEPTIONS__) +#endif // !__USING_SJLJ_EXCEPTIONS__ && !__USING_WASM_EXCEPTIONS__ const uint8_t* classInfo = NULL; // Note: See JITDwarfEmitter::EmitExceptionTable(...) for corresponding // dwarf emission // Parse LSDA header. uint8_t lpStartEncoding = *lsda++; - const uint8_t* lpStart = - (const uint8_t*)readEncodedPointer(&lsda, lpStartEncoding, base); - if (lpStart == 0) - lpStart = (const uint8_t*)funcStart; + const uint8_t* lpStart = lpStartEncoding == DW_EH_PE_omit + ? (const uint8_t*)funcStart + : (const uint8_t*)readEncodedPointer(&lsda, lpStartEncoding, base); + (void)(lpStart); // Unused when using SjLj/Wasm exceptions uint8_t ttypeEncoding = *lsda++; if (ttypeEncoding != DW_EH_PE_omit) { @@ -678,8 +676,8 @@ static void scan_eh_tab(scan_results &results, _Unwind_Action actions, // Walk call-site table looking for range that // includes current PC. uint8_t callSiteEncoding = *lsda++; -#ifdef __USING_SJLJ_OR_WASM_EXCEPTIONS__ - (void)callSiteEncoding; // When using SjLj exceptions, callSiteEncoding is never used +#if defined(__USING_SJLJ_EXCEPTIONS__) || defined(__USING_WASM_EXCEPTIONS__) + (void)callSiteEncoding; // Unused when using SjLj/Wasm exceptions #endif uint32_t callSiteTableLength = static_cast<uint32_t>(readULEB128(&lsda)); const uint8_t* callSiteTableStart = lsda; @@ -689,7 +687,7 @@ static void scan_eh_tab(scan_results &results, _Unwind_Action actions, while (callSitePtr < callSiteTableEnd) { // There is one entry per call site. -#ifndef __USING_SJLJ_OR_WASM_EXCEPTIONS__ +#if !defined(__USING_SJLJ_EXCEPTIONS__) && !defined(__USING_WASM_EXCEPTIONS__) // The call sites are non-overlapping in [start, start+length) // The call sites are ordered in increasing value of start uintptr_t start = readEncodedPointer(&callSitePtr, callSiteEncoding); @@ -697,15 +695,15 @@ static void scan_eh_tab(scan_results &results, _Unwind_Action actions, uintptr_t landingPad = readEncodedPointer(&callSitePtr, callSiteEncoding); uintptr_t actionEntry = readULEB128(&callSitePtr); if ((start <= ipOffset) && (ipOffset < (start + length))) -#else // __USING_SJLJ_OR_WASM_EXCEPTIONS__ +#else // __USING_SJLJ_EXCEPTIONS__ || __USING_WASM_EXCEPTIONS__ // ip is 1-based index into this table uintptr_t landingPad = readULEB128(&callSitePtr); uintptr_t actionEntry = readULEB128(&callSitePtr); if (--ip == 0) -#endif // __USING_SJLJ_OR_WASM_EXCEPTIONS__ +#endif // __USING_SJLJ_EXCEPTIONS__ || __USING_WASM_EXCEPTIONS__ { // Found the call site containing ip. -#ifndef __USING_SJLJ_OR_WASM_EXCEPTIONS__ +#if !defined(__USING_SJLJ_EXCEPTIONS__) && !defined(__USING_WASM_EXCEPTIONS__) if (landingPad == 0) { // No handler here @@ -713,9 +711,9 @@ static void scan_eh_tab(scan_results &results, _Unwind_Action actions, return; } landingPad = (uintptr_t)lpStart + landingPad; -#else // __USING_SJLJ_OR_WASM_EXCEPTIONS__ +#else // __USING_SJLJ_EXCEPTIONS__ || __USING_WASM_EXCEPTIONS__ ++landingPad; -#endif // __USING_SJLJ_OR_WASM_EXCEPTIONS__ +#endif // __USING_SJLJ_EXCEPTIONS__ || __USING_WASM_EXCEPTIONS__ results.landingPad = landingPad; if (actionEntry == 0) { @@ -843,7 +841,7 @@ static void scan_eh_tab(scan_results &results, _Unwind_Action actions, action += actionOffset; } // there is no break out of this loop, only return } -#ifndef __USING_SJLJ_OR_WASM_EXCEPTIONS__ +#if !defined(__USING_SJLJ_EXCEPTIONS__) && !defined(__USING_WASM_EXCEPTIONS__) else if (ipOffset < start) { // There is no call site for this ip @@ -851,7 +849,7 @@ static void scan_eh_tab(scan_results &results, _Unwind_Action actions, // Possible stack corruption. call_terminate(native_exception, unwind_exception); } -#endif // !__USING_SJLJ_OR_WASM_EXCEPTIONS__ +#endif // !__USING_SJLJ_EXCEPTIONS__ && !__USING_WASM_EXCEPTIONS__ } // there might be some tricky cases which break out of this loop // It is possible that no eh table entry specify how to handle @@ -1313,7 +1311,7 @@ _LIBCXXABI_FUNC_VIS _Unwind_Reason_Code __xlcxx_personality_v1( __attribute__((__alias__("__gxx_personality_v0"))); #endif -} // extern "C" +} // extern "C" } // __cxxabiv1 diff --git a/contrib/libs/cxxsupp/libcxxabi/src/cxa_vector.cpp b/contrib/libs/cxxsupp/libcxxabi/src/cxa_vector.cpp index 099f9f0c1e9..17d942a6e61 100644 --- a/contrib/libs/cxxsupp/libcxxabi/src/cxa_vector.cpp +++ b/contrib/libs/cxxsupp/libcxxabi/src/cxa_vector.cpp @@ -416,6 +416,6 @@ __cxa_vec_delete3(void *array_address, size_t element_size, size_t padding_size, } -} // extern "C" +} // extern "C" } // abi diff --git a/contrib/libs/cxxsupp/libcxxabi/src/demangle/DemangleConfig.h b/contrib/libs/cxxsupp/libcxxabi/src/demangle/DemangleConfig.h index d5e11432d98..d67d89bdb06 100644 --- a/contrib/libs/cxxsupp/libcxxabi/src/demangle/DemangleConfig.h +++ b/contrib/libs/cxxsupp/libcxxabi/src/demangle/DemangleConfig.h @@ -19,7 +19,7 @@ #include "../abort_message.h" #endif -#include <ciso646> +#include <version> #ifdef _MSC_VER // snprintf is implemented in VS 2015 @@ -99,6 +99,11 @@ #define DEMANGLE_FALLTHROUGH #endif +#ifndef DEMANGLE_ASSERT +#include <cassert> +#define DEMANGLE_ASSERT(__expr, __msg) assert((__expr) && (__msg)) +#endif + #define DEMANGLE_NAMESPACE_BEGIN namespace { namespace itanium_demangle { #define DEMANGLE_NAMESPACE_END } } diff --git a/contrib/libs/cxxsupp/libcxxabi/src/demangle/ItaniumDemangle.h b/contrib/libs/cxxsupp/libcxxabi/src/demangle/ItaniumDemangle.h index 54272785977..03282ac68ca 100644 --- a/contrib/libs/cxxsupp/libcxxabi/src/demangle/ItaniumDemangle.h +++ b/contrib/libs/cxxsupp/libcxxabi/src/demangle/ItaniumDemangle.h @@ -21,7 +21,6 @@ #include "Utility.h" #include <__cxxabi_config.h> #include <algorithm> -#include <cassert> #include <cctype> #include <cstdio> #include <cstdlib> @@ -61,13 +60,13 @@ template <class T, size_t N> class PODSmallVector { if (isInline()) { auto *Tmp = static_cast<T *>(std::malloc(NewCap * sizeof(T))); if (Tmp == nullptr) - std::terminate(); + std::abort(); std::copy(First, Last, Tmp); First = Tmp; } else { First = static_cast<T *>(std::realloc(First, NewCap * sizeof(T))); if (First == nullptr) - std::terminate(); + std::abort(); } Last = First + S; Cap = First + NewCap; @@ -129,12 +128,12 @@ public: // NOLINTNEXTLINE(readability-identifier-naming) void pop_back() { - assert(Last != First && "Popping empty vector!"); + DEMANGLE_ASSERT(Last != First, "Popping empty vector!"); --Last; } - void dropBack(size_t Index) { - assert(Index <= size() && "dropBack() can't expand!"); + void shrinkToSize(size_t Index) { + DEMANGLE_ASSERT(Index <= size(), "shrinkToSize() can't expand!"); Last = First + Index; } @@ -144,11 +143,11 @@ public: bool empty() const { return First == Last; } size_t size() const { return static_cast<size_t>(Last - First); } T &back() { - assert(Last != First && "Calling back() on empty vector!"); + DEMANGLE_ASSERT(Last != First, "Calling back() on empty vector!"); return *(Last - 1); } T &operator[](size_t Index) { - assert(Index < size() && "Invalid access!"); + DEMANGLE_ASSERT(Index < size(), "Invalid access!"); return *(begin() + Index); } void clear() { Last = First; } @@ -534,6 +533,23 @@ public: } }; +class TransformedType : public Node { + std::string_view Transform; + Node *BaseType; +public: + TransformedType(std::string_view Transform_, Node *BaseType_) + : Node(KTransformedType), Transform(Transform_), BaseType(BaseType_) {} + + template<typename Fn> void match(Fn F) const { F(Transform, BaseType); } + + void printLeft(OutputBuffer &OB) const override { + OB += Transform; + OB += '('; + BaseType->print(OB); + OB += ')'; + } +}; + struct AbiTagAttr : Node { Node *Base; std::string_view Tag; @@ -873,26 +889,53 @@ public: } }; +/// Represents the explicitly named object parameter. +/// E.g., +/// \code{.cpp} +/// struct Foo { +/// void bar(this Foo && self); +/// }; +/// \endcode +class ExplicitObjectParameter final : public Node { + Node *Base; + +public: + ExplicitObjectParameter(Node *Base_) + : Node(KExplicitObjectParameter), Base(Base_) { + DEMANGLE_ASSERT( + Base != nullptr, + "Creating an ExplicitObjectParameter without a valid Base Node."); + } + + template <typename Fn> void match(Fn F) const { F(Base); } + + void printLeft(OutputBuffer &OB) const override { + OB += "this "; + Base->print(OB); + } +}; + class FunctionEncoding final : public Node { const Node *Ret; const Node *Name; NodeArray Params; const Node *Attrs; + const Node *Requires; Qualifiers CVQuals; FunctionRefQual RefQual; public: FunctionEncoding(const Node *Ret_, const Node *Name_, NodeArray Params_, - const Node *Attrs_, Qualifiers CVQuals_, - FunctionRefQual RefQual_) + const Node *Attrs_, const Node *Requires_, + Qualifiers CVQuals_, FunctionRefQual RefQual_) : Node(KFunctionEncoding, /*RHSComponentCache=*/Cache::Yes, /*ArrayCache=*/Cache::No, /*FunctionCache=*/Cache::Yes), Ret(Ret_), Name(Name_), Params(Params_), Attrs(Attrs_), - CVQuals(CVQuals_), RefQual(RefQual_) {} + Requires(Requires_), CVQuals(CVQuals_), RefQual(RefQual_) {} template<typename Fn> void match(Fn F) const { - F(Ret, Name, Params, Attrs, CVQuals, RefQual); + F(Ret, Name, Params, Attrs, Requires, CVQuals, RefQual); } Qualifiers getCVQuals() const { return CVQuals; } @@ -935,6 +978,11 @@ public: if (Attrs != nullptr) Attrs->print(OB); + + if (Requires != nullptr) { + OB += " requires "; + Requires->print(OB); + } } }; @@ -1006,6 +1054,24 @@ struct NestedName : Node { } }; +struct MemberLikeFriendName : Node { + Node *Qual; + Node *Name; + + MemberLikeFriendName(Node *Qual_, Node *Name_) + : Node(KMemberLikeFriendName), Qual(Qual_), Name(Name_) {} + + template<typename Fn> void match(Fn F) const { F(Qual, Name); } + + std::string_view getBaseName() const override { return Name->getBaseName(); } + + void printLeft(OutputBuffer &OB) const override { + Qual->print(OB); + OB += "::friend "; + Name->print(OB); + } +}; + struct ModuleName : Node { ModuleName *Parent; Node *Name; @@ -1171,6 +1237,24 @@ public: } }; +class TemplateParamQualifiedArg final : public Node { + Node *Param; + Node *Arg; + +public: + TemplateParamQualifiedArg(Node *Param_, Node *Arg_) + : Node(KTemplateParamQualifiedArg), Param(Param_), Arg(Arg_) {} + + template <typename Fn> void match(Fn F) const { F(Param, Arg); } + + Node *getArg() { return Arg; } + + void printLeft(OutputBuffer &OB) const override { + // Don't print Param to keep the output consistent. + Arg->print(OB); + } +}; + /// A template type parameter declaration, 'typename T'. class TypeTemplateParamDecl final : public Node { Node *Name; @@ -1186,6 +1270,26 @@ public: void printRight(OutputBuffer &OB) const override { Name->print(OB); } }; +/// A constrained template type parameter declaration, 'C<U> T'. +class ConstrainedTypeTemplateParamDecl final : public Node { + Node *Constraint; + Node *Name; + +public: + ConstrainedTypeTemplateParamDecl(Node *Constraint_, Node *Name_) + : Node(KConstrainedTypeTemplateParamDecl, Cache::Yes), + Constraint(Constraint_), Name(Name_) {} + + template<typename Fn> void match(Fn F) const { F(Constraint, Name); } + + void printLeft(OutputBuffer &OB) const override { + Constraint->print(OB); + OB += " "; + } + + void printRight(OutputBuffer &OB) const override { Name->print(OB); } +}; + /// A non-type template parameter declaration, 'int N'. class NonTypeTemplateParamDecl final : public Node { Node *Name; @@ -1214,13 +1318,14 @@ public: class TemplateTemplateParamDecl final : public Node { Node *Name; NodeArray Params; + Node *Requires; public: - TemplateTemplateParamDecl(Node *Name_, NodeArray Params_) + TemplateTemplateParamDecl(Node *Name_, NodeArray Params_, Node *Requires_) : Node(KTemplateTemplateParamDecl, Cache::Yes), Name(Name_), - Params(Params_) {} + Params(Params_), Requires(Requires_) {} - template<typename Fn> void match(Fn F) const { F(Name, Params); } + template <typename Fn> void match(Fn F) const { F(Name, Params, Requires); } void printLeft(OutputBuffer &OB) const override { ScopedOverride<unsigned> LT(OB.GtIsGt, 0); @@ -1229,7 +1334,13 @@ public: OB += "> typename "; } - void printRight(OutputBuffer &OB) const override { Name->print(OB); } + void printRight(OutputBuffer &OB) const override { + Name->print(OB); + if (Requires != nullptr) { + OB += " requires "; + Requires->print(OB); + } + } }; /// A template parameter pack declaration, 'typename ...T'. @@ -1326,7 +1437,7 @@ public: /// A variadic template argument. This node represents an occurrence of /// J<something>E in some <template-args>. It isn't itself unexpanded, unless -/// one of it's Elements is. The parser inserts a ParameterPack into the +/// one of its Elements is. The parser inserts a ParameterPack into the /// TemplateParams table if the <template-args> this pack belongs to apply to an /// <encoding>. class TemplateArgumentPack final : public Node { @@ -1392,11 +1503,13 @@ public: class TemplateArgs final : public Node { NodeArray Params; + Node *Requires; public: - TemplateArgs(NodeArray Params_) : Node(KTemplateArgs), Params(Params_) {} + TemplateArgs(NodeArray Params_, Node *Requires_) + : Node(KTemplateArgs), Params(Params_), Requires(Requires_) {} - template<typename Fn> void match(Fn F) const { F(Params); } + template<typename Fn> void match(Fn F) const { F(Params, Requires); } NodeArray getParams() { return Params; } @@ -1405,6 +1518,7 @@ public: OB += "<"; Params.printWithComma(OB); OB += ">"; + // Don't print the requires clause to keep the output simple. } }; @@ -1589,7 +1703,7 @@ public: std::string_view SV = ExpandedSpecialSubstitution::getBaseName(); if (isInstantiation()) { // The instantiations are typedefs that drop the "basic_" prefix. - assert(starts_with(SV, "basic_")); + DEMANGLE_ASSERT(starts_with(SV, "basic_"), ""); SV.remove_prefix(sizeof("basic_") - 1); } return SV; @@ -1655,17 +1769,21 @@ public: class ClosureTypeName : public Node { NodeArray TemplateParams; + const Node *Requires1; NodeArray Params; + const Node *Requires2; std::string_view Count; public: - ClosureTypeName(NodeArray TemplateParams_, NodeArray Params_, + ClosureTypeName(NodeArray TemplateParams_, const Node *Requires1_, + NodeArray Params_, const Node *Requires2_, std::string_view Count_) : Node(KClosureTypeName), TemplateParams(TemplateParams_), - Params(Params_), Count(Count_) {} + Requires1(Requires1_), Params(Params_), Requires2(Requires2_), + Count(Count_) {} template<typename Fn> void match(Fn F) const { - F(TemplateParams, Params, Count); + F(TemplateParams, Requires1, Params, Requires2, Count); } void printDeclarator(OutputBuffer &OB) const { @@ -1675,12 +1793,22 @@ public: TemplateParams.printWithComma(OB); OB += ">"; } + if (Requires1 != nullptr) { + OB += " requires "; + Requires1->print(OB); + OB += " "; + } OB.printOpen(); Params.printWithComma(OB); OB.printClose(); + if (Requires2 != nullptr) { + OB += " requires "; + Requires2->print(OB); + } } void printLeft(OutputBuffer &OB) const override { + // FIXME: This demangling is not particularly readable. OB += "\'lambda"; OB += Count; OB += "\'"; @@ -2309,6 +2437,95 @@ public: } }; +class RequiresExpr : public Node { + NodeArray Parameters; + NodeArray Requirements; +public: + RequiresExpr(NodeArray Parameters_, NodeArray Requirements_) + : Node(KRequiresExpr), Parameters(Parameters_), + Requirements(Requirements_) {} + + template<typename Fn> void match(Fn F) const { F(Parameters, Requirements); } + + void printLeft(OutputBuffer &OB) const override { + OB += "requires"; + if (!Parameters.empty()) { + OB += ' '; + OB.printOpen(); + Parameters.printWithComma(OB); + OB.printClose(); + } + OB += ' '; + OB.printOpen('{'); + for (const Node *Req : Requirements) { + Req->print(OB); + } + OB += ' '; + OB.printClose('}'); + } +}; + +class ExprRequirement : public Node { + const Node *Expr; + bool IsNoexcept; + const Node *TypeConstraint; +public: + ExprRequirement(const Node *Expr_, bool IsNoexcept_, + const Node *TypeConstraint_) + : Node(KExprRequirement), Expr(Expr_), IsNoexcept(IsNoexcept_), + TypeConstraint(TypeConstraint_) {} + + template <typename Fn> void match(Fn F) const { + F(Expr, IsNoexcept, TypeConstraint); + } + + void printLeft(OutputBuffer &OB) const override { + OB += " "; + if (IsNoexcept || TypeConstraint) + OB.printOpen('{'); + Expr->print(OB); + if (IsNoexcept || TypeConstraint) + OB.printClose('}'); + if (IsNoexcept) + OB += " noexcept"; + if (TypeConstraint) { + OB += " -> "; + TypeConstraint->print(OB); + } + OB += ';'; + } +}; + +class TypeRequirement : public Node { + const Node *Type; +public: + TypeRequirement(const Node *Type_) + : Node(KTypeRequirement), Type(Type_) {} + + template <typename Fn> void match(Fn F) const { F(Type); } + + void printLeft(OutputBuffer &OB) const override { + OB += " typename "; + Type->print(OB); + OB += ';'; + } +}; + +class NestedRequirement : public Node { + const Node *Constraint; +public: + NestedRequirement(const Node *Constraint_) + : Node(KNestedRequirement), Constraint(Constraint_) {} + + template <typename Fn> void match(Fn F) const { F(Constraint); } + + void printLeft(OutputBuffer &OB) const override { + OB += " requires "; + Constraint->print(OB); + OB += ';'; + } +}; + template <class Float> struct FloatData; namespace float_literal_impl { @@ -2377,7 +2594,7 @@ void Node::visit(Fn F) const { return F(static_cast<const X *>(this)); #include "ItaniumNodes.def" } - assert(0 && "unknown mangling node kind"); + DEMANGLE_ASSERT(0, "unknown mangling node kind"); } /// Determine the kind of a node from its type. @@ -2403,6 +2620,8 @@ template <typename Derived, typename Alloc> struct AbstractManglingParser { // table. PODSmallVector<Node *, 32> Subs; + // A list of template argument values corresponding to a template parameter + // list. using TemplateParamList = PODSmallVector<Node *, 8>; class ScopedTemplateParamList { @@ -2417,9 +2636,11 @@ template <typename Derived, typename Alloc> struct AbstractManglingParser { Parser->TemplateParams.push_back(&Params); } ~ScopedTemplateParamList() { - assert(Parser->TemplateParams.size() >= OldNumTemplateParamLists); - Parser->TemplateParams.dropBack(OldNumTemplateParamLists); + DEMANGLE_ASSERT(Parser->TemplateParams.size() >= OldNumTemplateParamLists, + ""); + Parser->TemplateParams.shrinkToSize(OldNumTemplateParamLists); } + TemplateParamList *params() { return &Params; } }; // Template parameter table. Like the above, but referenced like "T42_". @@ -2434,12 +2655,31 @@ template <typename Derived, typename Alloc> struct AbstractManglingParser { // parameter list, the corresponding parameter list pointer will be null. PODSmallVector<TemplateParamList *, 4> TemplateParams; + class SaveTemplateParams { + AbstractManglingParser *Parser; + decltype(TemplateParams) OldParams; + decltype(OuterTemplateParams) OldOuterParams; + + public: + SaveTemplateParams(AbstractManglingParser *TheParser) : Parser(TheParser) { + OldParams = std::move(Parser->TemplateParams); + OldOuterParams = std::move(Parser->OuterTemplateParams); + Parser->TemplateParams.clear(); + Parser->OuterTemplateParams.clear(); + } + ~SaveTemplateParams() { + Parser->TemplateParams = std::move(OldParams); + Parser->OuterTemplateParams = std::move(OldOuterParams); + } + }; + // Set of unresolved forward <template-param> references. These can occur in a // conversion operator's type, and are resolved in the enclosing <encoding>. PODSmallVector<ForwardTemplateReference *, 4> ForwardTemplateRefs; bool TryToParseTemplateArgs = true; bool PermitForwardTemplateReferences = false; + bool InConstraintExpr = false; size_t ParsingLambdaParamsAtLevel = (size_t)-1; unsigned NumSyntheticTemplateParameters[3] = {}; @@ -2478,10 +2718,10 @@ template <typename Derived, typename Alloc> struct AbstractManglingParser { } NodeArray popTrailingNodeArray(size_t FromPosition) { - assert(FromPosition <= Names.size()); + DEMANGLE_ASSERT(FromPosition <= Names.size(), ""); NodeArray res = makeNodeArray(Names.begin() + (long)FromPosition, Names.end()); - Names.dropBack(FromPosition); + Names.shrinkToSize(FromPosition); return res; } @@ -2519,11 +2759,16 @@ template <typename Derived, typename Alloc> struct AbstractManglingParser { bool parseSeqId(size_t *Out); Node *parseSubstitution(); Node *parseTemplateParam(); - Node *parseTemplateParamDecl(); + Node *parseTemplateParamDecl(TemplateParamList *Params); Node *parseTemplateArgs(bool TagTemplates = false); Node *parseTemplateArg(); - /// Parse the <expr> production. + bool isTemplateParamDecl() { + return look() == 'T' && + std::string_view("yptnk").find(look(1)) != std::string_view::npos; + } + + /// Parse the <expression> production. Node *parseExpr(); Node *parsePrefixExpr(std::string_view Kind, Node::Prec Prec); Node *parseBinaryExpr(std::string_view Kind, Node::Prec Prec); @@ -2536,6 +2781,8 @@ template <typename Derived, typename Alloc> struct AbstractManglingParser { Node *parseFoldExpr(); Node *parsePointerToMemberConversionExpr(Node::Prec Prec); Node *parseSubobjectExpr(); + Node *parseConstraintExpr(); + Node *parseRequiresExpr(); /// Parse the <type> production. Node *parseType(); @@ -2547,7 +2794,7 @@ template <typename Derived, typename Alloc> struct AbstractManglingParser { Node *parseClassEnumType(); Node *parseQualifiedType(); - Node *parseEncoding(); + Node *parseEncoding(bool ParseParams = true); bool parseCallOffset(); Node *parseSpecialName(); @@ -2559,6 +2806,7 @@ template <typename Derived, typename Alloc> struct AbstractManglingParser { Qualifiers CVQualifiers = QualNone; FunctionRefQual ReferenceQualifier = FrefQualNone; size_t ForwardTemplateRefsBegin; + bool HasExplicitObjectParameter = false; NameState(AbstractManglingParser *Enclosing) : ForwardTemplateRefsBegin(Enclosing->ForwardTemplateRefs.size()) {} @@ -2574,7 +2822,7 @@ template <typename Derived, typename Alloc> struct AbstractManglingParser { return true; ForwardTemplateRefs[I]->Ref = (*TemplateParams[0])[Idx]; } - ForwardTemplateRefs.dropBack(State.ForwardTemplateRefsBegin); + ForwardTemplateRefs.shrinkToSize(State.ForwardTemplateRefsBegin); return false; } @@ -2638,8 +2886,8 @@ template <typename Derived, typename Alloc> struct AbstractManglingParser { std::string_view getSymbol() const { std::string_view Res = Name; if (Kind < Unnameable) { - assert(starts_with(Res, "operator") && - "operator name does not start with 'operator'"); + DEMANGLE_ASSERT(starts_with(Res, "operator"), + "operator name does not start with 'operator'"); Res.remove_prefix(sizeof("operator") - 1); if (starts_with(Res, ' ')) Res.remove_prefix(1); @@ -2663,7 +2911,7 @@ template <typename Derived, typename Alloc> struct AbstractManglingParser { Node *parseDestructorName(); /// Top-level entry point into the parser. - Node *parse(); + Node *parse(bool ParseParams = true); }; const char* parse_discriminator(const char* first, const char* last); @@ -2727,6 +2975,10 @@ Node *AbstractManglingParser<Derived, Alloc>::parseLocalName(NameState *State) { return make<LocalName>(Encoding, StringLitName); } + // The template parameters of the inner name are unrelated to those of the + // enclosing context. + SaveTemplateParams SaveTemplateParamsScope(this); + if (consumeIf('d')) { parseNumber(true); if (!consumeIf('_')) @@ -2782,9 +3034,9 @@ AbstractManglingParser<Derived, Alloc>::parseUnscopedName(NameState *State, return Res; } -// <unqualified-name> ::= [<module-name>] L? <operator-name> [<abi-tags>] +// <unqualified-name> ::= [<module-name>] F? L? <operator-name> [<abi-tags>] // ::= [<module-name>] <ctor-dtor-name> [<abi-tags>] -// ::= [<module-name>] L? <source-name> [<abi-tags>] +// ::= [<module-name>] F? L? <source-name> [<abi-tags>] // ::= [<module-name>] L? <unnamed-type-name> [<abi-tags>] // # structured binding declaration // ::= [<module-name>] L? DC <source-name>+ E @@ -2794,6 +3046,8 @@ Node *AbstractManglingParser<Derived, Alloc>::parseUnqualifiedName( if (getDerived().parseModuleNameOpt(Module)) return nullptr; + bool IsMemberLikeFriend = Scope && consumeIf('F'); + consumeIf('L'); Node *Result; @@ -2824,7 +3078,9 @@ Node *AbstractManglingParser<Derived, Alloc>::parseUnqualifiedName( Result = make<ModuleEntity>(Module, Result); if (Result != nullptr) Result = getDerived().parseAbiTags(Result); - if (Result != nullptr && Scope != nullptr) + if (Result != nullptr && IsMemberLikeFriend) + Result = make<MemberLikeFriendName>(Scope, Result); + else if (Result != nullptr && Scope != nullptr) Result = make<NestedName>(Scope, Result); return Result; @@ -2856,7 +3112,8 @@ bool AbstractManglingParser<Derived, Alloc>::parseModuleNameOpt( // // <closure-type-name> ::= Ul <lambda-sig> E [ <nonnegative number> ] _ // -// <lambda-sig> ::= <parameter type>+ # Parameter types or "v" if the lambda has no parameters +// <lambda-sig> ::= <template-param-decl>* [Q <requires-clause expression>] +// <parameter type>+ # or "v" if the lambda has no parameters template <typename Derived, typename Alloc> Node * AbstractManglingParser<Derived, Alloc>::parseUnnamedTypeName(NameState *State) { @@ -2877,10 +3134,10 @@ AbstractManglingParser<Derived, Alloc>::parseUnnamedTypeName(NameState *State) { ScopedTemplateParamList LambdaTemplateParams(this); size_t ParamsBegin = Names.size(); - while (look() == 'T' && - std::string_view("yptn").find(look(1)) != std::string_view::npos) { - Node *T = parseTemplateParamDecl(); - if (!T) + while (getDerived().isTemplateParamDecl()) { + Node *T = + getDerived().parseTemplateParamDecl(LambdaTemplateParams.params()); + if (T == nullptr) return nullptr; Names.push_back(T); } @@ -2911,20 +3168,38 @@ AbstractManglingParser<Derived, Alloc>::parseUnnamedTypeName(NameState *State) { if (TempParams.empty()) TemplateParams.pop_back(); - if (!consumeIf("vE")) { + Node *Requires1 = nullptr; + if (consumeIf('Q')) { + Requires1 = getDerived().parseConstraintExpr(); + if (Requires1 == nullptr) + return nullptr; + } + + if (!consumeIf("v")) { do { Node *P = getDerived().parseType(); if (P == nullptr) return nullptr; Names.push_back(P); - } while (!consumeIf('E')); + } while (look() != 'E' && look() != 'Q'); } NodeArray Params = popTrailingNodeArray(ParamsBegin); + Node *Requires2 = nullptr; + if (consumeIf('Q')) { + Requires2 = getDerived().parseConstraintExpr(); + if (Requires2 == nullptr) + return nullptr; + } + + if (!consumeIf('E')) + return nullptr; + std::string_view Count = parseNumber(); if (!consumeIf('_')) return nullptr; - return make<ClosureTypeName>(TempParams, Params, Count); + return make<ClosureTypeName>(TempParams, Requires1, Params, Requires2, + Count); } if (consumeIf("Ub")) { (void)parseNumber(); @@ -3190,15 +3465,25 @@ AbstractManglingParser<Derived, Alloc>::parseNestedName(NameState *State) { if (!consumeIf('N')) return nullptr; - Qualifiers CVTmp = parseCVQualifiers(); - if (State) State->CVQualifiers = CVTmp; + // 'H' specifies that the encoding that follows + // has an explicit object parameter. + if (!consumeIf('H')) { + Qualifiers CVTmp = parseCVQualifiers(); + if (State) + State->CVQualifiers = CVTmp; - if (consumeIf('O')) { - if (State) State->ReferenceQualifier = FrefQualRValue; - } else if (consumeIf('R')) { - if (State) State->ReferenceQualifier = FrefQualLValue; - } else { - if (State) State->ReferenceQualifier = FrefQualNone; + if (consumeIf('O')) { + if (State) + State->ReferenceQualifier = FrefQualRValue; + } else if (consumeIf('R')) { + if (State) + State->ReferenceQualifier = FrefQualLValue; + } else { + if (State) + State->ReferenceQualifier = FrefQualNone; + } + } else if (State) { + State->HasExplicitObjectParameter = true; } Node *SoFar = nullptr; @@ -3446,7 +3731,7 @@ Node *AbstractManglingParser<Derived, Alloc>::parseUnresolvedName(bool Global) { } } - assert(SoFar != nullptr); + DEMANGLE_ASSERT(SoFar != nullptr, ""); Node *Base = getDerived().parseBaseUnresolvedName(); if (Base == nullptr) @@ -3894,7 +4179,15 @@ Node *AbstractManglingParser<Derived, Alloc>::parseType() { // Typically, <builtin-type>s are not considered substitution candidates, // but the exception to that exception is vendor extended types (Itanium C++ // ABI 5.9.1). - Result = make<NameType>(Res); + if (consumeIf('I')) { + Node *BaseType = parseType(); + if (BaseType == nullptr) + return nullptr; + if (!consumeIf('E')) + return nullptr; + Result = make<TransformedType>(Res, BaseType); + } else + Result = make<NameType>(Res); break; } case 'D': @@ -3961,6 +4254,17 @@ Node *AbstractManglingParser<Derived, Alloc>::parseType() { case 'c': First += 2; return make<NameType>("decltype(auto)"); + // ::= Dk <type-constraint> # constrained auto + // ::= DK <type-constraint> # constrained decltype(auto) + case 'k': + case 'K': { + std::string_view Kind = look(1) == 'k' ? " auto" : " decltype(auto)"; + First += 2; + Node *Constraint = getDerived().parseName(); + if (!Constraint) + return nullptr; + return make<PostfixQualifiedType>(Constraint, Kind); + } // ::= Dn # std::nullptr_t (i.e., decltype(nullptr)) case 'n': First += 2; @@ -4512,6 +4816,75 @@ Node *AbstractManglingParser<Derived, Alloc>::parseSubobjectExpr() { Ty, Expr, Offset, popTrailingNodeArray(SelectorsBegin), OnePastTheEnd); } +template <typename Derived, typename Alloc> +Node *AbstractManglingParser<Derived, Alloc>::parseConstraintExpr() { + // Within this expression, all enclosing template parameter lists are in + // scope. + ScopedOverride<bool> SaveInConstraintExpr(InConstraintExpr, true); + return getDerived().parseExpr(); +} + +template <typename Derived, typename Alloc> +Node *AbstractManglingParser<Derived, Alloc>::parseRequiresExpr() { + NodeArray Params; + if (consumeIf("rQ")) { + // <expression> ::= rQ <bare-function-type> _ <requirement>+ E + size_t ParamsBegin = Names.size(); + while (!consumeIf('_')) { + Node *Type = getDerived().parseType(); + if (Type == nullptr) + return nullptr; + Names.push_back(Type); + } + Params = popTrailingNodeArray(ParamsBegin); + } else if (!consumeIf("rq")) { + // <expression> ::= rq <requirement>+ E + return nullptr; + } + + size_t ReqsBegin = Names.size(); + do { + Node *Constraint = nullptr; + if (consumeIf('X')) { + // <requirement> ::= X <expression> [N] [R <type-constraint>] + Node *Expr = getDerived().parseExpr(); + if (Expr == nullptr) + return nullptr; + bool Noexcept = consumeIf('N'); + Node *TypeReq = nullptr; + if (consumeIf('R')) { + TypeReq = getDerived().parseName(); + if (TypeReq == nullptr) + return nullptr; + } + Constraint = make<ExprRequirement>(Expr, Noexcept, TypeReq); + } else if (consumeIf('T')) { + // <requirement> ::= T <type> + Node *Type = getDerived().parseType(); + if (Type == nullptr) + return nullptr; + Constraint = make<TypeRequirement>(Type); + } else if (consumeIf('Q')) { + // <requirement> ::= Q <constraint-expression> + // + // FIXME: We use <expression> instead of <constraint-expression>. Either + // the requires expression is already inside a constraint expression, in + // which case it makes no difference, or we're in a requires-expression + // that might be partially-substituted, where the language behavior is + // not yet settled and clang mangles after substitution. + Node *NestedReq = getDerived().parseExpr(); + if (NestedReq == nullptr) + return nullptr; + Constraint = make<NestedRequirement>(NestedReq); + } + if (Constraint == nullptr) + return nullptr; + Names.push_back(Constraint); + } while (!consumeIf('E')); + + return make<RequiresExpr>(Params, popTrailingNodeArray(ReqsBegin)); +} + // <expression> ::= <unary operator-name> <expression> // ::= <binary operator-name> <expression> <expression> // ::= <ternary operator-name> <expression> <expression> <expression> @@ -4748,6 +5121,8 @@ Node *AbstractManglingParser<Derived, Alloc>::parseExpr() { return Ex; return make<EnclosingExpr>("noexcept ", Ex, Node::Prec::Unary); } + if (look() == 'r' && (look(1) == 'q' || look(1) == 'Q')) + return parseRequiresExpr(); if (consumeIf("so")) return parseSubobjectExpr(); if (consumeIf("sp")) { @@ -5026,29 +5401,14 @@ Node *AbstractManglingParser<Derived, Alloc>::parseSpecialName() { } // <encoding> ::= <function name> <bare-function-type> +// [`Q` <requires-clause expr>] // ::= <data name> // ::= <special-name> template <typename Derived, typename Alloc> -Node *AbstractManglingParser<Derived, Alloc>::parseEncoding() { +Node *AbstractManglingParser<Derived, Alloc>::parseEncoding(bool ParseParams) { // The template parameters of an encoding are unrelated to those of the // enclosing context. - class SaveTemplateParams { - AbstractManglingParser *Parser; - decltype(TemplateParams) OldParams; - decltype(OuterTemplateParams) OldOuterParams; - - public: - SaveTemplateParams(AbstractManglingParser *TheParser) : Parser(TheParser) { - OldParams = std::move(Parser->TemplateParams); - OldOuterParams = std::move(Parser->OuterTemplateParams); - Parser->TemplateParams.clear(); - Parser->OuterTemplateParams.clear(); - } - ~SaveTemplateParams() { - Parser->TemplateParams = std::move(OldParams); - Parser->OuterTemplateParams = std::move(OldOuterParams); - } - } SaveTemplateParams(this); + SaveTemplateParams SaveTemplateParamsScope(this); if (look() == 'G' || look() == 'T') return getDerived().parseSpecialName(); @@ -5071,6 +5431,16 @@ Node *AbstractManglingParser<Derived, Alloc>::parseEncoding() { if (IsEndOfEncoding()) return Name; + // ParseParams may be false at the top level only, when called from parse(). + // For example in the mangled name _Z3fooILZ3BarEET_f, ParseParams may be + // false when demangling 3fooILZ3BarEET_f but is always true when demangling + // 3Bar. + if (!ParseParams) { + while (consume()) + ; + return Name; + } + Node *Attrs = nullptr; if (consumeIf("Ua9enable_ifI")) { size_t BeforeArgs = Names.size(); @@ -5092,22 +5462,35 @@ Node *AbstractManglingParser<Derived, Alloc>::parseEncoding() { return nullptr; } - if (consumeIf('v')) - return make<FunctionEncoding>(ReturnType, Name, NodeArray(), - Attrs, NameInfo.CVQualifiers, - NameInfo.ReferenceQualifier); + NodeArray Params; + if (!consumeIf('v')) { + size_t ParamsBegin = Names.size(); + do { + Node *Ty = getDerived().parseType(); + if (Ty == nullptr) + return nullptr; - size_t ParamsBegin = Names.size(); - do { - Node *Ty = getDerived().parseType(); - if (Ty == nullptr) + const bool IsFirstParam = ParamsBegin == Names.size(); + if (NameInfo.HasExplicitObjectParameter && IsFirstParam) + Ty = make<ExplicitObjectParameter>(Ty); + + if (Ty == nullptr) + return nullptr; + + Names.push_back(Ty); + } while (!IsEndOfEncoding() && look() != 'Q'); + Params = popTrailingNodeArray(ParamsBegin); + } + + Node *Requires = nullptr; + if (consumeIf('Q')) { + Requires = getDerived().parseConstraintExpr(); + if (!Requires) return nullptr; - Names.push_back(Ty); - } while (!IsEndOfEncoding()); + } - return make<FunctionEncoding>(ReturnType, Name, - popTrailingNodeArray(ParamsBegin), - Attrs, NameInfo.CVQualifiers, + return make<FunctionEncoding>(ReturnType, Name, Params, Attrs, Requires, + NameInfo.CVQualifiers, NameInfo.ReferenceQualifier); } @@ -5134,7 +5517,8 @@ template <> struct FloatData<long double> { #if defined(__mips__) && defined(__mips_n64) || defined(__aarch64__) || \ - defined(__wasm__) || defined(__riscv) || defined(__loongarch__) + defined(__wasm__) || defined(__riscv) || defined(__loongarch__) || \ + defined(__ve__) static const size_t mangled_size = 32; #elif defined(__arm__) || defined(__mips__) || defined(__hexagon__) static const size_t mangled_size = 16; @@ -5268,6 +5652,7 @@ Node *AbstractManglingParser<Derived, Alloc>::parseSubstitution() { // ::= TL <level-1> _ <parameter-2 non-negative number> _ template <typename Derived, typename Alloc> Node *AbstractManglingParser<Derived, Alloc>::parseTemplateParam() { + const char *Begin = First; if (!consumeIf('T')) return nullptr; @@ -5289,6 +5674,14 @@ Node *AbstractManglingParser<Derived, Alloc>::parseTemplateParam() { return nullptr; } + // We don't track enclosing template parameter levels well enough to reliably + // substitute them all within a <constraint-expression>, so print the + // parameter numbering instead for now. + // TODO: Track all enclosing template parameters and substitute them here. + if (InConstraintExpr) { + return make<NameType>(std::string_view(Begin, First - 1 - Begin)); + } + // If we're in a context where this <template-param> refers to a // <template-arg> further ahead in the mangled name (currently just conversion // operator types), then we should only look it up in the right context. @@ -5297,7 +5690,8 @@ Node *AbstractManglingParser<Derived, Alloc>::parseTemplateParam() { Node *ForwardRef = make<ForwardTemplateReference>(Index); if (!ForwardRef) return nullptr; - assert(ForwardRef->getKind() == Node::KForwardTemplateReference); + DEMANGLE_ASSERT(ForwardRef->getKind() == Node::KForwardTemplateReference, + ""); ForwardTemplateRefs.push_back( static_cast<ForwardTemplateReference *>(ForwardRef)); return ForwardRef; @@ -5326,11 +5720,13 @@ Node *AbstractManglingParser<Derived, Alloc>::parseTemplateParam() { // ::= Tt <template-param-decl>* E # template parameter // ::= Tp <template-param-decl> # parameter pack template <typename Derived, typename Alloc> -Node *AbstractManglingParser<Derived, Alloc>::parseTemplateParamDecl() { +Node *AbstractManglingParser<Derived, Alloc>::parseTemplateParamDecl( + TemplateParamList *Params) { auto InventTemplateParamName = [&](TemplateParamKind Kind) { unsigned Index = NumSyntheticTemplateParameters[(int)Kind]++; Node *N = make<SyntheticTemplateParamName>(Kind, Index); - if (N) TemplateParams.back()->push_back(N); + if (N && Params) + Params->push_back(N); return N; }; @@ -5341,6 +5737,16 @@ Node *AbstractManglingParser<Derived, Alloc>::parseTemplateParamDecl() { return make<TypeTemplateParamDecl>(Name); } + if (consumeIf("Tk")) { + Node *Constraint = getDerived().parseName(); + if (!Constraint) + return nullptr; + Node *Name = InventTemplateParamName(TemplateParamKind::Type); + if (!Name) + return nullptr; + return make<ConstrainedTypeTemplateParamDecl>(Constraint, Name); + } + if (consumeIf("Tn")) { Node *Name = InventTemplateParamName(TemplateParamKind::NonType); if (!Name) @@ -5357,18 +5763,25 @@ Node *AbstractManglingParser<Derived, Alloc>::parseTemplateParamDecl() { return nullptr; size_t ParamsBegin = Names.size(); ScopedTemplateParamList TemplateTemplateParamParams(this); - while (!consumeIf("E")) { - Node *P = parseTemplateParamDecl(); + Node *Requires = nullptr; + while (!consumeIf('E')) { + Node *P = parseTemplateParamDecl(TemplateTemplateParamParams.params()); if (!P) return nullptr; Names.push_back(P); + if (consumeIf('Q')) { + Requires = getDerived().parseConstraintExpr(); + if (Requires == nullptr || !consumeIf('E')) + return nullptr; + break; + } } - NodeArray Params = popTrailingNodeArray(ParamsBegin); - return make<TemplateTemplateParamDecl>(Name, Params); + NodeArray InnerParams = popTrailingNodeArray(ParamsBegin); + return make<TemplateTemplateParamDecl>(Name, InnerParams, Requires); } if (consumeIf("Tp")) { - Node *P = parseTemplateParamDecl(); + Node *P = parseTemplateParamDecl(Params); if (!P) return nullptr; return make<TemplateParamPackDecl>(P); @@ -5382,6 +5795,7 @@ Node *AbstractManglingParser<Derived, Alloc>::parseTemplateParamDecl() { // ::= <expr-primary> # simple expressions // ::= J <template-arg>* E # argument pack // ::= LZ <encoding> E # extension +// ::= <template-param-decl> <template-arg> template <typename Derived, typename Alloc> Node *AbstractManglingParser<Derived, Alloc>::parseTemplateArg() { switch (look()) { @@ -5416,6 +5830,18 @@ Node *AbstractManglingParser<Derived, Alloc>::parseTemplateArg() { // ::= <expr-primary> # simple expressions return getDerived().parseExprPrimary(); } + case 'T': { + // Either <template-param> or a <template-param-decl> <template-arg>. + if (!getDerived().isTemplateParamDecl()) + return getDerived().parseType(); + Node *Param = getDerived().parseTemplateParamDecl(nullptr); + if (!Param) + return nullptr; + Node *Arg = getDerived().parseTemplateArg(); + if (!Arg) + return nullptr; + return make<TemplateParamQualifiedArg>(Param, Arg); + } default: return getDerived().parseType(); } @@ -5438,30 +5864,39 @@ AbstractManglingParser<Derived, Alloc>::parseTemplateArgs(bool TagTemplates) { } size_t ArgsBegin = Names.size(); + Node *Requires = nullptr; while (!consumeIf('E')) { if (TagTemplates) { - auto OldParams = std::move(TemplateParams); Node *Arg = getDerived().parseTemplateArg(); - TemplateParams = std::move(OldParams); if (Arg == nullptr) return nullptr; Names.push_back(Arg); Node *TableEntry = Arg; + if (Arg->getKind() == Node::KTemplateParamQualifiedArg) { + TableEntry = + static_cast<TemplateParamQualifiedArg *>(TableEntry)->getArg(); + } if (Arg->getKind() == Node::KTemplateArgumentPack) { TableEntry = make<ParameterPack>( static_cast<TemplateArgumentPack*>(TableEntry)->getElements()); if (!TableEntry) return nullptr; } - TemplateParams.back()->push_back(TableEntry); + OuterTemplateParams.push_back(TableEntry); } else { Node *Arg = getDerived().parseTemplateArg(); if (Arg == nullptr) return nullptr; Names.push_back(Arg); } + if (consumeIf('Q')) { + Requires = getDerived().parseConstraintExpr(); + if (!Requires || !consumeIf('E')) + return nullptr; + break; + } } - return make<TemplateArgs>(popTrailingNodeArray(ArgsBegin)); + return make<TemplateArgs>(popTrailingNodeArray(ArgsBegin), Requires); } // <mangled-name> ::= _Z <encoding> @@ -5470,9 +5905,9 @@ AbstractManglingParser<Derived, Alloc>::parseTemplateArgs(bool TagTemplates) { // extension ::= ___Z <encoding> _block_invoke<decimal-digit>+ // extension ::= ___Z <encoding> _block_invoke_<decimal-digit>+ template <typename Derived, typename Alloc> -Node *AbstractManglingParser<Derived, Alloc>::parse() { +Node *AbstractManglingParser<Derived, Alloc>::parse(bool ParseParams) { if (consumeIf("_Z") || consumeIf("__Z")) { - Node *Encoding = getDerived().parseEncoding(); + Node *Encoding = getDerived().parseEncoding(ParseParams); if (Encoding == nullptr) return nullptr; if (look() == '.') { @@ -5486,7 +5921,7 @@ Node *AbstractManglingParser<Derived, Alloc>::parse() { } if (consumeIf("___Z") || consumeIf("____Z")) { - Node *Encoding = getDerived().parseEncoding(); + Node *Encoding = getDerived().parseEncoding(ParseParams); if (Encoding == nullptr || !consumeIf("_block_invoke")) return nullptr; bool RequireNumber = consumeIf('_'); diff --git a/contrib/libs/cxxsupp/libcxxabi/src/demangle/ItaniumNodes.def b/contrib/libs/cxxsupp/libcxxabi/src/demangle/ItaniumNodes.def index f615cb9fadb..18f5d52b47e 100644 --- a/contrib/libs/cxxsupp/libcxxabi/src/demangle/ItaniumNodes.def +++ b/contrib/libs/cxxsupp/libcxxabi/src/demangle/ItaniumNodes.def @@ -19,6 +19,7 @@ NODE(QualType) NODE(ConversionOperatorType) NODE(PostfixQualifiedType) NODE(ElaboratedTypeSpefType) +NODE(TransformedType) NODE(NameType) NODE(AbiTagAttr) NODE(EnableIfAttr) @@ -36,6 +37,7 @@ NODE(SpecialName) NODE(CtorVtableSpecialName) NODE(QualifiedName) NODE(NestedName) +NODE(MemberLikeFriendName) NODE(LocalName) NODE(ModuleName) NODE(ModuleEntity) @@ -44,7 +46,9 @@ NODE(PixelVectorType) NODE(BinaryFPType) NODE(BitIntType) NODE(SyntheticTemplateParamName) +NODE(TemplateParamQualifiedArg) NODE(TypeTemplateParamDecl) +NODE(ConstrainedTypeTemplateParamDecl) NODE(NonTypeTemplateParamDecl) NODE(TemplateTemplateParamDecl) NODE(TemplateParamPackDecl) @@ -91,5 +95,10 @@ NODE(DoubleLiteral) NODE(LongDoubleLiteral) NODE(BracedExpr) NODE(BracedRangeExpr) +NODE(RequiresExpr) +NODE(ExprRequirement) +NODE(TypeRequirement) +NODE(NestedRequirement) +NODE(ExplicitObjectParameter) #undef NODE diff --git a/contrib/libs/cxxsupp/libcxxabi/src/demangle/README.txt b/contrib/libs/cxxsupp/libcxxabi/src/demangle/README.txt index 76470f61f95..d38f6abbb02 100644 --- a/contrib/libs/cxxsupp/libcxxabi/src/demangle/README.txt +++ b/contrib/libs/cxxsupp/libcxxabi/src/demangle/README.txt @@ -34,7 +34,7 @@ differences, we want to keep the "core" generic demangling library identical between both copies to simplify development and testing. If you're working on the generic library, then do the work first in -libcxxabi, then run the cp-to-llvm.sh script in src/demangle. This +libcxxabi, then run libcxxabi/src/demangle/cp-to-llvm.sh. This script takes as an optional argument the path to llvm, and copies the changes you made to libcxxabi over. Note that this script just blindly overwrites all changes to the generic library in llvm, so be diff --git a/contrib/libs/cxxsupp/libcxxabi/src/demangle/Utility.h b/contrib/libs/cxxsupp/libcxxabi/src/demangle/Utility.h index 8370633aceb..f1fad35d60d 100644 --- a/contrib/libs/cxxsupp/libcxxabi/src/demangle/Utility.h +++ b/contrib/libs/cxxsupp/libcxxabi/src/demangle/Utility.h @@ -19,11 +19,9 @@ #include "DemangleConfig.h" #include <array> -#include <cassert> #include <cstdint> #include <cstdlib> #include <cstring> -#include <exception> #include <limits> #include <string_view> @@ -49,7 +47,7 @@ class OutputBuffer { BufferCapacity = Need; Buffer = static_cast<char *>(std::realloc(Buffer, BufferCapacity)); if (Buffer == nullptr) - std::terminate(); + std::abort(); } } @@ -160,7 +158,7 @@ public: } void insert(size_t Pos, const char *S, size_t N) { - assert(Pos <= CurrentPosition); + DEMANGLE_ASSERT(Pos <= CurrentPosition, ""); if (N == 0) return; grow(N); @@ -173,7 +171,7 @@ public: void setCurrentPosition(size_t NewPos) { CurrentPosition = NewPos; } char back() const { - assert(CurrentPosition); + DEMANGLE_ASSERT(CurrentPosition, ""); return Buffer[CurrentPosition - 1]; } diff --git a/contrib/libs/cxxsupp/libcxxabi/src/fallback_malloc.cpp b/contrib/libs/cxxsupp/libcxxabi/src/fallback_malloc.cpp index 11e79740ffb..76bd2e9bcd9 100644 --- a/contrib/libs/cxxsupp/libcxxabi/src/fallback_malloc.cpp +++ b/contrib/libs/cxxsupp/libcxxabi/src/fallback_malloc.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include "fallback_malloc.h" +#include "abort_message.h" #include <__thread/support.h> #ifndef _LIBCXXABI_HAS_NO_THREADS @@ -16,7 +17,7 @@ #endif #include <__memory/aligned_alloc.h> -#include <assert.h> +#include <__assert> #include <stdlib.h> // for malloc, calloc, free #include <string.h> // for memset @@ -142,7 +143,7 @@ void* fallback_malloc(size_t len) { // Check the invariant that all heap_nodes pointers 'p' are aligned // so that 'p + 1' has an alignment of at least RequiredAlignment - assert(reinterpret_cast<size_t>(p + 1) % RequiredAlignment == 0); + _LIBCXXABI_ASSERT(reinterpret_cast<size_t>(p + 1) % RequiredAlignment == 0, ""); // Calculate the number of extra padding elements needed in order // to split 'p' and create a properly aligned heap_node from the tail @@ -163,7 +164,7 @@ void* fallback_malloc(size_t len) { q->next_node = 0; q->len = static_cast<heap_size>(aligned_nelems); void* ptr = q + 1; - assert(reinterpret_cast<size_t>(ptr) % RequiredAlignment == 0); + _LIBCXXABI_ASSERT(reinterpret_cast<size_t>(ptr) % RequiredAlignment == 0, ""); return ptr; } @@ -176,7 +177,7 @@ void* fallback_malloc(size_t len) { prev->next_node = p->next_node; p->next_node = 0; void* ptr = p + 1; - assert(reinterpret_cast<size_t>(ptr) % RequiredAlignment == 0); + _LIBCXXABI_ASSERT(reinterpret_cast<size_t>(ptr) % RequiredAlignment == 0, ""); return ptr; } } diff --git a/contrib/libs/cxxsupp/libcxxabi/src/private_typeinfo.cpp b/contrib/libs/cxxsupp/libcxxabi/src/private_typeinfo.cpp index 85232a494ad..88fa7b5d65e 100644 --- a/contrib/libs/cxxsupp/libcxxabi/src/private_typeinfo.cpp +++ b/contrib/libs/cxxsupp/libcxxabi/src/private_typeinfo.cpp @@ -42,6 +42,7 @@ // is_equal() with use_strcmp=false so the string names are not compared. #include <cstdint> +#include <cassert> #include <string.h> #ifdef _LIBCXXABI_FORGIVING_DYNAMIC_CAST @@ -75,6 +76,242 @@ static inline ptrdiff_t update_offset_to_base(const char* vtable, namespace __cxxabiv1 { +namespace { + +struct derived_object_info { + const void* dynamic_ptr; + const __class_type_info* dynamic_type; + std::ptrdiff_t offset_to_derived; +}; + +/// A helper function that gets (dynamic_ptr, dynamic_type, offset_to_derived) from static_ptr. +void dyn_cast_get_derived_info(derived_object_info* info, const void* static_ptr) +{ +#if __has_feature(cxx_abi_relative_vtable) + // The vtable address will point to the first virtual function, which is 8 + // bytes after the start of the vtable (4 for the offset from top + 4 for + // the typeinfo component). + const int32_t* vtable = + *reinterpret_cast<const int32_t* const*>(static_ptr); + info->offset_to_derived = static_cast<std::ptrdiff_t>(vtable[-2]); + info->dynamic_ptr = static_cast<const char*>(static_ptr) + info->offset_to_derived; + + // The typeinfo component is now a relative offset to a proxy. + int32_t offset_to_ti_proxy = vtable[-1]; + const uint8_t* ptr_to_ti_proxy = + reinterpret_cast<const uint8_t*>(vtable) + offset_to_ti_proxy; + info->dynamic_type = *(reinterpret_cast<const __class_type_info* const*>(ptr_to_ti_proxy)); +#else + void **vtable = *static_cast<void ** const *>(static_ptr); + info->offset_to_derived = reinterpret_cast<ptrdiff_t>(vtable[-2]); + info->dynamic_ptr = static_cast<const char*>(static_ptr) + info->offset_to_derived; + info->dynamic_type = static_cast<const __class_type_info*>(vtable[-1]); +#endif +} + +/// A helper function for __dynamic_cast that casts a base sub-object pointer +/// to the object's dynamic type. +/// +/// This function returns the casting result directly. No further processing +/// required. +/// +/// Specifically, this function can only be called if the following pre- +/// condition holds: +/// * The dynamic type of the object pointed to by `static_ptr` is exactly +/// the same as `dst_type`. +const void* dyn_cast_to_derived(const void* static_ptr, + const void* dynamic_ptr, + const __class_type_info* static_type, + const __class_type_info* dst_type, + std::ptrdiff_t offset_to_derived, + std::ptrdiff_t src2dst_offset) +{ + // We're downcasting from src_type to the complete object's dynamic type. + // This is a really hot path that can be further optimized with the + // `src2dst_offset` hint. + // In such a case, dynamic_ptr already gives the casting result if the + // casting ever succeeds. All we have to do now is to check static_ptr + // points to a public base sub-object of dynamic_ptr. + + if (src2dst_offset >= 0) + { + // The static type is a unique public non-virtual base type of + // dst_type at offset `src2dst_offset` from the origin of dst. + // Note that there might be other non-public static_type bases. The + // hint only guarantees that the public base is non-virtual and + // unique. So we have to check whether static_ptr points to that + // unique public base sub-object. + if (offset_to_derived != -src2dst_offset) + return nullptr; + return dynamic_ptr; + } + + if (src2dst_offset == -2) + { + // static_type is not a public base of dst_type. + return nullptr; + } + + // If src2dst_offset == -3, then: + // src_type is a multiple public base type but never a virtual + // base type. We can't conclude that static_ptr points to those + // public base sub-objects because there might be other non- + // public static_type bases. The search is inevitable. + + // Fallback to the slow path to check that static_type is a public + // base type of dynamic_type. + // Using giant short cut. Add that information to info. + __dynamic_cast_info info = {dst_type, static_ptr, static_type, src2dst_offset, 0, 0, 0, 0, 0, 0, 0, 0, + 1, // number_of_dst_type + false, false, false, true, nullptr}; + // Do the search + dst_type->search_above_dst(&info, dynamic_ptr, dynamic_ptr, public_path, false); +#ifdef _LIBCXXABI_FORGIVING_DYNAMIC_CAST + // The following if should always be false because we should + // definitely find (static_ptr, static_type), either on a public + // or private path + if (info.path_dst_ptr_to_static_ptr == unknown) + { + // We get here only if there is some kind of visibility problem + // in client code. + static_assert(std::atomic<size_t>::is_always_lock_free, ""); + static std::atomic<size_t> error_count(0); + size_t error_count_snapshot = error_count.fetch_add(1, std::memory_order_relaxed); + if ((error_count_snapshot & (error_count_snapshot-1)) == 0) + syslog(LOG_ERR, "dynamic_cast error 1: Both of the following type_info's " + "should have public visibility. At least one of them is hidden. %s" + ", %s.\n", static_type->name(), dst_type->name()); + // Redo the search comparing type_info's using strcmp + info = {dst_type, static_ptr, static_type, src2dst_offset, 0, 0, 0, 0, 0, 0, + 0, 0, 0, false, false, false, true, nullptr}; + info.number_of_dst_type = 1; + dst_type->search_above_dst(&info, dynamic_ptr, dynamic_ptr, public_path, true); + } +#endif // _LIBCXXABI_FORGIVING_DYNAMIC_CAST + // Query the search. + if (info.path_dst_ptr_to_static_ptr != public_path) + return nullptr; + + return dynamic_ptr; +} + +/// A helper function for __dynamic_cast that tries to perform a downcast +/// before giving up and falling back to the slow path. +const void* dyn_cast_try_downcast(const void* static_ptr, + const void* dynamic_ptr, + const __class_type_info* dst_type, + const __class_type_info* dynamic_type, + std::ptrdiff_t src2dst_offset) +{ + if (src2dst_offset < 0) + { + // We can only optimize the case if the static type is a unique public + // base of dst_type. Give up. + return nullptr; + } + + // Pretend there is a dst_type object that leads to static_ptr. Later we + // will check whether this imagined dst_type object exists. If it exists + // then it will be the casting result. + const void* dst_ptr_to_static = reinterpret_cast<const char*>(static_ptr) - src2dst_offset; + + if (reinterpret_cast<std::intptr_t>(dst_ptr_to_static) < reinterpret_cast<std::intptr_t>(dynamic_ptr)) + { + // The imagined dst_type object does not exist. Bail-out quickly. + return nullptr; + } + + // Try to search a path from dynamic_type to dst_type. + __dynamic_cast_info dynamic_to_dst_info = {dynamic_type, + dst_ptr_to_static, + dst_type, + src2dst_offset, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, // number_of_dst_type + false, + false, + false, + true, + nullptr}; + dynamic_type->search_above_dst(&dynamic_to_dst_info, dynamic_ptr, dynamic_ptr, public_path, false); + if (dynamic_to_dst_info.path_dst_ptr_to_static_ptr != unknown) { + // We have found at least one path from dynamic_ptr to dst_ptr. The + // downcast can succeed. + return dst_ptr_to_static; + } + + return nullptr; +} + +const void* dyn_cast_slow(const void* static_ptr, + const void* dynamic_ptr, + const __class_type_info* static_type, + const __class_type_info* dst_type, + const __class_type_info* dynamic_type, + std::ptrdiff_t src2dst_offset) +{ + // Not using giant short cut. Do the search + + // Initialize info struct for this search. + __dynamic_cast_info info = {dst_type, static_ptr, static_type, src2dst_offset, 0, 0, 0, 0, 0, 0, + 0, 0, 0, false, false, false, true, nullptr}; + + dynamic_type->search_below_dst(&info, dynamic_ptr, public_path, false); +#ifdef _LIBCXXABI_FORGIVING_DYNAMIC_CAST + // The following if should always be false because we should + // definitely find (static_ptr, static_type), either on a public + // or private path + if (info.path_dst_ptr_to_static_ptr == unknown && + info.path_dynamic_ptr_to_static_ptr == unknown) + { + static_assert(std::atomic<size_t>::is_always_lock_free, ""); + static std::atomic<size_t> error_count(0); + size_t error_count_snapshot = error_count.fetch_add(1, std::memory_order_relaxed); + if ((error_count_snapshot & (error_count_snapshot-1)) == 0) + syslog(LOG_ERR, "dynamic_cast error 2: One or more of the following type_info's " + "has hidden visibility or is defined in more than one translation " + "unit. They should all have public visibility. " + "%s, %s, %s.\n", static_type->name(), dynamic_type->name(), + dst_type->name()); + // Redo the search comparing type_info's using strcmp + info = {dst_type, static_ptr, static_type, src2dst_offset, 0, 0, 0, 0, 0, 0, + 0, 0, 0, false, false, false, true, nullptr}; + dynamic_type->search_below_dst(&info, dynamic_ptr, public_path, true); + } +#endif // _LIBCXXABI_FORGIVING_DYNAMIC_CAST + // Query the search. + switch (info.number_to_static_ptr) + { + case 0: + if (info.number_to_dst_ptr == 1 && + info.path_dynamic_ptr_to_static_ptr == public_path && + info.path_dynamic_ptr_to_dst_ptr == public_path) + return info.dst_ptr_not_leading_to_static_ptr; + break; + case 1: + if (info.path_dst_ptr_to_static_ptr == public_path || + ( + info.number_to_dst_ptr == 0 && + info.path_dynamic_ptr_to_static_ptr == public_path && + info.path_dynamic_ptr_to_dst_ptr == public_path + ) + ) + return info.dst_ptr_leading_to_static_ptr; + break; + } + + return nullptr; +} + +} // namespace + // __shim_type_info __shim_type_info::~__shim_type_info() @@ -233,7 +470,8 @@ __class_type_info::can_catch(const __shim_type_info* thrown_type, if (thrown_class_type == 0) return false; // bullet 2 - __dynamic_cast_info info = {thrown_class_type, 0, this, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,}; + assert(adjustedPtr && "catching a class without an object?"); + __dynamic_cast_info info = {thrown_class_type, 0, this, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, true, nullptr}; info.number_of_dst_type = 1; thrown_class_type->has_unambiguous_public_base(&info, adjustedPtr, public_path); if (info.path_dst_ptr_to_static_ptr == public_path) @@ -248,32 +486,46 @@ __class_type_info::can_catch(const __shim_type_info* thrown_type, #pragma clang diagnostic pop #endif +// When we have an object to inspect - we just pass the pointer to the sub- +// object that matched the static_type we just checked. If that is different +// from any previously recorded pointer to that object type, then we have +// an ambiguous case. + +// When we have no object to inspect, we need to account for virtual bases +// explicitly. +// info->vbase_cookie is a pointer to the name of the innermost virtual base +// type, or nullptr if there is no virtual base on the path so far. +// adjustedPtr points to the subobject we just found. +// If vbase_cookie != any previously recorded (including the case of nullptr +// representing an already-found static sub-object) then we have an ambiguous +// case. Assuming that the vbase_cookie values agree; if then we have a +// different offset (adjustedPtr) from any previously recorded, this indicates +// an ambiguous case within the virtual base. + void __class_type_info::process_found_base_class(__dynamic_cast_info* info, void* adjustedPtr, int path_below) const { - if (info->dst_ptr_leading_to_static_ptr == 0) - { - // First time here - info->dst_ptr_leading_to_static_ptr = adjustedPtr; - info->path_dst_ptr_to_static_ptr = path_below; - info->number_to_static_ptr = 1; - } - else if (info->dst_ptr_leading_to_static_ptr == adjustedPtr) - { - // We've been here before. Update path to "most public" - if (info->path_dst_ptr_to_static_ptr == not_public_path) - info->path_dst_ptr_to_static_ptr = path_below; - } - else - { - // We've detected an ambiguous cast from (thrown_class_type, adjustedPtr) - // to a static_type - info->number_to_static_ptr += 1; - info->path_dst_ptr_to_static_ptr = not_public_path; - info->search_done = true; - } + if (info->number_to_static_ptr == 0) { + // First time we found this base + info->dst_ptr_leading_to_static_ptr = adjustedPtr; + info->path_dst_ptr_to_static_ptr = path_below; + // stash the virtual base cookie. + info->dst_ptr_not_leading_to_static_ptr = info->vbase_cookie; + info->number_to_static_ptr = 1; + } else if (info->dst_ptr_not_leading_to_static_ptr == info->vbase_cookie && + info->dst_ptr_leading_to_static_ptr == adjustedPtr) { + // We've been here before. Update path to "most public" + if (info->path_dst_ptr_to_static_ptr == not_public_path) + info->path_dst_ptr_to_static_ptr = path_below; + } else { + // We've detected an ambiguous cast from (thrown_class_type, adjustedPtr) + // to a static_type. + info->number_to_static_ptr += 1; + info->path_dst_ptr_to_static_ptr = not_public_path; + info->search_done = true; + } } void @@ -301,16 +553,30 @@ __base_class_type_info::has_unambiguous_public_base(__dynamic_cast_info* info, void* adjustedPtr, int path_below) const { - ptrdiff_t offset_to_base = 0; - if (adjustedPtr != nullptr) - { - offset_to_base = __offset_flags >> __offset_shift; - if (__offset_flags & __virtual_mask) - { - const char* vtable = *static_cast<const char*const*>(adjustedPtr); - offset_to_base = update_offset_to_base(vtable, offset_to_base); - } + bool is_virtual = __offset_flags & __virtual_mask; + ptrdiff_t offset_to_base = 0; + if (info->have_object) { + /* We have an object to inspect, we can look through its vtables to + find the layout. */ + offset_to_base = __offset_flags >> __offset_shift; + if (is_virtual) { + const char* vtable = *static_cast<const char* const*>(adjustedPtr); + offset_to_base = update_offset_to_base(vtable, offset_to_base); } + } else if (!is_virtual) { + /* We have no object; however, for non-virtual bases, (since we do not + need to inspect any content) we can pretend to have an object based + at '0'. */ + offset_to_base = __offset_flags >> __offset_shift; + } else { + /* No object to inspect, and the next base is virtual. + We cannot indirect through the vtable to find the actual object offset. + So, update vbase_cookie to the new innermost virtual base using the + pointer to the typeinfo name as a key. */ + info->vbase_cookie = static_cast<const void*>(__base_type->name()); + // .. and reset the pointer. + adjustedPtr = nullptr; + } __base_type->has_unambiguous_public_base( info, static_cast<char*>(adjustedPtr) + offset_to_base, @@ -431,14 +697,22 @@ __pointer_type_info::can_catch(const __shim_type_info* thrown_type, dynamic_cast<const __class_type_info*>(thrown_pointer_type->__pointee); if (thrown_class_type == 0) return false; - __dynamic_cast_info info = {thrown_class_type, 0, catch_class_type, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,}; + bool have_object = adjustedPtr != nullptr; + __dynamic_cast_info info = {thrown_class_type, 0, catch_class_type, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + have_object, nullptr}; info.number_of_dst_type = 1; thrown_class_type->has_unambiguous_public_base(&info, adjustedPtr, public_path); if (info.path_dst_ptr_to_static_ptr == public_path) { - if (adjustedPtr != NULL) - adjustedPtr = const_cast<void*>(info.dst_ptr_leading_to_static_ptr); - return true; + // In the case of a thrown null pointer, we have no object but we might + // well have computed the offset to where a public sub-object would be. + // However, we do not want to return that offset to the user; we still + // want them to catch a null ptr. + if (have_object) + adjustedPtr = const_cast<void*>(info.dst_ptr_leading_to_static_ptr); + else + adjustedPtr = nullptr; + return true; } return false; } @@ -623,174 +897,46 @@ extern "C" _LIBCXXABI_FUNC_VIS void * __dynamic_cast(const void *static_ptr, const __class_type_info *static_type, const __class_type_info *dst_type, std::ptrdiff_t src2dst_offset) { - // Possible future optimization: Take advantage of src2dst_offset - // Get (dynamic_ptr, dynamic_type) from static_ptr -#if __has_feature(cxx_abi_relative_vtable) - // The vtable address will point to the first virtual function, which is 8 - // bytes after the start of the vtable (4 for the offset from top + 4 for the typeinfo component). - const int32_t* vtable = - *reinterpret_cast<const int32_t* const*>(static_ptr); - int32_t offset_to_derived = vtable[-2]; - const void* dynamic_ptr = static_cast<const char*>(static_ptr) + offset_to_derived; - - // The typeinfo component is now a relative offset to a proxy. - int32_t offset_to_ti_proxy = vtable[-1]; - const uint8_t* ptr_to_ti_proxy = - reinterpret_cast<const uint8_t*>(vtable) + offset_to_ti_proxy; - const __class_type_info* dynamic_type = - *(reinterpret_cast<const __class_type_info* const*>(ptr_to_ti_proxy)); -#else - void **vtable = *static_cast<void ** const *>(static_ptr); - ptrdiff_t offset_to_derived = reinterpret_cast<ptrdiff_t>(vtable[-2]); - const void* dynamic_ptr = static_cast<const char*>(static_ptr) + offset_to_derived; - const __class_type_info* dynamic_type = static_cast<const __class_type_info*>(vtable[-1]); -#endif + derived_object_info derived_info; + dyn_cast_get_derived_info(&derived_info, static_ptr); // Initialize answer to nullptr. This will be changed from the search // results if a non-null answer is found. Regardless, this is what will // be returned. const void* dst_ptr = 0; - // Initialize info struct for this search. - __dynamic_cast_info info = {dst_type, static_ptr, static_type, src2dst_offset, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,}; // Find out if we can use a giant short cut in the search - if (is_equal(dynamic_type, dst_type, false)) + if (is_equal(derived_info.dynamic_type, dst_type, false)) { - // We're downcasting from src_type to the complete object's dynamic - // type. This is a really hot path that can be further optimized - // with the `src2dst_offset` hint. - // In such a case, dynamic_ptr already gives the casting result if the - // casting ever succeeds. All we have to do now is to check - // static_ptr points to a public base sub-object of dynamic_ptr. - - if (src2dst_offset >= 0) - { - // The static type is a unique public non-virtual base type of - // dst_type at offset `src2dst_offset` from the origin of dst. - // Note that there might be other non-public static_type bases. The - // hint only guarantees that the public base is non-virtual and - // unique. So we have to check whether static_ptr points to that - // unique public base sub-object. - if (offset_to_derived == -src2dst_offset) - dst_ptr = dynamic_ptr; - } - else if (src2dst_offset == -2) - { - // static_type is not a public base of dst_type. - dst_ptr = nullptr; - } - else - { - // If src2dst_offset == -3, then: - // src_type is a multiple public base type but never a virtual - // base type. We can't conclude that static_ptr points to those - // public base sub-objects because there might be other non- - // public static_type bases. The search is inevitable. - - // Fallback to the slow path to check that static_type is a public - // base type of dynamic_type. - // Using giant short cut. Add that information to info. - info.number_of_dst_type = 1; - // Do the search - dynamic_type->search_above_dst(&info, dynamic_ptr, dynamic_ptr, public_path, false); -#ifdef _LIBCXXABI_FORGIVING_DYNAMIC_CAST - // The following if should always be false because we should - // definitely find (static_ptr, static_type), either on a public - // or private path - if (info.path_dst_ptr_to_static_ptr == unknown) - { - // We get here only if there is some kind of visibility problem - // in client code. - static_assert(std::atomic<size_t>::is_always_lock_free, ""); - static std::atomic<size_t> error_count(0); - size_t error_count_snapshot = error_count.fetch_add(1, std::memory_order_relaxed); - if ((error_count_snapshot & (error_count_snapshot-1)) == 0) - syslog(LOG_ERR, "dynamic_cast error 1: Both of the following type_info's " - "should have public visibility. At least one of them is hidden. %s" - ", %s.\n", static_type->name(), dynamic_type->name()); - // Redo the search comparing type_info's using strcmp - info = {dst_type, static_ptr, static_type, src2dst_offset, 0}; - info.number_of_dst_type = 1; - dynamic_type->search_above_dst(&info, dynamic_ptr, dynamic_ptr, public_path, true); - } -#endif // _LIBCXXABI_FORGIVING_DYNAMIC_CAST - // Query the search. - if (info.path_dst_ptr_to_static_ptr == public_path) - dst_ptr = dynamic_ptr; - } + dst_ptr = dyn_cast_to_derived(static_ptr, + derived_info.dynamic_ptr, + static_type, + dst_type, + derived_info.offset_to_derived, + src2dst_offset); } else { - if (src2dst_offset >= 0) - { - // Optimize toward downcasting: dst_type has one unique public - // static_type bases. Let's first try to do a downcast before - // falling back to the slow path. The downcast succeeds if there - // is at least one path regardless of visibility from - // dynamic_type to dst_type. - const void* dst_ptr_to_static = reinterpret_cast<const char*>(static_ptr) - src2dst_offset; - if (reinterpret_cast<std::intptr_t>(dst_ptr_to_static) >= reinterpret_cast<std::intptr_t>(dynamic_ptr)) - { - // Try to search a path from dynamic_type to dst_type. - __dynamic_cast_info dynamic_to_dst_info = {dynamic_type, dst_ptr_to_static, dst_type, src2dst_offset}; - dynamic_to_dst_info.number_of_dst_type = 1; - dynamic_type->search_above_dst(&dynamic_to_dst_info, dynamic_ptr, dynamic_ptr, public_path, false); - if (dynamic_to_dst_info.path_dst_ptr_to_static_ptr != unknown) { - // We have found at least one path from dynamic_ptr to - // dst_ptr. The downcast can succeed. - dst_ptr = dst_ptr_to_static; - } - } - } + // Optimize toward downcasting: let's first try to do a downcast before + // falling back to the slow path. + dst_ptr = dyn_cast_try_downcast(static_ptr, + derived_info.dynamic_ptr, + dst_type, + derived_info.dynamic_type, + src2dst_offset); if (!dst_ptr) { - // Not using giant short cut. Do the search - dynamic_type->search_below_dst(&info, dynamic_ptr, public_path, false); -#ifdef _LIBCXXABI_FORGIVING_DYNAMIC_CAST - // The following if should always be false because we should - // definitely find (static_ptr, static_type), either on a public - // or private path - if (info.path_dst_ptr_to_static_ptr == unknown && - info.path_dynamic_ptr_to_static_ptr == unknown) - { - static_assert(std::atomic<size_t>::is_always_lock_free, ""); - static std::atomic<size_t> error_count(0); - size_t error_count_snapshot = error_count.fetch_add(1, std::memory_order_relaxed); - if ((error_count_snapshot & (error_count_snapshot-1)) == 0) - syslog(LOG_ERR, "dynamic_cast error 2: One or more of the following type_info's " - "has hidden visibility or is defined in more than one translation " - "unit. They should all have public visibility. " - "%s, %s, %s.\n", static_type->name(), dynamic_type->name(), - dst_type->name()); - // Redo the search comparing type_info's using strcmp - info = {dst_type, static_ptr, static_type, src2dst_offset, 0}; - dynamic_type->search_below_dst(&info, dynamic_ptr, public_path, true); - } -#endif // _LIBCXXABI_FORGIVING_DYNAMIC_CAST - // Query the search. - switch (info.number_to_static_ptr) - { - case 0: - if (info.number_to_dst_ptr == 1 && - info.path_dynamic_ptr_to_static_ptr == public_path && - info.path_dynamic_ptr_to_dst_ptr == public_path) - dst_ptr = info.dst_ptr_not_leading_to_static_ptr; - break; - case 1: - if (info.path_dst_ptr_to_static_ptr == public_path || - ( - info.number_to_dst_ptr == 0 && - info.path_dynamic_ptr_to_static_ptr == public_path && - info.path_dynamic_ptr_to_dst_ptr == public_path - ) - ) - dst_ptr = info.dst_ptr_leading_to_static_ptr; - break; - } + dst_ptr = dyn_cast_slow(static_ptr, + derived_info.dynamic_ptr, + static_type, + dst_type, + derived_info.dynamic_type, + src2dst_offset); } } + return const_cast<void*>(dst_ptr); } @@ -1075,7 +1221,7 @@ __vmi_class_type_info::search_below_dst(__dynamic_cast_info* info, if (info->search_done) break; // If we just found a dst_type with a public path to (static_ptr, static_type), - // then the only reason to continue the search is to make sure sure + // then the only reason to continue the search is to make sure // no other dst_type points to (static_ptr, static_type). // If !diamond, then we don't need to search here. // if we just found a dst_type with a private path to (static_ptr, static_type), diff --git a/contrib/libs/cxxsupp/libcxxabi/src/private_typeinfo.h b/contrib/libs/cxxsupp/libcxxabi/src/private_typeinfo.h index 622e09cc242..328a02edef5 100644 --- a/contrib/libs/cxxsupp/libcxxabi/src/private_typeinfo.h +++ b/contrib/libs/cxxsupp/libcxxabi/src/private_typeinfo.h @@ -110,6 +110,13 @@ struct _LIBCXXABI_HIDDEN __dynamic_cast_info bool found_any_static_type; // Set whenever a search can be stopped bool search_done; + + // Data that modifies the search mechanism. + + // There is no object (seen when we throw a null pointer to object). + bool have_object; + // Virtual base + const void* vbase_cookie; }; // Has no base class diff --git a/contrib/libs/cxxsupp/libcxxabi/src/stdlib_new_delete.cpp b/contrib/libs/cxxsupp/libcxxabi/src/stdlib_new_delete.cpp index 25cfb2b824c..b802559d479 100644 --- a/contrib/libs/cxxsupp/libcxxabi/src/stdlib_new_delete.cpp +++ b/contrib/libs/cxxsupp/libcxxabi/src/stdlib_new_delete.cpp @@ -7,7 +7,10 @@ //===----------------------------------------------------------------------===// #include "__cxxabi_config.h" +#include "abort_message.h" +#include "include/overridable_function.h" // from libc++ #include <__memory/aligned_alloc.h> +#include <cstddef> #include <cstdlib> #include <new> @@ -25,249 +28,216 @@ # error libc++ and libc++abi seem to disagree on whether exceptions are enabled #endif -// ------------------ BEGIN COPY ------------------ -// Implement all new and delete operators as weak definitions -// in this shared library, so that they can be overridden by programs -// that define non-weak copies of the functions. - -_LIBCPP_WEAK -void * -operator new(std::size_t size) _THROW_BAD_ALLOC -{ - if (size == 0) - size = 1; - void* p; - while ((p = std::malloc(size)) == nullptr) - { - // If malloc fails and there is a new_handler, - // call it to try free up memory. - std::new_handler nh = std::get_new_handler(); - if (nh) - nh(); - else +inline void __throw_bad_alloc_shim() { #ifndef _LIBCPP_HAS_NO_EXCEPTIONS - throw std::bad_alloc(); -#ifdef __EMSCRIPTEN__ - // Abort here so that when exceptions are disabled, we do not just - // return 0 when malloc returns 0. - // We could also do this with set_new_handler, but that adds a - // global constructor and a table entry, overhead that we can avoid - // by doing it this way. - abort(); + throw std::bad_alloc(); #else - break; -#endif + abort_message("bad_alloc was thrown in -fno-exceptions mode"); #endif - } - return p; -} - -_LIBCPP_WEAK -void* -operator new(size_t size, const std::nothrow_t&) noexcept -{ - void* p = nullptr; -#ifndef _LIBCPP_HAS_NO_EXCEPTIONS - try - { -#endif // _LIBCPP_HAS_NO_EXCEPTIONS - p = ::operator new(size); -#ifndef _LIBCPP_HAS_NO_EXCEPTIONS - } - catch (...) - { - } -#endif // _LIBCPP_HAS_NO_EXCEPTIONS - return p; -} - -_LIBCPP_WEAK -void* -operator new[](size_t size) _THROW_BAD_ALLOC -{ - return ::operator new(size); -} - -_LIBCPP_WEAK -void* -operator new[](size_t size, const std::nothrow_t&) noexcept -{ - void* p = nullptr; -#ifndef _LIBCPP_HAS_NO_EXCEPTIONS - try - { -#endif // _LIBCPP_HAS_NO_EXCEPTIONS - p = ::operator new[](size); -#ifndef _LIBCPP_HAS_NO_EXCEPTIONS - } - catch (...) - { - } -#endif // _LIBCPP_HAS_NO_EXCEPTIONS - return p; } -_LIBCPP_WEAK -void -operator delete(void* ptr) noexcept -{ - std::free(ptr); -} +#define _LIBCPP_ASSERT_SHIM(expr, str) \ + do { \ + if (!expr) \ + abort_message(str); \ + } while (false) -_LIBCPP_WEAK -void -operator delete(void* ptr, const std::nothrow_t&) noexcept -{ - ::operator delete(ptr); -} - -_LIBCPP_WEAK -void -operator delete(void* ptr, size_t) noexcept -{ - ::operator delete(ptr); -} - -_LIBCPP_WEAK -void -operator delete[] (void* ptr) noexcept -{ - ::operator delete(ptr); -} +// ------------------ BEGIN COPY ------------------ +// Implement all new and delete operators as weak definitions +// in this shared library, so that they can be overridden by programs +// that define non-weak copies of the functions. -_LIBCPP_WEAK -void -operator delete[] (void* ptr, const std::nothrow_t&) noexcept -{ - ::operator delete[](ptr); +static void* operator_new_impl(std::size_t size) { + if (size == 0) + size = 1; + void* p; + while ((p = std::malloc(size)) == nullptr) { + // If malloc fails and there is a new_handler, + // call it to try free up memory. + std::new_handler nh = std::get_new_handler(); + if (nh) + nh(); + else + break; + } + return p; +} + +_LIBCPP_MAKE_OVERRIDABLE_FUNCTION_DETECTABLE _LIBCPP_WEAK void* operator new(std::size_t size) _THROW_BAD_ALLOC { + void* p = operator_new_impl(size); + if (p == nullptr) + __throw_bad_alloc_shim(); + return p; +} + +_LIBCPP_WEAK void* operator new(size_t size, const std::nothrow_t&) noexcept { +#ifdef _LIBCPP_HAS_NO_EXCEPTIONS +# if _LIBCPP_CAN_DETECT_OVERRIDDEN_FUNCTION + _LIBCPP_ASSERT_SHIM( + !std::__is_function_overridden(static_cast<void* (*)(std::size_t)>(&operator new)), + "libc++ was configured with exceptions disabled and `operator new(size_t)` has been overridden, " + "but `operator new(size_t, nothrow_t)` has not been overridden. This is problematic because " + "`operator new(size_t, nothrow_t)` must call `operator new(size_t)`, which will terminate in case " + "it fails to allocate, making it impossible for `operator new(size_t, nothrow_t)` to fulfill its " + "contract (since it should return nullptr upon failure). Please make sure you override " + "`operator new(size_t, nothrow_t)` as well."); +# endif + + return operator_new_impl(size); +#else + void* p = nullptr; + try { + p = ::operator new(size); + } catch (...) { + } + return p; +#endif } -_LIBCPP_WEAK -void -operator delete[] (void* ptr, size_t) noexcept -{ - ::operator delete[](ptr); +_LIBCPP_MAKE_OVERRIDABLE_FUNCTION_DETECTABLE _LIBCPP_WEAK void* operator new[](size_t size) _THROW_BAD_ALLOC { + return ::operator new(size); } -#if !defined(_LIBCPP_HAS_NO_LIBRARY_ALIGNED_ALLOCATION) +_LIBCPP_WEAK void* operator new[](size_t size, const std::nothrow_t&) noexcept { +#ifdef _LIBCPP_HAS_NO_EXCEPTIONS +# if _LIBCPP_CAN_DETECT_OVERRIDDEN_FUNCTION + _LIBCPP_ASSERT_SHIM( + !std::__is_function_overridden(static_cast<void* (*)(std::size_t)>(&operator new[])), + "libc++ was configured with exceptions disabled and `operator new[](size_t)` has been overridden, " + "but `operator new[](size_t, nothrow_t)` has not been overridden. This is problematic because " + "`operator new[](size_t, nothrow_t)` must call `operator new[](size_t)`, which will terminate in case " + "it fails to allocate, making it impossible for `operator new[](size_t, nothrow_t)` to fulfill its " + "contract (since it should return nullptr upon failure). Please make sure you override " + "`operator new[](size_t, nothrow_t)` as well."); +# endif -_LIBCPP_WEAK -void * -operator new(std::size_t size, std::align_val_t alignment) _THROW_BAD_ALLOC -{ - if (size == 0) - size = 1; - if (static_cast<size_t>(alignment) < sizeof(void*)) - alignment = std::align_val_t(sizeof(void*)); - - // Try allocating memory. If allocation fails and there is a new_handler, - // call it to try free up memory, and try again until it succeeds, or until - // the new_handler decides to terminate. - // - // If allocation fails and there is no new_handler, we throw bad_alloc - // (or return nullptr if exceptions are disabled). - void* p; - while ((p = std::__libcpp_aligned_alloc(static_cast<std::size_t>(alignment), size)) == nullptr) - { - std::new_handler nh = std::get_new_handler(); - if (nh) - nh(); - else { -#ifndef _LIBCPP_HAS_NO_EXCEPTIONS - throw std::bad_alloc(); + return operator_new_impl(size); #else - break; + void* p = nullptr; + try { + p = ::operator new[](size); + } catch (...) { + } + return p; #endif - } - } - return p; -} - -_LIBCPP_WEAK -void* -operator new(size_t size, std::align_val_t alignment, const std::nothrow_t&) noexcept -{ - void* p = nullptr; -#ifndef _LIBCPP_HAS_NO_EXCEPTIONS - try - { -#endif // _LIBCPP_HAS_NO_EXCEPTIONS - p = ::operator new(size, alignment); -#ifndef _LIBCPP_HAS_NO_EXCEPTIONS - } - catch (...) - { - } -#endif // _LIBCPP_HAS_NO_EXCEPTIONS - return p; } -_LIBCPP_WEAK -void* -operator new[](size_t size, std::align_val_t alignment) _THROW_BAD_ALLOC -{ - return ::operator new(size, alignment); -} +_LIBCPP_WEAK void operator delete(void* ptr) noexcept { std::free(ptr); } -_LIBCPP_WEAK -void* -operator new[](size_t size, std::align_val_t alignment, const std::nothrow_t&) noexcept -{ - void* p = nullptr; -#ifndef _LIBCPP_HAS_NO_EXCEPTIONS - try - { -#endif // _LIBCPP_HAS_NO_EXCEPTIONS - p = ::operator new[](size, alignment); -#ifndef _LIBCPP_HAS_NO_EXCEPTIONS - } - catch (...) - { - } -#endif // _LIBCPP_HAS_NO_EXCEPTIONS - return p; -} +_LIBCPP_WEAK void operator delete(void* ptr, const std::nothrow_t&) noexcept { ::operator delete(ptr); } -_LIBCPP_WEAK -void -operator delete(void* ptr, std::align_val_t) noexcept -{ - std::__libcpp_aligned_free(ptr); -} +_LIBCPP_WEAK void operator delete(void* ptr, size_t) noexcept { ::operator delete(ptr); } -_LIBCPP_WEAK -void -operator delete(void* ptr, std::align_val_t alignment, const std::nothrow_t&) noexcept -{ - ::operator delete(ptr, alignment); -} +_LIBCPP_WEAK void operator delete[](void* ptr) noexcept { ::operator delete(ptr); } -_LIBCPP_WEAK -void -operator delete(void* ptr, size_t, std::align_val_t alignment) noexcept -{ - ::operator delete(ptr, alignment); -} +_LIBCPP_WEAK void operator delete[](void* ptr, const std::nothrow_t&) noexcept { ::operator delete[](ptr); } -_LIBCPP_WEAK -void -operator delete[] (void* ptr, std::align_val_t alignment) noexcept -{ - ::operator delete(ptr, alignment); -} +_LIBCPP_WEAK void operator delete[](void* ptr, size_t) noexcept { ::operator delete[](ptr); } -_LIBCPP_WEAK -void -operator delete[] (void* ptr, std::align_val_t alignment, const std::nothrow_t&) noexcept -{ - ::operator delete[](ptr, alignment); -} +#if !defined(_LIBCPP_HAS_NO_LIBRARY_ALIGNED_ALLOCATION) -_LIBCPP_WEAK -void -operator delete[] (void* ptr, size_t, std::align_val_t alignment) noexcept -{ - ::operator delete[](ptr, alignment); +static void* operator_new_aligned_impl(std::size_t size, std::align_val_t alignment) { + if (size == 0) + size = 1; + if (static_cast<size_t>(alignment) < sizeof(void*)) + alignment = std::align_val_t(sizeof(void*)); + + // Try allocating memory. If allocation fails and there is a new_handler, + // call it to try free up memory, and try again until it succeeds, or until + // the new_handler decides to terminate. + void* p; + while ((p = std::__libcpp_aligned_alloc(static_cast<std::size_t>(alignment), size)) == nullptr) { + std::new_handler nh = std::get_new_handler(); + if (nh) + nh(); + else + break; + } + return p; +} + +_LIBCPP_MAKE_OVERRIDABLE_FUNCTION_DETECTABLE _LIBCPP_WEAK void* +operator new(std::size_t size, std::align_val_t alignment) _THROW_BAD_ALLOC { + void* p = operator_new_aligned_impl(size, alignment); + if (p == nullptr) + __throw_bad_alloc_shim(); + return p; +} + +_LIBCPP_WEAK void* operator new(size_t size, std::align_val_t alignment, const std::nothrow_t&) noexcept { +# ifdef _LIBCPP_HAS_NO_EXCEPTIONS +# if _LIBCPP_CAN_DETECT_OVERRIDDEN_FUNCTION + _LIBCPP_ASSERT_SHIM( + !std::__is_function_overridden(static_cast<void* (*)(std::size_t, std::align_val_t)>(&operator new)), + "libc++ was configured with exceptions disabled and `operator new(size_t, align_val_t)` has been overridden, " + "but `operator new(size_t, align_val_t, nothrow_t)` has not been overridden. This is problematic because " + "`operator new(size_t, align_val_t, nothrow_t)` must call `operator new(size_t, align_val_t)`, which will " + "terminate in case it fails to allocate, making it impossible for `operator new(size_t, align_val_t, nothrow_t)` " + "to fulfill its contract (since it should return nullptr upon failure). Please make sure you override " + "`operator new(size_t, align_val_t, nothrow_t)` as well."); +# endif + + return operator_new_aligned_impl(size, alignment); +# else + void* p = nullptr; + try { + p = ::operator new(size, alignment); + } catch (...) { + } + return p; +# endif +} + +_LIBCPP_MAKE_OVERRIDABLE_FUNCTION_DETECTABLE _LIBCPP_WEAK void* +operator new[](size_t size, std::align_val_t alignment) _THROW_BAD_ALLOC { + return ::operator new(size, alignment); +} + +_LIBCPP_WEAK void* operator new[](size_t size, std::align_val_t alignment, const std::nothrow_t&) noexcept { +# ifdef _LIBCPP_HAS_NO_EXCEPTIONS +# if _LIBCPP_CAN_DETECT_OVERRIDDEN_FUNCTION + _LIBCPP_ASSERT_SHIM( + !std::__is_function_overridden(static_cast<void* (*)(std::size_t, std::align_val_t)>(&operator new[])), + "libc++ was configured with exceptions disabled and `operator new[](size_t, align_val_t)` has been overridden, " + "but `operator new[](size_t, align_val_t, nothrow_t)` has not been overridden. This is problematic because " + "`operator new[](size_t, align_val_t, nothrow_t)` must call `operator new[](size_t, align_val_t)`, which will " + "terminate in case it fails to allocate, making it impossible for `operator new[](size_t, align_val_t, " + "nothrow_t)` to fulfill its contract (since it should return nullptr upon failure). Please make sure you " + "override " + "`operator new[](size_t, align_val_t, nothrow_t)` as well."); +# endif + + return operator_new_aligned_impl(size, alignment); +# else + void* p = nullptr; + try { + p = ::operator new[](size, alignment); + } catch (...) { + } + return p; +# endif +} + +_LIBCPP_WEAK void operator delete(void* ptr, std::align_val_t) noexcept { std::__libcpp_aligned_free(ptr); } + +_LIBCPP_WEAK void operator delete(void* ptr, std::align_val_t alignment, const std::nothrow_t&) noexcept { + ::operator delete(ptr, alignment); +} + +_LIBCPP_WEAK void operator delete(void* ptr, size_t, std::align_val_t alignment) noexcept { + ::operator delete(ptr, alignment); +} + +_LIBCPP_WEAK void operator delete[](void* ptr, std::align_val_t alignment) noexcept { + ::operator delete(ptr, alignment); +} + +_LIBCPP_WEAK void operator delete[](void* ptr, std::align_val_t alignment, const std::nothrow_t&) noexcept { + ::operator delete[](ptr, alignment); +} + +_LIBCPP_WEAK void operator delete[](void* ptr, size_t, std::align_val_t alignment) noexcept { + ::operator delete[](ptr, alignment); } #endif // !_LIBCPP_HAS_NO_LIBRARY_ALIGNED_ALLOCATION diff --git a/contrib/libs/cxxsupp/libcxxabi/ya.make b/contrib/libs/cxxsupp/libcxxabi/ya.make index a96ac77959a..ea3a68d8921 100644 --- a/contrib/libs/cxxsupp/libcxxabi/ya.make +++ b/contrib/libs/cxxsupp/libcxxabi/ya.make @@ -11,9 +11,9 @@ LICENSE( LICENSE_TEXTS(.yandex_meta/licenses.list.txt) -VERSION(17.0.6) +VERSION(18.1.8) -ORIGINAL_SOURCE(https://github.com/llvm/llvm-project/archive/llvmorg-17.0.6.tar.gz) +ORIGINAL_SOURCE(https://github.com/llvm/llvm-project/archive/llvmorg-18.1.8.tar.gz) PEERDIR( contrib/libs/libunwind @@ -74,6 +74,7 @@ ELSEIF (OS_EMSCRIPTEN AND ARCH_WASM32) CFLAGS( -D_LIBCPP_SAFE_STATIC= -D_LIBCXXABI_DTOR_FUNC= + -D__USING_WASM_EXCEPTIONS__ ) ENDIF() diff --git a/contrib/libs/cxxsupp/libcxxcuda11/ya.make b/contrib/libs/cxxsupp/libcxxcuda11/ya.make index f4c59b74608..e3f58f3349c 100644 --- a/contrib/libs/cxxsupp/libcxxcuda11/ya.make +++ b/contrib/libs/cxxsupp/libcxxcuda11/ya.make @@ -19,8 +19,6 @@ VERSION(2023-10-05) ORIGINAL_SOURCE(https://github.com/llvm/llvm-project/archive/dc129d6f715cf83a2072fc8de8b4e4c70bca6935.tar.gz) SUBSCRIBER( - halyavin - somov g:cpp-committee g:cpp-contrib ) diff --git a/contrib/libs/cxxsupp/stdc/ya.make b/contrib/libs/cxxsupp/stdc/ya.make index 3a4e52aed45..b025e390394 100644 --- a/contrib/libs/cxxsupp/stdc/ya.make +++ b/contrib/libs/cxxsupp/stdc/ya.make @@ -9,7 +9,6 @@ LICENSE(Service-Sourceless-Library) SUBSCRIBER( g:contrib g:cpp-contrib - pg ) NO_PLATFORM() diff --git a/contrib/libs/cxxsupp/system_stl/ya.make b/contrib/libs/cxxsupp/system_stl/ya.make index d536c83ef57..187c28d53e1 100644 --- a/contrib/libs/cxxsupp/system_stl/ya.make +++ b/contrib/libs/cxxsupp/system_stl/ya.make @@ -9,7 +9,6 @@ LICENSE(Service-Sourceless-Library) SUBSCRIBER( g:contrib g:cpp-contrib - somov ) NO_PLATFORM() diff --git a/contrib/libs/lcms2/.yandex_meta/__init__.py b/contrib/libs/lcms2/.yandex_meta/__init__.py index fa80c60ee26..1d48cb38a6f 100644 --- a/contrib/libs/lcms2/.yandex_meta/__init__.py +++ b/contrib/libs/lcms2/.yandex_meta/__init__.py @@ -22,7 +22,7 @@ def post_install(self): lcms2 = NixProject( - owners=["kikht", "shindo", "g:mds", "g:cpp-contrib"], + owners=["shindo", "g:mds", "g:cpp-contrib"], arcdir="contrib/libs/lcms2", nixattr="lcms2", post_build=post_build, diff --git a/contrib/libs/libevent/.yandex_meta/__init__.py b/contrib/libs/libevent/.yandex_meta/__init__.py index 71462737f07..a37a7a8a155 100644 --- a/contrib/libs/libevent/.yandex_meta/__init__.py +++ b/contrib/libs/libevent/.yandex_meta/__init__.py @@ -85,7 +85,7 @@ def libevent_post_install(self): libevent = GNUMakeNixProject( - owners=["g:cpp-contrib", "kikht", "dldmitry", "efmv"], + owners=["g:cpp-contrib", "dldmitry"], arcdir="contrib/libs/libevent", nixattr="libevent", ignore_commands=["bash", "sed"], diff --git a/contrib/restricted/boost/context/.yandex_meta/devtools.copyrights.report b/contrib/restricted/boost/context/.yandex_meta/devtools.copyrights.report index 5ab98d2b00c..60d046ffecf 100644 --- a/contrib/restricted/boost/context/.yandex_meta/devtools.copyrights.report +++ b/contrib/restricted/boost/context/.yandex_meta/devtools.copyrights.report @@ -95,16 +95,6 @@ BELONGS ya.make src/asm/ontop_x86_64_sysv_macho_gas.S [2:4] src/untested.cpp [2:4] -KEEP COPYRIGHT_SERVICE_LABEL 9ed7b7468c684d33492a4176e9cf31b1 -BELONGS ya.make - Note: matched license text is too long. Read it in the source files. - Scancode info: - Original SPDX id: COPYRIGHT_SERVICE_LABEL - Score : 100.00 - Match type : COPYRIGHT - Files with this license: - src/fcontext.cpp [1:3] - KEEP COPYRIGHT_SERVICE_LABEL c3ce9643507aa0942a28211fe8f95eff BELONGS ya.make License text: diff --git a/contrib/restricted/boost/context/.yandex_meta/devtools.licenses.report b/contrib/restricted/boost/context/.yandex_meta/devtools.licenses.report index 2259eba7085..87fc6196123 100644 --- a/contrib/restricted/boost/context/.yandex_meta/devtools.licenses.report +++ b/contrib/restricted/boost/context/.yandex_meta/devtools.licenses.report @@ -127,7 +127,6 @@ BELONGS ya.make include/boost/context/stack_traits.hpp [3:5] include/boost/context/windows/protected_fixedsize_stack.hpp [3:5] src/continuation.cpp [3:5] - src/fcontext.cpp [2:4] src/fiber.cpp [3:5] src/posix/stack_traits.cpp [3:5] src/untested.cpp [3:5] diff --git a/contrib/restricted/boost/context/.yandex_meta/licenses.list.txt b/contrib/restricted/boost/context/.yandex_meta/licenses.list.txt index dbd93236711..6eecddf257b 100644 --- a/contrib/restricted/boost/context/.yandex_meta/licenses.list.txt +++ b/contrib/restricted/boost/context/.yandex_meta/licenses.list.txt @@ -56,9 +56,3 @@ // Copyright Oliver Kowalke 2017. // Distributed under the Boost Software License, Version 1.0. // (See accompanying file LICENSE_1_0.txt or copy at - - -====================COPYRIGHT==================== -// SPDX-FileCopyrightText: Copyright 2024 Arm Limited and/or its affiliates <open-source-office@arm.com> -// Distributed under the Boost Software License, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at diff --git a/contrib/restricted/boost/context/fcontext_impl/ya.make b/contrib/restricted/boost/context/fcontext_impl/ya.make index d0b2bdffa7b..da427b0f305 100644 --- a/contrib/restricted/boost/context/fcontext_impl/ya.make +++ b/contrib/restricted/boost/context/fcontext_impl/ya.make @@ -4,9 +4,9 @@ LIBRARY() WITHOUT_LICENSE_TEXTS() -VERSION(1.87.0) +VERSION(1.86.0) -ORIGINAL_SOURCE(https://github.com/boostorg/context/archive/boost-1.87.0.tar.gz) +ORIGINAL_SOURCE(https://github.com/boostorg/context/archive/boost-1.86.0.tar.gz) LICENSE(BSL-1.0) @@ -68,7 +68,6 @@ SRCS( src/asm/make_${FCONTEXT_ARCH}_${FCONTEXT_ABI}_${FCONTEXT_FMT}_${FCONTEXT_SUF} src/asm/jump_${FCONTEXT_ARCH}_${FCONTEXT_ABI}_${FCONTEXT_FMT}_${FCONTEXT_SUF} src/asm/ontop_${FCONTEXT_ARCH}_${FCONTEXT_ABI}_${FCONTEXT_FMT}_${FCONTEXT_SUF} - src/fcontext.cpp ) END() diff --git a/contrib/restricted/boost/context/impl_common/ya.make b/contrib/restricted/boost/context/impl_common/ya.make index 3a9a118a9cc..61f75161188 100644 --- a/contrib/restricted/boost/context/impl_common/ya.make +++ b/contrib/restricted/boost/context/impl_common/ya.make @@ -4,9 +4,9 @@ LIBRARY() WITHOUT_LICENSE_TEXTS() -VERSION(1.87.0) +VERSION(1.86.0) -ORIGINAL_SOURCE(https://github.com/boostorg/context/archive/boost-1.87.0.tar.gz) +ORIGINAL_SOURCE(https://github.com/boostorg/context/archive/boost-1.86.0.tar.gz) LICENSE(BSL-1.0) diff --git a/contrib/restricted/boost/context/include/boost/context/detail/fcontext.hpp b/contrib/restricted/boost/context/include/boost/context/detail/fcontext.hpp index 52be6083afd..00cb24d9415 100644 --- a/contrib/restricted/boost/context/include/boost/context/detail/fcontext.hpp +++ b/contrib/restricted/boost/context/include/boost/context/detail/fcontext.hpp @@ -27,11 +27,14 @@ struct transfer_t { void * data; }; -BOOST_CONTEXT_DECL transfer_t jump_fcontext( fcontext_t const to, void * vp); -BOOST_CONTEXT_DECL fcontext_t make_fcontext( void * sp, std::size_t size, void (* fn)( transfer_t) ); +extern "C" BOOST_CONTEXT_DECL +transfer_t BOOST_CONTEXT_CALLDECL jump_fcontext( fcontext_t const to, void * vp); +extern "C" BOOST_CONTEXT_DECL +fcontext_t BOOST_CONTEXT_CALLDECL make_fcontext( void * sp, std::size_t size, void (* fn)( transfer_t) ); // based on an idea of Giovanni Derreta -BOOST_CONTEXT_DECL transfer_t ontop_fcontext( fcontext_t const to, void * vp, transfer_t (* fn)( transfer_t) ); +extern "C" BOOST_CONTEXT_DECL +transfer_t BOOST_CONTEXT_CALLDECL ontop_fcontext( fcontext_t const to, void * vp, transfer_t (* fn)( transfer_t) ); }}} diff --git a/contrib/restricted/boost/context/src/asm/jump_arm64_aapcs_elf_gas.S b/contrib/restricted/boost/context/src/asm/jump_arm64_aapcs_elf_gas.S index 7c1f0753559..cefd1830d71 100644 --- a/contrib/restricted/boost/context/src/asm/jump_arm64_aapcs_elf_gas.S +++ b/contrib/restricted/boost/context/src/asm/jump_arm64_aapcs_elf_gas.S @@ -55,7 +55,6 @@ .text .align 2 .global jump_fcontext -.hidden jump_fcontext .type jump_fcontext, %function jump_fcontext: # prepare stack for GP + FPU diff --git a/contrib/restricted/boost/context/src/asm/jump_arm64_aapcs_macho_gas.S b/contrib/restricted/boost/context/src/asm/jump_arm64_aapcs_macho_gas.S index 12cc0215868..31738f74531 100644 --- a/contrib/restricted/boost/context/src/asm/jump_arm64_aapcs_macho_gas.S +++ b/contrib/restricted/boost/context/src/asm/jump_arm64_aapcs_macho_gas.S @@ -52,7 +52,6 @@ *******************************************************/ .text -.private_extern _jump_fcontext .globl _jump_fcontext .balign 16 _jump_fcontext: diff --git a/contrib/restricted/boost/context/src/asm/jump_arm_aapcs_elf_gas.S b/contrib/restricted/boost/context/src/asm/jump_arm_aapcs_elf_gas.S index 9934c0897c8..86efe9d8214 100644 --- a/contrib/restricted/boost/context/src/asm/jump_arm_aapcs_elf_gas.S +++ b/contrib/restricted/boost/context/src/asm/jump_arm_aapcs_elf_gas.S @@ -41,7 +41,6 @@ .file "jump_arm_aapcs_elf_gas.S" .text .globl jump_fcontext -.hidden jump_fcontext .align 2 .type jump_fcontext,%function .syntax unified diff --git a/contrib/restricted/boost/context/src/asm/jump_arm_aapcs_macho_gas.S b/contrib/restricted/boost/context/src/asm/jump_arm_aapcs_macho_gas.S index 44e7f2a4227..077c36409e8 100644 --- a/contrib/restricted/boost/context/src/asm/jump_arm_aapcs_macho_gas.S +++ b/contrib/restricted/boost/context/src/asm/jump_arm_aapcs_macho_gas.S @@ -39,7 +39,6 @@ *******************************************************/ .text -.private_extern _jump_fcontext .globl _jump_fcontext .align 2 _jump_fcontext: diff --git a/contrib/restricted/boost/context/src/asm/jump_i386_sysv_elf_gas.S b/contrib/restricted/boost/context/src/asm/jump_i386_sysv_elf_gas.S index d00ac4fffd4..ed83717ce2b 100644 --- a/contrib/restricted/boost/context/src/asm/jump_i386_sysv_elf_gas.S +++ b/contrib/restricted/boost/context/src/asm/jump_i386_sysv_elf_gas.S @@ -31,7 +31,6 @@ .file "jump_i386_sysv_elf_gas.S" .text .globl jump_fcontext -.hidden jump_fcontext .align 2 .type jump_fcontext,@function jump_fcontext: diff --git a/contrib/restricted/boost/context/src/asm/jump_i386_sysv_macho_gas.S b/contrib/restricted/boost/context/src/asm/jump_i386_sysv_macho_gas.S index 12aa702be98..8ab7c6f29c6 100644 --- a/contrib/restricted/boost/context/src/asm/jump_i386_sysv_macho_gas.S +++ b/contrib/restricted/boost/context/src/asm/jump_i386_sysv_macho_gas.S @@ -25,7 +25,6 @@ ****************************************************************************************/ .text -.private_extern _jump_fcontext .globl _jump_fcontext .align 2 _jump_fcontext: diff --git a/contrib/restricted/boost/context/src/asm/jump_x86_64_sysv_elf_gas.S b/contrib/restricted/boost/context/src/asm/jump_x86_64_sysv_elf_gas.S index 2eff59a380c..be264bdc2e2 100644 --- a/contrib/restricted/boost/context/src/asm/jump_x86_64_sysv_elf_gas.S +++ b/contrib/restricted/boost/context/src/asm/jump_x86_64_sysv_elf_gas.S @@ -44,7 +44,6 @@ .file "jump_x86_64_sysv_elf_gas.S" .text .globl jump_fcontext -.hidden jump_fcontext .type jump_fcontext,@function .align 16 jump_fcontext: @@ -74,6 +73,14 @@ jump_fcontext: /* read the current SSP and store it */ rdsspq %rcx movq %rcx, (%rsp) +#endif + +#if BOOST_CONTEXT_SHADOW_STACK + /* grow the stack to reserve space for shadow stack pointer(SSP) */ + leaq -0x8(%rsp), %rsp + /* read the current SSP and store it */ + rdsspq %rcx + movq %rcx, (%rsp) # endif /* store RSP (pointing to context-data) in RAX */ diff --git a/contrib/restricted/boost/context/src/asm/jump_x86_64_sysv_macho_gas.S b/contrib/restricted/boost/context/src/asm/jump_x86_64_sysv_macho_gas.S index 673daa61dfa..afc3e5c126f 100644 --- a/contrib/restricted/boost/context/src/asm/jump_x86_64_sysv_macho_gas.S +++ b/contrib/restricted/boost/context/src/asm/jump_x86_64_sysv_macho_gas.S @@ -25,7 +25,6 @@ ****************************************************************************************/ .text -.private_extern _jump_fcontext .globl _jump_fcontext .align 8 _jump_fcontext: diff --git a/contrib/restricted/boost/context/src/asm/make_arm64_aapcs_elf_gas.S b/contrib/restricted/boost/context/src/asm/make_arm64_aapcs_elf_gas.S index 8ac825bf546..66cfb2da17e 100644 --- a/contrib/restricted/boost/context/src/asm/make_arm64_aapcs_elf_gas.S +++ b/contrib/restricted/boost/context/src/asm/make_arm64_aapcs_elf_gas.S @@ -55,7 +55,6 @@ .text .align 2 .global make_fcontext -.hidden make_fcontext .type make_fcontext, %function make_fcontext: # shift address in x0 (allocated stack) to lower 16 byte boundary diff --git a/contrib/restricted/boost/context/src/asm/make_arm64_aapcs_macho_gas.S b/contrib/restricted/boost/context/src/asm/make_arm64_aapcs_macho_gas.S index a6a1314c0dc..b30b1e3e5bb 100644 --- a/contrib/restricted/boost/context/src/asm/make_arm64_aapcs_macho_gas.S +++ b/contrib/restricted/boost/context/src/asm/make_arm64_aapcs_macho_gas.S @@ -52,7 +52,6 @@ *******************************************************/ .text -.private_extern _make_fcontext .globl _make_fcontext .balign 16 diff --git a/contrib/restricted/boost/context/src/asm/make_arm_aapcs_elf_gas.S b/contrib/restricted/boost/context/src/asm/make_arm_aapcs_elf_gas.S index 9616e566af6..98ae64b43f9 100644 --- a/contrib/restricted/boost/context/src/asm/make_arm_aapcs_elf_gas.S +++ b/contrib/restricted/boost/context/src/asm/make_arm_aapcs_elf_gas.S @@ -41,7 +41,6 @@ .file "make_arm_aapcs_elf_gas.S" .text .globl make_fcontext -.hidden make_fcontext .align 2 .type make_fcontext,%function .syntax unified diff --git a/contrib/restricted/boost/context/src/asm/make_arm_aapcs_macho_gas.S b/contrib/restricted/boost/context/src/asm/make_arm_aapcs_macho_gas.S index de934075d24..c909ae9d43a 100644 --- a/contrib/restricted/boost/context/src/asm/make_arm_aapcs_macho_gas.S +++ b/contrib/restricted/boost/context/src/asm/make_arm_aapcs_macho_gas.S @@ -39,7 +39,6 @@ *******************************************************/ .text -.private_extern _make_fcontext .globl _make_fcontext .align 2 _make_fcontext: diff --git a/contrib/restricted/boost/context/src/asm/make_i386_sysv_elf_gas.S b/contrib/restricted/boost/context/src/asm/make_i386_sysv_elf_gas.S index 992bae019af..c6e0b36558a 100644 --- a/contrib/restricted/boost/context/src/asm/make_i386_sysv_elf_gas.S +++ b/contrib/restricted/boost/context/src/asm/make_i386_sysv_elf_gas.S @@ -31,7 +31,6 @@ .file "make_i386_sysv_elf_gas.S" .text .globl make_fcontext -.hidden make_fcontext .align 2 .type make_fcontext,@function make_fcontext: diff --git a/contrib/restricted/boost/context/src/asm/make_i386_sysv_macho_gas.S b/contrib/restricted/boost/context/src/asm/make_i386_sysv_macho_gas.S index a5890d75f67..519e406248b 100644 --- a/contrib/restricted/boost/context/src/asm/make_i386_sysv_macho_gas.S +++ b/contrib/restricted/boost/context/src/asm/make_i386_sysv_macho_gas.S @@ -25,7 +25,6 @@ ****************************************************************************************/ .text -.private_extern _make_fcontext .globl _make_fcontext .align 2 _make_fcontext: diff --git a/contrib/restricted/boost/context/src/asm/make_x86_64_sysv_elf_gas.S b/contrib/restricted/boost/context/src/asm/make_x86_64_sysv_elf_gas.S index 7561c0896ba..b0d0c0341ef 100644 --- a/contrib/restricted/boost/context/src/asm/make_x86_64_sysv_elf_gas.S +++ b/contrib/restricted/boost/context/src/asm/make_x86_64_sysv_elf_gas.S @@ -44,7 +44,6 @@ .file "make_x86_64_sysv_elf_gas.S" .text .globl make_fcontext -.hidden make_fcontext .type make_fcontext,@function .align 16 make_fcontext: @@ -93,6 +92,35 @@ make_fcontext: movq %rcx, 0x38(%rax) #if BOOST_CONTEXT_SHADOW_STACK + /* Populate the shadow stack and normal stack */ + /* get original SSP */ + rdsspq %r8 + /* restore new shadow stack */ + rstorssp -0x8(%r9) + /* save the restore token on the original shadow stack */ + saveprevssp + /* push the address of "jmp trampoline" to the new shadow stack */ + /* as well as the stack */ + call 1f + jmp trampoline +1: + /* save address of "jmp trampoline" as return-address */ + /* for context-function */ + pop 0x38(%rax) + /* Get the new SSP. */ + rdsspq %r9 + /* restore original shadow stack */ + rstorssp -0x8(%r8) + /* save the restore token on the new shadow stack. */ + saveprevssp + + /* reserve space for the new SSP */ + leaq -0x8(%rax), %rax + /* save the new SSP to this fcontext */ + movq %r9, (%rax) +#endif + +#if BOOST_CONTEXT_SHADOW_STACK /* Populate the shadow stack */ /* get original SSP */ diff --git a/contrib/restricted/boost/context/src/asm/make_x86_64_sysv_macho_gas.S b/contrib/restricted/boost/context/src/asm/make_x86_64_sysv_macho_gas.S index 06357f678ac..5d6c5431c59 100644 --- a/contrib/restricted/boost/context/src/asm/make_x86_64_sysv_macho_gas.S +++ b/contrib/restricted/boost/context/src/asm/make_x86_64_sysv_macho_gas.S @@ -25,7 +25,6 @@ ****************************************************************************************/ .text -.private_extern _make_fcontext .globl _make_fcontext .align 8 _make_fcontext: diff --git a/contrib/restricted/boost/context/src/asm/ontop_arm64_aapcs_elf_gas.S b/contrib/restricted/boost/context/src/asm/ontop_arm64_aapcs_elf_gas.S index 8e40fc7d365..665ca5a2c11 100644 --- a/contrib/restricted/boost/context/src/asm/ontop_arm64_aapcs_elf_gas.S +++ b/contrib/restricted/boost/context/src/asm/ontop_arm64_aapcs_elf_gas.S @@ -55,7 +55,6 @@ .text .align 2 .global ontop_fcontext -.hidden ontop_fcontext .type ontop_fcontext, %function ontop_fcontext: # prepare stack for GP + FPU diff --git a/contrib/restricted/boost/context/src/asm/ontop_arm64_aapcs_macho_gas.S b/contrib/restricted/boost/context/src/asm/ontop_arm64_aapcs_macho_gas.S index 8babe470233..a387d06dd29 100644 --- a/contrib/restricted/boost/context/src/asm/ontop_arm64_aapcs_macho_gas.S +++ b/contrib/restricted/boost/context/src/asm/ontop_arm64_aapcs_macho_gas.S @@ -52,7 +52,6 @@ *******************************************************/ .text -.private_extern _ontop_fcontext .global _ontop_fcontext .balign 16 _ontop_fcontext: diff --git a/contrib/restricted/boost/context/src/asm/ontop_arm_aapcs_elf_gas.S b/contrib/restricted/boost/context/src/asm/ontop_arm_aapcs_elf_gas.S index 8c6c9020de8..59ad5ca9ce7 100644 --- a/contrib/restricted/boost/context/src/asm/ontop_arm_aapcs_elf_gas.S +++ b/contrib/restricted/boost/context/src/asm/ontop_arm_aapcs_elf_gas.S @@ -41,7 +41,6 @@ .file "ontop_arm_aapcs_elf_gas.S" .text .globl ontop_fcontext -.hidden ontop_fcontext .align 2 .type ontop_fcontext,%function .syntax unified diff --git a/contrib/restricted/boost/context/src/asm/ontop_arm_aapcs_macho_gas.S b/contrib/restricted/boost/context/src/asm/ontop_arm_aapcs_macho_gas.S index 07e63fb24f2..3633aca641a 100644 --- a/contrib/restricted/boost/context/src/asm/ontop_arm_aapcs_macho_gas.S +++ b/contrib/restricted/boost/context/src/asm/ontop_arm_aapcs_macho_gas.S @@ -39,7 +39,6 @@ *******************************************************/ .text -.private_extern _ontop_fcontext .globl _ontop_fcontext .align 2 _ontop_fcontext: diff --git a/contrib/restricted/boost/context/src/asm/ontop_i386_sysv_elf_gas.S b/contrib/restricted/boost/context/src/asm/ontop_i386_sysv_elf_gas.S index ea7a75b90f4..0cb6168fab7 100644 --- a/contrib/restricted/boost/context/src/asm/ontop_i386_sysv_elf_gas.S +++ b/contrib/restricted/boost/context/src/asm/ontop_i386_sysv_elf_gas.S @@ -24,14 +24,9 @@ * * ****************************************************************************************/ -#ifdef __x86_64__ -#include "ontop_x86_64_sysv_elf_gas.S" -#else - .file "ontop_i386_sysv_elf_gas.S" .text .globl ontop_fcontext -.hidden ontop_fcontext .align 2 .type ontop_fcontext,@function ontop_fcontext: @@ -103,5 +98,3 @@ ontop_fcontext: /* Mark that we don't need executable stack. */ .section .note.GNU-stack,"",%progbits - -#endif diff --git a/contrib/restricted/boost/context/src/asm/ontop_i386_sysv_macho_gas.S b/contrib/restricted/boost/context/src/asm/ontop_i386_sysv_macho_gas.S index c129e8e0c76..3a88372b3ab 100644 --- a/contrib/restricted/boost/context/src/asm/ontop_i386_sysv_macho_gas.S +++ b/contrib/restricted/boost/context/src/asm/ontop_i386_sysv_macho_gas.S @@ -25,7 +25,6 @@ ****************************************************************************************/ .text -.private_extern _ontop_fcontext .globl _ontop_fcontext .align 2 _ontop_fcontext: diff --git a/contrib/restricted/boost/context/src/asm/ontop_x86_64_sysv_elf_gas.S b/contrib/restricted/boost/context/src/asm/ontop_x86_64_sysv_elf_gas.S index e3e4bdf870a..c3892b8ba0a 100644 --- a/contrib/restricted/boost/context/src/asm/ontop_x86_64_sysv_elf_gas.S +++ b/contrib/restricted/boost/context/src/asm/ontop_x86_64_sysv_elf_gas.S @@ -40,7 +40,6 @@ .file "ontop_x86_64_sysv_elf_gas.S" .text .globl ontop_fcontext -.hidden ontop_fcontext .type ontop_fcontext,@function .align 16 ontop_fcontext: diff --git a/contrib/restricted/boost/context/src/asm/ontop_x86_64_sysv_macho_gas.S b/contrib/restricted/boost/context/src/asm/ontop_x86_64_sysv_macho_gas.S index 7d15b5ba243..49755c69b07 100644 --- a/contrib/restricted/boost/context/src/asm/ontop_x86_64_sysv_macho_gas.S +++ b/contrib/restricted/boost/context/src/asm/ontop_x86_64_sysv_macho_gas.S @@ -25,7 +25,6 @@ ****************************************************************************************/ .text -.private_extern _ontop_fcontext .globl _ontop_fcontext .align 8 _ontop_fcontext: diff --git a/contrib/restricted/boost/context/src/fcontext.cpp b/contrib/restricted/boost/context/src/fcontext.cpp deleted file mode 100644 index 9fe0a4892dd..00000000000 --- a/contrib/restricted/boost/context/src/fcontext.cpp +++ /dev/null @@ -1,39 +0,0 @@ -// SPDX-FileCopyrightText: Copyright 2024 Arm Limited and/or its affiliates <open-source-office@arm.com> -// Distributed under the Boost Software License, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at -// http://www.boost.org/LICENSE_1_0.txt) - -// This code wraps the plain C asm symbols into properly -// namespaced and mangled C++ symbols that are safe to -// export from a library. - -#include <boost/context/detail/fcontext.hpp> - -extern "C" -boost::context::detail::transfer_t BOOST_CONTEXT_CALLDECL jump_fcontext( boost::context::detail::fcontext_t const to, void * vp); -extern "C" -boost::context::detail::fcontext_t BOOST_CONTEXT_CALLDECL make_fcontext( void * sp, std::size_t size, void (* fn)( boost::context::detail::transfer_t) ); - -// based on an idea of Giovanni Derreta -extern "C" -boost::context::detail::transfer_t BOOST_CONTEXT_CALLDECL ontop_fcontext( boost::context::detail::fcontext_t const to, void * vp, boost::context::detail::transfer_t (* fn)( boost::context::detail::transfer_t) ); - -namespace boost { -namespace context { -namespace detail { - -transfer_t jump_fcontext( fcontext_t const to, void * vp) -{ - return ::jump_fcontext(to, vp); -} - -fcontext_t make_fcontext( void * sp, std::size_t size, void (* fn)( transfer_t) ) -{ - return ::make_fcontext(sp, size, fn); -} - -transfer_t ontop_fcontext( fcontext_t const to, void * vp, transfer_t (* fn)( transfer_t) ) -{ - return ::ontop_fcontext(to, vp, fn); -} -}}} diff --git a/contrib/restricted/boost/context/ucontext_impl/ya.make b/contrib/restricted/boost/context/ucontext_impl/ya.make index 8c9ce98e9b1..61ba1464c04 100644 --- a/contrib/restricted/boost/context/ucontext_impl/ya.make +++ b/contrib/restricted/boost/context/ucontext_impl/ya.make @@ -4,9 +4,9 @@ LIBRARY() WITHOUT_LICENSE_TEXTS() -VERSION(1.87.0) +VERSION(1.86.0) -ORIGINAL_SOURCE(https://github.com/boostorg/context/archive/boost-1.87.0.tar.gz) +ORIGINAL_SOURCE(https://github.com/boostorg/context/archive/boost-1.86.0.tar.gz) LICENSE(BSL-1.0) diff --git a/contrib/restricted/boost/context/ya.make b/contrib/restricted/boost/context/ya.make index d3c74aa9863..f70cdc034c2 100644 --- a/contrib/restricted/boost/context/ya.make +++ b/contrib/restricted/boost/context/ya.make @@ -6,9 +6,9 @@ LICENSE(BSL-1.0) LICENSE_TEXTS(.yandex_meta/licenses.list.txt) -VERSION(1.87.0) +VERSION(1.86.0) -ORIGINAL_SOURCE(https://github.com/boostorg/context/archive/boost-1.87.0.tar.gz) +ORIGINAL_SOURCE(https://github.com/boostorg/context/archive/boost-1.86.0.tar.gz) IF (SANITIZER_TYPE) PEERDIR( diff --git a/contrib/restricted/uriparser/.yandex_meta/__init__.py b/contrib/restricted/uriparser/.yandex_meta/__init__.py index a5e4630685c..b0736eb2830 100644 --- a/contrib/restricted/uriparser/.yandex_meta/__init__.py +++ b/contrib/restricted/uriparser/.yandex_meta/__init__.py @@ -23,7 +23,7 @@ def uriparser_post_install(self): uriparser = CMakeNinjaNixProject( - owners=["kikht", "shindo", "g:mds", "g:cpp-contrib"], + owners=["shindo", "g:mds", "g:cpp-contrib"], arcdir="contrib/restricted/uriparser", nixattr="uriparser", put={"testrunner": "test"}, diff --git a/library/cpp/monlib/encode/prometheus/prometheus_decoder_ut.cpp b/library/cpp/monlib/encode/prometheus/prometheus_decoder_ut.cpp index 9299076f933..72e13a39dda 100644 --- a/library/cpp/monlib/encode/prometheus/prometheus_decoder_ut.cpp +++ b/library/cpp/monlib/encode/prometheus/prometheus_decoder_ut.cpp @@ -68,12 +68,13 @@ Y_UNIT_TEST_SUITE(TPrometheusDecoderTest) { Y_UNIT_TEST(Minimal) { auto samples = Decode( "minimal_metric 1.234\n" + "big_num 1.04671344e+10\n" "another_metric -3e3 103948\n" "# Even that:\n" "no_labels{} 3\n" "# HELP line for non-existing metric will be ignored.\n"); - UNIT_ASSERT_EQUAL(samples.SamplesSize(), 3); + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 4); { auto& s = samples.GetSamples(0); UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); @@ -84,12 +85,19 @@ Y_UNIT_TEST_SUITE(TPrometheusDecoderTest) { { auto& s = samples.GetSamples(1); UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); + UNIT_ASSERT_EQUAL(1, s.LabelsSize()); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "big_num"); + ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 1.04671344e+10); + } + { + auto& s = samples.GetSamples(2); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); UNIT_ASSERT_EQUAL(s.LabelsSize(), 1); ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "another_metric"); ASSERT_DOUBLE_POINT(s, TInstant::MilliSeconds(103948), -3000.0); } { - auto& s = samples.GetSamples(2); + auto& s = samples.GetSamples(3); UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); UNIT_ASSERT_EQUAL(1, s.LabelsSize()); ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "no_labels"); diff --git a/yql/essentials/core/services/yql_lineage.cpp b/yql/essentials/core/services/yql_lineage.cpp index c78bbb6b820..12f624c5c17 100644 --- a/yql/essentials/core/services/yql_lineage.cpp +++ b/yql/essentials/core/services/yql_lineage.cpp @@ -543,9 +543,11 @@ private: if (lambda.GetTypeAnn()->GetKind() == ETypeAnnotationKind::List) { value = &body; while(value->IsCallable({"FlatMap", "OrderedFlatMap"})) { - if (value->Head().IsCallable("Member") && &value->Head().Head() == &arg) { - TString field(value->Head().Tail().Content()); - flattenColumns.emplace(value->Tail().Head().HeadPtr().Get(), field); + TNodeMap<TMaybe<TFieldsLineage>> visited; + if (auto res = ScanExprLineage(value->Head(), &arg, &innerLineage, visited, {})) { + for (const auto& f: res->Items) { + flattenColumns.emplace(value->Tail().Head().HeadPtr().Get(), f.Field); + } } value = &value->Tail().Tail(); } diff --git a/yql/essentials/minikql/comp_nodes/mkql_block_map_join.cpp b/yql/essentials/minikql/comp_nodes/mkql_block_map_join.cpp index 189f8a0d2df..fad21299c81 100644 --- a/yql/essentials/minikql/comp_nodes/mkql_block_map_join.cpp +++ b/yql/essentials/minikql/comp_nodes/mkql_block_map_join.cpp @@ -239,24 +239,147 @@ private: TVector<NYql::NUdf::IBlockItemHasher::TPtr> Hashers_; }; -class TBlockIndex : public TComputationValue<TBlockIndex> { - struct TIndexEntry { +template <typename TDerived> +class TBlockStorageBase : public TComputationValue<TDerived> { + using TBase = TComputationValue<TDerived>; + +public: + struct TBlock { + size_t Size; + std::vector<arrow::Datum> Columns; + + TBlock() = default; + TBlock(size_t size, std::vector<arrow::Datum> columns) + : Size(size) + , Columns(std::move(columns)) + {} + }; + + struct TRowEntry { ui32 BlockOffset; ui32 ItemOffset; - TIndexEntry() = default; - TIndexEntry(ui32 blockOffset, ui32 itemOffset) + TRowEntry() = default; + TRowEntry(ui32 blockOffset, ui32 itemOffset) : BlockOffset(blockOffset) , ItemOffset(itemOffset) {} }; + TBlockStorageBase( + TMemoryUsageInfo* memInfo, + const TVector<TType*>& itemTypes, + NUdf::TUnboxedValue stream, + arrow::MemoryPool* pool + ) + : TBase(memInfo) + , InputsDescr_(ToValueDescr(itemTypes)) + , Stream_(stream) + , Inputs_(itemTypes.size()) + { + TBlockTypeHelper helper; + for (size_t i = 0; i < itemTypes.size(); i++) { + TType* blockItemType = AS_TYPE(TBlockType, itemTypes[i])->GetItemType(); + Readers_.push_back(MakeBlockReader(TTypeInfoHelper(), blockItemType)); + Hashers_.push_back(helper.MakeHasher(blockItemType)); + Comparators_.push_back(helper.MakeComparator(blockItemType)); + Trimmers_.push_back(MakeBlockTrimmer(TTypeInfoHelper(), blockItemType, pool)); + } + } + + NUdf::EFetchStatus FetchStream() { + switch (Stream_.WideFetch(Inputs_.data(), Inputs_.size())) { + case NUdf::EFetchStatus::Yield: + return NUdf::EFetchStatus::Yield; + case NUdf::EFetchStatus::Finish: + return NUdf::EFetchStatus::Finish; + case NUdf::EFetchStatus::Ok: + break; + } + + std::vector<arrow::Datum> blockColumns; + for (size_t i = 0; i < Inputs_.size() - 1; i++) { + auto& datum = TArrowBlock::From(Inputs_[i]).GetDatum(); + ARROW_DEBUG_CHECK_DATUM_TYPES(InputsDescr_[i], datum.descr()); + if (datum.is_scalar()) { + blockColumns.push_back(datum); + } else { + MKQL_ENSURE(datum.is_array(), "Expecting array"); + blockColumns.push_back(Trimmers_[i]->Trim(datum.array())); + } + } + + auto blockSize = ::GetBlockCount(Inputs_[Inputs_.size() - 1]); + Data_.emplace_back(blockSize, std::move(blockColumns)); + RowCount_ += blockSize; + + return NUdf::EFetchStatus::Ok; + } + + const TBlock& GetBlock(size_t blockOffset) const { + Y_ENSURE(blockOffset < GetBlockCount()); + return Data_[blockOffset]; + } + + size_t GetBlockCount() const { + return Data_.size(); + } + + TBlockItem GetItem(TRowEntry entry, ui32 columnIdx) const { + Y_ENSURE(columnIdx < Inputs_.size() - 1); + return GetItemFromBlock(GetBlock(entry.BlockOffset), columnIdx, entry.ItemOffset); + } + + void GetRow(TRowEntry entry, const TVector<ui32>& ioMap, std::vector<NYql::NUdf::TBlockItem>& row) const { + Y_ENSURE(row.size() == ioMap.size()); + for (size_t i = 0; i < row.size(); i++) { + row[i] = GetItem(entry, ioMap[i]); + } + } + +protected: + TBlockItem GetItemFromBlock(const TBlock& block, ui32 columnIdx, size_t offset) const { + Y_ENSURE(offset < block.Size); + const auto& datum = block.Columns[columnIdx]; + if (datum.is_scalar()) { + return Readers_[columnIdx]->GetScalarItem(*datum.scalar()); + } else { + MKQL_ENSURE(datum.is_array(), "Expecting array"); + return Readers_[columnIdx]->GetItem(*datum.array(), offset); + } + } + +protected: + const std::vector<arrow::ValueDescr> InputsDescr_; + + TVector<std::unique_ptr<IBlockReader>> Readers_; + TVector<NUdf::IBlockItemHasher::TPtr> Hashers_; + TVector<NUdf::IBlockItemComparator::TPtr> Comparators_; + TVector<IBlockTrimmer::TPtr> Trimmers_; + + std::vector<TBlock> Data_; + size_t RowCount_ = 0; + + NUdf::TUnboxedValue Stream_; + TUnboxedValueVector Inputs_; +}; + +class TBlockStorage: public TBlockStorageBase<TBlockStorage> { +private: + using TBase = TBlockStorageBase<TBlockStorage>; +public: + using TBase::TBase; +}; + +class TIndexedBlockStorage : public TBlockStorageBase<TIndexedBlockStorage> { + using TBase = TBlockStorageBase<TIndexedBlockStorage>; + struct TIndexNode { - TIndexEntry Entry; + TRowEntry Entry; TIndexNode* Next; TIndexNode() = delete; - TIndexNode(TIndexEntry entry, TIndexNode* next = nullptr) + TIndexNode(TRowEntry entry, TIndexNode* next = nullptr) : Entry(entry) , Next(next) {} @@ -268,7 +391,7 @@ class TBlockIndex : public TComputationValue<TBlockIndex> { : Raw(0) {} - TIndexMapValue(TIndexEntry entry) { + TIndexMapValue(TRowEntry entry) { TIndexEntryUnion un; un.Entry = entry; @@ -289,7 +412,7 @@ class TBlockIndex : public TComputationValue<TBlockIndex> { return EntryList; } - TIndexEntry GetEntry() const { + TRowEntry GetEntry() const { Y_ENSURE(IsInplace()); TIndexEntryUnion un; @@ -299,7 +422,7 @@ class TBlockIndex : public TComputationValue<TBlockIndex> { private: union TIndexEntryUnion { - TIndexEntry Entry; + TRowEntry Entry; ui64 Raw; }; @@ -309,13 +432,12 @@ class TBlockIndex : public TComputationValue<TBlockIndex> { }; }; - using TBase = TComputationValue<TBlockIndex>; using TIndexMap = TRobinHoodHashFixedMap< ui64, TIndexMapValue, std::equal_to<ui64>, std::hash<ui64>, - TMKQLAllocator<char> + TMKQLHugeAllocator<char> >; static_assert(sizeof(TIndexMapValue) == 8); @@ -323,6 +445,8 @@ class TBlockIndex : public TComputationValue<TBlockIndex> { public: class TIterator { + friend class TIndexedBlockStorage; + enum class EIteratorType { EMPTY, INPLACE, @@ -332,26 +456,6 @@ public: public: TIterator() = default; - TIterator(const TBlockIndex* blockIndex) - : Type_(EIteratorType::EMPTY) - , BlockIndex_(blockIndex) - {} - - TIterator(const TBlockIndex* blockIndex, TIndexEntry entry, std::vector<NYql::NUdf::TBlockItem> itemsToLookup) - : Type_(EIteratorType::INPLACE) - , BlockIndex_(blockIndex) - , Entry_(entry) - , EntryConsumed_(false) - , ItemsToLookup_(std::move(itemsToLookup)) - {} - - TIterator(const TBlockIndex* blockIndex, TIndexNode* node, std::vector<NYql::NUdf::TBlockItem> itemsToLookup) - : Type_(EIteratorType::LIST) - , BlockIndex_(blockIndex) - , Node_(node) - , ItemsToLookup_(std::move(itemsToLookup)) - {} - TIterator(const TIterator&) = delete; TIterator& operator=(const TIterator&) = delete; @@ -384,7 +488,7 @@ public: return *this; } - TMaybe<TIndexEntry> Next() { + TMaybe<TRowEntry> Next() { Y_ENSURE(IsValid()); switch (Type_) { @@ -397,7 +501,7 @@ public: } EntryConsumed_ = true; - return BlockIndex_->IsKeyEquals(Entry_, ItemsToLookup_) ? TMaybe<TIndexEntry>(Entry_) : Nothing(); + return BlockIndex_->IsKeyEquals(Entry_, ItemsToLookup_) ? TMaybe<TRowEntry>(Entry_) : Nothing(); case EIteratorType::LIST: for (; Node_ != nullptr; Node_ = Node_->Next) { @@ -434,13 +538,34 @@ public: } private: + TIterator(const TIndexedBlockStorage* blockIndex) + : Type_(EIteratorType::EMPTY) + , BlockIndex_(blockIndex) + {} + + TIterator(const TIndexedBlockStorage* blockIndex, TRowEntry entry, std::vector<NYql::NUdf::TBlockItem> itemsToLookup) + : Type_(EIteratorType::INPLACE) + , BlockIndex_(blockIndex) + , Entry_(entry) + , EntryConsumed_(false) + , ItemsToLookup_(std::move(itemsToLookup)) + {} + + TIterator(const TIndexedBlockStorage* blockIndex, TIndexNode* node, std::vector<NYql::NUdf::TBlockItem> itemsToLookup) + : Type_(EIteratorType::LIST) + , BlockIndex_(blockIndex) + , Node_(node) + , ItemsToLookup_(std::move(itemsToLookup)) + {} + + private: EIteratorType Type_; - const TBlockIndex* BlockIndex_ = nullptr; + const TIndexedBlockStorage* BlockIndex_ = nullptr; union { TIndexNode* Node_; struct { - TIndexEntry Entry_; + TRowEntry Entry_; bool EntryConsumed_; }; }; @@ -449,7 +574,7 @@ public: }; public: - TBlockIndex( + TIndexedBlockStorage( TMemoryUsageInfo* memInfo, const TVector<TType*>& itemTypes, const TVector<ui32>& keyColumns, @@ -457,106 +582,75 @@ public: bool any, arrow::MemoryPool* pool ) - : TBase(memInfo) - , InputsDescr_(ToValueDescr(itemTypes)) + : TBase(memInfo, itemTypes, stream, pool) , KeyColumns_(keyColumns) - , Stream_(stream) - , Inputs_(itemTypes.size()) , Any_(any) - { - TBlockTypeHelper helper; - for (size_t i = 0; i < itemTypes.size(); i++) { - TType* blockItemType = AS_TYPE(TBlockType, itemTypes[i])->GetItemType(); - Readers_.push_back(MakeBlockReader(TTypeInfoHelper(), blockItemType)); - Hashers_.push_back(helper.MakeHasher(blockItemType)); - Comparators_.push_back(helper.MakeComparator(blockItemType)); - Trimmers_.push_back(MakeBlockTrimmer(TTypeInfoHelper(), blockItemType, pool)); - } - } + {} NUdf::EFetchStatus FetchStream() { - switch (Stream_.WideFetch(Inputs_.data(), Inputs_.size())) { - case NUdf::EFetchStatus::Yield: - return NUdf::EFetchStatus::Yield; - case NUdf::EFetchStatus::Finish: - return NUdf::EFetchStatus::Finish; - case NUdf::EFetchStatus::Ok: - break; - } + Y_ENSURE(!Index_, "Data fetch shouldn't be done after the index has been built"); + return TBase::FetchStream(); + } - { - std::vector<arrow::Datum> block; - for (size_t i = 0; i < Inputs_.size() - 1; i++) { - auto& datum = TArrowBlock::From(Inputs_[i]).GetDatum(); - ARROW_DEBUG_CHECK_DATUM_TYPES(InputsDescr_[i], datum.descr()); - if (datum.is_scalar()) { - block.push_back(datum); - } else { - MKQL_ENSURE(datum.is_array(), "Expecting array"); - block.push_back(Trimmers_[i]->Trim(datum.array())); - } - } - Data_.push_back(std::move(block)); - } + void BuildIndex() { + Index_ = std::make_unique<TIndexMap>(CalculateRHHashTableCapacity(RowCount_)); + for (size_t blockOffset = 0; blockOffset < Data_.size(); blockOffset++) { + const auto& block = GetBlock(blockOffset); + auto blockSize = block.Size; + + std::array<TRobinHoodBatchRequestItem<ui64>, PrefetchBatchSize> insertBatch; + std::array<TRowEntry, PrefetchBatchSize> insertBatchEntries; + std::array<std::vector<NYql::NUdf::TBlockItem>, PrefetchBatchSize> insertBatchKeys; + ui32 insertBatchLen = 0; + + auto processInsertBatch = [&]() { + Index_->BatchInsert({insertBatch.data(), insertBatchLen}, [&](size_t i, TIndexMap::iterator iter, bool isNew) { + auto value = static_cast<TIndexMapValue*>(Index_->GetMutablePayload(iter)); + if (isNew) { + // Store single entry inplace + *value = TIndexMapValue(insertBatchEntries[i]); + Index_->CheckGrow(); + } else { + if (Any_ && ContainsKey(value, insertBatchKeys[i])) { + return; + } - const auto& block = Data_.back(); - auto blockOffset = Data_.size() - 1; - auto blockSize = GetBlockCount(Inputs_[Inputs_.size() - 1]); - - std::array<TRobinHoodBatchRequestItem<ui64>, PrefetchBatchSize> insertBatch; - std::array<TIndexEntry, PrefetchBatchSize> insertBatchEntries; - std::array<std::vector<NYql::NUdf::TBlockItem>, PrefetchBatchSize> insertBatchKeys; - ui32 insertBatchLen = 0; - - auto processInsertBatch = [&]() { - Index_.BatchInsert({insertBatch.data(), insertBatchLen}, [&](size_t i, TIndexMap::iterator iter, bool isNew) { - auto value = static_cast<TIndexMapValue*>(Index_.GetMutablePayload(iter)); - if (isNew) { - // Store single entry inplace - *value = TIndexMapValue(insertBatchEntries[i]); - Index_.CheckGrow(); - } else { - if (Any_ && ContainsKey(value, insertBatchKeys[i])) { - return; - } + // Store as list + if (value->IsInplace()) { + *value = TIndexMapValue(InsertIndexNode(value->GetEntry())); + } - // Store as list - if (value->IsInplace()) { - *value = TIndexMapValue(InsertIndexNode(value->GetEntry())); + *value = TIndexMapValue(InsertIndexNode(insertBatchEntries[i], value->GetList())); } + }); + }; - *value = TIndexMapValue(InsertIndexNode(insertBatchEntries[i], value->GetList())); + Y_ENSURE(blockOffset <= std::numeric_limits<ui32>::max()); + Y_ENSURE(blockSize <= std::numeric_limits<ui32>::max()); + for (size_t itemOffset = 0; itemOffset < blockSize; itemOffset++) { + ui64 keyHash = GetKey(block, itemOffset, insertBatchKeys[insertBatchLen]); + if (!keyHash) { + continue; } - }); - }; - Y_ENSURE(blockOffset <= std::numeric_limits<ui32>::max()); - Y_ENSURE(blockSize <= std::numeric_limits<ui32>::max()); - for (size_t itemOffset = 0; itemOffset < blockSize; itemOffset++) { - ui64 keyHash = GetKey(block, itemOffset, insertBatchKeys[insertBatchLen]); - if (!keyHash) { - continue; - } + insertBatchEntries[insertBatchLen] = TRowEntry(blockOffset, itemOffset); + insertBatch[insertBatchLen].ConstructKey(keyHash); + insertBatchLen++; - insertBatchEntries[insertBatchLen] = TIndexEntry(blockOffset, itemOffset); - insertBatch[insertBatchLen].ConstructKey(keyHash); - insertBatchLen++; + if (insertBatchLen == PrefetchBatchSize) { + processInsertBatch(); + insertBatchLen = 0; + } + } - if (insertBatchLen == PrefetchBatchSize) { + if (insertBatchLen > 0) { processInsertBatch(); - insertBatchLen = 0; } } - - if (insertBatchLen > 0) { - processInsertBatch(); - } - - return NUdf::EFetchStatus::Ok; } template<typename TGetKey> - void BatchLookup(size_t batchSize, std::array<TBlockIndex::TIterator, PrefetchBatchSize>& iterators, TGetKey&& getKey) { + void BatchLookup(size_t batchSize, std::array<TIndexedBlockStorage::TIterator, PrefetchBatchSize>& iterators, TGetKey&& getKey) { Y_ENSURE(batchSize <= PrefetchBatchSize); std::array<TRobinHoodBatchRequestItem<ui64>, PrefetchBatchSize> lookupBatch; @@ -568,14 +662,14 @@ public: itemsBatch[i] = items; } - Index_.BatchLookup({lookupBatch.data(), batchSize}, [&](size_t i, TIndexMap::iterator iter) { + Index_->BatchLookup({lookupBatch.data(), batchSize}, [&](size_t i, TIndexMap::iterator iter) { if (!iter) { // Empty iterator iterators[i] = TIterator(this); return; } - auto value = static_cast<const TIndexMapValue*>(Index_.GetPayload(iter)); + auto value = static_cast<const TIndexMapValue*>(Index_->GetPayload(iter)); if (value->IsInplace()) { iterators[i] = TIterator(this, value->GetEntry(), std::move(itemsBatch[i])); } else { @@ -584,20 +678,7 @@ public: }); } - TBlockItem GetItem(TIndexEntry entry, ui32 columnIdx) const { - Y_ENSURE(entry.BlockOffset < Data_.size()); - Y_ENSURE(columnIdx < Inputs_.size() - 1); - return GetItemFromBlock(Data_[entry.BlockOffset], columnIdx, entry.ItemOffset); - } - - void GetRow(TIndexEntry entry, const TVector<ui32>& ioMap, std::vector<NYql::NUdf::TBlockItem>& row) const { - Y_ENSURE(row.size() == ioMap.size()); - for (size_t i = 0; i < row.size(); i++) { - row[i] = GetItem(entry, ioMap[i]); - } - } - - bool IsKeyEquals(TIndexEntry entry, const std::vector<NYql::NUdf::TBlockItem>& keyItems) const { + bool IsKeyEquals(TRowEntry entry, const std::vector<NYql::NUdf::TBlockItem>& keyItems) const { Y_ENSURE(keyItems.size() == KeyColumns_.size()); for (size_t i = 0; i < KeyColumns_.size(); i++) { auto indexItem = GetItem(entry, KeyColumns_[i]); @@ -610,7 +691,7 @@ public: } private: - ui64 GetKey(const std::vector<arrow::Datum>& block, size_t offset, std::vector<NYql::NUdf::TBlockItem>& keyItems) const { + ui64 GetKey(const TBlock& block, size_t offset, std::vector<NYql::NUdf::TBlockItem>& keyItems) const { ui64 keyHash = 0; keyItems.clear(); for (ui32 keyColumn : KeyColumns_) { @@ -627,17 +708,7 @@ private: return keyHash; } - TBlockItem GetItemFromBlock(const std::vector<arrow::Datum>& block, ui32 columnIdx, size_t offset) const { - const auto& datum = block[columnIdx]; - if (datum.is_scalar()) { - return Readers_[columnIdx]->GetScalarItem(*datum.scalar()); - } else { - MKQL_ENSURE(datum.is_array(), "Expecting array"); - return Readers_[columnIdx]->GetItem(*datum.array(), offset); - } - } - - TIndexNode* InsertIndexNode(TIndexEntry entry, TIndexNode* currentHead = nullptr) { + TIndexNode* InsertIndexNode(TRowEntry entry, TIndexNode* currentHead = nullptr) { return &IndexNodes_.emplace_back(entry, currentHead); } @@ -658,22 +729,11 @@ private: } private: - const std::vector<arrow::ValueDescr> InputsDescr_; const TVector<ui32>& KeyColumns_; - TVector<std::unique_ptr<IBlockReader>> Readers_; - TVector<NUdf::IBlockItemHasher::TPtr> Hashers_; - TVector<NUdf::IBlockItemComparator::TPtr> Comparators_; - TVector<IBlockTrimmer::TPtr> Trimmers_; - - std::vector<std::vector<arrow::Datum>> Data_; - - TIndexMap Index_; + std::unique_ptr<TIndexMap> Index_; std::deque<TIndexNode> IndexNodes_; - NUdf::TUnboxedValue Stream_; - TUnboxedValueVector Inputs_; - const bool Any_; }; @@ -682,7 +742,7 @@ class TBlockMapJoinCoreWraper : public TMutableComputationNode<TBlockMapJoinCore { using TBaseComputation = TMutableComputationNode<TBlockMapJoinCoreWraper<WithoutRight, RightRequired, RightAny>>; using TJoinState = TBlockJoinState<RightRequired>; -using TIndexState = TBlockIndex; +using TIndexState = TIndexedBlockStorage; public: TBlockMapJoinCoreWraper( TComputationMutables& mutables, @@ -728,10 +788,8 @@ public: std::move(joinState), std::move(indexState), std::move(LeftStream_->GetValue(ctx)), - LeftItemTypes_, LeftKeyColumns_, std::move(RightStream_->GetValue(ctx)), - RightItemTypes_, RightKeyColumns_, RightIOMap_ ); @@ -747,10 +805,8 @@ private: NUdf::TUnboxedValue&& joinState, NUdf::TUnboxedValue&& indexState, NUdf::TUnboxedValue&& leftStream, - const TVector<TType*>& leftTypes, const TVector<ui32>& leftKeyColumns, NUdf::TUnboxedValue&& rightStream, - const TVector<TType*>& rightTypes, const TVector<ui32>& rightKeyColumns, const TVector<ui32>& rightIOMap ) @@ -758,10 +814,8 @@ private: , JoinState_(joinState) , IndexState_(indexState) , LeftStream_(leftStream) - , LeftItemTypes_(leftTypes) , LeftKeyColumns_(leftKeyColumns) , RightStream_(rightStream) - , RightItemTypes_(rightTypes) , RightKeyColumns_(rightKeyColumns) , RightIOMap_(rightIOMap) , HolderFactory_(holderFactory) @@ -781,6 +835,7 @@ private: } } + indexState.BuildIndex(); RightStreamConsumed_ = true; } @@ -882,11 +937,9 @@ private: NUdf::TUnboxedValue IndexState_; NUdf::TUnboxedValue LeftStream_; - const TVector<TType*>& LeftItemTypes_; const TVector<ui32>& LeftKeyColumns_; NUdf::TUnboxedValue RightStream_; - const TVector<TType*>& RightItemTypes_; const TVector<ui32>& RightKeyColumns_; const TVector<ui32>& RightIOMap_; bool RightStreamConsumed_ = false; diff --git a/yql/essentials/minikql/mkql_alloc.h b/yql/essentials/minikql/mkql_alloc.h index cf20e066e98..24c18f41a8f 100644 --- a/yql/essentials/minikql/mkql_alloc.h +++ b/yql/essentials/minikql/mkql_alloc.h @@ -164,7 +164,7 @@ struct TMkqlPAllocHeader { ui64 Self; // should be placed right before pointer to allocated area, see GetMemoryChunkContext }; -static_assert(sizeof(TMkqlPAllocHeader) == +static_assert(sizeof(TMkqlPAllocHeader) == sizeof(size_t) + sizeof(TAllocState::TListEntry) + sizeof(void*), "Padding is not allowed"); @@ -497,6 +497,38 @@ struct TMKQLAllocator using TWithDefaultMiniKQLAlloc = TWithMiniKQLAlloc<EMemorySubPool::Default>; using TWithTemporaryMiniKQLAlloc = TWithMiniKQLAlloc<EMemorySubPool::Temporary>; +template <typename Type> +struct TMKQLHugeAllocator +{ + typedef Type value_type; + typedef Type* pointer; + typedef const Type* const_pointer; + typedef Type& reference; + typedef const Type& const_reference; + typedef size_t size_type; + typedef ptrdiff_t difference_type; + + TMKQLHugeAllocator() noexcept = default; + ~TMKQLHugeAllocator() noexcept = default; + + template<typename U> TMKQLHugeAllocator(const TMKQLHugeAllocator<U>&) noexcept {} + template<typename U> struct rebind { typedef TMKQLHugeAllocator<U> other; }; + template<typename U> bool operator==(const TMKQLHugeAllocator<U>&) const { return true; } + template<typename U> bool operator!=(const TMKQLHugeAllocator<U>&) const { return false; } + + static pointer allocate(size_type n, const void* = nullptr) + { + size_t size = Max(n * sizeof(value_type), TAllocState::POOL_PAGE_SIZE); + return static_cast<pointer>(TlsAllocState->GetBlock(size)); + } + + static void deallocate(const_pointer p, size_type n) noexcept + { + size_t size = Max(n * sizeof(value_type), TAllocState::POOL_PAGE_SIZE); + TlsAllocState->ReturnBlock(const_cast<pointer>(p), size); + } +}; + template <typename T> class TPagedList { diff --git a/yql/essentials/providers/common/provider/yql_provider.cpp b/yql/essentials/providers/common/provider/yql_provider.cpp index 79156cd8d37..0b7a1ef6609 100644 --- a/yql/essentials/providers/common/provider/yql_provider.cpp +++ b/yql/essentials/providers/common/provider/yql_provider.cpp @@ -604,6 +604,60 @@ TWriteReplicationSettings ParseWriteReplicationSettings(TExprList node, TExprCon return ret; } +TWriteTransferSettings ParseWriteTransferSettings(TExprList node, TExprContext& ctx) { + TMaybeNode<TCoAtom> mode; + TMaybeNode<TCoAtom> source; + TMaybeNode<TCoAtom> target; + TMaybeNode<TCoAtom> transformLambda; + TVector<TCoNameValueTuple> settings; + TVector<TCoNameValueTuple> other; + + for (auto child : node) { + if (auto maybeTuple = child.Maybe<TCoNameValueTuple>()) { + auto tuple = maybeTuple.Cast(); + auto name = tuple.Name().Value(); + + if (name == "mode") { + YQL_ENSURE(tuple.Value().Maybe<TCoAtom>()); + mode = tuple.Value().Cast<TCoAtom>(); + } else if (name == "source") { + YQL_ENSURE(tuple.Value().Maybe<TCoAtom>()); + source = tuple.Value().Cast<TCoAtom>(); + } else if (name == "target") { + YQL_ENSURE(tuple.Value().Maybe<TCoAtom>()); + target = tuple.Value().Cast<TCoAtom>(); + } else if (name == "transformLambda") { + YQL_ENSURE(tuple.Value().Maybe<TCoAtom>()); + transformLambda = tuple.Value().Cast<TCoAtom>(); + } else if (name == "settings") { + YQL_ENSURE(tuple.Value().Maybe<TCoNameValueTupleList>()); + for (const auto& item : tuple.Value().Cast<TCoNameValueTupleList>()) { + settings.push_back(item); + } + } else { + other.push_back(tuple); + } + } + } + + const auto& builtSettings = Build<TCoNameValueTupleList>(ctx, node.Pos()) + .Add(settings) + .Done(); + + const auto& builtOther = Build<TCoNameValueTupleList>(ctx, node.Pos()) + .Add(other) + .Done(); + + TWriteTransferSettings ret(builtOther); + ret.Mode = mode; + ret.Source = source; + ret.Target = target; + ret.TransformLambda = transformLambda; + ret.TransferSettings = builtSettings; + + return ret; +} + TWriteRoleSettings ParseWriteRoleSettings(TExprList node, TExprContext& ctx) { TMaybeNode<TCoAtom> mode; TVector<TCoAtom> roles; @@ -858,7 +912,7 @@ bool FillUsedFilesImpl( if (node.GetTypeAnn()) { usedPgExtensions |= node.GetTypeAnn()->GetUsedPgExtensions(); } - + if (node.IsCallable("PgResolvedCall")) { auto procId = FromString<ui32>(node.Child(1)->Content()); const auto& proc = NPg::LookupProc(procId); @@ -1072,7 +1126,7 @@ void FillSecureParams( } } -bool AddPgFile(bool isPath, const TString& pathOrContent, const TString& md5, const TString& alias, TUserDataTable& files, +bool AddPgFile(bool isPath, const TString& pathOrContent, const TString& md5, const TString& alias, TUserDataTable& files, const TTypeAnnotationContext& types, TPositionHandle pos, TExprContext& ctx) { TUserDataBlock block; @@ -1140,7 +1194,7 @@ bool FillUsedFiles( return false; } } - + Y_ENSURE(remainingPgExtensions == 0); if (!needFullPgCatalog) { return true; diff --git a/yql/essentials/providers/common/provider/yql_provider.h b/yql/essentials/providers/common/provider/yql_provider.h index 978621d4b16..ea9992dd5e2 100644 --- a/yql/essentials/providers/common/provider/yql_provider.h +++ b/yql/essentials/providers/common/provider/yql_provider.h @@ -97,6 +97,19 @@ struct TWriteReplicationSettings { {} }; +struct TWriteTransferSettings { + NNodes::TMaybeNode<NNodes::TCoAtom> Mode; + NNodes::TMaybeNode<NNodes::TCoAtom> Source; + NNodes::TMaybeNode<NNodes::TCoAtom> Target; + NNodes::TMaybeNode<NNodes::TCoAtom> TransformLambda; + NNodes::TMaybeNode<NNodes::TCoNameValueTupleList> TransferSettings; + NNodes::TCoNameValueTupleList Other; + + TWriteTransferSettings(const NNodes::TCoNameValueTupleList& other) + : Other(other) + {} +}; + struct TWriteRoleSettings { NNodes::TMaybeNode<NNodes::TCoAtom> Mode; NNodes::TMaybeNode<NNodes::TCoAtomList> Roles; @@ -168,6 +181,7 @@ TVector<TString> GetResOrPullColumnHints(const TExprNode& node); TWriteTableSettings ParseWriteTableSettings(NNodes::TExprList node, TExprContext& ctx); TWriteTopicSettings ParseWriteTopicSettings(NNodes::TExprList node, TExprContext& ctx); TWriteReplicationSettings ParseWriteReplicationSettings(NNodes::TExprList node, TExprContext& ctx); +TWriteTransferSettings ParseWriteTransferSettings(NNodes::TExprList node, TExprContext& ctx); TWriteRoleSettings ParseWriteRoleSettings(NNodes::TExprList node, TExprContext& ctx); TWriteObjectSettings ParseWriteObjectSettings(NNodes::TExprList node, TExprContext& ctx); diff --git a/yql/essentials/sql/sql.cpp b/yql/essentials/sql/sql.cpp index 55409d5ba3f..6ecd365a906 100644 --- a/yql/essentials/sql/sql.cpp +++ b/yql/essentials/sql/sql.cpp @@ -144,6 +144,10 @@ namespace NSQLTranslation { } NYql::TAstParseResult SqlASTToYql(const google::protobuf::Message& protoAst, const TSQLHints& hints, const TTranslationSettings& settings) { + return SqlASTToYql("", protoAst, hints, settings); + } + + NYql::TAstParseResult SqlASTToYql(const TString& query, const google::protobuf::Message& protoAst, const TSQLHints& hints, const TTranslationSettings& settings) { NYql::TAstParseResult result; switch (settings.SyntaxVersion) { case 0: @@ -161,7 +165,7 @@ namespace NSQLTranslation { return NSQLTranslationV0::SqlASTToYql(protoAst, settings); case 1: - return NSQLTranslationV1::SqlASTToYql(protoAst, hints, settings); + return NSQLTranslationV1::SqlASTToYql(query, protoAst, hints, settings); default: result.Issues.AddIssue(NYql::YqlIssue(NYql::TPosition(), NYql::TIssuesIds::DEFAULT_ERROR, TStringBuilder() << "Unknown SQL syntax version: " << settings.SyntaxVersion)); diff --git a/yql/essentials/sql/sql.h b/yql/essentials/sql/sql.h index 26a61b22313..f0f844c88a1 100644 --- a/yql/essentials/sql/sql.h +++ b/yql/essentials/sql/sql.h @@ -21,7 +21,11 @@ namespace NSQLTranslation { google::protobuf::Message* SqlAST(const TString& query, const TString& queryName, NYql::TIssues& issues, size_t maxErrors, const TTranslationSettings& settings = {}, ui16* actualSyntaxVersion = nullptr); ILexer::TPtr SqlLexer(const TString& query, NYql::TIssues& issues, const TTranslationSettings& settings = {}, ui16* actualSyntaxVersion = nullptr); + + /*[[deprecated]] Use SqlASTToYql(query, protoAst, hints, settings)*/ NYql::TAstParseResult SqlASTToYql(const google::protobuf::Message& protoAst, const TSQLHints& hints, const TTranslationSettings& settings); + NYql::TAstParseResult SqlASTToYql(const TString& query, const google::protobuf::Message& protoAst, const TSQLHints& hints, const TTranslationSettings& settings); + TVector<NYql::TAstParseResult> SqlToAstStatements(const TString& query, const TTranslationSettings& settings, NYql::TWarningRules* warningRules = nullptr, ui16* actualSyntaxVersion = nullptr, TVector<NYql::TStmtParseInfo>* stmtParseInfo = nullptr); diff --git a/yql/essentials/sql/v1/SQLv1.g.in b/yql/essentials/sql/v1/SQLv1.g.in index 192a1a6026f..510afaa501e 100644 --- a/yql/essentials/sql/v1/SQLv1.g.in +++ b/yql/essentials/sql/v1/SQLv1.g.in @@ -76,6 +76,9 @@ sql_stmt_core: | backup_stmt | restore_stmt | alter_sequence_stmt + | create_transfer_stmt + | alter_transfer_stmt + | drop_transfer_stmt ; expr: @@ -906,6 +909,29 @@ alter_replication_set_setting: SET LPAREN replication_settings RPAREN; drop_replication_stmt: DROP ASYNC REPLICATION object_ref CASCADE?; +create_transfer_stmt: CREATE TRANSFER object_ref + FROM object_ref TO object_ref (USING lambda_or_parameter)? + WITH LPAREN transfer_settings RPAREN +; + +lambda_or_parameter: + lambda + | bind_parameter +; +transfer_settings: transfer_settings_entry (COMMA transfer_settings_entry)*; +transfer_settings_entry: an_id EQUALS expr; + +alter_transfer_stmt: ALTER TRANSFER object_ref alter_transfer_action (COMMA alter_transfer_action)*; +alter_transfer_action: + alter_transfer_set_setting + | alter_transfer_set_using +; + +alter_transfer_set_setting: SET LPAREN transfer_settings RPAREN; +alter_transfer_set_using: SET USING lambda_or_parameter; + +drop_transfer_stmt: DROP TRANSFER object_ref CASCADE?; + action_or_subquery_args: opt_bind_parameter (COMMA opt_bind_parameter)*; define_action_or_subquery_stmt: DEFINE (ACTION|SUBQUERY) bind_parameter LPAREN action_or_subquery_args? RPAREN AS define_action_or_subquery_body END DEFINE; @@ -1466,6 +1492,7 @@ keyword_as_compat: | TO | TOPIC | TRANSACTION + | TRANSFER | TRIGGER | TYPE | UNCONDITIONAL @@ -1693,6 +1720,7 @@ keyword_compat: ( | TO | TOPIC | TRANSACTION + | TRANSFER | TRIGGER | TYPE | UNCONDITIONAL @@ -2072,6 +2100,7 @@ TIES: T I E S; TO: T O; TOPIC: T O P I C; TRANSACTION: T R A N S A C T I O N; +TRANSFER: T R A N S F E R; TRIGGER: T R I G G E R; TRUE: T R U E; TUPLE: T U P L E; diff --git a/yql/essentials/sql/v1/SQLv1Antlr4.g.in b/yql/essentials/sql/v1/SQLv1Antlr4.g.in index 3ae4ad6492d..2ddfd0c748c 100644 --- a/yql/essentials/sql/v1/SQLv1Antlr4.g.in +++ b/yql/essentials/sql/v1/SQLv1Antlr4.g.in @@ -75,6 +75,9 @@ sql_stmt_core: | backup_stmt | restore_stmt | alter_sequence_stmt + | create_transfer_stmt + | alter_transfer_stmt + | drop_transfer_stmt ; expr: @@ -905,6 +908,30 @@ alter_replication_set_setting: SET LPAREN replication_settings RPAREN; drop_replication_stmt: DROP ASYNC REPLICATION object_ref CASCADE?; +lambda_or_parameter: + lambda + | bind_parameter +; + +create_transfer_stmt: CREATE TRANSFER object_ref + FROM object_ref TO object_ref (USING lambda_or_parameter)? + WITH LPAREN transfer_settings RPAREN +; + +transfer_settings: transfer_settings_entry (COMMA transfer_settings_entry)*; +transfer_settings_entry: an_id EQUALS expr; + +alter_transfer_stmt: ALTER TRANSFER object_ref alter_transfer_action (COMMA alter_transfer_action)*; +alter_transfer_action: + alter_transfer_set_setting + | alter_transfer_set_using +; + +alter_transfer_set_setting: SET LPAREN transfer_settings RPAREN; +alter_transfer_set_using: SET USING lambda_or_parameter; + +drop_transfer_stmt: DROP TRANSFER object_ref CASCADE?; + action_or_subquery_args: opt_bind_parameter (COMMA opt_bind_parameter)*; define_action_or_subquery_stmt: DEFINE (ACTION|SUBQUERY) bind_parameter LPAREN action_or_subquery_args? RPAREN AS define_action_or_subquery_body END DEFINE; @@ -1465,6 +1492,7 @@ keyword_as_compat: | TO | TOPIC | TRANSACTION + | TRANSFER | TRIGGER | TYPE | UNCONDITIONAL @@ -1692,6 +1720,7 @@ keyword_compat: ( | TO | TOPIC | TRANSACTION + | TRANSFER | TRIGGER | TYPE | UNCONDITIONAL @@ -2071,6 +2100,7 @@ TIES: T I E S; TO: T O; TOPIC: T O P I C; TRANSACTION: T R A N S A C T I O N; +TRANSFER: T R A N S F E R; TRIGGER: T R I G G E R; TRUE: T R U E; TUPLE: T U P L E; diff --git a/yql/essentials/sql/v1/context.cpp b/yql/essentials/sql/v1/context.cpp index 43afc53c38b..d28c31469b4 100644 --- a/yql/essentials/sql/v1/context.cpp +++ b/yql/essentials/sql/v1/context.cpp @@ -83,12 +83,14 @@ THashMap<TStringBuf, TPragmaMaybeField> CTX_PRAGMA_MAYBE_FIELDS = { TContext::TContext(const NSQLTranslation::TTranslationSettings& settings, const NSQLTranslation::TSQLHints& hints, - TIssues& issues) + TIssues& issues, + const TString& query) : ClusterMapping(settings.ClusterMapping) , PathPrefix(settings.PathPrefix) , ClusterPathPrefixes(settings.ClusterPathPrefixes) , SQLHints(hints) , Settings(settings) + , Query(query) , Pool(new TMemoryPool(4096)) , Issues(issues) , IncrementMonCounterFunction(settings.IncrementCounter) diff --git a/yql/essentials/sql/v1/context.h b/yql/essentials/sql/v1/context.h index a5001fd157c..85a4739eaf5 100644 --- a/yql/essentials/sql/v1/context.h +++ b/yql/essentials/sql/v1/context.h @@ -92,7 +92,8 @@ namespace NSQLTranslationV1 { public: TContext(const NSQLTranslation::TTranslationSettings& settings, const NSQLTranslation::TSQLHints& hints, - NYql::TIssues& issues); + NYql::TIssues& issues, + const TString& query = {}); virtual ~TContext(); @@ -237,6 +238,7 @@ namespace NSQLTranslationV1 { THashMap<TString, std::pair<TPosition, TNodePtr>> Variables; THashSet<TString> WeakVariables; NSQLTranslation::TTranslationSettings Settings; + const TString Query; std::unique_ptr<TMemoryPool> Pool; NYql::TIssues& Issues; TMap<TString, TNodePtr> UniversalAliases; @@ -328,6 +330,7 @@ namespace NSQLTranslationV1 { bool DistinctOverWindow = false; bool SeqMode = false; bool EmitUnionMerge = false; + TVector<size_t> ForAllStatementsParts; }; class TColumnRefScope { diff --git a/yql/essentials/sql/v1/format/sql_format.cpp b/yql/essentials/sql/v1/format/sql_format.cpp index b6986310aa1..26d7eb9287b 100644 --- a/yql/essentials/sql/v1/format/sql_format.cpp +++ b/yql/essentials/sql/v1/format/sql_format.cpp @@ -165,144 +165,6 @@ bool Validate(const TParsedTokenList& query, const TParsedTokenList& formattedQu return in == inEnd && out == outEnd && parenthesesBalance == 0; } -enum EParenType { - Open, - Close, - None -}; - -using TAdvanceCallback = std::function<EParenType(TTokenIterator& curr, TTokenIterator end)>; - -TTokenIterator SkipToNextBalanced(TTokenIterator begin, TTokenIterator end, const TAdvanceCallback& advance) { - i64 level = 0; - TTokenIterator curr = begin; - while (curr != end) { - switch (advance(curr, end)) { - case EParenType::Open: { - ++level; - break; - } - case EParenType::Close: { - --level; - if (level < 0) { - return end; - } else if (level == 0) { - return curr; - } - break; - } - case EParenType::None: - break; - } - } - return curr; -} - -TTokenIterator GetNextStatementBegin(TTokenIterator begin, TTokenIterator end) { - TAdvanceCallback advanceLambdaBody = [](TTokenIterator& curr, TTokenIterator end) -> EParenType { - Y_UNUSED(end); - if (curr->Name == "LBRACE_CURLY") { - ++curr; - return EParenType::Open; - } else if (curr->Name == "RBRACE_CURLY") { - ++curr; - return EParenType::Close; - } else { - ++curr; - return EParenType::None; - } - }; - - TAdvanceCallback advanceAction = [](TTokenIterator& curr, TTokenIterator end) -> EParenType { - auto tmp = curr; - if (curr->Name == "DEFINE") { - ++curr; - curr = SkipWSOrComment(curr, end); - if (curr != end && (curr->Name == "ACTION" || curr->Name == "SUBQUERY")) { - ++curr; - return EParenType::Open; - } - } else if (curr->Name == "END") { - ++curr; - curr = SkipWSOrComment(curr, end); - if (curr != end && curr->Name == "DEFINE") { - ++curr; - return EParenType::Close; - } - } - - curr = tmp; - ++curr; - return EParenType::None; - }; - - TAdvanceCallback advanceInlineAction = [](TTokenIterator& curr, TTokenIterator end) -> EParenType { - auto tmp = curr; - if (curr->Name == "DO") { - ++curr; - curr = SkipWSOrComment(curr, end); - if (curr != end && curr->Name == "BEGIN") { - ++curr; - return EParenType::Open; - } - } else if (curr->Name == "END") { - ++curr; - curr = SkipWSOrComment(curr, end); - if (curr != end && curr->Name == "DO") { - ++curr; - return EParenType::Close; - } - } - - curr = tmp; - ++curr; - return EParenType::None; - }; - - TTokenIterator curr = begin; - while (curr != end) { - bool matched = false; - for (auto cb : {advanceLambdaBody, advanceAction, advanceInlineAction}) { - TTokenIterator tmp = curr; - if (cb(tmp, end) == EParenType::Open) { - curr = SkipToNextBalanced(curr, end, cb); - matched = true; - if (curr == end) { - return curr; - } - } - } - if (matched) { - continue; - } - if (curr->Name == "SEMICOLON") { - auto next = SkipWS(curr + 1, end); - while (next != end && next->Name == "COMMENT" && curr->Line == next->Line) { - curr = next; - next = SkipWS(next + 1, end); - } - ++curr; - break; - } - ++curr; - } - - return curr; -} - -void SplitByStatements(TTokenIterator begin, TTokenIterator end, TVector<TTokenIterator>& output) { - output.clear(); - if (begin == end) { - return; - } - output.push_back(begin); - auto curr = begin; - while (curr != end) { - curr = GetNextStatementBegin(curr, end); - output.push_back(curr); - } -} - enum class EScope { Default, TypeName, @@ -833,6 +695,7 @@ private: case TRule_sql_stmt_core::kAltSqlStmtCore14: // export case TRule_sql_stmt_core::kAltSqlStmtCore32: // drop external data source case TRule_sql_stmt_core::kAltSqlStmtCore34: // drop replication + case TRule_sql_stmt_core::kAltSqlStmtCore60: // drop transfer return true; case TRule_sql_stmt_core::kAltSqlStmtCore3: { // named nodes const auto& stmt = msg.GetAlt_sql_stmt_core3().GetRule_named_nodes_stmt1(); @@ -1541,6 +1404,21 @@ private: VisitAllFields(TRule_drop_replication_stmt::GetDescriptor(), msg); } + void VisitCreateTransfer(const TRule_create_transfer_stmt& msg) { + NewLine(); + VisitAllFields(TRule_create_transfer_stmt::GetDescriptor(), msg); + } + + void VisitAlterTransfer(const TRule_alter_transfer_stmt& msg) { + NewLine(); + VisitAllFields(TRule_alter_transfer_stmt::GetDescriptor(), msg); + } + + void VisitDropTransfer(const TRule_drop_transfer_stmt& msg) { + NewLine(); + VisitAllFields(TRule_drop_transfer_stmt::GetDescriptor(), msg); + } + void VisitCreateResourcePool(const TRule_create_resource_pool_stmt& msg) { NewLine(); VisitAllFields(TRule_create_resource_pool_stmt::GetDescriptor(), msg); @@ -3107,6 +2985,9 @@ TStaticData::TStaticData() {TRule_create_replication_stmt::GetDescriptor(), MakePrettyFunctor(&TPrettyVisitor::VisitCreateAsyncReplication)}, {TRule_alter_replication_stmt::GetDescriptor(), MakePrettyFunctor(&TPrettyVisitor::VisitAlterAsyncReplication)}, {TRule_drop_replication_stmt::GetDescriptor(), MakePrettyFunctor(&TPrettyVisitor::VisitDropAsyncReplication)}, + {TRule_create_transfer_stmt::GetDescriptor(), MakePrettyFunctor(&TPrettyVisitor::VisitCreateTransfer)}, + {TRule_alter_transfer_stmt::GetDescriptor(), MakePrettyFunctor(&TPrettyVisitor::VisitAlterTransfer)}, + {TRule_drop_transfer_stmt::GetDescriptor(), MakePrettyFunctor(&TPrettyVisitor::VisitDropTransfer)}, {TRule_create_topic_stmt::GetDescriptor(), MakePrettyFunctor(&TPrettyVisitor::VisitCreateTopic)}, {TRule_alter_topic_stmt::GetDescriptor(), MakePrettyFunctor(&TPrettyVisitor::VisitAlterTopic)}, {TRule_drop_topic_stmt::GetDescriptor(), MakePrettyFunctor(&TPrettyVisitor::VisitDropTopic)}, @@ -3188,42 +3069,15 @@ public: } auto lexer = NSQLTranslationV1::MakeLexer(parsedSettings.AnsiLexer, parsedSettings.Antlr4Parser); - TParsedTokenList allTokens; - auto onNextToken = [&](NSQLTranslation::TParsedToken&& token) { - if (token.Name != "EOF") { - allTokens.push_back(token); - } - }; - - if (!lexer->Tokenize(query, "Query", onNextToken, issues, NSQLTranslation::SQL_MAX_PARSER_ERRORS)) { + TVector<TString> statements; + if (!NSQLTranslationV1::SplitQueryToStatements(query, lexer, statements, issues)) { return false; } - TVector<TTokenIterator> statements; - SplitByStatements(allTokens.begin(), allTokens.end(), statements); TStringBuilder finalFormattedQuery; bool prevAddLine = false; TMaybe<ui32> prevStmtCoreAltCase; - for (size_t i = 1; i < statements.size(); ++i) { - TStringBuilder currentQueryBuilder; - for (auto it = statements[i - 1]; it != statements[i]; ++it) { - currentQueryBuilder << it->Content; - } - - TString currentQuery = currentQueryBuilder; - currentQuery = StripStringLeft(currentQuery); - bool isBlank = true; - for (auto c : currentQuery) { - if (c != ';') { - isBlank = false; - break; - } - }; - - if (isBlank) { - continue; - } - + for (const TString& currentQuery : statements) { TVector<NSQLTranslation::TParsedToken> comments; TParsedTokenList parsedTokens, stmtTokens; auto onNextRawToken = [&](NSQLTranslation::TParsedToken&& token) { diff --git a/yql/essentials/sql/v1/format/sql_format_ut.h b/yql/essentials/sql/v1/format/sql_format_ut.h index bd62ddf3687..a8fd6b7b3ff 100644 --- a/yql/essentials/sql/v1/format/sql_format_ut.h +++ b/yql/essentials/sql/v1/format/sql_format_ut.h @@ -383,6 +383,26 @@ Y_UNIT_TEST(AsyncReplication) { setup.Run(cases); } +Y_UNIT_TEST(Transfer) { + TCases cases = { + {"create transfer user from topic1 to table1 with (user='foo')", + "CREATE TRANSFER user FROM topic1 TO table1 WITH (user = 'foo');\n"}, + {"alter transfer user set (user='foo')", + "ALTER TRANSFER user SET (user = 'foo');\n"}, + {"drop transfer user", + "DROP TRANSFER user;\n"}, + {"drop transfer user cascade", + "DROP TRANSFER user CASCADE;\n"}, + {"create transfer user from topic1 to table1 using ($x) -> { $y = cast($x as String); return $y ; } with (user='foo')", + "CREATE TRANSFER user FROM topic1 TO table1 USING ($x) -> {\n $y = CAST($x AS String);\n RETURN $y;\n} WITH (user = 'foo');\n"}, + {"create transfer user from topic1 to table1 using $xxx with (user='foo')", + "CREATE TRANSFER user FROM topic1 TO table1 USING $xxx WITH (user = 'foo');\n"}, + }; + + TSetup setup; + setup.Run(cases); +} + Y_UNIT_TEST(ExternalTableOperations) { TCases cases = { {"creAte exTernAl TabLe usEr (a int) With (a = \"b\")", diff --git a/yql/essentials/sql/v1/lexer/lexer.cpp b/yql/essentials/sql/v1/lexer/lexer.cpp index 1d38ec3d8b0..2609e0f7f6f 100644 --- a/yql/essentials/sql/v1/lexer/lexer.cpp +++ b/yql/essentials/sql/v1/lexer/lexer.cpp @@ -8,8 +8,11 @@ #include <yql/essentials/parser/proto_ast/gen/v1_ansi/SQLv1Lexer.h> #include <yql/essentials/parser/proto_ast/gen/v1_antlr4/SQLv1Antlr4Lexer.h> #include <yql/essentials/parser/proto_ast/gen/v1_ansi_antlr4/SQLv1Antlr4Lexer.h> +#include <yql/essentials/sql/v1/sql.h> #include <util/string/ascii.h> +#include <util/string/builder.h> +#include <util/string/strip.h> #if defined(_tsan_enabled_) #include <util/system/mutex.h> @@ -80,4 +83,204 @@ bool IsProbablyKeyword(const NSQLTranslation::TParsedToken& token) { return AsciiEqualsIgnoreCase(token.Name, token.Content); } +using NSQLTranslation::TParsedTokenList; +using TTokenIterator = TParsedTokenList::const_iterator; + +namespace { + +enum EParenType { + Open, + Close, + None +}; + +using TAdvanceCallback = std::function<EParenType(TTokenIterator& curr, TTokenIterator end)>; + +TTokenIterator SkipWS(TTokenIterator curr, TTokenIterator end) { + while (curr != end && curr->Name == "WS") { + ++curr; + } + return curr; +} + +TTokenIterator SkipWSOrComment(TTokenIterator curr, TTokenIterator end) { + while (curr != end && (curr->Name == "WS" || curr->Name == "COMMENT")) { + ++curr; + } + return curr; +} + +TTokenIterator SkipToNextBalanced(TTokenIterator begin, TTokenIterator end, const TAdvanceCallback& advance) { + i64 level = 0; + TTokenIterator curr = begin; + while (curr != end) { + switch (advance(curr, end)) { + case EParenType::Open: { + ++level; + break; + } + case EParenType::Close: { + --level; + if (level < 0) { + return end; + } else if (level == 0) { + return curr; + } + break; + } + case EParenType::None: + break; + } + } + return curr; +} + +TTokenIterator GetNextStatementBegin(TTokenIterator begin, TTokenIterator end) { + TAdvanceCallback advanceLambdaBody = [](TTokenIterator& curr, TTokenIterator end) -> EParenType { + Y_UNUSED(end); + if (curr->Name == "LBRACE_CURLY") { + ++curr; + return EParenType::Open; + } else if (curr->Name == "RBRACE_CURLY") { + ++curr; + return EParenType::Close; + } else { + ++curr; + return EParenType::None; + } + }; + + TAdvanceCallback advanceAction = [](TTokenIterator& curr, TTokenIterator end) -> EParenType { + auto tmp = curr; + if (curr->Name == "DEFINE") { + ++curr; + curr = SkipWSOrComment(curr, end); + if (curr != end && (curr->Name == "ACTION" || curr->Name == "SUBQUERY")) { + ++curr; + return EParenType::Open; + } + } else if (curr->Name == "END") { + ++curr; + curr = SkipWSOrComment(curr, end); + if (curr != end && curr->Name == "DEFINE") { + ++curr; + return EParenType::Close; + } + } + + curr = tmp; + ++curr; + return EParenType::None; + }; + + TAdvanceCallback advanceInlineAction = [](TTokenIterator& curr, TTokenIterator end) -> EParenType { + auto tmp = curr; + if (curr->Name == "DO") { + ++curr; + curr = SkipWSOrComment(curr, end); + if (curr != end && curr->Name == "BEGIN") { + ++curr; + return EParenType::Open; + } + } else if (curr->Name == "END") { + ++curr; + curr = SkipWSOrComment(curr, end); + if (curr != end && curr->Name == "DO") { + ++curr; + return EParenType::Close; + } + } + + curr = tmp; + ++curr; + return EParenType::None; + }; + + TTokenIterator curr = begin; + while (curr != end) { + bool matched = false; + for (auto cb : {advanceLambdaBody, advanceAction, advanceInlineAction}) { + TTokenIterator tmp = curr; + if (cb(tmp, end) == EParenType::Open) { + curr = SkipToNextBalanced(curr, end, cb); + matched = true; + if (curr == end) { + return curr; + } + } + } + if (matched) { + continue; + } + if (curr->Name == "SEMICOLON") { + auto next = SkipWS(curr + 1, end); + while (next != end && next->Name == "COMMENT" && curr->Line == next->Line) { + curr = next; + next = SkipWS(next + 1, end); + } + ++curr; + break; + } + ++curr; + } + + return curr; +} + +void SplitByStatements(TTokenIterator begin, TTokenIterator end, TVector<TTokenIterator>& output) { + output.clear(); + if (begin == end) { + return; + } + output.push_back(begin); + auto curr = begin; + while (curr != end) { + curr = GetNextStatementBegin(curr, end); + output.push_back(curr); + } +} + +} + +bool SplitQueryToStatements(const TString& query, NSQLTranslation::ILexer::TPtr& lexer, TVector<TString>& statements, NYql::TIssues& issues) { + TParsedTokenList allTokens; + auto onNextToken = [&](NSQLTranslation::TParsedToken&& token) { + if (token.Name != "EOF") { + allTokens.push_back(token); + } + }; + + if (!lexer->Tokenize(query, "Query", onNextToken, issues, NSQLTranslation::SQL_MAX_PARSER_ERRORS)) { + return false; + } + + TVector<TTokenIterator> statementsTokens; + SplitByStatements(allTokens.begin(), allTokens.end(), statementsTokens); + + for (size_t i = 1; i < statementsTokens.size(); ++i) { + TStringBuilder currentQueryBuilder; + for (auto it = statementsTokens[i - 1]; it != statementsTokens[i]; ++it) { + currentQueryBuilder << it->Content; + } + TString statement = currentQueryBuilder; + statement = StripStringLeft(statement); + + bool isBlank = true; + for (auto c : statement) { + if (c != ';') { + isBlank = false; + break; + } + }; + + if (isBlank) { + continue; + } + + statements.push_back(statement); + } + + return true; +} + } // namespace NSQLTranslationV1 diff --git a/yql/essentials/sql/v1/lexer/lexer.h b/yql/essentials/sql/v1/lexer/lexer.h index 25bfe28f81f..d43efc6387f 100644 --- a/yql/essentials/sql/v1/lexer/lexer.h +++ b/yql/essentials/sql/v1/lexer/lexer.h @@ -12,4 +12,6 @@ NSQLTranslation::ILexer::TPtr MakeLexer(bool ansi, bool antlr4); // in SELECT * FROM ... GROUP BY ... - group is a keyword. bool IsProbablyKeyword(const NSQLTranslation::TParsedToken& token); +bool SplitQueryToStatements(const TString& query, NSQLTranslation::ILexer::TPtr& lexer, + TVector<TString>& statements, NYql::TIssues& issues); } diff --git a/yql/essentials/sql/v1/node.h b/yql/essentials/sql/v1/node.h index 609cd82dd48..e19245f695b 100644 --- a/yql/essentials/sql/v1/node.h +++ b/yql/essentials/sql/v1/node.h @@ -1548,6 +1548,14 @@ namespace NSQLTranslationV1 { std::map<TString, TNodePtr>&& settings, const TObjectOperatorContext& context); TNodePtr BuildDropAsyncReplication(TPosition pos, const TString& id, bool cascade, const TObjectOperatorContext& context); + TNodePtr BuildCreateTransfer(TPosition pos, const TString& id, const TString&& source, const TString&& target, + const TString&& transformLambda, + std::map<TString, TNodePtr>&& settings, + const TObjectOperatorContext& context); + TNodePtr BuildAlterTransfer(TPosition pos, const TString& id, std::optional<TString>&& transformLambda, + std::map<TString, TNodePtr>&& settings, + const TObjectOperatorContext& context); + TNodePtr BuildDropTransfer(TPosition pos, const TString& id, bool cascade, const TObjectOperatorContext& context); TNodePtr BuildWriteResult(TPosition pos, const TString& label, TNodePtr settings); TNodePtr BuildCommitClusters(TPosition pos); TNodePtr BuildRollbackClusters(TPosition pos); diff --git a/yql/essentials/sql/v1/query.cpp b/yql/essentials/sql/v1/query.cpp index c74f96a27ca..a1323972c44 100644 --- a/yql/essentials/sql/v1/query.cpp +++ b/yql/essentials/sql/v1/query.cpp @@ -2628,6 +2628,159 @@ TNodePtr BuildAlterAsyncReplication(TPosition pos, const TString& id, return new TAlterAsyncReplication(pos, id, std::move(settings), context); } +class TTransfer + : public TAstListNode + , protected TObjectOperatorContext +{ +protected: + virtual INode::TPtr FillOptions(INode::TPtr options) const = 0; + +public: + explicit TTransfer(TPosition pos, const TString& id, const TString& mode, const TObjectOperatorContext& context) + : TAstListNode(pos) + , TObjectOperatorContext(context) + , Id(id) + , Mode(mode) + { + } + + bool DoInit(TContext& ctx, ISource* src) override { + Scoped->UseCluster(ServiceId, Cluster); + + auto keys = Y("Key", Q(Y(Q("transfer"), Y("String", BuildQuotedAtom(Pos, Id))))); + auto options = FillOptions(Y(Q(Y(Q("mode"), Q(Mode))))); + + Add("block", Q(Y( + Y("let", "sink", Y("DataSink", BuildQuotedAtom(Pos, ServiceId), Scoped->WrapCluster(Cluster, ctx))), + Y("let", "world", Y(TString(WriteName), "world", "sink", keys, Y("Void"), Q(options))), + Y("return", ctx.PragmaAutoCommit ? Y(TString(CommitName), "world", "sink") : AstNode("world")) + ))); + + return TAstListNode::DoInit(ctx, src); + } + + TPtr DoClone() const final { + return {}; + } + +private: + const TString Id; + const TString Mode; + +}; // TTransfer + +class TCreateTransfer final: public TTransfer { +public: + explicit TCreateTransfer(TPosition pos, const TString& id, const TString&& source, const TString&& target, + const TString&& transformLambda, + std::map<TString, TNodePtr>&& settings, + const TObjectOperatorContext& context) + : TTransfer(pos, id, "create", context) + , Source(std::move(source)) + , Target(std::move(target)) + , TransformLambda(std::move(transformLambda)) + , Settings(std::move(settings)) + { + } + +protected: + INode::TPtr FillOptions(INode::TPtr options) const override { + options = L(options, Q(Y(Q("source"), Q(Source)))); + options = L(options, Q(Y(Q("target"), Q(Target)))); + options = L(options, Q(Y(Q("transformLambda"), Q(TransformLambda)))); + + if (!Settings.empty()) { + auto settings = Y(); + for (auto&& [k, v] : Settings) { + if (v) { + settings = L(settings, Q(Y(BuildQuotedAtom(Pos, k), v))); + } else { + settings = L(settings, Q(Y(BuildQuotedAtom(Pos, k)))); + } + } + options = L(options, Q(Y(Q("settings"), Q(settings)))); + } + + return options; + } + +private: + const TString Source; + const TString Target; + const TString TransformLambda; + std::map<TString, TNodePtr> Settings; + +}; // TCreateTransfer + +TNodePtr BuildCreateTransfer(TPosition pos, const TString& id, const TString&& source, const TString&& target, + const TString&& transformLambda, + std::map<TString, TNodePtr>&& settings, + const TObjectOperatorContext& context) +{ + return new TCreateTransfer(pos, id, std::move(source), std::move(target), std::move(transformLambda), std::move(settings), context); +} + +class TDropTransfer final: public TTransfer { +public: + explicit TDropTransfer(TPosition pos, const TString& id, bool cascade, const TObjectOperatorContext& context) + : TTransfer(pos, id, cascade ? "dropCascade" : "drop", context) + { + } + +protected: + INode::TPtr FillOptions(INode::TPtr options) const override { + return options; + } + +}; // TDropTransfer + +TNodePtr BuildDropTransfer(TPosition pos, const TString& id, bool cascade, const TObjectOperatorContext& context) { + return new TDropTransfer(pos, id, cascade, context); +} + +class TAlterTransfer final: public TTransfer { +public: + explicit TAlterTransfer(TPosition pos, const TString& id, std::optional<TString>&& transformLambda, + std::map<TString, TNodePtr>&& settings, + const TObjectOperatorContext& context) + : TTransfer(pos, id, "alter", context) + , TransformLambda(std::move(transformLambda)) + , Settings(std::move(settings)) + { + } + +protected: + INode::TPtr FillOptions(INode::TPtr options) const override { + options = L(options, Q(Y(Q("transformLambda"), Q(TransformLambda ? TransformLambda.value() : "")))); + + if (!Settings.empty()) { + auto settings = Y(); + for (auto&& [k, v] : Settings) { + if (v) { + settings = L(settings, Q(Y(BuildQuotedAtom(Pos, k), v))); + } else { + settings = L(settings, Q(Y(BuildQuotedAtom(Pos, k)))); + } + } + options = L(options, Q(Y(Q("settings"), Q(settings)))); + } + + return options; + } + +private: + const std::optional<TString> TransformLambda; + std::map<TString, TNodePtr> Settings; + +}; // TAlterTransfer + +TNodePtr BuildAlterTransfer(TPosition pos, const TString& id, std::optional<TString>&& transformLambda, + std::map<TString, TNodePtr>&& settings, + const TObjectOperatorContext& context) +{ + return new TAlterTransfer(pos, id, std::move(transformLambda), std::move(settings), context); +} + static const TMap<EWriteColumnMode, TString> columnModeToStrMapMR { {EWriteColumnMode::Default, ""}, {EWriteColumnMode::Insert, "append"}, diff --git a/yql/essentials/sql/v1/sql.cpp b/yql/essentials/sql/v1/sql.cpp index 3e5dba78f3c..503f3e45da9 100644 --- a/yql/essentials/sql/v1/sql.cpp +++ b/yql/essentials/sql/v1/sql.cpp @@ -75,13 +75,21 @@ void SqlASTsToYqlsImpl(NYql::TAstParseResult& res, const std::vector<::NSQLv1Gen } } -NYql::TAstParseResult SqlASTToYql(const google::protobuf::Message& protoAst, +NYql::TAstParseResult SqlASTToYql( + const google::protobuf::Message& protoAst, + const NSQLTranslation::TSQLHints& hints, + const NSQLTranslation::TTranslationSettings& settings) { + return SqlASTToYql("", protoAst, hints, settings); +} + +NYql::TAstParseResult SqlASTToYql(const TString& query, + const google::protobuf::Message& protoAst, const NSQLTranslation::TSQLHints& hints, const NSQLTranslation::TTranslationSettings& settings) { YQL_ENSURE(IsQueryMode(settings.Mode)); TAstParseResult res; - TContext ctx(settings, hints, res.Issues); + TContext ctx(settings, hints, res.Issues, query); SqlASTToYqlImpl(res, protoAst, ctx); res.ActualSyntaxType = NYql::ESyntaxType::YQLv1; return res; @@ -99,7 +107,7 @@ NYql::TAstParseResult SqlToYql(const TString& query, const NSQLTranslation::TTra return res; } - TContext ctx(settings, hints, res.Issues); + TContext ctx(settings, hints, res.Issues, query); NSQLTranslation::TErrorCollectorOverIssues collector(res.Issues, settings.MaxErrors, settings.File); google::protobuf::Message* ast(SqlAST(query, queryName, collector, settings.AnsiLexer, settings.Antlr4Parser, settings.TestAntlr4, settings.Arena)); @@ -177,11 +185,14 @@ bool NeedUseForAllStatements(const TRule_sql_stmt_core::AltCase& subquery) { case TRule_sql_stmt_core::kAltSqlStmtCore55: // backup case TRule_sql_stmt_core::kAltSqlStmtCore56: // restore case TRule_sql_stmt_core::kAltSqlStmtCore57: // alter sequence + case TRule_sql_stmt_core::kAltSqlStmtCore58: // create transfer + case TRule_sql_stmt_core::kAltSqlStmtCore59: // alter transfer + case TRule_sql_stmt_core::kAltSqlStmtCore60: // drop transfer return false; } } -TVector<NYql::TAstParseResult> SqlToAstStatements(const TString& query, const NSQLTranslation::TTranslationSettings& settings, NYql::TWarningRules* warningRules, +TVector<NYql::TAstParseResult> SqlToAstStatements(const TString& queryText, const NSQLTranslation::TTranslationSettings& settings, NYql::TWarningRules* warningRules, TVector<NYql::TStmtParseInfo>* stmtParseInfo) { TVector<TAstParseResult> result; @@ -191,14 +202,14 @@ TVector<NYql::TAstParseResult> SqlToAstStatements(const TString& query, const NS NSQLTranslation::TSQLHints hints; auto lexer = MakeLexer(settings.AnsiLexer, settings.Antlr4Parser); YQL_ENSURE(lexer); - if (!CollectSqlHints(*lexer, query, queryName, settings.File, hints, issues, settings.MaxErrors, settings.Antlr4Parser)) { + if (!CollectSqlHints(*lexer, queryText, queryName, settings.File, hints, issues, settings.MaxErrors, settings.Antlr4Parser)) { return result; } TContext ctx(settings, hints, issues); NSQLTranslation::TErrorCollectorOverIssues collector(issues, settings.MaxErrors, settings.File); - google::protobuf::Message* astProto(SqlAST(query, queryName, collector, settings.AnsiLexer, settings.Antlr4Parser, settings.TestAntlr4, settings.Arena)); + google::protobuf::Message* astProto(SqlAST(queryText, queryName, collector, settings.AnsiLexer, settings.Antlr4Parser, settings.TestAntlr4, settings.Arena)); if (astProto) { auto ast = static_cast<const TSQLv1ParserAST&>(*astProto); const auto& query = ast.GetRule_sql_query(); @@ -209,7 +220,7 @@ TVector<NYql::TAstParseResult> SqlToAstStatements(const TString& query, const NS if (NeedUseForAllStatements(statements.GetRule_sql_stmt2().GetRule_sql_stmt_core2().Alt_case())) { commonStates.push_back(statements.GetRule_sql_stmt2().GetRule_sql_stmt_core2()); } else { - TContext ctx(settings, hints, issues); + TContext ctx(settings, hints, issues, queryText); result.emplace_back(); if (stmtParseInfo) { stmtParseInfo->push_back({}); @@ -223,7 +234,7 @@ TVector<NYql::TAstParseResult> SqlToAstStatements(const TString& query, const NS commonStates.push_back(block.GetRule_sql_stmt2().GetRule_sql_stmt_core2()); continue; } - TContext ctx(settings, hints, issues); + TContext ctx(settings, hints, issues, queryText); result.emplace_back(); if (stmtParseInfo) { stmtParseInfo->push_back({}); @@ -245,4 +256,28 @@ TVector<NYql::TAstParseResult> SqlToAstStatements(const TString& query, const NS return result; } +bool SplitQueryToStatements(const TString& query, TVector<TString>& statements, NYql::TIssues& issues, + const NSQLTranslation::TTranslationSettings& settings) { + auto lexer = NSQLTranslationV1::MakeLexer(settings.AnsiLexer, settings.Antlr4Parser); + + TVector<TString> parts; + if (!SplitQueryToStatements(query, lexer, parts, issues)) { + return false; + } + + for (auto& currentQuery : parts) { + NYql::TIssues parserIssues; + auto message = NSQLTranslationV1::SqlAST(currentQuery, "Query", parserIssues, NSQLTranslation::SQL_MAX_PARSER_ERRORS, + settings.AnsiLexer, settings.Antlr4Parser, settings.TestAntlr4, settings.Arena); + if (!message) { + // Skip empty statements + continue; + } + + statements.push_back(std::move(currentQuery)); + } + + return true; +} + } // namespace NSQLTranslationV1 diff --git a/yql/essentials/sql/v1/sql.h b/yql/essentials/sql/v1/sql.h index 0a12c45d308..6e59aaf6914 100644 --- a/yql/essentials/sql/v1/sql.h +++ b/yql/essentials/sql/v1/sql.h @@ -3,6 +3,7 @@ #include <yql/essentials/ast/yql_ast.h> #include <yql/essentials/parser/lexer_common/hints.h> #include <yql/essentials/parser/proto_ast/common.h> +#include <yql/essentials/parser/proto_ast/gen/v1_proto_split/SQLv1Parser.pb.main.h> #include <yql/essentials/public/issue/yql_warning.h> #include <yql/essentials/public/issue/yql_issue_manager.h> #include <yql/essentials/sql/settings/translation_settings.h> @@ -16,7 +17,16 @@ namespace NSQLTranslation { namespace NSQLTranslationV1 { NYql::TAstParseResult SqlToYql(const TString& query, const NSQLTranslation::TTranslationSettings& settings, NYql::TWarningRules* warningRules = nullptr); + + /*[[deprecated]] Use SqlASTToYql(query, protoAst, hints, settings)*/ NYql::TAstParseResult SqlASTToYql(const google::protobuf::Message& protoAst, const NSQLTranslation::TSQLHints& hints, const NSQLTranslation::TTranslationSettings& settings); + NYql::TAstParseResult SqlASTToYql(const TString& query, const google::protobuf::Message& protoAst, const NSQLTranslation::TSQLHints& hints, const NSQLTranslation::TTranslationSettings& settings); + TVector<NYql::TAstParseResult> SqlToAstStatements(const TString& query, const NSQLTranslation::TTranslationSettings& settings, NYql::TWarningRules* warningRules, TVector<NYql::TStmtParseInfo>* stmtParseInfo = nullptr); + bool NeedUseForAllStatements(const NSQLv1Generated::TRule_sql_stmt_core::AltCase& subquery); + + bool SplitQueryToStatements(const TString& query, TVector<TString>& statements, NYql::TIssues& issues, + const NSQLTranslation::TTranslationSettings& settings); + } // namespace NSQLTranslationV1 diff --git a/yql/essentials/sql/v1/sql_expression.cpp b/yql/essentials/sql/v1/sql_expression.cpp index a7a9877ccbd..199f15f2939 100644 --- a/yql/essentials/sql/v1/sql_expression.cpp +++ b/yql/essentials/sql/v1/sql_expression.cpp @@ -34,6 +34,31 @@ TNodePtr TSqlExpression::Build(const TRule_expr& node) { } } +TNodePtr TSqlExpression::Build(const TRule_lambda_or_parameter& node) { + // lambda_or_parameter: + // lambda + // | bind_parameter + switch (node.Alt_case()) { + case TRule_lambda_or_parameter::kAltLambdaOrParameter1: { + return LambdaRule(node.alt_lambda_or_parameter1().GetRule_lambda1()); + } + case TRule_lambda_or_parameter::kAltLambdaOrParameter2: { + TString named; + if (!NamedNodeImpl(node.GetAlt_lambda_or_parameter2().GetRule_bind_parameter1(), named, *this)) { + return nullptr; + } + auto namedNode = GetNamedNode(named); + if (!namedNode) { + return nullptr; + } + + return namedNode; + } + case TRule_lambda_or_parameter::ALT_NOT_SET: + Y_ABORT("You should change implementation according to grammar changes"); + } + } + TNodePtr TSqlExpression::SubExpr(const TRule_mul_subexpr& node, const TTrailingQuestions& tail) { // mul_subexpr: con_subexpr (DOUBLE_PIPE con_subexpr)*; auto getNode = [](const TRule_mul_subexpr::TBlock2& b) -> const TRule_con_subexpr& { return b.GetRule_con_subexpr2(); }; diff --git a/yql/essentials/sql/v1/sql_expression.h b/yql/essentials/sql/v1/sql_expression.h index 64b9dd8a690..4f9100722ca 100644 --- a/yql/essentials/sql/v1/sql_expression.h +++ b/yql/essentials/sql/v1/sql_expression.h @@ -22,6 +22,7 @@ public: } TNodePtr Build(const TRule_expr& node); + TNodePtr Build(const TRule_lambda_or_parameter& node); void SetSmartParenthesisMode(ESmartParenthesis mode) { SmartParenthesisMode = mode; diff --git a/yql/essentials/sql/v1/sql_query.cpp b/yql/essentials/sql/v1/sql_query.cpp index eb1a174b30c..a04ce659861 100644 --- a/yql/essentials/sql/v1/sql_query.cpp +++ b/yql/essentials/sql/v1/sql_query.cpp @@ -143,7 +143,68 @@ static bool AsyncReplicationAlterAction(std::map<TString, TNodePtr>& settings, return AsyncReplicationSettings(settings, in.GetRule_alter_replication_set_setting1().GetRule_replication_settings3(), ctx, false); } -bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core& core) { +static bool TransferSettingsEntry(std::map<TString, TNodePtr>& out, + const TRule_transfer_settings_entry& in, TSqlExpression& ctx, bool create) +{ + auto key = IdEx(in.GetRule_an_id1(), ctx); + auto value = ctx.Build(in.GetRule_expr3()); + + if (!value) { + ctx.Context().Error() << "Invalid transfer setting: " << key.Name; + return false; + } + + TSet<TString> configSettings = { + "connection_string", + "endpoint", + "database", + "token", + "token_secret_name", + "user", + "password", + "password_secret_name", + }; + + TSet<TString> stateSettings = { + "state", + "failover_mode", + }; + + const auto keyName = to_lower(key.Name); + if (!configSettings.count(keyName) && !stateSettings.contains(keyName)) { + ctx.Context().Error() << "Unknown transfer setting: " << key.Name; + return false; + } + + if (create && stateSettings.count(keyName)) { + ctx.Context().Error() << key.Name << " is not supported in CREATE"; + return false; + } + + if (!out.emplace(keyName, value).second) { + ctx.Context().Error() << "Duplicate transfer setting: " << key.Name; + } + + return true; +} + +static bool TransferSettings(std::map<TString, TNodePtr>& out, + const TRule_transfer_settings& in, TSqlExpression& ctx, bool create) +{ + if (!TransferSettingsEntry(out, in.GetRule_transfer_settings_entry1(), ctx, create)) { + return false; + } + + for (auto& block : in.GetBlock2()) { + if (!TransferSettingsEntry(out, block.GetRule_transfer_settings_entry2(), ctx, create)) { + return false; + } + } + + return true; +} + +bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core& core, size_t statementNumber) { TString internalStatementName; TString humanStatementName; ParseStatementName(core, internalStatementName, humanStatementName); @@ -161,6 +222,10 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core& return false; } + if (NeedUseForAllStatements(altCase)) { + Ctx.ForAllStatementsParts.push_back(statementNumber); + } + switch (altCase) { case TRule_sql_stmt_core::kAltSqlStmtCore1: { bool success = false; @@ -1763,6 +1828,103 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core& AddStatementToBlocks(blocks, BuildAlterSequence(pos, service, cluster, id, params, Ctx.Scoped)); break; } + case TRule_sql_stmt_core::kAltSqlStmtCore58: { + // create_transfer_stmt: CREATE TRANSFER + + auto& node = core.GetAlt_sql_stmt_core58().GetRule_create_transfer_stmt1(); + TObjectOperatorContext context(Ctx.Scoped); + if (node.GetRule_object_ref3().HasBlock1()) { + const auto& cluster = node.GetRule_object_ref3().GetBlock1().GetRule_cluster_expr1(); + if (!ClusterExpr(cluster, false, context.ServiceId, context.Cluster)) { + return false; + } + } + + auto prefixPath = Ctx.GetPrefixPath(context.ServiceId, context.Cluster); + + std::map<TString, TNodePtr> settings; + TSqlExpression expr(Ctx, Mode); + if (!TransferSettings(settings, node.GetRule_transfer_settings11(), expr, true)) { + return false; + } + + const TString id = Id(node.GetRule_object_ref3().GetRule_id_or_at2(), *this).second; + const TString source = Id(node.GetRule_object_ref5().GetRule_id_or_at2(), *this).second; + const TString target = Id(node.GetRule_object_ref7().GetRule_id_or_at2(), *this).second; + TString transformLambda; + if (node.GetBlock8().HasRule_lambda_or_parameter2()) { + if (!ParseTransferLambda(transformLambda, node.GetBlock8().GetRule_lambda_or_parameter2())) { + return false; + } + } + + AddStatementToBlocks(blocks, BuildCreateTransfer(Ctx.Pos(), BuildTablePath(prefixPath, id), + std::move(source), std::move(target), std::move(transformLambda), std::move(settings), context)); + break; + } + case TRule_sql_stmt_core::kAltSqlStmtCore59: { + // alter_transfer_stmt: ALTER TRANSFER + auto& node = core.GetAlt_sql_stmt_core59().GetRule_alter_transfer_stmt1(); + TObjectOperatorContext context(Ctx.Scoped); + if (node.GetRule_object_ref3().HasBlock1()) { + const auto& cluster = node.GetRule_object_ref3().GetBlock1().GetRule_cluster_expr1(); + if (!ClusterExpr(cluster, false, context.ServiceId, context.Cluster)) { + return false; + } + } + + std::map<TString, TNodePtr> settings; + std::optional<TString> transformLambda; + TSqlExpression expr(Ctx, Mode); + + auto transferAlterAction = [&](std::optional<TString>& transformLambda, const TRule_alter_transfer_action& in) + { + if (in.HasAlt_alter_transfer_action1()) { + return TransferSettings(settings, in.GetAlt_alter_transfer_action1().GetRule_alter_transfer_set_setting1().GetRule_transfer_settings3(), expr, false); + } else if (in.HasAlt_alter_transfer_action2()) { + TString lb; + if (!ParseTransferLambda(lb, in.GetAlt_alter_transfer_action2().GetRule_alter_transfer_set_using1().GetRule_lambda_or_parameter3())) { + return false; + } + transformLambda = lb; + return true; + } + + return false; + }; + + if (!transferAlterAction(transformLambda, node.GetRule_alter_transfer_action4())) { + return false; + } + for (auto& block : node.GetBlock5()) { + if (!transferAlterAction(transformLambda, block.GetRule_alter_transfer_action2())) { + return false; + } + } + + const TString id = Id(node.GetRule_object_ref3().GetRule_id_or_at2(), *this).second; + AddStatementToBlocks(blocks, BuildAlterTransfer(Ctx.Pos(), + BuildTablePath(Ctx.GetPrefixPath(context.ServiceId, context.Cluster), id), + std::move(transformLambda), std::move(settings), context)); + break; + } + case TRule_sql_stmt_core::kAltSqlStmtCore60: { + // drop_transfer_stmt: DROP TRANSFER + auto& node = core.GetAlt_sql_stmt_core60().GetRule_drop_transfer_stmt1(); + TObjectOperatorContext context(Ctx.Scoped); + if (node.GetRule_object_ref3().HasBlock1()) { + const auto& cluster = node.GetRule_object_ref3().GetBlock1().GetRule_cluster_expr1(); + if (!ClusterExpr(cluster, false, context.ServiceId, context.Cluster)) { + return false; + } + } + + const TString id = Id(node.GetRule_object_ref3().GetRule_id_or_at2(), *this).second; + AddStatementToBlocks(blocks, BuildDropTransfer(Ctx.Pos(), + BuildTablePath(Ctx.GetPrefixPath(context.ServiceId, context.Cluster), id), + node.HasBlock4(), context)); + break; + } case TRule_sql_stmt_core::ALT_NOT_SET: Ctx.IncrementMonCounter("sql_errors", "UnknownStatement" + internalStatementName); AltNotImplemented("sql_stmt_core", core); @@ -3471,12 +3633,13 @@ TNodePtr TSqlQuery::Build(const TSQLv1ParserAST& ast) { Ctx.PopCurrentBlocks(); }; if (query.Alt_case() == TRule_sql_query::kAltSqlQuery1) { + size_t statementNumber = 0; const auto& statements = query.GetAlt_sql_query1().GetRule_sql_stmt_list1(); - if (!Statement(blocks, statements.GetRule_sql_stmt2().GetRule_sql_stmt_core2())) { + if (!Statement(blocks, statements.GetRule_sql_stmt2().GetRule_sql_stmt_core2(), statementNumber++)) { return nullptr; } for (auto block: statements.GetBlock3()) { - if (!Statement(blocks, block.GetRule_sql_stmt2().GetRule_sql_stmt_core2())) { + if (!Statement(blocks, block.GetRule_sql_stmt2().GetRule_sql_stmt_core2(), statementNumber++)) { return nullptr; } } @@ -3546,8 +3709,10 @@ TNodePtr TSqlQuery::Build(const std::vector<::NSQLv1Generated::TRule_sql_stmt_co Y_DEFER { Ctx.PopCurrentBlocks(); }; + + size_t statementNumber = 0; for (const auto& statement : statements) { - if (!Statement(blocks, statement)) { + if (!Statement(blocks, statement, statementNumber++)) { return nullptr; } } diff --git a/yql/essentials/sql/v1/sql_query.h b/yql/essentials/sql/v1/sql_query.h index 03fd85df6b3..ea5b917f8f2 100644 --- a/yql/essentials/sql/v1/sql_query.h +++ b/yql/essentials/sql/v1/sql_query.h @@ -20,7 +20,7 @@ public: TNodePtr Build(const TSQLv1ParserAST& ast); TNodePtr Build(const std::vector<::NSQLv1Generated::TRule_sql_stmt_core>& ast); - bool Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core& core); + bool Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core& core, size_t statementNumber); private: bool DeclareStatement(const TRule_declare_stmt& stmt); bool ExportStatement(const TRule_export_stmt& stmt); diff --git a/yql/essentials/sql/v1/sql_translation.cpp b/yql/essentials/sql/v1/sql_translation.cpp index 9989295291a..a4c55cad71d 100644 --- a/yql/essentials/sql/v1/sql_translation.cpp +++ b/yql/essentials/sql/v1/sql_translation.cpp @@ -83,7 +83,7 @@ TNodePtr BuildViewSelect( const TString& contextRecreationQuery ) { TIssues issues; - TContext context(parentContext.Settings, {}, issues); + TContext context(parentContext.Settings, {}, issues, parentContext.Query); if (!RecreateContext(context, context.Settings, contextRecreationQuery)) { parentContext.Issues.AddIssues(issues); return nullptr; @@ -4651,13 +4651,15 @@ bool TSqlTranslation::DefineActionOrSubqueryBody(TSqlQuery& query, TBlocks& bloc Y_DEFER { Ctx.PopCurrentBlocks(); }; - if (!query.Statement(blocks, body.GetBlock2().GetRule_sql_stmt_core1())) { + + size_t statementNumber = 0; + if (!query.Statement(blocks, body.GetBlock2().GetRule_sql_stmt_core1(), statementNumber++)) { return false; } for (const auto& nestedStmtItem : body.GetBlock2().GetBlock2()) { const auto& nestedStmt = nestedStmtItem.GetRule_sql_stmt_core2(); - if (!query.Statement(blocks, nestedStmt)) { + if (!query.Statement(blocks, nestedStmt, statementNumber++)) { return false; } } @@ -5096,6 +5098,101 @@ bool TSqlTranslation::ParseViewQuery( return true; } +namespace { + +static std::string::size_type GetQueryPosition(const TString& query, const NSQLv1Generated::TToken& token, bool antlr4) { + if (1 == token.GetLine() && 0 == token.GetColumn()) { + return 0; + } + + TPosition pos = {0, 1}; + TTextWalker walker(pos, antlr4); + + std::string::size_type position = 0; + for (char c : query) { + walker.Advance(c); + ++position; + + if (pos.Row == token.GetLine() && pos.Column == token.GetColumn()) { + return position; + } + } + + return std::string::npos; +} + +static TString GetLambdaText(TTranslation& ctx, TContext& Ctx, const TRule_lambda_or_parameter& lambdaOrParameter) { + static const TString statementSeparator = ";\n"; + + TVector<TString> statements; + NYql::TIssues issues; + if (!SplitQueryToStatements(Ctx.Query, statements, issues, Ctx.Settings)) { + return {}; + } + + TStringBuilder result; + for (const auto id : Ctx.ForAllStatementsParts) { + result << statements[id] << "\n"; + } + + switch (lambdaOrParameter.Alt_case()) { + case NSQLv1Generated::TRule_lambda_or_parameter::kAltLambdaOrParameter1: { + const auto& lambda = lambdaOrParameter.GetAlt_lambda_or_parameter1().GetRule_lambda1(); + + auto& beginToken = lambda.GetRule_smart_parenthesis1().GetToken1(); + const NSQLv1Generated::TToken* endToken = nullptr; + switch (lambda.GetBlock2().GetBlock2().GetAltCase()) { + case TRule_lambda_TBlock2_TBlock2::AltCase::kAlt1: + endToken = &lambda.GetBlock2().GetBlock2().GetAlt1().GetToken3(); + break; + case TRule_lambda_TBlock2_TBlock2::AltCase::kAlt2: + endToken = &lambda.GetBlock2().GetBlock2().GetAlt2().GetToken3(); + break; + case TRule_lambda_TBlock2_TBlock2::AltCase::ALT_NOT_SET: + Y_ABORT("You should change implementation according to grammar changes"); + } + + auto begin = GetQueryPosition(Ctx.Query, beginToken, Ctx.Settings.Antlr4Parser); + auto end = GetQueryPosition(Ctx.Query, *endToken, Ctx.Settings.Antlr4Parser); + if (begin == std::string::npos || end == std::string::npos) { + return {}; + } + + result << "$__ydb_transfer_lambda = " << Ctx.Query.substr(begin, end - begin + endToken->value().size()) << statementSeparator; + + return result; + } + case NSQLv1Generated::TRule_lambda_or_parameter::kAltLambdaOrParameter2: { + const auto& valueBlock = lambdaOrParameter.GetAlt_lambda_or_parameter2().GetRule_bind_parameter1().GetBlock2(); + const auto id = Id(valueBlock.GetAlt1().GetRule_an_id_or_type1(), ctx); + result << "$__ydb_transfer_lambda = $" << id << statementSeparator; + return result; + } + case NSQLv1Generated::TRule_lambda_or_parameter::ALT_NOT_SET: + Y_ABORT("You should change implementation according to grammar changes"); + } +} + +} + +bool TSqlTranslation::ParseTransferLambda( + TString& lambdaText, + const TRule_lambda_or_parameter& lambdaOrParameter) { + + TSqlExpression expr(Ctx, Ctx.Settings.Mode); + auto result = expr.Build(lambdaOrParameter); + if (!result) { + return false; + } + + lambdaText = GetLambdaText(*this, Ctx, lambdaOrParameter); + if (lambdaText.empty()) { + Ctx.Error() << "Cannot parse lambda correctly"; + } + + return !lambdaText.empty(); +} + class TReturningListColumns : public INode { public: TReturningListColumns(TPosition pos) diff --git a/yql/essentials/sql/v1/sql_translation.h b/yql/essentials/sql/v1/sql_translation.h index 75e9c2f7d48..74b78ae3306 100644 --- a/yql/essentials/sql/v1/sql_translation.h +++ b/yql/essentials/sql/v1/sql_translation.h @@ -263,6 +263,7 @@ protected: TVector<TDeferredAtom>& addTables, TVector<TDeferredAtom>& removeTables, const TRule_alter_backup_collection_entries& entries); + bool ParseTransferLambda(TString& lambdaText, const TRule_lambda_or_parameter& lambdaOrParameter); bool ValidateAuthMethod(const std::map<TString, TDeferredAtom>& result); bool ValidateExternalTable(const TCreateTableParameters& params); diff --git a/yql/essentials/sql/v1/sql_ut.cpp b/yql/essentials/sql/v1/sql_ut.cpp index e7b30928c32..9baf35f8f4e 100644 --- a/yql/essentials/sql/v1/sql_ut.cpp +++ b/yql/essentials/sql/v1/sql_ut.cpp @@ -4,6 +4,7 @@ #include <yql/essentials/providers/common/provider/yql_provider_names.h> #include <yql/essentials/sql/sql.h> +#include <yql/essentials/sql/v1/sql.h> #include <util/generic/map.h> #include <library/cpp/testing/unittest/registar.h> @@ -287,6 +288,11 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) { UNIT_ASSERT(SqlToYql("USE plato; SELECT REPLICATION FROM REPLICATION").IsOk()); } + Y_UNIT_TEST(TransferKeywordNotReservedForNames) { + UNIT_ASSERT(SqlToYql("USE plato; CREATE TABLE TRANSFER (TRANSFER Uint32, PRIMARY KEY (TRANSFER));").IsOk()); + UNIT_ASSERT(SqlToYql("USE plato; SELECT TRANSFER FROM TRANSFER").IsOk()); + } + Y_UNIT_TEST(SecondsKeywordNotReservedForNames) { UNIT_ASSERT(SqlToYql("USE plato; CREATE TABLE SECONDS (SECONDS Uint32, PRIMARY KEY (SECONDS));").IsOk()); UNIT_ASSERT(SqlToYql("USE plato; SELECT SECONDS FROM SECONDS").IsOk()); @@ -8056,3 +8062,60 @@ Y_UNIT_TEST_SUITE(ColumnFamily) { UNIT_ASSERT_STRING_CONTAINS(res.Issues.ToString(), "COMPRESSION_LEVEL value should be an integer"); } } + +Y_UNIT_TEST_SUITE(QuerySplit) { + Y_UNIT_TEST(Simple) { + TString query = R"( + ; + -- Comment 1 + SELECT * From Input; -- Comment 2 + -- Comment 3 + $a = "a"; + + -- Comment 9 + ; + + -- Comment 10 + + -- Comment 8 + + $b = ($x) -> { + -- comment 4 + return /* Comment 5 */ $x; + -- Comment 6 + }; + + // Comment 7 + + + + )"; + + google::protobuf::Arena Arena; + + NSQLTranslation::TTranslationSettings settings; + settings.AnsiLexer = false; + settings.Antlr4Parser = true; + settings.Arena = &Arena; + + TVector<TString> statements; + NYql::TIssues issues; + + UNIT_ASSERT(NSQLTranslationV1::SplitQueryToStatements(query, statements, issues, settings)); + + UNIT_ASSERT_VALUES_EQUAL(statements.size(), 3); + + UNIT_ASSERT_VALUES_EQUAL(statements[0], "-- Comment 1\n SELECT * From Input; -- Comment 2\n"); + UNIT_ASSERT_VALUES_EQUAL(statements[1], R"(-- Comment 3 + $a = "a";)"); + UNIT_ASSERT_VALUES_EQUAL(statements[2], R"(-- Comment 10 + + -- Comment 8 + + $b = ($x) -> { + -- comment 4 + return /* Comment 5 */ $x; + -- Comment 6 + };)"); + } +} diff --git a/yql/essentials/sql/v1/sql_ut_antlr4.cpp b/yql/essentials/sql/v1/sql_ut_antlr4.cpp index 5337f573510..05e85605ef4 100644 --- a/yql/essentials/sql/v1/sql_ut_antlr4.cpp +++ b/yql/essentials/sql/v1/sql_ut_antlr4.cpp @@ -287,6 +287,11 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) { UNIT_ASSERT(SqlToYql("USE plato; SELECT REPLICATION FROM REPLICATION").IsOk()); } + Y_UNIT_TEST(TransferKeywordNotReservedForNames) { + UNIT_ASSERT(SqlToYql("USE plato; CREATE TABLE TRANSFER (TRANSFER Uint32, PRIMARY KEY (TRANSFER));").IsOk()); + UNIT_ASSERT(SqlToYql("USE plato; SELECT TRANSFER FROM TRANSFER").IsOk()); + } + Y_UNIT_TEST(SecondsKeywordNotReservedForNames) { UNIT_ASSERT(SqlToYql("USE plato; CREATE TABLE SECONDS (SECONDS Uint32, PRIMARY KEY (SECONDS));").IsOk()); UNIT_ASSERT(SqlToYql("USE plato; SELECT SECONDS FROM SECONDS").IsOk()); @@ -8028,3 +8033,48 @@ Y_UNIT_TEST_SUITE(ColumnFamily) { UNIT_ASSERT_STRING_CONTAINS(res.Issues.ToString(), "COMPRESSION_LEVEL value should be an integer"); } } + +Y_UNIT_TEST_SUITE(Transfer) { + Y_UNIT_TEST(Lambda) { + NYql::TAstParseResult res = SqlToYql(R"( use plato; + -- Русский коммент, empty statement + ; + + -- befor comment + $a = "А"; + + SELECT * FROM Input; + + $b = ($x) -> { return $a || $x; }; + + CREATE TRANSFER `TransferName` + FROM `TopicName` TO `TableName` + USING ($x) -> { + -- internal comment + return $b($x); + } + WITH ( + CONNECTION_STRING = "grpc://localhost:2135/?database=/Root" + ); + )"); + + UNIT_ASSERT_C(res.IsOk(), res.Issues.ToString()); + UNIT_ASSERT_VALUES_EQUAL_C(res.Issues.Size(), 0, res.Issues.ToString()); + + const auto programm = GetPrettyPrint(res); + + Cerr << ">>>>> Root " << programm << Endl; + auto expected = R"('transformLambda 'use plato; +-- befor comment + $a = "А"; +$b = ($x) -> { return $a || $x; }; +$__ydb_transfer_lambda = ($x) -> { + -- internal comment + return $b($x); + }; +))"; + + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, programm.find(expected)); + + } +} diff --git a/yql/essentials/tools/sql2yql/sql2yql.cpp b/yql/essentials/tools/sql2yql/sql2yql.cpp index dd313b451cb..ba30d95e4d2 100644 --- a/yql/essentials/tools/sql2yql/sql2yql.cpp +++ b/yql/essentials/tools/sql2yql/sql2yql.cpp @@ -317,7 +317,7 @@ int BuildAST(int argc, char* argv[]) { auto lexer = SqlLexer(query, parseRes.Issues, settings); if (lexer && CollectSqlHints(*lexer, query, queryFile, settings.File, hints, parseRes.Issues, settings.MaxErrors, settings.Antlr4Parser)) { - parseRes = NSQLTranslation::SqlASTToYql(*ast, hints, settings); + parseRes = NSQLTranslation::SqlASTToYql(query, *ast, hints, settings); } } } else { diff --git a/yql/essentials/utils/log/log.cpp b/yql/essentials/utils/log/log.cpp index 7f26d131787..9ecf09a827e 100644 --- a/yql/essentials/utils/log/log.cpp +++ b/yql/essentials/utils/log/log.cpp @@ -161,6 +161,7 @@ NYql::NLog::EComponent ConvertComponent(NYql::NProto::TLoggingConfig::EComponent case TLoggingConfig::PROVIDER_GENERIC: return EComponent::ProviderGeneric; case TLoggingConfig::PROVIDER_PG: return EComponent::ProviderPg; case TLoggingConfig::PROVIDER_PURE: return EComponent::ProviderPure; + case TLoggingConfig::FAST_MAP_REDUCE: return EComponent::FastMapReduce; } ythrow yexception() << "unknown log component: " diff --git a/yql/essentials/utils/log/log_component.h b/yql/essentials/utils/log/log_component.h index bed3fb06728..ad345a9949f 100644 --- a/yql/essentials/utils/log/log_component.h +++ b/yql/essentials/utils/log/log_component.h @@ -36,6 +36,7 @@ enum class EComponent { ProviderGeneric, ProviderPg, ProviderPure, + FastMapReduce, // <--- put other log components here MaxValue }; @@ -81,6 +82,7 @@ struct EComponentHelpers { case EComponent::ProviderGeneric: return TStringBuf("generic"); case EComponent::ProviderPg: return TStringBuf("PG"); case EComponent::ProviderPure: return TStringBuf("pure"); + case EComponent::FastMapReduce: return TStringBuf("fast map reduce"); default: ythrow yexception() << "invalid log component value: " << ToInt(component); @@ -115,6 +117,7 @@ struct EComponentHelpers { if (str == TStringBuf("generic")) return EComponent::ProviderGeneric; if (str == TStringBuf("PG")) return EComponent::ProviderPg; if (str == TStringBuf("pure")) return EComponent::ProviderPure; + if (str == TStringBuf("fast map reduce")) return EComponent::FastMapReduce; ythrow yexception() << "unknown log component: '" << str << '\''; } diff --git a/yql/essentials/utils/log/proto/logger_config.proto b/yql/essentials/utils/log/proto/logger_config.proto index f9433b67751..17dff082821 100644 --- a/yql/essentials/utils/log/proto/logger_config.proto +++ b/yql/essentials/utils/log/proto/logger_config.proto @@ -48,6 +48,7 @@ message TLoggingConfig { PROVIDER_GENERIC = 24; PROVIDER_PG = 25; PROVIDER_PURE = 26; + FAST_MAP_REDUCE = 27; } message TComponentLevel { diff --git a/yt/cpp/mapreduce/raw_client/raw_client.cpp b/yt/cpp/mapreduce/raw_client/raw_client.cpp index f25fe5a4ee7..3586751191e 100644 --- a/yt/cpp/mapreduce/raw_client/raw_client.cpp +++ b/yt/cpp/mapreduce/raw_client/raw_client.cpp @@ -49,7 +49,7 @@ TNode THttpRawClient::TryGet( if (!error.IsResolveError()) { throw; } - return TNode(); + return {}; } } diff --git a/yt/yql/providers/yt/provider/ya.make b/yt/yql/providers/yt/provider/ya.make index 719850bb8b0..247095b2ca2 100644 --- a/yt/yql/providers/yt/provider/ya.make +++ b/yt/yql/providers/yt/provider/ya.make @@ -16,6 +16,7 @@ SRCS( yql_yt_datasource_type_ann.cpp yql_yt_datasource.cpp yql_yt_epoch.cpp + yql_yt_forwarding_gateway.cpp yql_yt_gateway.cpp yql_yt_horizontal_join.cpp yql_yt_helpers.cpp diff --git a/yt/yql/providers/yt/provider/yql_yt_datasink.cpp b/yt/yql/providers/yt/provider/yql_yt_datasink.cpp index ca309ebf34b..4738a5f5a72 100644 --- a/yt/yql/providers/yt/provider/yql_yt_datasink.cpp +++ b/yt/yql/providers/yt/provider/yql_yt_datasink.cpp @@ -266,12 +266,16 @@ public: } if (auto columnGroup = NYql::GetSetting(*res->Child(TYtWriteTable::idx_Settings), EYtSettingType::ColumnGroups)) { const TString normalized = NormalizeColumnGroupSpec(columnGroup->Tail().Content()); - res = ctx.ChangeChild(*res, TYtWriteTable::idx_Settings, - NYql::UpdateSettingValue(*res->Child(TYtWriteTable::idx_Settings), - EYtSettingType::ColumnGroups, - ctx.NewAtom(res->Child(TYtWriteTable::idx_Settings)->Pos(), normalized, TNodeFlags::MultilineContent), - ctx) - ); + if (normalized) { + res = ctx.ChangeChild(*res, TYtWriteTable::idx_Settings, + NYql::UpdateSettingValue(*res->Child(TYtWriteTable::idx_Settings), + EYtSettingType::ColumnGroups, + ctx.NewAtom(res->Child(TYtWriteTable::idx_Settings)->Pos(), normalized, TNodeFlags::MultilineContent), + ctx) + ); + } else { + res = ctx.ChangeChild(*res, TYtWriteTable::idx_Settings, NYql::RemoveSetting(*res->Child(TYtWriteTable::idx_Settings), EYtSettingType::ColumnGroups, ctx)); + } } else if (NYql::HasSetting(*res->Child(TYtWriteTable::idx_Table)->Child(TYtTable::idx_Settings), EYtSettingType::Anonymous)) { if (const auto mode = State_->Configuration->ColumnGroupMode.Get().GetOrElse(EColumnGroupMode::Disable); mode != EColumnGroupMode::Disable) { res = ctx.ChangeChild(*res, TYtWriteTable::idx_Settings, diff --git a/yt/yql/providers/yt/provider/yql_yt_forwarding_gateway.cpp b/yt/yql/providers/yt/provider/yql_yt_forwarding_gateway.cpp new file mode 100644 index 00000000000..8749937a977 --- /dev/null +++ b/yt/yql/providers/yt/provider/yql_yt_forwarding_gateway.cpp @@ -0,0 +1,140 @@ +#include "yql_yt_forwarding_gateway.h" + +using namespace NThreading; + +namespace NYql { + +TYtForwardingGatewayBase::TYtForwardingGatewayBase(IYtGateway::TPtr&& slave) + : Slave_(std::move(slave)) +{ +} + +void TYtForwardingGatewayBase::OpenSession(TOpenSessionOptions&& options) { + Slave_->OpenSession(std::move(options)); +} + +TFuture<void> TYtForwardingGatewayBase::CloseSession(TCloseSessionOptions&& options) { + return Slave_->CloseSession(std::move(options)); +} + +TFuture<void> TYtForwardingGatewayBase::CleanupSession(TCleanupSessionOptions&& options) { + return Slave_->CleanupSession(std::move(options)); +} + +TFuture<IYtGateway::TFinalizeResult> TYtForwardingGatewayBase::Finalize(TFinalizeOptions&& options) { + return Slave_->Finalize(std::move(options)); +} + +TFuture<IYtGateway::TCanonizePathsResult> TYtForwardingGatewayBase::CanonizePaths(TCanonizePathsOptions&& options) { + return Slave_->CanonizePaths(std::move(options)); +} + +TFuture<IYtGateway::TTableInfoResult> TYtForwardingGatewayBase::GetTableInfo(TGetTableInfoOptions&& options) { + return Slave_->GetTableInfo(std::move(options)); +} + +TFuture<IYtGateway::TTableRangeResult> TYtForwardingGatewayBase::GetTableRange(TTableRangeOptions&& options) { + return Slave_->GetTableRange(std::move(options)); +} + +TFuture<IYtGateway::TFolderResult> TYtForwardingGatewayBase::GetFolder(TFolderOptions&& options) { + return Slave_->GetFolder(std::move(options)); +} + +TFuture<IYtGateway::TBatchFolderResult> TYtForwardingGatewayBase::ResolveLinks(TResolveOptions&& options) { + return Slave_->ResolveLinks(std::move(options)); +} + +TFuture<IYtGateway::TBatchFolderResult> TYtForwardingGatewayBase::GetFolders(TBatchFolderOptions&& options) { + return Slave_->GetFolders(std::move(options)); +} + +TFuture<IYtGateway::TResOrPullResult> TYtForwardingGatewayBase::ResOrPull(const TExprNode::TPtr& node, TExprContext& ctx, TResOrPullOptions&& options) { + return Slave_->ResOrPull(node, ctx, std::move(options)); +} + +TFuture<IYtGateway::TRunResult> TYtForwardingGatewayBase::Run(const TExprNode::TPtr& node, TExprContext& ctx, TRunOptions&& options) { + return Slave_->Run(node, ctx, std::move(options)); +} + +TFuture<IYtGateway::TRunResult> TYtForwardingGatewayBase::Prepare(const TExprNode::TPtr& node, TExprContext& ctx, TPrepareOptions&& options) const { + return Slave_->Prepare(node, ctx, std::move(options)); +} + +TFuture<IYtGateway::TCalcResult> TYtForwardingGatewayBase::Calc(const TExprNode::TListType& nodes, TExprContext& ctx, TCalcOptions&& options) { + return Slave_->Calc(nodes, ctx, std::move(options)); +} + +TFuture<IYtGateway::TPublishResult> TYtForwardingGatewayBase::Publish(const TExprNode::TPtr& node, TExprContext& ctx, TPublishOptions&& options) { + return Slave_->Publish(node, ctx, std::move(options)); +} + +TFuture<IYtGateway::TCommitResult> TYtForwardingGatewayBase::Commit(TCommitOptions&& options) { + return Slave_->Commit(std::move(options)); +} + +TFuture<IYtGateway::TDropTrackablesResult> TYtForwardingGatewayBase::DropTrackables(TDropTrackablesOptions&& options) { + return Slave_->DropTrackables(std::move(options)); +} + +TFuture<IYtGateway::TPathStatResult> TYtForwardingGatewayBase::PathStat(TPathStatOptions&& options) { + return Slave_->PathStat(std::move(options)); +} + +IYtGateway::TPathStatResult TYtForwardingGatewayBase::TryPathStat(TPathStatOptions&& options) { + return Slave_->TryPathStat(std::move(options)); +} + +bool TYtForwardingGatewayBase::TryParseYtUrl(const TString& url, TString* cluster, TString* path) const { + return Slave_->TryParseYtUrl(url, cluster, path); +} + +TString TYtForwardingGatewayBase::GetDefaultClusterName() const { + return Slave_->GetDefaultClusterName(); +} + +TString TYtForwardingGatewayBase::GetClusterServer(const TString& cluster) const { + return Slave_->GetClusterServer(cluster); +} + +NYT::TRichYPath TYtForwardingGatewayBase::GetRealTable(const TString& sessionId, const TString& cluster, const TString& table, ui32 epoch, const TString& tmpFolder) const { + return Slave_->GetRealTable(sessionId, cluster, table, epoch, tmpFolder); +} + +NYT::TRichYPath TYtForwardingGatewayBase::GetWriteTable(const TString& sessionId, const TString& cluster, const TString& table, const TString& tmpFolder) const { + return Slave_->GetWriteTable(sessionId, cluster, table, tmpFolder); +} + +TFuture<IYtGateway::TDownloadTablesResult> TYtForwardingGatewayBase::DownloadTables(TDownloadTablesOptions&& options) { + return Slave_->DownloadTables(std::move(options)); +} + +TFuture<IYtGateway::TUploadTableResult> TYtForwardingGatewayBase::UploadTable(TUploadTableOptions&& options) { + return Slave_->UploadTable(std::move(options)); +} + +NThreading::TFuture<IYtGateway::TRunResult> TYtForwardingGatewayBase::GetTableStat(const TExprNode::TPtr& node, TExprContext& ctx, TPrepareOptions&& options) { + return Slave_->GetTableStat(node, ctx, std::move(options)); +} + +IYtGateway::TFullResultTableResult TYtForwardingGatewayBase::PrepareFullResultTable(TFullResultTableOptions&& options) { + return Slave_->PrepareFullResultTable(std::move(options)); +} + +void TYtForwardingGatewayBase::SetStatUploader(IStatUploader::TPtr statUploader) { + Slave_->SetStatUploader(statUploader); +} + +void TYtForwardingGatewayBase::RegisterMkqlCompiler(NCommon::TMkqlCallableCompilerBase& compiler) { + Slave_->RegisterMkqlCompiler(compiler); +} + +IYtGateway::TGetTablePartitionsResult TYtForwardingGatewayBase::GetTablePartitions(TGetTablePartitionsOptions&& options) { + return Slave_->GetTablePartitions(std::move(options)); +} + +void TYtForwardingGatewayBase::AddCluster(const TYtClusterConfig& config) { + Slave_->AddCluster(config); +} + +} // namspace NYql diff --git a/yt/yql/providers/yt/provider/yql_yt_forwarding_gateway.h b/yt/yql/providers/yt/provider/yql_yt_forwarding_gateway.h new file mode 100644 index 00000000000..afc926fe2ee --- /dev/null +++ b/yt/yql/providers/yt/provider/yql_yt_forwarding_gateway.h @@ -0,0 +1,78 @@ +#include "yql_yt_provider.h" + +using namespace NThreading; + +namespace NYql { + +class TYtForwardingGatewayBase: public IYtGateway { +public: + TYtForwardingGatewayBase(IYtGateway::TPtr&& slave); + + void OpenSession(TOpenSessionOptions&& options) override; + + TFuture<void> CloseSession(TCloseSessionOptions&& options) override; + + TFuture<void> CleanupSession(TCleanupSessionOptions&& options) override; + + TFuture<TFinalizeResult> Finalize(TFinalizeOptions&& options) override; + + TFuture<TCanonizePathsResult> CanonizePaths(TCanonizePathsOptions&& options) override; + + TFuture<TTableInfoResult> GetTableInfo(TGetTableInfoOptions&& options) override; + TFuture<TTableRangeResult> GetTableRange(TTableRangeOptions&& options) override; + + TFuture<TFolderResult> GetFolder(TFolderOptions&& options) override; + + TFuture<TBatchFolderResult> ResolveLinks(TResolveOptions&& options) override; + + TFuture<TBatchFolderResult> GetFolders(TBatchFolderOptions&& options) override; + + TFuture<TResOrPullResult> ResOrPull(const TExprNode::TPtr& node, TExprContext& ctx, TResOrPullOptions&& options) override; + + TFuture<TRunResult> Run(const TExprNode::TPtr& node, TExprContext& ctx, TRunOptions&& options) override; + + TFuture<TRunResult> Prepare(const TExprNode::TPtr& node, TExprContext& ctx, TPrepareOptions&& options) const override; + + TFuture<TCalcResult> Calc(const TExprNode::TListType& nodes, TExprContext& ctx, TCalcOptions&& options) override; + + TFuture<TPublishResult> Publish(const TExprNode::TPtr& node, TExprContext& ctx, TPublishOptions&& options) override; + + TFuture<TCommitResult> Commit(TCommitOptions&& options) override; + + TFuture<TDropTrackablesResult> DropTrackables(TDropTrackablesOptions&& options) override; + + TFuture<TPathStatResult> PathStat(TPathStatOptions&& options) override; + + TPathStatResult TryPathStat(TPathStatOptions&& options) override; + + bool TryParseYtUrl(const TString& url, TString* cluster, TString* path) const override; + + TString GetDefaultClusterName() const override; + + TString GetClusterServer(const TString& cluster) const override; + + NYT::TRichYPath GetRealTable(const TString& sessionId, const TString& cluster, const TString& table, ui32 epoch, const TString& tmpFolder) const override; + + NYT::TRichYPath GetWriteTable(const TString& sessionId, const TString& cluster, const TString& table, const TString& tmpFolder) const override; + + TFuture<TDownloadTablesResult> DownloadTables(TDownloadTablesOptions&& options) override; + + TFuture<TUploadTableResult> UploadTable(TUploadTableOptions&& options) override; + + NThreading::TFuture<TRunResult> GetTableStat(const TExprNode::TPtr& node, TExprContext& ctx, TPrepareOptions&& options) override; + + TFullResultTableResult PrepareFullResultTable(TFullResultTableOptions&& options) override; + + void SetStatUploader(IStatUploader::TPtr statUploader) override; + + void RegisterMkqlCompiler(NCommon::TMkqlCallableCompilerBase& compiler) override; + + TGetTablePartitionsResult GetTablePartitions(TGetTablePartitionsOptions&& options) override; + + void AddCluster(const TYtClusterConfig& config) override; + +protected: + IYtGateway::TPtr Slave_; +}; + +} // namspace NYql diff --git a/yt/yql/providers/yt/provider/yql_yt_op_settings.cpp b/yt/yql/providers/yt/provider/yql_yt_op_settings.cpp index 2d1ebbc5025..5ef1da9eb01 100644 --- a/yt/yql/providers/yt/provider/yql_yt_op_settings.cpp +++ b/yt/yql/providers/yt/provider/yql_yt_op_settings.cpp @@ -1005,6 +1005,9 @@ bool ValidateColumnGroups(const TExprNode& setting, const TStructExprType& rowTy TString NormalizeColumnGroupSpec(const TStringBuf spec) { try { auto columnGroups = NYT::NodeFromYsonString(spec); + if (columnGroups.AsMap().empty()) { + return {}; + } for (auto& grp: columnGroups.AsMap()) { if (!grp.second.IsEntity()) { std::stable_sort(grp.second.AsList().begin(), grp.second.AsList().end(), [](const auto& l, const auto& r) { return l.AsString() < r.AsString(); }); diff --git a/yt/yql/tests/sql/suites/column_group/hint-disable.cfg b/yt/yql/tests/sql/suites/column_group/hint-disable.cfg index eb97f5b81d8..97707c9e373 100644 --- a/yt/yql/tests/sql/suites/column_group/hint-disable.cfg +++ b/yt/yql/tests/sql/suites/column_group/hint-disable.cfg @@ -3,6 +3,7 @@ out Output1 output1.txt out Output2 output2.txt out Output3 output3.txt out Output4 output4.txt +out Output5 output5.txt providers yt pragma yt.ColumnGroupMode="disable" pragma yt.OptimizeFor="scan" diff --git a/yt/yql/tests/sql/suites/column_group/hint-perusage.cfg b/yt/yql/tests/sql/suites/column_group/hint-perusage.cfg index c4261930389..9a3a2daa2c6 100644 --- a/yt/yql/tests/sql/suites/column_group/hint-perusage.cfg +++ b/yt/yql/tests/sql/suites/column_group/hint-perusage.cfg @@ -3,6 +3,7 @@ out Output1 output1.txt out Output2 output2.txt out Output3 output3.txt out Output4 output4.txt +out Output5 output5.txt providers yt pragma yt.ColumnGroupMode="perusage" pragma yt.OptimizeFor="scan" diff --git a/yt/yql/tests/sql/suites/column_group/hint-single.cfg b/yt/yql/tests/sql/suites/column_group/hint-single.cfg index 2aab65b91e7..e58a2ccde3a 100644 --- a/yt/yql/tests/sql/suites/column_group/hint-single.cfg +++ b/yt/yql/tests/sql/suites/column_group/hint-single.cfg @@ -3,6 +3,7 @@ out Output1 output1.txt out Output2 output2.txt out Output3 output3.txt out Output4 output4.txt +out Output5 output5.txt providers yt pragma yt.ColumnGroupMode="single" pragma yt.OptimizeFor="scan" diff --git a/yt/yql/tests/sql/suites/column_group/hint.sql b/yt/yql/tests/sql/suites/column_group/hint.sql index 5a04cc74f04..3e21451fbb7 100644 --- a/yt/yql/tests/sql/suites/column_group/hint.sql +++ b/yt/yql/tests/sql/suites/column_group/hint.sql @@ -4,6 +4,7 @@ $i1 = select * from Input where a > "a"; -- several publish consumers with same $i2 = select * from Input where a > "a1"; -- several publish consumers with different groups $i3 = select * from Input where a < "a2"; -- several consumers including publish $i4 = select * from Input where a != "a"; -- several publish consumers with and without groups +$i5 = select * from Input where a != "b"; -- single publish consumer with with no groups (special case) -- test column group spec normalization insert into Output1 with column_groups="{g1=[a;b;c];def=#}" select * from $i1; @@ -17,4 +18,6 @@ insert into Output3 with column_groups="{g1=[a;b;c];def=#}" select * from $i4; insert into Output4 select * from $i4; +insert into Output5 with column_groups="{}" select * from $i5; + select a,b,c,d from $i3; diff --git a/yt/yt/client/api/rpc_proxy/client_impl.cpp b/yt/yt/client/api/rpc_proxy/client_impl.cpp index 824d11488e2..ee2fdbc2d30 100644 --- a/yt/yt/client/api/rpc_proxy/client_impl.cpp +++ b/yt/yt/client/api/rpc_proxy/client_impl.cpp @@ -1709,7 +1709,7 @@ TFuture<std::vector<TColumnarStatistics>> TClient::GetColumnarStatistics( auto req = proxy.GetColumnarStatistics(); SetTimeoutOptions(*req, options); - for (const auto& subPath: path) { + for (const auto& subPath : path) { req->add_paths(ConvertToYsonString(subPath).ToString()); } diff --git a/yt/yt/client/api/rpc_proxy/helpers.cpp b/yt/yt/client/api/rpc_proxy/helpers.cpp index d5703d97392..41f88cbcbf7 100644 --- a/yt/yt/client/api/rpc_proxy/helpers.cpp +++ b/yt/yt/client/api/rpc_proxy/helpers.cpp @@ -279,19 +279,19 @@ void ToProto( if (result.PoolTreeCounts) { auto* poolTreeCounts = proto->mutable_pool_tree_counts()->mutable_entries(); - for (const auto& entry: *result.PoolTreeCounts) { + for (const auto& entry : *result.PoolTreeCounts) { (*poolTreeCounts)[entry.first] = entry.second; } } if (result.PoolCounts) { - for (const auto& entry: *result.PoolCounts) { + for (const auto& entry : *result.PoolCounts) { auto* newPoolCount = proto->mutable_pool_counts()->add_entries(); newPoolCount->set_pool(entry.first); newPoolCount->set_count(entry.second); } } if (result.UserCounts) { - for (const auto& entry: *result.UserCounts) { + for (const auto& entry : *result.UserCounts) { auto* newUserCount = proto->mutable_user_counts()->add_entries(); newUserCount->set_user(entry.first); newUserCount->set_count(entry.second); @@ -299,7 +299,7 @@ void ToProto( } if (result.StateCounts) { - for (const auto& state: TEnumTraits<NScheduler::EOperationState>::GetDomainValues()) { + for (const auto& state : TEnumTraits<NScheduler::EOperationState>::GetDomainValues()) { if ((*result.StateCounts)[state] != 0) { auto* newStateCount = proto->mutable_state_counts()->add_entries(); newStateCount->set_state(ConvertOperationStateToProto(state)); @@ -308,7 +308,7 @@ void ToProto( } } if (result.TypeCounts) { - for (const auto& type: TEnumTraits<NScheduler::EOperationType>::GetDomainValues()) { + for (const auto& type : TEnumTraits<NScheduler::EOperationType>::GetDomainValues()) { if ((*result.TypeCounts)[type] != 0) { auto* newTypeCount = proto->mutable_type_counts()->add_entries(); newTypeCount->set_type(ConvertOperationTypeToProto(type)); @@ -341,7 +341,7 @@ void FromProto( if (proto.has_pool_counts()) { result->PoolCounts.emplace(); - for (const auto& poolCount: proto.pool_counts().entries()) { + for (const auto& poolCount : proto.pool_counts().entries()) { auto pool = poolCount.pool(); YT_VERIFY((*result->PoolCounts)[pool] == 0); (*result->PoolCounts)[pool] = poolCount.count(); @@ -351,7 +351,7 @@ void FromProto( } if (proto.has_user_counts()) { result->UserCounts.emplace(); - for (const auto& userCount: proto.user_counts().entries()) { + for (const auto& userCount : proto.user_counts().entries()) { auto user = userCount.user(); YT_VERIFY((*result->UserCounts)[user] == 0); (*result->UserCounts)[user] = userCount.count(); @@ -363,7 +363,7 @@ void FromProto( if (proto.has_state_counts()) { result->StateCounts.emplace(); std::fill(result->StateCounts->begin(), result->StateCounts->end(), 0); - for (const auto& stateCount: proto.state_counts().entries()) { + for (const auto& stateCount : proto.state_counts().entries()) { auto state = ConvertOperationStateFromProto(stateCount.state()); YT_VERIFY(result->StateCounts->IsValidIndex(state)); YT_VERIFY((*result->StateCounts)[state] == 0); @@ -375,7 +375,7 @@ void FromProto( if (proto.has_type_counts()) { result->TypeCounts.emplace(); std::fill(result->TypeCounts->begin(), result->TypeCounts->end(), 0); - for (const auto& typeCount: proto.type_counts().entries()) { + for (const auto& typeCount : proto.type_counts().entries()) { auto type = ConvertOperationTypeFromProto(typeCount.type()); YT_VERIFY(result->TypeCounts->IsValidIndex(type)); YT_VERIFY((*result->TypeCounts)[type] == 0); @@ -1170,7 +1170,7 @@ void ToProto( const NApi::TListJobsStatistics& statistics) { protoStatistics->mutable_state_counts()->clear_entries(); - for (const auto& state: TEnumTraits<NJobTrackerClient::EJobState>::GetDomainValues()) { + for (const auto& state : TEnumTraits<NJobTrackerClient::EJobState>::GetDomainValues()) { if (statistics.StateCounts[state] != 0) { auto* newStateCount = protoStatistics->mutable_state_counts()->add_entries(); newStateCount->set_state(ConvertJobStateToProto(state)); @@ -1179,7 +1179,7 @@ void ToProto( } protoStatistics->mutable_type_counts()->clear_entries(); - for (const auto& type: TEnumTraits<NJobTrackerClient::EJobType>::GetDomainValues()) { + for (const auto& type : TEnumTraits<NJobTrackerClient::EJobType>::GetDomainValues()) { if (statistics.TypeCounts[type] != 0) { auto* newTypeCount = protoStatistics->mutable_type_counts()->add_entries(); newTypeCount->set_type(ConvertJobTypeToProto(type)); @@ -1193,7 +1193,7 @@ void FromProto( const NProto::TListJobsStatistics& protoStatistics) { std::fill(statistics->StateCounts.begin(), statistics->StateCounts.end(), 0); - for (const auto& stateCount: protoStatistics.state_counts().entries()) { + for (const auto& stateCount : protoStatistics.state_counts().entries()) { auto state = ConvertJobStateFromProto(stateCount.state()); YT_VERIFY(statistics->StateCounts.IsValidIndex(state)); YT_VERIFY(statistics->StateCounts[state] == 0); @@ -1201,7 +1201,7 @@ void FromProto( } std::fill(statistics->TypeCounts.begin(), statistics->TypeCounts.end(), 0); - for (const auto& typeCount: protoStatistics.type_counts().entries()) { + for (const auto& typeCount : protoStatistics.type_counts().entries()) { auto type = ConvertJobTypeFromProto(typeCount.type()); YT_VERIFY(statistics->TypeCounts.IsValidIndex(type)); YT_VERIFY(statistics->TypeCounts[type] == 0); diff --git a/yt/yt/client/formats/config.cpp b/yt/yt/client/formats/config.cpp index ed4ee65a73d..eff67b828da 100644 --- a/yt/yt/client/formats/config.cpp +++ b/yt/yt/client/formats/config.cpp @@ -272,7 +272,7 @@ void TProtobufTableConfig::Register(TRegistrar registrar) registrar.Postprocessor([] (TThis* config) { bool hasOtherColumns = false; - for (const auto& column: config->Columns) { + for (const auto& column : config->Columns) { if (column->ProtoType == EProtobufType::OtherColumns) { if (hasOtherColumns) { THROW_ERROR_EXCEPTION("Multiple \"other_columns\" in protobuf config are not allowed"); diff --git a/yt/yt/client/signature/generator.cpp b/yt/yt/client/signature/generator.cpp index 75b69621067..98e3fe0d7e6 100644 --- a/yt/yt/client/signature/generator.cpp +++ b/yt/yt/client/signature/generator.cpp @@ -2,20 +2,68 @@ #include "signature.h" +#include <yt/yt/core/ytree/convert.h> + namespace NYT::NSignature { +using namespace NYson; + //////////////////////////////////////////////////////////////////////////////// -NYson::TYsonString& ISignatureGenerator::GetHeader(const TSignaturePtr& signature) +TSignaturePtr TSignatureGeneratorBase::Sign(TYsonString data) +{ + auto signature = New<TSignature>(); + signature->Payload_ = std::move(data); + Sign(signature); + return signature; +} + +//////////////////////////////////////////////////////////////////////////////// + +TYsonString& TSignatureGeneratorBase::GetHeader(const TSignaturePtr& signature) { return signature->Header_; } -std::vector<std::byte>& ISignatureGenerator::GetSignature(const TSignaturePtr& signature) +std::vector<std::byte>& TSignatureGeneratorBase::GetSignature(const TSignaturePtr& signature) { return signature->Signature_; } //////////////////////////////////////////////////////////////////////////////// +class TDummySignatureGenerator + : public TSignatureGeneratorBase +{ +public: + void Sign(const TSignaturePtr& signature) override + { + GetHeader(signature) = NYson::TYsonString("DummySignature"_sb); + } +}; + +TSignatureGeneratorBasePtr CreateDummySignatureGenerator() +{ + return New<TDummySignatureGenerator>(); +} + +//////////////////////////////////////////////////////////////////////////////// + +class TAlwaysThrowingSignatureGenerator + : public TSignatureGeneratorBase +{ +public: + void Sign(const TSignaturePtr& /*signature*/) override + { + THROW_ERROR_EXCEPTION("Signature generation is unsupported"); + } +}; + +TSignatureGeneratorBasePtr CreateAlwaysThrowingSignatureGenerator() +{ + return New<TAlwaysThrowingSignatureGenerator>(); +} + +//////////////////////////////////////////////////////////////////////////////// + } // namespace NYT::NSignature diff --git a/yt/yt/client/signature/generator.h b/yt/yt/client/signature/generator.h index 523f35d4c38..f93f41c0cba 100644 --- a/yt/yt/client/signature/generator.h +++ b/yt/yt/client/signature/generator.h @@ -8,7 +8,7 @@ namespace NYT::NSignature { //////////////////////////////////////////////////////////////////////////////// -class ISignatureGenerator +class TSignatureGeneratorBase : public TRefCounted { public: @@ -16,7 +16,9 @@ public: //! based on its payload. virtual void Sign(const TSignaturePtr& signature) = 0; - virtual ~ISignatureGenerator() = default; + [[nodiscard]] TSignaturePtr Sign(NYson::TYsonString data); + + virtual ~TSignatureGeneratorBase() = default; protected: NYson::TYsonString& GetHeader(const TSignaturePtr& signature); @@ -24,7 +26,13 @@ protected: std::vector<std::byte>& GetSignature(const TSignaturePtr& signature); }; -DEFINE_REFCOUNTED_TYPE(ISignatureGenerator) +DEFINE_REFCOUNTED_TYPE(TSignatureGeneratorBase) + +//////////////////////////////////////////////////////////////////////////////// + +TSignatureGeneratorBasePtr CreateDummySignatureGenerator(); + +TSignatureGeneratorBasePtr CreateAlwaysThrowingSignatureGenerator(); //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/client/signature/public.h b/yt/yt/client/signature/public.h index 845445d9e93..bf0a10cdbb4 100644 --- a/yt/yt/client/signature/public.h +++ b/yt/yt/client/signature/public.h @@ -10,8 +10,8 @@ DECLARE_REFCOUNTED_CLASS(TSignature) /////////////////////////////////////////////////////////////////////////////// -DECLARE_REFCOUNTED_CLASS(ISignatureGenerator) -DECLARE_REFCOUNTED_CLASS(ISignatureValidator) +DECLARE_REFCOUNTED_CLASS(TSignatureGeneratorBase) +DECLARE_REFCOUNTED_CLASS(TSignatureValidatorBase) /////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/client/signature/signature.cpp b/yt/yt/client/signature/signature.cpp index 4c071c14073..0a71e0a02c6 100644 --- a/yt/yt/client/signature/signature.cpp +++ b/yt/yt/client/signature/signature.cpp @@ -14,12 +14,6 @@ using namespace NYTree; //////////////////////////////////////////////////////////////////////////////// -TSignature::TSignature(NYson::TYsonString payload) - : Payload_(std::move(payload)) -{ } - -//////////////////////////////////////////////////////////////////////////////// - const TYsonString& TSignature::Payload() const { return Payload_; diff --git a/yt/yt/client/signature/signature.h b/yt/yt/client/signature/signature.h index 6cb739bd3c0..8f37c81261f 100644 --- a/yt/yt/client/signature/signature.h +++ b/yt/yt/client/signature/signature.h @@ -20,20 +20,15 @@ public: //! Constructs an empty TSignature. TSignature() = default; - //! Creates a TSignature containing the given payload without an actual signature. - explicit TSignature(NYson::TYsonString payload); - [[nodiscard]] const NYson::TYsonString& Payload() const; private: - // TODO(arkady-e1ppa): Whenever trivial generator/validators are added - // remove initialization. - NYson::TYsonString Header_ = NYson::TYsonString(TStringBuf("")); + NYson::TYsonString Header_; NYson::TYsonString Payload_; std::vector<std::byte> Signature_; - friend class ISignatureGenerator; - friend class ISignatureValidator; + friend class TSignatureGeneratorBase; + friend class TSignatureValidatorBase; friend void Serialize(const TSignature& signature, NYson::IYsonConsumer* consumer); friend void Deserialize(TSignature& signature, NYTree::INodePtr node); diff --git a/yt/yt/client/signature/unittests/dummy_ut.cpp b/yt/yt/client/signature/unittests/dummy_ut.cpp new file mode 100644 index 00000000000..1a5d483ad8e --- /dev/null +++ b/yt/yt/client/signature/unittests/dummy_ut.cpp @@ -0,0 +1,73 @@ +#include <yt/yt/core/test_framework/framework.h> + +#include <yt/yt/client/signature/generator.h> +#include <yt/yt/client/signature/validator.h> +#include <yt/yt/client/signature/signature.h> + +#include <yt/yt/core/yson/string.h> + +#include <yt/yt/core/ytree/convert.h> + +namespace NYT::NSignature { +namespace { + +//////////////////////////////////////////////////////////////////////////////// + +using namespace NYson; +using namespace NYTree; + +const auto YsonSignature = TYsonString( + R"({"header"="DummySignature";"payload"="payload";"signature"="";})"_sb); + +//////////////////////////////////////////////////////////////////////////////// + +TEST(TDummySignatureGeneratorTest, Generate) +{ + auto generator = CreateDummySignatureGenerator(); + auto signature = generator->Sign(TYsonString("payload"_sb)); + EXPECT_EQ(ConvertToYsonString(signature, EYsonFormat::Text), YsonSignature); + generator->Sign(signature); + EXPECT_EQ(ConvertToYsonString(signature, EYsonFormat::Text), YsonSignature); +} + +//////////////////////////////////////////////////////////////////////////////// + +TEST(TDummySignatureValidatorTest, ValidateGood) +{ + auto signature = ConvertTo<TSignaturePtr>(YsonSignature); + auto validator = CreateDummySignatureValidator(); + EXPECT_TRUE(validator->Validate(signature).Get().Value()); +} + +//////////////////////////////////////////////////////////////////////////////// + +TEST(TDummySignatureValidatorTest, GenerateValidate) +{ + auto generator = CreateDummySignatureGenerator(); + auto validator = CreateDummySignatureValidator(); + auto signature = generator->Sign(TYsonString("payload"_sb)); + EXPECT_TRUE(validator->Validate(signature).Get().Value()); +} + +//////////////////////////////////////////////////////////////////////////////// + +TEST(TAlwaysThrowingSignatureGeneratorTest, Generate) +{ + auto generator = CreateAlwaysThrowingSignatureGenerator(); + EXPECT_THROW_WITH_SUBSTRING(generator->Sign(New<TSignature>()), "unsupported"); +} + +//////////////////////////////////////////////////////////////////////////////// + +TEST(TAlwaysThrowingSignatureValidatorTest, Validate) +{ + auto validator = CreateAlwaysThrowingSignatureValidator(); + EXPECT_THROW_WITH_SUBSTRING( + YT_UNUSED_FUTURE(validator->Validate(New<TSignature>())), + "unsupported"); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace +} // namespace NYT::NSignature diff --git a/yt/yt/client/signature/unittests/signature_ut.cpp b/yt/yt/client/signature/unittests/signature_ut.cpp index 75ae1733fab..26397157469 100644 --- a/yt/yt/client/signature/unittests/signature_ut.cpp +++ b/yt/yt/client/signature/unittests/signature_ut.cpp @@ -16,14 +16,6 @@ using namespace NYTree; //////////////////////////////////////////////////////////////////////////////// -TEST(TSignatureTest, PayloadConstruct) -{ - TSignature signature(TYsonString("payload"_sb)); - EXPECT_EQ(signature.Payload().ToString(), "payload"); -} - -//////////////////////////////////////////////////////////////////////////////// - TEST(TSignatureTest, DeserializeSerialize) { // SignatureSize bytes. diff --git a/yt/yt/client/signature/unittests/ya.make b/yt/yt/client/signature/unittests/ya.make index b7f6fd5fe56..ee92b119d94 100644 --- a/yt/yt/client/signature/unittests/ya.make +++ b/yt/yt/client/signature/unittests/ya.make @@ -3,6 +3,7 @@ GTEST(unittester-client-signature) INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc) SRCS( + dummy_ut.cpp signature_ut.cpp ) diff --git a/yt/yt/client/signature/validator.cpp b/yt/yt/client/signature/validator.cpp index 710303bf851..ef57ba0d251 100644 --- a/yt/yt/client/signature/validator.cpp +++ b/yt/yt/client/signature/validator.cpp @@ -2,21 +2,58 @@ #include "signature.h" +#include <yt/yt/core/actions/future.h> + namespace NYT::NSignature { //////////////////////////////////////////////////////////////////////////////// -const NYson::TYsonString& ISignatureValidator::GetHeader(const TSignaturePtr& signature) +const NYson::TYsonString& TSignatureValidatorBase::GetHeader(const TSignaturePtr& signature) { return signature->Header_; } -const std::vector<std::byte>& ISignatureValidator::GetSignature(const TSignaturePtr& signature) +const std::vector<std::byte>& TSignatureValidatorBase::GetSignature(const TSignaturePtr& signature) { return signature->Signature_; } //////////////////////////////////////////////////////////////////////////////// +class TDummySignatureValidator + : public TSignatureValidatorBase +{ +public: + TFuture<bool> Validate(const TSignaturePtr& signature) override + { + YT_VERIFY(GetHeader(signature).ToString() == "DummySignature"); + return TrueFuture; + } +}; + +TSignatureValidatorBasePtr CreateDummySignatureValidator() +{ + return New<TDummySignatureValidator>(); +} + +//////////////////////////////////////////////////////////////////////////////// + +class TAlwaysThrowingSignatureValidator + : public TSignatureValidatorBase +{ +public: + TFuture<bool> Validate(const TSignaturePtr& /*signature*/) override + { + THROW_ERROR_EXCEPTION("Signature validation is unsupported"); + } +}; + +TSignatureValidatorBasePtr CreateAlwaysThrowingSignatureValidator() +{ + return New<TAlwaysThrowingSignatureValidator>(); +} + +//////////////////////////////////////////////////////////////////////////////// + } // namespace NYT::NSignature diff --git a/yt/yt/client/signature/validator.h b/yt/yt/client/signature/validator.h index d3f3e76a695..bde89c44bbe 100644 --- a/yt/yt/client/signature/validator.h +++ b/yt/yt/client/signature/validator.h @@ -9,13 +9,13 @@ namespace NYT::NSignature { //////////////////////////////////////////////////////////////////////////////// -class ISignatureValidator +class TSignatureValidatorBase : public TRefCounted { public: virtual TFuture<bool> Validate(const TSignaturePtr& signature) = 0; - virtual ~ISignatureValidator() = default; + virtual ~TSignatureValidatorBase() = default; protected: const NYson::TYsonString& GetHeader(const TSignaturePtr& signature); @@ -23,7 +23,13 @@ protected: const std::vector<std::byte>& GetSignature(const TSignaturePtr& signature); }; -DEFINE_REFCOUNTED_TYPE(ISignatureValidator) +DEFINE_REFCOUNTED_TYPE(TSignatureValidatorBase) + +//////////////////////////////////////////////////////////////////////////////// + +TSignatureValidatorBasePtr CreateDummySignatureValidator(); + +TSignatureValidatorBasePtr CreateAlwaysThrowingSignatureValidator(); //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/client/table_client/merge_table_schemas.cpp b/yt/yt/client/table_client/merge_table_schemas.cpp index d41e58baf53..ef7d10fcfc9 100644 --- a/yt/yt/client/table_client/merge_table_schemas.cpp +++ b/yt/yt/client/table_client/merge_table_schemas.cpp @@ -100,7 +100,7 @@ TTableSchemaPtr MergeTableSchemas( auto getDeletedColumnsStableNames = [] (const std::vector<TDeletedColumn>& deletedColumns) { THashSet<TColumnStableName> stableNames; - for (const auto& column: deletedColumns) { + for (const auto& column : deletedColumns) { stableNames.insert(column.StableName()); } return stableNames; diff --git a/yt/yt/client/table_client/schema.cpp b/yt/yt/client/table_client/schema.cpp index 4b826077e03..9cdf7119583 100644 --- a/yt/yt/client/table_client/schema.cpp +++ b/yt/yt/client/table_client/schema.cpp @@ -1625,7 +1625,7 @@ void TTableSchemaTruncatedFormatter::operator()(TStringBuilderBase* builder) con if (!Schema_) { builder->AppendString(ToString(nullptr)); } else if (Threshold_ > 0 && Schema_->GetMemoryUsage(Threshold_) < Threshold_) { - FormatValue(builder, *Schema_, "%v"); + FormatValue(builder, *Schema_, "v"); } else { builder->AppendString("<schema memory usage is over the logging threshold>"); } diff --git a/yt/yt/core/misc/arithmetic_formula.cpp b/yt/yt/core/misc/arithmetic_formula.cpp index d4231991ec1..54404cd6667 100644 --- a/yt/yt/core/misc/arithmetic_formula.cpp +++ b/yt/yt/core/misc/arithmetic_formula.cpp @@ -868,7 +868,7 @@ void TArithmeticFormula::Load(TStreamLoadContext& context) TBooleanFormulaTags::TBooleanFormulaTags(THashSet<std::string> tags) : Tags_(std::move(tags)) { - for (const auto& key: Tags_) { + for (const auto& key : Tags_) { PreparedTags_[key] = 1; } } @@ -955,7 +955,7 @@ std::string TBooleanFormula::GetFormula() const bool TBooleanFormula::IsSatisfiedBy(const std::vector<std::string>& value) const { THashMap<std::string, i64> values; - for (const auto& key: value) { + for (const auto& key : value) { values[key] = 1; } return Impl_->Eval(values, EEvaluationContext::Boolean); diff --git a/yt/yt/core/misc/unittests/consistent_hashing_ut.cpp b/yt/yt/core/misc/unittests/consistent_hashing_ut.cpp index 8c95d6e961d..2f96372a61c 100644 --- a/yt/yt/core/misc/unittests/consistent_hashing_ut.cpp +++ b/yt/yt/core/misc/unittests/consistent_hashing_ut.cpp @@ -313,7 +313,7 @@ double GetPercentageInconsistentFiles( auto countDisplaced = [&] (const std::vector<std::pair<EQueryType, std::pair<TString, int>>>& queries) { std::map<TCrpItemWithToken, TCompactVector<TString, 1>> serversBefore; - for (const auto& file: files) { + for (const auto& file : files) { auto candidates = ring.GetServersForFile(file.first, file.second); candidates.resize(std::min(static_cast<size_t>(file.second), candidateCount)); serversBefore[file] = candidates; @@ -339,7 +339,7 @@ double GetPercentageInconsistentFiles( } int result = 0; - for (const auto& elem: files) { + for (const auto& elem : files) { auto candidates = ring.GetServersForFile(elem.first, elem.second); candidates.resize(std::min(static_cast<size_t>(elem.second), candidateCount)); result += (serversBefore[elem] != candidates); diff --git a/yt/yt/core/rpc/unittests/viable_peer_registry_ut.cpp b/yt/yt/core/rpc/unittests/viable_peer_registry_ut.cpp index cad9357c471..9df0436e169 100644 --- a/yt/yt/core/rpc/unittests/viable_peer_registry_ut.cpp +++ b/yt/yt/core/rpc/unittests/viable_peer_registry_ut.cpp @@ -190,7 +190,7 @@ IViablePeerRegistryPtr CreateTestRegistry( std::vector<std::string> AddressesFromChannels(const std::vector<IChannelPtr>& channels) { std::vector<std::string> result; - for (const auto& channel: channels) { + for (const auto& channel : channels) { result.push_back(channel->GetEndpointDescription()); } return result; diff --git a/yt/yt/core/ya.make b/yt/yt/core/ya.make index 857e56920b8..a958bb1c6a2 100644 --- a/yt/yt/core/ya.make +++ b/yt/yt/core/ya.make @@ -269,6 +269,7 @@ SRCS( yson/string_merger.cpp yson/ypath_designated_consumer.cpp yson/ypath_filtering_consumer.cpp + yson/yson_builder.cpp yson/depth_limiting_yson_consumer.cpp yson/list_verb_lazy_yson_consumer.cpp yson/attributes_stripper.cpp diff --git a/yt/yt/core/yson/protobuf_interop.cpp b/yt/yt/core/yson/protobuf_interop.cpp index dd019603843..12f138442b1 100644 --- a/yt/yt/core/yson/protobuf_interop.cpp +++ b/yt/yt/core/yson/protobuf_interop.cpp @@ -3051,6 +3051,11 @@ TProtobufElementResolveResult ResolveProtobufElementByYPath( } tokenizer.Advance(); + if (options.AllowAsterisks && tokenizer.GetType() == NYPath::ETokenType::Asterisk) { + tokenizer.Advance(); + tokenizer.Expect(NYPath::ETokenType::Slash); + tokenizer.Advance(); + } tokenizer.Expect(NYPath::ETokenType::Literal); const auto& fieldName = tokenizer.GetLiteralValue(); @@ -3115,7 +3120,9 @@ TProtobufElementResolveResult ResolveProtobufElementByYPath( tokenizer.Expect(NYPath::ETokenType::Slash); tokenizer.Advance(); - tokenizer.ExpectListIndex(); + if (!options.AllowAsterisks || tokenizer.GetType() != NYPath::ETokenType::Asterisk) { + tokenizer.ExpectListIndex(); + } if (!field->IsMessage()) { return GetProtobufElementFromField( diff --git a/yt/yt/core/yson/protobuf_interop_options.h b/yt/yt/core/yson/protobuf_interop_options.h index 68a45c4b296..40a53d44c7a 100644 --- a/yt/yt/core/yson/protobuf_interop_options.h +++ b/yt/yt/core/yson/protobuf_interop_options.h @@ -14,6 +14,7 @@ namespace NYT::NYson { struct TResolveProtobufElementByYPathOptions { bool AllowUnknownYsonFields = false; + bool AllowAsterisks = false; }; struct TProtobufWriterOptions diff --git a/yt/yt/core/yson/unittests/protobuf_yson_ut.cpp b/yt/yt/core/yson/unittests/protobuf_yson_ut.cpp index c70e71828bd..6c24070f16c 100644 --- a/yt/yt/core/yson/unittests/protobuf_yson_ut.cpp +++ b/yt/yt/core/yson/unittests/protobuf_yson_ut.cpp @@ -1023,7 +1023,7 @@ TEST(TYsonToProtobufTest, Entities) TEST(TYsonToProtobufTest, ValidUtf8StringCheck) { - for (auto configOption: {EUtf8Check::Disable, EUtf8Check::LogOnFail, EUtf8Check::ThrowOnFail}) { + for (auto configOption : {EUtf8Check::Disable, EUtf8Check::LogOnFail, EUtf8Check::ThrowOnFail}) { for (auto option: std::vector<std::optional<EUtf8Check>>{ std::nullopt, EUtf8Check::Disable, EUtf8Check::LogOnFail, EUtf8Check::ThrowOnFail}) { @@ -2675,7 +2675,7 @@ TEST(TPackedRepeatedProtobufTest, TestSerializeDeserialize) TEST(TEnumYsonStorageTypeTest, TestDeserializeSerialize) { - for (auto storageType: {EEnumYsonStorageType::String, EEnumYsonStorageType::Int}) { + for (auto storageType : {EEnumYsonStorageType::String, EEnumYsonStorageType::Int}) { auto config = New<TProtobufInteropConfig>(); config->DefaultEnumYsonStorageType = storageType; SetProtobufInteropConfig(config); diff --git a/yt/yt/core/yson/unittests/ya.make b/yt/yt/core/yson/unittests/ya.make index 59a807fca42..8f8fc7dfc02 100644 --- a/yt/yt/core/yson/unittests/ya.make +++ b/yt/yt/core/yson/unittests/ya.make @@ -14,6 +14,7 @@ SRCS( protobuf_yson_ut.cpp ypath_designated_yson_consumer_ut.cpp ypath_filtering_yson_consumer_ut.cpp + yson_builder_ut.cpp yson_parser_ut.cpp yson_pull_parser_ut.cpp yson_token_writer_ut.cpp diff --git a/yt/yt/core/yson/unittests/yson_builder_ut.cpp b/yt/yt/core/yson/unittests/yson_builder_ut.cpp new file mode 100644 index 00000000000..c3353a56dec --- /dev/null +++ b/yt/yt/core/yson/unittests/yson_builder_ut.cpp @@ -0,0 +1,87 @@ +#include <yt/yt/core/yson/consumer.h> +#include <yt/yt/core/yson/string.h> +#include <yt/yt/core/yson/yson_builder.h> + +#include <yt/yt/core/test_framework/framework.h> + +namespace NYT::NYson { +namespace { + +//////////////////////////////////////////////////////////////////////////////// + +TEST(TYsonStringBuilderTest, Simple) +{ + NYson::TYsonStringBuilder builder; + builder->OnStringScalar("some_scalar"); + ASSERT_EQ(builder.Flush().ToString(), TString{"\1\x16some_scalar"}); + ASSERT_TRUE(builder.IsEmpty()); +} + +TEST(TYsonStringBuilderTest, Reusing) +{ + NYson::TYsonStringBuilder builder; + builder->OnStringScalar("some_scalar1"); + ASSERT_EQ(builder.Flush().ToString(), TString{"\1\x18some_scalar1"}); + ASSERT_TRUE(builder.IsEmpty()); + + builder->OnStringScalar("some_scalar2"); + ASSERT_EQ(builder.Flush().ToString(), TString{"\1\x18some_scalar2"}); + ASSERT_TRUE(builder.IsEmpty()); +} + +TEST(TYsonStringBuilderTest, Checkpoints) +{ + NYson::TYsonStringBuilder builder; + builder->OnStringScalar("some_scalar"); + + auto checkpoint = builder.CreateCheckpoint(); + builder.CreateCheckpoint(); + builder.RestoreCheckpoint(checkpoint); + builder.RestoreCheckpoint(checkpoint); + + ASSERT_EQ(builder.Flush().ToString(), TString{"\1\x16some_scalar"}); + ASSERT_TRUE(builder.IsEmpty()); +} + +TEST(TYsonStringBuilderTest, MapCheckpoints) +{ + NYson::TYsonStringBuilder builder(NYson::EYsonFormat::Text); + + builder->OnBeginMap(); + + builder->OnKeyedItem("key1"); + builder->OnEntity(); + + auto checkpoint = builder.CreateCheckpoint(); + builder->OnKeyedItem("key2"); + builder->OnEntity(); + builder.RestoreCheckpoint(checkpoint); + + builder->OnEndMap(); + + ASSERT_EQ(builder.Flush().ToString(), TString{R"({"key1"=#;})"}); +} + +//////////////////////////////////////////////////////////////////////////////// + +TEST(TYsonBuilderTest, Forwarding) +{ + NYson::TYsonStringBuilder stringBuilder(NYson::EYsonFormat::Text); + NYson::TYsonBuilder builder( + NYson::EYsonBuilderForwardingPolicy::Forward, + &stringBuilder, + stringBuilder.GetConsumer()); + builder->OnBeginMap(); + auto checkpoint = builder.CreateCheckpoint(); + builder->OnKeyedItem("key"); + builder->OnEntity(); + builder.RestoreCheckpoint(checkpoint); + builder->OnEndMap(); + + ASSERT_EQ(stringBuilder.Flush().ToString(), TString{R"({})"}); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace +} // namespace NYT::NYson diff --git a/yt/yt/core/yson/yson_builder.cpp b/yt/yt/core/yson/yson_builder.cpp new file mode 100644 index 00000000000..a17b9b87721 --- /dev/null +++ b/yt/yt/core/yson/yson_builder.cpp @@ -0,0 +1,97 @@ +#include "yson_builder.h" + +#include "writer.h" + +namespace NYT::NYson { + +//////////////////////////////////////////////////////////////////////////////// + +IYsonConsumer* IYsonBuilder::operator->() +{ + return GetConsumer(); +} + +//////////////////////////////////////////////////////////////////////////////// + +TYsonStringBuilder::TYsonStringBuilder(EYsonFormat format, EYsonType type, bool enableRaw) + : Output_(ValueString_) + , Writer_(CreateYsonWriter(&Output_, format, type, enableRaw)) +{ } + +IYsonConsumer* TYsonStringBuilder::GetConsumer() +{ + return Writer_.get(); +} + +IYsonBuilder::TCheckpoint TYsonStringBuilder::CreateCheckpoint() +{ + Writer_->Flush(); + return IYsonBuilder::TCheckpoint(std::ssize(ValueString_)); +} + +void TYsonStringBuilder::RestoreCheckpoint(IYsonBuilder::TCheckpoint checkpoint) +{ + Writer_->Flush(); + int checkpointSize = checkpoint.Underlying(); + YT_VERIFY(checkpointSize >= 0 && checkpointSize <= std::ssize(ValueString_)); + ValueString_.resize(static_cast<size_t>(checkpointSize)); +} + +TYsonString TYsonStringBuilder::Flush() +{ + Writer_->Flush(); + auto result = TYsonString(ValueString_); + ValueString_.clear(); + return result; +} + +bool TYsonStringBuilder::IsEmpty() +{ + Writer_->Flush(); + return ValueString_.empty(); +} + +//////////////////////////////////////////////////////////////////////////////// + +TYsonBuilder::TYsonBuilder( + EYsonBuilderForwardingPolicy policy, + IYsonBuilder* underlying, + IYsonConsumer* consumer) + : Policy_(policy) + , Underlying_(underlying) + , Consumer_(consumer) +{ } + +IYsonConsumer* TYsonBuilder::GetConsumer() +{ + return Consumer_; +} + +IYsonBuilder::TCheckpoint TYsonBuilder::CreateCheckpoint() +{ + switch (Policy_) { + case EYsonBuilderForwardingPolicy::Forward: + return Underlying_->CreateCheckpoint(); + case EYsonBuilderForwardingPolicy::Crash: + return IYsonBuilder::TCheckpoint{}; + case EYsonBuilderForwardingPolicy::Ignore: + return IYsonBuilder::TCheckpoint{}; + } +} + +void TYsonBuilder::RestoreCheckpoint(TCheckpoint checkpoint) +{ + switch (Policy_) { + case EYsonBuilderForwardingPolicy::Forward: + Underlying_->RestoreCheckpoint(checkpoint); + break; + case EYsonBuilderForwardingPolicy::Crash: + YT_ABORT(); + case EYsonBuilderForwardingPolicy::Ignore: + break; + } +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NYson diff --git a/yt/yt/core/yson/yson_builder.h b/yt/yt/core/yson/yson_builder.h new file mode 100644 index 00000000000..5f483124e8f --- /dev/null +++ b/yt/yt/core/yson/yson_builder.h @@ -0,0 +1,76 @@ +#pragma once + +#include "public.h" + +#include <library/cpp/yt/misc/strong_typedef.h> + +namespace NYT::NYson { + +//////////////////////////////////////////////////////////////////////////////// + +struct IYsonBuilder +{ + YT_DEFINE_STRONG_TYPEDEF(TCheckpoint, int); + + virtual ~IYsonBuilder() = default; + + virtual IYsonConsumer* GetConsumer() = 0; + virtual TCheckpoint CreateCheckpoint() = 0; + virtual void RestoreCheckpoint(TCheckpoint checkpoint) = 0; + + IYsonConsumer* operator->(); +}; + +//////////////////////////////////////////////////////////////////////////////// + +class TYsonStringBuilder + : public IYsonBuilder +{ +public: + TYsonStringBuilder( + NYson::EYsonFormat format = NYson::EYsonFormat::Binary, + NYson::EYsonType type = NYson::EYsonType::Node, + bool enableRaw = true); + + NYson::IYsonConsumer* GetConsumer() override; + IYsonBuilder::TCheckpoint CreateCheckpoint() override; + void RestoreCheckpoint(IYsonBuilder::TCheckpoint checkpoint) override; + + NYson::TYsonString Flush(); + bool IsEmpty(); + +private: + TString ValueString_; + TStringOutput Output_; + const std::unique_ptr<NYson::IFlushableYsonConsumer> Writer_; +}; + +//////////////////////////////////////////////////////////////////////////////// + +DEFINE_ENUM(EYsonBuilderForwardingPolicy, + (Forward) + (Ignore) + (Crash) +); + +//////////////////////////////////////////////////////////////////////////////// + +class TYsonBuilder + : public IYsonBuilder +{ +public: + TYsonBuilder(EYsonBuilderForwardingPolicy policy, IYsonBuilder* underlying, IYsonConsumer* consumer); + + IYsonConsumer* GetConsumer() override; + IYsonBuilder::TCheckpoint CreateCheckpoint() override; + void RestoreCheckpoint(TCheckpoint checkpoint) override; + +private: + const EYsonBuilderForwardingPolicy Policy_; + IYsonBuilder* const Underlying_; + NYson::IYsonConsumer* const Consumer_; +}; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NYson diff --git a/yt/yt/library/formats/protobuf.cpp b/yt/yt/library/formats/protobuf.cpp index 648f4c27eb1..46f87d4b15d 100644 --- a/yt/yt/library/formats/protobuf.cpp +++ b/yt/yt/library/formats/protobuf.cpp @@ -895,7 +895,7 @@ void TProtobufFormatDescriptionBase<TProtobufWriterType>::InitEmbeddedColumn( { auto embeddingIndex = tableType->AddEmbedding(parentEmbeddingIndex, columnConfig); - for (auto& fieldConfig: columnConfig->Type->Fields) { + for (auto& fieldConfig : columnConfig->Type->Fields) { InitColumn(fieldIndex, tableSchema, typeBuilder, tableType, fieldConfig, parent, embeddingIndex); } } @@ -924,7 +924,7 @@ void TProtobufFormatDescriptionBase<TProtobufParserType>::InitEmbeddedColumn( std::move(child), //KMP fieldIndex); - for (auto& fieldConfig: columnConfig->Type->Fields) { + for (auto& fieldConfig : columnConfig->Type->Fields) { InitColumn(fieldIndex, tableSchema, typeBuilder, tableType, fieldConfig, childPtr->Type, parentEmbeddingIndex); } } @@ -1600,7 +1600,7 @@ static int Process( const std::unique_ptr<TProtobufParserFieldDescription>& child) { if (child->Type->ProtoType == EProtobufType::EmbeddedMessage) { - for (const auto& grandChild: child->Type->Children) { + for (const auto& grandChild : child->Type->Children) { globalChildIndex = Process(ids, globalChildIndex, nameTable, child->Type, grandChild); } } else { @@ -1620,7 +1620,7 @@ std::vector<std::pair<ui16, TProtobufParserFieldDescription*>> TProtobufParserFo std::vector<std::pair<ui16, TProtobufParserFieldDescription*>> ids; int globalChildIndex = 0; - for (const auto& child: TableType_->Children) { + for (const auto& child : TableType_->Children) { globalChildIndex = Process(ids, globalChildIndex, nameTable, TableType_, child); } diff --git a/yt/yt/library/program/program.cpp b/yt/yt/library/program/program.cpp index 2dc58dc1755..6b2e012afe1 100644 --- a/yt/yt/library/program/program.cpp +++ b/yt/yt/library/program/program.cpp @@ -29,10 +29,6 @@ #include <util/system/thread.h> #include <util/system/sigset.h> -#include <util/string/subst.h> - -#include <thread> - #include <stdlib.h> #ifdef _unix_ @@ -46,11 +42,6 @@ #include <sys/prctl.h> #endif -#if defined(_linux_) && defined(CLANG_COVERAGE) -extern "C" int __llvm_profile_write_file(void); -extern "C" void __llvm_profile_set_filename(const char* name); -#endif - namespace NYT { using namespace NYson; @@ -96,8 +87,6 @@ TProgram::TProgram() .NoArgument() .StoreValue(&PrintBuild_, true); Opts_.SetFreeArgsNum(0); - - ConfigureCoverageOutput(); } TProgram::~TProgram() = default; @@ -276,19 +265,6 @@ void ConfigureUids() #endif } -void ConfigureCoverageOutput() -{ -#if defined(_linux_) && defined(CLANG_COVERAGE) - // YT tests use pid namespaces. We can't use process id as unique identifier for output file. - if (auto profileFile = getenv("LLVM_PROFILE_FILE")) { - TString fixedProfile{profileFile}; - SubstGlobal(fixedProfile, "%e", "ytserver-all"); - SubstGlobal(fixedProfile, "%p", ToString(TInstant::Now().NanoSeconds())); - __llvm_profile_set_filename(fixedProfile.c_str()); - } -#endif -} - void ConfigureIgnoreSigpipe() { #ifdef _unix_ diff --git a/yt/yt/library/program/program.h b/yt/yt/library/program/program.h index 7104326bdac..e60d374574e 100644 --- a/yt/yt/library/program/program.h +++ b/yt/yt/library/program/program.h @@ -126,8 +126,6 @@ NYson::TYsonString CheckYsonArgMapper(const TString& arg); //! Drop privileges and save them if running with suid-bit. void ConfigureUids(); -void ConfigureCoverageOutput(); - void ConfigureIgnoreSigpipe(); //! Intercepts standard crash signals (see signal_registry.h for full list) with a nice handler. |