diff options
| author | Alexander Smirnov <[email protected]> | 2025-02-05 00:51:36 +0000 |
|---|---|---|
| committer | Alexander Smirnov <[email protected]> | 2025-02-05 00:51:36 +0000 |
| commit | 21c282998b41e7edec2cae094f996be2feb7eb37 (patch) | |
| tree | 985f632cd3809ddc5ba06449701db2f1a2ae48c3 | |
| parent | 8587d8117d7b1b539a00f9e214a749d1d538691b (diff) | |
| parent | 1d3a1b1e4201d7a3cbb150b881aeffc3535c2995 (diff) | |
Merge branch 'rightlib' into merge-libs-250205-0050
111 files changed, 1139 insertions, 233 deletions
diff --git a/build/conf/go.conf b/build/conf/go.conf index 896f287bd21..2a7b00dd656 100644 --- a/build/conf/go.conf +++ b/build/conf/go.conf @@ -675,7 +675,7 @@ module _GO_BASE_UNIT: _BASE_UNIT { } default ? { GO_PROTO_OUTS+=${hide;output;norel;nopath;noext;suf=.pb.go:File} - GO_PROTO_OPTS+=--plugin=protoc-gen-go=${tool:_TOOL_PROTOC_GEN_GO} --go_out=$GO_PROTO_GEN_PLUGINS:${ARCADIA_BUILD_ROOT}/$PROTO_NAMESPACE + GO_PROTO_OPTS+=--plugin=protoc-gen-go=${tool:_TOOL_PROTOC_GEN_GO} --go_opt=${GO_PROTO_GEN_PLUGINS} --go_out=${ARCADIA_BUILD_ROOT}/$PROTO_NAMESPACE } } diff --git a/build/conf/java.conf b/build/conf/java.conf index c9aa0cad394..38c1ead5e3b 100644 --- a/build/conf/java.conf +++ b/build/conf/java.conf @@ -2092,10 +2092,6 @@ macro JAVA_DEPENDENCIES_CONFIGURATION(Args...) { } # tag:java-specific -# TODO(DEVTOOLS-6901): remove this variable when there are no more references to it in jbuild -JAVA_FORBIDDEN_LIBRARIES_VALUE= - -# tag:java-specific JAR_INCLUDE_FILTER_VALUE= JAR_EXCLUDE_FILTER_VALUE= diff --git a/build/conf/settings.conf b/build/conf/settings.conf index 23186f34778..a9be913dd36 100644 --- a/build/conf/settings.conf +++ b/build/conf/settings.conf @@ -125,6 +125,8 @@ _FOLDABLE_VARS=\ # end of _FOLDABLE_VARS list +_MODULE_SCOPE_ONLY_VARS=PIC + ARCADIA_TEST_ROOT=../arcadia_tests_data/ DEFAULT_REQUIREMENTS=network:restricted cpu:1 ram:32 diff --git a/build/conf/sysincl.conf b/build/conf/sysincl.conf index 8945843f6c9..ce34f83e4c6 100644 --- a/build/conf/sysincl.conf +++ b/build/conf/sysincl.conf @@ -97,6 +97,9 @@ when ($USE_ARCADIA_COMPILER_RUNTIME == "no") { when ($OS_EMSCRIPTEN) { SYSINCL+=build/sysincl/emscripten.yml } +otherwise { + SYSINCL+=build/sysincl/emscripten-to-nothing.yml +} when ($CUDA_VERSION == "10.2") { SYSINCL+=build/sysincl/nvidia-cub-1.8.0.yml diff --git a/build/conf/ts/ts.conf b/build/conf/ts/ts.conf index e4b6c88642a..52d47a84db0 100644 --- a/build/conf/ts/ts.conf +++ b/build/conf/ts/ts.conf @@ -218,7 +218,10 @@ macro TS_LARGE_FILES(DESTINATION="$_TS_REQUIRED_MISSING", FILES...) { _TS_LARGE_FILES($DESTINATION $FILES) } - +### @usage RUN_JAVASCRIPT_AFTER_BUILD(script_path [args...] [IN inputs...] [OUTDIR outdir]) +### +### Run JS script after build of TS_* module. +### Documentation: https://docs.yandex-team.ru/frontend-in-arcadia/references/macros#run-javascript-after-build macro RUN_JAVASCRIPT_AFTER_BUILD(JS_SCRIPT, OUTDIR="", IN{input}[], ARGS...) { # --after-build-args uses `arg=val` format because args can start with --, so value will be parsed as another key. # With `arg=val` format parser knows that after `=` is a value of the --after-build-args. diff --git a/build/mapping.conf.json b/build/mapping.conf.json index 173c0090bd5..57ac821fa5d 100644 --- a/build/mapping.conf.json +++ b/build/mapping.conf.json @@ -280,7 +280,11 @@ "6576113077": "https://devtools-registry.s3.yandex.net/6576113077", "6576113078": "https://devtools-registry.s3.yandex.net/6576113078", "6576113095": "https://devtools-registry.s3.yandex.net/6576113095", + "7709336951": "https://devtools-registry.s3.yandex.net/7709336951", + "7709336970": "https://devtools-registry.s3.yandex.net/7709336970", + "7709337139": "https://devtools-registry.s3.yandex.net/7709337139", "6576113106": "https://devtools-registry.s3.yandex.net/6576113106", + "7709337153": "https://devtools-registry.s3.yandex.net/7709337153", "6148089711": "https://devtools-registry.s3.yandex.net/6148089711", "6512097221": "https://devtools-registry.s3.yandex.net/6512097221", "6812263131": "https://devtools-registry.s3.yandex.net/6812263131", @@ -1492,7 +1496,11 @@ "6576113077": "devtools/ktlint_build/package.json", "6576113078": "devtools/ktlint_build/package.json", "6576113095": "devtools/ktlint_build/package.json", + "7709336951": "devtools/ktlint_build/package.json", + "7709336970": "devtools/ktlint_build/package.json", + "7709337139": "devtools/ktlint_build/package.json", "6576113106": "devtools/ktlint_build/package_win.json", + "7709337153": "devtools/ktlint_build/package_win.json", "6148089711": "devtools/local_cache/toolscache/server/ya-tc for darwin", "6512097221": "devtools/local_cache/toolscache/server/ya-tc for darwin", "6812263131": "devtools/local_cache/toolscache/server/ya-tc for darwin", diff --git a/build/plugins/java.py b/build/plugins/java.py index d58b5f9d6c3..12789097c62 100644 --- a/build/plugins/java.py +++ b/build/plugins/java.py @@ -52,15 +52,11 @@ def onjava_module(unit, *args): 'MANAGED_PEERS': '${MANAGED_PEERS}', 'MANAGED_PEERS_CLOSURE': '${MANAGED_PEERS_CLOSURE}', 'NON_NAMAGEABLE_PEERS': '${NON_NAMAGEABLE_PEERS}', - 'EXCLUDE': extract_macro_calls(unit, 'EXCLUDE_VALUE', args_delim), 'JAVA_SRCS': extract_macro_calls(unit, 'JAVA_SRCS_VALUE', args_delim), 'JAVAC_FLAGS': extract_macro_calls(unit, 'JAVAC_FLAGS_VALUE', args_delim), 'ANNOTATION_PROCESSOR': extract_macro_calls(unit, 'ANNOTATION_PROCESSOR_VALUE', args_delim), - 'JAR_INCLUDE_FILTER': extract_macro_calls(unit, 'JAR_INCLUDE_FILTER_VALUE', args_delim), - 'JAR_EXCLUDE_FILTER': extract_macro_calls(unit, 'JAR_EXCLUDE_FILTER_VALUE', args_delim), # TODO remove when java test dart is in prod 'UNITTEST_DIR': unit.get('UNITTEST_DIR'), - 'SYSTEM_PROPERTIES': extract_macro_calls(unit, 'SYSTEM_PROPERTIES_VALUE', args_delim), 'JVM_ARGS': extract_macro_calls(unit, 'JVM_ARGS_VALUE', args_delim), 'TEST_CWD': extract_macro_calls(unit, 'TEST_CWD_VALUE', args_delim), 'TEST_FORK_MODE': extract_macro_calls(unit, 'TEST_FORK_MODE', args_delim), @@ -72,9 +68,7 @@ def onjava_module(unit, *args): 'IDEA_EXCLUDE': extract_macro_calls(unit, 'IDEA_EXCLUDE_DIRS_VALUE', args_delim), 'IDEA_RESOURCE': extract_macro_calls(unit, 'IDEA_RESOURCE_DIRS_VALUE', args_delim), 'IDEA_MODULE_NAME': extract_macro_calls(unit, 'IDEA_MODULE_NAME_VALUE', args_delim), - 'FAKEID': extract_macro_calls(unit, 'FAKEID', args_delim), 'TEST_DATA': extract_macro_calls(unit, 'TEST_DATA_VALUE', args_delim), - 'JAVA_FORBIDDEN_LIBRARIES': extract_macro_calls(unit, 'JAVA_FORBIDDEN_LIBRARIES_VALUE', args_delim), 'JDK_RESOURCE': 'JDK' + (unit.get('JDK_VERSION') or unit.get('JDK_REAL_VERSION') or '_DEFAULT'), } if unit.get('ENABLE_PREVIEW_VALUE') == 'yes' and (unit.get('JDK_VERSION') or unit.get('JDK_REAL_VERSION')) in ( diff --git a/build/scripts/go_proto_wrapper.py b/build/scripts/go_proto_wrapper.py index 8f2a6130e05..c5d6c48bd17 100644 --- a/build/scripts/go_proto_wrapper.py +++ b/build/scripts/go_proto_wrapper.py @@ -51,7 +51,6 @@ def main(args): if m: out_dir_flag = m.group(1) index = len(out_dir_flag) - index = max(index, args[i].find(':', index) + 1) out_dir = args[i][index:] if out_dir_orig: assert out_dir_orig == out_dir, 'Output directories do not match: [{}] and [{}]'.format( diff --git a/build/sysincl/emscripten-to-nothing.yml b/build/sysincl/emscripten-to-nothing.yml new file mode 100644 index 00000000000..30edef0e4c5 --- /dev/null +++ b/build/sysincl/emscripten-to-nothing.yml @@ -0,0 +1,4 @@ +- includes: + - emscripten.h + - emscripten/console.h + - emscripten/version.h diff --git a/build/ymake.core.conf b/build/ymake.core.conf index b793c6d0603..41dcbab4f92 100644 --- a/build/ymake.core.conf +++ b/build/ymake.core.conf @@ -14,7 +14,7 @@ GO_FAKEID=11100371 ANDROID_FAKEID=2023-05-17 CLANG_TIDY_FAKEID=2023-06-06 CYTHON_FAKE_ID=10784829 -JAVA_FAKEID=20240627 +JAVA_FAKEID=14386852 PROTO_FAKEID=0 FBS_FAKEID=2024-03-13 @@ -2611,7 +2611,7 @@ macro ADD_COMPILABLE_TRANSLIT(TranslitTable, NGrams, Name, Options...) { RUN_PROGRAM(dict/tools/make_ngrams -i ${NGrams} ${Options} ${__translitname_lower} IN ${NGrams} STDOUT ${__ngrams_table}) } -_COPY_FILE_CONTEXT= +_COPY_FILE_CONTEXT=TEXT macro _COPY_FILE_IMPL(TEXT[], AUTO_DST="", NOAUTO_DST="", OUTPUT_INCLUDES[], INDUCED_DEPS[], OUTPUT_INCLUDES_INP[], FILE...) { .CMD=$COPY_CMD ${input:FILE} ${context=TEXT;input:TEXT} ${output:AUTO_DST} ${noauto;output:NOAUTO_DST} ${hide;output_include:OUTPUT_INCLUDES} ${output_include;from_input;hide:OUTPUT_INCLUDES_INP} $INDUCED_DEPS ${hide;kv:"p CP"} ${hide;kv:"pc light-cyan"} diff --git a/contrib/libs/cxxsupp/libcxx/include/__config b/contrib/libs/cxxsupp/libcxx/include/__config index 9505d9b556f..b5c2fb4f21a 100644 --- a/contrib/libs/cxxsupp/libcxx/include/__config +++ b/contrib/libs/cxxsupp/libcxx/include/__config @@ -49,16 +49,6 @@ # define _LIBCPP_CLANG_VER (__clang_major__ * 100 + __clang_minor__) #elif defined(__GNUC__) # define _LIBCPP_COMPILER_GCC -#elif defined(_MSC_VER) -# define _LIBCPP_COMPILER_MSVC - -# if _MSVC_LANG == 201705L -# define _LIBCPP_STD_VER 20 -# elif _MSVC_LANG == 201703L -# define _LIBCPP_STD_VER 17 -# else -# define _LIBCPP_STD_VER 14 -# endif #endif #ifdef __cplusplus diff --git a/contrib/libs/cxxsupp/libcxx/include/deque b/contrib/libs/cxxsupp/libcxx/include/deque index c830b6d5f74..9559235abf6 100644 --- a/contrib/libs/cxxsupp/libcxx/include/deque +++ b/contrib/libs/cxxsupp/libcxx/include/deque @@ -2429,8 +2429,7 @@ typename deque<_Tp, _Allocator>::iterator deque<_Tp, _Allocator>::erase(const_it template <class _Tp, class _Allocator> typename deque<_Tp, _Allocator>::iterator deque<_Tp, _Allocator>::erase(const_iterator __f, const_iterator __l) { - // temporary disable this check for maps - // _LIBCPP_ASSERT_VALID_INPUT_RANGE(__f <= __l, "deque::erase(first, last) called with an invalid range"); + _LIBCPP_ASSERT_VALID_INPUT_RANGE(__f <= __l, "deque::erase(first, last) called with an invalid range"); size_type __old_sz = size(); size_type __old_start = __start_; difference_type __n = __l - __f; diff --git a/contrib/libs/cxxsupp/libcxx/patches/22__config.patch b/contrib/libs/cxxsupp/libcxx/patches/22__config.patch index 99f19a50c1e..ce7c96782e8 100644 --- a/contrib/libs/cxxsupp/libcxx/patches/22__config.patch +++ b/contrib/libs/cxxsupp/libcxx/patches/22__config.patch @@ -1,8 +1,8 @@ diff --git a/include/__config b/include/__config -index d32ad24..a7d7799 100644 +index d32ad24..1bb99ee 100644 --- a/include/__config +++ b/include/__config -@@ -39,13 +39,35 @@ +@@ -39,13 +39,25 @@ # define _LIBCPP_GCC_VER (__GNUC__ * 100 + __GNUC_MINOR__) #endif @@ -16,16 +16,6 @@ index d32ad24..a7d7799 100644 +# define _LIBCPP_CLANG_VER (__clang_major__ * 100 + __clang_minor__) +#elif defined(__GNUC__) +# define _LIBCPP_COMPILER_GCC -+#elif defined(_MSC_VER) -+# define _LIBCPP_COMPILER_MSVC -+ -+# if _MSVC_LANG == 201705L -+# define _LIBCPP_STD_VER 20 -+# elif _MSVC_LANG == 201703L -+# define _LIBCPP_STD_VER 17 -+# else -+# define _LIBCPP_STD_VER 14 -+# endif +#endif + #ifdef __cplusplus @@ -39,7 +29,7 @@ index d32ad24..a7d7799 100644 # endif # elif defined(_LIBCPP_APPLE_CLANG_VER) # if _LIBCPP_APPLE_CLANG_VER < 1500 -@@ -94,7 +116,7 @@ +@@ -94,7 +106,7 @@ # define _LIBCPP_OBJECT_FORMAT_ELF 1 # elif defined(__MACH__) # define _LIBCPP_OBJECT_FORMAT_MACHO 1 @@ -48,7 +38,7 @@ index d32ad24..a7d7799 100644 # define _LIBCPP_OBJECT_FORMAT_COFF 1 # elif defined(__wasm__) # define _LIBCPP_OBJECT_FORMAT_WASM 1 -@@ -819,7 +841,7 @@ typedef __char32_t char32_t; +@@ -819,7 +831,7 @@ typedef __char32_t char32_t; // TODO: We provide a escape hatch with _LIBCPP_NO_ABI_TAG for folks who want to avoid increasing // the length of symbols with an ABI tag. In practice, we should remove the escape hatch and // use compression mangling instead, see https://github.com/itanium-cxx-abi/cxx-abi/issues/70. @@ -57,7 +47,7 @@ index d32ad24..a7d7799 100644 # define _LIBCPP_HIDE_FROM_ABI \ _LIBCPP_HIDDEN _LIBCPP_EXCLUDE_FROM_EXPLICIT_INSTANTIATION \ __attribute__((__abi_tag__(_LIBCPP_TOSTRING(_LIBCPP_ODR_SIGNATURE)))) -@@ -1101,7 +1123,8 @@ __sanitizer_verify_double_ended_contiguous_container(const void*, const void*, c +@@ -1101,7 +1113,8 @@ __sanitizer_verify_double_ended_contiguous_container(const void*, const void*, c defined(__APPLE__) || \ defined(__MVS__) || \ defined(_AIX) || \ @@ -67,7 +57,7 @@ index d32ad24..a7d7799 100644 // clang-format on # define _LIBCPP_HAS_THREAD_API_PTHREAD # elif defined(__Fuchsia__) -@@ -1267,6 +1290,10 @@ __sanitizer_verify_double_ended_contiguous_container(const void*, const void*, c +@@ -1267,6 +1280,10 @@ __sanitizer_verify_double_ended_contiguous_container(const void*, const void*, c # define _LIBCPP_ENABLE_CXX20_REMOVED_TYPE_TRAITS # endif // _LIBCPP_ENABLE_CXX20_REMOVED_FEATURES @@ -78,7 +68,7 @@ index d32ad24..a7d7799 100644 // clang-format off # define _LIBCPP_PUSH_MACROS _Pragma("push_macro(\"min\")") _Pragma("push_macro(\"max\")") _Pragma("push_macro(\"refresh\")") _Pragma("push_macro(\"move\")") _Pragma("push_macro(\"erase\")") # define _LIBCPP_POP_MACROS _Pragma("pop_macro(\"min\")") _Pragma("pop_macro(\"max\")") _Pragma("pop_macro(\"refresh\")") _Pragma("pop_macro(\"move\")") _Pragma("pop_macro(\"erase\")") -@@ -1452,6 +1479,11 @@ __sanitizer_verify_double_ended_contiguous_container(const void*, const void*, c +@@ -1452,6 +1469,11 @@ __sanitizer_verify_double_ended_contiguous_container(const void*, const void*, c # define _LIBCPP_NO_DESTROY # endif @@ -90,7 +80,7 @@ index d32ad24..a7d7799 100644 # if __has_attribute(__diagnose_if__) && !defined(_LIBCPP_DISABLE_ADDITIONAL_DIAGNOSTICS) # define _LIBCPP_DIAGNOSE_WARNING(...) __attribute__((__diagnose_if__(__VA_ARGS__, "warning"))) # else -@@ -1473,6 +1505,12 @@ __sanitizer_verify_double_ended_contiguous_container(const void*, const void*, c +@@ -1473,6 +1495,12 @@ __sanitizer_verify_double_ended_contiguous_container(const void*, const void*, c # define _LIBCPP_LIFETIMEBOUND # endif diff --git a/contrib/libs/cxxsupp/libcxx/patches/68-asan-contiguous-container.patch b/contrib/libs/cxxsupp/libcxx/patches/68-asan-contiguous-container.patch index cabdbe83665..35cfcb90532 100644 --- a/contrib/libs/cxxsupp/libcxx/patches/68-asan-contiguous-container.patch +++ b/contrib/libs/cxxsupp/libcxx/patches/68-asan-contiguous-container.patch @@ -1,8 +1,8 @@ diff --git a/include/__config b/include/__config -index a7d7799..e78d8e9 100644 +index 1bb99ee..465d784 100644 --- a/include/__config +++ b/include/__config -@@ -1091,10 +1091,12 @@ typedef __char32_t char32_t; +@@ -1081,10 +1081,12 @@ typedef __char32_t char32_t; # ifndef _LIBCPP_HAS_NO_ASAN extern "C" _LIBCPP_EXPORTED_FROM_ABI void __sanitizer_annotate_contiguous_container(const void*, const void*, const void*, const void*); diff --git a/contrib/libs/cxxsupp/libcxx/patches/73-deque-disable-harderning.patch b/contrib/libs/cxxsupp/libcxx/patches/73-deque-disable-harderning.patch deleted file mode 100644 index 99a0a7d17b6..00000000000 --- a/contrib/libs/cxxsupp/libcxx/patches/73-deque-disable-harderning.patch +++ /dev/null @@ -1,14 +0,0 @@ -diff --git a/include/deque b/include/deque -index 9559235..c830b6d 100644 ---- a/include/deque -+++ b/include/deque -@@ -2429,7 +2429,8 @@ typename deque<_Tp, _Allocator>::iterator deque<_Tp, _Allocator>::erase(const_it - - template <class _Tp, class _Allocator> - typename deque<_Tp, _Allocator>::iterator deque<_Tp, _Allocator>::erase(const_iterator __f, const_iterator __l) { -- _LIBCPP_ASSERT_VALID_INPUT_RANGE(__f <= __l, "deque::erase(first, last) called with an invalid range"); -+ // temporary disable this check for maps -+ // _LIBCPP_ASSERT_VALID_INPUT_RANGE(__f <= __l, "deque::erase(first, last) called with an invalid range"); - size_type __old_sz = size(); - size_type __old_start = __start_; - difference_type __n = __l - __f; diff --git a/contrib/libs/cxxsupp/libcxx/patches/xxx-config-epilogue.patch b/contrib/libs/cxxsupp/libcxx/patches/xxx-config-epilogue.patch index e959bd323f9..c9244a9f6b5 100644 --- a/contrib/libs/cxxsupp/libcxx/patches/xxx-config-epilogue.patch +++ b/contrib/libs/cxxsupp/libcxx/patches/xxx-config-epilogue.patch @@ -1,8 +1,8 @@ diff --git a/include/__config b/include/__config -index e78d8e9..9505d9b 100644 +index 465d784..b5c2fb4 100644 --- a/include/__config +++ b/include/__config -@@ -1578,4 +1578,6 @@ __sanitizer_verify_double_ended_contiguous_container(const void*, const void*, c +@@ -1568,4 +1568,6 @@ __sanitizer_verify_double_ended_contiguous_container(const void*, const void*, c #endif // __cplusplus diff --git a/contrib/python/importlib-metadata/py3/.dist-info/METADATA b/contrib/python/importlib-metadata/py3/.dist-info/METADATA index d1ad5b7cddd..b5a602cbff7 100644 --- a/contrib/python/importlib-metadata/py3/.dist-info/METADATA +++ b/contrib/python/importlib-metadata/py3/.dist-info/METADATA @@ -1,6 +1,6 @@ -Metadata-Version: 2.1 +Metadata-Version: 2.2 Name: importlib_metadata -Version: 8.5.0 +Version: 8.6.1 Summary: Read metadata from Python packages Author-email: "Jason R. Coombs" <[email protected]> Project-URL: Source, https://github.com/python/importlib_metadata @@ -9,36 +9,36 @@ Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: Apache Software License Classifier: Programming Language :: Python :: 3 Classifier: Programming Language :: Python :: 3 :: Only -Requires-Python: >=3.8 +Requires-Python: >=3.9 Description-Content-Type: text/x-rst License-File: LICENSE -Requires-Dist: typing-extensions >=3.6.4 ; python_version < "3.8" +Requires-Dist: typing-extensions>=3.6.4; python_version < "3.8" +Provides-Extra: test +Requires-Dist: pytest!=8.1.*,>=6; extra == "test" +Requires-Dist: importlib_resources>=1.3; python_version < "3.9" and extra == "test" +Requires-Dist: packaging; extra == "test" +Requires-Dist: pyfakefs; extra == "test" +Requires-Dist: flufl.flake8; extra == "test" +Requires-Dist: pytest-perf>=0.9.2; extra == "test" +Requires-Dist: jaraco.test>=5.4; extra == "test" +Provides-Extra: doc +Requires-Dist: sphinx>=3.5; extra == "doc" +Requires-Dist: jaraco.packaging>=9.3; extra == "doc" +Requires-Dist: rst.linker>=1.9; extra == "doc" +Requires-Dist: furo; extra == "doc" +Requires-Dist: sphinx-lint; extra == "doc" +Requires-Dist: jaraco.tidelift>=1.4; extra == "doc" +Provides-Extra: perf +Requires-Dist: ipython; extra == "perf" Provides-Extra: check -Requires-Dist: pytest-checkdocs >=2.4 ; extra == 'check' -Requires-Dist: pytest-ruff >=0.2.1 ; (sys_platform != "cygwin") and extra == 'check' +Requires-Dist: pytest-checkdocs>=2.4; extra == "check" +Requires-Dist: pytest-ruff>=0.2.1; sys_platform != "cygwin" and extra == "check" Provides-Extra: cover -Requires-Dist: pytest-cov ; extra == 'cover' -Provides-Extra: doc -Requires-Dist: sphinx >=3.5 ; extra == 'doc' -Requires-Dist: jaraco.packaging >=9.3 ; extra == 'doc' -Requires-Dist: rst.linker >=1.9 ; extra == 'doc' -Requires-Dist: furo ; extra == 'doc' -Requires-Dist: sphinx-lint ; extra == 'doc' -Requires-Dist: jaraco.tidelift >=1.4 ; extra == 'doc' +Requires-Dist: pytest-cov; extra == "cover" Provides-Extra: enabler -Requires-Dist: pytest-enabler >=2.2 ; extra == 'enabler' -Provides-Extra: perf -Requires-Dist: ipython ; extra == 'perf' -Provides-Extra: test -Requires-Dist: pytest !=8.1.*,>=6 ; extra == 'test' -Requires-Dist: packaging ; extra == 'test' -Requires-Dist: pyfakefs ; extra == 'test' -Requires-Dist: flufl.flake8 ; extra == 'test' -Requires-Dist: pytest-perf >=0.9.2 ; extra == 'test' -Requires-Dist: jaraco.test >=5.4 ; extra == 'test' -Requires-Dist: importlib-resources >=1.3 ; (python_version < "3.9") and extra == 'test' +Requires-Dist: pytest-enabler>=2.2; extra == "enabler" Provides-Extra: type -Requires-Dist: pytest-mypy ; extra == 'type' +Requires-Dist: pytest-mypy; extra == "type" .. image:: https://img.shields.io/pypi/v/importlib_metadata.svg :target: https://pypi.org/project/importlib_metadata diff --git a/contrib/python/importlib-metadata/py3/importlib_metadata/_adapters.py b/contrib/python/importlib-metadata/py3/importlib_metadata/_adapters.py index 3b516a2d066..f5b30dd92cd 100644 --- a/contrib/python/importlib-metadata/py3/importlib_metadata/_adapters.py +++ b/contrib/python/importlib-metadata/py3/importlib_metadata/_adapters.py @@ -1,11 +1,58 @@ import email.message +import email.policy import re import textwrap from ._text import FoldedCase +class RawPolicy(email.policy.EmailPolicy): + def fold(self, name, value): + folded = self.linesep.join( + textwrap.indent(value, prefix=' ' * 8, predicate=lambda line: True) + .lstrip() + .splitlines() + ) + return f'{name}: {folded}{self.linesep}' + + class Message(email.message.Message): + r""" + Specialized Message subclass to handle metadata naturally. + + Reads values that may have newlines in them and converts the + payload to the Description. + + >>> msg_text = textwrap.dedent(''' + ... Name: Foo + ... Version: 3.0 + ... License: blah + ... de-blah + ... <BLANKLINE> + ... First line of description. + ... Second line of description. + ... <BLANKLINE> + ... Fourth line! + ... ''').lstrip().replace('<BLANKLINE>', '') + >>> msg = Message(email.message_from_string(msg_text)) + >>> msg['Description'] + 'First line of description.\nSecond line of description.\n\nFourth line!\n' + + Message should render even if values contain newlines. + + >>> print(msg) + Name: Foo + Version: 3.0 + License: blah + de-blah + Description: First line of description. + Second line of description. + <BLANKLINE> + Fourth line! + <BLANKLINE> + <BLANKLINE> + """ + multiple_use_keys = set( map( FoldedCase, @@ -57,15 +104,20 @@ class Message(email.message.Message): def _repair_headers(self): def redent(value): "Correct for RFC822 indentation" - if not value or '\n' not in value: + indent = ' ' * 8 + if not value or '\n' + indent not in value: return value - return textwrap.dedent(' ' * 8 + value) + return textwrap.dedent(indent + value) headers = [(key, redent(value)) for key, value in vars(self)['_headers']] if self._payload: headers.append(('Description', self.get_payload())) + self.set_payload('') return headers + def as_string(self): + return super().as_string(policy=RawPolicy()) + @property def json(self): """ diff --git a/contrib/python/importlib-metadata/py3/patches/01-add-arcadia-support.patch b/contrib/python/importlib-metadata/py3/patches/01-add-arcadia-support.patch index 33e804574e8..4cefb340037 100644 --- a/contrib/python/importlib-metadata/py3/patches/01-add-arcadia-support.patch +++ b/contrib/python/importlib-metadata/py3/patches/01-add-arcadia-support.patch @@ -1,7 +1,7 @@ --- contrib/python/importlib-metadata/py3/.dist-info/METADATA (index) +++ contrib/python/importlib-metadata/py3/.dist-info/METADATA (working tree) @@ -15,1 +15,0 @@ Classifier: License :: OSI Approved :: Apache Software License --Requires-Dist: zipp >=3.20 +-Requires-Dist: zipp>=3.20 --- contrib/python/importlib-metadata/py3/importlib_metadata/__init__.py (index) +++ contrib/python/importlib-metadata/py3/importlib_metadata/__init__.py (working tree) @@ -796,6 +795,59 @@ class PathDistribution(Distribution): diff --git a/contrib/python/importlib-metadata/py3/ya.make b/contrib/python/importlib-metadata/py3/ya.make index c9de3138bda..bd8f1d06d9a 100644 --- a/contrib/python/importlib-metadata/py3/ya.make +++ b/contrib/python/importlib-metadata/py3/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(8.5.0) +VERSION(8.6.1) LICENSE(Apache-2.0) diff --git a/contrib/python/prompt-toolkit/py3/.dist-info/METADATA b/contrib/python/prompt-toolkit/py3/.dist-info/METADATA index eae81f9c1d1..8d4f5d343d2 100644 --- a/contrib/python/prompt-toolkit/py3/.dist-info/METADATA +++ b/contrib/python/prompt-toolkit/py3/.dist-info/METADATA @@ -1,6 +1,6 @@ -Metadata-Version: 2.1 +Metadata-Version: 2.2 Name: prompt_toolkit -Version: 3.0.48 +Version: 3.0.50 Summary: Library for building powerful interactive command lines in Python Home-page: https://github.com/prompt-toolkit/python-prompt-toolkit Author: Jonathan Slenders @@ -9,20 +9,28 @@ Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: BSD License Classifier: Operating System :: OS Independent Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python :: 3.7 Classifier: Programming Language :: Python :: 3.8 Classifier: Programming Language :: Python :: 3.9 Classifier: Programming Language :: Python :: 3.10 Classifier: Programming Language :: Python :: 3.11 Classifier: Programming Language :: Python :: 3.12 +Classifier: Programming Language :: Python :: 3.13 Classifier: Programming Language :: Python :: 3 :: Only Classifier: Programming Language :: Python Classifier: Topic :: Software Development -Requires-Python: >=3.7.0 +Requires-Python: >=3.8.0 Description-Content-Type: text/x-rst License-File: LICENSE License-File: AUTHORS.rst Requires-Dist: wcwidth +Dynamic: author +Dynamic: classifier +Dynamic: description +Dynamic: description-content-type +Dynamic: home-page +Dynamic: requires-dist +Dynamic: requires-python +Dynamic: summary Python Prompt Toolkit ===================== diff --git a/contrib/python/prompt-toolkit/py3/prompt_toolkit/__init__.py b/contrib/python/prompt-toolkit/py3/prompt_toolkit/__init__.py index 80da72d1ecb..94727e7cb22 100644 --- a/contrib/python/prompt-toolkit/py3/prompt_toolkit/__init__.py +++ b/contrib/python/prompt-toolkit/py3/prompt_toolkit/__init__.py @@ -28,7 +28,7 @@ from .formatted_text import ANSI, HTML from .shortcuts import PromptSession, print_formatted_text, prompt # Don't forget to update in `docs/conf.py`! -__version__ = "3.0.48" +__version__ = "3.0.50" assert pep440.match(__version__) diff --git a/contrib/python/prompt-toolkit/py3/prompt_toolkit/application/current.py b/contrib/python/prompt-toolkit/py3/prompt_toolkit/application/current.py index 7e2cf480ba8..3f7eb4bd46c 100644 --- a/contrib/python/prompt-toolkit/py3/prompt_toolkit/application/current.py +++ b/contrib/python/prompt-toolkit/py3/prompt_toolkit/application/current.py @@ -143,8 +143,8 @@ def create_app_session( """ Create a separate AppSession. - This is useful if there can be multiple individual `AppSession`s going on. - Like in the case of an Telnet/SSH server. + This is useful if there can be multiple individual ``AppSession``'s going + on. Like in the case of a Telnet/SSH server. """ # If no input/output is specified, fall back to the current input/output, # if there was one that was set/created for the current session. diff --git a/contrib/python/prompt-toolkit/py3/prompt_toolkit/application/run_in_terminal.py b/contrib/python/prompt-toolkit/py3/prompt_toolkit/application/run_in_terminal.py index 18a3dadeb9c..1f5e18ea786 100644 --- a/contrib/python/prompt-toolkit/py3/prompt_toolkit/application/run_in_terminal.py +++ b/contrib/python/prompt-toolkit/py3/prompt_toolkit/application/run_in_terminal.py @@ -111,4 +111,7 @@ async def in_terminal(render_cli_done: bool = False) -> AsyncGenerator[None, Non app._request_absolute_cursor_position() app._redraw() finally: - new_run_in_terminal_f.set_result(None) + # (Check for `.done()`, because it can be that this future was + # cancelled.) + if not new_run_in_terminal_f.done(): + new_run_in_terminal_f.set_result(None) diff --git a/contrib/python/prompt-toolkit/py3/prompt_toolkit/filters/base.py b/contrib/python/prompt-toolkit/py3/prompt_toolkit/filters/base.py index 410749db475..cd95424dc38 100644 --- a/contrib/python/prompt-toolkit/py3/prompt_toolkit/filters/base.py +++ b/contrib/python/prompt-toolkit/py3/prompt_toolkit/filters/base.py @@ -81,8 +81,7 @@ class Filter(metaclass=ABCMeta): instead of for instance ``filter1 or Always()``. """ raise ValueError( - "The truth value of a Filter is ambiguous. " - "Instead, call it as a function." + "The truth value of a Filter is ambiguous. Instead, call it as a function." ) diff --git a/contrib/python/prompt-toolkit/py3/prompt_toolkit/input/win32.py b/contrib/python/prompt-toolkit/py3/prompt_toolkit/input/win32.py index 322d7c0d72f..1ff3234a398 100644 --- a/contrib/python/prompt-toolkit/py3/prompt_toolkit/input/win32.py +++ b/contrib/python/prompt-toolkit/py3/prompt_toolkit/input/win32.py @@ -16,7 +16,7 @@ if not SPHINX_AUTODOC_RUNNING: import msvcrt from ctypes import windll -from ctypes import Array, pointer +from ctypes import Array, byref, pointer from ctypes.wintypes import DWORD, HANDLE from typing import Callable, ContextManager, Iterable, Iterator, TextIO @@ -35,6 +35,7 @@ from prompt_toolkit.win32_types import ( from .ansi_escape_sequences import REVERSE_ANSI_SEQUENCES from .base import Input +from .vt100_parser import Vt100Parser __all__ = [ "Win32Input", @@ -52,6 +53,9 @@ RIGHTMOST_BUTTON_PRESSED = 0x2 MOUSE_MOVED = 0x0001 MOUSE_WHEELED = 0x0004 +# See: https://msdn.microsoft.com/pl-pl/library/windows/desktop/ms686033(v=vs.85).aspx +ENABLE_VIRTUAL_TERMINAL_INPUT = 0x0200 + class _Win32InputBase(Input): """ @@ -74,7 +78,14 @@ class Win32Input(_Win32InputBase): def __init__(self, stdin: TextIO | None = None) -> None: super().__init__() - self.console_input_reader = ConsoleInputReader() + self._use_virtual_terminal_input = _is_win_vt100_input_enabled() + + self.console_input_reader: Vt100ConsoleInputReader | ConsoleInputReader + + if self._use_virtual_terminal_input: + self.console_input_reader = Vt100ConsoleInputReader() + else: + self.console_input_reader = ConsoleInputReader() def attach(self, input_ready_callback: Callable[[], None]) -> ContextManager[None]: """ @@ -101,7 +112,9 @@ class Win32Input(_Win32InputBase): return False def raw_mode(self) -> ContextManager[None]: - return raw_mode() + return raw_mode( + use_win10_virtual_terminal_input=self._use_virtual_terminal_input + ) def cooked_mode(self) -> ContextManager[None]: return cooked_mode() @@ -555,6 +568,102 @@ class ConsoleInputReader: return [KeyPress(Keys.WindowsMouseEvent, data)] +class Vt100ConsoleInputReader: + """ + Similar to `ConsoleInputReader`, but for usage when + `ENABLE_VIRTUAL_TERMINAL_INPUT` is enabled. This assumes that Windows sends + us the right vt100 escape sequences and we parse those with our vt100 + parser. + + (Using this instead of `ConsoleInputReader` results in the "data" attribute + from the `KeyPress` instances to be more correct in edge cases, because + this responds to for instance the terminal being in application cursor keys + mode.) + """ + + def __init__(self) -> None: + self._fdcon = None + + self._buffer: list[KeyPress] = [] # Buffer to collect the Key objects. + self._vt100_parser = Vt100Parser( + lambda key_press: self._buffer.append(key_press) + ) + + # When stdin is a tty, use that handle, otherwise, create a handle from + # CONIN$. + self.handle: HANDLE + if sys.stdin.isatty(): + self.handle = HANDLE(windll.kernel32.GetStdHandle(STD_INPUT_HANDLE)) + else: + self._fdcon = os.open("CONIN$", os.O_RDWR | os.O_BINARY) + self.handle = HANDLE(msvcrt.get_osfhandle(self._fdcon)) + + def close(self) -> None: + "Close fdcon." + if self._fdcon is not None: + os.close(self._fdcon) + + def read(self) -> Iterable[KeyPress]: + """ + Return a list of `KeyPress` instances. It won't return anything when + there was nothing to read. (This function doesn't block.) + + http://msdn.microsoft.com/en-us/library/windows/desktop/ms684961(v=vs.85).aspx + """ + max_count = 2048 # Max events to read at the same time. + + read = DWORD(0) + arrtype = INPUT_RECORD * max_count + input_records = arrtype() + + # Check whether there is some input to read. `ReadConsoleInputW` would + # block otherwise. + # (Actually, the event loop is responsible to make sure that this + # function is only called when there is something to read, but for some + # reason this happened in the asyncio_win32 loop, and it's better to be + # safe anyway.) + if not wait_for_handles([self.handle], timeout=0): + return [] + + # Get next batch of input event. + windll.kernel32.ReadConsoleInputW( + self.handle, pointer(input_records), max_count, pointer(read) + ) + + # First, get all the keys from the input buffer, in order to determine + # whether we should consider this a paste event or not. + for key_data in self._get_keys(read, input_records): + self._vt100_parser.feed(key_data) + + # Return result. + result = self._buffer + self._buffer = [] + return result + + def _get_keys( + self, read: DWORD, input_records: Array[INPUT_RECORD] + ) -> Iterator[str]: + """ + Generator that yields `KeyPress` objects from the input records. + """ + for i in range(read.value): + ir = input_records[i] + + # Get the right EventType from the EVENT_RECORD. + # (For some reason the Windows console application 'cmder' + # [http://gooseberrycreative.com/cmder/] can return '0' for + # ir.EventType. -- Just ignore that.) + if ir.EventType in EventTypes: + ev = getattr(ir.Event, EventTypes[ir.EventType]) + + # Process if this is a key event. (We also have mouse, menu and + # focus events.) + if isinstance(ev, KEY_EVENT_RECORD) and ev.KeyDown: + u_char = ev.uChar.UnicodeChar + if u_char != "\x00": + yield u_char + + class _Win32Handles: """ Utility to keep track of which handles are connectod to which callbacks. @@ -700,8 +809,11 @@ class raw_mode: `raw_input` method of `.vt100_input`. """ - def __init__(self, fileno: int | None = None) -> None: + def __init__( + self, fileno: int | None = None, use_win10_virtual_terminal_input: bool = False + ) -> None: self.handle = HANDLE(windll.kernel32.GetStdHandle(STD_INPUT_HANDLE)) + self.use_win10_virtual_terminal_input = use_win10_virtual_terminal_input def __enter__(self) -> None: # Remember original mode. @@ -717,12 +829,15 @@ class raw_mode: ENABLE_LINE_INPUT = 0x0002 ENABLE_PROCESSED_INPUT = 0x0001 - windll.kernel32.SetConsoleMode( - self.handle, - self.original_mode.value - & ~(ENABLE_ECHO_INPUT | ENABLE_LINE_INPUT | ENABLE_PROCESSED_INPUT), + new_mode = self.original_mode.value & ~( + ENABLE_ECHO_INPUT | ENABLE_LINE_INPUT | ENABLE_PROCESSED_INPUT ) + if self.use_win10_virtual_terminal_input: + new_mode |= ENABLE_VIRTUAL_TERMINAL_INPUT + + windll.kernel32.SetConsoleMode(self.handle, new_mode) + def __exit__(self, *a: object) -> None: # Restore original mode windll.kernel32.SetConsoleMode(self.handle, self.original_mode) @@ -747,3 +862,25 @@ class cooked_mode(raw_mode): self.original_mode.value | (ENABLE_ECHO_INPUT | ENABLE_LINE_INPUT | ENABLE_PROCESSED_INPUT), ) + + +def _is_win_vt100_input_enabled() -> bool: + """ + Returns True when we're running Windows and VT100 escape sequences are + supported. + """ + hconsole = HANDLE(windll.kernel32.GetStdHandle(STD_INPUT_HANDLE)) + + # Get original console mode. + original_mode = DWORD(0) + windll.kernel32.GetConsoleMode(hconsole, byref(original_mode)) + + try: + # Try to enable VT100 sequences. + result: int = windll.kernel32.SetConsoleMode( + hconsole, DWORD(ENABLE_VIRTUAL_TERMINAL_INPUT) + ) + + return result == 1 + finally: + windll.kernel32.SetConsoleMode(hconsole, original_mode) diff --git a/contrib/python/prompt-toolkit/py3/prompt_toolkit/layout/controls.py b/contrib/python/prompt-toolkit/py3/prompt_toolkit/layout/controls.py index 222e471c572..5083c8286d4 100644 --- a/contrib/python/prompt-toolkit/py3/prompt_toolkit/layout/controls.py +++ b/contrib/python/prompt-toolkit/py3/prompt_toolkit/layout/controls.py @@ -667,7 +667,11 @@ class BufferControl(UIControl): merged_processor = merge_processors(input_processors) - def transform(lineno: int, fragments: StyleAndTextTuples) -> _ProcessedLine: + def transform( + lineno: int, + fragments: StyleAndTextTuples, + get_line: Callable[[int], StyleAndTextTuples], + ) -> _ProcessedLine: "Transform the fragments for a given line number." # Get cursor position at this line. @@ -679,7 +683,14 @@ class BufferControl(UIControl): transformation = merged_processor.apply_transformation( TransformationInput( - self, document, lineno, source_to_display, fragments, width, height + self, + document, + lineno, + source_to_display, + fragments, + width, + height, + get_line, ) ) @@ -697,7 +708,7 @@ class BufferControl(UIControl): try: return cache[i] except KeyError: - processed_line = transform(i, get_line(i)) + processed_line = transform(i, get_line(i), get_line) cache[i] = processed_line return processed_line diff --git a/contrib/python/prompt-toolkit/py3/prompt_toolkit/layout/processors.py b/contrib/python/prompt-toolkit/py3/prompt_toolkit/layout/processors.py index b10ecf71844..666e79c66da 100644 --- a/contrib/python/prompt-toolkit/py3/prompt_toolkit/layout/processors.py +++ b/contrib/python/prompt-toolkit/py3/prompt_toolkit/layout/processors.py @@ -86,6 +86,9 @@ class TransformationInput: previous processors into account.) :param fragments: List of fragments that we can transform. (Received from the previous processor.) + :param get_line: Optional ; a callable that returns the fragments of another + line in the current buffer; This can be used to create processors capable + of affecting transforms across multiple lines. """ def __init__( @@ -97,6 +100,7 @@ class TransformationInput: fragments: StyleAndTextTuples, width: int, height: int, + get_line: Callable[[int], StyleAndTextTuples] | None = None, ) -> None: self.buffer_control = buffer_control self.document = document @@ -105,6 +109,7 @@ class TransformationInput: self.fragments = fragments self.width = width self.height = height + self.get_line = get_line def unpack( self, @@ -842,9 +847,9 @@ class ReverseSearchProcessor(Processor): def apply_transformation(self, ti: TransformationInput) -> Transformation: from .controls import SearchBufferControl - assert isinstance( - ti.buffer_control, SearchBufferControl - ), "`ReverseSearchProcessor` should be applied to a `SearchBufferControl` only." + assert isinstance(ti.buffer_control, SearchBufferControl), ( + "`ReverseSearchProcessor` should be applied to a `SearchBufferControl` only." + ) source_to_display: SourceToDisplay | None display_to_source: DisplayToSource | None @@ -987,6 +992,7 @@ class _MergedProcessor(Processor): fragments, ti.width, ti.height, + ti.get_line, ) ) fragments = transformation.fragments diff --git a/contrib/python/prompt-toolkit/py3/prompt_toolkit/output/vt100.py b/contrib/python/prompt-toolkit/py3/prompt_toolkit/output/vt100.py index 069636b8c30..90df21e558c 100644 --- a/contrib/python/prompt-toolkit/py3/prompt_toolkit/output/vt100.py +++ b/contrib/python/prompt-toolkit/py3/prompt_toolkit/output/vt100.py @@ -436,6 +436,11 @@ class Vt100_Output(Output): # default, we don't change them.) self._cursor_shape_changed = False + # Don't hide/show the cursor when this was already done. + # (`None` means that we don't know whether the cursor is visible or + # not.) + self._cursor_visible: bool | None = None + @classmethod def from_pty( cls, @@ -651,10 +656,14 @@ class Vt100_Output(Output): self.write_raw("\x1b[%iD" % amount) def hide_cursor(self) -> None: - self.write_raw("\x1b[?25l") + if self._cursor_visible in (True, None): + self._cursor_visible = False + self.write_raw("\x1b[?25l") def show_cursor(self) -> None: - self.write_raw("\x1b[?12l\x1b[?25h") # Stop blinking cursor and show. + if self._cursor_visible in (False, None): + self._cursor_visible = True + self.write_raw("\x1b[?12l\x1b[?25h") # Stop blinking cursor and show. def set_cursor_shape(self, cursor_shape: CursorShape) -> None: if cursor_shape == CursorShape._NEVER_CHANGE: diff --git a/contrib/python/prompt-toolkit/py3/prompt_toolkit/output/windows10.py b/contrib/python/prompt-toolkit/py3/prompt_toolkit/output/windows10.py index c39f3ecfd19..2b7e596e468 100644 --- a/contrib/python/prompt-toolkit/py3/prompt_toolkit/output/windows10.py +++ b/contrib/python/prompt-toolkit/py3/prompt_toolkit/output/windows10.py @@ -66,15 +66,20 @@ class Windows10_Output: return False # We don't need this on Windows. def __getattr__(self, name: str) -> Any: + # NOTE: Now that we use "virtual terminal input" on + # Windows, both input and output are done through + # ANSI escape sequences on Windows. This means, we + # should enable bracketed paste like on Linux, and + # enable mouse support by calling the vt100_output. if name in ( "get_size", "get_rows_below_cursor_position", - "enable_mouse_support", - "disable_mouse_support", "scroll_buffer_to_prompt", "get_win32_screen_buffer_info", - "enable_bracketed_paste", - "disable_bracketed_paste", + # "enable_mouse_support", + # "disable_mouse_support", + # "enable_bracketed_paste", + # "disable_bracketed_paste", ): return getattr(self.win32_output, name) else: diff --git a/contrib/python/prompt-toolkit/py3/prompt_toolkit/renderer.py b/contrib/python/prompt-toolkit/py3/prompt_toolkit/renderer.py index 3f92303a819..8d5e03c19e8 100644 --- a/contrib/python/prompt-toolkit/py3/prompt_toolkit/renderer.py +++ b/contrib/python/prompt-toolkit/py3/prompt_toolkit/renderer.py @@ -257,7 +257,7 @@ def _output_screen_diff( # give weird artifacts on resize events.) reset_attributes() - if screen.show_cursor or is_done: + if screen.show_cursor: output.show_cursor() return current_pos, last_style @@ -353,6 +353,11 @@ class Renderer: self.mouse_support = to_filter(mouse_support) self.cpr_not_supported_callback = cpr_not_supported_callback + # TODO: Move following state flags into `Vt100_Output`, similar to + # `_cursor_shape_changed` and `_cursor_visible`. But then also + # adjust the `Win32Output` to not call win32 APIs if nothing has + # to be changed. + self._in_alternate_screen = False self._mouse_support_enabled = False self._bracketed_paste_enabled = False @@ -416,6 +421,7 @@ class Renderer: self._bracketed_paste_enabled = False self.output.reset_cursor_shape() + self.output.show_cursor() # NOTE: No need to set/reset cursor key mode here. diff --git a/contrib/python/prompt-toolkit/py3/tests/test_cli.py b/contrib/python/prompt-toolkit/py3/tests/test_cli.py index c155325f981..a876f2993b2 100644 --- a/contrib/python/prompt-toolkit/py3/tests/test_cli.py +++ b/contrib/python/prompt-toolkit/py3/tests/test_cli.py @@ -870,11 +870,11 @@ def test_vi_temp_navigation_mode(): """ feed = partial(_feed_cli_with_input, editing_mode=EditingMode.VI) - result, cli = feed("abcde" "\x0f" "3h" "x\r") # c-o # 3 times to the left. + result, cli = feed("abcde\x0f3hx\r") # c-o # 3 times to the left. assert result.text == "axbcde" assert result.cursor_position == 2 - result, cli = feed("abcde" "\x0f" "b" "x\r") # c-o # One word backwards. + result, cli = feed("abcde\x0fbx\r") # c-o # One word backwards. assert result.text == "xabcde" assert result.cursor_position == 1 diff --git a/contrib/python/prompt-toolkit/py3/ya.make b/contrib/python/prompt-toolkit/py3/ya.make index b1cb3e19b0b..5eed9c2519b 100644 --- a/contrib/python/prompt-toolkit/py3/ya.make +++ b/contrib/python/prompt-toolkit/py3/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(3.0.48) +VERSION(3.0.50) LICENSE(BSD-3-Clause) diff --git a/contrib/restricted/abseil-cpp-tstring/y_absl/base/config.h b/contrib/restricted/abseil-cpp-tstring/y_absl/base/config.h index dce0ab09a5a..19b7b71c7a7 100644 --- a/contrib/restricted/abseil-cpp-tstring/y_absl/base/config.h +++ b/contrib/restricted/abseil-cpp-tstring/y_absl/base/config.h @@ -975,7 +975,7 @@ static_assert(Y_ABSL_INTERNAL_INLINE_NAMESPACE_STR[0] != 'h' || #error Y_ABSL_INTERNAL_EMSCRIPTEN_VERSION cannot be directly set #endif #ifdef __EMSCRIPTEN__ -#error #include <emscripten/version.h> +#include <emscripten/version.h> #ifdef __EMSCRIPTEN_major__ #if __EMSCRIPTEN_minor__ >= 1000 #error __EMSCRIPTEN_minor__ is too big to fit in Y_ABSL_INTERNAL_EMSCRIPTEN_VERSION diff --git a/contrib/restricted/abseil-cpp-tstring/y_absl/base/internal/raw_logging.cc b/contrib/restricted/abseil-cpp-tstring/y_absl/base/internal/raw_logging.cc index 7ddf743de3f..75534f157ba 100644 --- a/contrib/restricted/abseil-cpp-tstring/y_absl/base/internal/raw_logging.cc +++ b/contrib/restricted/abseil-cpp-tstring/y_absl/base/internal/raw_logging.cc @@ -22,7 +22,7 @@ #include <util/generic/string.h> #ifdef __EMSCRIPTEN__ -#error #include <emscripten/console.h> +#include <emscripten/console.h> #endif #include "y_absl/base/attributes.h" diff --git a/contrib/restricted/abseil-cpp-tstring/y_absl/debugging/internal/stacktrace_emscripten-inl.inc b/contrib/restricted/abseil-cpp-tstring/y_absl/debugging/internal/stacktrace_emscripten-inl.inc index ea86e5b0e10..e1c46233acc 100644 --- a/contrib/restricted/abseil-cpp-tstring/y_absl/debugging/internal/stacktrace_emscripten-inl.inc +++ b/contrib/restricted/abseil-cpp-tstring/y_absl/debugging/internal/stacktrace_emscripten-inl.inc @@ -20,7 +20,7 @@ #ifndef Y_ABSL_DEBUGGING_INTERNAL_STACKTRACE_EMSCRIPTEN_INL_H_ #define Y_ABSL_DEBUGGING_INTERNAL_STACKTRACE_EMSCRIPTEN_INL_H_ -#error #include <emscripten.h> +#include <emscripten.h> #include <atomic> #include <cstring> diff --git a/contrib/restricted/abseil-cpp-tstring/y_absl/debugging/symbolize_emscripten.inc b/contrib/restricted/abseil-cpp-tstring/y_absl/debugging/symbolize_emscripten.inc index 558160f9705..c6f07f1a5e0 100644 --- a/contrib/restricted/abseil-cpp-tstring/y_absl/debugging/symbolize_emscripten.inc +++ b/contrib/restricted/abseil-cpp-tstring/y_absl/debugging/symbolize_emscripten.inc @@ -13,7 +13,7 @@ // limitations under the License. #include <cxxabi.h> -#error #include <emscripten.h> +#include <emscripten.h> #include <algorithm> #include <cstring> diff --git a/contrib/restricted/abseil-cpp-tstring/y_absl/log/internal/globals.cc b/contrib/restricted/abseil-cpp-tstring/y_absl/log/internal/globals.cc index 5ee68c2f45a..84ec102d443 100644 --- a/contrib/restricted/abseil-cpp-tstring/y_absl/log/internal/globals.cc +++ b/contrib/restricted/abseil-cpp-tstring/y_absl/log/internal/globals.cc @@ -18,7 +18,7 @@ #include <cstdio> #if defined(__EMSCRIPTEN__) -#error #include <emscripten/console.h> +#include <emscripten/console.h> #endif #include "y_absl/base/attributes.h" diff --git a/contrib/restricted/abseil-cpp/.yandex_meta/__init__.py b/contrib/restricted/abseil-cpp/.yandex_meta/__init__.py index e7c7f6d2550..8ffd340d627 100644 --- a/contrib/restricted/abseil-cpp/.yandex_meta/__init__.py +++ b/contrib/restricted/abseil-cpp/.yandex_meta/__init__.py @@ -102,8 +102,6 @@ abseil_cpp = CMakeNinjaNixProject( arcdir="contrib/restricted/abseil-cpp", nixattr="abseil-cpp", disable_includes=[ - "emscripten.h", - "emscripten/*.h", # if defined(__myriad2__) "rtems.h", # if defined(__Fuchsia__) diff --git a/contrib/restricted/abseil-cpp/absl/base/config.h b/contrib/restricted/abseil-cpp/absl/base/config.h index 10747bc5f78..46e6fbcb9b2 100644 --- a/contrib/restricted/abseil-cpp/absl/base/config.h +++ b/contrib/restricted/abseil-cpp/absl/base/config.h @@ -968,7 +968,7 @@ static_assert(ABSL_INTERNAL_INLINE_NAMESPACE_STR[0] != 'h' || #error ABSL_INTERNAL_EMSCRIPTEN_VERSION cannot be directly set #endif #ifdef __EMSCRIPTEN__ -#error #include <emscripten/version.h> +#include <emscripten/version.h> #ifdef __EMSCRIPTEN_major__ #if __EMSCRIPTEN_minor__ >= 1000 #error __EMSCRIPTEN_minor__ is too big to fit in ABSL_INTERNAL_EMSCRIPTEN_VERSION diff --git a/contrib/restricted/abseil-cpp/absl/base/internal/raw_logging.cc b/contrib/restricted/abseil-cpp/absl/base/internal/raw_logging.cc index 6f68002a3cb..d32b40a8bb7 100644 --- a/contrib/restricted/abseil-cpp/absl/base/internal/raw_logging.cc +++ b/contrib/restricted/abseil-cpp/absl/base/internal/raw_logging.cc @@ -22,7 +22,7 @@ #include <string> #ifdef __EMSCRIPTEN__ -#error #include <emscripten/console.h> +#include <emscripten/console.h> #endif #include "absl/base/attributes.h" diff --git a/contrib/restricted/abseil-cpp/absl/debugging/internal/stacktrace_emscripten-inl.inc b/contrib/restricted/abseil-cpp/absl/debugging/internal/stacktrace_emscripten-inl.inc index 16522102f64..0f444514386 100644 --- a/contrib/restricted/abseil-cpp/absl/debugging/internal/stacktrace_emscripten-inl.inc +++ b/contrib/restricted/abseil-cpp/absl/debugging/internal/stacktrace_emscripten-inl.inc @@ -20,7 +20,7 @@ #ifndef ABSL_DEBUGGING_INTERNAL_STACKTRACE_EMSCRIPTEN_INL_H_ #define ABSL_DEBUGGING_INTERNAL_STACKTRACE_EMSCRIPTEN_INL_H_ -#error #include <emscripten.h> +#include <emscripten.h> #include <atomic> #include <cstring> diff --git a/contrib/restricted/abseil-cpp/absl/debugging/symbolize_emscripten.inc b/contrib/restricted/abseil-cpp/absl/debugging/symbolize_emscripten.inc index 925cbb9b2ac..a0f344dd9b5 100644 --- a/contrib/restricted/abseil-cpp/absl/debugging/symbolize_emscripten.inc +++ b/contrib/restricted/abseil-cpp/absl/debugging/symbolize_emscripten.inc @@ -13,7 +13,7 @@ // limitations under the License. #include <cxxabi.h> -#error #include <emscripten.h> +#include <emscripten.h> #include <algorithm> #include <cstring> diff --git a/contrib/restricted/abseil-cpp/absl/log/internal/globals.cc b/contrib/restricted/abseil-cpp/absl/log/internal/globals.cc index 47896923108..359858f15ef 100644 --- a/contrib/restricted/abseil-cpp/absl/log/internal/globals.cc +++ b/contrib/restricted/abseil-cpp/absl/log/internal/globals.cc @@ -18,7 +18,7 @@ #include <cstdio> #if defined(__EMSCRIPTEN__) -#error #include <emscripten/console.h> +#include <emscripten/console.h> #endif #include "absl/base/attributes.h" diff --git a/library/cpp/yt/memory/shared_range.h b/library/cpp/yt/memory/shared_range.h index 113378670a1..ffe28a7094c 100644 --- a/library/cpp/yt/memory/shared_range.h +++ b/library/cpp/yt/memory/shared_range.h @@ -58,7 +58,7 @@ public: //! Constructs an empty TSharedRange from a nullptr expression. TSharedRange(std::nullptr_t) - : TRange<T>(nullptr, 0UL) + : TRange<T>(nullptr, static_cast<size_t>(0)) , Holder_(nullptr) { } diff --git a/util/memory/pool.h b/util/memory/pool.h index a7f09cd37b0..c16c3b47fce 100644 --- a/util/memory/pool.h +++ b/util/memory/pool.h @@ -131,6 +131,29 @@ public: } }; + // When a bookmark is destroyed, the memory pool returns to the state when the bookmark was created. + class TBookmark { + public: + inline TBookmark(TMemoryPool& memoryPool) + : OwnerPoolRef_(memoryPool) + , BookmarkChunk_(memoryPool.Current_) + , BookmarkChunkDataSize_(memoryPool.Current_->DataSize()) + { + } + + inline ~TBookmark() { + OwnerPoolRef_.Current_->ResetChunk(); + if (OwnerPoolRef_.Current_ == BookmarkChunk_) { + Y_UNUSED(BookmarkChunk_->Allocate(BookmarkChunkDataSize_)); + } + } + + private: + TMemoryPool& OwnerPoolRef_; + TMemoryPool::TChunk* BookmarkChunk_; + size_t BookmarkChunkDataSize_; + }; + inline TMemoryPool(size_t initial, IGrowPolicy* grow = TExpGrow::Instance(), IAllocator* alloc = TDefaultAllocator::Instance(), const TOptions& options = TOptions()) : Current_(&Empty_) , BlockSize_(initial) diff --git a/util/memory/pool_ut.cpp b/util/memory/pool_ut.cpp index e7c055f36a2..98dbb082d5f 100644 --- a/util/memory/pool_ut.cpp +++ b/util/memory/pool_ut.cpp @@ -84,6 +84,7 @@ class TMemPoolTest: public TTestBase { UNIT_TEST(TestLargeStartingAlign) UNIT_TEST(TestMoveAlloc) UNIT_TEST(TestRoundUpToNextPowerOfTwoOption) + UNIT_TEST(TestMemoryPoolBookmark) UNIT_TEST_SUITE_END(); private: @@ -280,6 +281,43 @@ private: pool.Allocate(1); UNIT_ASSERT_VALUES_EQUAL(2 * EXPECTED_ALLOCATION_SIZE, allocator.GetSize()); } + + void TestMemoryPoolBookmark() { + TCheckedAllocator alloc; + + { + TMemoryPool pool(200U, TMemoryPool::TExpGrow::Instance(), &alloc); + ui64* someData = pool.Allocate<ui64>(); + static const ui64 TESTING{0x123456789ABCDEF}; + *someData = TESTING; + + const auto ma = pool.MemoryAllocated(); + const auto chunkOverhead = ma - sizeof(ui64); + const auto firstChunkTotal = pool.MemoryWaste() + ma; + + // Allocate some memory in pool but not enough to need new chunks: + { + TMemoryPool::TBookmark bookmarkA(pool); + for (size_t i = 0U; i != 10; ++i) { + UNIT_ASSERT(pool.Allocate<ui64>()); + } + } + UNIT_ASSERT_VALUES_EQUAL(pool.MemoryAllocated(), ma); + UNIT_ASSERT_VALUES_EQUAL(*someData, TESTING); + + // Allocate some memory in pool enough to need a new single chunk: + { + TMemoryPool::TBookmark bookmarkB(pool); + for (size_t i = 0U; i != 50; ++i) { + UNIT_ASSERT(pool.Allocate<ui64>()); + } + } + UNIT_ASSERT_VALUES_EQUAL(pool.MemoryAllocated(), firstChunkTotal + chunkOverhead); // The last (second) chunk is completely free + UNIT_ASSERT_VALUES_EQUAL(*someData, TESTING); + } + + alloc.CheckAtEnd(); + } }; UNIT_TEST_SUITE_REGISTRATION(TMemPoolTest); diff --git a/yql/essentials/ast/yql_type_string.cpp b/yql/essentials/ast/yql_type_string.cpp index 8d38a751c80..fc360d2f90e 100644 --- a/yql/essentials/ast/yql_type_string.cpp +++ b/yql/essentials/ast/yql_type_string.cpp @@ -84,6 +84,13 @@ enum EToken TOKEN_DYNUMBER = -49, TOKEN_SCALAR = -50, TOKEN_BLOCK = -51, + TOKEN_DATE32 = -52, + TOKEN_DATETIME64 = -53, + TOKEN_TIMESTAMP64 = -54, + TOKEN_INTERVAL64 = -55, + TOKEN_TZDATE32 = -56, + TOKEN_TZDATETIME64 = -57, + TOKEN_TZTIMESTAMP64 = -58, // identifiers TOKEN_IDENTIFIER = -100, @@ -148,6 +155,13 @@ EToken TokenTypeFromStr(TStringBuf str) { TStringBuf("DyNumber"), TOKEN_DYNUMBER }, { TStringBuf("Block"), TOKEN_BLOCK}, { TStringBuf("Scalar"), TOKEN_SCALAR}, + { TStringBuf("Date32"), TOKEN_DATE32 }, + { TStringBuf("Datetime64"), TOKEN_DATETIME64}, + { TStringBuf("Timestamp64"), TOKEN_TIMESTAMP64 }, + { TStringBuf("Interval64"), TOKEN_INTERVAL64 }, + { TStringBuf("TzDate32"), TOKEN_TZDATE32 }, + { TStringBuf("TzDatetime64"), TOKEN_TZDATETIME64}, + { TStringBuf("TzTimestamp64"), TOKEN_TZTIMESTAMP64 }, }; auto it = map.find(str); @@ -218,6 +232,13 @@ private: case TOKEN_UUID: case TOKEN_JSON_DOCUMENT: case TOKEN_DYNUMBER: + case TOKEN_DATE32: + case TOKEN_DATETIME64: + case TOKEN_TIMESTAMP64: + case TOKEN_INTERVAL64: + case TOKEN_TZDATE32: + case TOKEN_TZDATETIME64: + case TOKEN_TZTIMESTAMP64: type = MakeDataType(Identifier); GetNextToken(); break; diff --git a/yql/essentials/ast/yql_type_string_ut.cpp b/yql/essentials/ast/yql_type_string_ut.cpp index 676f6c23972..2d20e05fc7b 100644 --- a/yql/essentials/ast/yql_type_string_ut.cpp +++ b/yql/essentials/ast/yql_type_string_ut.cpp @@ -54,9 +54,16 @@ Y_UNIT_TEST_SUITE(TTypeString) TestOk("Datetime", "(DataType 'Datetime)"); TestOk("Timestamp", "(DataType 'Timestamp)"); TestOk("Interval", "(DataType 'Interval)"); + TestOk("Date32", "(DataType 'Date32)"); + TestOk("Datetime64", "(DataType 'Datetime64)"); + TestOk("Timestamp64", "(DataType 'Timestamp64)"); + TestOk("Interval64", "(DataType 'Interval64)"); TestOk("TzDate", "(DataType 'TzDate)"); TestOk("TzDatetime", "(DataType 'TzDatetime)"); TestOk("TzTimestamp", "(DataType 'TzTimestamp)"); + TestOk("TzDate32", "(DataType 'TzDate32)"); + TestOk("TzDatetime64", "(DataType 'TzDatetime64)"); + TestOk("TzTimestamp64", "(DataType 'TzTimestamp64)"); TestOk("Uuid", "(DataType 'Uuid)"); TestOk("Decimal(10,2)", "(DataType 'Decimal '10 '2)"); } diff --git a/yql/essentials/core/common_opt/yql_co_flow2.cpp b/yql/essentials/core/common_opt/yql_co_flow2.cpp index ec1ea8f6488..0a39c3a266c 100644 --- a/yql/essentials/core/common_opt/yql_co_flow2.cpp +++ b/yql/essentials/core/common_opt/yql_co_flow2.cpp @@ -18,8 +18,8 @@ using namespace NNodes; bool AllowSubsetFieldsForNode(const TExprNode& node, const TOptimizeContext& optCtx) { YQL_ENSURE(optCtx.Types); - static const TString multiUsageFlags = to_lower(TString("FieldSubsetEnableMultiusage")); - return optCtx.IsSingleUsage(node) || optCtx.Types->OptimizerFlags.contains(multiUsageFlags); + static const char flag[] = "FieldSubsetEnableMultiusage"; + return !IsOptimizerDisabled<flag>(*optCtx.Types) || optCtx.IsSingleUsage(node); } bool AllowComplexFiltersOverAggregatePushdown(const TOptimizeContext& optCtx) { diff --git a/yql/essentials/core/type_ann/type_ann_join.cpp b/yql/essentials/core/type_ann/type_ann_join.cpp index 498237b3561..78c1c89f282 100644 --- a/yql/essentials/core/type_ann/type_ann_join.cpp +++ b/yql/essentials/core/type_ann/type_ann_join.cpp @@ -1006,9 +1006,9 @@ namespace NTypeAnnImpl { } const auto joinKind = input->Child(2)->Content(); - if (joinKind != "Inner" && joinKind != "Left" && joinKind != "LeftSemi" && joinKind != "LeftOnly") { + if (joinKind != "Inner" && joinKind != "Left" && joinKind != "LeftSemi" && joinKind != "LeftOnly"&& joinKind != "Cross") { ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(2)->Pos()), TStringBuilder() << "Unknown join kind: " << joinKind - << ", supported: Inner, Left, LeftSemi, LeftOnly")); + << ", supported: Inner, Left, LeftSemi, LeftOnly, Cross")); return IGraphTransformer::TStatus::Error; } @@ -1018,6 +1018,10 @@ namespace NTypeAnnImpl { } auto checkKeyColumns = [&](std::unordered_set<ui32>& keyColumns, bool isLeft, const TExprNode& keyColumnsNode, const TMultiExprType* itemType) { + if (joinKind == "Cross" && !keyColumnsNode.Children().empty()) { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(keyColumnsNode.Pos()), "Specifying key columns is not allowed for cross join")); + return false; + } for (const auto& keyColumnNode : keyColumnsNode.Children()) { auto position = GetWideBlockFieldPosition(*itemType, keyColumnNode->Content()); if (!position) { @@ -1030,6 +1034,10 @@ namespace NTypeAnnImpl { }; auto checkKeyDrops = [&](std::unordered_set<ui32>& keyDrops, bool isLeft, const std::unordered_set<ui32>& keyColumns, const TExprNode& keyDropsNode, const TMultiExprType* itemType) { + if (joinKind == "Cross" && !keyDropsNode.Children().empty()) { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(keyDropsNode.Pos()), "Specifying key drops is not allowed for cross join")); + return false; + } for (const auto& keyDropNode : keyDropsNode.Children()) { auto position = GetWideBlockFieldPosition(*itemType, keyDropNode->Content()); if (!position) { diff --git a/yql/essentials/minikql/comp_nodes/mkql_grace_join.cpp b/yql/essentials/minikql/comp_nodes/mkql_grace_join.cpp index 0a511e61ad7..9af653b9027 100644 --- a/yql/essentials/minikql/comp_nodes/mkql_grace_join.cpp +++ b/yql/essentials/minikql/comp_nodes/mkql_grace_join.cpp @@ -755,10 +755,6 @@ private: } } - if (resultLeft == EFetchResult::Yield || resultRight == EFetchResult::Yield) { - return EFetchResult::Yield; - } - if (resultLeft == EFetchResult::Finish ) { *HaveMoreLeftRows = false; } @@ -768,6 +764,14 @@ private: *HaveMoreRightRows = false; } + if (!*HaveMoreLeftRows && !*HaveMoreRightRows) { + return EFetchResult::Finish; + } + + if ((resultLeft == EFetchResult::Yield || !*HaveMoreLeftRows) && (resultRight == EFetchResult::Yield || !*HaveMoreRightRows)) { + return EFetchResult::Yield; + } + return EFetchResult::Finish; } diff --git a/yql/essentials/sql/v1/format/sql_format.cpp b/yql/essentials/sql/v1/format/sql_format.cpp index 19619bf6e89..1240e073537 100644 --- a/yql/essentials/sql/v1/format/sql_format.cpp +++ b/yql/essentials/sql/v1/format/sql_format.cpp @@ -2671,10 +2671,11 @@ private: } void VisitNeqSubexpr(const TRule_neq_subexpr& msg) { - VisitNeqSubexprImpl(msg, false); + bool pushedIndent = false; + VisitNeqSubexprImpl(msg, pushedIndent, true); } - void VisitNeqSubexprImpl(const TRule_neq_subexpr& msg, bool pushedIndent) { + void VisitNeqSubexprImpl(const TRule_neq_subexpr& msg, bool& pushedIndent, bool top) { auto getExpr = [](const TRule_neq_subexpr::TBlock2& b) -> const TRule_bit_subexpr& { return b.GetRule_bit_subexpr2(); }; auto getOp = [](const TRule_neq_subexpr::TBlock2& b) -> const TRule_neq_subexpr::TBlock2::TBlock1& { return b.GetBlock1(); }; VisitBinaryOp(msg.GetRule_bit_subexpr1(), getOp, getExpr, msg.GetBlock2().begin(), msg.GetBlock2().end()); @@ -2706,9 +2707,10 @@ private: } } - VisitNeqSubexprImpl(alt.GetRule_neq_subexpr2(), pushedIndent); - if (pushedIndent) { + VisitNeqSubexprImpl(alt.GetRule_neq_subexpr2(), pushedIndent, false); + if (pushedIndent && top) { PopCurrentIndent(); + pushedIndent = false; } break; diff --git a/yql/essentials/sql/v1/format/sql_format_ut.h b/yql/essentials/sql/v1/format/sql_format_ut.h index b22bf1cb21c..b7fbc66f937 100644 --- a/yql/essentials/sql/v1/format/sql_format_ut.h +++ b/yql/essentials/sql/v1/format/sql_format_ut.h @@ -1632,6 +1632,14 @@ Y_UNIT_TEST(OperatorNewlines) { "$x = 1\n\t>>|\n\t2;\n"}, {"$x = 1\n?? 2 ??\n3\n??\n4 +\n5\n*\n6 +\n7 ??\n8;", "$x = 1 ??\n\t2 ??\n\t3\n\t??\n\t4\n\t+ 5\n\t*\n\t6\n\t+ 7 ??\n\t8;\n"}, + {"select 1 ??\n2 ?? 3,\n4;", + "SELECT\n\t1 ??\n\t\t2 ?? 3,\n\t4\n;\n"}, + {"select 1\n?? 2 ?? 3,\n4;", + "SELECT\n\t1 ??\n\t\t2 ?? 3,\n\t4\n;\n"}, + {"select 1\n?? 2 ??\n3 ?? 4,\n5;", + "SELECT\n\t1 ??\n\t\t2 ??\n\t\t3 ?? 4,\n\t5\n;\n"}, + {"select 1\n?? 2 ?? 3 ??\n4 ?? 5,\n6;", + "SELECT\n\t1 ??\n\t\t2 ?? 3 ??\n\t\t4 ?? 5,\n\t6\n;\n"}, }; TSetup setup; diff --git a/yt/cpp/mapreduce/http_client/rpc_parameters_serialization.cpp b/yt/cpp/mapreduce/http_client/rpc_parameters_serialization.cpp index c986e4c025b..413a09ab978 100644 --- a/yt/cpp/mapreduce/http_client/rpc_parameters_serialization.cpp +++ b/yt/cpp/mapreduce/http_client/rpc_parameters_serialization.cpp @@ -555,6 +555,9 @@ TNode SerializeParamsForListJobs( if (options.WithMonitoringDescriptor_) { result["with_monitoring_descriptor"] = *options.WithMonitoringDescriptor_; } + if (options.OperationIncarnation_) { + result["operation_incarnation"] = *options.OperationIncarnation_; + } if (options.FromTime_) { result["from_time"] = ToString(options.FromTime_); } diff --git a/yt/cpp/mapreduce/interface/operation.h b/yt/cpp/mapreduce/interface/operation.h index aa109575a8a..218ead0572d 100644 --- a/yt/cpp/mapreduce/interface/operation.h +++ b/yt/cpp/mapreduce/interface/operation.h @@ -2880,6 +2880,10 @@ struct TListJobsOptions FLUENT_FIELD_OPTION(bool, WithMonitoringDescriptor); /// + /// @brief Return only jobs with given operation incarnation. + FLUENT_FIELD_OPTION(TString, OperationIncarnation); + + /// /// @brief Search for jobs with start time >= `FromTime`. FLUENT_FIELD_OPTION(TInstant, FromTime); diff --git a/yt/yql/providers/yt/gateway/fmr/ya.make b/yt/yql/providers/yt/gateway/fmr/ya.make new file mode 100644 index 00000000000..4f805b1aca7 --- /dev/null +++ b/yt/yql/providers/yt/gateway/fmr/ya.make @@ -0,0 +1,14 @@ +LIBRARY() + +SRCS( + yql_yt_fmr.cpp +) + +PEERDIR( + yql/essentials/utils/log + yt/yql/providers/yt/provider +) + +YQL_LAST_ABI_VERSION() + +END() diff --git a/yt/yql/providers/yt/gateway/fmr/yql_yt_fmr.cpp b/yt/yql/providers/yt/gateway/fmr/yql_yt_fmr.cpp new file mode 100644 index 00000000000..81d42245304 --- /dev/null +++ b/yt/yql/providers/yt/gateway/fmr/yql_yt_fmr.cpp @@ -0,0 +1,27 @@ +#include "yql_yt_fmr.h" + +#include <yql/essentials/utils/log/profile.h> + +#include <util/generic/ptr.h> + +using namespace NThreading; + +namespace NYql { + +namespace { + +class TFmrYtGateway final: public TYtForwardingGatewayBase { +public: + TFmrYtGateway(IYtGateway::TPtr&& slave) + : TYtForwardingGatewayBase(std::move(slave)) + { + } +}; + +} // namespace + +IYtGateway::TPtr CreateYtFmrGateway(IYtGateway::TPtr slave) { + return MakeIntrusive<TFmrYtGateway>(std::move(slave)); +} + +} // namspace NYql diff --git a/yt/yql/providers/yt/gateway/fmr/yql_yt_fmr.h b/yt/yql/providers/yt/gateway/fmr/yql_yt_fmr.h new file mode 100644 index 00000000000..97e26b63b1e --- /dev/null +++ b/yt/yql/providers/yt/gateway/fmr/yql_yt_fmr.h @@ -0,0 +1,9 @@ +#pragma once + +#include <yt/yql/providers/yt/provider/yql_yt_forwarding_gateway.h> + +namespace NYql { + +IYtGateway::TPtr CreateYtFmrGateway(IYtGateway::TPtr slave); + +} // namspace NYql diff --git a/yt/yql/providers/yt/provider/yql_yt_join_impl.cpp b/yt/yql/providers/yt/provider/yql_yt_join_impl.cpp index 5d9949a257d..5002d3cac2a 100644 --- a/yt/yql/providers/yt/provider/yql_yt_join_impl.cpp +++ b/yt/yql/providers/yt/provider/yql_yt_join_impl.cpp @@ -274,8 +274,6 @@ TStatus UpdateInMemorySizeUsingBlocksSetting(TMapJoinSettings& settings, TYtSect const TStructExprType* itemType, const TVector<TString>& joinKeyList, const TYtState::TPtr& state, const TString& cluster, const TVector<TYtPathInfo::TPtr>& tables) { - Y_ENSURE(!op.JoinKind->IsAtom("Cross")); - ui64 dataSize = 0; auto status = CalculateJoinLeafSize(dataSize, settings, inputSection, op, ctx, isLeft, itemType, joinKeyList, state, cluster, tables); if (status != TStatus::Ok) { @@ -2087,46 +2085,56 @@ bool RewriteYtMapJoin(TYtEquiJoin equiJoin, const TJoinLabels& labels, bool isLo } } + if (useBlocks) { + for (auto& [_, columnType] : columnTypes) { + if (!IsSupportedAsBlockType(pos, *columnType, ctx, *state->Types)) { + useBlocks = false; + YQL_CLOG(INFO, ProviderYt) << "Block mapjoin won't be used because of unsupported type: " << *columnType; + break; + } + } + } + TExprNode::TPtr joined; - if (!isCross) { + if (useBlocks) { TExprNode::TListType leftKeyColumnNodes; TExprNode::TListType leftKeyColumnNodesNullable; - auto mapInput = RemapNonConvertibleItems(listArg, mainLabel, *leftKeyColumns, outputKeyType, leftKeyColumnNodes, leftKeyColumnNodesNullable, ctx); - if (useBlocks) { - for (auto& [_, columnType] : columnTypes) { - if (!IsSupportedAsBlockType(pos, *columnType, ctx, *state->Types)) { - useBlocks = false; - YQL_CLOG(INFO, ProviderYt) << "Block mapjoin won't be used because of unsupported type: " << *columnType; - break; - } - } + TExprNode::TPtr mapInput; + if (!isCross) { + mapInput = RemapNonConvertibleItems(listArg, mainLabel, *leftKeyColumns, outputKeyType, leftKeyColumnNodes, leftKeyColumnNodesNullable, ctx); + } else { + YQL_ENSURE(remappedMembers.empty()); + mapInput = listArg; } - if (useBlocks) { - if (!mapJoinUseFlow) { - mapInput = ctx.Builder(pos) - .Callable("ToFlow") - .Add(0, std::move(mapInput)) - .Seal() - .Build(); - } - - tableContent = ctx.Builder(pos) + if (!mapJoinUseFlow) { + mapInput = ctx.Builder(pos) .Callable("ToFlow") - .Add(0, std::move(tableContent)) - .Callable(1, "DependsOn") - .Add(0, listArg) - .Seal() + .Add(0, std::move(mapInput)) .Seal() .Build(); + } - joined = BuildBlockMapJoin(std::move(mapInput), std::move(tableContent), - leftKeyColumnNodes, leftOutputColumns, leftOutputColumnSources, leftUsedSourceColumns, - remappedMembers, rightOutputColumns, rightOutputColumnSources, rightUsedSourceColumns, - outItemType, joinType, pos, needPayload, isUniqueKey, ctx - ); - } else { + tableContent = ctx.Builder(pos) + .Callable("ToFlow") + .Add(0, std::move(tableContent)) + .Callable(1, "DependsOn") + .Add(0, listArg) + .Seal() + .Seal() + .Build(); + + joined = BuildBlockMapJoin(std::move(mapInput), std::move(tableContent), + leftKeyColumnNodes, leftOutputColumns, leftOutputColumnSources, leftUsedSourceColumns, + remappedMembers, rightOutputColumns, rightOutputColumnSources, rightUsedSourceColumns, + outItemType, joinType, pos, needPayload, isUniqueKey, ctx + ); + } else { + if (!isCross) { + TExprNode::TListType leftKeyColumnNodes; + TExprNode::TListType leftKeyColumnNodesNullable; + auto mapInput = RemapNonConvertibleItems(listArg, mainLabel, *leftKeyColumns, outputKeyType, leftKeyColumnNodes, leftKeyColumnNodesNullable, ctx); if (mapJoinUseFlow) { joined = ctx.Builder(pos) .Callable("FlatMap") @@ -2180,31 +2188,31 @@ bool RewriteYtMapJoin(TYtEquiJoin equiJoin, const TJoinLabels& labels, bool isLo .Seal() .Build(); } - } - } else { - auto joinedOut = ctx.NewCallable(pos, "AsStruct", std::move(joinedOutNodes)); - auto joinedBody = ctx.Builder(pos) - .Callable("Map") - .Callable(0, "ToFlow") - .Add(0, std::move(tableContent)) - .Callable(1, "DependsOn") - .Add(0, listArg) + } else { + auto joinedOut = ctx.NewCallable(pos, "AsStruct", std::move(joinedOutNodes)); + auto joinedBody = ctx.Builder(pos) + .Callable("Map") + .Callable(0, "ToFlow") + .Add(0, std::move(tableContent)) + .Callable(1, "DependsOn") + .Add(0, listArg) + .Seal() + .Seal() + .Lambda(1) + .Param("smallRow") + .ApplyPartial(nullptr, std::move(joinedOut)).WithNode(*lookupArg, "smallRow").Seal() .Seal() .Seal() - .Lambda(1) - .Param("smallRow") - .ApplyPartial(nullptr, std::move(joinedOut)).WithNode(*lookupArg, "smallRow").Seal() - .Seal() - .Seal() - .Build(); + .Build(); - auto joinedLambda = ctx.NewLambda(pos, ctx.NewArguments(pos, { mainArg }), std::move(joinedBody)); - joined = ctx.Builder(pos) - .Callable("FlatMap") - .Add(0, listArg) - .Add(1, std::move(joinedLambda)) - .Seal() - .Build(); + auto joinedLambda = ctx.NewLambda(pos, ctx.NewArguments(pos, { mainArg }), std::move(joinedBody)); + joined = ctx.Builder(pos) + .Callable("FlatMap") + .Add(0, listArg) + .Add(1, std::move(joinedLambda)) + .Seal() + .Build(); + } } auto mapLambda = ctx.NewLambda(pos, ctx.NewArguments(pos, {std::move(listArg)}), std::move(joined)); @@ -3605,9 +3613,6 @@ TStatus RewriteYtEquiJoinLeaf(TYtEquiJoin equiJoin, TYtJoinNodeOp& op, TYtJoinNo bool mapJoinUseFlow = state->Configuration->MapJoinUseFlow.Get().GetOrElse(DEFAULT_MAP_JOIN_USE_FLOW); bool mapJoinUseBlocks = state->Configuration->BlockMapJoin.Get().GetOrElse(state->Types->UseBlocks); - if (joinType == "Cross") { - mapJoinUseBlocks = false; - } if (leftTablesReady) { auto status = UpdateInMemorySizeSetting(mapSettings, leftLeaf.Section, labels, op, ctx, true, leftItemType, leftJoinKeyList, state, cluster, leftTables, mapJoinUseFlow); diff --git a/yt/yql/tools/mrjob/mrjob.cpp b/yt/yql/tools/mrjob/mrjob.cpp new file mode 100644 index 00000000000..410c44d5af1 --- /dev/null +++ b/yt/yql/tools/mrjob/mrjob.cpp @@ -0,0 +1,29 @@ +#include <yt/yql/providers/yt/job/yql_job_registry.h> + +#include <yql/essentials/utils/backtrace/backtrace.h> + +#include <yt/cpp/mapreduce/client/init.h> + +#include <util/system/yassert.h> +#include <util/system/mlock.h> + +int main(int argc, const char *argv[]) { + Y_UNUSED(NKikimr::NUdf::GetStaticSymbols()); + try { + LockAllMemory(LockCurrentMemory | LockFutureMemory); + } catch (yexception&) { + Cerr << "mlockall failed, but that's fine" << Endl; + } + + NYql::NBacktrace::RegisterKikimrFatalActions(); + NYql::NBacktrace::EnableKikimrSymbolize(); + + try { + NYT::Initialize(argc, argv); + } catch (...) { + Cerr << CurrentExceptionMessage(); + return -1; + } + + Y_ABORT("This binary should not be called directly"); +} diff --git a/yt/yql/tools/mrjob/test/test.py b/yt/yql/tools/mrjob/test/test.py new file mode 100644 index 00000000000..41d53874612 --- /dev/null +++ b/yt/yql/tools/mrjob/test/test.py @@ -0,0 +1,29 @@ +import os +import os.path +import re +import subprocess + +import yatest.common + + +def test_libc(): + mrjob_dir = yatest.common.binary_path('yt/yql/tools/mrjob') + mrjob_path = os.path.join(mrjob_dir, 'mrjob') + tools_path = os.path.dirname(yatest.common.cxx_compiler_path()) + nm_path = os.path.join(tools_path, 'llvm-nm') + readelf_path = os.path.join(tools_path, 'readelf') + if os.path.isfile(nm_path): + result = subprocess.check_output([nm_path, mrjob_path]) + elif os.path.isfile(readelf_path): + result = subprocess.check_output([readelf_path, '-a', mrjob_path]) + else: + assert False, 'neither llvm-nm nor readelf found, checked paths: %s' % str((readelf_path, nm_path)) + + glibc_tag_count = 0 + for line in result.decode().split('\n'): + glibc_tag = re.search(r'GLIBC_[0-9\.]+', line) + if glibc_tag: + glibc_tag_count += 1 + parts = glibc_tag.group().split('.') + assert len(parts) > 1 + assert int(parts[1]) <= 11 diff --git a/yt/yql/tools/mrjob/test/ya.make b/yt/yql/tools/mrjob/test/ya.make new file mode 100644 index 00000000000..f3122797f7e --- /dev/null +++ b/yt/yql/tools/mrjob/test/ya.make @@ -0,0 +1,10 @@ +IF(OS_LINUX) + PY3TEST() + TEST_SRCS(test.py) + + DEPENDS( + yt/yql/tools/mrjob + ) + + END() +ENDIF() diff --git a/yt/yql/tools/mrjob/ya.make b/yt/yql/tools/mrjob/ya.make new file mode 100644 index 00000000000..4d88e6b5b59 --- /dev/null +++ b/yt/yql/tools/mrjob/ya.make @@ -0,0 +1,36 @@ +PROGRAM(mrjob) + +ALLOCATOR(J) + +SRCS( + mrjob.cpp +) + +IF (OS_LINUX) + # prevent external python extensions to lookup protobuf symbols (and maybe + # other common stuff) in main binary + EXPORTS_SCRIPT(${ARCADIA_ROOT}/yql/essentials/tools/exports.symlist) +ENDIF() + +PEERDIR( + yt/cpp/mapreduce/client + yql/essentials/public/udf/service/terminate_policy + yql/essentials/providers/common/gateway + yql/essentials/utils/backtrace + yql/essentials/parser/pg_wrapper + yql/essentials/sql/pg + yt/yql/providers/yt/job + yt/yql/providers/yt/codec/codegen + yt/yql/providers/yt/comp_nodes/llvm16 + yql/essentials/minikql/computation/llvm16 + yql/essentials/minikql/invoke_builtins/llvm16 + yql/essentials/minikql/comp_nodes/llvm16 +) + +YQL_LAST_ABI_VERSION() + +END() + +RECURSE_FOR_TESTS( + test +) diff --git a/yt/yql/tools/ytrun/lib/ya.make b/yt/yql/tools/ytrun/lib/ya.make new file mode 100644 index 00000000000..12bcaa20fd3 --- /dev/null +++ b/yt/yql/tools/ytrun/lib/ya.make @@ -0,0 +1,34 @@ +LIBRARY() + +SRCS( + ytrun_lib.cpp +) + +PEERDIR( + yt/yql/providers/yt/provider + yt/yql/providers/yt/gateway/native + yt/yql/providers/yt/gateway/fmr + yt/yql/providers/yt/lib/config_clusters + yt/yql/providers/yt/lib/yt_download + yt/yql/providers/yt/lib/yt_url_lister + yt/yql/providers/yt/lib/log + + yql/essentials/providers/common/provider + yql/essentials/core/cbo + yql/essentials/core/peephole_opt + yql/essentials/core/cbo/simple + yql/essentials/core/services + yql/essentials/utils/backtrace + yql/essentials/tools/yql_facade_run + + yt/cpp/mapreduce/client + yt/cpp/mapreduce/interface + + library/cpp/digest/md5 + library/cpp/malloc/api + library/cpp/sighandler +) + +YQL_LAST_ABI_VERSION() + +END() diff --git a/yt/yql/tools/ytrun/lib/ytrun_lib.cpp b/yt/yql/tools/ytrun/lib/ytrun_lib.cpp new file mode 100644 index 00000000000..9a4b5d0377a --- /dev/null +++ b/yt/yql/tools/ytrun/lib/ytrun_lib.cpp @@ -0,0 +1,223 @@ +#include "ytrun_lib.h" + +#include <yt/yql/providers/yt/provider/yql_yt_provider_impl.h> +#include <yt/yql/providers/yt/provider/yql_yt_provider.h> +#include <yt/yql/providers/yt/lib/config_clusters/config_clusters.h> +#include <yt/yql/providers/yt/lib/yt_download/yt_download.h> +#include <yt/yql/providers/yt/lib/yt_url_lister/yt_url_lister.h> +#include <yt/yql/providers/yt/lib/log/yt_logger.h> +#include <yt/yql/providers/yt/gateway/native/yql_yt_native.h> +#include <yt/yql/providers/yt/gateway/fmr/yql_yt_fmr.h> +#include <yql/essentials/providers/common/provider/yql_provider_names.h> +#include <yql/essentials/core/peephole_opt/yql_opt_peephole_physical.h> +#include <yql/essentials/core/services/yql_transform_pipeline.h> +#include <yql/essentials/core/cbo/simple/cbo_simple.h> +#include <yql/essentials/utils/backtrace/backtrace.h> + +#include <yt/cpp/mapreduce/client/init.h> +#include <yt/cpp/mapreduce/interface/config.h> + +#include <library/cpp/digest/md5/md5.h> +#include <library/cpp/malloc/api/malloc.h> +#include <library/cpp/sighandler/async_signals_handler.h> + +#include <util/folder/path.h> +#include <util/stream/file.h> + +namespace { + +class TPeepHolePipelineConfigurator : public NYql::IPipelineConfigurator { +public: + TPeepHolePipelineConfigurator() = default; + + void AfterCreate(NYql::TTransformationPipeline* pipeline) const final { + Y_UNUSED(pipeline); + } + + void AfterTypeAnnotation(NYql::TTransformationPipeline* pipeline) const final { + Y_UNUSED(pipeline); + } + + void AfterOptimize(NYql::TTransformationPipeline* pipeline) const final { + pipeline->Add(NYql::CreateYtWideFlowTransformer(nullptr), "WideFlow"); + pipeline->Add(NYql::CreateYtBlockInputTransformer(nullptr), "BlockInput"); + pipeline->Add(NYql::MakePeepholeOptimization(pipeline->GetTypeAnnotationContext()), "PeepHole"); + pipeline->Add(NYql::CreateYtBlockOutputTransformer(nullptr), "BlockOutput"); + } +}; + +TPeepHolePipelineConfigurator PEEPHOLE_CONFIG_INSTANCE; + +void FlushYtDebugLogOnSignal() { + if (!NMalloc::IsAllocatorCorrupted) { + NYql::FlushYtDebugLog(); + } +} + +} // unnamed + +namespace NYql { + +TYtRunTool::TYtRunTool(TString name) + : TFacadeRunner(std::move(name)) +{ + GetRunOptions().EnableResultPosition = true; + GetRunOptions().EnableCredentials = true; + GetRunOptions().EnableQPlayer = true; + GetRunOptions().ResultStream = &Cout; + + GetRunOptions().AddOptExtension([this](NLastGetopt::TOpts& opts) { + opts.AddLongOption("user", "MR user") + .Optional() + .RequiredArgument("USER") + .StoreResult(&GetRunOptions().User); + + opts.AddLongOption("mrjob-bin", "Path to mrjob binary") + .Optional() + .StoreResult(&MrJobBin_); + opts.AddLongOption("mrjob-udfsdir", "Path to udfs for mr jobs") + .Optional() + .StoreResult(&MrJobUdfsDir_); + opts.AddLongOption("show-progress", "Report operation progress").NoArgument() + .Handler0([&]() { + SetOperationProgressWriter([](const TOperationProgress& progress) { + TStringBuilder remoteId; + if (progress.RemoteId) { + remoteId << ", remoteId: " << progress.RemoteId; + } + TStringBuilder counters; + if (progress.Counters) { + if (progress.Counters->Running) { + counters << ' ' << progress.Counters->Running; + } + if (progress.Counters->Total) { + counters << TStringBuf(" (") << (100ul * progress.Counters->Completed / progress.Counters->Total) << TStringBuf("%)"); + } + } + Cerr << "Operation: [" << progress.Category << "] " << progress.Id + << ", state: " << progress.State << remoteId << counters + << ", current stage: " << progress.Stage.first << Endl; + }); + }); + opts.AddLongOption("threads", "gateway threads") + .Optional() + .RequiredArgument("COUNT") + .StoreResult(&NumThreads_); + opts.AddLongOption("keep-temp", "keep temporary tables") + .Optional() + .NoArgument() + .SetFlag(&KeepTemp_); + opts.AddLongOption("use-graph-meta", "Use tables metadata from graph") + .Optional() + .NoArgument() + .SetFlag(&GetRunOptions().UseMetaFromGrpah); + }); + + GetRunOptions().AddOptHandler([this](const NLastGetopt::TOptsParseResult& res) { + Y_UNUSED(res); + + if (!GetRunOptions().GatewaysConfig) { + GetRunOptions().GatewaysConfig = MakeHolder<TGatewaysConfig>(); + } + + auto ytConfig = GetRunOptions().GatewaysConfig->MutableYt(); + ytConfig->SetGatewayThreads(NumThreads_); + if (MrJobBin_.empty()) { + ytConfig->ClearMrJobBin(); + } else { + ytConfig->SetMrJobBin(MrJobBin_); + ytConfig->SetMrJobBinMd5(MD5::File(MrJobBin_)); + } + + if (MrJobUdfsDir_.empty()) { + ytConfig->ClearMrJobUdfsDir(); + } else { + ytConfig->SetMrJobUdfsDir(MrJobUdfsDir_); + } + auto attr = ytConfig->MutableDefaultSettings()->Add(); + attr->SetName("KeepTempTables"); + attr->SetValue(KeepTemp_ ? "yes" : "no"); + + FillClusterMapping(*ytConfig, TString{YtProviderName}); + + DefYtServer_ = NYql::TConfigClusters::GetDefaultYtServer(*ytConfig); + }); + + GetRunOptions().SetSupportedGateways({TString{YtProviderName}}); + GetRunOptions().GatewayTypes.emplace(YtProviderName); + + AddFsDownloadFactory([this]() -> NFS::IDownloaderPtr { + return MakeYtDownloader(*GetRunOptions().FsConfig, DefYtServer_); + }); + + AddUrlListerFactory([]() -> IUrlListerPtr { + return MakeYtUrlLister(); + }); + + AddProviderFactory([this]() -> NYql::TDataProviderInitializer { + if (GetRunOptions().GatewayTypes.contains(YtProviderName) && GetRunOptions().GatewaysConfig->HasYt()) { + return GetYtNativeDataProviderInitializer(CreateYtGateway(), CreateCboFactory(), CreateDqHelper()); + } + return {}; + }); + + SetPeepholePipelineConfigurator(&PEEPHOLE_CONFIG_INSTANCE); +} + +IYtGateway::TPtr TYtRunTool::CreateYtGateway() { + TYtNativeServices services; + services.FunctionRegistry = GetFuncRegistry().Get(); + services.FileStorage = GetFileStorage(); + services.Config = std::make_shared<TYtGatewayConfig>(GetRunOptions().GatewaysConfig->GetYt()); + auto ytGateway = CreateYtNativeGateway(services); + return GetRunOptions().GatewayTypes.contains(FastMapReduceGatewayName) ? CreateYtFmrGateway(ytGateway): ytGateway; +} + +IOptimizerFactory::TPtr TYtRunTool::CreateCboFactory() { + return MakeSimpleCBOOptimizerFactory(); +} + +IDqHelper::TPtr TYtRunTool::CreateDqHelper() { + return {}; +} + +int TYtRunTool::DoMain(int argc, const char *argv[]) { + // Init MR/YT for proper work of embedded agent + NYT::Initialize(argc, argv); + + NYql::NBacktrace::AddAfterFatalCallback([](int){ FlushYtDebugLogOnSignal(); }); + NYql::SetYtLoggerGlobalBackend(LOG_DEF_PRIORITY); + + if (NYT::TConfig::Get()->Prefix.empty()) { + NYT::TConfig::Get()->Prefix = "//"; + } + + int res = TFacadeRunner::DoMain(argc, argv); + if (0 == res) { + NYql::DropYtDebugLog(); + } + return res; +} + +TProgram::TStatus TYtRunTool::DoRunProgram(TProgramPtr program) { + auto sigHandler = [program](int) { + Cerr << "Aborting..." << Endl; + try { + program->Abort().GetValueSync(); + } catch (...) { + Cerr << CurrentExceptionMessage(); + } + }; + SetAsyncSignalFunction(SIGINT, sigHandler); + SetAsyncSignalFunction(SIGTERM, sigHandler); + + TProgram::TStatus status = TFacadeRunner::DoRunProgram(program); + + auto dummySigHandler = [](int) { }; + SetAsyncSignalFunction(SIGINT, dummySigHandler); + SetAsyncSignalFunction(SIGTERM, dummySigHandler); + + return status; +} + +} // NYql diff --git a/yt/yql/tools/ytrun/lib/ytrun_lib.h b/yt/yql/tools/ytrun/lib/ytrun_lib.h new file mode 100644 index 00000000000..de3c0dfd643 --- /dev/null +++ b/yt/yql/tools/ytrun/lib/ytrun_lib.h @@ -0,0 +1,37 @@ +#pragma once + +#include <yt/yql/providers/yt/provider/yql_yt_gateway.h> + +#include <yql/essentials/tools/yql_facade_run/yql_facade_run.h> +#include <yql/essentials/core/cbo/cbo_optimizer_new.h> +#include <yql/essentials/core/dq_integration/yql_dq_helper.h> + +#include <util/generic/string.h> +#include <util/generic/hash.h> + +namespace NYql { + +constexpr TStringBuf FastMapReduceGatewayName = "fmr"; + +class TYtRunTool: public TFacadeRunner { +public: + TYtRunTool(TString name = "ytrun"); + ~TYtRunTool() = default; + +protected: + int DoMain(int argc, const char *argv[]) override; + TProgram::TStatus DoRunProgram(TProgramPtr program) override; + + virtual IYtGateway::TPtr CreateYtGateway(); + virtual IOptimizerFactory::TPtr CreateCboFactory(); + virtual IDqHelper::TPtr CreateDqHelper(); + +protected: + TString MrJobBin_; + TString MrJobUdfsDir_; + size_t NumThreads_ = 1; + bool KeepTemp_ = false; + TString DefYtServer_; +}; + +} // NYql diff --git a/yt/yql/tools/ytrun/ya.make b/yt/yql/tools/ytrun/ya.make new file mode 100644 index 00000000000..d1b9a412a66 --- /dev/null +++ b/yt/yql/tools/ytrun/ya.make @@ -0,0 +1,35 @@ +PROGRAM() + +ALLOCATOR(J) + +SRCS( + ytrun.cpp +) + +IF (OS_LINUX) + # prevent external python extensions to lookup protobuf symbols (and maybe + # other common stuff) in main binary + EXPORTS_SCRIPT(${ARCADIA_ROOT}/yql/essentials/tools/exports.symlist) +ENDIF() + +PEERDIR( + yt/yql/tools/ytrun/lib + + yt/yql/providers/yt/codec/codegen + yt/yql/providers/yt/comp_nodes/llvm16 + yql/essentials/minikql/invoke_builtins/llvm16 + yql/essentials/minikql/comp_nodes/llvm16 + yql/essentials/parser/pg_wrapper + yql/essentials/public/udf/service/terminate_policy + yql/essentials/sql/pg +) + +YQL_LAST_ABI_VERSION() + +RESOURCE( + yql/essentials/cfg/tests/gateways.conf gateways.conf + yql/essentials/cfg/tests/fs.conf fs.conf + yql/essentials/cfg/tests/fs_arc.conf fs_arc.conf +) + +END() diff --git a/yt/yql/tools/ytrun/ytrun.cpp b/yt/yql/tools/ytrun/ytrun.cpp new file mode 100644 index 00000000000..d405eb419e9 --- /dev/null +++ b/yt/yql/tools/ytrun/ytrun.cpp @@ -0,0 +1,11 @@ +#include <yt/yql/tools/ytrun/lib/ytrun_lib.h> + +int main(int argc, const char *argv[]) { + try { + return NYql::TYtRunTool().Main(argc, argv); + } + catch (...) { + Cerr << CurrentExceptionMessage() << Endl; + return 1; + } +} diff --git a/yt/yt/client/api/delegating_client.h b/yt/yt/client/api/delegating_client.h index ecb92de39e3..f2c3c903616 100644 --- a/yt/yt/client/api/delegating_client.h +++ b/yt/yt/client/api/delegating_client.h @@ -282,6 +282,11 @@ public: const TUnfreezeTableOptions& options), (path, options)) + DELEGATE_METHOD(TFuture<void>, CancelTabletTransition, ( + NTabletClient::TTabletId tabletId, + const TCancelTabletTransitionOptions& options), + (tabletId, options)) + DELEGATE_METHOD(TFuture<void>, ReshardTable, ( const NYPath::TYPath& path, const std::vector<NTableClient::TLegacyOwningKey>& pivotKeys, diff --git a/yt/yt/client/api/operation_client.cpp b/yt/yt/client/api/operation_client.cpp index 391740c7eb3..625b55fe611 100644 --- a/yt/yt/client/api/operation_client.cpp +++ b/yt/yt/client/api/operation_client.cpp @@ -311,6 +311,7 @@ void Serialize(const TJob& job, NYson::IYsonConsumer* consumer, TStringBuf idKey .OptionalItem("is_stale", job.IsStale) .OptionalItem("job_cookie", job.JobCookie) .OptionalItem("archive_features", job.ArchiveFeatures) + .OptionalItem("operation_incarnation", job.OperationIncarnation) .EndMap(); } diff --git a/yt/yt/client/api/operation_client.h b/yt/yt/client/api/operation_client.h index 92f963c13d0..22ddbeb856e 100644 --- a/yt/yt/client/api/operation_client.h +++ b/yt/yt/client/api/operation_client.h @@ -209,6 +209,7 @@ struct TListJobsOptions std::optional<bool> WithCompetitors; std::optional<bool> WithMonitoringDescriptor; std::optional<TString> TaskName; + std::optional<std::string> OperationIncarnation; std::optional<TInstant> FromTime; std::optional<TInstant> ToTime; @@ -386,6 +387,7 @@ struct TJob std::optional<TString> MonitoringDescriptor; std::optional<ui64> JobCookie; NYson::TYsonString ArchiveFeatures; + std::optional<std::string> OperationIncarnation; std::optional<bool> IsStale; diff --git a/yt/yt/client/api/rpc_proxy/client_impl.cpp b/yt/yt/client/api/rpc_proxy/client_impl.cpp index ebd61803194..bc032c68d77 100644 --- a/yt/yt/client/api/rpc_proxy/client_impl.cpp +++ b/yt/yt/client/api/rpc_proxy/client_impl.cpp @@ -319,6 +319,13 @@ TFuture<void> TClient::UnfreezeTable( return req->Invoke().As<void>(); } +TFuture<void> TClient::CancelTabletTransition( + NTabletClient::TTabletId /*tabletId*/, + const TCancelTabletTransitionOptions& /*options*/) +{ + ThrowUnimplemented("CancelTabletTransition"); +} + TFuture<void> TClient::ReshardTable( const TYPath& path, const std::vector<TLegacyOwningKey>& pivotKeys, @@ -1516,6 +1523,9 @@ TFuture<TListJobsResult> TClient::ListJobs( if (options.TaskName) { req->set_task_name(*options.TaskName); } + if (options.OperationIncarnation) { + req->set_operation_incarnation(*options.OperationIncarnation); + } if (options.FromTime) { req->set_from_time(NYT::ToProto(*options.FromTime)); } diff --git a/yt/yt/client/api/rpc_proxy/client_impl.h b/yt/yt/client/api/rpc_proxy/client_impl.h index 906b141c73e..db95372369f 100644 --- a/yt/yt/client/api/rpc_proxy/client_impl.h +++ b/yt/yt/client/api/rpc_proxy/client_impl.h @@ -53,6 +53,10 @@ public: const NYPath::TYPath& path, const NApi::TUnfreezeTableOptions& options) override; + TFuture<void> CancelTabletTransition( + NTabletClient::TTabletId tabletId, + const NApi::TCancelTabletTransitionOptions& options) override; + TFuture<void> ReshardTable( const NYPath::TYPath& path, const std::vector<NTableClient::TLegacyOwningKey>& pivotKeys, diff --git a/yt/yt/client/api/rpc_proxy/helpers.cpp b/yt/yt/client/api/rpc_proxy/helpers.cpp index 41f88cbcbf7..7bbdc3a3800 100644 --- a/yt/yt/client/api/rpc_proxy/helpers.cpp +++ b/yt/yt/client/api/rpc_proxy/helpers.cpp @@ -1014,6 +1014,9 @@ void ToProto(NProto::TJob* protoJob, const NApi::TJob& job) if (job.MonitoringDescriptor) { protoJob->set_monitoring_descriptor(*job.MonitoringDescriptor); } + if (job.OperationIncarnation) { + protoJob->set_operation_incarnation(*job.OperationIncarnation); + } } void FromProto(NApi::TJob* job, const NProto::TJob& protoJob) @@ -1163,6 +1166,11 @@ void FromProto(NApi::TJob* job, const NProto::TJob& protoJob) } else { job->MonitoringDescriptor.reset(); } + if (protoJob.has_operation_incarnation()) { + job->OperationIncarnation = protoJob.operation_incarnation(); + } else { + job->OperationIncarnation.reset(); + } } void ToProto( diff --git a/yt/yt/client/api/rpc_proxy/transaction.cpp b/yt/yt/client/api/rpc_proxy/transaction.cpp index 9e0c7423a1c..454cbf6e6db 100644 --- a/yt/yt/client/api/rpc_proxy/transaction.cpp +++ b/yt/yt/client/api/rpc_proxy/transaction.cpp @@ -22,7 +22,7 @@ NApi::ITransactionPtr CreateTransaction( i64 sequenceNumberSourceId, TStringBuf capitalizedCreationReason) { - return New<TTransaction>( + auto transaction = New<TTransaction>( std::move(connection), std::move(client), std::move(channel), @@ -37,9 +37,12 @@ NApi::ITransactionPtr CreateTransaction( std::move(stickyParameters), sequenceNumberSourceId, capitalizedCreationReason); + + transaction->Initialize(); + + return transaction; } //////////////////////////////////////////////////////////////////////////////// } // namespace NYT::NApi::NRpcProxy - diff --git a/yt/yt/client/api/rpc_proxy/transaction_impl.cpp b/yt/yt/client/api/rpc_proxy/transaction_impl.cpp index d4426488f41..cf732951ba7 100644 --- a/yt/yt/client/api/rpc_proxy/transaction_impl.cpp +++ b/yt/yt/client/api/rpc_proxy/transaction_impl.cpp @@ -81,7 +81,10 @@ TTransaction::TTransaction( PingPeriod_, /*sticky*/ stickyParameters.has_value(), StickyProxyAddress_); +} +void TTransaction::Initialize() +{ // TODO(babenko): don't run periodic pings if client explicitly disables them in options RunPeriodicPings(); } diff --git a/yt/yt/client/api/rpc_proxy/transaction_impl.h b/yt/yt/client/api/rpc_proxy/transaction_impl.h index e191a6fb79f..300f3a9bd9b 100644 --- a/yt/yt/client/api/rpc_proxy/transaction_impl.h +++ b/yt/yt/client/api/rpc_proxy/transaction_impl.h @@ -48,6 +48,8 @@ public: i64 sequenceNumberSourceId, TStringBuf capitalizedCreationReason); + void Initialize(); + // ITransaction implementation. NApi::IConnectionPtr GetConnection() override; NApi::IClientPtr GetClient() const override; diff --git a/yt/yt/client/api/table_client.h b/yt/yt/client/api/table_client.h index a1eb9b31525..076ad62b74a 100644 --- a/yt/yt/client/api/table_client.h +++ b/yt/yt/client/api/table_client.h @@ -90,6 +90,11 @@ struct TUnfreezeTableOptions , public TTabletRangeOptions { }; +struct TCancelTabletTransitionOptions + : public TTimeoutOptions + , public TMutatingOptions +{ }; + struct TReshardTableOptions : public TTimeoutOptions , public TMutatingOptions @@ -377,6 +382,10 @@ struct ITableClient const NYPath::TYPath& path, const TUnfreezeTableOptions& options = {}) = 0; + virtual TFuture<void> CancelTabletTransition( + NTabletClient::TTabletId tabletId, + const TCancelTabletTransitionOptions& options = {}) = 0; + virtual TFuture<void> ReshardTable( const NYPath::TYPath& path, const std::vector<NTableClient::TLegacyOwningKey>& pivotKeys, diff --git a/yt/yt/client/driver/driver.cpp b/yt/yt/client/driver/driver.cpp index 2141e3eff5d..a77f6d2c33a 100644 --- a/yt/yt/client/driver/driver.cpp +++ b/yt/yt/client/driver/driver.cpp @@ -230,6 +230,7 @@ public: REGISTER (TRemountTableCommand, "remount_table", Null, Structured, true, false, ApiVersion4); REGISTER (TFreezeTableCommand, "freeze_table", Null, Structured, true, false, ApiVersion4); REGISTER (TUnfreezeTableCommand, "unfreeze_table", Null, Structured, true, false, ApiVersion4); + REGISTER (TCancelTabletTransitionCommand, "cancel_tablet_transition", Null, Structured, true, false, ApiVersion4); REGISTER (TReshardTableCommand, "reshard_table", Null, Structured, true, false, ApiVersion4); REGISTER (TAlterTableCommand, "alter_table", Null, Structured, true, false, ApiVersion4); diff --git a/yt/yt/client/driver/scheduler_commands.cpp b/yt/yt/client/driver/scheduler_commands.cpp index 5002ccfb8ec..4fb6a48d759 100644 --- a/yt/yt/client/driver/scheduler_commands.cpp +++ b/yt/yt/client/driver/scheduler_commands.cpp @@ -490,6 +490,11 @@ void TListJobsCommand::Register(TRegistrar registrar) [] (TThis* command) -> auto& { return command->Options.WithMonitoringDescriptor; }) .Optional(/*init*/ false); + registrar.ParameterWithUniversalAccessor<std::optional<std::string>>( + "operation_incarnation", + [] (TThis* command) -> auto& { return command->Options.OperationIncarnation; }) + .Optional(/*init*/ false); + registrar.ParameterWithUniversalAccessor<std::optional<TInstant>>( "from_time", [] (TThis* command) -> auto& { return command->Options.FromTime; }) diff --git a/yt/yt/client/driver/shuffle_commands.cpp b/yt/yt/client/driver/shuffle_commands.cpp index 67c93746d7f..d1cb605ad9d 100644 --- a/yt/yt/client/driver/shuffle_commands.cpp +++ b/yt/yt/client/driver/shuffle_commands.cpp @@ -10,7 +10,6 @@ #include <yt/yt/client/table_client/table_output.h> #include <yt/yt/client/table_client/value_consumer.h> - namespace NYT::NDriver { using namespace NConcurrency; diff --git a/yt/yt/client/driver/table_commands.cpp b/yt/yt/client/driver/table_commands.cpp index c05964a50ff..3cef7004d1e 100644 --- a/yt/yt/client/driver/table_commands.cpp +++ b/yt/yt/client/driver/table_commands.cpp @@ -575,6 +575,19 @@ void TUnfreezeTableCommand::DoExecute(ICommandContextPtr context) //////////////////////////////////////////////////////////////////////////////// +void TCancelTabletTransitionCommand::DoExecute(ICommandContextPtr context) +{ + auto asyncResult = context->GetClient()->CancelTabletTransition( + TabletId, + Options); + WaitFor(asyncResult) + .ThrowOnError(); + + ProduceEmptyOutput(context); +} + +//////////////////////////////////////////////////////////////////////////////// + void TReshardTableCommand::Register(TRegistrar registrar) { registrar.Parameter("pivot_keys", &TThis::PivotKeys) diff --git a/yt/yt/client/driver/table_commands.h b/yt/yt/client/driver/table_commands.h index 97f52e44fd3..26dacebf7e6 100644 --- a/yt/yt/client/driver/table_commands.h +++ b/yt/yt/client/driver/table_commands.h @@ -247,6 +247,24 @@ public: //////////////////////////////////////////////////////////////////////////////// +class TCancelTabletTransitionCommand + : public TTypedCommand<NApi::TCancelTabletTransitionOptions> +{ + NTabletClient::TTabletId TabletId; + + REGISTER_YSON_STRUCT_LITE(TCancelTabletTransitionCommand); + + static void Register(TRegistrar registrar) + { + registrar.Parameter("tablet_id", &TThis::TabletId); + } + +public: + void DoExecute(ICommandContextPtr context) override; +}; + +//////////////////////////////////////////////////////////////////////////////// + class TReshardTableCommand : public TTabletCommandBase<NApi::TReshardTableOptions> { diff --git a/yt/yt/client/federated/cache.h b/yt/yt/client/federated/cache.h index db6110b0f52..c7c173942de 100644 --- a/yt/yt/client/federated/cache.h +++ b/yt/yt/client/federated/cache.h @@ -5,7 +5,6 @@ #include <yt/yt/client/cache/cache.h> #include <yt/yt/client/cache/config.h> - namespace NYT::NClient::NFederated { //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/client/federated/client.cpp b/yt/yt/client/federated/client.cpp index 86ba307c1a9..092ad1f9301 100644 --- a/yt/yt/client/federated/client.cpp +++ b/yt/yt/client/federated/client.cpp @@ -379,6 +379,7 @@ public: UNIMPLEMENTED_METHOD(TFuture<void>, RemountTable, (const NYPath::TYPath&, const TRemountTableOptions&)); UNIMPLEMENTED_METHOD(TFuture<void>, FreezeTable, (const NYPath::TYPath&, const TFreezeTableOptions&)); UNIMPLEMENTED_METHOD(TFuture<void>, UnfreezeTable, (const NYPath::TYPath&, const TUnfreezeTableOptions&)); + UNIMPLEMENTED_METHOD(TFuture<void>, CancelTabletTransition, (NTabletClient::TTabletId, const TCancelTabletTransitionOptions&)); UNIMPLEMENTED_METHOD(TFuture<void>, ReshardTable, (const NYPath::TYPath&, const std::vector<NTableClient::TUnversionedOwningRow>&, const TReshardTableOptions&)); UNIMPLEMENTED_METHOD(TFuture<void>, ReshardTable, (const NYPath::TYPath&, int, const TReshardTableOptions&)); UNIMPLEMENTED_METHOD(TFuture<std::vector<NTabletClient::TTabletActionId>>, ReshardTableAutomatic, (const NYPath::TYPath&, const TReshardTableAutomaticOptions&)); diff --git a/yt/yt/client/federated/unittests/client_ut.cpp b/yt/yt/client/federated/unittests/client_ut.cpp index f57aeb670b8..5caba8db1a7 100644 --- a/yt/yt/client/federated/unittests/client_ut.cpp +++ b/yt/yt/client/federated/unittests/client_ut.cpp @@ -13,7 +13,6 @@ #include <util/datetime/base.h> - namespace NYT::NClient::NFederated { namespace { diff --git a/yt/yt/client/hedging/cache.h b/yt/yt/client/hedging/cache.h index 51719f786b5..c5a5528fd67 100644 --- a/yt/yt/client/hedging/cache.h +++ b/yt/yt/client/hedging/cache.h @@ -8,7 +8,6 @@ #include <util/generic/strbuf.h> - namespace NYT::NClient::NHedging::NRpc { //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/client/hedging/hedging.cpp b/yt/yt/client/hedging/hedging.cpp index 7fa807b6f1e..e58a5fd5619 100644 --- a/yt/yt/client/hedging/hedging.cpp +++ b/yt/yt/client/hedging/hedging.cpp @@ -125,6 +125,7 @@ public: UNSUPPORTED_METHOD(TFuture<void>, RemountTable, (const TYPath&, const TRemountTableOptions&)); UNSUPPORTED_METHOD(TFuture<void>, FreezeTable, (const TYPath&, const TFreezeTableOptions&)); UNSUPPORTED_METHOD(TFuture<void>, UnfreezeTable, (const TYPath&, const TUnfreezeTableOptions&)); + UNSUPPORTED_METHOD(TFuture<void>, CancelTabletTransition, (NTabletClient::TTabletId, const TCancelTabletTransitionOptions&)); UNSUPPORTED_METHOD(TFuture<void>, ReshardTable, (const TYPath&, const std::vector<NTableClient::TUnversionedOwningRow>&, const TReshardTableOptions&)); UNSUPPORTED_METHOD(TFuture<void>, ReshardTable, (const TYPath&, int, const TReshardTableOptions&)); UNSUPPORTED_METHOD(TFuture<std::vector<NTabletClient::TTabletActionId>>, ReshardTableAutomatic, (const TYPath&, const TReshardTableAutomaticOptions&)); diff --git a/yt/yt/client/hedging/hedging_executor.cpp b/yt/yt/client/hedging/hedging_executor.cpp index 5541464e704..073499c1aa4 100644 --- a/yt/yt/client/hedging/hedging_executor.cpp +++ b/yt/yt/client/hedging/hedging_executor.cpp @@ -4,7 +4,6 @@ #include <yt/yt/core/logging/log.h> - namespace NYT::NClient::NHedging::NRpc { //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/client/hedging/hedging_executor.h b/yt/yt/client/hedging/hedging_executor.h index dd0b4f5c8e4..3b08bf5034e 100644 --- a/yt/yt/client/hedging/hedging_executor.h +++ b/yt/yt/client/hedging/hedging_executor.h @@ -16,7 +16,6 @@ #include <vector> - namespace NYT::NClient::NHedging::NRpc { //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/client/queue_client/producer_client.cpp b/yt/yt/client/queue_client/producer_client.cpp index 77172ee4fdd..ec6f0f72165 100644 --- a/yt/yt/client/queue_client/producer_client.cpp +++ b/yt/yt/client/queue_client/producer_client.cpp @@ -345,6 +345,7 @@ private: if (Options_.AutoSequenceNumber) { pushQueueProducerOptions.SequenceNumber = TQueueProducerSequenceNumber{LastSequenceNumber_.load().Underlying() + 1}; } + pushQueueProducerOptions.RequireSyncReplica = Options_.RequireSyncReplica; return transaction->PushQueueProducer(ProducerPath_, QueuePath_, SessionId_, Epoch_, NameTable_, buffer.SerializedRows, pushQueueProducerOptions) .Apply(BIND([=, this, this_ = MakeStrong(this)] (const TPushQueueProducerResult& pushQueueProducerResult) { diff --git a/yt/yt/client/queue_client/producer_client.h b/yt/yt/client/queue_client/producer_client.h index 50b056b202e..19e53a8cb00 100644 --- a/yt/yt/client/queue_client/producer_client.h +++ b/yt/yt/client/queue_client/producer_client.h @@ -48,6 +48,10 @@ struct TProducerSessionOptions //! Acknowledgment callback. TAckCallback AckCallback; + + //! If this happens to be a push into a replicated table queue, + //! controls if at least one sync replica is required. + bool RequireSyncReplica = true; }; struct IProducerSession diff --git a/yt/yt/client/unittests/mock/client.h b/yt/yt/client/unittests/mock/client.h index ea6fed2987f..6885b3c4732 100644 --- a/yt/yt/client/unittests/mock/client.h +++ b/yt/yt/client/unittests/mock/client.h @@ -403,6 +403,11 @@ public: const TUnfreezeTableOptions& options), (override)); + MOCK_METHOD(TFuture<void>, CancelTabletTransition, ( + NTabletClient::TTabletId tabletId, + const TCancelTabletTransitionOptions& options), + (override)); + MOCK_METHOD(TFuture<void>, ReshardTable, ( const NYPath::TYPath& path, const std::vector<NTableClient::TLegacyOwningKey>& pivotKeys, diff --git a/yt/yt/client/unittests/validate_logical_type_ut.cpp b/yt/yt/client/unittests/validate_logical_type_ut.cpp index 3790ee4ef8a..67ff680f635 100644 --- a/yt/yt/client/unittests/validate_logical_type_ut.cpp +++ b/yt/yt/client/unittests/validate_logical_type_ut.cpp @@ -7,7 +7,6 @@ #include <util/string/escape.h> - namespace NYT::NTableClient { //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/core/concurrency/async_stream_pipe.h b/yt/yt/core/concurrency/async_stream_pipe.h index 897eb865ea8..518fb6db6fb 100644 --- a/yt/yt/core/concurrency/async_stream_pipe.h +++ b/yt/yt/core/concurrency/async_stream_pipe.h @@ -5,7 +5,6 @@ #include <yt/yt/core/concurrency/async_stream.h> #include <yt/yt/core/concurrency/nonblocking_queue.h> - namespace NYT::NConcurrency { //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/core/concurrency/moody_camel_concurrent_queue.h b/yt/yt/core/concurrency/moody_camel_concurrent_queue.h index 939122ae8a0..e61c924ba37 100644 --- a/yt/yt/core/concurrency/moody_camel_concurrent_queue.h +++ b/yt/yt/core/concurrency/moody_camel_concurrent_queue.h @@ -417,7 +417,6 @@ template<typename T, typename Traits> class ConcurrentQueue; template<typename T, typename Traits> class BlockingConcurrentQueue; class ConcurrentQueueTests; - namespace details { struct ConcurrentQueueProducerTypelessBase diff --git a/yt/yt/core/misc/isa_crc64/checksum.cpp b/yt/yt/core/misc/isa_crc64/checksum.cpp index 6bf5c65283e..7b63bd3d6f5 100644 --- a/yt/yt/core/misc/isa_crc64/checksum.cpp +++ b/yt/yt/core/misc/isa_crc64/checksum.cpp @@ -23,7 +23,6 @@ uint64_t IsaCrcImplBase(const void* data, size_t length, uint64_t seed) } // namespace - namespace NYT::NIsaCrc64 { diff --git a/yt/yt/core/misc/object_pool-inl.h b/yt/yt/core/misc/object_pool-inl.h index 68b64c2a69c..ecfa3d45acc 100644 --- a/yt/yt/core/misc/object_pool-inl.h +++ b/yt/yt/core/misc/object_pool-inl.h @@ -4,7 +4,6 @@ #include "object_pool.h" #endif - namespace NYT { //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/core/ytalloc/bindings.cpp b/yt/yt/core/ytalloc/bindings.cpp index c68e2bd8e28..5844d89745a 100644 --- a/yt/yt/core/ytalloc/bindings.cpp +++ b/yt/yt/core/ytalloc/bindings.cpp @@ -202,7 +202,6 @@ void EnableYTProfiling(const TYTProfilingConfigPtr& config) //////////////////////////////////////////////////////////////////////////////// - namespace { std::atomic<bool> ConfiguredFromEnv_; diff --git a/yt/yt/core/ytree/unittests/yson_schema_ut.cpp b/yt/yt/core/ytree/unittests/yson_schema_ut.cpp index 88df8268a29..4c4cb107a02 100644 --- a/yt/yt/core/ytree/unittests/yson_schema_ut.cpp +++ b/yt/yt/core/ytree/unittests/yson_schema_ut.cpp @@ -8,7 +8,6 @@ #include <yt/yt/core/ytree/unittests/proto/test.pb.h> - namespace NYT::NYTree { namespace { diff --git a/yt/yt/core/ytree/ypath_service.h b/yt/yt/core/ytree/ypath_service.h index d0ea869e8f9..6df44dba804 100644 --- a/yt/yt/core/ytree/ypath_service.h +++ b/yt/yt/core/ytree/ypath_service.h @@ -17,7 +17,6 @@ #include <variant> - namespace NYT::NYTree { //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/core/ytree/yson_struct.cpp b/yt/yt/core/ytree/yson_struct.cpp index d101854ca0f..53b875c2e72 100644 --- a/yt/yt/core/ytree/yson_struct.cpp +++ b/yt/yt/core/ytree/yson_struct.cpp @@ -277,7 +277,6 @@ DEFINE_REFCOUNTED_TYPE(TYsonStruct) } // namespace NYT::NYTree - namespace NYT { using namespace NYTree; diff --git a/yt/yt/library/formats/arrow_writer.h b/yt/yt/library/formats/arrow_writer.h index ba05ce3c921..020090152cc 100644 --- a/yt/yt/library/formats/arrow_writer.h +++ b/yt/yt/library/formats/arrow_writer.h @@ -8,7 +8,6 @@ #include <yt/yt/core/ytree/public.h> - namespace NYT::NFormats { //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/library/formats/protobuf_parser.cpp b/yt/yt/library/formats/protobuf_parser.cpp index 4444daa316a..2afb51003bb 100644 --- a/yt/yt/library/formats/protobuf_parser.cpp +++ b/yt/yt/library/formats/protobuf_parser.cpp @@ -31,7 +31,6 @@ using namespace NComplexTypes; using ::google::protobuf::internal::WireFormatLite; - namespace { //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/library/formats/unittests/protobuf_format_ut.cpp b/yt/yt/library/formats/unittests/protobuf_format_ut.cpp index 42df9187bfb..5d27001009b 100644 --- a/yt/yt/library/formats/unittests/protobuf_format_ut.cpp +++ b/yt/yt/library/formats/unittests/protobuf_format_ut.cpp @@ -35,7 +35,6 @@ using namespace std::string_view_literals; - namespace NYT { namespace { diff --git a/yt/yt/library/formats/unittests/yamred_dsv_writer_ut.cpp b/yt/yt/library/formats/unittests/yamred_dsv_writer_ut.cpp index d61c214e996..c4a59f07e0c 100644 --- a/yt/yt/library/formats/unittests/yamred_dsv_writer_ut.cpp +++ b/yt/yt/library/formats/unittests/yamred_dsv_writer_ut.cpp @@ -11,7 +11,6 @@ #include <cstdio> - namespace NYT::NFormats { namespace { diff --git a/yt/yt/library/numeric/unittests/piecewise_linear_function_ut.cpp b/yt/yt/library/numeric/unittests/piecewise_linear_function_ut.cpp index 4ea3e3e815c..a16f410326c 100644 --- a/yt/yt/library/numeric/unittests/piecewise_linear_function_ut.cpp +++ b/yt/yt/library/numeric/unittests/piecewise_linear_function_ut.cpp @@ -6,7 +6,6 @@ #include <util/string/cast.h> - namespace NYT { using ::ToString; diff --git a/yt/yt/library/tcmalloc/config.h b/yt/yt/library/tcmalloc/config.h index 3c140b44308..86d52781466 100644 --- a/yt/yt/library/tcmalloc/config.h +++ b/yt/yt/library/tcmalloc/config.h @@ -73,10 +73,10 @@ DEFINE_REFCOUNTED_TYPE(TDynamicHeapSizeLimitConfig) struct TTCMallocConfig : public NYTree::TYsonStruct { - //! Threshold in bytes + //! Threshold in bytes. i64 AggressiveReleaseThreshold; - //! Threshold in fractions of total memory of the container + //! Threshold in fractions of total memory of the container. std::optional<double> AggressiveReleaseThresholdRatio; i64 AggressiveReleaseSize; diff --git a/yt/yt_proto/yt/client/api/rpc_proxy/proto/api_service.proto b/yt/yt_proto/yt/client/api/rpc_proxy/proto/api_service.proto index 82e2d17d48c..f0c94b33ef3 100644 --- a/yt/yt_proto/yt/client/api/rpc_proxy/proto/api_service.proto +++ b/yt/yt_proto/yt/client/api/rpc_proxy/proto/api_service.proto @@ -2435,6 +2435,8 @@ message TReqListJobs optional bool with_monitoring_descriptor = 21; + optional string operation_incarnation = 25; + optional uint64 from_time = 22; // TInstant optional uint64 to_time = 23; // TInstant @@ -3172,6 +3174,7 @@ message TJob optional uint64 job_cookie = 27; optional bytes archive_features = 29; // YSON optional string monitoring_descriptor = 30; + optional string operation_incarnation = 31; } message TListJobsStatistics diff --git a/yt/yt_proto/yt/formats/extension.proto b/yt/yt_proto/yt/formats/extension.proto index 80a529ba953..6b34717390e 100644 --- a/yt/yt_proto/yt/formats/extension.proto +++ b/yt/yt_proto/yt/formats/extension.proto @@ -91,6 +91,8 @@ extend google.protobuf.FieldOptions repeated EWrapperFieldFlag.Enum flags = 56788; optional string column_name = 56789; optional string key_column_name = 56790; + // TODO(ignat): use consistent tag number. + optional uint64 max_inline_hunk_size = 60130; } extend google.protobuf.MessageOptions |