diff options
| author | Alexander Smirnov <alex@ydb.tech> | 2024-09-26 14:44:45 +0000 |
|---|---|---|
| committer | Alexander Smirnov <alex@ydb.tech> | 2024-09-26 14:44:45 +0000 |
| commit | c853e78c5e416ede1d99665049657f2e304872e4 (patch) | |
| tree | b6e2e59629b8e0f863eec9a700121ca3f7416f28 | |
| parent | 7d208c76420539a7e44f2393c66e7d0444744517 (diff) | |
| parent | 47e779420bb80722978c1ffd518245bffd886a50 (diff) | |
| download | ydb-c853e78c5e416ede1d99665049657f2e304872e4.tar.gz | |
Merge branch 'rightlib' into mergelibs-240926-1443
264 files changed, 2929 insertions, 1551 deletions
diff --git a/build/conf/a.yaml b/build/conf/a.yaml index e15f1892c6..cde8d7dc68 100644 --- a/build/conf/a.yaml +++ b/build/conf/a.yaml @@ -7,7 +7,7 @@ arcanum: requirements: - system: arcanum type: merge_time_fits - alias: Merge time intervals (YMAKE-1194) + alias: Merge is forbidden from 11:00 to 20:00 MSK (YMAKE-1194) data: merge_intervals_utc: # 20:00..11:00 MSK diff --git a/build/conf/go.conf b/build/conf/go.conf index c75e3ec324..23dc650cb5 100644 --- a/build/conf/go.conf +++ b/build/conf/go.conf @@ -816,6 +816,7 @@ module GO_LIBRARY: _GO_BASE_UNIT { SET(MODULE_TYPE LIBRARY) SET(PEERDIR_TAGS GO GO_PROTO GO_FBS __EMPTY__ RESOURCE_LIB PACKAGE_UNION) + _MAKEFILE_INCLUDE_LIKE_DEPS+=.yo.snapshot.json MODULE_SUFFIX=.a } diff --git a/build/conf/java.conf b/build/conf/java.conf index ed38d79a31..7b9e2f93d8 100644 --- a/build/conf/java.conf +++ b/build/conf/java.conf @@ -472,7 +472,7 @@ when($JDK_VERSION == "8") { } # tag:java-specific -JDK_LATEST_VERSION=22 +JDK_LATEST_VERSION=23 JDK_LATEST_PEERDIR=build/platform/java/jdk/jdk${JDK_LATEST_VERSION} # tag:java-specific @@ -1074,6 +1074,13 @@ module JAR_LIBRARY: _COMPILABLE_JAR_BASE { when($WITH_KOTLINC_PLUGIN_SERIALIZATION) { KOTLINC_OPTS_VALUE+=-Xplugin=${KOTLIN_COMPILER_RESOURCE_GLOBAL}/plugins/kotlin-serialization-plugin.jar } + when($WITH_KOTLINC_PLUGIN_DETEKT) { + KOTLINC_OPTS_VALUE+=-Xplugin=${tool:"devtools/detekt-compiler-plugin"} + KOTLINC_OPTS_VALUE+=-P plugin:detekt-compiler-plugin:report=xml:${BINDIR}/misc/detekt-report.xml + # If there are no kt sources, the report won't be generated, however the file must be always present. + ALL_KT_COMMANDS+=&& $YMAKE_PYTHON ${input:"build/scripts/touch.py"} ${BINDIR}/misc/detekt-report.xml + ALL_KT_COMMANDS+=&& $FS_TOOLS copy ${BINDIR}/misc/detekt-report.xml ${output;noauto:"detekt-report.xml"} + } # Must be in sync with KT_CLASSES_DIR! # There are problems in JDK13 with abs paths in classpath baked into jar file manifest. Using relative path # here assumes that jar file with classpath for javac located in the $ARCADIA_BUILD_ROOT @@ -1147,6 +1154,11 @@ module JAR_LIBRARY: _COMPILABLE_JAR_BASE { JAVAC_OPTS+=$ERROR_PRONE_JDK16PLUS_ADD_OPENS } + when($ERROR_PRONE_VALUE && $JDK_REAL_VERSION == "23") { + JAVAC_FLAGS_VALUE+=$ERROR_PRONE_JDK16PLUS_ADD_OPENS + JAVAC_OPTS+=$ERROR_PRONE_JDK16PLUS_ADD_OPENS + } + when($ERROR_PRONE_VALUE && $ERROR_PRONE_VERSION == "2.7.1") { JAVAC_FLAGS_VALUE+=$ERROR_PRONE_2_7_1_FORCED_OPTS JAVAC_OPTS+=$ERROR_PRONE_2_7_1_FORCED_OPTS @@ -1165,6 +1177,7 @@ module JAR_LIBRARY: _COMPILABLE_JAR_BASE { ENABLE(YMAKE_JAVA_TEST) _ADD_KOTLIN_STYLE_CHECKS($(BUILD_ROOT)/$MODDIR/all-kt-sources.txt REQUIREMENTS cpu:2) _ADD_JAVA_STYLE_CHECKS($(BUILD_ROOT)/$MODDIR/lint-java.srclst::$(SOURCE_ROOT)) + _ADD_DETEKT_REPORT_CHECK($(BUILD_ROOT)/$MODDIR/detekt-report.xml) _ADD_CLASSPATH_CLASH_CHECK() JAVA_MODULE() } @@ -1284,7 +1297,10 @@ JAVA_SWIG_DELIM=JAVA_SWIG_DELIM JDK_REAL_VERSION= JDK_VERSION= -when ($JDK_VERSION == "22") { +when ($JDK_VERSION == "23") { + JDK_REAL_VERSION=23 +} +elsewhen ($JDK_VERSION == "22") { JDK_REAL_VERSION=22 } elsewhen ($JDK_VERSION == "21") { @@ -1312,6 +1328,9 @@ when ($USE_SYSTEM_JDK) { JDK_RESOURCE_PEERDIR= } otherwise { + when ($JDK_REAL_VERSION == "23") { + JDK_RESOURCE_PEERDIR=build/platform/java/jdk/jdk23 build/platform/java/jdk/jdk17 + } when ($JDK_REAL_VERSION == "22") { JDK_RESOURCE_PEERDIR=build/platform/java/jdk/jdk22 build/platform/java/jdk/jdk17 } @@ -1331,6 +1350,9 @@ otherwise { } # tag:java-specific +when ($JDK_REAL_VERSION == "23") { + UBERJAR_RESOURCE_PEERDIR=build/platform/java/uberjar/uberjar23 +} when ($JDK_REAL_VERSION == "22") { UBERJAR_RESOURCE_PEERDIR=build/platform/java/uberjar/uberjar22 } @@ -1353,6 +1375,9 @@ when ($USE_SYSTEM_JDK) { JDK_RESOURCE=$USE_SYSTEM_JDK } otherwise { + when ($JDK_REAL_VERSION == "23") { + JDK_RESOURCE=$JDK23_RESOURCE_GLOBAL + } when ($JDK_REAL_VERSION == "22") { JDK_RESOURCE=$JDK22_RESOURCE_GLOBAL } @@ -1398,6 +1423,11 @@ when (!$USE_SYSTEM_ERROR_PRONE) { ERROR_PRONE_PEERDIR=build/platform/java/error_prone/2.30.0 ERROR_PRONE_RESOURCE=$ERROR_PRONE_2_30_0_RESOURCE_GLOBAL } + elsewhen ($JDK_REAL_VERSION == "23") { + ERROR_PRONE_VERSION=2.30.0 + ERROR_PRONE_PEERDIR=build/platform/java/error_prone/2.30.0 + ERROR_PRONE_RESOURCE=$ERROR_PRONE_2_30_0_RESOURCE_GLOBAL + } otherwise { ERROR_PRONE_VERSION=2.7.1 ERROR_PRONE_PEERDIR=build/platform/java/error_prone @@ -1415,6 +1445,9 @@ when ($USE_SYSTEM_UBERJAR) { UBERJAR_RESOURCE=$USE_SYSTEM_UBERJAR } otherwise { + when ($JDK_REAL_VERSION == "23") { + UBERJAR_RESOURCE=$UBERJAR23_RESOURCE_GLOBAL + } when ($JDK_REAL_VERSION == "22") { UBERJAR_RESOURCE=$UBERJAR22_RESOURCE_GLOBAL } @@ -1433,6 +1466,9 @@ otherwise { } # tag:java-specific +when ($JDK_REAL_VERSION == "23") { + WITH_JDK_RESOURCE=$WITH_JDK23_RESOURCE_GLOBAL +} when ($JDK_REAL_VERSION == "22") { WITH_JDK_RESOURCE=$WITH_JDK22_RESOURCE_GLOBAL } @@ -1938,11 +1974,25 @@ macro WITH_KOTLINC_SERIALIZATION() { } # tag:kotlin-specific +WITH_KOTLINC_PLUGIN_DETEKT= +_WITH_KOTLINC_PLUGIN_DETEKT_SEM= +### @usage: WITH_KOTLINC_DETEKT(-flags) +### +### Enable detekt kotlin compiler plugin https://detekt.dev/docs/gettingstarted/compilerplugin/ +macro WITH_KOTLINC_DETEKT(Options...) { + SET_APPEND(KOTLINC_OPTS_VALUE ${pre=-P plugin\:detekt-compiler-plugin\::Options}) + SET(WITH_KOTLINC_PLUGIN_DETEKT yes) + SET(_WITH_KOTLINC_PLUGIN_DETEKT_SEM && with_kotlinc_plugin_detekt) +} + + +# tag:kotlin-specific ### Also search for _KAPT_OPTS and change version there _KOTLIN_VERSION=1.9.24 KOTLIN_VERSION=1.9.24 KOTLIN_BOM_FILE=${ARCADIA_ROOT}/contrib/java/org/jetbrains/kotlin/kotlin-bom/1.9.24/ya.dependency_management.inc GROOVY_VERSION=3.0.5 +DETEKT_VERSION=1.23.7 _KOTLIN_SEM= \ ${_WITH_KOTLIN_SEM} \ @@ -1950,7 +2000,8 @@ _KOTLIN_SEM= \ ${_WITH_KOTLINC_PLUGIN_ALLOPEN_SEM} \ ${_WITH_KOTLINC_PLUGIN_LOMBOK_SEM} \ ${_WITH_KOTLINC_PLUGIN_NOARG_SEM} \ - ${_WITH_KOTLINC_PLUGIN_SERIALIZATION_SEM} + ${_WITH_KOTLINC_PLUGIN_SERIALIZATION_SEM} \ + ${_WITH_KOTLINC_PLUGIN_DETEKT_SEM} _JAVA_PROTO_GRPC_SEM= @@ -2083,6 +2134,11 @@ elsewhen ($JDK_REAL_VERSION == "22") { # see also devtools/ya/jbuild/gen/actions/idea.py KOTLIN_JVM_TARGET=21 } +elsewhen ($JDK_REAL_VERSION == "23") { + # remove when kotlin starts supporting jdk21 bytecode + # see also devtools/ya/jbuild/gen/actions/idea.py + KOTLIN_JVM_TARGET=21 +} otherwise { KOTLIN_JVM_TARGET=$JDK_REAL_VERSION } diff --git a/build/conf/linkers/ld.conf b/build/conf/linkers/ld.conf index eca74c3a7d..bab91a3ea3 100644 --- a/build/conf/linkers/ld.conf +++ b/build/conf/linkers/ld.conf @@ -141,9 +141,8 @@ when ($TIME_TRACE_GRANULARITY && $TIME_TRACE_GRANULARITY != "") { _LINKER_TIME_TRACE_FLAG= when (($TIME_TRACE == "yes" || $LINKER_TIME_TRACE == "yes") && $_DEFAULT_LINKER_ID == "lld") { _LINKER_TIME_TRACE_FLAG=\ - -Wl,--time-trace \ - -Wl,--time-trace-granularity=$_LINKER_TIME_TRACE_GRANULARITY \ - -Wl,--time-trace-file=${rootrel;output;pre=$MODULE_PREFIX;suf=$MODULE_SUFFIX.time_trace.json:REALPRJNAME} + -Wl,--time-trace=${rootrel;output;pre=$MODULE_PREFIX;suf=$MODULE_SUFFIX.time_trace.json:REALPRJNAME} \ + -Wl,--time-trace-granularity=$_LINKER_TIME_TRACE_GRANULARITY } _EXE_FLAGS=\ diff --git a/build/conf/linkers/msvc_linker.conf b/build/conf/linkers/msvc_linker.conf index b2cdfa4cb7..7bba3060ad 100644 --- a/build/conf/linkers/msvc_linker.conf +++ b/build/conf/linkers/msvc_linker.conf @@ -1,5 +1,5 @@ -LINK_EXE_CMD=${_WINE_LINK_PREFIX} ${_MSVC_LINK_OLD} -LINK_LIB_CMD=${_WINE_LIB_PREFIX} ${_MSVC_LIB_OLD} +LINK_EXE_CMD=${_WINE_LINK_PREFIX} ${_MSVC_LINK} +LINK_LIB_CMD=${_WINE_LIB_PREFIX} ${_MSVC_LIB} _MSVC_ARCH= when ($ARCH_I386 == "yes" || $ARCH_I686 == "yes") { @@ -46,9 +46,9 @@ when (!$NO_DEBUGINFO || $NO_DEBUGINFO == "no" || $NO_DEBUGINFO == "0") { _FLAGS_COMMON_LIBPATHS= when ($IDE_MSVS != "yes") { _FLAGS_COMMON_LIBPATHS=\ - /LIBPATH:\"${quo:_MSVC_TC_KIT_LIBS}/um/$_MSVC_ARCH\" \ - /LIBPATH:\"${quo:_MSVC_TC_KIT_LIBS}/ucrt/$_MSVC_ARCH\" \ - /LIBPATH:\"${quo:_MSVC_TC_VC_ROOT}/lib/$_MSVC_ARCH\" + /LIBPATH:\""${_MSVC_TC_KIT_LIBS}/um/$_MSVC_ARCH"\" \ + /LIBPATH:\""${_MSVC_TC_KIT_LIBS}/ucrt/$_MSVC_ARCH"\" \ + /LIBPATH:\""${_MSVC_TC_VC_ROOT}/lib/$_MSVC_ARCH"\" } _FLAGS_COMMON =/NOLOGO /ERRORREPORT:PROMPT /SUBSYSTEM:CONSOLE /TLBID:1 $MSVC_DYNAMICBASE /NXCOMPAT @@ -107,7 +107,7 @@ OBJADDE_LIB_GLOBAL= LINK_EXE_FLAGS=$LINK_EXE_FLAGS_PER_TYPE LINK_EXE_VFS_FLAGS= -when ($_UNDER_WINE_LINK != "yes") { +when ($_UNDER_WINE_LINK != "yes" && $USE_MSVC_LINKER != "yes") { LINK_EXE_VFS_FLAGS=/vfsoverlay:${BINDIR}/vfsoverlay.yaml } LINK_EXE_FLAGS+=${LINK_EXE_VFS_FLAGS} @@ -118,7 +118,7 @@ GENERATE_COMMON_LIB_VFS_CMD=\ ${_FLAGS_COMMON_LIBPATHS} GENERATE_COMMON_LIB_VFS= -when ($_UNDER_WINE_LINK != "yes") { +when ($_UNDER_WINE_LINK != "yes" && $USE_MSVC_LINKER != "yes") { GENERATE_COMMON_LIB_VFS=${GENERATE_COMMON_LIB_VFS_CMD} } diff --git a/build/conf/python.conf b/build/conf/python.conf index ec03c9bebf..eaa2c402c0 100644 --- a/build/conf/python.conf +++ b/build/conf/python.conf @@ -179,9 +179,10 @@ macro NO_YMAKE_PYTHON3() { SET(YMAKE_PYTHON3_PEERDIR) } +PYTHON_LINTERS_DEFAULT_CONFIGS=build/config/tests/py_style/default_configs.json + FLAKE_EXTRA_PARAMS= FLAKE_MIGRATIONS_CONFIG= -FLAKE_CONFIG_FILES=build/config/tests/flake8/flake8.conf $FLAKE_MIGRATIONS_CONFIG when ($DISABLE_FLAKE8_MIGRATIONS == "yes") { FLAKE_EXTRA_PARAMS="DISABLE_FLAKE8_MIGRATIONS=yes" } @@ -265,24 +266,38 @@ multimodule PY3_PROGRAM { } } +# tag:python-specific tag:internal +### @usage: _ADD_PY_LINTER_CHECK(NAME name LINTER linter [DEPENDS deps] CONFIGS configs_file [GLOBAL_RESOURCES gr] [FILE_PROCESSING_TIME fpt] [EXTRA_PARAMS params] [PROJECT_TO_CONFIG_MAP pcm] [FLAKE_MIGRATIONS_CONFIG fmc] [CUSTOM_CONFIG cc]) +### +### Triggers respective plugin +macro _ADD_PY_LINTER_CHECK(Args...) { + SET_APPEND(_MAKEFILE_INCLUDE_LIKE_DEPS ${ARCADIA_ROOT}/${PYTHON_LINTERS_DEFAULT_CONFIGS}) +} + +# tag:python-specific tag:internal +### @usage: STYLE_DUMMY() +### +### Not an actual linter, used for dummy linter demonstration +macro STYLE_DUMMY() { + _ADD_PY_LINTER_CHECK(NAME dummy_linter LINTER tools/dummy_linter/dummy_linter CONFIGS $PYTHON_LINTERS_DEFAULT_CONFIGS) +} + # tag:python-specific tag:test ### @usage: STYLE_PYTHON([pyproject]) ### ### Check python3 sources for style issues using black. -BLACK_CONFIG_FILES= macro STYLE_PYTHON(pyproject...) { - BLACK_CONFIG_FILES=$pyproject build/config/tests/py_style/config.toml - _ADD_PY_LINTER_CHECK(NAME black LINTER tools/black_linter/black_linter FILE_PROCESSING_TIME $BLACK_FILE_PROCESSING_TIME CONFIGS $BLACK_CONFIG_FILES) + _ADD_PY_LINTER_CHECK(NAME black LINTER tools/black_linter/black_linter FILE_PROCESSING_TIME $BLACK_FILE_PROCESSING_TIME CONFIGS $PYTHON_LINTERS_DEFAULT_CONFIGS CUSTOM_CONFIG $pyproject) } # tag:python-specific tag:test ### @usage: STYLE_RUFF() ### ### Check python3 sources for style issues using ruff. -RUFF_CONFIG_PATHS_FILE=build/config/tests/ruff/ruff_config_paths.json +RUFF_PROJECT_TO_CONFIG_MAP=build/config/tests/ruff/ruff_config_paths.json macro STYLE_RUFF() { - SET_APPEND(_MAKEFILE_INCLUDE_LIKE_DEPS ${ARCADIA_ROOT}/${RUFF_CONFIG_PATHS_FILE}) - _ADD_PY_LINTER_CHECK(NAME ruff LINTER tools/ruff_linter/bin/ruff_linter GLOBAL_RESOURCES build/external_resources/ruff CONFIGS $RUFF_CONFIG_PATHS_FILE) + SET_APPEND(_MAKEFILE_INCLUDE_LIKE_DEPS ${ARCADIA_ROOT}/${RUFF_PROJECT_TO_CONFIG_MAP}) + _ADD_PY_LINTER_CHECK(NAME ruff LINTER tools/ruff_linter/bin/ruff_linter GLOBAL_RESOURCES build/external_resources/ruff CONFIGS $PYTHON_LINTERS_DEFAULT_CONFIGS PROJECT_TO_CONFIG_MAP $RUFF_PROJECT_TO_CONFIG_MAP) } # tag:python-specific tag:test @@ -290,7 +305,7 @@ macro STYLE_RUFF() { ### ### Check python3 sources for style issues using flake8. macro STYLE_FLAKE8() { - _ADD_PY_LINTER_CHECK(NAME flake8 LINTER tools/flake8_linter/flake8_linter GLOBAL_RESOURCES build/external_resources/flake8_py3 FILE_PROCESSING_TIME $FLAKE8_FILE_PROCESSING_TIME CONFIGS $FLAKE_CONFIG_FILES EXTRA_PARAMS $FLAKE_EXTRA_PARAMS) + _ADD_PY_LINTER_CHECK(NAME flake8 LINTER tools/flake8_linter/flake8_linter GLOBAL_RESOURCES build/external_resources/flake8_py3 FILE_PROCESSING_TIME $FLAKE8_FILE_PROCESSING_TIME CONFIGS $PYTHON_LINTERS_DEFAULT_CONFIGS FLAKE_MIGRATIONS_CONFIG $FLAKE_MIGRATIONS_CONFIG EXTRA_PARAMS $FLAKE_EXTRA_PARAMS) } # tag:python-specific tag:test @@ -298,7 +313,7 @@ macro STYLE_FLAKE8() { ### ### Check python3 sources for style issues using flake8. macro STYLE_PY2_FLAKE8() { - _ADD_PY_LINTER_CHECK(NAME py2_flake8 LINTER tools/flake8_linter/flake8_linter GLOBAL_RESOURCES build/external_resources/flake8_py2 FILE_PROCESSING_TIME $FLAKE8_FILE_PROCESSING_TIME CONFIGS $FLAKE_CONFIG_FILES EXTRA_PARAMS $FLAKE_EXTRA_PARAMS) + _ADD_PY_LINTER_CHECK(NAME py2_flake8 LINTER tools/flake8_linter/flake8_linter GLOBAL_RESOURCES build/external_resources/flake8_py2 FILE_PROCESSING_TIME $FLAKE8_FILE_PROCESSING_TIME CONFIGS $PYTHON_LINTERS_DEFAULT_CONFIGS FLAKE_MIGRATIONS_CONFIG $FLAKE_MIGRATIONS_CONFIG EXTRA_PARAMS $FLAKE_EXTRA_PARAMS) } # tag:python-specific tag:test diff --git a/build/conf/settings.conf b/build/conf/settings.conf index c77ca51561..880f89940c 100644 --- a/build/conf/settings.conf +++ b/build/conf/settings.conf @@ -14,14 +14,11 @@ FORCE_RESOLVE_MACRO_INCLS=yes ENABLE_RERESOLVE_FOR_GENERATED_FILES=yes REPORT_ALL_DUPSRC=yes DEPS_CACHE_CONTROL_UIDS_CACHE=yes -INCLUDE_BLACKLIST_TO_CONF_HASH=no -INCLUDE_ISOLATED_PROJECTS_TO_CONF_HASH=no -USE_GRAND_BYPASS=no USE_GLOBAL_CMD=yes YMAKE_USE_NEW_UIDS=yes -USE_GRAPH_CHANGES_PREDICTOR=yes PIC_NO_PIE=no FAIL_PY2=no +MAIN_OUTPUT_AS_EXTRA=no when ($YMAKE_USE_OLD_UIDS == "yes") { YMAKE_USE_NEW_UIDS=no @@ -32,7 +29,6 @@ when ($OS_WINDOWS == "yes") { } when ($AUTOCHECK == "yes") { - USE_GRAPH_CHANGES_PREDICTOR=no PIC_NO_PIE=yes } diff --git a/build/config/tests/py_style/config_paths.json b/build/config/tests/py_style/config_paths.json deleted file mode 100644 index 7b3c51669b..0000000000 --- a/build/config/tests/py_style/config_paths.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "": "build/config/tests/py_style/config.toml", - "devtools/contrib/piglet/template_generator/": "maps/infra/pyproject.toml", - "devtools/dwh/intrasearch/utils/": "devtools/dwh/pyproject.toml", - "devtools/rnd/": "devtools/rnd/pyproject.toml", - "devtools/yamaker/": "devtools/yamaker/pyproject.toml", - "dj/services/arcanum/": "dj/services/arcanum/pyproject.toml", - "intranet/search/": "intranet/search/pyproject.toml", - "maps/": "maps/infra/pyproject.toml", - "ml/tensorflow/models/userbert/pipeline/multipart/data/": "ml/tensorflow/models/userbert/pipeline/multipart/data/pyproject.toml", - "partner/python/": "partner/python/pyproject.toml" -} diff --git a/build/config/tests/py_style/default_configs.json b/build/config/tests/py_style/default_configs.json new file mode 100644 index 0000000000..1c8f2ab585 --- /dev/null +++ b/build/config/tests/py_style/default_configs.json @@ -0,0 +1,6 @@ +{ + "flake8": "build/config/tests/flake8/flake8.conf", + "py2_flake8": "build/config/tests/flake8/flake8.conf", + "black": "build/config/tests/py_style/config.toml", + "dummy_linter": "build/config/tests/py_style/dummy_linter_config.json" +} diff --git a/build/config/tests/py_style/dummy_linter_config.json b/build/config/tests/py_style/dummy_linter_config.json new file mode 100644 index 0000000000..0967ef424b --- /dev/null +++ b/build/config/tests/py_style/dummy_linter_config.json @@ -0,0 +1 @@ +{} diff --git a/build/config/tests/py_style/ya.make b/build/config/tests/py_style/ya.make index 37dcd2ec82..6a6a287558 100644 --- a/build/config/tests/py_style/ya.make +++ b/build/config/tests/py_style/ya.make @@ -2,7 +2,6 @@ LIBRARY() RESOURCE_FILES( config.toml - config_paths.json ) END() diff --git a/build/export_generators/cmake/cmake/conan-profiles/android.arm64.profile b/build/export_generators/cmake/cmake/conan-profiles/android.arm64.profile index 6147d99225..0f08282c77 100644 --- a/build/export_generators/cmake/cmake/conan-profiles/android.arm64.profile +++ b/build/export_generators/cmake/cmake/conan-profiles/android.arm64.profile @@ -11,5 +11,3 @@ os.api_level=21 [tool_requires] android-ndk/r25 [options] -[env] - diff --git a/build/export_generators/cmake/cmake/conan-profiles/android.armv7.profile b/build/export_generators/cmake/cmake/conan-profiles/android.armv7.profile index 57d8f55785..9419ed1ba8 100644 --- a/build/export_generators/cmake/cmake/conan-profiles/android.armv7.profile +++ b/build/export_generators/cmake/cmake/conan-profiles/android.armv7.profile @@ -11,5 +11,3 @@ os.api_level=21 [tool_requires] android-ndk/r25 [options] -[env] - diff --git a/build/export_generators/cmake/cmake/conan-profiles/android.x86.profile b/build/export_generators/cmake/cmake/conan-profiles/android.x86.profile index d8b73b992c..101b70af76 100644 --- a/build/export_generators/cmake/cmake/conan-profiles/android.x86.profile +++ b/build/export_generators/cmake/cmake/conan-profiles/android.x86.profile @@ -11,5 +11,3 @@ os.api_level=21 [tool_requires] android-ndk/r25 [options] -[env] - diff --git a/build/export_generators/cmake/cmake/conan-profiles/android.x86_64.profile b/build/export_generators/cmake/cmake/conan-profiles/android.x86_64.profile index 50ad337afa..7cb2b0c907 100644 --- a/build/export_generators/cmake/cmake/conan-profiles/android.x86_64.profile +++ b/build/export_generators/cmake/cmake/conan-profiles/android.x86_64.profile @@ -11,5 +11,3 @@ os.api_level=21 [tool_requires] android-ndk/r25 [options] -[env] - diff --git a/build/export_generators/cmake/cmake/conan-profiles/linux.aarch64.profile b/build/export_generators/cmake/cmake/conan-profiles/linux.aarch64.profile index 6945b1d777..533bfbdd1e 100644 --- a/build/export_generators/cmake/cmake/conan-profiles/linux.aarch64.profile +++ b/build/export_generators/cmake/cmake/conan-profiles/linux.aarch64.profile @@ -1,6 +1,6 @@ -include(default) +{% set target_host = 'aarch64-linux-gnu' %} -target_host=aarch64-linux-gnu +include(default) [settings] arch=armv8 @@ -12,17 +12,18 @@ compiler.cppstd=20 os=Linux [options] -[env] -CONAN_CMAKE_FIND_ROOT_PATH=/usr/$target_host -CONAN_CMAKE_SYSROOT=/usr/$target_host -SYSROOT=/usr/$target_host -CC=$target_host-gcc -CXX=$target_host-g++ -CXXFLAGS="-I/usr/$target_host/include/" -CFLAGS="-I/usr/$target_host/include/" -CHOST=$target_host -AR=$target_host-ar -AS=$target_host-as -RANLIB=$target_host-ranlib -LD=$target_host-ld -STRIP=$target_host-strip +[buildenv] +PATH=+(path)/usr/{{target_host}}/bin +CONAN_CMAKE_FIND_ROOT_PATH=/usr/{{target_host}} +CONAN_CMAKE_SYSROOT=/usr/{{target_host}} +SYSROOT=/usr/{{target_host}} +CC={{target_host}}-gcc +CXX={{target_host}}-g++ +CXXFLAGS=-I/usr/{{target_host}}/include/ +CFLAGS=-I/usr/{{target_host}}/include/ +CHOST={{target_host}} +AR={{target_host}}-ar +AS={{target_host}}-as +RANLIB={{target_host}}-ranlib +LD={{target_host}}-ld +STRIP={{target_host}}-strip diff --git a/build/export_generators/cmake/cmake/conan-profiles/linux.ppc64le.profile b/build/export_generators/cmake/cmake/conan-profiles/linux.ppc64le.profile index 5ecd24d6ce..e688123b00 100644 --- a/build/export_generators/cmake/cmake/conan-profiles/linux.ppc64le.profile +++ b/build/export_generators/cmake/cmake/conan-profiles/linux.ppc64le.profile @@ -1,6 +1,6 @@ -include(default) +{% set target_host = 'powerpc64le-linux-gnu' %} -target_host=powerpc64le-linux-gnu +include(default) [settings] arch=ppc64le @@ -12,17 +12,18 @@ compiler.cppstd=20 os=Linux [options] -[env] -CONAN_CMAKE_FIND_ROOT_PATH=/usr/$target_host -CONAN_CMAKE_SYSROOT=/usr/$target_host -SYSROOT=/usr/$target_host -CC=$target_host-gcc -CXX=$target_host-g++ -CXXFLAGS="-I/usr/$target_host/include/" -CFLAGS="-I/usr/$target_host/include/" -CHOST=$target_host -AR=$target_host-ar -AS=$target_host-as -RANLIB=$target_host-ranlib -LD=$target_host-ld -STRIP=$target_host-strip +[buildenv] +PATH=+(path)/usr/{{target_host}}/bin +CONAN_CMAKE_FIND_ROOT_PATH=/usr/{{target_host}} +CONAN_CMAKE_SYSROOT=/usr/{{target_host}} +SYSROOT=/usr/{{target_host}} +CC={{target_host}}-gcc +CXX={{target_host}}-g++ +CXXFLAGS=-I/usr/{{target_host}}/include/ +CFLAGS=-I/usr/{{target_host}}/include/ +CHOST={{target_host}} +AR={{target_host}}-ar +AS={{target_host}}-as +RANLIB={{target_host}}-ranlib +LD={{target_host}}-ld +STRIP={{target_host}}-strip diff --git a/build/export_generators/cmake/cmake/conan-profiles/macos.arm64.profile b/build/export_generators/cmake/cmake/conan-profiles/macos.arm64.profile index 52ba46ab97..f215fd1592 100644 --- a/build/export_generators/cmake/cmake/conan-profiles/macos.arm64.profile +++ b/build/export_generators/cmake/cmake/conan-profiles/macos.arm64.profile @@ -8,4 +8,3 @@ compiler.version=14 compiler.cppstd=20 os=Macos [options] -[env] diff --git a/build/export_generators/cmake/cmake/conan-profiles/macos.x86_64.profile b/build/export_generators/cmake/cmake/conan-profiles/macos.x86_64.profile index 3370b88548..a741d197e1 100644 --- a/build/export_generators/cmake/cmake/conan-profiles/macos.x86_64.profile +++ b/build/export_generators/cmake/cmake/conan-profiles/macos.x86_64.profile @@ -8,4 +8,3 @@ compiler.version=14 compiler.cppstd=20 os=Macos [options] -[env] diff --git a/build/external_resources/ymake/public.resources.json b/build/external_resources/ymake/public.resources.json index a4331a44f9..8e38b4a9ce 100644 --- a/build/external_resources/ymake/public.resources.json +++ b/build/external_resources/ymake/public.resources.json @@ -1,19 +1,19 @@ { "by_platform": { "darwin": { - "uri": "sbr:7095381929" + "uri": "sbr:7129654082" }, "darwin-arm64": { - "uri": "sbr:7095381018" + "uri": "sbr:7129653224" }, "linux": { - "uri": "sbr:7095384078" + "uri": "sbr:7129655749" }, "linux-aarch64": { - "uri": "sbr:7095380204" + "uri": "sbr:7129652122" }, "win32-clang-cl": { - "uri": "sbr:7095382771" + "uri": "sbr:7129654809" } } } diff --git a/build/external_resources/ymake/resources.json b/build/external_resources/ymake/resources.json index 4e758b6909..88af245122 100644 --- a/build/external_resources/ymake/resources.json +++ b/build/external_resources/ymake/resources.json @@ -1,19 +1,19 @@ { "by_platform": { "darwin": { - "uri": "sbr:7095368415" + "uri": "sbr:7129644338" }, "darwin-arm64": { - "uri": "sbr:7095367498" + "uri": "sbr:7129644098" }, "linux": { - "uri": "sbr:7095370400" + "uri": "sbr:7129644725" }, "linux-aarch64": { - "uri": "sbr:7095366278" + "uri": "sbr:7129643797" }, "win32-clang-cl": { - "uri": "sbr:7095369426" + "uri": "sbr:7129644525" } } } diff --git a/build/mapping.conf.json b/build/mapping.conf.json index d2e418a62a..09a1a2e8bc 100644 --- a/build/mapping.conf.json +++ b/build/mapping.conf.json @@ -398,6 +398,8 @@ "7017113894": "https://devtools-registry.s3.yandex.net/7017113894", "7083801854": "https://devtools-registry.s3.yandex.net/7083801854", "7083802536": "https://devtools-registry.s3.yandex.net/7083802536", + "7120328912": "https://devtools-registry.s3.yandex.net/7120328912", + "7120353733": "https://devtools-registry.s3.yandex.net/7120353733", "5486731632": "https://devtools-registry.s3.yandex.net/5486731632", "5514350352": "https://devtools-registry.s3.yandex.net/5514350352", "5514360398": "https://devtools-registry.s3.yandex.net/5514360398", @@ -547,6 +549,7 @@ "7070261862": "https://devtools-registry.s3.yandex.net/7070261862", "7083582755": "https://devtools-registry.s3.yandex.net/7083582755", "7095381929": "https://devtools-registry.s3.yandex.net/7095381929", + "7129654082": "https://devtools-registry.s3.yandex.net/7129654082", "5766171800": "https://devtools-registry.s3.yandex.net/5766171800", "5805430761": "https://devtools-registry.s3.yandex.net/5805430761", "5829025456": "https://devtools-registry.s3.yandex.net/5829025456", @@ -586,6 +589,7 @@ "7070261345": "https://devtools-registry.s3.yandex.net/7070261345", "7083581957": "https://devtools-registry.s3.yandex.net/7083581957", "7095381018": "https://devtools-registry.s3.yandex.net/7095381018", + "7129653224": "https://devtools-registry.s3.yandex.net/7129653224", "5766173070": "https://devtools-registry.s3.yandex.net/5766173070", "5805432830": "https://devtools-registry.s3.yandex.net/5805432830", "5829031598": "https://devtools-registry.s3.yandex.net/5829031598", @@ -625,6 +629,7 @@ "7070262933": "https://devtools-registry.s3.yandex.net/7070262933", "7083584557": "https://devtools-registry.s3.yandex.net/7083584557", "7095384078": "https://devtools-registry.s3.yandex.net/7095384078", + "7129655749": "https://devtools-registry.s3.yandex.net/7129655749", "5766171341": "https://devtools-registry.s3.yandex.net/5766171341", "5805430188": "https://devtools-registry.s3.yandex.net/5805430188", "5829023352": "https://devtools-registry.s3.yandex.net/5829023352", @@ -664,6 +669,7 @@ "7070260816": "https://devtools-registry.s3.yandex.net/7070260816", "7083581112": "https://devtools-registry.s3.yandex.net/7083581112", "7095380204": "https://devtools-registry.s3.yandex.net/7095380204", + "7129652122": "https://devtools-registry.s3.yandex.net/7129652122", "5766172695": "https://devtools-registry.s3.yandex.net/5766172695", "5805432230": "https://devtools-registry.s3.yandex.net/5805432230", "5829029743": "https://devtools-registry.s3.yandex.net/5829029743", @@ -703,6 +709,7 @@ "7070262364": "https://devtools-registry.s3.yandex.net/7070262364", "7083583698": "https://devtools-registry.s3.yandex.net/7083583698", "7095382771": "https://devtools-registry.s3.yandex.net/7095382771", + "7129654809": "https://devtools-registry.s3.yandex.net/7129654809", "4307890075": "https://devtools-registry.s3.yandex.net/4307890075", "5517245192": "https://devtools-registry.s3.yandex.net/5517245192", "4307901240": "https://devtools-registry.s3.yandex.net/4307901240", @@ -740,6 +747,7 @@ "6941864137": "https://devtools-registry.s3.yandex.net/6941864137", "6956612793": "https://devtools-registry.s3.yandex.net/6956612793", "6957908969": "https://devtools-registry.s3.yandex.net/6957908969", + "7101981380": "https://devtools-registry.s3.yandex.net/7101981380", "5776379446": "https://devtools-registry.s3.yandex.net/5776379446", "5777100597": "https://devtools-registry.s3.yandex.net/5777100597", "5909067709": "https://devtools-registry.s3.yandex.net/5909067709", @@ -750,6 +758,7 @@ "6941861330": "https://devtools-registry.s3.yandex.net/6941861330", "6956611400": "https://devtools-registry.s3.yandex.net/6956611400", "6957906780": "https://devtools-registry.s3.yandex.net/6957906780", + "7101978818": "https://devtools-registry.s3.yandex.net/7101978818", "5776375952": "https://devtools-registry.s3.yandex.net/5776375952", "5777098178": "https://devtools-registry.s3.yandex.net/5777098178", "5909065014": "https://devtools-registry.s3.yandex.net/5909065014", @@ -760,6 +769,7 @@ "6941857195": "https://devtools-registry.s3.yandex.net/6941857195", "6956608703": "https://devtools-registry.s3.yandex.net/6956608703", "6957901545": "https://devtools-registry.s3.yandex.net/6957901545", + "7101973818": "https://devtools-registry.s3.yandex.net/7101973818", "5776374505": "https://devtools-registry.s3.yandex.net/5776374505", "5777096988": "https://devtools-registry.s3.yandex.net/5777096988", "5909063641": "https://devtools-registry.s3.yandex.net/5909063641", @@ -770,6 +780,7 @@ "6941855347": "https://devtools-registry.s3.yandex.net/6941855347", "6956607525": "https://devtools-registry.s3.yandex.net/6956607525", "6957899481": "https://devtools-registry.s3.yandex.net/6957899481", + "7101971416": "https://devtools-registry.s3.yandex.net/7101971416", "5776377955": "https://devtools-registry.s3.yandex.net/5776377955", "5777099502": "https://devtools-registry.s3.yandex.net/5777099502", "5909066324": "https://devtools-registry.s3.yandex.net/5909066324", @@ -780,6 +791,7 @@ "6941859166": "https://devtools-registry.s3.yandex.net/6941859166", "6956610092": "https://devtools-registry.s3.yandex.net/6956610092", "6957903888": "https://devtools-registry.s3.yandex.net/6957903888", + "7101976574": "https://devtools-registry.s3.yandex.net/7101976574", "6391354461": "https://devtools-registry.s3.yandex.net/6391354461", "6990868751": "https://devtools-registry.s3.yandex.net/6990868751", "6990860705": "https://devtools-registry.s3.yandex.net/6990860705", @@ -1277,6 +1289,8 @@ "7017113894": "devtools/ya/test/programs/test_tool/bin/test_tool for linux", "7083801854": "devtools/ya/test/programs/test_tool/bin/test_tool for linux", "7083802536": "devtools/ya/test/programs/test_tool/bin/test_tool for linux", + "7120328912": "devtools/ya/test/programs/test_tool/bin/test_tool for linux", + "7120353733": "devtools/ya/test/programs/test_tool/bin/test_tool for linux", "5486731632": "devtools/ya/test/programs/test_tool/bin3/test_tool3 for linux", "5514350352": "devtools/ya/test/programs/test_tool/bin3/test_tool3 for linux", "5514360398": "devtools/ya/test/programs/test_tool/bin3/test_tool3 for linux", @@ -1426,6 +1440,7 @@ "7070261862": "devtools/ymake/bin/ymake for darwin", "7083582755": "devtools/ymake/bin/ymake for darwin", "7095381929": "devtools/ymake/bin/ymake for darwin", + "7129654082": "devtools/ymake/bin/ymake for darwin", "5766171800": "devtools/ymake/bin/ymake for darwin-arm64", "5805430761": "devtools/ymake/bin/ymake for darwin-arm64", "5829025456": "devtools/ymake/bin/ymake for darwin-arm64", @@ -1465,6 +1480,7 @@ "7070261345": "devtools/ymake/bin/ymake for darwin-arm64", "7083581957": "devtools/ymake/bin/ymake for darwin-arm64", "7095381018": "devtools/ymake/bin/ymake for darwin-arm64", + "7129653224": "devtools/ymake/bin/ymake for darwin-arm64", "5766173070": "devtools/ymake/bin/ymake for linux", "5805432830": "devtools/ymake/bin/ymake for linux", "5829031598": "devtools/ymake/bin/ymake for linux", @@ -1504,6 +1520,7 @@ "7070262933": "devtools/ymake/bin/ymake for linux", "7083584557": "devtools/ymake/bin/ymake for linux", "7095384078": "devtools/ymake/bin/ymake for linux", + "7129655749": "devtools/ymake/bin/ymake for linux", "5766171341": "devtools/ymake/bin/ymake for linux-aarch64", "5805430188": "devtools/ymake/bin/ymake for linux-aarch64", "5829023352": "devtools/ymake/bin/ymake for linux-aarch64", @@ -1543,6 +1560,7 @@ "7070260816": "devtools/ymake/bin/ymake for linux-aarch64", "7083581112": "devtools/ymake/bin/ymake for linux-aarch64", "7095380204": "devtools/ymake/bin/ymake for linux-aarch64", + "7129652122": "devtools/ymake/bin/ymake for linux-aarch64", "5766172695": "devtools/ymake/bin/ymake for win32-clang-cl", "5805432230": "devtools/ymake/bin/ymake for win32-clang-cl", "5829029743": "devtools/ymake/bin/ymake for win32-clang-cl", @@ -1582,6 +1600,7 @@ "7070262364": "devtools/ymake/bin/ymake for win32-clang-cl", "7083583698": "devtools/ymake/bin/ymake for win32-clang-cl", "7095382771": "devtools/ymake/bin/ymake for win32-clang-cl", + "7129654809": "devtools/ymake/bin/ymake for win32-clang-cl", "4307890075": "flake8_linter for linux", "5517245192": "flake8_linter for linux", "4307901240": "flake8_linter for linux-aarch64", @@ -1619,6 +1638,7 @@ "6941864137": "none-none-none-result_resources/jdk-darwin-aarch64.yandex.tgz", "6956612793": "none-none-none-result_resources/jdk-darwin-aarch64.yandex.tgz", "6957908969": "none-none-none-result_resources/jdk-darwin-aarch64.yandex.tgz", + "7101981380": "none-none-none-result_resources/jdk-darwin-aarch64.yandex.tgz", "5776379446": "none-none-none-result_resources/jdk-darwin-x86_64.yandex.tgz", "5777100597": "none-none-none-result_resources/jdk-darwin-x86_64.yandex.tgz", "5909067709": "none-none-none-result_resources/jdk-darwin-x86_64.yandex.tgz", @@ -1629,6 +1649,7 @@ "6941861330": "none-none-none-result_resources/jdk-darwin-x86_64.yandex.tgz", "6956611400": "none-none-none-result_resources/jdk-darwin-x86_64.yandex.tgz", "6957906780": "none-none-none-result_resources/jdk-darwin-x86_64.yandex.tgz", + "7101978818": "none-none-none-result_resources/jdk-darwin-x86_64.yandex.tgz", "5776375952": "none-none-none-result_resources/jdk-linux-aarch64.yandex.tgz", "5777098178": "none-none-none-result_resources/jdk-linux-aarch64.yandex.tgz", "5909065014": "none-none-none-result_resources/jdk-linux-aarch64.yandex.tgz", @@ -1639,6 +1660,7 @@ "6941857195": "none-none-none-result_resources/jdk-linux-aarch64.yandex.tgz", "6956608703": "none-none-none-result_resources/jdk-linux-aarch64.yandex.tgz", "6957901545": "none-none-none-result_resources/jdk-linux-aarch64.yandex.tgz", + "7101973818": "none-none-none-result_resources/jdk-linux-aarch64.yandex.tgz", "5776374505": "none-none-none-result_resources/jdk-linux-x86_64.yandex.tgz", "5777096988": "none-none-none-result_resources/jdk-linux-x86_64.yandex.tgz", "5909063641": "none-none-none-result_resources/jdk-linux-x86_64.yandex.tgz", @@ -1649,6 +1671,7 @@ "6941855347": "none-none-none-result_resources/jdk-linux-x86_64.yandex.tgz", "6956607525": "none-none-none-result_resources/jdk-linux-x86_64.yandex.tgz", "6957899481": "none-none-none-result_resources/jdk-linux-x86_64.yandex.tgz", + "7101971416": "none-none-none-result_resources/jdk-linux-x86_64.yandex.tgz", "5776377955": "none-none-none-result_resources/jdk-windows-amd64.yandex.tgz", "5777099502": "none-none-none-result_resources/jdk-windows-amd64.yandex.tgz", "5909066324": "none-none-none-result_resources/jdk-windows-amd64.yandex.tgz", @@ -1659,6 +1682,7 @@ "6941859166": "none-none-none-result_resources/jdk-windows-amd64.yandex.tgz", "6956610092": "none-none-none-result_resources/jdk-windows-amd64.yandex.tgz", "6957903888": "none-none-none-result_resources/jdk-windows-amd64.yandex.tgz", + "7101976574": "none-none-none-result_resources/jdk-windows-amd64.yandex.tgz", "6391354461": "none-none-none-result_resources/protoc-linux-x86_64.tgz", "6990868751": "none-none-none-sandbox/backup/3527d100-e2d0-4b0e-bb7a-905010853d98/yfm-docs.tar", "6990860705": "none-none-none-sandbox/backup/d386643e-58f8-43e1-8760-341d73801df8/yfm-docs.tar", diff --git a/build/platform/java/jdk/ya.make b/build/platform/java/jdk/ya.make index 983dfbba15..a978c66900 100644 --- a/build/platform/java/jdk/ya.make +++ b/build/platform/java/jdk/ya.make @@ -3,6 +3,9 @@ RESOURCES_LIBRARY() # Add new JDK to build/plugins/java.py (2 times) IF(USE_SYSTEM_JDK) MESSAGE(WARNING DEFAULT_JDK are disabled) +ELSEIF(JDK_REAL_VERSION == "23") + DECLARE_EXTERNAL_HOST_RESOURCES_BUNDLE_BY_JSON(JDK_DEFAULT jdk23/jdk.json) + SET_RESOURCE_URI_FROM_JSON(WITH_JDK_URI jdk23/jdk.json) ELSEIF(JDK_REAL_VERSION == "22") DECLARE_EXTERNAL_HOST_RESOURCES_BUNDLE_BY_JSON(JDK_DEFAULT jdk22/jdk.json) SET_RESOURCE_URI_FROM_JSON(WITH_JDK_URI jdk22/jdk.json) diff --git a/build/platform/lld/lld14.json b/build/platform/lld/lld14.json deleted file mode 100644 index d5638f4581..0000000000 --- a/build/platform/lld/lld14.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "by_platform": { - "darwin-arm64": { - "uri": "sbr:6683655545" - }, - "darwin-x86_64": { - "uri": "sbr:6683694178" - }, - "linux-aarch64": { - "uri": "sbr:6683720769" - }, - "linux-x86_64": { - "uri": "sbr:6683733162" - } - } -} - diff --git a/build/platform/lld/ya.make b/build/platform/lld/ya.make index 7ea02fc2ec..b3459211f2 100644 --- a/build/platform/lld/ya.make +++ b/build/platform/lld/ya.make @@ -5,12 +5,8 @@ DEFAULT(LLD_VERSION ${CLANG_VER}) TOOLCHAIN(lld) VERSION(${LLD_VERSION}) -IF (LLD_VERSION == 14) - DECLARE_EXTERNAL_HOST_RESOURCES_BUNDLE_BY_JSON(LLD_ROOT lld14.json) -ELSE() - # fallback on latest version - DECLARE_EXTERNAL_HOST_RESOURCES_BUNDLE_BY_JSON(LLD_ROOT lld16.json) -ENDIF() +# lld16 is the only supported version at the time +DECLARE_EXTERNAL_HOST_RESOURCES_BUNDLE_BY_JSON(LLD_ROOT lld16.json) IF (OS_ANDROID) # Use LLD shipped with Android NDK. @@ -52,8 +48,6 @@ ELSEIF (OS_DARWIN OR OS_IOS) LDFLAGS( -fuse-ld=${LLD_ROOT_RESOURCE_GLOBAL}/bin/ld64.lld ) - ELSEIF (XCODE) - LDFLAGS(-DYA_XCODE) ELSE() LDFLAGS( -fuse-ld=lld diff --git a/build/platform/test_tool/host.ya.make.inc b/build/platform/test_tool/host.ya.make.inc index f44f5f2df2..69a66014eb 100644 --- a/build/platform/test_tool/host.ya.make.inc +++ b/build/platform/test_tool/host.ya.make.inc @@ -1,12 +1,12 @@ IF (HOST_OS_DARWIN AND HOST_ARCH_X86_64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7083800804) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7120352053) ELSEIF (HOST_OS_DARWIN AND HOST_ARCH_ARM64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7083799383) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7120351367) ELSEIF (HOST_OS_LINUX AND HOST_ARCH_X86_64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7083802536) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7120353733) ELSEIF (HOST_OS_LINUX AND HOST_ARCH_AARCH64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7083798661) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7120350948) ELSEIF (HOST_OS_WINDOWS AND HOST_ARCH_X86_64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7083801684) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7120352672) ENDIF() diff --git a/build/platform/test_tool/host_os.ya.make.inc b/build/platform/test_tool/host_os.ya.make.inc index 43fc98f8c7..f82b95639d 100644 --- a/build/platform/test_tool/host_os.ya.make.inc +++ b/build/platform/test_tool/host_os.ya.make.inc @@ -1,12 +1,12 @@ IF (HOST_OS_DARWIN AND HOST_ARCH_X86_64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7083800164) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7120327883) ELSEIF (HOST_OS_DARWIN AND HOST_ARCH_ARM64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7083799217) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7120327375) ELSEIF (HOST_OS_LINUX AND HOST_ARCH_X86_64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7083801854) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7120328912) ELSEIF (HOST_OS_LINUX AND HOST_ARCH_AARCH64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7083798603) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7120326904) ELSEIF (HOST_OS_WINDOWS AND HOST_ARCH_X86_64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7083801030) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7120328512) ENDIF() diff --git a/build/plugins/_common.py b/build/plugins/_common.py index a2b19b5b3d..e40798aaf8 100644 --- a/build/plugins/_common.py +++ b/build/plugins/_common.py @@ -22,6 +22,20 @@ def lazy(func): return wrapper +def cache_by_second_arg(func): + result = {} + + def wrapper(arg0, arg1, *args, **kwargs): + try: + return result[arg1] + except KeyError: + result[arg1] = func(arg0, arg1, *args, **kwargs) + + return result[arg1] + + return wrapper + + def pathid(path): return six.ensure_str(base64.b32encode(hashlib.md5(six.ensure_binary(path)).digest()).lower().strip(b'=')) diff --git a/build/plugins/_dart_fields.py b/build/plugins/_dart_fields.py index 51349aa4ae..34f13579a2 100644 --- a/build/plugins/_dart_fields.py +++ b/build/plugins/_dart_fields.py @@ -259,13 +259,28 @@ def _get_ts_test_data_dirs(unit): ) -@_common.lazy +@_common.cache_by_second_arg def get_linter_configs(unit, config_paths): rel_config_path = _common.rootrel_arc_src(config_paths, unit) arc_config_path = unit.resolve_arc_path(rel_config_path) abs_config_path = unit.resolve(arc_config_path) with open(abs_config_path, 'r') as fd: - return list(json.load(fd).values()) + return json.load(fd) + + +def _reference_group_var(varname: str, extensions: list[str] | None = None) -> str: + if extensions is None: + return f'"${{join=\\;:{varname}}}"' + + return serialize_list(f'${{ext={ext};join=\\;:{varname}}}' for ext in extensions) + + +def assert_file_exists(unit, path): + path = unit.resolve(SOURCE_ROOT_SHORT + path) + if not os.path.exists(path): + message = 'File {} is not found'.format(path) + ymake.report_configure_error(message) + raise DartValueError() class AndroidApkTestActivity: @@ -552,25 +567,56 @@ class LintConfigs: KEY = 'LINT-CONFIGS' @classmethod - def value(cls, unit, flat_args, spec_args): + def python_configs(cls, unit, flat_args, spec_args): resolved_configs = [] - configs = spec_args.get('CONFIGS', []) - for cfg in configs: - filename = unit.resolve(SOURCE_ROOT_SHORT + cfg) - if not os.path.exists(filename): - message = 'Configuration file {} is not found'.format(filename) - raise DartValueError(message) - resolved_configs.append(cfg) - if os.path.splitext(filename)[-1] == '.json': - cfgs = get_linter_configs(unit, cfg) - for c in cfgs: - filename = unit.resolve(SOURCE_ROOT_SHORT + c) - if not os.path.exists(filename): - message = 'Configuration file {} is not found'.format(filename) - raise DartValueError(message) - resolved_configs.append(c) + + project_to_config_map = spec_args.get('PROJECT_TO_CONFIG_MAP', []) + if project_to_config_map: + # ruff, TODO rewrite once custom configs migrated to autoincludes scheme + project_to_config_map = project_to_config_map[0] + assert_file_exists(unit, project_to_config_map) + resolved_configs.append(project_to_config_map) + cfgs = get_linter_configs(unit, project_to_config_map).values() + for c in cfgs: + assert_file_exists(unit, c) + resolved_configs.append(c) + return {cls.KEY: serialize_list(resolved_configs)} + + custom_config = spec_args.get('CUSTOM_CONFIG', []) + if custom_config: + # black if custom config is passed + # TODO rewrite once custom configs migrated to autoincludes scheme + custom_config = custom_config[0] + assert_file_exists(unit, custom_config) + resolved_configs.append(custom_config) + return {cls.KEY: serialize_list(resolved_configs)} + + config = spec_args['CONFIGS'][0] + # black without custom config or flake8, using default configs file + assert_file_exists(unit, config) + name = spec_args['NAME'][0] + cfg = get_linter_configs(unit, config)[name] + assert_file_exists(unit, cfg) + resolved_configs.append(cfg) + if name in ('flake8', 'py2_flake8'): + resolved_configs.extend(spec_args.get('FLAKE_MIGRATIONS_CONFIG', [])) return {cls.KEY: serialize_list(resolved_configs)} + @classmethod + def cpp_configs(cls, unit, flat_args, spec_args): + custom_config = spec_args.get('CUSTOM_CONFIG') + if custom_config: + config = custom_config[0] + assert_file_exists(unit, config) + else: + # file with default configs + config = spec_args.get('CONFIGS')[0] + assert_file_exists(unit, config) + name = spec_args['NAME'][0] + config = get_linter_configs(unit, config)[name] + assert_file_exists(unit, config) + return {cls.KEY: serialize_list([config])} + class LintExtraParams: KEY = 'LINT-EXTRA-PARAMS' @@ -581,7 +627,8 @@ class LintExtraParams: for arg in extra_params: if '=' not in arg: message = 'Wrong EXTRA_PARAMS value: "{}". Values must have format "name=value".'.format(arg) - raise DartValueError(message) + ymake.report_configure_error(message) + raise DartValueError() return {cls.KEY: serialize_list(extra_params)} @@ -600,7 +647,8 @@ class LintName: def value(cls, unit, flat_args, spec_args): lint_name = spec_args['NAME'][0] if lint_name in ('flake8', 'py2_flake8') and (unit.get('DISABLE_FLAKE8') or 'no') == 'yes': - raise DartValueError('Flake8 linting is disabled by `DISABLE_FLAKE8`') + unit.message(['INFO', 'Flake8 linting is disabled by `DISABLE_FLAKE8`']) + raise DartValueError() return {cls.KEY: lint_name} @@ -1144,6 +1192,11 @@ class TestFiles: test_files = serialize_list(test_files) return {cls.KEY: test_files, cls.KEY2: test_files} + @classmethod + def cpp_linter_files(cls, unit, flat_args, spec_args): + files_dart = _reference_group_var("ALL_SRCS", consts.STYLE_CPP_ALL_EXTS) + return {cls.KEY: files_dart, cls.KEY2: files_dart} + class TestEnv: KEY = 'TEST-ENV' diff --git a/build/plugins/a.yaml b/build/plugins/a.yaml index 3c652e093a..48447c2fc3 100644 --- a/build/plugins/a.yaml +++ b/build/plugins/a.yaml @@ -7,7 +7,7 @@ arcanum: requirements: - system: arcanum type: merge_time_fits - alias: Merge time intervals (YMAKE-1194) + alias: Merge is forbidden from 11:00 to 20:00 MSK (YMAKE-1194) data: merge_intervals_utc: # 20:00..11:00 MSK diff --git a/build/plugins/java.py b/build/plugins/java.py index 60d70919b4..b6565ce082 100644 --- a/build/plugins/java.py +++ b/build/plugins/java.py @@ -102,6 +102,7 @@ def onjava_module(unit, *args): '20', '21', '22', + '23', ): data['ENABLE_PREVIEW'] = extract_macro_calls(unit, 'ENABLE_PREVIEW_VALUE', args_delim) @@ -231,6 +232,11 @@ def on_add_classpath_clash_check(unit, *args): unit.onjava_test_deps(jdeps_val) +def on_add_detekt_report_check(unit, *args): + if unit.get('WITH_KOTLIN_VALUE') == 'yes' and unit.get('WITH_KOTLINC_PLUGIN_DETEKT') == 'yes': + unit.onadd_check(['detekt.report'] + list(args)) + + # Ymake java modules related macroses @@ -373,6 +379,7 @@ def on_jdk_version_macro_check(unit, *args): '20', '21', '22', + '23', ) if jdk_version not in available_versions: ymake.report_configure_error( diff --git a/build/plugins/nots.py b/build/plugins/nots.py index 625b1af5bf..6dcff79bf7 100644 --- a/build/plugins/nots.py +++ b/build/plugins/nots.py @@ -603,6 +603,8 @@ def _setup_stylelint(unit): if not test_files: return + unit.on_peerdir_ts_resource("stylelint") + from lib.nots.package_manager import constants recipes_value = unit.get("TEST_RECIPES_VALUE") diff --git a/build/plugins/res.py b/build/plugins/res.py index 24c183ad7f..8dcea72d48 100644 --- a/build/plugins/res.py +++ b/build/plugins/res.py @@ -13,7 +13,7 @@ def remove_prefix(text, prefix): def onresource_files(unit, *args): """ - @usage: RESOURCE_FILES([PREFIX {prefix}] [STRIP prefix_to_strip] {path}) + @usage: RESOURCE_FILES([DONT_COMPRESS] [PREFIX {prefix}] [STRIP prefix_to_strip] {path}) This macro expands into RESOURCE(DONT_PARSE {path} resfs/file/{prefix}{path} @@ -26,8 +26,7 @@ def onresource_files(unit, *args): resfs/file/{key} stores any value whose source was a file on a filesystem. resfs/src/resfs/file/{key} must store its path. - DONT_PARSE disables parsing for source code files (determined by extension) - Please don't abuse: use separate DONT_PARSE macro call only for files subject to parsing + DONT_COMPRESS allows optionally disable resource compression on platforms where it is supported This form is for use from other plugins: RESOURCE_FILES([DEST {dest}] {path}) expands into RESOURCE({path} resfs/file/{dest}) @@ -43,9 +42,14 @@ def onresource_files(unit, *args): # GO_RESOURCE currently doesn't support DONT_PARSE res.append('DONT_PARSE') + if args and not unit.enabled('_GO_MODULE') and 'DONT_COMPRESS' in args: + res.append('DONT_COMPRESS') + args = iter(args) for arg in args: - if arg == 'PREFIX': + if arg == 'DONT_COMPRESS': + pass + elif arg == 'PREFIX': prefix, dest = next(args), None elif arg == 'DEST': dest, prefix = next(args), None @@ -58,7 +62,7 @@ def onresource_files(unit, *args): ) if key in res: unit.message( - ['warn', "Dublicated resource file {} in RESOURCE_FILES() macro. Skipped it.".format(path)] + ['warn', "Duplicated resource file {} in RESOURCE_FILES() macro. Skipped it.".format(path)] ) continue src = 'resfs/src/{}={}'.format(key, rootrel_arc_src(path, unit)) @@ -106,8 +110,15 @@ def on_ya_conf_json(unit, conf_file): with open(conf_abs_path) as f: conf = json.load(f) formulas = set() - for bottle_name, bottle in conf['bottles'].items(): - formula = bottle['formula'] + + def _iter_bottles(config): + if "simple_tools" in config: + for name, info in config["simple_tools"].items(): + yield name, f"build/external_resources/{info.get('resource', name)}/resources.json" + for name, bottle in config["bottles"].items(): + yield name, bottle["formula"] + + for bottle_name, formula in _iter_bottles(conf): if isinstance(formula, six.string_types): if formula.startswith(valid_dirs): abs_path = unit.resolve('$S/' + formula) diff --git a/build/plugins/ytest.py b/build/plugins/ytest.py index 6b1a77e235..f24ba674a0 100644 --- a/build/plugins/ytest.py +++ b/build/plugins/ytest.py @@ -24,7 +24,6 @@ from _dart_fields import ( serialize_list, get_unit_list_variable, deserialize_list, - prepare_env, create_dart_record, ) @@ -99,6 +98,19 @@ CHECK_FIELDS_BASE = ( df.UseArcadiaPython.value, ) +LINTER_FIELDS_BASE = ( + df.LintName.value, + df.LintExtraParams.from_macro_args, + df.TestName.name_from_macro_args, + df.TestedProjectName.unit_name, + df.SourceFolderPath.normalized, + df.TestEnv.value, + df.UseArcadiaPython.value, + df.LintFileProcessingTime.from_macro_args, + df.Linter.value, + df.CustomDependencies.depends_with_linter, +) + tidy_config_map = None @@ -397,13 +409,6 @@ def dump_test(unit, kw): return data -def reference_group_var(varname: str, extensions: list[str] | None = None) -> str: - if extensions is None: - return f'"${{join=\\;:{varname}}}"' - - return serialize_list(f'${{ext={ext};join=\\;:{varname}}}' for ext in extensions) - - def count_entries(x): # see (de)serialize_list assert x is None or isinstance(x, str), type(x) @@ -693,6 +698,39 @@ def govet(fields, unit, *args): unit.set_property(["DART_DATA", data]) +@df.with_fields( + CHECK_FIELDS_BASE + + ( + df.TestedProjectName.normalized_basename, + df.SourceFolderPath.normalized, + df.TestFiles.flat_args_wo_first, + df.ModuleLang.value, + ) +) +def detekt_report(fields, unit, *args): + flat_args, spec_args = _common.sort_by_keywords( + { + "DEPENDS": -1, + "TIMEOUT": 1, + "DATA": -1, + "TAG": -1, + "REQUIREMENTS": -1, + "FORK_MODE": 1, + "SPLIT_FACTOR": 1, + "FORK_SUBTESTS": 0, + "FORK_TESTS": 0, + "SIZE": 1, + }, + args, + ) + + dart_record = create_dart_record(fields, unit, flat_args, spec_args) + + data = dump_test(unit, dart_record) + if data: + unit.set_property(["DART_DATA", data]) + + def onadd_check(unit, *args): if unit.get("TIDY") == "yes": # graph changed for clang_tidy tests @@ -727,6 +765,8 @@ def onadd_check(unit, *args): gofmt(unit, *args) elif check_type == "govet": govet(unit, *args) + elif check_type == "detekt.report": + detekt_report(unit, *args) def on_register_no_check_imports(unit): @@ -950,32 +990,14 @@ def onsetup_run_python(unit): unit.ondepends('contrib/tools/python') -@_common.lazy -def get_linter_configs(unit, config_paths): - rel_config_path = _common.rootrel_arc_src(config_paths, unit) - arc_config_path = unit.resolve_arc_path(rel_config_path) - abs_config_path = unit.resolve(arc_config_path) - with open(abs_config_path, 'r') as fd: - return list(json.load(fd).values()) - - @df.with_fields( ( - df.LintName.value, - df.TestFiles.py_linter_files, - df.LintConfigs.value, - df.LintExtraParams.from_macro_args, - df.TestName.name_from_macro_args, - df.TestedProjectName.unit_name, - df.SourceFolderPath.normalized, - df.TestEnv.value, - df.UseArcadiaPython.value, - df.LintFileProcessingTime.from_macro_args, - df.Linter.value, - df.CustomDependencies.depends_with_linter, + df.TestFiles.cpp_linter_files, + df.LintConfigs.cpp_configs, ) + + LINTER_FIELDS_BASE ) -def on_add_py_linter_check(fields, unit, *args): +def on_add_cpp_linter_check(fields, unit, *args): if unit.get("TIDY") == "yes": return @@ -988,8 +1010,8 @@ def on_add_py_linter_check(fields, unit, *args): "NAME": 1, "LINTER": 1, "DEPENDS": unlimited, - "FILES": unlimited, - "CONFIGS": unlimited, + "CONFIGS": 1, + "CUSTOM_CONFIG": 1, "GLOBAL_RESOURCES": unlimited, "FILE_PROCESSING_TIME": 1, "EXTRA_PARAMS": unlimited, @@ -1012,91 +1034,48 @@ def on_add_py_linter_check(fields, unit, *args): unit.set_property(["DART_DATA", data]) -def on_add_linter_check(unit, *args): +@df.with_fields( + ( + df.TestFiles.py_linter_files, + df.LintConfigs.python_configs, + ) + + LINTER_FIELDS_BASE +) +def on_add_py_linter_check(fields, unit, *args): if unit.get("TIDY") == "yes": return - source_root_from_prefix = '${ARCADIA_ROOT}/' - source_root_to_prefix = '$S/' - unlimited = -1 no_lint_value = _common.get_no_lint_value(unit) if no_lint_value in ("none", "none_internal"): return + unlimited = -1 keywords = { + "NAME": 1, + "LINTER": 1, "DEPENDS": unlimited, - "FILES": unlimited, - "CONFIGS": unlimited, + "CONFIGS": 1, "GLOBAL_RESOURCES": unlimited, "FILE_PROCESSING_TIME": 1, "EXTRA_PARAMS": unlimited, + "PROJECT_TO_CONFIG_MAP": 1, + "FLAKE_MIGRATIONS_CONFIG": 1, + "CUSTOM_CONFIG": 1, } - flat_args, spec_args = _common.sort_by_keywords(keywords, args) - if len(flat_args) != 2: - unit.message(['ERROR', '_ADD_LINTER_CHECK params: expected 2 free parameters']) - return - - configs = [] - for cfg in spec_args.get('CONFIGS', []): - filename = unit.resolve(source_root_to_prefix + cfg) - if not os.path.exists(filename): - unit.message(['ERROR', 'Configuration file {} is not found'.format(filename)]) - return - configs.append(cfg) - deps = [] - - lint_name, linter = flat_args - deps.append(os.path.dirname(linter)) - - test_files = [] - for path in spec_args.get('FILES', []): - if path.startswith(source_root_from_prefix): - test_files.append(path.replace(source_root_from_prefix, source_root_to_prefix, 1)) - elif path.startswith(source_root_to_prefix): - test_files.append(path) - - if lint_name == 'cpp_style': - files_dart = reference_group_var("ALL_SRCS", consts.STYLE_CPP_ALL_EXTS) - else: - if not test_files: - unit.message(['WARN', 'No files to lint for {}'.format(lint_name)]) - return - files_dart = serialize_list(test_files) - - for arg in spec_args.get('EXTRA_PARAMS', []): - if '=' not in arg: - unit.message(['WARN', 'Wrong EXTRA_PARAMS value: "{}". Values must have format "name=value".'.format(arg)]) - return - - deps += spec_args.get('DEPENDS', []) - - for dep in deps: - unit.ondepends(dep) + _, spec_args = _common.sort_by_keywords(keywords, args) - for resource in spec_args.get('GLOBAL_RESOURCES', []): + global_resources = spec_args.get('GLOBAL_RESOURCES', []) + for resource in global_resources: unit.onpeerdir(resource) + try: + dart_record = create_dart_record(fields, unit, (), spec_args) + except df.DartValueError as e: + if msg := str(e): + unit.message(['WARN', msg]) + return + dart_record[df.ScriptRelPath.KEY] = 'custom_lint' - test_record = { - 'TEST-NAME': lint_name, - 'SCRIPT-REL-PATH': 'custom_lint', - 'TESTED-PROJECT-NAME': unit.name(), - 'SOURCE-FOLDER-PATH': _common.get_norm_unit_path(unit), - 'CUSTOM-DEPENDENCIES': " ".join(deps), - 'TEST-ENV': prepare_env(unit.get("TEST_ENV_VALUE")), - 'USE_ARCADIA_PYTHON': unit.get('USE_ARCADIA_PYTHON') or '', - # TODO remove FILES, see DEVTOOLS-7052 - 'FILES': files_dart, - 'TEST-FILES': files_dart, - # Linter specific parameters - # TODO Add configs to DATA. See YMAKE-427 - 'LINT-CONFIGS': serialize_list(configs), - 'LINT-NAME': lint_name, - 'LINT-FILE-PROCESSING-TIME': spec_args.get('FILE_PROCESSING_TIME', [''])[0], - 'LINT-EXTRA-PARAMS': serialize_list(spec_args.get('EXTRA_PARAMS', [])), - 'LINTER': linter, - } - - data = dump_test(unit, test_record) + data = dump_test(unit, dart_record) if data: unit.set_property(["DART_DATA", data]) diff --git a/build/scripts/build_catboost.py b/build/scripts/build_catboost.py index 81d4e795a0..6455bb59bb 100755 --- a/build/scripts/build_catboost.py +++ b/build/scripts/build_catboost.py @@ -1,3 +1,4 @@ +from __future__ import print_function import sys import os import shutil @@ -45,7 +46,7 @@ class BuildCbBase(object): class BuildCb(BuildCbBase): def run(self, argv): if len(argv) < 5: - print >> sys.stderr, "BuildCb.Run(<ARCADIA_ROOT> <archiver> <mninfo> <mnname> <cppOutput> [params...])" + print("BuildCb.Run(<ARCADIA_ROOT> <archiver> <mninfo> <mnname> <cppOutput> [params...])", file=sys.stderr) sys.exit(1) self.SrcRoot = argv[0] @@ -64,7 +65,7 @@ def build_cb_f(argv): if __name__ == '__main__': if len(sys.argv) < 2: - print >> sys.stderr, "Usage: build_cb.py <funcName> <args...>" + print("Usage: build_cb.py <funcName> <args...>", file=sys.stderr) sys.exit(1) if sys.argv[2:]: diff --git a/build/scripts/clang_tidy.py b/build/scripts/clang_tidy.py index bbb223b206..c7636b8bf2 100644 --- a/build/scripts/clang_tidy.py +++ b/build/scripts/clang_tidy.py @@ -1,3 +1,4 @@ +from __future__ import print_function import argparse import json import os diff --git a/build/scripts/collect_java_srcs.py b/build/scripts/collect_java_srcs.py deleted file mode 100644 index f361f271d1..0000000000 --- a/build/scripts/collect_java_srcs.py +++ /dev/null @@ -1,51 +0,0 @@ -import os -import sys -import contextlib -import tarfile -import zipfile - - -if __name__ == '__main__': - build_root = sys.argv[1] - root = os.path.normpath(sys.argv[2]) - dest = os.path.normpath(sys.argv[3]) - srcs = sys.argv[4:] - - for src in srcs: - src = os.path.normpath(src) - if src.endswith('.java') or src.endswith('.kt'): - src_rel_path = os.path.relpath(src, root) - - if os.path.join(root, src_rel_path) == src: - # Inside root - dst = os.path.join(dest, src_rel_path) - - else: - # Outside root - print('External src file "{}" is outside of srcdir {}, ignore'.format( - os.path.relpath(src, build_root), - os.path.relpath(root, build_root), - ) - continue - - if os.path.exists(dst): - print >> sys.stderr, 'Duplicate external src file {}, choice is undefined'.format( - os.path.relpath(dst, root) - ) - - else: - destdir = os.path.dirname(dst) - if destdir and not os.path.exists(destdir): - os.makedirs(destdir) - os.rename(src, dst) - - elif src.endswith('.jsr'): - with contextlib.closing(tarfile.open(src, 'r')) as tf: - tf.extractall(dst) - - elif src.endswith('-sources.jar'): - with zipfile.ZipFile(src) as zf: - zf.extractall(dst) - - else: - print >> sys.stderr, 'Unrecognized file type', os.path.relpath(src, build_root) diff --git a/build/scripts/compile_cuda.py b/build/scripts/compile_cuda.py index 9660300069..8e08d17ad2 100644 --- a/build/scripts/compile_cuda.py +++ b/build/scripts/compile_cuda.py @@ -1,3 +1,4 @@ +from __future__ import print_function import sys import subprocess import os @@ -60,7 +61,7 @@ def main(): executable = command[0] if not os.path.exists(executable): - print >> sys.stderr, '{} not found'.format(executable) + print('{} not found'.format(executable), file=sys.stderr) sys.exit(1) if is_clang(command): diff --git a/build/scripts/configure_file.py b/build/scripts/configure_file.py index 5ab7467391..257593dc9b 100755 --- a/build/scripts/configure_file.py +++ b/build/scripts/configure_file.py @@ -1,5 +1,6 @@ #!/usr/bin/env python2.7 +from __future__ import print_function import sys import os.path import re diff --git a/build/scripts/coverage-info.py b/build/scripts/coverage-info.py index ddc5f275f8..114e8f4b03 100644 --- a/build/scripts/coverage-info.py +++ b/build/scripts/coverage-info.py @@ -1,3 +1,4 @@ +from __future__ import print_function import argparse import os import sys @@ -72,12 +73,12 @@ def print_stat(da, fnda, teamcity_stat_output): func_total = len(fnda.values()) func_coverage = 100.0 * func_hit / func_total if func_total else 0 - print >> sys.stderr, '[[imp]]Lines[[rst]] {: >16} {: >16} {: >16.1f}%'.format( + print('[[imp]]Lines[[rst]] {: >16} {: >16} {: >16.1f}%'.format( lines_hit, lines_total, lines_coverage - ) - print >> sys.stderr, '[[imp]]Functions[[rst]] {: >16} {: >16} {: >16.1f}%'.format( + ), file=sys.stderr) + print('[[imp]]Functions[[rst]] {: >16} {: >16} {: >16.1f}%'.format( func_hit, func_total, func_coverage - ) + ), file=sys.stderr) if teamcity_stat_output: with open(teamcity_stat_output, 'w') as tc_file: @@ -119,7 +120,7 @@ def combine_info_files(lcov, files, out_file): for trace in chunk: assert os.path.exists(trace), "Trace file does not exist: {} (cwd={})".format(trace, os.getcwd()) combine_cmd += ["-a", os.path.abspath(trace)] - print >> sys.stderr, '## lcov', ' '.join(combine_cmd[1:]) + print('## lcov', ' '.join(combine_cmd[1:]), file=sys.stderr) out_file_tmp = "combined.tmp" with open(out_file_tmp, "w") as stdout: subprocess.check_call(combine_cmd, stdout=stdout) @@ -157,7 +158,7 @@ def update_stat_global(src_file, line, fnda, da): def gen_info_global(cmd, cov_info, probe_path, update_stat, lcov_args): - print >> sys.stderr, '## geninfo', ' '.join(cmd) + print('## geninfo', ' '.join(cmd), file=sys.stderr) subprocess.check_call(cmd) if recast(cov_info + '.tmp', cov_info, probe_path, update_stat): lcov_args.append(cov_info) @@ -297,7 +298,7 @@ def main( output_dir, output_trace, ] - print >> sys.stderr, '## genhtml', ' '.join(cmd) + print('## genhtml', ' '.join(cmd), file=sys.stderr) subprocess.check_call(cmd) if lcov_cobertura: gen_cobertura(lcov_cobertura, gcov_report, output_trace) diff --git a/build/scripts/create_jcoverage_report.py b/build/scripts/create_jcoverage_report.py index f24827d8ae..0dd75bb0a8 100644 --- a/build/scripts/create_jcoverage_report.py +++ b/build/scripts/create_jcoverage_report.py @@ -1,3 +1,4 @@ +from __future__ import print_function import argparse import tarfile import zipfile @@ -77,12 +78,12 @@ def main( else: continue - entry.filename = entry.filename.encode('utf-8') + entry.filename = entry.filename jf.extract(entry, dest) timer.step("Jar files extracted") if not agent_disposition: - print >> sys.stderr, 'Can\'t find jacoco agent. Will not generate html report for java coverage.' + print('Can\'t find jacoco agent. Will not generate html report for java coverage.', file=sys.stderr) if tar_output: report_dir = 'java.report.temp' diff --git a/build/scripts/custom_link_green_mysql.py b/build/scripts/custom_link_green_mysql.py index f754135b8f..1ebff0adf7 100644 --- a/build/scripts/custom_link_green_mysql.py +++ b/build/scripts/custom_link_green_mysql.py @@ -1,3 +1,4 @@ +from __future__ import print_function import os import shutil import subprocess @@ -91,7 +92,7 @@ def main(): name = os.path.basename(sys.argv[0]) command = ' '.join(args) message = '{name} failed: {error}\nCommand line: {command}' - print >> sys.stderr, message.format(**locals()) + print(message.format(**locals()), file=sys.stderr) if __name__ == '__main__': diff --git a/build/scripts/decimal_md5.py b/build/scripts/decimal_md5.py index 684d39e767..2a125cca60 100644 --- a/build/scripts/decimal_md5.py +++ b/build/scripts/decimal_md5.py @@ -1,6 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +from __future__ import print_function import hashlib import struct import sys @@ -20,9 +21,9 @@ def ensure_paths_exist(paths): if not os.path.exists(path) ) if bad_paths: - print >> sys.stderr, "decimal_md5 inputs do not exist:" + print("decimal_md5 inputs do not exist:", file=sys.stderr) for path in bad_paths: - print >> sys.stderr, path + print(path, file=sys.stderr) sys.exit(1) diff --git a/build/scripts/error.py b/build/scripts/error.py index 9911ee7cc1..0681e48679 100644 --- a/build/scripts/error.py +++ b/build/scripts/error.py @@ -63,14 +63,17 @@ def is_temporary_error(exc): logger.debug("Getaddrinfo exception: %s", exc) return True - import urllib2 + try: + import urllib2 + import httplib + except ImportError: + import urllib.request as urllib2 + import http.client as httplib if isinstance(exc, urllib2.HTTPError) and exc.code in (429,): logger.debug("urllib2.HTTPError: %s", exc) return True - import httplib - if isinstance(exc, httplib.IncompleteRead): logger.debug("IncompleteRead exception: %s", exc) return True diff --git a/build/scripts/f2c.py b/build/scripts/f2c.py index 878580e4d4..fc1c1d0f8a 100644 --- a/build/scripts/f2c.py +++ b/build/scripts/f2c.py @@ -1,3 +1,4 @@ +from __future__ import print_function import sys import subprocess import argparse @@ -49,11 +50,11 @@ if __name__ == '__main__': ret = p.wait() if ret: - print >> sys.stderr, 'f2c failed: %s, %s' % (stderr, ret) + print('f2c failed: %s, %s' % (stderr, ret), file=sys.stderr) sys.exit(ret) if 'Error' in stderr: - print >> sys.stderr, stderr + print(stderr, file=sys.stderr) with open(args.output, 'w') as f: f.write(header) diff --git a/build/scripts/fetch_from.py b/build/scripts/fetch_from.py index ca3e7b3684..bd27d75d8e 100755 --- a/build/scripts/fetch_from.py +++ b/build/scripts/fetch_from.py @@ -1,3 +1,5 @@ +from __future__ import print_function + import datetime as dt import errno import hashlib @@ -11,7 +13,15 @@ import socket import string import sys import tarfile -import urllib2 + +try: + # Python 2 + import urllib2 as urllib_request + from urllib2 import HTTPError, URLError +except (ImportError, ModuleNotFoundError): + # Python 3 + import urllib.request as urllib_request + from urllib.error import HTTPError, URLError import retry @@ -122,12 +132,12 @@ def setup_logging(args, base_name): def is_temporary(e): def is_broken(e): - return isinstance(e, urllib2.HTTPError) and e.code in (410, 404) + return isinstance(e, HTTPError) and e.code in (410, 404) if is_broken(e): return False - if isinstance(e, (BadChecksumFetchError, IncompleteFetchError, urllib2.URLError, socket.error)): + if isinstance(e, (BadChecksumFetchError, IncompleteFetchError, URLError, socket.error)): return True import error @@ -147,7 +157,7 @@ def report_to_snowden(value): 'value': json.dumps(value), } - urllib2.urlopen( + urllib_request.urlopen( 'https://back-snowden.qloud.yandex-team.ru/report/add', json.dumps( [ @@ -198,8 +208,8 @@ def git_like_hash_with_size(filepath): file_size += len(block) sha.update(block) - sha.update('\0') - sha.update(str(file_size)) + sha.update(b'\0') + sha.update(str(file_size).encode('utf-8')) return sha.hexdigest(), file_size @@ -213,9 +223,9 @@ def size_printer(display_name, size): now = dt.datetime.now() if last_stamp[0] + dt.timedelta(seconds=10) < now: if size: - print >> sys.stderr, "##status##{} - [[imp]]{:.1f}%[[rst]]".format( + print("##status##{} - [[imp]]{:.1f}%[[rst]]".format( display_name, 100.0 * sz[0] / size if size else 0 - ) + ), file=sys.stderr) last_stamp[0] = now return printer @@ -225,9 +235,9 @@ def fetch_url(url, unpack, resource_file_name, expected_md5=None, expected_sha1= logging.info('Downloading from url %s name %s and expected md5 %s', url, resource_file_name, expected_md5) tmp_file_name = uniq_string_generator() - request = urllib2.Request(url, headers={'User-Agent': make_user_agent()}) - req = retry.retry_func(lambda: urllib2.urlopen(request, timeout=30), tries=tries, delay=5, backoff=1.57079) - logging.debug('Headers: %s', req.headers.headers) + request = urllib_request.Request(url, headers={'User-Agent': make_user_agent()}) + req = retry.retry_func(lambda: urllib_request.urlopen(request, timeout=30), tries=tries, delay=5, backoff=1.57079) + logging.debug('Headers: %s', req.headers) expected_file_size = int(req.headers.get('Content-Length', 0)) real_md5 = hashlib.md5() real_sha1 = hashlib.sha1() @@ -244,8 +254,8 @@ def fetch_url(url, unpack, resource_file_name, expected_md5=None, expected_sha1= real_md5 = real_md5.hexdigest() real_file_size = os.path.getsize(tmp_file_name) - real_sha1.update('\0') - real_sha1.update(str(real_file_size)) + real_sha1.update(b'\0') + real_sha1.update(str(real_file_size).encode('utf-8')) real_sha1 = real_sha1.hexdigest() if unpack: diff --git a/build/scripts/fetch_from_archive.py b/build/scripts/fetch_from_archive.py index e7bbe23362..3214b78dc8 100644 --- a/build/scripts/fetch_from_archive.py +++ b/build/scripts/fetch_from_archive.py @@ -1,3 +1,4 @@ +from __future__ import print_function import os import sys import logging @@ -30,7 +31,7 @@ if __name__ == '__main__': main(args) except Exception as e: logging.exception(e) - print >> sys.stderr, open(args.abs_log_path).read() + print(open(args.abs_log_path).read(), file=sys.stderr) sys.stderr.flush() import error diff --git a/build/scripts/fetch_from_mds.py b/build/scripts/fetch_from_mds.py index 3fd264ebb1..559cf8350d 100644 --- a/build/scripts/fetch_from_mds.py +++ b/build/scripts/fetch_from_mds.py @@ -1,3 +1,6 @@ +from __future__ import print_function +from __future__ import print_function + import os import sys import logging @@ -43,7 +46,7 @@ if __name__ == '__main__': main(args) except Exception as e: logging.exception(e) - print >> sys.stderr, open(args.abs_log_path).read() + print(open(args.abs_log_path).read(), file=sys.stderr) sys.stderr.flush() import error diff --git a/build/scripts/fetch_from_npm.py b/build/scripts/fetch_from_npm.py index 3941b59d43..d8f0c41f26 100644 --- a/build/scripts/fetch_from_npm.py +++ b/build/scripts/fetch_from_npm.py @@ -1,3 +1,5 @@ +from __future__ import print_function +from future.utils import raise_ import os import sys import time @@ -66,7 +68,7 @@ def fetch(tarball_url, sky_id, integrity, integrity_algorithm, file_name, tries= time.sleep(i) if exc_info: - raise exc_info[0], exc_info[1], exc_info[2] + raise_(exc_info[0], exc_info[1], exc_info[2]) return fetched_file @@ -104,7 +106,7 @@ if __name__ == "__main__": main(args) except Exception as e: logging.exception(e) - print >>sys.stderr, open(args.abs_log_path).read() + print(open(args.abs_log_path).read(), file=sys.stderr) sys.stderr.flush() import error diff --git a/build/scripts/fetch_from_sandbox.py b/build/scripts/fetch_from_sandbox.py index 6af180b4b0..a8f0fc5997 100755 --- a/build/scripts/fetch_from_sandbox.py +++ b/build/scripts/fetch_from_sandbox.py @@ -1,3 +1,4 @@ +from __future__ import print_function import itertools import json import logging @@ -210,7 +211,10 @@ def fetch(resource_id, custom_fetcher): time.sleep(i) else: if exc_info: - raise exc_info[0], exc_info[1], exc_info[2] + if sys.version_info[0] == 2: + raise exc_info[0], exc_info[1], exc_info[2] + else: + raise exc_info[1].with_traceback(exc_info[2]) else: raise Exception("No available protocol and/or server to fetch resource") @@ -265,7 +269,7 @@ if __name__ == '__main__': main(args) except Exception as e: logging.exception(e) - print >>sys.stderr, open(args.abs_log_path).read() + print(open(args.abs_log_path).read(), file=sys.stderr) sys.stderr.flush() import error diff --git a/build/scripts/gen_java_codenav_entry.py b/build/scripts/gen_java_codenav_entry.py index 2959dc4843..0429fc5398 100644 --- a/build/scripts/gen_java_codenav_entry.py +++ b/build/scripts/gen_java_codenav_entry.py @@ -1,3 +1,4 @@ +from __future__ import print_function import argparse import datetime import os @@ -20,23 +21,23 @@ def just_do_it(java, kythe, kythe_to_proto, out_name, binding_only, kindexes): open(temp_out_name, 'w').close() start = datetime.datetime.now() for kindex in kindex_inputs: - print >> sys.stderr, '[INFO] Processing:', kindex + print('[INFO] Processing:', kindex, file=sys.stderr) indexer_start = datetime.datetime.now() p = subprocess.Popen( [java, '-jar', os.path.join(kythe, 'indexers/java_indexer.jar'), kindex], stdout=subprocess.PIPE ) indexer_out, _ = p.communicate() - print >> sys.stderr, '[INFO] Indexer execution time:', ( + print('[INFO] Indexer execution time:', ( datetime.datetime.now() - indexer_start - ).total_seconds(), 'seconds' + ).total_seconds(), 'seconds', file=sys.stderr) if p.returncode: raise Exception('java_indexer failed with exit code {}'.format(p.returncode)) dedup_start = datetime.datetime.now() p = subprocess.Popen([os.path.join(kythe, 'tools/dedup_stream')], stdin=subprocess.PIPE, stdout=subprocess.PIPE) dedup_out, _ = p.communicate(indexer_out) - print >> sys.stderr, '[INFO] Dedup execution time:', ( + print('[INFO] Dedup execution time:', ( datetime.datetime.now() - dedup_start - ).total_seconds(), 'seconds' + ).total_seconds(), 'seconds', file=sys.stderr) if p.returncode: raise Exception('dedup_stream failed with exit code {}'.format(p.returncode)) entrystream_start = datetime.datetime.now() @@ -48,18 +49,18 @@ def just_do_it(java, kythe, kythe_to_proto, out_name, binding_only, kindexes): p.communicate(dedup_out) if p.returncode: raise Exception('entrystream failed with exit code {}'.format(p.returncode)) - print >> sys.stderr, '[INFO] Entrystream execution time:', ( + print('[INFO] Entrystream execution time:', ( datetime.datetime.now() - entrystream_start - ).total_seconds(), 'seconds' + ).total_seconds(), 'seconds', file=sys.stderr) preprocess_start = datetime.datetime.now() subprocess.check_call( [kythe_to_proto, '--preprocess-entry', '--entries', temp_out_name, '--out', out_name] + (['--only-binding-data'] if binding_only else []) ) - print >> sys.stderr, '[INFO] Preprocessing execution time:', ( + print('[INFO] Preprocessing execution time:', ( datetime.datetime.now() - preprocess_start - ).total_seconds(), 'seconds' - print >> sys.stderr, '[INFO] Total execution time:', (datetime.datetime.now() - start).total_seconds(), 'seconds' + ).total_seconds(), 'seconds', file=sys.stderr) + print('[INFO] Total execution time:', (datetime.datetime.now() - start).total_seconds(), 'seconds', file=sys.stderr) if __name__ == '__main__': diff --git a/build/scripts/gen_py3_reg.py b/build/scripts/gen_py3_reg.py index ff6bf0de56..ee6c6baf9b 100644 --- a/build/scripts/gen_py3_reg.py +++ b/build/scripts/gen_py3_reg.py @@ -1,3 +1,4 @@ +from __future__ import print_function import sys template = ''' @@ -24,8 +25,8 @@ def mangle(name): if __name__ == '__main__': if len(sys.argv) != 3: - print >> sys.stderr, 'Usage: <path/to/gen_py_reg.py> <python_module_name> <output_file>' - print >> sys.stderr, 'Passed: ' + ' '.join(sys.argv) + print('Usage: <path/to/gen_py_reg.py> <python_module_name> <output_file>', file=sys.stderr) + print('Passed: ' + ' '.join(sys.argv), file=sys.stderr) sys.exit(1) with open(sys.argv[2], 'w') as f: diff --git a/build/scripts/gen_py_reg.py b/build/scripts/gen_py_reg.py index 0f38dfffe3..be52418964 100644 --- a/build/scripts/gen_py_reg.py +++ b/build/scripts/gen_py_reg.py @@ -1,3 +1,4 @@ +from __future__ import print_function import sys template = ''' @@ -22,8 +23,8 @@ def mangle(name): if __name__ == '__main__': if len(sys.argv) != 3: - print >> sys.stderr, 'Usage: <path/to/gen_py_reg.py> <python_module_name> <output_file>' - print >> sys.stderr, 'Passed: ' + ' '.join(sys.argv) + print('Usage: <path/to/gen_py_reg.py> <python_module_name> <output_file>', file=sys.stderr) + print('Passed: ' + ' '.join(sys.argv), file=sys.stderr) sys.exit(1) with open(sys.argv[2], 'w') as f: diff --git a/build/scripts/go_tool.py b/build/scripts/go_tool.py index 39d55df4d3..f1dbc9731e 100644 --- a/build/scripts/go_tool.py +++ b/build/scripts/go_tool.py @@ -75,8 +75,14 @@ def preprocess_args(args): args.output_root = os.path.normpath(args.output_root) args.import_map = {} args.module_map = {} + + def is_valid_cgo_peer(peer): + if peer.endswith('.fake.pkg') or peer.endswith('.fake'): + return False + return True + if args.cgo_peers: - args.cgo_peers = [x for x in args.cgo_peers if not x.endswith('.fake.pkg')] + args.cgo_peers = list(filter(is_valid_cgo_peer, args.cgo_peers)) srcs = [] for f in args.srcs: diff --git a/build/scripts/ios_wrapper.py b/build/scripts/ios_wrapper.py index eeb0a78d26..f8bdcce0a2 100644 --- a/build/scripts/ios_wrapper.py +++ b/build/scripts/ios_wrapper.py @@ -1,3 +1,4 @@ +from __future__ import print_function import errno import json import os @@ -49,13 +50,13 @@ def just_do_it(args): elif i.endswith('.strings_tar'): strings.append(i) else: - print >> sys.stderr, 'Unknown input:', i, 'ignoring' + print('Unknown input:', i, 'ignoring', file=sys.stderr) if not plists: raise Exception("Can't find plist files") if not plists[0].endswith('.plist'): - print >> sys.stderr, "Main plist file can be defined incorretly" + print("Main plist file can be defined incorretly", file=sys.stderr) if not storyboards: - print >> sys.stderr, "Storyboards list are empty" + print("Storyboards list are empty", file=sys.stderr) if len(signs) > 1: raise Exception("Too many .xcent files") app_dir = os.path.join(module_dir, app_name + '.app') diff --git a/build/scripts/link_dyn_lib.py b/build/scripts/link_dyn_lib.py index 72937544b6..3e62f4c407 100644 --- a/build/scripts/link_dyn_lib.py +++ b/build/scripts/link_dyn_lib.py @@ -1,3 +1,4 @@ +from __future__ import print_function import sys import os import subprocess @@ -264,8 +265,8 @@ if __name__ == '__main__': thinlto_cache.postprocess(opts) if proc.returncode: - print >> sys.stderr, 'linker has failed with retcode:', proc.returncode - print >> sys.stderr, 'linker command:', shlex_join(cmd) + print('linker has failed with retcode:', proc.returncode, file=sys.stderr) + print('linker command:', shlex_join(cmd), file=sys.stderr) sys.exit(proc.returncode) if opts.fix_elf: @@ -274,8 +275,8 @@ if __name__ == '__main__': proc.communicate() if proc.returncode: - print >> sys.stderr, 'fix_elf has failed with retcode:', proc.returncode - print >> sys.stderr, 'fix_elf command:', shlex_join(cmd) + print('fix_elf has failed with retcode:', proc.returncode, file=sys.stderr) + print('fix_elf command:', shlex_join(cmd), file=sys.stderr) sys.exit(proc.returncode) if opts.soname and opts.soname != opts.target: diff --git a/build/scripts/link_fat_obj.py b/build/scripts/link_fat_obj.py index ea143ec902..aeafea58c4 100644 --- a/build/scripts/link_fat_obj.py +++ b/build/scripts/link_fat_obj.py @@ -76,13 +76,8 @@ def main(): linker = groups['linker'] archiver = groups['archiver'] - if 'YA_XCODE' in str(sys.argv): - no_pie = '-Wl,-no_pie' - else: - no_pie = '-Wl,-no-pie' - do_link = ( - linker + ['-o', obj_output, '-Wl,-r', '-nodefaultlibs', '-nostartfiles', no_pie] + global_srcs + auto_input + linker + ['-o', obj_output, '-Wl,-r', '-nodefaultlibs', '-nostartfiles', '-Wl,-no-pie'] + global_srcs + auto_input ) do_archive = archiver + [lib_output] + peers do_globals = None diff --git a/build/scripts/list.py b/build/scripts/list.py index 7c3b2ae695..5c8390c52c 100644 --- a/build/scripts/list.py +++ b/build/scripts/list.py @@ -1,4 +1,5 @@ -import sys
-
-if __name__ == "__main__":
- print(' '.join(sys.argv[1:]))
+from __future__ import print_function +import sys + +if __name__ == "__main__": + print(' '.join(sys.argv[1:])) diff --git a/build/scripts/mkver.py b/build/scripts/mkver.py index 7bdbb88514..2101032770 100755 --- a/build/scripts/mkver.py +++ b/build/scripts/mkver.py @@ -1,3 +1,4 @@ +from __future__ import print_function import sys if __name__ == '__main__': diff --git a/build/scripts/pack_ios.py b/build/scripts/pack_ios.py index 350220be7b..3ecd18f188 100644 --- a/build/scripts/pack_ios.py +++ b/build/scripts/pack_ios.py @@ -1,3 +1,4 @@ +from __future__ import print_function import argparse import os import shutil @@ -15,7 +16,7 @@ def just_do_it(): args = parser.parse_args() app_tar = [p for p in args.peers if p.endswith('.ios.interface')] if not app_tar: - print >> sys.stderr, 'No one IOS_INTERFACE module found' + print('No one IOS_INTERFACE module found', file=sys.stderr) shutil.copyfile(args.binary, os.path.join(args.temp_dir, 'bin')) if os.path.exists(args.target): os.remove(args.target) @@ -27,7 +28,7 @@ def just_do_it(): if len(app_tar) > 1: app_tar = [p for p in args.peers if not p.endswith('.default.ios.interface')] if len(app_tar) > 1: - print >> sys.stderr, 'Many IOS_INTERFACE modules found, {} will be used'.format(app_tar[-1]) + print('Many IOS_INTERFACE modules found, {} will be used'.format(app_tar[-1]), file=sys.stderr) app_tar = app_tar[-1] with tarfile.open(app_tar) as tf: tf.extractall(args.temp_dir) diff --git a/build/scripts/pack_jcoverage_resources.py b/build/scripts/pack_jcoverage_resources.py index 5881d90153..6ed3b1320d 100644 --- a/build/scripts/pack_jcoverage_resources.py +++ b/build/scripts/pack_jcoverage_resources.py @@ -1,3 +1,4 @@ +from __future__ import print_function import sys import tarfile import os @@ -11,7 +12,7 @@ def main(args): res = subprocess.call(args[args.index('-end') + 1 :]) if not os.path.exists(report_file): - print >> sys.stderr, 'Can\'t find jacoco exec file' + print('Can\'t find jacoco exec file', file=sys.stderr) return res with tarfile.open(output_file, 'w') as outf: diff --git a/build/scripts/python_yndexer.py b/build/scripts/python_yndexer.py index 3180665387..276ed20a27 100644 --- a/build/scripts/python_yndexer.py +++ b/build/scripts/python_yndexer.py @@ -1,3 +1,4 @@ +from __future__ import print_function import os import sys import threading @@ -49,5 +50,5 @@ if __name__ == '__main__': result = process.wait(timeout=timeout) if result != 0: - print >> sys.stderr, 'Yndexing process finished with code', result + print('Yndexing process finished with code', result, file=sys.stderr) touch(output_file) diff --git a/build/scripts/run_ios_simulator.py b/build/scripts/run_ios_simulator.py index b69ef81ccb..576ef9cee3 100644 --- a/build/scripts/run_ios_simulator.py +++ b/build/scripts/run_ios_simulator.py @@ -1,3 +1,4 @@ +from __future__ import print_function import argparse import json import os @@ -55,7 +56,7 @@ def action_spawn(simctl, profiles, device_dir, name, args): def action_kill(simctl, profiles, device_dir, name): device = filter(lambda x: x["name"] == name, get_all_devices(simctl, profiles, device_dir)) if not device: - print >> sys.stderr, "Device named {} not found; do nothing".format(name) + print("Device named {} not found; do nothing".format(name), file=sys.stderr) return if len(device) > 1: raise Exception("Can't remove: too many devices named {}:\n{}".format(name, '\n'.join(i for i in device))) diff --git a/build/scripts/run_msvc_wine.py b/build/scripts/run_msvc_wine.py index 38ffa1ffb0..3a498d7388 100644 --- a/build/scripts/run_msvc_wine.py +++ b/build/scripts/run_msvc_wine.py @@ -1,3 +1,4 @@ +from __future__ import print_function import sys import os import re @@ -41,7 +42,7 @@ def run_subprocess_with_timeout(timeout, args): stdout, stderr = p.communicate(timeout=timeout) return p, stdout, stderr except subprocess.TimeoutExpired as e: - print >> sys.stderr, 'timeout running {0}, error {1}, delay {2} seconds'.format(args, str(e), delay) + print('timeout running {0}, error {1}, delay {2} seconds'.format(args, str(e), delay), file=sys.stderr) if p is not None: try: p.kill() @@ -93,7 +94,7 @@ def call_wine_cmd_once(wine, cmd, env, mode): error = e if error is not None: - print >> sys.stderr, 'Output {} already exists and we have failed to remove it: {}'.format(output, error) + print('Output {} already exists and we have failed to remove it: {}'.format(output, error), file=sys.stderr) # print >>sys.stderr, cmd, env, wine @@ -164,7 +165,7 @@ def call_wine_cmd_once(wine, cmd, env, mode): stdout_and_stderr = '\n'.join(filter_lines()).strip() if stdout_and_stderr: - print >> sys.stderr, stdout_and_stderr + print(stdout_and_stderr, file=sys.stderr) return return_code @@ -175,7 +176,7 @@ def prepare_vc(fr, to): to_p = os.path.join(to, p) if not os.path.exists(to_p): - print >> sys.stderr, 'install %s -> %s' % (fr_p, to_p) + print('install %s -> %s' % (fr_p, to_p), file=sys.stderr) os.link(fr_p, to_p) @@ -196,7 +197,7 @@ def run_slave(): try: return call_wine_cmd_once([wine], args['cmd'], args['env'], args['mode']) except Exception as e: - print >> sys.stderr, '%s, will retry in %s' % (str(e), tout) + print('%s, will retry in %s' % (str(e), tout), file=sys.stderr) time.sleep(tout) tout = min(2 * tout, 4) @@ -508,7 +509,7 @@ def run_main(): return if mode == 'cxx': log = colorize(log) - print >> sys.stderr, log + print(log, file=sys.stderr) tout = 200 @@ -517,26 +518,26 @@ def run_main(): if rc in (-signal.SIGALRM, signal.SIGALRM): print_err_log(out) - print >> sys.stderr, '##append_tag##time out' + print('##append_tag##time out', file=sys.stderr) elif out and ' stack overflow ' in out: - print >> sys.stderr, '##append_tag##stack overflow' + print('##append_tag##stack overflow', file=sys.stderr) elif out and 'recvmsg: Connection reset by peer' in out: - print >> sys.stderr, '##append_tag##wine gone' + print('##append_tag##wine gone', file=sys.stderr) elif out and 'D8037' in out: - print >> sys.stderr, '##append_tag##repair wine' + print('##append_tag##repair wine', file=sys.stderr) try: os.unlink(os.path.join(os.environ['WINEPREFIX'], '.update-timestamp')) except Exception as e: - print >> sys.stderr, e + print(e, file=sys.stderr) else: print_err_log(out) # non-zero return code - bad, return it immediately if rc: - print >> sys.stderr, '##win_cmd##' + ' '.join(cmd) - print >> sys.stderr, '##args##' + ' '.join(free_args) + print('##win_cmd##' + ' '.join(cmd), file=sys.stderr) + print('##args##' + ' '.join(free_args), file=sys.stderr) return rc # check for output existence(if we expect it!) and real length @@ -545,7 +546,7 @@ def run_main(): return 0 else: # retry! - print >> sys.stderr, '##append_tag##no output' + print('##append_tag##no output', file=sys.stderr) else: return 0 @@ -575,7 +576,7 @@ def main(): except KeyboardInterrupt: sys.exit(4) except Exception as e: - print >> sys.stderr, str(e) + print(str(e), file=sys.stderr) sys.exit(3) diff --git a/build/scripts/symlink.py b/build/scripts/symlink.py index 9636bf3929..a19891df40 100755 --- a/build/scripts/symlink.py +++ b/build/scripts/symlink.py @@ -1,5 +1,6 @@ #!/usr/bin/env python +from __future__ import print_function import sys import os import platform @@ -8,7 +9,7 @@ from subprocess import call def symlink(): if len(sys.argv) < 3: - print >> sys.stderr, "Usage: symlink.py <source> <target>" + print("Usage: symlink.py <source> <target>", file=sys.stderr) sys.exit(1) source = sys.argv[1] diff --git a/build/scripts/with_crash_on_timeout.py b/build/scripts/with_crash_on_timeout.py index bde864ed29..915ed51c51 100644 --- a/build/scripts/with_crash_on_timeout.py +++ b/build/scripts/with_crash_on_timeout.py @@ -1,3 +1,4 @@ +from __future__ import print_function # TODO prettyboy remove after ya-bin release import os @@ -13,7 +14,7 @@ def main(args): with open(meta_path) as f: meta_info = json.loads(f.read()) if meta_info["exit_code"] == timeout_code: - print >> sys.stderr, meta_info["project"], 'crashed by timeout, use --test-disable-timeout option' + print(meta_info["project"], 'crashed by timeout, use --test-disable-timeout option', file=sys.stderr) return 1 return 0 diff --git a/build/scripts/yndexer.py b/build/scripts/yndexer.py index c8127de711..4e766e0fc8 100644 --- a/build/scripts/yndexer.py +++ b/build/scripts/yndexer.py @@ -1,3 +1,4 @@ +from __future__ import print_function import sys import subprocess import threading @@ -86,5 +87,5 @@ if __name__ == '__main__': result = process.wait(timeout=timeout) if result != 0: - print >> sys.stderr, 'Yndexing process finished with code', result + print('Yndexing process finished with code', result, file=sys.stderr) touch(output_file) diff --git a/build/ymake.core.conf b/build/ymake.core.conf index 388705f63d..0916dddbba 100644 --- a/build/ymake.core.conf +++ b/build/ymake.core.conf @@ -133,6 +133,8 @@ when ($USE_PREBUILT_TOOLS == "yes") { } } +CPP_LINTERS_DEFAULT_CONFIGS=build/config/tests/cpp_style/default_configs.json + ### @usage: FUNCTION_ORDERING_FILE(VAR_NAME) ### ### Select file for function reordering. Works only with lld linker. @@ -3029,22 +3031,22 @@ macro _SRC("rodata", SRC, SRCFLAGS...) { .SEM=target_rodata_sources PRIVATE ${input:SRC} ${hide;output;suf=${OBJ_SUF}.o:SRC} ${hide;input:"build/scripts/rodata2cpp.py"} && target_macroses-ITEM && target_macroses-macro target_rodata_sources && target_macroses-args PRIVATE ${input:SRC} ${hide;output;suf=${OBJ_SUF}.o:SRC} ${hide;input:"build/scripts/rodata2cpp.py"} } -macro _SRS_S_ASM(SRC, SRCFLAGS...) { - .CMD=$C_COMPILER $C_FLAGS_PLATFORM $CFLAGS $SFLAGS $SRCFLAGS -c -o ${output;suf=.o:SRC} ${input:SRC} $TOOLCHAIN_ENV ${pre=-I:_C__INCLUDE} +macro _SRC_S_ASM(SRC, SRCFLAGS...) { + .CMD=$C_COMPILER $C_FLAGS_PLATFORM $CFLAGS $SFLAGS $SRCFLAGS -c -o ${output;suf=.o:SRC} ${input:SRC} $TOOLCHAIN_ENV ${pre=-I:_C__INCLUDE} ${hide;kv:"p AS"} ${hide;kv:"pc light-green"} .SEM=target_sources PRIVATE ${input:SRC} ${hide;output;suf=${OBJ_SUF}.o:SRC} $_ADD_EXTRA_FLAGS($SRCFLAGS) && target_options-privates-ITEM && target_options-privates-option target_sources && target_options-privates-args ${input:SRC} ${hide;output;suf=${OBJ_SUF}.o:SRC} && add_language ASM && project_languages ASM } # tag:src-processing macro _SRC("S", SRC, SRCFLAGS...) { - .CMD=$_SRS_S_ASM($SRC $SRCFLAGS) - .SEM=$_SRS_S_ASM($SRC, $SRCFLAGS) + .CMD=$_SRC_S_ASM($SRC $SRCFLAGS) + .SEM=$_SRC_S_ASM($SRC, $SRCFLAGS) .STRUCT_CMD=yes } # tag:src-processing macro _SRC("s", SRC, SRCFLAGS...) { - .CMD=$_SRS_S_ASM($SRC $SRCFLAGS) - .SEM=$_SRS_S_ASM($SRC, $SRCFLAGS) + .CMD=$_SRC_S_ASM($SRC $SRCFLAGS) + .SEM=$_SRC_S_ASM($SRC, $SRCFLAGS) .STRUCT_CMD=yes } @@ -3272,7 +3274,6 @@ when ($IDE_MSVS_CALL == "yes") { NVCC_CFLAGS+=/Fd$(TEMP) } -NVCC_STD_VER=17 when ($MSVC == "yes") { NVCC_STD=/std:c++${NVCC_STD_VER} } @@ -4934,7 +4935,7 @@ NASM_FLAGS= # tag:nasm-specific macro _SRC_nasm_helper(SRC, SRCFLAGS...) { - .CMD=${NASM_RESOURCE_GLOBAL}/bin/nasm -f ${_YASM_FMT_VALUE}${HARDWARE_ARCH} -D ${pre=_;suf=_:HARDWARE_TYPE} $NASM_FLAGS ${pre=-I :_ASM__INCLUDE} $SRCFLAGS -o ${output;noext;suf=${OBJECT_SUF}:SRC} ${SRC} + .CMD=${NASM_RESOURCE_GLOBAL}/bin/nasm -f ${_YASM_FMT_VALUE}${HARDWARE_ARCH} -D ${pre=_;suf=_:HARDWARE_TYPE} $NASM_FLAGS ${pre=-I :_ASM__INCLUDE} $SRCFLAGS -o ${output;noext;suf=${OBJECT_SUF}:SRC} ${SRC} ${hide;kv:"p AS"} ${hide;kv:"pc light-green"} } # tag:nasm-specific @@ -5671,6 +5672,7 @@ when ($CUDA11) { CUDA_VERSION=11.4 CUDNN_VERSION=8.0.5 TENSORRT_VERSION=7 + NVCC_STD_VER=17 } when ($CUDA12) { @@ -5678,6 +5680,7 @@ when ($CUDA12) { CUDNN_VERSION=8.6.0 TENSORRT_VERSION=8 CUDA_ARCHITECTURES= + NVCC_STD_VER=20 } when ($TENSORFLOW_WITH_CUDA) { @@ -5830,33 +5833,30 @@ macro YA_CONF_JSON(File) { } # tag:internal -# Don't use directly - wrap with custom macros -macro ADD_LINTER_CHECK(CheckName, Linter, DEPENDS[], EXCLUDE[], FILES[], CONFIGS[]) { - SET(_FILES_VAR uniq_${hash:VAR_SALT}) - _GLOB($_FILES_VAR $FILES EXCLUDE $EXCLUDE) - _ADD_LINTER_CHECK($CheckName $Linter ${pre=DEPENDS :DEPENDS} FILES $$_FILES_VAR CONFIGS $CONFIGS) -} - -# tag:internal -# Draft. Don't use. -macro _CPP_STYLE(EXCLUDE[], Files...) { - ADD_LINTER_CHECK(cpp_style tools/cpp_style_checker/cpp_style_checker DEPENDS contrib/libs/clang16/tools/clang-format EXCLUDE $EXCLUDE FILES $Files CONFIGS devtools/ya/handlers/style/style_config) +### @usage: _STYLE_CPP([CONFIG]) +### +### Proxy. Don't use. Call _ADD_CPP_LINTER_CHECK directly if you need a new macro, see STYLE_CPP +macro _STYLE_CPP(CONFIG...) { + _ADD_CPP_LINTER_CHECK(NAME clang_format LINTER tools/cpp_style_checker/cpp_style_checker DEPENDS contrib/libs/clang16/tools/clang-format CONFIGS $CPP_LINTERS_DEFAULT_CONFIGS CUSTOM_CONFIG $CONFIG) } # tag:internal -# A convenient proxy -macro _STYLE_CPP(CONFIG) { +### @usage: _ADD_CPP_LINTER_CHECK(NAME name LINTER linter [DEPENDS deps] CONFIGS configs_file [GLOBAL_RESOURCES gr] [FILE_PROCESSING_TIME fpt] [EXTRA_PARAMS params] [CUSTOM_CONFIG cc]) +### +### Triggers respective plugin +macro _ADD_CPP_LINTER_CHECK(Args...) { SET(USE_ALL_SRCS yes) HEADERS(.) - _ADD_LINTER_CHECK(cpp_style tools/cpp_style_checker/cpp_style_checker DEPENDS contrib/libs/clang16/tools/clang-format CONFIGS $CONFIG) + SET_APPEND(_MAKEFILE_INCLUDE_LIKE_DEPS $CPP_LINTERS_DEFAULT_CONFIGS) } -#tag:test +# tag:test ### @usage STYLE_CPP() ### ### Run 'ya tool clang-format' test on all cpp sources and headers of the current module macro STYLE_CPP() { - _STYLE_CPP("build/config/tests/cpp_style/config.clang-format") + .ALLOWED_IN_COMMON=yes + _ADD_CPP_LINTER_CHECK(NAME clang_format LINTER tools/cpp_style_checker/cpp_style_checker DEPENDS contrib/libs/clang16/tools/clang-format CONFIGS $CPP_LINTERS_DEFAULT_CONFIGS) } ### @usage: HEADERS([Dir1 Dir2 ...]) diff --git a/build/ymake_conf.py b/build/ymake_conf.py index 670ff24337..6e53ddf856 100755 --- a/build/ymake_conf.py +++ b/build/ymake_conf.py @@ -2175,11 +2175,7 @@ class MSVCLinker(MSVC, Linker): linker_lib = self.tc.lib emit('_MSVC_LIB', '"{}"'.format(linker_lib)) - emit('_MSVC_LIB_OLD_UNQUOTED', linker_lib) - emit('_MSVC_LIB_OLD', '${quo:_MSVC_LIB_OLD_UNQUOTED}') emit('_MSVC_LINK', '"{}"'.format(linker)) - emit('_MSVC_LINK_OLD_UNQUOTED', linker) - emit('_MSVC_LINK_OLD', '${quo:_MSVC_LINK_OLD_UNQUOTED}') if self.build.is_release: emit('LINK_EXE_FLAGS_PER_TYPE', '$LINK_EXE_FLAGS_RELEASE') @@ -2425,6 +2421,13 @@ class Cuda(object): emit('NVCC_ENV', '${env:_NVCC_ENV}') emit('_NVCC_ENV', 'PATH=$CUDA_ROOT/nvvm/bin:$CUDA_ROOT/bin') + if self.cuda_version.value.startswith('10.'): + emit('NVCC_STD_VER', '17') + elif self.cuda_version.value.startswith('11.'): + emit('NVCC_STD_VER', '17') + else: + emit('NVCC_STD_VER', '20') + def print_macros(self): mtime = ' ' custom_pid = ' ' @@ -2453,7 +2456,7 @@ class Cuda(object): if not self.cuda_version.from_user: return False - if self.cuda_version.value in ('11.4', '11.8', '12.1', '12.2'): + if self.cuda_version.value in ('11.4', '11.8', '12.1', '12.2', '12.6'): return True elif self.cuda_version.value in ('10.2', '11.4.19') and target.is_linux_armv8: return True diff --git a/contrib/libs/croaring/include/roaring/memory.h b/contrib/libs/croaring/include/roaring/memory.h index ad9a64f5e2..6ec3d348dc 100644 --- a/contrib/libs/croaring/include/roaring/memory.h +++ b/contrib/libs/croaring/include/roaring/memory.h @@ -1,12 +1,12 @@ #ifndef INCLUDE_ROARING_MEMORY_H_ #define INCLUDE_ROARING_MEMORY_H_ +#include <stddef.h> // for size_t + #ifdef __cplusplus extern "C" { #endif -#include <stddef.h> // for size_t - typedef void* (*roaring_malloc_p)(size_t); typedef void* (*roaring_realloc_p)(void*, size_t); typedef void* (*roaring_calloc_p)(size_t, size_t); diff --git a/contrib/libs/croaring/include/roaring/portability.h b/contrib/libs/croaring/include/roaring/portability.h index 5aebe193c9..897feedbf8 100644 --- a/contrib/libs/croaring/include/roaring/portability.h +++ b/contrib/libs/croaring/include/roaring/portability.h @@ -585,6 +585,8 @@ static inline uint32_t croaring_refcount_get(const croaring_refcount_t *val) { #if defined(__GNUC__) || defined(__clang__) #define CROARING_DEPRECATED __attribute__((deprecated)) +#elif defined(_MSC_VER) +#define CROARING_DEPRECATED __declspec(deprecated) #else #define CROARING_DEPRECATED #endif // defined(__GNUC__) || defined(__clang__) diff --git a/contrib/libs/croaring/include/roaring/roaring64.h b/contrib/libs/croaring/include/roaring/roaring64.h index c71bccb917..fd89feb5e0 100644 --- a/contrib/libs/croaring/include/roaring/roaring64.h +++ b/contrib/libs/croaring/include/roaring/roaring64.h @@ -1,6 +1,7 @@ #ifndef ROARING64_H #define ROARING64_H +#include <roaring.h> #include <stdbool.h> #include <stddef.h> #include <stdint.h> @@ -93,6 +94,14 @@ roaring64_bitmap_t *roaring64_bitmap_of_ptr(size_t n_args, #endif /** + * Create a new bitmap by moving containers from a 32 bit roaring bitmap. + * + * After calling this function, the original bitmap will be empty, and the + * returned bitmap will contain all the values from the original bitmap. + */ +roaring64_bitmap_t *roaring64_bitmap_move_from_roaring32(roaring_bitmap_t *r); + +/** * Create a new bitmap containing all the values in [min, max) that are at a * distance k*step from min. */ diff --git a/contrib/libs/croaring/include/roaring/roaring_version.h b/contrib/libs/croaring/include/roaring/roaring_version.h index 3f7519449e..0d136bbbd3 100644 --- a/contrib/libs/croaring/include/roaring/roaring_version.h +++ b/contrib/libs/croaring/include/roaring/roaring_version.h @@ -2,11 +2,11 @@ // /include/roaring/roaring_version.h automatically generated by release.py, do not change by hand #ifndef ROARING_INCLUDE_ROARING_VERSION #define ROARING_INCLUDE_ROARING_VERSION -#define ROARING_VERSION "4.1.1" +#define ROARING_VERSION "4.1.2" enum { ROARING_VERSION_MAJOR = 4, ROARING_VERSION_MINOR = 1, - ROARING_VERSION_REVISION = 1 + ROARING_VERSION_REVISION = 2 }; #endif // ROARING_INCLUDE_ROARING_VERSION // clang-format on
\ No newline at end of file diff --git a/contrib/libs/croaring/src/roaring64.c b/contrib/libs/croaring/src/roaring64.c index d41507b3c4..e63d3d965c 100644 --- a/contrib/libs/croaring/src/roaring64.c +++ b/contrib/libs/croaring/src/roaring64.c @@ -178,6 +178,43 @@ roaring64_bitmap_t *roaring64_bitmap_copy(const roaring64_bitmap_t *r) { return result; } +/** + * Steal the containers from a 32-bit bitmap and insert them into a 64-bit + * bitmap (with an offset) + * + * After calling this function, the original bitmap will be empty, and the + * returned bitmap will contain all the values from the original bitmap. + */ +static void move_from_roaring32_offset(roaring64_bitmap_t *dst, + roaring_bitmap_t *src, + uint32_t high_bits) { + uint64_t key_base = ((uint64_t)high_bits) << 32; + uint32_t r32_size = ra_get_size(&src->high_low_container); + for (uint32_t i = 0; i < r32_size; ++i) { + uint16_t key = ra_get_key_at_index(&src->high_low_container, i); + uint8_t typecode; + container_t *container = ra_get_container_at_index( + &src->high_low_container, (uint16_t)i, &typecode); + + uint8_t high48[ART_KEY_BYTES]; + uint64_t high48_bits = key_base | ((uint64_t)key << 16); + split_key(high48_bits, high48); + leaf_t *leaf = create_leaf(container, typecode); + art_insert(&dst->art, high48, (art_val_t *)leaf); + } + // We stole all the containers, so leave behind a size of zero + src->high_low_container.size = 0; +} + +roaring64_bitmap_t *roaring64_bitmap_move_from_roaring32( + roaring_bitmap_t *bitmap32) { + roaring64_bitmap_t *result = roaring64_bitmap_create(); + + move_from_roaring32_offset(result, bitmap32, 0); + + return result; +} + roaring64_bitmap_t *roaring64_bitmap_from_range(uint64_t min, uint64_t max, uint64_t step) { if (step == 0 || max <= min) { @@ -1947,22 +1984,8 @@ roaring64_bitmap_t *roaring64_bitmap_portable_deserialize_safe( read_bytes += bitmap32_size; // Insert all containers of the 32-bit bitmap into the 64-bit bitmap. - uint32_t r32_size = ra_get_size(&bitmap32->high_low_container); - for (size_t i = 0; i < r32_size; ++i) { - uint16_t key16 = - ra_get_key_at_index(&bitmap32->high_low_container, (uint16_t)i); - uint8_t typecode; - container_t *container = ra_get_container_at_index( - &bitmap32->high_low_container, (uint16_t)i, &typecode); - - uint64_t high48_bits = - (((uint64_t)high32) << 32) | (((uint64_t)key16) << 16); - uint8_t high48[ART_KEY_BYTES]; - split_key(high48_bits, high48); - leaf_t *leaf = create_leaf(container, typecode); - art_insert(&r->art, high48, (art_val_t *)leaf); - } - roaring_bitmap_free_without_containers(bitmap32); + move_from_roaring32_offset(r, bitmap32, high32); + roaring_bitmap_free(bitmap32); } return r; } diff --git a/contrib/libs/croaring/ya.make b/contrib/libs/croaring/ya.make index f6a661c0b7..55acab5673 100644 --- a/contrib/libs/croaring/ya.make +++ b/contrib/libs/croaring/ya.make @@ -10,9 +10,9 @@ LICENSE( LICENSE_TEXTS(.yandex_meta/licenses.list.txt) -VERSION(4.1.1) +VERSION(4.1.2) -ORIGINAL_SOURCE(https://github.com/RoaringBitmap/CRoaring/archive/v4.1.1.tar.gz) +ORIGINAL_SOURCE(https://github.com/RoaringBitmap/CRoaring/archive/v4.1.2.tar.gz) ADDINCL( GLOBAL contrib/libs/croaring/include diff --git a/contrib/libs/cxxsupp/libcxx/include/__iterator/reverse_iterator.h b/contrib/libs/cxxsupp/libcxx/include/__iterator/reverse_iterator.h index beb10f7f4f..60969e1751 100644 --- a/contrib/libs/cxxsupp/libcxx/include/__iterator/reverse_iterator.h +++ b/contrib/libs/cxxsupp/libcxx/include/__iterator/reverse_iterator.h @@ -144,7 +144,7 @@ public: _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_SINCE_CXX17 reference operator*() const {_Iter __tmp = current; return *--__tmp;} -#if _LIBCPP_STD_VER >= 20 +#if (_LIBCPP_STD_VER >= 20) && !defined(__NVCC__) _LIBCPP_INLINE_VISIBILITY constexpr pointer operator->() const requires is_pointer_v<_Iter> || requires(const _Iter __i) { __i.operator->(); } diff --git a/contrib/libs/cxxsupp/libcxxrt/exception.cc b/contrib/libs/cxxsupp/libcxxrt/exception.cc index 6f8c223b34..2fe90c3e1c 100644 --- a/contrib/libs/cxxsupp/libcxxrt/exception.cc +++ b/contrib/libs/cxxsupp/libcxxrt/exception.cc @@ -282,7 +282,7 @@ namespace std { // Forward declaration of standard library terminate() function used to // abort execution. - void terminate(void) _LIBCXXRT_NOEXCEPT; + [[noreturn]] void terminate(void) _LIBCXXRT_NOEXCEPT; } using namespace ABI_NAMESPACE; @@ -1636,7 +1636,7 @@ namespace std * Terminates the program, calling a custom terminate implementation if * required. */ - void terminate() _LIBCXXRT_NOEXCEPT + [[noreturn]] void terminate() _LIBCXXRT_NOEXCEPT { static __cxa_thread_info *info = thread_info(); if (0 != info && 0 != info->terminateHandler) diff --git a/contrib/libs/cxxsupp/libcxxrt/ya.make b/contrib/libs/cxxsupp/libcxxrt/ya.make index a3632f35d4..bb32702ba5 100644 --- a/contrib/libs/cxxsupp/libcxxrt/ya.make +++ b/contrib/libs/cxxsupp/libcxxrt/ya.make @@ -11,9 +11,9 @@ LICENSE( LICENSE_TEXTS(.yandex_meta/licenses.list.txt) -VERSION(2024-08-06) +VERSION(2024-09-24) -ORIGINAL_SOURCE(https://github.com/libcxxrt/libcxxrt/archive/7a3ef57f64be0f2f2a156af011adfbe76c7dce74.tar.gz) +ORIGINAL_SOURCE(https://github.com/libcxxrt/libcxxrt/archive/40e4fa2049930412a2c43cdf0c39b6b5aa735341.tar.gz) ADDINCL( contrib/libs/cxxsupp/libcxxrt diff --git a/contrib/libs/protobuf/src/google/protobuf/extension_set.h b/contrib/libs/protobuf/src/google/protobuf/extension_set.h index c17bf4dc8a..a68db9efc1 100644 --- a/contrib/libs/protobuf/src/google/protobuf/extension_set.h +++ b/contrib/libs/protobuf/src/google/protobuf/extension_set.h @@ -1564,4 +1564,5 @@ void LinkExtensionReflection( } // namespace google #include "google/protobuf/port_undef.inc" + #endif // GOOGLE_PROTOBUF_EXTENSION_SET_H__ diff --git a/contrib/libs/protobuf/src/google/protobuf/json/internal/parser.cc b/contrib/libs/protobuf/src/google/protobuf/json/internal/parser.cc index 0da57a1f05..3bacf60c3b 100644 --- a/contrib/libs/protobuf/src/google/protobuf/json/internal/parser.cc +++ b/contrib/libs/protobuf/src/google/protobuf/json/internal/parser.cc @@ -972,11 +972,9 @@ y_absl::Status ParseFieldMask(JsonLexer& lex, const Desc<Traits>& desc, // Assume approximately six-letter words, so add one extra space for an // underscore for every six bytes. snake_path.reserve(path.size() * 7 / 6); - - // Port from protobuf 21.x for entity/ugc (TODO remove?). + // Port from protobuf 21.x bool is_quoted = false; bool is_escaping = false; - for (char c : path) { // Outputs quoted string as-is. if (is_quoted) { diff --git a/contrib/libs/protobuf/ya.make b/contrib/libs/protobuf/ya.make index ef458741f3..e72de69ce9 100644 --- a/contrib/libs/protobuf/ya.make +++ b/contrib/libs/protobuf/ya.make @@ -43,9 +43,9 @@ ENDIF() PEERDIR( contrib/libs/zlib - library/cpp/sanitizer/include - contrib/restricted/abseil-cpp-tstring/y_absl/status contrib/restricted/abseil-cpp-tstring/y_absl/log + contrib/restricted/abseil-cpp-tstring/y_absl/status + library/cpp/sanitizer/include ) ADDINCL( diff --git a/contrib/python/cffi/py3/.dist-info/METADATA b/contrib/python/cffi/py3/.dist-info/METADATA index a0d90a31e9..60b0779f68 100644 --- a/contrib/python/cffi/py3/.dist-info/METADATA +++ b/contrib/python/cffi/py3/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: cffi -Version: 1.17.0 +Version: 1.17.1 Summary: Foreign Function Interface for Python calling C code. Home-page: http://cffi.readthedocs.org Author: Armin Rigo, Maciej Fijalkowski diff --git a/contrib/python/cffi/py3/c/_cffi_backend.c b/contrib/python/cffi/py3/c/_cffi_backend.c index aa4ec48ba9..4ba773a175 100644 --- a/contrib/python/cffi/py3/c/_cffi_backend.c +++ b/contrib/python/cffi/py3/c/_cffi_backend.c @@ -2,7 +2,7 @@ #include <Python.h> #include "structmember.h" -#define CFFI_VERSION "1.17.0" +#define CFFI_VERSION "1.17.1" #ifdef MS_WIN32 #include <windows.h> diff --git a/contrib/python/cffi/py3/cffi/__init__.py b/contrib/python/cffi/py3/cffi/__init__.py index deeacc57cd..2e35a38c9c 100644 --- a/contrib/python/cffi/py3/cffi/__init__.py +++ b/contrib/python/cffi/py3/cffi/__init__.py @@ -5,8 +5,8 @@ from .api import FFI from .error import CDefError, FFIError, VerificationError, VerificationMissing from .error import PkgConfigError -__version__ = "1.17.0" -__version_info__ = (1, 17, 0) +__version__ = "1.17.1" +__version_info__ = (1, 17, 1) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/contrib/python/cffi/py3/cffi/_embedding.h b/contrib/python/cffi/py3/cffi/_embedding.h index 0eeeea831d..94d8b30a9e 100644 --- a/contrib/python/cffi/py3/cffi/_embedding.h +++ b/contrib/python/cffi/py3/cffi/_embedding.h @@ -225,7 +225,7 @@ static int _cffi_initialize_python(void) if (f != NULL && f != Py_None) { PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME - "\ncompiled with cffi version: 1.17.0" + "\ncompiled with cffi version: 1.17.1" "\n_cffi_backend module: ", f); modules = PyImport_GetModuleDict(); mod = PyDict_GetItemString(modules, "_cffi_backend"); diff --git a/contrib/python/cffi/py3/cffi/_shimmed_dist_utils.py b/contrib/python/cffi/py3/cffi/_shimmed_dist_utils.py index 611bf40f40..c3d2312818 100644 --- a/contrib/python/cffi/py3/cffi/_shimmed_dist_utils.py +++ b/contrib/python/cffi/py3/cffi/_shimmed_dist_utils.py @@ -30,7 +30,11 @@ try: from distutils.log import set_threshold, set_verbosity if sys.platform == 'win32': - from distutils.msvc9compiler import MSVCCompiler + try: + # FUTURE: msvc9compiler module was removed in setuptools 74; consider removing, as it's only used by an ancient patch in `recompiler` + from distutils.msvc9compiler import MSVCCompiler + except ImportError: + MSVCCompiler = None except Exception as ex: if sys.version_info >= (3, 12): raise Exception("This CFFI feature requires setuptools on Python >= 3.12. Please install the setuptools package.") from ex diff --git a/contrib/python/cffi/py3/cffi/api.py b/contrib/python/cffi/py3/cffi/api.py index edeb792810..5a474f3da9 100644 --- a/contrib/python/cffi/py3/cffi/api.py +++ b/contrib/python/cffi/py3/cffi/api.py @@ -693,7 +693,8 @@ class FFI(object): raise TypeError("emit_c_code() is only for C extension modules, " "not for dlopen()-style pure Python modules") recompile(self, module_name, source, - c_file=filename, call_c_compiler=False, **kwds) + c_file=filename, call_c_compiler=False, + uses_ffiplatform=False, **kwds) def emit_python_code(self, filename): from .recompiler import recompile @@ -705,7 +706,8 @@ class FFI(object): raise TypeError("emit_python_code() is only for dlopen()-style " "pure Python modules, not for C extension modules") recompile(self, module_name, source, - c_file=filename, call_c_compiler=False, **kwds) + c_file=filename, call_c_compiler=False, + uses_ffiplatform=False, **kwds) def compile(self, tmpdir='.', verbose=0, target=None, debug=None): """The 'target' argument gives the final file name of the diff --git a/contrib/python/cffi/py3/cffi/recompiler.py b/contrib/python/cffi/py3/cffi/recompiler.py index dd22b21c2c..c236807fe8 100644 --- a/contrib/python/cffi/py3/cffi/recompiler.py +++ b/contrib/python/cffi/py3/cffi/recompiler.py @@ -1417,6 +1417,10 @@ else: s = s.encode('ascii') super(NativeIO, self).write(s) +def _is_file_like(maybefile): + # compare to xml.etree.ElementTree._get_writer + return hasattr(maybefile, 'write') + def _make_c_or_py_source(ffi, module_name, preamble, target_file, verbose): if verbose: print("generating %s" % (target_file,)) @@ -1424,6 +1428,9 @@ def _make_c_or_py_source(ffi, module_name, preamble, target_file, verbose): target_is_python=(preamble is None)) recompiler.collect_type_table() recompiler.collect_step_tables() + if _is_file_like(target_file): + recompiler.write_source_to_f(target_file, preamble) + return True f = NativeIO() recompiler.write_source_to_f(f, preamble) output = f.getvalue() @@ -1481,9 +1488,12 @@ def _unpatch_meths(patchlist): def _patch_for_embedding(patchlist): if sys.platform == 'win32': # we must not remove the manifest when building for embedding! + # FUTURE: this module was removed in setuptools 74; this is likely dead code and should be removed, + # since the toolchain it supports (VS2005-2008) is also long dead. from cffi._shimmed_dist_utils import MSVCCompiler - _patch_meth(patchlist, MSVCCompiler, '_remove_visual_c_ref', - lambda self, manifest_file: manifest_file) + if MSVCCompiler is not None: + _patch_meth(patchlist, MSVCCompiler, '_remove_visual_c_ref', + lambda self, manifest_file: manifest_file) if sys.platform == 'darwin': # we must not make a '-bundle', but a '-dynamiclib' instead @@ -1517,12 +1527,16 @@ def _patch_for_target(patchlist, target): def recompile(ffi, module_name, preamble, tmpdir='.', call_c_compiler=True, c_file=None, source_extension='.c', extradir=None, - compiler_verbose=1, target=None, debug=None, **kwds): + compiler_verbose=1, target=None, debug=None, + uses_ffiplatform=True, **kwds): if not isinstance(module_name, str): module_name = module_name.encode('ascii') if ffi._windows_unicode: ffi._apply_windows_unicode(kwds) if preamble is not None: + if call_c_compiler and _is_file_like(c_file): + raise TypeError("Writing to file-like objects is not supported " + "with call_c_compiler=True") embedding = (ffi._embedding is not None) if embedding: ffi._apply_embedding_fix(kwds) @@ -1541,7 +1555,10 @@ def recompile(ffi, module_name, preamble, tmpdir='.', call_c_compiler=True, else: target = '*' # - ext = ffiplatform.get_extension(ext_c_file, module_name, **kwds) + if uses_ffiplatform: + ext = ffiplatform.get_extension(ext_c_file, module_name, **kwds) + else: + ext = None updated = make_c_source(ffi, module_name, preamble, c_file, verbose=compiler_verbose) if call_c_compiler: diff --git a/contrib/python/cffi/py3/ya.make b/contrib/python/cffi/py3/ya.make index 30e438d17e..90ddf448de 100644 --- a/contrib/python/cffi/py3/ya.make +++ b/contrib/python/cffi/py3/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(1.17.0) +VERSION(1.17.1) LICENSE(MIT) diff --git a/contrib/python/executing/.dist-info/METADATA b/contrib/python/executing/.dist-info/METADATA index b598e4907d..45ff9aa881 100644 --- a/contrib/python/executing/.dist-info/METADATA +++ b/contrib/python/executing/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: executing -Version: 2.0.1 +Version: 2.1.0 Summary: Get the currently executing AST node of a frame, and other information Home-page: https://github.com/alexmojaki/executing Author: Alex Hall @@ -9,25 +9,23 @@ License: MIT Classifier: License :: OSI Approved :: MIT License Classifier: Programming Language :: Python Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python :: 3.5 -Classifier: Programming Language :: Python :: 3.6 -Classifier: Programming Language :: Python :: 3.7 Classifier: Programming Language :: Python :: 3.8 Classifier: Programming Language :: Python :: 3.9 Classifier: Programming Language :: Python :: 3.10 Classifier: Programming Language :: Python :: 3.11 Classifier: Programming Language :: Python :: 3.12 -Requires-Python: >=3.5 +Classifier: Programming Language :: Python :: 3.13 +Requires-Python: >=3.8 Description-Content-Type: text/markdown License-File: LICENSE.txt Provides-Extra: tests -Requires-Dist: asttokens >=2.1.0 ; extra == 'tests' -Requires-Dist: ipython ; extra == 'tests' -Requires-Dist: pytest ; extra == 'tests' -Requires-Dist: coverage ; extra == 'tests' -Requires-Dist: coverage-enable-subprocess ; extra == 'tests' -Requires-Dist: littleutils ; extra == 'tests' -Requires-Dist: rich ; (python_version >= "3.11") and extra == 'tests' +Requires-Dist: asttokens>=2.1.0; extra == "tests" +Requires-Dist: ipython; extra == "tests" +Requires-Dist: pytest; extra == "tests" +Requires-Dist: coverage; extra == "tests" +Requires-Dist: coverage-enable-subprocess; extra == "tests" +Requires-Dist: littleutils; extra == "tests" +Requires-Dist: rich; python_version >= "3.11" and extra == "tests" # executing diff --git a/contrib/python/executing/executing/_position_node_finder.py b/contrib/python/executing/executing/_position_node_finder.py index 8ca21a67bd..7a814150da 100644 --- a/contrib/python/executing/executing/_position_node_finder.py +++ b/contrib/python/executing/executing/_position_node_finder.py @@ -72,7 +72,7 @@ def mangled_name(node: EnhancedAST) -> str: @lru_cache(128) # pragma: no mutate def get_instructions(code: CodeType) -> list[dis.Instruction]: - return list(dis.get_instructions(code, show_caches=True)) + return list(dis.get_instructions(code)) types_cmp_issue_fix = ( @@ -114,7 +114,7 @@ class PositionNodeFinder(object): """ def __init__(self, frame: FrameType, stmts: Set[EnhancedAST], tree: ast.Module, lasti: int, source: Source): - self.bc_list = get_instructions(frame.f_code) + self.bc_dict={bc.offset:bc for bc in get_instructions(frame.f_code) } self.source = source self.decorator: Optional[EnhancedAST] = None @@ -141,7 +141,7 @@ class PositionNodeFinder(object): # we ignore here the start position and try to find the ast-node just by end position and expected node type # This is save, because there can only be one attribute ending at a specific point in the source code. typ = (ast.Attribute,) - elif self.opname(lasti) == "CALL": + elif self.opname(lasti) in ("CALL", "CALL_KW"): # A CALL instruction can be a method call, in which case the lineno and col_offset gets changed by the compiler. # Therefore we ignoring here this attributes and searchnig for a Call-node only by end_col_offset and end_lineno. # This is save, because there can only be one method ending at a specific point in the source code. @@ -156,13 +156,18 @@ class PositionNodeFinder(object): typ=typ, ) - self.known_issues(self.result, self.instruction(lasti)) + instruction = self.instruction(lasti) + assert instruction is not None + + self.result = self.fix_result(self.result, instruction) + + self.known_issues(self.result, instruction) self.test_for_decorator(self.result, lasti) # verify if self.decorator is None: - self.verify(self.result, self.instruction(lasti)) + self.verify(self.result, instruction) else: assert_(self.decorator in self.result.decorator_list) @@ -213,6 +218,32 @@ class PositionNodeFinder(object): if sys.version_info < (3, 12): index += 4 + def fix_result( + self, node: EnhancedAST, instruction: dis.Instruction + ) -> EnhancedAST: + if ( + sys.version_info >= (3, 12, 5) + and instruction.opname in ("GET_ITER", "FOR_ITER") + and isinstance(node.parent, ast.For) + and node is node.parent.iter + ): + # node positions have changed in 3.12.5 + # https://github.com/python/cpython/issues/93691 + # `for` calls __iter__ and __next__ during execution, the calling + # expression of these calls was the ast.For node since cpython 3.11 (see test_iter). + # cpython 3.12.5 changed this to the `iter` node of the loop, to make tracebacks easier to read. + # This keeps backward compatibility with older executing versions. + + # there are also cases like: + # + # for a in iter(l): pass + # + # where `iter(l)` would be otherwise the resulting node for the `iter()` call and the __iter__ call of the for implementation. + # keeping the old behaviour makes it possible to distinguish both cases. + + return node.parent + return node + def known_issues(self, node: EnhancedAST, instruction: dis.Instruction) -> None: if instruction.opname in ("COMPARE_OP", "IS_OP", "CONTAINS_OP") and isinstance( node, types_cmp_issue @@ -324,6 +355,35 @@ class PositionNodeFinder(object): ): raise KnownIssue("exception generation maps to condition") + if sys.version_info >= (3, 13): + if instruction.opname in ( + "STORE_FAST_STORE_FAST", + "STORE_FAST_LOAD_FAST", + "LOAD_FAST_LOAD_FAST", + ): + raise KnownIssue(f"can not map {instruction.opname} to two ast nodes") + + if instruction.opname == "LOAD_FAST" and instruction.argval == "__class__": + # example: + # class T: + # def a(): + # super() + # some_node # <- there is a LOAD_FAST for this node because we use super() + + raise KnownIssue( + f"loading of __class__ is accociated with a random node at the end of a class if you use super()" + ) + + if ( + instruction.opname == "COMPARE_OP" + and isinstance(node, ast.UnaryOp) + and isinstance(node.operand,ast.Compare) + and isinstance(node.op, ast.Not) + ): + # work around for + # https://github.com/python/cpython/issues/114671 + self.result = node.operand + @staticmethod def is_except_cleanup(inst: dis.Instruction, node: EnhancedAST) -> bool: if inst.opname not in ( @@ -703,6 +763,52 @@ class PositionNodeFinder(object): if node_match(ast.FormattedValue) and inst_match("FORMAT_VALUE"): return + if sys.version_info >= (3, 13): + + if inst_match("NOP"): + return + + if inst_match("TO_BOOL") and node_match(ast.BoolOp): + return + + if inst_match("CALL_KW") and node_match((ast.Call, ast.ClassDef)): + return + + if inst_match("LOAD_FAST", argval=".type_params"): + return + + if inst_match("LOAD_FAST", argval="__classdict__"): + return + + if inst_match("LOAD_FAST") and node_match( + ( + ast.FunctionDef, + ast.ClassDef, + ast.TypeAlias, + ast.TypeVar, + ast.Lambda, + ast.AsyncFunctionDef, + ) + ): + # These are loads for closure variables. + # It is difficult to check that this is actually closure variable, see: + # https://github.com/alexmojaki/executing/pull/80#discussion_r1716027317 + return + + if ( + inst_match("LOAD_FAST") + and node_match(ast.TypeAlias) + and node.name.id == instruction.argval + ): + return + + if inst_match("STORE_NAME",argval="__static_attributes__"): + # the node is the first node in the body + return + + if inst_match("LOAD_FAST") and isinstance(node.parent,ast.TypeVar): + return + # old verifier @@ -771,11 +877,14 @@ class PositionNodeFinder(object): raise VerifierFailure(title, node, instruction) - def instruction(self, index: int) -> dis.Instruction: - return self.bc_list[index // 2] + def instruction(self, index: int) -> Optional[dis.Instruction]: + return self.bc_dict.get(index,None) def opname(self, index: int) -> str: - return self.instruction(index).opname + i=self.instruction(index) + if i is None: + return "CACHE" + return i.opname extra_node_types=() if sys.version_info >= (3,12): @@ -798,7 +907,10 @@ class PositionNodeFinder(object): *extra_node_types, ), ) -> EnhancedAST: - position = self.instruction(index).positions + instruction = self.instruction(index) + assert instruction is not None + + position = instruction.positions assert position is not None and position.lineno is not None return only( diff --git a/contrib/python/executing/executing/executing.py b/contrib/python/executing/executing/executing.py index 7727c42232..5cf117e18c 100644 --- a/contrib/python/executing/executing/executing.py +++ b/contrib/python/executing/executing/executing.py @@ -273,16 +273,15 @@ class Source(object): node_finder = NodeFinder(frame, stmts, tree, lasti, source) node = node_finder.result decorator = node_finder.decorator + + if node: + new_stmts = {statement_containing_node(node)} + assert_(new_stmts <= stmts) + stmts = new_stmts except Exception: if TESTING: raise - assert stmts is not None - if node: - new_stmts = {statement_containing_node(node)} - assert_(new_stmts <= stmts) - stmts = new_stmts - executing_cache[key] = args = source, node, stmts, decorator return Executing(frame, *args) diff --git a/contrib/python/executing/executing/version.py b/contrib/python/executing/executing/version.py index 9d909dcc3c..b15121b0fe 100644 --- a/contrib/python/executing/executing/version.py +++ b/contrib/python/executing/executing/version.py @@ -1 +1 @@ -__version__ = '2.0.1'
\ No newline at end of file +__version__ = '2.1.0'
\ No newline at end of file diff --git a/contrib/python/executing/ya.make b/contrib/python/executing/ya.make index b676502282..b437b26981 100644 --- a/contrib/python/executing/ya.make +++ b/contrib/python/executing/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(2.0.1) +VERSION(2.1.0) LICENSE(MIT) diff --git a/contrib/python/hypothesis/py3/.dist-info/METADATA b/contrib/python/hypothesis/py3/.dist-info/METADATA index d1e09fb0ef..b78a2eeab3 100644 --- a/contrib/python/hypothesis/py3/.dist-info/METADATA +++ b/contrib/python/hypothesis/py3/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: hypothesis -Version: 6.111.2 +Version: 6.112.0 Summary: A library for property-based testing Home-page: https://hypothesis.works Author: David R. MacIver and Zac Hatfield-Dodds diff --git a/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/data.py b/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/data.py index 18f3bd9b8a..0f4f1138f3 100644 --- a/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/data.py +++ b/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/data.py @@ -111,11 +111,12 @@ class FloatKWargs(TypedDict): class StringKWargs(TypedDict): intervals: IntervalSet min_size: int - max_size: Optional[int] + max_size: int class BytesKWargs(TypedDict): - size: int + min_size: int + max_size: int class BooleanKWargs(TypedDict): @@ -206,7 +207,7 @@ NASTY_FLOATS.extend([-x for x in NASTY_FLOATS]) FLOAT_INIT_LOGIC_CACHE = LRUCache(4096) POOLED_KWARGS_CACHE = LRUCache(4096) -DRAW_STRING_DEFAULT_MAX_SIZE = 10**10 # "arbitrarily large" +COLLECTION_DEFAULT_MAX_SIZE = 10**10 # "arbitrarily large" class Example: @@ -1036,7 +1037,7 @@ class IRNode: return self.value == (minimal_char * self.kwargs["min_size"]) if self.ir_type == "bytes": # smallest size and all-zero value. - return len(self.value) == self.kwargs["size"] and not any(self.value) + return len(self.value) == self.kwargs["min_size"] and not any(self.value) raise NotImplementedError(f"unhandled ir_type {self.ir_type}") @@ -1095,7 +1096,9 @@ def ir_value_permitted(value, ir_type, kwargs): return False return all(ord(c) in kwargs["intervals"] for c in value) elif ir_type == "bytes": - return len(value) == kwargs["size"] + if len(value) < kwargs["min_size"]: + return False + return kwargs["max_size"] is None or len(value) <= kwargs["max_size"] elif ir_type == "boolean": if kwargs["p"] <= 2 ** (-64): return value is False @@ -1314,7 +1317,7 @@ class PrimitiveProvider(abc.ABC): intervals: IntervalSet, *, min_size: int = 0, - max_size: Optional[int] = None, + max_size: int = COLLECTION_DEFAULT_MAX_SIZE, forced: Optional[str] = None, fake_forced: bool = False, ) -> str: @@ -1322,7 +1325,12 @@ class PrimitiveProvider(abc.ABC): @abc.abstractmethod def draw_bytes( - self, size: int, *, forced: Optional[bytes] = None, fake_forced: bool = False + self, + min_size: int = 0, + max_size: int = COLLECTION_DEFAULT_MAX_SIZE, + *, + forced: Optional[bytes] = None, + fake_forced: bool = False, ) -> bytes: raise NotImplementedError @@ -1606,14 +1614,10 @@ class HypothesisProvider(PrimitiveProvider): intervals: IntervalSet, *, min_size: int = 0, - max_size: Optional[int] = None, + max_size: int = COLLECTION_DEFAULT_MAX_SIZE, forced: Optional[str] = None, fake_forced: bool = False, ) -> str: - if max_size is None: - max_size = DRAW_STRING_DEFAULT_MAX_SIZE - - assert forced is None or min_size <= len(forced) <= max_size assert self._cd is not None average_size = min( @@ -1663,17 +1667,40 @@ class HypothesisProvider(PrimitiveProvider): return "".join(chars) def draw_bytes( - self, size: int, *, forced: Optional[bytes] = None, fake_forced: bool = False + self, + min_size: int = 0, + max_size: int = COLLECTION_DEFAULT_MAX_SIZE, + *, + forced: Optional[bytes] = None, + fake_forced: bool = False, ) -> bytes: - forced_i = None - if forced is not None: - forced_i = int_from_bytes(forced) - size = len(forced) - assert self._cd is not None - return self._cd.draw_bits( - 8 * size, forced=forced_i, fake_forced=fake_forced - ).to_bytes(size, "big") + + buf = bytearray() + average_size = min( + max(min_size * 2, min_size + 5), + 0.5 * (min_size + max_size), + ) + elements = many( + self._cd, + min_size=min_size, + max_size=max_size, + average_size=average_size, + forced=None if forced is None else len(forced), + fake_forced=fake_forced, + observe=False, + ) + while elements.more(): + forced_i: Optional[int] = None + if forced is not None: + # implicit conversion from bytes to int by indexing here + forced_i = forced[elements.count - 1] + + buf += self._cd.draw_bits( + 8, forced=forced_i, fake_forced=fake_forced + ).to_bytes(1, "big") + + return bytes(buf) def _draw_float( self, @@ -2216,12 +2243,13 @@ class ConjectureData: intervals: IntervalSet, *, min_size: int = 0, - max_size: Optional[int] = None, + max_size: int = COLLECTION_DEFAULT_MAX_SIZE, forced: Optional[str] = None, fake_forced: bool = False, observe: bool = True, ) -> str: - assert forced is None or min_size <= len(forced) + assert forced is None or min_size <= len(forced) <= max_size + assert min_size >= 0 kwargs: StringKWargs = self._pooled_kwargs( "string", @@ -2255,17 +2283,19 @@ class ConjectureData: def draw_bytes( self, - # TODO move to min_size and max_size here. - size: int, + min_size: int = 0, + max_size: int = COLLECTION_DEFAULT_MAX_SIZE, *, forced: Optional[bytes] = None, fake_forced: bool = False, observe: bool = True, ) -> bytes: - assert forced is None or len(forced) == size - assert size >= 0 + assert forced is None or min_size <= len(forced) <= max_size + assert min_size >= 0 - kwargs: BytesKWargs = self._pooled_kwargs("bytes", {"size": size}) + kwargs: BytesKWargs = self._pooled_kwargs( + "bytes", {"min_size": min_size, "max_size": max_size} + ) if self.ir_tree_nodes is not None and observe: node_value = self._pop_ir_tree_node("bytes", kwargs, forced=forced) diff --git a/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/datatree.py b/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/datatree.py index 60c1610bc3..87a0c988c8 100644 --- a/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/datatree.py +++ b/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/datatree.py @@ -146,9 +146,31 @@ class Conclusion: MAX_CHILDREN_EFFECTIVELY_INFINITE = 100_000 -def compute_max_children(ir_type, kwargs): - from hypothesis.internal.conjecture.data import DRAW_STRING_DEFAULT_MAX_SIZE +def _count_distinct_strings(*, alphabet_size, min_size, max_size): + # We want to estimate if we're going to have more children than + # MAX_CHILDREN_EFFECTIVELY_INFINITE, without computing a potentially + # extremely expensive pow. We'll check if the number of strings in + # the largest string size alone is enough to put us over this limit. + # We'll also employ a trick of estimating against log, which is cheaper + # than computing a pow. + # + # x = max_size + # y = alphabet_size + # n = MAX_CHILDREN_EFFECTIVELY_INFINITE + # + # x**y > n + # <=> log(x**y) > log(n) + # <=> y * log(x) > log(n) + definitely_too_large = max_size * math.log(alphabet_size) > math.log( + MAX_CHILDREN_EFFECTIVELY_INFINITE + ) + if definitely_too_large: + return MAX_CHILDREN_EFFECTIVELY_INFINITE + return sum(alphabet_size**k for k in range(min_size, max_size + 1)) + + +def compute_max_children(ir_type, kwargs): if ir_type == "integer": min_value = kwargs["min_value"] max_value = kwargs["max_value"] @@ -178,50 +200,27 @@ def compute_max_children(ir_type, kwargs): return 1 return 2 elif ir_type == "bytes": - return 2 ** (8 * kwargs["size"]) + return _count_distinct_strings( + alphabet_size=2**8, min_size=kwargs["min_size"], max_size=kwargs["max_size"] + ) elif ir_type == "string": min_size = kwargs["min_size"] max_size = kwargs["max_size"] intervals = kwargs["intervals"] - if max_size is None: - max_size = DRAW_STRING_DEFAULT_MAX_SIZE - if len(intervals) == 0: # Special-case the empty alphabet to avoid an error in math.log(0). # Only possibility is the empty string. return 1 - # We want to estimate if we're going to have more children than - # MAX_CHILDREN_EFFECTIVELY_INFINITE, without computing a potentially - # extremely expensive pow. We'll check if the number of strings in - # the largest string size alone is enough to put us over this limit. - # We'll also employ a trick of estimating against log, which is cheaper - # than computing a pow. - # - # x = max_size - # y = len(intervals) - # n = MAX_CHILDREN_EFFECTIVELY_INFINITE - # - # x**y > n - # <=> log(x**y) > log(n) - # <=> y * log(x) > log(n) - - # avoid math.log(1) == 0 and incorrectly failing the below estimate, - # even when we definitely are too large. - if len(intervals) == 1: - definitely_too_large = max_size > MAX_CHILDREN_EFFECTIVELY_INFINITE - else: - definitely_too_large = max_size * math.log(len(intervals)) > math.log( - MAX_CHILDREN_EFFECTIVELY_INFINITE - ) - - if definitely_too_large: + # avoid math.log(1) == 0 and incorrectly failing our effectively_infinite + # estimate, even when we definitely are too large. + if len(intervals) == 1 and max_size > MAX_CHILDREN_EFFECTIVELY_INFINITE: return MAX_CHILDREN_EFFECTIVELY_INFINITE - # number of strings of length k, for each k in [min_size, max_size]. - return sum(len(intervals) ** k for k in range(min_size, max_size + 1)) - + return _count_distinct_strings( + alphabet_size=len(intervals), min_size=min_size, max_size=max_size + ) elif ir_type == "float": min_value = kwargs["min_value"] max_value = kwargs["max_value"] @@ -306,8 +305,8 @@ def all_children(ir_type, kwargs): else: yield from [False, True] if ir_type == "bytes": - size = kwargs["size"] - yield from (int_to_bytes(i, size) for i in range(2 ** (8 * size))) + for size in range(kwargs["min_size"], kwargs["max_size"] + 1): + yield from (int_to_bytes(i, size) for i in range(2 ** (8 * size))) if ir_type == "string": min_size = kwargs["min_size"] max_size = kwargs["max_size"] diff --git a/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/shrinker.py b/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/shrinker.py index 3be703a18e..d1084e4cdd 100644 --- a/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/shrinker.py +++ b/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/shrinker.py @@ -1075,10 +1075,9 @@ class Shrinker: return False # pragma: no cover if node.ir_type in {"string", "bytes"}: - size_kwarg = "min_size" if node.ir_type == "string" else "size" # if the size *increased*, we would have to guess what to pad with # in order to try fixing up this attempt. Just give up. - if node.kwargs[size_kwarg] <= attempt_kwargs[size_kwarg]: + if node.kwargs["min_size"] <= attempt_kwargs["min_size"]: return False # the size decreased in our attempt. Try again, but replace with # the min_size that we would have gotten, and truncate the value @@ -1089,7 +1088,7 @@ class Shrinker: initial_attempt[node.index].copy( with_kwargs=attempt_kwargs, with_value=initial_attempt[node.index].value[ - : attempt_kwargs[size_kwarg] + : attempt_kwargs["min_size"] ], ) ] diff --git a/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/shrinking/bytes.py b/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/shrinking/bytes.py index 3ba75a2719..7fbc26fd24 100644 --- a/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/shrinking/bytes.py +++ b/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/shrinking/bytes.py @@ -8,17 +8,16 @@ # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -from hypothesis.internal.compat import int_from_bytes, int_to_bytes +from hypothesis.internal.conjecture.shrinking.collection import Collection from hypothesis.internal.conjecture.shrinking.integer import Integer -class Bytes(Integer): +class Bytes(Collection): def __init__(self, initial, predicate, **kwargs): - # shrink by interpreting the bytes as an integer. - # move to Collection.shrink when we support variable-size bytes, - # because b'\x00\x02' could shrink to either b'\x00\x01' or b'\x02'. super().__init__( - int_from_bytes(initial), - lambda n: predicate(int_to_bytes(n, len(initial))), + # implicit conversion from bytes to list of integers here + list(initial), + lambda val: predicate(bytes(val)), + ElementShrinker=Integer, **kwargs, ) diff --git a/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/strings.py b/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/strings.py index b6e6dd8deb..53d81b7394 100644 --- a/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/strings.py +++ b/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/strings.py @@ -12,15 +12,16 @@ import copy import re import warnings from functools import lru_cache, partial +from typing import Optional from hypothesis.errors import HypothesisWarning, InvalidArgument from hypothesis.internal import charmap +from hypothesis.internal.conjecture.data import COLLECTION_DEFAULT_MAX_SIZE from hypothesis.internal.filtering import max_len, min_len from hypothesis.internal.intervalsets import IntervalSet from hypothesis.internal.reflection import get_pretty_function_description from hypothesis.strategies._internal.collections import ListStrategy from hypothesis.strategies._internal.lazy import unwrap_strategies -from hypothesis.strategies._internal.numbers import IntegersStrategy from hypothesis.strategies._internal.strategies import ( OneOfStrategy, SampledFromStrategy, @@ -158,7 +159,13 @@ class TextStrategy(ListStrategy): elems = unwrap_strategies(self.element_strategy) if isinstance(elems, OneCharStringStrategy): return data.draw_string( - elems.intervals, min_size=self.min_size, max_size=self.max_size + elems.intervals, + min_size=self.min_size, + max_size=( + COLLECTION_DEFAULT_MAX_SIZE + if self.max_size == float("inf") + else self.max_size + ), ) return "".join(super().do_draw(data)) @@ -224,9 +231,13 @@ def _string_filter_rewrite(self, kind, condition): stacklevel=2, ) - elems = unwrap_strategies(self.element_strategy) if ( - (kind is bytes or isinstance(elems, OneCharStringStrategy)) + ( + kind is bytes + or isinstance( + unwrap_strategies(self.element_strategy), OneCharStringStrategy + ) + ) and isinstance(pattern := getattr(condition, "__self__", None), re.Pattern) and isinstance(pattern.pattern, kind) ): @@ -331,15 +342,15 @@ def _identifier_characters(): return id_start, id_continue -class BytesStrategy(ListStrategy): - def __init__(self, min_size, max_size): - super().__init__(IntegersStrategy(0, 255), min_size=min_size, max_size=max_size) +class BytesStrategy(SearchStrategy): + def __init__(self, min_size: int, max_size: Optional[int]): + self.min_size = min_size + self.max_size = ( + max_size if max_size is not None else COLLECTION_DEFAULT_MAX_SIZE + ) def do_draw(self, data): - # TODO: refactor the underlying provider to support variable-length bytes - if self.min_size == self.max_size: - return bytes(data.draw_bytes(self.min_size)) - return bytes(super().do_draw(data)) + return data.draw_bytes(self.min_size, self.max_size) _nonempty_filters = ( *ListStrategy._nonempty_filters, @@ -353,4 +364,4 @@ class BytesStrategy(ListStrategy): def filter(self, condition): if (new := _string_filter_rewrite(self, bytes, condition)) is not None: return new - return super().filter(condition) + return ListStrategy.filter(self, condition) diff --git a/contrib/python/hypothesis/py3/hypothesis/version.py b/contrib/python/hypothesis/py3/hypothesis/version.py index af1b866e64..2c69dd8977 100644 --- a/contrib/python/hypothesis/py3/hypothesis/version.py +++ b/contrib/python/hypothesis/py3/hypothesis/version.py @@ -8,5 +8,5 @@ # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -__version_info__ = (6, 111, 2) +__version_info__ = (6, 112, 0) __version__ = ".".join(map(str, __version_info__)) diff --git a/contrib/python/hypothesis/py3/ya.make b/contrib/python/hypothesis/py3/ya.make index f7183e57f1..ce520eaa5f 100644 --- a/contrib/python/hypothesis/py3/ya.make +++ b/contrib/python/hypothesis/py3/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(6.111.2) +VERSION(6.112.0) LICENSE(MPL-2.0) diff --git a/contrib/python/more-itertools/py3/.dist-info/METADATA b/contrib/python/more-itertools/py3/.dist-info/METADATA index c346b40880..a06c9b0a57 100644 --- a/contrib/python/more-itertools/py3/.dist-info/METADATA +++ b/contrib/python/more-itertools/py3/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: more-itertools -Version: 10.4.0 +Version: 10.5.0 Summary: More routines for operating on iterables, beyond itertools Keywords: itertools,iterator,iteration,filter,peek,peekable,chunk,chunked Author-email: Erik Rose <erikrose@grinchcentral.com> diff --git a/contrib/python/more-itertools/py3/more_itertools/__init__.py b/contrib/python/more-itertools/py3/more_itertools/__init__.py index 2e2fcbbe7b..583fb57457 100644 --- a/contrib/python/more-itertools/py3/more_itertools/__init__.py +++ b/contrib/python/more-itertools/py3/more_itertools/__init__.py @@ -3,4 +3,4 @@ from .more import * # noqa from .recipes import * # noqa -__version__ = '10.4.0' +__version__ = '10.5.0' diff --git a/contrib/python/more-itertools/py3/more_itertools/more.py b/contrib/python/more-itertools/py3/more_itertools/more.py index 3bf2c76b76..64fab26185 100644 --- a/contrib/python/more-itertools/py3/more_itertools/more.py +++ b/contrib/python/more-itertools/py3/more_itertools/more.py @@ -3017,7 +3017,7 @@ def circular_shifts(iterable, steps=1): n = len(buffer) n //= math.gcd(n, steps) - for __ in repeat(None, n): + for _ in repeat(None, n): buffer.rotate(steps) yield tuple(buffer) diff --git a/contrib/python/more-itertools/py3/more_itertools/more.pyi b/contrib/python/more-itertools/py3/more_itertools/more.pyi index f1a155dce7..66e6938e13 100644 --- a/contrib/python/more-itertools/py3/more_itertools/more.pyi +++ b/contrib/python/more-itertools/py3/more_itertools/more.pyi @@ -3,8 +3,8 @@ from __future__ import annotations import sys +import types -from types import TracebackType from typing import ( Any, Callable, @@ -42,7 +42,7 @@ _Raisable = BaseException | Type[BaseException] # The type of isinstance's second argument (from typeshed builtins) if sys.version_info >= (3, 10): - _ClassInfo = type | UnionType | tuple[_ClassInfo, ...] + _ClassInfo = type | types.UnionType | tuple[_ClassInfo, ...] else: _ClassInfo = type | tuple[_ClassInfo, ...] @@ -619,7 +619,7 @@ class callback_iter(Generic[_T], Iterator[_T]): self, exc_type: Type[BaseException] | None, exc_value: BaseException | None, - traceback: TracebackType | None, + traceback: types.TracebackType | None, ) -> bool | None: ... def __iter__(self) -> callback_iter[_T]: ... def __next__(self) -> _T: ... diff --git a/contrib/python/more-itertools/py3/more_itertools/recipes.py b/contrib/python/more-itertools/py3/more_itertools/recipes.py index a21a1f5d88..67f76fa899 100644 --- a/contrib/python/more-itertools/py3/more_itertools/recipes.py +++ b/contrib/python/more-itertools/py3/more_itertools/recipes.py @@ -218,7 +218,12 @@ def all_equal(iterable, key=None): True """ - return len(list(islice(groupby(iterable, key), 2))) <= 1 + iterator = groupby(iterable, key) + for first in iterator: + for second in iterator: + return False + return True + return True def quantify(iterable, pred=bool): diff --git a/contrib/python/more-itertools/py3/tests/test_recipes.py b/contrib/python/more-itertools/py3/tests/test_recipes.py index d3762d49db..684a6fcd0b 100644 --- a/contrib/python/more-itertools/py3/tests/test_recipes.py +++ b/contrib/python/more-itertools/py3/tests/test_recipes.py @@ -2,11 +2,12 @@ from decimal import Decimal from doctest import DocTestSuite from fractions import Fraction from functools import reduce -from itertools import combinations, count, permutations +from itertools import combinations, count, groupby, permutations from operator import mul from math import factorial from sys import version_info from unittest import TestCase, skipIf +from unittest.mock import patch import more_itertools as mi @@ -158,6 +159,22 @@ class AllEqualTests(TestCase): self.assertTrue(mi.all_equal('4٤໔４৪', key=int)) self.assertFalse(mi.all_equal('Abc', key=str.casefold)) + @patch('more_itertools.recipes.groupby', autospec=True) + def test_groupby_calls(self, mock_groupby): + next_count = 0 + + class _groupby(groupby): + def __next__(true_self): + nonlocal next_count + next_count += 1 + return super().__next__() + + mock_groupby.side_effect = _groupby + iterable = iter('aaaaa') + self.assertTrue(mi.all_equal(iterable)) + self.assertEqual(list(iterable), []) + self.assertEqual(next_count, 2) + class QuantifyTests(TestCase): """Tests for ``quantify()``""" diff --git a/contrib/python/more-itertools/py3/ya.make b/contrib/python/more-itertools/py3/ya.make index ee8f86bc14..45df93175b 100644 --- a/contrib/python/more-itertools/py3/ya.make +++ b/contrib/python/more-itertools/py3/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(10.4.0) +VERSION(10.5.0) LICENSE(MIT) diff --git a/contrib/python/types-protobuf/.dist-info/METADATA b/contrib/python/types-protobuf/.dist-info/METADATA index c7742acc49..ce1d526999 100644 --- a/contrib/python/types-protobuf/.dist-info/METADATA +++ b/contrib/python/types-protobuf/.dist-info/METADATA @@ -1,9 +1,9 @@ Metadata-Version: 2.1 Name: types-protobuf -Version: 5.27.0.20240626 +Version: 5.27.0.20240907 Summary: Typing stubs for protobuf Home-page: https://github.com/python/typeshed -License: Apache-2.0 license +License: Apache-2.0 Project-URL: GitHub, https://github.com/python/typeshed Project-URL: Changes, https://github.com/typeshed-internal/stub_uploader/blob/main/data/changelogs/protobuf.md Project-URL: Issue tracker, https://github.com/python/typeshed/issues @@ -38,6 +38,7 @@ If you find that annotations are missing, feel free to contribute and help compl See https://github.com/python/typeshed/blob/main/README.md for more details. -This package was generated from typeshed commit `b13bb947c3f7a000d4d4ec6ad2868726a289b2b4` and was tested -with mypy 1.10.0, pyright 1.1.369, and +This package was generated from typeshed commit +[`e8e9291c76f50c3bcde79e7bb61060f5c24c054e`](https://github.com/python/typeshed/commit/e8e9291c76f50c3bcde79e7bb61060f5c24c054e) and was tested +with mypy 1.11.1, pyright 1.1.379, and pytype 2024.4.11. diff --git a/contrib/python/types-protobuf/google-stubs/protobuf/internal/containers.pyi b/contrib/python/types-protobuf/google-stubs/protobuf/internal/containers.pyi index 30a37353c1..aaa9704392 100644 --- a/contrib/python/types-protobuf/google-stubs/protobuf/internal/containers.pyi +++ b/contrib/python/types-protobuf/google-stubs/protobuf/internal/containers.pyi @@ -33,7 +33,7 @@ class RepeatedScalarFieldContainer(BaseContainer[_ScalarV]): def append(self, value: _ScalarV) -> None: ... def insert(self, key: int, value: _ScalarV) -> None: ... def extend(self, elem_seq: Iterable[_ScalarV] | None) -> None: ... - def MergeFrom(self: _M, other: _M) -> None: ... + def MergeFrom(self: _M, other: _M | Iterable[_ScalarV]) -> None: ... def remove(self, elem: _ScalarV) -> None: ... def pop(self, key: int = -1) -> _ScalarV: ... @overload @@ -49,7 +49,7 @@ class RepeatedCompositeFieldContainer(BaseContainer[_MessageV]): def append(self, value: _MessageV) -> None: ... def insert(self, key: int, value: _MessageV) -> None: ... def extend(self, elem_seq: Iterable[_MessageV]) -> None: ... - def MergeFrom(self: _M, other: _M) -> None: ... + def MergeFrom(self: _M, other: _M | Iterable[_MessageV]) -> None: ... def remove(self, elem: _MessageV) -> None: ... def pop(self, key: int = -1) -> _MessageV: ... def __delitem__(self, key: int | slice) -> None: ... diff --git a/contrib/python/types-protobuf/ya.make b/contrib/python/types-protobuf/ya.make index a4a8846b2c..f2e94d656a 100644 --- a/contrib/python/types-protobuf/ya.make +++ b/contrib/python/types-protobuf/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(5.27.0.20240626) +VERSION(5.27.0.20240907) LICENSE(Apache-2.0) diff --git a/contrib/python/ydb/py3/.dist-info/METADATA b/contrib/python/ydb/py3/.dist-info/METADATA index db2f0036b3..7c7c7cdbaf 100644 --- a/contrib/python/ydb/py3/.dist-info/METADATA +++ b/contrib/python/ydb/py3/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: ydb -Version: 3.17.1 +Version: 3.17.2 Summary: YDB Python SDK Home-page: http://github.com/ydb-platform/ydb-python-sdk Author: Yandex LLC diff --git a/contrib/python/ydb/py3/ya.make b/contrib/python/ydb/py3/ya.make index c1ab6d4472..b3e9929330 100644 --- a/contrib/python/ydb/py3/ya.make +++ b/contrib/python/ydb/py3/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(3.17.1) +VERSION(3.17.2) LICENSE(Apache-2.0) diff --git a/contrib/python/ydb/py3/ydb/table.py b/contrib/python/ydb/py3/ydb/table.py index cfcffb17af..01f5e52b6a 100644 --- a/contrib/python/ydb/py3/ydb/table.py +++ b/contrib/python/ydb/py3/ydb/table.py @@ -290,6 +290,7 @@ class TableIndex(object): self._pb.name = name self.name = name self.index_columns = [] + self.data_columns = [] # output only. self.status = None @@ -307,6 +308,12 @@ class TableIndex(object): self.index_columns.append(column) return self + def with_data_columns(self, *columns): + for column in columns: + self._pb.data_columns.append(column) + self.data_columns.append(column) + return self + def to_pb(self): return self._pb diff --git a/contrib/python/ydb/py3/ydb/ydb_version.py b/contrib/python/ydb/py3/ydb/ydb_version.py index b0ef9f368d..1115cbbcb9 100644 --- a/contrib/python/ydb/py3/ydb/ydb_version.py +++ b/contrib/python/ydb/py3/ydb/ydb_version.py @@ -1 +1 @@ -VERSION = "3.17.1" +VERSION = "3.17.2" diff --git a/contrib/restricted/abseil-cpp-tstring/y_absl/debugging/ya.make b/contrib/restricted/abseil-cpp-tstring/y_absl/debugging/ya.make index b4b0267449..b8fee8b070 100644 --- a/contrib/restricted/abseil-cpp-tstring/y_absl/debugging/ya.make +++ b/contrib/restricted/abseil-cpp-tstring/y_absl/debugging/ya.make @@ -19,21 +19,21 @@ ADDINCL( NO_COMPILER_WARNINGS() -IF(NOT Y_ABSL_DONT_USE_DEBUG) -SRCS( - failure_signal_handler.cc - internal/address_is_readable.cc - internal/decode_rust_punycode.cc - internal/demangle.cc - internal/demangle_rust.cc - internal/elf_mem_image.cc - internal/examine_stack.cc - internal/utf8_for_code_point.cc - internal/vdso_support.cc - leak_check.cc - stacktrace.cc - symbolize.cc -) +IF (NOT Y_ABSL_DONT_USE_DEBUG) + SRCS( + failure_signal_handler.cc + internal/address_is_readable.cc + internal/decode_rust_punycode.cc + internal/demangle.cc + internal/demangle_rust.cc + internal/elf_mem_image.cc + internal/examine_stack.cc + internal/utf8_for_code_point.cc + internal/vdso_support.cc + leak_check.cc + stacktrace.cc + symbolize.cc + ) ENDIF() END() diff --git a/contrib/restricted/abseil-cpp-tstring/y_absl/log/ya.make b/contrib/restricted/abseil-cpp-tstring/y_absl/log/ya.make index 74000ae8ee..746cf7a0cd 100644 --- a/contrib/restricted/abseil-cpp-tstring/y_absl/log/ya.make +++ b/contrib/restricted/abseil-cpp-tstring/y_absl/log/ya.make @@ -32,8 +32,10 @@ ENDIF() NO_COMPILER_WARNINGS() -IF(Y_ABSL_DONT_USE_DEBUG) - CFLAGS(-DY_ABSL_DONT_USE_DEBUG_LIBRARY=1) +IF (Y_ABSL_DONT_USE_DEBUG) + CFLAGS( + -DY_ABSL_DONT_USE_DEBUG_LIBRARY=1 + ) ENDIF() SRCS( diff --git a/contrib/restricted/abseil-cpp-tstring/y_absl/strings/internal/resize_uninitialized.h b/contrib/restricted/abseil-cpp-tstring/y_absl/strings/internal/resize_uninitialized.h index 984562257d..8451cc504b 100644 --- a/contrib/restricted/abseil-cpp-tstring/y_absl/strings/internal/resize_uninitialized.h +++ b/contrib/restricted/abseil-cpp-tstring/y_absl/strings/internal/resize_uninitialized.h @@ -49,6 +49,16 @@ struct ResizeUninitializedTraits< } }; +template <typename string_type> +struct ResizeUninitializedTraits< + string_type, y_absl::void_t<decltype(std::declval<string_type&>() + .ReserveAndResize(237))> > { + using HasMember = std::true_type; + static void Resize(string_type* s, size_t new_size) { + s->ReserveAndResize(new_size); + } +}; + // Returns true if the TString implementation supports a resize where // the new characters added to the TString are left untouched. // @@ -98,6 +108,15 @@ struct AppendUninitializedTraits< } }; +template <typename string_type> +struct AppendUninitializedTraits< + string_type, y_absl::void_t<decltype(std::declval<string_type&>() + .ReserveAndResize(237))> > { + static void Append(string_type* s, size_t n) { + s->ReserveAndResize(s->size() + n); + } +}; + // Like STLStringResizeUninitialized(str, new_size), except guaranteed to use // exponential growth so that the amortized complexity of increasing the string // size by a small amount is O(1), in contrast to O(str->size()) in the case of diff --git a/contrib/restricted/abseil-cpp-tstring/y_absl/strings/ya.make b/contrib/restricted/abseil-cpp-tstring/y_absl/strings/ya.make index 74d25c05f5..a2d57d8f20 100644 --- a/contrib/restricted/abseil-cpp-tstring/y_absl/strings/ya.make +++ b/contrib/restricted/abseil-cpp-tstring/y_absl/strings/ya.make @@ -19,12 +19,14 @@ ADDINCL( NO_COMPILER_WARNINGS() -SRCDIR(contrib/restricted/abseil-cpp-tstring/y_absl) - -IF(Y_ABSL_DONT_USE_DEBUG) - CFLAGS(-DY_ABSL_DONT_USE_DEBUG_LIBRARY=1) +IF (Y_ABSL_DONT_USE_DEBUG) + CFLAGS( + -DY_ABSL_DONT_USE_DEBUG_LIBRARY=1 + ) ENDIF() +SRCDIR(contrib/restricted/abseil-cpp-tstring/y_absl) + SRCS( crc/crc32c.cc crc/internal/cpu_detect.cc diff --git a/contrib/restricted/abseil-cpp-tstring/y_absl/synchronization/mutex.cc b/contrib/restricted/abseil-cpp-tstring/y_absl/synchronization/mutex.cc index 8fb7c97e39..ac0cd05141 100644 --- a/contrib/restricted/abseil-cpp-tstring/y_absl/synchronization/mutex.cc +++ b/contrib/restricted/abseil-cpp-tstring/y_absl/synchronization/mutex.cc @@ -434,14 +434,10 @@ static SynchEvent* GetSynchEvent(const void* addr) { // if event recording is on static void PostSynchEvent(void* obj, int ev) { SynchEvent* e = GetSynchEvent(obj); -#ifdef Y_ABSL_DONT_USE_DEBUG_LIBRARY - constexpr bool DONT_COLLECT_STACK_TRACE = 1; -#else - constexpr bool DONT_COLLECT_STACK_TRACE = 0; -#endif +#ifndef Y_ABSL_DONT_USE_DEBUG_LIBRARY // logging is on if event recording is on and either there's no event struct, // or it explicitly says to log - if ((e == nullptr || e->log) && !DONT_COLLECT_STACK_TRACE) { + if (e == nullptr || e->log) { void* pcs[40]; int n = y_absl::GetStackTrace(pcs, Y_ABSL_ARRAYSIZE(pcs), 1); // A buffer with enough space for the ASCII for all the PCs, even on a @@ -460,6 +456,7 @@ static void PostSynchEvent(void* obj, int ev) { Y_ABSL_RAW_LOG(INFO, "%s%p %s %s", event_properties[ev].msg, obj, (e == nullptr ? "" : e->name), buffer); } +#endif const int flags = event_properties[ev].flags; if ((flags & SYNCH_F_LCK) != 0 && e != nullptr && e->invariant != nullptr) { // Calling the invariant as is causes problems under ThreadSanitizer. @@ -1325,6 +1322,7 @@ static inline void DebugOnlyLockLeave(Mutex* mu) { static char* StackString(void** pcs, int n, char* buf, int maxlen, bool symbolize) { +#ifndef Y_ABSL_DONT_USE_DEBUG_LIBRARY static constexpr int kSymLen = 200; char sym[kSymLen]; int len = 0; @@ -1344,12 +1342,21 @@ static char* StackString(void** pcs, int n, char* buf, int maxlen, len += strlen(&buf[len]); } return buf; +#else + buf[0] = 0; + return buf; +#endif } static char* CurrentStackString(char* buf, int maxlen, bool symbolize) { +#ifndef Y_ABSL_DONT_USE_DEBUG_LIBRARY void* pcs[40]; return StackString(pcs, y_absl::GetStackTrace(pcs, Y_ABSL_ARRAYSIZE(pcs), 2), buf, maxlen, symbolize); +#else + buf[0] = 0; + return buf; +#endif } namespace { @@ -1375,7 +1382,11 @@ struct ScopedDeadlockReportBuffers { // Helper to pass to GraphCycles::UpdateStackTrace. int GetStack(void** stack, int max_depth) { +#ifndef Y_ABSL_DONT_USE_DEBUG_LIBRARY return y_absl::GetStackTrace(stack, max_depth, 3); +#else + return 0; +#endif } } // anonymous namespace diff --git a/contrib/restricted/abseil-cpp-tstring/y_absl/synchronization/ya.make b/contrib/restricted/abseil-cpp-tstring/y_absl/synchronization/ya.make index a4eee05708..82ecd4b5ff 100644 --- a/contrib/restricted/abseil-cpp-tstring/y_absl/synchronization/ya.make +++ b/contrib/restricted/abseil-cpp-tstring/y_absl/synchronization/ya.make @@ -20,12 +20,14 @@ ADDINCL( GLOBAL contrib/restricted/abseil-cpp-tstring ) -IF(Y_ABSL_DONT_USE_DEBUG) - CFLAGS(-DY_ABSL_DONT_USE_DEBUG_LIBRARY=1) -ENDIF() - NO_COMPILER_WARNINGS() +IF (Y_ABSL_DONT_USE_DEBUG) + CFLAGS( + -DY_ABSL_DONT_USE_DEBUG_LIBRARY=1 + ) +ENDIF() + SRCS( barrier.cc blocking_counter.cc diff --git a/contrib/restricted/boost/asio/include/boost/asio/detail/config.hpp b/contrib/restricted/boost/asio/include/boost/asio/detail/config.hpp index 3e810c89f7..8e83197321 100644 --- a/contrib/restricted/boost/asio/include/boost/asio/detail/config.hpp +++ b/contrib/restricted/boost/asio/include/boost/asio/detail/config.hpp @@ -583,7 +583,9 @@ #if defined(BOOST_ASIO_HAS_ALIGNOF) # define BOOST_ASIO_ALIGNOF(T) alignof(T) -# if defined(__GNUC__) +# if defined(__STDCPP_DEFAULT_NEW_ALIGNMENT__) +# define BOOST_ASIO_DEFAULT_ALIGN __STDCPP_DEFAULT_NEW_ALIGNMENT__ +# elif defined(__GNUC__) # if ((__GNUC__ == 4) && (__GNUC_MINOR__ >= 9)) || (__GNUC__ > 4) # define BOOST_ASIO_DEFAULT_ALIGN alignof(std::max_align_t) # else // ((__GNUC__ == 4) && (__GNUC_MINOR__ >= 9)) || (__GNUC__ > 4) @@ -603,9 +605,11 @@ # if (__cplusplus >= 201703) # if defined(__clang__) # if defined(BOOST_ASIO_HAS_CLANG_LIBCXX) -# if (_LIBCPP_STD_VER > 14) && defined(_LIBCPP_HAS_ALIGNED_ALLOC) +# if (_LIBCPP_STD_VER > 14) && defined(_LIBCPP_HAS_ALIGNED_ALLOC) \ + && !defined(_LIBCPP_MSVCRT) && !defined(__MINGW32__) # define BOOST_ASIO_HAS_STD_ALIGNED_ALLOC 1 # endif // (_LIBCPP_STD_VER > 14) && defined(_LIBCPP_HAS_ALIGNED_ALLOC) + // && !defined(_LIBCPP_MSVCRT) && !defined(__MINGW32__) # elif defined(_GLIBCXX_HAVE_ALIGNED_ALLOC) # define BOOST_ASIO_HAS_STD_ALIGNED_ALLOC 1 # endif // defined(_GLIBCXX_HAVE_ALIGNED_ALLOC) @@ -1471,6 +1475,13 @@ # endif // !defined(BOOST_ASIO_HAS_TIMERFD) #endif // defined(__linux__) +// Linux: io_uring is used instead of epoll. +#if !defined(BOOST_ASIO_HAS_IO_URING_AS_DEFAULT) +# if !defined(BOOST_ASIO_HAS_EPOLL) && defined(BOOST_ASIO_HAS_IO_URING) +# define BOOST_ASIO_HAS_IO_URING_AS_DEFAULT 1 +# endif // !defined(BOOST_ASIO_HAS_EPOLL) && defined(BOOST_ASIO_HAS_IO_URING) +#endif // !defined(BOOST_ASIO_HAS_IO_URING_AS_DEFAULT) + // Mac OS X, FreeBSD, NetBSD, OpenBSD: kqueue. #if (defined(__MACH__) && defined(__APPLE__)) \ || defined(__FreeBSD__) \ @@ -1572,6 +1583,34 @@ # endif // !defined(BOOST_ASIO_DISABLE_LOCAL_SOCKETS) #endif // !defined(BOOST_ASIO_HAS_LOCAL_SOCKETS) +// Files. +#if !defined(BOOST_ASIO_HAS_FILE) +# if !defined(BOOST_ASIO_DISABLE_FILE) +# if defined(BOOST_ASIO_HAS_WINDOWS_RANDOM_ACCESS_HANDLE) +# define BOOST_ASIO_HAS_FILE 1 +# elif defined(BOOST_ASIO_HAS_IO_URING) +# define BOOST_ASIO_HAS_FILE 1 +# endif // defined(BOOST_ASIO_HAS_IO_URING) +# endif // !defined(BOOST_ASIO_DISABLE_FILE) +#endif // !defined(BOOST_ASIO_HAS_FILE) + +// Pipes. +#if !defined(BOOST_ASIO_HAS_PIPE) +# if defined(BOOST_ASIO_HAS_IOCP) \ + || !defined(BOOST_ASIO_WINDOWS) \ + && !defined(BOOST_ASIO_WINDOWS_RUNTIME) \ + && !defined(__CYGWIN__) +# if !defined(__SYMBIAN32__) +# if !defined(BOOST_ASIO_DISABLE_PIPE) +# define BOOST_ASIO_HAS_PIPE 1 +# endif // !defined(BOOST_ASIO_DISABLE_PIPE) +# endif // !defined(__SYMBIAN32__) +# endif // defined(BOOST_ASIO_HAS_IOCP) + // || !defined(BOOST_ASIO_WINDOWS) + // && !defined(BOOST_ASIO_WINDOWS_RUNTIME) + // && !defined(__CYGWIN__) +#endif // !defined(BOOST_ASIO_HAS_PIPE) + // Can use sigaction() instead of signal(). #if !defined(BOOST_ASIO_HAS_SIGACTION) # if !defined(BOOST_ASIO_DISABLE_SIGACTION) diff --git a/contrib/restricted/boost/asio/include/boost/asio/detail/is_buffer_sequence.hpp b/contrib/restricted/boost/asio/include/boost/asio/detail/is_buffer_sequence.hpp index 53d33ad445..402e680199 100644 --- a/contrib/restricted/boost/asio/include/boost/asio/detail/is_buffer_sequence.hpp +++ b/contrib/restricted/boost/asio/include/boost/asio/detail/is_buffer_sequence.hpp @@ -25,6 +25,8 @@ namespace asio { class mutable_buffer; class const_buffer; +class mutable_registered_buffer; +class const_registered_buffer; namespace detail { @@ -259,6 +261,30 @@ struct is_buffer_sequence<const_buffer, mutable_buffer> { }; +template <> +struct is_buffer_sequence<mutable_registered_buffer, mutable_buffer> + : true_type +{ +}; + +template <> +struct is_buffer_sequence<mutable_registered_buffer, const_buffer> + : true_type +{ +}; + +template <> +struct is_buffer_sequence<const_registered_buffer, const_buffer> + : true_type +{ +}; + +template <> +struct is_buffer_sequence<const_registered_buffer, mutable_buffer> + : false_type +{ +}; + template <typename T> struct is_dynamic_buffer_class_v1 : integral_constant<bool, diff --git a/contrib/restricted/boost/asio/include/boost/asio/detail/pop_options.hpp b/contrib/restricted/boost/asio/include/boost/asio/detail/pop_options.hpp index c34eb983b4..89008ccdb9 100644 --- a/contrib/restricted/boost/asio/include/boost/asio/detail/pop_options.hpp +++ b/contrib/restricted/boost/asio/include/boost/asio/detail/pop_options.hpp @@ -146,4 +146,8 @@ # endif # endif +# pragma pop_macro ("emit") +# pragma pop_macro ("signal") +# pragma pop_macro ("slot") + #endif diff --git a/contrib/restricted/boost/asio/include/boost/asio/detail/push_options.hpp b/contrib/restricted/boost/asio/include/boost/asio/detail/push_options.hpp index 1a709cb1d4..020e97be44 100644 --- a/contrib/restricted/boost/asio/include/boost/asio/detail/push_options.hpp +++ b/contrib/restricted/boost/asio/include/boost/asio/detail/push_options.hpp @@ -169,6 +169,7 @@ # pragma warning (disable:4127) # pragma warning (disable:4180) # pragma warning (disable:4244) +# pragma warning (disable:4265) # pragma warning (disable:4355) # pragma warning (disable:4510) # pragma warning (disable:4512) diff --git a/contrib/restricted/boost/asio/ya.make b/contrib/restricted/boost/asio/ya.make index 58792d2350..3a2909b016 100644 --- a/contrib/restricted/boost/asio/ya.make +++ b/contrib/restricted/boost/asio/ya.make @@ -9,9 +9,9 @@ LICENSE( LICENSE_TEXTS(.yandex_meta/licenses.list.txt) -VERSION(1.77.0) +VERSION(1.78.0) -ORIGINAL_SOURCE(https://github.com/boostorg/asio/archive/boost-1.77.0.tar.gz) +ORIGINAL_SOURCE(https://github.com/boostorg/asio/archive/boost-1.78.0.tar.gz) PEERDIR( contrib/libs/openssl @@ -37,4 +37,8 @@ ADDINCL( GLOBAL contrib/restricted/boost/asio/include ) +NO_COMPILER_WARNINGS() + +NO_UTIL() + END() diff --git a/contrib/restricted/fast_float/README.md b/contrib/restricted/fast_float/README.md index 9c2e188fa9..316f813e3a 100644 --- a/contrib/restricted/fast_float/README.md +++ b/contrib/restricted/fast_float/README.md @@ -144,6 +144,51 @@ print the number 22250738585072012 three times: std::cout << "parsed the number "<< i << std::endl; ``` +## Behavior of result_out_of_range + +When parsing floating-point values, the numbers can sometimes be too small (e.g., `1e-1000`) or +too large (e.g., `1e1000`). The C language established the precedent that these small values are out of range. +In such cases, it is customary to parse small values to zero and large +values to infinity. That is the behaviour of the C language (e.g., `stdtod`). That is the behaviour followed by the fast_float library. + + + +Specifically, we follow Jonathan Wakely's interpretation of the standard: + +> In any case, the resulting value is one of at most two floating-point values closest to the value of the string matching the pattern. + +It is also the approach taken by the [Microsoft C++ library](https://github.com/microsoft/STL/blob/62205ab155d093e71dd9588a78f02c5396c3c14b/tests/std/tests/P0067R5_charconv/test.cpp#L943-L946). + +Hence, we have the following examples: + +```cpp + double result = -1; + std::string str = "3e-1000"; + auto r = fast_float::from_chars(str.data(), str.data() + str.size(), result); + // r.ec == std::errc::result_out_of_range + // r.ptr == str.data() + 7 + // result == 0 +``` + + +```cpp + double result = -1; + std::string str = "3e1000"; + auto r = fast_float::from_chars(str.data(), str.data() + str.size(), result); + // r.ec == std::errc::result_out_of_range + // r.ptr == str.data() + 6 + // result == std::numeric_limits<double>::infinity() +``` + +Users who wish for the value to be left unmodified given `std::errc::result_out_of_range` may do so by adding two lines of code: + +```cpp + double old_result = result; // make copy + auto r = fast_float::from_chars(start, end, result); + if(r.ec == std::errc::result_out_of_range) { result = old_result; } +``` + + ## C++20: compile-time evaluation (constexpr) In C++20, you may use `fast_float::from_chars` to parse strings @@ -290,6 +335,7 @@ int main() { The fast_float library is part of: - GCC (as of version 12): the `from_chars` function in GCC relies on fast_float. +- [Chromium](https://github.com/Chromium/Chromium), the engine behind Google Chrome and Microsoft Edge, - [WebKit](https://github.com/WebKit/WebKit), the engine behind Safari (Apple's web browser) - [DuckDB](https://duckdb.org) - [Apache Arrow](https://github.com/apache/arrow/pull/8494) where it multiplied the number parsing speed by two or three times @@ -369,6 +415,16 @@ target_link_libraries(myprogram PUBLIC fast_float) You should change the `GIT_TAG` line so that you recover the version you wish to use. +You may also use [CPM](https://github.com/cpm-cmake/CPM.cmake), like so: + +``` +CPMAddPackage( + NAME fast_float + GITHUB_REPOSITORY "fastfloat/fast_float" + GIT_TAG v6.1.4) +``` + + ## Using as single header The script `script/amalgamate.py` may be used to generate a single header @@ -379,7 +435,13 @@ the command line help. You may directly download automatically generated single-header files: -https://github.com/fastfloat/fast_float/releases/download/v6.1.4/fast_float.h +https://github.com/fastfloat/fast_float/releases/download/v6.1.5/fast_float.h + +## Packages + +- The fast_float library is part of the [Conan package manager](https://conan.io/center/recipes/fast_float). +- It is part of the [brew package manager](https://formulae.brew.sh/formula/fast_float). +- Some Linux distribution like Fedora include fast_float (e.g., as `fast_float-devel`). ## RFC 7159 diff --git a/contrib/restricted/fast_float/include/fast_float/bigint.h b/contrib/restricted/fast_float/include/fast_float/bigint.h index 92c3d5b192..03a5caa4a5 100644 --- a/contrib/restricted/fast_float/include/fast_float/bigint.h +++ b/contrib/restricted/fast_float/include/fast_float/bigint.h @@ -404,12 +404,16 @@ template <typename = void> struct pow5_tables { #endif }; +#if FASTFLOAT_DETAIL_MUST_DEFINE_CONSTEXPR_VARIABLE + template <typename T> constexpr uint32_t pow5_tables<T>::large_step; template <typename T> constexpr uint64_t pow5_tables<T>::small_power_of_5[]; template <typename T> constexpr limb pow5_tables<T>::large_power_of_5[]; +#endif + // big integer type. implements a small subset of big integer // arithmetic, using simple algorithms since asymptotically // faster algorithms are slower for a small number of limbs. diff --git a/contrib/restricted/fast_float/include/fast_float/constexpr_feature_detect.h b/contrib/restricted/fast_float/include/fast_float/constexpr_feature_detect.h index 18daf40941..7624beafca 100644 --- a/contrib/restricted/fast_float/include/fast_float/constexpr_feature_detect.h +++ b/contrib/restricted/fast_float/include/fast_float/constexpr_feature_detect.h @@ -37,4 +37,10 @@ #define FASTFLOAT_IS_CONSTEXPR 0 #endif +#if __cplusplus >= 201703L || (defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) +#define FASTFLOAT_DETAIL_MUST_DEFINE_CONSTEXPR_VARIABLE 0 +#else +#define FASTFLOAT_DETAIL_MUST_DEFINE_CONSTEXPR_VARIABLE 1 +#endif + #endif // FASTFLOAT_CONSTEXPR_FEATURE_DETECT_H diff --git a/contrib/restricted/fast_float/include/fast_float/fast_table.h b/contrib/restricted/fast_float/include/fast_float/fast_table.h index 097e27b14b..69f9b2c924 100644 --- a/contrib/restricted/fast_float/include/fast_float/fast_table.h +++ b/contrib/restricted/fast_float/include/fast_float/fast_table.h @@ -693,10 +693,14 @@ template <class unused = void> struct powers_template { }; }; +#if FASTFLOAT_DETAIL_MUST_DEFINE_CONSTEXPR_VARIABLE + template <class unused> constexpr uint64_t powers_template<unused>::power_of_five_128[number_of_entries]; +#endif + using powers = powers_template<>; } // namespace fast_float diff --git a/contrib/restricted/fast_float/include/fast_float/float_common.h b/contrib/restricted/fast_float/include/fast_float/float_common.h index 909765450d..82ab7b0c6b 100644 --- a/contrib/restricted/fast_float/include/fast_float/float_common.h +++ b/contrib/restricted/fast_float/include/fast_float/float_common.h @@ -343,7 +343,8 @@ full_multiplication(uint64_t a, uint64_t b) { // But MinGW on ARM64 doesn't have native support for 64-bit multiplications answer.high = __umulh(a, b); answer.low = a * b; -#elif defined(FASTFLOAT_32BIT) || (defined(_WIN64) && !defined(__clang__)) +#elif defined(FASTFLOAT_32BIT) || \ + (defined(_WIN64) && !defined(__clang__) && !defined(_M_ARM64)) answer.low = _umul128(a, b, &answer.high); // _umul128 not available on ARM64 #elif defined(FASTFLOAT_64BIT) && defined(__SIZEOF_INT128__) __uint128_t r = ((__uint128_t)a) * b; @@ -442,12 +443,16 @@ template <typename U> struct binary_format_lookup_tables<double, U> { constant_55555 * 5 * 5 * 5 * 5)}; }; +#if FASTFLOAT_DETAIL_MUST_DEFINE_CONSTEXPR_VARIABLE + template <typename U> constexpr double binary_format_lookup_tables<double, U>::powers_of_ten[]; template <typename U> constexpr uint64_t binary_format_lookup_tables<double, U>::max_mantissa[]; +#endif + template <typename U> struct binary_format_lookup_tables<float, U> { static constexpr float powers_of_ten[] = {1e0f, 1e1f, 1e2f, 1e3f, 1e4f, 1e5f, 1e6f, 1e7f, 1e8f, 1e9f, 1e10f}; @@ -469,12 +474,16 @@ template <typename U> struct binary_format_lookup_tables<float, U> { 0x1000000 / (constant_55555 * constant_55555 * 5)}; }; +#if FASTFLOAT_DETAIL_MUST_DEFINE_CONSTEXPR_VARIABLE + template <typename U> constexpr float binary_format_lookup_tables<float, U>::powers_of_ten[]; template <typename U> constexpr uint64_t binary_format_lookup_tables<float, U>::max_mantissa[]; +#endif + template <> inline constexpr int binary_format<double>::min_exponent_fast_path() { #if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) @@ -677,8 +686,12 @@ template <typename = void> struct space_lut { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; }; +#if FASTFLOAT_DETAIL_MUST_DEFINE_CONSTEXPR_VARIABLE + template <typename T> constexpr bool space_lut<T>::value[]; +#endif + inline constexpr bool is_space(uint8_t c) { return space_lut<>::value[c]; } #endif @@ -759,12 +772,16 @@ template <typename = void> struct int_luts { 3379220508056640625, 4738381338321616896}; }; +#if FASTFLOAT_DETAIL_MUST_DEFINE_CONSTEXPR_VARIABLE + template <typename T> constexpr uint8_t int_luts<T>::chdigit[]; template <typename T> constexpr size_t int_luts<T>::maxdigits_u64[]; template <typename T> constexpr uint64_t int_luts<T>::min_safe_u64[]; +#endif + template <typename UC> fastfloat_really_inline constexpr uint8_t ch_to_digit(UC c) { return int_luts<>::chdigit[static_cast<unsigned char>(c)]; diff --git a/contrib/restricted/fast_float/ya.make b/contrib/restricted/fast_float/ya.make index 2149e7dfee..92f9329ca0 100644 --- a/contrib/restricted/fast_float/ya.make +++ b/contrib/restricted/fast_float/ya.make @@ -10,9 +10,9 @@ LICENSE( LICENSE_TEXTS(.yandex_meta/licenses.list.txt) -VERSION(6.1.4) +VERSION(6.1.5) -ORIGINAL_SOURCE(https://github.com/fastfloat/fast_float/archive/v6.1.4.tar.gz) +ORIGINAL_SOURCE(https://github.com/fastfloat/fast_float/archive/v6.1.5.tar.gz) NO_COMPILER_WARNINGS() diff --git a/library/cpp/lwtrace/trace.cpp b/library/cpp/lwtrace/trace.cpp index 76e64a3025..691ffc3a7a 100644 --- a/library/cpp/lwtrace/trace.cpp +++ b/library/cpp/lwtrace/trace.cpp @@ -246,7 +246,7 @@ namespace NLWTrace { class TReceiver: public TOperand<T, OT_VARIABLE> { public: TReceiver(TSession::TTraceVariables& traceVariables, const TString& name) - : TOperand<T, OT_VARIABLE>(traceVariables, name, nullptr, 0) + : TOperand<T, OT_VARIABLE>(traceVariables, name, {}, 0) { } }; diff --git a/library/cpp/netliba/socket/protocols.h b/library/cpp/netliba/socket/protocols.h index ec6896ab9b..e364c3368a 100644 --- a/library/cpp/netliba/socket/protocols.h +++ b/library/cpp/netliba/socket/protocols.h @@ -18,12 +18,15 @@ namespace NNetlibaSocket { ACK_CANCELED, ACK_RESEND_NOSHMEM, - PING, - PONG, + PING, // 10 + PONG, // 11 PONG_IB, KILL, + XS_PING, // 14 + XS_PONG, // 15 + CMD_END, }; } diff --git a/library/cpp/tld/tlds-alpha-by-domain.txt b/library/cpp/tld/tlds-alpha-by-domain.txt index 100d6aaa17..dde82486ef 100644 --- a/library/cpp/tld/tlds-alpha-by-domain.txt +++ b/library/cpp/tld/tlds-alpha-by-domain.txt @@ -1,4 +1,4 @@ -# Version 2024091600, Last Updated Mon Sep 16 07:07:02 2024 UTC +# Version 2024092200, Last Updated Sun Sep 22 07:07:01 2024 UTC AAA AARP ABB diff --git a/library/cpp/xml/document/xml-document.cpp b/library/cpp/xml/document/xml-document.cpp index 18a554d732..25c0ed6a17 100644 --- a/library/cpp/xml/document/xml-document.cpp +++ b/library/cpp/xml/document/xml-document.cpp @@ -304,6 +304,7 @@ namespace NXml { xmlNode* nodePtr = GetPtr(); xmlUnlinkNode(nodePtr); xmlFreeNode(nodePtr); + NodePointer = nullptr; } static int XmlWriteToOstream(void* context, const char* buffer, int len) { diff --git a/library/cpp/yt/error/origin_attributes.h b/library/cpp/yt/error/origin_attributes.h index 05815c1e38..d98782469b 100644 --- a/library/cpp/yt/error/origin_attributes.h +++ b/library/cpp/yt/error/origin_attributes.h @@ -41,7 +41,7 @@ struct TOriginAttributes static constexpr size_t ExtensionDataByteSizeCap = 64; using TErasedExtensionData = TErasedStorage<ExtensionDataByteSizeCap>; - TProcessId Pid; + TProcessId Pid = 0; NThreading::TThreadId Tid; TThreadName ThreadName; diff --git a/library/cpp/yt/memory/atomic_intrusive_ptr-inl.h b/library/cpp/yt/memory/atomic_intrusive_ptr-inl.h index 21cd8b9412..37a73c2c24 100644 --- a/library/cpp/yt/memory/atomic_intrusive_ptr-inl.h +++ b/library/cpp/yt/memory/atomic_intrusive_ptr-inl.h @@ -16,35 +16,113 @@ TAtomicIntrusivePtr<T>::TAtomicIntrusivePtr(std::nullptr_t) { } template <class T> +TAtomicIntrusivePtr<T>& TAtomicIntrusivePtr<T>::operator=(TIntrusivePtr<T> other) +{ + Store(std::move(other)); + return *this; +} + +template <class T> +TAtomicIntrusivePtr<T>& TAtomicIntrusivePtr<T>::operator=(std::nullptr_t) +{ + Reset(); + return *this; +} + +template <class T> +TAtomicIntrusivePtr<T>::operator bool() const +{ + return Get(); +} + +#ifdef _lsan_enabled_ + +template <class T> TAtomicIntrusivePtr<T>::TAtomicIntrusivePtr(TIntrusivePtr<T> other) - : Ptr_(AcquireObject(other.Release(), true)) + : Ptr_(std::move(other)) { } template <class T> TAtomicIntrusivePtr<T>::TAtomicIntrusivePtr(TAtomicIntrusivePtr&& other) - : Ptr_(other.Ptr_.load(std::memory_order::relaxed)) + : Ptr_(std::move(other)) +{ } + +template <class T> +TAtomicIntrusivePtr<T>::~TAtomicIntrusivePtr() = default; + +template <class T> +TIntrusivePtr<T> TAtomicIntrusivePtr<T>::Acquire() const { - other.Ptr_.store(nullptr, std::memory_order::relaxed); + auto guard = Guard(Lock_); + return Ptr_; } template <class T> -TAtomicIntrusivePtr<T>::~TAtomicIntrusivePtr() +TIntrusivePtr<T> TAtomicIntrusivePtr<T>::Exchange(TIntrusivePtr<T> other) { - ReleaseObject(Ptr_.load()); + auto guard = Guard(Lock_); + Ptr_.Swap(other); + return other; } template <class T> -TAtomicIntrusivePtr<T>& TAtomicIntrusivePtr<T>::operator=(TIntrusivePtr<T> other) +void TAtomicIntrusivePtr<T>::Store(TIntrusivePtr<T> other) { - Store(std::move(other)); - return *this; + Exchange(std::move(other)); } template <class T> -TAtomicIntrusivePtr<T>& TAtomicIntrusivePtr<T>::operator=(std::nullptr_t) +void TAtomicIntrusivePtr<T>::Reset() { - Reset(); - return *this; + Exchange(nullptr); +} + +template <class T> +bool TAtomicIntrusivePtr<T>::CompareAndSwap(TRawPtr& comparePtr, T* target) +{ + auto guard = Guard(Lock_); + if (Ptr_.Get() != comparePtr) { + comparePtr = Ptr_.Get(); + return false; + } + auto targetPtr = TIntrusivePtr<T>(target, /*addReference*/ false); + Ptr_.Swap(targetPtr); + guard.Release(); + // targetPtr will die here. + return true; +} + +template <class T> +bool TAtomicIntrusivePtr<T>::CompareAndSwap(TRawPtr& comparePtr, TIntrusivePtr<T> target) +{ + return CompareAndSwap(comparePtr, target.Release()); +} + +template <class T> +typename TAtomicIntrusivePtr<T>::TRawPtr TAtomicIntrusivePtr<T>::Get() const +{ + auto guard = Guard(Lock_); + return Ptr_.Get(); +} + +#else + +template <class T> +TAtomicIntrusivePtr<T>::TAtomicIntrusivePtr(TIntrusivePtr<T> other) + : Ptr_(AcquireObject(other.Release(), /*consumeRef*/ true)) +{ } + +template <class T> +TAtomicIntrusivePtr<T>::TAtomicIntrusivePtr(TAtomicIntrusivePtr&& other) + : Ptr_(other.Ptr_.load(std::memory_order::relaxed)) +{ + other.Ptr_.store(nullptr, std::memory_order::relaxed); +} + +template <class T> +TAtomicIntrusivePtr<T>::~TAtomicIntrusivePtr() +{ + ReleaseObject(Ptr_.load()); } template <class T> @@ -100,7 +178,7 @@ TIntrusivePtr<T> TAtomicIntrusivePtr<T>::Acquire() const template <class T> TIntrusivePtr<T> TAtomicIntrusivePtr<T>::Exchange(TIntrusivePtr<T> other) { - auto [obj, localRefs] = TTaggedPtr<T>::Unpack(Ptr_.exchange(AcquireObject(other.Release(), true))); + auto [obj, localRefs] = TTaggedPtr<T>::Unpack(Ptr_.exchange(AcquireObject(other.Release(), /*consumeRef*/ true))); DoRelease(obj, localRefs + 1); return TIntrusivePtr<T>(obj, false); } @@ -108,7 +186,7 @@ TIntrusivePtr<T> TAtomicIntrusivePtr<T>::Exchange(TIntrusivePtr<T> other) template <class T> void TAtomicIntrusivePtr<T>::Store(TIntrusivePtr<T> other) { - ReleaseObject(Ptr_.exchange(AcquireObject(other.Release(), true))); + ReleaseObject(Ptr_.exchange(AcquireObject(other.Release(), /*consumeRef*/ true))); } template <class T> @@ -120,7 +198,7 @@ void TAtomicIntrusivePtr<T>::Reset() template <class T> bool TAtomicIntrusivePtr<T>::CompareAndSwap(TRawPtr& comparePtr, T* target) { - auto* targetPtr = AcquireObject(target, false); + auto* targetPtr = AcquireObject(target, /*consumeRef*/ false); auto currentPtr = Ptr_.load(); if (UnpackPointer<T>(currentPtr).Ptr == comparePtr && Ptr_.compare_exchange_strong(currentPtr, targetPtr)) { @@ -138,7 +216,7 @@ template <class T> bool TAtomicIntrusivePtr<T>::CompareAndSwap(TRawPtr& comparePtr, TIntrusivePtr<T> target) { // TODO(lukyan): Make helper for packed owning ptr? - auto targetPtr = AcquireObject(target.Release(), true); + auto targetPtr = AcquireObject(target.Release(), /*consumeRef*/ true); auto currentPtr = Ptr_.load(); if (TTaggedPtr<T>::Unpack(currentPtr).Ptr == comparePtr && Ptr_.compare_exchange_strong(currentPtr, targetPtr)) { @@ -159,16 +237,10 @@ typename TAtomicIntrusivePtr<T>::TRawPtr TAtomicIntrusivePtr<T>::Get() const } template <class T> -TAtomicIntrusivePtr<T>::operator bool() const -{ - return Get(); -} - -template <class T> TPackedPtr TAtomicIntrusivePtr<T>::AcquireObject(T* obj, bool consumeRef) { if (obj) { - Ref(obj, static_cast<int>(ReservedRefCount - consumeRef)); + Ref(obj, ReservedRefCount - static_cast<int>(consumeRef)); } return TTaggedPtr(obj).Pack(); @@ -185,10 +257,12 @@ template <class T> void TAtomicIntrusivePtr<T>::DoRelease(T* obj, int refs) { if (obj) { - Unref(obj, static_cast<int>(ReservedRefCount - refs)); + Unref(obj, ReservedRefCount - refs); } } +#endif + //////////////////////////////////////////////////////////////////////////////// template <class T> diff --git a/library/cpp/yt/memory/atomic_intrusive_ptr.h b/library/cpp/yt/memory/atomic_intrusive_ptr.h index ae0a307182..91d220f7a9 100644 --- a/library/cpp/yt/memory/atomic_intrusive_ptr.h +++ b/library/cpp/yt/memory/atomic_intrusive_ptr.h @@ -2,17 +2,20 @@ #include "intrusive_ptr.h" -namespace NYT { - -//////////////////////////////////////////////////////////////////////////////// +#include <util/system/compiler.h> -// Atomic ptr based on https://github.com/facebook/folly/blob/main/folly/concurrency/AtomicSharedPtr.h +#ifdef _lsan_enabled_ +#include <util/system/spinlock.h> +#endif -// Operators * and -> for TAtomicIntrusivePtr are useless because it is not safe to work with atomic ptr such way -// Safe usage is to convert to TIntrusivePtr. +namespace NYT { -// Max TAtomicIntrusivePtr count per object is (2**16 = 2**32 / 2**16). +//////////////////////////////////////////////////////////////////////////////// +//! Atomic pointer with split reference counting. +/* + * \see https://github.com/facebook/folly/blob/main/folly/concurrency/AtomicSharedPtr.h +*/ template <class T> class TAtomicIntrusivePtr { @@ -39,7 +42,7 @@ public: bool CompareAndSwap(TRawPtr& comparePtr, T* target); bool CompareAndSwap(TRawPtr& comparePtr, TIntrusivePtr<T> target); - // Result is suitable only for comparison. Not dereference. + //! Result is only suitable for comparison, not dereference. TRawPtr Get() const; explicit operator bool() const; @@ -57,6 +60,10 @@ private: template <class U> friend bool operator!=(const TIntrusivePtr<U>& lhs, const TAtomicIntrusivePtr<U>& rhs); +#ifdef _lsan_enabled_ + ::TSpinLock Lock_; + TIntrusivePtr<T> Ptr_; +#else // Keeps packed pointer (localRefCount, objectPtr). // Atomic ptr holds N references, where N = ReservedRefCount - localRefCount. // LocalRefCount is incremented in Acquire method. @@ -68,11 +75,10 @@ private: constexpr static int ReservedRefCount = (1 << CounterBits) - 1; // Consume ref if ownership is transferred. - // AcquireObject(ptr.Release(), true) - // AcquireObject(ptr.Get(), false) static TPackedPtr AcquireObject(T* obj, bool consumeRef = false); static void ReleaseObject(TPackedPtr packedPtr); static void DoRelease(T* obj, int refs); +#endif }; //////////////////////////////////////////////////////////////////////////////// diff --git a/library/cpp/yt/memory/blob.cpp b/library/cpp/yt/memory/blob.cpp index 9c17cffad0..260cfb698b 100644 --- a/library/cpp/yt/memory/blob.cpp +++ b/library/cpp/yt/memory/blob.cpp @@ -14,7 +14,7 @@ static constexpr double BlobCapacityMultiplier = 1.5; TBlob::TBlob( TRefCountedTypeCookie tagCookie, size_t size, - bool initiailizeStorage, + bool initializeStorage, bool pageAligned) : PageAligned_(pageAligned) { @@ -24,7 +24,7 @@ TBlob::TBlob( } else { Allocate(std::max(size, InitialBlobCapacity)); Size_ = size; - if (initiailizeStorage) { + if (initializeStorage) { ::memset(Begin_, 0, Size_); } } diff --git a/library/cpp/yt/memory/blob.h b/library/cpp/yt/memory/blob.h index 42cab83017..a14ac742b2 100644 --- a/library/cpp/yt/memory/blob.h +++ b/library/cpp/yt/memory/blob.h @@ -26,7 +26,7 @@ public: explicit TBlob( TRefCountedTypeCookie tagCookie = GetRefCountedTypeCookie<TDefaultBlobTag>(), size_t size = 0, - bool initiailizeStorage = true, + bool initializeStorage = true, bool pageAligned = false); //! Copies a chunk of memory into a new instance. diff --git a/library/cpp/yt/memory/chunked_memory_pool.cpp b/library/cpp/yt/memory/chunked_memory_pool.cpp index a10a6fe724..d8673f5d6c 100644 --- a/library/cpp/yt/memory/chunked_memory_pool.cpp +++ b/library/cpp/yt/memory/chunked_memory_pool.cpp @@ -15,6 +15,8 @@ TAllocationHolder::TAllocationHolder(TMutableRef ref, TRefCountedTypeCookie cook TRefCountedTrackerFacade::AllocateTagInstance(Cookie_); TRefCountedTrackerFacade::AllocateSpace(Cookie_, Ref_.Size()); } +#else + Y_UNUSED(cookie); #endif } diff --git a/library/cpp/yt/memory/ref.cpp b/library/cpp/yt/memory/ref.cpp index c01094846e..4d02ef6875 100644 --- a/library/cpp/yt/memory/ref.cpp +++ b/library/cpp/yt/memory/ref.cpp @@ -59,6 +59,8 @@ public: #ifdef YT_ENABLE_REF_COUNTED_TRACKING TRefCountedTrackerFacade::AllocateTagInstance(Cookie_); TRefCountedTrackerFacade::AllocateSpace(Cookie_, String_.length()); +#else + Y_UNUSED(cookie); #endif } ~TStringHolder() @@ -119,7 +121,11 @@ protected: TRefCountedTypeCookie cookie) { Size_ = size; +#ifdef YT_ENABLE_REF_COUNTED_TRACKING Cookie_ = cookie; +#else + Y_UNUSED(cookie); +#endif if (options.InitializeStorage) { ::memset(static_cast<TDerived*>(this)->GetBegin(), 0, Size_); } diff --git a/library/cpp/yt/memory/ref_counted-inl.h b/library/cpp/yt/memory/ref_counted-inl.h index f86f634ffd..6c20db8c41 100644 --- a/library/cpp/yt/memory/ref_counted-inl.h +++ b/library/cpp/yt/memory/ref_counted-inl.h @@ -90,7 +90,7 @@ Y_FORCE_INLINE void DestroyRefCountedImpl(T* obj) return; } - YT_ASSERT(offset < std::numeric_limits<ui16>::max()); + YT_ASSERT(offset < (1ULL << PackedPtrTagBits)); auto* vTablePtr = reinterpret_cast<TPackedPtr*>(basePtr); *vTablePtr = TTaggedPtr<void(void*, ui16)>(&NYT::NDetail::TMemoryReleaser<T>::Do, offset).Pack(); @@ -182,8 +182,24 @@ Y_FORCE_INLINE int TRefCounter::GetRefCount() const noexcept Y_FORCE_INLINE void TRefCounter::Ref(int n) const noexcept { + YT_ASSERT(n >= 0); + // It is safe to use relaxed here, since new reference is always created from another live reference. - StrongCount_.fetch_add(n, std::memory_order::relaxed); + auto value = StrongCount_.fetch_add(n, std::memory_order::relaxed); + YT_ASSERT(value > 0); + YT_ASSERT(value <= std::numeric_limits<TRefCount>::max() - n); + + YT_ASSERT(WeakCount_.load(std::memory_order::relaxed) > 0); +} + +Y_FORCE_INLINE void TRefCounter::DangerousRef(int n) const noexcept +{ + YT_ASSERT(n >= 0); + + // Relaxed is fine as per lukyan@, the caller guarantees object liveness. + auto value = StrongCount_.fetch_add(n, std::memory_order::relaxed); + YT_ASSERT(value >= 0); + YT_ASSERT(value <= std::numeric_limits<TRefCount>::max() - n); YT_ASSERT(WeakCount_.load(std::memory_order::relaxed) > 0); } @@ -191,6 +207,7 @@ Y_FORCE_INLINE void TRefCounter::Ref(int n) const noexcept Y_FORCE_INLINE bool TRefCounter::TryRef() const noexcept { auto value = StrongCount_.load(std::memory_order::relaxed); + YT_ASSERT(value >= 0 && value < std::numeric_limits<TRefCount>::max()); YT_ASSERT(WeakCount_.load(std::memory_order::relaxed) > 0); while (value != 0 && !StrongCount_.compare_exchange_weak(value, value + 1)); @@ -199,6 +216,8 @@ Y_FORCE_INLINE bool TRefCounter::TryRef() const noexcept Y_FORCE_INLINE bool TRefCounter::Unref(int n) const { + YT_ASSERT(n >= 0); + // We must properly synchronize last access to object with it destruction. // Otherwise compiler might reorder access to object past this decrement. // diff --git a/library/cpp/yt/memory/ref_counted.h b/library/cpp/yt/memory/ref_counted.h index 6ef546de0d..6abef3bf05 100644 --- a/library/cpp/yt/memory/ref_counted.h +++ b/library/cpp/yt/memory/ref_counted.h @@ -40,8 +40,13 @@ public: int GetRefCount() const noexcept; //! Increments the strong reference counter. + //! The current strong RC must be positive. void Ref(int n = 1) const noexcept; + //! Increments the strong reference counter. + //! The current strong RC may be zero. + void DangerousRef(int n = 1) const noexcept; + //! Increments the strong reference counter if it is not null. bool TryRef() const noexcept; @@ -58,8 +63,10 @@ public: bool WeakUnref() const; private: - mutable std::atomic<int> StrongCount_ = 1; - mutable std::atomic<int> WeakCount_ = 1; + // NB: Must we 64 bit as TAtomicIntrusivePtr grabs refs in 64K batches. + using TRefCount = i64; + mutable std::atomic<TRefCount> StrongCount_ = 1; + mutable std::atomic<TRefCount> WeakCount_ = 1; }; //////////////////////////////////////////////////////////////////////////////// diff --git a/library/cpp/yt/memory/unittests/atomic_intrusive_ptr_ut.cpp b/library/cpp/yt/memory/unittests/atomic_intrusive_ptr_ut.cpp index 68489fcdf7..47db5325e1 100644 --- a/library/cpp/yt/memory/unittests/atomic_intrusive_ptr_ut.cpp +++ b/library/cpp/yt/memory/unittests/atomic_intrusive_ptr_ut.cpp @@ -3,12 +3,17 @@ #include <library/cpp/yt/memory/new.h> #include <library/cpp/yt/memory/ref_counted.h> #include <library/cpp/yt/memory/atomic_intrusive_ptr.h> +#include <library/cpp/yt/memory/leaky_singleton.h> + +#include <util/system/compiler.h> namespace NYT { namespace { //////////////////////////////////////////////////////////////////////////////// +#ifndef _lsan_enabled_ + using ::testing::IsNull; using ::testing::NotNull; using ::testing::InSequence; @@ -168,16 +173,25 @@ private: //////////////////////////////////////////////////////////////////////////////// -TEST(TAtomicPtrTest, Empty) +TEST(TIntrusiveAtomicPtrTest, Empty) { TIntricateObjectPtr emptyPointer; EXPECT_EQ(nullptr, emptyPointer.Get()); } -// Reserved ref count. -constexpr int RRC = 65535; +constexpr int ReservedRefCount = 65535; -TEST(TAtomicPtrTest, Basic) +TEST(TIntrusiveAtomicPtrTest, Reset) +{ + TIntricateObject object; + TIntricateObjectPtr owningPointer(&object); + TAtomicIntrusivePtr<TIntricateObject> atomicPointer(owningPointer); + atomicPointer.Reset(); + EXPECT_EQ(nullptr, atomicPointer.Get()); + EXPECT_THAT(object, HasRefCounts(1 + ReservedRefCount, ReservedRefCount, 0)); +} + +TEST(TIntrusiveAtomicPtrTest, Basic) { TIntricateObject object; @@ -196,27 +210,27 @@ TEST(TAtomicPtrTest, Basic) TIntricateObjectPtr owningPointer(&object); TAtomicIntrusivePtr<TIntricateObject> atomicPointer(owningPointer); - EXPECT_THAT(object, HasRefCounts(2 + RRC, 1, 1)); + EXPECT_THAT(object, HasRefCounts(2 + ReservedRefCount, 1, 1)); EXPECT_EQ(&object, owningPointer.Get()); auto p1 = atomicPointer.Acquire(); - EXPECT_THAT(object, HasRefCounts(2 + RRC, 1, 1)); + EXPECT_THAT(object, HasRefCounts(2 + ReservedRefCount, 1, 1)); p1.Reset(); - EXPECT_THAT(object, HasRefCounts(2 + RRC, 2, 1)); + EXPECT_THAT(object, HasRefCounts(2 + ReservedRefCount, 2, 1)); owningPointer.Reset(); - EXPECT_THAT(object, HasRefCounts(2 + RRC, 3, 1)); + EXPECT_THAT(object, HasRefCounts(2 + ReservedRefCount, 3, 1)); } - EXPECT_THAT(object, HasRefCounts(2 + RRC, 2 + RRC, 2)); + EXPECT_THAT(object, HasRefCounts(2 + ReservedRefCount, 2 + ReservedRefCount, 2)); } -TEST(TAtomicPtrTest, BasicConst) +TEST(TIntrusiveAtomicPtrTest, BasicConst) { const TIntricateObject object; @@ -235,86 +249,86 @@ TEST(TAtomicPtrTest, BasicConst) TConstIntricateObjectPtr owningPointer(&object); TAtomicIntrusivePtr<const TIntricateObject> atomicPointer(owningPointer); - EXPECT_THAT(object, HasRefCounts(2 + RRC, 1, 1)); + EXPECT_THAT(object, HasRefCounts(2 + ReservedRefCount, 1, 1)); EXPECT_EQ(&object, owningPointer.Get()); auto p1 = atomicPointer.Acquire(); - EXPECT_THAT(object, HasRefCounts(2 + RRC, 1, 1)); + EXPECT_THAT(object, HasRefCounts(2 + ReservedRefCount, 1, 1)); p1.Reset(); - EXPECT_THAT(object, HasRefCounts(2 + RRC, 2, 1)); + EXPECT_THAT(object, HasRefCounts(2 + ReservedRefCount, 2, 1)); owningPointer.Reset(); - EXPECT_THAT(object, HasRefCounts(2 + RRC, 3, 1)); + EXPECT_THAT(object, HasRefCounts(2 + ReservedRefCount, 3, 1)); } - EXPECT_THAT(object, HasRefCounts(2 + RRC, 2 + RRC, 2)); + EXPECT_THAT(object, HasRefCounts(2 + ReservedRefCount, 2 + ReservedRefCount, 2)); } -TEST(TAtomicPtrTest, Acquire) +TEST(TIntrusiveAtomicPtrTest, Acquire) { TIntricateObject object; { TAtomicIntrusivePtr<TIntricateObject> atomicPtr{TIntricateObjectPtr(&object)}; - EXPECT_THAT(object, HasRefCounts(RRC, 0, 0)); + EXPECT_THAT(object, HasRefCounts(ReservedRefCount, 0, 0)); - for (int i = 0; i < RRC / 2; ++i) { + for (int i = 0; i < ReservedRefCount / 2; ++i) { { auto tmp = atomicPtr.Acquire(); - EXPECT_THAT(object, HasRefCounts(RRC, i, 0)); + EXPECT_THAT(object, HasRefCounts(ReservedRefCount, i, 0)); } - EXPECT_THAT(object, HasRefCounts(RRC, i + 1, 0)); + EXPECT_THAT(object, HasRefCounts(ReservedRefCount, i + 1, 0)); } { auto tmp = atomicPtr.Acquire(); - EXPECT_THAT(object, HasRefCounts(RRC + RRC / 2, RRC / 2, 0)); + EXPECT_THAT(object, HasRefCounts(ReservedRefCount + ReservedRefCount / 2, ReservedRefCount / 2, 0)); } - EXPECT_THAT(object, HasRefCounts(RRC + RRC / 2, RRC / 2 + 1, 0)); + EXPECT_THAT(object, HasRefCounts(ReservedRefCount + ReservedRefCount / 2, ReservedRefCount / 2 + 1, 0)); } - EXPECT_THAT(object, HasRefCounts(RRC + RRC / 2, RRC + RRC / 2, 1)); + EXPECT_THAT(object, HasRefCounts(ReservedRefCount + ReservedRefCount / 2, ReservedRefCount + ReservedRefCount / 2, 1)); } -TEST(TAtomicPtrTest, AcquireConst) +TEST(TIntrusiveAtomicPtrTest, AcquireConst) { const TIntricateObject object; { TAtomicIntrusivePtr<const TIntricateObject> atomicPtr{TConstIntricateObjectPtr(&object)}; - EXPECT_THAT(object, HasRefCounts(RRC, 0, 0)); + EXPECT_THAT(object, HasRefCounts(ReservedRefCount, 0, 0)); - for (int i = 0; i < RRC / 2; ++i) { + for (int i = 0; i < ReservedRefCount / 2; ++i) { { auto tmp = atomicPtr.Acquire(); - EXPECT_THAT(object, HasRefCounts(RRC, i, 0)); + EXPECT_THAT(object, HasRefCounts(ReservedRefCount, i, 0)); } - EXPECT_THAT(object, HasRefCounts(RRC, i + 1, 0)); + EXPECT_THAT(object, HasRefCounts(ReservedRefCount, i + 1, 0)); } { auto tmp = atomicPtr.Acquire(); - EXPECT_THAT(object, HasRefCounts(RRC + RRC / 2, RRC / 2, 0)); + EXPECT_THAT(object, HasRefCounts(ReservedRefCount + ReservedRefCount / 2, ReservedRefCount / 2, 0)); } - EXPECT_THAT(object, HasRefCounts(RRC + RRC / 2, RRC / 2 + 1, 0)); + EXPECT_THAT(object, HasRefCounts(ReservedRefCount + ReservedRefCount / 2, ReservedRefCount / 2 + 1, 0)); } - EXPECT_THAT(object, HasRefCounts(RRC + RRC / 2, RRC + RRC / 2, 1)); + EXPECT_THAT(object, HasRefCounts(ReservedRefCount + ReservedRefCount / 2, ReservedRefCount + ReservedRefCount / 2, 1)); } -TEST(TAtomicPtrTest, CAS) +TEST(TIntrusiveAtomicPtrTest, CAS) { TIntricateObject o1; TIntricateObject o2; { TAtomicIntrusivePtr<TIntricateObject> atomicPtr{TIntricateObjectPtr(&o1)}; - EXPECT_THAT(o1, HasRefCounts(RRC, 0, 0)); + EXPECT_THAT(o1, HasRefCounts(ReservedRefCount, 0, 0)); TIntricateObjectPtr p2(&o2); EXPECT_THAT(o2, HasRefCounts(1, 0, 0)); @@ -323,28 +337,28 @@ TEST(TAtomicPtrTest, CAS) EXPECT_TRUE(atomicPtr.CompareAndSwap(rawPtr, std::move(p2))); EXPECT_EQ(rawPtr, &o1); - EXPECT_THAT(o1, HasRefCounts(RRC, RRC, 1)); - EXPECT_THAT(o2, HasRefCounts(RRC, 0, 0)); + EXPECT_THAT(o1, HasRefCounts(ReservedRefCount, ReservedRefCount, 1)); + EXPECT_THAT(o2, HasRefCounts(ReservedRefCount, 0, 0)); rawPtr = nullptr; EXPECT_FALSE(atomicPtr.CompareAndSwap(rawPtr, TIntricateObjectPtr(&o1))); EXPECT_EQ(rawPtr, &o2); - EXPECT_THAT(o1, HasRefCounts(2 * RRC, 2 * RRC, 2)); - EXPECT_THAT(o2, HasRefCounts(RRC, 0, 0)); + EXPECT_THAT(o1, HasRefCounts(2 * ReservedRefCount, 2 * ReservedRefCount, 2)); + EXPECT_THAT(o2, HasRefCounts(ReservedRefCount, 0, 0)); } - EXPECT_THAT(o2, HasRefCounts(RRC, RRC, 1)); + EXPECT_THAT(o2, HasRefCounts(ReservedRefCount, ReservedRefCount, 1)); } -TEST(TAtomicPtrTest, CASConst) +TEST(TIntrusiveAtomicPtrTest, CASConst) { const TIntricateObject o1; const TIntricateObject o2; { TAtomicIntrusivePtr<const TIntricateObject> atomicPtr{TConstIntricateObjectPtr(&o1)}; - EXPECT_THAT(o1, HasRefCounts(RRC, 0, 0)); + EXPECT_THAT(o1, HasRefCounts(ReservedRefCount, 0, 0)); TConstIntricateObjectPtr p2(&o2); EXPECT_THAT(o2, HasRefCounts(1, 0, 0)); @@ -353,20 +367,45 @@ TEST(TAtomicPtrTest, CASConst) EXPECT_TRUE(atomicPtr.CompareAndSwap(rawPtr, std::move(p2))); EXPECT_EQ(rawPtr, &o1); - EXPECT_THAT(o1, HasRefCounts(RRC, RRC, 1)); - EXPECT_THAT(o2, HasRefCounts(RRC, 0, 0)); + EXPECT_THAT(o1, HasRefCounts(ReservedRefCount, ReservedRefCount, 1)); + EXPECT_THAT(o2, HasRefCounts(ReservedRefCount, 0, 0)); rawPtr = nullptr; EXPECT_FALSE(atomicPtr.CompareAndSwap(rawPtr, TConstIntricateObjectPtr(&o1))); EXPECT_EQ(rawPtr, &o2); - EXPECT_THAT(o1, HasRefCounts(2 * RRC, 2 * RRC, 2)); - EXPECT_THAT(o2, HasRefCounts(RRC, 0, 0)); + EXPECT_THAT(o1, HasRefCounts(2 * ReservedRefCount, 2 * ReservedRefCount, 2)); + EXPECT_THAT(o2, HasRefCounts(ReservedRefCount, 0, 0)); } - EXPECT_THAT(o2, HasRefCounts(RRC, RRC, 1)); + EXPECT_THAT(o2, HasRefCounts(ReservedRefCount, ReservedRefCount, 1)); } +TEST(TIntrusiveAtomicPtrTest, LSan) +{ + struct S final + { }; + + struct TSingleton + { + TSingleton() + { + for (auto& ptr : Ptrs) { + ptr.Store(New<S>()); + // Clobber pointer bits to prevent LSan from tracing the pointer. + ptr.Acquire(); + } + } + + // LSan has some issues detecting leaks when just one allocation is made. + std::array<TAtomicIntrusivePtr<S>, 100> Ptrs; + }; + + LeakySingleton<TSingleton>(); +} + +#endif + //////////////////////////////////////////////////////////////////////////////// } // namespace diff --git a/library/cpp/yt/misc/port.h b/library/cpp/yt/misc/port.h index fe1c5e96de..fd7df53855 100644 --- a/library/cpp/yt/misc/port.h +++ b/library/cpp/yt/misc/port.h @@ -7,9 +7,11 @@ #error YT requires 64-bit platform #endif -// This define enables tracking of reference-counted objects to provide -// various insightful information on memory usage and object creation patterns. -#define YT_ENABLE_REF_COUNTED_TRACKING +#ifndef YT_DISABLE_REF_COUNTED_TRACKING + // This define enables tracking of reference-counted objects to provide + // various insightful information on memory usage and object creation patterns. + #define YT_ENABLE_REF_COUNTED_TRACKING +#endif // This define enables logging with TRACE level. You can still disable trace logging // for particular TU by discarding this macro identifier. diff --git a/library/cpp/yt/misc/source_location.cpp b/library/cpp/yt/misc/source_location.cpp index 598126f5a7..8d22d43636 100644 --- a/library/cpp/yt/misc/source_location.cpp +++ b/library/cpp/yt/misc/source_location.cpp @@ -1,32 +1,11 @@ #include "source_location.h" -#include <library/cpp/yt/string/format.h> - #include <string.h> namespace NYT { //////////////////////////////////////////////////////////////////////////////// -#ifdef __cpp_lib_source_location - -void FormatValue(TStringBuilderBase* builder, const std::source_location& location, TStringBuf /*spec*/) -{ - if (location.file_name() != nullptr) { - builder->AppendFormat( - "%v:%v:%v", - location.file_name(), - location.line(), - location.column()); - } else { - builder->AppendString("<unknown>"); - } -} - -#endif // __cpp_lib_source_location - -//////////////////////////////////////////////////////////////////////////////// - const char* TSourceLocation::GetFileName() const { return FileName_; @@ -70,18 +49,6 @@ bool TSourceLocation::operator==(const TSourceLocation& other) const Line_ == other.Line_; } -void FormatValue(TStringBuilderBase* builder, const TSourceLocation& location, TStringBuf /*spec*/) -{ - if (location.GetFileName() != nullptr) { - builder->AppendFormat( - "%v:%v", - location.GetFileName(), - location.GetLine()); - } else { - builder->AppendString("<unknown>"); - } -} - //////////////////////////////////////////////////////////////////////////////// } // namespace NYT diff --git a/library/cpp/yt/misc/source_location.h b/library/cpp/yt/misc/source_location.h index 286527887a..7a47502262 100644 --- a/library/cpp/yt/misc/source_location.h +++ b/library/cpp/yt/misc/source_location.h @@ -10,15 +10,6 @@ namespace NYT { //////////////////////////////////////////////////////////////////////////////// -class TStringBuilderBase; - -// TODO(dgolear): Drop when LLVM-14 is eradicated. -#ifdef __cpp_lib_source_location -void FormatValue(TStringBuilderBase* builder, const std::source_location& location, TStringBuf /*spec*/); -#endif // __cpp_lib_source_location - -//////////////////////////////////////////////////////////////////////////////// - class TSourceLocation { public: @@ -47,8 +38,6 @@ private: #define YT_CURRENT_SOURCE_LOCATION ::NYT::TSourceLocation(__FILE__, __LINE__) #endif // __cpp_lib_source_location -void FormatValue(TStringBuilderBase* builder, const TSourceLocation& location, TStringBuf spec); - //////////////////////////////////////////////////////////////////////////////// } // namespace NYT diff --git a/library/cpp/yt/misc/variant-inl.h b/library/cpp/yt/misc/variant-inl.h deleted file mode 100644 index 4b0e04c92c..0000000000 --- a/library/cpp/yt/misc/variant-inl.h +++ /dev/null @@ -1,50 +0,0 @@ -#ifndef VARIANT_INL_H_ -#error "Direct inclusion of this file is not allowed, include variant.h" -// For the sake of sane code completion. -#include "variant.h" -#endif - -#include <type_traits> - -namespace NYT { - -//////////////////////////////////////////////////////////////////////////////// - -namespace NDetail { - -template <size_t Index, class... Ts> -struct TVariantFormatter; - -template <size_t Index> -struct TVariantFormatter<Index> -{ - template <class TVariant> - static void Do(TStringBuilderBase* /*builder*/, const TVariant& /*variant*/, TStringBuf /*spec*/) - { } -}; - -template <size_t Index, class T, class... Ts> -struct TVariantFormatter<Index, T, Ts...> -{ - template <class TVariant> - static void Do(TStringBuilderBase* builder, const TVariant& variant, TStringBuf spec) - { - if (variant.index() == Index) { - FormatValue(builder, std::get<Index>(variant), spec); - } else { - TVariantFormatter<Index + 1, Ts...>::Do(builder, variant, spec); - } - } -}; - -} // namespace NDetail - -template <class... Ts> -void FormatValue(TStringBuilderBase* builder, const std::variant<Ts...>& variant, TStringBuf spec) -{ - NDetail::TVariantFormatter<0, Ts...>::Do(builder, variant, spec); -} - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT diff --git a/library/cpp/yt/misc/variant.cpp b/library/cpp/yt/misc/variant.cpp deleted file mode 100644 index df78563d0c..0000000000 --- a/library/cpp/yt/misc/variant.cpp +++ /dev/null @@ -1,16 +0,0 @@ -#include "variant.h" - -#include <library/cpp/yt/string/string_builder.h> - -namespace NYT { - -//////////////////////////////////////////////////////////////////////////////// - -void FormatValue(TStringBuilderBase* builder, const std::monostate&, TStringBuf /*spec*/) -{ - builder->AppendString(TStringBuf("<monostate>")); -} - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT diff --git a/library/cpp/yt/misc/variant.h b/library/cpp/yt/misc/variant.h index 9e8e122089..7cc3d8ddda 100644 --- a/library/cpp/yt/misc/variant.h +++ b/library/cpp/yt/misc/variant.h @@ -7,15 +7,6 @@ namespace NYT { //////////////////////////////////////////////////////////////////////////////// -class TStringBuilderBase; - -template <class... Ts> -void FormatValue(TStringBuilderBase* builder, const std::variant<Ts...>& variant, TStringBuf spec); - -void FormatValue(TStringBuilderBase* builder, const std::monostate&, TStringBuf /*spec*/); - -//////////////////////////////////////////////////////////////////////////////// - //! A concise way of creating a functor with an overloaded operator(). /*! * Very useful for std::visit-ing variants. For example: @@ -50,7 +41,3 @@ auto Visit(T&& variant, U&&... visitorOverloads) //////////////////////////////////////////////////////////////////////////////// } // namespace NYT - -#define VARIANT_INL_H_ -#include "variant-inl.h" -#undef VARIANT_INL_H_ diff --git a/library/cpp/yt/misc/ya.make b/library/cpp/yt/misc/ya.make index 841930aaa2..3135887e95 100644 --- a/library/cpp/yt/misc/ya.make +++ b/library/cpp/yt/misc/ya.make @@ -6,7 +6,6 @@ SRCS( guid.cpp source_location.cpp thread_name.cpp - variant.cpp ) PEERDIR( diff --git a/library/cpp/yt/string/format-inl.h b/library/cpp/yt/string/format-inl.h index 67c8affa6e..188a699a3d 100644 --- a/library/cpp/yt/string/format-inl.h +++ b/library/cpp/yt/string/format-inl.h @@ -16,7 +16,7 @@ #include <library/cpp/yt/misc/concepts.h> #include <library/cpp/yt/misc/enum.h> -#include <library/cpp/yt/misc/wrapper_traits.h> +#include <library/cpp/yt/misc/source_location.h> #include <util/generic/maybe.h> @@ -30,6 +30,10 @@ #include <filesystem> #endif +#ifdef __cpp_lib_source_location +#include <source_location> +#endif // __cpp_lib_source_location + namespace NYT { //////////////////////////////////////////////////////////////////////////////// @@ -592,6 +596,58 @@ inline void FormatValue(TStringBuilderBase* builder, const std::filesystem::path } #endif +#ifdef __cpp_lib_source_location +// std::source_location +inline void FormatValue(TStringBuilderBase* builder, const std::source_location& location, TStringBuf /*spec*/) +{ + if (location.file_name() != nullptr) { + builder->AppendFormat( + "%v:%v:%v", + location.file_name(), + location.line(), + location.column()); + } else { + builder->AppendString("<unknown>"); + } +} +#endif // __cpp_lib_source_location + +// TSourceLocation +inline void FormatValue(TStringBuilderBase* builder, const TSourceLocation& location, TStringBuf /*spec*/) +{ + if (location.GetFileName() != nullptr) { + builder->AppendFormat( + "%v:%v", + location.GetFileName(), + location.GetLine()); + } else { + builder->AppendString("<unknown>"); + } +} + +// std::monostate +inline void FormatValue(TStringBuilderBase* builder, const std::monostate&, TStringBuf /*spec*/) +{ + builder->AppendString(TStringBuf("<monostate>")); +} + +// std::variant +template <class... Ts> + requires (CFormattable<Ts> && ...) +void FormatValue(TStringBuilderBase* builder, const std::variant<Ts...>& variant, TStringBuf spec) +{ + [&] <size_t... Ids> (std::index_sequence<Ids...>) { + ([&] { + if (variant.index() == Ids) { + FormatValue(builder, std::get<Ids>(variant), spec); + return false; + } + + return true; + } () && ...); + } (std::index_sequence_for<Ts...>()); +} + // char inline void FormatValue(TStringBuilderBase* builder, char value, TStringBuf spec) { diff --git a/library/cpp/yt/string/unittests/format_ut.cpp b/library/cpp/yt/string/unittests/format_ut.cpp index aa820fd5af..ac3be99ad9 100644 --- a/library/cpp/yt/string/unittests/format_ut.cpp +++ b/library/cpp/yt/string/unittests/format_ut.cpp @@ -78,6 +78,7 @@ static_assert(CFormattable<TInstant>); struct TUnformattable { }; static_assert(!CFormattable<TUnformattable>); +static_assert(!CFormattable<std::variant<TUnformattable>>); static_assert(CFormattable<TWithCustomFlags>); diff --git a/util/folder/path.cpp b/util/folder/path.cpp index 940f6dffe0..321776cb0c 100644 --- a/util/folder/path.cpp +++ b/util/folder/path.cpp @@ -406,8 +406,7 @@ void TFsPath::DeleteIfExists() const { return; } - ::unlink(this->c_str()); - ::rmdir(this->c_str()); + NFs::Remove(GetPath()); if (Exists()) { ythrow TIoException() << "failed to delete " << Path_; } @@ -441,7 +440,7 @@ void TFsPath::ForceDelete() const { return; } - TFileStat stat(GetPath().c_str(), true); + TFileStat stat(GetPath(), true); if (stat.IsNull()) { const int err = LastSystemError(); #ifdef _win_ @@ -455,19 +454,15 @@ void TFsPath::ForceDelete() const { } } - bool succ; if (stat.IsDir()) { TVector<TFsPath> children; List(children); for (auto& i : children) { i.ForceDelete(); } - succ = ::rmdir(this->c_str()) == 0; - } else { - succ = ::unlink(this->c_str()) == 0; } - if (!succ && LastSystemError()) { + if (!NFs::Remove(GetPath())) { ythrow TIoException() << "failed to delete " << Path_; } } diff --git a/util/folder/path_ut.cpp b/util/folder/path_ut.cpp index 0fbcac8263..f73bd7f57a 100644 --- a/util/folder/path_ut.cpp +++ b/util/folder/path_ut.cpp @@ -663,7 +663,7 @@ Y_UNIT_TEST_SUITE(TFsPathTests) { }; #endif - Y_UNIT_TEST(TestForceDeleteErrorUnlink) { + Y_UNIT_TEST(TestForceDeleteErrorRemove) { TTempDir tempDir; const TFsPath testDir = TFsPath(tempDir()).Child("dir"); @@ -680,14 +680,17 @@ Y_UNIT_TEST_SUITE(TFsPathTests) { Y_DEFER { Chmod(testFile.c_str(), MODE0777); }; + // Checks that dir/file with readonly attribute will be deleted + // on Windows + UNIT_ASSERT_NO_EXCEPTION(testFile.ForceDelete()); #else Chmod(testDir.c_str(), S_IRUSR | S_IXUSR); Y_DEFER { Chmod(testDir.c_str(), MODE0777); }; + UNIT_ASSERT_EXCEPTION_CONTAINS(testFile.ForceDelete(), TIoException, + "failed to delete"); #endif - - UNIT_ASSERT_EXCEPTION_CONTAINS(testFile.ForceDelete(), TIoException, "failed to delete"); } Y_UNIT_TEST(TestForceDeleteErrorRmdir) { diff --git a/util/generic/maybe.h b/util/generic/maybe.h index 46468e5166..ceaf4d66f0 100644 --- a/util/generic/maybe.h +++ b/util/generic/maybe.h @@ -20,7 +20,7 @@ namespace NMaybe { }; } // namespace NMaybe -struct TNothing { +struct [[nodiscard]] TNothing { explicit constexpr TNothing(int) noexcept { } }; diff --git a/util/generic/string.h b/util/generic/string.h index b71bb72304..3476f71b68 100644 --- a/util/generic/string.h +++ b/util/generic/string.h @@ -861,7 +861,8 @@ public: return this->ConstRef(); } - operator TStringType&() { + template <typename T, typename = std::enable_if_t<std::is_same_v<T, TStringType>>> + operator T&() & { return this->MutRef(); } diff --git a/util/generic/string_ut.cpp b/util/generic/string_ut.cpp index bfd86bb07e..01c7a01188 100644 --- a/util/generic/string_ut.cpp +++ b/util/generic/string_ut.cpp @@ -1243,4 +1243,16 @@ Y_UNIT_TEST_SUITE(Interop) { Y_UNIT_TEST(TestTemp) { UNIT_ASSERT_VALUES_EQUAL("x" + ConstRef(TString("y")), "xy"); } + + static void ComparePointers(const std::string& s, const void* expected, TStringBuf descr) { + UNIT_ASSERT_VALUES_EQUAL_C(static_cast<const void*>(s.c_str()), expected, descr); + } + + Y_UNIT_TEST(TestConstShared) { + TString s(600, 'a'); + const void* stringStart = s.c_str(); + ComparePointers(s, stringStart, "unique"); + TString shared{s}; + ComparePointers(s, stringStart, "shared"); // converting a TString to a `const std::string&` should not cause data cloning + } } // Y_UNIT_TEST_SUITE(Interop) diff --git a/vendor/github.com/spf13/cobra/.yo.snapshot.json b/vendor/github.com/spf13/cobra/.yo.snapshot.json new file mode 100644 index 0000000000..595b4ffca5 --- /dev/null +++ b/vendor/github.com/spf13/cobra/.yo.snapshot.json @@ -0,0 +1,75 @@ +{ + "Version": "v1.8.1", + "GoVersion": "1.15", + "Sum": "h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM=", + "Packages": { + "github.com/spf13/cobra": { + "Imports": [ + "bytes", + "context", + "errors", + "fmt", + "github.com/inconshreveable/mousetrap", + "github.com/spf13/pflag", + "io", + "os", + "path/filepath", + "reflect", + "regexp", + "sort", + "strconv", + "strings", + "sync", + "text/template", + "time", + "unicode" + ], + "TestImports": [ + "bytes", + "context", + "errors", + "fmt", + "github.com/spf13/pflag", + "io", + "os", + "os/exec", + "path/filepath", + "reflect", + "regexp", + "strings", + "sync", + "testing", + "text/template" + ] + }, + "github.com/spf13/cobra/doc": { + "Imports": [ + "bytes", + "fmt", + "github.com/cpuguy83/go-md2man/v2/md2man", + "github.com/spf13/cobra", + "github.com/spf13/pflag", + "gopkg.in/yaml.v3", + "io", + "os", + "path/filepath", + "sort", + "strconv", + "strings", + "time" + ], + "TestImports": [ + "bufio", + "bytes", + "fmt", + "github.com/spf13/cobra", + "github.com/spf13/cobra/doc", + "io/ioutil", + "os", + "path/filepath", + "strings", + "testing" + ] + } + } +}
\ No newline at end of file diff --git a/vendor/github.com/spf13/pflag/.yo.snapshot.json b/vendor/github.com/spf13/pflag/.yo.snapshot.json new file mode 100644 index 0000000000..40b89fd397 --- /dev/null +++ b/vendor/github.com/spf13/pflag/.yo.snapshot.json @@ -0,0 +1,44 @@ +{ + "Version": "v1.0.6-0.20201009195203-85dd5c8bc61c", + "GoVersion": "1.12", + "Sum": "h1:zqmyTlQyufRC65JnImJ6H1Sf7BDj8bG31EV919NVEQc=", + "Packages": { + "github.com/spf13/pflag": { + "Imports": [ + "bytes", + "encoding/base64", + "encoding/csv", + "encoding/hex", + "errors", + "flag", + "fmt", + "io", + "net", + "os", + "reflect", + "sort", + "strconv", + "strings", + "time" + ], + "TestImports": [ + "bytes", + "encoding/base64", + "encoding/csv", + "flag", + "fmt", + "github.com/spf13/pflag", + "io", + "io/ioutil", + "net", + "os", + "reflect", + "sort", + "strconv", + "strings", + "testing", + "time" + ] + } + } +}
\ No newline at end of file @@ -39,33 +39,33 @@ REGISTRY_ENDPOINT = os.environ.get("YA_REGISTRY_ENDPOINT", "https://devtools-reg PLATFORM_MAP = { "data": { "darwin": { - "md5": "0d542fef71e6428ade85f0da9f52cb5a", + "md5": "30824e0eea5c1de99cad870142f16922", "urls": [ - f"{REGISTRY_ENDPOINT}/7083801705" + f"{REGISTRY_ENDPOINT}/7120326728" ] }, "darwin-arm64": { - "md5": "22390e745628eb3cf06f1bf7af30687c", + "md5": "36b8d3ae595d998a6156e60fa3064d1d", "urls": [ - f"{REGISTRY_ENDPOINT}/7083801096" + f"{REGISTRY_ENDPOINT}/7120326401" ] }, "linux-aarch64": { - "md5": "8e8d1e623d0b7e90edf87c572dfa435a", + "md5": "4a05a464bcfe750c20c6cbb117bbc0fa", "urls": [ - f"{REGISTRY_ENDPOINT}/7083800513" + f"{REGISTRY_ENDPOINT}/7120325932" ] }, "win32-clang-cl": { - "md5": "687a54f15098a398a5cdc2c4a7133e65", + "md5": "df49f4b133b8948eef46e65b7b0adf32", "urls": [ - f"{REGISTRY_ENDPOINT}/7083802244" + f"{REGISTRY_ENDPOINT}/7120327156" ] }, "linux": { - "md5": "8ed590932df65eb577f76f543ee53a78", + "md5": "fb7ae3201abf5b7891e20be90c955a35", "urls": [ - f"{REGISTRY_ENDPOINT}/7083802745" + f"{REGISTRY_ENDPOINT}/7120327554" ] } } diff --git a/yt/cpp/mapreduce/interface/common.cpp b/yt/cpp/mapreduce/interface/common.cpp index 831762fa95..966be8341f 100644 --- a/yt/cpp/mapreduce/interface/common.cpp +++ b/yt/cpp/mapreduce/interface/common.cpp @@ -348,6 +348,7 @@ TColumnSchema& TColumnSchema::Type(const NTi::TTypePtr& type) & { Y_ABORT_UNLESS(type.Get(), "Cannot create column schema with nullptr type"); TypeV3_ = type; + RawTypeV3_ = {}; return *this; } @@ -355,6 +356,7 @@ TColumnSchema TColumnSchema::Type(const NTi::TTypePtr& type) && { Y_ABORT_UNLESS(type.Get(), "Cannot create column schema with nullptr type"); TypeV3_ = type; + RawTypeV3_ = {}; return *this; } @@ -388,6 +390,25 @@ TColumnSchema TColumnSchema::Type(EValueType type, bool required) && return Type(ToTypeV3(type, required)); } +const TMaybe<TNode>& TColumnSchema::RawTypeV3() const +{ + return RawTypeV3_; +} + +TColumnSchema& TColumnSchema::RawTypeV3(TNode rawTypeV3) & +{ + RawTypeV3_ = std::move(rawTypeV3); + TypeV3_ = nullptr; + return *this; +} + +TColumnSchema TColumnSchema::RawTypeV3(TNode rawTypeV3) && +{ + RawTypeV3_ = std::move(rawTypeV3); + TypeV3_ = nullptr; + return *this; +} + bool operator==(const TColumnSchema& lhs, const TColumnSchema& rhs) { return diff --git a/yt/cpp/mapreduce/interface/common.h b/yt/cpp/mapreduce/interface/common.h index 557f4ea6c4..d8365c32c0 100644 --- a/yt/cpp/mapreduce/interface/common.h +++ b/yt/cpp/mapreduce/interface/common.h @@ -649,11 +649,6 @@ public: NTi::TTypePtr TypeV3() const; /// @} - /// - /// @brief Raw yson representation of column type - /// @deprecated Prefer to use `TypeV3` methods. - FLUENT_FIELD_OPTION_ENCAPSULATED(TNode, RawTypeV3); - /// Column sort order FLUENT_FIELD_OPTION_ENCAPSULATED(ESortOrder, SortOrder); @@ -696,10 +691,21 @@ public: TColumnSchema Type(EValueType type, bool required) &&; /// @} + /// + /// @{ + /// + /// @brief Raw yson representation of column type + /// @deprecated Prefer to use `TypeV3` methods. + const TMaybe<TNode>& RawTypeV3() const; + TColumnSchema& RawTypeV3(TNode rawTypeV3)&; + TColumnSchema RawTypeV3(TNode rawTypeV3)&&; + /// @} + + private: friend void Deserialize(TColumnSchema& columnSchema, const TNode& node); NTi::TTypePtr TypeV3_; - bool Required_ = false; + TMaybe<TNode> RawTypeV3_; }; /// Equality check checks all fields of column schema. diff --git a/yt/cpp/mapreduce/interface/ut/common_ut.cpp b/yt/cpp/mapreduce/interface/ut/common_ut.cpp index 85122a97ec..196ef73839 100644 --- a/yt/cpp/mapreduce/interface/ut/common_ut.cpp +++ b/yt/cpp/mapreduce/interface/ut/common_ut.cpp @@ -351,3 +351,13 @@ TEST(TCommonTest, TableSchemaEquality) other.UniqueKeys(false); ASSERT_SERIALIZABLES_NE(other, schema); } + +TEST(TCommonTest, ModificationLoadedSchema) +{ + auto schema = TTableSchema::FromNode(NodeFromYsonString(R"""( + [{name=foo;type_v3=string}] + )""")); + schema.MutableColumns()[0].Type(VT_INT64, true); + + ASSERT_EQ(schema.ToNode()[0]["type"].AsString(), "int64"); +} diff --git a/yt/yt/client/api/client_common.h b/yt/yt/client/api/client_common.h index 873be58c0c..a5569fb074 100644 --- a/yt/yt/client/api/client_common.h +++ b/yt/yt/client/api/client_common.h @@ -144,6 +144,8 @@ struct TSelectRowsOptionsBase ui64 RangeExpansionLimit = 200000; //! Limits maximum parallel subqueries. int MaxSubqueries = std::numeric_limits<int>::max(); + //! Limits parallel subqueries by row count. + ui64 MinRowCountPerSubquery = 100'000; //! Path in Cypress with UDFs. std::optional<TString> UdfRegistryPath; //! If |true| then logging is more verbose. diff --git a/yt/yt/client/api/public.h b/yt/yt/client/api/public.h index 847a60ebe5..be710a04ce 100644 --- a/yt/yt/client/api/public.h +++ b/yt/yt/client/api/public.h @@ -203,8 +203,8 @@ inline const TString BannedAttributeName("banned"); inline const TString RoleAttributeName("role"); inline const TString AddressesAttributeName("addresses"); inline const TString BalancersAttributeName("balancers"); -inline const TString DefaultRpcProxyRole("default"); -inline const TString DefaultHttpProxyRole("data"); +inline const std::string DefaultRpcProxyRole("default"); +inline const std::string DefaultHttpProxyRole("data"); inline const TString JournalPayloadKey("payload"); inline const TString HunkPayloadKey("payload"); diff --git a/yt/yt/client/api/rpc_proxy/client_impl.cpp b/yt/yt/client/api/rpc_proxy/client_impl.cpp index cc9e093381..b9e4944381 100644 --- a/yt/yt/client/api/rpc_proxy/client_impl.cpp +++ b/yt/yt/client/api/rpc_proxy/client_impl.cpp @@ -471,19 +471,27 @@ TFuture<void> TClient::AlterTableReplica( if (options.Enabled) { req->set_enabled(*options.Enabled); } + if (options.Mode) { req->set_mode(static_cast<NProto::ETableReplicaMode>(*options.Mode)); } + if (options.PreserveTimestamps) { req->set_preserve_timestamps(*options.PreserveTimestamps); } + if (options.Atomicity) { req->set_atomicity(static_cast<NProto::EAtomicity>(*options.Atomicity)); } + if (options.EnableReplicatedTableTracker) { req->set_enable_replicated_table_tracker(*options.EnableReplicatedTableTracker); } + if (options.ReplicaPath) { + req->set_replica_path(*options.ReplicaPath); + } + ToProto(req->mutable_mutating_options(), options); return req->Invoke().As<void>(); diff --git a/yt/yt/client/api/rpc_proxy/row_batch_reader.cpp b/yt/yt/client/api/rpc_proxy/row_batch_reader.cpp new file mode 100644 index 0000000000..22223f6945 --- /dev/null +++ b/yt/yt/client/api/rpc_proxy/row_batch_reader.cpp @@ -0,0 +1,156 @@ +#include "row_batch_reader.h" +#include "helpers.h" +#include "row_stream.h" +#include "wire_row_stream.h" + +#include <yt/yt/client/table_client/name_table.h> + +#include <yt_proto/yt/client/api/rpc_proxy/proto/api_service.pb.h> + +namespace NYT::NApi::NRpcProxy { + +using namespace NConcurrency; +using namespace NTableClient; + +//////////////////////////////////////////////////////////////////////////////// + +TRowBatchReader::TRowBatchReader( + IAsyncZeroCopyInputStreamPtr underlying, + bool isStreamWithStatistics) + : Underlying_(std::move(underlying)) + , Decoder_(CreateWireRowStreamDecoder(NameTable_)) + , IsStreamWithStatistics_(isStreamWithStatistics) +{ + YT_VERIFY(Underlying_); + + RowsFuture_ = GetRows(); + ReadyEvent_.TrySetFrom(RowsFuture_); +} + +IUnversionedRowBatchPtr TRowBatchReader::Read(const TRowBatchReadOptions& options) +{ + StoredRows_.clear(); + + if (!ReadyEvent_.IsSet() || !ReadyEvent_.Get().IsOK()) { + return CreateEmptyUnversionedRowBatch(); + } + + if (!Finished_) { + ReadyEvent_ = NewPromise<void>(); + } + + std::vector<TUnversionedRow> rows; + rows.reserve(options.MaxRowsPerRead); + i64 dataWeight = 0; + + while (RowsFuture_ && + RowsFuture_.IsSet() && + RowsFuture_.Get().IsOK() && + !Finished_ && + std::ssize(rows) < options.MaxRowsPerRead && + dataWeight < options.MaxDataWeightPerRead) + { + const auto& currentRows = RowsFuture_.Get().Value(); + + if (currentRows.Empty()) { + ReadyEvent_.Set(); + Finished_ = true; + continue; + } + + while (CurrentRowsOffset_ < std::ssize(currentRows) && + std::ssize(rows) < options.MaxRowsPerRead && + dataWeight < options.MaxDataWeightPerRead) + { + auto row = currentRows[CurrentRowsOffset_++]; + rows.push_back(row); + dataWeight += GetDataWeight(row); + } + + StoredRows_.push_back(currentRows); + + if (CurrentRowsOffset_ == std::ssize(currentRows)) { + RowsFuture_ = GetRows(); + CurrentRowsOffset_ = 0; + } + } + + RowCount_ += rows.size(); + DataWeight_ += dataWeight; + + ReadyEvent_.TrySetFrom(RowsFuture_); + return rows.empty() + ? nullptr + : CreateBatchFromUnversionedRows(MakeSharedRange(std::move(rows), MakeStrong(this))); +} + +TFuture<void> TRowBatchReader::GetReadyEvent() const +{ + return ReadyEvent_; +} + +const TNameTablePtr& TRowBatchReader::GetNameTable() const +{ + return NameTable_; +} + +TFuture<TSharedRange<TUnversionedRow>> TRowBatchReader::GetRows() +{ + return Underlying_->Read() + .Apply(BIND([this, weakThis = MakeWeak(this)](const TSharedRef& block) { + auto this_ = weakThis.Lock(); + if (!this_) { + THROW_ERROR_EXCEPTION(NYT::EErrorCode::Canceled, "Reader destroyed"); + } + + NProto::TRowsetDescriptor descriptor; + NProto::TRowsetStatistics statistics; + auto payloadRef = DeserializeRowStreamBlockEnvelope( + block, + &descriptor, + IsStreamWithStatistics_ ? &statistics : nullptr); + + ValidateRowsetDescriptor( + descriptor, + CurrentWireFormatVersion, + NProto::RK_UNVERSIONED, + NProto::ERowsetFormat::RF_YT_WIRE); + + if (descriptor.rowset_format() != NApi::NRpcProxy::NProto::RF_YT_WIRE) { + THROW_ERROR_EXCEPTION( + "Unsupported rowset format %Qv", + NApi::NRpcProxy::NProto::ERowsetFormat_Name(descriptor.rowset_format())); + } + + auto batch = Decoder_->Decode(payloadRef, descriptor); + auto rows = batch->MaterializeRows(); + + if (IsStreamWithStatistics_) { + ApplyStatistics(statistics); + } + + if (rows.Empty()) { + return ExpectEndOfStream(Underlying_).Apply(BIND([=] { + return std::move(rows); + })); + } + return MakeFuture(std::move(rows)); + })); +} + +void TRowBatchReader::ApplyStatistics(const NProto::TRowsetStatistics& /*statistics*/) +{ +} + +//////////////////////////////////////////////////////////////////////////////// + +IRowBatchReaderPtr CreateRowBatchReader( + IAsyncZeroCopyInputStreamPtr inputStream, + bool isStreamWithStatistics) +{ + return New<TRowBatchReader>(std::move(inputStream), isStreamWithStatistics); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NApi::NRpcProxy diff --git a/yt/yt/client/api/rpc_proxy/row_batch_reader.h b/yt/yt/client/api/rpc_proxy/row_batch_reader.h new file mode 100644 index 0000000000..b7e30c02b6 --- /dev/null +++ b/yt/yt/client/api/rpc_proxy/row_batch_reader.h @@ -0,0 +1,58 @@ +#pragma once + +#include <yt/yt/client/api/row_batch_reader.h> +#include <yt/yt/client/api/rpc_proxy/public.h> + +#include <yt/yt_proto/yt/client/api/rpc_proxy/proto/api_service.pb.h> + +namespace NYT::NApi::NRpcProxy { + +//////////////////////////////////////////////////////////////////////////////// + +class TRowBatchReader + : public virtual IRowBatchReader +{ +public: + TRowBatchReader( + NConcurrency::IAsyncZeroCopyInputStreamPtr underlying, + bool isStreamWithStatistics); + + NTableClient::IUnversionedRowBatchPtr Read(const NTableClient::TRowBatchReadOptions& options) override; + + TFuture<void> GetReadyEvent() const override; + + const NTableClient::TNameTablePtr& GetNameTable() const override; + +protected: + i64 RowCount_ = 0; + i64 DataWeight_ = 0; + + virtual void ApplyStatistics(const NProto::TRowsetStatistics& statistics); + +private: + const NConcurrency::IAsyncZeroCopyInputStreamPtr Underlying_; + const NTableClient::TNameTablePtr NameTable_ = New<NTableClient::TNameTable>(); + const IRowStreamDecoderPtr Decoder_; + + TFuture<TSharedRange<NTableClient::TUnversionedRow>> RowsFuture_; + TPromise<void> ReadyEvent_ = NewPromise<void>(); + + std::vector<TSharedRange<NTableClient::TUnversionedRow>> StoredRows_; + + bool Finished_ = false; + i64 CurrentRowsOffset_ = 0; + + const bool IsStreamWithStatistics_; + + TFuture<TSharedRange<NTableClient::TUnversionedRow>> GetRows(); +}; + +//////////////////////////////////////////////////////////////////////////////// + +IRowBatchReaderPtr CreateRowBatchReader( + NConcurrency::IAsyncZeroCopyInputStreamPtr inputStream, + bool isStreamWithStatistics); + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NApi::NRpcProxy diff --git a/yt/yt/client/api/rpc_proxy/row_batch_writer.cpp b/yt/yt/client/api/rpc_proxy/row_batch_writer.cpp new file mode 100644 index 0000000000..7d1cbe5a4d --- /dev/null +++ b/yt/yt/client/api/rpc_proxy/row_batch_writer.cpp @@ -0,0 +1,68 @@ +#include "row_batch_writer.h" +#include "row_stream.h" +#include "wire_row_stream.h" + +#include <yt/yt/client/table_client/name_table.h> +#include <yt/yt/client/table_client/row_batch.h> + +#include <yt/yt/core/concurrency/async_stream.h> + +namespace NYT::NApi::NRpcProxy { + +using namespace NConcurrency; +using namespace NTableClient; + +//////////////////////////////////////////////////////////////////////////////// + +TRowBatchWriter::TRowBatchWriter(IAsyncZeroCopyOutputStreamPtr underlying) + : Underlying_(std::move(underlying)) + , Encoder_(CreateWireRowStreamEncoder(NameTable_)) +{ + YT_VERIFY(Underlying_); + NameTable_->SetEnableColumnNameValidation(); +} + +bool TRowBatchWriter::Write(TRange<TUnversionedRow> rows) +{ + YT_VERIFY(!Closed_); + YT_VERIFY(ReadyEvent_.IsSet() && ReadyEvent_.Get().IsOK()); + + auto batch = CreateBatchFromUnversionedRows(TSharedRange<TUnversionedRow>(rows, nullptr)); + + auto block = Encoder_->Encode(batch, nullptr); + + ReadyEvent_ = NewPromise<void>(); + ReadyEvent_.TrySetFrom(Underlying_->Write(std::move(block))); + + return ReadyEvent_.IsSet() && ReadyEvent_.Get().IsOK(); +} + +TFuture<void> TRowBatchWriter::GetReadyEvent() +{ + return ReadyEvent_; +} + +TFuture<void> TRowBatchWriter::Close() +{ + YT_VERIFY(!Closed_); + Closed_ = true; + + return Underlying_->Close(); +} + +const TNameTablePtr& TRowBatchWriter::GetNameTable() const +{ + return NameTable_; +} + +//////////////////////////////////////////////////////////////////////////////// + +IRowBatchWriterPtr CreateRowBatchWriter( + IAsyncZeroCopyOutputStreamPtr outputStream) +{ + return New<TRowBatchWriter>(std::move(outputStream)); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NApi::NRpcProxy diff --git a/yt/yt/client/api/rpc_proxy/row_batch_writer.h b/yt/yt/client/api/rpc_proxy/row_batch_writer.h new file mode 100644 index 0000000000..1476f08ca6 --- /dev/null +++ b/yt/yt/client/api/rpc_proxy/row_batch_writer.h @@ -0,0 +1,41 @@ +#pragma once + +#include "public.h" + +#include <yt/yt/client/api/row_batch_writer.h> + +namespace NYT::NApi::NRpcProxy { + +//////////////////////////////////////////////////////////////////////////////// + +class TRowBatchWriter + : public virtual IRowBatchWriter +{ +public: + explicit TRowBatchWriter(NConcurrency::IAsyncZeroCopyOutputStreamPtr underlying); + + bool Write(TRange<NTableClient::TUnversionedRow> rows) override; + + TFuture<void> GetReadyEvent() override; + + TFuture<void> Close() override; + + const NTableClient::TNameTablePtr& GetNameTable() const override; + +private: + const NConcurrency::IAsyncZeroCopyOutputStreamPtr Underlying_; + const NTableClient::TNameTablePtr NameTable_ = New<NTableClient::TNameTable>(); + const IRowStreamEncoderPtr Encoder_; + + TPromise<void> ReadyEvent_ = MakePromise<void>(TError()); + bool Closed_ = false; +}; + +//////////////////////////////////////////////////////////////////////////////// + +IRowBatchWriterPtr CreateRowBatchWriter( + NConcurrency::IAsyncZeroCopyOutputStreamPtr outputStream); + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NApi::NRpcProxy diff --git a/yt/yt/client/api/rpc_proxy/table_reader.cpp b/yt/yt/client/api/rpc_proxy/table_reader.cpp index 7bf5238da3..bb1f571332 100644 --- a/yt/yt/client/api/rpc_proxy/table_reader.cpp +++ b/yt/yt/client/api/rpc_proxy/table_reader.cpp @@ -1,20 +1,12 @@ #include "table_reader.h" +#include "row_batch_reader.h" #include "helpers.h" -#include "row_stream.h" -#include "wire_row_stream.h" -#include <yt/yt/client/api/rowset.h> #include <yt/yt/client/api/table_reader.h> +#include <yt/yt/client/table_client/schema.h> #include <yt/yt_proto/yt/client/chunk_client/proto/data_statistics.pb.h> -#include <yt/yt/client/table_client/name_table.h> -#include <yt/yt/client/table_client/unversioned_reader.h> -#include <yt/yt/client/table_client/unversioned_row.h> -#include <yt/yt/client/table_client/row_batch.h> - -#include <yt/yt/core/rpc/stream.h> - namespace NYT::NApi::NRpcProxy { using namespace NConcurrency; @@ -23,7 +15,8 @@ using namespace NTableClient; //////////////////////////////////////////////////////////////////////////////// class TTableReader - : public ITableReader + : public TRowBatchReader + , public ITableReader { public: TTableReader( @@ -31,19 +24,13 @@ public: i64 startRowIndex, const std::vector<TString>& omittedInaccessibleColumns, TTableSchemaPtr schema, - const NApi::NRpcProxy::NProto::TRowsetStatistics& statistics) - : Underlying_(std::move(underlying)) + const NProto::TRowsetStatistics& statistics) + : TRowBatchReader(std::move(underlying), /*isStreamWithStatistics*/ true) , StartRowIndex_(startRowIndex) , TableSchema_(std::move(schema)) , OmittedInaccessibleColumns_(omittedInaccessibleColumns) - , Decoder_(CreateWireRowStreamDecoder(NameTable_)) { - YT_VERIFY(Underlying_); - - ApplyReaderStatistics(statistics); - - RowsWithStatisticsFuture_ = GetRowsWithStatistics(); - ReadyEvent_.TrySetFrom(RowsWithStatisticsFuture_); + ApplyStatistics(statistics); } i64 GetStartRowIndex() const override @@ -65,76 +52,6 @@ public: return dataStatistics; } - TFuture<void> GetReadyEvent() const override - { - return ReadyEvent_; - } - - IUnversionedRowBatchPtr Read(const TRowBatchReadOptions& options) override - { - StoredRows_.clear(); - - if (!ReadyEvent_.IsSet() || !ReadyEvent_.Get().IsOK()) { - return CreateEmptyUnversionedRowBatch(); - } - - if (!Finished_) { - ReadyEvent_ = NewPromise<void>(); - } - - std::vector<TUnversionedRow> rows; - rows.reserve(options.MaxRowsPerRead); - i64 dataWeight = 0; - - while (RowsWithStatisticsFuture_ && - RowsWithStatisticsFuture_.IsSet() && - RowsWithStatisticsFuture_.Get().IsOK() && - !Finished_ && - std::ssize(rows) < options.MaxRowsPerRead && - dataWeight < options.MaxDataWeightPerRead) - { - const auto& currentRows = RowsWithStatisticsFuture_.Get().Value().Rows; - const auto& currentStatistics = RowsWithStatisticsFuture_.Get().Value().Statistics; - - if (currentRows.Empty()) { - ReadyEvent_.Set(); - Finished_ = true; - ApplyReaderStatistics(currentStatistics); - continue; - } - - while (CurrentRowsOffset_ < std::ssize(currentRows) && - std::ssize(rows) < options.MaxRowsPerRead && - dataWeight < options.MaxDataWeightPerRead) - { - auto row = currentRows[CurrentRowsOffset_++]; - rows.push_back(row); - dataWeight += GetDataWeight(row); - } - - StoredRows_.push_back(currentRows); - ApplyReaderStatistics(currentStatistics); - - if (CurrentRowsOffset_ == std::ssize(currentRows)) { - RowsWithStatisticsFuture_ = GetRowsWithStatistics(); - CurrentRowsOffset_ = 0; - } - } - - RowCount_ += rows.size(); - DataWeight_ += dataWeight; - - ReadyEvent_.TrySetFrom(RowsWithStatisticsFuture_); - return rows.empty() - ? nullptr - : CreateBatchFromUnversionedRows(MakeSharedRange(std::move(rows), MakeStrong(this))); - } - - const TNameTablePtr& GetNameTable() const override - { - return NameTable_; - } - const TTableSchemaPtr& GetTableSchema() const override { return TableSchema_; @@ -146,86 +63,18 @@ public: } private: - struct TRowsWithStatistics - { - TSharedRange<TUnversionedRow> Rows; - NApi::NRpcProxy::NProto::TRowsetStatistics Statistics; - }; - - const IAsyncZeroCopyInputStreamPtr Underlying_; const i64 StartRowIndex_; const TTableSchemaPtr TableSchema_; const std::vector<TString> OmittedInaccessibleColumns_; - const TNameTablePtr NameTable_ = New<TNameTable>(); - const IRowStreamDecoderPtr Decoder_; - NChunkClient::NProto::TDataStatistics DataStatistics_; i64 TotalRowCount_; - i64 RowCount_ = 0; - i64 DataWeight_ = 0; - - TNameTableToSchemaIdMapping IdMapping_; - - TPromise<void> ReadyEvent_ = NewPromise<void>(); - - std::vector<TSharedRange<TUnversionedRow>> StoredRows_; - TFuture<TRowsWithStatistics> RowsWithStatisticsFuture_; - i64 CurrentRowsOffset_ = 0; - - bool Finished_ = false; - - void ApplyReaderStatistics(const NApi::NRpcProxy::NProto::TRowsetStatistics& statistics) + void ApplyStatistics(const NProto::TRowsetStatistics& statistics) override { TotalRowCount_ = statistics.total_row_count(); DataStatistics_ = statistics.data_statistics(); } - - TFuture<TRowsWithStatistics> GetRowsWithStatistics() - { - return Underlying_->Read() - .Apply(BIND([this, weakThis = MakeWeak(this)] (const TSharedRef& block) { - auto this_ = weakThis.Lock(); - if (!this_) { - THROW_ERROR_EXCEPTION(NYT::EErrorCode::Canceled, "Reader destroyed"); - } - - NApi::NRpcProxy::NProto::TRowsetDescriptor descriptor; - NApi::NRpcProxy::NProto::TRowsetStatistics statistics; - auto payloadRef = DeserializeRowStreamBlockEnvelope(block, &descriptor, &statistics); - - ValidateRowsetDescriptor( - descriptor, - NApi::NRpcProxy::CurrentWireFormatVersion, - NApi::NRpcProxy::NProto::RK_UNVERSIONED, - NApi::NRpcProxy::NProto::ERowsetFormat::RF_YT_WIRE); - - auto decoder = GetOrCreateDecoder(descriptor.rowset_format()); - auto batch = decoder->Decode(payloadRef, descriptor); - auto rows = batch->MaterializeRows(); - auto rowsWithStatistics = TRowsWithStatistics{ - std::move(rows), - std::move(statistics) - }; - - if (rowsWithStatistics.Rows.Empty()) { - return ExpectEndOfStream(Underlying_).Apply(BIND([=] { - return std::move(rowsWithStatistics); - })); - } - return MakeFuture(std::move(rowsWithStatistics)); - })); - } - - IRowStreamDecoderPtr GetOrCreateDecoder(NApi::NRpcProxy::NProto::ERowsetFormat format) - { - if (format != NApi::NRpcProxy::NProto::RF_YT_WIRE) { - THROW_ERROR_EXCEPTION("Unsupported rowset format %Qv", - NApi::NRpcProxy::NProto::ERowsetFormat_Name(format)); - } - return Decoder_; - } }; TFuture<ITableReaderPtr> CreateTableReader(IAsyncZeroCopyInputStreamPtr inputStream) diff --git a/yt/yt/client/api/rpc_proxy/table_writer.cpp b/yt/yt/client/api/rpc_proxy/table_writer.cpp index 31197018f6..d957c67023 100644 --- a/yt/yt/client/api/rpc_proxy/table_writer.cpp +++ b/yt/yt/client/api/rpc_proxy/table_writer.cpp @@ -1,15 +1,8 @@ #include "table_writer.h" -#include "helpers.h" -#include "row_stream.h" -#include "wire_row_stream.h" +#include "row_batch_writer.h" #include <yt/yt/client/api/table_writer.h> - -#include <yt/yt/client/table_client/name_table.h> #include <yt/yt/client/table_client/schema.h> -#include <yt/yt/client/table_client/row_batch.h> - -#include <yt/yt/core/rpc/stream.h> namespace NYT::NApi::NRpcProxy { @@ -19,51 +12,16 @@ using namespace NTableClient; //////////////////////////////////////////////////////////////////////////////// class TTableWriter - : public ITableWriter + : public TRowBatchWriter + , public ITableWriter { public: TTableWriter( IAsyncZeroCopyOutputStreamPtr underlying, TTableSchemaPtr schema) - : Underlying_(std::move(underlying)) + : TRowBatchWriter(std::move(underlying)) , Schema_(std::move(schema)) - , Encoder_(CreateWireRowStreamEncoder(NameTable_)) { - YT_VERIFY(Underlying_); - NameTable_->SetEnableColumnNameValidation(); - } - - bool Write(TRange<TUnversionedRow> rows) override - { - YT_VERIFY(!Closed_); - YT_VERIFY(ReadyEvent_.IsSet() && ReadyEvent_.Get().IsOK()); - - auto batch = CreateBatchFromUnversionedRows(TSharedRange<TUnversionedRow>(rows, nullptr)); - - auto block = Encoder_->Encode(batch, nullptr); - - ReadyEvent_ = NewPromise<void>(); - ReadyEvent_.TrySetFrom(Underlying_->Write(std::move(block))); - - return ReadyEvent_.IsSet() && ReadyEvent_.Get().IsOK(); - } - - TFuture<void> GetReadyEvent() override - { - return ReadyEvent_; - } - - TFuture<void> Close() override - { - YT_VERIFY(!Closed_); - Closed_ = true; - - return Underlying_->Close(); - } - - const TNameTablePtr& GetNameTable() const override - { - return NameTable_; } const TTableSchemaPtr& GetSchema() const override @@ -72,14 +30,7 @@ public: } private: - const IAsyncZeroCopyOutputStreamPtr Underlying_; const TTableSchemaPtr Schema_; - - const TNameTablePtr NameTable_ = New<TNameTable>(); - const IRowStreamEncoderPtr Encoder_; - - TPromise<void> ReadyEvent_ = MakePromise<void>(TError()); - bool Closed_ = false; }; ITableWriterPtr CreateTableWriter( diff --git a/yt/yt/client/api/shuffle_client.h b/yt/yt/client/api/shuffle_client.h index eb07ccc6d2..d7972cd680 100644 --- a/yt/yt/client/api/shuffle_client.h +++ b/yt/yt/client/api/shuffle_client.h @@ -9,7 +9,7 @@ namespace NYT::NApi { struct TShuffleHandle : public NYTree::TYsonStruct { - TGuid TransactionId; + NObjectClient::TTransactionId TransactionId; TString CoordinatorAddress; TString Account; int PartitionCount; diff --git a/yt/yt/client/api/table_client.h b/yt/yt/client/api/table_client.h index f577d4a4b4..581412eba7 100644 --- a/yt/yt/client/api/table_client.h +++ b/yt/yt/client/api/table_client.h @@ -135,6 +135,7 @@ struct TAlterTableReplicaOptions std::optional<bool> PreserveTimestamps; std::optional<NTransactionClient::EAtomicity> Atomicity; std::optional<bool> EnableReplicatedTableTracker; + std::optional<TString> ReplicaPath; }; struct TGetTablePivotKeysOptions diff --git a/yt/yt/client/api/table_reader.h b/yt/yt/client/api/table_reader.h index 927f8dcb9f..b26e970657 100644 --- a/yt/yt/client/api/table_reader.h +++ b/yt/yt/client/api/table_reader.h @@ -10,7 +10,7 @@ namespace NYT::NApi { //////////////////////////////////////////////////////////////////////////////// struct ITableReader - : public IRowBatchReader + : public virtual IRowBatchReader { //! Returns the starting row index within the table. virtual i64 GetStartRowIndex() const = 0; diff --git a/yt/yt/client/api/table_writer.h b/yt/yt/client/api/table_writer.h index fd22dbd1a0..f4d42c3c78 100644 --- a/yt/yt/client/api/table_writer.h +++ b/yt/yt/client/api/table_writer.h @@ -8,7 +8,7 @@ namespace NYT::NApi { //////////////////////////////////////////////////////////////////////////////// struct ITableWriter - : public IRowBatchWriter + : public virtual IRowBatchWriter { //! Returns the schema to be used for constructing rows. virtual const NTableClient::TTableSchemaPtr& GetSchema() const = 0; diff --git a/yt/yt/client/driver/command-inl.h b/yt/yt/client/driver/command-inl.h index a27f748e71..4b595a57df 100644 --- a/yt/yt/client/driver/command-inl.h +++ b/yt/yt/client/driver/command-inl.h @@ -392,6 +392,14 @@ void TSelectRowsCommandBase< .GreaterThan(0) .Optional(/*init*/ false); + registrar.template ParameterWithUniversalAccessor<ui64>( + "min_row_count_per_subquery", + [] (TThis* command) -> auto& { + return command->Options.MinRowCountPerSubquery; + }) + .GreaterThan(0) + .Optional(/*init*/ false); + registrar.template ParameterWithUniversalAccessor<std::optional<TString>>( "udf_registry_path", [] (TThis* command) -> auto& { diff --git a/yt/yt/client/driver/proxy_discovery_cache.cpp b/yt/yt/client/driver/proxy_discovery_cache.cpp index 772f526231..a17894a93e 100644 --- a/yt/yt/client/driver/proxy_discovery_cache.cpp +++ b/yt/yt/client/driver/proxy_discovery_cache.cpp @@ -133,7 +133,7 @@ private: continue; } - if (proxyNode->Attributes().Get<TString>(RoleAttributeName, DefaultRpcProxyRole) != request.Role) { + if (proxyNode->Attributes().Get<std::string>(RoleAttributeName, DefaultRpcProxyRole) != request.Role) { continue; } diff --git a/yt/yt/client/driver/table_commands.cpp b/yt/yt/client/driver/table_commands.cpp index 9b5d7cae49..0ff27ef95f 100644 --- a/yt/yt/client/driver/table_commands.cpp +++ b/yt/yt/client/driver/table_commands.cpp @@ -1564,6 +1564,14 @@ void TAlterTableReplicaCommand::Register(TRegistrar registrar) return command->Options.EnableReplicatedTableTracker; }) .Optional(/*init*/ false); + + registrar.ParameterWithUniversalAccessor<std::optional<TString>>( + "replica_path", + [] (TThis* command) -> auto& { + return command->Options.ReplicaPath; + }) + .Optional(/*init*/ false); + } void TAlterTableReplicaCommand::DoExecute(ICommandContextPtr context) diff --git a/yt/yt/client/node_tracker_client/node_directory.cpp b/yt/yt/client/node_tracker_client/node_directory.cpp index 0ef4ee4561..7d377febbc 100644 --- a/yt/yt/client/node_tracker_client/node_directory.cpp +++ b/yt/yt/client/node_tracker_client/node_directory.cpp @@ -58,9 +58,9 @@ namespace { constexpr int TypicalTagCount = 16; // Cf. YT-10645 -TCompactVector<TStringBuf, TypicalTagCount> GetSortedTags(const std::vector<TString>& tags) +TCompactVector<std::string, TypicalTagCount> GetSortedTags(const std::vector<std::string>& tags) { - TCompactVector<TStringBuf, TypicalTagCount> result; + TCompactVector<std::string, TypicalTagCount> result; result.reserve(tags.size()); for (const auto& tag : tags) { result.push_back(tag); @@ -94,7 +94,7 @@ TNodeDescriptor::TNodeDescriptor( const std::optional<std::string>& host, const std::optional<std::string>& rack, const std::optional<std::string>& dc, - const std::vector<TString>& tags, + const std::vector<std::string>& tags, std::optional<TInstant> lastSeenTime) : Addresses_(std::move(addresses)) , DefaultAddress_(NNodeTrackerClient::GetDefaultAddress(Addresses_)) @@ -145,7 +145,7 @@ const std::optional<std::string>& TNodeDescriptor::GetDataCenter() const return DataCenter_; } -const std::vector<TString>& TNodeDescriptor::GetTags() const +const std::vector<std::string>& TNodeDescriptor::GetTags() const { return Tags_; } @@ -380,7 +380,7 @@ void FromProto(NNodeTrackerClient::TNodeDescriptor* descriptor, const NNodeTrack protoDescriptor.has_host() ? std::make_optional(protoDescriptor.host()) : std::nullopt, protoDescriptor.has_rack() ? std::make_optional(protoDescriptor.rack()) : std::nullopt, protoDescriptor.has_data_center() ? std::make_optional(protoDescriptor.data_center()) : std::nullopt, - FromProto<std::vector<TString>>(protoDescriptor.tags()), + FromProto<std::vector<std::string>>(protoDescriptor.tags()), protoDescriptor.has_last_seen_time() ? std::make_optional(FromProto<TInstant>(protoDescriptor.last_seen_time())) : std::nullopt); } @@ -434,7 +434,7 @@ bool operator == (const TNodeDescriptor& lhs, const NProto::TNodeDescriptor& rhs } const auto& lhsTags = lhs.GetTags(); - auto rhsTags = FromProto<std::vector<TString>>(rhs.tags()); + auto rhsTags = FromProto<std::vector<std::string>>(rhs.tags()); if (GetSortedTags(lhsTags) != GetSortedTags(rhsTags)) { return false; } diff --git a/yt/yt/client/node_tracker_client/node_directory.h b/yt/yt/client/node_tracker_client/node_directory.h index f8ebc1ba0c..eafef00686 100644 --- a/yt/yt/client/node_tracker_client/node_directory.h +++ b/yt/yt/client/node_tracker_client/node_directory.h @@ -35,7 +35,7 @@ public: const std::optional<std::string>& host = {}, const std::optional<std::string>& rack = {}, const std::optional<std::string>& dc = {}, - const std::vector<TString>& tags = {}, + const std::vector<std::string>& tags = {}, std::optional<TInstant> lastSeenTime = {}); TNodeDescriptor& operator=(const TNodeDescriptor& other) = default; @@ -55,7 +55,7 @@ public: const std::optional<std::string>& GetRack() const; const std::optional<std::string>& GetDataCenter() const; - const std::vector<TString>& GetTags() const; + const std::vector<std::string>& GetTags() const; //! GetLastSeenTime returns last instant when node was seen online on some master. /*! @@ -76,7 +76,7 @@ private: std::optional<std::string> Host_; std::optional<std::string> Rack_; std::optional<std::string> DataCenter_; - std::vector<TString> Tags_; + std::vector<std::string> Tags_; // Not persisted. mutable TCopyableAtomic<TCpuInstant> LastSeenTime_; diff --git a/yt/yt/client/object_client/helpers-inl.h b/yt/yt/client/object_client/helpers-inl.h index 1aaa08b251..9d487edb19 100644 --- a/yt/yt/client/object_client/helpers-inl.h +++ b/yt/yt/client/object_client/helpers-inl.h @@ -123,14 +123,14 @@ inline TObjectId MakeSequoiaId( inline TObjectId MakeWellKnownId( EObjectType type, TCellTag cellTag, - ui64 counter /*= 0xffffffffffffffff*/) + ui64 counter) { YT_VERIFY(counter & WellKnownCounterMask); return MakeId( type, cellTag, counter, - static_cast<ui32>(cellTag.Underlying() * 901517) ^ 0x140a8383); + static_cast<ui32>(static_cast<ui32>(cellTag.Underlying()) * 901517ULL ^ 0x140a8383ULL)); } inline TObjectId MakeSchemaObjectId( diff --git a/yt/yt/client/query_client/query_builder.cpp b/yt/yt/client/query_client/query_builder.cpp index 0c2100e899..4cdce71596 100644 --- a/yt/yt/client/query_client/query_builder.cpp +++ b/yt/yt/client/query_client/query_builder.cpp @@ -67,6 +67,11 @@ void TQueryBuilder::AddGroupByExpression(TString expression, TString alias) }); } +void TQueryBuilder::SetWithTotals(EWithTotalsMode withTotalsMode) +{ + WithTotalsMode_ = withTotalsMode; +} + void TQueryBuilder::AddHavingConjunct(TString expression) { HavingConjuncts_.push_back(std::move(expression)); @@ -151,6 +156,10 @@ TString TQueryBuilder::Build() parts.push_back(JoinSeq(", ", GroupByEntries_)); } + if (WithTotalsMode_ == EWithTotalsMode::BeforeHaving) { + parts.push_back("WITH TOTALS"); + } + if (!HavingConjuncts_.empty()) { if (GroupByEntries_.empty()) { THROW_ERROR_EXCEPTION("Having without group by is not valid"); @@ -159,6 +168,10 @@ TString TQueryBuilder::Build() parts.push_back(JoinSeq(" AND ", Parenthesize(HavingConjuncts_))); } + if (WithTotalsMode_ == EWithTotalsMode::AfterHaving) { + parts.push_back("WITH TOTALS"); + } + if (!OrderByEntries_.empty()) { parts.push_back("ORDER BY"); parts.push_back(JoinSeq(", ", OrderByEntries_)); diff --git a/yt/yt/client/query_client/query_builder.h b/yt/yt/client/query_client/query_builder.h index ab509d970c..56ec58458a 100644 --- a/yt/yt/client/query_client/query_builder.h +++ b/yt/yt/client/query_client/query_builder.h @@ -18,6 +18,12 @@ DEFINE_ENUM(ETableJoinType, (Left) ); +DEFINE_ENUM(EWithTotalsMode, + (None) + (BeforeHaving) + (AfterHaving) +); + //////////////////////////////////////////////////////////////////////////////// class TQueryBuilder @@ -34,6 +40,8 @@ public: void AddGroupByExpression(TString expression); void AddGroupByExpression(TString expression, TString alias); + void SetWithTotals(EWithTotalsMode withTotalsMode); + void AddHavingConjunct(TString expression); void AddOrderByExpression(TString expression); @@ -76,6 +84,7 @@ private: std::vector<TString> WhereConjuncts_; std::vector<TOrderByEntry> OrderByEntries_; std::vector<TEntryWithAlias> GroupByEntries_; + EWithTotalsMode WithTotalsMode_ = EWithTotalsMode::None; std::vector<TString> HavingConjuncts_; std::vector<TJoinEntry> JoinEntries_; std::optional<i64> Limit_; diff --git a/yt/yt/client/scheduler/public.h b/yt/yt/client/scheduler/public.h index d07e8469df..d92ae73f78 100644 --- a/yt/yt/client/scheduler/public.h +++ b/yt/yt/client/scheduler/public.h @@ -133,6 +133,7 @@ DEFINE_ENUM(EAbortReason, ((UnresolvedNodeId) ( 53)) ((RootVolumePreparationFailed) ( 54)) ((InterruptionFailed) ( 55)) + ((OperationIncarnationChanged) ( 56)) ((SchedulingFirst) (100)) ((SchedulingTimeout) (101)) ((SchedulingResourceOvercommit) (102)) diff --git a/yt/yt/client/table_client/adapters.cpp b/yt/yt/client/table_client/adapters.cpp index 0309bbbc48..43ac145369 100644 --- a/yt/yt/client/table_client/adapters.cpp +++ b/yt/yt/client/table_client/adapters.cpp @@ -1,4 +1,5 @@ #include "adapters.h" +#include "private.h" #include "row_batch.h" @@ -21,7 +22,7 @@ using NProfiling::TWallTimer; //////////////////////////////////////////////////////////////////////////////// -YT_DEFINE_GLOBAL(const NLogging::TLogger, Logger, "TableClientAdapters"); +static constexpr auto& Logger = TableClientLogger; //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/client/table_client/check_schema_compatibility.cpp b/yt/yt/client/table_client/check_schema_compatibility.cpp index a2297bf05d..d2fd29b903 100644 --- a/yt/yt/client/table_client/check_schema_compatibility.cpp +++ b/yt/yt/client/table_client/check_schema_compatibility.cpp @@ -2,6 +2,7 @@ #include "logical_type.h" #include "schema.h" #include "comparator.h" +#include "versioned_io_options.h" #include <yt/yt/client/complex_types/check_type_compatibility.h> @@ -27,6 +28,14 @@ std::pair<ESchemaCompatibility, TError> CheckTableSchemaCompatibilityImpl( for (const auto& inputColumn : inputSchema.Columns()) { if (!outputSchema.FindColumn(inputColumn.Name())) { + if (options.AllowTimestampColumns) { + if (auto originalColumnName = GetTimestampColumnOriginalNameOrNull(inputColumn.Name())) { + if (outputSchema.FindColumn(*originalColumnName)) { + continue; + } + } + } + return { ESchemaCompatibility::Incompatible, TError("Column %v is found in input schema but is missing in output schema", diff --git a/yt/yt/client/table_client/check_schema_compatibility.h b/yt/yt/client/table_client/check_schema_compatibility.h index 2fbb823ff0..78ebb088cc 100644 --- a/yt/yt/client/table_client/check_schema_compatibility.h +++ b/yt/yt/client/table_client/check_schema_compatibility.h @@ -11,6 +11,7 @@ struct TTableSchemaCompatibilityOptions bool IgnoreSortOrder = false; bool ForbidExtraComputedColumns = true; bool IgnoreStableNamesDifference = false; + bool AllowTimestampColumns = false; }; // Validates that values from table with inputSchema also match outputSchema. diff --git a/yt/yt/client/table_client/comparator.cpp b/yt/yt/client/table_client/comparator.cpp index a6037004f3..3e18b63848 100644 --- a/yt/yt/client/table_client/comparator.cpp +++ b/yt/yt/client/table_client/comparator.cpp @@ -1,6 +1,7 @@ #include "comparator.h" #include "key_bound.h" +#include "private.h" #include "serialize.h" #include <yt/yt/core/logging/log.h> @@ -15,7 +16,7 @@ using namespace NYson; using namespace NYTree; //! Used only for YT_LOG_FATAL below. -YT_DEFINE_GLOBAL(const NLogging::TLogger, Logger, "TableClientComparator"); +static constexpr auto& Logger = TableClientLogger; //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/client/table_client/composite_compare.cpp b/yt/yt/client/table_client/composite_compare.cpp index b41b0cfe0e..168a311a99 100644 --- a/yt/yt/client/table_client/composite_compare.cpp +++ b/yt/yt/client/table_client/composite_compare.cpp @@ -1,5 +1,7 @@ #include "composite_compare.h" +#include "private.h" + #include <yt/yt/client/table_client/row_base.h> #include <yt/yt/core/yson/pull_parser.h> @@ -20,7 +22,7 @@ using namespace NYson; //////////////////////////////////////////////////////////////////////////////// -YT_DEFINE_GLOBAL(const NLogging::TLogger, Logger, "YsonCompositeCompare"); +static constexpr auto& Logger = TableClientLogger; //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/client/table_client/config.cpp b/yt/yt/client/table_client/config.cpp index a580572880..96ff8b5af6 100644 --- a/yt/yt/client/table_client/config.cpp +++ b/yt/yt/client/table_client/config.cpp @@ -467,6 +467,8 @@ void TChunkWriterOptions::Register(TRegistrar registrar) registrar.Parameter("schema_modification", &TThis::SchemaModification) .Default(ETableSchemaModification::None); + registrar.Parameter("versioned_write_options", &TThis::VersionedWriteOptions) + .Default(); registrar.Parameter("max_heavy_columns", &TThis::MaxHeavyColumns) .Default(0); diff --git a/yt/yt/client/table_client/config.h b/yt/yt/client/table_client/config.h index e1bc37043d..a8accf974b 100644 --- a/yt/yt/client/table_client/config.h +++ b/yt/yt/client/table_client/config.h @@ -1,6 +1,7 @@ #pragma once #include "public.h" +#include "versioned_io_options.h" #include <yt/yt/client/chunk_client/config.h> @@ -433,6 +434,8 @@ public: ETableSchemaModification SchemaModification; + TVersionedWriteOptions VersionedWriteOptions; + EOptimizeFor OptimizeFor; std::optional<NChunkClient::EChunkFormat> ChunkFormat; NChunkClient::EChunkFormat GetEffectiveChunkFormat(bool versioned) const; diff --git a/yt/yt/client/table_client/key.cpp b/yt/yt/client/table_client/key.cpp index 7ba78aa88e..fc693f865a 100644 --- a/yt/yt/client/table_client/key.cpp +++ b/yt/yt/client/table_client/key.cpp @@ -1,5 +1,6 @@ #include "key.h" +#include "private.h" #include "serialize.h" #include <yt/yt/core/ytree/fluent.h> @@ -12,8 +13,8 @@ using namespace NYson; //////////////////////////////////////////////////////////////////////////////// -//! Used only for YT_LOG_FATAL below. -YT_DEFINE_GLOBAL(const NLogging::TLogger, Logger, "TableClientKey"); +//! Used only for YT_LOG_FATAL below in debug mode. +[[maybe_unused]] static constexpr auto& Logger = TableClientLogger; //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/client/table_client/key_bound.cpp b/yt/yt/client/table_client/key_bound.cpp index e02b894afd..e666e0d532 100644 --- a/yt/yt/client/table_client/key_bound.cpp +++ b/yt/yt/client/table_client/key_bound.cpp @@ -1,6 +1,7 @@ #include "key_bound.h" #include "helpers.h" +#include "private.h" #include "row_buffer.h" #include "serialize.h" @@ -15,7 +16,7 @@ using namespace NLogging; //////////////////////////////////////////////////////////////////////////////// //! Used only for YT_LOG_FATAL below. -YT_DEFINE_GLOBAL(const NLogging::TLogger, Logger, "TableClientKey"); +[[maybe_unused]] static constexpr auto& Logger = TableClientLogger; //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/client/table_client/private.h b/yt/yt/client/table_client/private.h new file mode 100644 index 0000000000..56efb84387 --- /dev/null +++ b/yt/yt/client/table_client/private.h @@ -0,0 +1,15 @@ +#pragma once + +#include "public.h" + +#include <yt/yt/core/logging/log.h> + +namespace NYT::NTableClient { + +//////////////////////////////////////////////////////////////////////////////// + +YT_DEFINE_GLOBAL(const NLogging::TLogger, TableClientLogger, "TableClient"); + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NTableClient diff --git a/yt/yt/client/table_client/public.h b/yt/yt/client/table_client/public.h index 45a241afff..e52eaecf48 100644 --- a/yt/yt/client/table_client/public.h +++ b/yt/yt/client/table_client/public.h @@ -47,6 +47,7 @@ class TColumnMetaExt; class TVersionedRowDigestExt; class TCompressionDictionaryExt; class TVersionedReadOptions; +class TVersionedWriteOptions; } // namespace NProto @@ -443,6 +444,7 @@ static_assert(sizeof(TDynamicTableKeyMask) * 8 == MaxKeyColumnCountInDynamicTabl using TUUComparerSignature = int(const TUnversionedValue*, const TUnversionedValue*, int); struct TVersionedReadOptions; +struct TVersionedWriteOptions; //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/client/table_client/schema.cpp b/yt/yt/client/table_client/schema.cpp index ac530ed758..f3762d0951 100644 --- a/yt/yt/client/table_client/schema.cpp +++ b/yt/yt/client/table_client/schema.cpp @@ -4,6 +4,7 @@ #include "comparator.h" #include "logical_type.h" #include "unversioned_row.h" +#include "versioned_io_options.h" #include <optional> @@ -518,7 +519,11 @@ TColumnStableName TTableSchema::TNameMapping::NameToStableName(TStringBuf name) auto* column = Schema_.FindColumn(name); if (!column) { if (Schema_.GetStrict()) { - THROW_ERROR_EXCEPTION("No column with name %Qv in strict schema", name); + if (auto originalColumnName = GetTimestampColumnOriginalNameOrNull(name); + !originalColumnName || !Schema_.FindColumn(*originalColumnName)) + { + THROW_ERROR_EXCEPTION("No column with name %Qv in strict schema", name); + } } return TColumnStableName(TString(name)); } diff --git a/yt/yt/client/table_client/versioned_io_options.cpp b/yt/yt/client/table_client/versioned_io_options.cpp index 69bd8fceff..7a067d8a96 100644 --- a/yt/yt/client/table_client/versioned_io_options.cpp +++ b/yt/yt/client/table_client/versioned_io_options.cpp @@ -12,6 +12,12 @@ void TVersionedReadOptions::Register(TRegistrar registrar) .Default(EVersionedIOMode::Default); } +void TVersionedWriteOptions::Register(TRegistrar registrar) +{ + registrar.Parameter("write_mode", &TThis::WriteMode) + .Default(EVersionedIOMode::Default); +} + void ToProto( NProto::TVersionedReadOptions* protoOptions, const TVersionedReadOptions& options) @@ -26,6 +32,20 @@ void FromProto( options->ReadMode = CheckedEnumCast<EVersionedIOMode>(protoOptions.read_mode()); } +void ToProto( + NProto::TVersionedWriteOptions* protoOptions, + const NTableClient::TVersionedWriteOptions& options) +{ + protoOptions->set_write_mode(static_cast<i32>(options.WriteMode)); +} + +void FromProto( + NTableClient::TVersionedWriteOptions* options, + const NProto::TVersionedWriteOptions& protoOptions) +{ + options->WriteMode = CheckedEnumCast<EVersionedIOMode>(protoOptions.write_mode()); +} + std::optional<TString> GetTimestampColumnOriginalNameOrNull(TStringBuf name) { auto prefixEnd = name.begin() + ssize(TimestampColumnPrefix); diff --git a/yt/yt/client/table_client/versioned_io_options.h b/yt/yt/client/table_client/versioned_io_options.h index 6e1f2f7a7b..219c97a064 100644 --- a/yt/yt/client/table_client/versioned_io_options.h +++ b/yt/yt/client/table_client/versioned_io_options.h @@ -23,6 +23,16 @@ struct TVersionedReadOptions static void Register(TRegistrar registrar); }; +struct TVersionedWriteOptions + : public NYTree::TYsonStructLite +{ + EVersionedIOMode WriteMode; + + REGISTER_YSON_STRUCT_LITE(TVersionedWriteOptions); + + static void Register(TRegistrar registrar); +}; + void ToProto( NProto::TVersionedReadOptions* protoOptions, const NTableClient::TVersionedReadOptions& options); @@ -31,6 +41,14 @@ void FromProto( NTableClient::TVersionedReadOptions* options, const NProto::TVersionedReadOptions& protoOptions); +void ToProto( + NProto::TVersionedWriteOptions* protoOptions, + const NTableClient::TVersionedWriteOptions& options); + +void FromProto( + NTableClient::TVersionedWriteOptions* options, + const NProto::TVersionedWriteOptions& protoOptions); + std::optional<TString> GetTimestampColumnOriginalNameOrNull(TStringBuf name); //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/client/table_client/wire_protocol.cpp b/yt/yt/client/table_client/wire_protocol.cpp index c4b11c926b..d473573986 100644 --- a/yt/yt/client/table_client/wire_protocol.cpp +++ b/yt/yt/client/table_client/wire_protocol.cpp @@ -1,5 +1,7 @@ #include "wire_protocol.h" +#include "private.h" + #include <yt/yt_proto/yt/client/table_chunk_format/proto/chunk_meta.pb.h> #include <yt/yt/client/table_client/row_buffer.h> #include <yt/yt/client/table_client/schema.h> @@ -36,7 +38,7 @@ using NCrypto::TMD5Hash; //////////////////////////////////////////////////////////////////////////////// -const static NLogging::TLogger Logger("WireProtocol"); +static constexpr auto& Logger = TableClientLogger; struct TWireProtocolWriterTag { }; diff --git a/yt/yt/client/unittests/connection_ut.cpp b/yt/yt/client/unittests/connection_ut.cpp index 336d1a5b6d..8e9a0dc10e 100644 --- a/yt/yt/client/unittests/connection_ut.cpp +++ b/yt/yt/client/unittests/connection_ut.cpp @@ -23,13 +23,13 @@ TEST_F(TProxyUrlTest, ParseProxyUrlAliasingRules) TEST_F(TProxyUrlTest, ApplyProxyUrlAliasingRules) { { - TString url = "markov"; + std::string url = "markov"; ApplyProxyUrlAliasingRules(url, THashMap<std::string, std::string>({{"primary", "localhost:12345"}})); ASSERT_EQ(url, "markov"); } // See ENV in ya.make { - TString url = "primary"; + std::string url = "primary"; ApplyProxyUrlAliasingRules(url, THashMap<std::string, std::string>({{"primary", "localhost:12345"}})); ASSERT_EQ(url, "localhost:12345"); } diff --git a/yt/yt/client/unittests/query_builder_ut.cpp b/yt/yt/client/unittests/query_builder_ut.cpp index 2a6313c495..687f95d5c0 100644 --- a/yt/yt/client/unittests/query_builder_ut.cpp +++ b/yt/yt/client/unittests/query_builder_ut.cpp @@ -27,6 +27,8 @@ TEST(TQueryBuilderTest, Simple) b.AddGroupByExpression("x + y * z", "group_expr"); b.AddGroupByExpression("x - 1"); + b.SetWithTotals(EWithTotalsMode::BeforeHaving); + b.AddHavingConjunct("group_expr > 42"); b.AddHavingConjunct("group_expr < 420"); @@ -46,6 +48,7 @@ TEST(TQueryBuilderTest, Simple) "LEFT JOIN [table2] AS [lookup2] ON (idx) = (lookup2.idx) " "WHERE (x > y_alias) AND (y = 177 OR y % 2 = 0) " "GROUP BY (x + y * z) AS group_expr, (x - 1) " + "WITH TOTALS " "HAVING (group_expr > 42) AND (group_expr < 420) " "ORDER BY (z) ASC, (x) DESC, (x + y) DESC, (z - y_alias) " "LIMIT 43"); @@ -62,6 +65,23 @@ TEST(TQueryBuilderTest, SourceAlias) "FROM [//t] AS t_alias"); } +TEST(TQueryBuilderTest, TotalsAfterHaving) +{ + TQueryBuilder b; + b.AddSelectExpression("x"); + b.SetSource("//t"); + b.AddGroupByExpression("x - 1"); + b.SetWithTotals(EWithTotalsMode::AfterHaving); + b.AddHavingConjunct("group_expr > 42"); + + EXPECT_EQ(b.Build(), + "(x) " + "FROM [//t] " + "GROUP BY (x - 1) " + "HAVING (group_expr > 42) " + "WITH TOTALS"); +} + //////////////////////////////////////////////////////////////////////////////// } // namespace diff --git a/yt/yt/client/ya.make b/yt/yt/client/ya.make index ec0e33b3a8..a0cc4caf88 100644 --- a/yt/yt/client/ya.make +++ b/yt/yt/client/ya.make @@ -51,7 +51,9 @@ SRCS( api/rpc_proxy/timestamp_provider.cpp api/rpc_proxy/transaction.cpp api/rpc_proxy/transaction_impl.cpp + api/rpc_proxy/row_batch_reader.cpp api/rpc_proxy/row_stream.cpp + api/rpc_proxy/row_batch_writer.cpp api/rpc_proxy/wire_row_stream.cpp bundle_controller_client/bundle_controller_client.cpp diff --git a/yt/yt/client/ypath/rich.cpp b/yt/yt/client/ypath/rich.cpp index b275c2ea04..ddd386df64 100644 --- a/yt/yt/client/ypath/rich.cpp +++ b/yt/yt/client/ypath/rich.cpp @@ -686,6 +686,11 @@ TVersionedReadOptions TRichYPath::GetVersionedReadOptions() const return GetAttribute(*this, "versioned_read_options", TVersionedReadOptions()); } +TVersionedWriteOptions TRichYPath::GetVersionedWriteOptions() const +{ + return GetAttribute(*this, "versioned_write_options", TVersionedWriteOptions()); +} + //////////////////////////////////////////////////////////////////////////////// TString ConvertToString(const TRichYPath& path, EYsonFormat ysonFormat) @@ -787,6 +792,7 @@ const std::vector<TString>& GetWellKnownRichYPathAttributes() "create", "read_via_exec_node", "versioned_read_options", + "versioned_write_options", }; return WellKnownAttributes; } diff --git a/yt/yt/client/ypath/rich.h b/yt/yt/client/ypath/rich.h index 0e483b4838..b6d7d60ccf 100644 --- a/yt/yt/client/ypath/rich.h +++ b/yt/yt/client/ypath/rich.h @@ -170,6 +170,9 @@ public: // "versioned_read_options" NTableClient::TVersionedReadOptions GetVersionedReadOptions() const; + // "versioned_write_options" + NTableClient::TVersionedWriteOptions GetVersionedWriteOptions() const; + private: TYPath Path_; NYTree::IAttributeDictionaryPtr Attributes_; diff --git a/yt/yt/core/bus/tcp/connection.cpp b/yt/yt/core/bus/tcp/connection.cpp index b71fbb9363..83faf6cb42 100644 --- a/yt/yt/core/bus/tcp/connection.cpp +++ b/yt/yt/core/bus/tcp/connection.cpp @@ -621,18 +621,18 @@ void TTcpConnection::ConnectSocket(const TNetworkAddress& address) DialerSession_->Dial(); } -void TTcpConnection::OnDialerFinished(const TErrorOr<SOCKET>& socketOrError) +void TTcpConnection::OnDialerFinished(const TErrorOr<TFileDescriptor>& fdOrError) { YT_LOG_DEBUG("Dialer finished"); DialerSession_.Reset(); - if (!socketOrError.IsOK()) { + if (!fdOrError.IsOK()) { Abort(TError( NBus::EErrorCode::TransportError, "Error connecting to %v", EndpointDescription_) - << socketOrError); + << fdOrError); return; } @@ -643,7 +643,7 @@ void TTcpConnection::OnDialerFinished(const TErrorOr<SOCKET>& socketOrError) return; } - Socket_ = socketOrError.Value(); + Socket_ = fdOrError.Value(); auto tosLevel = TosLevel_.load(); if (tosLevel != DefaultTosLevel) { diff --git a/yt/yt/core/bus/tcp/connection.h b/yt/yt/core/bus/tcp/connection.h index d5d631f3b2..2948d20b67 100644 --- a/yt/yt/core/bus/tcp/connection.h +++ b/yt/yt/core/bus/tcp/connection.h @@ -297,7 +297,7 @@ private: int GetSocketPort(); void ConnectSocket(const NNet::TNetworkAddress& address); - void OnDialerFinished(const TErrorOr<SOCKET>& socketOrError); + void OnDialerFinished(const TErrorOr<TFileDescriptor>& fdOrError); void ResolveAddress(); void OnAddressResolveFinished(const TErrorOr<NNet::TNetworkAddress>& result); diff --git a/yt/yt/core/concurrency/fiber_scheduler_thread.cpp b/yt/yt/core/concurrency/fiber_scheduler_thread.cpp index 1add139c3f..4c1509d8ca 100644 --- a/yt/yt/core/concurrency/fiber_scheduler_thread.cpp +++ b/yt/yt/core/concurrency/fiber_scheduler_thread.cpp @@ -1091,13 +1091,13 @@ YT_DEFINE_THREAD_LOCAL(TFiberId, CurrentFiberId); TFiberId GetCurrentFiberId() { - NYT::NOrigin::EnableOriginOverrides(); + NYT::NDetail::EnableErrorOriginOverrides(); return CurrentFiberId(); } void SetCurrentFiberId(TFiberId id) { - NYT::NOrigin::EnableOriginOverrides(); + NYT::NDetail::EnableErrorOriginOverrides(); CurrentFiberId() = id; } diff --git a/yt/yt/core/concurrency/new_fair_share_thread_pool.cpp b/yt/yt/core/concurrency/new_fair_share_thread_pool.cpp index d27a146059..3437188486 100644 --- a/yt/yt/core/concurrency/new_fair_share_thread_pool.cpp +++ b/yt/yt/core/concurrency/new_fair_share_thread_pool.cpp @@ -443,11 +443,11 @@ public: } // Detach under lock. - auto* poolDangerousPtr = bucket->Pool.Release(); + auto* rawPool = bucket->Pool.Release(); // Do not want use NewWithDeleter and keep pointer to TTwoLevelFairShareQueue in each execution pool. - if (NYT::GetRefCounter(poolDangerousPtr)->Unref(1)) { - auto poolsToRemove = DetachPool(poolDangerousPtr); + if (NYT::GetRefCounter(rawPool)->Unref(1)) { + auto poolsToRemove = DetachPool(rawPool); guard.Release(); while (poolsToRemove.GetSize() > 0) { @@ -466,19 +466,21 @@ public: if (!inserted) { YT_ASSERT(mappingIt->second->PoolName == poolName); - auto* pool = mappingIt->second; + auto* rawPool = mappingIt->second; // If RetainPoolQueue_ contains only one element its LinkedListNode will be null. // Determine that pool is in RetainPoolQueue_ by checking its ref count. - if (NYT::GetRefCounter(pool)->GetRefCount() == 0) { - RetainPoolQueue_.Remove(pool); - pool->LinkedListNode = {}; + if (NYT::GetRefCounter(rawPool)->GetRefCount() == 0) { + RetainPoolQueue_.Remove(rawPool); + rawPool->LinkedListNode = {}; - YT_LOG_TRACE("Restoring pool (PoolName: %v)", pool->PoolName); + YT_LOG_TRACE("Restoring pool (PoolName: %v)", rawPool->PoolName); } - YT_LOG_TRACE("Reusing pool (PoolName: %v)", pool->PoolName); + YT_LOG_TRACE("Reusing pool (PoolName: %v)", rawPool->PoolName); - return pool; + // NB: Strong RC could be zero, see above. + NYT::GetRefCounter(rawPool)->DangerousRef(); + return TExecutionPoolPtr(rawPool, /*addReference*/ false); } else { YT_LOG_TRACE("Creating pool (PoolName: %v)", poolName); auto pool = New<TExecutionPool>(poolName, GetPoolProfiler(poolName)); diff --git a/yt/yt/core/crypto/tls.cpp b/yt/yt/core/crypto/tls.cpp index ee3d1399b6..38779e21a4 100644 --- a/yt/yt/core/crypto/tls.cpp +++ b/yt/yt/core/crypto/tls.cpp @@ -57,7 +57,7 @@ constexpr auto TlsBufferSize = 1_MB; struct TSslContextImpl : public TRefCounted { - SSL_CTX* Ctx = nullptr; + SSL_CTX* Context = nullptr; TSslContextImpl() { @@ -66,36 +66,36 @@ struct TSslContextImpl ~TSslContextImpl() { - if (Ctx) { - SSL_CTX_free(Ctx); + if (Context) { + SSL_CTX_free(Context); } - if (ActiveCtx_) { - SSL_CTX_free(ActiveCtx_); + if (ActiveContext_) { + SSL_CTX_free(ActiveContext_); } } void Reset() { - if (Ctx) { - SSL_CTX_free(Ctx); + if (Context) { + SSL_CTX_free(Context); } #if OPENSSL_VERSION_NUMBER >= 0x10100000L - Ctx = SSL_CTX_new(TLS_method()); - if (!Ctx) { + Context = SSL_CTX_new(TLS_method()); + if (!Context) { THROW_ERROR_EXCEPTION("SSL_CTX_new(TLS_method()) failed") << GetSslErrors(); } - if (SSL_CTX_set_min_proto_version(Ctx, TLS1_2_VERSION) == 0) { + if (SSL_CTX_set_min_proto_version(Context, TLS1_2_VERSION) == 0) { THROW_ERROR_EXCEPTION("SSL_CTX_set_min_proto_version failed") << GetSslErrors(); } - if (SSL_CTX_set_max_proto_version(Ctx, TLS1_2_VERSION) == 0) { + if (SSL_CTX_set_max_proto_version(Context, TLS1_2_VERSION) == 0) { THROW_ERROR_EXCEPTION("SSL_CTX_set_max_proto_version failed") << GetSslErrors(); } #else Ctx = SSL_CTX_new(TLSv1_2_method()); - if (!Ctx) { + if (!Context) { THROW_ERROR_EXCEPTION("SSL_CTX_new(TLSv1_2_method()) failed") << GetSslErrors(); } @@ -104,17 +104,17 @@ struct TSslContextImpl void Commit(TInstant time) { - SSL_CTX* oldCtx; - YT_ASSERT(Ctx); + SSL_CTX* oldContext; + YT_ASSERT(Context); { auto guard = WriterGuard(Lock_); - oldCtx = ActiveCtx_; - ActiveCtx_ = Ctx; - Ctx = nullptr; + oldContext = ActiveContext_; + ActiveContext_ = Context; + Context = nullptr; CommitTime_ = time; } - if (oldCtx) { - SSL_CTX_free(oldCtx); + if (oldContext) { + SSL_CTX_free(oldContext); } } @@ -127,19 +127,19 @@ struct TSslContextImpl SSL* NewSsl() { auto guard = ReaderGuard(Lock_); - YT_ASSERT(ActiveCtx_); - return SSL_new(ActiveCtx_); + YT_ASSERT(ActiveContext_); + return SSL_new(ActiveContext_); } bool IsActive(const SSL* ssl) { auto guard = ReaderGuard(Lock_); - return SSL_get_SSL_CTX(ssl) == ActiveCtx_; + return SSL_get_SSL_CTX(ssl) == ActiveContext_; } private: YT_DECLARE_SPIN_LOCK(NThreading::TReaderWriterSpinLock, Lock_); - SSL_CTX* ActiveCtx_ = nullptr; + SSL_CTX* ActiveContext_ = nullptr; TInstant CommitTime_; }; @@ -155,14 +155,14 @@ class TTlsConnection { public: TTlsConnection( - TSslContextImplPtr ctx, + TSslContextImplPtr context, IPollerPtr poller, IConnectionPtr connection) - : Ctx_(std::move(ctx)) + : Context_(std::move(context)) , Invoker_(CreateSerializedInvoker(poller->GetInvoker(), "crypto_tls_connection")) , Underlying_(std::move(connection)) { - Ssl_ = Ctx_->NewSsl(); + Ssl_ = Context_->NewSsl(); if (!Ssl_) { THROW_ERROR_EXCEPTION("SSL_new failed") << GetSslErrors(); @@ -229,14 +229,19 @@ public: return Underlying_->GetWriteByteCount(); } - const TNetworkAddress& LocalAddress() const override + TConnectionId GetId() const override + { + return Underlying_->GetId(); + } + + const TNetworkAddress& GetLocalAddress() const override { - return Underlying_->LocalAddress(); + return Underlying_->GetLocalAddress(); } - const TNetworkAddress& RemoteAddress() const override + const TNetworkAddress& GetRemoteAddress() const override { - return Underlying_->RemoteAddress(); + return Underlying_->GetRemoteAddress(); } TConnectionStatistics GetWriteStatistics() const override @@ -356,7 +361,7 @@ public: } private: - const TSslContextImplPtr Ctx_; + const TSslContextImplPtr Context_; const IInvokerPtr Invoker_; const IConnectionPtr Underlying_; @@ -570,18 +575,18 @@ public: TSslContextImplPtr ctx, IDialerPtr dialer, IPollerPtr poller) - : Ctx_(std::move(ctx)) + : Context_(std::move(ctx)) , Underlying_(std::move(dialer)) , Poller_(std::move(poller)) { } - TFuture<IConnectionPtr> Dial(const TNetworkAddress& remote, TDialerContextPtr context) override + TFuture<IConnectionPtr> Dial(const TNetworkAddress& remoteAddress, TDialerContextPtr context) override { - return Underlying_->Dial(remote) - .Apply(BIND([ctx = Ctx_, poller = Poller_, context = std::move(context)] (const IConnectionPtr& underlying) -> IConnectionPtr { + return Underlying_->Dial(remoteAddress) + .Apply(BIND([ctx = Context_, poller = Poller_, context = std::move(context)] (const IConnectionPtr& underlying) -> IConnectionPtr { auto connection = New<TTlsConnection>(ctx, poller, underlying); - if (context != nullptr && context->Host != std::nullopt) { - connection->SetHost(*(context->Host)); + if (context && context->Host) { + connection->SetHost(*context->Host); } connection->StartClient(); return connection; @@ -589,7 +594,7 @@ public: } private: - const TSslContextImplPtr Ctx_; + const TSslContextImplPtr Context_; const IDialerPtr Underlying_; const IPollerPtr Poller_; }; @@ -658,12 +663,12 @@ TInstant TSslContext::GetCommitTime() const void TSslContext::UseBuiltinOpenSslX509Store() { - SSL_CTX_set_cert_store(Impl_->Ctx, GetBuiltinOpenSslX509Store().Release()); + SSL_CTX_set_cert_store(Impl_->Context, GetBuiltinOpenSslX509Store().Release()); } void TSslContext::SetCipherList(const TString& list) { - if (SSL_CTX_set_cipher_list(Impl_->Ctx, list.data()) == 0) { + if (SSL_CTX_set_cipher_list(Impl_->Context, list.data()) == 0) { THROW_ERROR_EXCEPTION("SSL_CTX_set_cipher_list failed") << TErrorAttribute("cipher_list", list) << GetSslErrors(); @@ -672,7 +677,7 @@ void TSslContext::SetCipherList(const TString& list) void TSslContext::AddCertificateFromFile(const TString& path) { - if (SSL_CTX_use_certificate_file(Impl_->Ctx, path.c_str(), SSL_FILETYPE_PEM) != 1) { + if (SSL_CTX_use_certificate_file(Impl_->Context, path.c_str(), SSL_FILETYPE_PEM) != 1) { THROW_ERROR_EXCEPTION("SSL_CTX_use_certificate_file failed") << TErrorAttribute("path", path) << GetSslErrors(); @@ -681,7 +686,7 @@ void TSslContext::AddCertificateFromFile(const TString& path) void TSslContext::AddCertificateChainFromFile(const TString& path) { - if (SSL_CTX_use_certificate_chain_file(Impl_->Ctx, path.c_str()) != 1) { + if (SSL_CTX_use_certificate_chain_file(Impl_->Context, path.c_str()) != 1) { THROW_ERROR_EXCEPTION("SSL_CTX_use_certificate_chain_file failed") << TErrorAttribute("path", path) << GetSslErrors(); @@ -690,7 +695,7 @@ void TSslContext::AddCertificateChainFromFile(const TString& path) void TSslContext::AddPrivateKeyFromFile(const TString& path) { - if (SSL_CTX_use_PrivateKey_file(Impl_->Ctx, path.c_str(), SSL_FILETYPE_PEM) != 1) { + if (SSL_CTX_use_PrivateKey_file(Impl_->Context, path.c_str(), SSL_FILETYPE_PEM) != 1) { THROW_ERROR_EXCEPTION("SSL_CTX_use_PrivateKey_file failed") << TErrorAttribute("path", path) << GetSslErrors(); @@ -714,12 +719,12 @@ void TSslContext::AddCertificateChain(const TString& certificateChain) X509_free(certificateObject); }); - if (SSL_CTX_use_certificate(Impl_->Ctx, certificateObject) != 1) { + if (SSL_CTX_use_certificate(Impl_->Context, certificateObject) != 1) { THROW_ERROR_EXCEPTION("SSL_CTX_use_certificate failed") << GetSslErrors(); } - SSL_CTX_clear_chain_certs(Impl_->Ctx); + SSL_CTX_clear_chain_certs(Impl_->Context); while (true) { auto chainCertificateObject = PEM_read_bio_X509(bio, nullptr, nullptr, nullptr); if (!chainCertificateObject) { @@ -733,7 +738,7 @@ void TSslContext::AddCertificateChain(const TString& certificateChain) << GetSslErrors(); } - int result = SSL_CTX_add0_chain_cert(Impl_->Ctx, chainCertificateObject); + int result = SSL_CTX_add0_chain_cert(Impl_->Context, chainCertificateObject); if (!result) { X509_free(chainCertificateObject); THROW_ERROR_EXCEPTION("SSL_CTX_add0_chain_cert") @@ -759,7 +764,7 @@ void TSslContext::AddCertificate(const TString& certificate) X509_free(certificateObject); }); - if (SSL_CTX_use_certificate(Impl_->Ctx, certificateObject) != 1) { + if (SSL_CTX_use_certificate(Impl_->Context, certificateObject) != 1) { THROW_ERROR_EXCEPTION("SSL_CTX_use_certificate failed") << GetSslErrors(); } @@ -782,7 +787,7 @@ void TSslContext::AddPrivateKey(const TString& privateKey) EVP_PKEY_free(privateKeyObject); }); - if (SSL_CTX_use_PrivateKey(Impl_->Ctx, privateKeyObject) != 1) { + if (SSL_CTX_use_PrivateKey(Impl_->Context, privateKeyObject) != 1) { THROW_ERROR_EXCEPTION("SSL_CTX_use_PrivateKey failed") << GetSslErrors(); } diff --git a/yt/yt/core/http/client.cpp b/yt/yt/core/http/client.cpp index dc7bf3077d..d21358b2c6 100644 --- a/yt/yt/core/http/client.cpp +++ b/yt/yt/core/http/client.cpp @@ -12,6 +12,7 @@ #include <yt/yt/core/net/connection.h> #include <yt/yt/core/concurrency/poller.h> + #include <util/string/cast.h> namespace NYT::NHttp { @@ -26,13 +27,13 @@ class TClient { public: TClient( - const TClientConfigPtr& config, - const IDialerPtr& dialer, - const IInvokerPtr& invoker) - : Config_(config) - , Dialer_(dialer) - , Invoker_(invoker) - , ConnectionPool_(New<TConnectionPool>(dialer, config, invoker)) + TClientConfigPtr config, + IDialerPtr dialer, + IInvokerPtr invoker) + : Config_(std::move(config)) + , Dialer_(std::move(dialer)) + , Invoker_(std::move(invoker)) + , ConnectionPool_(New<TConnectionPool>(Dialer_, Config_, Invoker_)) { } TFuture<IResponsePtr> Get( @@ -109,7 +110,7 @@ private: const TClientConfigPtr Config_; const IDialerPtr Dialer_; const IInvokerPtr Invoker_; - TConnectionPoolPtr ConnectionPool_; + const TConnectionPoolPtr ConnectionPool_; static int GetDefaultPort(const TUrlRef& parsedUrl) { @@ -125,9 +126,8 @@ private: auto host = parsedUrl.Host; TNetworkAddress address; - auto tryIP = TNetworkAddress::TryParse(host); - if (tryIP.IsOK()) { - address = tryIP.Value(); + if (auto ipOrError = TNetworkAddress::TryParse(host); ipOrError.IsOK()) { + address = ipOrError.Value(); } else { auto asyncAddress = TAddressResolver::Get()->Resolve(ToString(host)); address = WaitFor(asyncAddress) @@ -137,15 +137,17 @@ private: return TNetworkAddress(address, parsedUrl.Port.value_or(GetDefaultPort(parsedUrl))); } - std::pair<THttpOutputPtr, THttpInputPtr> OpenHttp(const TUrlRef& urlRef) + std::pair<THttpOutputPtr, THttpInputPtr> Connect(const TUrlRef& urlRef) { auto context = New<TDialerContext>(); context->Host = urlRef.Host; + auto address = GetAddress(urlRef); // TODO(aleexfi): Enable connection pool by default if (Config_->MaxIdleConnections == 0) { - auto connection = WaitFor(Dialer_->Dial(address, std::move(context))).ValueOrThrow(); + auto connection = WaitFor(Dialer_->Dial(address, std::move(context))) + .ValueOrThrow(); auto input = New<THttpInput>( connection, @@ -161,9 +163,10 @@ private: return {std::move(output), std::move(input)}; } else { - auto connection = WaitFor(ConnectionPool_->Connect(address, std::move(context))).ValueOrThrow(); + auto connection = WaitFor(ConnectionPool_->Connect(address, std::move(context))) + .ValueOrThrow(); - auto reuseSharedState = New<NDetail::TReusableConnectionState>(connection, ConnectionPool_); + auto reusableState = New<NDetail::TReusableConnectionState>(connection, ConnectionPool_); auto input = New<NDetail::TConnectionReuseWrapper<THttpInput>>( connection, @@ -171,13 +174,13 @@ private: Invoker_, EMessageType::Response, Config_); - input->SetReusableState(reuseSharedState); + input->SetReusableState(reusableState); auto output = New<NDetail::TConnectionReuseWrapper<THttpOutput>>( connection, EMessageType::Request, Config_); - output->SetReusableState(reuseSharedState); + output->SetReusableState(reusableState); return {std::move(output), std::move(input)}; } @@ -186,7 +189,7 @@ private: template <typename T> TFuture<T> WrapError(const TString& url, TCallback<T()> action) { - return BIND([=, this_ = MakeStrong(this)] { + return BIND([=, this_ = MakeStrong(this), action = std::move(action)] { try { return action(); } catch (const std::exception& ex) { @@ -223,7 +226,7 @@ private: // Waits for response headers internally. Response_->GetStatusCode(); - return IResponsePtr{Response_}; + return IResponsePtr(Response_); })); } @@ -238,10 +241,10 @@ private: } private: - THttpOutputPtr Request_; - THttpInputPtr Response_; - TIntrusivePtr<TClient> Client_; - TString Url_; + const THttpOutputPtr Request_; + const THttpInputPtr Response_; + const TIntrusivePtr<TClient> Client_; + const TString Url_; }; std::pair<THttpOutputPtr, THttpInputPtr> StartAndWriteHeaders( @@ -254,7 +257,7 @@ private: auto urlRef = ParseUrl(url); - std::tie(request, response) = OpenHttp(urlRef); + std::tie(request, response) = Connect(urlRef); request->SetHost(urlRef.Host, urlRef.PortStr); if (headers) { @@ -276,7 +279,7 @@ private: { return WrapError(url, BIND([=, this, this_ = MakeStrong(this)] { auto [request, response] = StartAndWriteHeaders(method, url, headers); - return IActiveRequestPtr{New<TActiveRequest>(request, response, this_, url)}; + return IActiveRequestPtr(New<TActiveRequest>(request, response, this_, url)); })); } @@ -297,34 +300,44 @@ private: .ThrowOnError(); } + if (Config_->IgnoreContinueResponses) { + while (response->GetStatusCode() == EStatusCode::Continue) { + response->Reset(); + } + } + // Waits for response headers internally. auto redirectUrl = response->TryGetRedirectUrl(); if (redirectUrl && redirectCount < Config_->MaxRedirectCount) { return DoRequest(method, *redirectUrl, body, headers, redirectCount + 1); } - return IResponsePtr(response); + return response; } }; //////////////////////////////////////////////////////////////////////////////// IClientPtr CreateClient( - const TClientConfigPtr& config, - const IDialerPtr& dialer, - const IInvokerPtr& invoker) + TClientConfigPtr config, + IDialerPtr dialer, + IInvokerPtr invoker) { - return New<TClient>(config, dialer, invoker); + return New<TClient>( + std::move(config), + std::move(dialer), + std::move(invoker)); } IClientPtr CreateClient( - const TClientConfigPtr& config, - const IPollerPtr& poller) + TClientConfigPtr config, + IPollerPtr poller) { + auto invoker = poller->GetInvoker(); return CreateClient( - config, - CreateDialer(New<TDialerConfig>(), poller, HttpLogger()), - poller->GetInvoker()); + std::move(config), + CreateDialer(New<TDialerConfig>(), std::move(poller), HttpLogger()), + std::move(invoker)); } //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/core/http/client.h b/yt/yt/core/http/client.h index a75df31142..2745762633 100644 --- a/yt/yt/core/http/client.h +++ b/yt/yt/core/http/client.h @@ -78,12 +78,12 @@ DEFINE_REFCOUNTED_TYPE(IClient) //////////////////////////////////////////////////////////////////////////////// IClientPtr CreateClient( - const TClientConfigPtr& config, - const NNet::IDialerPtr& dialer, - const IInvokerPtr& invoker); + TClientConfigPtr config, + NNet::IDialerPtr dialer, + IInvokerPtr invoker); IClientPtr CreateClient( - const TClientConfigPtr& config, - const NConcurrency::IPollerPtr& poller); + TClientConfigPtr config, + NConcurrency::IPollerPtr poller); //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/core/http/config.cpp b/yt/yt/core/http/config.cpp index c43ea1fe9d..2fb5b909aa 100644 --- a/yt/yt/core/http/config.cpp +++ b/yt/yt/core/http/config.cpp @@ -14,6 +14,9 @@ void THttpIOConfig::Register(TRegistrar registrar) registrar.Parameter("max_redirect_count", &TThis::MaxRedirectCount) .Default(0); + registrar.Parameter("ignore_continue_responses", &TThis::IgnoreContinueResponses) + .Default(false); + registrar.Parameter("connection_idle_timeout", &TThis::ConnectionIdleTimeout) .Default(TDuration::Minutes(5)); diff --git a/yt/yt/core/http/config.h b/yt/yt/core/http/config.h index ba3826e226..41ca0a1cda 100644 --- a/yt/yt/core/http/config.h +++ b/yt/yt/core/http/config.h @@ -18,6 +18,8 @@ public: int MaxRedirectCount; + bool IgnoreContinueResponses; + TDuration ConnectionIdleTimeout; TDuration HeaderReadTimeout; diff --git a/yt/yt/core/http/connection_pool.cpp b/yt/yt/core/http/connection_pool.cpp index a14891164c..24dc08d5b2 100644 --- a/yt/yt/core/http/connection_pool.cpp +++ b/yt/yt/core/http/connection_pool.cpp @@ -1,5 +1,7 @@ #include "connection_pool.h" +#include "private.h" + #include <yt/yt/core/concurrency/periodic_executor.h> #include <yt/yt/core/net/connection.h> @@ -11,6 +13,10 @@ using namespace NConcurrency; //////////////////////////////////////////////////////////////////////////////// +static constexpr auto& Logger = HttpLogger; + +//////////////////////////////////////////////////////////////////////////////// + TDuration TIdleConnection::GetIdleTime() const { return TInstant::Now() - InsertionTime; @@ -56,7 +62,11 @@ TFuture<IConnectionPtr> TConnectionPool::Connect( while (auto item = Connections_.Extract(address)) { if (item->GetIdleTime() < Config_->ConnectionIdleTimeout && item->IsOK()) { - return MakeFuture<IConnectionPtr>(std::move(item->Connection)); + auto&& connection = item->Connection; + YT_LOG_DEBUG("Connection is extracted from cache (Address: %v, ConnectionId: %v)", + address, + connection->GetId()); + return MakeFuture<IConnectionPtr>(std::move(connection)); } } } @@ -66,21 +76,28 @@ TFuture<IConnectionPtr> TConnectionPool::Connect( void TConnectionPool::Release(const IConnectionPtr& connection) { + YT_LOG_DEBUG("Connection is put to cache (Address: %v, ConnectionId: %v)", + connection->GetRemoteAddress(), + connection->GetId()); + auto guard = Guard(SpinLock_); - Connections_.Insert(connection->RemoteAddress(), {connection, TInstant::Now()}); + Connections_.Insert(connection->GetRemoteAddress(), {connection, TInstant::Now()}); } void TConnectionPool::DropExpiredConnections() { auto guard = Guard(SpinLock_); - TMultiLruCache<TNetworkAddress, TIdleConnection> validConnections( - Config_->MaxIdleConnections); + decltype(Connections_) validConnections(Config_->MaxIdleConnections); while (Connections_.GetSize() > 0) { auto idleConnection = Connections_.Pop(); if (idleConnection.GetIdleTime() < Config_->ConnectionIdleTimeout && idleConnection.IsOK()) { - validConnections.Insert(idleConnection.Connection->RemoteAddress(), idleConnection); + validConnections.Insert(idleConnection.Connection->GetRemoteAddress(), idleConnection); + } else { + YT_LOG_DEBUG("Connection expired from cache (Address: %v, ConnectionId: %v)", + idleConnection.Connection->GetRemoteAddress(), + idleConnection.Connection->GetId()); } } diff --git a/yt/yt/core/http/connection_reuse_helpers.h b/yt/yt/core/http/connection_reuse_helpers.h index bb3762bb6d..5d7039ebad 100644 --- a/yt/yt/core/http/connection_reuse_helpers.h +++ b/yt/yt/core/http/connection_reuse_helpers.h @@ -1,6 +1,6 @@ #pragma once -#include "public.h" +#include "private.h" #include <yt/yt/core/net/public.h> @@ -9,7 +9,7 @@ namespace NYT::NHttp::NDetail { //////////////////////////////////////////////////////////////////////////////// //! Responsible for returning the connection to the owning pool -//! if it could be reused +//! if it could be reused. struct TReusableConnectionState final { std::atomic<bool> Reusable = true; @@ -20,10 +20,12 @@ struct TReusableConnectionState final ~TReusableConnectionState(); }; -using TReusableConnectionStatePtr = TIntrusivePtr<TReusableConnectionState>; +DEFINE_REFCOUNTED_TYPE(TReusableConnectionState) + +//////////////////////////////////////////////////////////////////////////////// //! Reports to the shared state whether the connection could be reused -//! (by calling T::IsSafeToReuse() in the destructor) +//! (by calling T::IsSafeToReuse() in the destructor). template <class T> class TConnectionReuseWrapper : public T diff --git a/yt/yt/core/http/helpers.cpp b/yt/yt/core/http/helpers.cpp index 96a9c67245..e5bead9511 100644 --- a/yt/yt/core/http/helpers.cpp +++ b/yt/yt/core/http/helpers.cpp @@ -413,7 +413,7 @@ bool TryParseTraceParent(const TString& traceParent, NTracing::TSpanContext& spa // Now we have exactly three parts: traceId-spanId-options. // Parse trace context. - if (!TGuid::FromStringHex32(parts[0], &spanContext.TraceId)) { + if (!NTracing::TTraceId::FromStringHex32(parts[0], &spanContext.TraceId)) { return false; } diff --git a/yt/yt/core/http/http.h b/yt/yt/core/http/http.h index afd6c061f6..0de762c235 100644 --- a/yt/yt/core/http/http.h +++ b/yt/yt/core/http/http.h @@ -233,8 +233,9 @@ struct IRequest virtual const NNet::TNetworkAddress& GetRemoteAddress() const = 0; - virtual TGuid GetConnectionId() const = 0; - virtual TGuid GetRequestId() const = 0; + virtual TConnectionId GetConnectionId() const = 0; + virtual TRequestId GetRequestId() const = 0; + virtual i64 GetReadByteCount() const = 0; virtual TInstant GetStartTime() const = 0; diff --git a/yt/yt/core/http/mock/http.h b/yt/yt/core/http/mock/http.h index 27e62e561f..84ee0356ea 100644 --- a/yt/yt/core/http/mock/http.h +++ b/yt/yt/core/http/mock/http.h @@ -21,8 +21,8 @@ public: MOCK_METHOD(const TUrlRef&, GetUrl, (), (override)); MOCK_METHOD(const THeadersPtr&, GetHeaders, (), (override)); MOCK_METHOD(const NNet::TNetworkAddress&, GetRemoteAddress, (), (const, override)); - MOCK_METHOD(TGuid, GetConnectionId, (), (const, override)); - MOCK_METHOD(TGuid, GetRequestId, (), (const, override)); + MOCK_METHOD(TConnectionId, GetConnectionId, (), (const, override)); + MOCK_METHOD(TRequestId, GetRequestId, (), (const, override)); MOCK_METHOD(i64, GetReadByteCount, (), (const, override)); MOCK_METHOD(TInstant, GetStartTime, (), (const, override)); MOCK_METHOD(int, GetPort, (), (const, override)); diff --git a/yt/yt/core/http/private.h b/yt/yt/core/http/private.h index 691d022bbd..a156303350 100644 --- a/yt/yt/core/http/private.h +++ b/yt/yt/core/http/private.h @@ -20,4 +20,12 @@ DECLARE_REFCOUNTED_CLASS(THttpOutput) //////////////////////////////////////////////////////////////////////////////// +namespace NDetail { + +DECLARE_REFCOUNTED_STRUCT(TReusableConnectionState) + +} // namespace NDetail + +//////////////////////////////////////////////////////////////////////////////// + } // namespace NYT::NHttp diff --git a/yt/yt/core/http/public.h b/yt/yt/core/http/public.h index 74f0e8d29d..b904f89675 100644 --- a/yt/yt/core/http/public.h +++ b/yt/yt/core/http/public.h @@ -33,6 +33,8 @@ DECLARE_REFCOUNTED_CLASS(TSharedRefOutputStream) //////////////////////////////////////////////////////////////////////////////// using TContentEncoding = TString; +using TConnectionId = TGuid; +using TRequestId = TGuid; //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/core/http/server.cpp b/yt/yt/core/http/server.cpp index 7464803307..ae7df4b620 100644 --- a/yt/yt/core/http/server.cpp +++ b/yt/yt/core/http/server.cpp @@ -165,20 +165,19 @@ private: ConnectionsDropped_.Increment(); ActiveConnections_--; YT_LOG_WARNING("Server is over max active connection limit (RemoteAddress: %v)", - connection->RemoteAddress()); + connection->GetRemoteAddress()); return; } ConnectionsActive_.Update(count); ConnectionsAccepted_.Increment(); - auto connectionId = TGuid::Create(); YT_LOG_DEBUG("Connection accepted (ConnectionId: %v, RemoteAddress: %v, LocalAddress: %v)", - connectionId, - connection->RemoteAddress(), - connection->LocalAddress()); + connection->GetId(), + connection->GetRemoteAddress(), + connection->GetLocalAddress()); Invoker_->Invoke( - BIND(&TServer::HandleConnection, MakeStrong(this), std::move(connection), connectionId)); + BIND(&TServer::HandleConnection, MakeStrong(this), std::move(connection))); } bool HandleRequest(const THttpInputPtr& request, const THttpOutputPtr& response) @@ -271,10 +270,10 @@ private: return true; } - void HandleConnection(const IConnectionPtr& connection, TGuid connectionId) + void HandleConnection(const IConnectionPtr& connection) { try { - connection->SubscribePeerDisconnect(BIND([config = Config_, canceler = GetCurrentFiberCanceler(), connectionId = connectionId] { + connection->SubscribePeerDisconnect(BIND([config = Config_, canceler = GetCurrentFiberCanceler(), connectionId = connection->GetId()] { YT_LOG_DEBUG("Client closed TCP socket (ConnectionId: %v)", connectionId); if (config->CancelFiberOnConnectionClose.value_or(false)) { @@ -291,17 +290,17 @@ private: connection->SetNoDelay(); } - DoHandleConnection(connection, connectionId); + DoHandleConnection(connection); } catch (const std::exception& ex) { - YT_LOG_ERROR(ex, "Unhandled exception (ConnectionId: %v)", connectionId); + YT_LOG_ERROR(ex, "Unhandled exception (ConnectionId: %v)", connection->GetId()); } } - void DoHandleConnection(const IConnectionPtr& connection, TGuid connectionId) + void DoHandleConnection(const IConnectionPtr& connection) { auto request = New<THttpInput>( connection, - connection->RemoteAddress(), + connection->GetRemoteAddress(), GetCurrentInvoker(), EMessageType::Request, Config_); @@ -317,11 +316,8 @@ private: EMessageType::Response, Config_); - request->SetConnectionId(connectionId); - response->SetConnectionId(connectionId); - while (true) { - auto requestId = TGuid::Create(); + auto requestId = TRequestId::Create(); request->SetRequestId(requestId); response->SetRequestId(requestId); @@ -332,7 +328,7 @@ private: auto logDrop = [&] (auto reason) { YT_LOG_DEBUG("Dropping HTTP connection (ConnectionId: %v, Reason: %v)", - connectionId, + connection->GetId(), reason); }; @@ -381,10 +377,10 @@ private: auto connectionResult = WaitFor(connection->Close()); if (connectionResult.IsOK()) { YT_LOG_DEBUG("HTTP connection closed (ConnectionId: %v)", - connectionId); + connection->GetId()); } else { YT_LOG_DEBUG(connectionResult, "Error closing HTTP connection (ConnectionId: %v)", - connectionId); + connection->GetId()); } } }; diff --git a/yt/yt/core/http/stream.cpp b/yt/yt/core/http/stream.cpp index db16ffdf53..632e035a4c 100644 --- a/yt/yt/core/http/stream.cpp +++ b/yt/yt/core/http/stream.cpp @@ -295,22 +295,17 @@ const TNetworkAddress& THttpInput::GetRemoteAddress() const return RemoteAddress_; } -TGuid THttpInput::GetConnectionId() const +TConnectionId THttpInput::GetConnectionId() const { - return ConnectionId_; + return Connection_->GetId(); } -void THttpInput::SetConnectionId(TGuid connectionId) -{ - ConnectionId_ = connectionId; -} - -TGuid THttpInput::GetRequestId() const +TRequestId THttpInput::GetRequestId() const { return RequestId_; } -void THttpInput::SetRequestId(TGuid requestId) +void THttpInput::SetRequestId(TRequestId requestId) { RequestId_ = requestId; } @@ -633,12 +628,7 @@ void THttpOutput::Reset() Trailers_.Reset(); } -void THttpOutput::SetConnectionId(TGuid connectionId) -{ - ConnectionId_ = connectionId; -} - -void THttpOutput::SetRequestId(TGuid requestId) +void THttpOutput::SetRequestId(TRequestId requestId) { RequestId_ = requestId; } diff --git a/yt/yt/core/http/stream.h b/yt/yt/core/http/stream.h index b01da702f1..9efcea7f7a 100644 --- a/yt/yt/core/http/stream.h +++ b/yt/yt/core/http/stream.h @@ -105,11 +105,10 @@ public: const NNet::TNetworkAddress& GetRemoteAddress() const override; - TGuid GetConnectionId() const override; - void SetConnectionId(TGuid connectionId); + TConnectionId GetConnectionId() const override; - TGuid GetRequestId() const override; - void SetRequestId(TGuid requestId); + TRequestId GetRequestId() const override; + void SetRequestId(TRequestId requestId); i64 GetReadByteCount() const override; @@ -149,8 +148,7 @@ private: THeadersPtr Headers_; // Debug. - TGuid ConnectionId_; - TGuid RequestId_; + TRequestId RequestId_; i64 StartByteCount_ = 0; NNet::TConnectionStatistics StartStatistics_; TInstant LastProgressLogTime_; @@ -215,8 +213,7 @@ public: bool IsSafeToReuse() const; void Reset(); - void SetConnectionId(TGuid connectionId); - void SetRequestId(TGuid requestId); + void SetRequestId(TRequestId requestId); i64 GetWriteByteCount() const override; @@ -228,8 +225,7 @@ private: TClosure OnWriteFinish_; //! Debugging. - TGuid ConnectionId_; - TGuid RequestId_; + TRequestId RequestId_; i64 StartByteCount_ = 0; NNet::TConnectionStatistics StartStatistics_; bool HeadersLogged_ = false; diff --git a/yt/yt/core/http/unittests/http_ut.cpp b/yt/yt/core/http/unittests/http_ut.cpp index e5bc967d73..cda7dea65f 100644 --- a/yt/yt/core/http/unittests/http_ut.cpp +++ b/yt/yt/core/http/unittests/http_ut.cpp @@ -159,6 +159,11 @@ struct TFakeConnection TString Input; TString Output; + TConnectionId GetId() const override + { + return {}; + } + bool SetNoDelay() override { return true; @@ -216,12 +221,12 @@ struct TFakeConnection THROW_ERROR_EXCEPTION("Not implemented"); } - const TNetworkAddress& LocalAddress() const override + const TNetworkAddress& GetLocalAddress() const override { THROW_ERROR_EXCEPTION("Not implemented"); } - const TNetworkAddress& RemoteAddress() const override + const TNetworkAddress& GetRemoteAddress() const override { THROW_ERROR_EXCEPTION("Not implemented"); } @@ -1149,7 +1154,7 @@ TEST_P(THttpServerTest, ConnectionKeepAlive) auto response = New<THttpInput>( connection, - connection->RemoteAddress(), + connection->GetRemoteAddress(), Poller->GetInvoker(), EMessageType::Response, New<THttpIOConfig>()); @@ -1183,7 +1188,7 @@ TEST_P(THttpServerTest, ConnectionKeepAlive) auto response = New<THttpInput>( connection, - connection->RemoteAddress(), + connection->GetRemoteAddress(), Poller->GetInvoker(), EMessageType::Response, New<THttpIOConfig>()); diff --git a/yt/yt/core/misc/arithmetic_formula.cpp b/yt/yt/core/misc/arithmetic_formula.cpp index 8fe147ef65..143e3de8f6 100644 --- a/yt/yt/core/misc/arithmetic_formula.cpp +++ b/yt/yt/core/misc/arithmetic_formula.cpp @@ -42,7 +42,7 @@ bool IsSymbolAllowedInName(char c, EEvaluationContext context, bool isFirst) return false; } -void ValidateFormulaVariable(const TString& variable, EEvaluationContext context) +void ValidateFormulaVariable(const std::string& variable, EEvaluationContext context) { if (variable.empty()) { THROW_ERROR_EXCEPTION("Variable should not be empty"); @@ -67,12 +67,12 @@ void ValidateFormulaVariable(const TString& variable, EEvaluationContext context //////////////////////////////////////////////////////////////////////////////// -void ValidateArithmeticFormulaVariable(const TString& variable) +void ValidateArithmeticFormulaVariable(const std::string& variable) { ValidateFormulaVariable(variable, EEvaluationContext::Arithmetic); } -void ValidateBooleanFormulaVariable(const TString& variable) +void ValidateBooleanFormulaVariable(const std::string& variable) { ValidateFormulaVariable(variable, EEvaluationContext::Boolean); } @@ -81,12 +81,12 @@ void ValidateBooleanFormulaVariable(const TString& variable) namespace { -void ThrowError(const TString& formula, int position, const TString& message, EEvaluationContext evaluationContext) +void ThrowError(const std::string& formula, int position, const std::string& message, EEvaluationContext evaluationContext) { const static int maxContextSize = 30; int contextStart = std::max(0, position - maxContextSize / 2); - TString context = formula.substr(contextStart, maxContextSize); + std::string context = formula.substr(contextStart, maxContextSize); int contextPosition = std::min(position, maxContextSize / 2); TStringBuilder builder; @@ -154,7 +154,7 @@ struct TFormulaToken { EFormulaTokenType Type; int Position; - TString Name; + std::string Name; i64 Number = 0; }; @@ -171,11 +171,11 @@ class TGenericFormulaImpl : public TRefCounted { public: - DEFINE_BYVAL_RO_PROPERTY(TString, Formula); + DEFINE_BYVAL_RO_PROPERTY(std::string, Formula); DEFINE_BYVAL_RO_PROPERTY(size_t, Hash); public: - TGenericFormulaImpl(const TString& formula, size_t hash, std::vector<TFormulaToken> parsedFormula); + TGenericFormulaImpl(const std::string& formula, size_t hash, std::vector<TFormulaToken> parsedFormula); bool operator==(const TGenericFormulaImpl& other) const; @@ -183,31 +183,31 @@ public: int Size() const; - i64 Eval(const THashMap<TString, i64>& values, EEvaluationContext context) const; + i64 Eval(const THashMap<std::string, i64>& values, EEvaluationContext context) const; - THashSet<TString> GetVariables() const; + THashSet<std::string> GetVariables() const; private: std::vector<TFormulaToken> ParsedFormula_; - static std::vector<TFormulaToken> Tokenize(const TString& formula, EEvaluationContext context); + static std::vector<TFormulaToken> Tokenize(const std::string& formula, EEvaluationContext context); static std::vector<TFormulaToken> Parse( - const TString& formula, + const std::string& formula, const std::vector<TFormulaToken>& tokens, EEvaluationContext context); static size_t CalculateHash(const std::vector<TFormulaToken>& tokens); static void CheckTypeConsistency( - const TString& formula, + const std::string& formula, const std::vector<TFormulaToken>& tokens, EEvaluationContext context); - friend TIntrusivePtr<TGenericFormulaImpl> MakeGenericFormulaImpl(const TString& formula, EEvaluationContext context); + friend TIntrusivePtr<TGenericFormulaImpl> MakeGenericFormulaImpl(const std::string& formula, EEvaluationContext context); }; //////////////////////////////////////////////////////////////////////////////// TGenericFormulaImpl::TGenericFormulaImpl( - const TString& formula, + const std::string& formula, size_t hash, std::vector<TFormulaToken> parsedFormula) : Formula_(formula) @@ -234,9 +234,9 @@ int TGenericFormulaImpl::Size() const return ParsedFormula_.size(); } -i64 TGenericFormulaImpl::Eval(const THashMap<TString, i64>& values, EEvaluationContext context) const +i64 TGenericFormulaImpl::Eval(const THashMap<std::string, i64>& values, EEvaluationContext context) const { - auto variableValue = [&] (const TString& var) -> i64 { + auto variableValue = [&] (const std::string& var) -> i64 { auto iter = values.find(var); if (iter == values.end()) { if (context == EEvaluationContext::Boolean) { @@ -389,9 +389,9 @@ i64 TGenericFormulaImpl::Eval(const THashMap<TString, i64>& values, EEvaluationC #undef APPLY_BINARY_OP } -THashSet<TString> TGenericFormulaImpl::GetVariables() const +THashSet<std::string> TGenericFormulaImpl::GetVariables() const { - THashSet<TString> variables; + THashSet<std::string> variables; for (const auto& token : ParsedFormula_) { if (token.Type == EFormulaTokenType::Variable) { variables.insert(token.Name); @@ -400,12 +400,12 @@ THashSet<TString> TGenericFormulaImpl::GetVariables() const return variables; } -std::vector<TFormulaToken> TGenericFormulaImpl::Tokenize(const TString& formula, EEvaluationContext context) +std::vector<TFormulaToken> TGenericFormulaImpl::Tokenize(const std::string& formula, EEvaluationContext context) { std::vector<TFormulaToken> result; size_t pos = 0; - auto throwError = [&] (int position, const TString& message) { + auto throwError = [&] (int position, const std::string& message) { ThrowError(formula, position, message, context); }; @@ -526,7 +526,7 @@ std::vector<TFormulaToken> TGenericFormulaImpl::Tokenize(const TString& formula, }; auto extractVariable = [&] { - TString name; + std::string name; while (pos < formula.size() && IsSymbolAllowedInName(formula[pos], context, /*isFirst*/ name.empty())) { name += formula[pos++]; } @@ -595,7 +595,7 @@ std::vector<TFormulaToken> TGenericFormulaImpl::Tokenize(const TString& formula, } std::vector<TFormulaToken> TGenericFormulaImpl::Parse( - const TString& formula, + const std::string& formula, const std::vector<TFormulaToken>& tokens, EEvaluationContext context) { @@ -607,7 +607,7 @@ std::vector<TFormulaToken> TGenericFormulaImpl::Parse( return result; } - auto throwError = [&] (int position, const TString& message) { + auto throwError = [&] (int position, const std::string& message) { ThrowError(formula, position, message, context); }; @@ -712,7 +712,7 @@ size_t TGenericFormulaImpl::CalculateHash(const std::vector<TFormulaToken>& toke } void TGenericFormulaImpl::CheckTypeConsistency( - const TString& formula, + const std::string& formula, const std::vector<TFormulaToken>& tokens, EEvaluationContext context) { @@ -775,7 +775,7 @@ void TGenericFormulaImpl::CheckTypeConsistency( } } -TIntrusivePtr<TGenericFormulaImpl> MakeGenericFormulaImpl(const TString& formula, EEvaluationContext context) +TIntrusivePtr<TGenericFormulaImpl> MakeGenericFormulaImpl(const std::string& formula, EEvaluationContext context) { auto tokens = TGenericFormulaImpl::Tokenize(formula, context); auto parsed = TGenericFormulaImpl::Parse(formula, tokens, context); @@ -786,7 +786,7 @@ TIntrusivePtr<TGenericFormulaImpl> MakeGenericFormulaImpl(const TString& formula //////////////////////////////////////////////////////////////////////////////// TArithmeticFormula::TArithmeticFormula() - : Impl_(MakeGenericFormulaImpl(TString(), EEvaluationContext::Arithmetic)) + : Impl_(MakeGenericFormulaImpl(std::string(), EEvaluationContext::Arithmetic)) { } TArithmeticFormula::TArithmeticFormula(TIntrusivePtr<TGenericFormulaImpl> impl) @@ -819,22 +819,22 @@ size_t TArithmeticFormula::GetHash() const return Impl_->GetHash(); } -TString TArithmeticFormula::GetFormula() const +std::string TArithmeticFormula::GetFormula() const { return Impl_->GetFormula(); } -i64 TArithmeticFormula::Eval(const THashMap<TString, i64>& values) const +i64 TArithmeticFormula::Eval(const THashMap<std::string, i64>& values) const { return Impl_->Eval(values, EEvaluationContext::Arithmetic); } -THashSet<TString> TArithmeticFormula::GetVariables() const +THashSet<std::string> TArithmeticFormula::GetVariables() const { return Impl_->GetVariables(); } -TArithmeticFormula MakeArithmeticFormula(const TString& formula) +TArithmeticFormula MakeArithmeticFormula(const std::string& formula) { auto impl = MakeGenericFormulaImpl(formula, EEvaluationContext::Arithmetic); return TArithmeticFormula(std::move(impl)); @@ -860,13 +860,13 @@ void TArithmeticFormula::Save(TStreamSaveContext& context) const void TArithmeticFormula::Load(TStreamLoadContext& context) { using NYT::Load; - auto formula = Load<TString>(context); + auto formula = Load<std::string>(context); Impl_ = MakeGenericFormulaImpl(formula, EEvaluationContext::Arithmetic); } //////////////////////////////////////////////////////////////////////////////// -TBooleanFormulaTags::TBooleanFormulaTags(THashSet<TString> tags) +TBooleanFormulaTags::TBooleanFormulaTags(THashSet<std::string> tags) : Tags_(std::move(tags)) { for (const auto& key: Tags_) { @@ -874,7 +874,7 @@ TBooleanFormulaTags::TBooleanFormulaTags(THashSet<TString> tags) } } -const THashSet<TString>& TBooleanFormulaTags::GetSourceTags() const +const THashSet<std::string>& TBooleanFormulaTags::GetSourceTags() const { return Tags_; } @@ -888,7 +888,7 @@ void TBooleanFormulaTags::Save(TStreamSaveContext& context) const void TBooleanFormulaTags::Load(TStreamLoadContext& context) { using NYT::Load; - *this = TBooleanFormulaTags(Load<THashSet<TString>>(context)); + *this = TBooleanFormulaTags(Load<THashSet<std::string>>(context)); } bool TBooleanFormulaTags::operator==(const TBooleanFormulaTags& other) const @@ -904,7 +904,7 @@ void Serialize(const TBooleanFormulaTags& tags, NYson::IYsonConsumer* consumer) void Deserialize(TBooleanFormulaTags& tags, NYTree::INodePtr node) { - tags = TBooleanFormulaTags(ConvertTo<THashSet<TString>>(node)); + tags = TBooleanFormulaTags(ConvertTo<THashSet<std::string>>(node)); } void FormatValue(TStringBuilderBase* builder, const TBooleanFormulaTags& tags, TStringBuf /*spec*/) @@ -915,7 +915,7 @@ void FormatValue(TStringBuilderBase* builder, const TBooleanFormulaTags& tags, T //////////////////////////////////////////////////////////////////////////////// TBooleanFormula::TBooleanFormula() - : Impl_(MakeGenericFormulaImpl(TString(), EEvaluationContext::Boolean)) + : Impl_(MakeGenericFormulaImpl(std::string(), EEvaluationContext::Boolean)) { } TBooleanFormula::TBooleanFormula(TIntrusivePtr<TGenericFormulaImpl> impl) @@ -948,23 +948,23 @@ size_t TBooleanFormula::GetHash() const return Impl_->GetHash(); } -TString TBooleanFormula::GetFormula() const +std::string TBooleanFormula::GetFormula() const { return Impl_->GetFormula(); } -bool TBooleanFormula::IsSatisfiedBy(const std::vector<TString>& value) const +bool TBooleanFormula::IsSatisfiedBy(const std::vector<std::string>& value) const { - THashMap<TString, i64> values; + THashMap<std::string, i64> values; for (const auto& key: value) { values[key] = 1; } return Impl_->Eval(values, EEvaluationContext::Boolean); } -bool TBooleanFormula::IsSatisfiedBy(const THashSet<TString>& value) const +bool TBooleanFormula::IsSatisfiedBy(const THashSet<std::string>& value) const { - return IsSatisfiedBy(std::vector<TString>(value.begin(), value.end())); + return IsSatisfiedBy(std::vector<std::string>(value.begin(), value.end())); } bool TBooleanFormula::IsSatisfiedBy(const TBooleanFormulaTags& tags) const @@ -972,7 +972,7 @@ bool TBooleanFormula::IsSatisfiedBy(const TBooleanFormulaTags& tags) const return Impl_->Eval(tags.PreparedTags_, EEvaluationContext::Boolean); } -TBooleanFormula MakeBooleanFormula(const TString& formula) +TBooleanFormula MakeBooleanFormula(const std::string& formula) { auto impl = MakeGenericFormulaImpl(formula, EEvaluationContext::Boolean); return TBooleanFormula(std::move(impl)); @@ -1020,7 +1020,7 @@ void Deserialize(TBooleanFormula& booleanFormula, TYsonPullParserCursor* cursor) { MaybeSkipAttributes(cursor); EnsureYsonToken("TBooleanFormula", *cursor, EYsonItemType::StringValue); - booleanFormula = MakeBooleanFormula(ExtractTo<TString>(cursor)); + booleanFormula = MakeBooleanFormula(ExtractTo<std::string>(cursor)); } void TBooleanFormula::Save(TStreamSaveContext& context) const @@ -1032,7 +1032,7 @@ void TBooleanFormula::Save(TStreamSaveContext& context) const void TBooleanFormula::Load(TStreamLoadContext& context) { using NYT::Load; - auto formula = Load<TString>(context); + auto formula = Load<std::string>(context); Impl_ = MakeGenericFormulaImpl(formula, EEvaluationContext::Boolean); } @@ -1070,7 +1070,7 @@ size_t TTimeFormula::GetHash() const return Formula_.GetHash(); } -TString TTimeFormula::GetFormula() const +std::string TTimeFormula::GetFormula() const { return Formula_.GetFormula(); } @@ -1088,9 +1088,9 @@ TTimeFormula::TTimeFormula(TArithmeticFormula&& arithmeticFormula) : Formula_(std::move(arithmeticFormula)) { } -TTimeFormula MakeTimeFormula(const TString& formula) +TTimeFormula MakeTimeFormula(const std::string& formula) { - const static THashSet<TString> allowedVariables{"minutes", "hours"}; + const static THashSet<std::string> allowedVariables{"minutes", "hours"}; auto arithmeticFormula = MakeArithmeticFormula(formula); @@ -1119,7 +1119,7 @@ void Deserialize(TTimeFormula& timeFormula, TYsonPullParserCursor* cursor) { MaybeSkipAttributes(cursor); EnsureYsonToken("TTimeFormula", *cursor, EYsonItemType::StringValue); - timeFormula = MakeTimeFormula(ExtractTo<TString>(cursor)); + timeFormula = MakeTimeFormula(ExtractTo<std::string>(cursor)); } void TTimeFormula::Save(TStreamSaveContext& context) const diff --git a/yt/yt/core/misc/arithmetic_formula.h b/yt/yt/core/misc/arithmetic_formula.h index ddc15d9def..fdb2bcfc00 100644 --- a/yt/yt/core/misc/arithmetic_formula.h +++ b/yt/yt/core/misc/arithmetic_formula.h @@ -12,10 +12,10 @@ namespace NYT { //////////////////////////////////////////////////////////////////////////////// //! Validates that a string is a correct arithmetic formula variable name. -void ValidateArithmeticFormulaVariable(const TString& variable); +void ValidateArithmeticFormulaVariable(const std::string& variable); //! Validates that a string is a correct boolean formula variable name. -void ValidateBooleanFormulaVariable(const TString& variable); +void ValidateBooleanFormulaVariable(const std::string& variable); //////////////////////////////////////////////////////////////////////////////// @@ -47,13 +47,13 @@ public: size_t GetHash() const; //! Returns a human-readable representation of the formula. - TString GetFormula() const; + std::string GetFormula() const; //! Evaluate the formula given values of variables. - i64 Eval(const THashMap<TString, i64>& values) const; + i64 Eval(const THashMap<std::string, i64>& values) const; //! Returns the list of variables used in the formula. - THashSet<TString> GetVariables() const; + THashSet<std::string> GetVariables() const; void Save(TStreamSaveContext& context) const; void Load(TStreamLoadContext& context); @@ -63,11 +63,11 @@ private: explicit TArithmeticFormula(TIntrusivePtr<TGenericFormulaImpl> impl); - friend TArithmeticFormula MakeArithmeticFormula(const TString& formula); + friend TArithmeticFormula MakeArithmeticFormula(const std::string& formula); }; //! Parse string and return arithmetic formula. -TArithmeticFormula MakeArithmeticFormula(const TString& formula); +TArithmeticFormula MakeArithmeticFormula(const std::string& formula); void Serialize(const TArithmeticFormula& arithmeticFormula, NYson::IYsonConsumer* consumer); void Deserialize(TArithmeticFormula& arithmeticFormula, NYTree::INodePtr node); @@ -78,9 +78,9 @@ class TBooleanFormulaTags { public: TBooleanFormulaTags() = default; - explicit TBooleanFormulaTags(THashSet<TString> tags); + explicit TBooleanFormulaTags(THashSet<std::string> tags); - const THashSet<TString>& GetSourceTags() const; + const THashSet<std::string>& GetSourceTags() const; void Save(TStreamSaveContext& context) const; void Load(TStreamLoadContext& context); @@ -88,8 +88,8 @@ public: bool operator==(const TBooleanFormulaTags& other) const; private: - THashSet<TString> Tags_; - THashMap<TString, i64> PreparedTags_; + THashSet<std::string> Tags_; + THashMap<std::string, i64> PreparedTags_; friend class TBooleanFormula; }; @@ -123,11 +123,11 @@ public: size_t GetHash() const; //! Returns a human-readable representation of the formula. - TString GetFormula() const; + std::string GetFormula() const; //! Check that a given set of true-variables satisfies the formula. - bool IsSatisfiedBy(const std::vector<TString>& value) const; - bool IsSatisfiedBy(const THashSet<TString>& value) const; + bool IsSatisfiedBy(const std::vector<std::string>& value) const; + bool IsSatisfiedBy(const THashSet<std::string>& value) const; bool IsSatisfiedBy(const TBooleanFormulaTags& tags) const; void Save(TStreamSaveContext& context) const; @@ -138,11 +138,11 @@ private: explicit TBooleanFormula(TIntrusivePtr<TGenericFormulaImpl> impl); - friend TBooleanFormula MakeBooleanFormula(const TString& formula); + friend TBooleanFormula MakeBooleanFormula(const std::string& formula); }; //! Parse string and return boolean formula. -TBooleanFormula MakeBooleanFormula(const TString& formula); +TBooleanFormula MakeBooleanFormula(const std::string& formula); //! Make conjunction, disjunction and negation of formulas. TBooleanFormula operator&(const TBooleanFormula& lhs, const TBooleanFormula& rhs); @@ -179,7 +179,7 @@ public: size_t GetHash() const; //! Returns a human-readable representation of the formula. - TString GetFormula() const; + std::string GetFormula() const; //! Check that given time satisfies the formula. bool IsSatisfiedBy(TInstant time) const; @@ -192,11 +192,11 @@ private: explicit TTimeFormula(TArithmeticFormula&& arithmeticFormula); - friend TTimeFormula MakeTimeFormula(const TString& formula); + friend TTimeFormula MakeTimeFormula(const std::string& formula); }; //! Parse string and return time formula. -TTimeFormula MakeTimeFormula(const TString& formula); +TTimeFormula MakeTimeFormula(const std::string& formula); void Serialize(const TTimeFormula& timeFormula, NYson::IYsonConsumer* consumer); void Deserialize(TTimeFormula& timeFormula, NYTree::INodePtr node); diff --git a/yt/yt/core/misc/atomic_object-inl.h b/yt/yt/core/misc/atomic_object-inl.h index f75722cb32..1e76cbf5ab 100644 --- a/yt/yt/core/misc/atomic_object-inl.h +++ b/yt/yt/core/misc/atomic_object-inl.h @@ -54,7 +54,7 @@ bool TAtomicObject<T>::CompareExchange(T& expected, const T& desired) } template <class T> -template <class F> +template <CInvocable<void(T&)> F> void TAtomicObject<T>::Transform(const F& func) { auto guard = WriterGuard(Spinlock_); @@ -62,6 +62,14 @@ void TAtomicObject<T>::Transform(const F& func) } template <class T> +template <class R, CInvocable<R(const T&)> F> +R TAtomicObject<T>::Read(const F& func) const +{ + auto guard = ReaderGuard(Spinlock_); + return func(Object_); +} + +template <class T> T TAtomicObject<T>::Load() const { auto guard = ReaderGuard(Spinlock_); diff --git a/yt/yt/core/misc/atomic_object.h b/yt/yt/core/misc/atomic_object.h index d75e9cfd7d..91ef9278b0 100644 --- a/yt/yt/core/misc/atomic_object.h +++ b/yt/yt/core/misc/atomic_object.h @@ -2,6 +2,8 @@ #include <library/cpp/yt/threading/rw_spin_lock.h> +#include <library/cpp/yt/misc/concepts.h> + namespace NYT { //////////////////////////////////////////////////////////////////////////////// @@ -30,9 +32,13 @@ public: bool CompareExchange(T& expected, const T& desired); //! Atomically transforms the value with function #func. - template <class F> + template <CInvocable<void(T&)> F> void Transform(const F& func); + //! Atomicaly reads the value with function #func. + template <class R = void, CInvocable<R(const T&)> F> + R Read(const F& func) const; + T Load() const; private: diff --git a/yt/yt/core/misc/error.cpp b/yt/yt/core/misc/error.cpp index c707ddc2e0..cc5fcbf7a4 100644 --- a/yt/yt/core/misc/error.cpp +++ b/yt/yt/core/misc/error.cpp @@ -34,7 +34,7 @@ constexpr TStringBuf OriginalErrorDepthAttribute = "original_error_depth"; //////////////////////////////////////////////////////////////////////////////// -namespace NOrigin { +namespace NDetail { namespace { @@ -72,8 +72,6 @@ void TryExtractHost(const TOriginAttributes& attributes) attributes.Host = name; } -} // namespace - //////////////////////////////////////////////////////////////////////////////// bool HasHost(const TOriginAttributes& attributes) noexcept @@ -187,9 +185,11 @@ TOriginAttributes ExtractFromDictionaryOverride(const NYTree::IAttributeDictiona return result; } +} // namespace + //////////////////////////////////////////////////////////////////////////////// -void EnableOriginOverrides() +void EnableErrorOriginOverrides() { static NGlobal::TVariable<std::byte> getExtensionDataOverride{ NYT::NDetail::GetExtensionDataTag, @@ -214,14 +214,14 @@ void EnableOriginOverrides() extractFromDictionaryOverride.Get(); } -} // namespace NOrigin +} // namespace NDetail //////////////////////////////////////////////////////////////////////////////// bool HasHost(const TError& error) noexcept { if (auto* attributes = error.MutableOriginAttributes()) { - return NOrigin::HasHost(*attributes); + return NYT::NDetail::HasHost(*attributes); } return false; } @@ -229,7 +229,7 @@ bool HasHost(const TError& error) noexcept TStringBuf GetHost(const TError& error) noexcept { if (auto* attributes = error.MutableOriginAttributes()) { - return NOrigin::GetHost(*attributes); + return NYT::NDetail::GetHost(*attributes); } return {}; } @@ -237,7 +237,7 @@ TStringBuf GetHost(const TError& error) noexcept NConcurrency::TFiberId GetFid(const TError& error) noexcept { if (auto* attributes = error.MutableOriginAttributes()) { - return NOrigin::GetFid(*attributes); + return NYT::NDetail::GetFid(*attributes); } return NConcurrency::InvalidFiberId; } @@ -245,7 +245,7 @@ NConcurrency::TFiberId GetFid(const TError& error) noexcept bool HasTracingAttributes(const TError& error) noexcept { if (auto* attributes = error.MutableOriginAttributes()) { - return NOrigin::HasTracingAttributes(*attributes); + return NYT::NDetail::HasTracingAttributes(*attributes); } return false; } @@ -253,7 +253,7 @@ bool HasTracingAttributes(const TError& error) noexcept NTracing::TTraceId GetTraceId(const TError& error) noexcept { if (auto* attributes = error.MutableOriginAttributes()) { - return NOrigin::GetTraceId(*attributes); + return NYT::NDetail::GetTraceId(*attributes); } return NTracing::InvalidTraceId; } @@ -261,7 +261,7 @@ NTracing::TTraceId GetTraceId(const TError& error) noexcept NTracing::TSpanId GetSpanId(const TError& error) noexcept { if (auto* attributes = error.MutableOriginAttributes()) { - return NOrigin::GetSpanId(*attributes); + return NYT::NDetail::GetSpanId(*attributes); } return NTracing::InvalidSpanId; } @@ -274,7 +274,7 @@ void SetTracingAttributes(TError* error, const NTracing::TTracingAttributes& att return; } - NOrigin::UpdateTracingAttributes(originAttributes, attributes); + NYT::NDetail::UpdateTracingAttributes(originAttributes, attributes); } //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/core/misc/error.h b/yt/yt/core/misc/error.h index 18c984803d..f3c773da33 100644 --- a/yt/yt/core/misc/error.h +++ b/yt/yt/core/misc/error.h @@ -17,11 +17,11 @@ namespace NYT { //////////////////////////////////////////////////////////////////////////////// -namespace NOrigin { +namespace NDetail { -void EnableOriginOverrides(); +void EnableErrorOriginOverrides(); -} // namespace NOrigin +} // namespace NDetail //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/core/misc/id_generator.h b/yt/yt/core/misc/id_generator.h index 8fc1eff48b..8e0bd71554 100644 --- a/yt/yt/core/misc/id_generator.h +++ b/yt/yt/core/misc/id_generator.h @@ -21,7 +21,6 @@ public: private: std::atomic<ui64> Current_ = 0; - }; //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/core/misc/unittests/boolean_formula_ut.cpp b/yt/yt/core/misc/unittests/boolean_formula_ut.cpp index 0bae531297..94e43cbf2d 100644 --- a/yt/yt/core/misc/unittests/boolean_formula_ut.cpp +++ b/yt/yt/core/misc/unittests/boolean_formula_ut.cpp @@ -12,8 +12,9 @@ class TBooleanFormulaTest : public ::testing::Test , public ::testing::WithParamInterface<std::tuple< const char*, - std::vector<TString>, - bool>> + std::vector<std::string>, + bool + >> { }; TEST_P(TBooleanFormulaTest, Test) @@ -35,31 +36,31 @@ INSTANTIATE_TEST_SUITE_P( TBooleanFormulaTest, TBooleanFormulaTest, ::testing::Values( - std::tuple("", std::vector<TString>{}, true), - std::tuple("", std::vector<TString>{"b"}, true), - std::tuple("a", std::vector<TString>{"b"}, false), - std::tuple("!a", std::vector<TString>{"b"}, true), - std::tuple("b", std::vector<TString>{"b"}, true), - std::tuple("a|b", std::vector<TString>{"b"}, true), - std::tuple("a & b", std::vector<TString>{"b"}, false), - std::tuple("(b)", std::vector<TString>{"b"}, true), - std::tuple("a|(a|b)", std::vector<TString>{"b"}, true), - std::tuple("(a|b)&(!a&b)", std::vector<TString>{"b"}, true), - std::tuple("a&b", std::vector<TString>{"a", "b"}, true), - std::tuple("(a|c)&(b|c)", std::vector<TString>{"a", "b"}, true), - std::tuple("(a|b)&c", std::vector<TString>{"a", "b"}, false), - std::tuple("a|b|c", std::vector<TString>{"b"}, true), - std::tuple("!a & b & !c", std::vector<TString>{"b"}, true), - std::tuple("var-1 | !var/2", std::vector<TString>{"var-1"}, true), - std::tuple("var-1 | !var/2", std::vector<TString>{"var/2"}, false), - std::tuple("var-1 | !var/2", std::vector<TString>{}, true), - std::tuple("!in-", std::vector<TString>{}, true), - std::tuple("in/|x", std::vector<TString>{"in/"}, true), - std::tuple("%true", std::vector<TString>{""}, true), - std::tuple("%false", std::vector<TString>{"false"}, false), - std::tuple("%true|%false", std::vector<TString>{""}, true), - std::tuple("a.b.c-d.e:1234", std::vector<TString>{"a.b.c-d.e:1234"}, true), - std::tuple("!a.b.c-d.e:1234", std::vector<TString>{"a.b.c-d.e:1234"}, false) + std::tuple("", std::vector<std::string>{}, true), + std::tuple("", std::vector<std::string>{"b"}, true), + std::tuple("a", std::vector<std::string>{"b"}, false), + std::tuple("!a", std::vector<std::string>{"b"}, true), + std::tuple("b", std::vector<std::string>{"b"}, true), + std::tuple("a|b", std::vector<std::string>{"b"}, true), + std::tuple("a & b", std::vector<std::string>{"b"}, false), + std::tuple("(b)", std::vector<std::string>{"b"}, true), + std::tuple("a|(a|b)", std::vector<std::string>{"b"}, true), + std::tuple("(a|b)&(!a&b)", std::vector<std::string>{"b"}, true), + std::tuple("a&b", std::vector<std::string>{"a", "b"}, true), + std::tuple("(a|c)&(b|c)", std::vector<std::string>{"a", "b"}, true), + std::tuple("(a|b)&c", std::vector<std::string>{"a", "b"}, false), + std::tuple("a|b|c", std::vector<std::string>{"b"}, true), + std::tuple("!a & b & !c", std::vector<std::string>{"b"}, true), + std::tuple("var-1 | !var/2", std::vector<std::string>{"var-1"}, true), + std::tuple("var-1 | !var/2", std::vector<std::string>{"var/2"}, false), + std::tuple("var-1 | !var/2", std::vector<std::string>{}, true), + std::tuple("!in-", std::vector<std::string>{}, true), + std::tuple("in/|x", std::vector<std::string>{"in/"}, true), + std::tuple("%true", std::vector<std::string>{""}, true), + std::tuple("%false", std::vector<std::string>{"false"}, false), + std::tuple("%true|%false", std::vector<std::string>{""}, true), + std::tuple("a.b.c-d.e:1234", std::vector<std::string>{"a.b.c-d.e:1234"}, true), + std::tuple("!a.b.c-d.e:1234", std::vector<std::string>{"a.b.c-d.e:1234"}, false) )); //////////////////////////////////////////////////////////////////////////////// @@ -165,7 +166,7 @@ TEST(TBooleanFormulaTest, ExternalOperators) auto aOrB = formulaA | formulaB; auto notA = !formulaA; - for (auto vars : std::vector<std::vector<TString>>{{}, {"a"}, {"b"}, {"a", "b"}}) { + for (auto vars : std::vector<std::vector<std::string>>{{}, {"a"}, {"b"}, {"a", "b"}}) { bool resA = formulaA.IsSatisfiedBy(vars); bool resB = formulaB.IsSatisfiedBy(vars); @@ -175,25 +176,25 @@ TEST(TBooleanFormulaTest, ExternalOperators) } EXPECT_FALSE((!MakeBooleanFormula("a | b")) - .IsSatisfiedBy(std::vector<TString>{"b"})); + .IsSatisfiedBy(std::vector<std::string>{"b"})); EXPECT_EQ((formulaA & formulaB).GetFormula(), "(a) & (b)"); EXPECT_EQ((formulaA | formulaB).GetFormula(), "(a) | (b)"); EXPECT_EQ((!formulaA).GetFormula(), "!(a)"); auto empty = MakeBooleanFormula(""); - EXPECT_TRUE(empty.IsSatisfiedBy(THashSet<TString>{})); - EXPECT_FALSE((!empty).IsSatisfiedBy(THashSet<TString>{})); - EXPECT_TRUE((empty | !empty).IsSatisfiedBy(THashSet<TString>{})); - - EXPECT_TRUE((empty | formulaA).IsSatisfiedBy(THashSet<TString>{})); - EXPECT_TRUE((empty | formulaA).IsSatisfiedBy(THashSet<TString>{"a"})); - EXPECT_TRUE((formulaA | empty).IsSatisfiedBy(THashSet<TString>{})); - EXPECT_TRUE((formulaA | empty).IsSatisfiedBy(THashSet<TString>{"a"})); - EXPECT_FALSE((empty & formulaA).IsSatisfiedBy(THashSet<TString>{})); - EXPECT_TRUE((empty & formulaA).IsSatisfiedBy(THashSet<TString>{"a"})); - EXPECT_FALSE((formulaA & empty).IsSatisfiedBy(THashSet<TString>{})); - EXPECT_TRUE((formulaA & empty).IsSatisfiedBy(THashSet<TString>{"a"})); + EXPECT_TRUE(empty.IsSatisfiedBy(THashSet<std::string>{})); + EXPECT_FALSE((!empty).IsSatisfiedBy(THashSet<std::string>{})); + EXPECT_TRUE((empty | !empty).IsSatisfiedBy(THashSet<std::string>{})); + + EXPECT_TRUE((empty | formulaA).IsSatisfiedBy(THashSet<std::string>{})); + EXPECT_TRUE((empty | formulaA).IsSatisfiedBy(THashSet<std::string>{"a"})); + EXPECT_TRUE((formulaA | empty).IsSatisfiedBy(THashSet<std::string>{})); + EXPECT_TRUE((formulaA | empty).IsSatisfiedBy(THashSet<std::string>{"a"})); + EXPECT_FALSE((empty & formulaA).IsSatisfiedBy(THashSet<std::string>{})); + EXPECT_TRUE((empty & formulaA).IsSatisfiedBy(THashSet<std::string>{"a"})); + EXPECT_FALSE((formulaA & empty).IsSatisfiedBy(THashSet<std::string>{})); + EXPECT_TRUE((formulaA & empty).IsSatisfiedBy(THashSet<std::string>{"a"})); } //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/core/net/connection.cpp b/yt/yt/core/net/connection.cpp index c18c6195c4..fa0c88d4d2 100644 --- a/yt/yt/core/net/connection.cpp +++ b/yt/yt/core/net/connection.cpp @@ -628,8 +628,7 @@ public: TFuture<void> Close() { - auto error = TError("Connection closed") - << TErrorAttribute("connection", Name_); + auto error = AnnotateError(TError("Connection closed")); return AbortIO(error); } @@ -665,12 +664,17 @@ public: return future; } - const TNetworkAddress& LocalAddress() const + TConnectionId GetId() const + { + return Id_; + } + + const TNetworkAddress& GetLocalAddress() const { return LocalAddress_; } - const TNetworkAddress& RemoteAddress() const + const TNetworkAddress& GetRemoteAddress() const { return RemoteAddress_; } @@ -746,7 +750,8 @@ public: } private: - const TString Name_; + const TConnectionId Id_ = TConnectionId::Create(); + const TString Endpoint_; const TString LoggingTag_; const TNetworkAddress LocalAddress_; const TNetworkAddress RemoteAddress_; @@ -768,7 +773,8 @@ private: TString filePath, const IPollerPtr& poller, bool useDeliveryFence) - : Name_(Format("File{%v}", filePath)) + : Endpoint_(Format("File{%v}", filePath)) + , LoggingTag_(Format("ConnectionId: %v %v", Id_, Endpoint_)) , FD_(fd) , Poller_(std::move(poller)) , UseDeliveryFence_(useDeliveryFence) @@ -780,8 +786,8 @@ private: const TNetworkAddress& localAddress, const TNetworkAddress& remoteAddress, IPollerPtr poller) - : Name_(Format("FD{%v<->%v}", localAddress, remoteAddress)) - , LoggingTag_(Format("ConnectionId: %v", Name_)) + : Endpoint_(Format("FD{%v<->%v}", localAddress, remoteAddress)) + , LoggingTag_(Format("ConnectionId: %v %v", Id_, Endpoint_)) , LocalAddress_(localAddress) , RemoteAddress_(remoteAddress) , FD_(fd) @@ -894,6 +900,13 @@ private: TDelayedExecutorCookie ReadTimeoutCookie_; TDelayedExecutorCookie WriteTimeoutCookie_; + TError AnnotateError(const TError& error) const + { + return error + << TErrorAttribute("connection_id", Id_) + << TErrorAttribute("connection_endpoint", Endpoint_); + } + TFuture<void> DoWrite(const TSharedRef& data) { auto write = std::make_unique<TWriteOperation>(data); @@ -959,8 +972,7 @@ private: if (error.IsOK()) { if (direction->Operation) { - THROW_ERROR_EXCEPTION("Another IO operation is in progress") - << TErrorAttribute("connection", Name_); + THROW_ERROR(AnnotateError(TError("Another IO operation is in progress"))); } YT_VERIFY(!direction->Running); @@ -1010,7 +1022,7 @@ private: if (result.IsOK()) { direction->BytesTransferred += result.Value().ByteCount; } else { - result = result << TErrorAttribute("connection", Name_); + result = AnnotateError(result); } bool needUnregister = false; @@ -1154,14 +1166,19 @@ public: YT_UNUSED_FUTURE(Impl_->Abort(TError("Connection is abandoned"))); } - const TNetworkAddress& LocalAddress() const override + TConnectionId GetId() const override + { + return Impl_->GetId(); + } + + const TNetworkAddress& GetLocalAddress() const override { - return Impl_->LocalAddress(); + return Impl_->GetLocalAddress(); } - const TNetworkAddress& RemoteAddress() const override + const TNetworkAddress& GetRemoteAddress() const override { - return Impl_->RemoteAddress(); + return Impl_->GetRemoteAddress(); } int GetHandle() const override diff --git a/yt/yt/core/net/connection.h b/yt/yt/core/net/connection.h index 24b28872d5..894b748222 100644 --- a/yt/yt/core/net/connection.h +++ b/yt/yt/core/net/connection.h @@ -68,8 +68,10 @@ struct IConnection : public IConnectionReader , public IConnectionWriter { - virtual const TNetworkAddress& LocalAddress() const = 0; - virtual const TNetworkAddress& RemoteAddress() const = 0; + virtual TConnectionId GetId() const = 0; + + virtual const TNetworkAddress& GetLocalAddress() const = 0; + virtual const TNetworkAddress& GetRemoteAddress() const = 0; // Returns true if connection is not is failed state and has no // active IO operations. diff --git a/yt/yt/core/net/dialer.cpp b/yt/yt/core/net/dialer.cpp index d21b9b0140..5bc8d3db92 100644 --- a/yt/yt/core/net/dialer.cpp +++ b/yt/yt/core/net/dialer.cpp @@ -1,6 +1,7 @@ #include "dialer.h" #include "connection.h" #include "config.h" +#include "private.h" #include <yt/yt/core/concurrency/pollable_detail.h> @@ -15,6 +16,10 @@ using namespace NConcurrency; //////////////////////////////////////////////////////////////////////////////// +static constexpr auto& Logger = NetLogger; + +//////////////////////////////////////////////////////////////////////////////// + class TDialSession : public TRefCounted { @@ -23,48 +28,55 @@ public: const TNetworkAddress& remoteAddress, const IAsyncDialerPtr& asyncDialer, IPollerPtr poller) - : Name_(Format("dialer[%v]", remoteAddress)) - , RemoteAddress_(remoteAddress) + : RemoteAddress_(remoteAddress) , Poller_(std::move(poller)) , Session_(asyncDialer->CreateSession( remoteAddress, BIND(&TDialSession::OnDialerFinished, MakeWeak(this)))) + { } + + TFuture<IConnectionPtr> Run() { + YT_LOG_DEBUG("Dial started (Address: %v)", + RemoteAddress_); + Session_->Dial(); Promise_.OnCanceled(BIND([this, this_ = MakeStrong(this)] (const TError& error) { Promise_.TrySet(TError(NYT::EErrorCode::Canceled, "Dial canceled") - << TErrorAttribute("dialer", Name_) + << TErrorAttribute("remote_address", ToString(RemoteAddress_)) << error); })); - } - TFuture<IConnectionPtr> GetFuture() const - { return Promise_.ToFuture(); } private: - const TString Name_; const TNetworkAddress RemoteAddress_; const IPollerPtr Poller_; const IAsyncDialerSessionPtr Session_; const TPromise<IConnectionPtr> Promise_ = NewPromise<IConnectionPtr>(); - void OnDialerFinished(const TErrorOr<SOCKET>& socketOrError) + void OnDialerFinished(const TErrorOr<TFileDescriptor>& fdOrError) { - if (socketOrError.IsOK()) { - auto socket = socketOrError.Value(); - Promise_.TrySet(CreateConnectionFromFD( - socket, - GetSocketName(socket), - RemoteAddress_, - Poller_)); - } else { - Promise_.TrySet(socketOrError - << TErrorAttribute("dialer", Name_)); + if (!fdOrError.IsOK()) { + Promise_.TrySet(TError("Dial failed") + << TErrorAttribute("remote_address", ToString(RemoteAddress_)) + << fdOrError); + return; } + + auto socket = fdOrError.Value(); + YT_LOG_DEBUG("Dial completed (Address: %v, FD: %v)", + RemoteAddress_, + socket); + + Promise_.TrySet(CreateConnectionFromFD( + socket, + GetSocketName(socket), + RemoteAddress_, + Poller_)); } }; @@ -86,14 +98,14 @@ public: { } TFuture<IConnectionPtr> Dial( - const TNetworkAddress& remote, + const TNetworkAddress& remoteAddress, TDialerContextPtr /*context*/) override { auto session = New<TDialSession>( - remote, + remoteAddress, AsyncDialer_, Poller_); - return session->GetFuture(); + return session->Run(); } private: diff --git a/yt/yt/core/net/dialer.h b/yt/yt/core/net/dialer.h index b48c782cf5..ac145d28a8 100644 --- a/yt/yt/core/net/dialer.h +++ b/yt/yt/core/net/dialer.h @@ -4,6 +4,8 @@ #include <yt/yt/core/net/address.h> +#include <yt/yt/core/misc/proc.h> + #include <library/cpp/yt/logging/public.h> #include <library/cpp/yt/memory/ref.h> @@ -14,8 +16,7 @@ namespace NYT::NNet { //////////////////////////////////////////////////////////////////////////////// -struct TDialerContext - : public TRefCounted +struct TDialerContext final { //! Host is used for TlsDialer. std::optional<TString> Host; @@ -24,12 +25,11 @@ struct TDialerContext DEFINE_REFCOUNTED_TYPE(TDialerContext) //! Dialer establishes connection to a (resolved) network address. - struct IDialer : public virtual TRefCounted { virtual TFuture<IConnectionPtr> Dial( - const TNetworkAddress& remote, + const TNetworkAddress& remoteAddress, TDialerContextPtr context = nullptr) = 0; }; @@ -43,7 +43,7 @@ IDialerPtr CreateDialer( //////////////////////////////////////////////////////////////////////////////// //! Async dialer notifies caller via callback for better performance. -using TAsyncDialerCallback = TCallback<void(const TErrorOr<SOCKET>&)>; +using TAsyncDialerCallback = TCallback<void(const TErrorOr<TFileDescriptor>&)>; //! Dialer session interface. //! Caller should hold a reference to a session until callback is called. diff --git a/yt/yt/core/net/local_address.cpp b/yt/yt/core/net/local_address.cpp index 03aa4e6650..541b152cdb 100644 --- a/yt/yt/core/net/local_address.cpp +++ b/yt/yt/core/net/local_address.cpp @@ -50,7 +50,7 @@ std::atomic<bool> IPv6Enabled_ = false; const char* ReadLocalHostName() noexcept { - NYT::NOrigin::EnableOriginOverrides(); + NYT::NDetail::EnableErrorOriginOverrides(); // Writer-side imposes AcqRel ordering, so all preceding writes must be visible. char* ptr = LocalHostNamePtr.load(std::memory_order::relaxed); return ptr ? ptr : LocalHostNameData; @@ -86,7 +86,7 @@ void GuardedWriteString(std::atomic<char*>& storage, char* initial, TStringBuf s void WriteLocalHostName(TStringBuf hostName) noexcept { - NYT::NOrigin::EnableOriginOverrides(); + NYT::NDetail::EnableErrorOriginOverrides(); static NThreading::TForkAwareSpinLock Lock; auto guard = Guard(Lock); diff --git a/yt/yt/core/net/public.h b/yt/yt/core/net/public.h index b28a2f65d2..fde7eea47a 100644 --- a/yt/yt/core/net/public.h +++ b/yt/yt/core/net/public.h @@ -12,6 +12,8 @@ class TNetworkAddress; class TIP6Address; class TIP6Network; +using TConnectionId = TGuid; + DECLARE_REFCOUNTED_STRUCT(IConnection) DECLARE_REFCOUNTED_STRUCT(IPacketConnection) DECLARE_REFCOUNTED_STRUCT(IConnectionReader) diff --git a/yt/yt/core/rpc/grpc/helpers.cpp b/yt/yt/core/rpc/grpc/helpers.cpp index a42d958a11..bfe01bf540 100644 --- a/yt/yt/core/rpc/grpc/helpers.cpp +++ b/yt/yt/core/rpc/grpc/helpers.cpp @@ -42,6 +42,11 @@ TGprString MakeGprString(char* str) return TGprString(str, gpr_free); } +TX509Ptr MakeX509Ptr(X509* cert) +{ + return TX509Ptr(cert, X509_free); +} + TStringBuf ToStringBuf(const grpc_slice& slice) { return TStringBuf( @@ -522,25 +527,21 @@ TGrpcServerCredentialsPtr LoadServerCredentials(const TServerCredentialsConfigPt nullptr)); } -std::optional<TString> ParseIssuerFromX509(TStringBuf x509String) +TX509Ptr ParsePemCertToX509(TStringBuf pemCert) { auto* bio = BIO_new(BIO_s_mem()); auto bioGuard = Finally([&] { BIO_free(bio); }); - BIO_write(bio, x509String.data(), x509String.length()); - - auto* x509 = PEM_read_bio_X509(bio, nullptr, nullptr, nullptr); - auto x509Guard = Finally([&] { - X509_free(x509); - }); + BIO_write(bio, pemCert.data(), pemCert.length()); - if (!x509) { - return std::nullopt; - } + return MakeX509Ptr(PEM_read_bio_X509(bio, nullptr, nullptr, nullptr)); +} - auto* issuerName = X509_get_issuer_name(x509); +std::optional<TString> ParseIssuerFromX509(const TX509Ptr& pemCertX509) +{ + auto* issuerName = X509_get_issuer_name(pemCertX509.get()); std::array<char, 1024> buf; auto* issuerString = X509_NAME_oneline(issuerName, buf.data(), buf.size()); @@ -551,6 +552,29 @@ std::optional<TString> ParseIssuerFromX509(TStringBuf x509String) return TString(issuerString); } +std::optional<TString> ParseSerialNumberFromX509(const TX509Ptr& pemCertX509) +{ + ASN1_STRING* serialNumber = X509_get_serialNumber(pemCertX509.get()); + if (!serialNumber) { + return std::nullopt; + } + BIGNUM* bn = ASN1_INTEGER_to_BN(serialNumber, nullptr); + auto bnGuard = Finally([&] { + BN_free(bn); + }); + if (!bn) { + return std::nullopt; + } + char* hexSerialNumber = BN_bn2hex(bn); + auto serialNumberGuard = Finally([&] { + OPENSSL_free(hexSerialNumber); + }); + if (!hexSerialNumber) { + return std::nullopt; + } + return TString(hexSerialNumber); +} + //////////////////////////////////////////////////////////////////////////////// TGuardedGrpcCompletionQueue::TGuardedGrpcCompletionQueue(TGrpcCompletionQueuePtr completionQueue) diff --git a/yt/yt/core/rpc/grpc/helpers.h b/yt/yt/core/rpc/grpc/helpers.h index 8c3d2ec565..b6b775c483 100644 --- a/yt/yt/core/rpc/grpc/helpers.h +++ b/yt/yt/core/rpc/grpc/helpers.h @@ -16,6 +16,8 @@ #include <contrib/libs/grpc/include/grpc/impl/codegen/grpc_types.h> #include <contrib/libs/grpc/include/grpc/byte_buffer_reader.h> +typedef struct x509_st X509; + namespace NYT::NRpc::NGrpc { //////////////////////////////////////////////////////////////////////////////// @@ -23,6 +25,9 @@ namespace NYT::NRpc::NGrpc { using TGprString = std::unique_ptr<char, void(*)(void*)>; TGprString MakeGprString(char* str); +using TX509Ptr = std::unique_ptr<X509, void(*)(X509*)>; +TX509Ptr MakeX509Ptr(X509* cert); + TStringBuf ToStringBuf(const grpc_slice& slice); //////////////////////////////////////////////////////////////////////////////// @@ -289,7 +294,9 @@ TError DeserializeError(TStringBuf serializedError); TGrpcPemKeyCertPair LoadPemKeyCertPair(const TSslPemKeyCertPairConfigPtr& config); TGrpcChannelCredentialsPtr LoadChannelCredentials(const TChannelCredentialsConfigPtr& config); TGrpcServerCredentialsPtr LoadServerCredentials(const TServerCredentialsConfigPtr& config); -std::optional<TString> ParseIssuerFromX509(TStringBuf x509String); +TX509Ptr ParsePemCertToX509(TStringBuf pemCert); +std::optional<TString> ParseIssuerFromX509(const TX509Ptr& pemCertX509); +std::optional<TString> ParseSerialNumberFromX509(const TX509Ptr& pemCertX509); //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/core/rpc/grpc/proto/grpc.proto b/yt/yt/core/rpc/grpc/proto/grpc.proto index 6ec01e35f9..cde26cec75 100644 --- a/yt/yt/core/rpc/grpc/proto/grpc.proto +++ b/yt/yt/core/rpc/grpc/proto/grpc.proto @@ -13,6 +13,7 @@ message TSslCredentialsExt optional string peer_identity = 1; optional string issuer = 2; + optional string serial_number = 3; } //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/core/rpc/grpc/server.cpp b/yt/yt/core/rpc/grpc/server.cpp index bb693aea89..c90f488740 100644 --- a/yt/yt/core/rpc/grpc/server.cpp +++ b/yt/yt/core/rpc/grpc/server.cpp @@ -779,7 +779,7 @@ private: std::optional<NGrpc::NProto::TSslCredentialsExt> sslCredentialsExtension; ParsePeerIdentity(authContext, &sslCredentialsExtension); - ParseIssuer(authContext, &sslCredentialsExtension); + ParseIssuerAndSerialNumber(authContext, &sslCredentialsExtension); return sslCredentialsExtension; } @@ -803,7 +803,7 @@ private: (*sslCredentialsExtension)->set_peer_identity(TString(peerIdentityProperty->value, peerIdentityProperty->value_length)); } - static void ParseIssuer(const TGrpcAuthContextPtr& authContext, std::optional<NGrpc::NProto::TSslCredentialsExt>* sslCredentialsExtension) + static void ParseIssuerAndSerialNumber(const TGrpcAuthContextPtr& authContext, std::optional<NGrpc::NProto::TSslCredentialsExt>* sslCredentialsExtension) { const char* peerIdentityPropertyName = grpc_auth_context_peer_identity_property_name(authContext.Unwrap()); if (!peerIdentityPropertyName) { @@ -816,15 +816,23 @@ private: return; } - auto issuer = ParseIssuerFromX509(TStringBuf(pemCertProperty->value, pemCertProperty->value_length)); - if (!issuer) { + auto pemCertX509 = ParsePemCertToX509(TStringBuf(pemCertProperty->value, pemCertProperty->value_length)); + if (!pemCertX509) { return; } - if (!sslCredentialsExtension->has_value()) { - sslCredentialsExtension->emplace(); + if (auto issuer = ParseIssuerFromX509(pemCertX509)) { + if (!sslCredentialsExtension->has_value()) { + sslCredentialsExtension->emplace(); + } + (*sslCredentialsExtension)->set_issuer(std::move(*issuer)); + } + if (auto serialNumber = ParseSerialNumberFromX509(pemCertX509)) { + if (!sslCredentialsExtension->has_value()) { + sslCredentialsExtension->emplace(); + } + (*sslCredentialsExtension)->set_serial_number(std::move(*serialNumber)); } - (*sslCredentialsExtension)->set_issuer(std::move(*issuer)); } void ParseTimeout() diff --git a/yt/yt/core/rpc/unittests/lib/test_service.cpp b/yt/yt/core/rpc/unittests/lib/test_service.cpp index f0994b55e3..afd60131da 100644 --- a/yt/yt/core/rpc/unittests/lib/test_service.cpp +++ b/yt/yt/core/rpc/unittests/lib/test_service.cpp @@ -20,6 +20,10 @@ using namespace NConcurrency; //////////////////////////////////////////////////////////////////////////////// +YT_DEFINE_GLOBAL(std::unique_ptr<NThreading::TEvent>, Latch_); + +//////////////////////////////////////////////////////////////////////////////// + class TTestService : public ITestService , public TServiceBase @@ -54,6 +58,10 @@ public: .SetQueueSizeLimit(20) .SetConcurrencyByteLimit(10_MB) .SetQueueByteSizeLimit(20_MB)); + RegisterMethod(RPC_SERVICE_METHOD_DESC(LatchedCall) + .SetCancelable(true) + .SetConcurrencyLimit(10) + .SetQueueSizeLimit(20)); RegisterMethod(RPC_SERVICE_METHOD_DESC(SlowCanceledCall) .SetCancelable(true)); RegisterMethod(RPC_SERVICE_METHOD_DESC(RequestBytesThrottledCall)); @@ -166,6 +174,15 @@ public: context->Reply(); } + DECLARE_RPC_SERVICE_METHOD(NTestRpc, LatchedCall) + { + context->SetRequestInfo(); + if (request->wait_on_latch()) { + Latch_()->Wait(); + } + context->Reply(); + } + DECLARE_RPC_SERVICE_METHOD(NTestRpc, SlowCanceledCall) { try { @@ -396,4 +413,29 @@ ITestServicePtr CreateTestService( //////////////////////////////////////////////////////////////////////////////// +void ReleaseLatchedCalls() +{ + if (!Latch_()) { + return; + } + + Latch_()->NotifyAll(); +} + +void MaybeInitLatch() +{ + if (!Latch_()) { + Latch_() = std::make_unique<NThreading::TEvent>(); + } +} + +void ResetLatch() +{ + if (Latch_()) { + Latch_().reset(); + } +} + +//////////////////////////////////////////////////////////////////////////////// + } // namespace NYT::NRpc diff --git a/yt/yt/core/rpc/unittests/lib/test_service.h b/yt/yt/core/rpc/unittests/lib/test_service.h index 3bc224eee8..bfb9ebd149 100644 --- a/yt/yt/core/rpc/unittests/lib/test_service.h +++ b/yt/yt/core/rpc/unittests/lib/test_service.h @@ -33,6 +33,7 @@ public: DEFINE_RPC_PROXY_METHOD(NTestRpc, NotRegistered); DEFINE_RPC_PROXY_METHOD(NTestRpc, SlowCall); DEFINE_RPC_PROXY_METHOD(NTestRpc, SlowCanceledCall); + DEFINE_RPC_PROXY_METHOD(NTestRpc, LatchedCall); DEFINE_RPC_PROXY_METHOD(NTestRpc, NoReply); DEFINE_RPC_PROXY_METHOD(NTestRpc, FlakyCall); DEFINE_RPC_PROXY_METHOD(NTestRpc, RequireCoolFeature); @@ -75,4 +76,10 @@ ITestServicePtr CreateTestService( //////////////////////////////////////////////////////////////////////////////// +void ReleaseLatchedCalls(); +void MaybeInitLatch(); +void ResetLatch(); + +//////////////////////////////////////////////////////////////////////////////// + } // namespace NYT::NRpc diff --git a/yt/yt/core/rpc/unittests/lib/test_service.proto b/yt/yt/core/rpc/unittests/lib/test_service.proto index 8ef02ef1f5..d126bea68f 100644 --- a/yt/yt/core/rpc/unittests/lib/test_service.proto +++ b/yt/yt/core/rpc/unittests/lib/test_service.proto @@ -130,6 +130,17 @@ message TRspSlowCanceledCall //////////////////////////////////////////////////////////////////////////////// +message TReqLatchedCall +{ + optional bool wait_on_latch = 1 [default = true]; +} + +message TRspLatchedCall +{ +} + +//////////////////////////////////////////////////////////////////////////////// + message TReqRequestBytesThrottledCall { } diff --git a/yt/yt/core/rpc/unittests/rpc_ut.cpp b/yt/yt/core/rpc/unittests/rpc_ut.cpp index 6b67d5f117..146a3cad28 100644 --- a/yt/yt/core/rpc/unittests/rpc_ut.cpp +++ b/yt/yt/core/rpc/unittests/rpc_ut.cpp @@ -787,6 +787,8 @@ TYPED_TEST(TRpcTest, SlowCall) TYPED_TEST(TRpcTest, RequestQueueSizeLimit) { + MaybeInitLatch(); + std::vector<TFuture<void>> futures; std::vector<TTestProxy> proxies; @@ -798,7 +800,7 @@ TYPED_TEST(TRpcTest, RequestQueueSizeLimit) } for (int i = 0; i <= 30; ++i) { - auto req = proxies[i].SlowCall(); + auto req = proxies[i].LatchedCall(); futures.push_back(req->Invoke().AsVoid()); } @@ -806,11 +808,14 @@ TYPED_TEST(TRpcTest, RequestQueueSizeLimit) { TTestProxy proxy(this->CreateChannel()); proxy.SetDefaultTimeout(TDuration::Seconds(60.0)); - auto req = proxy.SlowCall(); + auto req = proxy.LatchedCall(); EXPECT_EQ(NRpc::EErrorCode::RequestQueueSizeLimitExceeded, req->Invoke().Get().GetCode()); } + ReleaseLatchedCalls(); EXPECT_TRUE(AllSucceeded(std::move(futures)).Get().IsOK()); + + ResetLatch(); } TYPED_TEST(TNotGrpcTest, RequesMemoryPressureException) @@ -968,19 +973,15 @@ TYPED_TEST(TNotGrpcTest, RequestQueueByteSizeLimit) TYPED_TEST(TRpcTest, ConcurrencyLimit) { - auto shared_counter = std::make_shared<std::atomic<int>>(0); + MaybeInitLatch(); std::vector<TFuture<void>> futures; for (int i = 0; i < 10; ++i) { TTestProxy proxy(this->CreateChannel()); proxy.SetDefaultTimeout(TDuration::Seconds(10.0)); - auto req = proxy.SlowCall(); + auto req = proxy.LatchedCall(); futures.push_back( - req->Invoke() - .AsVoid() - .Apply(BIND([counter = shared_counter] { - counter->fetch_add(1); - }))); + req->Invoke().AsVoid()); } Sleep(TDuration::MilliSeconds(200)); @@ -988,18 +989,21 @@ TYPED_TEST(TRpcTest, ConcurrencyLimit) TFuture<void> backlogFuture; { TTestProxy proxy(this->CreateChannel()); - auto req = proxy.SlowCall(); + auto req = proxy.LatchedCall(); + req->set_wait_on_latch(false); backlogFuture = - req->Invoke() - .AsVoid() - .Apply(BIND([counter = shared_counter] { - EXPECT_EQ(counter->load(), 10); - })); + req->Invoke().AsVoid(); } + Sleep(TDuration::Seconds(2)); + EXPECT_FALSE(backlogFuture.IsSet()); + ReleaseLatchedCalls(); + EXPECT_TRUE(AllSucceeded(std::move(futures)).Get().IsOK()); EXPECT_TRUE(backlogFuture.Get().IsOK()); + + ResetLatch(); } TYPED_TEST(TRpcTest, NoReply) diff --git a/yt/yt/core/threading/thread.cpp b/yt/yt/core/threading/thread.cpp index e3222f8336..389cd88237 100644 --- a/yt/yt/core/threading/thread.cpp +++ b/yt/yt/core/threading/thread.cpp @@ -295,10 +295,10 @@ YT_PREVENT_TLS_CACHING void TThread::ConfigureSignalHandlerStack() // The size of of the custom stack to be provided for signal handlers. constexpr size_t SignalHandlerStackSize = 16_KB; - thread_local std::unique_ptr<char[]> Stack = std::make_unique<char[]>(SignalHandlerStackSize); + SignalHandlerStack_ = std::make_unique<char[]>(SignalHandlerStackSize); stack_t stack{ - .ss_sp = Stack.get(), + .ss_sp = SignalHandlerStack_.get(), .ss_flags = 0, .ss_size = SignalHandlerStackSize, }; diff --git a/yt/yt/core/threading/thread.h b/yt/yt/core/threading/thread.h index ac9c9885ec..83f40a2e41 100644 --- a/yt/yt/core/threading/thread.h +++ b/yt/yt/core/threading/thread.h @@ -75,6 +75,7 @@ private: TThreadId ThreadId_ = InvalidThreadId; ::TThread UnderlyingThread_; + std::unique_ptr<char[]> SignalHandlerStack_; void SetThreadPriority(); void ConfigureSignalHandlerStack(); diff --git a/yt/yt/core/tracing/trace_context.cpp b/yt/yt/core/tracing/trace_context.cpp index 1e3006b60b..b65e537a92 100644 --- a/yt/yt/core/tracing/trace_context.cpp +++ b/yt/yt/core/tracing/trace_context.cpp @@ -115,7 +115,7 @@ namespace NDetail { thread_local TTraceContext *CurrentTraceContextData{}; YT_PREVENT_TLS_CACHING TTraceContext*& CurrentTraceContext() { - NYT::NOrigin::EnableOriginOverrides(); + NYT::NDetail::EnableErrorOriginOverrides(); asm volatile(""); return CurrentTraceContextData; } diff --git a/yt/yt/core/ytree/attribute_consumer.cpp b/yt/yt/core/ytree/attribute_consumer.cpp index 7e28cb13f9..5fbe929182 100644 --- a/yt/yt/core/ytree/attribute_consumer.cpp +++ b/yt/yt/core/ytree/attribute_consumer.cpp @@ -9,23 +9,26 @@ using namespace NYson; //////////////////////////////////////////////////////////////////////////////// -TAttributeConsumer::TAttributeConsumer(IAttributeDictionary* attributes) - : Attributes(attributes) +TAttributeConsumer::TAttributeConsumer(IAttributeDictionary* attributes, std::optional<THashSet<TString>> keyWhitelist) + : Attributes_(attributes) + , KeyWhitelist_(std::move(keyWhitelist)) { } IAttributeDictionary* TAttributeConsumer::GetAttributes() const { - return Attributes; + return Attributes_; } void TAttributeConsumer::OnMyKeyedItem(TStringBuf key) { - Writer.reset(new TBufferedBinaryYsonWriter(&Output)); - Forward(Writer.get(), [this, key = TString(key)] { - Writer->Flush(); - Writer.reset(); - Attributes->SetYson(key, TYsonString(Output.Str())); - Output.clear(); + Writer_.reset(new TBufferedBinaryYsonWriter(&Output_)); + Forward(Writer_.get(), [this, key = TString(key)] { + Writer_->Flush(); + Writer_.reset(); + if (!KeyWhitelist_ || KeyWhitelist_->contains(key)) { + Attributes_->SetYson(key, TYsonString(Output_.Str())); + } + Output_.clear(); }); } diff --git a/yt/yt/core/ytree/attribute_consumer.h b/yt/yt/core/ytree/attribute_consumer.h index cbcaf031d2..4c1fd206df 100644 --- a/yt/yt/core/ytree/attribute_consumer.h +++ b/yt/yt/core/ytree/attribute_consumer.h @@ -14,7 +14,7 @@ class TAttributeConsumer : public NYson::TForwardingYsonConsumer { public: - explicit TAttributeConsumer(IAttributeDictionary* attributes); + explicit TAttributeConsumer(IAttributeDictionary* attributes, std::optional<THashSet<TString>> keyWhitelist = {}); IAttributeDictionary* GetAttributes() const; protected: @@ -33,10 +33,12 @@ protected: void OnMyEndAttributes()override; private: - IAttributeDictionary* const Attributes; + IAttributeDictionary* const Attributes_; - TStringStream Output; - std::unique_ptr<NYson::TBufferedBinaryYsonWriter> Writer; + const std::optional<THashSet<TString>> KeyWhitelist_; + + TStringStream Output_; + std::unique_ptr<NYson::TBufferedBinaryYsonWriter> Writer_; void ThrowMapExpected(); diff --git a/yt/yt/core/ytree/unittests/yson_schema_ut.cpp b/yt/yt/core/ytree/unittests/yson_schema_ut.cpp index 220e38975a..88df8268a2 100644 --- a/yt/yt/core/ytree/unittests/yson_schema_ut.cpp +++ b/yt/yt/core/ytree/unittests/yson_schema_ut.cpp @@ -157,6 +157,20 @@ struct TTestStructWithCustomType } }; +struct TTestStructWithUndefinedType + : public TYsonStruct +{ + NYT::NYTree::TYsonStructPtr UndefinedTypeField; + + REGISTER_YSON_STRUCT(TTestStructWithUndefinedType); + + static void Register(TRegistrar registrar) + { + registrar.Parameter("undefined_type_field", &TThis::UndefinedTypeField) + .Optional(); + } +}; + //////////////////////////////////////////////////////////////////////////////// void CheckSchema(const TYsonStructPtr& ysonStruct, TStringBuf expected) @@ -236,6 +250,30 @@ TEST(TYsonStructSchemaTest, TestYsonStructWithCustomType) ]})"); } +TEST(TYsonStructSchemaTest, TestYsonStructWithUndefinedType) +{ + auto ysonStruct = New<TTestStructWithUndefinedType>(); + CheckSchema( + ysonStruct, + R"({type_name="struct"; + members=[ + { + name="undefined_type_field"; + type={type_name="optional";item={type_name="struct";members=[];};}; + }; + ]})"); + ysonStruct->UndefinedTypeField = New<TTestSubStruct>(); + CheckSchema( + ysonStruct, + R"({type_name="struct"; + members=[ + { + name="undefined_type_field"; + type={type_name="optional";item={type_name="struct";members=[{name="my_uint";type="uint32";}]}}; + }; + ]})"); +} + //////////////////////////////////////////////////////////////////////////////// } // namespace diff --git a/yt/yt/core/ytree/virtual-inl.h b/yt/yt/core/ytree/virtual-inl.h index d3cf979930..6b24ab8306 100644 --- a/yt/yt/core/ytree/virtual-inl.h +++ b/yt/yt/core/ytree/virtual-inl.h @@ -44,13 +44,13 @@ public: return 0; } - std::vector<TString> GetKeys(i64 limit) const override + std::vector<std::string> GetKeys(i64 limit) const override { - std::vector<TString> keys; + std::vector<std::string> keys; if (auto collection = Collection_.lock()) { keys.reserve(limit); for (const auto& [key, value] : *collection) { - if (static_cast<i64>(keys.size()) >= limit) { + if (std::ssize(keys) >= limit) { break; } keys.emplace_back(TConversionTraits::ConvertKeyToString(key)); @@ -59,7 +59,7 @@ public: return keys; } - IYPathServicePtr FindItemService(TStringBuf key) const override + IYPathServicePtr FindItemService(const std::string& key) const override { if (auto collection = Collection_.lock()) { auto it = collection->find(TConversionTraits::ConvertStringToKey(key)); diff --git a/yt/yt/core/ytree/virtual.cpp b/yt/yt/core/ytree/virtual.cpp index 4cff5fa7bb..32292e5df4 100644 --- a/yt/yt/core/ytree/virtual.cpp +++ b/yt/yt/core/ytree/virtual.cpp @@ -168,6 +168,11 @@ void TVirtualMapBase::GetSelf( attributeFilter, limit); + if (limit < 0) { + THROW_ERROR_EXCEPTION("Limit is negative") + << TErrorAttribute("limit", limit); + } + auto keys = GetKeys(limit); i64 size = GetSize(); @@ -248,6 +253,11 @@ void TVirtualMapBase::ListSelf( attributeFilter, limit); + if (limit < 0) { + THROW_ERROR_EXCEPTION("Limit is negative") + << TErrorAttribute("limit", limit); + } + auto keys = GetKeys(limit); i64 size = GetSize(); @@ -358,15 +368,14 @@ class TCompositeMapService::TImpl : public TRefCounted { public: - std::vector<TString> GetKeys(i64 limit) const + std::vector<std::string> GetKeys(i64 limit) const { - std::vector<TString> keys; - int index = 0; - auto it = Services_.begin(); - while (it != Services_.end() && index < limit) { - keys.push_back(it->first); - ++it; - ++index; + std::vector<std::string> keys; + for (const auto& [key, _] : Services_) { + if (std::ssize(keys) >= limit) { + break; + } + keys.push_back(key); } return keys; } @@ -376,7 +385,7 @@ public: return Services_.size(); } - IYPathServicePtr FindItemService(TStringBuf key) const + IYPathServicePtr FindItemService(const std::string& key) const { auto it = Services_.find(key); return it != Services_.end() ? it->second : nullptr; @@ -423,7 +432,7 @@ TCompositeMapService::TCompositeMapService() TCompositeMapService::~TCompositeMapService() = default; -std::vector<TString> TCompositeMapService::GetKeys(i64 limit) const +std::vector<std::string> TCompositeMapService::GetKeys(i64 limit) const { return Impl_->GetKeys(limit); } @@ -433,7 +442,7 @@ i64 TCompositeMapService::GetSize() const return Impl_->GetSize(); } -IYPathServicePtr TCompositeMapService::FindItemService(TStringBuf key) const +IYPathServicePtr TCompositeMapService::FindItemService(const std::string& key) const { return Impl_->FindItemService(key); } diff --git a/yt/yt/core/ytree/virtual.h b/yt/yt/core/ytree/virtual.h index 0ed73bc9ad..a94b3cd9ee 100644 --- a/yt/yt/core/ytree/virtual.h +++ b/yt/yt/core/ytree/virtual.h @@ -31,11 +31,11 @@ protected: virtual std::optional<TVirtualCompositeNodeReadOffloadParams> GetReadOffloadParams() const; - virtual std::vector<TString> GetKeys(i64 limit = std::numeric_limits<i64>::max()) const = 0; + virtual std::vector<std::string> GetKeys(i64 limit = std::numeric_limits<i64>::max()) const = 0; virtual i64 GetSize() const = 0; - virtual IYPathServicePtr FindItemService(TStringBuf key) const = 0; + virtual IYPathServicePtr FindItemService(const std::string& key) const = 0; bool DoInvoke(const IYPathServiceContextPtr& context) override; @@ -74,9 +74,9 @@ public: TCompositeMapService(); ~TCompositeMapService(); - std::vector<TString> GetKeys(i64 limit = std::numeric_limits<i64>::max()) const override; + std::vector<std::string> GetKeys(i64 limit = std::numeric_limits<i64>::max()) const override; i64 GetSize() const override; - IYPathServicePtr FindItemService(TStringBuf key) const override; + IYPathServicePtr FindItemService(const std::string& key) const override; void ListSystemAttributes(std::vector<TAttributeDescriptor>* descriptors) override; bool GetBuiltinAttribute(TInternedAttributeKey key, NYson::IYsonConsumer* consumer) override; diff --git a/yt/yt/core/ytree/yson_schema-inl.h b/yt/yt/core/ytree/yson_schema-inl.h index 821048f19f..2f7b98912a 100644 --- a/yt/yt/core/ytree/yson_schema-inl.h +++ b/yt/yt/core/ytree/yson_schema-inl.h @@ -90,13 +90,32 @@ void WriteSchema(const T&, NYson::IYsonConsumer* consumer) } template <CYsonStructDerived T> +void WriteSchemaForNull(NYson::IYsonConsumer* consumer) +{ + if constexpr (std::is_same_v<T, TYsonStruct>) { + // It is not allowed to instantiate object of type `TYsonStruct`. + BuildYsonFluently(consumer) + .BeginMap() + .Item("type_name").Value("struct") + .Item("members").BeginList().EndList() + .EndMap(); + } else { + New<T>()->WriteSchema(consumer); + } +} + +template <CYsonStructDerived T> void WriteSchema(const NYT::TIntrusivePtr<T>& value, NYson::IYsonConsumer* consumer) { BuildYsonFluently(consumer) .BeginMap() .Item("type_name").Value("optional") .Item("item").Do([&] (auto fluent) { - (value ? value : New<T>())->WriteSchema(fluent.GetConsumer()); + if (value) { + value->WriteSchema(fluent.GetConsumer()); + } else { + WriteSchemaForNull<T>(fluent.GetConsumer()); + } }) .EndMap(); } diff --git a/yt/yt/library/profiling/solomon/sensor.h b/yt/yt/library/profiling/solomon/sensor.h index 3ac491ef59..c81055ea14 100644 --- a/yt/yt/library/profiling/solomon/sensor.h +++ b/yt/yt/library/profiling/solomon/sensor.h @@ -62,7 +62,7 @@ private: std::atomic<i64> Value_ = 0; }; -static_assert(sizeof(TSimpleCounter) == 24); +static_assert(sizeof(TSimpleCounter) == 32); //////////////////////////////////////////////////////////////////////////////// @@ -75,10 +75,10 @@ public: TDuration GetValue() override; private: - std::atomic<TDuration::TValue> Value_{0}; + std::atomic<TDuration::TValue> Value_ = 0; }; -static_assert(sizeof(TSimpleTimeCounter) == 24); +static_assert(sizeof(TSimpleTimeCounter) == 32); //////////////////////////////////////////////////////////////////////////////// @@ -108,7 +108,7 @@ class THistogram , public IHistogramImpl { public: - THistogram(const TSensorOptions& options); + explicit THistogram(const TSensorOptions& options); void Record(TDuration value) override; diff --git a/yt/yt_proto/yt/client/api/rpc_proxy/proto/api_service.proto b/yt/yt_proto/yt/client/api/rpc_proxy/proto/api_service.proto index 784f7c7da0..54321bcc96 100644 --- a/yt/yt_proto/yt/client/api/rpc_proxy/proto/api_service.proto +++ b/yt/yt_proto/yt/client/api/rpc_proxy/proto/api_service.proto @@ -638,6 +638,7 @@ message TReqSelectRows optional int32 execution_backend = 23; // EExecutionBackend optional NYT.NTableClient.NProto.TVersionedReadOptions versioned_read_options = 25; optional bool use_lookup_cache = 26; + optional int64 min_row_count_per_subquery = 27; optional TSuppressableAccessTrackingOptions suppressable_access_tracking_options = 104; @@ -836,6 +837,7 @@ message TReqExplainQuery optional string execution_pool = 11; optional bool new_range_inference = 12; optional int32 syntax_version = 13 [default = 1]; + optional int64 min_row_count_per_subquery = 14; } message TRspExplainQuery @@ -1986,6 +1988,7 @@ message TReqAlterTableReplica optional bool preserve_timestamps = 4; optional EAtomicity atomicity = 5; optional bool enable_replicated_table_tracker = 6; + optional string replica_path = 7; optional TMutatingOptions mutating_options = 100; } diff --git a/yt/yt_proto/yt/client/table_client/proto/versioned_io_options.proto b/yt/yt_proto/yt/client/table_client/proto/versioned_io_options.proto index 53e1ada19e..9172331d81 100644 --- a/yt/yt_proto/yt/client/table_client/proto/versioned_io_options.proto +++ b/yt/yt_proto/yt/client/table_client/proto/versioned_io_options.proto @@ -9,4 +9,9 @@ message TVersionedReadOptions required int32 read_mode = 1; // EVersionedIOMode } +message TVersionedWriteOptions +{ + required int32 write_mode = 1; // EVersionedIOMode +} + //////////////////////////////////////////////////////////////////////////////// |