diff options
| author | Alexander Smirnov <alex@ydb.tech> | 2025-03-04 16:15:41 +0000 |
|---|---|---|
| committer | Alexander Smirnov <alex@ydb.tech> | 2025-03-04 16:15:41 +0000 |
| commit | b21a377d1f5b24149cf65fd1f8feb44411ae38f9 (patch) | |
| tree | 0459a651275d60cf60489d8142f20a8bd5e6a199 | |
| parent | 827cd39b843ead1adfaa20f8a55e2e17da62a4eb (diff) | |
| parent | 00325857a11f51ad6b43a4d35f57e85e06866ab6 (diff) | |
| download | ydb-b21a377d1f5b24149cf65fd1f8feb44411ae38f9.tar.gz | |
Merge pull request #15307 from ydb-platform/merge-libs-250304-1328
165 files changed, 2825 insertions, 2403 deletions
diff --git a/build/conf/autoincludes.json b/build/conf/autoincludes.json index 3d47744839..c5db9dcd61 100644 --- a/build/conf/autoincludes.json +++ b/build/conf/autoincludes.json @@ -1,6 +1,7 @@ [ "devtools/ya", "library/cpp/geo", - "util" + "util", + "yql/essentials" ] diff --git a/build/conf/java.conf b/build/conf/java.conf index f76b0cfc90..8a06527ba7 100644 --- a/build/conf/java.conf +++ b/build/conf/java.conf @@ -404,7 +404,7 @@ macro _JAVA_SRCS(RESOURCES?"yes":"no", SRCDIR=".", PACKAGE_PREFIX="", EXCLUDE[], SET(VAR_SALT $SRCDIR $Globs $EXCLUDE $PACKAGE_PREFIX $RESOURCES $FILES) SET(JAR_SRCS_GLOB uniq_${hash:VAR_SALT}) _LATE_GLOB(${JAR_SRCS_GLOB} ${pre=${SRCDIR}/:Globs} EXCLUDE ${EXCLUDE}) - SET_APPEND(LINT_JAVA_SOURCES \${rootrel;input;ext=.java:${JAR_SRCS_GLOB}}) + SET_APPEND(LINT_JAVA_SOURCES \${rootrel;ext=.java;input:${JAR_SRCS_GLOB}}) SET_APPEND(ALL_JAR_SOURCES --jsources ${BINDIR}/misc/${tolower:JAR_SRCS_GLOB}.src.txt --resources ${BINDIR}/misc/${tolower:JAR_SRCS_GLOB}.res.txt --srcdir ${quo:SRCDIR} \${input:${JAR_SRCS_GLOB}} ${pre=\$\{input\:\";suf=\"\}:FILES}) _FILL_JAR_COPY_RESOURCES_CMD(LINK_JAR_RESOURCES ${quo:SRCDIR} ${BINDIR}/cls ${PACKAGE_PREFIX} ${BINDIR}/misc/${tolower:JAR_SRCS_GLOB}.res.txt) _FILL_JAR_COPY_RESOURCES_CMD(LINK_JAR_JSOURCES ${quo:SRCDIR} ${BINDIR}/src ${PACKAGE_PREFIX} ${BINDIR}/misc/${tolower:JAR_SRCS_GLOB}.src.txt) @@ -503,7 +503,7 @@ macro JAR_ANNOTATION_PROCESSOR(Classes...) { # tag:java-specific macro _JAR_ANN_PROC_OPTS(Classes...) { - .CMD=$_JAR_ANN_PROC_OPT_PREFIX ${join=,:Classes} $_USE_ANNOTATION_PROCESSOR_OPT + .CMD=$_JAR_ANN_PROC_OPT_PREFIX ${hideempty;join=,:Classes} $_USE_ANNOTATION_PROCESSOR_OPT } # tag:java-specific @@ -531,7 +531,8 @@ macro _NOOP_MACRO(Args...) { # tag:java-specific module _JAR_BASE: _BARE_UNIT { .NODE_TYPE=Bundle - .CMD=TOUCH_UNIT + .CMD=$TOUCH_UNIT + .STRUCT_CMD=yes .PEERDIR_POLICY=as_build_from .FINAL_TARGET=no .ALIASES=SRCS=_SRCS_NO_GLOBAL @@ -581,7 +582,7 @@ JAR_RESOURCE_ID= SRC_RESOURCE_ID= FETCH_SRCS_JAR= FETCH_TARGET_JAR= -FETCH_CONTRIB_JAR=${hide:JAVA_FAKEID} $FETCH_TARGET_JAR $FETCH_SRCS_JAR +FETCH_CONTRIB_JAR=${hide:JAVA_FAKEID} && $FETCH_TARGET_JAR && $FETCH_SRCS_JAR # tag:java-specific macro JAR_RESOURCE(Id) { @@ -613,14 +614,14 @@ _JAVA_CONTRIB_SEM= \ # tag:java-specific module JAVA_CONTRIB: _JAR_BASE { - .CMD=FETCH_CONTRIB_JAR + .CMD=$FETCH_CONTRIB_JAR .PEERDIR_POLICY=as_include .SEM=_JAVA_CONTRIB_SEM .FINAL_TARGET=yes .GLOBAL=MAVEN_EXPORT_COORDS when ($JAR_RESOURCE_ID) { - FETCH_TARGET_JAR= && $_FETCH_CONTRIB($JAR_RESOURCE_ID ${BINDIR}/${MODULE_PREFIX}${REALPRJNAME}${MODULE_SUFFIX}) + FETCH_TARGET_JAR= && $_FETCH_CONTRIB($JAR_RESOURCE_ID ${MODULE_PREFIX}${REALPRJNAME}${MODULE_SUFFIX}) } otherwise { when ($LOCAL_JAR_PATH) { @@ -770,14 +771,14 @@ macro _ADD_OPTS_IF_NON_EMPTY(Opt, Args...) { .CMD=${pre=$Opt :Args} } macro _ADD_GEN_POM_FROM_COORD_FILES_ARGS(Deps...) { - .CMD=${pre=--deps-coords ;ext=.jar;suf=.mvn_coords:Deps} + .CMD=${pre=--deps-coords ;suf=.mvn_coords;ext=.jar:Deps} } MAVEN_BIN=$MAVEN_RESOURCE_GLOBAL/bin/mvn MAVEN_EXPORT_OUT_DIR_FLAG=$_ADD_OPTS_IF_NON_EMPTY(--output-dir ${MAVEN_EXPORT_OUT_DIR}) MAVEN_EXPORT_SOURCE_DIRS=$_ADD_OPTS_IF_NON_EMPTY(--source-dirs ${ALL_SRCDIRS}) -MAVEN_EXPORT_DEPS_COORS=$_ADD_GEN_POM_FROM_COORD_FILES_ARGS(${MANAGED_PEERS_CLOSURE}) +MAVEN_EXPORT_DEPS_COORS=$_ADD_GEN_POM_FROM_COORD_FILES_ARGS($MANAGED_PEERS_CLOSURE) MAVEN_EXPORT_OUT_DIR= MAVEN_EXPORT=no @@ -1010,7 +1011,7 @@ _BUILD_PROTO_JAR_SEM= \ ### Reimplementation of the JAVA_LIBRARY with ymake.core.conf and ymake based dependency management module JAR_LIBRARY: _COMPILABLE_JAR_BASE { .EXTS=.jsrc .java .jar .mf .gentar .kt - .CMD=LINK_JAR + .CMD=$LINK_JAR .SEM=_BUILD_JAR_SEM .FINAL_TARGET=yes .ALIASES=JAVA_SRCS=FULL_JAVA_SRCS ANNOTATION_PROCESSOR=JAR_ANNOTATION_PROCESSOR @@ -1047,7 +1048,7 @@ module JAR_LIBRARY: _COMPILABLE_JAR_BASE { # For Kapt usage see: https://kotlinlang.org/docs/kapt.html#using-in-cli # See for kapt.kotlin.generated: https://github.com/JetBrains/kotlin/blob/master/plugins/kapt3/kapt3-cli/testData/integration/kotlinFileGeneration/build.txt _KAPT_OPTS=-Xplugin=${tool:"contrib/java/org/jetbrains/kotlin/kotlin-annotation-processing/1.9.24"} $KT_KAPT_PLUGIN_OPTS - _RUN_KAPT=${YMAKE_PYTHON} ${input:"build/scripts/with_kapt_args.py"} ${pre=--ap-classpath :KT_KAPT_AP_CLASSPATH} -- $COMPILE_KT $_KAPT_OPTS + _RUN_KAPT=${YMAKE_PYTHON} ${input:"build/scripts/with_kapt_args.py"} --ap-classpath ${KT_KAPT_AP_CLASSPATH} -- $COMPILE_KT $_KAPT_OPTS _APPEND_KAPT_GENERATED_SRCS=$YMAKE_PYTHON3 ${input:"build/scripts/resolve_java_srcs.py"} -d $KT_KAPT_SOURCES_DIR --include-patterns '**/*.java' '**/*.kt' --resolve-kotlin --append -s ${BINDIR}/all-java.srclst -k $KT_SRCLIST -r ${BINDIR}/not-used.txt ALL_KT_COMMANDS+=&& $_RUN_KAPT && $_APPEND_KAPT_GENERATED_SRCS @@ -1179,7 +1180,7 @@ module JAR_LIBRARY: _COMPILABLE_JAR_BASE { # tag:java-specific _SCRIPTGEN_FLAGS= macro _GEN_JAVA_SCRIPT_IMPL(Out, Template, Props...) { - .CMD=$SCRIPTGEN_RESOURCE_GLOBAL/scriptgen --java $JDK_RESOURCE/bin/java --output ${output:Out} --template ${input:Template} ${_SCRIPTGEN_FLAGS} -D JAR_NAME=${REALPRJNAME}.jar -D CLASSPATH=${join;pre="::";nopath;ext=.jar:MANAGED_PEERS_CLOSURE} -D PROJECT_DIR=${MODDIR} -D JAR_BASENAME=${REALPRJNAME} -D MAIN_CLASS=${_JAR_MAIN_CLASS} -D ENABLE_PREVIEW_VALUE=${ENABLE_PREVIEW_VALUE} $Props + .CMD=$SCRIPTGEN_RESOURCE_GLOBAL/scriptgen --java $JDK_RESOURCE/bin/java --output ${output:Out} --template ${input:Template} ${_SCRIPTGEN_FLAGS} -D JAR_NAME=${REALPRJNAME}.jar -D CLASSPATH=${join=;pre=\:\:;nopath;ext=.jar:MANAGED_PEERS_CLOSURE} -D PROJECT_DIR=${MODDIR} -D JAR_BASENAME=${REALPRJNAME} -D MAIN_CLASS=${_JAR_MAIN_CLASS} -D ENABLE_PREVIEW_VALUE=${ENABLE_PREVIEW_VALUE} $Props } # tag:java-specific @@ -1202,12 +1203,17 @@ when ($HOST_OS_WINDOWS == "yes") { GEN_JAVA_RUN_SH=$SCRIPTGEN_RESOURCE_GLOBAL/scriptgen --java $JDK_RESOURCE/bin/java --output ${output:_GEN_JAVA_RUN_SH_OUTPUT} \ -D GENERATE_DEFAULT_RUNNER=yes \ -D JAR_NAME=${REALPRJNAME}.jar \ - -D CLASSPATH=${join;pre="::";nopath:MANAGED_PEERS_CLOSURE} \ + -D CLASSPATH=${join=;pre=\:\:;nopath:MANAGED_PEERS_CLOSURE} \ -D PROJECT_DIR=${REALPRJNAME} \ -D JAR_BASENAME=${REALPRJNAME} \ -D MAIN_CLASS=${_JAR_MAIN_CLASS} \ -D ENABLE_PREVIEW=${ENABLE_PREVIEW_VALUE} GEN_RUN_CP=${WRITER_PY} --file ${BINDIR}/run-bf.txt -Q -m --ya-start-command-file ${qe;pre=$REALPRJNAME/;nopath:MANAGED_PEERS_CLOSURE} --ya-end-command-file && ${YMAKE_PYTHON} ${input:"build/scripts/make_manifest_from_bf.py"} ${BINDIR}/run-bf.txt ${TARGET} +COLLECT_JAR_PROGRAM_CP__LATEOUT__=\ + ${pre=$BINDIR/$REALPRJNAME/;nopath;ext=.jar:MANAGED_PEERS_CLOSURE} \ + ${pre=$BINDIR/$REALPRJNAME/;nopath;ext=.so:MANAGED_PEERS_CLOSURE} \ + ${pre=$BINDIR/$REALPRJNAME/;nopath;ext=.dll:MANAGED_PEERS_CLOSURE} \ + ${pre=$BINDIR/$REALPRJNAME/;nopath;ext=.dylib:MANAGED_PEERS_CLOSURE} COLLECT_JAR_PROGRAM_CP=$FS_TOOLS link_or_copy_to_dir \ --ya-start-command-file \ ${ext=.jar:MANAGED_PEERS_CLOSURE} \ @@ -1217,10 +1223,7 @@ COLLECT_JAR_PROGRAM_CP=$FS_TOOLS link_or_copy_to_dir \ ${_SOURCE_JARS} \ --ya-end-command-file \ ${BINDIR}/${REALPRJNAME} \ - ${hide;late_out;pre=$BINDIR/$REALPRJNAME/;nopath;ext=.jar:MANAGED_PEERS_CLOSURE} \ - ${hide;late_out;pre=$BINDIR/$REALPRJNAME/;nopath;ext=.so:MANAGED_PEERS_CLOSURE} \ - ${hide;late_out;pre=$BINDIR/$REALPRJNAME/;nopath;ext=.dll:MANAGED_PEERS_CLOSURE} \ - ${hide;late_out;pre=$BINDIR/$REALPRJNAME/;nopath;ext=.dylib:MANAGED_PEERS_CLOSURE} + ${hide;late_out:COLLECT_JAR_PROGRAM_CP__LATEOUT__} MAKE_JAR_PROGRAM_CPLST=${MAKE_JAVA_CLASSPATH_FILE} --from-args ${output;pre=$MODULE_PREFIX;suf=${MODULE_SUFFIX}.cplst:REALPRJNAME} --ya-start-command-file ${rootrel;pre=$BINDIR/$REALPRJNAME/;nopath;ext=.jar:MANAGED_PEERS_CLOSURE} ${_SOURCE_JARS_CPLIST} --ya-end-command-file TAR_CLASSPATH= && $YMAKE_PYTHON ${input:"build/scripts/find_and_tar.py"} ${output;pre=$MODULE_PREFIX;suf=.tar:REALPRJNAME} ${cwd;pre=$BINDIR/:REALPRJNAME} DO_TAR_CLASSPATH= @@ -1258,7 +1261,7 @@ macro _MARK_JAVA_PROG_WITH_SOURCES(Args...) { # tag:java-specific module _JAR_RUNNABLE: _COMPILABLE_JAR_BASE { .FINAL_TARGET=yes - .CMD=LINK_JAR_PROGRAM + .CMD=$LINK_JAR_PROGRAM .SEM=_SEM_IGNORED .ALIASES=JAVA_SRCS=_MARK_JAVA_PROG_WITH_SOURCES .ALLOWED=WITH_JDK GENERATE_SCRIPT @@ -1299,7 +1302,7 @@ module _JAR_RUNNABLE: _COMPILABLE_JAR_BASE { LINK_JAR_TEST=${hide:JAVA_FAKEID} ${WRITER_PY} --file ${BINDIR}/run-bf.txt -Q -m --ya-start-command-file ${ext=.jar:MANAGED_PEERS_CLOSURE} --ya-end-command-file && ${YMAKE_PYTHON} ${input:"build/scripts/make_manifest_from_bf.py"} ${BINDIR}/run-bf.txt ${TARGET} ${hide;kv:"p JT"} module _JAR_TEST: _COMPILABLE_JAR_BASE { .FINAL_TARGET=yes - .CMD=LINK_JAR_TEST + .CMD=$LINK_JAR_TEST .DEFAULT_NAME_GENERATOR=FullPath CONSUME_NON_MANAGEABLE_PEERS=yes diff --git a/build/conf/proto.conf b/build/conf/proto.conf index 356293d9d9..f79cd7ebdb 100644 --- a/build/conf/proto.conf +++ b/build/conf/proto.conf @@ -508,7 +508,7 @@ otherwise { KOTLIN_PROTO_FLAGS= # tag:proto tag:java-specific macro _JAVA_PROTO_CMD(File) { - .CMD=${cwd;rootdir;input:File} $YMAKE_PYTHON ${input:"build/scripts/tared_protoc.py"} --tar-output ${output;norel;nopath;noext;suf=.jsrc:File} --protoc-out-dir $ARCADIA_BUILD_ROOT/java_out $JAVA_PROTOC -I=./$PROTO_NAMESPACE ${pre=-I=:_PROTO__INCLUDE} -I=$ARCADIA_ROOT --java_out=${_JAVA_PROTO_LITE_ARG}$ARCADIA_BUILD_ROOT/java_out ${KOTLIN_PROTO_FLAGS} $_PROTOC_FLAGS ${input;rootrel:File} ${hide;kv:"p PB"} ${hide;kv:"pc yellow"} $JAVA_PROTO_ARGS ${hide:PROTO_FAKEID} + .CMD=${cwd;rootdir;input:File} $YMAKE_PYTHON ${input:"build/scripts/tared_protoc.py"} --tar-output ${output;norel;nopath;noext;suf=.jsrc:File} --protoc-out-dir $ARCADIA_BUILD_ROOT/java_out $JAVA_PROTOC -I=./$PROTO_NAMESPACE ${pre=-I=:_PROTO__INCLUDE} -I=$ARCADIA_ROOT --java_out=${_JAVA_PROTO_LITE_ARG}$ARCADIA_BUILD_ROOT/java_out ${KOTLIN_PROTO_FLAGS} $_PROTOC_FLAGS ${input;rootrel:File} ${hide;kv:"p PB"} ${hide;kv:"pc yellow"} $JAVA_PROTO_ARGS ${hide:PROTO_FAKEID} ${hide:"UID_BANHAMMER"} .SEM=proto_files ${input;rootrel:File} ${hide;output:File.jsrc} } diff --git a/build/export_generators/cmake/conanfile.py.jinja b/build/export_generators/cmake/conanfile.py.jinja index ca78850911..13717d007d 100644 --- a/build/export_generators/cmake/conanfile.py.jinja +++ b/build/export_generators/cmake/conanfile.py.jinja @@ -33,11 +33,13 @@ class App(ConanFile): {%- endif -%} {%- if (has_conan_os_depends_requires) -%} -{%- for conan_os_depend in conan.os_depends|selectattr('requires') %} -{%- if (conan_os_depend.requires|length) %} - if self.settings.os == "{{ conan_os_depend.os }}": -{%- for conan_require in conan_os_depend.requires %} - self.requires("{{ conan_require }}") +{%- set oses = conan.os_depends|selectattr('os')|map(attribute='os')|unique -%} +{%- for os in oses %} +{%- set os_requires = conan.os_depends|selectattr('os', 'eq', os)|selectattr('requires')|map(attribute='requires')|sum|unique -%} +{%- if (os_requires|length) %} + if self.settings.os == "{{ os }}": +{%- for os_require in os_requires %} + self.requires("{{ os_require }}") {%- endfor -%} {%- endif -%} {%- endfor -%} @@ -55,11 +57,13 @@ class App(ConanFile): {%- endif -%} {%- if (has_conan_os_depends_tool_requires) -%} -{%- for conan_os_depend in conan.os_depends|selectattr('tool_requires') %} -{%- if (conan_os_depend.tool_requires|length) %} - if self.settings.os == "{{ conan_os_depend.os }}": -{%- for conan_tool_require in conan_os_depend.tool_requires %} - self.tool_requires("{{ conan_tool_require }}") +{%- set oses = conan.os_depends|selectattr('os')|map(attribute='os')|unique -%} +{%- for os in oses %} +{%- set os_tool_requires = conan.os_depends|selectattr('os', 'eq', os)|selectattr('tool_requires')|map(attribute='tool_requires')|sum|unique -%} +{%- if (os_tool_requires|length) %} + if self.settings.os == "{{ os }}": +{%- for os_tool_require in os_tool_requires %} + self.requires("{{ os_tool_require }}") {%- endfor -%} {%- endif -%} {%- endfor -%} diff --git a/build/external_resources/ymake/public.resources.json b/build/external_resources/ymake/public.resources.json index 885ecc49de..2094162ad8 100644 --- a/build/external_resources/ymake/public.resources.json +++ b/build/external_resources/ymake/public.resources.json @@ -1,19 +1,19 @@ { "by_platform": { "darwin": { - "uri": "sbr:8103757184" + "uri": "sbr:8157966268" }, "darwin-arm64": { - "uri": "sbr:8103755745" + "uri": "sbr:8157965432" }, "linux": { - "uri": "sbr:8103760334" + "uri": "sbr:8157967911" }, "linux-aarch64": { - "uri": "sbr:8103754551" + "uri": "sbr:8157964700" }, "win32-clang-cl": { - "uri": "sbr:8103758621" + "uri": "sbr:8157967266" } } } diff --git a/build/external_resources/ymake/resources.json b/build/external_resources/ymake/resources.json index 934feca767..d52afb7ac7 100644 --- a/build/external_resources/ymake/resources.json +++ b/build/external_resources/ymake/resources.json @@ -1,19 +1,19 @@ { "by_platform": { "darwin": { - "uri": "sbr:8103761804" + "uri": "sbr:8157959616" }, "darwin-arm64": { - "uri": "sbr:8103760168" + "uri": "sbr:8157958728" }, "linux": { - "uri": "sbr:8103765600" + "uri": "sbr:8157961256" }, "linux-aarch64": { - "uri": "sbr:8103758578" + "uri": "sbr:8157958024" }, "win32-clang-cl": { - "uri": "sbr:8103763594" + "uri": "sbr:8157960421" } } } diff --git a/build/mapping.conf.json b/build/mapping.conf.json index 1e01a3d75c..51df82af27 100644 --- a/build/mapping.conf.json +++ b/build/mapping.conf.json @@ -509,6 +509,7 @@ "8029671029": "{registry_endpoint}/8029671029", "8067063302": "{registry_endpoint}/8067063302", "8119415565": "{registry_endpoint}/8119415565", + "8161441586": "{registry_endpoint}/8161441586", "5486731632": "{registry_endpoint}/5486731632", "5514350352": "{registry_endpoint}/5514350352", "5514360398": "{registry_endpoint}/5514360398", @@ -701,6 +702,7 @@ "8000013577": "{registry_endpoint}/8000013577", "8069587979": "{registry_endpoint}/8069587979", "8103757184": "{registry_endpoint}/8103757184", + "8157966268": "{registry_endpoint}/8157966268", "5766171800": "{registry_endpoint}/5766171800", "5805430761": "{registry_endpoint}/5805430761", "5829025456": "{registry_endpoint}/5829025456", @@ -768,6 +770,7 @@ "8000012780": "{registry_endpoint}/8000012780", "8069587058": "{registry_endpoint}/8069587058", "8103755745": "{registry_endpoint}/8103755745", + "8157965432": "{registry_endpoint}/8157965432", "5766173070": "{registry_endpoint}/5766173070", "5805432830": "{registry_endpoint}/5805432830", "5829031598": "{registry_endpoint}/5829031598", @@ -835,6 +838,7 @@ "8000014871": "{registry_endpoint}/8000014871", "8069590186": "{registry_endpoint}/8069590186", "8103760334": "{registry_endpoint}/8103760334", + "8157967911": "{registry_endpoint}/8157967911", "5766171341": "{registry_endpoint}/5766171341", "5805430188": "{registry_endpoint}/5805430188", "5829023352": "{registry_endpoint}/5829023352", @@ -902,6 +906,7 @@ "8000012216": "{registry_endpoint}/8000012216", "8069586009": "{registry_endpoint}/8069586009", "8103754551": "{registry_endpoint}/8103754551", + "8157964700": "{registry_endpoint}/8157964700", "5766172695": "{registry_endpoint}/5766172695", "5805432230": "{registry_endpoint}/5805432230", "5829029743": "{registry_endpoint}/5829029743", @@ -969,6 +974,7 @@ "8000014239": "{registry_endpoint}/8000014239", "8069588665": "{registry_endpoint}/8069588665", "8103758621": "{registry_endpoint}/8103758621", + "8157967266": "{registry_endpoint}/8157967266", "4307890075": "{registry_endpoint}/4307890075", "5517245192": "{registry_endpoint}/5517245192", "4307901240": "{registry_endpoint}/4307901240", @@ -1763,6 +1769,7 @@ "8029671029": "devtools/ya/test/programs/test_tool/bin/test_tool for linux", "8067063302": "devtools/ya/test/programs/test_tool/bin/test_tool for linux", "8119415565": "devtools/ya/test/programs/test_tool/bin/test_tool for linux", + "8161441586": "devtools/ya/test/programs/test_tool/bin/test_tool for linux", "5486731632": "devtools/ya/test/programs/test_tool/bin3/test_tool3 for linux", "5514350352": "devtools/ya/test/programs/test_tool/bin3/test_tool3 for linux", "5514360398": "devtools/ya/test/programs/test_tool/bin3/test_tool3 for linux", @@ -1955,6 +1962,7 @@ "8000013577": "devtools/ymake/bin/ymake for darwin", "8069587979": "devtools/ymake/bin/ymake for darwin", "8103757184": "devtools/ymake/bin/ymake for darwin", + "8157966268": "devtools/ymake/bin/ymake for darwin", "5766171800": "devtools/ymake/bin/ymake for darwin-arm64", "5805430761": "devtools/ymake/bin/ymake for darwin-arm64", "5829025456": "devtools/ymake/bin/ymake for darwin-arm64", @@ -2022,6 +2030,7 @@ "8000012780": "devtools/ymake/bin/ymake for darwin-arm64", "8069587058": "devtools/ymake/bin/ymake for darwin-arm64", "8103755745": "devtools/ymake/bin/ymake for darwin-arm64", + "8157965432": "devtools/ymake/bin/ymake for darwin-arm64", "5766173070": "devtools/ymake/bin/ymake for linux", "5805432830": "devtools/ymake/bin/ymake for linux", "5829031598": "devtools/ymake/bin/ymake for linux", @@ -2089,6 +2098,7 @@ "8000014871": "devtools/ymake/bin/ymake for linux", "8069590186": "devtools/ymake/bin/ymake for linux", "8103760334": "devtools/ymake/bin/ymake for linux", + "8157967911": "devtools/ymake/bin/ymake for linux", "5766171341": "devtools/ymake/bin/ymake for linux-aarch64", "5805430188": "devtools/ymake/bin/ymake for linux-aarch64", "5829023352": "devtools/ymake/bin/ymake for linux-aarch64", @@ -2156,6 +2166,7 @@ "8000012216": "devtools/ymake/bin/ymake for linux-aarch64", "8069586009": "devtools/ymake/bin/ymake for linux-aarch64", "8103754551": "devtools/ymake/bin/ymake for linux-aarch64", + "8157964700": "devtools/ymake/bin/ymake for linux-aarch64", "5766172695": "devtools/ymake/bin/ymake for win32-clang-cl", "5805432230": "devtools/ymake/bin/ymake for win32-clang-cl", "5829029743": "devtools/ymake/bin/ymake for win32-clang-cl", @@ -2223,6 +2234,7 @@ "8000014239": "devtools/ymake/bin/ymake for win32-clang-cl", "8069588665": "devtools/ymake/bin/ymake for win32-clang-cl", "8103758621": "devtools/ymake/bin/ymake for win32-clang-cl", + "8157967266": "devtools/ymake/bin/ymake for win32-clang-cl", "4307890075": "flake8_linter for linux", "5517245192": "flake8_linter for linux", "4307901240": "flake8_linter for linux-aarch64", diff --git a/build/platform/test_tool/host.ya.make.inc b/build/platform/test_tool/host.ya.make.inc index 0a710764cf..1d16478d48 100644 --- a/build/platform/test_tool/host.ya.make.inc +++ b/build/platform/test_tool/host.ya.make.inc @@ -1,12 +1,12 @@ IF (HOST_OS_DARWIN AND HOST_ARCH_X86_64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8119446934) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8161442120) ELSEIF (HOST_OS_DARWIN AND HOST_ARCH_ARM64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8119444678) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8161441534) ELSEIF (HOST_OS_LINUX AND HOST_ARCH_X86_64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8119452256) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8161443391) ELSEIF (HOST_OS_LINUX AND HOST_ARCH_AARCH64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8119442724) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8161440859) ELSEIF (HOST_OS_WINDOWS AND HOST_ARCH_X86_64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8119449312) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8161442654) ENDIF() diff --git a/build/platform/test_tool/host_os.ya.make.inc b/build/platform/test_tool/host_os.ya.make.inc index 39a59b4039..3f8266caf3 100644 --- a/build/platform/test_tool/host_os.ya.make.inc +++ b/build/platform/test_tool/host_os.ya.make.inc @@ -1,12 +1,12 @@ IF (HOST_OS_DARWIN AND HOST_ARCH_X86_64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8119412037) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8161440522) ELSEIF (HOST_OS_DARWIN AND HOST_ARCH_ARM64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8119409839) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8161440178) ELSEIF (HOST_OS_LINUX AND HOST_ARCH_X86_64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8119415565) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8161441586) ELSEIF (HOST_OS_LINUX AND HOST_ARCH_AARCH64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8119408227) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8161439551) ELSEIF (HOST_OS_WINDOWS AND HOST_ARCH_X86_64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8119413751) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8161440986) ENDIF() diff --git a/build/plugins/_dart_fields.py b/build/plugins/_dart_fields.py index e102154a34..6575f84ecf 100644 --- a/build/plugins/_dart_fields.py +++ b/build/plugins/_dart_fields.py @@ -670,7 +670,7 @@ class LintConfigs: class LintExtraParams: KEY = 'LINT-EXTRA-PARAMS' - _CUSTOM_CLANG_FORMAT_BIN_ALLOWED_PATHS = ('ads', 'bigrt', 'grut') + _CUSTOM_CLANG_FORMAT_BIN_ALLOWED_PATHS = ('ads', 'bigrt', 'grut', 'yabs') @classmethod def from_macro_args(cls, unit, flat_args, spec_args): @@ -1137,6 +1137,8 @@ class TestFiles: # https://a.yandex-team.ru/arcadia/devtools/ya/test/dartfile/__init__.py?rev=r14292146#L10 KEY2 = 'FILES' + # XXX: this is a workaround to support very specific linting settings. + # Do not use it as a general mechanism! _GRUT_PREFIX = 'grut' _GRUT_INCLUDE_LINTER_TEST_PATHS = ( 'grut/libs/bigrt/clients', @@ -1151,6 +1153,36 @@ class TestFiles: 'grut/libs/shooter', ) + # XXX: this is a workaround to support very specific linting settings. + # Do not use it as a general mechanism! + _MAPS_RENDERER_PREFIX = 'maps/renderer' + _MAPS_RENDERER_INCLUDE_LINTER_TEST_PATHS = ( + 'maps/renderer/cartograph', + 'maps/renderer/denormalization', + 'maps/renderer/libs/api', + 'maps/renderer/libs/data_sets/geojson_data_set', + 'maps/renderer/libs/data_sets/yt_data_set', + 'maps/renderer/libs/design', + 'maps/renderer/libs/geosx', + 'maps/renderer/libs/gltf', + 'maps/renderer/libs/golden', + 'maps/renderer/libs/hd3d', + 'maps/renderer/libs/image', + 'maps/renderer/libs/kv_storage', + 'maps/renderer/libs/marking', + 'maps/renderer/libs/mesh', + 'maps/renderer/libs/serializers', + 'maps/renderer/libs/style2', + 'maps/renderer/libs/style2_layer_bundle', + 'maps/renderer/libs/terrain', + 'maps/renderer/libs/vec', + 'maps/renderer/tilemill', + 'maps/renderer/tools/fontograph', + 'maps/renderer/tools/terrain_cli', + 'maps/renderer/tools/mapcheck2/lib', + 'maps/renderer/tools/mapcheck2/tests', + ) + @classmethod def value(cls, unit, flat_args, spec_args): data_re = re.compile(r"sbr:/?/?(\d+)=?.*") @@ -1251,6 +1283,14 @@ class TestFiles: break else: raise DartValueError() + + if upath.startswith(cls._MAPS_RENDERER_PREFIX): + for path in cls._MAPS_RENDERER_INCLUDE_LINTER_TEST_PATHS: + if os.path.commonpath([upath, path]) == path: + break + else: + raise DartValueError() + files_dart = _reference_group_var("ALL_SRCS", consts.STYLE_CPP_ALL_EXTS) return {cls.KEY: files_dart, cls.KEY2: files_dart} diff --git a/build/plugins/_requirements.py b/build/plugins/_requirements.py index 40c50f8791..c358914e0e 100644 --- a/build/plugins/_requirements.py +++ b/build/plugins/_requirements.py @@ -4,6 +4,7 @@ import lib._metric_resolvers as mr CANON_SB_VAULT_REGEX = re.compile(r"\w+=(value|file):[-\w]+:\w+") CANON_YAV_REGEX = re.compile(r"\w+=(value|file):sec-[a-z0-9]+:\w+") +PORTO_LAYERS_REGEX = re.compile(r"^\d+(,\d+)*$") VALID_DNS_REQUIREMENTS = ("default", "local", "dns64") VALID_NETWORK_REQUIREMENTS = ("full", "restricted") @@ -87,6 +88,11 @@ def validate_yav_vault(name, value): return "yav value '{}' should follow pattern <ENV_NAME>=<value|file>:<sec-id>:<key>".format(value) +def validate_porto_layers(name, value): + if not PORTO_LAYERS_REGEX.match(value): + return "porto layers '{}' should follow pattern porto_layers=<layerId1>[,<layerIdN>]*".format(value) + + def validate_numerical_requirement(name, value): if mr.resolve_value(value) is None: return "Cannot convert [[imp]]{}[[rst]] to the proper [[imp]]{}[[rst]] requirement value".format(value, name) @@ -133,6 +139,7 @@ def validate_requirement( ): req_checks = { 'container': validate_numerical_requirement, + 'porto_layers': validate_porto_layers, 'cpu': lambda n, v: validate_force_sandbox_requirement( n, v, test_size, is_force_sandbox, in_autocheck, is_fuzzing, is_kvm, is_ytexec_run, check_cpu ), @@ -166,7 +173,7 @@ def validate_requirement( req_name, ", ".join(sorted(req_checks)) ) - if req_name in ('container', 'disk') and not is_force_sandbox: + if req_name in ('container', 'disk', 'porto_layers') and not is_force_sandbox: return "Only [[imp]]LARGE[[rst]] tests without [[imp]]ya:force_distbuild[[rst]] tag can have [[imp]]{}[[rst]] requirement".format( req_name ) diff --git a/build/plugins/lib/nots/typescript/ts_config.py b/build/plugins/lib/nots/typescript/ts_config.py index 608e9b8de2..435ff4e78b 100644 --- a/build/plugins/lib/nots/typescript/ts_config.py +++ b/build/plugins/lib/nots/typescript/ts_config.py @@ -64,7 +64,7 @@ class TsConfig(object): def read(self): try: - with open(self.path, encoding="utf-8") as f: + with open(self.path) as f: self.data = self.rj.load(f, parse_mode=(self.rj.PM_COMMENTS | self.rj.PM_TRAILING_COMMAS)) except Exception as e: diff --git a/build/plugins/lib/test_const/__init__.py b/build/plugins/lib/test_const/__init__.py index 293b514d01..1319ec0ead 100644 --- a/build/plugins/lib/test_const/__init__.py +++ b/build/plugins/lib/test_const/__init__.py @@ -231,6 +231,7 @@ class TestRequirements(Enum): Dns = 'dns' Kvm = 'kvm' Network = 'network' + PortoLayers = 'porto_layers' Ram = 'ram' RamDisk = 'ram_disk' SbVault = 'sb_vault' @@ -450,6 +451,7 @@ class PythonLinterName(Enum): class CppLinterName(Enum): ClangFormat = "clang_format" ClangFormatYT = "clang_format_yt" + ClangFormat15 = "clang_format_15" class DefaultLinterConfig(Enum): @@ -464,6 +466,7 @@ class LinterConfigsValidationRules(Enum): LINTER_CONFIG_TYPES = { CppLinterName.ClangFormat: (".clang-format",), + CppLinterName.ClangFormat15: (".clang-format",), CppLinterName.ClangFormatYT: (".clang-format",), PythonLinterName.Black: ("pyproject.toml",), PythonLinterName.Ruff: ("pyproject.toml", "ruff.toml"), diff --git a/build/plugins/pybuild.py b/build/plugins/pybuild.py index 2a1add15ae..441579c561 100644 --- a/build/plugins/pybuild.py +++ b/build/plugins/pybuild.py @@ -170,6 +170,7 @@ def add_python_lint_checks(unit, py_ver, files): "passport/backend/oauth/", # PASSP-35982 "sdg/sdc/contrib/", # SDC contrib "sdg/sdc/third_party/", # SDC contrib + "smart_devices/third_party/", # smart_devices contrib "yt/yt/", # YT-20053 "yt/python/", # YT-20053 "yt/python_py2/", @@ -724,6 +725,8 @@ def onpy_register(unit, *args): unit.oncflags(['-DPyInit_{}=PyInit_{}'.format(shortname, mangle(name))]) else: unit.oncflags(['-Dinit{}=init{}'.format(shortname, mangle(name))]) + # BOOST_PYTHON_MODULE case + unit.oncflags(['-Dinit_module_{}=init_module_{}'.format(shortname, mangle(name))]) def py_main(unit, arg): diff --git a/build/plugins/ytest.py b/build/plugins/ytest.py index 4ae288e062..937c049aff 100644 --- a/build/plugins/ytest.py +++ b/build/plugins/ytest.py @@ -268,8 +268,11 @@ def validate_test(unit, kw): if in_autocheck and size == consts.TestSize.Large: errors.append("LARGE test must have ya:fat tag") - if consts.YaTestTags.Privileged in tags and 'container' not in requirements: - errors.append("Only tests with 'container' requirement can have 'ya:privileged' tag") + if 'container' in requirements and 'porto_layers' in requirements: + errors.append("Only one of 'container', 'porto_layers' can be set, not both") + + if consts.YaTestTags.Privileged in tags and 'container' not in requirements and 'porto_layers' not in requirements: + errors.append("Only tests with 'container' or 'porto_layers' requirement can have 'ya:privileged' tag") if size not in size_timeout: errors.append( diff --git a/build/ymake.core.conf b/build/ymake.core.conf index b96a5a412b..73e1c196cf 100644 --- a/build/ymake.core.conf +++ b/build/ymake.core.conf @@ -5906,3 +5906,7 @@ macro GENERATE_IMPLIB(Lib, Path, SONAME="") { LINK_EXCLUDE_LIBRARIES($Lib) } + +when ($OS_LINUX == "yes") { + YMAKE_USE_OBJCOPY=yes +} diff --git a/contrib/libs/cxxsupp/libcxx/.yandex_meta/build.ym b/contrib/libs/cxxsupp/libcxx/.yandex_meta/build.ym index 23f291c8a4..85545f7c95 100644 --- a/contrib/libs/cxxsupp/libcxx/.yandex_meta/build.ym +++ b/contrib/libs/cxxsupp/libcxx/.yandex_meta/build.ym @@ -5,7 +5,6 @@ {% block keep_sources %} .yandex_meta/scripts/sysincls.py -glibcxx_eh_cxx17.cpp include/__config_site include/__config_epilogue.h include/__memory/pointer_safety.h @@ -114,12 +113,7 @@ ELSEIF (OS_IOS) ELSEIF (OS_LINUX OR OS_DARWIN) IF (ARCH_ARM7) # libcxxrt support for ARM is currently broken, use libcxxabi instead - # But allow switching back to glibcxx_static via -DCXX_RT=glibcxx_static - DEFAULT(CXX_RT "libcxxabi") - # ARM7 OS_SDK has old libstdc++ without aligned allocation support - CFLAGS( - GLOBAL -fno-aligned-new - ) + SET(CXX_RT "libcxxabi") ELSE() SET(CXX_RT "libcxxrt") ENDIF() @@ -156,13 +150,8 @@ ENDIF() # * libcxxabi - https://github.com/llvm/llvm-project/tree/main/libcxxabi from Arcadia # * libcxxabi_dynamic - https://github.com/llvm/llvm-project/tree/main/libcxxabi from SDK, dynamically linked # * libcxxrt - https://github.com/libcxxrt/libcxxrt from Arcadia (with some parts hijacked from libcxxabi) -# * glibcxx_static - https://github.com/gcc-mirror/gcc/tree/master/libstdc++-v3/libsupc++ from SDK, statically linked # * msvcrt - Visual C++ runtime library from SDK, dynamically linked -DISABLE(NEED_GLIBCXX_CXX17_SHIMS) - -DISABLE(NEED_CXX_RT_ADDINCL) - IF (CXX_RT == "libcxxrt") PEERDIR( contrib/libs/cxxsupp/libcxxabi-parts @@ -175,20 +164,6 @@ IF (CXX_RT == "libcxxrt") CFLAGS( -DLIBCXXRT ) - # These builtins are equivalent to clang -rtlib=compiler_rt and - # are needed by potentially any code generated by clang. - # With glibcxx runtime, builtins are provided by libgcc -ELSEIF (CXX_RT == "glibcxx_static") - LDFLAGS( - -Wl,-Bstatic - -lsupc++ - -lgcc - -lgcc_eh - -Wl,-Bdynamic - ) - CXXFLAGS(-D__GLIBCXX__=1) - ENABLE(NEED_GLIBCXX_CXX17_SHIMS) - ENABLE(NEED_CXX_RT_ADDINCL) ELSEIF (CXX_RT == "libcxxabi") PEERDIR( contrib/libs/cxxsupp/builtins @@ -213,27 +188,6 @@ ELSE() MESSAGE(FATAL_ERROR "Unexpected CXX_RT value: \${CXX_RT}") ENDIF() -IF (NEED_GLIBCXX_CXX17_SHIMS) - IF (GCC) - # Assume GCC is bundled with a modern enough version of C++ runtime - ELSEIF (OS_SDK == "ubuntu-12" OR OS_SDK == "ubuntu-14" OR OS_SDK == "ubuntu-16") - # Prior to ubuntu-18, system C++ runtime for C++17 is incomplete - # and requires std::uncaught_exceptions() to be implemented. - SRCS( - glibcxx_eh_cxx17.cpp - ) - ENDIF() -ENDIF() - -IF (NEED_CXX_RT_ADDINCL) - # FIXME: - # This looks extremely weird and we have to use cxxabi.h from libsupc++ instead. - # This ADDINCL is placed here just to fix the status quo - ADDINCL( - GLOBAL contrib/libs/cxxsupp/libcxxrt/include - ) -ENDIF() - NO_UTIL() NO_RUNTIME() diff --git a/contrib/libs/cxxsupp/libcxx/glibcxx_eh_cxx17.cpp b/contrib/libs/cxxsupp/libcxx/glibcxx_eh_cxx17.cpp deleted file mode 100644 index 5e5ad083ba..0000000000 --- a/contrib/libs/cxxsupp/libcxx/glibcxx_eh_cxx17.cpp +++ /dev/null @@ -1,17 +0,0 @@ -#include <cxxabi.h> - -/** - * libc++ expects std::uncaught_exceptions() to be provided by C++ runtime library. - * - * GCC versions prior to GCC 6 did not provide this function yet, but it can be - * implemented using its API. - * - * This implementation should cover ubuntu-12, ubuntu-14, ubuntu-16. - */ - -namespace std { - int uncaught_exceptions() noexcept { - const auto* globals{__cxxabiv1::__cxa_get_globals()}; - return static_cast<int>(globals->uncaughtExceptions); - } -} diff --git a/contrib/libs/cxxsupp/libcxx/ya.make b/contrib/libs/cxxsupp/libcxx/ya.make index 122c16d4f7..c4b545164e 100644 --- a/contrib/libs/cxxsupp/libcxx/ya.make +++ b/contrib/libs/cxxsupp/libcxx/ya.make @@ -36,12 +36,7 @@ ELSEIF (OS_IOS) ELSEIF (OS_LINUX OR OS_DARWIN) IF (ARCH_ARM7) # libcxxrt support for ARM is currently broken, use libcxxabi instead - # But allow switching back to glibcxx_static via -DCXX_RT=glibcxx_static - DEFAULT(CXX_RT "libcxxabi") - # ARM7 OS_SDK has old libstdc++ without aligned allocation support - CFLAGS( - GLOBAL -fno-aligned-new - ) + SET(CXX_RT "libcxxabi") ELSE() SET(CXX_RT "libcxxrt") ENDIF() @@ -74,13 +69,8 @@ ENDIF() # * libcxxabi - https://github.com/llvm/llvm-project/tree/main/libcxxabi from Arcadia # * libcxxabi_dynamic - https://github.com/llvm/llvm-project/tree/main/libcxxabi from SDK, dynamically linked # * libcxxrt - https://github.com/libcxxrt/libcxxrt from Arcadia (with some parts hijacked from libcxxabi) -# * glibcxx_static - https://github.com/gcc-mirror/gcc/tree/master/libstdc++-v3/libsupc++ from SDK, statically linked # * msvcrt - Visual C++ runtime library from SDK, dynamically linked -DISABLE(NEED_GLIBCXX_CXX17_SHIMS) - -DISABLE(NEED_CXX_RT_ADDINCL) - IF (CXX_RT == "libcxxrt") PEERDIR( contrib/libs/cxxsupp/libcxxabi-parts @@ -93,20 +83,6 @@ IF (CXX_RT == "libcxxrt") CFLAGS( -DLIBCXXRT ) - # These builtins are equivalent to clang -rtlib=compiler_rt and - # are needed by potentially any code generated by clang. - # With glibcxx runtime, builtins are provided by libgcc -ELSEIF (CXX_RT == "glibcxx_static") - LDFLAGS( - -Wl,-Bstatic - -lsupc++ - -lgcc - -lgcc_eh - -Wl,-Bdynamic - ) - CXXFLAGS(-D__GLIBCXX__=1) - ENABLE(NEED_GLIBCXX_CXX17_SHIMS) - ENABLE(NEED_CXX_RT_ADDINCL) ELSEIF (CXX_RT == "libcxxabi") PEERDIR( contrib/libs/cxxsupp/builtins @@ -129,27 +105,6 @@ ELSE() MESSAGE(FATAL_ERROR "Unexpected CXX_RT value: ${CXX_RT}") ENDIF() -IF (NEED_GLIBCXX_CXX17_SHIMS) - IF (GCC) - # Assume GCC is bundled with a modern enough version of C++ runtime - ELSEIF (OS_SDK == "ubuntu-12" OR OS_SDK == "ubuntu-14" OR OS_SDK == "ubuntu-16") - # Prior to ubuntu-18, system C++ runtime for C++17 is incomplete - # and requires std::uncaught_exceptions() to be implemented. - SRCS( - glibcxx_eh_cxx17.cpp - ) - ENDIF() -ENDIF() - -IF (NEED_CXX_RT_ADDINCL) - # FIXME: - # This looks extremely weird and we have to use cxxabi.h from libsupc++ instead. - # This ADDINCL is placed here just to fix the status quo - ADDINCL( - GLOBAL contrib/libs/cxxsupp/libcxxrt/include - ) -ENDIF() - NO_UTIL() NO_RUNTIME() diff --git a/contrib/libs/poco/Foundation/include/Poco/BufferedStreamBuf.h b/contrib/libs/poco/Foundation/include/Poco/BufferedStreamBuf.h index 3e39215262..182d8085e7 100644 --- a/contrib/libs/poco/Foundation/include/Poco/BufferedStreamBuf.h +++ b/contrib/libs/poco/Foundation/include/Poco/BufferedStreamBuf.h @@ -25,6 +25,10 @@ #include <iosfwd> #include <ios> +namespace DB +{ +class ReadBufferFromIStream; +} namespace Poco { @@ -124,6 +128,8 @@ protected: } private: + friend class DB::ReadBufferFromIStream; + virtual int readFromDevice(char_type* /*buffer*/, std::streamsize /*length*/) { return 0; diff --git a/contrib/libs/re2/.yandex_meta/__init__.py b/contrib/libs/re2/.yandex_meta/__init__.py index cd11be23f4..3e7fe27381 100644 --- a/contrib/libs/re2/.yandex_meta/__init__.py +++ b/contrib/libs/re2/.yandex_meta/__init__.py @@ -83,6 +83,8 @@ re2 = CMakeNinjaNixProject( "include/re2": [ "re2/re2.h", "re2/stringpiece.h", + "re2/regexp.h", + "re2/walker-inl.h", ], "include/util": [ "util/logging.h", diff --git a/contrib/libs/re2/include/re2/re2.h b/contrib/libs/re2/include/re2/re2.h index 31cfa08363..a0c4744d8f 100644 --- a/contrib/libs/re2/include/re2/re2.h +++ b/contrib/libs/re2/include/re2/re2.h @@ -1 +1 @@ -#include "../../re2/re2.h" /* inclink generated by yamaker */ +#include "../../re2/re2.h" /* inclink generated by yamaker */
\ No newline at end of file diff --git a/contrib/libs/re2/include/re2/regexp.h b/contrib/libs/re2/include/re2/regexp.h new file mode 100644 index 0000000000..c1a566141c --- /dev/null +++ b/contrib/libs/re2/include/re2/regexp.h @@ -0,0 +1 @@ +#include "../../re2/regexp.h" /* inclink generated by yamaker */ diff --git a/contrib/libs/re2/include/re2/walker-inl.h b/contrib/libs/re2/include/re2/walker-inl.h new file mode 100644 index 0000000000..be1858437d --- /dev/null +++ b/contrib/libs/re2/include/re2/walker-inl.h @@ -0,0 +1 @@ +#include "../../re2/walker-inl.h" /* inclink generated by yamaker */ diff --git a/contrib/python/psutil/py3/.dist-info/METADATA b/contrib/python/psutil/py3/.dist-info/METADATA index eced53b9c3..685d2e23f3 100644 --- a/contrib/python/psutil/py3/.dist-info/METADATA +++ b/contrib/python/psutil/py3/.dist-info/METADATA @@ -1,7 +1,7 @@ Metadata-Version: 2.1 Name: psutil -Version: 6.1.1 -Summary: Cross-platform lib for process and system monitoring in Python. +Version: 7.0.0 +Summary: Cross-platform lib for process and system monitoring in Python. NOTE: the syntax of this script MUST be kept compatible with Python 2.7. Home-page: https://github.com/giampaolo/psutil Author: Giampaolo Rodola Author-email: g.rodola@gmail.com @@ -34,8 +34,6 @@ Classifier: Operating System :: POSIX :: Linux Classifier: Operating System :: POSIX :: SunOS/Solaris Classifier: Operating System :: POSIX Classifier: Programming Language :: C -Classifier: Programming Language :: Python :: 2 -Classifier: Programming Language :: Python :: 2.7 Classifier: Programming Language :: Python :: 3 Classifier: Programming Language :: Python :: Implementation :: CPython Classifier: Programming Language :: Python :: Implementation :: PyPy @@ -52,12 +50,15 @@ Classifier: Topic :: System :: Networking Classifier: Topic :: System :: Operating System Classifier: Topic :: System :: Systems Administration Classifier: Topic :: Utilities -Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.* +Requires-Python: >=3.6 Description-Content-Type: text/x-rst License-File: LICENSE Provides-Extra: dev +Requires-Dist: pytest ; extra == 'dev' +Requires-Dist: pytest-xdist ; extra == 'dev' +Requires-Dist: setuptools ; extra == 'dev' Requires-Dist: abi3audit ; extra == 'dev' -Requires-Dist: black ; extra == 'dev' +Requires-Dist: black (==24.10.0) ; extra == 'dev' Requires-Dist: check-manifest ; extra == 'dev' Requires-Dist: coverage ; extra == 'dev' Requires-Dist: packaging ; extra == 'dev' @@ -82,7 +83,7 @@ Requires-Dist: setuptools ; extra == 'test' | |downloads| |stars| |forks| |contributors| |coverage| | |version| |py-versions| |packages| |license| -| |github-actions-wheels| |github-actions-bsd| |appveyor| |doc| |twitter| |tidelift| +| |github-actions-wheels| |github-actions-bsd| |doc| |twitter| |tidelift| .. |downloads| image:: https://img.shields.io/pypi/dm/psutil.svg :target: https://pepy.tech/project/psutil @@ -108,10 +109,6 @@ Requires-Dist: setuptools ; extra == 'test' :target: https://github.com/giampaolo/psutil/actions?query=workflow%3Absd-tests :alt: FreeBSD, NetBSD, OpenBSD -.. |appveyor| image:: https://img.shields.io/appveyor/build/giampaolo/psutil/master.svg?maxAge=3600&label=Windows%20(py2) - :target: https://ci.appveyor.com/project/giampaolo/psutil - :alt: Windows (Appveyor) - .. |coverage| image:: https://coveralls.io/repos/github/giampaolo/psutil/badge.svg?branch=master :target: https://coveralls.io/github/giampaolo/psutil?branch=master :alt: Test coverage (coverall.io) @@ -177,8 +174,9 @@ psutil currently supports the following platforms: - **Sun Solaris** - **AIX** -Supported Python versions are **2.7**, **3.6+** and -`PyPy <http://pypy.org/>`__. +Supported Python versions are cPython 3.6+ and `PyPy <https://pypy.org/>`__. +Latest psutil version supporting Python 2.7 is +`psutil 6.1.1 <https://pypi.org/project/psutil/6.1.1/>`__. Funding ======= diff --git a/contrib/python/psutil/py3/README.rst b/contrib/python/psutil/py3/README.rst index 3fc6e601b1..16c756c50e 100644 --- a/contrib/python/psutil/py3/README.rst +++ b/contrib/python/psutil/py3/README.rst @@ -1,6 +1,6 @@ | |downloads| |stars| |forks| |contributors| |coverage| | |version| |py-versions| |packages| |license| -| |github-actions-wheels| |github-actions-bsd| |appveyor| |doc| |twitter| |tidelift| +| |github-actions-wheels| |github-actions-bsd| |doc| |twitter| |tidelift| .. |downloads| image:: https://img.shields.io/pypi/dm/psutil.svg :target: https://pepy.tech/project/psutil @@ -26,10 +26,6 @@ :target: https://github.com/giampaolo/psutil/actions?query=workflow%3Absd-tests :alt: FreeBSD, NetBSD, OpenBSD -.. |appveyor| image:: https://img.shields.io/appveyor/build/giampaolo/psutil/master.svg?maxAge=3600&label=Windows%20(py2) - :target: https://ci.appveyor.com/project/giampaolo/psutil - :alt: Windows (Appveyor) - .. |coverage| image:: https://coveralls.io/repos/github/giampaolo/psutil/badge.svg?branch=master :target: https://coveralls.io/github/giampaolo/psutil?branch=master :alt: Test coverage (coverall.io) @@ -98,8 +94,9 @@ psutil currently supports the following platforms: - **Sun Solaris** - **AIX** -Supported Python versions are **2.7**, **3.6+** and -`PyPy <http://pypy.org/>`__. +Supported Python versions are cPython 3.6+ and `PyPy <https://pypy.org/>`__. +Latest psutil version supporting Python 2.7 is +`psutil 6.1.1 <https://pypi.org/project/psutil/6.1.1/>`__. Funding ======= diff --git a/contrib/python/psutil/py3/patches/01-arcadia.patch b/contrib/python/psutil/py3/patches/01-arcadia.patch index 54b53d1db5..6a0b3aff1d 100644 --- a/contrib/python/psutil/py3/patches/01-arcadia.patch +++ b/contrib/python/psutil/py3/patches/01-arcadia.patch @@ -1,17 +1,3 @@ ---- contrib/python/psutil/py3/psutil/_pslinux.py (index) -+++ contrib/python/psutil/py3/psutil/_pslinux.py (working tree) -@@ -322,7 +322,10 @@ try: - except ImportError: - import ctypes - -- libc = ctypes.CDLL(None, use_errno=True) -+ try: -+ libc = ctypes.CDLL(None, use_errno=True) -+ except: -+ libc = None - - if hasattr(libc, "prlimit"): - --- contrib/python/psutil/py3/psutil/arch/windows/disk.c (index) +++ contrib/python/psutil/py3/psutil/arch/windows/disk.c (working tree) @@ -7,6 +7,7 @@ diff --git a/contrib/python/psutil/py3/psutil/__init__.py b/contrib/python/psutil/py3/psutil/__init__.py index 5648339382..cf4a58057f 100644 --- a/contrib/python/psutil/py3/psutil/__init__.py +++ b/contrib/python/psutil/py3/psutil/__init__.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -17,17 +15,16 @@ sensors) in Python. Supported platforms: - Sun Solaris - AIX -Works with Python versions 2.7 and 3.6+. +Supported Python versions are cPython 3.6+ and PyPy. """ -from __future__ import division - import collections import contextlib import datetime import functools import os import signal +import socket import subprocess import sys import threading @@ -54,16 +51,16 @@ from ._common import CONN_NONE from ._common import CONN_SYN_RECV from ._common import CONN_SYN_SENT from ._common import CONN_TIME_WAIT -from ._common import FREEBSD # NOQA +from ._common import FREEBSD from ._common import LINUX from ._common import MACOS -from ._common import NETBSD # NOQA +from ._common import NETBSD from ._common import NIC_DUPLEX_FULL from ._common import NIC_DUPLEX_HALF from ._common import NIC_DUPLEX_UNKNOWN -from ._common import OPENBSD # NOQA +from ._common import OPENBSD from ._common import OSX # deprecated alias -from ._common import POSIX # NOQA +from ._common import POSIX from ._common import POWER_TIME_UNKNOWN from ._common import POWER_TIME_UNLIMITED from ._common import STATUS_DEAD @@ -88,11 +85,6 @@ from ._common import ZombieProcess from ._common import debug from ._common import memoize_when_activated from ._common import wrap_numbers as _wrap_numbers -from ._compat import PY3 as _PY3 -from ._compat import PermissionError -from ._compat import ProcessLookupError -from ._compat import SubprocessTimeoutExpired as _SubprocessTimeoutExpired -from ._compat import long if LINUX: @@ -101,24 +93,24 @@ if LINUX: PROCFS_PATH = "/proc" from . import _pslinux as _psplatform - from ._pslinux import IOPRIO_CLASS_BE # NOQA - from ._pslinux import IOPRIO_CLASS_IDLE # NOQA - from ._pslinux import IOPRIO_CLASS_NONE # NOQA - from ._pslinux import IOPRIO_CLASS_RT # NOQA + from ._pslinux import IOPRIO_CLASS_BE # noqa: F401 + from ._pslinux import IOPRIO_CLASS_IDLE # noqa: F401 + from ._pslinux import IOPRIO_CLASS_NONE # noqa: F401 + from ._pslinux import IOPRIO_CLASS_RT # noqa: F401 elif WINDOWS: from . import _pswindows as _psplatform - from ._psutil_windows import ABOVE_NORMAL_PRIORITY_CLASS # NOQA - from ._psutil_windows import BELOW_NORMAL_PRIORITY_CLASS # NOQA - from ._psutil_windows import HIGH_PRIORITY_CLASS # NOQA - from ._psutil_windows import IDLE_PRIORITY_CLASS # NOQA - from ._psutil_windows import NORMAL_PRIORITY_CLASS # NOQA - from ._psutil_windows import REALTIME_PRIORITY_CLASS # NOQA - from ._pswindows import CONN_DELETE_TCB # NOQA - from ._pswindows import IOPRIO_HIGH # NOQA - from ._pswindows import IOPRIO_LOW # NOQA - from ._pswindows import IOPRIO_NORMAL # NOQA - from ._pswindows import IOPRIO_VERYLOW # NOQA + from ._psutil_windows import ABOVE_NORMAL_PRIORITY_CLASS # noqa: F401 + from ._psutil_windows import BELOW_NORMAL_PRIORITY_CLASS # noqa: F401 + from ._psutil_windows import HIGH_PRIORITY_CLASS # noqa: F401 + from ._psutil_windows import IDLE_PRIORITY_CLASS # noqa: F401 + from ._psutil_windows import NORMAL_PRIORITY_CLASS # noqa: F401 + from ._psutil_windows import REALTIME_PRIORITY_CLASS # noqa: F401 + from ._pswindows import CONN_DELETE_TCB # noqa: F401 + from ._pswindows import IOPRIO_HIGH # noqa: F401 + from ._pswindows import IOPRIO_LOW # noqa: F401 + from ._pswindows import IOPRIO_NORMAL # noqa: F401 + from ._pswindows import IOPRIO_VERYLOW # noqa: F401 elif MACOS: from . import _psosx as _psplatform @@ -128,8 +120,8 @@ elif BSD: elif SUNOS: from . import _pssunos as _psplatform - from ._pssunos import CONN_BOUND # NOQA - from ._pssunos import CONN_IDLE # NOQA + from ._pssunos import CONN_BOUND # noqa: F401 + from ._pssunos import CONN_IDLE # noqa: F401 # This is public writable API which is read from _pslinux.py and # _pssunos.py via sys.modules. @@ -143,7 +135,8 @@ elif AIX: PROCFS_PATH = "/proc" else: # pragma: no cover - raise NotImplementedError('platform %s is not supported' % sys.platform) + msg = f"platform {sys.platform} is not supported" + raise NotImplementedError(msg) # fmt: off @@ -214,8 +207,8 @@ if hasattr(_psplatform.Process, "rlimit"): AF_LINK = _psplatform.AF_LINK __author__ = "Giampaolo Rodola'" -__version__ = "6.1.1" -version_info = tuple([int(num) for num in __version__.split('.')]) +__version__ = "7.0.0" +version_info = tuple(int(num) for num in __version__.split('.')) _timer = getattr(time, 'monotonic', time.time) _TOTAL_PHYMEM = None @@ -231,22 +224,19 @@ _SENTINEL = object() if int(__version__.replace('.', '')) != getattr( _psplatform.cext, 'version', None ): - msg = "version conflict: %r C extension " % _psplatform.cext.__file__ + msg = f"version conflict: {_psplatform.cext.__file__!r} C extension " msg += "module was built for another version of psutil" if hasattr(_psplatform.cext, 'version'): - msg += " (%s instead of %s)" % ( - '.'.join([x for x in str(_psplatform.cext.version)]), - __version__, - ) + v = ".".join(list(str(_psplatform.cext.version))) + msg += f" ({v} instead of {__version__})" else: - msg += " (different than %s)" % __version__ - msg += "; you may try to 'pip uninstall psutil', manually remove %s" % ( - getattr( - _psplatform.cext, - "__file__", - "the existing psutil install directory", - ) + msg += f" (different than {__version__})" + what = getattr( + _psplatform.cext, + "__file__", + "the existing psutil install directory", ) + msg += f"; you may try to 'pip uninstall psutil', manually remove {what}" msg += " or clean the virtual env somehow, then reinstall" raise ImportError(msg) @@ -282,12 +272,20 @@ def _pprint_secs(secs): return datetime.datetime.fromtimestamp(secs).strftime(fmt) +def _check_conn_kind(kind): + """Check net_connections()'s `kind` parameter.""" + kinds = tuple(_common.conn_tmap) + if kind not in kinds: + msg = f"invalid kind argument {kind!r}; valid ones are: {kinds}" + raise ValueError(msg) + + # ===================================================================== # --- Process class # ===================================================================== -class Process(object): # noqa: UP004 +class Process: """Represents an OS process with the given PID. If PID is omitted current process PID (os.getpid()) is used. Raise NoSuchProcess if PID does not exist. @@ -322,17 +320,14 @@ class Process(object): # noqa: UP004 if pid is None: pid = os.getpid() else: - if not _PY3 and not isinstance(pid, (int, long)): - msg = "pid must be an integer (got %r)" % pid - raise TypeError(msg) if pid < 0: - msg = "pid must be a positive integer (got %s)" % pid + msg = f"pid must be a positive integer (got {pid})" raise ValueError(msg) try: _psplatform.cext.check_pid_range(pid) - except OverflowError: - msg = "process PID out of range (got %s)" % pid - raise NoSuchProcess(pid, msg=msg) + except OverflowError as err: + msg = "process PID out of range" + raise NoSuchProcess(pid, msg=msg) from err self._pid = pid self._name = None @@ -365,9 +360,8 @@ class Process(object): # noqa: UP004 except NoSuchProcess: if not _ignore_nsp: msg = "process PID not found" - raise NoSuchProcess(pid, msg=msg) - else: - self._gone = True + raise NoSuchProcess(pid, msg=msg) from None + self._gone = True def _get_ident(self): """Return a (pid, uid) tuple which is supposed to identify a @@ -419,10 +413,10 @@ class Process(object): # noqa: UP004 if self._create_time is not None: info['started'] = _pprint_secs(self._create_time) - return "%s.%s(%s)" % ( + return "{}.{}({})".format( self.__class__.__module__, self.__class__.__name__, - ", ".join(["%s=%r" % (k, v) for k, v in info.items()]), + ", ".join([f"{k}={v!r}" for k, v in info.items()]), ) __repr__ = __str__ @@ -556,12 +550,12 @@ class Process(object): # noqa: UP004 valid_names = _as_dict_attrnames if attrs is not None: if not isinstance(attrs, (list, tuple, set, frozenset)): - msg = "invalid attrs type %s" % type(attrs) + msg = f"invalid attrs type {type(attrs)}" raise TypeError(msg) attrs = set(attrs) invalid_names = attrs - valid_names if invalid_names: - msg = "invalid attr name%s %s" % ( + msg = "invalid attr name{} {}".format( "s" if len(invalid_names) > 1 else "", ", ".join(map(repr, invalid_names)), ) @@ -1049,7 +1043,7 @@ class Process(object): # noqa: UP004 """ blocking = interval is not None and interval > 0.0 if interval is not None and interval < 0: - msg = "interval is not positive (got %r)" % interval + msg = f"interval is not positive (got {interval!r})" raise ValueError(msg) num_cpus = cpu_count() or 1 @@ -1127,10 +1121,6 @@ class Process(object): # noqa: UP004 """ return self._proc.memory_info() - @_common.deprecated_method(replacement="memory_info") - def memory_info_ex(self): - return self.memory_info() - def memory_full_info(self): """This method returns the same information as memory_info(), plus, on some platform (Linux, macOS, Windows), also provides @@ -1159,9 +1149,9 @@ class Process(object): # noqa: UP004 """ valid_types = list(_psplatform.pfullmem._fields) if memtype not in valid_types: - msg = "invalid memtype %r; valid types are %r" % ( - memtype, - tuple(valid_types), + msg = ( + f"invalid memtype {memtype!r}; valid types are" + f" {tuple(valid_types)!r}" ) raise ValueError(msg) fun = ( @@ -1178,7 +1168,7 @@ class Process(object): # noqa: UP004 # we should never get here msg = ( "can't calculate process memory percent because total physical" - " system memory is not positive (%r)" % (total_phymem) + f" system memory is not positive ({total_phymem!r})" ) raise ValueError(msg) return (value / float(total_phymem)) * 100 @@ -1203,11 +1193,11 @@ class Process(object): # noqa: UP004 path = tupl[2] nums = tupl[3:] try: - d[path] = map(lambda x, y: x + y, d[path], nums) + d[path] = list(map(lambda x, y: x + y, d[path], nums)) except KeyError: d[path] = nums nt = _psplatform.pmmap_grouped - return [nt(path, *d[path]) for path in d] # NOQA + return [nt(path, *d[path]) for path in d] else: nt = _psplatform.pmmap_ext return [nt(*x) for x in it] @@ -1241,6 +1231,7 @@ class Process(object): # noqa: UP004 | all | the sum of all the possible families and protocols | +------------+----------------------------------------------------+ """ + _check_conn_kind(kind) return self._proc.net_connections(kind) @_common.deprecated_method(replacement="net_connections") @@ -1254,7 +1245,9 @@ class Process(object): # noqa: UP004 def _send_signal(self, sig): assert not self.pid < 0, self.pid self._raise_if_pid_reused() - if self.pid == 0: + + pid, ppid, name = self.pid, self._ppid, self._name + if pid == 0: # see "man 2 kill" msg = ( "preventing sending signal to process with PID 0 as it " @@ -1263,17 +1256,16 @@ class Process(object): # noqa: UP004 ) raise ValueError(msg) try: - os.kill(self.pid, sig) - except ProcessLookupError: - if OPENBSD and pid_exists(self.pid): + os.kill(pid, sig) + except ProcessLookupError as err: + if OPENBSD and pid_exists(pid): # We do this because os.kill() lies in case of # zombie processes. - raise ZombieProcess(self.pid, self._name, self._ppid) - else: - self._gone = True - raise NoSuchProcess(self.pid, self._name) - except PermissionError: - raise AccessDenied(self.pid, self._name) + raise ZombieProcess(pid, name, ppid) from err + self._gone = True + raise NoSuchProcess(pid, name) from err + except PermissionError as err: + raise AccessDenied(pid, name) from err def send_signal(self, sig): """Send a signal *sig* to process pre-emptively checking @@ -1358,11 +1350,12 @@ class Process(object): # noqa: UP004 # The valid attr names which can be processed by Process.as_dict(). # fmt: off -_as_dict_attrnames = set( - [x for x in dir(Process) if not x.startswith('_') and x not in +_as_dict_attrnames = { + x for x in dir(Process) if not x.startswith("_") and x not in {'send_signal', 'suspend', 'resume', 'terminate', 'kill', 'wait', 'is_running', 'as_dict', 'parent', 'parents', 'children', 'rlimit', - 'memory_info_ex', 'connections', 'oneshot'}]) + 'connections', 'oneshot'} +} # fmt: on @@ -1439,16 +1432,13 @@ class Popen(Process): try: return object.__getattribute__(self.__subproc, name) except AttributeError: - msg = "%s instance has no attribute '%s'" % ( - self.__class__.__name__, - name, - ) - raise AttributeError(msg) + msg = f"{self.__class__!r} has no attribute {name!r}" + raise AttributeError(msg) from None def wait(self, timeout=None): if self.__subproc.returncode is not None: return self.__subproc.returncode - ret = super(Popen, self).wait(timeout) # noqa + ret = super().wait(timeout) self.__subproc.returncode = ret return ret @@ -1525,7 +1515,7 @@ def process_iter(attrs=None, ad_value=None): remove(pid) while _pids_reused: pid = _pids_reused.pop() - debug("refreshing Process instance for reused PID %s" % pid) + debug(f"refreshing Process instance for reused PID {pid}") remove(pid) try: ls = sorted(list(pmap.items()) + list(dict.fromkeys(new_pids).items())) @@ -1542,7 +1532,7 @@ def process_iter(attrs=None, ad_value=None): _pmap = pmap -process_iter.cache_clear = lambda: _pmap.clear() # noqa +process_iter.cache_clear = lambda: _pmap.clear() # noqa: PLW0108 process_iter.cache_clear.__doc__ = "Clear process_iter() internal cache." @@ -1586,9 +1576,7 @@ def wait_procs(procs, timeout=None, callback=None): def check_gone(proc, timeout): try: returncode = proc.wait(timeout=timeout) - except TimeoutExpired: - pass - except _SubprocessTimeoutExpired: + except (TimeoutExpired, subprocess.TimeoutExpired): pass else: if returncode is not None or not proc.is_running(): @@ -1599,12 +1587,12 @@ def wait_procs(procs, timeout=None, callback=None): callback(proc) if timeout is not None and not timeout >= 0: - msg = "timeout must be a positive integer, got %s" % timeout + msg = f"timeout must be a positive integer, got {timeout}" raise ValueError(msg) gone = set() alive = set(procs) if callback is not None and not callable(callback): - msg = "callback %r is not a callable" % callback + msg = f"callback {callback!r} is not a callable" raise TypeError(msg) if timeout is not None: deadline = _timer() + timeout @@ -1627,7 +1615,7 @@ def wait_procs(procs, timeout=None, callback=None): check_gone(proc, timeout) else: check_gone(proc, max_timeout) - alive = alive - gone # noqa PLR6104 + alive = alive - gone # noqa: PLR6104 if alive: # Last attempt over processes survived so far. @@ -1646,7 +1634,7 @@ def wait_procs(procs, timeout=None, callback=None): def cpu_count(logical=True): """Return the number of logical CPUs in the system (same as - os.cpu_count() in Python 3.4). + os.cpu_count()). If *logical* is False return the number of physical cores only (e.g. hyper thread CPUs are excluded). @@ -1804,7 +1792,7 @@ def cpu_percent(interval=None, percpu=False): tid = threading.current_thread().ident blocking = interval is not None and interval > 0.0 if interval is not None and interval < 0: - msg = "interval is not positive (got %r)" % interval + msg = f"interval is not positive (got {interval})" raise ValueError(msg) def calculate(t1, t2): @@ -1864,7 +1852,7 @@ def cpu_times_percent(interval=None, percpu=False): tid = threading.current_thread().ident blocking = interval is not None and interval > 0.0 if interval is not None and interval < 0: - msg = "interval is not positive (got %r)" % interval + msg = f"interval is not positive (got {interval!r})" raise ValueError(msg) def calculate(t1, t2): @@ -2202,6 +2190,7 @@ def net_connections(kind='inet'): On macOS this function requires root privileges. """ + _check_conn_kind(kind) return _psplatform.net_connections(kind) @@ -2223,35 +2212,46 @@ def net_if_addrs(): Note: you can have more than one address of the same family associated with each interface. """ - has_enums = _PY3 - if has_enums: - import socket rawlist = _psplatform.net_if_addrs() rawlist.sort(key=lambda x: x[1]) # sort by family ret = collections.defaultdict(list) for name, fam, addr, mask, broadcast, ptp in rawlist: - if has_enums: - try: - fam = socket.AddressFamily(fam) - except ValueError: - if WINDOWS and fam == -1: - fam = _psplatform.AF_LINK - elif ( - hasattr(_psplatform, "AF_LINK") - and fam == _psplatform.AF_LINK - ): - # Linux defines AF_LINK as an alias for AF_PACKET. - # We re-set the family here so that repr(family) - # will show AF_LINK rather than AF_PACKET - fam = _psplatform.AF_LINK + try: + fam = socket.AddressFamily(fam) + except ValueError: + if WINDOWS and fam == -1: + fam = _psplatform.AF_LINK + elif ( + hasattr(_psplatform, "AF_LINK") and fam == _psplatform.AF_LINK + ): + # Linux defines AF_LINK as an alias for AF_PACKET. + # We re-set the family here so that repr(family) + # will show AF_LINK rather than AF_PACKET + fam = _psplatform.AF_LINK + if fam == _psplatform.AF_LINK: # The underlying C function may return an incomplete MAC # address in which case we fill it with null bytes, see: # https://github.com/giampaolo/psutil/issues/786 separator = ":" if POSIX else "-" while addr.count(separator) < 5: - addr += "%s00" % separator - ret[name].append(_common.snicaddr(fam, addr, mask, broadcast, ptp)) + addr += f"{separator}00" + + nt = _common.snicaddr(fam, addr, mask, broadcast, ptp) + + # On Windows broadcast is None, so we determine it via + # ipaddress module. + if WINDOWS and fam in {socket.AF_INET, socket.AF_INET6}: + try: + broadcast = _common.broadcast_addr(nt) + except Exception as err: # noqa: BLE001 + debug(err) + else: + if broadcast is not None: + nt._replace(broadcast=broadcast) + + ret[name].append(nt) + return dict(ret) @@ -2404,83 +2404,4 @@ def _set_debug(value): _psplatform.cext.set_debug(bool(value)) -def test(): # pragma: no cover - from ._common import bytes2human - from ._compat import get_terminal_size - - today_day = datetime.date.today() - # fmt: off - templ = "%-10s %5s %5s %7s %7s %5s %6s %6s %6s %s" - attrs = ['pid', 'memory_percent', 'name', 'cmdline', 'cpu_times', - 'create_time', 'memory_info', 'status', 'nice', 'username'] - print(templ % ("USER", "PID", "%MEM", "VSZ", "RSS", "NICE", # NOQA - "STATUS", "START", "TIME", "CMDLINE")) - # fmt: on - for p in process_iter(attrs, ad_value=None): - if p.info['create_time']: - ctime = datetime.datetime.fromtimestamp(p.info['create_time']) - if ctime.date() == today_day: - ctime = ctime.strftime("%H:%M") - else: - ctime = ctime.strftime("%b%d") - else: - ctime = '' - if p.info['cpu_times']: - cputime = time.strftime( - "%M:%S", time.localtime(sum(p.info['cpu_times'])) - ) - else: - cputime = '' - - user = p.info['username'] or '' - if not user and POSIX: - try: - user = p.uids()[0] - except Error: - pass - if user and WINDOWS and '\\' in user: - user = user.split('\\')[1] - user = user[:9] - vms = ( - bytes2human(p.info['memory_info'].vms) - if p.info['memory_info'] is not None - else '' - ) - rss = ( - bytes2human(p.info['memory_info'].rss) - if p.info['memory_info'] is not None - else '' - ) - memp = ( - round(p.info['memory_percent'], 1) - if p.info['memory_percent'] is not None - else '' - ) - nice = int(p.info['nice']) if p.info['nice'] else '' - if p.info['cmdline']: - cmdline = ' '.join(p.info['cmdline']) - else: - cmdline = p.info['name'] - status = p.info['status'][:5] if p.info['status'] else '' - - line = templ % ( - user[:10], - p.info['pid'], - memp, - vms, - rss, - nice, - status, - ctime, - cputime, - cmdline, - ) - print(line[: get_terminal_size()[0]]) # NOQA - - -del memoize_when_activated, division -if sys.version_info[0] < 3: - del num, x # noqa - -if __name__ == "__main__": - test() +del memoize_when_activated diff --git a/contrib/python/psutil/py3/psutil/_common.py b/contrib/python/psutil/py3/psutil/_common.py index 4b99b093e3..4096c0a18c 100644 --- a/contrib/python/psutil/py3/psutil/_common.py +++ b/contrib/python/psutil/py3/psutil/_common.py @@ -2,17 +2,14 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -"""Common objects shared by __init__.py and _ps*.py modules.""" +"""Common objects shared by __init__.py and _ps*.py modules. -# Note: this module is imported by setup.py so it should not import -# psutil or third-party modules. - -from __future__ import division -from __future__ import print_function +Note: this module is imported by setup.py, so it should not import +psutil or third-party modules. +""" import collections -import contextlib -import errno +import enum import functools import os import socket @@ -36,14 +33,6 @@ except ImportError: AF_UNIX = None -# can't take it from _common.py as this script is imported by setup.py -PY3 = sys.version_info[0] >= 3 -if PY3: - import enum -else: - enum = None - - PSUTIL_DEBUG = bool(os.getenv('PSUTIL_DEBUG')) _DEFAULT = object() @@ -57,7 +46,7 @@ __all__ = [ 'CONN_FIN_WAIT1', 'CONN_FIN_WAIT2', 'CONN_LAST_ACK', 'CONN_LISTEN', 'CONN_NONE', 'CONN_SYN_RECV', 'CONN_SYN_SENT', 'CONN_TIME_WAIT', # net constants - 'NIC_DUPLEX_FULL', 'NIC_DUPLEX_HALF', 'NIC_DUPLEX_UNKNOWN', + 'NIC_DUPLEX_FULL', 'NIC_DUPLEX_HALF', 'NIC_DUPLEX_UNKNOWN', # noqa: F822 # process status constants 'STATUS_DEAD', 'STATUS_DISK_SLEEP', 'STATUS_IDLE', 'STATUS_LOCKED', 'STATUS_RUNNING', 'STATUS_SLEEPING', 'STATUS_STOPPED', 'STATUS_SUSPENDED', @@ -134,42 +123,29 @@ CONN_LISTEN = "LISTEN" CONN_CLOSING = "CLOSING" CONN_NONE = "NONE" + # net_if_stats() -if enum is None: +class NicDuplex(enum.IntEnum): NIC_DUPLEX_FULL = 2 NIC_DUPLEX_HALF = 1 NIC_DUPLEX_UNKNOWN = 0 -else: - class NicDuplex(enum.IntEnum): - NIC_DUPLEX_FULL = 2 - NIC_DUPLEX_HALF = 1 - NIC_DUPLEX_UNKNOWN = 0 - globals().update(NicDuplex.__members__) +globals().update(NicDuplex.__members__) + # sensors_battery() -if enum is None: +class BatteryTime(enum.IntEnum): POWER_TIME_UNKNOWN = -1 POWER_TIME_UNLIMITED = -2 -else: - class BatteryTime(enum.IntEnum): - POWER_TIME_UNKNOWN = -1 - POWER_TIME_UNLIMITED = -2 - globals().update(BatteryTime.__members__) +globals().update(BatteryTime.__members__) # --- others ENCODING = sys.getfilesystemencoding() -if not PY3: - ENCODING_ERRS = "replace" -else: - try: - ENCODING_ERRS = sys.getfilesystemencodeerrors() # py 3.6 - except AttributeError: - ENCODING_ERRS = "surrogateescape" if POSIX else "replace" +ENCODING_ERRS = sys.getfilesystemencodeerrors() # =================================================================== @@ -273,7 +249,7 @@ if AF_INET6 is not None: "udp6": ([AF_INET6], [SOCK_DGRAM]), }) -if AF_UNIX is not None: +if AF_UNIX is not None and not SUNOS: conn_tmap.update({"unix": ([AF_UNIX], [SOCK_STREAM, SOCK_DGRAM])}) @@ -293,9 +269,7 @@ class Error(Exception): info = collections.OrderedDict() for name in attrs: value = getattr(self, name, None) - if value: # noqa - info[name] = value - elif name == "pid" and value == 0: + if value or (name == "pid" and value == 0): info[name] = value return info @@ -303,8 +277,8 @@ class Error(Exception): # invoked on `raise Error` info = self._infodict(("pid", "ppid", "name")) if info: - details = "(%s)" % ", ".join( - ["%s=%r" % (k, v) for k, v in info.items()] + details = "({})".format( + ", ".join([f"{k}={v!r}" for k, v in info.items()]) ) else: details = None @@ -313,8 +287,8 @@ class Error(Exception): def __repr__(self): # invoked on `repr(Error)` info = self._infodict(("pid", "ppid", "name", "seconds", "msg")) - details = ", ".join(["%s=%r" % (k, v) for k, v in info.items()]) - return "psutil.%s(%s)" % (self.__class__.__name__, details) + details = ", ".join([f"{k}={v!r}" for k, v in info.items()]) + return f"psutil.{self.__class__.__name__}({details})" class NoSuchProcess(Error): @@ -380,7 +354,7 @@ class TimeoutExpired(Error): self.seconds = seconds self.pid = pid self.name = name - self.msg = "timeout after %s seconds" % seconds + self.msg = f"timeout after {seconds} seconds" def __reduce__(self): return (self.__class__, (self.seconds, self.pid, self.name)) @@ -391,26 +365,6 @@ class TimeoutExpired(Error): # =================================================================== -# This should be in _compat.py rather than here, but does not work well -# with setup.py importing this module via a sys.path trick. -if PY3: - if isinstance(__builtins__, dict): # cpython - exec_ = __builtins__["exec"] - else: # pypy - exec_ = getattr(__builtins__, "exec") # noqa - - exec_("""def raise_from(value, from_value): - try: - raise value from from_value - finally: - value = None - """) -else: - - def raise_from(value, from_value): - raise value - - def usage_percent(used, total, round_=None): """Calculate percentage usage of 'used' against 'total'.""" try: @@ -456,7 +410,7 @@ def memoize(fun): try: ret = cache[key] = fun(*args, **kwargs) except Exception as err: # noqa: BLE001 - raise raise_from(err, None) + raise err from None return ret def cache_clear(): @@ -505,14 +459,14 @@ def memoize_when_activated(fun): try: return fun(self) except Exception as err: # noqa: BLE001 - raise raise_from(err, None) + raise err from None except KeyError: # case 3: we entered oneshot() ctx but there's no cache # for this entry yet try: ret = fun(self) except Exception as err: # noqa: BLE001 - raise raise_from(err, None) + raise err from None try: self._cache[fun] = ret except AttributeError: @@ -546,9 +500,9 @@ def isfile_strict(path): """ try: st = os.stat(path) - except OSError as err: - if err.errno in {errno.EPERM, errno.EACCES}: - raise + except PermissionError: + raise + except OSError: return False else: return stat.S_ISREG(st.st_mode) @@ -561,9 +515,9 @@ def path_exists_strict(path): """ try: os.stat(path) - except OSError as err: - if err.errno in {errno.EPERM, errno.EACCES}: - raise + except PermissionError: + raise + except OSError: return False else: return True @@ -575,11 +529,10 @@ def supports_ipv6(): if not socket.has_ipv6 or AF_INET6 is None: return False try: - sock = socket.socket(AF_INET6, socket.SOCK_STREAM) - with contextlib.closing(sock): + with socket.socket(AF_INET6, socket.SOCK_STREAM) as sock: sock.bind(("::1", 0)) return True - except socket.error: + except OSError: return False @@ -615,26 +568,20 @@ def sockfam_to_enum(num): """Convert a numeric socket family value to an IntEnum member. If it's not a known member, return the numeric value itself. """ - if enum is None: + try: + return socket.AddressFamily(num) + except ValueError: return num - else: # pragma: no cover - try: - return socket.AddressFamily(num) - except ValueError: - return num def socktype_to_enum(num): """Convert a numeric socket type value to an IntEnum member. If it's not a known member, return the numeric value itself. """ - if enum is None: + try: + return socket.SocketKind(num) + except ValueError: return num - else: # pragma: no cover - try: - return socket.SocketKind(num) - except ValueError: - return num def conn_to_ntuple(fd, fam, type_, laddr, raddr, status, status_map, pid=None): @@ -656,15 +603,37 @@ def conn_to_ntuple(fd, fam, type_, laddr, raddr, status, status_map, pid=None): return sconn(fd, fam, type_, laddr, raddr, status, pid) +def broadcast_addr(addr): + """Given the address ntuple returned by ``net_if_addrs()`` + calculates the broadcast address. + """ + import ipaddress + + if not addr.address or not addr.netmask: + return None + if addr.family == socket.AF_INET: + return str( + ipaddress.IPv4Network( + f"{addr.address}/{addr.netmask}", strict=False + ).broadcast_address + ) + if addr.family == socket.AF_INET6: + return str( + ipaddress.IPv6Network( + f"{addr.address}/{addr.netmask}", strict=False + ).broadcast_address + ) + + def deprecated_method(replacement): """A decorator which can be used to mark a method as deprecated 'replcement' is the method name which will be called instead. """ def outer(fun): - msg = "%s() is deprecated and will be removed; use %s() instead" % ( - fun.__name__, - replacement, + msg = ( + f"{fun.__name__}() is deprecated and will be removed; use" + f" {replacement}() instead" ) if fun.__doc__ is None: fun.__doc__ = msg @@ -789,8 +758,6 @@ wrap_numbers.cache_info = _wn.cache_info # is 8K. We use a bigger buffer (32K) in order to have more consistent # results when reading /proc pseudo files on Linux, see: # https://github.com/giampaolo/psutil/issues/2050 -# On Python 2 this also speeds up the reading of big files: -# (namely /proc/{pid}/smaps and /proc/net/*): # https://github.com/giampaolo/psutil/issues/708 FILE_READ_BUFFER_SIZE = 32 * 1024 @@ -800,17 +767,13 @@ def open_binary(fname): def open_text(fname): - """On Python 3 opens a file in text mode by using fs encoding and - a proper en/decoding errors handler. - On Python 2 this is just an alias for open(name, 'rt'). + """Open a file in text mode by using the proper FS encoding and + en/decoding error handlers. """ - if not PY3: - return open(fname, buffering=FILE_READ_BUFFER_SIZE) - # See: # https://github.com/giampaolo/psutil/issues/675 # https://github.com/giampaolo/psutil/pull/733 - fobj = open( + fobj = open( # noqa: SIM115 fname, buffering=FILE_READ_BUFFER_SIZE, encoding=ENCODING, @@ -842,7 +805,7 @@ def cat(fname, fallback=_DEFAULT, _open=open_text): try: with _open(fname) as f: return f.read() - except (IOError, OSError): + except OSError: return fallback @@ -852,7 +815,7 @@ def bcat(fname, fallback=_DEFAULT): def bytes2human(n, format="%(value).1f%(symbol)s"): - """Used by various scripts. See: http://goo.gl/zeJZl. + """Used by various scripts. See: https://code.activestate.com/recipes/578019-bytes-to-human-human-to-bytes-converter/?in=user-4178764. >>> bytes2human(10000) '9.8K' @@ -875,15 +838,8 @@ def get_procfs_path(): return sys.modules['psutil'].PROCFS_PATH -if PY3: - - def decode(s): - return s.decode(encoding=ENCODING, errors=ENCODING_ERRS) - -else: - - def decode(s): - return s +def decode(s): + return s.decode(encoding=ENCODING, errors=ENCODING_ERRS) # ===================================================================== @@ -927,13 +883,12 @@ def hilite(s, color=None, bold=False): # pragma: no cover try: color = colors[color] except KeyError: - raise ValueError( - "invalid color %r; choose between %s" % (list(colors.keys())) - ) + msg = f"invalid color {color!r}; choose amongst {list(colors.keys())}" + raise ValueError(msg) from None attr.append(color) if bold: attr.append('1') - return '\x1b[%sm%s\x1b[0m' % (';'.join(attr), s) + return f"\x1b[{';'.join(attr)}m{s}\x1b[0m" def print_color( @@ -941,9 +896,9 @@ def print_color( ): # pragma: no cover """Print a colorized version of string.""" if not term_supports_colors(): - print(s, file=file) # NOQA + print(s, file=file) elif POSIX: - print(hilite(s, color, bold), file=file) # NOQA + print(hilite(s, color, bold), file=file) else: import ctypes @@ -958,10 +913,11 @@ def print_color( try: color = colors[color] except KeyError: - raise ValueError( - "invalid color %r; choose between %r" - % (color, list(colors.keys())) + msg = ( + f"invalid color {color!r}; choose between" + f" {list(colors.keys())!r}" ) + raise ValueError(msg) from None if bold and color <= 7: color += 8 @@ -970,7 +926,7 @@ def print_color( handle = GetStdHandle(handle_id) SetConsoleTextAttribute(handle, color) try: - print(s, file=file) # NOQA + print(s, file=file) finally: SetConsoleTextAttribute(handle, DEFAULT_COLOR) @@ -984,11 +940,11 @@ def debug(msg): inspect.currentframe().f_back ) if isinstance(msg, Exception): - if isinstance(msg, (OSError, IOError, EnvironmentError)): + if isinstance(msg, OSError): # ...because str(exc) may contain info about the file name - msg = "ignoring %s" % msg + msg = f"ignoring {msg}" else: - msg = "ignoring %r" % msg - print( # noqa - "psutil-debug [%s:%s]> %s" % (fname, lineno, msg), file=sys.stderr + msg = f"ignoring {msg!r}" + print( # noqa: T201 + f"psutil-debug [{fname}:{lineno}]> {msg}", file=sys.stderr ) diff --git a/contrib/python/psutil/py3/psutil/_compat.py b/contrib/python/psutil/py3/psutil/_compat.py deleted file mode 100644 index 92e01713c3..0000000000 --- a/contrib/python/psutil/py3/psutil/_compat.py +++ /dev/null @@ -1,477 +0,0 @@ -# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -"""Module which provides compatibility with older Python versions. -This is more future-compatible rather than the opposite (prefer latest -Python 3 way of doing things). -""" - -import collections -import contextlib -import errno -import functools -import os -import sys -import types - - -# fmt: off -__all__ = [ - # constants - "PY3", - # builtins - "long", "range", "super", "unicode", "basestring", - # literals - "b", - # collections module - "lru_cache", - # shutil module - "which", "get_terminal_size", - # contextlib module - "redirect_stderr", - # python 3 exceptions - "FileNotFoundError", "PermissionError", "ProcessLookupError", - "InterruptedError", "ChildProcessError", "FileExistsError", -] -# fmt: on - - -PY3 = sys.version_info[0] >= 3 -_SENTINEL = object() - -if PY3: - long = int - xrange = range - unicode = str - basestring = str - range = range - - def b(s): - return s.encode("latin-1") - -else: - long = long - range = xrange - unicode = unicode - basestring = basestring - - def b(s): - return s - - -# --- builtins - - -# Python 3 super(). -# Taken from "future" package. -# Credit: Ryan Kelly -if PY3: - super = super -else: - _builtin_super = super - - def super(type_=_SENTINEL, type_or_obj=_SENTINEL, framedepth=1): - """Like Python 3 builtin super(). If called without any arguments - it attempts to infer them at runtime. - """ - if type_ is _SENTINEL: - f = sys._getframe(framedepth) - try: - # Get the function's first positional argument. - type_or_obj = f.f_locals[f.f_code.co_varnames[0]] - except (IndexError, KeyError): - msg = 'super() used in a function with no args' - raise RuntimeError(msg) - try: - # Get the MRO so we can crawl it. - mro = type_or_obj.__mro__ - except (AttributeError, RuntimeError): - try: - mro = type_or_obj.__class__.__mro__ - except AttributeError: - msg = 'super() used in a non-newstyle class' - raise RuntimeError(msg) - for type_ in mro: - # Find the class that owns the currently-executing method. - for meth in type_.__dict__.values(): - # Drill down through any wrappers to the underlying func. - # This handles e.g. classmethod() and staticmethod(). - try: - while not isinstance(meth, types.FunctionType): - if isinstance(meth, property): - # Calling __get__ on the property will invoke - # user code which might throw exceptions or - # have side effects - meth = meth.fget - else: - try: - meth = meth.__func__ - except AttributeError: - meth = meth.__get__(type_or_obj, type_) - except (AttributeError, TypeError): - continue - if meth.func_code is f.f_code: - break # found - else: - # Not found. Move onto the next class in MRO. - continue - break # found - else: - msg = 'super() called outside a method' - raise RuntimeError(msg) - - # Dispatch to builtin super(). - if type_or_obj is not _SENTINEL: - return _builtin_super(type_, type_or_obj) - return _builtin_super(type_) - - -# --- exceptions - - -if PY3: - FileNotFoundError = FileNotFoundError # NOQA - PermissionError = PermissionError # NOQA - ProcessLookupError = ProcessLookupError # NOQA - InterruptedError = InterruptedError # NOQA - ChildProcessError = ChildProcessError # NOQA - FileExistsError = FileExistsError # NOQA -else: - # https://github.com/PythonCharmers/python-future/blob/exceptions/ - # src/future/types/exceptions/pep3151.py - import platform - - def _instance_checking_exception(base_exception=Exception): - def wrapped(instance_checker): - class TemporaryClass(base_exception): - def __init__(self, *args, **kwargs): - if len(args) == 1 and isinstance(args[0], TemporaryClass): - unwrap_me = args[0] - for attr in dir(unwrap_me): - if not attr.startswith('__'): - setattr(self, attr, getattr(unwrap_me, attr)) - else: - super(TemporaryClass, self).__init__( # noqa - *args, **kwargs - ) - - class __metaclass__(type): - def __instancecheck__(cls, inst): - return instance_checker(inst) - - def __subclasscheck__(cls, classinfo): - value = sys.exc_info()[1] - return isinstance(value, cls) - - TemporaryClass.__name__ = instance_checker.__name__ - TemporaryClass.__doc__ = instance_checker.__doc__ - return TemporaryClass - - return wrapped - - @_instance_checking_exception(EnvironmentError) - def FileNotFoundError(inst): - return getattr(inst, 'errno', _SENTINEL) == errno.ENOENT - - @_instance_checking_exception(EnvironmentError) - def ProcessLookupError(inst): - return getattr(inst, 'errno', _SENTINEL) == errno.ESRCH - - @_instance_checking_exception(EnvironmentError) - def PermissionError(inst): - return getattr(inst, 'errno', _SENTINEL) in {errno.EACCES, errno.EPERM} - - @_instance_checking_exception(EnvironmentError) - def InterruptedError(inst): - return getattr(inst, 'errno', _SENTINEL) == errno.EINTR - - @_instance_checking_exception(EnvironmentError) - def ChildProcessError(inst): - return getattr(inst, 'errno', _SENTINEL) == errno.ECHILD - - @_instance_checking_exception(EnvironmentError) - def FileExistsError(inst): - return getattr(inst, 'errno', _SENTINEL) == errno.EEXIST - - if platform.python_implementation() != "CPython": - try: - raise OSError(errno.EEXIST, "perm") - except FileExistsError: - pass - except OSError: - msg = ( - "broken or incompatible Python implementation, see: " - "https://github.com/giampaolo/psutil/issues/1659" - ) - raise RuntimeError(msg) - - -# --- stdlib additions - - -# py 3.2 functools.lru_cache -# Taken from: http://code.activestate.com/recipes/578078 -# Credit: Raymond Hettinger -try: - from functools import lru_cache -except ImportError: - try: - from threading import RLock - except ImportError: - from dummy_threading import RLock - - _CacheInfo = collections.namedtuple( - "CacheInfo", ["hits", "misses", "maxsize", "currsize"] - ) - - class _HashedSeq(list): # noqa: FURB189 - __slots__ = ('hashvalue',) - - def __init__(self, tup, hash=hash): - self[:] = tup - self.hashvalue = hash(tup) - - def __hash__(self): - return self.hashvalue - - def _make_key( - args, - kwds, - typed, - kwd_mark=(_SENTINEL,), - fasttypes=set((int, str, frozenset, type(None))), # noqa - sorted=sorted, - tuple=tuple, - type=type, - len=len, - ): - key = args - if kwds: - sorted_items = sorted(kwds.items()) - key += kwd_mark - for item in sorted_items: - key += item - if typed: - key += tuple(type(v) for v in args) - if kwds: - key += tuple(type(v) for k, v in sorted_items) - elif len(key) == 1 and type(key[0]) in fasttypes: - return key[0] - return _HashedSeq(key) - - def lru_cache(maxsize=100, typed=False): - """Least-recently-used cache decorator, see: - http://docs.python.org/3/library/functools.html#functools.lru_cache. - """ - - def decorating_function(user_function): - cache = {} - stats = [0, 0] - HITS, MISSES = 0, 1 - make_key = _make_key - cache_get = cache.get - _len = len - lock = RLock() - root = [] - root[:] = [root, root, None, None] - nonlocal_root = [root] - PREV, NEXT, KEY, RESULT = 0, 1, 2, 3 - if maxsize == 0: - - def wrapper(*args, **kwds): - result = user_function(*args, **kwds) - stats[MISSES] += 1 - return result - - elif maxsize is None: - - def wrapper(*args, **kwds): - key = make_key(args, kwds, typed) - result = cache_get(key, root) - if result is not root: - stats[HITS] += 1 - return result - result = user_function(*args, **kwds) - cache[key] = result - stats[MISSES] += 1 - return result - - else: - - def wrapper(*args, **kwds): - if kwds or typed: - key = make_key(args, kwds, typed) - else: - key = args - lock.acquire() - try: - link = cache_get(key) - if link is not None: - (root,) = nonlocal_root - link_prev, link_next, key, result = link - link_prev[NEXT] = link_next - link_next[PREV] = link_prev - last = root[PREV] - last[NEXT] = root[PREV] = link - link[PREV] = last - link[NEXT] = root - stats[HITS] += 1 - return result - finally: - lock.release() - result = user_function(*args, **kwds) - lock.acquire() - try: - (root,) = nonlocal_root - if key in cache: - pass - elif _len(cache) >= maxsize: - oldroot = root - oldroot[KEY] = key - oldroot[RESULT] = result - root = nonlocal_root[0] = oldroot[NEXT] - oldkey = root[KEY] - root[KEY] = root[RESULT] = None - del cache[oldkey] - cache[key] = oldroot - else: - last = root[PREV] - link = [last, root, key, result] - last[NEXT] = root[PREV] = cache[key] = link - stats[MISSES] += 1 - finally: - lock.release() - return result - - def cache_info(): - """Report cache statistics.""" - lock.acquire() - try: - return _CacheInfo( - stats[HITS], stats[MISSES], maxsize, len(cache) - ) - finally: - lock.release() - - def cache_clear(): - """Clear the cache and cache statistics.""" - lock.acquire() - try: - cache.clear() - root = nonlocal_root[0] - root[:] = [root, root, None, None] - stats[:] = [0, 0] - finally: - lock.release() - - wrapper.__wrapped__ = user_function - wrapper.cache_info = cache_info - wrapper.cache_clear = cache_clear - return functools.update_wrapper(wrapper, user_function) - - return decorating_function - - -# python 3.3 -try: - from shutil import which -except ImportError: - - def which(cmd, mode=os.F_OK | os.X_OK, path=None): - """Given a command, mode, and a PATH string, return the path which - conforms to the given mode on the PATH, or None if there is no such - file. - - `mode` defaults to os.F_OK | os.X_OK. `path` defaults to the result - of os.environ.get("PATH"), or can be overridden with a custom search - path. - """ - - def _access_check(fn, mode): - return ( - os.path.exists(fn) - and os.access(fn, mode) - and not os.path.isdir(fn) - ) - - if os.path.dirname(cmd): - if _access_check(cmd, mode): - return cmd - return None - - if path is None: - path = os.environ.get("PATH", os.defpath) - if not path: - return None - path = path.split(os.pathsep) - - if sys.platform == "win32": - if os.curdir not in path: - path.insert(0, os.curdir) - - pathext = os.environ.get("PATHEXT", "").split(os.pathsep) - if any(cmd.lower().endswith(ext.lower()) for ext in pathext): - files = [cmd] - else: - files = [cmd + ext for ext in pathext] - else: - files = [cmd] - - seen = set() - for dir in path: - normdir = os.path.normcase(dir) - if normdir not in seen: - seen.add(normdir) - for thefile in files: - name = os.path.join(dir, thefile) - if _access_check(name, mode): - return name - return None - - -# python 3.3 -try: - from shutil import get_terminal_size -except ImportError: - - def get_terminal_size(fallback=(80, 24)): - try: - import fcntl - import struct - import termios - except ImportError: - return fallback - else: - try: - # This should work on Linux. - res = struct.unpack( - 'hh', fcntl.ioctl(1, termios.TIOCGWINSZ, '1234') - ) - return (res[1], res[0]) - except Exception: # noqa: BLE001 - return fallback - - -# python 3.3 -try: - from subprocess import TimeoutExpired as SubprocessTimeoutExpired -except ImportError: - - class SubprocessTimeoutExpired(Exception): - pass - - -# python 3.5 -try: - from contextlib import redirect_stderr -except ImportError: - - @contextlib.contextmanager - def redirect_stderr(new_target): - original = sys.stderr - try: - sys.stderr = new_target - yield new_target - finally: - sys.stderr = original diff --git a/contrib/python/psutil/py3/psutil/_psaix.py b/contrib/python/psutil/py3/psutil/_psaix.py index 2ccc638bce..ba2725fdc1 100644 --- a/contrib/python/psutil/py3/psutil/_psaix.py +++ b/contrib/python/psutil/py3/psutil/_psaix.py @@ -28,10 +28,6 @@ from ._common import conn_to_ntuple from ._common import get_procfs_path from ._common import memoize_when_activated from ._common import usage_percent -from ._compat import PY3 -from ._compat import FileNotFoundError -from ._compat import PermissionError -from ._compat import ProcessLookupError __extra__all__ = ["PROCFS_PATH"] @@ -148,12 +144,10 @@ def cpu_count_cores(): cmd = ["lsdev", "-Cc", "processor"] p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = p.communicate() - if PY3: - stdout, stderr = ( - x.decode(sys.stdout.encoding) for x in (stdout, stderr) - ) + stdout, stderr = (x.decode(sys.stdout.encoding) for x in (stdout, stderr)) if p.returncode != 0: - raise RuntimeError("%r command error\n%s" % (cmd, stderr)) + msg = f"{cmd!r} command error\n{stderr}" + raise RuntimeError(msg) processors = stdout.strip().splitlines() return len(processors) or None @@ -211,12 +205,6 @@ def net_connections(kind, _pid=-1): """Return socket connections. If pid == -1 return system-wide connections (as opposed to connections opened by one process only). """ - cmap = _common.conn_tmap - if kind not in cmap: - raise ValueError( - "invalid %r kind argument; choose between %s" - % (kind, ', '.join([repr(x) for x in cmap])) - ) families, types = _common.conn_tmap[kind] rawlist = cext.net_connections(_pid) ret = [] @@ -243,7 +231,7 @@ def net_connections(kind, _pid=-1): def net_if_stats(): """Get NIC stats (isup, duplex, speed, mtu).""" duplex_map = {"Full": NIC_DUPLEX_FULL, "Half": NIC_DUPLEX_HALF} - names = set([x[0] for x in net_if_addrs()]) + names = {x[0] for x in net_if_addrs()} ret = {} for name in names: mtu = cext_posix.net_if_mtu(name) @@ -260,10 +248,9 @@ def net_if_stats(): stderr=subprocess.PIPE, ) stdout, stderr = p.communicate() - if PY3: - stdout, stderr = ( - x.decode(sys.stdout.encoding) for x in (stdout, stderr) - ) + stdout, stderr = ( + x.decode(sys.stdout.encoding) for x in (stdout, stderr) + ) if p.returncode == 0: re_result = re.search( r"Running: (\d+) Mbps.*?(\w+) Duplex", stdout @@ -330,18 +317,18 @@ def wrap_exceptions(fun): @functools.wraps(fun) def wrapper(self, *args, **kwargs): + pid, ppid, name = self.pid, self._ppid, self._name try: return fun(self, *args, **kwargs) - except (FileNotFoundError, ProcessLookupError): + except (FileNotFoundError, ProcessLookupError) as err: # ENOENT (no such file or directory) gets raised on open(). # ESRCH (no such process) can get raised on read() if # process is gone in meantime. - if not pid_exists(self.pid): - raise NoSuchProcess(self.pid, self._name) - else: - raise ZombieProcess(self.pid, self._name, self._ppid) - except PermissionError: - raise AccessDenied(self.pid, self._name) + if not pid_exists(pid): + raise NoSuchProcess(pid, name) from err + raise ZombieProcess(pid, name, ppid) from err + except PermissionError as err: + raise AccessDenied(pid, name) from err return wrapper @@ -444,7 +431,7 @@ class Process: # is no longer there. if not retlist: # will raise NSP if process is gone - os.stat('%s/%s' % (self._procfs_path, self.pid)) + os.stat(f"{self._procfs_path}/{self.pid}") return retlist @wrap_exceptions @@ -457,7 +444,7 @@ class Process: # is no longer there. if not ret: # will raise NSP if process is gone - os.stat('%s/%s' % (self._procfs_path, self.pid)) + os.stat(f"{self._procfs_path}/{self.pid}") return ret @wrap_exceptions @@ -503,10 +490,10 @@ class Process: def cwd(self): procfs_path = self._procfs_path try: - result = os.readlink("%s/%s/cwd" % (procfs_path, self.pid)) + result = os.readlink(f"{procfs_path}/{self.pid}/cwd") return result.rstrip('/') except FileNotFoundError: - os.stat("%s/%s" % (procfs_path, self.pid)) # raise NSP or AD + os.stat(f"{procfs_path}/{self.pid}") # raise NSP or AD return "" @wrap_exceptions @@ -533,10 +520,9 @@ class Process: stderr=subprocess.PIPE, ) stdout, stderr = p.communicate() - if PY3: - stdout, stderr = ( - x.decode(sys.stdout.encoding) for x in (stdout, stderr) - ) + stdout, stderr = ( + x.decode(sys.stdout.encoding) for x in (stdout, stderr) + ) if "no such process" in stderr.lower(): raise NoSuchProcess(self.pid, self._name) procfiles = re.findall(r"(\d+): S_IFREG.*name:(.*)\n", stdout) @@ -554,7 +540,7 @@ class Process: def num_fds(self): if self.pid == 0: # no /proc/0/fd return 0 - return len(os.listdir("%s/%s/fd" % (self._procfs_path, self.pid))) + return len(os.listdir(f"{self._procfs_path}/{self.pid}/fd")) @wrap_exceptions def num_ctx_switches(self): @@ -570,10 +556,10 @@ class Process: def io_counters(self): try: rc, wc, rb, wb = cext.proc_io_counters(self.pid) - except OSError: + except OSError as err: # if process is terminated, proc_io_counters returns OSError # instead of NSP if not pid_exists(self.pid): - raise NoSuchProcess(self.pid, self._name) + raise NoSuchProcess(self.pid, self._name) from err raise return _common.pio(rc, wc, rb, wb) diff --git a/contrib/python/psutil/py3/psutil/_psbsd.py b/contrib/python/psutil/py3/psutil/_psbsd.py index 77b99c0fd8..13bd926fab 100644 --- a/contrib/python/psutil/py3/psutil/_psbsd.py +++ b/contrib/python/psutil/py3/psutil/_psbsd.py @@ -10,7 +10,7 @@ import functools import os from collections import defaultdict from collections import namedtuple -from xml.etree import ElementTree # noqa ICN001 +from xml.etree import ElementTree # noqa: ICN001 from . import _common from . import _psposix @@ -28,10 +28,6 @@ from ._common import debug from ._common import memoize from ._common import memoize_when_activated from ._common import usage_percent -from ._compat import FileNotFoundError -from ._compat import PermissionError -from ._compat import ProcessLookupError -from ._compat import which __extra__all__ = [] @@ -196,8 +192,8 @@ def virtual_memory(): # #2233), so zabbix seems to be wrong. Htop calculates it # differently, and the used value seem more realistic, so let's # match htop. - # https://github.com/htop-dev/htop/blob/e7f447b/netbsd/NetBSDProcessList.c#L162 # noqa - # https://github.com/zabbix/zabbix/blob/af5e0f8/src/libs/zbxsysinfo/netbsd/memory.c#L135 # noqa + # https://github.com/htop-dev/htop/blob/e7f447b/netbsd/NetBSDProcessList.c#L162 + # https://github.com/zabbix/zabbix/blob/af5e0f8/src/libs/zbxsysinfo/netbsd/memory.c#L135 used = active + wired avail = total - used else: @@ -206,7 +202,7 @@ def virtual_memory(): # * https://people.freebsd.org/~rse/dist/freebsd-memory # * https://www.cyberciti.biz/files/scripts/freebsd-memory.pl.txt # matches zabbix: - # * https://github.com/zabbix/zabbix/blob/af5e0f8/src/libs/zbxsysinfo/freebsd/memory.c#L143 # noqa + # * https://github.com/zabbix/zabbix/blob/af5e0f8/src/libs/zbxsysinfo/freebsd/memory.c#L143 avail = inactive + cached + free used = active + wired + cached @@ -439,14 +435,8 @@ def net_if_stats(): def net_connections(kind): """System-wide network connections.""" - if kind not in _common.conn_tmap: - raise ValueError( - "invalid %r kind argument; choose between %s" - % (kind, ', '.join([repr(x) for x in conn_tmap])) - ) families, types = conn_tmap[kind] ret = set() - if OPENBSD: rawlist = cext.net_connections(-1, families, types) elif NETBSD: @@ -495,7 +485,7 @@ if FREEBSD: current, high = cext.sensors_cpu_temperature(cpu) if high <= 0: high = None - name = "Core %s" % cpu + name = f"Core {cpu}" ret["coretemp"].append( _common.shwtemp(name, current, high, high) ) @@ -600,21 +590,18 @@ def wrap_exceptions(fun): @functools.wraps(fun) def wrapper(self, *args, **kwargs): + pid, ppid, name = self.pid, self._ppid, self._name try: return fun(self, *args, **kwargs) - except ProcessLookupError: - if is_zombie(self.pid): - raise ZombieProcess(self.pid, self._name, self._ppid) - else: - raise NoSuchProcess(self.pid, self._name) - except PermissionError: - raise AccessDenied(self.pid, self._name) - except OSError: - if self.pid == 0: - if 0 in pids(): - raise AccessDenied(self.pid, self._name) - else: - raise + except ProcessLookupError as err: + if is_zombie(pid): + raise ZombieProcess(pid, name, ppid) from err + raise NoSuchProcess(pid, name) from err + except PermissionError as err: + raise AccessDenied(pid, name) from err + except OSError as err: + if pid == 0 and 0 in pids(): + raise AccessDenied(pid, name) from err raise return wrapper @@ -623,18 +610,19 @@ def wrap_exceptions(fun): @contextlib.contextmanager def wrap_exceptions_procfs(inst): """Same as above, for routines relying on reading /proc fs.""" + pid, name, ppid = inst.pid, inst._name, inst._ppid try: yield - except (ProcessLookupError, FileNotFoundError): + except (ProcessLookupError, FileNotFoundError) as err: # ENOENT (no such file or directory) gets raised on open(). # ESRCH (no such process) can get raised on read() if # process is gone in meantime. if is_zombie(inst.pid): - raise ZombieProcess(inst.pid, inst._name, inst._ppid) + raise ZombieProcess(pid, name, ppid) from err else: - raise NoSuchProcess(inst.pid, inst._name) - except PermissionError: - raise AccessDenied(inst.pid, inst._name) + raise NoSuchProcess(pid, name) from err + except PermissionError as err: + raise AccessDenied(pid, name) from err class Process: @@ -683,16 +671,18 @@ class Process: # /proc/0 dir exists but /proc/0/exe doesn't return "" with wrap_exceptions_procfs(self): - return os.readlink("/proc/%s/exe" % self.pid) + return os.readlink(f"/proc/{self.pid}/exe") else: # OpenBSD: exe cannot be determined; references: # https://chromium.googlesource.com/chromium/src/base/+/ # master/base_paths_posix.cc # We try our best guess by using which against the first # cmdline arg (may return None). + import shutil + cmdline = self.cmdline() if cmdline: - return which(cmdline[0]) or "" + return shutil.which(cmdline[0]) or "" else: return "" @@ -709,15 +699,15 @@ class Process: return cext.proc_cmdline(self.pid) except OSError as err: if err.errno == errno.EINVAL: + pid, name, ppid = self.pid, self._name, self._ppid if is_zombie(self.pid): - raise ZombieProcess(self.pid, self._name, self._ppid) - elif not pid_exists(self.pid): - raise NoSuchProcess(self.pid, self._name, self._ppid) - else: - # XXX: this happens with unicode tests. It means the C - # routine is unable to decode invalid unicode chars. - debug("ignoring %r and returning an empty list" % err) - return [] + raise ZombieProcess(pid, name, ppid) from err + if not pid_exists(self.pid): + raise NoSuchProcess(pid, name, ppid) from err + # XXX: this happens with unicode tests. It means the C + # routine is unable to decode invalid unicode chars. + debug(f"ignoring {err!r} and returning an empty list") + return [] else: raise else: @@ -822,11 +812,6 @@ class Process: @wrap_exceptions def net_connections(self, kind='inet'): - if kind not in conn_tmap: - raise ValueError( - "invalid %r kind argument; choose between %s" - % (kind, ', '.join([repr(x) for x in conn_tmap])) - ) families, types = conn_tmap[kind] ret = [] @@ -945,12 +930,11 @@ class Process: # Pre-emptively check if CPUs are valid because the C # function has a weird behavior in case of invalid CPUs, # see: https://github.com/giampaolo/psutil/issues/586 - allcpus = tuple(range(len(per_cpu_times()))) + allcpus = set(range(len(per_cpu_times()))) for cpu in cpus: if cpu not in allcpus: - raise ValueError( - "invalid CPU #%i (choose between %s)" % (cpu, allcpus) - ) + msg = f"invalid CPU {cpu!r} (choose between {allcpus})" + raise ValueError(msg) try: cext.proc_cpu_affinity_set(self.pid, cpus) except OSError as err: @@ -961,10 +945,11 @@ class Process: if err.errno in {errno.EINVAL, errno.EDEADLK}: for cpu in cpus: if cpu not in allcpus: - raise ValueError( - "invalid CPU #%i (choose between %s)" - % (cpu, allcpus) + msg = ( + f"invalid CPU {cpu!r} (choose between" + f" {allcpus})" ) + raise ValueError(msg) from err raise @wrap_exceptions @@ -977,9 +962,10 @@ class Process: return cext.proc_getrlimit(self.pid, resource) else: if len(limits) != 2: - raise ValueError( - "second argument must be a (soft, hard) tuple, got %s" - % repr(limits) + msg = ( + "second argument must be a (soft, hard) tuple, got" + f" {limits!r}" ) + raise ValueError(msg) soft, hard = limits return cext.proc_setrlimit(self.pid, resource, soft, hard) diff --git a/contrib/python/psutil/py3/psutil/_pslinux.py b/contrib/python/psutil/py3/psutil/_pslinux.py index 3e7f3e402a..8cc64e9a10 100644 --- a/contrib/python/psutil/py3/psutil/_pslinux.py +++ b/contrib/python/psutil/py3/psutil/_pslinux.py @@ -4,15 +4,16 @@ """Linux platform implementation.""" -from __future__ import division import base64 import collections +import enum import errno import functools import glob import os import re +import resource import socket import struct import sys @@ -24,6 +25,7 @@ from . import _common from . import _psposix from . import _psutil_linux as cext from . import _psutil_posix as cext_posix +from ._common import ENCODING from ._common import NIC_DUPLEX_FULL from ._common import NIC_DUPLEX_HALF from ._common import NIC_DUPLEX_UNKNOWN @@ -44,18 +46,6 @@ from ._common import parse_environ_block from ._common import path_exists_strict from ._common import supports_ipv6 from ._common import usage_percent -from ._compat import PY3 -from ._compat import FileNotFoundError -from ._compat import PermissionError -from ._compat import ProcessLookupError -from ._compat import b -from ._compat import basestring - - -if PY3: - import enum -else: - enum = None # fmt: off @@ -69,6 +59,11 @@ __extra__all__ = [ "CONN_FIN_WAIT2", "CONN_TIME_WAIT", "CONN_CLOSE", "CONN_CLOSE_WAIT", "CONN_LAST_ACK", "CONN_LISTEN", "CONN_CLOSING", ] + +if hasattr(resource, "prlimit"): + __extra__all__.extend( + [x for x in dir(cext) if x.startswith('RLIM') and x.isupper()] + ) # fmt: on @@ -78,8 +73,8 @@ __extra__all__ = [ POWER_SUPPLY_PATH = "/sys/class/power_supply" -HAS_PROC_SMAPS = os.path.exists('/proc/%s/smaps' % os.getpid()) -HAS_PROC_SMAPS_ROLLUP = os.path.exists('/proc/%s/smaps_rollup' % os.getpid()) +HAS_PROC_SMAPS = os.path.exists(f"/proc/{os.getpid()}/smaps") +HAS_PROC_SMAPS_ROLLUP = os.path.exists(f"/proc/{os.getpid()}/smaps_rollup") HAS_PROC_IO_PRIORITY = hasattr(cext, "proc_ioprio_get") HAS_CPU_AFFINITY = hasattr(cext, "proc_cpu_affinity_get") @@ -102,29 +97,21 @@ LITTLE_ENDIAN = sys.byteorder == 'little' # * https://lkml.org/lkml/2015/8/17/234 DISK_SECTOR_SIZE = 512 -if enum is None: - AF_LINK = socket.AF_PACKET -else: - AddressFamily = enum.IntEnum( - 'AddressFamily', {'AF_LINK': int(socket.AF_PACKET)} - ) - AF_LINK = AddressFamily.AF_LINK +AddressFamily = enum.IntEnum( + 'AddressFamily', {'AF_LINK': int(socket.AF_PACKET)} +) +AF_LINK = AddressFamily.AF_LINK + # ioprio_* constants http://linux.die.net/man/2/ioprio_get -if enum is None: +class IOPriority(enum.IntEnum): IOPRIO_CLASS_NONE = 0 IOPRIO_CLASS_RT = 1 IOPRIO_CLASS_BE = 2 IOPRIO_CLASS_IDLE = 3 -else: - class IOPriority(enum.IntEnum): - IOPRIO_CLASS_NONE = 0 - IOPRIO_CLASS_RT = 1 - IOPRIO_CLASS_BE = 2 - IOPRIO_CLASS_IDLE = 3 - globals().update(IOPriority.__members__) +globals().update(IOPriority.__members__) # See: # https://github.com/torvalds/linux/blame/master/fs/proc/array.c @@ -211,7 +198,7 @@ pcputimes = namedtuple('pcputimes', def readlink(path): """Wrapper around os.readlink().""" - assert isinstance(path, basestring), path + assert isinstance(path, str), path path = os.readlink(path) # readlink() might return paths containing null bytes ('\x00') # resulting in "TypeError: must be encoded string without NULL @@ -255,9 +242,9 @@ def is_storage_device(name): name = name.replace('/', '!') including_virtual = True if including_virtual: - path = "/sys/block/%s" % name + path = f"/sys/block/{name}" else: - path = "/sys/block/%s/device" % name + path = f"/sys/block/{name}/device" return os.access(path, os.F_OK) @@ -270,7 +257,7 @@ def set_scputimes_ntuple(procfs_path): Used by cpu_times() function. """ global scputimes - with open_binary('%s/stat' % procfs_path) as f: + with open_binary(f"{procfs_path}/stat") as f: values = f.readline().split()[1:] fields = ['user', 'nice', 'system', 'idle', 'iowait', 'irq', 'softirq'] vlen = len(values) @@ -290,66 +277,11 @@ try: set_scputimes_ntuple("/proc") except Exception as err: # noqa: BLE001 # Don't want to crash at import time. - debug("ignoring exception on import: %r" % err) + debug(f"ignoring exception on import: {err!r}") scputimes = namedtuple('scputimes', 'user system idle')(0.0, 0.0, 0.0) # ===================================================================== -# --- prlimit -# ===================================================================== - -# Backport of resource.prlimit() for Python 2. Originally this was done -# in C, but CentOS-6 which we use to create manylinux wheels is too old -# and does not support prlimit() syscall. As such the resulting wheel -# would not include prlimit(), even when installed on newer systems. -# This is the only part of psutil using ctypes. - -prlimit = None -try: - from resource import prlimit # python >= 3.4 -except ImportError: - import ctypes - - try: - libc = ctypes.CDLL(None, use_errno=True) - except: - libc = None - - if hasattr(libc, "prlimit"): - - def prlimit(pid, resource_, limits=None): - class StructRlimit(ctypes.Structure): - _fields_ = [ - ('rlim_cur', ctypes.c_longlong), - ('rlim_max', ctypes.c_longlong), - ] - - current = StructRlimit() - if limits is None: - # get - ret = libc.prlimit(pid, resource_, None, ctypes.byref(current)) - else: - # set - new = StructRlimit() - new.rlim_cur = limits[0] - new.rlim_max = limits[1] - ret = libc.prlimit( - pid, resource_, ctypes.byref(new), ctypes.byref(current) - ) - - if ret != 0: - errno_ = ctypes.get_errno() - raise OSError(errno_, os.strerror(errno_)) - return (current.rlim_cur, current.rlim_max) - - -if prlimit is not None: - __extra__all__.extend( - [x for x in dir(cext) if x.startswith('RLIM') and x.isupper()] - ) - - -# ===================================================================== # --- system memory # ===================================================================== @@ -392,14 +324,13 @@ def calculate_avail_vmem(mems): slab_reclaimable = mems[b'SReclaimable:'] except KeyError as err: debug( - "%s is missing from /proc/meminfo; using an approximation for " - "calculating available memory" - % err.args[0] + f"{err.args[0]} is missing from /proc/meminfo; using an" + " approximation for calculating available memory" ) return fallback try: - f = open_binary('%s/zoneinfo' % get_procfs_path()) - except IOError: + f = open_binary(f"{get_procfs_path()}/zoneinfo") + except OSError: return fallback # kernel 2.6.13 watermark_low = 0 @@ -428,7 +359,7 @@ def virtual_memory(): """ missing_fields = [] mems = {} - with open_binary('%s/meminfo' % get_procfs_path()) as f: + with open_binary(f"{get_procfs_path()}/meminfo") as f: for line in f: fields = line.split() mems[fields[0]] = int(fields[1]) * 1024 @@ -530,7 +461,7 @@ def virtual_memory(): # Warn about missing metrics which are set to 0. if missing_fields: - msg = "%s memory stats couldn't be determined and %s set to 0" % ( + msg = "{} memory stats couldn't be determined and {} set to 0".format( ", ".join(missing_fields), "was" if len(missing_fields) == 1 else "were", ) @@ -554,7 +485,7 @@ def virtual_memory(): def swap_memory(): """Return swap memory metrics.""" mems = {} - with open_binary('%s/meminfo' % get_procfs_path()) as f: + with open_binary(f"{get_procfs_path()}/meminfo") as f: for line in f: fields = line.split() mems[fields[0]] = int(fields[1]) * 1024 @@ -574,12 +505,12 @@ def swap_memory(): percent = usage_percent(used, total, round_=1) # get pgin/pgouts try: - f = open_binary("%s/vmstat" % get_procfs_path()) - except IOError as err: + f = open_binary(f"{get_procfs_path()}/vmstat") + except OSError as err: # see https://github.com/giampaolo/psutil/issues/722 msg = ( "'sin' and 'sout' swap memory stats couldn't " - + "be determined and were set to 0 (%s)" % str(err) + f"be determined and were set to 0 ({err})" ) warnings.warn(msg, RuntimeWarning, stacklevel=2) sin = sout = 0 @@ -620,7 +551,7 @@ def cpu_times(): """ procfs_path = get_procfs_path() set_scputimes_ntuple(procfs_path) - with open_binary('%s/stat' % procfs_path) as f: + with open_binary(f"{procfs_path}/stat") as f: values = f.readline().split() fields = values[1 : len(scputimes._fields) + 1] fields = [float(x) / CLOCK_TICKS for x in fields] @@ -634,7 +565,7 @@ def per_cpu_times(): procfs_path = get_procfs_path() set_scputimes_ntuple(procfs_path) cpus = [] - with open_binary('%s/stat' % procfs_path) as f: + with open_binary(f"{procfs_path}/stat") as f: # get rid of the first line which refers to system wide CPU stats f.readline() for line in f: @@ -654,7 +585,7 @@ def cpu_count_logical(): except ValueError: # as a second fallback we try to parse /proc/cpuinfo num = 0 - with open_binary('%s/cpuinfo' % get_procfs_path()) as f: + with open_binary(f"{get_procfs_path()}/cpuinfo") as f: for line in f: if line.lower().startswith(b'processor'): num += 1 @@ -664,7 +595,7 @@ def cpu_count_logical(): # try to parse /proc/stat as a last resort if num == 0: search = re.compile(r'cpu\d') - with open_text('%s/stat' % get_procfs_path()) as f: + with open_text(f"{get_procfs_path()}/stat") as f: for line in f: line = line.split(' ')[0] if search.match(line): @@ -697,7 +628,7 @@ def cpu_count_cores(): # Method #2 mapping = {} current_info = {} - with open_binary('%s/cpuinfo' % get_procfs_path()) as f: + with open_binary(f"{get_procfs_path()}/cpuinfo") as f: for line in f: line = line.strip().lower() if not line: @@ -720,7 +651,7 @@ def cpu_count_cores(): def cpu_stats(): """Return various CPU stats as a named tuple.""" - with open_binary('%s/stat' % get_procfs_path()) as f: + with open_binary(f"{get_procfs_path()}/stat") as f: ctx_switches = None interrupts = None soft_interrupts = None @@ -745,12 +676,12 @@ def cpu_stats(): def _cpu_get_cpuinfo_freq(): """Return current CPU frequency from cpuinfo if available.""" - ret = [] - with open_binary('%s/cpuinfo' % get_procfs_path()) as f: - for line in f: - if line.lower().startswith(b'cpu mhz'): - ret.append(float(line.split(b':', 1)[1])) - return ret + with open_binary(f"{get_procfs_path()}/cpuinfo") as f: + return [ + float(line.split(b':', 1)[1]) + for line in f + if line.lower().startswith(b'cpu mhz') + ] if os.path.exists("/sys/devices/system/cpu/cpufreq/policy0") or os.path.exists( @@ -781,9 +712,7 @@ if os.path.exists("/sys/devices/system/cpu/cpufreq/policy0") or os.path.exists( # https://github.com/giampaolo/psutil/issues/1071 curr = bcat(pjoin(path, "cpuinfo_cur_freq"), fallback=None) if curr is None: - online_path = ( - "/sys/devices/system/cpu/cpu{}/online".format(i) - ) + online_path = f"/sys/devices/system/cpu/cpu{i}/online" # if cpu core is offline, set to all zeroes if cat(online_path, fallback=None) == "0\n": ret.append(_common.scpufreq(0.0, 0.0, 0.0)) @@ -854,12 +783,12 @@ class NetConnections: def get_proc_inodes(self, pid): inodes = defaultdict(list) - for fd in os.listdir("%s/%s/fd" % (self._procfs_path, pid)): + for fd in os.listdir(f"{self._procfs_path}/{pid}/fd"): try: - inode = readlink("%s/%s/fd/%s" % (self._procfs_path, pid, fd)) + inode = readlink(f"{self._procfs_path}/{pid}/fd/{fd}") except (FileNotFoundError, ProcessLookupError): # ENOENT == file which is gone in the meantime; - # os.stat('/proc/%s' % self.pid) will be done later + # os.stat(f"/proc/{self.pid}") will be done later # to force NSP (if it's the case) continue except OSError as err: @@ -917,8 +846,7 @@ class NetConnections: # no end-points connected if not port: return () - if PY3: - ip = ip.encode('ascii') + ip = ip.encode('ascii') if family == socket.AF_INET: # see: https://github.com/giampaolo/psutil/issues/201 if LITTLE_ENDIAN: @@ -942,9 +870,8 @@ class NetConnections: except ValueError: # see: https://github.com/giampaolo/psutil/issues/623 if not supports_ipv6(): - raise _Ipv6UnsupportedError - else: - raise + raise _Ipv6UnsupportedError from None + raise return _common.addr(ip, port) @staticmethod @@ -961,10 +888,11 @@ class NetConnections: line.split()[:10] ) except ValueError: - raise RuntimeError( - "error while parsing %s; malformed line %s %r" - % (file, lineno, line) + msg = ( + f"error while parsing {file}; malformed line" + f" {lineno} {line!r}" ) + raise RuntimeError(msg) from None if inode in inodes: # # We assume inet sockets are unique, so we error # # out if there are multiple references to the @@ -1002,11 +930,11 @@ class NetConnections: if ' ' not in line: # see: https://github.com/giampaolo/psutil/issues/766 continue - raise RuntimeError( - "error while parsing %s; malformed line %r" - % (file, line) + msg = ( + f"error while parsing {file}; malformed line {line!r}" ) - if inode in inodes: # noqa + raise RuntimeError(msg) # noqa: B904 + if inode in inodes: # noqa: SIM108 # With UNIX sockets we can have a single inode # referencing many file descriptors. pairs = inodes[inode] @@ -1026,11 +954,6 @@ class NetConnections: yield (fd, family, type_, path, raddr, status, pid) def retrieve(self, kind, pid=None): - if kind not in self.tmap: - raise ValueError( - "invalid %r kind argument; choose between %s" - % (kind, ', '.join([repr(x) for x in self.tmap])) - ) self._procfs_path = get_procfs_path() if pid is not None: inodes = self.get_proc_inodes(pid) @@ -1041,7 +964,7 @@ class NetConnections: inodes = self.get_all_inodes() ret = set() for proto_name, family, type_ in self.tmap[kind]: - path = "%s/net/%s" % (self._procfs_path, proto_name) + path = f"{self._procfs_path}/net/{proto_name}" if family in {socket.AF_INET, socket.AF_INET6}: ls = self.process_inet( path, family, type_, inodes, filter_pid=pid @@ -1073,7 +996,7 @@ def net_io_counters(): """Return network I/O statistics for every network interface installed on the system as a dict of raw tuples. """ - with open_text("%s/net/dev" % get_procfs_path()) as f: + with open_text(f"{get_procfs_path()}/net/dev") as f: lines = f.readlines() retdict = {} for line in lines[2:]: @@ -1134,8 +1057,7 @@ def net_if_stats(): # https://github.com/giampaolo/psutil/issues/1279 if err.errno != errno.ENODEV: raise - else: - debug(err) + debug(err) else: output_flags = ','.join(flags) isup = 'running' in flags @@ -1175,7 +1097,7 @@ def disk_io_counters(perdisk=False): # See: # https://www.kernel.org/doc/Documentation/iostats.txt # https://www.kernel.org/doc/Documentation/ABI/testing/procfs-diskstats - with open_text("%s/diskstats" % get_procfs_path()) as f: + with open_text(f"{get_procfs_path()}/diskstats") as f: lines = f.readlines() for line in lines: fields = line.split() @@ -1198,7 +1120,8 @@ def disk_io_counters(perdisk=False): reads, rbytes, writes, wbytes = map(int, fields[3:]) rtime = wtime = reads_merged = writes_merged = busy_time = 0 else: - raise ValueError("not sure how to interpret line %r" % line) + msg = f"not sure how to interpret line {line!r}" + raise ValueError(msg) yield (name, reads, writes, rbytes, wbytes, rtime, wtime, reads_merged, writes_merged, busy_time) # fmt: on @@ -1218,16 +1141,16 @@ def disk_io_counters(perdisk=False): wtime, reads_merged, writes_merged, busy_time) # fmt: on - if os.path.exists('%s/diskstats' % get_procfs_path()): + if os.path.exists(f"{get_procfs_path()}/diskstats"): gen = read_procfs() elif os.path.exists('/sys/block'): gen = read_sysfs() else: - raise NotImplementedError( - "%s/diskstats nor /sys/block filesystem are available on this " - "system" - % get_procfs_path() + msg = ( + f"{get_procfs_path()}/diskstats nor /sys/block are available on" + " this system" ) + raise NotImplementedError(msg) retdict = {} for entry in gen: @@ -1273,7 +1196,7 @@ class RootFsDeviceFinder: self.minor = os.minor(dev) def ask_proc_partitions(self): - with open_text("%s/partitions" % get_procfs_path()) as f: + with open_text(f"{get_procfs_path()}/partitions") as f: for line in f.readlines()[2:]: fields = line.split() if len(fields) < 4: # just for extra safety @@ -1283,19 +1206,19 @@ class RootFsDeviceFinder: name = fields[3] if major == self.major and minor == self.minor: if name: # just for extra safety - return "/dev/%s" % name + return f"/dev/{name}" def ask_sys_dev_block(self): - path = "/sys/dev/block/%s:%s/uevent" % (self.major, self.minor) + path = f"/sys/dev/block/{self.major}:{self.minor}/uevent" with open_text(path) as f: for line in f: if line.startswith("DEVNAME="): name = line.strip().rpartition("DEVNAME=")[2] if name: # just for extra safety - return "/dev/%s" % name + return f"/dev/{name}" def ask_sys_class_block(self): - needle = "%s:%s" % (self.major, self.minor) + needle = f"{self.major}:{self.minor}" files = glob.iglob("/sys/class/block/*/dev") for file in files: try: @@ -1307,24 +1230,24 @@ class RootFsDeviceFinder: data = f.read().strip() if data == needle: name = os.path.basename(os.path.dirname(file)) - return "/dev/%s" % name + return f"/dev/{name}" def find(self): path = None if path is None: try: path = self.ask_proc_partitions() - except (IOError, OSError) as err: + except OSError as err: debug(err) if path is None: try: path = self.ask_sys_dev_block() - except (IOError, OSError) as err: + except OSError as err: debug(err) if path is None: try: path = self.ask_sys_class_block() - except (IOError, OSError) as err: + except OSError as err: debug(err) # We use exists() because the "/dev/*" part of the path is hard # coded, so we want to be sure. @@ -1337,7 +1260,7 @@ def disk_partitions(all=False): fstypes = set() procfs_path = get_procfs_path() if not all: - with open_text("%s/filesystems" % procfs_path) as f: + with open_text(f"{procfs_path}/filesystems") as f: for line in f: line = line.strip() if not line.startswith("nodev"): @@ -1352,7 +1275,7 @@ def disk_partitions(all=False): if procfs_path == "/proc" and os.path.isfile('/etc/mtab'): mounts_path = os.path.realpath("/etc/mtab") else: - mounts_path = os.path.realpath("%s/self/mounts" % procfs_path) + mounts_path = os.path.realpath(f"{procfs_path}/self/mounts") retlist = [] partitions = cext.disk_partitions(mounts_path) @@ -1395,7 +1318,7 @@ def sensors_temperatures(): # https://github.com/giampaolo/psutil/issues/971 # https://github.com/nicolargo/glances/issues/1060 basenames.extend(glob.glob('/sys/class/hwmon/hwmon*/device/temp*_*')) - basenames = sorted(set([x.split('_')[0] for x in basenames])) + basenames = sorted({x.split('_')[0] for x in basenames}) # Only add the coretemp hwmon entries if they're not already in # /sys/class/hwmon/ @@ -1404,7 +1327,7 @@ def sensors_temperatures(): basenames2 = glob.glob( '/sys/devices/platform/coretemp.*/hwmon/hwmon*/temp*_*' ) - repl = re.compile('/sys/devices/platform/coretemp.*/hwmon/') + repl = re.compile(r"/sys/devices/platform/coretemp.*/hwmon/") for name in basenames2: altname = repl.sub('/sys/class/hwmon/', name) if altname not in basenames: @@ -1416,7 +1339,7 @@ def sensors_temperatures(): current = float(bcat(path)) / 1000.0 path = os.path.join(os.path.dirname(base), 'name') unit_name = cat(path).strip() - except (IOError, OSError, ValueError): + except (OSError, ValueError): # A lot of things can go wrong here, so let's just skip the # whole entry. Sure thing is Linux's /sys/class/hwmon really # is a stinky broken mess. @@ -1455,15 +1378,15 @@ def sensors_temperatures(): current = float(bcat(path)) / 1000.0 path = os.path.join(base, 'type') unit_name = cat(path).strip() - except (IOError, OSError, ValueError) as err: + except (OSError, ValueError) as err: debug(err) continue trip_paths = glob.glob(base + '/trip_point*') - trip_points = set([ + trip_points = { '_'.join(os.path.basename(p).split('_')[0:3]) for p in trip_paths - ]) + } critical = None high = None for trip_point in trip_points: @@ -1511,11 +1434,11 @@ def sensors_fans(): # https://github.com/giampaolo/psutil/issues/971 basenames = glob.glob('/sys/class/hwmon/hwmon*/device/fan*_*') - basenames = sorted(set([x.split('_')[0] for x in basenames])) + basenames = sorted({x.split("_")[0] for x in basenames}) for base in basenames: try: current = int(bcat(base + '_input')) - except (IOError, OSError) as err: + except OSError as err: debug(err) continue unit_name = cat(os.path.join(os.path.dirname(base), 'name')).strip() @@ -1557,7 +1480,7 @@ def sensors_battery(): # Get the first available battery. Usually this is "BAT0", except # some rare exceptions: # https://github.com/giampaolo/psutil/issues/1238 - root = os.path.join(POWER_SUPPLY_PATH, sorted(bats)[0]) + root = os.path.join(POWER_SUPPLY_PATH, min(bats)) # Base metrics. energy_now = multi_bcat(root + "/energy_now", root + "/charge_now") @@ -1634,14 +1557,15 @@ def users(): def boot_time(): """Return the system boot time expressed in seconds since the epoch.""" global BOOT_TIME - path = '%s/stat' % get_procfs_path() + path = f"{get_procfs_path()}/stat" with open_binary(path) as f: for line in f: if line.startswith(b'btime'): ret = float(line.strip().split()[1]) BOOT_TIME = ret return ret - raise RuntimeError("line 'btime' not found in %s" % path) + msg = f"line 'btime' not found in {path}" + raise RuntimeError(msg) # ===================================================================== @@ -1651,7 +1575,8 @@ def boot_time(): def pids(): """Returns a list of PIDs currently running on the system.""" - return [int(x) for x in os.listdir(b(get_procfs_path())) if x.isdigit()] + path = get_procfs_path().encode(ENCODING) + return [int(x) for x in os.listdir(path) if x.isdigit()] def pid_exists(pid): @@ -1673,7 +1598,7 @@ def pid_exists(pid): # Note: already checked that this is faster than using a # regular expr. Also (a lot) faster than doing # 'return pid in pids()' - path = "%s/%s/status" % (get_procfs_path(), pid) + path = f"{get_procfs_path()}/{pid}/status" with open_binary(path) as f: for line in f: if line.startswith(b"Tgid:"): @@ -1681,8 +1606,9 @@ def pid_exists(pid): # If tgid and pid are the same then we're # dealing with a process PID. return tgid == pid - raise ValueError("'Tgid' line not found in %s" % path) - except (EnvironmentError, ValueError): + msg = f"'Tgid' line not found in {path}" + raise ValueError(msg) + except (OSError, ValueError): return pid in pids() @@ -1694,7 +1620,7 @@ def ppid_map(): procfs_path = get_procfs_path() for pid in pids(): try: - with open_binary("%s/%s/stat" % (procfs_path, pid)) as f: + with open_binary(f"{procfs_path}/{pid}/stat") as f: data = f.read() except (FileNotFoundError, ProcessLookupError): # Note: we should be able to access /stat for all processes @@ -1709,28 +1635,27 @@ def ppid_map(): def wrap_exceptions(fun): - """Decorator which translates bare OSError and IOError exceptions + """Decorator which translates bare OSError and OSError exceptions into NoSuchProcess and AccessDenied. """ @functools.wraps(fun) def wrapper(self, *args, **kwargs): + pid, name = self.pid, self._name try: return fun(self, *args, **kwargs) - except PermissionError: - raise AccessDenied(self.pid, self._name) - except ProcessLookupError: + except PermissionError as err: + raise AccessDenied(pid, name) from err + except ProcessLookupError as err: self._raise_if_zombie() - raise NoSuchProcess(self.pid, self._name) - except FileNotFoundError: + raise NoSuchProcess(pid, name) from err + except FileNotFoundError as err: self._raise_if_zombie() # /proc/PID directory may still exist, but the files within # it may not, indicating the process is gone, see: # https://github.com/giampaolo/psutil/issues/2418 - if not os.path.exists( - "%s/%s/stat" % (self._procfs_path, self.pid) - ): - raise NoSuchProcess(self.pid, self._name) + if not os.path.exists(f"{self._procfs_path}/{pid}/stat"): + raise NoSuchProcess(pid, name) from err raise return wrapper @@ -1755,8 +1680,8 @@ class Process: # it's empty. Instead of returning a "null" value we'll raise an # exception. try: - data = bcat("%s/%s/stat" % (self._procfs_path, self.pid)) - except (IOError, OSError): + data = bcat(f"{self._procfs_path}/{self.pid}/stat") + except OSError: return False else: rpar = data.rfind(b')') @@ -1771,7 +1696,7 @@ class Process: """Raise NSP if the process disappeared on us.""" # For those C function who do not raise NSP, possibly returning # incorrect or incomplete result. - os.stat('%s/%s' % (self._procfs_path, self.pid)) + os.stat(f"{self._procfs_path}/{self.pid}") @wrap_exceptions @memoize_when_activated @@ -1784,7 +1709,7 @@ class Process: The return value is cached in case oneshot() ctx manager is in use. """ - data = bcat("%s/%s/stat" % (self._procfs_path, self.pid)) + data = bcat(f"{self._procfs_path}/{self.pid}/stat") # Process name is between parentheses. It can contain spaces and # other parentheses. This is taken into account by looking for # the first occurrence of "(" and the last occurrence of ")". @@ -1819,13 +1744,13 @@ class Process: The return value is cached in case oneshot() ctx manager is in use. """ - with open_binary("%s/%s/status" % (self._procfs_path, self.pid)) as f: + with open_binary(f"{self._procfs_path}/{self.pid}/status") as f: return f.read() @wrap_exceptions @memoize_when_activated def _read_smaps_file(self): - with open_binary("%s/%s/smaps" % (self._procfs_path, self.pid)) as f: + with open_binary(f"{self._procfs_path}/{self.pid}/smaps") as f: return f.read().strip() def oneshot_enter(self): @@ -1840,28 +1765,25 @@ class Process: @wrap_exceptions def name(self): - name = self._parse_stat_file()['name'] - if PY3: - name = decode(name) # XXX - gets changed later and probably needs refactoring - return name + return decode(self._parse_stat_file()['name']) @wrap_exceptions def exe(self): try: - return readlink("%s/%s/exe" % (self._procfs_path, self.pid)) + return readlink(f"{self._procfs_path}/{self.pid}/exe") except (FileNotFoundError, ProcessLookupError): self._raise_if_zombie() # no such file error; might be raised also if the # path actually exists for system processes with # low pids (about 0-20) - if os.path.lexists("%s/%s" % (self._procfs_path, self.pid)): + if os.path.lexists(f"{self._procfs_path}/{self.pid}"): return "" raise @wrap_exceptions def cmdline(self): - with open_text("%s/%s/cmdline" % (self._procfs_path, self.pid)) as f: + with open_text(f"{self._procfs_path}/{self.pid}/cmdline") as f: data = f.read() if not data: # may happen in case of zombie process @@ -1887,7 +1809,7 @@ class Process: @wrap_exceptions def environ(self): - with open_text("%s/%s/environ" % (self._procfs_path, self.pid)) as f: + with open_text(f"{self._procfs_path}/{self.pid}/environ") as f: data = f.read() return parse_environ_block(data) @@ -1901,11 +1823,11 @@ class Process: return None # May not be available on old kernels. - if os.path.exists('/proc/%s/io' % os.getpid()): + if os.path.exists(f"/proc/{os.getpid()}/io"): @wrap_exceptions def io_counters(self): - fname = "%s/%s/io" % (self._procfs_path, self.pid) + fname = f"{self._procfs_path}/{self.pid}/io" fields = {} with open_binary(fname) as f: for line in f: @@ -1920,7 +1842,8 @@ class Process: else: fields[name] = int(value) if not fields: - raise RuntimeError("%s file was empty" % fname) + msg = f"{fname} file was empty" + raise RuntimeError(msg) try: return pio( fields[b'syscr'], # read syscalls @@ -1931,10 +1854,11 @@ class Process: fields[b'wchar'], # write chars ) except KeyError as err: - raise ValueError( - "%r field was not found in %s; found fields are %r" - % (err.args[0], fname, fields) + msg = ( + f"{err.args[0]!r} field was not found in {fname}; found" + f" fields are {fields!r}" ) + raise ValueError(msg) from None @wrap_exceptions def cpu_times(self): @@ -1979,7 +1903,7 @@ class Process: # | data | data + stack | drs | DATA | # | dirty | dirty pages (unused in Linux 2.6) | dt | | # ============================================================ - with open_binary("%s/%s/statm" % (self._procfs_path, self.pid)) as f: + with open_binary(f"{self._procfs_path}/{self.pid}/statm") as f: vms, rss, shared, text, lib, data, dirty = ( int(x) * PAGESIZE for x in f.readline().split()[:7] ) @@ -1998,7 +1922,7 @@ class Process: # compared to /proc/pid/smaps_rollup. uss = pss = swap = 0 with open_binary( - "{}/{}/smaps_rollup".format(self._procfs_path, self.pid) + f"{self._procfs_path}/{self.pid}/smaps_rollup" ) as f: for line in f: if line.startswith(b"Private_"): @@ -2023,8 +1947,6 @@ class Process: # Note: using 3 regexes is faster than reading the file # line by line. - # XXX: on Python 3 the 2 regexes are 30% slower than on - # Python 2 though. Figure out why. # # You might be tempted to calculate USS by subtracting # the "shared" value from the "resident" value in @@ -2062,7 +1984,7 @@ class Process: def memory_maps(self): """Return process's mapped memory regions as a list of named tuples. Fields are explained in 'man proc'; here is an updated - (Apr 2012) version: http://goo.gl/fmebo. + (Apr 2012) version: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/Documentation/filesystems/proc.txt?id=b76437579d1344b612cf1851ae610c636cec7db0. /proc/{PID}/smaps does not exist on kernels < 2.6.14 or if CONFIG_MMU kernel configuration option is not enabled. @@ -2083,11 +2005,8 @@ class Process: if fields[0].startswith(b'VmFlags:'): # see issue #369 continue - else: - raise ValueError( - "don't know how to interpret line %r" - % line - ) + msg = f"don't know how to interpret line {line!r}" + raise ValueError(msg) from None yield (current_block.pop(), data) data = self._read_smaps_file() @@ -2109,8 +2028,7 @@ class Process: if not path: path = '[anon]' else: - if PY3: - path = decode(path) + path = decode(path) path = path.strip() if path.endswith(' (deleted)') and not path_exists_strict( path @@ -2136,7 +2054,7 @@ class Process: @wrap_exceptions def cwd(self): - return readlink("%s/%s/cwd" % (self._procfs_path, self.pid)) + return readlink(f"{self._procfs_path}/{self.pid}/cwd") @wrap_exceptions def num_ctx_switches( @@ -2145,34 +2063,29 @@ class Process: data = self._read_status_file() ctxsw = _ctxsw_re.findall(data) if not ctxsw: - raise NotImplementedError( - "'voluntary_ctxt_switches' and 'nonvoluntary_ctxt_switches'" - "lines were not found in %s/%s/status; the kernel is " - "probably older than 2.6.23" % (self._procfs_path, self.pid) + msg = ( + "'voluntary_ctxt_switches' and" + " 'nonvoluntary_ctxt_switches'lines were not found in" + f" {self._procfs_path}/{self.pid}/status; the kernel is" + " probably older than 2.6.23" ) - else: - return _common.pctxsw(int(ctxsw[0]), int(ctxsw[1])) + raise NotImplementedError(msg) + return _common.pctxsw(int(ctxsw[0]), int(ctxsw[1])) @wrap_exceptions def num_threads(self, _num_threads_re=re.compile(br'Threads:\t(\d+)')): - # Note: on Python 3 using a re is faster than iterating over file - # line by line. On Python 2 is the exact opposite, and iterating - # over a file on Python 3 is slower than on Python 2. + # Using a re is faster than iterating over file line by line. data = self._read_status_file() return int(_num_threads_re.findall(data)[0]) @wrap_exceptions def threads(self): - thread_ids = os.listdir("%s/%s/task" % (self._procfs_path, self.pid)) + thread_ids = os.listdir(f"{self._procfs_path}/{self.pid}/task") thread_ids.sort() retlist = [] hit_enoent = False for thread_id in thread_ids: - fname = "%s/%s/task/%s/stat" % ( - self._procfs_path, - self.pid, - thread_id, - ) + fname = f"{self._procfs_path}/{self.pid}/task/{thread_id}/stat" try: with open_binary(fname) as f: st = f.read().strip() @@ -2194,7 +2107,7 @@ class Process: @wrap_exceptions def nice_get(self): - # with open_text('%s/%s/stat' % (self._procfs_path, self.pid)) as f: + # with open_text(f"{self._procfs_path}/{self.pid}/stat") as f: # data = f.read() # return int(data.split()[18]) @@ -2233,15 +2146,17 @@ class Process: all_cpus = tuple(range(len(per_cpu_times()))) for cpu in cpus: if cpu not in all_cpus: - raise ValueError( - "invalid CPU number %r; choose between %s" - % (cpu, eligible_cpus) + msg = ( + f"invalid CPU {cpu!r}; choose between" + f" {eligible_cpus!r}" ) + raise ValueError(msg) from None if cpu not in eligible_cpus: - raise ValueError( - "CPU number %r is not eligible; choose " - "between %s" % (cpu, eligible_cpus) + msg = ( + f"CPU number {cpu} is not eligible; choose" + f" between {eligible_cpus}" ) + raise ValueError(msg) from err raise # only starting from kernel 2.6.13 @@ -2250,22 +2165,25 @@ class Process: @wrap_exceptions def ionice_get(self): ioclass, value = cext.proc_ioprio_get(self.pid) - if enum is not None: - ioclass = IOPriority(ioclass) + ioclass = IOPriority(ioclass) return _common.pionice(ioclass, value) @wrap_exceptions def ionice_set(self, ioclass, value): if value is None: value = 0 - if value and ioclass in {IOPRIO_CLASS_IDLE, IOPRIO_CLASS_NONE}: - raise ValueError("%r ioclass accepts no value" % ioclass) + if value and ioclass in { + IOPriority.IOPRIO_CLASS_IDLE, + IOPriority.IOPRIO_CLASS_NONE, + }: + msg = f"{ioclass!r} ioclass accepts no value" + raise ValueError(msg) if value < 0 or value > 7: msg = "value not in 0-7 range" raise ValueError(msg) return cext.proc_ioprio_set(self.pid, ioclass, value) - if prlimit is not None: + if hasattr(resource, "prlimit"): @wrap_exceptions def rlimit(self, resource_, limits=None): @@ -2278,16 +2196,16 @@ class Process: try: if limits is None: # get - return prlimit(self.pid, resource_) + return resource.prlimit(self.pid, resource_) else: # set if len(limits) != 2: msg = ( "second argument must be a (soft, hard) " - + "tuple, got %s" % repr(limits) + f"tuple, got {limits!r}" ) raise ValueError(msg) - prlimit(self.pid, resource_, limits) + resource.prlimit(self.pid, resource_, limits) except OSError as err: if err.errno == errno.ENOSYS: # I saw this happening on Travis: @@ -2298,18 +2216,17 @@ class Process: @wrap_exceptions def status(self): letter = self._parse_stat_file()['status'] - if PY3: - letter = letter.decode() + letter = letter.decode() # XXX is '?' legit? (we're not supposed to return it anyway) return PROC_STATUSES.get(letter, '?') @wrap_exceptions def open_files(self): retlist = [] - files = os.listdir("%s/%s/fd" % (self._procfs_path, self.pid)) + files = os.listdir(f"{self._procfs_path}/{self.pid}/fd") hit_enoent = False for fd in files: - file = "%s/%s/fd/%s" % (self._procfs_path, self.pid, fd) + file = f"{self._procfs_path}/{self.pid}/fd/{fd}" try: path = readlink(file) except (FileNotFoundError, ProcessLookupError): @@ -2332,11 +2249,7 @@ class Process: # absolute path though. if path.startswith('/') and isfile_strict(path): # Get file position and flags. - file = "%s/%s/fdinfo/%s" % ( - self._procfs_path, - self.pid, - fd, - ) + file = f"{self._procfs_path}/{self.pid}/fdinfo/{fd}" try: with open_binary(file) as f: pos = int(f.readline().split()[1]) @@ -2363,7 +2276,7 @@ class Process: @wrap_exceptions def num_fds(self): - return len(os.listdir("%s/%s/fd" % (self._procfs_path, self.pid))) + return len(os.listdir(f"{self._procfs_path}/{self.pid}/fd")) @wrap_exceptions def ppid(self): diff --git a/contrib/python/psutil/py3/psutil/_psosx.py b/contrib/python/psutil/py3/psutil/_psosx.py index ed9b319de7..620497b30a 100644 --- a/contrib/python/psutil/py3/psutil/_psosx.py +++ b/contrib/python/psutil/py3/psutil/_psosx.py @@ -22,8 +22,6 @@ from ._common import isfile_strict from ._common import memoize_when_activated from ._common import parse_environ_block from ._common import usage_percent -from ._compat import PermissionError -from ._compat import ProcessLookupError __extra__all__ = [] @@ -114,8 +112,7 @@ def virtual_memory(): """System virtual memory as a namedtuple.""" total, active, inactive, wired, free, speculative = cext.virtual_mem() # This is how Zabbix calculate avail and used mem: - # https://github.com/zabbix/zabbix/blob/trunk/src/libs/zbxsysinfo/ - # osx/memory.c + # https://github.com/zabbix/zabbix/blob/master/src/libs/zbxsysinfo/osx/memory.c # Also see: https://github.com/giampaolo/psutil/issues/1277 avail = inactive + free used = active + wired @@ -345,15 +342,15 @@ def wrap_exceptions(fun): @functools.wraps(fun) def wrapper(self, *args, **kwargs): + pid, ppid, name = self.pid, self._ppid, self._name try: return fun(self, *args, **kwargs) - except ProcessLookupError: - if is_zombie(self.pid): - raise ZombieProcess(self.pid, self._name, self._ppid) - else: - raise NoSuchProcess(self.pid, self._name) - except PermissionError: - raise AccessDenied(self.pid, self._name) + except ProcessLookupError as err: + if is_zombie(pid): + raise ZombieProcess(pid, name, ppid) from err + raise NoSuchProcess(pid, name) from err + except PermissionError as err: + raise AccessDenied(pid, name) from err return wrapper @@ -502,11 +499,6 @@ class Process: @wrap_exceptions def net_connections(self, kind='inet'): - if kind not in conn_tmap: - raise ValueError( - "invalid %r kind argument; choose between %s" - % (kind, ', '.join([repr(x) for x in conn_tmap])) - ) families, types = conn_tmap[kind] rawlist = cext.proc_net_connections(self.pid, families, types) ret = [] diff --git a/contrib/python/psutil/py3/psutil/_psposix.py b/contrib/python/psutil/py3/psutil/_psposix.py index 42bdfa7ef6..88703fdbd2 100644 --- a/contrib/python/psutil/py3/psutil/_psposix.py +++ b/contrib/python/psutil/py3/psutil/_psposix.py @@ -4,10 +4,10 @@ """Routines common to all posix systems.""" +import enum import glob import os import signal -import sys import time from ._common import MACOS @@ -15,25 +15,12 @@ from ._common import TimeoutExpired from ._common import memoize from ._common import sdiskusage from ._common import usage_percent -from ._compat import PY3 -from ._compat import ChildProcessError -from ._compat import FileNotFoundError -from ._compat import InterruptedError -from ._compat import PermissionError -from ._compat import ProcessLookupError -from ._compat import unicode if MACOS: from . import _psutil_osx -if PY3: - import enum -else: - enum = None - - __all__ = ['pid_exists', 'wait_pid', 'disk_usage', 'get_terminal_map'] @@ -59,23 +46,16 @@ def pid_exists(pid): return True -# Python 3.5 signals enum (contributed by me ^^): -# https://bugs.python.org/issue21076 -if enum is not None and hasattr(signal, "Signals"): - Negsignal = enum.IntEnum( - 'Negsignal', dict([(x.name, -x.value) for x in signal.Signals]) - ) - - def negsig_to_enum(num): - """Convert a negative signal value to an enum.""" - try: - return Negsignal(num) - except ValueError: - return num +Negsignal = enum.IntEnum( + 'Negsignal', {x.name: -x.value for x in signal.Signals} +) -else: # pragma: no cover - def negsig_to_enum(num): +def negsig_to_enum(num): + """Convert a negative signal value to an enum.""" + try: + return Negsignal(num) + except ValueError: return num @@ -139,7 +119,7 @@ def wait_pid( # can't determine its exit status code. while _pid_exists(pid): interval = sleep(interval) - return + return None else: if retpid == 0: # WNOHANG flag was used and PID is still running. @@ -171,7 +151,8 @@ def wait_pid( # continue else: # Should never happen. - raise ValueError("unknown process exit status %r" % status) + msg = f"unknown process exit status {status!r}" + raise ValueError(msg) def disk_usage(path): @@ -181,24 +162,7 @@ def disk_usage(path): total and used disk space whereas "free" and "percent" represent the "free" and "used percent" user disk space. """ - if PY3: - st = os.statvfs(path) - else: # pragma: no cover - # os.statvfs() does not support unicode on Python 2: - # - https://github.com/giampaolo/psutil/issues/416 - # - http://bugs.python.org/issue18695 - try: - st = os.statvfs(path) - except UnicodeEncodeError: - if isinstance(path, unicode): - try: - path = path.encode(sys.getfilesystemencoding()) - except UnicodeEncodeError: - pass - st = os.statvfs(path) - else: - raise - + st = os.statvfs(path) # Total space which is only available to root (unless changed # at system level). total = st.f_blocks * st.f_frsize diff --git a/contrib/python/psutil/py3/psutil/_pssunos.py b/contrib/python/psutil/py3/psutil/_pssunos.py index a38d939d79..78d941cc5f 100644 --- a/contrib/python/psutil/py3/psutil/_pssunos.py +++ b/contrib/python/psutil/py3/psutil/_pssunos.py @@ -18,6 +18,7 @@ from . import _psposix from . import _psutil_posix as cext_posix from . import _psutil_sunos as cext from ._common import AF_INET6 +from ._common import ENCODING from ._common import AccessDenied from ._common import NoSuchProcess from ._common import ZombieProcess @@ -28,11 +29,6 @@ from ._common import memoize_when_activated from ._common import sockfam_to_enum from ._common import socktype_to_enum from ._common import usage_percent -from ._compat import PY3 -from ._compat import FileNotFoundError -from ._compat import PermissionError -from ._compat import ProcessLookupError -from ._compat import b __extra__all__ = ["CONN_IDLE", "CONN_BOUND", "PROCFS_PATH"] @@ -147,17 +143,17 @@ def swap_memory(): p = subprocess.Popen( [ '/usr/bin/env', - 'PATH=/usr/sbin:/sbin:%s' % os.environ['PATH'], + f"PATH=/usr/sbin:/sbin:{os.environ['PATH']}", 'swap', '-l', ], stdout=subprocess.PIPE, ) stdout, _ = p.communicate() - if PY3: - stdout = stdout.decode(sys.stdout.encoding) + stdout = stdout.decode(sys.stdout.encoding) if p.returncode != 0: - raise RuntimeError("'swap -l' failed (retcode=%s)" % p.returncode) + msg = f"'swap -l' failed (retcode={p.returncode})" + raise RuntimeError(msg) lines = stdout.strip().split('\n')[1:] if not lines: @@ -244,7 +240,7 @@ def disk_partitions(all=False): continue except OSError as err: # https://github.com/giampaolo/psutil/issues/1674 - debug("skipping %r: %s" % (mountpoint, err)) + debug(f"skipping {mountpoint!r}: {err}") continue ntuple = _common.sdiskpart(device, mountpoint, fstype, opts) retlist.append(ntuple) @@ -265,14 +261,6 @@ def net_connections(kind, _pid=-1): connections (as opposed to connections opened by one process only). Only INET sockets are returned (UNIX are not). """ - cmap = _common.conn_tmap.copy() - if _pid == -1: - cmap.pop('unix', 0) - if kind not in cmap: - raise ValueError( - "invalid %r kind argument; choose between %s" - % (kind, ', '.join([repr(x) for x in cmap])) - ) families, types = _common.conn_tmap[kind] rawlist = cext.net_connections(_pid) ret = set() @@ -346,7 +334,8 @@ def users(): def pids(): """Returns a list of PIDs currently running on the system.""" - return [int(x) for x in os.listdir(b(get_procfs_path())) if x.isdigit()] + path = get_procfs_path().encode(ENCODING) + return [int(x) for x in os.listdir(path) if x.isdigit()] def pid_exists(pid): @@ -361,24 +350,23 @@ def wrap_exceptions(fun): @functools.wraps(fun) def wrapper(self, *args, **kwargs): + pid, ppid, name = self.pid, self._ppid, self._name try: return fun(self, *args, **kwargs) - except (FileNotFoundError, ProcessLookupError): + except (FileNotFoundError, ProcessLookupError) as err: # ENOENT (no such file or directory) gets raised on open(). # ESRCH (no such process) can get raised on read() if # process is gone in meantime. - if not pid_exists(self.pid): - raise NoSuchProcess(self.pid, self._name) - else: - raise ZombieProcess(self.pid, self._name, self._ppid) - except PermissionError: - raise AccessDenied(self.pid, self._name) - except OSError: - if self.pid == 0: + if not pid_exists(pid): + raise NoSuchProcess(pid, name) from err + raise ZombieProcess(pid, name, ppid) from err + except PermissionError as err: + raise AccessDenied(pid, name) from err + except OSError as err: + if pid == 0: if 0 in pids(): - raise AccessDenied(self.pid, self._name) - else: - raise + raise AccessDenied(pid, name) from err + raise raise return wrapper @@ -399,7 +387,7 @@ class Process: """Raise NSP if the process disappeared on us.""" # For those C function who do not raise NSP, possibly returning # incorrect or incomplete result. - os.stat('%s/%s' % (self._procfs_path, self.pid)) + os.stat(f"{self._procfs_path}/{self.pid}") def oneshot_enter(self): self._proc_name_and_args.cache_activate(self) @@ -420,7 +408,7 @@ class Process: @memoize_when_activated def _proc_basic_info(self): if self.pid == 0 and not os.path.exists( - '%s/%s/psinfo' % (self._procfs_path, self.pid) + f"{self._procfs_path}/{self.pid}/psinfo" ): raise AccessDenied(self.pid) ret = cext.proc_basic_info(self.pid, self._procfs_path) @@ -440,9 +428,7 @@ class Process: @wrap_exceptions def exe(self): try: - return os.readlink( - "%s/%s/path/a.out" % (self._procfs_path, self.pid) - ) + return os.readlink(f"{self._procfs_path}/{self.pid}/path/a.out") except OSError: pass # continue and guess the exe name from the cmdline # Will be guessed later from cmdline but we want to explicitly @@ -539,9 +525,7 @@ class Process: if tty != cext.PRNODEV: for x in (0, 1, 2, 255): try: - return os.readlink( - '%s/%d/path/%d' % (procfs_path, self.pid, x) - ) + return os.readlink(f"{procfs_path}/{self.pid}/path/{x}") except FileNotFoundError: hit_enoent = True continue @@ -553,12 +537,12 @@ class Process: # /proc/PID/path/cwd may not be resolved by readlink() even if # it exists (ls shows it). If that's the case and the process # is still alive return None (we can return None also on BSD). - # Reference: http://goo.gl/55XgO + # Reference: https://groups.google.com/g/comp.unix.solaris/c/tcqvhTNFCAs procfs_path = self._procfs_path try: - return os.readlink("%s/%s/path/cwd" % (procfs_path, self.pid)) + return os.readlink(f"{procfs_path}/{self.pid}/path/cwd") except FileNotFoundError: - os.stat("%s/%s" % (procfs_path, self.pid)) # raise NSP or AD + os.stat(f"{procfs_path}/{self.pid}") # raise NSP or AD return "" @wrap_exceptions @@ -580,7 +564,7 @@ class Process: def threads(self): procfs_path = self._procfs_path ret = [] - tids = os.listdir('%s/%d/lwp' % (procfs_path, self.pid)) + tids = os.listdir(f"{procfs_path}/{self.pid}/lwp") hit_enoent = False for tid in tids: tid = int(tid) @@ -588,7 +572,7 @@ class Process: utime, stime = cext.query_process_thread( self.pid, tid, procfs_path ) - except EnvironmentError as err: + except OSError as err: if err.errno == errno.EOVERFLOW and not IS_64_BIT: # We may get here if we attempt to query a 64bit process # with a 32bit python. @@ -615,8 +599,8 @@ class Process: retlist = [] hit_enoent = False procfs_path = self._procfs_path - pathdir = '%s/%d/path' % (procfs_path, self.pid) - for fd in os.listdir('%s/%d/fd' % (procfs_path, self.pid)): + pathdir = f"{procfs_path}/{self.pid}/path" + for fd in os.listdir(f"{procfs_path}/{self.pid}/fd"): path = os.path.join(pathdir, fd) if os.path.islink(path): try: @@ -640,16 +624,16 @@ class Process: cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) stdout, stderr = p.communicate() - if PY3: - stdout, stderr = ( - x.decode(sys.stdout.encoding) for x in (stdout, stderr) - ) + stdout, stderr = ( + x.decode(sys.stdout.encoding) for x in (stdout, stderr) + ) if p.returncode != 0: if 'permission denied' in stderr.lower(): raise AccessDenied(self.pid, self._name) if 'no such process' in stderr.lower(): raise NoSuchProcess(self.pid, self._name) - raise RuntimeError("%r command error\n%s" % (cmd, stderr)) + msg = f"{cmd!r} command error\n{stderr}" + raise RuntimeError(msg) lines = stdout.split('\n')[2:] for i, line in enumerate(lines): @@ -675,7 +659,7 @@ class Process: # is no longer there. if not ret: # will raise NSP if process is gone - os.stat('%s/%s' % (self._procfs_path, self.pid)) + os.stat(f"{self._procfs_path}/{self.pid}") # UNIX sockets if kind in {'all', 'unix'}: @@ -691,9 +675,8 @@ class Process: @wrap_exceptions def memory_maps(self): def toaddr(start, end): - return '%s-%s' % ( - hex(start)[2:].strip('L'), - hex(end)[2:].strip('L'), + return "{}-{}".format( + hex(start)[2:].strip('L'), hex(end)[2:].strip('L') ) procfs_path = self._procfs_path @@ -718,9 +701,7 @@ class Process: addr = toaddr(addr, addrsize) if not name.startswith('['): try: - name = os.readlink( - '%s/%s/path/%s' % (procfs_path, self.pid, name) - ) + name = os.readlink(f"{procfs_path}/{self.pid}/path/{name}") except OSError as err: if err.errno == errno.ENOENT: # sometimes the link may not be resolved by @@ -729,7 +710,7 @@ class Process: # unresolved link path. # This seems an inconsistency with /proc similar # to: http://goo.gl/55XgO - name = '%s/%s/path/%s' % (procfs_path, self.pid, name) + name = f"{procfs_path}/{self.pid}/path/{name}" hit_enoent = True else: raise @@ -740,7 +721,7 @@ class Process: @wrap_exceptions def num_fds(self): - return len(os.listdir("%s/%s/fd" % (self._procfs_path, self.pid))) + return len(os.listdir(f"{self._procfs_path}/{self.pid}/fd")) @wrap_exceptions def num_ctx_switches(self): diff --git a/contrib/python/psutil/py3/psutil/_psutil_common.c b/contrib/python/psutil/py3/psutil/_psutil_common.c index 941f6bff1b..2a56ae0f4a 100644 --- a/contrib/python/psutil/py3/psutil/_psutil_common.c +++ b/contrib/python/psutil/py3/psutil/_psutil_common.c @@ -69,15 +69,6 @@ PyErr_SetExcFromWindowsErrWithFilenameObject(PyObject *type, return PyErr_SetFromWindowsErrWithFilename(ierr, NULL); } #endif // !defined(PyErr_SetExcFromWindowsErrWithFilenameObject) - - -// PyPy 2.7 -#if !defined(PyErr_SetFromWindowsErr) -PyObject * -PyErr_SetFromWindowsErr(int winerr) { - return PyErr_SetFromWindowsErrWithFilename(winerr, ""); -} -#endif // !defined(PyErr_SetFromWindowsErr) #endif // defined(PSUTIL_WINDOWS) && defined(PYPY_VERSION) diff --git a/contrib/python/psutil/py3/psutil/_psutil_common.h b/contrib/python/psutil/py3/psutil/_psutil_common.h index 2cdfa9d4d6..024452630f 100644 --- a/contrib/python/psutil/py3/psutil/_psutil_common.h +++ b/contrib/python/psutil/py3/psutil/_psutil_common.h @@ -23,14 +23,6 @@ static const int PSUTIL_CONN_NONE = 128; // --- Backward compatibility with missing Python.h APIs // ==================================================================== -#if PY_MAJOR_VERSION < 3 - // On Python 2 we just return a plain byte string, which is never - // supposed to raise decoding errors, see: - // https://github.com/giampaolo/psutil/issues/1040 - #define PyUnicode_DecodeFSDefault PyString_FromString - #define PyUnicode_DecodeFSDefaultAndSize PyString_FromStringAndSize -#endif - #if defined(PSUTIL_WINDOWS) && defined(PYPY_VERSION) #if !defined(PyErr_SetFromWindowsErrWithFilename) PyObject *PyErr_SetFromWindowsErrWithFilename(int ierr, @@ -45,7 +37,6 @@ static const int PSUTIL_CONN_NONE = 128; // --- _Py_PARSE_PID // SIZEOF_INT|LONG is missing on Linux + PyPy (only?). -// SIZEOF_PID_T is missing on Windows + Python2. // In this case we guess it from setup.py. It's not 100% bullet proof, // If wrong we'll probably get compiler warnings. // FWIW on all UNIX platforms I've seen pid_t is defined as an int. @@ -60,8 +51,8 @@ static const int PSUTIL_CONN_NONE = 128; #define SIZEOF_PID_T PSUTIL_SIZEOF_PID_T // set as a macro in setup.py #endif -// _Py_PARSE_PID is Python 3 only, but since it's private make sure it's -// always present. +// _Py_PARSE_PID was added in Python 3, but since it's private we make +// sure it's always present. #ifndef _Py_PARSE_PID #if SIZEOF_PID_T == SIZEOF_INT #define _Py_PARSE_PID "i" @@ -75,14 +66,10 @@ static const int PSUTIL_CONN_NONE = 128; #endif #endif -// Python 2 or PyPy on Windows +// PyPy on Windows #ifndef PyLong_FromPid #if ((SIZEOF_PID_T == SIZEOF_INT) || (SIZEOF_PID_T == SIZEOF_LONG)) - #if PY_MAJOR_VERSION >= 3 - #define PyLong_FromPid PyLong_FromLong - #else - #define PyLong_FromPid PyInt_FromLong - #endif + #define PyLong_FromPid PyLong_FromLong #elif defined(SIZEOF_LONG_LONG) && SIZEOF_PID_T == SIZEOF_LONG_LONG #define PyLong_FromPid PyLong_FromLongLong #else diff --git a/contrib/python/psutil/py3/psutil/_psutil_linux.c b/contrib/python/psutil/py3/psutil/_psutil_linux.c index 46244c5792..fcd886bfb6 100644 --- a/contrib/python/psutil/py3/psutil/_psutil_linux.c +++ b/contrib/python/psutil/py3/psutil/_psutil_linux.c @@ -19,6 +19,13 @@ #include "arch/linux/proc.h" #include "arch/linux/users.h" +// May happen on old RedHat versions, see: +// https://github.com/giampaolo/psutil/issues/607 +#ifndef DUPLEX_UNKNOWN + #define DUPLEX_UNKNOWN 0xff +#endif + +#define INITERR return NULL static PyMethodDef mod_methods[] = { // --- per-process functions @@ -41,40 +48,24 @@ static PyMethodDef mod_methods[] = { {"set_debug", psutil_set_debug, METH_VARARGS}, {NULL, NULL, 0, NULL} }; -// May happen on old RedHat versions, see: -// https://github.com/giampaolo/psutil/issues/607 -#ifndef DUPLEX_UNKNOWN - #define DUPLEX_UNKNOWN 0xff -#endif - -#if PY_MAJOR_VERSION >= 3 - #define INITERR return NULL - static struct PyModuleDef moduledef = { - PyModuleDef_HEAD_INIT, - "_psutil_linux", - NULL, - -1, - mod_methods, - NULL, - NULL, - NULL, - NULL - }; +static struct PyModuleDef moduledef = { + PyModuleDef_HEAD_INIT, + "_psutil_linux", + NULL, + -1, + mod_methods, + NULL, + NULL, + NULL, + NULL +}; - PyObject *PyInit__psutil_linux(void) -#else /* PY_MAJOR_VERSION */ - #define INITERR return - void init_psutil_linux(void) -#endif /* PY_MAJOR_VERSION */ -{ -#if PY_MAJOR_VERSION >= 3 +PyObject * +PyInit__psutil_linux(void) { PyObject *mod = PyModule_Create(&moduledef); -#else - PyObject *mod = Py_InitModule("_psutil_linux", mod_methods); -#endif if (mod == NULL) INITERR; @@ -91,7 +82,5 @@ static PyMethodDef mod_methods[] = { if (mod == NULL) INITERR; -#if PY_MAJOR_VERSION >= 3 return mod; -#endif } diff --git a/contrib/python/psutil/py3/psutil/_psutil_osx.c b/contrib/python/psutil/py3/psutil/_psutil_osx.c index 09fa267a98..b16103379b 100644 --- a/contrib/python/psutil/py3/psutil/_psutil_osx.c +++ b/contrib/python/psutil/py3/psutil/_psutil_osx.c @@ -7,6 +7,7 @@ */ #include <Python.h> +#include <sys/time.h> // needed for old macOS versions #include <sys/proc.h> #include <netinet/tcp_fsm.h> @@ -20,6 +21,8 @@ #include "arch/osx/sys.h" +#define INITERR return NULL + static PyMethodDef mod_methods[] = { // --- per-process functions {"proc_cmdline", psutil_proc_cmdline, METH_VARARGS}, @@ -61,33 +64,22 @@ static PyMethodDef mod_methods[] = { }; -#if PY_MAJOR_VERSION >= 3 - #define INITERR return NULL - - static struct PyModuleDef moduledef = { - PyModuleDef_HEAD_INIT, - "_psutil_osx", - NULL, - -1, - mod_methods, - NULL, - NULL, - NULL, - NULL - }; - - PyObject *PyInit__psutil_osx(void) -#else /* PY_MAJOR_VERSION */ - #define INITERR return - - void init_psutil_osx(void) -#endif /* PY_MAJOR_VERSION */ -{ -#if PY_MAJOR_VERSION >= 3 +static struct PyModuleDef moduledef = { + PyModuleDef_HEAD_INIT, + "_psutil_osx", + NULL, + -1, + mod_methods, + NULL, + NULL, + NULL, + NULL +}; + + +PyObject * +PyInit__psutil_osx(void) { PyObject *mod = PyModule_Create(&moduledef); -#else - PyObject *mod = Py_InitModule("_psutil_osx", mod_methods); -#endif if (mod == NULL) INITERR; @@ -140,7 +132,5 @@ static PyMethodDef mod_methods[] = { if (mod == NULL) INITERR; -#if PY_MAJOR_VERSION >= 3 return mod; -#endif } diff --git a/contrib/python/psutil/py3/psutil/_psutil_posix.c b/contrib/python/psutil/py3/psutil/_psutil_posix.c index 7e6eaac46c..de10f7854a 100644 --- a/contrib/python/psutil/py3/psutil/_psutil_posix.c +++ b/contrib/python/psutil/py3/psutil/_psutil_posix.c @@ -52,6 +52,9 @@ #include "_psutil_common.h" +#define INITERR return NULL + + // ==================================================================== // --- Utils // ==================================================================== @@ -434,11 +437,7 @@ append_flag(PyObject *py_retlist, const char * flag_name) { PyObject *py_str = NULL; -#if PY_MAJOR_VERSION >= 3 py_str = PyUnicode_FromString(flag_name); -#else - py_str = PyString_FromString(flag_name); -#endif if (! py_str) return 0; if (PyList_Append(py_retlist, py_str)) { @@ -883,33 +882,21 @@ static PyMethodDef mod_methods[] = { }; -#if PY_MAJOR_VERSION >= 3 - #define INITERR return NULL - - static struct PyModuleDef moduledef = { - PyModuleDef_HEAD_INIT, - "_psutil_posix", - NULL, - -1, - mod_methods, - NULL, - NULL, - NULL, - NULL - }; - - PyObject *PyInit__psutil_posix(void) -#else /* PY_MAJOR_VERSION */ - #define INITERR return +static struct PyModuleDef moduledef = { + PyModuleDef_HEAD_INIT, + "_psutil_posix", + NULL, + -1, + mod_methods, + NULL, + NULL, + NULL, + NULL +}; - void init_psutil_posix(void) -#endif /* PY_MAJOR_VERSION */ -{ -#if PY_MAJOR_VERSION >= 3 +PyObject * +PyInit__psutil_posix(void) { PyObject *mod = PyModule_Create(&moduledef); -#else - PyObject *mod = Py_InitModule("_psutil_posix", mod_methods); -#endif if (mod == NULL) INITERR; @@ -1022,9 +1009,7 @@ static PyMethodDef mod_methods[] = { if (mod == NULL) INITERR; -#if PY_MAJOR_VERSION >= 3 return mod; -#endif } #ifdef __cplusplus diff --git a/contrib/python/psutil/py3/psutil/_psutil_windows.c b/contrib/python/psutil/py3/psutil/_psutil_windows.c index 0c221bdc23..0af18f3e26 100644 --- a/contrib/python/psutil/py3/psutil/_psutil_windows.c +++ b/contrib/python/psutil/py3/psutil/_psutil_windows.c @@ -34,6 +34,10 @@ #include "arch/windows/wmi.h" +#define INITERROR return NULL +#define GETSTATE(m) ((struct module_state*)PyModule_GetState(m)) + + // ------------------------ Python init --------------------------- static PyMethodDef @@ -116,21 +120,15 @@ struct module_state { PyObject *error; }; -#if PY_MAJOR_VERSION >= 3 -#define GETSTATE(m) ((struct module_state*)PyModule_GetState(m)) -#else -#define GETSTATE(m) (&_state) -static struct module_state _state; -#endif - -#if PY_MAJOR_VERSION >= 3 -static int psutil_windows_traverse(PyObject *m, visitproc visit, void *arg) { +static int +psutil_windows_traverse(PyObject *m, visitproc visit, void *arg) { Py_VISIT(GETSTATE(m)->error); return 0; } -static int psutil_windows_clear(PyObject *m) { +static int +psutil_windows_clear(PyObject *m) { Py_CLEAR(GETSTATE(m)->error); return 0; } @@ -147,21 +145,12 @@ static struct PyModuleDef moduledef = { NULL }; -#define INITERROR return NULL - -PyMODINIT_FUNC PyInit__psutil_windows(void) -#else -#define INITERROR return -void init_psutil_windows(void) -#endif -{ +PyMODINIT_FUNC +PyInit__psutil_windows(void) { struct module_state *st = NULL; -#if PY_MAJOR_VERSION >= 3 + PyObject *module = PyModule_Create(&moduledef); -#else - PyObject *module = Py_InitModule("_psutil_windows", PsutilMethods); -#endif if (module == NULL) INITERROR; @@ -283,7 +272,5 @@ void init_psutil_windows(void) PyModule_AddIntConstant( module, "WINDOWS_10", PSUTIL_WINDOWS_10); -#if PY_MAJOR_VERSION >= 3 return module; -#endif } diff --git a/contrib/python/psutil/py3/psutil/_pswindows.py b/contrib/python/psutil/py3/psutil/_pswindows.py index e39ba711fa..e5af3c90f4 100644 --- a/contrib/python/psutil/py3/psutil/_pswindows.py +++ b/contrib/python/psutil/py3/psutil/_pswindows.py @@ -5,7 +5,7 @@ """Windows platform implementation.""" import contextlib -import errno +import enum import functools import os import signal @@ -15,7 +15,6 @@ from collections import namedtuple from . import _common from ._common import ENCODING -from ._common import ENCODING_ERRS from ._common import AccessDenied from ._common import NoSuchProcess from ._common import TimeoutExpired @@ -27,11 +26,6 @@ from ._common import memoize from ._common import memoize_when_activated from ._common import parse_environ_block from ._common import usage_percent -from ._compat import PY3 -from ._compat import long -from ._compat import lru_cache -from ._compat import range -from ._compat import unicode from ._psutil_windows import ABOVE_NORMAL_PRIORITY_CLASS from ._psutil_windows import BELOW_NORMAL_PRIORITY_CLASS from ._psutil_windows import HIGH_PRIORITY_CLASS @@ -54,14 +48,10 @@ except ImportError as err: msg = "this Windows version is too old (< Windows Vista); " msg += "psutil 3.4.2 is the latest version which supports Windows " msg += "2000, XP and 2003 server" - raise RuntimeError(msg) + raise RuntimeError(msg) from err else: raise -if PY3: - import enum -else: - enum = None # process priority constants, import from __init__.py: # http://msdn.microsoft.com/en-us/library/ms686219(v=vs.85).aspx @@ -88,11 +78,8 @@ CONN_DELETE_TCB = "DELETE_TCB" ERROR_PARTIAL_COPY = 299 PYPY = '__pypy__' in sys.builtin_module_names -if enum is None: - AF_LINK = -1 -else: - AddressFamily = enum.IntEnum('AddressFamily', {'AF_LINK': -1}) - AF_LINK = AddressFamily.AF_LINK +AddressFamily = enum.IntEnum('AddressFamily', {'AF_LINK': -1}) +AF_LINK = AddressFamily.AF_LINK TCP_STATUSES = { cext.MIB_TCP_STATE_ESTAB: _common.CONN_ESTABLISHED, @@ -110,32 +97,27 @@ TCP_STATUSES = { cext.PSUTIL_CONN_NONE: _common.CONN_NONE, } -if enum is not None: - class Priority(enum.IntEnum): - ABOVE_NORMAL_PRIORITY_CLASS = ABOVE_NORMAL_PRIORITY_CLASS - BELOW_NORMAL_PRIORITY_CLASS = BELOW_NORMAL_PRIORITY_CLASS - HIGH_PRIORITY_CLASS = HIGH_PRIORITY_CLASS - IDLE_PRIORITY_CLASS = IDLE_PRIORITY_CLASS - NORMAL_PRIORITY_CLASS = NORMAL_PRIORITY_CLASS - REALTIME_PRIORITY_CLASS = REALTIME_PRIORITY_CLASS +class Priority(enum.IntEnum): + ABOVE_NORMAL_PRIORITY_CLASS = ABOVE_NORMAL_PRIORITY_CLASS + BELOW_NORMAL_PRIORITY_CLASS = BELOW_NORMAL_PRIORITY_CLASS + HIGH_PRIORITY_CLASS = HIGH_PRIORITY_CLASS + IDLE_PRIORITY_CLASS = IDLE_PRIORITY_CLASS + NORMAL_PRIORITY_CLASS = NORMAL_PRIORITY_CLASS + REALTIME_PRIORITY_CLASS = REALTIME_PRIORITY_CLASS - globals().update(Priority.__members__) -if enum is None: +globals().update(Priority.__members__) + + +class IOPriority(enum.IntEnum): IOPRIO_VERYLOW = 0 IOPRIO_LOW = 1 IOPRIO_NORMAL = 2 IOPRIO_HIGH = 3 -else: - class IOPriority(enum.IntEnum): - IOPRIO_VERYLOW = 0 - IOPRIO_LOW = 1 - IOPRIO_NORMAL = 2 - IOPRIO_HIGH = 3 - globals().update(IOPriority.__members__) +globals().update(IOPriority.__members__) pinfo_map = dict( num_handles=0, @@ -199,7 +181,7 @@ pio = namedtuple('pio', ['read_count', 'write_count', # ===================================================================== -@lru_cache(maxsize=512) +@functools.lru_cache(maxsize=512) def convert_dos_path(s): r"""Convert paths using native DOS format like: "\Device\HarddiskVolume1\Windows\systemew\file.txt" @@ -212,18 +194,6 @@ def convert_dos_path(s): return os.path.join(driveletter, remainder) -def py2_strencode(s): - """Encode a unicode string to a byte string by using the default fs - encoding + "replace" error handler. - """ - if PY3: - return s - if isinstance(s, str): - return s - else: - return s.encode(ENCODING, ENCODING_ERRS) - - @memoize def getpagesize(): return cext.getpagesize() @@ -282,7 +252,7 @@ disk_io_counters = cext.disk_io_counters def disk_usage(path): """Return disk usage associated with path.""" - if PY3 and isinstance(path, bytes): + if isinstance(path, bytes): # XXX: do we want to use "strict"? Probably yes, in order # to fail immediately. After all we are accepting input here... path = path.decode(ENCODING, errors="strict") @@ -367,7 +337,7 @@ def getloadavg(): # Drop to 2 decimal points which is what Linux does raw_loads = cext.getloadavg() - return tuple([round(load, 2) for load in raw_loads]) + return tuple(round(load, 2) for load in raw_loads) # ===================================================================== @@ -379,11 +349,6 @@ def net_connections(kind, _pid=-1): """Return socket connections. If pid == -1 return system-wide connections (as opposed to connections opened by one process only). """ - if kind not in conn_tmap: - raise ValueError( - "invalid %r kind argument; choose between %s" - % (kind, ', '.join([repr(x) for x in conn_tmap])) - ) families, types = conn_tmap[kind] rawlist = cext.net_connections(_pid, families, types) ret = set() @@ -408,9 +373,6 @@ def net_if_stats(): ret = {} rawdict = cext.net_if_stats() for name, items in rawdict.items(): - if not PY3: - assert isinstance(name, unicode), type(name) - name = py2_strencode(name) isup, duplex, speed, mtu = items if hasattr(_common, 'NicDuplex'): duplex = _common.NicDuplex(duplex) @@ -422,18 +384,12 @@ def net_io_counters(): """Return network I/O statistics for every network interface installed on the system as a dict of raw tuples. """ - ret = cext.net_io_counters() - return dict([(py2_strencode(k), v) for k, v in ret.items()]) + return cext.net_io_counters() def net_if_addrs(): """Return the addresses associated to each NIC.""" - ret = [] - for items in cext.net_if_addrs(): - items = list(items) - items[0] = py2_strencode(items[0]) - ret.append(items) - return ret + return cext.net_if_addrs() # ===================================================================== @@ -489,7 +445,6 @@ def users(): rawlist = cext.users() for item in rawlist: user, hostname, tstamp = item - user = py2_strencode(user) nt = _common.suser(user, None, hostname, tstamp, None) retlist.append(nt) return retlist @@ -503,7 +458,7 @@ def users(): def win_service_iter(): """Yields a list of WindowsService instances.""" for name, display_name in cext.winservice_enumerate(): - yield WindowsService(py2_strencode(name), py2_strencode(display_name)) + yield WindowsService(name, display_name) def win_service_get(name): @@ -521,14 +476,11 @@ class WindowsService: # noqa: PLW1641 self._display_name = display_name def __str__(self): - details = "(name=%r, display_name=%r)" % ( - self._name, - self._display_name, - ) - return "%s%s" % (self.__class__.__name__, details) + details = f"(name={self._name!r}, display_name={self._display_name!r})" + return f"{self.__class__.__name__}{details}" def __repr__(self): - return "<%s at %s>" % (self.__str__(), id(self)) + return f"<{self.__str__()} at {id(self)}>" def __eq__(self, other): # Test for equality with another WindosService object based @@ -547,10 +499,10 @@ class WindowsService: # noqa: PLW1641 ) # XXX - update _self.display_name? return dict( - display_name=py2_strencode(display_name), - binpath=py2_strencode(binpath), - username=py2_strencode(username), - start_type=py2_strencode(start_type), + display_name=display_name, + binpath=binpath, + username=username, + start_type=start_type, ) def _query_status(self): @@ -568,18 +520,18 @@ class WindowsService: # noqa: PLW1641 try: yield except OSError as err: + name = self._name if is_permission_err(err): msg = ( - "service %r is not querable (not enough privileges)" - % self._name + f"service {name!r} is not querable (not enough privileges)" ) - raise AccessDenied(pid=None, name=self._name, msg=msg) + raise AccessDenied(pid=None, name=name, msg=msg) from err elif err.winerror in { cext.ERROR_INVALID_NAME, cext.ERROR_SERVICE_DOES_NOT_EXIST, }: - msg = "service %r does not exist" % self._name - raise NoSuchProcess(pid=None, name=self._name, msg=msg) + msg = f"service {name!r} does not exist" + raise NoSuchProcess(pid=None, name=name, msg=msg) from err else: raise @@ -628,7 +580,7 @@ class WindowsService: # noqa: PLW1641 def description(self): """Service long description.""" - return py2_strencode(cext.winservice_query_descr(self.name())) + return cext.winservice_query_descr(self.name()) # utils @@ -696,12 +648,7 @@ ppid_map = cext.ppid_map # used internally by Process.children() def is_permission_err(exc): """Return True if this is a permission error.""" assert isinstance(exc, OSError), exc - if exc.errno in {errno.EPERM, errno.EACCES}: - return True - # On Python 2 OSError doesn't always have 'winerror'. Sometimes - # it does, in which case the original exception was WindowsError - # (which is a subclass of OSError). - return getattr(exc, "winerror", -1) in { + return isinstance(exc, PermissionError) or exc.winerror in { cext.ERROR_ACCESS_DENIED, cext.ERROR_PRIVILEGE_NOT_HELD, } @@ -712,7 +659,7 @@ def convert_oserror(exc, pid=None, name=None): assert isinstance(exc, OSError), exc if is_permission_err(exc): return AccessDenied(pid=pid, name=name) - if exc.errno == errno.ESRCH: + if isinstance(exc, ProcessLookupError): return NoSuchProcess(pid=pid, name=name) raise exc @@ -725,7 +672,7 @@ def wrap_exceptions(fun): try: return fun(self, *args, **kwargs) except OSError as err: - raise convert_oserror(err, pid=self.pid, name=self._name) + raise convert_oserror(err, pid=self.pid, name=self._name) from err return wrapper @@ -742,7 +689,7 @@ def retry_error_partial_copy(fun): for _ in range(times): # retries for roughly 1 second try: return fun(self, *args, **kwargs) - except WindowsError as _: + except OSError as _: err = _ if err.winerror == ERROR_PARTIAL_COPY: time.sleep(delay) @@ -750,8 +697,8 @@ def retry_error_partial_copy(fun): continue raise msg = ( - "{} retried {} times, converted to AccessDenied as it's still" - "returning {}".format(fun, times, err) + f"{fun} retried {times} times, converted to AccessDenied as it's " + f"still returning {err}" ) raise AccessDenied(pid=self.pid, name=self._name, msg=msg) @@ -805,17 +752,15 @@ class Process: if PYPY: try: exe = cext.proc_exe(self.pid) - except WindowsError as err: + except OSError as err: # 24 = ERROR_TOO_MANY_OPEN_FILES. Not sure why this happens # (perhaps PyPy's JIT delaying garbage collection of files?). if err.errno == 24: - debug("%r translated into AccessDenied" % err) - raise AccessDenied(self.pid, self._name) + debug(f"{err!r} translated into AccessDenied") + raise AccessDenied(self.pid, self._name) from err raise else: exe = cext.proc_exe(self.pid) - if not PY3: - exe = py2_strencode(exe) if exe.startswith('\\'): return convert_dos_path(exe) return exe # May be "Registry", "MemCompression", ... @@ -827,32 +772,26 @@ class Process: # PEB method detects cmdline changes but requires more # privileges: https://github.com/giampaolo/psutil/pull/1398 try: - ret = cext.proc_cmdline(self.pid, use_peb=True) + return cext.proc_cmdline(self.pid, use_peb=True) except OSError as err: if is_permission_err(err): - ret = cext.proc_cmdline(self.pid, use_peb=False) + return cext.proc_cmdline(self.pid, use_peb=False) else: raise else: - ret = cext.proc_cmdline(self.pid, use_peb=True) - if PY3: - return ret - else: - return [py2_strencode(s) for s in ret] + return cext.proc_cmdline(self.pid, use_peb=True) @wrap_exceptions @retry_error_partial_copy def environ(self): - ustr = cext.proc_environ(self.pid) - if ustr and not PY3: - assert isinstance(ustr, unicode), type(ustr) - return parse_environ_block(py2_strencode(ustr)) + s = cext.proc_environ(self.pid) + return parse_environ_block(s) def ppid(self): try: return ppid_map()[self.pid] except KeyError: - raise NoSuchProcess(self.pid, self._name) + raise NoSuchProcess(self.pid, self._name) from None def _get_raw_meminfo(self): try: @@ -900,12 +839,10 @@ class Process: except OSError as err: # XXX - can't use wrap_exceptions decorator as we're # returning a generator; probably needs refactoring. - raise convert_oserror(err, self.pid, self._name) + raise convert_oserror(err, self.pid, self._name) from err else: for addr, perm, path, rss in raw: path = convert_dos_path(path) - if not PY3: - path = py2_strencode(path) addr = hex(addr) yield (addr, perm, path, rss) @@ -917,11 +854,7 @@ class Process: def send_signal(self, sig): if sig == signal.SIGTERM: cext.proc_kill(self.pid) - # py >= 2.7 - elif sig in { - getattr(signal, "CTRL_C_EVENT", object()), - getattr(signal, "CTRL_BREAK_EVENT", object()), - }: + elif sig in {signal.CTRL_C_EVENT, signal.CTRL_BREAK_EVENT}: os.kill(self.pid, sig) else: msg = ( @@ -946,9 +879,9 @@ class Process: # May also be None if OpenProcess() failed with # ERROR_INVALID_PARAMETER, meaning PID is already gone. exit_code = cext.proc_wait(self.pid, cext_timeout) - except cext.TimeoutExpired: + except cext.TimeoutExpired as err: # WaitForSingleObject() returned WAIT_TIMEOUT. Just raise. - raise TimeoutExpired(timeout, self.pid, self._name) + raise TimeoutExpired(timeout, self.pid, self._name) from err except cext.TimeoutAbandoned: # WaitForSingleObject() returned WAIT_ABANDONED, see: # https://github.com/giampaolo/psutil/issues/1224 @@ -978,7 +911,7 @@ class Process: if self.pid in {0, 4}: return 'NT AUTHORITY\\SYSTEM' domain, user = cext.proc_username(self.pid) - return py2_strencode(domain) + '\\' + py2_strencode(user) + return f"{domain}\\{user}" @wrap_exceptions def create_time(self, fast_only=False): @@ -1038,7 +971,7 @@ class Process: # return a normalized pathname since the native C function appends # "\\" at the and of the path path = cext.proc_cwd(self.pid) - return py2_strencode(os.path.normpath(path)) + return os.path.normpath(path) @wrap_exceptions def open_files(self): @@ -1050,12 +983,10 @@ class Process: # Convert the first part in the corresponding drive letter # (e.g. "C:\") by using Windows's QueryDosDevice() raw_file_names = cext.proc_open_files(self.pid) - for _file in raw_file_names: - _file = convert_dos_path(_file) - if isfile_strict(_file): - if not PY3: - _file = py2_strencode(_file) - ntuple = _common.popenfile(_file, -1) + for file in raw_file_names: + file = convert_dos_path(file) + if isfile_strict(file): + ntuple = _common.popenfile(file, -1) ret.add(ntuple) return list(ret) @@ -1066,8 +997,7 @@ class Process: @wrap_exceptions def nice_get(self): value = cext.proc_priority_get(self.pid) - if enum is not None: - value = Priority(value) + value = Priority(value) return value @wrap_exceptions @@ -1077,8 +1007,7 @@ class Process: @wrap_exceptions def ionice_get(self): ret = cext.proc_io_priority_get(self.pid) - if enum is not None: - ret = IOPriority(ret) + ret = IOPriority(ret) return ret @wrap_exceptions @@ -1087,12 +1016,13 @@ class Process: msg = "value argument not accepted on Windows" raise TypeError(msg) if ioclass not in { - IOPRIO_VERYLOW, - IOPRIO_LOW, - IOPRIO_NORMAL, - IOPRIO_HIGH, + IOPriority.IOPRIO_VERYLOW, + IOPriority.IOPRIO_LOW, + IOPriority.IOPRIO_NORMAL, + IOPriority.IOPRIO_HIGH, }: - raise ValueError("%s is not a valid priority" % ioclass) + msg = f"{ioclass} is not a valid priority" + raise ValueError(msg) cext.proc_io_priority_set(self.pid, ioclass) @wrap_exceptions @@ -1134,7 +1064,8 @@ class Process: def cpu_affinity_set(self, value): def to_bitmask(ls): if not ls: - raise ValueError("invalid argument %r" % ls) + msg = f"invalid argument {ls!r}" + raise ValueError(msg) out = 0 for b in ls: out |= 2**b @@ -1146,12 +1077,11 @@ class Process: allcpus = list(range(len(per_cpu_times()))) for cpu in value: if cpu not in allcpus: - if not isinstance(cpu, (int, long)): - raise TypeError( - "invalid CPU %r; an integer is required" % cpu - ) - else: - raise ValueError("invalid CPU %r" % cpu) + if not isinstance(cpu, int): + msg = f"invalid CPU {cpu!r}; an integer is required" + raise TypeError(msg) + msg = f"invalid CPU {cpu!r}" + raise ValueError(msg) bitmask = to_bitmask(value) cext.proc_cpu_affinity_set(self.pid, bitmask) diff --git a/contrib/python/psutil/py3/psutil/arch/linux/proc.c b/contrib/python/psutil/py3/psutil/arch/linux/proc.c index b58a3ce2a2..f8230ee75b 100644 --- a/contrib/python/psutil/py3/psutil/arch/linux/proc.c +++ b/contrib/python/psutil/py3/psutil/arch/linux/proc.c @@ -118,11 +118,7 @@ psutil_proc_cpu_affinity_get(PyObject *self, PyObject *args) { cpucount_s = CPU_COUNT_S(setsize, mask); for (cpu = 0, count = cpucount_s; count; cpu++) { if (CPU_ISSET_S(cpu, setsize, mask)) { -#if PY_MAJOR_VERSION >= 3 PyObject *cpu_num = PyLong_FromLong(cpu); -#else - PyObject *cpu_num = PyInt_FromLong(cpu); -#endif if (cpu_num == NULL) goto error; if (PyList_Append(py_list, cpu_num)) { @@ -159,11 +155,7 @@ psutil_proc_cpu_affinity_set(PyObject *self, PyObject *args) { if (!PySequence_Check(py_cpu_set)) { return PyErr_Format( PyExc_TypeError, -#if PY_MAJOR_VERSION >= 3 "sequence argument expected, got %R", Py_TYPE(py_cpu_set) -#else - "sequence argument expected, got %s", Py_TYPE(py_cpu_set)->tp_name -#endif ); } @@ -177,11 +169,7 @@ psutil_proc_cpu_affinity_set(PyObject *self, PyObject *args) { if (!item) { return NULL; } -#if PY_MAJOR_VERSION >= 3 long value = PyLong_AsLong(item); -#else - long value = PyInt_AsLong(item); -#endif Py_XDECREF(item); if ((value == -1) || PyErr_Occurred()) { if (!PyErr_Occurred()) diff --git a/contrib/python/psutil/py3/psutil/arch/osx/disk.c b/contrib/python/psutil/py3/psutil/arch/osx/disk.c index d02cf794d5..e1a8f5a492 100644 --- a/contrib/python/psutil/py3/psutil/arch/osx/disk.c +++ b/contrib/python/psutil/py3/psutil/arch/osx/disk.c @@ -168,7 +168,6 @@ psutil_disk_usage_used(PyObject *self, PyObject *args) { PyObject *py_mount_point_bytes = NULL; char* mount_point; -#if PY_MAJOR_VERSION >= 3 if (!PyArg_ParseTuple(args, "O&O", PyUnicode_FSConverter, &py_mount_point_bytes, &py_default_value)) { return NULL; } @@ -177,11 +176,6 @@ psutil_disk_usage_used(PyObject *self, PyObject *args) { Py_XDECREF(py_mount_point_bytes); return NULL; } -#else - if (!PyArg_ParseTuple(args, "sO", &mount_point, &py_default_value)) { - return NULL; - } -#endif #ifdef ATTR_VOL_SPACEUSED /* Call getattrlist(ATTR_VOL_SPACEUSED) to get used space info. */ diff --git a/contrib/python/psutil/py3/psutil/arch/osx/mem.c b/contrib/python/psutil/py3/psutil/arch/osx/mem.c index 53493065c2..8103876070 100644 --- a/contrib/python/psutil/py3/psutil/arch/osx/mem.c +++ b/contrib/python/psutil/py3/psutil/arch/osx/mem.c @@ -8,6 +8,9 @@ // from psutil/_psutil_osx.c in 2023. This is the GIT blame before the move: // https://github.com/giampaolo/psutil/blame/efd7ed3/psutil/_psutil_osx.c +// See: +// https://github.com/apple-open-source/macos/blob/master/system_cmds/vm_stat/vm_stat.c + #include <Python.h> #include <mach/host_info.h> #include <sys/sysctl.h> @@ -17,12 +20,12 @@ static int -psutil_sys_vminfo(vm_statistics_data_t *vmstat) { +psutil_sys_vminfo(vm_statistics64_t vmstat) { kern_return_t ret; - mach_msg_type_number_t count = sizeof(*vmstat) / sizeof(integer_t); + unsigned int count = HOST_VM_INFO64_COUNT; mach_port_t mport = mach_host_self(); - ret = host_statistics(mport, HOST_VM_INFO, (host_info_t)vmstat, &count); + ret = host_statistics64(mport, HOST_VM_INFO64, (host_info64_t)vmstat, &count); if (ret != KERN_SUCCESS) { PyErr_Format( PyExc_RuntimeError, @@ -46,7 +49,7 @@ psutil_virtual_mem(PyObject *self, PyObject *args) { int mib[2]; uint64_t total; size_t len = sizeof(total); - vm_statistics_data_t vm; + vm_statistics64_data_t vm; long pagesize = psutil_getpagesize(); // physical mem mib[0] = CTL_HW; @@ -86,7 +89,7 @@ psutil_swap_mem(PyObject *self, PyObject *args) { int mib[2]; size_t size; struct xsw_usage totals; - vm_statistics_data_t vmstat; + vm_statistics64_data_t vmstat; long pagesize = psutil_getpagesize(); mib[0] = CTL_VM; diff --git a/contrib/python/psutil/py3/psutil/arch/osx/proc.c b/contrib/python/psutil/py3/psutil/arch/osx/proc.c index 136311ecee..681642c3ad 100644 --- a/contrib/python/psutil/py3/psutil/arch/osx/proc.c +++ b/contrib/python/psutil/py3/psutil/arch/osx/proc.c @@ -763,7 +763,7 @@ error: /* * Return process open files as a Python tuple. * References: - * - lsof source code: http://goo.gl/SYW79 and http://goo.gl/m78fd + * - lsof source code: https://github.com/apple-opensource/lsof/blob/28/lsof/dialects/darwin/libproc/dproc.c#L342 * - /usr/include/sys/proc_info.h */ PyObject * @@ -854,7 +854,7 @@ error: * Return process TCP and UDP connections as a list of tuples. * Raises NSP in case of zombie process. * References: - * - lsof source code: http://goo.gl/SYW79 and http://goo.gl/wNrC0 + * - lsof source code: https://github.com/apple-opensource/lsof/blob/28/lsof/dialects/darwin/libproc/dproc.c#L342 * - /usr/include/sys/proc_info.h */ PyObject * diff --git a/contrib/python/psutil/py3/psutil/arch/windows/disk.c b/contrib/python/psutil/py3/psutil/arch/windows/disk.c index 88469e3a61..d0405a98f8 100644 --- a/contrib/python/psutil/py3/psutil/arch/windows/disk.c +++ b/contrib/python/psutil/py3/psutil/arch/windows/disk.c @@ -45,33 +45,6 @@ PyObject * psutil_disk_usage(PyObject *self, PyObject *args) { BOOL retval; ULARGE_INTEGER _, total, free; - -#if PY_MAJOR_VERSION <= 2 - char *path; - - if (PyArg_ParseTuple(args, "u", &path)) { - Py_BEGIN_ALLOW_THREADS - retval = GetDiskFreeSpaceExW((LPCWSTR)path, &_, &total, &free); - Py_END_ALLOW_THREADS - goto return_; - } - - // on Python 2 we also want to accept plain strings other - // than Unicode - PyErr_Clear(); // drop the argument parsing error - if (PyArg_ParseTuple(args, "s", &path)) { - Py_BEGIN_ALLOW_THREADS - retval = GetDiskFreeSpaceEx(path, &_, &total, &free); - Py_END_ALLOW_THREADS - goto return_; - } - - return NULL; - -return_: - if (retval == 0) - return PyErr_SetFromWindowsErrWithFilename(0, path); -#else PyObject *py_path; wchar_t *path; @@ -92,7 +65,7 @@ return_: if (retval == 0) return PyErr_SetExcFromWindowsErrWithFilenameObject(PyExc_OSError, 0, py_path); -#endif + return Py_BuildValue("(LL)", total.QuadPart, free.QuadPart); } diff --git a/contrib/python/psutil/py3/psutil/arch/windows/net.c b/contrib/python/psutil/py3/psutil/arch/windows/net.c index 8d8f7d1c0a..9a5634b069 100644 --- a/contrib/python/psutil/py3/psutil/arch/windows/net.c +++ b/contrib/python/psutil/py3/psutil/arch/windows/net.c @@ -255,21 +255,14 @@ psutil_net_if_addrs(PyObject *self, PyObject *args) { continue; } -#if PY_MAJOR_VERSION >= 3 py_address = PyUnicode_FromString(buff_addr); -#else - py_address = PyString_FromString(buff_addr); -#endif if (py_address == NULL) goto error; if (netmaskIntRet != NULL) { -#if PY_MAJOR_VERSION >= 3 py_netmask = PyUnicode_FromString(buff_netmask); -#else - py_netmask = PyString_FromString(buff_netmask); -#endif - } else { + } + else { Py_INCREF(Py_None); py_netmask = Py_None; } diff --git a/contrib/python/psutil/py3/psutil/arch/windows/proc.c b/contrib/python/psutil/py3/psutil/arch/windows/proc.c index 05fb502557..41fa9dda68 100644 --- a/contrib/python/psutil/py3/psutil/arch/windows/proc.c +++ b/contrib/python/psutil/py3/psutil/arch/windows/proc.c @@ -192,11 +192,7 @@ psutil_proc_wait(PyObject *self, PyObject *args) { CloseHandle(hProcess); -#if PY_MAJOR_VERSION >= 3 return PyLong_FromLong((long) ExitCode); -#else - return PyInt_FromLong((long) ExitCode); -#endif } diff --git a/contrib/python/psutil/py3/ya.make b/contrib/python/psutil/py3/ya.make index c81c1dffb8..fd48170ac2 100644 --- a/contrib/python/psutil/py3/ya.make +++ b/contrib/python/psutil/py3/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(6.1.1) +VERSION(7.0.0) LICENSE(BSD-3-Clause) @@ -27,7 +27,7 @@ NO_CHECK_IMPORTS( NO_UTIL() CFLAGS( - -DPSUTIL_VERSION=611 + -DPSUTIL_VERSION=700 ) SRCS( @@ -125,7 +125,6 @@ PY_SRCS( TOP_LEVEL psutil/__init__.py psutil/_common.py - psutil/_compat.py psutil/_psaix.py psutil/_psbsd.py psutil/_pslinux.py diff --git a/contrib/python/simplejson/py2/.dist-info/METADATA b/contrib/python/simplejson/py2/.dist-info/METADATA index 84d46e5eca..9a1e6c550b 100644 --- a/contrib/python/simplejson/py2/.dist-info/METADATA +++ b/contrib/python/simplejson/py2/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: simplejson -Version: 3.19.3 +Version: 3.20.1 Summary: Simple, fast, extensible JSON encoder/decoder for Python Home-page: https://github.com/simplejson/simplejson Author: Bob Ippolito diff --git a/contrib/python/simplejson/py2/simplejson/__init__.py b/contrib/python/simplejson/py2/simplejson/__init__.py index d3ff141d00..8bb46f15ed 100644 --- a/contrib/python/simplejson/py2/simplejson/__init__.py +++ b/contrib/python/simplejson/py2/simplejson/__init__.py @@ -118,7 +118,7 @@ Serializing multiple objects to JSON lines (newline-delimited JSON):: """ from __future__ import absolute_import -__version__ = '3.19.3' +__version__ = '3.20.1' __all__ = [ 'dump', 'dumps', 'load', 'loads', 'JSONDecoder', 'JSONDecodeError', 'JSONEncoder', diff --git a/contrib/python/simplejson/py2/simplejson/_speedups.c b/contrib/python/simplejson/py2/simplejson/_speedups.c index bd56b4dc87..4133b9bc76 100644 --- a/contrib/python/simplejson/py2/simplejson/_speedups.c +++ b/contrib/python/simplejson/py2/simplejson/_speedups.c @@ -10,11 +10,13 @@ #define JSON_UNICHR Py_UCS4 #define JSON_InternFromString PyUnicode_InternFromString #define PyString_GET_SIZE PyUnicode_GET_LENGTH +#define JSON_StringCheck PyUnicode_Check #define PY2_UNUSED #define PY3_UNUSED UNUSED #else /* PY_MAJOR_VERSION >= 3 */ #define PY2_UNUSED UNUSED #define PY3_UNUSED +#define JSON_StringCheck(obj) (PyString_Check(obj) || PyUnicode_Check(obj)) #define PyBytes_Check PyString_Check #define PyUnicode_READY(obj) 0 #define PyUnicode_KIND(obj) (sizeof(Py_UNICODE)) @@ -3034,25 +3036,29 @@ encoder_listencode_dict(PyEncoderObject *s, JSON_Accu *rval, PyObject *dct, Py_s if (value == NULL) goto bail; - encoded = PyDict_GetItem(s->key_memo, key); - if (encoded != NULL) { - Py_INCREF(encoded); - } else { - kstr = encoder_stringify_key(s, key); - if (kstr == NULL) - goto bail; - else if (kstr == Py_None) { - /* skipkeys */ - Py_DECREF(item); - Py_DECREF(kstr); - continue; - } + kstr = encoder_stringify_key(s, key); + if (kstr == NULL) + goto bail; + else if (kstr == Py_None) { + /* skipkeys */ + Py_DECREF(item); + Py_DECREF(kstr); + continue; } if (idx) { if (JSON_Accu_Accumulate(rval, s->item_separator)) goto bail; } - if (encoded == NULL) { + /* + * Only cache the encoding of string keys. False and True are + * indistinguishable from 0 and 1 in a dictionary lookup and there + * may be other quirks with user defined subclasses. + */ + encoded = PyDict_GetItem(s->key_memo, kstr); + if (encoded != NULL) { + Py_INCREF(encoded); + Py_CLEAR(kstr); + } else { encoded = encoder_encode_string(s, kstr); Py_CLEAR(kstr); if (encoded == NULL) diff --git a/contrib/python/simplejson/py2/simplejson/tests/test_dump.py b/contrib/python/simplejson/py2/simplejson/tests/test_dump.py index eff24c299c..5628489f26 100644 --- a/contrib/python/simplejson/py2/simplejson/tests/test_dump.py +++ b/contrib/python/simplejson/py2/simplejson/tests/test_dump.py @@ -73,6 +73,12 @@ class TestDump(TestCase): {True: False, False: True}, sort_keys=True), '{"false": true, "true": false}') self.assertEqual( + # load first because the keys are not sorted + json.loads(json.dumps({'k1': {False: 5}, 'k2': {0: 5}})), + {'k1': {'false': 5}, 'k2': {'0': 5}}, + ) + + self.assertEqual( json.dumps( {2: 3.0, 4.0: long_type(5), diff --git a/contrib/python/simplejson/py2/ya.make b/contrib/python/simplejson/py2/ya.make index 7397dd761d..f565e0ee79 100644 --- a/contrib/python/simplejson/py2/ya.make +++ b/contrib/python/simplejson/py2/ya.make @@ -2,7 +2,7 @@ PY2_LIBRARY() -VERSION(3.19.3) +VERSION(3.20.1) LICENSE(MIT) diff --git a/contrib/python/simplejson/py3/.dist-info/METADATA b/contrib/python/simplejson/py3/.dist-info/METADATA index 689e566f7f..bee5586b12 100644 --- a/contrib/python/simplejson/py3/.dist-info/METADATA +++ b/contrib/python/simplejson/py3/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: simplejson -Version: 3.19.3 +Version: 3.20.1 Summary: Simple, fast, extensible JSON encoder/decoder for Python Home-page: https://github.com/simplejson/simplejson Author: Bob Ippolito diff --git a/contrib/python/simplejson/py3/simplejson/__init__.py b/contrib/python/simplejson/py3/simplejson/__init__.py index d3ff141d00..8bb46f15ed 100644 --- a/contrib/python/simplejson/py3/simplejson/__init__.py +++ b/contrib/python/simplejson/py3/simplejson/__init__.py @@ -118,7 +118,7 @@ Serializing multiple objects to JSON lines (newline-delimited JSON):: """ from __future__ import absolute_import -__version__ = '3.19.3' +__version__ = '3.20.1' __all__ = [ 'dump', 'dumps', 'load', 'loads', 'JSONDecoder', 'JSONDecodeError', 'JSONEncoder', diff --git a/contrib/python/simplejson/py3/simplejson/_speedups.c b/contrib/python/simplejson/py3/simplejson/_speedups.c index bd56b4dc87..4133b9bc76 100644 --- a/contrib/python/simplejson/py3/simplejson/_speedups.c +++ b/contrib/python/simplejson/py3/simplejson/_speedups.c @@ -10,11 +10,13 @@ #define JSON_UNICHR Py_UCS4 #define JSON_InternFromString PyUnicode_InternFromString #define PyString_GET_SIZE PyUnicode_GET_LENGTH +#define JSON_StringCheck PyUnicode_Check #define PY2_UNUSED #define PY3_UNUSED UNUSED #else /* PY_MAJOR_VERSION >= 3 */ #define PY2_UNUSED UNUSED #define PY3_UNUSED +#define JSON_StringCheck(obj) (PyString_Check(obj) || PyUnicode_Check(obj)) #define PyBytes_Check PyString_Check #define PyUnicode_READY(obj) 0 #define PyUnicode_KIND(obj) (sizeof(Py_UNICODE)) @@ -3034,25 +3036,29 @@ encoder_listencode_dict(PyEncoderObject *s, JSON_Accu *rval, PyObject *dct, Py_s if (value == NULL) goto bail; - encoded = PyDict_GetItem(s->key_memo, key); - if (encoded != NULL) { - Py_INCREF(encoded); - } else { - kstr = encoder_stringify_key(s, key); - if (kstr == NULL) - goto bail; - else if (kstr == Py_None) { - /* skipkeys */ - Py_DECREF(item); - Py_DECREF(kstr); - continue; - } + kstr = encoder_stringify_key(s, key); + if (kstr == NULL) + goto bail; + else if (kstr == Py_None) { + /* skipkeys */ + Py_DECREF(item); + Py_DECREF(kstr); + continue; } if (idx) { if (JSON_Accu_Accumulate(rval, s->item_separator)) goto bail; } - if (encoded == NULL) { + /* + * Only cache the encoding of string keys. False and True are + * indistinguishable from 0 and 1 in a dictionary lookup and there + * may be other quirks with user defined subclasses. + */ + encoded = PyDict_GetItem(s->key_memo, kstr); + if (encoded != NULL) { + Py_INCREF(encoded); + Py_CLEAR(kstr); + } else { encoded = encoder_encode_string(s, kstr); Py_CLEAR(kstr); if (encoded == NULL) diff --git a/contrib/python/simplejson/py3/simplejson/tests/test_dump.py b/contrib/python/simplejson/py3/simplejson/tests/test_dump.py index eff24c299c..5628489f26 100644 --- a/contrib/python/simplejson/py3/simplejson/tests/test_dump.py +++ b/contrib/python/simplejson/py3/simplejson/tests/test_dump.py @@ -73,6 +73,12 @@ class TestDump(TestCase): {True: False, False: True}, sort_keys=True), '{"false": true, "true": false}') self.assertEqual( + # load first because the keys are not sorted + json.loads(json.dumps({'k1': {False: 5}, 'k2': {0: 5}})), + {'k1': {'false': 5}, 'k2': {'0': 5}}, + ) + + self.assertEqual( json.dumps( {2: 3.0, 4.0: long_type(5), diff --git a/contrib/python/simplejson/py3/ya.make b/contrib/python/simplejson/py3/ya.make index 7537b289e7..1d192847a5 100644 --- a/contrib/python/simplejson/py3/ya.make +++ b/contrib/python/simplejson/py3/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(3.19.3) +VERSION(3.20.1) LICENSE(MIT) diff --git a/library/cpp/http/simple/http_client.cpp b/library/cpp/http/simple/http_client.cpp index 2be5a14582..6fb0475539 100644 --- a/library/cpp/http/simple/http_client.cpp +++ b/library/cpp/http/simple/http_client.cpp @@ -195,28 +195,28 @@ void TSimpleHttpClient::EnableVerificationForHttps() { HttpsVerification = true; } -void TSimpleHttpClient::DoGet(const TStringBuf relativeUrl, IOutputStream* output, const THeaders& headers, NThreading::TCancellationToken cancellation) const { +void TSimpleHttpClient::DoGet(const TStringBuf relativeUrl, IOutputStream* output, const THeaders& headers, THttpHeaders* outHeaders, NThreading::TCancellationToken cancellation) const { TKeepAliveHttpClient cl = CreateClient(); - TKeepAliveHttpClient::THttpCode code = cl.DoGet(relativeUrl, output, headers, nullptr, std::move(cancellation)); + TKeepAliveHttpClient::THttpCode code = cl.DoGet(relativeUrl, output, headers, outHeaders, std::move(cancellation)); Y_ENSURE(cl.GetHttpInput()); ProcessResponse(relativeUrl, *cl.GetHttpInput(), output, code); } -void TSimpleHttpClient::DoPost(const TStringBuf relativeUrl, TStringBuf body, IOutputStream* output, const THashMap<TString, TString>& headers, NThreading::TCancellationToken cancellation) const { +void TSimpleHttpClient::DoPost(const TStringBuf relativeUrl, TStringBuf body, IOutputStream* output, const THashMap<TString, TString>& headers, THttpHeaders* outHeaders, NThreading::TCancellationToken cancellation) const { TKeepAliveHttpClient cl = CreateClient(); - TKeepAliveHttpClient::THttpCode code = cl.DoPost(relativeUrl, body, output, headers, nullptr, std::move(cancellation)); + TKeepAliveHttpClient::THttpCode code = cl.DoPost(relativeUrl, body, output, headers, outHeaders, std::move(cancellation)); Y_ENSURE(cl.GetHttpInput()); ProcessResponse(relativeUrl, *cl.GetHttpInput(), output, code); } -void TSimpleHttpClient::DoPostRaw(const TStringBuf relativeUrl, const TStringBuf rawRequest, IOutputStream* output, NThreading::TCancellationToken cancellation) const { +void TSimpleHttpClient::DoPostRaw(const TStringBuf relativeUrl, const TStringBuf rawRequest, IOutputStream* output, THttpHeaders* outHeaders, NThreading::TCancellationToken cancellation) const { TKeepAliveHttpClient cl = CreateClient(); - TKeepAliveHttpClient::THttpCode code = cl.DoRequestRaw(rawRequest, output, nullptr, std::move(cancellation)); + TKeepAliveHttpClient::THttpCode code = cl.DoRequestRaw(rawRequest, output, outHeaders, std::move(cancellation)); Y_ENSURE(cl.GetHttpInput()); ProcessResponse(relativeUrl, *cl.GetHttpInput(), output, code); diff --git a/library/cpp/http/simple/http_client.h b/library/cpp/http/simple/http_client.h index 86a8cb4e99..3860862698 100644 --- a/library/cpp/http/simple/http_client.h +++ b/library/cpp/http/simple/http_client.h @@ -172,13 +172,13 @@ public: void EnableVerificationForHttps(); - void DoGet(const TStringBuf relativeUrl, IOutputStream* output, const THeaders& headers = THeaders(), NThreading::TCancellationToken cancellation = NThreading::TCancellationToken::Default()) const; + void DoGet(const TStringBuf relativeUrl, IOutputStream* output, const THeaders& headers = THeaders(), THttpHeaders* outHeaders = nullptr, NThreading::TCancellationToken cancellation = NThreading::TCancellationToken::Default()) const; // builds post request from headers and body - void DoPost(const TStringBuf relativeUrl, TStringBuf body, IOutputStream* output, const THeaders& headers = THeaders(), NThreading::TCancellationToken cancellation = NThreading::TCancellationToken::Default()) const; + void DoPost(const TStringBuf relativeUrl, TStringBuf body, IOutputStream* output, const THeaders& headers = THeaders(), THttpHeaders* outHeaders = nullptr, NThreading::TCancellationToken cancellation = NThreading::TCancellationToken::Default()) const; // requires already well-formed post request - void DoPostRaw(const TStringBuf relativeUrl, TStringBuf rawRequest, IOutputStream* output, NThreading::TCancellationToken cancellation = NThreading::TCancellationToken::Default()) const; + void DoPostRaw(const TStringBuf relativeUrl, TStringBuf rawRequest, IOutputStream* output, THttpHeaders* outHeaders = nullptr, NThreading::TCancellationToken cancellation = NThreading::TCancellationToken::Default()) const; virtual ~TSimpleHttpClient(); diff --git a/library/cpp/monlib/encode/buffered/buffered_encoder_base.cpp b/library/cpp/monlib/encode/buffered/buffered_encoder_base.cpp index 87c832d642..c0449a10ff 100644 --- a/library/cpp/monlib/encode/buffered/buffered_encoder_base.cpp +++ b/library/cpp/monlib/encode/buffered/buffered_encoder_base.cpp @@ -144,6 +144,12 @@ void TBufferedEncoderBase::OnLogHistogram(TInstant time, TLogHistogramSnapshotPt metric.TimeSeries.Add(time, s.Get()); } +void TBufferedEncoderBase::OnMemOnly(bool isMemOnly) { + State_.Expect(TEncoderState::EState::METRIC); + TMetric& metric = Metrics_.back(); + metric.IsMemOnly = isMemOnly; +} + TString TBufferedEncoderBase::FormatLabels(const TPooledLabels& labels) const { auto formattedLabels = TVector<TString>(Reserve(labels.size() + CommonLabels_.size())); auto addLabel = [&](const TPooledLabel& l) { diff --git a/library/cpp/monlib/encode/buffered/buffered_encoder_base.h b/library/cpp/monlib/encode/buffered/buffered_encoder_base.h index fe3714e58f..dab5671ad4 100644 --- a/library/cpp/monlib/encode/buffered/buffered_encoder_base.h +++ b/library/cpp/monlib/encode/buffered/buffered_encoder_base.h @@ -37,6 +37,8 @@ public: void OnSummaryDouble(TInstant time, ISummaryDoubleSnapshotPtr snapshot) override; void OnLogHistogram(TInstant, TLogHistogramSnapshotPtr) override; + void OnMemOnly(bool isMemOnly) override; + protected: using TPooledStr = TStringPoolBuilder::TValue; @@ -80,6 +82,7 @@ protected: EMetricType MetricType = EMetricType::UNKNOWN; TPooledLabels Labels; TMetricTimeSeries TimeSeries; + bool IsMemOnly; }; protected: diff --git a/library/cpp/monlib/encode/fake/fake.cpp b/library/cpp/monlib/encode/fake/fake.cpp index 69d691361a..e3db20cbd1 100644 --- a/library/cpp/monlib/encode/fake/fake.cpp +++ b/library/cpp/monlib/encode/fake/fake.cpp @@ -43,6 +43,9 @@ namespace NMonitoring { void Close() override { } + + void OnMemOnly(bool) override { + } }; IMetricEncoderPtr EncoderFake() { diff --git a/library/cpp/monlib/encode/protobuf/protobuf_encoder.cpp b/library/cpp/monlib/encode/protobuf/protobuf_encoder.cpp index 2d11b9d5ba..2cffda13ac 100644 --- a/library/cpp/monlib/encode/protobuf/protobuf_encoder.cpp +++ b/library/cpp/monlib/encode/protobuf/protobuf_encoder.cpp @@ -227,6 +227,11 @@ namespace NMonitoring { FillLogHistogram(*snapshot, point->MutableLogHistogram()); } + void OnMemOnly(bool isMemOnly) override { + Y_ENSURE(Sample_, "metric not started"); + Sample_->SetIsMemOnly(isMemOnly); + } + void Close() override { } diff --git a/library/cpp/monlib/encode/protobuf/protos/samples.proto b/library/cpp/monlib/encode/protobuf/protos/samples.proto index 371f4181d2..a3ca917221 100644 --- a/library/cpp/monlib/encode/protobuf/protos/samples.proto +++ b/library/cpp/monlib/encode/protobuf/protos/samples.proto @@ -59,7 +59,7 @@ message TPoint { message TSingleSample { repeated TLabel Labels = 1; EMetricType MetricType = 2; - + bool IsMemOnly = 10; // inlined TPoint uint64 Time = 3; oneof Value { @@ -76,6 +76,7 @@ message TMultiSample { repeated TLabel Labels = 1; EMetricType MetricType = 2; repeated TPoint Points = 3; + bool IsMemOnly = 4; } message TSingleSamplesList { diff --git a/library/cpp/monlib/encode/spack/spack_v1_decoder.cpp b/library/cpp/monlib/encode/spack/spack_v1_decoder.cpp index 384ef456dd..a236bb9a50 100644 --- a/library/cpp/monlib/encode/spack/spack_v1_decoder.cpp +++ b/library/cpp/monlib/encode/spack/spack_v1_decoder.cpp @@ -107,15 +107,15 @@ namespace NMonitoring { c->OnMetricBegin(metricType); - // TODO: use it - ReadFixed<ui8>(); // skip flags byte + // (5.2) flags byte + c->OnMemOnly(ReadFixed<ui8>() & 0x01); auto metricNameValueIndex = std::numeric_limits<ui32>::max(); if (Header_.Version >= SV1_02) { metricNameValueIndex = ReadVarint(); } - // (5.2) labels + // (5.3) labels ui32 labelsCount = ReadVarint(); DECODE_ENSURE(Header_.Version >= SV1_02 || labelsCount > 0, "metric #" << i << " has no labels"); c->OnLabelsBegin(); @@ -125,7 +125,7 @@ namespace NMonitoring { ReadLabels(labelNames, labelValues, labelsCount, c); c->OnLabelsEnd(); - // (5.3) values + // (5.4) values switch (valueType) { case EValueType::NONE: break; diff --git a/library/cpp/monlib/encode/spack/spack_v1_encoder.cpp b/library/cpp/monlib/encode/spack/spack_v1_encoder.cpp index 7e13c3292b..70c5bba551 100644 --- a/library/cpp/monlib/encode/spack/spack_v1_encoder.cpp +++ b/library/cpp/monlib/encode/spack/spack_v1_encoder.cpp @@ -68,6 +68,10 @@ namespace NMonitoring { TBufferedEncoderBase::OnLogHistogram(time, snapshot); } + void OnMemOnly(bool isMemOnly) override { + TBufferedEncoderBase::OnMemOnly(isMemOnly); + } + void Close() override { if (Closed_) { return; @@ -128,8 +132,8 @@ namespace NMonitoring { ui8 typesByte = PackTypes(metric); Out_->Write(&typesByte, sizeof(typesByte)); - // TODO: implement - ui8 flagsByte = 0x00; + // (5.2) flags byte + ui8 flagsByte = metric.IsMemOnly & 0x01; Out_->Write(&flagsByte, sizeof(flagsByte)); // v1.2 format addition — metric name @@ -143,10 +147,10 @@ namespace NMonitoring { WriteVarUInt32(Out_, it->Value->Index); } - // (5.2) labels + // (5.3) labels WriteLabels(metric.Labels, MetricName_); - // (5.3) values + // (5.4) values switch (metric.TimeSeries.Size()) { case 0: break; diff --git a/library/cpp/monlib/encode/spack/spack_v1_ut.cpp b/library/cpp/monlib/encode/spack/spack_v1_ut.cpp index fe778eb7e0..bbc3b8712b 100644 --- a/library/cpp/monlib/encode/spack/spack_v1_ut.cpp +++ b/library/cpp/monlib/encode/spack/spack_v1_ut.cpp @@ -104,7 +104,7 @@ Y_UNIT_TEST_SUITE(TSpackTest) { ui8 expectedMetric1[] = { 0x0C, // types (RATE | NONE) (fixed ui8) - 0x00, // flags (fixed ui8) + 0x01, // flags (fixed ui8) 0x01, // metric labels count (varint) 0x00, // label name index (varint) 0x01, // label value index (varint) @@ -282,6 +282,7 @@ Y_UNIT_TEST_SUITE(TSpackTest) { e->OnLabelsBegin(); e->OnLabel("name", "q1"); e->OnLabelsEnd(); + e->OnMemOnly(true); } e->OnMetricEnd(); } @@ -513,12 +514,14 @@ Y_UNIT_TEST_SUITE(TSpackTest) { { const NProto::TMultiSample& s = samples.GetSamples(0); UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::RATE); + UNIT_ASSERT_EQUAL(s.GetIsMemOnly(), true); UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); AssertLabelEqual(s.GetLabels(0), "name", "q1"); } { const NProto::TMultiSample& s = samples.GetSamples(1); UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::COUNTER); + UNIT_ASSERT_EQUAL(s.GetIsMemOnly(), false); UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); AssertLabelEqual(s.GetLabels(0), "name", "q2"); @@ -528,6 +531,7 @@ Y_UNIT_TEST_SUITE(TSpackTest) { { const NProto::TMultiSample& s = samples.GetSamples(2); UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::COUNTER); + UNIT_ASSERT_EQUAL(s.GetIsMemOnly(), false); UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); AssertLabelEqual(s.GetLabels(0), "name", "q3"); @@ -537,6 +541,7 @@ Y_UNIT_TEST_SUITE(TSpackTest) { { const NProto::TMultiSample& s = samples.GetSamples(3); UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE); + UNIT_ASSERT_EQUAL(s.GetIsMemOnly(), false); UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); AssertLabelEqual(s.GetLabels(0), "name", "answer"); @@ -547,6 +552,7 @@ Y_UNIT_TEST_SUITE(TSpackTest) { { const NProto::TMultiSample& s = samples.GetSamples(4); UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::HISTOGRAM); + UNIT_ASSERT_EQUAL(s.GetIsMemOnly(), false); UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); AssertLabelEqual(s.GetLabels(0), "name", "responseTimeMillis"); @@ -570,6 +576,7 @@ Y_UNIT_TEST_SUITE(TSpackTest) { { const NProto::TMultiSample& s = samples.GetSamples(5); UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::IGAUGE); + UNIT_ASSERT_EQUAL(s.GetIsMemOnly(), false); UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); AssertLabelEqual(s.GetLabels(0), "name", "bytes"); @@ -579,6 +586,7 @@ Y_UNIT_TEST_SUITE(TSpackTest) { { const NProto::TMultiSample& s = samples.GetSamples(6); UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::DSUMMARY); + UNIT_ASSERT_EQUAL(s.GetIsMemOnly(), false); UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); AssertLabelEqual(s.GetLabels(0), "name", "temperature"); @@ -601,6 +609,7 @@ Y_UNIT_TEST_SUITE(TSpackTest) { { const NProto::TMultiSample& s = samples.GetSamples(7); UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::LOGHISTOGRAM); + UNIT_ASSERT_EQUAL(s.GetIsMemOnly(), false); UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); AssertLabelEqual(s.GetLabels(0), "name", "ms"); diff --git a/library/cpp/monlib/metrics/metric_consumer.h b/library/cpp/monlib/metrics/metric_consumer.h index f7a727585a..1152b51c4a 100644 --- a/library/cpp/monlib/metrics/metric_consumer.h +++ b/library/cpp/monlib/metrics/metric_consumer.h @@ -33,6 +33,10 @@ namespace NMonitoring { virtual void OnHistogram(TInstant time, IHistogramSnapshotPtr snapshot) = 0; virtual void OnLogHistogram(TInstant time, TLogHistogramSnapshotPtr snapshot) = 0; virtual void OnSummaryDouble(TInstant time, ISummaryDoubleSnapshotPtr snapshot) = 0; + + virtual void OnMemOnly(bool isMemOnly) { + Y_UNUSED(isMemOnly); + } }; using IMetricConsumerPtr = THolder<IMetricConsumer>; diff --git a/library/cpp/monlib/metrics/metric_registry.cpp b/library/cpp/monlib/metrics/metric_registry.cpp index c747ae2b74..245f65702d 100644 --- a/library/cpp/monlib/metrics/metric_registry.cpp +++ b/library/cpp/monlib/metrics/metric_registry.cpp @@ -11,16 +11,17 @@ namespace NMonitoring { } template <typename TLabelsConsumer> - void ConsumeMetric(TInstant time, IMetricConsumer* consumer, IMetric* metric, TLabelsConsumer&& labelsConsumer) { + void ConsumeMetric(TInstant time, IMetricConsumer* consumer, IMetric* metric, TLabelsConsumer&& labelsConsumer, TMetricOpts opts = {}) { consumer->OnMetricBegin(metric->Type()); // (1) add labels consumer->OnLabelsBegin(); labelsConsumer(); consumer->OnLabelsEnd(); - // (2) add time and value metric->Accept(time, consumer); + // (3) add flag + consumer->OnMemOnly(opts.MemOnly); consumer->OnMetricEnd(); } } @@ -55,104 +56,177 @@ namespace NMonitoring { } TGauge* TMetricRegistry::Gauge(TLabels labels) { - return Metric<TGauge, EMetricType::GAUGE>(std::move(labels)); + return GaugeWithOpts(std::move(labels)); + } + TGauge* TMetricRegistry::GaugeWithOpts(TLabels labels, TMetricOpts opts) { + return Metric<TGauge, EMetricType::GAUGE>(std::move(labels), std::move(opts)); } TGauge* TMetricRegistry::Gauge(ILabelsPtr labels) { - return Metric<TGauge, EMetricType::GAUGE>(std::move(labels)); + return GaugeWithOpts(std::move(labels)); + } + TGauge* TMetricRegistry::GaugeWithOpts(ILabelsPtr labels, TMetricOpts opts) { + return Metric<TGauge, EMetricType::GAUGE>(std::move(labels), std::move(opts)); } TLazyGauge* TMetricRegistry::LazyGauge(TLabels labels, std::function<double()> supplier) { - return Metric<TLazyGauge, EMetricType::GAUGE>(std::move(labels), std::move(supplier)); + return LazyGaugeWithOpts(std::move(labels), std::move(supplier)); + } + TLazyGauge* TMetricRegistry::LazyGaugeWithOpts(TLabels labels, std::function<double()> supplier, TMetricOpts opts) { + return Metric<TLazyGauge, EMetricType::GAUGE>(std::move(labels),std::move(opts), std::move(supplier)); } TLazyGauge* TMetricRegistry::LazyGauge(ILabelsPtr labels, std::function<double()> supplier) { - return Metric<TLazyGauge, EMetricType::GAUGE>(std::move(labels), std::move(supplier)); + return LazyGaugeWithOpts(std::move(labels), std::move(supplier)); + } + TLazyGauge* TMetricRegistry::LazyGaugeWithOpts(ILabelsPtr labels, std::function<double()> supplier, TMetricOpts opts) { + return Metric<TLazyGauge, EMetricType::GAUGE>(std::move(labels), std::move(opts), std::move(supplier)); } TIntGauge* TMetricRegistry::IntGauge(TLabels labels) { - return Metric<TIntGauge, EMetricType::IGAUGE>(std::move(labels)); + return IntGaugeWithOpts(std::move(labels)); + } + TIntGauge* TMetricRegistry::IntGaugeWithOpts(TLabels labels, TMetricOpts opts) { + return Metric<TIntGauge, EMetricType::IGAUGE>(std::move(labels), std::move(opts)); } TIntGauge* TMetricRegistry::IntGauge(ILabelsPtr labels) { - return Metric<TIntGauge, EMetricType::IGAUGE>(std::move(labels)); + return IntGaugeWithOpts(std::move(labels)); + } + TIntGauge* TMetricRegistry::IntGaugeWithOpts(ILabelsPtr labels, TMetricOpts opts) { + return Metric<TIntGauge, EMetricType::IGAUGE>(std::move(labels), std::move(opts)); } TLazyIntGauge* TMetricRegistry::LazyIntGauge(TLabels labels, std::function<i64()> supplier) { - return Metric<TLazyIntGauge, EMetricType::IGAUGE>(std::move(labels), std::move(supplier)); + return LazyIntGaugeWithOpts(std::move(labels), std::move(supplier)); + } + TLazyIntGauge* TMetricRegistry::LazyIntGaugeWithOpts(TLabels labels, std::function<i64()> supplier, TMetricOpts opts) { + return Metric<TLazyIntGauge, EMetricType::IGAUGE>(std::move(labels), std::move(opts), std::move(supplier)); } TLazyIntGauge* TMetricRegistry::LazyIntGauge(ILabelsPtr labels, std::function<i64()> supplier) { - return Metric<TLazyIntGauge, EMetricType::IGAUGE>(std::move(labels), std::move(supplier)); + return LazyIntGaugeWithOpts(std::move(labels), std::move(supplier)); + } + TLazyIntGauge* TMetricRegistry::LazyIntGaugeWithOpts(ILabelsPtr labels, std::function<i64()> supplier, TMetricOpts opts) { + return Metric<TLazyIntGauge, EMetricType::IGAUGE>(std::move(labels), std::move(opts), std::move(supplier)); } TCounter* TMetricRegistry::Counter(TLabels labels) { - return Metric<TCounter, EMetricType::COUNTER>(std::move(labels)); + return CounterWithOpts(std::move(labels)); + } + TCounter* TMetricRegistry::CounterWithOpts(TLabels labels, TMetricOpts opts) { + return Metric<TCounter, EMetricType::COUNTER>(std::move(labels), std::move(opts)); } TCounter* TMetricRegistry::Counter(ILabelsPtr labels) { - return Metric<TCounter, EMetricType::COUNTER>(std::move(labels)); + return CounterWithOpts(std::move(labels)); + } + TCounter* TMetricRegistry::CounterWithOpts(ILabelsPtr labels, TMetricOpts opts) { + return Metric<TCounter, EMetricType::COUNTER>(std::move(labels), std::move(opts)); } TLazyCounter* TMetricRegistry::LazyCounter(TLabels labels, std::function<ui64()> supplier) { - return Metric<TLazyCounter, EMetricType::COUNTER>(std::move(labels), std::move(supplier)); + return LazyCounterWithOpts(std::move(labels), std::move(supplier)); + } + TLazyCounter* TMetricRegistry::LazyCounterWithOpts(TLabels labels, std::function<ui64()> supplier, TMetricOpts opts) { + return Metric<TLazyCounter, EMetricType::COUNTER>(std::move(labels), std::move(opts), std::move(supplier)); } TLazyCounter* TMetricRegistry::LazyCounter(ILabelsPtr labels, std::function<ui64()> supplier) { - return Metric<TLazyCounter, EMetricType::COUNTER>(std::move(labels), std::move(supplier)); + return LazyCounterWithOpts(std::move(labels), std::move(supplier)); + } + TLazyCounter* TMetricRegistry::LazyCounterWithOpts(ILabelsPtr labels, std::function<ui64()> supplier, TMetricOpts opts) { + return Metric<TLazyCounter, EMetricType::COUNTER>(std::move(labels), std::move(opts), std::move(supplier)); } TRate* TMetricRegistry::Rate(TLabels labels) { - return Metric<TRate, EMetricType::RATE>(std::move(labels)); + return RateWithOpts(std::move(labels)); + } + TRate* TMetricRegistry::RateWithOpts(TLabels labels, TMetricOpts opts) { + return Metric<TRate, EMetricType::RATE>(std::move(labels), std::move(opts)); } TRate* TMetricRegistry::Rate(ILabelsPtr labels) { - return Metric<TRate, EMetricType::RATE>(std::move(labels)); + return RateWithOpts(std::move(labels)); + } + TRate* TMetricRegistry::RateWithOpts(ILabelsPtr labels, TMetricOpts opts) { + return Metric<TRate, EMetricType::RATE>(std::move(labels), std::move(opts)); } TLazyRate* TMetricRegistry::LazyRate(TLabels labels, std::function<ui64()> supplier) { - return Metric<TLazyRate, EMetricType::RATE>(std::move(labels), std::move(supplier)); + return LazyRateWithOpts(std::move(labels), std::move(supplier)); + } + TLazyRate* TMetricRegistry::LazyRateWithOpts(TLabels labels, std::function<ui64()> supplier, TMetricOpts opts) { + return Metric<TLazyRate, EMetricType::RATE>(std::move(labels), std::move(opts), std::move(supplier)); } TLazyRate* TMetricRegistry::LazyRate(ILabelsPtr labels, std::function<ui64()> supplier) { - return Metric<TLazyRate, EMetricType::RATE>(std::move(labels), std::move(supplier)); + return LazyRateWithOpts(std::move(labels), std::move(supplier)); + } + TLazyRate* TMetricRegistry::LazyRateWithOpts(ILabelsPtr labels, std::function<ui64()> supplier, TMetricOpts opts) { + return Metric<TLazyRate, EMetricType::RATE>(std::move(labels), std::move(opts), std::move(supplier)); } THistogram* TMetricRegistry::HistogramCounter(TLabels labels, IHistogramCollectorPtr collector) { - return Metric<THistogram, EMetricType::HIST>(std::move(labels), std::move(collector), false); + return HistogramCounterWithOpts(std::move(labels), std::move(collector)); + } + THistogram* TMetricRegistry::HistogramCounterWithOpts(TLabels labels, IHistogramCollectorPtr collector, TMetricOpts opts) { + return Metric<THistogram, EMetricType::HIST>(std::move(labels), std::move(opts), std::move(collector), false); } THistogram* TMetricRegistry::HistogramCounter(ILabelsPtr labels, IHistogramCollectorPtr collector) { - return Metric<THistogram, EMetricType::HIST>(std::move(labels), std::move(collector), false); + return HistogramCounterWithOpts(std::move(labels), std::move(collector)); + } + THistogram* TMetricRegistry::HistogramCounterWithOpts(ILabelsPtr labels, IHistogramCollectorPtr collector, TMetricOpts opts) { + return Metric<THistogram, EMetricType::HIST>(std::move(labels), std::move(opts), std::move(collector), false); } THistogram* TMetricRegistry::HistogramCounter(TLabels labels, std::function<IHistogramCollectorPtr()> supplier) { - return Metric<THistogram, EMetricType::HIST>(std::move(labels), std::move(supplier), false); + return HistogramCounterWithOpts(std::move(labels), std::move(supplier)); + } + THistogram* TMetricRegistry::HistogramCounterWithOpts(TLabels labels, std::function<IHistogramCollectorPtr()> supplier, TMetricOpts opts) { + return Metric<THistogram, EMetricType::HIST>(std::move(labels), std::move(opts), std::move(supplier), false); } THistogram* TMetricRegistry::HistogramCounter(ILabelsPtr labels, std::function<IHistogramCollectorPtr()> supplier) { - return Metric<THistogram, EMetricType::HIST>(std::move(labels), std::move(supplier), false); + return HistogramCounterWithOpts(std::move(labels), std::move(supplier)); + } + THistogram* TMetricRegistry::HistogramCounterWithOpts(ILabelsPtr labels, std::function<IHistogramCollectorPtr()> supplier, TMetricOpts opts) { + return Metric<THistogram, EMetricType::HIST>(std::move(labels), std::move(opts), std::move(supplier), false); } THistogram* TMetricRegistry::HistogramRate(TLabels labels, IHistogramCollectorPtr collector) { - return Metric<THistogram, EMetricType::HIST_RATE>(std::move(labels), std::move(collector), true); + return HistogramRateWithOpts(std::move(labels), std::move(collector)); + } + THistogram* TMetricRegistry::HistogramRateWithOpts(TLabels labels, IHistogramCollectorPtr collector, TMetricOpts opts) { + return Metric<THistogram, EMetricType::HIST_RATE>(std::move(labels), std::move(opts), std::move(collector), true); } THistogram* TMetricRegistry::HistogramRate(ILabelsPtr labels, IHistogramCollectorPtr collector) { - return Metric<THistogram, EMetricType::HIST_RATE>(std::move(labels), std::move(collector), true); + return HistogramRateWithOpts(std::move(labels), std::move(collector)); + } + THistogram* TMetricRegistry::HistogramRateWithOpts(ILabelsPtr labels, IHistogramCollectorPtr collector, TMetricOpts opts) { + return Metric<THistogram, EMetricType::HIST_RATE>(std::move(labels), std::move(opts), std::move(collector), true); } THistogram* TMetricRegistry::HistogramRate(TLabels labels, std::function<IHistogramCollectorPtr()> supplier) { - return Metric<THistogram, EMetricType::HIST_RATE>(std::move(labels), std::move(supplier), true); + return HistogramRateWithOpts(std::move(labels), std::move(supplier)); + } + THistogram* TMetricRegistry::HistogramRateWithOpts(TLabels labels, std::function<IHistogramCollectorPtr()> supplier, TMetricOpts opts) { + return Metric<THistogram, EMetricType::HIST_RATE>(std::move(labels), std::move(opts), std::move(supplier), true); } THistogram* TMetricRegistry::HistogramRate(ILabelsPtr labels, std::function<IHistogramCollectorPtr()> supplier) { - return Metric<THistogram, EMetricType::HIST_RATE>(std::move(labels), std::move(supplier), true); + return HistogramRateWithOpts(std::move(labels), std::move(supplier)); + } + THistogram* TMetricRegistry::HistogramRateWithOpts(ILabelsPtr labels, std::function<IHistogramCollectorPtr()> supplier, TMetricOpts opts) { + return Metric<THistogram, EMetricType::HIST_RATE>(std::move(labels), std::move(opts), std::move(supplier), true); } void TMetricRegistry::Reset() { TWriteGuard g{*Lock_}; - for (auto& [label, metric] : Metrics_) { + for (auto& [label, metricValue] : Metrics_) { + auto metric = metricValue.Metric; switch (metric->Type()) { case EMetricType::GAUGE: static_cast<TGauge*>(metric.Get())->Set(.0); @@ -184,16 +258,19 @@ namespace NMonitoring { } template <typename TMetric, EMetricType type, typename TLabelsType, typename... Args> - TMetric* TMetricRegistry::Metric(TLabelsType&& labels, Args&&... args) { + TMetric* TMetricRegistry::Metric(TLabelsType&& labels, TMetricOpts&& opts, Args&&... args) { { TReadGuard g{*Lock_}; auto it = Metrics_.find(labels); if (it != Metrics_.end()) { - Y_ENSURE(it->second->Type() == type, "cannot create metric " << labels + Y_ENSURE(it->second.Metric->Type() == type, "cannot create metric " << labels << " with type " << MetricTypeToStr(type) - << ", because registry already has same metric with type " << MetricTypeToStr(it->second->Type())); - return static_cast<TMetric*>(it->second.Get()); + << ", because registry already has same metric with type " << MetricTypeToStr(it->second.Metric->Type())); + Y_ENSURE(it->second.Opts.MemOnly == opts.MemOnly,"cannot create metric " << labels + << " with memOnly=" << opts.MemOnly + << ", because registry already has same metric with memOnly=" << it->second.Opts.MemOnly); + return static_cast<TMetric*>(it->second.Metric.Get()); } } @@ -202,14 +279,15 @@ namespace NMonitoring { TWriteGuard g{*Lock_}; // decltype(Metrics_)::iterator breaks build on windows - THashMap<ILabelsPtr, IMetricPtr>::iterator it; + THashMap<ILabelsPtr, TMetricValue>::iterator it; + TMetricValue metricValue = {metric, opts}; if constexpr (!std::is_convertible_v<TLabelsType, ILabelsPtr>) { - it = Metrics_.emplace(new TLabels{std::forward<TLabelsType>(labels)}, std::move(metric)).first; + it = Metrics_.emplace(new TLabels{std::forward<TLabelsType>(labels)}, std::move(metricValue)).first; } else { - it = Metrics_.emplace(std::forward<TLabelsType>(labels), std::move(metric)).first; + it = Metrics_.emplace(std::forward<TLabelsType>(labels), std::move(metricValue)).first; } - return static_cast<TMetric*>(it->second.Get()); + return static_cast<TMetric*>(it->second.Metric.Get()); } } @@ -227,7 +305,7 @@ namespace NMonitoring { consumer->OnLabelsEnd(); } - TVector<std::pair<ILabelsPtr, IMetricPtr>> tmpMetrics; + TVector<std::pair<ILabelsPtr, TMetricValue>> tmpMetrics; { TReadGuard g{*Lock_}; @@ -239,10 +317,17 @@ namespace NMonitoring { for (const auto& it: tmpMetrics) { ILabels* labels = it.first.Get(); - IMetric* metric = it.second.Get(); - ConsumeMetric(time, consumer, metric, [&]() { - ConsumeLabels(consumer, *labels); - }); + IMetric* metric = it.second.Metric.Get(); + TMetricOpts opts = it.second.Opts; + ConsumeMetric( + time, + consumer, + metric, + [&]() { + ConsumeLabels(consumer, *labels); + }, + opts + ); } consumer->OnStreamEnd(); @@ -253,11 +338,18 @@ namespace NMonitoring { for (const auto& it: Metrics_) { ILabels* labels = it.first.Get(); - IMetric* metric = it.second.Get(); - ConsumeMetric(time, consumer, metric, [&]() { - ConsumeLabels(consumer, CommonLabels_); - ConsumeLabels(consumer, *labels); - }); + IMetric* metric = it.second.Metric.Get(); + TMetricOpts opts = it.second.Opts; + ConsumeMetric( + time, + consumer, + metric, + [&]() { + ConsumeLabels(consumer, CommonLabels_); + ConsumeLabels(consumer, *labels); + }, + opts + ); } } } diff --git a/library/cpp/monlib/metrics/metric_registry.h b/library/cpp/monlib/metrics/metric_registry.h index 7b3a7aab70..f60467cf91 100644 --- a/library/cpp/monlib/metrics/metric_registry.h +++ b/library/cpp/monlib/metrics/metric_registry.h @@ -9,35 +9,109 @@ namespace NMonitoring { + + struct TMetricOpts { + bool MemOnly = false; + }; + class IMetricFactory { public: virtual ~IMetricFactory() = default; virtual IGauge* Gauge(ILabelsPtr labels) = 0; + virtual IGauge* GaugeWithOpts(ILabelsPtr labels, TMetricOpts opts = {}) { + Y_UNUSED(opts); + return Gauge(std::move(labels)); + } + virtual ILazyGauge* LazyGauge(ILabelsPtr labels, std::function<double()> supplier) = 0; + virtual ILazyGauge* LazyGaugeWithOpts(ILabelsPtr labels, std::function<double()> supplier, TMetricOpts opts = {}) { + Y_UNUSED(opts); + return LazyGauge(std::move(labels), std::move(supplier)); + } + virtual IIntGauge* IntGauge(ILabelsPtr labels) = 0; + virtual IIntGauge* IntGaugeWithOpts(ILabelsPtr labels, TMetricOpts opts = {}) { + Y_UNUSED(opts); + return IntGauge(std::move(labels)); + } virtual ILazyIntGauge* LazyIntGauge(ILabelsPtr labels, std::function<i64()> supplier) = 0; + virtual ILazyIntGauge* LazyIntGaugeWithOpts(ILabelsPtr labels, std::function<i64()> supplier, TMetricOpts opts = {}) { + Y_UNUSED(opts); + return LazyIntGauge(std::move(labels), std::move(supplier)); + } + virtual ICounter* Counter(ILabelsPtr labels) = 0; + virtual ICounter* CounterWithOpts(ILabelsPtr labels, TMetricOpts opts = {}) { + Y_UNUSED(opts); + return Counter(std::move(labels)); + } + virtual ILazyCounter* LazyCounter(ILabelsPtr labels, std::function<ui64()> supplier) = 0; + virtual ILazyCounter* LazyCounterWithOpts(ILabelsPtr labels, std::function<ui64()> supplier, TMetricOpts opts = {}) { + Y_UNUSED(opts); + return LazyCounter(std::move(labels), std::move(supplier)); + } virtual IRate* Rate(ILabelsPtr labels) = 0; + virtual IRate* RateWithOpts(ILabelsPtr labels, TMetricOpts opts = {}) { + Y_UNUSED(opts); + return Rate(std::move(labels)); + } + virtual ILazyRate* LazyRate(ILabelsPtr labels, std::function<ui64()> supplier) = 0; + virtual ILazyRate* LazyRateWithOpts(ILabelsPtr labels, std::function<ui64()> supplier, TMetricOpts opts = {}) { + Y_UNUSED(opts); + return LazyRate(std::move(labels), std::move(supplier)); + } virtual IHistogram* HistogramCounter( ILabelsPtr labels, IHistogramCollectorPtr collector) = 0; + virtual IHistogram* HistogramCounterWithOpts( + ILabelsPtr labels, + IHistogramCollectorPtr collector, + TMetricOpts opts = {}) + { + Y_UNUSED(opts); + return HistogramCounter(std::move(labels), std::move(collector)); + } virtual IHistogram* HistogramRate( ILabelsPtr labels, IHistogramCollectorPtr collector) = 0; + virtual IHistogram* HistogramRateWithOpts( + ILabelsPtr labels, + IHistogramCollectorPtr collector, + TMetricOpts opts = {}) + { + Y_UNUSED(opts); + return HistogramRate(std::move(labels), std::move(collector)); + } virtual IHistogram* HistogramCounter( ILabelsPtr labels, std::function<IHistogramCollectorPtr()> makeHistogramCollector) = 0; + virtual IHistogram* HistogramCounterWithOpts( + ILabelsPtr labels, + std::function<IHistogramCollectorPtr()> makeHistogramCollector, + TMetricOpts opts = {}) + { + Y_UNUSED(opts); + return HistogramCounter(std::move(labels), std::move(makeHistogramCollector)); + } virtual IHistogram* HistogramRate( ILabelsPtr labels, std::function<IHistogramCollectorPtr()> makeHistogramCollector) = 0; + virtual IHistogram* HistogramRateWithOpts( + ILabelsPtr labels, + std::function<IHistogramCollectorPtr()> makeHistogramCollector, + TMetricOpts opts = {}) + { + Y_UNUSED(opts); + return HistogramRate(std::move(labels), std::move(makeHistogramCollector)); + } }; class IMetricSupplier { @@ -79,29 +153,53 @@ namespace NMonitoring { static std::shared_ptr<TMetricRegistry> SharedInstance(); TGauge* Gauge(TLabels labels); + TGauge* GaugeWithOpts(TLabels labels, TMetricOpts opts = {}); TLazyGauge* LazyGauge(TLabels labels, std::function<double()> supplier); + TLazyGauge* LazyGaugeWithOpts(TLabels labels, std::function<double()> supplier, TMetricOpts opts = {}); TIntGauge* IntGauge(TLabels labels); + TIntGauge* IntGaugeWithOpts(TLabels labels, TMetricOpts opts = {}); TLazyIntGauge* LazyIntGauge(TLabels labels, std::function<i64()> supplier); + TLazyIntGauge* LazyIntGaugeWithOpts(TLabels labels, std::function<i64()> supplier, TMetricOpts opts = {}); TCounter* Counter(TLabels labels); + TCounter* CounterWithOpts(TLabels labels, TMetricOpts opts = {}); TLazyCounter* LazyCounter(TLabels labels, std::function<ui64()> supplier); + TLazyCounter* LazyCounterWithOpts(TLabels labels, std::function<ui64()> supplier, TMetricOpts opts = {}); TRate* Rate(TLabels labels); + TRate* RateWithOpts(TLabels labels, TMetricOpts opts = {}); TLazyRate* LazyRate(TLabels labels, std::function<ui64()> supplier); + TLazyRate* LazyRateWithOpts(TLabels labels, std::function<ui64()> supplier, TMetricOpts opts = {}); THistogram* HistogramCounter( TLabels labels, IHistogramCollectorPtr collector); + THistogram* HistogramCounterWithOpts( + TLabels labels, + IHistogramCollectorPtr collector, + TMetricOpts opts = {}); THistogram* HistogramRate( TLabels labels, IHistogramCollectorPtr collector); + THistogram* HistogramRateWithOpts( + TLabels labels, + IHistogramCollectorPtr collector, + TMetricOpts opts = {}); THistogram* HistogramCounter( TLabels labels, std::function<IHistogramCollectorPtr()> makeHistogramCollector); + THistogram* HistogramCounterWithOpts( + TLabels labels, + std::function<IHistogramCollectorPtr()> makeHistogramCollector, + TMetricOpts opts = {}); THistogram* HistogramRate( TLabels labels, std::function<IHistogramCollectorPtr()> makeHistogramCollector); + THistogram* HistogramRateWithOpts( + TLabels labels, + std::function<IHistogramCollectorPtr()> makeHistogramCollector, + TMetricOpts opts = {}); /** * Set all registered metrics to zero @@ -123,36 +221,65 @@ namespace NMonitoring { private: TGauge* Gauge(ILabelsPtr labels) override; + TGauge* GaugeWithOpts(ILabelsPtr labels, TMetricOpts opts = {}) override; TLazyGauge* LazyGauge(ILabelsPtr labels, std::function<double()> supplier) override; + TLazyGauge* LazyGaugeWithOpts(ILabelsPtr labels, std::function<double()> supplier, TMetricOpts opts = {}) override; TIntGauge* IntGauge(ILabelsPtr labels) override; + TIntGauge* IntGaugeWithOpts(ILabelsPtr labels, TMetricOpts opts = {}) override; TLazyIntGauge* LazyIntGauge(ILabelsPtr labels, std::function<i64()> supplier) override; + TLazyIntGauge* LazyIntGaugeWithOpts(ILabelsPtr labels, std::function<i64()> supplier, TMetricOpts opts = {}) override; TCounter* Counter(ILabelsPtr labels) override; + TCounter* CounterWithOpts(ILabelsPtr labels, TMetricOpts opts = {}) override; TLazyCounter* LazyCounter(ILabelsPtr labels, std::function<ui64()> supplier) override; + TLazyCounter* LazyCounterWithOpts(ILabelsPtr labels, std::function<ui64()> supplier, TMetricOpts opts = {}) override; TRate* Rate(ILabelsPtr labels) override; + TRate* RateWithOpts(ILabelsPtr labels, TMetricOpts opts = {}) override; TLazyRate* LazyRate(ILabelsPtr labels, std::function<ui64()> supplier) override; + TLazyRate* LazyRateWithOpts(ILabelsPtr labels, std::function<ui64()> supplier, TMetricOpts opts = {}) override; THistogram* HistogramCounter( ILabelsPtr labels, IHistogramCollectorPtr collector) override; + THistogram* HistogramCounterWithOpts( + ILabelsPtr labels, + IHistogramCollectorPtr collector, + TMetricOpts opts = {}) override; THistogram* HistogramRate( ILabelsPtr labels, IHistogramCollectorPtr collector) override; + THistogram* HistogramRateWithOpts( + ILabelsPtr labels, + IHistogramCollectorPtr collector, + TMetricOpts opts = {}) override; THistogram* HistogramCounter( ILabelsPtr labels, std::function<IHistogramCollectorPtr()> makeHistogramCollector) override; + THistogram* HistogramCounterWithOpts( + ILabelsPtr labels, + std::function<IHistogramCollectorPtr()> makeHistogramCollector, + TMetricOpts opts = {}) override; THistogram* HistogramRate( ILabelsPtr labels, std::function<IHistogramCollectorPtr()> makeHistogramCollector) override; + THistogram* HistogramRateWithOpts( + ILabelsPtr labels, + std::function<IHistogramCollectorPtr()> makeHistogramCollector, + TMetricOpts opts = {}) override; + + struct TMetricValue { + IMetricPtr Metric; + TMetricOpts Opts; + }; private: THolder<TRWMutex> Lock_ = MakeHolder<TRWMutex>(); - THashMap<ILabelsPtr, IMetricPtr> Metrics_; + THashMap<ILabelsPtr, TMetricValue> Metrics_; template <typename TMetric, EMetricType type, typename TLabelsType, typename... Args> - TMetric* Metric(TLabelsType&& labels, Args&&... args); + TMetric* Metric(TLabelsType&& labels, TMetricOpts&& opts, Args&&... args); TLabels CommonLabels_; }; diff --git a/library/cpp/monlib/metrics/metric_registry_ut.cpp b/library/cpp/monlib/metrics/metric_registry_ut.cpp index 802c423264..89e6d56146 100644 --- a/library/cpp/monlib/metrics/metric_registry_ut.cpp +++ b/library/cpp/monlib/metrics/metric_registry_ut.cpp @@ -396,4 +396,76 @@ Y_UNIT_TEST_SUITE(TMetricRegistryTest) { UNIT_ASSERT(commonLabels[0].GetName() == "common"); UNIT_ASSERT(commonLabels[0].GetValue() == "label"); } + + Y_UNIT_TEST(MemOnlyMetric) { + TMetricRegistry registry; + i64 int_val = 0; + double double_val = 0; + + registry.GaugeWithOpts({{"some", "gauge"}}, {true}); + UNIT_ASSERT_EXCEPTION(registry.Gauge({{"some", "gauge"}}), yexception); + + registry.LazyGaugeWithOpts( + {{"some", "lazy_gauge"}}, + [&double_val](){return double_val;}, + {true}); + UNIT_ASSERT_EXCEPTION( + registry.LazyGauge( + {{"some", "lazy_gauge"}}, + [&double_val](){return double_val;}), + yexception); + + registry.IntGaugeWithOpts({{"some", "int_gauge"}}, {true}); + UNIT_ASSERT_EXCEPTION(registry.IntGauge({{"some", "int_gauge"}}), yexception); + + registry.LazyIntGaugeWithOpts( + {{"some", "lazy_int_gauge"}}, + [&int_val](){return int_val;}, + {true}); + UNIT_ASSERT_EXCEPTION( + registry.LazyIntGauge( + {{"some", "lazy_int_gauge"}}, + [&int_val](){return int_val;}), + yexception); + + registry.CounterWithOpts({{"some", "counter"}}, {true}); + UNIT_ASSERT_EXCEPTION(registry.Counter({{"some", "counter"}}), yexception); + + registry.LazyCounterWithOpts({{"some", "lazy_counter"}}, [&int_val](){return int_val;}, {true}); + UNIT_ASSERT_EXCEPTION( + registry.LazyCounter( + {{"some", "lazy_counter"}}, + [&int_val](){return int_val;}), + yexception); + + registry.RateWithOpts({{"some", "rate"}}, {true}); + UNIT_ASSERT_EXCEPTION(registry.Rate({{"some", "rate"}}), yexception); + + registry.LazyRateWithOpts({{"some", "lazy_rate"}}, [&double_val](){return double_val;}, {true}); + UNIT_ASSERT_EXCEPTION( + registry.LazyRate( + {{"some", "lazy_rate"}}, + [&double_val](){return double_val;}), + yexception); + + registry.HistogramCounterWithOpts( + {{"some", "histogram_counter"}}, + ExplicitHistogram({1, 5, 15, 20, 25}), + {true}); + UNIT_ASSERT_EXCEPTION( + registry.HistogramCounter( + {{"some", "histogram_counter"}}, + ExplicitHistogram({1, 5, 15, 20, 25})), + yexception); + + registry.HistogramRateWithOpts( + {{"some", "histogram_rate"}}, + ExponentialHistogram(5, 2), + {true}); + UNIT_ASSERT_EXCEPTION( + registry.HistogramRate( + {{"some", "histogram_rate"}}, + ExponentialHistogram(5, 2)), + yexception); + } } diff --git a/library/cpp/monlib/metrics/metric_sub_registry.h b/library/cpp/monlib/metrics/metric_sub_registry.h index d1860c00a9..c3481f338f 100644 --- a/library/cpp/monlib/metrics/metric_sub_registry.h +++ b/library/cpp/monlib/metrics/metric_sub_registry.h @@ -33,61 +33,110 @@ public: AddCommonLabels(labels.Get()); return DelegatePtr_->Gauge(std::move(labels)); } + IGauge* GaugeWithOpts(ILabelsPtr labels, TMetricOpts opts = {}) override { + AddCommonLabels(labels.Get()); + return DelegatePtr_->GaugeWithOpts(std::move(labels), std::move(opts)); + } ILazyGauge* LazyGauge(ILabelsPtr labels, std::function<double()> supplier) override { AddCommonLabels(labels.Get()); return DelegatePtr_->LazyGauge(std::move(labels), std::move(supplier)); } + ILazyGauge* LazyGaugeWithOpts(ILabelsPtr labels, std::function<double()> supplier, TMetricOpts opts = {}) override { + AddCommonLabels(labels.Get()); + return DelegatePtr_->LazyGaugeWithOpts(std::move(labels), std::move(supplier), std::move(opts)); + } IIntGauge* IntGauge(ILabelsPtr labels) override { AddCommonLabels(labels.Get()); return DelegatePtr_->IntGauge(std::move(labels)); } + IIntGauge* IntGaugeWithOpts(ILabelsPtr labels, TMetricOpts opts = {}) override { + AddCommonLabels(labels.Get()); + return DelegatePtr_->IntGaugeWithOpts(std::move(labels), std::move(opts)); + } ILazyIntGauge* LazyIntGauge(ILabelsPtr labels, std::function<i64()> supplier) override { AddCommonLabels(labels.Get()); return DelegatePtr_->LazyIntGauge(std::move(labels), std::move(supplier)); } + ILazyIntGauge* LazyIntGaugeWithOpts(ILabelsPtr labels, std::function<i64()> supplier, TMetricOpts opts = {}) override { + AddCommonLabels(labels.Get()); + return DelegatePtr_->LazyIntGaugeWithOpts(std::move(labels), std::move(supplier), std::move(opts)); + } ICounter* Counter(ILabelsPtr labels) override { AddCommonLabels(labels.Get()); return DelegatePtr_->Counter(std::move(labels)); } + ICounter* CounterWithOpts(ILabelsPtr labels, TMetricOpts opts = {}) override { + AddCommonLabels(labels.Get()); + return DelegatePtr_->CounterWithOpts(std::move(labels), std::move(opts)); + } ILazyCounter* LazyCounter(ILabelsPtr labels, std::function<ui64()> supplier) override { AddCommonLabels(labels.Get()); return DelegatePtr_->LazyCounter(std::move(labels), std::move(supplier)); } + ILazyCounter* LazyCounterWithOpts(ILabelsPtr labels, std::function<ui64()> supplier, TMetricOpts opts = {}) override { + AddCommonLabels(labels.Get()); + return DelegatePtr_->LazyCounterWithOpts(std::move(labels), std::move(supplier), std::move(opts)); + } IRate* Rate(ILabelsPtr labels) override { AddCommonLabels(labels.Get()); return DelegatePtr_->Rate(std::move(labels)); } + IRate* RateWithOpts(ILabelsPtr labels, TMetricOpts opts = {}) override { + AddCommonLabels(labels.Get()); + return DelegatePtr_->RateWithOpts(std::move(labels), std::move(opts)); + } + ILazyRate* LazyRate(ILabelsPtr labels, std::function<ui64()> supplier) override { AddCommonLabels(labels.Get()); return DelegatePtr_->LazyRate(std::move(labels), std::move(supplier)); } + ILazyRate* LazyRateWithOpts(ILabelsPtr labels, std::function<ui64()> supplier, TMetricOpts opts = {}) override { + AddCommonLabels(labels.Get()); + return DelegatePtr_->LazyRateWithOpts(std::move(labels), std::move(supplier), std::move(opts)); + } IHistogram* HistogramCounter(ILabelsPtr labels, IHistogramCollectorPtr collector) override { AddCommonLabels(labels.Get()); return DelegatePtr_->HistogramCounter(std::move(labels), std::move(collector)); } + IHistogram* HistogramCounterWithOpts(ILabelsPtr labels, IHistogramCollectorPtr collector, TMetricOpts opts = {}) override { + AddCommonLabels(labels.Get()); + return DelegatePtr_->HistogramCounterWithOpts(std::move(labels), std::move(collector), std::move(opts)); + } IHistogram* HistogramRate(ILabelsPtr labels, IHistogramCollectorPtr collector) override { AddCommonLabels(labels.Get()); return DelegatePtr_->HistogramRate(std::move(labels), std::move(collector)); } + IHistogram* HistogramRateWithOpts(ILabelsPtr labels, IHistogramCollectorPtr collector, TMetricOpts opts = {}) override { + AddCommonLabels(labels.Get()); + return DelegatePtr_->HistogramRateWithOpts(std::move(labels), std::move(collector), std::move(opts)); + } IHistogram* HistogramCounter(ILabelsPtr labels, std::function<IHistogramCollectorPtr()> collector) override { AddCommonLabels(labels.Get()); return DelegatePtr_->HistogramCounter(std::move(labels), std::move(collector)); } + IHistogram* HistogramCounterWithOpts(ILabelsPtr labels, std::function<IHistogramCollectorPtr()> collector, TMetricOpts opts = {}) override { + AddCommonLabels(labels.Get()); + return DelegatePtr_->HistogramCounterWithOpts(std::move(labels), std::move(collector), std::move(opts)); + } IHistogram* HistogramRate(ILabelsPtr labels, std::function<IHistogramCollectorPtr()> collector) override { AddCommonLabels(labels.Get()); return DelegatePtr_->HistogramRate(std::move(labels), std::move(collector)); } + IHistogram* HistogramRateWithOpts(ILabelsPtr labels, std::function<IHistogramCollectorPtr()> collector, TMetricOpts opts) override { + AddCommonLabels(labels.Get()); + return DelegatePtr_->HistogramRateWithOpts(std::move(labels), std::move(collector), std::move(opts)); + } void Accept(TInstant time, IMetricConsumer* consumer) const override { DelegatePtr_->Accept(time, consumer); diff --git a/library/cpp/yt/memory/non_null_ptr-inl.h b/library/cpp/yt/memory/non_null_ptr-inl.h index a14b1e9302..d4d00c5549 100644 --- a/library/cpp/yt/memory/non_null_ptr-inl.h +++ b/library/cpp/yt/memory/non_null_ptr-inl.h @@ -1,6 +1,6 @@ #pragma once #ifndef NON_NULL_PTR_H_ -#error "Direct inclusion of this file is not allowed, include helpers.h" +#error "Direct inclusion of this file is not allowed, include non_null_ptr.h" // For the sake of sane code completion. #include "non_null_ptr.h" #endif diff --git a/library/cpp/yt/memory/poison-inl.h b/library/cpp/yt/memory/poison-inl.h new file mode 100644 index 0000000000..c7563565a8 --- /dev/null +++ b/library/cpp/yt/memory/poison-inl.h @@ -0,0 +1,60 @@ +#pragma once +#ifndef POISON_INL_H_ +#error "Direct inclusion of this file is not allowed, include poison.h" +// For the sake of sane code completion. +#include "poison.h" +#endif + +#include <util/system/compiler.h> + +namespace NYT { + +//////////////////////////////////////////////////////////////////////////////// + +#if defined(_asan_enabled_) + +extern "C" { +void __asan_poison_memory_region(void const volatile *addr, size_t size); +void __asan_unpoison_memory_region(void const volatile *addr, size_t size); +} // extern "C" + +Y_FORCE_INLINE void PoisonMemory(TMutableRef ref) +{ + __asan_poison_memory_region(ref.data(), ref.size()); +} + +Y_FORCE_INLINE void UnpoisonMemory(TMutableRef ref) +{ + __asan_unpoison_memory_region(ref.data(), ref.size()); +} + +#elif defined(_msan_enabled_) + +extern "C" { +void __msan_unpoison(const volatile void* a, size_t size); +void __msan_poison(const volatile void* a, size_t size); +} // extern "C" + +Y_FORCE_INLINE void PoisonMemory(TMutableRef ref) +{ + __msan_poison(ref.data(), ref.size()); +} + +Y_FORCE_INLINE void UnpoisonMemory(TMutableRef ref) +{ + __msan_unpoison(ref.data(), ref.size()); +} + +#elif defined(NDEBUG) + +Y_FORCE_INLINE void PoisonMemory(TMutableRef /*ref*/) +{ } + +Y_FORCE_INLINE void UnpoisonMemory(TMutableRef /*ref*/) +{ } + +#endif + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT diff --git a/library/cpp/yt/memory/poison.cpp b/library/cpp/yt/memory/poison.cpp new file mode 100644 index 0000000000..bc4bcad4e0 --- /dev/null +++ b/library/cpp/yt/memory/poison.cpp @@ -0,0 +1,64 @@ +#include "poison.h" + +namespace NYT { + +//////////////////////////////////////////////////////////////////////////////// + +namespace { + +template <char Byte0, char Byte1, char Byte2, char Byte3, char Byte4, char Byte5, char Byte6, char Byte7> +void ClobberMemory(char* __restrict__ ptr, size_t size) +{ + while (size >= 8) { + *ptr++ = Byte0; + *ptr++ = Byte1; + *ptr++ = Byte2; + *ptr++ = Byte3; + *ptr++ = Byte4; + *ptr++ = Byte5; + *ptr++ = Byte6; + *ptr++ = Byte7; + size -= 8; + } + + switch (size) { + case 7: + *ptr++ = Byte0; + [[fallthrough]]; + case 6: + *ptr++ = Byte1; + [[fallthrough]]; + case 5: + *ptr++ = Byte2; + [[fallthrough]]; + case 4: + *ptr++ = Byte3; + [[fallthrough]]; + case 3: + *ptr++ = Byte4; + [[fallthrough]]; + case 2: + *ptr++ = Byte5; + [[fallthrough]]; + case 1: + *ptr++ = Byte6; + } +} + +} // namespace + +#if !defined(NDEBUG) && !defined(_asan_enabled_) && !defined(_msan_enabled_) +void PoisonMemory(TMutableRef ref) +{ + ClobberMemory<'d', 'e', 'a', 'd', 'b', 'e', 'e', 'f'>(ref.data(), ref.size()); +} + +void UnpoisonMemory(TMutableRef ref) +{ + ClobberMemory<'c', 'a', 'f', 'e', 'b', 'a', 'b', 'e'>(ref.data(), ref.size()); +} +#endif + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT diff --git a/library/cpp/yt/memory/poison.h b/library/cpp/yt/memory/poison.h new file mode 100644 index 0000000000..9519f3da10 --- /dev/null +++ b/library/cpp/yt/memory/poison.h @@ -0,0 +1,25 @@ +#pragma once + +#include "ref.h" + +namespace NYT { + +//////////////////////////////////////////////////////////////////////////////// + +//! In release builds, does nothing. +//! In checked builds, clobbers memory with garbage pattern. +//! In sanitized builds, invokes sanitizer poisoning. +void PoisonMemory(TMutableRef ref); + +//! In release builds, does nothing. +//! In checked builds, clobbers memory with (another) garbage pattern. +//! In sanitized builds, invokes sanitizer unpoisoning. +void UnpoisonMemory(TMutableRef ref); + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT + +#define POISON_INL_H_ +#include "poison-inl.h" +#undef POISON_INL_H_ diff --git a/library/cpp/yt/memory/range.h b/library/cpp/yt/memory/range.h index 420ee9b429..84ea2e7758 100644 --- a/library/cpp/yt/memory/range.h +++ b/library/cpp/yt/memory/range.h @@ -354,12 +354,23 @@ public: : TRange<T>(elements) { } + using TRange<T>::Data; using TRange<T>::Begin; using TRange<T>::End; using TRange<T>::Front; using TRange<T>::Back; using TRange<T>::operator[]; + T* Data() const + { + return const_cast<T*>(this->Data_); + } + + T* data() const + { + return Data(); + } + iterator Begin() const { return const_cast<T*>(this->Data_); diff --git a/library/cpp/yt/memory/ya.make b/library/cpp/yt/memory/ya.make index 5397dccf32..d0c377cf3d 100644 --- a/library/cpp/yt/memory/ya.make +++ b/library/cpp/yt/memory/ya.make @@ -16,6 +16,7 @@ SRCS( chunked_output_stream.cpp memory_tag.cpp new.cpp + poison.cpp ref.cpp ref_tracked.cpp safe_memory_reader.cpp diff --git a/tools/cpp_style_checker/__main__.py b/tools/cpp_style_checker/__main__.py index ab8534a0ad..089d9fa079 100644 --- a/tools/cpp_style_checker/__main__.py +++ b/tools/cpp_style_checker/__main__.py @@ -5,7 +5,7 @@ import subprocess import time import yaml -from build.plugins.lib.test_const import CLANG_FORMAT_RESOURCE +from build.plugins.lib.test_const import CLANG_FORMAT_RESOURCE, CLANG_FORMAT_15_RESOURCE from library.python.testing.custom_linter_util import linter_params, reporter from library.python.testing.style import rules @@ -16,6 +16,8 @@ def main(): if 'clang_format_bin' in params.extra_params: # custom clang-format clang_format_binary = params.depends[params.extra_params['clang_format_bin']] + elif 'use_clang_format_15' in params.extra_params: + clang_format_binary = os.path.join(params.global_resources[CLANG_FORMAT_15_RESOURCE], 'clang-format-15') else: clang_format_binary = os.path.join(params.global_resources[CLANG_FORMAT_RESOURCE], 'clang-format') @@ -39,33 +39,33 @@ REGISTRY_ENDPOINT = os.environ.get("YA_REGISTRY_ENDPOINT", "https://devtools-reg PLATFORM_MAP = { "data": { "darwin": { - "md5": "ed34b869e3a766cccca99388465df834", + "md5": "0a246880109da2b6d775b5daeb6bb2f2", "urls": [ - f"{REGISTRY_ENDPOINT}/8119502178" + f"{REGISTRY_ENDPOINT}/8161454854" ] }, "darwin-arm64": { - "md5": "a1cacfea98f9894d5581fe46b4eb27f8", + "md5": "4f2cf03181b6f11f9fa5e70319233ec7", "urls": [ - f"{REGISTRY_ENDPOINT}/8119500148" + f"{REGISTRY_ENDPOINT}/8161452021" ] }, "linux-aarch64": { - "md5": "76e502fc5106cd60cfaeaf54fc61c44d", + "md5": "bdf485f4384d5c49d076cd6f51549c52", "urls": [ - f"{REGISTRY_ENDPOINT}/8119498515" + f"{REGISTRY_ENDPOINT}/8161449965" ] }, "win32-clang-cl": { - "md5": "9d94a672b39ef7d913f5743fd901592f", + "md5": "e243509e7283a639cb34c83349e128be", "urls": [ - f"{REGISTRY_ENDPOINT}/8119504001" + f"{REGISTRY_ENDPOINT}/8161456097" ] }, "linux": { - "md5": "7222ec6e66035ce9e75c26df1aad868a", + "md5": "e4a4799a1caa797935daf3013bebe4e2", "urls": [ - f"{REGISTRY_ENDPOINT}/8119506702" + f"{REGISTRY_ENDPOINT}/8161456791" ] } } diff --git a/ydb/ci/rightlib.txt b/ydb/ci/rightlib.txt index b7b70c3a1b..1298102d24 100644 --- a/ydb/ci/rightlib.txt +++ b/ydb/ci/rightlib.txt @@ -1 +1 @@ -6678165e016ba474f1b8dd6d49af92b0d46350b9 +0ae3f82349eeb4f353c62dd726e4ba06bbc837f9 diff --git a/yql/essentials/core/facade/yql_facade.cpp b/yql/essentials/core/facade/yql_facade.cpp index f8d9c28ba1..547c3a292e 100644 --- a/yql/essentials/core/facade/yql_facade.cpp +++ b/yql/essentials/core/facade/yql_facade.cpp @@ -691,6 +691,7 @@ void TProgram::HandleTranslationSettings(NSQLTranslation::TTranslationSettings& loadedSettings.V0WarnAsError = NSQLTranslation::ISqlFeaturePolicy::Make(dataNode["V0WarnAsError"].AsBool()); loadedSettings.DqDefaultAuto = NSQLTranslation::ISqlFeaturePolicy::Make(dataNode["DqDefaultAuto"].AsBool()); loadedSettings.BlockDefaultAuto = NSQLTranslation::ISqlFeaturePolicy::Make(dataNode["BlockDefaultAuto"].AsBool()); + loadedSettings.IsReplay = true; currentSettings = &loadedSettings; } } diff --git a/yql/essentials/linters.make.inc b/yql/essentials/linters.make.inc new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/yql/essentials/linters.make.inc diff --git a/yql/essentials/minikql/aligned_page_pool.cpp b/yql/essentials/minikql/aligned_page_pool.cpp index 04aab1bd8a..bcbb3edd11 100644 --- a/yql/essentials/minikql/aligned_page_pool.cpp +++ b/yql/essentials/minikql/aligned_page_pool.cpp @@ -783,6 +783,40 @@ void* GetAlignedPage(ui64 size) { } template<typename TMmap> +void* GetAlignedPage() { + const auto size = TAlignedPagePool::POOL_PAGE_SIZE; + auto& globalPool = TGlobalPools<TMmap, false>::Instance(); + + if (auto* page = globalPool.Get(0).GetPage()) { + return page; + } + + auto allocSize = size * 2; + void* unalignedPtr = globalPool.DoMmap(allocSize); + if (Y_UNLIKELY(MAP_FAILED == unalignedPtr)) { + TStringStream mmaps; + const auto lastError = LastSystemError(); + if (lastError == ENOMEM) { + mmaps << GetMemoryMapsString(); + } + + ythrow yexception() << "Mmap failed to allocate " << allocSize << " bytes: " + << LastSystemErrorText(lastError) << mmaps.Str(); + } + + void* page = AlignUp(unalignedPtr, size); + + // Unmap unaligned prefix before offset and tail after aligned page + const size_t offset = (intptr_t)page - (intptr_t)unalignedPtr; + if (Y_UNLIKELY(offset)) { + globalPool.DoMunmap(unalignedPtr, offset); + globalPool.DoMunmap((ui8*)page + size, size - offset); + } + + return page; +} + +template<typename TMmap> void ReleaseAlignedPage(void* mem, ui64 size) { size = AlignUp(size, SYS_PAGE_SIZE); if (size < TAlignedPagePool::POOL_PAGE_SIZE) { @@ -801,6 +835,11 @@ void ReleaseAlignedPage(void* mem, ui64 size) { } template<typename TMmap> +void ReleaseAlignedPage(void* ptr) { + TGlobalPools<TMmap, false>::Instance().PushPage(0, ptr); +} + +template<typename TMmap> i64 GetTotalMmapedBytes() { return TGlobalPools<TMmap, true>::Instance().GetTotalMmappedBytes() + TGlobalPools<TMmap, false>::Instance().GetTotalMmappedBytes(); } @@ -822,10 +861,18 @@ template void* GetAlignedPage<>(ui64); template void* GetAlignedPage<TFakeAlignedMmap>(ui64); template void* GetAlignedPage<TFakeUnalignedMmap>(ui64); +template void* GetAlignedPage<>(); +template void* GetAlignedPage<TFakeAlignedMmap>(); +template void* GetAlignedPage<TFakeUnalignedMmap>(); + template void ReleaseAlignedPage<>(void*,ui64); template void ReleaseAlignedPage<TFakeAlignedMmap>(void*,ui64); template void ReleaseAlignedPage<TFakeUnalignedMmap>(void*,ui64); +template void ReleaseAlignedPage<>(void*); +template void ReleaseAlignedPage<TFakeAlignedMmap>(void*); +template void ReleaseAlignedPage<TFakeUnalignedMmap>(void*); + size_t GetMemoryMapsCount() { size_t lineCount = 0; TString line; diff --git a/yql/essentials/minikql/aligned_page_pool.h b/yql/essentials/minikql/aligned_page_pool.h index 4a5b1d2e55..511b99b4d7 100644 --- a/yql/essentials/minikql/aligned_page_pool.h +++ b/yql/essentials/minikql/aligned_page_pool.h @@ -309,9 +309,15 @@ template<typename TMmap = TSystemMmap> void* GetAlignedPage(ui64 size); template<typename TMmap = TSystemMmap> +void* GetAlignedPage(); + +template<typename TMmap = TSystemMmap> void ReleaseAlignedPage(void* mem, ui64 size); template<typename TMmap = TSystemMmap> +void ReleaseAlignedPage(void* mem); + +template<typename TMmap = TSystemMmap> i64 GetTotalMmapedBytes(); template<typename TMmap = TSystemMmap> i64 GetTotalFreeListBytes(); diff --git a/yql/essentials/minikql/comp_nodes/mkql_apply.cpp b/yql/essentials/minikql/comp_nodes/mkql_apply.cpp index 180a38176b..60fbe82ad4 100644 --- a/yql/essentials/minikql/comp_nodes/mkql_apply.cpp +++ b/yql/essentials/minikql/comp_nodes/mkql_apply.cpp @@ -206,9 +206,8 @@ private: } IComputationNode* WrapApply(TCallable& callable, const TComputationNodeFactoryContext& ctx) { - const bool withPos = callable.GetType()->GetName() == "Apply2"; - const ui32 deltaArgs = withPos ? 3 : 0; - MKQL_ENSURE(callable.GetInputsCount() >= 2 + deltaArgs, "Expected at least " << (2 + deltaArgs) << " arguments"); + MKQL_ENSURE(callable.GetInputsCount() >= 5, "Expected at least 5 arguments"); + constexpr size_t posArgs = 3; const auto function = callable.GetInput(0); MKQL_ENSURE(!function.IsImmediate() && function.GetNode()->GetType()->IsCallable(), @@ -218,11 +217,11 @@ IComputationNode* WrapApply(TCallable& callable, const TComputationNodeFactoryCo const auto returnType = functionCallable->GetType()->GetReturnType(); MKQL_ENSURE(returnType->IsCallable(), "Expected callable as return type"); - const TStringBuf file = withPos ? AS_VALUE(TDataLiteral, callable.GetInput(2))->AsValue().AsStringRef() : NUdf::TStringRef(); - const ui32 row = withPos ? AS_VALUE(TDataLiteral, callable.GetInput(3))->AsValue().Get<ui32>() : 0; - const ui32 column = withPos ? AS_VALUE(TDataLiteral, callable.GetInput(4))->AsValue().Get<ui32>() : 0; + const TStringBuf file = AS_VALUE(TDataLiteral, callable.GetInput(2))->AsValue().AsStringRef(); + const ui32 row = AS_VALUE(TDataLiteral, callable.GetInput(3))->AsValue().Get<ui32>(); + const ui32 column = AS_VALUE(TDataLiteral, callable.GetInput(4))->AsValue().Get<ui32>(); - const ui32 inputsCount = callable.GetInputsCount() - deltaArgs; + const ui32 inputsCount = callable.GetInputsCount() - posArgs; const ui32 argsCount = inputsCount - 2; const ui32 dependentCount = AS_VALUE(TDataLiteral, callable.GetInput(1))->AsValue().Get<ui32>(); @@ -235,11 +234,11 @@ IComputationNode* WrapApply(TCallable& callable, const TComputationNodeFactoryCo TComputationNodePtrVector argNodes(callableType->GetArgumentsCount() + dependentCount); for (ui32 i = 2; i < 2 + usedArgs; ++i) { - argNodes[i - 2] = LocateNode(ctx.NodeLocator, callable, i + deltaArgs); + argNodes[i - 2] = LocateNode(ctx.NodeLocator, callable, i + posArgs); } for (ui32 i = 2 + usedArgs; i < inputsCount; ++i) { - argNodes[callableType->GetArgumentsCount() + i - 2 - usedArgs] = LocateNode(ctx.NodeLocator, callable, i + deltaArgs); + argNodes[callableType->GetArgumentsCount() + i - 2 - usedArgs] = LocateNode(ctx.NodeLocator, callable, i + posArgs); } auto functionNode = LocateNode(ctx.NodeLocator, callable, 0); diff --git a/yql/essentials/minikql/comp_nodes/mkql_dictitems.cpp b/yql/essentials/minikql/comp_nodes/mkql_dictitems.cpp index 8790089972..295bee402b 100644 --- a/yql/essentials/minikql/comp_nodes/mkql_dictitems.cpp +++ b/yql/essentials/minikql/comp_nodes/mkql_dictitems.cpp @@ -277,24 +277,9 @@ private: } IComputationNode* WrapDictItems(TCallable& callable, const TComputationNodeFactoryContext& ctx) { - MKQL_ENSURE(callable.GetInputsCount() == 1 || callable.GetInputsCount() == 2, "Expected one or two args"); + MKQL_ENSURE(callable.GetInputsCount() == 1, "Expected one arg"); const auto node = LocateNode(ctx.NodeLocator, callable, 0); - - if (1U == callable.GetInputsCount()) { - return new TDictItemsWrapper(ctx.Mutables, node); - } - - const auto mode = AS_VALUE(TDataLiteral, callable.GetInput(1))->AsValue().Get<ui32>(); - switch (static_cast<EDictItems>(mode)) { - case EDictItems::Both: - return new TDictItemsWrapper(ctx.Mutables, node); - case EDictItems::Keys: - return new TDictHalfsWrapper<true>(ctx.Mutables, node); - case EDictItems::Payloads: - return new TDictHalfsWrapper<false>(ctx.Mutables, node); - default: - Y_ABORT("Unknown mode: %" PRIu32, mode); - } + return new TDictItemsWrapper(ctx.Mutables, node); } IComputationNode* WrapDictKeys(TCallable& callable, const TComputationNodeFactoryContext& ctx) { diff --git a/yql/essentials/minikql/comp_nodes/mkql_factory.cpp b/yql/essentials/minikql/comp_nodes/mkql_factory.cpp index 3b2119ba21..c6b198463b 100644 --- a/yql/essentials/minikql/comp_nodes/mkql_factory.cpp +++ b/yql/essentials/minikql/comp_nodes/mkql_factory.cpp @@ -219,7 +219,6 @@ struct TCallableComputationNodeBuilderFuncMapFiller { {"Invoke", &WrapInvoke}, {"Udf", &WrapUdf}, {"ScriptUdf", &WrapScriptUdf}, - {"Apply", &WrapApply}, {"Apply2", &WrapApply}, {"Callable", &WrapCallable}, {"Size", &WrapSize}, diff --git a/yql/essentials/minikql/comp_nodes/mkql_replicate.cpp b/yql/essentials/minikql/comp_nodes/mkql_replicate.cpp index 81996e07bf..2adac528b9 100644 --- a/yql/essentials/minikql/comp_nodes/mkql_replicate.cpp +++ b/yql/essentials/minikql/comp_nodes/mkql_replicate.cpp @@ -227,20 +227,17 @@ private: } IComputationNode* WrapReplicate(TCallable& callable, const TComputationNodeFactoryContext& ctx) { - MKQL_ENSURE(callable.GetInputsCount() == 2 || callable.GetInputsCount() == 5, "Expected 2 or 5 args"); + MKQL_ENSURE(callable.GetInputsCount() == 5, "Expected 5 args"); const auto countType = AS_TYPE(TDataType, callable.GetInput(1)); MKQL_ENSURE(countType->GetSchemeType() == NUdf::TDataType<ui64>::Id, "Expected ui64"); const auto list = LocateNode(ctx.NodeLocator, callable, 0); const auto count = LocateNode(ctx.NodeLocator, callable, 1); - NUdf::TSourcePosition pos; - if (callable.GetInputsCount() == 5) { - const TStringBuf file = AS_VALUE(TDataLiteral, callable.GetInput(2))->AsValue().AsStringRef(); - const ui32 row = AS_VALUE(TDataLiteral, callable.GetInput(3))->AsValue().Get<ui32>(); - const ui32 column = AS_VALUE(TDataLiteral, callable.GetInput(4))->AsValue().Get<ui32>(); - pos = NUdf::TSourcePosition(row, column, file); - } + const TStringBuf file = AS_VALUE(TDataLiteral, callable.GetInput(2))->AsValue().AsStringRef(); + const ui32 row = AS_VALUE(TDataLiteral, callable.GetInput(3))->AsValue().Get<ui32>(); + const ui32 column = AS_VALUE(TDataLiteral, callable.GetInput(4))->AsValue().Get<ui32>(); + const NUdf::TSourcePosition pos = NUdf::TSourcePosition(row, column, file); return new TReplicateWrapper(ctx.Mutables, list, count, pos); } diff --git a/yql/essentials/minikql/comp_nodes/ut/mkql_chopper_ut.cpp b/yql/essentials/minikql/comp_nodes/ut/mkql_chopper_ut.cpp index 8e314cfb9e..811927b14c 100644 --- a/yql/essentials/minikql/comp_nodes/ut/mkql_chopper_ut.cpp +++ b/yql/essentials/minikql/comp_nodes/ut/mkql_chopper_ut.cpp @@ -297,7 +297,7 @@ Y_UNIT_TEST_SUITE(TMiniKQLChopperStreamTest) { UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|0|0|0|0|0|"); } } -#if !defined(MKQL_RUNTIME_VERSION) || MKQL_RUNTIME_VERSION >= 9u + Y_UNIT_TEST_SUITE(TMiniKQLChopperFlowTest) { Y_UNIT_TEST_LLVM(TestEmpty) { TSetup<LLVM> setup; @@ -482,6 +482,6 @@ Y_UNIT_TEST_SUITE(TMiniKQLChopperFlowTest) { UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|0|0|0|0|0|"); } } -#endif + } // NMiniKQL } // NKikimr diff --git a/yql/essentials/minikql/comp_nodes/ut/mkql_combine_ut.cpp b/yql/essentials/minikql/comp_nodes/ut/mkql_combine_ut.cpp index 55cea5f9fa..ab80f54e00 100644 --- a/yql/essentials/minikql/comp_nodes/ut/mkql_combine_ut.cpp +++ b/yql/essentials/minikql/comp_nodes/ut/mkql_combine_ut.cpp @@ -799,7 +799,7 @@ Y_UNIT_TEST_SUITE(TMiniKQLCombineStreamPerfTest) { Cerr << "Runtime is " << t2 - t1 << " vs C++ " << cppTime << Endl; } } -#if !defined(MKQL_RUNTIME_VERSION) || MKQL_RUNTIME_VERSION >= 3u + Y_UNIT_TEST_SUITE(TMiniKQLCombineFlowTest) { Y_UNIT_TEST_LLVM(TestFullCombineWithOptOut) { TSetup<LLVM> setup(GetNodeFactory()); @@ -1530,6 +1530,6 @@ Y_UNIT_TEST_SUITE(TMiniKQLCombineFlowPerfTest) { Cerr << "Runtime is " << t2 - t1 << " vs C++ " << cppTime << Endl; } } -#endif + } // NMiniKQL } // NKikimr diff --git a/yql/essentials/minikql/comp_nodes/ut/mkql_flatmap_ut.cpp b/yql/essentials/minikql/comp_nodes/ut/mkql_flatmap_ut.cpp index 12c11e4813..e8282e94d7 100644 --- a/yql/essentials/minikql/comp_nodes/ut/mkql_flatmap_ut.cpp +++ b/yql/essentials/minikql/comp_nodes/ut/mkql_flatmap_ut.cpp @@ -522,7 +522,7 @@ Y_UNIT_TEST_SUITE(TMiniKQLFlatMapTest) { UNIT_ASSERT(!iterator.Next(item)); UNIT_ASSERT(!iterator.Next(item)); } -#if !defined(MKQL_RUNTIME_VERSION) || MKQL_RUNTIME_VERSION >= 18u + Y_UNIT_TEST_LLVM(TestNarrowWithList) { TSetup<LLVM> setup; TProgramBuilder& pb = *setup.PgmBuilder; @@ -846,7 +846,6 @@ Y_UNIT_TEST_SUITE(TMiniKQLFlatMapTest) { UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); } -#endif } } diff --git a/yql/essentials/minikql/comp_nodes/ut/mkql_join_ut.cpp b/yql/essentials/minikql/comp_nodes/ut/mkql_join_ut.cpp index d404644972..3731ff1539 100644 --- a/yql/essentials/minikql/comp_nodes/ut/mkql_join_ut.cpp +++ b/yql/essentials/minikql/comp_nodes/ut/mkql_join_ut.cpp @@ -116,7 +116,7 @@ Y_UNIT_TEST_SUITE(TMiniKQLCommonJoinCoreTupleTest) { UNIT_ASSERT(!iterator.Next(item)); } } -#if !defined(MKQL_RUNTIME_VERSION) || MKQL_RUNTIME_VERSION >= 18u + Y_UNIT_TEST_SUITE(TMiniKQLCommonJoinCoreWideTest) { Y_UNIT_TEST_LLVM(Inner) { TSetup<LLVM> setup; @@ -323,7 +323,7 @@ Y_UNIT_TEST_SUITE(TMiniKQLCommonJoinCoreWideTest) { UNIT_ASSERT(!iterator.Next(item)); } } -#endif + } } diff --git a/yql/essentials/minikql/comp_nodes/ut/mkql_map_join_ut.cpp b/yql/essentials/minikql/comp_nodes/ut/mkql_map_join_ut.cpp index e68d4bd46e..fe734804ea 100644 --- a/yql/essentials/minikql/comp_nodes/ut/mkql_map_join_ut.cpp +++ b/yql/essentials/minikql/comp_nodes/ut/mkql_map_join_ut.cpp @@ -585,7 +585,7 @@ Y_UNIT_TEST_SUITE(TMiniKQLMapJoinCoreTest) { } } } -#if !defined(MKQL_RUNTIME_VERSION) || MKQL_RUNTIME_VERSION >= 18u + Y_UNIT_TEST_SUITE(TMiniKQLWideMapJoinCoreTest) { Y_UNIT_TEST_LLVM(TestInner) { for (ui32 pass = 0; pass < 1; ++pass) { @@ -1144,7 +1144,7 @@ Y_UNIT_TEST_SUITE(TMiniKQLWideMapJoinCoreTest) { } } } -#endif + } } diff --git a/yql/essentials/minikql/comp_nodes/ut/mkql_multimap_ut.cpp b/yql/essentials/minikql/comp_nodes/ut/mkql_multimap_ut.cpp index 5dcb30db2e..d6735151f6 100644 --- a/yql/essentials/minikql/comp_nodes/ut/mkql_multimap_ut.cpp +++ b/yql/essentials/minikql/comp_nodes/ut/mkql_multimap_ut.cpp @@ -121,7 +121,7 @@ Y_UNIT_TEST_SUITE(TMiniKQLMultiMapTest) { UNIT_ASSERT(!iterator.Next(item)); UNIT_ASSERT(!iterator.Next(item)); } -#if !defined(MKQL_RUNTIME_VERSION) || MKQL_RUNTIME_VERSION >= 18u + Y_UNIT_TEST_LLVM(TestFlattenByNarrow) { TSetup<LLVM> setup; TProgramBuilder& pb = *setup.PgmBuilder; @@ -164,7 +164,6 @@ Y_UNIT_TEST_SUITE(TMiniKQLMultiMapTest) { UNIT_ASSERT(!iterator.Next(item)); UNIT_ASSERT(!iterator.Next(item)); } -#endif } } diff --git a/yql/essentials/minikql/comp_nodes/ut/mkql_wide_chopper_ut.cpp b/yql/essentials/minikql/comp_nodes/ut/mkql_wide_chopper_ut.cpp index 554dd8a3d3..95ce34a54c 100644 --- a/yql/essentials/minikql/comp_nodes/ut/mkql_wide_chopper_ut.cpp +++ b/yql/essentials/minikql/comp_nodes/ut/mkql_wide_chopper_ut.cpp @@ -3,7 +3,7 @@ namespace NKikimr { namespace NMiniKQL { -#if !defined(MKQL_RUNTIME_VERSION) || MKQL_RUNTIME_VERSION >= 18u + Y_UNIT_TEST_SUITE(TMiniKQLWideChopperTest) { Y_UNIT_TEST_LLVM(TestConcatKeyToItems) { TSetup<LLVM> setup; @@ -468,7 +468,7 @@ Y_UNIT_TEST_SUITE(TMiniKQLWideChopperTest) { UNIT_ASSERT(!iterator.Next(item)); } } -#endif + } } diff --git a/yql/essentials/minikql/comp_nodes/ut/mkql_wide_combine_ut.cpp b/yql/essentials/minikql/comp_nodes/ut/mkql_wide_combine_ut.cpp index 55cb6babbf..4e4d13163c 100644 --- a/yql/essentials/minikql/comp_nodes/ut/mkql_wide_combine_ut.cpp +++ b/yql/essentials/minikql/comp_nodes/ut/mkql_wide_combine_ut.cpp @@ -155,7 +155,6 @@ void CheckIfStreamHasExpectedStringValues(const NUdf::TUnboxedValue& streamValue } // unnamed -#if !defined(MKQL_RUNTIME_VERSION) || MKQL_RUNTIME_VERSION >= 18u Y_UNIT_TEST_SUITE(TMiniKQLWideCombinerTest) { Y_UNIT_TEST_LLVM(TestLongStringsRefCounting) { TSetup<LLVM> setup; @@ -1065,7 +1064,7 @@ Y_UNIT_TEST_SUITE(TMiniKQLWideCombinerPerfTest) { Cerr << "Runtime is " << t2 - t1 << " vs C++ " << cppTime << Endl; } } -#endif + #if !defined(MKQL_RUNTIME_VERSION) || MKQL_RUNTIME_VERSION >= 29u Y_UNIT_TEST_SUITE(TMiniKQLWideLastCombinerTest) { Y_UNIT_TEST_LLVM_SPILLING(TestLongStringsRefCounting) { diff --git a/yql/essentials/minikql/comp_nodes/ut/mkql_wide_condense_ut.cpp b/yql/essentials/minikql/comp_nodes/ut/mkql_wide_condense_ut.cpp index b231df462b..1313812732 100644 --- a/yql/essentials/minikql/comp_nodes/ut/mkql_wide_condense_ut.cpp +++ b/yql/essentials/minikql/comp_nodes/ut/mkql_wide_condense_ut.cpp @@ -3,7 +3,7 @@ namespace NKikimr { namespace NMiniKQL { -#if !defined(MKQL_RUNTIME_VERSION) || MKQL_RUNTIME_VERSION >= 18u + Y_UNIT_TEST_SUITE(TMiniKQLWideCondense1Test) { Y_UNIT_TEST_LLVM(TestConcatItemsToKey) { TSetup<LLVM> setup; @@ -167,7 +167,7 @@ Y_UNIT_TEST_SUITE(TMiniKQLWideCondense1Test) { UNIT_ASSERT(!iterator.Next(item)); } } -#endif + } } diff --git a/yql/essentials/minikql/comp_nodes/ut/mkql_wide_filter_ut.cpp b/yql/essentials/minikql/comp_nodes/ut/mkql_wide_filter_ut.cpp index 15c8516089..fd7b3f2f23 100644 --- a/yql/essentials/minikql/comp_nodes/ut/mkql_wide_filter_ut.cpp +++ b/yql/essentials/minikql/comp_nodes/ut/mkql_wide_filter_ut.cpp @@ -3,7 +3,7 @@ namespace NKikimr { namespace NMiniKQL { -#if !defined(MKQL_RUNTIME_VERSION) || MKQL_RUNTIME_VERSION >= 18u + Y_UNIT_TEST_SUITE(TMiniKQLWideFilterTest) { Y_UNIT_TEST_LLVM(TestPredicateExpression) { TSetup<LLVM> setup; @@ -381,6 +381,6 @@ Y_UNIT_TEST_SUITE(TMiniKQLWideFilterTest) { UNIT_ASSERT(!iterator.Next(item)); } } -#endif + } } diff --git a/yql/essentials/minikql/comp_nodes/ut/mkql_wide_map_ut.cpp b/yql/essentials/minikql/comp_nodes/ut/mkql_wide_map_ut.cpp index d1afec27f4..97c7d82d7b 100644 --- a/yql/essentials/minikql/comp_nodes/ut/mkql_wide_map_ut.cpp +++ b/yql/essentials/minikql/comp_nodes/ut/mkql_wide_map_ut.cpp @@ -3,7 +3,7 @@ namespace NKikimr { namespace NMiniKQL { -#if !defined(MKQL_RUNTIME_VERSION) || MKQL_RUNTIME_VERSION >= 18u + Y_UNIT_TEST_SUITE(TMiniKQLWideMapTest) { Y_UNIT_TEST_LLVM(TestSimpleSwap) { TSetup<LLVM> setup; @@ -330,6 +330,6 @@ Y_UNIT_TEST_SUITE(TMiniKQLWideMapTest) { UNIT_ASSERT(!iterator.Next(item)); } } -#endif + } } diff --git a/yql/essentials/minikql/comp_nodes/ut/mkql_wide_nodes_ut.cpp b/yql/essentials/minikql/comp_nodes/ut/mkql_wide_nodes_ut.cpp index 093810f0bc..1cbe7d774b 100644 --- a/yql/essentials/minikql/comp_nodes/ut/mkql_wide_nodes_ut.cpp +++ b/yql/essentials/minikql/comp_nodes/ut/mkql_wide_nodes_ut.cpp @@ -3,7 +3,7 @@ namespace NKikimr { namespace NMiniKQL { -#if !defined(MKQL_RUNTIME_VERSION) || MKQL_RUNTIME_VERSION >= 18u + Y_UNIT_TEST_SUITE(TMiniKQLWideNodesTest) { // TDOD: fixme #if 0 @@ -119,6 +119,6 @@ Y_UNIT_TEST_SUITE(TMiniKQLWideNodesTest) { } } -#endif + } } diff --git a/yql/essentials/minikql/computation/mkql_block_reader.cpp b/yql/essentials/minikql/computation/mkql_block_reader.cpp index 4e2060e739..5886e121c4 100644 --- a/yql/essentials/minikql/computation/mkql_block_reader.cpp +++ b/yql/essentials/minikql/computation/mkql_block_reader.cpp @@ -162,6 +162,19 @@ private: i32 TypeLen = 0; }; +class TSingularTypeItemConverter: public IBlockItemConverter { +public: + NUdf::TUnboxedValuePod MakeValue(TBlockItem item, const THolderFactory& holderFactory) const final { + Y_UNUSED(item, holderFactory); + return NUdf::TUnboxedValuePod::Zero(); + } + + TBlockItem MakeItem(const NUdf::TUnboxedValuePod& value) const final { + Y_UNUSED(value); + return TBlockItem::Zero(); + } +}; + template <bool Nullable> class TTupleBlockItemConverter : public IBlockItemConverter { public: @@ -285,6 +298,7 @@ struct TConverterTraits { using TExtOptional = TExternalOptionalBlockItemConverter; template<typename TTzDate, bool Nullable> using TTzDateConverter = TTzDateBlockItemConverter<TTzDate, Nullable>; + using TSingularType = TSingularTypeItemConverter; constexpr static bool PassType = false; @@ -325,6 +339,10 @@ struct TConverterTraits { return std::make_unique<TTzDateConverter<TTzDate, false>>(); } } + + static std::unique_ptr<TResult> MakeSingular() { + return std::make_unique<TSingularType>(); + } }; } // namespace diff --git a/yql/essentials/minikql/computation/mkql_block_transport.cpp b/yql/essentials/minikql/computation/mkql_block_transport.cpp index a03a5027e8..766df54889 100644 --- a/yql/essentials/minikql/computation/mkql_block_transport.cpp +++ b/yql/essentials/minikql/computation/mkql_block_transport.cpp @@ -13,7 +13,7 @@ namespace { using NYql::TChunkedBuffer; TChunkedBuffer MakeChunkedBufferAndUntrack(const std::shared_ptr<const arrow::Buffer>& owner, const char* data, size_t size) { - MKQLArrowUntrack(owner->data()); + MKQLArrowUntrack(owner->data(), owner->capacity()); return TChunkedBuffer(TStringBuf{data, size}, owner); } @@ -429,6 +429,49 @@ private: const std::unique_ptr<TBlockDeserializerBase> Inner_; }; +class TSingularTypeBlockSerializer final: public IBlockSerializer { +private: + size_t ArrayMetadataCount() const final { + return 0; + } + + void StoreMetadata(const arrow::ArrayData& data, const IBlockSerializer::TMetadataSink& metaSink) const final { + Y_UNUSED(data, metaSink); + } + + void StoreArray(const arrow::ArrayData& data, TChunkedBuffer& dst) const final { + Y_UNUSED(data, dst); + } +}; + +class TSingularTypeBlockDeserializer final: public TBlockDeserializerBase { +private: + void DoLoadMetadata(const TMetadataSource& metaSource) final { + Y_UNUSED(metaSource); + } + + std::shared_ptr<arrow::ArrayData> DoMakeDefaultValue(const std::shared_ptr<arrow::Buffer>& nulls, i64 nullsCount, ui64 blockLen, ui64 offset) const final { + Y_UNUSED(offset); + Y_ENSURE(nullsCount == 0); + Y_ENSURE(!nulls || nulls->size() == 0); + return arrow::NullArray(blockLen).data(); + } + + std::shared_ptr<arrow::ArrayData> DoLoadArray(TChunkedBuffer& src, const std::shared_ptr<arrow::Buffer>& nulls, i64 nullsCount, ui64 blockLen, ui64 offset) final { + Y_UNUSED(offset, src); + Y_ENSURE(nullsCount == 0); + Y_ENSURE(!nulls || nulls->size() == 0); + return arrow::NullArray(blockLen).data(); + } + + bool IsNullable() const final { + return false; + } + + void DoResetMetadata() final { + } +}; + template<bool Nullable, typename TDerived> class TTupleBlockSerializerBase : public IBlockSerializer { size_t ArrayMetadataCount() const final { @@ -632,7 +675,7 @@ struct TSerializerTraits { using TExtOptional = TExtOptionalBlockSerializer; template<typename TTzDateType, bool Nullable> using TTzDate = TTzDateBlockSerializer<TTzDateType, Nullable>; - + using TSingularType = TSingularTypeBlockSerializer; constexpr static bool PassType = false; static std::unique_ptr<TResult> MakePg(const NUdf::TPgTypeDescription& desc, const NUdf::IPgBuilder* pgBuilder) { @@ -648,6 +691,10 @@ struct TSerializerTraits { ythrow yexception() << "Serializer not implemented for block resources"; } + static std::unique_ptr<TResult> MakeSingular() { + return std::make_unique<TSingularType>(); + } + template<typename TTzDateType> static std::unique_ptr<TResult> MakeTzDate(bool isOptional) { if (isOptional) { @@ -670,6 +717,7 @@ struct TDeserializerTraits { using TExtOptional = TExtOptionalBlockDeserializer; template<typename TTzDateType, bool Nullable> using TTzDate = TTzDateBlockDeserializer<TTzDateType, Nullable>; + using TSingularType = TSingularTypeBlockDeserializer; constexpr static bool PassType = false; @@ -686,6 +734,10 @@ struct TDeserializerTraits { ythrow yexception() << "Deserializer not implemented for block resources"; } + static std::unique_ptr<TResult> MakeSingular() { + return std::make_unique<TSingularType>(); + } + template<typename TTzDateType> static std::unique_ptr<TResult> MakeTzDate(bool isOptional) { if (isOptional) { diff --git a/yql/essentials/minikql/computation/mkql_block_trimmer.cpp b/yql/essentials/minikql/computation/mkql_block_trimmer.cpp index b53a3890a4..0b53f91452 100644 --- a/yql/essentials/minikql/computation/mkql_block_trimmer.cpp +++ b/yql/essentials/minikql/computation/mkql_block_trimmer.cpp @@ -98,6 +98,17 @@ public: } }; +class TSingularBlockTrimmer: public TBlockTrimmerBase { +public: + TSingularBlockTrimmer(arrow::MemoryPool* pool) + : TBlockTrimmerBase(pool) { + } + + std::shared_ptr<arrow::ArrayData> Trim(const std::shared_ptr<arrow::ArrayData>& array) override { + return array; + } +}; + template<typename TStringType, bool Nullable> class TStringBlockTrimmer : public TBlockTrimmerBase { using TOffset = typename TStringType::offset_type; @@ -217,6 +228,7 @@ struct TTrimmerTraits { using TResource = TResourceBlockTrimmer<Nullable>; template<typename TTzDate, bool Nullable> using TTzDateReader = TTzDateBlockTrimmer<TTzDate, Nullable>; + using TSingular = TSingularBlockTrimmer; constexpr static bool PassType = false; @@ -237,6 +249,10 @@ struct TTrimmerTraits { } } + static TResult::TPtr MakeSingular(arrow::MemoryPool* pool) { + return std::make_unique<TSingular>(pool); + } + template<typename TTzDate> static TResult::TPtr MakeTzDate(bool isOptional, arrow::MemoryPool* pool) { if (isOptional) { diff --git a/yql/essentials/minikql/computation/mkql_computation_node_pack_ut.cpp b/yql/essentials/minikql/computation/mkql_computation_node_pack_ut.cpp index b689e4cf8b..cbff1c5722 100644 --- a/yql/essentials/minikql/computation/mkql_computation_node_pack_ut.cpp +++ b/yql/essentials/minikql/computation/mkql_computation_node_pack_ut.cpp @@ -674,6 +674,8 @@ protected: auto tzDateType = PgmBuilder.NewDataType(NUdf::EDataSlot::TzDate); auto blockTzDateType = PgmBuilder.NewBlockType(tzDateType, TBlockType::EShape::Many); + auto nullType = PgmBuilder.NewNullType(); + auto blockNullType = PgmBuilder.NewBlockType(nullType, TBlockType::EShape::Many); auto rowType = legacyStruct @@ -683,11 +685,12 @@ protected: {"_yql_block_length", scalarUi64Type}, {"a", scalarOptStrType}, {"b", blockOptTupleOptUi32StrType}, - {"c", blockTzDateType} + {"c", blockTzDateType}, + {"nill", blockNullType}, }) : PgmBuilder.NewMultiType( {blockUi32Type, blockOptStrType, scalarOptStrType, - blockOptTupleOptUi32StrType, blockTzDateType, scalarUi64Type}); + blockOptTupleOptUi32StrType, blockTzDateType, blockNullType, scalarUi64Type}); ui64 blockLen = 1000; UNIT_ASSERT_LE(offset + len, blockLen); @@ -696,6 +699,8 @@ protected: auto builder2 = MakeArrayBuilder(TTypeInfoHelper(), optStrType, *ArrowPool_, CalcBlockLen(CalcMaxBlockItemSize(optStrType)), nullptr); auto builder3 = MakeArrayBuilder(TTypeInfoHelper(), optTupleOptUi32StrType, *ArrowPool_, CalcBlockLen(CalcMaxBlockItemSize(optTupleOptUi32StrType)), nullptr); auto builder4 = MakeArrayBuilder(TTypeInfoHelper(), tzDateType, *ArrowPool_, CalcBlockLen(CalcMaxBlockItemSize(tzDateType)), nullptr); + auto builder5 = MakeArrayBuilder(TTypeInfoHelper(), nullType, *ArrowPool_, CalcBlockLen(CalcMaxBlockItemSize(nullType)), nullptr); + for (ui32 i = 0; i < blockLen; ++i) { TBlockItem b1(i); @@ -712,6 +717,7 @@ protected: TBlockItem tzDate {i}; tzDate.SetTimezoneId(i % 100); builder4->Add(tzDate); + builder5->Add(TBlockItem::Zero()); } std::string_view testScalarString = "foobar"; @@ -725,12 +731,14 @@ protected: datums.emplace_back(arrow::Datum(std::make_shared<arrow::BinaryScalar>(strbuf))); datums.emplace_back(builder3->Build(true)); datums.emplace_back(builder4->Build(true)); + datums.emplace_back(builder5->Build(true)); } else { datums.emplace_back(builder1->Build(true)); datums.emplace_back(builder2->Build(true)); datums.emplace_back(arrow::Datum(std::make_shared<arrow::BinaryScalar>(strbuf))); datums.emplace_back(builder3->Build(true)); datums.emplace_back(builder4->Build(true)); + datums.emplace_back(builder5->Build(true)); datums.emplace_back(arrow::Datum(std::make_shared<arrow::UInt64Scalar>(blockLen))); } @@ -785,6 +793,7 @@ protected: auto reader2 = MakeBlockReader(TTypeInfoHelper(), optStrType); auto reader3 = MakeBlockReader(TTypeInfoHelper(), optTupleOptUi32StrType); auto reader4 = MakeBlockReader(TTypeInfoHelper(), tzDateType); + auto reader5 = MakeBlockReader(TTypeInfoHelper(), nullType); for (ui32 i = offset; i < len; ++i) { TBlockItem b1 = reader1->GetItem(*TArrowBlock::From(unpackedColumns[0]).GetDatum().array(), i - offset); @@ -814,6 +823,8 @@ protected: TBlockItem b4 = reader4->GetItem(*TArrowBlock::From(unpackedColumns[legacyStruct ? 5 : 4]).GetDatum().array(), i - offset); UNIT_ASSERT(b4.Get<ui16>() == i); UNIT_ASSERT(b4.GetTimezoneId() == (i % 100)); + TBlockItem b5 = reader5->GetItem(*TArrowBlock::From(unpackedColumns[legacyStruct ? 6 : 5]).GetDatum().array(), i - offset); + UNIT_ASSERT(b5); } } } diff --git a/yql/essentials/minikql/mkql_alloc.cpp b/yql/essentials/minikql/mkql_alloc.cpp index 963f46a67e..8446522eda 100644 --- a/yql/essentials/minikql/mkql_alloc.cpp +++ b/yql/essentials/minikql/mkql_alloc.cpp @@ -7,6 +7,8 @@ namespace NKikimr { namespace NMiniKQL { +constexpr ui64 ArrowSizeForArena = (TAllocState::POOL_PAGE_SIZE >> 2); + Y_POD_THREAD(TAllocState*) TlsAllocState; TAllocPageHeader TAllocState::EmptyPageHeader = { 0, 0, 0, 0, nullptr, nullptr }; @@ -94,6 +96,10 @@ void TAllocState::KillAllBoxed() { OffloadedBlocksRoot.InitLinks(); } + if (CurrentArrowPages) { + MKQLArrowFree(CurrentArrowPages, 0); + CurrentArrowPages = nullptr; + } CleanupArrowList(&ArrowBlocksRoot); #ifndef NDEBUG @@ -253,7 +259,49 @@ void TPagedArena::Clear() noexcept { } } +void* MKQLArrowAllocateOnArena(ui64 size) { + TAllocState* state = TlsAllocState; + Y_ENSURE(state); + + auto alignedSize = AlignUp(size, ArrowAlignment); + auto& page = state->CurrentArrowPages; + + if (Y_UNLIKELY(!page || page->Offset + alignedSize > page->Size)) { + const auto pageSize = TAllocState::POOL_PAGE_SIZE; + + if (state->EnableArrowTracking) { + state->OffloadAlloc(pageSize); + } + + if (page) { + MKQLArrowFree(page, 0); + } + + page = (TMkqlArrowHeader*)GetAlignedPage(); + page->Offset = sizeof(TMkqlArrowHeader); + page->Size = pageSize; + page->UseCount = 1; + + if (state->EnableArrowTracking) { + page->Entry.Link(&state->ArrowBlocksRoot); + Y_ENSURE(state->ArrowBuffers.insert(page).second); + } else { + page->Entry.Clear(); + } + } + + void* ptr = (ui8*)page + page->Offset; + page->Offset += alignedSize; + ++page->UseCount; + + return ptr; +} + void* MKQLArrowAllocate(ui64 size) { + if (size <= ArrowSizeForArena) { + return MKQLArrowAllocateOnArena(size); + } + TAllocState* state = TlsAllocState; Y_ENSURE(state); auto fullSize = size + sizeof(TMkqlArrowHeader); @@ -276,6 +324,9 @@ void* MKQLArrowAllocate(ui64 size) { #endif auto* header = (TMkqlArrowHeader*)ptr; + header->Offset = 0; + header->UseCount = 0; + if (state->EnableArrowTracking) { header->Entry.Link(&state->ArrowBlocksRoot); Y_ENSURE(state->ArrowBuffers.insert(header + 1).second); @@ -294,7 +345,31 @@ void* MKQLArrowReallocate(const void* mem, ui64 prevSize, ui64 size) { return res; } +void MKQLArrowFreeOnArena(const void* ptr) { + auto* page = (TMkqlArrowHeader*)TAllocState::GetPageStart(ptr); + if (page->UseCount.fetch_sub(1) == 1) { + if (!page->Entry.IsUnlinked()) { + TAllocState* state = TlsAllocState; + Y_ENSURE(state); + state->OffloadFree(page->Size); + page->Entry.Unlink(); + + auto it = state->ArrowBuffers.find(page); + Y_ENSURE(it != state->ArrowBuffers.end()); + state->ArrowBuffers.erase(it); + } + + ReleaseAlignedPage(page); + } + + return; +} + void MKQLArrowFree(const void* mem, ui64 size) { + if (size <= ArrowSizeForArena) { + return MKQLArrowFreeOnArena(mem); + } + auto fullSize = size + sizeof(TMkqlArrowHeader); auto header = ((TMkqlArrowHeader*)mem) - 1; if (!header->Entry.IsUnlinked()) { @@ -318,19 +393,37 @@ void MKQLArrowFree(const void* mem, ui64 size) { ReleaseAlignedPage(header, fullSize); } -void MKQLArrowUntrack(const void* mem) { +void MKQLArrowUntrack(const void* mem, ui64 size) { TAllocState* state = TlsAllocState; Y_ENSURE(state); if (!state->EnableArrowTracking) { return; } + if (size <= ArrowSizeForArena) { + auto* page = (TMkqlArrowHeader*)TAllocState::GetPageStart(mem); + + auto it = state->ArrowBuffers.find(page); + if (it == state->ArrowBuffers.end()) { + return; + } + + if (!page->Entry.IsUnlinked()) { + page->Entry.Unlink(); // unlink page immediately so we don't accidentally free untracked memory within `TAllocState` + state->ArrowBuffers.erase(it); + state->OffloadFree(page->Size); + } + + return; + } + auto it = state->ArrowBuffers.find(mem); if (it == state->ArrowBuffers.end()) { return; } - auto header = ((TMkqlArrowHeader*)mem) - 1; + auto* header = ((TMkqlArrowHeader*)mem) - 1; + Y_ENSURE(header->UseCount == 0); if (!header->Entry.IsUnlinked()) { header->Entry.Unlink(); auto fullSize = header->Size + sizeof(TMkqlArrowHeader); diff --git a/yql/essentials/minikql/mkql_alloc.h b/yql/essentials/minikql/mkql_alloc.h index abcb6cc73d..24bbbb8e9e 100644 --- a/yql/essentials/minikql/mkql_alloc.h +++ b/yql/essentials/minikql/mkql_alloc.h @@ -41,6 +41,8 @@ constexpr ui32 MaxPageUserData = TAlignedPagePool::POOL_PAGE_SIZE - sizeof(TAllo static_assert(sizeof(TAllocPageHeader) % MKQL_ALIGNMENT == 0, "Incorrect size of header"); +struct TMkqlArrowHeader; + struct TAllocState : public TAlignedPagePool { struct TListEntry { @@ -90,6 +92,7 @@ struct TAllocState : public TAlignedPagePool TListEntry GlobalPAllocList; TListEntry* CurrentPAllocList; TListEntry ArrowBlocksRoot; + TMkqlArrowHeader* CurrentArrowPages = nullptr; // page arena for small arrow allocations std::unordered_set<const void*> ArrowBuffers; bool EnableArrowTracking = true; @@ -186,7 +189,9 @@ constexpr size_t ArrowAlignment = 64; struct TMkqlArrowHeader { TAllocState::TListEntry Entry; ui64 Size; - char Padding[ArrowAlignment - sizeof(TAllocState::TListEntry) - sizeof(ui64)]; + ui64 Offset; + std::atomic<ui64> UseCount; + char Padding[ArrowAlignment - sizeof(TAllocState::TListEntry) - sizeof(ui64) - sizeof(ui64) - sizeof(std::atomic<ui64>)]; }; static_assert(sizeof(TMkqlArrowHeader) == ArrowAlignment); @@ -441,7 +446,7 @@ inline void MKQLUnregisterObject(NUdf::TBoxedValue* value) noexcept { void* MKQLArrowAllocate(ui64 size); void* MKQLArrowReallocate(const void* mem, ui64 prevSize, ui64 size); void MKQLArrowFree(const void* mem, ui64 size); -void MKQLArrowUntrack(const void* mem); +void MKQLArrowUntrack(const void* mem, ui64 size); template <const EMemorySubPool MemoryPoolExt = EMemorySubPool::Default> struct TWithMiniKQLAlloc { diff --git a/yql/essentials/minikql/mkql_program_builder.cpp b/yql/essentials/minikql/mkql_program_builder.cpp index b139c24058..e03a5e3b28 100644 --- a/yql/essentials/minikql/mkql_program_builder.cpp +++ b/yql/essentials/minikql/mkql_program_builder.cpp @@ -20,6 +20,8 @@ using namespace std::string_view_literals; namespace NKikimr { namespace NMiniKQL { +static_assert(RuntimeVersion >= 20); + namespace { struct TDataFunctionFlags { @@ -403,10 +405,6 @@ TRuntimeNode TProgramBuilder::Arg(TType* type) const { } TRuntimeNode TProgramBuilder::WideFlowArg(TType* type) const { - if constexpr (RuntimeVersion < 18U) { - THROW yexception() << "Runtime version (" << RuntimeVersion << ") too old for " << __func__; - } - TCallableBuilder builder(Env, __func__, type, true); return TRuntimeNode(builder.Build(), false); } @@ -973,10 +971,6 @@ TRuntimeNode TProgramBuilder::ToList(TRuntimeNode optional) { } TRuntimeNode TProgramBuilder::Iterable(TZeroLambda lambda) { - if constexpr (RuntimeVersion < 19U) { - THROW yexception() << "Runtime version (" << RuntimeVersion << ") too old for " << __func__; - } - const auto itemArg = Arg(NewNull().GetStaticType()); auto lambdaRes = lambda(); const auto resultType = NewListType(AS_TYPE(TStreamType, lambdaRes.GetStaticType())->GetItemType()); @@ -1397,9 +1391,6 @@ TRuntimeNode TProgramBuilder::Iterator(TRuntimeNode list, const TArrayRef<const TRuntimeNode TProgramBuilder::EmptyIterator(TType* streamType) { MKQL_ENSURE(streamType->IsStream() || streamType->IsFlow(), "Expected stream or flow."); - if (RuntimeVersion < 7U && streamType->IsFlow()) { - return ToFlow(EmptyIterator(NewStreamType(AS_TYPE(TFlowType, streamType)->GetItemType()))); - } TCallableBuilder callableBuilder(Env, __func__, streamType); return TRuntimeNode(callableBuilder.Build(), false); } @@ -1435,11 +1426,6 @@ TRuntimeNode TProgramBuilder::LazyList(TRuntimeNode list) { TRuntimeNode TProgramBuilder::ForwardList(TRuntimeNode stream) { const auto type = stream.GetStaticType(); MKQL_ENSURE(type->IsStream() || type->IsFlow(), "Expected flow or stream."); - if constexpr (RuntimeVersion < 10U) { - if (type->IsFlow()) { - return ForwardList(FromFlow(stream)); - } - } TCallableBuilder callableBuilder(Env, __func__, NewListType(type->IsFlow() ? AS_TYPE(TFlowType, stream)->GetItemType() : AS_TYPE(TStreamType, stream)->GetItemType())); callableBuilder.Add(stream); return TRuntimeNode(callableBuilder.Build(), false); @@ -2054,29 +2040,7 @@ TRuntimeNode TProgramBuilder::Lookup(TRuntimeNode dict, TRuntimeNode key) { return TRuntimeNode(callableBuilder.Build(), false); } -TRuntimeNode TProgramBuilder::DictItems(TRuntimeNode dict, EDictItems mode) { - const auto dictTypeChecked = AS_TYPE(TDictType, dict.GetStaticType()); - TType* itemType; - switch (mode) { - case EDictItems::Both: { - const std::array<TType*, 2U> tupleTypes = {{ dictTypeChecked->GetKeyType(), dictTypeChecked->GetPayloadType() }}; - itemType = NewTupleType(tupleTypes); - break; - } - case EDictItems::Keys: itemType = dictTypeChecked->GetKeyType(); break; - case EDictItems::Payloads: itemType = dictTypeChecked->GetPayloadType(); break; - } - - TCallableBuilder callableBuilder(Env, __func__, NewListType(itemType)); - callableBuilder.Add(dict); - callableBuilder.Add(NewDataLiteral((ui32)mode)); - return TRuntimeNode(callableBuilder.Build(), false); -} - TRuntimeNode TProgramBuilder::DictItems(TRuntimeNode dict) { - if constexpr (RuntimeVersion < 6U) { - return DictItems(dict, EDictItems::Both); - } const auto dictTypeChecked = AS_TYPE(TDictType, dict.GetStaticType()); const auto itemType = NewTupleType({ dictTypeChecked->GetKeyType(), dictTypeChecked->GetPayloadType() }); TCallableBuilder callableBuilder(Env, __func__, NewListType(itemType)); @@ -2085,9 +2049,6 @@ TRuntimeNode TProgramBuilder::DictItems(TRuntimeNode dict) { } TRuntimeNode TProgramBuilder::DictKeys(TRuntimeNode dict) { - if constexpr (RuntimeVersion < 6U) { - return DictItems(dict, EDictItems::Keys); - } const auto dictTypeChecked = AS_TYPE(TDictType, dict.GetStaticType()); TCallableBuilder callableBuilder(Env, __func__, NewListType(dictTypeChecked->GetKeyType())); callableBuilder.Add(dict); @@ -2095,9 +2056,6 @@ TRuntimeNode TProgramBuilder::DictKeys(TRuntimeNode dict) { } TRuntimeNode TProgramBuilder::DictPayloads(TRuntimeNode dict) { - if constexpr (RuntimeVersion < 6U) { - return DictItems(dict, EDictItems::Payloads); - } const auto dictTypeChecked = AS_TYPE(TDictType, dict.GetStaticType()); TCallableBuilder callableBuilder(Env, __func__, NewListType(dictTypeChecked->GetPayloadType())); callableBuilder.Add(dict); @@ -3019,10 +2977,6 @@ TRuntimeNode TProgramBuilder::SourceOf(TType* returnType) { } TRuntimeNode TProgramBuilder::Source() { - if constexpr (RuntimeVersion < 18U) { - THROW yexception() << "Runtime version (" << RuntimeVersion << ") too old for " << __func__; - } - TCallableBuilder callableBuilder(Env, __func__, NewFlowType(NewMultiType({}))); return TRuntimeNode(callableBuilder.Build(), false); } @@ -3197,9 +3151,7 @@ TRuntimeNode TProgramBuilder::QueueCreate(TRuntimeNode initCapacity, TRuntimeNod auto resType = AS_TYPE(TResourceType, returnType); const auto tag = resType->GetTag(); - if (initCapacity.GetStaticType()->IsVoid()) { - MKQL_ENSURE(RuntimeVersion >= 13, "Unbounded queue is not supported in runtime version " << RuntimeVersion); - } else { + if (!initCapacity.GetStaticType()->IsVoid()) { auto initCapacityType = AS_TYPE(TDataType, initCapacity); MKQL_ENSURE(initCapacityType->GetSchemeType() == NUdf::TDataType<ui64>::Id, "init capcity must be ui64"); } @@ -3253,8 +3205,6 @@ TRuntimeNode TProgramBuilder::QueuePeek(TRuntimeNode resource, TRuntimeNode inde } TRuntimeNode TProgramBuilder::QueueRange(TRuntimeNode resource, TRuntimeNode begin, TRuntimeNode end, const TArrayRef<const TRuntimeNode>& dependentNodes, TType* returnType) { - MKQL_ENSURE(RuntimeVersion >= 14, "QueueRange is not supported in runtime version " << RuntimeVersion); - MKQL_ENSURE(returnType->IsList(), "Expected list type as result of QueueRange"); auto resType = AS_TYPE(TResourceType, resource); @@ -3291,8 +3241,6 @@ TRuntimeNode TProgramBuilder::PreserveStream(TRuntimeNode stream, TRuntimeNode q } TRuntimeNode TProgramBuilder::Seq(const TArrayRef<const TRuntimeNode>& args, TType* returnType) { - MKQL_ENSURE(RuntimeVersion >= 15, "Seq is not supported in runtime version " << RuntimeVersion); - TCallableBuilder callableBuilder(Env, __func__, returnType); for (auto node : args) { callableBuilder.Add(node); @@ -3738,16 +3686,6 @@ TRuntimeNode TProgramBuilder::BuildFlatMap(const std::string_view& callableName, TRuntimeNode TProgramBuilder::MultiMap(TRuntimeNode list, const TExpandLambda& handler) { - if constexpr (RuntimeVersion < 16U) { - const auto single = [=](TRuntimeNode item) -> TRuntimeNode { - const auto newList = handler(item); - const auto retItemType = newList.front().GetStaticType(); - MKQL_ENSURE(retItemType->IsSameType(*newList.back().GetStaticType()), "Must be same type."); - return NewList(retItemType, newList); - }; - return OrderedFlatMap(list, single); - } - const auto listType = list.GetStaticType(); MKQL_ENSURE(listType->IsFlow() || listType->IsList(), "Expected flow, list, stream or optional"); @@ -3770,10 +3708,6 @@ TRuntimeNode TProgramBuilder::MultiMap(TRuntimeNode list, const TExpandLambda& h } TRuntimeNode TProgramBuilder::NarrowMultiMap(TRuntimeNode flow, const TWideLambda& handler) { - if constexpr (RuntimeVersion < 18U) { - THROW yexception() << "Runtime version (" << RuntimeVersion << ") too old for " << __func__; - } - const auto wideComponents = GetWideComponents(AS_TYPE(TFlowType, flow.GetStaticType())); TRuntimeNode::TList itemArgs; @@ -3795,10 +3729,6 @@ TRuntimeNode TProgramBuilder::NarrowMultiMap(TRuntimeNode flow, const TWideLambd } TRuntimeNode TProgramBuilder::ExpandMap(TRuntimeNode flow, const TExpandLambda& handler) { - if constexpr (RuntimeVersion < 18U) { - THROW yexception() << "Runtime version (" << RuntimeVersion << ") too old for " << __func__; - } - const auto itemType = AS_TYPE(TFlowType, flow.GetStaticType())->GetItemType(); const auto itemArg = Arg(itemType); const auto newItems = handler(itemArg); @@ -3815,10 +3745,6 @@ TRuntimeNode TProgramBuilder::ExpandMap(TRuntimeNode flow, const TExpandLambda& } TRuntimeNode TProgramBuilder::WideMap(TRuntimeNode flow, const TWideLambda& handler) { - if constexpr (RuntimeVersion < 18U) { - THROW yexception() << "Runtime version (" << RuntimeVersion << ") too old for " << __func__; - } - const auto wideComponents = GetWideComponents(AS_TYPE(TFlowType, flow.GetStaticType())); TRuntimeNode::TList itemArgs; @@ -3875,10 +3801,6 @@ TRuntimeNode TProgramBuilder::WideChain1Map(TRuntimeNode flow, const TWideLambda } TRuntimeNode TProgramBuilder::NarrowMap(TRuntimeNode flow, const TNarrowLambda& handler) { - if constexpr (RuntimeVersion < 18U) { - THROW yexception() << "Runtime version (" << RuntimeVersion << ") too old for " << __func__; - } - const auto wideComponents = GetWideComponents(AS_TYPE(TFlowType, flow.GetStaticType())); TRuntimeNode::TList itemArgs; @@ -3896,10 +3818,6 @@ TRuntimeNode TProgramBuilder::NarrowMap(TRuntimeNode flow, const TNarrowLambda& } TRuntimeNode TProgramBuilder::NarrowFlatMap(TRuntimeNode flow, const TNarrowLambda& handler) { - if constexpr (RuntimeVersion < 18U) { - THROW yexception() << "Runtime version (" << RuntimeVersion << ") too old for " << __func__; - } - const auto wideComponents = GetWideComponents(AS_TYPE(TFlowType, flow.GetStaticType())); TRuntimeNode::TList itemArgs; @@ -3931,10 +3849,6 @@ TRuntimeNode TProgramBuilder::NarrowFlatMap(TRuntimeNode flow, const TNarrowLamb } TRuntimeNode TProgramBuilder::BuildWideFilter(const std::string_view& callableName, TRuntimeNode flow, const TNarrowLambda& handler) { - if constexpr (RuntimeVersion < 18U) { - THROW yexception() << "Runtime version (" << RuntimeVersion << ") too old for " << __func__; - } - const auto wideComponents = GetWideComponents(AS_TYPE(TFlowType, flow.GetStaticType())); TRuntimeNode::TList itemArgs; @@ -4019,10 +3933,6 @@ TRuntimeNode TProgramBuilder::BuildFilter(const std::string_view& callableName, TRuntimeNode TProgramBuilder::BuildFilter(const std::string_view& callableName, TRuntimeNode list, TRuntimeNode limit, const TUnaryLambda& handler, TType* resultType) { - if constexpr (RuntimeVersion < 4U) { - return Take(BuildFilter(callableName, list, handler, resultType), limit); - } - const auto listType = list.GetStaticType(); MKQL_ENSURE(listType->IsFlow() || listType->IsList() || listType->IsStream(), "Expected flow, list or stream."); MKQL_ENSURE(limit.GetStaticType()->IsData(), "Expected data"); @@ -4322,15 +4232,12 @@ TRuntimeNode TProgramBuilder::Apply(TRuntimeNode callableNode, const TArrayRef<c << " with static " << arg.GetStaticType()->GetKindAsStr()); } - TCallableBuilder callableBuilder(Env, RuntimeVersion >= 8 ? "Apply2" : "Apply", callableType->GetReturnType()); + TCallableBuilder callableBuilder(Env, "Apply2", callableType->GetReturnType()); callableBuilder.Add(callableNode); callableBuilder.Add(NewDataLiteral<ui32>(dependentCount)); - - if constexpr (RuntimeVersion >= 8) { - callableBuilder.Add(NewDataLiteral<NUdf::EDataSlot::String>(file)); - callableBuilder.Add(NewDataLiteral(row)); - callableBuilder.Add(NewDataLiteral(column)); - } + callableBuilder.Add(NewDataLiteral<NUdf::EDataSlot::String>(file)); + callableBuilder.Add(NewDataLiteral(row)); + callableBuilder.Add(NewDataLiteral(column)); for (const auto& arg: args) { callableBuilder.Add(arg); @@ -4365,12 +4272,11 @@ TRuntimeNode TProgramBuilder::Callable(TType* callableType, const TArrayLambda& } TRuntimeNode TProgramBuilder::NewNull() { - if (!UseNullType || RuntimeVersion < 11) { - TCallableBuilder callableBuilder(Env, "Null", NewOptionalType(Env.GetVoidLazy()->GetType())); - return TRuntimeNode(callableBuilder.Build(), false); - } else { + if (UseNullType) { return TRuntimeNode(Env.GetNullLazy(), true); } + TCallableBuilder callableBuilder(Env, "Null", NewOptionalType(Env.GetVoidLazy()->GetType())); + return TRuntimeNode(callableBuilder.Build(), false); } TRuntimeNode TProgramBuilder::Concat(TRuntimeNode data1, TRuntimeNode data2) { @@ -4403,18 +4309,10 @@ TRuntimeNode TProgramBuilder::RFind(TRuntimeNode haystack, TRuntimeNode needle, } TRuntimeNode TProgramBuilder::StartsWith(TRuntimeNode string, TRuntimeNode prefix) { - if constexpr (RuntimeVersion < 19U) { - THROW yexception() << "Runtime version (" << RuntimeVersion << ") too old for " << __func__; - } - return DataCompare(__func__, string, prefix); } TRuntimeNode TProgramBuilder::EndsWith(TRuntimeNode string, TRuntimeNode suffix) { - if constexpr (RuntimeVersion < 19U) { - THROW yexception() << "Runtime version (" << RuntimeVersion << ") too old for " << __func__; - } - return DataCompare(__func__, string, suffix); } @@ -4746,10 +4644,6 @@ TRuntimeNode TProgramBuilder::CommonJoinCore(TRuntimeNode flow, EJoinKind joinKi ui64 memLimit, std::optional<ui32> sortedTableOrder, EAnyJoinSettings anyJoinSettings, const ui32 tableIndexField, TType* returnType) { - if constexpr (RuntimeVersion < 17U) { - THROW yexception() << "Runtime version (" << RuntimeVersion << ") too old for " << __func__; - } - MKQL_ENSURE(leftColumns.size() % 2U == 0U, "Expected even count"); MKQL_ENSURE(rightColumns.size() % 2U == 0U, "Expected even count"); @@ -4796,10 +4690,6 @@ TRuntimeNode TProgramBuilder::CommonJoinCore(TRuntimeNode flow, EJoinKind joinKi } TRuntimeNode TProgramBuilder::WideCombiner(TRuntimeNode flow, i64 memLimit, const TWideLambda& extractor, const TBinaryWideLambda& init, const TTernaryWideLambda& update, const TBinaryWideLambda& finish) { - if constexpr (RuntimeVersion < 18U) { - THROW yexception() << "Runtime version (" << RuntimeVersion << ") too old for " << __func__; - } - if (memLimit < 0) { if constexpr (RuntimeVersion < 46U) { THROW yexception() << "Runtime version (" << RuntimeVersion << ") too old for " << __func__ << " with limit " << memLimit; @@ -4934,10 +4824,6 @@ TRuntimeNode TProgramBuilder::WideLastCombinerWithSpilling(TRuntimeNode flow, co } TRuntimeNode TProgramBuilder::WideCondense1(TRuntimeNode flow, const TWideLambda& init, const TWideSwitchLambda& switcher, const TBinaryWideLambda& update, bool useCtx) { - if constexpr (RuntimeVersion < 18U) { - THROW yexception() << "Runtime version (" << RuntimeVersion << ") too old for " << __func__; - } - const auto wideComponents = GetWideComponents(AS_TYPE(TFlowType, flow.GetStaticType())); TRuntimeNode::TList itemArgs; @@ -4983,10 +4869,6 @@ TRuntimeNode TProgramBuilder::CombineCore(TRuntimeNode stream, const TBinaryLambda& finish, ui64 memLimit) { - if constexpr (RuntimeVersion < 3U) { - THROW yexception() << "Runtime version (" << RuntimeVersion << ") too old for " << __func__; - } - const bool isStream = stream.GetStaticType()->IsStream(); const auto itemType = isStream ? AS_TYPE(TStreamType, stream)->GetItemType() : AS_TYPE(TFlowType, stream)->GetItemType(); @@ -5037,10 +4919,6 @@ TRuntimeNode TProgramBuilder::GroupingCore(TRuntimeNode stream, const TUnaryLambda& keyExtractor, const TUnaryLambda& handler) { - if (handler && RuntimeVersion < 20U) { - THROW yexception() << "Runtime version (" << RuntimeVersion << ") too old for " << __func__ << " with handler"; - } - auto itemType = AS_TYPE(TStreamType, stream)->GetItemType(); TRuntimeNode keyExtractorItemArg = Arg(itemType); @@ -5084,13 +4962,6 @@ TRuntimeNode TProgramBuilder::Chopper(TRuntimeNode flow, const TUnaryLambda& key const auto flowType = flow.GetStaticType(); MKQL_ENSURE(flowType->IsFlow() || flowType->IsStream(), "Expected flow or stream."); - - if constexpr (RuntimeVersion < 9U) { - return FlatMap(GroupingCore(flow, groupSwitch, keyExtractor), - [&](TRuntimeNode item) -> TRuntimeNode { return groupHandler(Nth(item, 0U), Nth(item, 1U)); } - ); - } - const bool isStream = flowType->IsStream(); const auto itemType = isStream ? AS_TYPE(TStreamType, flow)->GetItemType() : AS_TYPE(TFlowType, flow)->GetItemType(); @@ -5119,10 +4990,6 @@ TRuntimeNode TProgramBuilder::Chopper(TRuntimeNode flow, const TUnaryLambda& key TRuntimeNode TProgramBuilder::WideChopper(TRuntimeNode flow, const TWideLambda& extractor, const TWideSwitchLambda& groupSwitch, const std::function<TRuntimeNode (TRuntimeNode::TList, TRuntimeNode)>& groupHandler ) { - if constexpr (RuntimeVersion < 18U) { - THROW yexception() << "Runtime version (" << RuntimeVersion << ") too old for " << __func__; - } - const auto wideComponents = GetWideComponents(AS_TYPE(TFlowType, flow.GetStaticType())); TRuntimeNode::TList itemArgs, keyArgs; @@ -5564,11 +5431,9 @@ TRuntimeNode TProgramBuilder::Replicate(TRuntimeNode item, TRuntimeNode count, c TCallableBuilder callableBuilder(Env, __func__, listType); callableBuilder.Add(item); callableBuilder.Add(count); - if constexpr (RuntimeVersion >= 2) { - callableBuilder.Add(NewDataLiteral<NUdf::EDataSlot::String>(file)); - callableBuilder.Add(NewDataLiteral(row)); - callableBuilder.Add(NewDataLiteral(column)); - } + callableBuilder.Add(NewDataLiteral<NUdf::EDataSlot::String>(file)); + callableBuilder.Add(NewDataLiteral(row)); + callableBuilder.Add(NewDataLiteral(column)); return TRuntimeNode(callableBuilder.Build(), false); } diff --git a/yql/essentials/minikql/mkql_type_builder.cpp b/yql/essentials/minikql/mkql_type_builder.cpp index d1df31a97d..231231eee7 100644 --- a/yql/essentials/minikql/mkql_type_builder.cpp +++ b/yql/essentials/minikql/mkql_type_builder.cpp @@ -1522,6 +1522,17 @@ bool ConvertArrowTypeImpl(NUdf::EDataSlot slot, std::shared_ptr<arrow::DataType> } } +inline bool IsSingularType(const TType* type) { + return type->IsNull() || + type->IsVoid() || + type->IsEmptyDict() || + type->IsEmptyList(); +} + +inline bool NeedWrapWithExternalOptional(const TType* type) { + return type->IsPg() || IsSingularType(type); +} + bool ConvertArrowTypeImpl(TType* itemType, std::shared_ptr<arrow::DataType>& type, const TArrowConvertFailedCallback& onFail, bool output) { bool isOptional; auto unpacked = UnpackOptional(itemType, isOptional); @@ -1534,8 +1545,7 @@ bool ConvertArrowTypeImpl(TType* itemType, std::shared_ptr<arrow::DataType>& typ return false; } - if (unpacked->IsOptional() || isOptional && unpacked->IsPg()) { - // at least 2 levels of optionals + if (unpacked->IsOptional() || isOptional && NeedWrapWithExternalOptional(unpacked)) { ui32 nestLevel = 0; auto currentType = itemType; auto previousType = itemType; @@ -1545,12 +1555,11 @@ bool ConvertArrowTypeImpl(TType* itemType, std::shared_ptr<arrow::DataType>& typ currentType = AS_TYPE(TOptionalType, currentType)->GetItemType(); } while (currentType->IsOptional()); - if (currentType->IsPg()) { + if (NeedWrapWithExternalOptional(currentType)) { previousType = currentType; ++nestLevel; } - // previousType is always Optional std::shared_ptr<arrow::DataType> innerArrowType; if (!ConvertArrowTypeImpl(previousType, innerArrowType, onFail, output)) { return false; @@ -1618,6 +1627,11 @@ bool ConvertArrowTypeImpl(TType* itemType, std::shared_ptr<arrow::DataType>& typ return true; } + if (IsSingularType(unpacked)) { + type = arrow::null(); + return true; + } + if (!unpacked->IsData()) { if (onFail) { onFail(unpacked); @@ -2479,6 +2493,10 @@ size_t CalcMaxBlockItemSize(const TType* type) { return sizeof(NYql::NUdf::TUnboxedValue); } + if (IsSingularType(type)) { + return 0; + } + if (type->IsData()) { auto slot = *AS_TYPE(TDataType, type)->GetDataSlot(); switch (slot) { @@ -2552,6 +2570,7 @@ struct TComparatorTraits { using TExtOptional = NUdf::TExternalOptionalBlockItemComparator; template <typename T, bool Nullable> using TTzDateComparator = NUdf::TTzDateBlockItemComparator<T, Nullable>; + using TSingularType = NUdf::TSingularTypeBlockItemComparator; constexpr static bool PassType = false; @@ -2565,6 +2584,10 @@ struct TComparatorTraits { ythrow yexception() << "Comparator not implemented for block resources: "; } + static std::unique_ptr<TResult> MakeSingular() { + return std::make_unique<TSingularType>(); + } + template<typename TTzDate> static std::unique_ptr<TResult> MakeTzDate(bool isOptional) { if (isOptional) { @@ -2586,6 +2609,7 @@ struct THasherTraits { using TExtOptional = NUdf::TExternalOptionalBlockItemHasher; template <typename T, bool Nullable> using TTzDateHasher = NYql::NUdf::TTzDateBlockItemHasher<T, Nullable>; + using TSingularType = NUdf::TSingularTypeBlockItemHaser; constexpr static bool PassType = false; @@ -2607,6 +2631,10 @@ struct THasherTraits { return std::make_unique<TTzDateHasher<TTzDate, false>>(); } } + + static std::unique_ptr<TResult> MakeSingular() { + return std::make_unique<TSingularType>(); + } }; NUdf::IBlockItemComparator::TPtr TBlockTypeHelper::MakeComparator(NUdf::TType* type) const { @@ -2622,12 +2650,11 @@ TType* TTypeBuilder::NewVoidType() const { } TType* TTypeBuilder::NewNullType() const { - if (!UseNullType || RuntimeVersion < 11) { - TCallableBuilder callableBuilder(Env, "Null", NewOptionalType(NewVoidType())); - return TRuntimeNode(callableBuilder.Build(), false).GetStaticType(); - } else { + if (UseNullType) { return TRuntimeNode(Env.GetNullLazy(), true).GetStaticType(); } + TCallableBuilder callableBuilder(Env, "Null", NewOptionalType(NewVoidType())); + return TRuntimeNode(callableBuilder.Build(), false).GetStaticType(); } TType* TTypeBuilder::NewEmptyStructType() const { diff --git a/yql/essentials/providers/common/mkql/yql_provider_mkql.cpp b/yql/essentials/providers/common/mkql/yql_provider_mkql.cpp index 123b59cc04..983bf85542 100644 --- a/yql/essentials/providers/common/mkql/yql_provider_mkql.cpp +++ b/yql/essentials/providers/common/mkql/yql_provider_mkql.cpp @@ -2236,22 +2236,12 @@ TMkqlCommonCallableCompiler::TShared::TShared() { AddCallable("EmptyList", [](const TExprNode& node, TMkqlBuildContext& ctx) { Y_UNUSED(node); - if (RuntimeVersion < 11) { - return ctx.ProgramBuilder.NewEmptyList(ctx.ProgramBuilder.NewVoid().GetStaticType()); - } else { - return TRuntimeNode(ctx.ProgramBuilder.GetTypeEnvironment().GetEmptyListLazy(), true); - } + return TRuntimeNode(ctx.ProgramBuilder.GetTypeEnvironment().GetEmptyListLazy(), true); }); AddCallable("EmptyDict", [](const TExprNode& node, TMkqlBuildContext& ctx) { Y_UNUSED(node); - if (RuntimeVersion < 11) { - auto voidType = ctx.ProgramBuilder.NewVoid().GetStaticType(); - auto dictType = ctx.ProgramBuilder.NewDictType(voidType, voidType, false); - return ctx.ProgramBuilder.NewDict(dictType, {}); - } else { - return TRuntimeNode(ctx.ProgramBuilder.GetTypeEnvironment().GetEmptyDictLazy(), true); - } + return TRuntimeNode(ctx.ProgramBuilder.GetTypeEnvironment().GetEmptyDictLazy(), true); }); AddCallable("SourceOf", [](const TExprNode& node, TMkqlBuildContext& ctx) { diff --git a/yql/essentials/providers/common/mkql/yql_type_mkql.cpp b/yql/essentials/providers/common/mkql/yql_type_mkql.cpp index cb001c847b..372ba7a98e 100644 --- a/yql/essentials/providers/common/mkql/yql_type_mkql.cpp +++ b/yql/essentials/providers/common/mkql/yql_type_mkql.cpp @@ -184,20 +184,10 @@ NKikimr::NMiniKQL::TType* BuildTypeImpl(const TTypeAnnotationNode& annotation, c } case ETypeAnnotationKind::EmptyList: { - if (NKikimr::NMiniKQL::RuntimeVersion < 11) { - auto voidType = typeBuilder.NewVoidType(); - return typeBuilder.NewListType(voidType); - } - return typeBuilder.GetTypeEnvironment().GetTypeOfEmptyListLazy(); } case ETypeAnnotationKind::EmptyDict: { - if constexpr(NKikimr::NMiniKQL::RuntimeVersion < 11) { - auto voidType = typeBuilder.NewVoidType(); - return typeBuilder.NewDictType(voidType, voidType, false); - } - return typeBuilder.GetTypeEnvironment().GetTypeOfEmptyDictLazy(); } diff --git a/yql/essentials/providers/common/schema/mkql/yql_mkql_schema.cpp b/yql/essentials/providers/common/schema/mkql/yql_mkql_schema.cpp index c172a0a09f..8273070925 100644 --- a/yql/essentials/providers/common/schema/mkql/yql_mkql_schema.cpp +++ b/yql/essentials/providers/common/schema/mkql/yql_mkql_schema.cpp @@ -192,17 +192,9 @@ struct TRuntimeTypeLoader { return Builder.GetTypeEnvironment().GetTypeOfTypeLazy(); } TMaybe<TType> LoadEmptyListType(ui32 /*level*/) { - if (NKikimr::NMiniKQL::RuntimeVersion < 11) { - return Builder.NewListType(Builder.NewVoid().GetStaticType()); - } - return Builder.GetTypeEnvironment().GetTypeOfEmptyListLazy(); } TMaybe<TType> LoadEmptyDictType(ui32 /*level*/) { - if (NKikimr::NMiniKQL::RuntimeVersion < 11) { - return Builder.NewDictType(Builder.NewVoid().GetStaticType(), Builder.NewVoid().GetStaticType(), false); - } - return Builder.GetTypeEnvironment().GetTypeOfEmptyDictLazy(); } TMaybe<TType> LoadDataType(const TString& dataType, ui32 /*level*/) { diff --git a/yql/essentials/public/issue/yql_warning.cpp b/yql/essentials/public/issue/yql_warning.cpp index 881e63e95f..6364025ff3 100644 --- a/yql/essentials/public/issue/yql_warning.cpp +++ b/yql/essentials/public/issue/yql_warning.cpp @@ -31,6 +31,10 @@ TWarningRule::EParseResult TWarningRule::ParseFrom(const TString& codePattern, c return EParseResult::PARSE_OK; } +TWarningPolicy::TWarningPolicy(bool isReplay) + : IsReplay(isReplay) +{} + void TWarningPolicy::AddRule(const TWarningRule& rule) { TString pattern = rule.GetPattern(); @@ -38,6 +42,10 @@ void TWarningPolicy::AddRule(const TWarningRule& rule) return; } + if (pattern == "*" && IsReplay) { + return; + } + Rules.push_back(rule); EWarningAction action = rule.GetAction(); diff --git a/yql/essentials/public/issue/yql_warning.h b/yql/essentials/public/issue/yql_warning.h index d1d6a90922..7c3939d1d3 100644 --- a/yql/essentials/public/issue/yql_warning.h +++ b/yql/essentials/public/issue/yql_warning.h @@ -34,6 +34,8 @@ using TWarningRules = TVector<TWarningRule>; class TWarningPolicy { public: + TWarningPolicy(bool isReplay = false); + void AddRule(const TWarningRule& rule); EWarningAction GetAction(TIssueCode code) const; @@ -43,6 +45,7 @@ public: void Clear(); private: + const bool IsReplay; TWarningRules Rules; EWarningAction BaseAction = EWarningAction::DEFAULT; THashMap<TIssueCode, EWarningAction> Overrides; diff --git a/yql/essentials/public/udf/arrow/block_builder.h b/yql/essentials/public/udf/arrow/block_builder.h index 92f4f7e123..baac1842b9 100644 --- a/yql/essentials/public/udf/arrow/block_builder.h +++ b/yql/essentials/public/udf/arrow/block_builder.h @@ -10,6 +10,7 @@ #include <yql/essentials/public/udf/udf_value_builder.h> #include <yql/essentials/public/udf/udf_type_inspection.h> +#include <arrow/array/array_base.h> #include <arrow/datum.h> #include <arrow/c/bridge.h> @@ -1358,6 +1359,53 @@ private: std::unique_ptr<TTypedBufferBuilder<ui8>> NullBuilder; }; +class TSingularBlockBuilder final: public TArrayBuilderBase { +public: + TSingularBlockBuilder(const TType* type, const ITypeInfoHelper& typeInfoHelper, arrow::MemoryPool& pool, + size_t maxLen, const TParams& params = {}) + : TArrayBuilderBase(typeInfoHelper, type, pool, maxLen, params) { + Reserve(); + } + + void DoAdd(NUdf::TUnboxedValuePod value) final { + Y_UNUSED(value); + } + + void DoAdd(TBlockItem value) final { + Y_UNUSED(value); + } + + void DoAdd(TInputBuffer& input) final { + Y_UNUSED(input.PopChar()); + } + + void DoAddDefault() final {} + + void DoAddMany(const arrow::ArrayData& array, const ui8* sparseBitmap, size_t popCount) final { + Y_UNUSED(array, sparseBitmap, popCount); + } + + void DoAddMany(const arrow::ArrayData& array, ui64 beginIndex, size_t count) final { + Y_UNUSED(array, beginIndex, count); + } + + void DoAddMany(const arrow::ArrayData& array, const ui64* indexes, size_t count) final { + Y_UNUSED(array, indexes, count); + } + + TBlockArrayTree::Ptr DoBuildTree(bool finish) final { + TBlockArrayTree::Ptr result = std::make_shared<TBlockArrayTree>(); + Y_UNUSED(finish); + result->Payload.push_back(arrow::NullArray(GetCurrLen()).data()); + return result; + } + +private: + size_t DoReserve() final { + return 0; + } +}; + using TArrayBuilderParams = TArrayBuilderBase::TParams; struct TBuilderTraits { @@ -1373,6 +1421,7 @@ struct TBuilderTraits { using TResource = TResourceArrayBuilder<Nullable>; template<typename TTzDate, bool Nullable> using TTzDateReader = TTzDateArrayBuilder<TTzDate, Nullable>; + using TSingular = TSingularBlockBuilder; constexpr static bool PassType = true; @@ -1412,6 +1461,10 @@ struct TBuilderTraits { return std::make_unique<TTzDateReader<TTzDate, false>>(type, typeInfoHelper, pool, maxLen, params); } } + + static std::unique_ptr<TResult> MakeSingular(const TType* type, const ITypeInfoHelper& typeInfoHelper, arrow::MemoryPool& pool, size_t maxLen, const TArrayBuilderParams& params) { + return std::make_unique<TSingular>(type, typeInfoHelper, pool, maxLen, params); + } }; inline std::unique_ptr<IArrayBuilder> MakeArrayBuilder( diff --git a/yql/essentials/public/udf/arrow/block_item.h b/yql/essentials/public/udf/arrow/block_item.h index 2f9784cd3c..79686b3094 100644 --- a/yql/essentials/public/udf/arrow/block_item.h +++ b/yql/essentials/public/udf/arrow/block_item.h @@ -166,6 +166,18 @@ public: return &Raw; } + static inline TBlockItem Void() { + TBlockItem v; + v.Raw.Simple.Meta = static_cast<ui8>(EMarkers::Embedded); + return v; + } + + static inline TBlockItem Zero() { + TBlockItem v; + v.Raw.Simple.Meta = static_cast<ui8>(EMarkers::Embedded); + return v; + } + inline const void* GetRawPtr() const { return &Raw; diff --git a/yql/essentials/public/udf/arrow/block_item_comparator.h b/yql/essentials/public/udf/arrow/block_item_comparator.h index e185b63f66..ad803799c6 100644 --- a/yql/essentials/public/udf/arrow/block_item_comparator.h +++ b/yql/essentials/public/udf/arrow/block_item_comparator.h @@ -169,6 +169,24 @@ public: } }; +class TSingularTypeBlockItemComparator: public TBlockItemComparatorBase<TSingularTypeBlockItemComparator, /*Nullable=*/false> { +public: + i64 DoCompare(TBlockItem lhs, TBlockItem rhs) const { + Y_UNUSED(lhs, rhs); + return 0; + } + + bool DoEquals(TBlockItem lhs, TBlockItem rhs) const { + Y_UNUSED(lhs, rhs); + return true; + } + + bool DoLess(TBlockItem lhs, TBlockItem rhs) const { + Y_UNUSED(lhs, rhs); + return false; + } +}; + template<typename TTzType, bool Nullable> class TTzDateBlockItemComparator : public TBlockItemComparatorBase<TTzDateBlockItemComparator<TTzType, Nullable>, Nullable> { using TLayout = typename TDataType<TTzType>::TLayout; diff --git a/yql/essentials/public/udf/arrow/block_item_hasher.h b/yql/essentials/public/udf/arrow/block_item_hasher.h index 3f77e27b6f..9108d7b06e 100644 --- a/yql/essentials/public/udf/arrow/block_item_hasher.h +++ b/yql/essentials/public/udf/arrow/block_item_hasher.h @@ -76,6 +76,14 @@ public: } }; +class TSingularTypeBlockItemHaser : public TBlockItemHasherBase<TSingularTypeBlockItemHaser, /*Nullable=*/false> { +public: + ui64 DoHash(TBlockItem value) const { + Y_UNUSED(value); + return 0; + } +}; + template <bool Nullable> class TTupleBlockItemHasher : public TBlockItemHasherBase<TTupleBlockItemHasher<Nullable>, Nullable> { public: diff --git a/yql/essentials/public/udf/arrow/block_reader.h b/yql/essentials/public/udf/arrow/block_reader.h index 05dd3ce440..6652df2ac6 100644 --- a/yql/essentials/public/udf/arrow/block_reader.h +++ b/yql/essentials/public/udf/arrow/block_reader.h @@ -424,6 +424,48 @@ private: TFixedSizeBlockReader<ui16, /* Nullable */false> TimezoneReader_; }; +// NOTE: For any singular type we use arrow::null() data type. +// This data type DOES NOT support bit mask so for optional type +// we have to use |TExternalOptional| wrapper. +class TSingularTypeBlockReader: public IBlockReader { +public: + TSingularTypeBlockReader() = default; + + ~TSingularTypeBlockReader() override = default; + + TBlockItem GetItem(const arrow::ArrayData& data, size_t index) override { + Y_UNUSED(data, index); + return TBlockItem::Zero(); + } + + TBlockItem GetScalarItem(const arrow::Scalar& scalar) override { + Y_UNUSED(scalar); + return TBlockItem::Zero(); + } + + ui64 GetDataWeight(const arrow::ArrayData& data) const override { + Y_UNUSED(data); + return 0; + } + + ui64 GetDataWeight(TBlockItem item) const override { + Y_UNUSED(item); + return 0; + } + + ui64 GetDefaultValueWeight() const override { + return 0; + } + + void SaveItem(const arrow::ArrayData& data, size_t index, TOutputBuffer& out) const override { + Y_UNUSED(index, data, out); + } + + void SaveScalarItem(const arrow::Scalar& scalar, TOutputBuffer& out) const override { + Y_UNUSED(scalar, out); + } +}; + class TExternalOptionalBlockReader final : public IBlockReader { public: TExternalOptionalBlockReader(std::unique_ptr<IBlockReader>&& inner) @@ -498,6 +540,7 @@ struct TReaderTraits { using TResource = TResourceBlockReader<Nullable>; template<typename TTzDate, bool Nullable> using TTzDateReader = TTzDateBlockReader<TTzDate, Nullable>; + using TSingularType = TSingularTypeBlockReader; constexpr static bool PassType = false; @@ -518,6 +561,10 @@ struct TReaderTraits { } } + static std::unique_ptr<TResult> MakeSingular() { + return std::make_unique<TSingularType>(); + } + template<typename TTzDate> static std::unique_ptr<TResult> MakeTzDate(bool isOptional) { if (isOptional) { @@ -595,6 +642,10 @@ inline void UpdateBlockItemSerializeProps(const ITypeInfoHelper& typeInfoHelper, return; } + if (IsSingularType(typeInfoHelper, type)) { + return; + } + Y_ENSURE(false, "Unsupported type"); } diff --git a/yql/essentials/public/udf/arrow/dispatch_traits.h b/yql/essentials/public/udf/arrow/dispatch_traits.h index 88c303cc87..87c25b93f5 100644 --- a/yql/essentials/public/udf/arrow/dispatch_traits.h +++ b/yql/essentials/public/udf/arrow/dispatch_traits.h @@ -1,5 +1,6 @@ #pragma once +#include <yql/essentials/public/udf/arrow/util.h> #include <yql/essentials/public/udf/udf_type_inspection.h> #include <yql/essentials/public/udf/udf_value_builder.h> @@ -85,8 +86,7 @@ std::unique_ptr<typename TTraits::TResult> DispatchByArrowTraits(const ITypeInfo TOptionalTypeInspector unpackedOpt(typeInfoHelper, unpacked); TPgTypeInspector unpackedPg(typeInfoHelper, unpacked); - if (unpackedOpt || typeOpt && unpackedPg) { - // at least 2 levels of optionals + if (unpackedOpt || (typeOpt && NeedWrapWithExternalOptional(typeInfoHelper, unpacked))) { ui32 nestLevel = 0; auto currentType = type; auto previousType = type; @@ -103,7 +103,7 @@ std::unique_ptr<typename TTraits::TResult> DispatchByArrowTraits(const ITypeInfo } } - if (TPgTypeInspector(typeInfoHelper, currentType)) { + if (NeedWrapWithExternalOptional(typeInfoHelper, currentType)) { previousType = currentType; ++nestLevel; } @@ -118,8 +118,7 @@ std::unique_ptr<typename TTraits::TResult> DispatchByArrowTraits(const ITypeInfo } return reader; - } - else { + } else { type = unpacked; } @@ -230,6 +229,15 @@ std::unique_ptr<typename TTraits::TResult> DispatchByArrowTraits(const ITypeInfo } } + if (IsSingularType(typeInfoHelper, type)) { + Y_ENSURE(!isOptional, "Optional data types are not supported directly for singular type. Please use TExternalOptional wrapper."); + if constexpr (TTraits::PassType) { + return TTraits::MakeSingular(type, std::forward<TArgs>(args)...); + } else { + return TTraits::MakeSingular(std::forward<TArgs>(args)...); + } + } + Y_ENSURE(false, "Unsupported type"); } diff --git a/yql/essentials/public/udf/arrow/ut/array_builder_ut.cpp b/yql/essentials/public/udf/arrow/ut/array_builder_ut.cpp index bbb4c134c8..d0851c5e86 100644 --- a/yql/essentials/public/udf/arrow/ut/array_builder_ut.cpp +++ b/yql/essentials/public/udf/arrow/ut/array_builder_ut.cpp @@ -220,6 +220,46 @@ Y_UNIT_TEST_SUITE(TArrayBuilderTest) { UNIT_ASSERT_VALUES_EQUAL(item2AfterRead.GetStringRefFromValue(), "234"); } + Y_UNIT_TEST(TestSingularTypeValueBuilderReader) { + TArrayBuilderTestData data; + const auto nullType = data.PgmBuilder.NewNullType(); + + std::shared_ptr<arrow::ArrayData> arrayData = arrow::NullArray{42}.data(); + IArrayBuilder::TArrayDataItem arrayDataItem = {.Data = arrayData.get(), .StartOffset = 0}; + { + const auto arrayBuilder = MakeArrayBuilder(NMiniKQL::TTypeInfoHelper(), nullType, *data.ArrowPool, MAX_BLOCK_SIZE, /*pgBuilder=*/nullptr); + // Check builder. + arrayBuilder->Add(TUnboxedValuePod::Zero()); + arrayBuilder->Add(TBlockItem::Zero()); + arrayBuilder->Add(TBlockItem::Zero(), 4); + TInputBuffer inputBuffer("Just arbitrary string"); + arrayBuilder->Add(inputBuffer); + arrayBuilder->AddMany(*arrayData, /*popCount=*/3u, /*sparseBitmat=*/nullptr, /*bitmapSize=*/arrayData->length); + arrayBuilder->AddMany(&arrayDataItem, /*arrayCount=*/1, /*beginIndex=*/1, /*count=*/3u); + std::vector<ui64> indexes = {1, 5, 7, 10}; + arrayBuilder->AddMany(&arrayDataItem, /*arrayCount=*/1, /*beginIndex=*/indexes.data(), /*count=*/4u); + UNIT_ASSERT_VALUES_EQUAL(arrayBuilder->Build(true).array()->length, 1 + 1 + 4 + 1 + 3 + 3 + 4); + } + + { + // Check reader. + const auto blockReader = MakeBlockReader(NMiniKQL::TTypeInfoHelper(), nullType); + + UNIT_ASSERT(blockReader->GetItem(*arrayData, 0)); + UNIT_ASSERT(blockReader->GetScalarItem(arrow::Scalar(arrow::null()))); + UNIT_ASSERT_EQUAL(blockReader->GetDataWeight(*arrayData), 0); + UNIT_ASSERT_EQUAL(blockReader->GetDataWeight(TBlockItem::Zero()), 0); + UNIT_ASSERT_EQUAL(blockReader->GetDefaultValueWeight(), 0); + UNIT_ASSERT_EQUAL(blockReader->GetDefaultValueWeight(), 0); + + TOutputBuffer outputBuffer; + blockReader->SaveItem(*arrayData, 1, outputBuffer); + UNIT_ASSERT(outputBuffer.Finish().empty()); + blockReader->SaveScalarItem(arrow::Scalar(arrow::null()), outputBuffer); + UNIT_ASSERT(outputBuffer.Finish().empty()); + } + } + Y_UNIT_TEST(TestBuilderAllocatedSize) { TArrayBuilderTestData data; const auto optStringType = data.PgmBuilder.NewDataType(NUdf::EDataSlot::String, true); diff --git a/yql/essentials/public/udf/arrow/util.h b/yql/essentials/public/udf/arrow/util.h index f7bdb715f9..e899af26af 100644 --- a/yql/essentials/public/udf/arrow/util.h +++ b/yql/essentials/public/udf/arrow/util.h @@ -12,6 +12,9 @@ #include <functional> +#include <yql/essentials/public/udf/udf_type_inspection.h> +#include <yql/essentials/public/udf/udf_types.h> + namespace NYql { namespace NUdf { @@ -236,5 +239,17 @@ inline void ZeroMemoryContext(void* ptr) { SetMemoryContext(ptr, nullptr); } +inline bool IsSingularType(const ITypeInfoHelper& typeInfoHelper, const TType* type) { + auto kind = typeInfoHelper.GetTypeKind(type); + return kind == ETypeKind::Null || + kind == ETypeKind::Void || + kind == ETypeKind::EmptyDict || + kind == ETypeKind::EmptyList; +} + +inline bool NeedWrapWithExternalOptional(const ITypeInfoHelper& typeInfoHelper, const TType* type) { + return TPgTypeInspector(typeInfoHelper, type) || IsSingularType(typeInfoHelper, type); +} + } // namespace NUdf } // namespace NYql diff --git a/yql/essentials/sql/settings/translation_settings.h b/yql/essentials/sql/settings/translation_settings.h index 8c6e716343..98d537038e 100644 --- a/yql/essentials/sql/settings/translation_settings.h +++ b/yql/essentials/sql/settings/translation_settings.h @@ -129,6 +129,7 @@ namespace NSQLTranslation { NYql::IAutoParamBuilderFactory* AutoParamBuilderFactory = nullptr; bool EmitReadsForExists = false; bool AlwaysAllowExports = false; + bool IsReplay = false; }; bool ParseTranslationSettings(const TString& query, NSQLTranslation::TTranslationSettings& settings, NYql::TIssues& issues); diff --git a/yql/essentials/sql/v1/context.cpp b/yql/essentials/sql/v1/context.cpp index 9790e931dc..b72c673b0e 100644 --- a/yql/essentials/sql/v1/context.cpp +++ b/yql/essentials/sql/v1/context.cpp @@ -100,6 +100,7 @@ TContext::TContext(const TLexers& lexers, const TParsers& parsers, , HasPendingErrors(false) , DqEngineEnable(Settings.DqDefaultAuto->Allow()) , AnsiQuotedIdentifiers(settings.AnsiLexer) + , WarningPolicy(settings.IsReplay) , BlockEngineEnable(Settings.BlockDefaultAuto->Allow()) { for (auto lib : settings.Libraries) { diff --git a/yql/essentials/tests/sql/minirun/part0/canondata/result.json b/yql/essentials/tests/sql/minirun/part0/canondata/result.json index 3c1aa86fec..ffeebb57cc 100644 --- a/yql/essentials/tests/sql/minirun/part0/canondata/result.json +++ b/yql/essentials/tests/sql/minirun/part0/canondata/result.json @@ -275,6 +275,27 @@ "uri": "https://{canondata_backend}/1936842/8073eb626dd657fcbe20d34185c363a1a18c3e7c/resource.tar.gz#test.test_blocks-agg_all_mixed_distinct-default.txt-Results_/results.txt" } ], + "test.test[blocks-agg_singular_type_key_optional-default.txt-Debug]": [ + { + "checksum": "71ee94512d6ef28833fb6df3bace7b53", + "size": 2727, + "uri": "https://{canondata_backend}/1925842/7e03c084910acb6d9d50a1f7dc65eda3cdac3b45/resource.tar.gz#test.test_blocks-agg_singular_type_key_optional-default.txt-Debug_/opt.yql" + } + ], + "test.test[blocks-agg_singular_type_key_optional-default.txt-Peephole]": [ + { + "checksum": "db2e4bd6530b31b6efceb77a4a184b4e", + "size": 6606, + "uri": "https://{canondata_backend}/1925842/7e03c084910acb6d9d50a1f7dc65eda3cdac3b45/resource.tar.gz#test.test_blocks-agg_singular_type_key_optional-default.txt-Peephole_/opt.yql" + } + ], + "test.test[blocks-agg_singular_type_key_optional-default.txt-Results]": [ + { + "checksum": "4b79ad0d41612ad09d735f34513ee6ff", + "size": 7301, + "uri": "https://{canondata_backend}/1925842/7e03c084910acb6d9d50a1f7dc65eda3cdac3b45/resource.tar.gz#test.test_blocks-agg_singular_type_key_optional-default.txt-Results_/results.txt" + } + ], "test.test[blocks-and-default.txt-Debug]": [ { "checksum": "47525fa40526e04498f0c41e6bc48f59", diff --git a/yql/essentials/tests/sql/minirun/part2/canondata/result.json b/yql/essentials/tests/sql/minirun/part2/canondata/result.json index 48c0652311..73ff7dcd87 100644 --- a/yql/essentials/tests/sql/minirun/part2/canondata/result.json +++ b/yql/essentials/tests/sql/minirun/part2/canondata/result.json @@ -258,6 +258,27 @@ "uri": "https://{canondata_backend}/1937150/3d01c6ab2777fc3b99338655d39a5bcbb1ac89c3/resource.tar.gz#test.test_blocks-agg_by_key_only_distinct-default.txt-Results_/results.txt" } ], + "test.test[blocks-agg_singular_type_value_optional-default.txt-Debug]": [ + { + "checksum": "06774e6dab64198fc6cc5d173b0bba26", + "size": 2781, + "uri": "https://{canondata_backend}/1781765/b8d92d6ccf46e436b2e5b3b70ab511bab6d820b0/resource.tar.gz#test.test_blocks-agg_singular_type_value_optional-default.txt-Debug_/opt.yql" + } + ], + "test.test[blocks-agg_singular_type_value_optional-default.txt-Peephole]": [ + { + "checksum": "d22ed37889eea3b41eadb6164bf6d017", + "size": 3113, + "uri": "https://{canondata_backend}/1781765/b8d92d6ccf46e436b2e5b3b70ab511bab6d820b0/resource.tar.gz#test.test_blocks-agg_singular_type_value_optional-default.txt-Peephole_/opt.yql" + } + ], + "test.test[blocks-agg_singular_type_value_optional-default.txt-Results]": [ + { + "checksum": "98fbeb83e5295954045efa6fd159626f", + "size": 5301, + "uri": "https://{canondata_backend}/1781765/b8d92d6ccf46e436b2e5b3b70ab511bab6d820b0/resource.tar.gz#test.test_blocks-agg_singular_type_value_optional-default.txt-Results_/results.txt" + } + ], "test.test[blocks-exists-default.txt-Debug]": [ { "checksum": "a871029504a6d3f1c07342493b86d28d", diff --git a/yql/essentials/tests/sql/minirun/part7/canondata/result.json b/yql/essentials/tests/sql/minirun/part7/canondata/result.json index 399f6d226b..d018f1582d 100644 --- a/yql/essentials/tests/sql/minirun/part7/canondata/result.json +++ b/yql/essentials/tests/sql/minirun/part7/canondata/result.json @@ -220,7 +220,7 @@ { "checksum": "02e80809d3cbf91101d09d4ac1e87aa0", "size": 623, - "uri": "https://{canondata_backend}/1917492/b01930df0710eb10e4ce2d35cddca6be33ac8a9f/resource.tar.gz#test.test_blocks-as_tuple-default.txt-Peephole_/opt.yql" + "uri": "https://{canondata_backend}/1130705/f9eb075ce8fc54a57832e4ee918669601325c133/resource.tar.gz#test.test_blocks-as_tuple-default.txt-Peephole_/opt.yql" } ], "test.test[blocks-as_tuple-default.txt-Results]": [ diff --git a/yql/essentials/tests/sql/minirun/part8/canondata/result.json b/yql/essentials/tests/sql/minirun/part8/canondata/result.json index 6d3cbc281d..94d8c6547e 100644 --- a/yql/essentials/tests/sql/minirun/part8/canondata/result.json +++ b/yql/essentials/tests/sql/minirun/part8/canondata/result.json @@ -352,6 +352,48 @@ "uri": "https://{canondata_backend}/1031349/4d0c6ce1905689c65e264d15d770d36efcd9426f/resource.tar.gz#test.test_binding-named_expr_input-default.txt-Results_/results.txt" } ], + "test.test[blocks-agg_singular_type_key-default.txt-Debug]": [ + { + "checksum": "b97d36400fafea8f8f6670954d7ac139", + "size": 2685, + "uri": "https://{canondata_backend}/1936273/07450a3416f3c728f9a8a8fdde6e5f5a0ca2d9a6/resource.tar.gz#test.test_blocks-agg_singular_type_key-default.txt-Debug_/opt.yql" + } + ], + "test.test[blocks-agg_singular_type_key-default.txt-Peephole]": [ + { + "checksum": "f77f6136a2995c5b0e0ae3ed00274d36", + "size": 6564, + "uri": "https://{canondata_backend}/1936273/07450a3416f3c728f9a8a8fdde6e5f5a0ca2d9a6/resource.tar.gz#test.test_blocks-agg_singular_type_key-default.txt-Peephole_/opt.yql" + } + ], + "test.test[blocks-agg_singular_type_key-default.txt-Results]": [ + { + "checksum": "e2233558149bd3009f7f16412bf4838a", + "size": 6117, + "uri": "https://{canondata_backend}/1936273/07450a3416f3c728f9a8a8fdde6e5f5a0ca2d9a6/resource.tar.gz#test.test_blocks-agg_singular_type_key-default.txt-Results_/results.txt" + } + ], + "test.test[blocks-agg_singular_type_value-default.txt-Debug]": [ + { + "checksum": "4733abf71c9c62e30af77c6490d59334", + "size": 2358, + "uri": "https://{canondata_backend}/1130705/a25045513209436069d9f9a29831b732c13e1675/resource.tar.gz#test.test_blocks-agg_singular_type_value-default.txt-Debug_/opt.yql" + } + ], + "test.test[blocks-agg_singular_type_value-default.txt-Peephole]": [ + { + "checksum": "b34c4e8ca42e6232ef03f2ffeb05fd83", + "size": 3013, + "uri": "https://{canondata_backend}/1130705/a25045513209436069d9f9a29831b732c13e1675/resource.tar.gz#test.test_blocks-agg_singular_type_value-default.txt-Peephole_/opt.yql" + } + ], + "test.test[blocks-agg_singular_type_value-default.txt-Results]": [ + { + "checksum": "4718805e72274809e4ac6c07ee8dfd7d", + "size": 3489, + "uri": "https://{canondata_backend}/1130705/a25045513209436069d9f9a29831b732c13e1675/resource.tar.gz#test.test_blocks-agg_singular_type_value-default.txt-Results_/results.txt" + } + ], "test.test[blocks-and_scalar-default.txt-Debug]": [ { "checksum": "e5ccc5c53756e09ded8e82b6d662e5e9", diff --git a/yql/essentials/tests/sql/sql2yql/canondata/result.json b/yql/essentials/tests/sql/sql2yql/canondata/result.json index f345b0e389..0c9c63e161 100644 --- a/yql/essentials/tests/sql/sql2yql/canondata/result.json +++ b/yql/essentials/tests/sql/sql2yql/canondata/result.json @@ -1378,6 +1378,34 @@ "uri": "https://{canondata_backend}/1917492/7dd4bc86433f6173a26b62397e1ef41fa9471945/resource.tar.gz#test_sql2yql.test_blocks-agg_by_key_only_distinct_/sql.yql" } ], + "test_sql2yql.test[blocks-agg_singular_type_key]": [ + { + "checksum": "7cae7f556775597a0b451a875e77a1df", + "size": 7636, + "uri": "https://{canondata_backend}/1784117/5ff6ff6c0808bf39612567f492af1bc2db36da20/resource.tar.gz#test_sql2yql.test_blocks-agg_singular_type_key_/sql.yql" + } + ], + "test_sql2yql.test[blocks-agg_singular_type_key_optional]": [ + { + "checksum": "a3f91d7949791561f4972eafc1610499", + "size": 7678, + "uri": "https://{canondata_backend}/1781765/9e1dc7f8aa95db55a59c09f397a0634224d08363/resource.tar.gz#test_sql2yql.test_blocks-agg_singular_type_key_optional_/sql.yql" + } + ], + "test_sql2yql.test[blocks-agg_singular_type_value]": [ + { + "checksum": "cd58c3714a9d215fd1f4bea4e36f37a2", + "size": 3634, + "uri": "https://{canondata_backend}/1781765/9e1dc7f8aa95db55a59c09f397a0634224d08363/resource.tar.gz#test_sql2yql.test_blocks-agg_singular_type_value_/sql.yql" + } + ], + "test_sql2yql.test[blocks-agg_singular_type_value_optional]": [ + { + "checksum": "5ae70db766241594bfea9edd5e5dec34", + "size": 3676, + "uri": "https://{canondata_backend}/1781765/0dce37dc71c65fe553d73ed7cf98a62bdee9ddee/resource.tar.gz#test_sql2yql.test_blocks-agg_singular_type_value_optional_/sql.yql" + } + ], "test_sql2yql.test[blocks-and]": [ { "checksum": "e22a52b51ef20174c3b832acb09df01b", @@ -1410,7 +1438,7 @@ { "checksum": "601f02d489707b615a9ff16a4fe1d3f5", "size": 1304, - "uri": "https://{canondata_backend}/1900335/c447765ddbde200b8fe3ee8091f4d625b36b6bc6/resource.tar.gz#test_sql2yql.test_blocks-as_tuple_/sql.yql" + "uri": "https://{canondata_backend}/1784826/bb2033aff3202d2b68e04361e6d1bacbf4cbbed6/resource.tar.gz#test_sql2yql.test_blocks-as_tuple_/sql.yql" } ], "test_sql2yql.test[blocks-coalesce]": [ @@ -8299,6 +8327,26 @@ "uri": "file://test_sql_format.test_blocks-agg_by_key_only_distinct_/formatted.sql" } ], + "test_sql_format.test[blocks-agg_singular_type_key]": [ + { + "uri": "file://test_sql_format.test_blocks-agg_singular_type_key_/formatted.sql" + } + ], + "test_sql_format.test[blocks-agg_singular_type_key_optional]": [ + { + "uri": "file://test_sql_format.test_blocks-agg_singular_type_key_optional_/formatted.sql" + } + ], + "test_sql_format.test[blocks-agg_singular_type_value]": [ + { + "uri": "file://test_sql_format.test_blocks-agg_singular_type_value_/formatted.sql" + } + ], + "test_sql_format.test[blocks-agg_singular_type_value_optional]": [ + { + "uri": "file://test_sql_format.test_blocks-agg_singular_type_value_optional_/formatted.sql" + } + ], "test_sql_format.test[blocks-and]": [ { "uri": "file://test_sql_format.test_blocks-and_/formatted.sql" diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_blocks-agg_singular_type_key_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_blocks-agg_singular_type_key_/formatted.sql new file mode 100644 index 0000000000..fd6c96da8f --- /dev/null +++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_blocks-agg_singular_type_key_/formatted.sql @@ -0,0 +1,72 @@ +PRAGMA config.flags('PeepholeFlags', 'UseAggPhases'); + +$n = 3; + +$data = ListMap( + ListFromRange(1, $n), ($x) -> ( + <| + idx: $x, + empty_list: [], + empty_dict: {}, + nil: NULL, + val: $x + 5, + vid: Void(), + emtpy_tuple: AsTuple(), + empty_struct: AsStruct() + |> + ) +); + +SELECT + empty_list, + SOME(idx) +FROM + as_table($data) +GROUP BY + empty_list +; + +SELECT + empty_dict, + SOME(idx) +FROM + as_table($data) +GROUP BY + empty_dict +; + +SELECT + nil, + SOME(idx) +FROM + as_table($data) +GROUP BY + nil +; + +SELECT + vid, + SOME(idx) +FROM + as_table($data) +GROUP BY + vid +; + +SELECT + emtpy_tuple, + SOME(idx) +FROM + as_table($data) +GROUP BY + emtpy_tuple +; + +SELECT + empty_struct, + SOME(idx) +FROM + as_table($data) +GROUP BY + empty_struct +; diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_blocks-agg_singular_type_key_optional_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_blocks-agg_singular_type_key_optional_/formatted.sql new file mode 100644 index 0000000000..401f8117f5 --- /dev/null +++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_blocks-agg_singular_type_key_optional_/formatted.sql @@ -0,0 +1,72 @@ +PRAGMA config.flags('PeepholeFlags', 'UseAggPhases'); + +$n = 3; + +$data = ListMap( + ListFromRange(1, $n), ($x) -> ( + <| + idx: $x, + empty_list: Just([]), + empty_dict: Just({}), + nil: Just(NULL), + val: $x + 5, + vid: Just(Void()), + emtpy_tuple: Just(AsTuple()), + empty_struct: Just(AsStruct()) + |> + ) +); + +SELECT + empty_list, + SOME(idx) +FROM + as_table($data) +GROUP BY + empty_list +; + +SELECT + empty_dict, + SOME(idx) +FROM + as_table($data) +GROUP BY + empty_dict +; + +SELECT + nil, + SOME(idx) +FROM + as_table($data) +GROUP BY + nil +; + +SELECT + vid, + SOME(idx) +FROM + as_table($data) +GROUP BY + vid +; + +SELECT + emtpy_tuple, + SOME(idx) +FROM + as_table($data) +GROUP BY + emtpy_tuple +; + +SELECT + empty_struct, + SOME(idx) +FROM + as_table($data) +GROUP BY + empty_struct +; diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_blocks-agg_singular_type_value_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_blocks-agg_singular_type_value_/formatted.sql new file mode 100644 index 0000000000..f8836f06f6 --- /dev/null +++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_blocks-agg_singular_type_value_/formatted.sql @@ -0,0 +1,33 @@ +PRAGMA config.flags('PeepholeFlags', 'UseAggPhases'); + +$n = 3; + +$data = ListMap( + ListFromRange(1, $n), ($x) -> ( + <| + idx: $x, + empty_list: [], + empty_dict: {}, + nil: NULL, + val: $x + 5, + vid: Void(), + emtpy_tuple: AsTuple(), + empty_struct: AsStruct() + |> + ) +); + +SELECT + idx, + SOME(empty_dict), + SOME(empty_list), + SOME(nil), + SOME(empty_dict), + SOME(vid), + SOME(emtpy_tuple), + SOME(empty_struct), +FROM + as_table($data) +GROUP BY + idx +; diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_blocks-agg_singular_type_value_optional_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_blocks-agg_singular_type_value_optional_/formatted.sql new file mode 100644 index 0000000000..258de29fb0 --- /dev/null +++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_blocks-agg_singular_type_value_optional_/formatted.sql @@ -0,0 +1,33 @@ +PRAGMA config.flags('PeepholeFlags', 'UseAggPhases'); + +$n = 3; + +$data = ListMap( + ListFromRange(1, $n), ($x) -> ( + <| + idx: $x, + empty_list: Just([]), + empty_dict: Just({}), + nil: Just(NULL), + val: $x + 5, + vid: Just(Void()), + emtpy_tuple: Just(AsTuple()), + empty_struct: Just(AsStruct()) + |> + ) +); + +SELECT + idx, + SOME(empty_dict), + SOME(empty_list), + SOME(nil), + SOME(empty_dict), + SOME(vid), + SOME(emtpy_tuple), + SOME(empty_struct), +FROM + as_table($data) +GROUP BY + idx +; diff --git a/yql/essentials/tests/sql/suites/blocks/agg_singular_type_key.sql b/yql/essentials/tests/sql/suites/blocks/agg_singular_type_key.sql new file mode 100644 index 0000000000..34c1f31962 --- /dev/null +++ b/yql/essentials/tests/sql/suites/blocks/agg_singular_type_key.sql @@ -0,0 +1,23 @@ +PRAGMA config.flags('PeepholeFlags', 'UseAggPhases'); + +$n = 3; +$data = ListMap(ListFromRange(1, $n), ($x) -> (<|idx: $x, + empty_list: [], + empty_dict: {}, + nil: NULL, + val: $x + 5, + vid: Void(), + emtpy_tuple: AsTuple(), + empty_struct: AsStruct()|>)); + +SELECT empty_list, SOME(idx) FROM as_table($data) GROUP BY empty_list; + +SELECT empty_dict, SOME(idx) FROM as_table($data) GROUP BY empty_dict; + +SELECT nil, SOME(idx) FROM as_table($data) GROUP BY nil; + +SELECT vid, SOME(idx) FROM as_table($data) GROUP BY vid; + +SELECT emtpy_tuple, SOME(idx) FROM as_table($data) GROUP BY emtpy_tuple; + +SELECT empty_struct, SOME(idx) FROM as_table($data) GROUP BY empty_struct; diff --git a/yql/essentials/tests/sql/suites/blocks/agg_singular_type_key_optional.sql b/yql/essentials/tests/sql/suites/blocks/agg_singular_type_key_optional.sql new file mode 100644 index 0000000000..0b86e48184 --- /dev/null +++ b/yql/essentials/tests/sql/suites/blocks/agg_singular_type_key_optional.sql @@ -0,0 +1,24 @@ +PRAGMA config.flags('PeepholeFlags', 'UseAggPhases'); + +$n = 3; + +$data = ListMap(ListFromRange(1, $n), ($x) -> (<|idx: $x, + empty_list: Just([]), + empty_dict: Just({}), + nil: Just(NULL), + val: $x + 5, + vid: Just(Void()), + emtpy_tuple: Just(AsTuple()), + empty_struct: Just(AsStruct())|>)); + +SELECT empty_list, SOME(idx) FROM as_table($data) GROUP BY empty_list; + +SELECT empty_dict, SOME(idx) FROM as_table($data) GROUP BY empty_dict; + +SELECT nil, SOME(idx) FROM as_table($data) GROUP BY nil; + +SELECT vid, SOME(idx) FROM as_table($data) GROUP BY vid; + +SELECT emtpy_tuple, SOME(idx) FROM as_table($data) GROUP BY emtpy_tuple; + +SELECT empty_struct, SOME(idx) FROM as_table($data) GROUP BY empty_struct; diff --git a/yql/essentials/tests/sql/suites/blocks/agg_singular_type_value.sql b/yql/essentials/tests/sql/suites/blocks/agg_singular_type_value.sql new file mode 100644 index 0000000000..ac290e2d25 --- /dev/null +++ b/yql/essentials/tests/sql/suites/blocks/agg_singular_type_value.sql @@ -0,0 +1,26 @@ +PRAGMA config.flags('PeepholeFlags', 'UseAggPhases'); + +$n = 3; +$data = ListMap(ListFromRange(1, $n), ($x) -> (<|idx: $x, + empty_list: [], + empty_dict: {}, + nil: NULL, + val: $x + 5, + vid: Void(), + emtpy_tuple: AsTuple(), + empty_struct: AsStruct()|>)); + +SELECT + idx, + SOME(empty_dict), + SOME(empty_list), + SOME(nil), + SOME(empty_dict), + SOME(vid), + SOME(emtpy_tuple), + SOME(empty_struct), +FROM + as_table($data) +GROUP BY + idx +; diff --git a/yql/essentials/tests/sql/suites/blocks/agg_singular_type_value_optional.sql b/yql/essentials/tests/sql/suites/blocks/agg_singular_type_value_optional.sql new file mode 100644 index 0000000000..3214db1049 --- /dev/null +++ b/yql/essentials/tests/sql/suites/blocks/agg_singular_type_value_optional.sql @@ -0,0 +1,27 @@ +PRAGMA config.flags('PeepholeFlags', 'UseAggPhases'); + +$n = 3; +$data = ListMap(ListFromRange(1, $n), ($x) -> (<|idx: $x, + empty_list: Just([]), + empty_dict: Just({}), + nil: Just(NULL), + val: $x + 5, + vid: Just(Void()), + emtpy_tuple: Just(AsTuple()), + empty_struct: Just(AsStruct())|>)); + +SELECT + idx, + SOME(empty_dict), + SOME(empty_list), + SOME(nil), + SOME(empty_dict), + SOME(vid), + SOME(emtpy_tuple), + SOME(empty_struct), +FROM + as_table($data) +GROUP BY + idx +; + diff --git a/yql/essentials/types/binary_json/ut/entry_ut.cpp b/yql/essentials/types/binary_json/ut/entry_ut.cpp index 0f099ed59d..247fc5034c 100644 --- a/yql/essentials/types/binary_json/ut/entry_ut.cpp +++ b/yql/essentials/types/binary_json/ut/entry_ut.cpp @@ -17,6 +17,7 @@ public: UNIT_TEST(TestGetContainer); UNIT_TEST(TestGetString); UNIT_TEST(TestGetNumber); + UNIT_TEST(TestOutOfBounds); UNIT_TEST_SUITE_END(); void TestGetType() { @@ -93,6 +94,28 @@ public: UNIT_ASSERT_VALUES_EQUAL(container.GetElement(0).GetNumber(), testCase.second); } } + + void TestOutOfBounds() { + const TVector<std::pair<TString, double>> testCases = { + { "1e100000000", std::numeric_limits<double>::infinity() }, + { "-1e100000000", -std::numeric_limits<double>::infinity() }, + { "1.797693135e+308", std::numeric_limits<double>::infinity() }, + { "-1.797693135e+308", -std::numeric_limits<double>::infinity() }, + }; + + for (const auto& testCase : testCases) { + UNIT_ASSERT(std::holds_alternative<TString>(SerializeToBinaryJson(testCase.first))); + } + + for (const auto& testCase : testCases) { + const auto binaryJson = std::get<TBinaryJson>(SerializeToBinaryJson(testCase.first, true)); + const auto reader = TBinaryJsonReader::Make(binaryJson); + const auto container = reader->GetRootCursor(); + + UNIT_ASSERT_VALUES_EQUAL(container.GetElement(0).GetNumber(), testCase.second); + } + } }; UNIT_TEST_SUITE_REGISTRATION(TBinaryJsonEntryTest); + diff --git a/yql/essentials/types/binary_json/write.cpp b/yql/essentials/types/binary_json/write.cpp index 28dffea9db..39cd05949d 100644 --- a/yql/essentials/types/binary_json/write.cpp +++ b/yql/essentials/types/binary_json/write.cpp @@ -13,6 +13,8 @@ #include <util/generic/set.h> #include <util/generic/stack.h> #include <util/generic/vector.h> +#include <yql/essentials/minikql/dom/node.h> +#include <yql/essentials/utils/parse_double.h> #include <cmath> @@ -415,9 +417,8 @@ private: */ class TBinaryJsonCallbacks : public TJsonCallbacks { public: - TBinaryJsonCallbacks(bool throwException) - : TJsonCallbacks(/* throwException */ throwException) - { + TBinaryJsonCallbacks(bool throwException, bool allowInf) + : TJsonCallbacks(/* throwException */ throwException), AllowInf(allowInf) { } bool OnNull() override { @@ -445,7 +446,7 @@ public: } bool OnDouble(double value) override { - if (Y_UNLIKELY(std::isinf(value))) { + if (Y_UNLIKELY(std::isinf(value) && !AllowInf)) { if (ThrowException) { ythrow yexception() << "JSON number is infinite"; } else { @@ -492,6 +493,7 @@ public: private: TJsonIndex Json; + bool AllowInf; }; void DomToJsonIndex(const NUdf::TUnboxedValue& value, TBinaryJsonCallbacks& callbacks) { @@ -573,8 +575,14 @@ template <typename TOnDemandValue> switch (value.get_number_type()) { case simdjson::builtin::number_type::floating_point_number: { double v; - RETURN_IF_NOT_SUCCESS(value.get(v)); - callbacks.OnDouble(v); + if (const auto& error = value.get(v); Y_UNLIKELY(error != simdjson::SUCCESS)) { + if (!NYql::TryDoubleFromString((std::string_view)value.raw_json_token(), v)) { + return error; + } + }; + if (Y_UNLIKELY(!callbacks.OnDouble(v))) { + return simdjson::error_code::NUMBER_ERROR; + } break; } case simdjson::builtin::number_type::signed_integer: { @@ -592,7 +600,9 @@ template <typename TOnDemandValue> case simdjson::builtin::number_type::big_integer: double v; RETURN_IF_NOT_SUCCESS(value.get(v)); - callbacks.OnDouble(v); + if (Y_UNLIKELY(!callbacks.OnDouble(v))) { + return simdjson::error_code::NUMBER_ERROR; + } break; } break; @@ -600,7 +610,7 @@ template <typename TOnDemandValue> case simdjson::ondemand::json_type::null: { auto is_null = value.is_null(); RETURN_IF_NOT_SUCCESS(is_null.error()); - if (Y_UNLIKELY(!is_null.value_unsafe())) { + if (Y_UNLIKELY(!is_null.value_unsafe())) { return simdjson::error_code::N_ATOM_ERROR; } callbacks.OnNull(); @@ -644,7 +654,7 @@ template <typename TOnDemandValue> } // unused, left for performance comparison -[[maybe_unused]] [[nodiscard]] simdjson::error_code SimdJsonToJsonIndexImpl(const simdjson::dom::element& value, TBinaryJsonCallbacks& callbacks) { +[[nodiscard]] [[maybe_unused]] simdjson::error_code SimdJsonToJsonIndexImpl(const simdjson::dom::element& value, TBinaryJsonCallbacks& callbacks) { #define RETURN_IF_NOT_SUCCESS(status) \ if (Y_UNLIKELY(status != simdjson::SUCCESS)) { \ return status; \ @@ -715,9 +725,9 @@ template <typename TOnDemandValue> } } -std::variant<TBinaryJson, TString> SerializeToBinaryJsonImpl(const TStringBuf json) { +std::variant<TBinaryJson, TString> SerializeToBinaryJsonImpl(const TStringBuf json, bool allowInf) { std::variant<TBinaryJson, TString> res; - TBinaryJsonCallbacks callbacks(/* throwException */ false); + TBinaryJsonCallbacks callbacks(/* throwException */ false, allowInf); const simdjson::padded_string paddedJson(json); simdjson::ondemand::parser parser; try { @@ -739,15 +749,16 @@ std::variant<TBinaryJson, TString> SerializeToBinaryJsonImpl(const TStringBuf js return res; } -std::variant<TBinaryJson, TString> SerializeToBinaryJson(const TStringBuf json) { - return SerializeToBinaryJsonImpl(json); +std::variant<TBinaryJson, TString> SerializeToBinaryJson(const TStringBuf json, bool allowInf) { + return SerializeToBinaryJsonImpl(json, allowInf); } TBinaryJson SerializeToBinaryJson(const NUdf::TUnboxedValue& value) { - TBinaryJsonCallbacks callbacks(/* throwException */ false); + TBinaryJsonCallbacks callbacks(/* throwException */ false, /* allowInf */ false); DomToJsonIndex(value, callbacks); TBinaryJsonSerializer serializer(std::move(callbacks).GetResult()); return std::move(serializer).Serialize(); } } + diff --git a/yql/essentials/types/binary_json/write.h b/yql/essentials/types/binary_json/write.h index c4f8f89d2d..61dea9bfb4 100644 --- a/yql/essentials/types/binary_json/write.h +++ b/yql/essentials/types/binary_json/write.h @@ -2,23 +2,24 @@ #include "format.h" -#include <yql/essentials/minikql/dom/node.h> - #include <util/generic/maybe.h> #include <variant> +namespace NYql::NUdf { +class TUnboxedValue; +}; + namespace NKikimr::NBinaryJson { /** * @brief Translates textual JSON into BinaryJson */ -std::variant<TBinaryJson, TString> SerializeToBinaryJson(const TStringBuf json); +std::variant<TBinaryJson, TString> SerializeToBinaryJson(const TStringBuf json, bool allowInf = false); /** * @brief Translates DOM layout from `yql/library/dom` library into BinaryJson */ -TBinaryJson SerializeToBinaryJson(const NUdf::TUnboxedValue& value); - +TBinaryJson SerializeToBinaryJson(const NYql::NUdf::TUnboxedValue& value); } diff --git a/yql/essentials/udfs/language/yql/yql_language_udf.cpp b/yql/essentials/udfs/language/yql/yql_language_udf.cpp index 356ecfade3..ab1090a108 100644 --- a/yql/essentials/udfs/language/yql/yql_language_udf.cpp +++ b/yql/essentials/udfs/language/yql/yql_language_udf.cpp @@ -1,10 +1,13 @@ #include <yql/essentials/public/udf/udf_helpers.h> +#include <yql/essentials/sql/v1/context.h> +#include <yql/essentials/sql/v1/sql_translation.h> #include <yql/essentials/sql/v1/lexer/antlr4/lexer.h> #include <yql/essentials/sql/v1/lexer/antlr4_ansi/lexer.h> #include <yql/essentials/sql/v1/proto_parser/proto_parser.h> #include <yql/essentials/sql/v1/proto_parser/antlr4/proto_parser.h> #include <yql/essentials/sql/v1/proto_parser/antlr4_ansi/proto_parser.h> +#include <yql/essentials/parser/proto_ast/gen/v1_proto_split/SQLv1Parser.pb.main.h> #include <yql/essentials/sql/v1/format/sql_format.h> #include <yql/essentials/sql/settings/translation_settings.h> #include <library/cpp/protobuf/util/simple_reflection.h> @@ -12,18 +15,34 @@ using namespace NYql; using namespace NKikimr::NUdf; using namespace NSQLTranslation; +using namespace NSQLTranslationV1; +using namespace NSQLv1Generated; + +class TRuleFreqTranslation : public TSqlTranslation +{ +public: + TRuleFreqTranslation(TContext& ctx) + : TSqlTranslation(ctx, ctx.Settings.Mode) + {} +}; class TRuleFreqVisitor { public: - TRuleFreqVisitor() { + TRuleFreqVisitor(TContext& ctx) + : Translation(ctx) + { } void Visit(const NProtoBuf::Message& msg) { const NProtoBuf::Descriptor* descr = msg.GetDescriptor(); - if (descr->name() == "TToken") { + if (descr == TToken::GetDescriptor()) { return; } + if (descr == TRule_use_stmt::GetDescriptor()) { + VisitUseStmt(dynamic_cast<const TRule_use_stmt&>(msg)); + } + TStringBuf fullName = descr->full_name(); fullName.SkipPrefix("NSQLv1Generated."); for (int i = 0; i < descr->field_count(); ++i) { @@ -51,6 +70,17 @@ public: } private: + void VisitUseStmt(const TRule_use_stmt& msg) { + const auto& cluster = msg.GetRule_cluster_expr2(); + if (cluster.GetBlock2().Alt_case() == TRule_cluster_expr::TBlock2::kAlt1) { + const auto& val = cluster.GetBlock2().GetAlt1().GetRule_pure_column_or_named1(); + if (val.Alt_case() == TRule_pure_column_or_named::kAltPureColumnOrNamed2) { + const auto& id = val.GetAlt_pure_column_or_named2().GetRule_an_id1(); + Freqs[std::make_pair("USE", Id(id, Translation))] += 1; + } + } + } + void VisitAllFields(const NProtoBuf::Message& msg, const NProtoBuf::Descriptor* descr) { for (int i = 0; i < descr->field_count(); ++i) { const NProtoBuf::FieldDescriptor* fd = descr->field(i); @@ -64,6 +94,7 @@ private: } THashMap<std::pair<TString, TString>, ui64> Freqs; + TRuleFreqTranslation Translation; }; SIMPLE_UDF(TObfuscate, TOptional<char*>(TAutoMap<char*>)) { @@ -125,7 +156,8 @@ SIMPLE_UDF(TRuleFreq, TOptional<TRuleFreqResult>(TAutoMap<char*>)) { return {}; } - TRuleFreqVisitor visitor; + TContext ctx(lexers, parsers, settings, {}, issues, query); + TRuleFreqVisitor visitor(ctx); visitor.Visit(*msg); auto listBuilder = valueBuilder->NewListBuilder(); diff --git a/yt/yql/providers/yt/codec/yt_arrow_converter.cpp b/yt/yql/providers/yt/codec/yt_arrow_converter.cpp index 7d85493324..29bfc40ad9 100644 --- a/yt/yql/providers/yt/codec/yt_arrow_converter.cpp +++ b/yt/yql/providers/yt/codec/yt_arrow_converter.cpp @@ -471,6 +471,10 @@ struct TComplexTypeYsonReaderTraits { ythrow yexception() << "Complex type Yson reader not implemented for block resources"; } + static std::unique_ptr<TResult> MakeSingular() { + ythrow yexception() << "Complex type Yson reader not implemented for singular types."; + } + template<typename TTzDate> static std::unique_ptr<TResult> MakeTzDate(bool isOptional) { if (isOptional) { diff --git a/yt/yql/providers/yt/gateway/qplayer/yql_yt_qplayer_gateway.cpp b/yt/yql/providers/yt/gateway/qplayer/yql_yt_qplayer_gateway.cpp index 3a9bd3272b..f1f70ad43b 100644 --- a/yt/yql/providers/yt/gateway/qplayer/yql_yt_qplayer_gateway.cpp +++ b/yt/yql/providers/yt/gateway/qplayer/yql_yt_qplayer_gateway.cpp @@ -247,7 +247,7 @@ public: } } } - data.WriteLock = options.ReadOnly() ? false : valueNode["WriteLock"].AsBool(); + data.WriteLock = valueNode["WriteLock"].AsBool(); res.Data.push_back(data); } diff --git a/yt/yt/client/table_client/config.cpp b/yt/yt/client/table_client/config.cpp index b8bb96c164..64f4502e08 100644 --- a/yt/yt/client/table_client/config.cpp +++ b/yt/yt/client/table_client/config.cpp @@ -153,6 +153,9 @@ void TChunkWriterConfig::Register(TRegistrar registrar) .InRange(0.0, 0.001) .Default(0.0001); + registrar.Parameter("use_original_data_weight_in_samples", &TThis::UseOriginalDataWeightInSamples) + .Default(false); + registrar.Parameter("chunk_indexes", &TThis::ChunkIndexes) .DefaultNew(); diff --git a/yt/yt/client/table_client/config.h b/yt/yt/client/table_client/config.h index fa096afa53..0b1c970012 100644 --- a/yt/yt/client/table_client/config.h +++ b/yt/yt/client/table_client/config.h @@ -148,6 +148,7 @@ struct TChunkWriterConfig i64 MaxDataWeightBetweenBlocks; double SampleRate; + bool UseOriginalDataWeightInSamples; bool EnableLargeColumnarStatistics; diff --git a/yt/yt/client/table_client/helpers.cpp b/yt/yt/client/table_client/helpers.cpp index 857893c458..b2ef61558f 100644 --- a/yt/yt/client/table_client/helpers.cpp +++ b/yt/yt/client/table_client/helpers.cpp @@ -1607,13 +1607,16 @@ TUnversionedValueRangeTruncationResult TruncateUnversionedValues( std::vector<TUnversionedValue> truncatedValues; truncatedValues.reserve(values.size()); + i64 inputSize = 0; int truncatableValueCount = 0; i64 remainingSize = options.MaxTotalSize; for (const auto& value : values) { + auto valueSize = EstimateRowValueSize(value); + inputSize += valueSize; if (IsStringLikeType(value.Type)) { ++truncatableValueCount; } else { - remainingSize -= EstimateRowValueSize(value); + remainingSize -= valueSize; } } @@ -1656,7 +1659,9 @@ TUnversionedValueRangeTruncationResult TruncateUnversionedValues( } } - return {MakeSharedRange(std::move(truncatedValues), rowBuffer), resultSize, clipped}; + auto sampleSize = options.UseOriginalDataWeightInSamples ? inputSize : resultSize; + + return {MakeSharedRange(std::move(truncatedValues), rowBuffer), sampleSize, clipped}; } //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/client/table_client/helpers.h b/yt/yt/client/table_client/helpers.h index 36bbff3498..3b85e1edfb 100644 --- a/yt/yt/client/table_client/helpers.h +++ b/yt/yt/client/table_client/helpers.h @@ -377,6 +377,7 @@ struct TUnversionedValueRangeTruncationOptions //! Otherwise, all values of primitive (not string-like) types are preserved and the remaining size //! is uniformely distributed between truncated versions of the remaining string-like values. bool ClipAfterOverflow = false; + bool UseOriginalDataWeightInSamples = false; //! Limits the total size of the resulting value range. //! See value-preservation rules described above. i64 MaxTotalSize = NTableClient::MaxSampleSize; diff --git a/yt/yt/core/concurrency/fair_throttler.cpp b/yt/yt/core/concurrency/fair_throttler.cpp index 3899c949e7..9a5cf3c697 100644 --- a/yt/yt/core/concurrency/fair_throttler.cpp +++ b/yt/yt/core/concurrency/fair_throttler.cpp @@ -35,7 +35,7 @@ void TFairThrottlerConfig::Register(TRegistrar registrar) registrar.Parameter("global_accumulation_ticks", &TThis::GlobalAccumulationTicks) .Default(5); - registrar.Parameter("ipc_path", &TThis::IPCPath) + registrar.Parameter("ipc_path", &TThis::IpcPath) .Default(); } @@ -95,27 +95,27 @@ static constexpr TStringBuf BucketsFileName = "buckets.v1"; //////////////////////////////////////////////////////////////////////////////// -class TFileIPCBucket - : public IIPCBucket +class TFileIpcBucket + : public IIpcBucket { public: - TFileIPCBucket(const TString& path, bool create) + TFileIpcBucket(const TString& path, bool create) : File_(path, OpenAlways | RdWr) { if (create) { File_.Flock(LOCK_EX | LOCK_NB); } - File_.Resize(sizeof(TBucket)); + File_.Resize(sizeof(TState)); Map_ = std::make_unique<TFileMap>(File_, TMemoryMapCommon::oRdWr); Map_->Map(0, Map_->Length()); LockMemory(Map_->Ptr(), Map_->Length()); } - TBucket* State() override + TState* GetState() override { - return reinterpret_cast<TBucket*>(Map_->Ptr()); + return reinterpret_cast<TState*>(Map_->Ptr()); } private: @@ -123,15 +123,15 @@ private: std::unique_ptr<TFileMap> Map_; }; -DEFINE_REFCOUNTED_TYPE(TFileIPCBucket) +DEFINE_REFCOUNTED_TYPE(TFileIpcBucket) //////////////////////////////////////////////////////////////////////////////// -class TFileThrottlerIPC - : public IThrottlerIPC +class TFileThrottlerIpc + : public IThrottlerIpc { public: - explicit TFileThrottlerIPC(const TString& path) + explicit TFileThrottlerIpc(const TString& path) : Path_(path) { NFS::MakeDirRecursive(Path_); @@ -166,7 +166,7 @@ public: return reinterpret_cast<TSharedBucket*>(SharedBucketMap_->Ptr()); } - std::vector<IIPCBucketPtr> ListBuckets() override + std::vector<IIpcBucketPtr> ListBuckets() override { Lock_->Acquire(); auto release = Finally([this] { @@ -175,14 +175,14 @@ public: Reload(); - std::vector<IIPCBucketPtr> buckets; + std::vector<IIpcBucketPtr> buckets; for (const auto& bucket : OpenBuckets_) { buckets.push_back(bucket.second); } return buckets; } - IIPCBucketPtr AddBucket() override + IIpcBucketPtr AddBucket() override { Lock_->Acquire(); auto release = Finally([this] { @@ -191,7 +191,7 @@ public: auto id = TGuid::Create(); OwnedBuckets_.insert(ToString(id)); - return New<TFileIPCBucket>(Path_ + "/" + BucketsFileName + "/" + ToString(id), true); + return New<TFileIpcBucket>(Path_ + "/" + BucketsFileName + "/" + ToString(id), true); } private: @@ -202,8 +202,8 @@ private: TFile SharedBucketFile_; std::unique_ptr<TFileMap> SharedBucketMap_; - THashMap<TString, IIPCBucketPtr> OpenBuckets_; - THashSet<TString> OwnedBuckets_; + THashMap<std::string, IIpcBucketPtr> OpenBuckets_; + THashSet<std::string> OwnedBuckets_; void Reload() { @@ -230,7 +230,7 @@ private: continue; } - OpenBuckets_[fileName] = New<TFileIPCBucket>(bucketPath, false); + OpenBuckets_[fileName] = New<TFileIpcBucket>(bucketPath, false); } catch (const TSystemError& ex) { continue; } @@ -238,12 +238,12 @@ private: } }; -IThrottlerIPCPtr CreateFileThrottlerIPC(const TString& path) +IThrottlerIpcPtr CreateFileThrottlerIpc(const TString& path) { - return New<TFileThrottlerIPC>(path); + return New<TFileThrottlerIpc>(path); } -DEFINE_REFCOUNTED_TYPE(TFileThrottlerIPC) +DEFINE_REFCOUNTED_TYPE(TFileThrottlerIpc) //////////////////////////////////////////////////////////////////////////////// @@ -623,12 +623,12 @@ TFairThrottler::TFairThrottler( , SharedBucket_(New<TSharedBucket>(config->GlobalAccumulationTicks, Profiler_)) , Config_(std::move(config)) { - if (Config_->IPCPath) { - IPC_ = New<TFileThrottlerIPC>(*Config_->IPCPath); + if (Config_->IpcPath) { + Ipc_ = New<TFileThrottlerIpc>(*Config_->IpcPath); SharedBucket_->Limit.Value = std::shared_ptr<std::atomic<i64>>( - &IPC_->State()->Value, - [ipc = IPC_] (auto /*ptr*/) { }); + &Ipc_->State()->Value, + [state = Ipc_] (auto /*ptr*/) { }); Profiler_.AddFuncGauge("/leader", MakeStrong(this), [this] { return IsLeader_.load(); @@ -646,7 +646,7 @@ TFairThrottler::TFairThrottler( } IThroughputThrottlerPtr TFairThrottler::CreateBucketThrottler( - const TString& name, + const std::string& name, TFairThrottlerBucketConfigPtr config) { if (!config) { @@ -670,15 +670,15 @@ IThroughputThrottlerPtr TFairThrottler::CreateBucketThrottler( throttler->SetLimited(config->Limit || config->RelativeLimit); - IIPCBucketPtr ipc; - if (IPC_) { - ipc = IPC_->AddBucket(); + IIpcBucketPtr state; + if (Ipc_) { + state = Ipc_->AddBucket(); } Buckets_[name] = TBucket{ .Config = std::move(config), .Throttler = throttler, - .IPC = ipc, + .Ipc = state, }; return throttler; @@ -686,7 +686,7 @@ IThroughputThrottlerPtr TFairThrottler::CreateBucketThrottler( void TFairThrottler::Reconfigure( TFairThrottlerConfigPtr config, - const THashMap<TString, TFairThrottlerBucketConfigPtr>& buckets) + const THashMap<std::string, TFairThrottlerBucketConfigPtr>& buckets) { for (const auto& [name, config] : buckets) { CreateBucketThrottler(name, config); @@ -712,7 +712,7 @@ void TFairThrottler::DoUpdateLeader() std::vector<TBucketThrottler::TBucketState> states; states.reserve(Buckets_.size()); - THashMap<TString, i64> bucketDemands; + THashMap<std::string, i64> bucketDemands; for (const auto& [name, bucket] : Buckets_) { auto state = bucket.Throttler->Peek(); @@ -736,12 +736,12 @@ void TFairThrottler::DoUpdateLeader() states.push_back(state); } - std::vector<IIPCBucketPtr> remoteBuckets; - if (IPC_) { - remoteBuckets = IPC_->ListBuckets(); + std::vector<IIpcBucketPtr> remoteBuckets; + if (Ipc_) { + remoteBuckets = Ipc_->ListBuckets(); for (const auto& remote : remoteBuckets) { - auto state = remote->State(); + auto state = remote->GetState(); weights.push_back(state->Weight.load()); @@ -771,9 +771,9 @@ void TFairThrottler::DoUpdateLeader() } auto freeIncome = ComputeFairDistribution(freeQuota, weights, demands, limits); - THashMap<TString, i64> bucketUsage; - THashMap<TString, i64> bucketIncome; - THashMap<TString, i64> bucketQuota; + THashMap<std::string, i64> bucketUsage; + THashMap<std::string, i64> bucketIncome; + THashMap<std::string, i64> bucketQuota; i64 leakedQuota = 0; int i = 0; @@ -792,7 +792,7 @@ void TFairThrottler::DoUpdateLeader() } for (const auto& remote : remoteBuckets) { - auto state = remote->State(); + auto state = remote->GetState(); state->InFlow += tickIncome[i] + freeIncome[i]; @@ -828,41 +828,41 @@ void TFairThrottler::DoUpdateFollower() { auto guard = Guard(Lock_); - THashMap<TString, i64> bucketIncome; - THashMap<TString, i64> bucketUsage; - THashMap<TString, i64> bucketDemands; - THashMap<TString, i64> bucketQuota; + THashMap<std::string, i64> bucketIncome; + THashMap<std::string, i64> bucketUsage; + THashMap<std::string, i64> bucketDemands; + THashMap<std::string, i64> bucketQuota; i64 inFlow = 0; i64 outFlow = 0; for (const auto& [name, bucket] : Buckets_) { - auto ipc = bucket.IPC->State(); + auto* ipcState = bucket.Ipc->GetState(); - ipc->Weight = bucket.Config->Weight; + ipcState->Weight = bucket.Config->Weight; if (auto limit = bucket.Config->GetLimit(Config_->TotalLimit)) { - ipc->Limit = *limit; + ipcState->Limit = *limit; } else { - ipc->Limit = -1; + ipcState->Limit = -1; } - auto state = bucket.Throttler->Peek(); + auto bucketState = bucket.Throttler->Peek(); auto guarantee = bucket.Config->GetGuarantee(Config_->TotalLimit); - auto demand = state.Usage + state.Overdraft + state.QueueSize; + auto demand = bucketState.Usage + bucketState.Overdraft + bucketState.QueueSize; if (guarantee && *guarantee > demand) { demand = *guarantee; } - ipc->Demand = demand; + ipcState->Demand = demand; bucketDemands[name] = demand; - auto in = ipc->InFlow.exchange(0); - auto out = bucket.Throttler->Refill(in, ipc->GuaranteedQuota); - ipc->OutFlow += out; + auto in = ipcState->InFlow.exchange(0); + auto out = bucket.Throttler->Refill(in, ipcState->GuaranteedQuota); + ipcState->OutFlow += out; bucketIncome[name] = in; - bucketUsage[name] = state.Usage; - bucketQuota[name] = state.Quota; + bucketUsage[name] = bucketState.Usage; + bucketQuota[name] = bucketState.Quota; inFlow += in; outFlow += out; @@ -906,7 +906,7 @@ void TFairThrottler::RefillFromSharedBucket() void TFairThrottler::UpdateLimits(TInstant at) { - if (!IsLeader_ && IPC_->TryLock()) { + if (!IsLeader_ && Ipc_->TryLock()) { IsLeader_ = true; YT_LOG_DEBUG("Throttler is leader"); diff --git a/yt/yt/core/concurrency/fair_throttler.h b/yt/yt/core/concurrency/fair_throttler.h index ecf32ae14b..801ffaef6f 100644 --- a/yt/yt/core/concurrency/fair_throttler.h +++ b/yt/yt/core/concurrency/fair_throttler.h @@ -19,7 +19,7 @@ struct TFairThrottlerConfig int GlobalAccumulationTicks; - std::optional<TString> IPCPath; + std::optional<TString> IpcPath; REGISTER_YSON_STRUCT(TFairThrottlerConfig); @@ -52,11 +52,11 @@ DEFINE_REFCOUNTED_TYPE(TFairThrottlerBucketConfig) //////////////////////////////////////////////////////////////////////////////// -struct IIPCBucket +struct IIpcBucket : public TRefCounted { // NB: This struct is shared between processes. All changes must be backward compatible. - struct TBucket + struct TState { std::atomic<double> Weight; std::atomic<i64> Limit; @@ -66,14 +66,14 @@ struct IIPCBucket std::atomic<i64> GuaranteedQuota; }; - virtual TBucket* State() = 0; + virtual TState* GetState() = 0; }; -DEFINE_REFCOUNTED_TYPE(IIPCBucket) +DEFINE_REFCOUNTED_TYPE(IIpcBucket) //////////////////////////////////////////////////////////////////////////////// -struct IThrottlerIPC +struct IThrottlerIpc : public TRefCounted { // NB: This struct is shared between processes. All changes must be backward compatible. @@ -84,13 +84,13 @@ struct IThrottlerIPC virtual bool TryLock() = 0; virtual TSharedBucket* State() = 0; - virtual std::vector<IIPCBucketPtr> ListBuckets() = 0; - virtual IIPCBucketPtr AddBucket() = 0; + virtual std::vector<IIpcBucketPtr> ListBuckets() = 0; + virtual IIpcBucketPtr AddBucket() = 0; }; -IThrottlerIPCPtr CreateFileThrottlerIPC(const TString& path); +IThrottlerIpcPtr CreateFileThrottlerIpc(const TString& path); -DEFINE_REFCOUNTED_TYPE(IThrottlerIPC) +DEFINE_REFCOUNTED_TYPE(IThrottlerIpc) //////////////////////////////////////////////////////////////////////////////// @@ -116,12 +116,12 @@ public: NProfiling::TProfiler profiler); IThroughputThrottlerPtr CreateBucketThrottler( - const TString& name, + const std::string& name, TFairThrottlerBucketConfigPtr config); void Reconfigure( TFairThrottlerConfigPtr config, - const THashMap<TString, TFairThrottlerBucketConfigPtr>& bucketConfigs); + const THashMap<std::string, TFairThrottlerBucketConfigPtr>& bucketConfigs); static std::vector<i64> ComputeFairDistribution( i64 totalLimit, @@ -140,15 +140,15 @@ private: { TFairThrottlerBucketConfigPtr Config; TBucketThrottlerPtr Throttler; - IIPCBucketPtr IPC; + IIpcBucketPtr Ipc; }; // Protects all Config_ and Buckets_. YT_DECLARE_SPIN_LOCK(NThreading::TSpinLock, Lock_); TFairThrottlerConfigPtr Config_; - THashMap<TString, TBucket> Buckets_; + THashMap<std::string, TBucket> Buckets_; - IThrottlerIPCPtr IPC_; + IThrottlerIpcPtr Ipc_; void DoUpdateLeader(); void DoUpdateFollower(); diff --git a/yt/yt/core/concurrency/public.h b/yt/yt/core/concurrency/public.h index 10ffdda916..4b547a74fc 100644 --- a/yt/yt/core/concurrency/public.h +++ b/yt/yt/core/concurrency/public.h @@ -120,8 +120,8 @@ DECLARE_REFCOUNTED_STRUCT(TFiberManagerDynamicConfig) DECLARE_REFCOUNTED_STRUCT(TFairThrottlerConfig) DECLARE_REFCOUNTED_STRUCT(TFairThrottlerBucketConfig) -DECLARE_REFCOUNTED_STRUCT(IThrottlerIPC) -DECLARE_REFCOUNTED_STRUCT(IIPCBucket) +DECLARE_REFCOUNTED_STRUCT(IThrottlerIpc) +DECLARE_REFCOUNTED_STRUCT(IIpcBucket) DECLARE_REFCOUNTED_CLASS(TFairThrottler) DECLARE_REFCOUNTED_CLASS(TBucketThrottler) diff --git a/yt/yt/core/concurrency/unittests/fair_throttler_ut.cpp b/yt/yt/core/concurrency/unittests/fair_throttler_ut.cpp index 41ee845af7..2a6528bd31 100644 --- a/yt/yt/core/concurrency/unittests/fair_throttler_ut.cpp +++ b/yt/yt/core/concurrency/unittests/fair_throttler_ut.cpp @@ -274,7 +274,7 @@ TEST_F(TFairThrottlerTest, Release) //////////////////////////////////////////////////////////////////////////////// -struct TFairThrottlerIPCTest +struct TFairThrottlerIpcTest : public ::testing::Test { TFairThrottlerConfigPtr Config = New<TFairThrottlerConfig>(); @@ -282,11 +282,11 @@ struct TFairThrottlerIPCTest TFairThrottlerPtr DatNode, ExeNode; - TFairThrottlerIPCTest() + TFairThrottlerIpcTest() { - TString testName = ::testing::UnitTest::GetInstance()->current_test_info()->name(); + std::string testName = ::testing::UnitTest::GetInstance()->current_test_info()->name(); - Config->IPCPath = GetOutputPath() / (testName + ".throttler"); + Config->IpcPath = GetOutputPath() / (testName + ".throttler"); Config->TotalLimit = 100; auto logger = Logger().WithTag("Test: %v", testName); @@ -296,7 +296,7 @@ struct TFairThrottlerIPCTest } }; -TEST_F(TFairThrottlerIPCTest, TwoBucket) +TEST_F(TFairThrottlerIpcTest, TwoBucket) { auto first = DatNode->CreateBucketThrottler("first", BucketConfig); auto second = ExeNode->CreateBucketThrottler("second", BucketConfig); @@ -322,12 +322,12 @@ TEST_F(TFairThrottlerIPCTest, TwoBucket) //////////////////////////////////////////////////////////////////////////////// #ifndef _win_ -TEST(TFileIPC, Test) +TEST(TFileIpcTest, Test) { auto path = GetOutputPath() / "test_ipc"; - auto a = CreateFileThrottlerIPC(path); - auto b = CreateFileThrottlerIPC(path); + auto a = CreateFileThrottlerIpc(path); + auto b = CreateFileThrottlerIpc(path); ASSERT_TRUE(a->TryLock()); ASSERT_FALSE(b->TryLock()); diff --git a/yt/yt/core/http/server.cpp b/yt/yt/core/http/server.cpp index d2fda153b5..43ca58c795 100644 --- a/yt/yt/core/http/server.cpp +++ b/yt/yt/core/http/server.cpp @@ -12,6 +12,7 @@ #include <yt/yt/core/concurrency/thread_pool_poller.h> #include <yt/yt/core/misc/finally.h> +#include <yt/yt/core/misc/memory_usage_tracker.h> #include <yt/yt/core/misc/public.h> #include <yt/yt/core/ytree/convert.h> @@ -62,6 +63,7 @@ public: IPollerPtr poller, IPollerPtr acceptor, IInvokerPtr invoker, + IMemoryUsageTrackerPtr memoryUsageTracker, IRequestPathMatcherPtr requestPathMatcher, bool ownPoller = false) : Config_(std::move(config)) @@ -69,6 +71,7 @@ public: , Poller_(std::move(poller)) , Acceptor_(std::move(acceptor)) , Invoker_(std::move(invoker)) + , MemoryUsageTracker_(std::move(memoryUsageTracker)) , OwnPoller_(ownPoller) , RequestPathMatcher_(std::move(requestPathMatcher)) { } @@ -123,6 +126,7 @@ private: const IPollerPtr Poller_; const IPollerPtr Acceptor_; const IInvokerPtr Invoker_; + const IMemoryUsageTrackerPtr MemoryUsageTracker_; const bool OwnPoller_ = false; IRequestPathMatcherPtr RequestPathMatcher_; @@ -291,7 +295,8 @@ private: connection->GetRemoteAddress(), GetCurrentInvoker(), EMessageType::Request, - Config_); + Config_, + MemoryUsageTracker_); if (Config_->IsHttps) { request->SetHttps(); @@ -302,7 +307,8 @@ private: auto response = New<THttpOutput>( connection, EMessageType::Response, - Config_); + Config_, + MemoryUsageTracker_); while (true) { auto requestId = TRequestId::Create(); @@ -381,6 +387,7 @@ IServerPtr CreateServer( IPollerPtr poller, IPollerPtr acceptor, IInvokerPtr invoker, + IMemoryUsageTrackerPtr memoryUsageTracker, bool ownPoller) { auto handlers = New<TRequestPathMatcher>(); @@ -390,6 +397,7 @@ IServerPtr CreateServer( std::move(poller), std::move(acceptor), std::move(invoker), + std::move(memoryUsageTracker), std::move(handlers), ownPoller); } @@ -399,19 +407,15 @@ IServerPtr CreateServer( IPollerPtr poller, IPollerPtr acceptor, IInvokerPtr invoker, + IMemoryUsageTrackerPtr memoryUsageTracker, bool ownPoller) { auto address = TNetworkAddress::CreateIPv6Any(config->Port); + IListenerPtr listener; for (int i = 0;; ++i) { try { - auto listener = CreateListener(address, poller, acceptor, config->MaxBacklogSize); - return CreateServer( - std::move(config), - std::move(listener), - std::move(poller), - std::move(acceptor), - std::move(invoker), - ownPoller); + listener = CreateListener(address, poller, acceptor, config->MaxBacklogSize); + break; } catch (const std::exception& ex) { if (i + 1 == config->BindRetryCount) { throw; @@ -421,6 +425,14 @@ IServerPtr CreateServer( } } } + return CreateServer( + std::move(config), + std::move(listener), + std::move(poller), + std::move(acceptor), + std::move(invoker), + std::move(memoryUsageTracker), + ownPoller); } } // namespace @@ -440,6 +452,7 @@ IServerPtr CreateServer( std::move(poller), std::move(acceptor), std::move(invoker), + /*memoryUsageTracker*/ GetNullMemoryUsageTracker(), /*ownPoller*/ false); } @@ -447,7 +460,8 @@ IServerPtr CreateServer( TServerConfigPtr config, IListenerPtr listener, IPollerPtr poller, - IPollerPtr acceptor) + IPollerPtr acceptor, + IMemoryUsageTrackerPtr memoryUsageTracker) { auto invoker = poller->GetInvoker(); return CreateServer( @@ -456,13 +470,15 @@ IServerPtr CreateServer( std::move(poller), std::move(acceptor), std::move(invoker), + std::move(memoryUsageTracker), /*ownPoller*/ false); } IServerPtr CreateServer( TServerConfigPtr config, IPollerPtr poller, - IPollerPtr acceptor) + IPollerPtr acceptor, + IMemoryUsageTrackerPtr memoryUsageTracker) { auto invoker = poller->GetInvoker(); return CreateServer( @@ -470,6 +486,7 @@ IServerPtr CreateServer( std::move(poller), std::move(acceptor), std::move(invoker), + std::move(memoryUsageTracker), /*ownPoller*/ false); } @@ -499,6 +516,7 @@ IServerPtr CreateServer(TServerConfigPtr config, int pollerThreadCount) std::move(poller), std::move(acceptor), std::move(invoker), + /*memoryUsageTracker*/ GetNullMemoryUsageTracker(), /*ownPoller*/ true); } @@ -513,6 +531,7 @@ IServerPtr CreateServer( std::move(poller), std::move(acceptor), std::move(invoker), + /*memoryUsageTracker*/ GetNullMemoryUsageTracker(), /*ownPoller*/ false); } diff --git a/yt/yt/core/http/server.h b/yt/yt/core/http/server.h index e96720981b..171fb70399 100644 --- a/yt/yt/core/http/server.h +++ b/yt/yt/core/http/server.h @@ -89,14 +89,16 @@ IServerPtr CreateServer( TServerConfigPtr config, NNet::IListenerPtr listener, NConcurrency::IPollerPtr poller, - NConcurrency::IPollerPtr acceptor); + NConcurrency::IPollerPtr acceptor, + IMemoryUsageTrackerPtr memoryTracker = GetNullMemoryUsageTracker()); IServerPtr CreateServer( TServerConfigPtr config, NConcurrency::IPollerPtr poller); IServerPtr CreateServer( TServerConfigPtr config, NConcurrency::IPollerPtr poller, - NConcurrency::IPollerPtr acceptor); + NConcurrency::IPollerPtr acceptor, + IMemoryUsageTrackerPtr memoryTracker = GetNullMemoryUsageTracker()); IServerPtr CreateServer( int port, NConcurrency::IPollerPtr poller); diff --git a/yt/yt/core/http/stream.cpp b/yt/yt/core/http/stream.cpp index ec76d4e845..7f059df841 100644 --- a/yt/yt/core/http/stream.cpp +++ b/yt/yt/core/http/stream.cpp @@ -252,12 +252,14 @@ THttpInput::THttpInput( const TNetworkAddress& remoteAddress, IInvokerPtr readInvoker, EMessageType messageType, - THttpIOConfigPtr config) + THttpIOConfigPtr config, + IMemoryUsageTrackerPtr memoryUsageTracker) : Connection_(std::move(connection)) , RemoteAddress_(remoteAddress) , MessageType_(messageType) , Config_(std::move(config)) , ReadInvoker_(std::move(readInvoker)) + , MemoryUsageTracker_(std::move(memoryUsageTracker)) , InputBuffer_(TSharedMutableRef::Allocate<THttpParserTag>(Config_->ReadBufferSize)) , Parser_(messageType == EMessageType::Request ? HTTP_REQUEST : HTTP_RESPONSE) , StartByteCount_(Connection_->GetReadByteCount()) @@ -514,7 +516,8 @@ TSharedRef THttpInput::DoRead() auto chunk = Parser_.GetLastBodyChunk(); if (!chunk.Empty()) { Connection_->SetReadDeadline(std::nullopt); - return chunk; + auto trackedChunk = TrackMemory(MemoryUsageTracker_, chunk); + return trackedChunk; } bool eof = false; @@ -576,11 +579,13 @@ THttpOutput::THttpOutput( THeadersPtr headers, IConnectionPtr connection, EMessageType messageType, - THttpIOConfigPtr config) + THttpIOConfigPtr config, + IMemoryUsageTrackerPtr memoryUsageTracker) : Connection_(std::move(connection)) , MessageType_(messageType) , Config_(std::move(config)) , OnWriteFinish_(BIND_NO_PROPAGATE(&THttpOutput::OnWriteFinish, MakeWeak(this))) + , MemoryUsageTracker_(std::move(memoryUsageTracker)) , StartByteCount_(Connection_->GetWriteByteCount()) , StartStatistics_(Connection_->GetWriteStatistics()) , LastProgressLogTime_(TInstant::Now()) @@ -590,12 +595,14 @@ THttpOutput::THttpOutput( THttpOutput::THttpOutput( IConnectionPtr connection, EMessageType messageType, - THttpIOConfigPtr config) + THttpIOConfigPtr config, + IMemoryUsageTrackerPtr memoryUsageTracker) : THttpOutput( New<THeaders>(), std::move(connection), messageType, - std::move(config)) + std::move(config), + std::move(memoryUsageTracker)) { } const THeadersPtr& THttpOutput::GetHeaders() @@ -763,6 +770,8 @@ TFuture<void> THttpOutput::Write(const TSharedRef& data) THROW_ERROR(AnnotateError(TError("Cannot write to finished HTTP message"))); } + auto trackedData = TrackMemory(MemoryUsageTracker_, data); + std::vector<TSharedRef> writeRefs; if (!HeadersFlushed_) { HeadersFlushed_ = true; @@ -770,9 +779,9 @@ TFuture<void> THttpOutput::Write(const TSharedRef& data) writeRefs.push_back(CrLfBuffer()); } - if (!data.Empty()) { - writeRefs.push_back(GetChunkHeader(data.Size())); - writeRefs.push_back(data); + if (!trackedData.Empty()) { + writeRefs.push_back(GetChunkHeader(trackedData.Size())); + writeRefs.push_back(trackedData); writeRefs.push_back(CrLfBuffer()); } diff --git a/yt/yt/core/http/stream.h b/yt/yt/core/http/stream.h index a45ab15f0f..4c11b32e11 100644 --- a/yt/yt/core/http/stream.h +++ b/yt/yt/core/http/stream.h @@ -91,7 +91,8 @@ public: const NNet::TNetworkAddress& remoteAddress, IInvokerPtr readInvoker, EMessageType messageType, - THttpIOConfigPtr config); + THttpIOConfigPtr config, + IMemoryUsageTrackerPtr memoryUsageTracker = GetNullMemoryUsageTracker()); EMethod GetMethod() override; const TUrlRef& GetUrl() override; @@ -136,6 +137,7 @@ private: const EMessageType MessageType_; const THttpIOConfigPtr Config_; const IInvokerPtr ReadInvoker_; + const IMemoryUsageTrackerPtr MemoryUsageTracker_; TSharedMutableRef InputBuffer_; TSharedRef UnconsumedData_; @@ -183,12 +185,14 @@ public: THeadersPtr headers, NNet::IConnectionPtr connection, EMessageType messageType, - THttpIOConfigPtr config); + THttpIOConfigPtr config, + IMemoryUsageTrackerPtr memoryUsageTracker = GetNullMemoryUsageTracker()); THttpOutput( NNet::IConnectionPtr connection, EMessageType messageType, - THttpIOConfigPtr config); + THttpIOConfigPtr config, + IMemoryUsageTrackerPtr memoryUsageTracker = GetNullMemoryUsageTracker()); const THeadersPtr& GetHeaders() override; void SetHeaders(const THeadersPtr& headers); @@ -224,6 +228,7 @@ private: const THttpIOConfigPtr Config_; const TClosure OnWriteFinish_; + const IMemoryUsageTrackerPtr MemoryUsageTracker_; // Debug TRequestId RequestId_; diff --git a/yt/yt/core/https/server.cpp b/yt/yt/core/https/server.cpp index 92bdee379f..b97cb801e8 100644 --- a/yt/yt/core/https/server.cpp +++ b/yt/yt/core/https/server.cpp @@ -112,7 +112,8 @@ IServerPtr CreateServer( const TServerConfigPtr& config, const IPollerPtr& poller, const IPollerPtr& acceptor, - const IInvokerPtr& controlInvoker) + const IInvokerPtr& controlInvoker, + const IMemoryUsageTrackerPtr& memoryTracker) { auto sslContext = New<TSslContext>(); ApplySslConfig(sslContext, config->Credentials); @@ -164,7 +165,8 @@ IServerPtr CreateServer( configCopy, tlsListener, poller, - acceptor); + acceptor, + memoryTracker); return New<TServer>(std::move(httpServer), std::move(certificateUpdater)); } diff --git a/yt/yt/core/https/server.h b/yt/yt/core/https/server.h index c6c40eeec3..46994e8ea4 100644 --- a/yt/yt/core/https/server.h +++ b/yt/yt/core/https/server.h @@ -8,6 +8,8 @@ #include <yt/yt/core/http/public.h> +#include <yt/yt/core/misc/memory_usage_tracker.h> + namespace NYT::NHttps { //////////////////////////////////////////////////////////////////////////////// @@ -26,7 +28,8 @@ NHttp::IServerPtr CreateServer( const TServerConfigPtr& config, const NConcurrency::IPollerPtr& poller, const NConcurrency::IPollerPtr& acceptor, - const IInvokerPtr& controlInvoker); + const IInvokerPtr& controlInvoker, + const IMemoryUsageTrackerPtr& memoryTracker = GetNullMemoryUsageTracker()); //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/core/ytree/fluent.h b/yt/yt/core/ytree/fluent.h index 366ca9794d..ea460e2bcd 100644 --- a/yt/yt/core/ytree/fluent.h +++ b/yt/yt/core/ytree/fluent.h @@ -78,8 +78,7 @@ namespace NYT::NYTree { // `func` only if `condition` is true; // * DoFor(TCollection collection, TFuncAny func) -> TAny, same as Do() // but iterate over `collection` and pass each of its elements as a second -// argument to `func`. Instead of passing a collection you may it is possible -// to pass two iterators as an argument; +// argument to `func`. You may pass two iterators as an argument instead; // // * DoMap(TFuncMap func) -> TAny, open a map, delegate invocation to a separate // procedure and close map; diff --git a/yt/yt/core/ytree/unittests/yson_struct_update_ut.cpp b/yt/yt/core/ytree/unittests/yson_struct_update_ut.cpp index fd82265dd7..5d25807ada 100644 --- a/yt/yt/core/ytree/unittests/yson_struct_update_ut.cpp +++ b/yt/yt/core/ytree/unittests/yson_struct_update_ut.cpp @@ -132,8 +132,8 @@ TEST(TUpdateYsonStructTest, Inherited) auto configurator = TConfigurator<TSpecBase>(); { - TConfigurator<TSpecWithPool> parentRegistrar = configurator; - parentRegistrar.Field("pool", &TSpecBase::Pool) + TConfigurator<TSpecWithPool> parentConfigurator = configurator; + parentConfigurator.Field("pool", &TSpecBase::Pool) .Updater(BIND([&] (const std::string& newPool) { updatedPool = newPool; })); @@ -154,8 +154,8 @@ TEST(TUpdateYsonStructTest, Nested) auto configurator = TConfigurator<TSpecBase>(); { - TConfigurator<TSpecWithPool> parentRegistrar = configurator; - parentRegistrar.Field("pool", &TSpecBase::Pool) + TConfigurator<TSpecWithPool> parentConfigurator = configurator; + parentConfigurator.Field("pool", &TSpecBase::Pool) .Updater(BIND([&] (const std::string& newPool) { updatedPool = newPool; })); diff --git a/yt/yt/core/ytree/yson_struct_update-inl.h b/yt/yt/core/ytree/yson_struct_update-inl.h index c6ae5f1ead..46ad7fe717 100644 --- a/yt/yt/core/ytree/yson_struct_update-inl.h +++ b/yt/yt/core/ytree/yson_struct_update-inl.h @@ -15,24 +15,24 @@ const IYsonStructMeta* GetYsonStructMeta() { return New<TStruct>()->GetMeta(); } -struct TRegisteredFieldDirectory +struct TConfiguredFieldDirectory : public TRefCounted { template <class TStruct> - static TRegisteredFieldDirectoryPtr Create() + static TConfiguredFieldDirectoryPtr Create() { static auto* meta = GetYsonStructMeta<TStruct>(); - auto directory = New<TRegisteredFieldDirectory>(); + auto directory = New<TConfiguredFieldDirectory>(); directory->Meta = meta; return directory; } - THashMap<IYsonStructParameterPtr, IFieldRegistrarPtr> ParameterToFieldRegistrar; + THashMap<IYsonStructParameterPtr, IFieldConfiguratorPtr> ParameterToFieldConfigurator; const IYsonStructMeta* Meta; }; -DEFINE_REFCOUNTED_TYPE(TRegisteredFieldDirectory); +DEFINE_REFCOUNTED_TYPE(TConfiguredFieldDirectory); //////////////////////////////////////////////////////////////////////////////// @@ -49,7 +49,7 @@ struct TUnwrapYsonStructIntrusivePtr<TIntrusivePtr<T>> //////////////////////////////////////////////////////////////////////////////// template <class TValue> -TFieldRegistrar<TValue>& TFieldRegistrar<TValue>::Validator(TCallback<void(const TValue&, const TValue&)> validator) +TFieldConfigurator<TValue>& TFieldConfigurator<TValue>::Validator(TCallback<void(const TValue&, const TValue&)> validator) { VerifyEmptyValidator(); Validator_ = validator; @@ -57,7 +57,7 @@ TFieldRegistrar<TValue>& TFieldRegistrar<TValue>::Validator(TCallback<void(const } template <class TValue> -TFieldRegistrar<TValue>& TFieldRegistrar<TValue>::Validator(TCallback<void(const TValue&)> validator) +TFieldConfigurator<TValue>& TFieldConfigurator<TValue>::Validator(TCallback<void(const TValue&)> validator) { VerifyEmptyValidator(); Validator_ = BIND_NO_PROPAGATE([validator = std::move(validator)] (const TValue& /*oldValue*/, const TValue& newValue) { @@ -67,7 +67,7 @@ TFieldRegistrar<TValue>& TFieldRegistrar<TValue>::Validator(TCallback<void(const } template <class TValue> -TFieldRegistrar<TValue>& TFieldRegistrar<TValue>::Updater(TCallback<void(const TValue&, const TValue&)> updater) +TFieldConfigurator<TValue>& TFieldConfigurator<TValue>::Updater(TCallback<void(const TValue&, const TValue&)> updater) { VerifyEmptyUpdater(); Updater_ = updater; @@ -75,7 +75,7 @@ TFieldRegistrar<TValue>& TFieldRegistrar<TValue>::Updater(TCallback<void(const T } template <class TValue> -TFieldRegistrar<TValue>& TFieldRegistrar<TValue>::Updater(TCallback<void(const TValue&)> updater) +TFieldConfigurator<TValue>& TFieldConfigurator<TValue>::Updater(TCallback<void(const TValue&)> updater) { VerifyEmptyUpdater(); Updater_ = BIND_NO_PROPAGATE([updater = std::move(updater)] (const TValue& /*oldValue*/, const TValue& newValue) { @@ -100,7 +100,7 @@ TFieldRegistrar<TValue>& TFieldRegistrar<TValue>::Updater(TCallback<void(const T // written as TIntrusivePtr<TMyStruct>, so I need to unwrap that to TMyStruct. template <class TValue> template <CYsonStructDerived TUnwrappedValue> -TFieldRegistrar<TValue>& TFieldRegistrar<TValue>::NestedUpdater( +TFieldConfigurator<TValue>& TFieldConfigurator<TValue>::NestedUpdater( TCallback<TSealedConfigurator<TUnwrappedValue>()> configureCallback) { static_assert( @@ -117,7 +117,7 @@ TFieldRegistrar<TValue>& TFieldRegistrar<TValue>::NestedUpdater( } template <class TValue> -void TFieldRegistrar<TValue>::DoValidate( +void TFieldConfigurator<TValue>::DoValidate( IYsonStructParameterPtr parameter, TYsonStructBase* oldStruct, TYsonStructBase* newStruct) const @@ -134,7 +134,7 @@ void TFieldRegistrar<TValue>::DoValidate( } template <class TValue> -void TFieldRegistrar<TValue>::DoUpdate( +void TFieldConfigurator<TValue>::DoUpdate( IYsonStructParameterPtr parameter, TYsonStructBase* oldStruct, TYsonStructBase* newStruct) const @@ -151,13 +151,13 @@ void TFieldRegistrar<TValue>::DoUpdate( } template <class TValue> -void TFieldRegistrar<TValue>::VerifyEmptyValidator() const +void TFieldConfigurator<TValue>::VerifyEmptyValidator() const { YT_VERIFY(!Validator_); } template <class TValue> -void TFieldRegistrar<TValue>::VerifyEmptyUpdater() const +void TFieldConfigurator<TValue>::VerifyEmptyUpdater() const { YT_VERIFY(!Updater_); } @@ -169,32 +169,32 @@ void TFieldRegistrar<TValue>::VerifyEmptyUpdater() const //////////////////////////////////////////////////////////////////////////////// template <CYsonStructDerived TStruct> -TConfigurator<TStruct>::TConfigurator(NDetail::TRegisteredFieldDirectoryPtr registeredFields) - : RegisteredFields_(registeredFields) +TConfigurator<TStruct>::TConfigurator(NDetail::TConfiguredFieldDirectoryPtr configuredFields) + : ConfiguredFields_(configuredFields) { - // NB: Initialize TRegisteredFieldDirectory with + // NB: Initialize TConfiguredFieldDirectory with // the youngest child in the hierarchy. - if (!registeredFields) { - RegisteredFields_ = NDetail::TRegisteredFieldDirectory::Create<TStruct>(); + if (!configuredFields) { + ConfiguredFields_ = NDetail::TConfiguredFieldDirectory::Create<TStruct>(); } } template <CYsonStructDerived TStruct> template <class TValue> -NDetail::TFieldRegistrar<TValue>& TConfigurator<TStruct>::Field(const std::string& name, TYsonStructField<TStruct, TValue> field) +NDetail::TFieldConfigurator<TValue>& TConfigurator<TStruct>::Field(const std::string& name, TYsonStructField<TStruct, TValue> field) { IYsonStructParameterPtr parameter; try { - parameter = RegisteredFields_->Meta->GetParameter(name); + parameter = ConfiguredFields_->Meta->GetParameter(name); } catch (const std::exception& ex) { YT_ABORT(); } YT_VERIFY(parameter->HoldsField(CreateTypeErasedYsonStructField(field))); - auto fieldRegistrar = New<NDetail::TFieldRegistrar<TValue>>(); - RegisteredFields_->ParameterToFieldRegistrar.emplace(parameter, fieldRegistrar); - return *fieldRegistrar; + auto fieldConfigurator = New<NDetail::TFieldConfigurator<TValue>>(); + ConfiguredFields_->ParameterToFieldConfigurator.emplace(parameter, fieldConfigurator); + return *fieldConfigurator; } template <CYsonStructDerived TStruct> @@ -202,7 +202,7 @@ template <class TAncestor> TConfigurator<TStruct>::operator TConfigurator<TAncestor>() const { static_assert(std::derived_from<TStruct, TAncestor> && std::derived_from<TAncestor, TYsonStructBase>); - return TConfigurator<TAncestor>(RegisteredFields_); + return TConfigurator<TAncestor>(ConfiguredFields_); } template <CYsonStructDerived TStruct> @@ -215,7 +215,7 @@ TSealedConfigurator<TStruct> TConfigurator<TStruct>::Seal() && template <CYsonStructDerived TStruct> TSealedConfigurator<TStruct>::TSealedConfigurator(TConfigurator<TStruct> configurator) - : RegisteredFields_(std::move(configurator.RegisteredFields_)) + : ConfiguredFields_(std::move(configurator.ConfiguredFields_)) { } template <CYsonStructDerived TStruct> @@ -223,7 +223,7 @@ void TSealedConfigurator<TStruct>::Validate( TIntrusivePtr<TStruct> oldStruct, TIntrusivePtr<TStruct> newStruct) const { - Do(oldStruct, newStruct, &NDetail::IFieldRegistrar::DoValidate); + Do(oldStruct, newStruct, &NDetail::IFieldConfigurator::DoValidate); } template <CYsonStructDerived TStruct> @@ -231,26 +231,26 @@ void TSealedConfigurator<TStruct>::Update( TIntrusivePtr<TStruct> oldStruct, TIntrusivePtr<TStruct> newStruct) const { - Do(oldStruct, newStruct, &NDetail::IFieldRegistrar::DoUpdate); + Do(oldStruct, newStruct, &NDetail::IFieldConfigurator::DoUpdate); } template <CYsonStructDerived TStruct> void TSealedConfigurator<TStruct>::Do( TIntrusivePtr<TStruct> oldStruct, TIntrusivePtr<TStruct> newStruct, - TFieldRegistrarMethod fieldMethod) const + TFieldConfiguratorMethod fieldMethod) const { const auto* meta = oldStruct->GetMeta(); YT_VERIFY(meta == newStruct->GetMeta()); - const auto& parameterToFieldRegistrar = RegisteredFields_->ParameterToFieldRegistrar; - YT_VERIFY(RegisteredFields_->Meta == meta); + const auto& parameterToFieldConfigurator = ConfiguredFields_->ParameterToFieldConfigurator; + YT_VERIFY(ConfiguredFields_->Meta == meta); for (const auto& [name, parameter] : meta->GetParameterMap()) { if (parameter->CompareParameter(oldStruct.Get(), newStruct.Get())) { continue; } - auto fieldDescIter = parameterToFieldRegistrar.find(parameter); - if (fieldDescIter == parameterToFieldRegistrar.end()) { + auto fieldDescIter = parameterToFieldConfigurator.find(parameter); + if (fieldDescIter == parameterToFieldConfigurator.end()) { THROW_ERROR_EXCEPTION("Field %Qv is not marked as updatable, but was changed", name); } else { (*(fieldDescIter->second).*fieldMethod)(parameter, oldStruct.Get(), newStruct.Get()); diff --git a/yt/yt/core/ytree/yson_struct_update.h b/yt/yt/core/ytree/yson_struct_update.h index 7a517d6f57..1b762b9cc4 100644 --- a/yt/yt/core/ytree/yson_struct_update.h +++ b/yt/yt/core/ytree/yson_struct_update.h @@ -19,11 +19,11 @@ namespace NDetail { //////////////////////////////////////////////////////////////////////////////// -DECLARE_REFCOUNTED_STRUCT(TRegisteredFieldDirectory); +DECLARE_REFCOUNTED_STRUCT(TConfiguredFieldDirectory); //////////////////////////////////////////////////////////////////////////////// -struct IFieldRegistrar +struct IFieldConfigurator : public TRefCounted { virtual void DoValidate( @@ -37,31 +37,31 @@ struct IFieldRegistrar TYsonStructBase* newStruct) const = 0; }; -DECLARE_REFCOUNTED_STRUCT(IFieldRegistrar); -DEFINE_REFCOUNTED_TYPE(IFieldRegistrar); +DECLARE_REFCOUNTED_STRUCT(IFieldConfigurator); +DEFINE_REFCOUNTED_TYPE(IFieldConfigurator); //////////////////////////////////////////////////////////////////////////////// template <class TValue> -class TFieldRegistrar - : public IFieldRegistrar +class TFieldConfigurator + : public IFieldConfigurator { public: // Registers validator that accepts old and new values as arguments. - TFieldRegistrar& Validator(TCallback<void(const TValue&, const TValue&)> validator); + TFieldConfigurator& Validator(TCallback<void(const TValue&, const TValue&)> validator); // Registers validator that accepts only new value as an argument. - TFieldRegistrar& Validator(TCallback<void(const TValue&)> validator); + TFieldConfigurator& Validator(TCallback<void(const TValue&)> validator); // Registers updater that accepts old and new values as arguments. - TFieldRegistrar& Updater(TCallback<void(const TValue&, const TValue&)> updater); + TFieldConfigurator& Updater(TCallback<void(const TValue&, const TValue&)> updater); // Registers updater that accepts only new value as an argument. - TFieldRegistrar& Updater(TCallback<void(const TValue&)> updater); + TFieldConfigurator& Updater(TCallback<void(const TValue&)> updater); // Registers nested YsonStruct to be updated recursively. template <CYsonStructDerived TUnwrappedValue> - TFieldRegistrar& NestedUpdater( + TFieldConfigurator& NestedUpdater( TCallback<TSealedConfigurator<TUnwrappedValue>()> configureCallback); void DoUpdate( @@ -106,19 +106,19 @@ template <CYsonStructDerived TStruct> class TConfigurator { public: - explicit TConfigurator(NDetail::TRegisteredFieldDirectoryPtr state = {}); + explicit TConfigurator(NDetail::TConfiguredFieldDirectoryPtr state = {}); template <class TValue> - NDetail::TFieldRegistrar<TValue>& Field(const std::string& name, TYsonStructField<TStruct, TValue> field); + NDetail::TFieldConfigurator<TValue>& Field(const std::string& name, TYsonStructField<TStruct, TValue> field); - // Converts to a registrar of a base class + // Converts to a configurator of a base class template <class TAncestor> operator TConfigurator<TAncestor>() const; TSealedConfigurator<TStruct> Seal() &&; private: - NDetail::TRegisteredFieldDirectoryPtr RegisteredFields_; + NDetail::TConfiguredFieldDirectoryPtr ConfiguredFields_; template <CYsonStructDerived TStructForSealed> friend class TSealedConfigurator; @@ -141,7 +141,7 @@ public: TIntrusivePtr<TStruct> newStruct) const; private: - using TFieldRegistrarMethod = void(NDetail::IFieldRegistrar::*)( + using TFieldConfiguratorMethod = void(NDetail::IFieldConfigurator::*)( IYsonStructParameterPtr parameter, TYsonStructBase* oldStruct, TYsonStructBase* newStruct) const; @@ -149,9 +149,9 @@ private: void Do( TIntrusivePtr<TStruct> oldStruct, TIntrusivePtr<TStruct> newStruct, - TFieldRegistrarMethod fieldMethod) const; + TFieldConfiguratorMethod fieldMethod) const; - NDetail::TRegisteredFieldDirectoryPtr RegisteredFields_; + NDetail::TConfiguredFieldDirectoryPtr ConfiguredFields_; }; //////////////////////////////////////////////////////////////////////////////// |