diff options
author | Alexander Smirnov <alex@ydb.tech> | 2024-10-31 17:11:37 +0000 |
---|---|---|
committer | Alexander Smirnov <alex@ydb.tech> | 2024-10-31 17:11:37 +0000 |
commit | a4a4e847216dbe1e32056717eb466537d2128bce (patch) | |
tree | 27070fcd1102d57042f56b8eb602789b020a1d04 | |
parent | e13dea6e57e441f5dc2fe09409a2932cdc4f821c (diff) | |
parent | 6c9f2f9532a9812c29ae9f3ed08ad266b93993c0 (diff) | |
download | ydb-a4a4e847216dbe1e32056717eb466537d2128bce.tar.gz |
Merge branch 'rightlib' into mergelibs-241031-1710
206 files changed, 4243 insertions, 1271 deletions
diff --git a/build/export_generators/cmake/generator.toml b/build/export_generators/cmake/generator.toml index 4fcb521f2f..fd4ab84d8a 100644 --- a/build/export_generators/cmake/generator.toml +++ b/build/export_generators/cmake/generator.toml @@ -47,6 +47,10 @@ copy=[ template={ path="dir_cmake_lists.jinja", dest="CMakeLists{PLATFORM}.txt" } merge_platform_template={ path="merge_platforms_cmake_lists.jinja", dest="CMakeLists.txt" } +[targets.EXTRA_ONLY] +template={ path="dir_cmake_lists.jinja", dest="CMakeLists{PLATFORM}.txt" } +merge_platform_template={ path="merge_platforms_cmake_lists.jinja", dest="CMakeLists.txt" } + [targets.add_executable] template={ path="dir_cmake_lists.jinja", dest="CMakeLists{PLATFORM}.txt" } merge_platform_template={ path="merge_platforms_cmake_lists.jinja", dest="CMakeLists.txt" } diff --git a/build/export_generators/ide-gradle/build.gradle.kts.jinja b/build/export_generators/ide-gradle/build.gradle.kts.jinja index 3f7755b94e..3833801b66 100644 --- a/build/export_generators/ide-gradle/build.gradle.kts.jinja +++ b/build/export_generators/ide-gradle/build.gradle.kts.jinja @@ -1,20 +1,26 @@ -{%- set mainClass = target.app_main_class -%} +val baseBuildDir = "{{ export_root }}/gradle.build/" +buildDir = file(baseBuildDir + project.path.replaceFirst(":", "/").replace(":", ".")) +subprojects { + buildDir = file(baseBuildDir + project.path.replaceFirst(":", "/").replace(":", ".")) +} + +{% set mainClass = target.app_main_class -%} {%- set publish = target.publish -%} {%- set with_kotlin = target.with_kotlin -%} {%- set kotlin_version = target.kotlin_version -%} {%- set hasJunit5Test = extra_targets|selectattr('junit5_test') -%} {%- set errorprone_plugin_version = "4.0.0" -%} plugins { -{# some plugins configuration #} +{#- some plugins configuration -#} {%- for library in target.consumer if library.classpath -%} -{# error prone plugin configuration #} +{#- error prone plugin configuration -#} {%- if library.prebuilt and library.jar and (library.type != "contrib" or build_contribs) and "contrib/java/com/google/errorprone/error_prone_annotations/" in library.jar -%} id("net.ltgt.errorprone") version "{{ errorprone_plugin_version }}" {%- endif -%} {%- endfor -%} -{# lombok configuration #} +{#- lombok configuration -#} {%- if "lombok.launch.AnnotationProcessorHider$AnnotationProcessor" in target.annotation_processors %} id("io.freefair.lombok") version "8.6" {%- endif -%} @@ -43,7 +49,7 @@ plugins { {% endif -%} {%- endif %} } -{# language level #} +{#- language level -#} {%- if target.required_jdk is defined and target.required_jdk|length %} java { toolchain { @@ -98,7 +104,7 @@ tasks.withType<Javadoc> { } {% endif %} -{# javac flags #} +{#- javac flags -#} {%- if (target.javac.flags is defined) and (target.javac.flags|length) %} tasks.withType<JavaCompile> { {%- for javac_flag in target.javac.flags %} @@ -113,20 +119,18 @@ tasks.withType<JavaCompile> { val bucketUsername: String by project val bucketPassword: String by project repositories { - repositories { - maven { - url = uri("https://bucket.yandex-team.ru/v1/maven/central") - credentials { - username = "$bucketUsername" - password = "$bucketPassword" + repositories { + maven { + url = uri("https://bucket.yandex-team.ru/v1/maven/central") + credentials { + username = "$bucketUsername" + password = "$bucketPassword" + } } } } - -} - -val project_root="{%- if export_root.startswith(arcadia_root + '/') -%}{{ arcadia_root }}{%- else -%}{{ export_root }}/.hic_sunt_dracones{%- endif -%}" +val project_root = "{{ arcadia_root }}" {% if mainClass -%} application { @@ -269,13 +273,13 @@ tasks.named<Test>("test") { useJUnitPlatform() } -{% endif -%} +{% endif -%} -{# run_java_program #} -{# {% set runs = targets|selectattr("runs") -%} #} -{% set runs = target.runs -%} -{% if runs -%} -{% for run in runs -%} +{#- run_java_program -#} +{#- {% set runs = targets|selectattr("runs") -%} -#} +{%- set runs = target.runs -%} +{%- if runs -%} +{%- for run in runs -%} val runJav{{ loop.index }} = task<JavaExec>("runJavaProgram{{ loop.index }}") { group = "build" @@ -287,7 +291,7 @@ val runJav{{ loop.index }} = task<JavaExec>("runJavaProgram{{ loop.index }}") { {% set real_classpath = classpath|replace('@', '') -%} {% set real_classpath = real_classpath|replace('.run.cp', '') -%} {% set real_classpath = real_classpath|replace('.cplst', '') -%} -{% set real_classpath = real_classpath|replace(export_root, '') -%} +{% set real_classpath = real_classpath|replace(export_root, '')|replace(arcadia_root, '') -%} {% set real_gradle_classpath = real_classpath|replace('/', ':') %} val classPath = "{{ real_gradle_classpath }}" val classPathParts = classPath.split(":") @@ -321,15 +325,15 @@ val runJav{{ loop.index }} = task<JavaExec>("runJavaProgram{{ loop.index }}") { {% for dir in run.out_dir -%} outputs.dir("{{ dir }}") {% endfor -%} -{# +{#- Не использованы аттрибуты run-cwd="str" run-in_dirs_inputs="list" run-in_noparse="list" run-out_dir="list" run-tool="list" -#} -{% endif -%} +-#} +{%- endif -%} } tasks { build { diff --git a/build/export_generators/ide-gradle/build.gradle.kts.proto.jinja b/build/export_generators/ide-gradle/build.gradle.kts.proto.jinja index 9a930779d4..d81a7ea0b0 100644 --- a/build/export_generators/ide-gradle/build.gradle.kts.proto.jinja +++ b/build/export_generators/ide-gradle/build.gradle.kts.proto.jinja @@ -1,6 +1,12 @@ {%- set publish = target.publish -%} import com.google.protobuf.gradle.* +val baseBuildDir = "{{ export_root }}/gradle.build/" +buildDir = file(baseBuildDir + project.path.replaceFirst(":", "/").replace(":", ".")) +subprojects { + buildDir = file(baseBuildDir + project.path.replaceFirst(":", "/").replace(":", ".")) +} + val buildProtoDir = File("${buildDir}", "__proto__") plugins { @@ -9,7 +15,7 @@ plugins { {%- if publish %} `maven-publish` `signing` -{%- endif -%} +{%- endif %} } {%- if publish %} @@ -20,16 +26,16 @@ version = {% if target.publish_version -%}"{{ target.publish_version }}"{%- else val bucketUsername: String by project val bucketPassword: String by project repositories { - repositories { - maven { - url = uri("https://bucket.yandex-team.ru/v1/maven/central") - credentials { - username = "$bucketUsername" - password = "$bucketPassword" + repositories { + maven { + url = uri("https://bucket.yandex-team.ru/v1/maven/central") + credentials { + username = "$bucketUsername" + password = "$bucketPassword" + } } } } -} sourceSets { main { @@ -37,7 +43,7 @@ sourceSets { } } -val project_root="{%- if export_root.startswith(arcadia_root + '/') -%}{{ arcadia_root }}{%- else -%}{{ export_root }}{%- endif -%}" +val project_root = "{{ arcadia_root }}" java { withSourcesJar() @@ -58,10 +64,10 @@ configurations.testImplementation { {%- if target.jar_source_set is defined -%} {%- for source_set in target.jar_source_set -%} -{%- set srcdir_glob = split(source_set, ':') -%} +{%- set srcdir_glob = split(source_set, ':') %} sourceSets.main.java.srcDirs += "{{ srcdir_glob[0] }}" {% endfor -%} -{%- endif -%} +{%- endif %} dependencies { {%- for library in target.consumer if library.classpath -%} diff --git a/build/export_generators/ide-gradle/generator.toml b/build/export_generators/ide-gradle/generator.toml index ea589c40f5..1684f86626 100644 --- a/build/export_generators/ide-gradle/generator.toml +++ b/build/export_generators/ide-gradle/generator.toml @@ -8,9 +8,12 @@ copy=[ "gradlew.bat", "gradle/wrapper/gradle-wrapper.jar", "gradle/wrapper/gradle-wrapper.properties", - ".idea/vcs.xml" + ".idea/vcs.xml", ] +[targets.EXTRA_ONLY] +template="build.gradle.kts.jinja" + [targets.jar] template="build.gradle.kts.jinja" diff --git a/build/export_generators/ide-gradle/settings.gradle.kts.jinja b/build/export_generators/ide-gradle/settings.gradle.kts.jinja index da32f75ab1..a71aefe4f4 100644 --- a/build/export_generators/ide-gradle/settings.gradle.kts.jinja +++ b/build/export_generators/ide-gradle/settings.gradle.kts.jinja @@ -3,7 +3,7 @@ rootProject.name = "{{ project_name }}" {% for subdir in subdirs -%} {%- set classname = subdir | replace("/", ":") -%} include(":{{ classname }}") -project(":{{ classname }}").projectDir = file("{{ export_root }}/{{ subdir }}") +project(":{{ classname }}").projectDir = file("{{ arcadia_root }}/{{ subdir }}") {% endfor -%} {%- include "[generator]/debug.jinja" ignore missing -%} diff --git a/build/external_resources/yexport/public.resources.json b/build/external_resources/yexport/public.resources.json index f304fcf477..eca2298793 100644 --- a/build/external_resources/yexport/public.resources.json +++ b/build/external_resources/yexport/public.resources.json @@ -1,13 +1,13 @@ { "by_platform": { "darwin": { - "uri": "sbr:7196092605" + "uri": "sbr:7370212609" }, "darwin-arm64": { - "uri": "sbr:7196092155" + "uri": "sbr:7370211678" }, "linux": { - "uri": "sbr:7196091589" + "uri": "sbr:7370210775" } } } diff --git a/build/external_resources/yexport/resources.json b/build/external_resources/yexport/resources.json index d2b3ee6f17..2936b9695b 100644 --- a/build/external_resources/yexport/resources.json +++ b/build/external_resources/yexport/resources.json @@ -1,13 +1,13 @@ { "by_platform": { "darwin": { - "uri": "sbr:7196077522" + "uri": "sbr:7370190051" }, "darwin-arm64": { - "uri": "sbr:7196076872" + "uri": "sbr:7370188821" }, "linux": { - "uri": "sbr:7196076129" + "uri": "sbr:7370187741" } } } diff --git a/build/external_resources/ymake/public.resources.json b/build/external_resources/ymake/public.resources.json index 3d0dcf2c0c..5c5c163c9b 100644 --- a/build/external_resources/ymake/public.resources.json +++ b/build/external_resources/ymake/public.resources.json @@ -1,19 +1,19 @@ { "by_platform": { "darwin": { - "uri": "sbr:7299619998" + "uri": "sbr:7367909157" }, "darwin-arm64": { - "uri": "sbr:7299619667" + "uri": "sbr:7367907802" }, "linux": { - "uri": "sbr:7299620690" + "uri": "sbr:7367911510" }, "linux-aarch64": { - "uri": "sbr:7299619265" + "uri": "sbr:7367906163" }, "win32-clang-cl": { - "uri": "sbr:7299620278" + "uri": "sbr:7367910505" } } } diff --git a/build/external_resources/ymake/resources.json b/build/external_resources/ymake/resources.json index 91bc84b2d7..49b22c1888 100644 --- a/build/external_resources/ymake/resources.json +++ b/build/external_resources/ymake/resources.json @@ -1,19 +1,19 @@ { "by_platform": { "darwin": { - "uri": "sbr:7299618240" + "uri": "sbr:7367869442" }, "darwin-arm64": { - "uri": "sbr:7299617682" + "uri": "sbr:7367867142" }, "linux": { - "uri": "sbr:7299619235" + "uri": "sbr:7367873946" }, "linux-aarch64": { - "uri": "sbr:7299616994" + "uri": "sbr:7367864752" }, "win32-clang-cl": { - "uri": "sbr:7299618667" + "uri": "sbr:7367871824" } } } diff --git a/build/mapping.conf.json b/build/mapping.conf.json index da06c1f807..7d4ff8fec6 100644 --- a/build/mapping.conf.json +++ b/build/mapping.conf.json @@ -493,6 +493,7 @@ "6767780570": "https://devtools-registry.s3.yandex.net/6767780570", "6913307396": "https://devtools-registry.s3.yandex.net/6913307396", "7196092605": "https://devtools-registry.s3.yandex.net/7196092605", + "7370212609": "https://devtools-registry.s3.yandex.net/7370212609", "5811823398": "https://devtools-registry.s3.yandex.net/5811823398", "5840611310": "https://devtools-registry.s3.yandex.net/5840611310", "5860185593": "https://devtools-registry.s3.yandex.net/5860185593", @@ -515,6 +516,7 @@ "6767778005": "https://devtools-registry.s3.yandex.net/6767778005", "6913305359": "https://devtools-registry.s3.yandex.net/6913305359", "7196092155": "https://devtools-registry.s3.yandex.net/7196092155", + "7370211678": "https://devtools-registry.s3.yandex.net/7370211678", "5811822876": "https://devtools-registry.s3.yandex.net/5811822876", "5840610640": "https://devtools-registry.s3.yandex.net/5840610640", "5860184285": "https://devtools-registry.s3.yandex.net/5860184285", @@ -537,6 +539,7 @@ "6767775474": "https://devtools-registry.s3.yandex.net/6767775474", "6913303010": "https://devtools-registry.s3.yandex.net/6913303010", "7196091589": "https://devtools-registry.s3.yandex.net/7196091589", + "7370210775": "https://devtools-registry.s3.yandex.net/7370210775", "5766172292": "https://devtools-registry.s3.yandex.net/5766172292", "5805431504": "https://devtools-registry.s3.yandex.net/5805431504", "5829027626": "https://devtools-registry.s3.yandex.net/5829027626", @@ -582,6 +585,8 @@ "7211385890": "https://devtools-registry.s3.yandex.net/7211385890", "7255221832": "https://devtools-registry.s3.yandex.net/7255221832", "7299619998": "https://devtools-registry.s3.yandex.net/7299619998", + "7343906703": "https://devtools-registry.s3.yandex.net/7343906703", + "7367909157": "https://devtools-registry.s3.yandex.net/7367909157", "5766171800": "https://devtools-registry.s3.yandex.net/5766171800", "5805430761": "https://devtools-registry.s3.yandex.net/5805430761", "5829025456": "https://devtools-registry.s3.yandex.net/5829025456", @@ -627,6 +632,8 @@ "7211384971": "https://devtools-registry.s3.yandex.net/7211384971", "7255220942": "https://devtools-registry.s3.yandex.net/7255220942", "7299619667": "https://devtools-registry.s3.yandex.net/7299619667", + "7343906236": "https://devtools-registry.s3.yandex.net/7343906236", + "7367907802": "https://devtools-registry.s3.yandex.net/7367907802", "5766173070": "https://devtools-registry.s3.yandex.net/5766173070", "5805432830": "https://devtools-registry.s3.yandex.net/5805432830", "5829031598": "https://devtools-registry.s3.yandex.net/5829031598", @@ -672,6 +679,8 @@ "7211387031": "https://devtools-registry.s3.yandex.net/7211387031", "7255223649": "https://devtools-registry.s3.yandex.net/7255223649", "7299620690": "https://devtools-registry.s3.yandex.net/7299620690", + "7343907747": "https://devtools-registry.s3.yandex.net/7343907747", + "7367911510": "https://devtools-registry.s3.yandex.net/7367911510", "5766171341": "https://devtools-registry.s3.yandex.net/5766171341", "5805430188": "https://devtools-registry.s3.yandex.net/5805430188", "5829023352": "https://devtools-registry.s3.yandex.net/5829023352", @@ -717,6 +726,8 @@ "7211384106": "https://devtools-registry.s3.yandex.net/7211384106", "7255220285": "https://devtools-registry.s3.yandex.net/7255220285", "7299619265": "https://devtools-registry.s3.yandex.net/7299619265", + "7343905894": "https://devtools-registry.s3.yandex.net/7343905894", + "7367906163": "https://devtools-registry.s3.yandex.net/7367906163", "5766172695": "https://devtools-registry.s3.yandex.net/5766172695", "5805432230": "https://devtools-registry.s3.yandex.net/5805432230", "5829029743": "https://devtools-registry.s3.yandex.net/5829029743", @@ -762,6 +773,8 @@ "7211386552": "https://devtools-registry.s3.yandex.net/7211386552", "7255222862": "https://devtools-registry.s3.yandex.net/7255222862", "7299620278": "https://devtools-registry.s3.yandex.net/7299620278", + "7343907143": "https://devtools-registry.s3.yandex.net/7343907143", + "7367910505": "https://devtools-registry.s3.yandex.net/7367910505", "4307890075": "https://devtools-registry.s3.yandex.net/4307890075", "5517245192": "https://devtools-registry.s3.yandex.net/5517245192", "4307901240": "https://devtools-registry.s3.yandex.net/4307901240", @@ -1452,6 +1465,7 @@ "6767780570": "devtools/yexport/bin/yexport for darwin", "6913307396": "devtools/yexport/bin/yexport for darwin", "7196092605": "devtools/yexport/bin/yexport for darwin", + "7370212609": "devtools/yexport/bin/yexport for darwin", "5811823398": "devtools/yexport/bin/yexport for darwin-arm64", "5840611310": "devtools/yexport/bin/yexport for darwin-arm64", "5860185593": "devtools/yexport/bin/yexport for darwin-arm64", @@ -1474,6 +1488,7 @@ "6767778005": "devtools/yexport/bin/yexport for darwin-arm64", "6913305359": "devtools/yexport/bin/yexport for darwin-arm64", "7196092155": "devtools/yexport/bin/yexport for darwin-arm64", + "7370211678": "devtools/yexport/bin/yexport for darwin-arm64", "5811822876": "devtools/yexport/bin/yexport for linux", "5840610640": "devtools/yexport/bin/yexport for linux", "5860184285": "devtools/yexport/bin/yexport for linux", @@ -1496,6 +1511,7 @@ "6767775474": "devtools/yexport/bin/yexport for linux", "6913303010": "devtools/yexport/bin/yexport for linux", "7196091589": "devtools/yexport/bin/yexport for linux", + "7370210775": "devtools/yexport/bin/yexport for linux", "5766172292": "devtools/ymake/bin/ymake for darwin", "5805431504": "devtools/ymake/bin/ymake for darwin", "5829027626": "devtools/ymake/bin/ymake for darwin", @@ -1541,6 +1557,8 @@ "7211385890": "devtools/ymake/bin/ymake for darwin", "7255221832": "devtools/ymake/bin/ymake for darwin", "7299619998": "devtools/ymake/bin/ymake for darwin", + "7343906703": "devtools/ymake/bin/ymake for darwin", + "7367909157": "devtools/ymake/bin/ymake for darwin", "5766171800": "devtools/ymake/bin/ymake for darwin-arm64", "5805430761": "devtools/ymake/bin/ymake for darwin-arm64", "5829025456": "devtools/ymake/bin/ymake for darwin-arm64", @@ -1586,6 +1604,8 @@ "7211384971": "devtools/ymake/bin/ymake for darwin-arm64", "7255220942": "devtools/ymake/bin/ymake for darwin-arm64", "7299619667": "devtools/ymake/bin/ymake for darwin-arm64", + "7343906236": "devtools/ymake/bin/ymake for darwin-arm64", + "7367907802": "devtools/ymake/bin/ymake for darwin-arm64", "5766173070": "devtools/ymake/bin/ymake for linux", "5805432830": "devtools/ymake/bin/ymake for linux", "5829031598": "devtools/ymake/bin/ymake for linux", @@ -1631,6 +1651,8 @@ "7211387031": "devtools/ymake/bin/ymake for linux", "7255223649": "devtools/ymake/bin/ymake for linux", "7299620690": "devtools/ymake/bin/ymake for linux", + "7343907747": "devtools/ymake/bin/ymake for linux", + "7367911510": "devtools/ymake/bin/ymake for linux", "5766171341": "devtools/ymake/bin/ymake for linux-aarch64", "5805430188": "devtools/ymake/bin/ymake for linux-aarch64", "5829023352": "devtools/ymake/bin/ymake for linux-aarch64", @@ -1676,6 +1698,8 @@ "7211384106": "devtools/ymake/bin/ymake for linux-aarch64", "7255220285": "devtools/ymake/bin/ymake for linux-aarch64", "7299619265": "devtools/ymake/bin/ymake for linux-aarch64", + "7343905894": "devtools/ymake/bin/ymake for linux-aarch64", + "7367906163": "devtools/ymake/bin/ymake for linux-aarch64", "5766172695": "devtools/ymake/bin/ymake for win32-clang-cl", "5805432230": "devtools/ymake/bin/ymake for win32-clang-cl", "5829029743": "devtools/ymake/bin/ymake for win32-clang-cl", @@ -1721,6 +1745,8 @@ "7211386552": "devtools/ymake/bin/ymake for win32-clang-cl", "7255222862": "devtools/ymake/bin/ymake for win32-clang-cl", "7299620278": "devtools/ymake/bin/ymake for win32-clang-cl", + "7343907143": "devtools/ymake/bin/ymake for win32-clang-cl", + "7367910505": "devtools/ymake/bin/ymake for win32-clang-cl", "4307890075": "flake8_linter for linux", "5517245192": "flake8_linter for linux", "4307901240": "flake8_linter for linux-aarch64", diff --git a/build/plugins/nots.py b/build/plugins/nots.py index 2f49030375..7615849861 100644 --- a/build/plugins/nots.py +++ b/build/plugins/nots.py @@ -828,7 +828,7 @@ def on_node_modules_configure(unit: NotsUnitType) -> None: ymake.report_configure_error( "Project is configured to use @yatool/prebuilder. \n" + "Some packages in the pnpm-lock.yaml are misconfigured.\n" - + "Run {COLORS.green}`ya tool nots update-lockfile`{COLORS.reset} to fix lockfile.\n" + + f"Run {COLORS.green}`ya tool nots update-lockfile`{COLORS.reset} to fix lockfile.\n" + "All packages with `requiresBuild:true` have to be marked with `hasAddons:true/false`.\n" + "Misconfigured keys: \n" + " - " @@ -843,7 +843,7 @@ def on_node_modules_configure(unit: NotsUnitType) -> None: ymake.report_configure_error( "Project is configured to use @yatool/prebuilder. \n" + "Some packages are misconfigured.\n" - + "Run {COLORS.green}`ya tool nots update-lockfile`{COLORS.reset} to fix pnpm-lock.yaml and package.json.\n" + + f"Run {COLORS.green}`ya tool nots update-lockfile`{COLORS.reset} to fix pnpm-lock.yaml and package.json.\n" + "Validation details: \n" + "\n".join(validation_messages) ) diff --git a/build/scripts/_fake_src.cpp b/build/scripts/_fake_src.cpp index 139597f9cb..4f31cbffc7 100644 --- a/build/scripts/_fake_src.cpp +++ b/build/scripts/_fake_src.cpp @@ -1,2 +1 @@ - - +// DO_NOT_STYLE because this script is sometimes implicilty added as a styling target to STYLE_CPP diff --git a/build/ymake.core.conf b/build/ymake.core.conf index 7964201a89..d9c2d7a43d 100644 --- a/build/ymake.core.conf +++ b/build/ymake.core.conf @@ -5594,7 +5594,12 @@ macro _SRC("xib", SRC, SRCFLAGS...) { # tag:src-processing macro _SRC("msg", SRC, SRCFLAGS...) { - .CMD=$ROS_CMD($SRC) + .CMD=$ROS_MSG_CMD($SRC) +} + +# tag:src-processing +macro _SRC("srv", SRC, SRCFLAGS...) { + .CMD=$ROS_SRV_CMD($SRC) } ACTOOL_PATH=$XCODE_TOOLS_ROOT_RESOURCE_GLOBAL/Xcode/Contents/Developer/usr/bin/ibtool diff --git a/contrib/libs/cxxsupp/libcxxrt/ya.make b/contrib/libs/cxxsupp/libcxxrt/ya.make index bb32702ba5..568f018f17 100644 --- a/contrib/libs/cxxsupp/libcxxrt/ya.make +++ b/contrib/libs/cxxsupp/libcxxrt/ya.make @@ -11,9 +11,9 @@ LICENSE( LICENSE_TEXTS(.yandex_meta/licenses.list.txt) -VERSION(2024-09-24) +VERSION(2024-10-14) -ORIGINAL_SOURCE(https://github.com/libcxxrt/libcxxrt/archive/40e4fa2049930412a2c43cdf0c39b6b5aa735341.tar.gz) +ORIGINAL_SOURCE(https://github.com/libcxxrt/libcxxrt/archive/76435c4451aeb5e04e9500b090293347a38cef8d.tar.gz) ADDINCL( contrib/libs/cxxsupp/libcxxrt diff --git a/contrib/libs/protobuf/patches/dont-use-string-view.patch b/contrib/libs/protobuf/patches/dont-use-string-view.patch deleted file mode 100644 index 084341c464..0000000000 --- a/contrib/libs/protobuf/patches/dont-use-string-view.patch +++ /dev/null @@ -1,56 +0,0 @@ -commit 4a6dd4d9aeb3e3f9c033d092c86d5be63ca8cba9 -author: nechda -date: 2024-08-08T14:02:13+03:00 - - [PROTOBUF] Dont use string_view - ---- contrib/libs/protobuf/src/google/protobuf/arenastring.cc (e2819444222a1e9154b9c7b701eff6427206db7c) -+++ contrib/libs/protobuf/src/google/protobuf/arenastring.cc (4a6dd4d9aeb3e3f9c033d092c86d5be63ca8cba9) -@@ -115,7 +115,7 @@ TaggedStringPtr CreateArenaString(Arena& arena, absl::string_view s) { - - } // namespace - --void ArenaStringPtr::Set(absl::string_view value, Arena* arena) { -+void ArenaStringPtr::Set(const TProtoStringType& value, Arena* arena) { - ScopedCheckPtrInvariants check(&tagged_ptr_); - if (IsDefault()) { - // If we're not on an arena, skip straight to a true string to avoid ---- contrib/libs/protobuf/src/google/protobuf/arenastring.h (e2819444222a1e9154b9c7b701eff6427206db7c) -+++ contrib/libs/protobuf/src/google/protobuf/arenastring.h (4a6dd4d9aeb3e3f9c033d092c86d5be63ca8cba9) -@@ -259,7 +259,7 @@ struct PROTOBUF_EXPORT ArenaStringPtr { - // instance known to not carry any heap allocated value. - inline void InitAllocated(TProtoStringType* str, Arena* arena); - -- void Set(absl::string_view value, Arena* arena); -+ void Set(const TProtoStringType& value, Arena* arena); - void Set(TProtoStringType&& value, Arena* arena); - template <typename... OverloadDisambiguator> - void Set(const TProtoStringType& value, Arena* arena); -@@ -410,15 +410,15 @@ inline void ArenaStringPtr::InitAllocated(TProtoStringType* str, Arena* arena) { - } - - inline void ArenaStringPtr::Set(const char* s, Arena* arena) { -- Set(absl::string_view{s}, arena); -+ Set(TProtoStringType{s}, arena); - } - - inline void ArenaStringPtr::Set(const char* s, size_t n, Arena* arena) { -- Set(absl::string_view{s, n}, arena); -+ Set(TProtoStringType(s, s + n), arena); - } - - inline void ArenaStringPtr::SetBytes(absl::string_view value, Arena* arena) { -- Set(value, arena); -+ Set(TProtoStringType(value.data(), value.size()), arena); - } - - template <> -@@ -439,7 +439,7 @@ inline void ArenaStringPtr::SetBytes(const char* s, Arena* arena) { - } - - inline void ArenaStringPtr::SetBytes(const void* p, size_t n, Arena* arena) { -- Set(absl::string_view{static_cast<const char*>(p), n}, arena); -+ Set(TProtoStringType(static_cast<const char*>(p), static_cast<const char*>(p) + n), arena); - } - - // Make sure rhs_arena allocated rhs, and lhs_arena allocated lhs. diff --git a/contrib/libs/protobuf/patches/dot-generate-string-view-setters.patch b/contrib/libs/protobuf/patches/dot-generate-string-view-setters.patch deleted file mode 100644 index 88ec297018..0000000000 --- a/contrib/libs/protobuf/patches/dot-generate-string-view-setters.patch +++ /dev/null @@ -1,21 +0,0 @@ ---- contrib/libs/protobuf/src/google/protobuf/arenastring.h (d40b078e35f492f4ad417c8dd4a0103d4e577049) -+++ contrib/libs/protobuf/src/google/protobuf/arenastring.h (2babb01d30e285a96044041b98053d9edbb48b2a) -@@ -266,7 +266,6 @@ struct PROTOBUF_EXPORT ArenaStringPtr { - void Set(const char* s, Arena* arena); - void Set(const char* s, size_t n, Arena* arena); - -- void SetBytes(absl::string_view value, Arena* arena); - void SetBytes(TProtoStringType&& value, Arena* arena); - template <typename... OverloadDisambiguator> - void SetBytes(const TProtoStringType& value, Arena* arena); -@@ -417,10 +416,6 @@ inline void ArenaStringPtr::Set(const char* s, size_t n, Arena* arena) { - Set(TProtoStringType(s, s + n), arena); - } - --inline void ArenaStringPtr::SetBytes(absl::string_view value, Arena* arena) { -- Set(TProtoStringType(value.data(), value.size()), arena); --} -- - template <> - PROTOBUF_EXPORT void ArenaStringPtr::Set(const TProtoStringType& value, - Arena* arena); diff --git a/contrib/libs/protobuf/patches/fix-string-from-nullptr.patch b/contrib/libs/protobuf/patches/fix-string-from-nullptr.patch new file mode 100644 index 0000000000..27c1110066 --- /dev/null +++ b/contrib/libs/protobuf/patches/fix-string-from-nullptr.patch @@ -0,0 +1,19 @@ +commit 556f12d15e9e59a132af2d4605b7444c4e3b623e (wip) +author: vadim-xd +date: 2024-09-09T01:14:56+03:00 + + Fix ArenaStringPtr::Set(nullptr). + + TString(nullptr) is allowed (at least currently), string_view(nullptr) is UB. + +--- contrib/libs/protobuf/src/google/protobuf/arenastring.h (bff10d7f3e550777570ff58e01b9ae291d02490e) ++++ contrib/libs/protobuf/src/google/protobuf/arenastring.h (556f12d15e9e59a132af2d4605b7444c4e3b623e) +@@ -424,7 +424,7 @@ inline void ArenaStringPtr::InitAllocated(TProtoStringType* str, Arena* arena) { + } + + inline void ArenaStringPtr::Set(const char* s, Arena* arena) { +- Set(absl::string_view{s}, arena); ++ Set(TProtoStringType{s}, arena); + } + + inline void ArenaStringPtr::Set(const char* s, size_t n, Arena* arena) { diff --git a/contrib/libs/protobuf/patches/protoc-dont-generate-setters-with-string-view.patch b/contrib/libs/protobuf/patches/protoc-dont-generate-setters-with-string-view.patch deleted file mode 100644 index f2e020253a..0000000000 --- a/contrib/libs/protobuf/patches/protoc-dont-generate-setters-with-string-view.patch +++ /dev/null @@ -1,40 +0,0 @@ ---- a/src/google/protobuf/compiler/cpp/field_generators/string_field.cc (192810303a994f7cbd6bf596a3d56393bfefee6c) -+++ b/src/google/protobuf/compiler/cpp/field_generators/string_field.cc (2b6f1fdeb6f4d5c82840dbe69e4f0d3149c8fe5a) -@@ -781,13 +781,11 @@ void RepeatedString::GenerateAccessorDeclarations(io::Printer* p) const { - $DEPRECATED$ void $set_name$(int index, TProtoStringType&& value); - $DEPRECATED$ void $set_name$(int index, const char* value); - $DEPRECATED$ void $set_name$(int index, const $byte$* value, std::size_t size); -- $DEPRECATED$ void $set_name$(int index, absl::string_view value); - $DEPRECATED$ TProtoStringType* $add_name$(); - $DEPRECATED$ void $add_name$(const TProtoStringType& value); - $DEPRECATED$ void $add_name$(TProtoStringType&& value); - $DEPRECATED$ void $add_name$(const char* value); - $DEPRECATED$ void $add_name$(const $byte$* value, std::size_t size); -- $DEPRECATED$ void $add_name$(absl::string_view value); - $DEPRECATED$ const $pb$::RepeatedPtrField<TProtoStringType>& $name$() const; - $DEPRECATED$ $pb$::RepeatedPtrField<TProtoStringType>* $mutable_name$(); - -@@ -852,11 +850,6 @@ void RepeatedString::GenerateInlineAccessorDefinitions(io::Printer* p) const { - $annotate_set$; - // @@protoc_insertion_point(field_set_pointer:$pkg.Msg.field$) - } -- inline void $Msg$::set_$name$(int index, absl::string_view value) { -- $field_$.Mutable(index)->assign(value.data(), value.size()); -- $annotate_set$; -- // @@protoc_insertion_point(field_set_string_piece:$pkg.Msg.field$) -- } - inline TProtoStringType* $Msg$::_internal_add_$name$() { return $field_$.Add(); } - inline void $Msg$::add_$name$(const TProtoStringType& value) { - $field_$.Add()->assign(value); -@@ -879,11 +872,6 @@ void RepeatedString::GenerateInlineAccessorDefinitions(io::Printer* p) const { - $annotate_add$; - // @@protoc_insertion_point(field_add_pointer:$pkg.Msg.field$) - } -- inline void $Msg$::add_$name$(absl::string_view value) { -- $field_$.Add()->assign(value.data(), value.size()); -- $annotate_add$; -- // @@protoc_insertion_point(field_add_string_piece:$pkg.Msg.field$) -- } - inline const ::$proto_ns$::RepeatedPtrField<TProtoStringType>& - $Msg$::$name$() const { - $annotate_list$; diff --git a/contrib/libs/protobuf/patches/protoc-resolve-string-method-ambiguity.patch b/contrib/libs/protobuf/patches/protoc-resolve-string-method-ambiguity.patch new file mode 100644 index 0000000000..81df82ffae --- /dev/null +++ b/contrib/libs/protobuf/patches/protoc-resolve-string-method-ambiguity.patch @@ -0,0 +1,26 @@ +commit 3bc481cb6e0013371bdc01ae0aef7d85dc9abe1a (HEAD) +author: vadim-xd +date: 2024-10-28T13:19:23+03:00 + + Resolve ambiguity for repeated string methods + +--- a/src/google/protobuf/compiler/cpp/field_generators/string_field.cc (192810303a994f7cbd6bf596a3d56393bfefee6c) ++++ b/src/google/protobuf/compiler/cpp/field_generators/string_field.cc (3bc481cb6e0013371bdc01ae0aef7d85dc9abe1a) +@@ -782,12 +782,18 @@ void RepeatedString::GenerateAccessorDeclarations(io::Printer* p) const { + $DEPRECATED$ void $set_name$(int index, const char* value); + $DEPRECATED$ void $set_name$(int index, const $byte$* value, std::size_t size); + $DEPRECATED$ void $set_name$(int index, absl::string_view value); ++ $DEPRECATED$ void $set_name$(int index, std::string&& value) { ++ return $set_name$(index, TProtoStringType(std::move(value))); ++ } + $DEPRECATED$ TProtoStringType* $add_name$(); + $DEPRECATED$ void $add_name$(const TProtoStringType& value); + $DEPRECATED$ void $add_name$(TProtoStringType&& value); + $DEPRECATED$ void $add_name$(const char* value); + $DEPRECATED$ void $add_name$(const $byte$* value, std::size_t size); + $DEPRECATED$ void $add_name$(absl::string_view value); ++ $DEPRECATED$ void $add_name$(std::string&& value) { ++ return $add_name$(TProtoStringType(std::move(value))); ++ } + $DEPRECATED$ const $pb$::RepeatedPtrField<TProtoStringType>& $name$() const; + $DEPRECATED$ $pb$::RepeatedPtrField<TProtoStringType>* $mutable_name$(); diff --git a/contrib/libs/protobuf/patches/resolve-string-method-ambiguity.patch b/contrib/libs/protobuf/patches/resolve-string-method-ambiguity.patch new file mode 100644 index 0000000000..ad4f12f4e7 --- /dev/null +++ b/contrib/libs/protobuf/patches/resolve-string-method-ambiguity.patch @@ -0,0 +1,30 @@ +commit 4a55ea8892270807a046eb33fd5bc336b5fa93cb (HEAD) +author: vadim-xd +date: 2024-10-28T12:47:03+03:00 + + Resolve ambiguity in ArenaStringPtr methods with std::string&& + +--- contrib/libs/protobuf/src/google/protobuf/arenastring.h (e2819444222a1e9154b9c7b701eff6427206db7c) ++++ contrib/libs/protobuf/src/google/protobuf/arenastring.h (4a55ea8892270807a046eb33fd5bc336b5fa93cb) +@@ -266,6 +266,10 @@ struct PROTOBUF_EXPORT ArenaStringPtr { + void Set(const char* s, Arena* arena); + void Set(const char* s, size_t n, Arena* arena); + ++ void Set(std::string&& value, Arena* arena) { ++ return Set(TProtoStringType(std::move(value)), arena); ++ } ++ + void SetBytes(absl::string_view value, Arena* arena); + void SetBytes(TProtoStringType&& value, Arena* arena); + template <typename... OverloadDisambiguator> +@@ -273,6 +277,10 @@ struct PROTOBUF_EXPORT ArenaStringPtr { + void SetBytes(const char* s, Arena* arena); + void SetBytes(const void* p, size_t n, Arena* arena); + ++ void SetBytes(std::string&& value, Arena* arena) { ++ return SetBytes(TProtoStringType(std::move(value)), arena); ++ } ++ + template <typename RefWrappedType> + void Set(std::reference_wrapper<RefWrappedType> const_string_ref, + ::google::protobuf::Arena* arena) { diff --git a/contrib/libs/protobuf/patches/z_05_use_ref_count.patch b/contrib/libs/protobuf/patches/z_05_use_ref_count.patch index 391af0669c..1944eace13 100644 --- a/contrib/libs/protobuf/patches/z_05_use_ref_count.patch +++ b/contrib/libs/protobuf/patches/z_05_use_ref_count.patch @@ -1,14 +1,14 @@ --- contrib/libs/protobuf/src/google/protobuf/arenastring.cc (5de6c47c971aa5c751f17043b8ddf45bcc6daee0) +++ contrib/libs/protobuf/src/google/protobuf/arenastring.cc (c7e1b2fc457f6c68ed50780be734651758d4f548) -@@ -96,9 +96,9 @@ class ScopedCheckPtrInvariants { +@@ -96,9 +96,10 @@ class ScopedCheckPtrInvariants { #endif // NDEBUG || !GOOGLE_PROTOBUF_INTERNAL_DONATE_STEAL // Creates a heap allocated TProtoStringType value. -inline TaggedStringPtr CreateString(absl::string_view value) { -+inline TaggedStringPtr CreateString(const TProtoStringType& value) { ++template <typename TArg> ++inline TaggedStringPtr CreateString(const TArg& value) { TaggedStringPtr res; - res.SetAllocated(new TProtoStringType(value.data(), value.length())); + res.SetAllocated(new TProtoStringType(value)); return res; } - diff --git a/contrib/libs/protobuf/src/google/protobuf/arenastring.cc b/contrib/libs/protobuf/src/google/protobuf/arenastring.cc index 4c4eb75da1..f466d542d9 100644 --- a/contrib/libs/protobuf/src/google/protobuf/arenastring.cc +++ b/contrib/libs/protobuf/src/google/protobuf/arenastring.cc @@ -96,7 +96,8 @@ class ScopedCheckPtrInvariants { #endif // NDEBUG || !GOOGLE_PROTOBUF_INTERNAL_DONATE_STEAL // Creates a heap allocated TProtoStringType value. -inline TaggedStringPtr CreateString(const TProtoStringType& value) { +template <typename TArg> +inline TaggedStringPtr CreateString(const TArg& value) { TaggedStringPtr res; res.SetAllocated(new TProtoStringType(value)); return res; @@ -115,7 +116,7 @@ TaggedStringPtr CreateArenaString(Arena& arena, y_absl::string_view s) { } // namespace -void ArenaStringPtr::Set(const TProtoStringType& value, Arena* arena) { +void ArenaStringPtr::Set(y_absl::string_view value, Arena* arena) { ScopedCheckPtrInvariants check(&tagged_ptr_); if (IsDefault()) { // If we're not on an arena, skip straight to a true string to avoid diff --git a/contrib/libs/protobuf/src/google/protobuf/arenastring.h b/contrib/libs/protobuf/src/google/protobuf/arenastring.h index 62a1ae3d4a..edee6ef4cb 100644 --- a/contrib/libs/protobuf/src/google/protobuf/arenastring.h +++ b/contrib/libs/protobuf/src/google/protobuf/arenastring.h @@ -259,19 +259,28 @@ struct PROTOBUF_EXPORT ArenaStringPtr { // instance known to not carry any heap allocated value. inline void InitAllocated(TProtoStringType* str, Arena* arena); - void Set(const TProtoStringType& value, Arena* arena); + void Set(y_absl::string_view value, Arena* arena); void Set(TProtoStringType&& value, Arena* arena); template <typename... OverloadDisambiguator> void Set(const TProtoStringType& value, Arena* arena); void Set(const char* s, Arena* arena); void Set(const char* s, size_t n, Arena* arena); + void Set(std::string&& value, Arena* arena) { + return Set(TProtoStringType(std::move(value)), arena); + } + + void SetBytes(y_absl::string_view value, Arena* arena); void SetBytes(TProtoStringType&& value, Arena* arena); template <typename... OverloadDisambiguator> void SetBytes(const TProtoStringType& value, Arena* arena); void SetBytes(const char* s, Arena* arena); void SetBytes(const void* p, size_t n, Arena* arena); + void SetBytes(std::string&& value, Arena* arena) { + return SetBytes(TProtoStringType(std::move(value)), arena); + } + template <typename RefWrappedType> void Set(std::reference_wrapper<RefWrappedType> const_string_ref, ::google::protobuf::Arena* arena) { @@ -413,7 +422,11 @@ inline void ArenaStringPtr::Set(const char* s, Arena* arena) { } inline void ArenaStringPtr::Set(const char* s, size_t n, Arena* arena) { - Set(TProtoStringType(s, s + n), arena); + Set(y_absl::string_view{s, n}, arena); +} + +inline void ArenaStringPtr::SetBytes(y_absl::string_view value, Arena* arena) { + Set(value, arena); } template <> @@ -434,7 +447,7 @@ inline void ArenaStringPtr::SetBytes(const char* s, Arena* arena) { } inline void ArenaStringPtr::SetBytes(const void* p, size_t n, Arena* arena) { - Set(TProtoStringType(static_cast<const char*>(p), static_cast<const char*>(p) + n), arena); + Set(y_absl::string_view{static_cast<const char*>(p), n}, arena); } // Make sure rhs_arena allocated rhs, and lhs_arena allocated lhs. diff --git a/contrib/libs/protoc/src/google/protobuf/compiler/cpp/field_generators/string_field.cc b/contrib/libs/protoc/src/google/protobuf/compiler/cpp/field_generators/string_field.cc index 11b5b83ea6..0da0e56609 100644 --- a/contrib/libs/protoc/src/google/protobuf/compiler/cpp/field_generators/string_field.cc +++ b/contrib/libs/protoc/src/google/protobuf/compiler/cpp/field_generators/string_field.cc @@ -781,11 +781,19 @@ void RepeatedString::GenerateAccessorDeclarations(io::Printer* p) const { $DEPRECATED$ void $set_name$(int index, TProtoStringType&& value); $DEPRECATED$ void $set_name$(int index, const char* value); $DEPRECATED$ void $set_name$(int index, const $byte$* value, std::size_t size); + $DEPRECATED$ void $set_name$(int index, y_absl::string_view value); + $DEPRECATED$ void $set_name$(int index, std::string&& value) { + return $set_name$(index, TProtoStringType(std::move(value))); + } $DEPRECATED$ TProtoStringType* $add_name$(); $DEPRECATED$ void $add_name$(const TProtoStringType& value); $DEPRECATED$ void $add_name$(TProtoStringType&& value); $DEPRECATED$ void $add_name$(const char* value); $DEPRECATED$ void $add_name$(const $byte$* value, std::size_t size); + $DEPRECATED$ void $add_name$(y_absl::string_view value); + $DEPRECATED$ void $add_name$(std::string&& value) { + return $add_name$(TProtoStringType(std::move(value))); + } $DEPRECATED$ const $pb$::RepeatedPtrField<TProtoStringType>& $name$() const; $DEPRECATED$ $pb$::RepeatedPtrField<TProtoStringType>* $mutable_name$(); @@ -850,6 +858,11 @@ void RepeatedString::GenerateInlineAccessorDefinitions(io::Printer* p) const { $annotate_set$; // @@protoc_insertion_point(field_set_pointer:$pkg.Msg.field$) } + inline void $Msg$::set_$name$(int index, y_absl::string_view value) { + $field_$.Mutable(index)->assign(value.data(), value.size()); + $annotate_set$; + // @@protoc_insertion_point(field_set_string_piece:$pkg.Msg.field$) + } inline TProtoStringType* $Msg$::_internal_add_$name$() { return $field_$.Add(); } inline void $Msg$::add_$name$(const TProtoStringType& value) { $field_$.Add()->assign(value); @@ -872,6 +885,11 @@ void RepeatedString::GenerateInlineAccessorDefinitions(io::Printer* p) const { $annotate_add$; // @@protoc_insertion_point(field_add_pointer:$pkg.Msg.field$) } + inline void $Msg$::add_$name$(y_absl::string_view value) { + $field_$.Add()->assign(value.data(), value.size()); + $annotate_add$; + // @@protoc_insertion_point(field_add_string_piece:$pkg.Msg.field$) + } inline const ::$proto_ns$::RepeatedPtrField<TProtoStringType>& $Msg$::$name$() const { $annotate_list$; diff --git a/contrib/python/anyio/.dist-info/METADATA b/contrib/python/anyio/.dist-info/METADATA index e28bbd52d0..10d7aafc77 100644 --- a/contrib/python/anyio/.dist-info/METADATA +++ b/contrib/python/anyio/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: anyio -Version: 4.6.2 +Version: 4.6.2.post1 Summary: High level compatibility layer for multiple asynchronous event loop implementations Author-email: Alex Grönholm <alex.gronholm@nextday.fi> License: MIT @@ -15,13 +15,12 @@ Classifier: Framework :: AnyIO Classifier: Typing :: Typed Classifier: Programming Language :: Python Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python :: 3.8 Classifier: Programming Language :: Python :: 3.9 Classifier: Programming Language :: Python :: 3.10 Classifier: Programming Language :: Python :: 3.11 Classifier: Programming Language :: Python :: 3.12 Classifier: Programming Language :: Python :: 3.13 -Requires-Python: >=3.8 +Requires-Python: >=3.9 Description-Content-Type: text/x-rst License-File: LICENSE Requires-Dist: idna >=2.8 diff --git a/contrib/python/anyio/anyio/_backends/_asyncio.py b/contrib/python/anyio/anyio/_backends/_asyncio.py index fa5349a8c2..0a69e7ac61 100644 --- a/contrib/python/anyio/anyio/_backends/_asyncio.py +++ b/contrib/python/anyio/anyio/_backends/_asyncio.py @@ -20,9 +20,18 @@ from asyncio import ( ) from asyncio.base_events import _run_until_complete_cb # type: ignore[attr-defined] from collections import OrderedDict, deque -from collections.abc import AsyncIterator, Iterable +from collections.abc import ( + AsyncGenerator, + AsyncIterator, + Awaitable, + Callable, + Collection, + Coroutine, + Iterable, + Sequence, +) from concurrent.futures import Future -from contextlib import suppress +from contextlib import AbstractContextManager, suppress from contextvars import Context, copy_context from dataclasses import dataclass from functools import partial, wraps @@ -42,15 +51,7 @@ from types import TracebackType from typing import ( IO, Any, - AsyncGenerator, - Awaitable, - Callable, - Collection, - ContextManager, - Coroutine, Optional, - Sequence, - Tuple, TypeVar, cast, ) @@ -358,6 +359,14 @@ def _task_started(task: asyncio.Task) -> bool: # +def is_anyio_cancellation(exc: CancelledError) -> bool: + return ( + bool(exc.args) + and isinstance(exc.args[0], str) + and exc.args[0].startswith("Cancelled by cancel scope ") + ) + + class CancelScope(BaseCancelScope): def __new__( cls, *, deadline: float = math.inf, shield: bool = False @@ -416,6 +425,8 @@ class CancelScope(BaseCancelScope): exc_val: BaseException | None, exc_tb: TracebackType | None, ) -> bool | None: + del exc_tb + if not self._active: raise RuntimeError("This cancel scope is not active") if current_task() is not self._host_task: @@ -432,47 +443,93 @@ class CancelScope(BaseCancelScope): "current cancel scope" ) - self._active = False - if self._timeout_handle: - self._timeout_handle.cancel() - self._timeout_handle = None - - self._tasks.remove(self._host_task) - if self._parent_scope is not None: - self._parent_scope._child_scopes.remove(self) - self._parent_scope._tasks.add(self._host_task) + try: + self._active = False + if self._timeout_handle: + self._timeout_handle.cancel() + self._timeout_handle = None - host_task_state.cancel_scope = self._parent_scope + self._tasks.remove(self._host_task) + if self._parent_scope is not None: + self._parent_scope._child_scopes.remove(self) + self._parent_scope._tasks.add(self._host_task) - # Restart the cancellation effort in the closest directly cancelled parent - # scope if this one was shielded - self._restart_cancellation_in_parent() + host_task_state.cancel_scope = self._parent_scope - if self._cancel_called and exc_val is not None: - for exc in iterate_exceptions(exc_val): - if isinstance(exc, CancelledError): - self._cancelled_caught = self._uncancel(exc) - if self._cancelled_caught: + # Undo all cancellations done by this scope + if self._cancelling is not None: + while self._cancel_calls: + self._cancel_calls -= 1 + if self._host_task.uncancel() <= self._cancelling: break - return self._cancelled_caught + # We only swallow the exception iff it was an AnyIO CancelledError, either + # directly as exc_val or inside an exception group and there are no cancelled + # parent cancel scopes visible to us here + not_swallowed_exceptions = 0 + swallow_exception = False + if exc_val is not None: + for exc in iterate_exceptions(exc_val): + if self._cancel_called and isinstance(exc, CancelledError): + if not (swallow_exception := self._uncancel(exc)): + not_swallowed_exceptions += 1 + else: + not_swallowed_exceptions += 1 - return None + # Restart the cancellation effort in the closest visible, cancelled parent + # scope if necessary + self._restart_cancellation_in_parent() + return swallow_exception and not not_swallowed_exceptions + finally: + self._host_task = None + del exc_val + + @property + def _effectively_cancelled(self) -> bool: + cancel_scope: CancelScope | None = self + while cancel_scope is not None: + if cancel_scope._cancel_called: + return True + + if cancel_scope.shield: + return False + + cancel_scope = cancel_scope._parent_scope + + return False + + @property + def _parent_cancellation_is_visible_to_us(self) -> bool: + return ( + self._parent_scope is not None + and not self.shield + and self._parent_scope._effectively_cancelled + ) def _uncancel(self, cancelled_exc: CancelledError) -> bool: - if sys.version_info < (3, 9) or self._host_task is None: + if self._host_task is None: self._cancel_calls = 0 return True - # Undo all cancellations done by this scope - if self._cancelling is not None: - while self._cancel_calls: - self._cancel_calls -= 1 - if self._host_task.uncancel() <= self._cancelling: - return True + while True: + if is_anyio_cancellation(cancelled_exc): + # Only swallow the cancellation exception if it's an AnyIO cancel + # exception and there are no other cancel scopes down the line pending + # cancellation + self._cancelled_caught = ( + self._effectively_cancelled + and not self._parent_cancellation_is_visible_to_us + ) + return self._cancelled_caught - self._cancel_calls = 0 - return f"Cancelled by cancel scope {id(self):x}" in cancelled_exc.args + # Sometimes third party frameworks catch a CancelledError and raise a new + # one, so as a workaround we have to look at the previous ones in + # __context__ too for a matching cancel message + if isinstance(cancelled_exc.__context__, CancelledError): + cancelled_exc = cancelled_exc.__context__ + continue + + return False def _timeout(self) -> None: if self._deadline != math.inf: @@ -496,19 +553,17 @@ class CancelScope(BaseCancelScope): should_retry = False current = current_task() for task in self._tasks: + should_retry = True if task._must_cancel: # type: ignore[attr-defined] continue # The task is eligible for cancellation if it has started - should_retry = True if task is not current and (task is self._host_task or _task_started(task)): waiter = task._fut_waiter # type: ignore[attr-defined] if not isinstance(waiter, asyncio.Future) or not waiter.done(): - origin._cancel_calls += 1 - if sys.version_info >= (3, 9): - task.cancel(f"Cancelled by cancel scope {id(origin):x}") - else: - task.cancel() + task.cancel(f"Cancelled by cancel scope {id(origin):x}") + if task is origin._host_task: + origin._cancel_calls += 1 # Deliver cancellation to child scopes that aren't shielded or running their own # cancellation callbacks @@ -546,17 +601,6 @@ class CancelScope(BaseCancelScope): scope = scope._parent_scope - def _parent_cancelled(self) -> bool: - # Check whether any parent has been cancelled - cancel_scope = self._parent_scope - while cancel_scope is not None and not cancel_scope._shield: - if cancel_scope._cancel_called: - return True - else: - cancel_scope = cancel_scope._parent_scope - - return False - def cancel(self) -> None: if not self._cancel_called: if self._timeout_handle: @@ -645,6 +689,26 @@ class _AsyncioTaskStatus(abc.TaskStatus): _task_states[task].parent_id = self._parent_id +async def _wait(tasks: Iterable[asyncio.Task[object]]) -> None: + tasks = set(tasks) + waiter = get_running_loop().create_future() + + def on_completion(task: asyncio.Task[object]) -> None: + tasks.discard(task) + if not tasks and not waiter.done(): + waiter.set_result(None) + + for task in tasks: + task.add_done_callback(on_completion) + del task + + try: + await waiter + finally: + while tasks: + tasks.pop().remove_done_callback(on_completion) + + class TaskGroup(abc.TaskGroup): def __init__(self) -> None: self.cancel_scope: CancelScope = CancelScope() @@ -663,38 +727,53 @@ class TaskGroup(abc.TaskGroup): exc_val: BaseException | None, exc_tb: TracebackType | None, ) -> bool | None: - ignore_exception = self.cancel_scope.__exit__(exc_type, exc_val, exc_tb) - if exc_val is not None: - self.cancel_scope.cancel() - if not isinstance(exc_val, CancelledError): - self._exceptions.append(exc_val) - - cancelled_exc_while_waiting_tasks: CancelledError | None = None - while self._tasks: - try: - await asyncio.wait(self._tasks) - except CancelledError as exc: - # This task was cancelled natively; reraise the CancelledError later - # unless this task was already interrupted by another exception + try: + if exc_val is not None: self.cancel_scope.cancel() - if cancelled_exc_while_waiting_tasks is None: - cancelled_exc_while_waiting_tasks = exc + if not isinstance(exc_val, CancelledError): + self._exceptions.append(exc_val) - self._active = False - if self._exceptions: - raise BaseExceptionGroup( - "unhandled errors in a TaskGroup", self._exceptions - ) + try: + if self._tasks: + with CancelScope() as wait_scope: + while self._tasks: + try: + await _wait(self._tasks) + except CancelledError as exc: + # Shield the scope against further cancellation attempts, + # as they're not productive (#695) + wait_scope.shield = True + self.cancel_scope.cancel() + + # Set exc_val from the cancellation exception if it was + # previously unset. However, we should not replace a native + # cancellation exception with one raise by a cancel scope. + if exc_val is None or ( + isinstance(exc_val, CancelledError) + and not is_anyio_cancellation(exc) + ): + exc_val = exc + else: + # If there are no child tasks to wait on, run at least one checkpoint + # anyway + await AsyncIOBackend.cancel_shielded_checkpoint() - # Raise the CancelledError received while waiting for child tasks to exit, - # unless the context manager itself was previously exited with another - # exception, or if any of the child tasks raised an exception other than - # CancelledError - if cancelled_exc_while_waiting_tasks: - if exc_val is None or ignore_exception: - raise cancelled_exc_while_waiting_tasks + self._active = False + if self._exceptions: + raise BaseExceptionGroup( + "unhandled errors in a TaskGroup", self._exceptions + ) + elif exc_val: + raise exc_val + except BaseException as exc: + if self.cancel_scope.__exit__(type(exc), exc, exc.__traceback__): + return True - return ignore_exception + raise + + return self.cancel_scope.__exit__(exc_type, exc_val, exc_tb) + finally: + del exc_val, exc_tb, self._exceptions def _spawn( self, @@ -730,7 +809,7 @@ class TaskGroup(abc.TaskGroup): if not isinstance(exc, CancelledError): self._exceptions.append(exc) - if not self.cancel_scope._parent_cancelled(): + if not self.cancel_scope._effectively_cancelled: self.cancel_scope.cancel() else: task_status_future.set_exception(exc) @@ -806,7 +885,7 @@ class TaskGroup(abc.TaskGroup): # Threads # -_Retval_Queue_Type = Tuple[Optional[T_Retval], Optional[BaseException]] +_Retval_Queue_Type = tuple[Optional[T_Retval], Optional[BaseException]] class WorkerThread(Thread): @@ -955,7 +1034,7 @@ class Process(abc.Process): _stderr: StreamReaderWrapper | None async def aclose(self) -> None: - with CancelScope(shield=True): + with CancelScope(shield=True) as scope: if self._stdin: await self._stdin.aclose() if self._stdout: @@ -963,14 +1042,14 @@ class Process(abc.Process): if self._stderr: await self._stderr.aclose() - try: - await self.wait() - except BaseException: - self.kill() - with CancelScope(shield=True): + scope.shield = False + try: await self.wait() - - raise + except BaseException: + scope.shield = True + self.kill() + await self.wait() + raise async def wait(self) -> int: return await self._process.wait() @@ -2022,9 +2101,7 @@ class AsyncIOTaskInfo(TaskInfo): if task_state := _task_states.get(task): if cancel_scope := task_state.cancel_scope: - return cancel_scope.cancel_called or ( - not cancel_scope.shield and cancel_scope._parent_cancelled() - ) + return cancel_scope._effectively_cancelled return False @@ -2118,7 +2195,7 @@ class TestRunner(abc.TestRunner): ) -> T_Retval: if not self._runner_task: self._send_stream, receive_stream = create_memory_object_stream[ - Tuple[Awaitable[Any], asyncio.Future] + tuple[Awaitable[Any], asyncio.Future] ](1) self._runner_task = self.get_loop().create_task( self._run_tests_and_fixtures(receive_stream) @@ -2480,7 +2557,7 @@ class AsyncIOBackend(AsyncBackend): cls, host: str, port: int, local_address: IPSockAddrType | None = None ) -> abc.SocketStream: transport, protocol = cast( - Tuple[asyncio.Transport, StreamProtocol], + tuple[asyncio.Transport, StreamProtocol], await get_running_loop().create_connection( StreamProtocol, host, port, local_addr=local_address ), @@ -2659,7 +2736,7 @@ class AsyncIOBackend(AsyncBackend): @classmethod def open_signal_receiver( cls, *signals: Signals - ) -> ContextManager[AsyncIterator[Signals]]: + ) -> AbstractContextManager[AsyncIterator[Signals]]: return _SignalReceiver(signals) @classmethod diff --git a/contrib/python/anyio/anyio/_backends/_trio.py b/contrib/python/anyio/anyio/_backends/_trio.py index aee974deb6..24dcd74446 100644 --- a/contrib/python/anyio/anyio/_backends/_trio.py +++ b/contrib/python/anyio/anyio/_backends/_trio.py @@ -7,8 +7,18 @@ import socket import sys import types import weakref -from collections.abc import AsyncIterator, Iterable +from collections.abc import ( + AsyncGenerator, + AsyncIterator, + Awaitable, + Callable, + Collection, + Coroutine, + Iterable, + Sequence, +) from concurrent.futures import Future +from contextlib import AbstractContextManager from dataclasses import dataclass from functools import partial from io import IOBase @@ -19,15 +29,8 @@ from types import TracebackType from typing import ( IO, Any, - AsyncGenerator, - Awaitable, - Callable, - Collection, - ContextManager, - Coroutine, Generic, NoReturn, - Sequence, TypeVar, cast, overload, @@ -183,13 +186,12 @@ class TaskGroup(abc.TaskGroup): try: return await self._nursery_manager.__aexit__(exc_type, exc_val, exc_tb) except BaseExceptionGroup as exc: - _, rest = exc.split(trio.Cancelled) - if not rest: - cancelled_exc = trio.Cancelled._create() - raise cancelled_exc from exc + if not exc.split(trio.Cancelled)[1]: + raise trio.Cancelled._create() from exc raise finally: + del exc_val, exc_tb self._active = False def start_soon( @@ -1289,7 +1291,7 @@ class TrioBackend(AsyncBackend): @classmethod def open_signal_receiver( cls, *signals: Signals - ) -> ContextManager[AsyncIterator[Signals]]: + ) -> AbstractContextManager[AsyncIterator[Signals]]: return _SignalReceiver(signals) @classmethod diff --git a/contrib/python/anyio/anyio/_core/_fileio.py b/contrib/python/anyio/anyio/_core/_fileio.py index 214a90bfd8..53d3288c29 100644 --- a/contrib/python/anyio/anyio/_core/_fileio.py +++ b/contrib/python/anyio/anyio/_core/_fileio.py @@ -3,7 +3,7 @@ from __future__ import annotations import os import pathlib import sys -from collections.abc import Callable, Iterable, Iterator, Sequence +from collections.abc import AsyncIterator, Callable, Iterable, Iterator, Sequence from dataclasses import dataclass from functools import partial from os import PathLike @@ -12,7 +12,6 @@ from typing import ( TYPE_CHECKING, Any, AnyStr, - AsyncIterator, Final, Generic, overload, diff --git a/contrib/python/anyio/anyio/_core/_signals.py b/contrib/python/anyio/anyio/_core/_signals.py index 115c749bd9..f3451d302f 100644 --- a/contrib/python/anyio/anyio/_core/_signals.py +++ b/contrib/python/anyio/anyio/_core/_signals.py @@ -1,13 +1,15 @@ from __future__ import annotations from collections.abc import AsyncIterator +from contextlib import AbstractContextManager from signal import Signals -from typing import ContextManager from ._eventloop import get_async_backend -def open_signal_receiver(*signals: Signals) -> ContextManager[AsyncIterator[Signals]]: +def open_signal_receiver( + *signals: Signals, +) -> AbstractContextManager[AsyncIterator[Signals]]: """ Start receiving operating system signals. diff --git a/contrib/python/anyio/anyio/_core/_streams.py b/contrib/python/anyio/anyio/_core/_streams.py index aa6b0c222a..6a9814e5a9 100644 --- a/contrib/python/anyio/anyio/_core/_streams.py +++ b/contrib/python/anyio/anyio/_core/_streams.py @@ -1,7 +1,7 @@ from __future__ import annotations import math -from typing import Tuple, TypeVar +from typing import TypeVar from warnings import warn from ..streams.memory import ( @@ -14,7 +14,7 @@ T_Item = TypeVar("T_Item") class create_memory_object_stream( - Tuple[MemoryObjectSendStream[T_Item], MemoryObjectReceiveStream[T_Item]], + tuple[MemoryObjectSendStream[T_Item], MemoryObjectReceiveStream[T_Item]], ): """ Create a memory object stream. diff --git a/contrib/python/anyio/anyio/_core/_subprocesses.py b/contrib/python/anyio/anyio/_core/_subprocesses.py index 1ac2d549df..7ba41a5b03 100644 --- a/contrib/python/anyio/anyio/_core/_subprocesses.py +++ b/contrib/python/anyio/anyio/_core/_subprocesses.py @@ -160,38 +160,25 @@ async def open_process( child process prior to the execution of the subprocess. (POSIX only) :param pass_fds: sequence of file descriptors to keep open between the parent and child processes. (POSIX only) - :param user: effective user to run the process as (Python >= 3.9; POSIX only) - :param group: effective group to run the process as (Python >= 3.9; POSIX only) - :param extra_groups: supplementary groups to set in the subprocess (Python >= 3.9; - POSIX only) + :param user: effective user to run the process as (POSIX only) + :param group: effective group to run the process as (POSIX only) + :param extra_groups: supplementary groups to set in the subprocess (POSIX only) :param umask: if not negative, this umask is applied in the child process before - running the given command (Python >= 3.9; POSIX only) + running the given command (POSIX only) :return: an asynchronous process object """ kwargs: dict[str, Any] = {} if user is not None: - if sys.version_info < (3, 9): - raise TypeError("the 'user' argument requires Python 3.9 or later") - kwargs["user"] = user if group is not None: - if sys.version_info < (3, 9): - raise TypeError("the 'group' argument requires Python 3.9 or later") - kwargs["group"] = group if extra_groups is not None: - if sys.version_info < (3, 9): - raise TypeError("the 'extra_groups' argument requires Python 3.9 or later") - kwargs["extra_groups"] = group if umask >= 0: - if sys.version_info < (3, 9): - raise TypeError("the 'umask' argument requires Python 3.9 or later") - kwargs["umask"] = umask return await get_async_backend().open_process( diff --git a/contrib/python/anyio/anyio/abc/_eventloop.py b/contrib/python/anyio/anyio/abc/_eventloop.py index 2c73bb9ffb..93d0e9d25b 100644 --- a/contrib/python/anyio/anyio/abc/_eventloop.py +++ b/contrib/python/anyio/anyio/abc/_eventloop.py @@ -3,7 +3,8 @@ from __future__ import annotations import math import sys from abc import ABCMeta, abstractmethod -from collections.abc import AsyncIterator, Awaitable +from collections.abc import AsyncIterator, Awaitable, Callable, Sequence +from contextlib import AbstractContextManager from os import PathLike from signal import Signals from socket import AddressFamily, SocketKind, socket @@ -11,9 +12,6 @@ from typing import ( IO, TYPE_CHECKING, Any, - Callable, - ContextManager, - Sequence, TypeVar, Union, overload, @@ -352,7 +350,7 @@ class AsyncBackend(metaclass=ABCMeta): @abstractmethod def open_signal_receiver( cls, *signals: Signals - ) -> ContextManager[AsyncIterator[Signals]]: + ) -> AbstractContextManager[AsyncIterator[Signals]]: pass @classmethod diff --git a/contrib/python/anyio/anyio/abc/_sockets.py b/contrib/python/anyio/anyio/abc/_sockets.py index b321225a7b..1c6a450cdc 100644 --- a/contrib/python/anyio/anyio/abc/_sockets.py +++ b/contrib/python/anyio/anyio/abc/_sockets.py @@ -8,7 +8,7 @@ from io import IOBase from ipaddress import IPv4Address, IPv6Address from socket import AddressFamily from types import TracebackType -from typing import Any, Tuple, TypeVar, Union +from typing import Any, TypeVar, Union from .._core._typedattr import ( TypedAttributeProvider, @@ -19,10 +19,10 @@ from ._streams import ByteStream, Listener, UnreliableObjectStream from ._tasks import TaskGroup IPAddressType = Union[str, IPv4Address, IPv6Address] -IPSockAddrType = Tuple[str, int] +IPSockAddrType = tuple[str, int] SockAddrType = Union[IPSockAddrType, str] -UDPPacketType = Tuple[bytes, IPSockAddrType] -UNIXDatagramPacketType = Tuple[bytes, str] +UDPPacketType = tuple[bytes, IPSockAddrType] +UNIXDatagramPacketType = tuple[bytes, str] T_Retval = TypeVar("T_Retval") diff --git a/contrib/python/anyio/anyio/from_thread.py b/contrib/python/anyio/anyio/from_thread.py index b8785845ba..93a4cfe8e4 100644 --- a/contrib/python/anyio/anyio/from_thread.py +++ b/contrib/python/anyio/anyio/from_thread.py @@ -3,15 +3,17 @@ from __future__ import annotations import sys from collections.abc import Awaitable, Callable, Generator from concurrent.futures import Future -from contextlib import AbstractContextManager, contextmanager +from contextlib import ( + AbstractAsyncContextManager, + AbstractContextManager, + contextmanager, +) from dataclasses import dataclass, field from inspect import isawaitable from threading import Lock, Thread, get_ident from types import TracebackType from typing import ( Any, - AsyncContextManager, - ContextManager, Generic, TypeVar, cast, @@ -87,7 +89,9 @@ class _BlockingAsyncContextManager(Generic[T_co], AbstractContextManager): type[BaseException] | None, BaseException | None, TracebackType | None ] = (None, None, None) - def __init__(self, async_cm: AsyncContextManager[T_co], portal: BlockingPortal): + def __init__( + self, async_cm: AbstractAsyncContextManager[T_co], portal: BlockingPortal + ): self._async_cm = async_cm self._portal = portal @@ -374,8 +378,8 @@ class BlockingPortal: return f, task_status_future.result() def wrap_async_context_manager( - self, cm: AsyncContextManager[T_co] - ) -> ContextManager[T_co]: + self, cm: AbstractAsyncContextManager[T_co] + ) -> AbstractContextManager[T_co]: """ Wrap an async context manager as a synchronous context manager via this portal. diff --git a/contrib/python/anyio/anyio/pytest_plugin.py b/contrib/python/anyio/anyio/pytest_plugin.py index b7d9305614..4a0d59dd06 100644 --- a/contrib/python/anyio/anyio/pytest_plugin.py +++ b/contrib/python/anyio/anyio/pytest_plugin.py @@ -4,7 +4,7 @@ import sys from collections.abc import Generator, Iterator from contextlib import ExitStack, contextmanager from inspect import isasyncgenfunction, iscoroutinefunction, ismethod -from typing import Any, Dict, Tuple, cast +from typing import Any, cast import pytest import sniffio @@ -28,7 +28,7 @@ def extract_backend_and_options(backend: object) -> tuple[str, dict[str, Any]]: return backend, {} elif isinstance(backend, tuple) and len(backend) == 2: if isinstance(backend[0], str) and isinstance(backend[1], dict): - return cast(Tuple[str, Dict[str, Any]], backend) + return cast(tuple[str, dict[str, Any]], backend) raise TypeError("anyio_backend must be either a string or tuple of (string, dict)") diff --git a/contrib/python/anyio/anyio/streams/tls.py b/contrib/python/anyio/anyio/streams/tls.py index d01c8e6f4c..b6961bee16 100644 --- a/contrib/python/anyio/anyio/streams/tls.py +++ b/contrib/python/anyio/anyio/streams/tls.py @@ -7,7 +7,7 @@ import sys from collections.abc import Callable, Mapping from dataclasses import dataclass from functools import wraps -from typing import Any, Tuple, TypeVar +from typing import Any, TypeVar from .. import ( BrokenResourceError, @@ -25,8 +25,8 @@ else: T_Retval = TypeVar("T_Retval") PosArgsT = TypeVarTuple("PosArgsT") -_PCTRTT = Tuple[Tuple[str, str], ...] -_PCTRTTT = Tuple[_PCTRTT, ...] +_PCTRTT = tuple[tuple[str, str], ...] +_PCTRTTT = tuple[_PCTRTT, ...] class TLSAttribute(TypedAttributeSet): diff --git a/contrib/python/anyio/ya.make b/contrib/python/anyio/ya.make index bb56a53ce5..aadbb5b297 100644 --- a/contrib/python/anyio/ya.make +++ b/contrib/python/anyio/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(4.6.2) +VERSION(4.6.2.post1) LICENSE(MIT) diff --git a/contrib/python/pyparsing/py3/.dist-info/METADATA b/contrib/python/pyparsing/py3/.dist-info/METADATA index 1aa7a1fc04..ff6f9b6227 100644 --- a/contrib/python/pyparsing/py3/.dist-info/METADATA +++ b/contrib/python/pyparsing/py3/.dist-info/METADATA @@ -1,9 +1,9 @@ Metadata-Version: 2.1 Name: pyparsing -Version: 3.1.4 +Version: 3.2.0 Summary: pyparsing module - Classes and methods to define and execute parsing grammars Author-email: Paul McGuire <ptmcg.gm+pyparsing@gmail.com> -Requires-Python: >=3.6.8 +Requires-Python: >=3.9 Description-Content-Type: text/x-rst Classifier: Development Status :: 5 - Production/Stable Classifier: Intended Audience :: Developers @@ -12,9 +12,6 @@ Classifier: License :: OSI Approved :: MIT License Classifier: Operating System :: OS Independent Classifier: Programming Language :: Python Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python :: 3.6 -Classifier: Programming Language :: Python :: 3.7 -Classifier: Programming Language :: Python :: 3.8 Classifier: Programming Language :: Python :: 3.9 Classifier: Programming Language :: Python :: 3.10 Classifier: Programming Language :: Python :: 3.11 diff --git a/contrib/python/pyparsing/py3/pyparsing/__init__.py b/contrib/python/pyparsing/py3/pyparsing/__init__.py index a440cfbefa..543ceb62bd 100644 --- a/contrib/python/pyparsing/py3/pyparsing/__init__.py +++ b/contrib/python/pyparsing/py3/pyparsing/__init__.py @@ -120,8 +120,8 @@ class version_info(NamedTuple): return f"{__name__}.{type(self).__name__}({', '.join('{}={!r}'.format(*nv) for nv in zip(self._fields, self))})" -__version_info__ = version_info(3, 1, 4, "final", 1) -__version_time__ = "25 Aug 2024 14:40 UTC" +__version_info__ = version_info(3, 2, 0, "final", 1) +__version_time__ = "13 Oct 2024 09:46 UTC" __version__ = __version_info__.__version__ __versionTime__ = __version_time__ __author__ = "Paul McGuire <ptmcg.gm+pyparsing@gmail.com>" @@ -131,9 +131,9 @@ from .exceptions import * from .actions import * from .core import __diag__, __compat__ from .results import * -from .core import * # type: ignore[misc, assignment] +from .core import * from .core import _builtin_exprs as core_builtin_exprs -from .helpers import * # type: ignore[misc, assignment] +from .helpers import * from .helpers import _builtin_exprs as helper_builtin_exprs from .unicode import unicode_set, UnicodeRangeList, pyparsing_unicode as unicode @@ -147,9 +147,9 @@ from .common import ( if "pyparsing_unicode" not in globals(): pyparsing_unicode = unicode # type: ignore[misc] if "pyparsing_common" not in globals(): - pyparsing_common = common # type: ignore[misc] + pyparsing_common = common if "pyparsing_test" not in globals(): - pyparsing_test = testing # type: ignore[misc] + pyparsing_test = testing core_builtin_exprs += common_builtin_exprs + helper_builtin_exprs @@ -208,6 +208,7 @@ __all__ = [ "StringEnd", "StringStart", "Suppress", + "Tag", "Token", "TokenConverter", "White", diff --git a/contrib/python/pyparsing/py3/pyparsing/core.py b/contrib/python/pyparsing/py3/pyparsing/core.py index cbe73c987a..4f43c3bf99 100644 --- a/contrib/python/pyparsing/py3/pyparsing/core.py +++ b/contrib/python/pyparsing/py3/pyparsing/core.py @@ -1,7 +1,9 @@ # # core.py # +from __future__ import annotations +import collections.abc from collections import deque import os import typing @@ -9,12 +11,9 @@ from typing import ( Any, Callable, Generator, - List, NamedTuple, Sequence, - Set, TextIO, - Tuple, Union, cast, ) @@ -51,12 +50,7 @@ from .results import ParseResults, _ParseResultsWithOffset from .unicode import pyparsing_unicode _MAX_INT = sys.maxsize -str_type: Tuple[type, ...] = (str, bytes) - -if sys.version_info >= (3, 7): - _RePattern = re.Pattern -else: - _RePattern = typing.Pattern +str_type: tuple[type, ...] = (str, bytes) # # Copyright (c) 2003-2022 Paul T. McGuire @@ -81,18 +75,7 @@ else: # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # - -if sys.version_info >= (3, 8): - from functools import cached_property -else: - - class cached_property: - def __init__(self, func): - self._func = func - - def __get__(self, instance, owner=None): - ret = instance.__dict__[self._func.__name__] = self._func(instance) - return ret +from functools import cached_property class __compat__(__config_flags): @@ -230,7 +213,7 @@ _single_arg_builtins = { # fmt: on _generatorType = types.GeneratorType -ParseImplReturnType = Tuple[int, Any] +ParseImplReturnType = tuple[int, Any] PostParseReturnType = Union[ParseResults, Sequence[ParseResults]] ParseAction = Union[ Callable[[], Any], @@ -260,12 +243,26 @@ hexnums: str = nums + "ABCDEFabcdef" alphanums: str = alphas + nums printables: str = "".join([c for c in string.printable if c not in string.whitespace]) + +class _ParseActionIndexError(Exception): + """ + Internal wrapper around IndexError so that IndexErrors raised inside + parse actions aren't misinterpreted as IndexErrors raised inside + ParserElement parseImpl methods. + """ + + def __init__(self, msg: str, exc: BaseException): + self.msg: str = msg + self.exc: BaseException = exc + + _trim_arity_call_line: traceback.StackSummary = None # type: ignore[assignment] +pa_call_line_synth = () def _trim_arity(func, max_limit=3): """decorator to trim function calls to match the arity of the target""" - global _trim_arity_call_line + global _trim_arity_call_line, pa_call_line_synth if func in _single_arg_builtins: return lambda s, l, t: func(t) @@ -280,8 +277,8 @@ def _trim_arity(func, max_limit=3): LINE_DIFF = 9 # IF ANY CODE CHANGES, EVEN JUST COMMENTS OR BLANK LINES, BETWEEN THE NEXT LINE AND # THE CALL TO FUNC INSIDE WRAPPER, LINE_DIFF MUST BE MODIFIED!!!! - _trim_arity_call_line = (_trim_arity_call_line or traceback.extract_stack(limit=2)[-1]) - pa_call_line_synth = (_trim_arity_call_line[0], _trim_arity_call_line[1] + LINE_DIFF) + _trim_arity_call_line = _trim_arity_call_line or traceback.extract_stack(limit=2)[-1] + pa_call_line_synth = pa_call_line_synth or (_trim_arity_call_line[0], _trim_arity_call_line[1] + LINE_DIFF) def wrapper(*args): nonlocal found_arity, limit @@ -311,6 +308,11 @@ def _trim_arity(func, max_limit=3): continue raise + except IndexError as ie: + # wrap IndexErrors inside a _ParseActionIndexError + raise _ParseActionIndexError( + "IndexError raised in parse action", ie + ).with_traceback(None) # fmt: on # copy func name to wrapper for sensible debug output @@ -351,7 +353,7 @@ def condition_as_parse_action( def _default_start_debug_action( - instring: str, loc: int, expr: "ParserElement", cache_hit: bool = False + instring: str, loc: int, expr: ParserElement, cache_hit: bool = False ): cache_hit_str = "*" if cache_hit else "" print( @@ -367,7 +369,7 @@ def _default_success_debug_action( instring: str, startloc: int, endloc: int, - expr: "ParserElement", + expr: ParserElement, toks: ParseResults, cache_hit: bool = False, ): @@ -378,7 +380,7 @@ def _default_success_debug_action( def _default_exception_debug_action( instring: str, loc: int, - expr: "ParserElement", + expr: ParserElement, exc: Exception, cache_hit: bool = False, ): @@ -443,7 +445,7 @@ class ParserElement(ABC): @classmethod def using_each(cls, seq, **class_kwargs): """ - Yields a sequence of class(obj, **class_kwargs) for obj in seq. + Yields a sequence of ``class(obj, **class_kwargs)`` for obj in seq. Example:: @@ -458,7 +460,7 @@ class ParserElement(ABC): debug_fail: typing.Optional[DebugExceptionAction] def __init__(self, savelist: bool = False): - self.parseAction: List[ParseAction] = list() + self.parseAction: list[ParseAction] = list() self.failAction: typing.Optional[ParseFailAction] = None self.customName: str = None # type: ignore[assignment] self._defaultName: typing.Optional[str] = None @@ -470,7 +472,7 @@ class ParserElement(ABC): # used when checking for left-recursion self.mayReturnEmpty = False self.keepTabs = False - self.ignoreExprs: List["ParserElement"] = list() + self.ignoreExprs: list[ParserElement] = list() self.debug = False self.streamlined = False # optimize exception handling for subclasses that don't advance parse index @@ -483,9 +485,9 @@ class ParserElement(ABC): # avoid redundant calls to preParse self.callPreparse = True self.callDuringTry = False - self.suppress_warnings_: List[Diagnostics] = [] + self.suppress_warnings_: list[Diagnostics] = [] - def suppress_warning(self, warning_type: Diagnostics) -> "ParserElement": + def suppress_warning(self, warning_type: Diagnostics) -> ParserElement: """ Suppress warnings emitted for a particular diagnostic on this expression. @@ -518,7 +520,7 @@ class ParserElement(ABC): to_visit.extend(cur.recurse()) yield cur - def copy(self) -> "ParserElement": + def copy(self) -> ParserElement: """ Make a copy of this :class:`ParserElement`. Useful for defining different parse actions for the same parsing pattern, using copies of @@ -549,7 +551,7 @@ class ParserElement(ABC): def set_results_name( self, name: str, list_all_matches: bool = False, *, listAllMatches: bool = False - ) -> "ParserElement": + ) -> ParserElement: """ Define name for referencing matching tokens as a nested attribute of the returned parse results. @@ -581,7 +583,7 @@ class ParserElement(ABC): listAllMatches = listAllMatches or list_all_matches return self._setResultsName(name, listAllMatches) - def _setResultsName(self, name, list_all_matches=False) -> "ParserElement": + def _setResultsName(self, name, list_all_matches=False) -> ParserElement: if name is None: return self newself = self.copy() @@ -592,7 +594,7 @@ class ParserElement(ABC): newself.modalResults = not list_all_matches return newself - def set_break(self, break_flag: bool = True) -> "ParserElement": + def set_break(self, break_flag: bool = True) -> ParserElement: """ Method to invoke the Python pdb debugger when this element is about to be parsed. Set ``break_flag`` to ``True`` to enable, ``False`` to @@ -602,19 +604,17 @@ class ParserElement(ABC): _parseMethod = self._parse def breaker(instring, loc, do_actions=True, callPreParse=True): - import pdb - - # this call to pdb.set_trace() is intentional, not a checkin error - pdb.set_trace() + # this call to breakpoint() is intentional, not a checkin error + breakpoint() return _parseMethod(instring, loc, do_actions, callPreParse) breaker._originalParseMethod = _parseMethod # type: ignore [attr-defined] - self._parse = breaker # type: ignore [assignment] + self._parse = breaker # type: ignore [method-assign] elif hasattr(self._parse, "_originalParseMethod"): - self._parse = self._parse._originalParseMethod # type: ignore [attr-defined, assignment] + self._parse = self._parse._originalParseMethod # type: ignore [method-assign] return self - def set_parse_action(self, *fns: ParseAction, **kwargs: Any) -> "ParserElement": + def set_parse_action(self, *fns: ParseAction, **kwargs: Any) -> ParserElement: """ Define one or more actions to perform when successfully matching parse element definition. @@ -702,7 +702,7 @@ class ParserElement(ABC): return self - def add_parse_action(self, *fns: ParseAction, **kwargs: Any) -> "ParserElement": + def add_parse_action(self, *fns: ParseAction, **kwargs: Any) -> ParserElement: """ Add one or more parse actions to expression's list of parse actions. See :class:`set_parse_action`. @@ -714,7 +714,7 @@ class ParserElement(ABC): ) return self - def add_condition(self, *fns: ParseCondition, **kwargs: Any) -> "ParserElement": + def add_condition(self, *fns: ParseCondition, **kwargs: Any) -> ParserElement: """Add a boolean predicate function to expression's list of parse actions. See :class:`set_parse_action` for function call signatures. Unlike ``set_parse_action``, functions passed to ``add_condition`` need to return boolean success/fail of the condition. @@ -751,7 +751,7 @@ class ParserElement(ABC): ) return self - def set_fail_action(self, fn: ParseFailAction) -> "ParserElement": + def set_fail_action(self, fn: ParseFailAction) -> ParserElement: """ Define action to perform if parsing fails at this expression. Fail acton fn is a callable function that takes the arguments @@ -809,8 +809,7 @@ class ParserElement(ABC): # @profile def _parseNoCache( self, instring, loc, do_actions=True, callPreParse=True - ) -> Tuple[int, ParseResults]: - TRY, MATCH, FAIL = 0, 1, 2 + ) -> tuple[int, ParseResults]: debugging = self.debug # and do_actions) len_instring = len(instring) @@ -934,25 +933,42 @@ class ParserElement(ABC): # cache for left-recursion in Forward references recursion_lock = RLock() - recursion_memos: typing.Dict[ - Tuple[int, "Forward", bool], Tuple[int, Union[ParseResults, Exception]] + recursion_memos: collections.abc.MutableMapping[ + tuple[int, Forward, bool], tuple[int, Union[ParseResults, Exception]] ] = {} - class _CacheType(dict): + class _CacheType(typing.Protocol): """ - class to help type checking + Class to be used for packrat and left-recursion cacheing of results + and exceptions. """ not_in_cache: bool - def get(self, *args): ... + def get(self, *args) -> typing.Any: ... + + def set(self, *args) -> None: ... + + def clear(self) -> None: ... + + class NullCache(dict): + """ + A null cache type for initialization of the packrat_cache class variable. + If/when enable_packrat() is called, this null cache will be replaced by a + proper _CacheType class instance. + """ + + not_in_cache: bool = True + + def get(self, *args) -> typing.Any: ... - def set(self, *args): ... + def set(self, *args) -> None: ... - # argument cache for optimizing repeated calls when backtracking through recursive expressions - packrat_cache = ( - _CacheType() - ) # set later by enable_packrat(); this is here so that reset_cache() doesn't fail + def clear(self) -> None: ... + + # class-level argument cache for optimizing repeated calls when backtracking + # through recursive expressions + packrat_cache: _CacheType = NullCache() packrat_cache_lock = RLock() packrat_cache_stats = [0, 0] @@ -960,9 +976,8 @@ class ParserElement(ABC): # we can cache these arguments and save ourselves the trouble of re-parsing the contained expression def _parseCache( self, instring, loc, do_actions=True, callPreParse=True - ) -> Tuple[int, ParseResults]: + ) -> tuple[int, ParseResults]: HIT, MISS = 0, 1 - TRY, MATCH, FAIL = 0, 1, 2 lookup = (self, instring, loc, callPreParse, do_actions) with ParserElement.packrat_cache_lock: cache = ParserElement.packrat_cache @@ -995,7 +1010,7 @@ class ParserElement(ABC): pass raise value - value = cast(Tuple[int, ParseResults, int], value) + value = cast(tuple[int, ParseResults, int], value) loc_, result, endloc = value[0], value[1].copy(), value[2] if self.debug and self.debugActions.debug_match: try: @@ -1075,7 +1090,7 @@ class ParserElement(ABC): elif ParserElement._packratEnabled: raise RuntimeError("Packrat and Bounded Recursion are not compatible") if cache_size_limit is None: - ParserElement.recursion_memos = _UnboundedMemo() # type: ignore[assignment] + ParserElement.recursion_memos = _UnboundedMemo() elif cache_size_limit > 0: ParserElement.recursion_memos = _LRUMemo(capacity=cache_size_limit) # type: ignore[assignment] else: @@ -1128,7 +1143,7 @@ class ParserElement(ABC): if cache_size_limit is None: ParserElement.packrat_cache = _UnboundedCache() else: - ParserElement.packrat_cache = _FifoCache(cache_size_limit) # type: ignore[assignment] + ParserElement.packrat_cache = _FifoCache(cache_size_limit) ParserElement._parse = ParserElement._parseCache def parse_string( @@ -1191,12 +1206,14 @@ class ParserElement(ABC): loc = self.preParse(instring, loc) se = Empty() + StringEnd().set_debug(False) se._parse(instring, loc) + except _ParseActionIndexError as pa_exc: + raise pa_exc.exc except ParseBaseException as exc: if ParserElement.verbose_stacktrace: raise - else: - # catch and re-raise exception from here, clearing out pyparsing internal stack trace - raise exc.with_traceback(None) + + # catch and re-raise exception from here, clearing out pyparsing internal stack trace + raise exc.with_traceback(None) else: return tokens @@ -1205,10 +1222,11 @@ class ParserElement(ABC): instring: str, max_matches: int = _MAX_INT, overlap: bool = False, + always_skip_whitespace=True, *, debug: bool = False, maxMatches: int = _MAX_INT, - ) -> Generator[Tuple[ParseResults, int, int], None, None]: + ) -> Generator[tuple[ParseResults, int, int], None, None]: """ Scan the input string for expression matches. Each match will return the matching tokens, start location, and end location. May be called with optional @@ -1249,7 +1267,13 @@ class ParserElement(ABC): instring = str(instring).expandtabs() instrlen = len(instring) loc = 0 - preparseFn = self.preParse + if always_skip_whitespace: + preparser = Empty() + preparser.ignoreExprs = self.ignoreExprs + preparser.whiteChars = self.whiteChars + preparseFn = preparser.preParse + else: + preparseFn = self.preParse parseFn = self._parse ParserElement.resetCache() matches = 0 @@ -1311,14 +1335,15 @@ class ParserElement(ABC): Now Is The Winter Of Our Discontent Made Glorious Summer By This Sun Of York. """ - out: List[str] = [] + out: list[str] = [] lastE = 0 # force preservation of <TAB>s, to minimize unwanted transformation of string, and to # keep string locs straight between transform_string and scan_string self.keepTabs = True try: for t, s, e in self.scan_string(instring, debug=debug): - out.append(instring[lastE:s]) + if s > lastE: + out.append(instring[lastE:s]) lastE = e if not t: @@ -1372,7 +1397,12 @@ class ParserElement(ABC): maxMatches = min(maxMatches, max_matches) try: return ParseResults( - [t for t, s, e in self.scan_string(instring, maxMatches, debug=debug)] + [ + t + for t, s, e in self.scan_string( + instring, maxMatches, always_skip_whitespace=False, debug=debug + ) + ] ) except ParseBaseException as exc: if ParserElement.verbose_stacktrace: @@ -1413,7 +1443,7 @@ class ParserElement(ABC): last = e yield instring[last:] - def __add__(self, other) -> "ParserElement": + def __add__(self, other) -> ParserElement: """ Implementation of ``+`` operator - returns :class:`And`. Adding strings to a :class:`ParserElement` converts them to :class:`Literal`\\ s by default. @@ -1449,7 +1479,7 @@ class ParserElement(ABC): return NotImplemented return And([self, other]) - def __radd__(self, other) -> "ParserElement": + def __radd__(self, other) -> ParserElement: """ Implementation of ``+`` operator when left operand is not a :class:`ParserElement` """ @@ -1462,7 +1492,7 @@ class ParserElement(ABC): return NotImplemented return other + self - def __sub__(self, other) -> "ParserElement": + def __sub__(self, other) -> ParserElement: """ Implementation of ``-`` operator, returns :class:`And` with error stop """ @@ -1472,7 +1502,7 @@ class ParserElement(ABC): return NotImplemented return self + And._ErrorStop() + other - def __rsub__(self, other) -> "ParserElement": + def __rsub__(self, other) -> ParserElement: """ Implementation of ``-`` operator when left operand is not a :class:`ParserElement` """ @@ -1482,7 +1512,7 @@ class ParserElement(ABC): return NotImplemented return other - self - def __mul__(self, other) -> "ParserElement": + def __mul__(self, other) -> ParserElement: """ Implementation of ``*`` operator, allows use of ``expr * 3`` in place of ``expr + expr + expr``. Expressions may also be multiplied by a 2-integer @@ -1562,10 +1592,10 @@ class ParserElement(ABC): ret = And([self] * minElements) return ret - def __rmul__(self, other) -> "ParserElement": + def __rmul__(self, other) -> ParserElement: return self.__mul__(other) - def __or__(self, other) -> "ParserElement": + def __or__(self, other) -> ParserElement: """ Implementation of ``|`` operator - returns :class:`MatchFirst` """ @@ -1581,7 +1611,7 @@ class ParserElement(ABC): return NotImplemented return MatchFirst([self, other]) - def __ror__(self, other) -> "ParserElement": + def __ror__(self, other) -> ParserElement: """ Implementation of ``|`` operator when left operand is not a :class:`ParserElement` """ @@ -1591,7 +1621,7 @@ class ParserElement(ABC): return NotImplemented return other | self - def __xor__(self, other) -> "ParserElement": + def __xor__(self, other) -> ParserElement: """ Implementation of ``^`` operator - returns :class:`Or` """ @@ -1601,7 +1631,7 @@ class ParserElement(ABC): return NotImplemented return Or([self, other]) - def __rxor__(self, other) -> "ParserElement": + def __rxor__(self, other) -> ParserElement: """ Implementation of ``^`` operator when left operand is not a :class:`ParserElement` """ @@ -1611,7 +1641,7 @@ class ParserElement(ABC): return NotImplemented return other ^ self - def __and__(self, other) -> "ParserElement": + def __and__(self, other) -> ParserElement: """ Implementation of ``&`` operator - returns :class:`Each` """ @@ -1621,7 +1651,7 @@ class ParserElement(ABC): return NotImplemented return Each([self, other]) - def __rand__(self, other) -> "ParserElement": + def __rand__(self, other) -> ParserElement: """ Implementation of ``&`` operator when left operand is not a :class:`ParserElement` """ @@ -1631,7 +1661,7 @@ class ParserElement(ABC): return NotImplemented return other & self - def __invert__(self) -> "ParserElement": + def __invert__(self) -> ParserElement: """ Implementation of ``~`` operator - returns :class:`NotAny` """ @@ -1701,7 +1731,7 @@ class ParserElement(ABC): return ret - def __call__(self, name: typing.Optional[str] = None) -> "ParserElement": + def __call__(self, name: typing.Optional[str] = None) -> ParserElement: """ Shortcut for :class:`set_results_name`, with ``list_all_matches=False``. @@ -1721,14 +1751,14 @@ class ParserElement(ABC): return self.copy() - def suppress(self) -> "ParserElement": + def suppress(self) -> ParserElement: """ Suppresses the output of this :class:`ParserElement`; useful to keep punctuation from cluttering up returned output. """ return Suppress(self) - def ignore_whitespace(self, recursive: bool = True) -> "ParserElement": + def ignore_whitespace(self, recursive: bool = True) -> ParserElement: """ Enables the skipping of whitespace before matching the characters in the :class:`ParserElement`'s defined pattern. @@ -1738,7 +1768,7 @@ class ParserElement(ABC): self.skipWhitespace = True return self - def leave_whitespace(self, recursive: bool = True) -> "ParserElement": + def leave_whitespace(self, recursive: bool = True) -> ParserElement: """ Disables the skipping of whitespace before matching the characters in the :class:`ParserElement`'s defined pattern. This is normally only used internally by @@ -1750,8 +1780,8 @@ class ParserElement(ABC): return self def set_whitespace_chars( - self, chars: Union[Set[str], str], copy_defaults: bool = False - ) -> "ParserElement": + self, chars: Union[set[str], str], copy_defaults: bool = False + ) -> ParserElement: """ Overrides the default whitespace chars """ @@ -1760,7 +1790,7 @@ class ParserElement(ABC): self.copyDefaultWhiteChars = copy_defaults return self - def parse_with_tabs(self) -> "ParserElement": + def parse_with_tabs(self) -> ParserElement: """ Overrides default behavior to expand ``<TAB>`` s to spaces before parsing the input string. Must be called before ``parse_string`` when the input grammar contains elements that @@ -1769,7 +1799,7 @@ class ParserElement(ABC): self.keepTabs = True return self - def ignore(self, other: "ParserElement") -> "ParserElement": + def ignore(self, other: ParserElement) -> ParserElement: """ Define expression to be ignored (e.g., comments) while doing pattern matching; may be called repeatedly, to define multiple comment or other @@ -1800,7 +1830,7 @@ class ParserElement(ABC): start_action: DebugStartAction, success_action: DebugSuccessAction, exception_action: DebugExceptionAction, - ) -> "ParserElement": + ) -> ParserElement: """ Customize display of debugging messages while doing pattern matching: @@ -1821,7 +1851,7 @@ class ParserElement(ABC): self.debug = True return self - def set_debug(self, flag: bool = True, recurse: bool = False) -> "ParserElement": + def set_debug(self, flag: bool = True, recurse: bool = False) -> ParserElement: """ Enable display of debugging messages while doing pattern matching. Set ``flag`` to ``True`` to enable, ``False`` to disable. @@ -1886,7 +1916,7 @@ class ParserElement(ABC): Child classes must define this method, which defines how the ``default_name`` is set. """ - def set_name(self, name: typing.Optional[str]) -> "ParserElement": + def set_name(self, name: typing.Optional[str]) -> ParserElement: """ Define name for this expression, makes debugging and exception messages clearer. If `__diag__.enable_debug_on_named_expressions` is set to True, setting a name will also @@ -1903,7 +1933,7 @@ class ParserElement(ABC): integer.set_name("integer") integer.parse_string("ABC") # -> Exception: Expected integer (at char 0), (line:1, col:1) """ - self.customName = name + self.customName = name # type: ignore[assignment] self.errmsg = f"Expected {str(self)}" if __diag__.enable_debug_on_named_expressions: @@ -1926,12 +1956,12 @@ class ParserElement(ABC): def __repr__(self) -> str: return str(self) - def streamline(self) -> "ParserElement": + def streamline(self) -> ParserElement: self.streamlined = True self._defaultName = None return self - def recurse(self) -> List["ParserElement"]: + def recurse(self) -> list[ParserElement]: return [] def _checkRecursion(self, parseElementList): @@ -2018,9 +2048,9 @@ class ParserElement(ABC): def run_tests( self, - tests: Union[str, List[str]], + tests: Union[str, list[str]], parse_all: bool = True, - comment: typing.Optional[Union["ParserElement", str]] = "#", + comment: typing.Optional[Union[ParserElement, str]] = "#", full_dump: bool = True, print_results: bool = True, failure_tests: bool = False, @@ -2037,7 +2067,7 @@ class ParserElement(ABC): postParse: typing.Optional[ Callable[[str, ParseResults], typing.Optional[str]] ] = None, - ) -> Tuple[bool, List[Tuple[str, Union[ParseResults, Exception]]]]: + ) -> tuple[bool, list[tuple[str, Union[ParseResults, Exception]]]]: """ Execute the parse expression on a series of test strings, showing each test, the parsed results or where the parse failed. Quick and easy way to @@ -2155,8 +2185,8 @@ class ParserElement(ABC): print_ = file.write result: Union[ParseResults, Exception] - allResults: List[Tuple[str, Union[ParseResults, Exception]]] = [] - comments: List[str] = [] + allResults: list[tuple[str, Union[ParseResults, Exception]]] = [] + comments: list[str] = [] success = True NL = Literal(r"\n").add_parse_action(replace_with("\n")).ignore(quoted_string) BOM = "\ufeff" @@ -2187,7 +2217,18 @@ class ParserElement(ABC): success = success and failureTests result = pe except Exception as exc: - out.append(f"FAIL-EXCEPTION: {type(exc).__name__}: {exc}") + tag = "FAIL-EXCEPTION" + + # see if this exception was raised in a parse action + tb = exc.__traceback__ + it = iter(traceback.walk_tb(tb)) + for f, line in it: + if (f.f_code.co_filename, line) == pa_call_line_synth: + next_f = next(it)[0] + tag += f" (raised in parse action {next_f.f_code.co_name!r})" + break + + out.append(f"{tag}: {type(exc).__name__}: {exc}") if ParserElement.verbose_stacktrace: out.extend(traceback.format_tb(exc.__traceback__)) success = success and failureTests @@ -2323,7 +2364,7 @@ class _PendingSkip(ParserElement): def _generateDefaultName(self) -> str: return str(self.anchor + Empty()).replace("Empty", "...") - def __add__(self, other) -> "ParserElement": + def __add__(self, other) -> ParserElement: skipper = SkipTo(other).set_name("...")("_skipped*") if self.must_skip: @@ -2505,9 +2546,8 @@ class Keyword(Token): match_string = matchString or match_string self.match = match_string self.matchLen = len(match_string) - try: - self.firstMatchChar = match_string[0] - except IndexError: + self.firstMatchChar = match_string[:1] + if not self.firstMatchChar: raise ValueError("null string passed to Keyword; use Empty() instead") self.errmsg = f"Expected {type(self).__name__} {self.name}" self.mayReturnEmpty = False @@ -2522,7 +2562,7 @@ class Keyword(Token): return repr(self.match) def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType: - errmsg = self.errmsg + errmsg = self.errmsg or "" errloc = loc if self.caseless: if instring[loc : loc + self.matchLen].upper() == self.caselessmatch: @@ -2900,7 +2940,7 @@ class Word(Token): self.re = None # type: ignore[assignment] else: self.re_match = self.re.match - self.parseImpl = self.parseImpl_regex # type: ignore[assignment] + self.parseImpl = self.parseImpl_regex # type: ignore[method-assign] def _generateDefaultName(self) -> str: def charsAsStr(s): @@ -3040,48 +3080,65 @@ class Regex(Token): self._re = None self.reString = self.pattern = pattern - self.flags = flags elif hasattr(pattern, "pattern") and hasattr(pattern, "match"): self._re = pattern self.pattern = self.reString = pattern.pattern - self.flags = flags + + elif callable(pattern): + # defer creating this pattern until we really need it + self.pattern = pattern + self._re = None else: raise TypeError( - "Regex may only be constructed with a string or a compiled RE object" + "Regex may only be constructed with a string or a compiled RE object," + " or a callable that takes no arguments and returns a string or a" + " compiled RE object" ) + self.flags = flags self.errmsg = f"Expected {self.name}" self.mayIndexError = False self.asGroupList = asGroupList self.asMatch = asMatch if self.asGroupList: - self.parseImpl = self.parseImplAsGroupList # type: ignore [assignment] + self.parseImpl = self.parseImplAsGroupList # type: ignore [method-assign] if self.asMatch: - self.parseImpl = self.parseImplAsMatch # type: ignore [assignment] + self.parseImpl = self.parseImplAsMatch # type: ignore [method-assign] @cached_property - def re(self) -> _RePattern: + def re(self) -> re.Pattern: if self._re: return self._re + if callable(self.pattern): + # replace self.pattern with the string returned by calling self.pattern() + self.pattern = cast(Callable[[], str], self.pattern)() + + # see if we got a compiled RE back instead of a str - if so, we're done + if hasattr(self.pattern, "pattern") and hasattr(self.pattern, "match"): + self._re = cast(re.Pattern[str], self.pattern) + self.pattern = self.reString = self._re.pattern + return self._re + try: - return re.compile(self.pattern, self.flags) + self._re = re.compile(self.pattern, self.flags) + return self._re except re.error: raise ValueError(f"invalid pattern ({self.pattern!r}) passed to Regex") @cached_property - def re_match(self) -> Callable[[str], Any]: + def re_match(self) -> Callable[[str, int], Any]: return self.re.match @cached_property - def mayReturnEmpty(self) -> bool: - return self.re_match("") is not None + def mayReturnEmpty(self) -> bool: # type: ignore[override] + return self.re_match("", 0) is not None def _generateDefaultName(self) -> str: - unescaped = self.pattern.replace("\\\\", "\\") - return f"Re:({unescaped!r})" + unescaped = repr(self.pattern).replace("\\\\", "\\") + return f"Re:({unescaped})" def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType: result = self.re_match(instring, loc) @@ -3243,7 +3300,7 @@ class QuotedString(Token): # fmt: off # build up re pattern for the content between the quote delimiters - inner_pattern: List[str] = [] + inner_pattern: list[str] = [] if esc_quote: inner_pattern.append(rf"(?:{re.escape(esc_quote)})") @@ -3287,6 +3344,7 @@ class QuotedString(Token): if self.convert_whitespace_escapes: self.unquote_scan_re = re.compile( rf"({'|'.join(re.escape(k) for k in self.ws_map)})" + rf"|(\\[0-7]{3}|\\0|\\x[0-9a-fA-F]{2}|\\u[0-9a-fA-F]{4})" rf"|({re.escape(self.esc_char)}.)" rf"|(\n|.)", flags=self.re_flags, @@ -3333,6 +3391,16 @@ class QuotedString(Token): loc = result.end() ret = result.group() + def convert_escaped_numerics(s: str) -> str: + if s == "0": + return "\0" + if s.isdigit() and len(s) == 3: + return chr(int(s, base=8)) + elif s.startswith(("u", "x")): + return chr(int(s[1:], base=16)) + else: + return s + if self.unquote_results: # strip off quotes ret = ret[self.quote_char_len : -self.end_quote_char_len] @@ -3346,10 +3414,13 @@ class QuotedString(Token): ret = "".join( # match group 1 matches \t, \n, etc. self.ws_map[match.group(1)] if match.group(1) - # match group 2 matches escaped characters - else match.group(2)[-1] if match.group(2) - # match group 3 matches any character - else match.group(3) + # match group 2 matches escaped octal, null, hex, and Unicode + # sequences + else convert_escaped_numerics(match.group(2)[1:]) if match.group(2) + # match group 3 matches escaped characters + else match.group(3)[-1] if match.group(3) + # match group 4 matches any character + else match.group(4) for match in self.unquote_scan_re.finditer(ret) ) else: @@ -3754,6 +3825,7 @@ class Tag(Token): ['Hello,', 'World', '!'] - enthusiastic: True """ + def __init__(self, tag_name: str, value: Any = True): super().__init__() self.mayReturnEmpty = True @@ -3777,7 +3849,7 @@ class ParseExpression(ParserElement): def __init__(self, exprs: typing.Iterable[ParserElement], savelist: bool = False): super().__init__(savelist) - self.exprs: List[ParserElement] + self.exprs: list[ParserElement] if isinstance(exprs, _generatorType): exprs = list(exprs) @@ -3801,7 +3873,7 @@ class ParseExpression(ParserElement): self.exprs = [exprs] self.callPreparse = False - def recurse(self) -> List[ParserElement]: + def recurse(self) -> list[ParserElement]: return self.exprs[:] def append(self, other) -> ParserElement: @@ -3943,7 +4015,7 @@ class ParseExpression(ParserElement): class And(ParseExpression): """ - Requires all given :class:`ParseExpression` s to be found in the given order. + Requires all given :class:`ParserElement` s to be found in the given order. Expressions may be separated by whitespace. May be constructed using the ``'+'`` operator. May also be constructed using the ``'-'`` operator, which will @@ -3970,9 +4042,9 @@ class And(ParseExpression): def __init__( self, exprs_arg: typing.Iterable[ParserElement], savelist: bool = True ): - exprs: List[ParserElement] = list(exprs_arg) + exprs: list[ParserElement] = list(exprs_arg) if exprs and Ellipsis in exprs: - tmp: List[ParserElement] = [] + tmp: list[ParserElement] = [] for i, expr in enumerate(exprs): if expr is not Ellipsis: tmp.append(expr) @@ -4104,7 +4176,7 @@ class And(ParseExpression): class Or(ParseExpression): - """Requires that at least one :class:`ParseExpression` is found. If + """Requires that at least one :class:`ParserElement` is found. If two expressions match, the expression that matches the longest string will be used. May be constructed using the ``'^'`` operator. @@ -4144,8 +4216,8 @@ class Or(ParseExpression): def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType: maxExcLoc = -1 maxException = None - matches: List[Tuple[int, ParserElement]] = [] - fatals: List[ParseFatalException] = [] + matches: list[tuple[int, ParserElement]] = [] + fatals: list[ParseFatalException] = [] if all(e.callPreparse for e in self.exprs): loc = self.preParse(instring, loc) for e in self.exprs: @@ -4184,7 +4256,7 @@ class Or(ParseExpression): best_expr = matches[0][1] return best_expr._parse(instring, loc, do_actions) - longest = -1, None + longest: tuple[int, typing.Optional[ParseResults]] = -1, None for loc1, expr1 in matches: if loc1 <= longest[0]: # already have a longer match than this one will deliver, we are done @@ -4219,7 +4291,7 @@ class Or(ParseExpression): # infer from this check that all alternatives failed at the current position # so emit this collective error message instead of any single error message if maxExcLoc == loc: - maxException.msg = self.errmsg + maxException.msg = self.errmsg or "" raise maxException raise ParseException(instring, loc, "no defined alternatives to match", self) @@ -4259,7 +4331,7 @@ class Or(ParseExpression): class MatchFirst(ParseExpression): - """Requires that at least one :class:`ParseExpression` is found. If + """Requires that at least one :class:`ParserElement` is found. If more than one expression matches, the first one listed is the one that will match. May be constructed using the ``'|'`` operator. @@ -4326,7 +4398,7 @@ class MatchFirst(ParseExpression): # infer from this check that all alternatives failed at the current position # so emit this collective error message instead of any individual error message if maxExcLoc == loc: - maxException.msg = self.errmsg + maxException.msg = self.errmsg or "" raise maxException raise ParseException(instring, loc, "no defined alternatives to match", self) @@ -4366,7 +4438,7 @@ class MatchFirst(ParseExpression): class Each(ParseExpression): - """Requires all given :class:`ParseExpression` s to be found, but in + """Requires all given :class:`ParserElement` s to be found, but in any order. Expressions may be separated by whitespace. May be constructed using the ``'&'`` operator. @@ -4480,11 +4552,11 @@ class Each(ParseExpression): tmpReqd = self.required[:] tmpOpt = self.optionals[:] multis = self.multioptionals[:] - matchOrder: List[ParserElement] = [] + matchOrder: list[ParserElement] = [] keepMatching = True - failed: List[ParserElement] = [] - fatals: List[ParseFatalException] = [] + failed: list[ParserElement] = [] + fatals: list[ParseFatalException] = [] while keepMatching: tmpExprs = tmpReqd + tmpOpt + multis failed.clear() @@ -4567,7 +4639,7 @@ class ParseElementEnhance(ParserElement): self.callPreparse = expr.callPreparse self.ignoreExprs.extend(expr.ignoreExprs) - def recurse(self) -> List[ParserElement]: + def recurse(self) -> list[ParserElement]: return [self.expr] if self.expr is not None else [] def parseImpl(self, instring, loc, do_actions=True): @@ -4579,7 +4651,10 @@ class ParseElementEnhance(ParserElement): except ParseSyntaxException: raise except ParseBaseException as pbe: - if not isinstance(self, Forward) or self.customName is not None: + pbe.pstr = pbe.pstr or instring + pbe.loc = pbe.loc or loc + pbe.parser_element = pbe.parser_element or self + if not isinstance(self, Forward) and self.customName is not None: if self.errmsg: pbe.msg = self.errmsg raise @@ -4704,7 +4779,7 @@ class IndentedBlock(ParseElementEnhance): if self._grouped: wrapper = Group else: - wrapper = lambda expr: expr + wrapper = lambda expr: expr # type: ignore[misc, assignment] return (wrapper(block) + Optional(trailing_undent)).parseImpl( instring, anchor_loc, do_actions ) @@ -4830,9 +4905,7 @@ class PrecededBy(ParseElementEnhance): """ - def __init__( - self, expr: Union[ParserElement, str], retreat: typing.Optional[int] = None - ): + def __init__(self, expr: Union[ParserElement, str], retreat: int = 0): super().__init__(expr) self.expr = self.expr().leave_whitespace() self.mayReturnEmpty = True @@ -4859,7 +4932,7 @@ class PrecededBy(ParseElementEnhance): def parseImpl(self, instring, loc=0, do_actions=True) -> ParseImplReturnType: if self.exact: if loc < self.retreat: - raise ParseException(instring, loc, self.errmsg) + raise ParseException(instring, loc, self.errmsg, self) start = loc - self.retreat _, ret = self.expr._parse(instring, start) return loc, ret @@ -4867,7 +4940,7 @@ class PrecededBy(ParseElementEnhance): # retreat specified a maximum lookbehind window, iterate test_expr = self.expr + StringEnd() instring_slice = instring[max(0, loc - self.retreat) : loc] - last_expr = ParseException(instring, loc, self.errmsg) + last_expr: ParseBaseException = ParseException(instring, loc, self.errmsg, self) for offset in range(1, min(loc, self.retreat + 1) + 1): try: @@ -5236,7 +5309,9 @@ class Opt(ParseElementEnhance): def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType: self_expr = self.expr try: - loc, tokens = self_expr._parse(instring, loc, do_actions, callPreParse=False) + loc, tokens = self_expr._parse( + instring, loc, do_actions, callPreParse=False + ) except (ParseException, IndexError): default_value = self.defaultValue if default_value is not self.__optionalNotMatched: @@ -5244,9 +5319,9 @@ class Opt(ParseElementEnhance): tokens = ParseResults([default_value]) tokens[self_expr.resultsName] = default_value else: - tokens = [default_value] + tokens = [default_value] # type: ignore[assignment] else: - tokens = [] + tokens = [] # type: ignore[assignment] return loc, tokens def _generateDefaultName(self) -> str: @@ -5446,7 +5521,7 @@ class Forward(ParseElementEnhance): super().__init__(other, savelist=False) # type: ignore[arg-type] self.lshift_line = None - def __lshift__(self, other) -> "Forward": + def __lshift__(self, other) -> Forward: if hasattr(self, "caller_frame"): del self.caller_frame if isinstance(other, str_type): @@ -5468,13 +5543,13 @@ class Forward(ParseElementEnhance): self.lshift_line = traceback.extract_stack(limit=2)[-2] # type: ignore[assignment] return self - def __ilshift__(self, other) -> "Forward": + def __ilshift__(self, other) -> Forward: if not isinstance(other, ParserElement): return NotImplemented return self << other - def __or__(self, other) -> "ParserElement": + def __or__(self, other) -> ParserElement: caller_line = traceback.extract_stack(limit=2)[-2] if ( __diag__.warn_on_match_first_with_lshift_operator @@ -5585,9 +5660,9 @@ class Forward(ParseElementEnhance): # in case the action did backtrack prev_loc, prev_result = memo[peek_key] = memo[act_key] del memo[peek_key], memo[act_key] - return prev_loc, prev_result.copy() + return prev_loc, copy.copy(prev_result) del memo[peek_key] - return prev_loc, prev_peek.copy() + return prev_loc, copy.copy(prev_peek) # the match did get better: see if we can improve further if do_actions: try: @@ -5675,7 +5750,7 @@ class Forward(ParseElementEnhance): class TokenConverter(ParseElementEnhance): """ - Abstract subclass of :class:`ParseExpression`, for converting parsed results. + Abstract subclass of :class:`ParseElementEnhance`, for converting parsed results. """ def __init__(self, expr: Union[ParserElement, str], savelist=False): @@ -5900,13 +5975,13 @@ class Suppress(TokenConverter): expr = _PendingSkip(NoMatch()) super().__init__(expr) - def __add__(self, other) -> "ParserElement": + def __add__(self, other) -> ParserElement: if isinstance(self.expr, _PendingSkip): return Suppress(SkipTo(other)) + other return super().__add__(other) - def __sub__(self, other) -> "ParserElement": + def __sub__(self, other) -> ParserElement: if isinstance(self.expr, _PendingSkip): return Suppress(SkipTo(other)) - other @@ -6131,7 +6206,7 @@ punc8bit = srange(r"[\0xa1-\0xbf\0xd7\0xf7]") # build list of built-in expressions, for future reference if a global default value # gets updated -_builtin_exprs: List[ParserElement] = [ +_builtin_exprs: list[ParserElement] = [ v for v in vars().values() if isinstance(v, ParserElement) ] diff --git a/contrib/python/pyparsing/py3/pyparsing/diagram/__init__.py b/contrib/python/pyparsing/py3/pyparsing/diagram/__init__.py index 3275adafb6..7926f2c355 100644 --- a/contrib/python/pyparsing/py3/pyparsing/diagram/__init__.py +++ b/contrib/python/pyparsing/py3/pyparsing/diagram/__init__.py @@ -1,20 +1,20 @@ # mypy: ignore-errors +from __future__ import annotations + import railroad import pyparsing +import dataclasses import typing from typing import ( - List, - NamedTuple, Generic, TypeVar, - Dict, Callable, - Set, Iterable, ) from jinja2 import Template from io import StringIO import inspect +import re jinja2_template_source = """\ @@ -55,14 +55,23 @@ jinja2_template_source = """\ template = Template(jinja2_template_source) -# Note: ideally this would be a dataclass, but we're supporting Python 3.5+ so we can't do this yet -NamedDiagram = NamedTuple( - "NamedDiagram", - [("name", str), ("diagram", typing.Optional[railroad.DiagramItem]), ("index", int)], -) -""" -A simple structure for associating a name with a railroad diagram -""" + +def _collapse_verbose_regex(regex_str: str) -> str: + collapsed = pyparsing.Regex(r"#.*").suppress().transform_string(regex_str) + collapsed = re.sub(r"\s*\n\s*", "", collapsed) + return collapsed + + +@dataclasses.dataclass +class NamedDiagram: + """ + A simple structure for associating a name with a railroad diagram + """ + + name: str + index: int + diagram: railroad.DiagramItem = None + T = TypeVar("T") @@ -108,7 +117,7 @@ class EditablePartial(Generic[T]): self.kwargs = kwargs @classmethod - def from_call(cls, func: Callable[..., T], *args, **kwargs) -> "EditablePartial[T]": + def from_call(cls, func: Callable[..., T], *args, **kwargs) -> EditablePartial[T]: """ If you call this function in the same way that you would call the constructor, it will store the arguments as you expect. For example EditablePartial.from_call(Fraction, 1, 3)() == Fraction(1, 3) @@ -135,7 +144,7 @@ class EditablePartial(Generic[T]): return self.func(*args, **kwargs) -def railroad_to_html(diagrams: List[NamedDiagram], embed=False, **kwargs) -> str: +def railroad_to_html(diagrams: list[NamedDiagram], embed=False, **kwargs) -> str: """ Given a list of NamedDiagram, produce a single HTML string that visualises those diagrams :params kwargs: kwargs to be passed in to the template @@ -158,7 +167,7 @@ def railroad_to_html(diagrams: List[NamedDiagram], embed=False, **kwargs) -> str return template.render(diagrams=data, embed=embed, **kwargs) -def resolve_partial(partial: "EditablePartial[T]") -> T: +def resolve_partial(partial: EditablePartial[T]) -> T: """ Recursively resolves a collection of Partials into whatever type they are """ @@ -180,7 +189,7 @@ def to_railroad( vertical: int = 3, show_results_names: bool = False, show_groups: bool = False, -) -> List[NamedDiagram]: +) -> list[NamedDiagram]: """ Convert a pyparsing element tree into a list of diagrams. This is the recommended entrypoint to diagram creation if you want to access the Railroad tree before it is converted to HTML @@ -244,40 +253,31 @@ def _should_vertical( return len(_visible_exprs(exprs)) >= specification +@dataclasses.dataclass class ElementState: """ State recorded for an individual pyparsing Element """ - # Note: this should be a dataclass, but we have to support Python 3.5 - def __init__( - self, - element: pyparsing.ParserElement, - converted: EditablePartial, - parent: EditablePartial, - number: int, - name: str = None, - parent_index: typing.Optional[int] = None, - ): - #: The pyparsing element that this represents - self.element: pyparsing.ParserElement = element - #: The name of the element - self.name: typing.Optional[str] = name - #: The output Railroad element in an unconverted state - self.converted: EditablePartial = converted - #: The parent Railroad element, which we store so that we can extract this if it's duplicated - self.parent: EditablePartial = parent - #: The order in which we found this element, used for sorting diagrams if this is extracted into a diagram - self.number: int = number - #: The index of this inside its parent - self.parent_index: typing.Optional[int] = parent_index - #: If true, we should extract this out into a subdiagram - self.extract: bool = False - #: If true, all of this element's children have been filled out - self.complete: bool = False + #: The pyparsing element that this represents + element: pyparsing.ParserElement + #: The output Railroad element in an unconverted state + converted: EditablePartial + #: The parent Railroad element, which we store so that we can extract this if it's duplicated + parent: EditablePartial + #: The order in which we found this element, used for sorting diagrams if this is extracted into a diagram + number: int + #: The name of the element + name: str = None + #: The index of this inside its parent + parent_index: typing.Optional[int] = None + #: If true, we should extract this out into a subdiagram + extract: bool = False + #: If true, all of this element's children have been filled out + complete: bool = False def mark_for_extraction( - self, el_id: int, state: "ConverterState", name: str = None, force: bool = False + self, el_id: int, state: ConverterState, name: str = None, force: bool = False ): """ Called when this instance has been seen twice, and thus should eventually be extracted into a sub-diagram @@ -313,16 +313,16 @@ class ConverterState: def __init__(self, diagram_kwargs: typing.Optional[dict] = None): #: A dictionary mapping ParserElements to state relating to them - self._element_diagram_states: Dict[int, ElementState] = {} + self._element_diagram_states: dict[int, ElementState] = {} #: A dictionary mapping ParserElement IDs to subdiagrams generated from them - self.diagrams: Dict[int, EditablePartial[NamedDiagram]] = {} + self.diagrams: dict[int, EditablePartial[NamedDiagram]] = {} #: The index of the next unnamed element self.unnamed_index: int = 1 #: The index of the next element. This is used for sorting self.index: int = 0 #: Shared kwargs that are used to customize the construction of diagrams self.diagram_kwargs: dict = diagram_kwargs or {} - self.extracted_diagram_names: Set[str] = set() + self.extracted_diagram_names: set[str] = set() def __setitem__(self, key: int, value: ElementState): self._element_diagram_states[key] = value @@ -513,7 +513,7 @@ def _to_diagram_element( # If the element isn't worth extracting, we always treat it as the first time we say it if _worth_extracting(element): - if el_id in lookup: + if el_id in lookup and lookup[el_id].name is not None: # If we've seen this element exactly once before, we are only just now finding out that it's a duplicate, # so we have to extract it into a new diagram. looked_up = lookup[el_id] @@ -618,6 +618,11 @@ def _to_diagram_element( ret = EditablePartial.from_call(railroad.Sequence, items=[]) elif len(exprs) > 0 and not element_results_name: ret = EditablePartial.from_call(railroad.Group, item="", label=name) + elif isinstance(element, pyparsing.Regex): + patt = _collapse_verbose_regex(element.pattern) + element.pattern = patt + element._defaultName = None + ret = EditablePartial.from_call(railroad.Terminal, element.defaultName) elif len(exprs) > 0: ret = EditablePartial.from_call(railroad.Sequence, items=[]) else: diff --git a/contrib/python/pyparsing/py3/pyparsing/exceptions.py b/contrib/python/pyparsing/py3/pyparsing/exceptions.py index 8db34f195a..57a1579d12 100644 --- a/contrib/python/pyparsing/py3/pyparsing/exceptions.py +++ b/contrib/python/pyparsing/py3/pyparsing/exceptions.py @@ -1,17 +1,20 @@ # exceptions.py +from __future__ import annotations +import copy import re import sys import typing +from functools import cached_property +from .unicode import pyparsing_unicode as ppu from .util import ( + _collapse_string_to_ranges, col, line, lineno, - _collapse_string_to_ranges, replaced_by_pep8, ) -from .unicode import pyparsing_unicode as ppu class _ExceptionWordUnicodeSet( @@ -31,7 +34,7 @@ class ParseBaseException(Exception): msg: str pstr: str parser_element: typing.Any # "ParserElement" - args: typing.Tuple[str, int, typing.Optional[str]] + args: tuple[str, int, typing.Optional[str]] __slots__ = ( "loc", @@ -50,18 +53,17 @@ class ParseBaseException(Exception): msg: typing.Optional[str] = None, elem=None, ): - self.loc = loc if msg is None: - self.msg = pstr - self.pstr = "" - else: - self.msg = msg - self.pstr = pstr + msg, pstr = pstr, "" + + self.loc = loc + self.msg = msg + self.pstr = pstr self.parser_element = elem self.args = (pstr, loc, msg) @staticmethod - def explain_exception(exc, depth=16): + def explain_exception(exc: Exception, depth: int = 16) -> str: """ Method to take an exception and translate the Python internal traceback into a list of the pyparsing expressions that caused the exception to be raised. @@ -82,17 +84,17 @@ class ParseBaseException(Exception): if depth is None: depth = sys.getrecursionlimit() - ret = [] + ret: list[str] = [] if isinstance(exc, ParseBaseException): ret.append(exc.line) ret.append(f"{' ' * (exc.column - 1)}^") ret.append(f"{type(exc).__name__}: {exc}") - if depth <= 0: + if depth <= 0 or exc.__traceback__ is None: return "\n".join(ret) callers = inspect.getinnerframes(exc.__traceback__, context=depth) - seen = set() + seen: set[int] = set() for ff in callers[-depth:]: frm = ff[0] @@ -125,41 +127,58 @@ class ParseBaseException(Exception): return "\n".join(ret) @classmethod - def _from_exception(cls, pe): + def _from_exception(cls, pe) -> ParseBaseException: """ internal factory method to simplify creating one type of ParseException from another - avoids having __init__ signature conflicts among subclasses """ return cls(pe.pstr, pe.loc, pe.msg, pe.parser_element) - @property + @cached_property def line(self) -> str: """ Return the line of text where the exception occurred. """ return line(self.loc, self.pstr) - @property + @cached_property def lineno(self) -> int: """ Return the 1-based line number of text where the exception occurred. """ return lineno(self.loc, self.pstr) - @property + @cached_property def col(self) -> int: """ Return the 1-based column on the line of text where the exception occurred. """ return col(self.loc, self.pstr) - @property + @cached_property def column(self) -> int: """ Return the 1-based column on the line of text where the exception occurred. """ return col(self.loc, self.pstr) + @cached_property + def found(self) -> str: + if not self.pstr: + return "" + + if self.loc >= len(self.pstr): + return "end of text" + + # pull out next word at error location + found_match = _exception_word_extractor.match(self.pstr, self.loc) + if found_match is not None: + found_text = found_match.group(0) + else: + found_text = self.pstr[self.loc : self.loc + 1] + + return repr(found_text).replace(r"\\", "\\") + # pre-PEP8 compatibility @property def parserElement(self): @@ -169,21 +188,15 @@ class ParseBaseException(Exception): def parserElement(self, elem): self.parser_element = elem + def copy(self): + return copy.copy(self) + + def formatted_message(self) -> str: + found_phrase = f", found {self.found}" if self.found else "" + return f"{self.msg}{found_phrase} (at char {self.loc}), (line:{self.lineno}, col:{self.column})" + def __str__(self) -> str: - if self.pstr: - if self.loc >= len(self.pstr): - foundstr = ", found end of text" - else: - # pull out next word at error location - found_match = _exception_word_extractor.match(self.pstr, self.loc) - if found_match is not None: - found = found_match.group(0) - else: - found = self.pstr[self.loc : self.loc + 1] - foundstr = (", found %r" % found).replace(r"\\", "\\") - else: - foundstr = "" - return f"{self.msg}{foundstr} (at char {self.loc}), (line:{self.lineno}, col:{self.column})" + return self.formatted_message() def __repr__(self): return str(self) @@ -199,12 +212,10 @@ class ParseBaseException(Exception): line_str = self.line line_column = self.column - 1 if markerString: - line_str = "".join( - (line_str[:line_column], markerString, line_str[line_column:]) - ) + line_str = f"{line_str[:line_column]}{markerString}{line_str[line_column:]}" return line_str.strip() - def explain(self, depth=16) -> str: + def explain(self, depth: int = 16) -> str: """ Method to translate the Python internal traceback into a list of the pyparsing expressions that caused the exception to be raised. @@ -292,6 +303,8 @@ class RecursiveGrammarException(Exception): Exception thrown by :class:`ParserElement.validate` if the grammar could be left-recursive; parser may need to enable left recursion using :class:`ParserElement.enable_left_recursion<ParserElement.enable_left_recursion>` + + Deprecated: only used by deprecated method ParserElement.validate. """ def __init__(self, parseElementList): diff --git a/contrib/python/pyparsing/py3/pyparsing/helpers.py b/contrib/python/pyparsing/py3/pyparsing/helpers.py index d5d14a08d6..d2bd05f3d3 100644 --- a/contrib/python/pyparsing/py3/pyparsing/helpers.py +++ b/contrib/python/pyparsing/py3/pyparsing/helpers.py @@ -1,5 +1,6 @@ # helpers.py import html.entities +import operator import re import sys import typing @@ -10,6 +11,7 @@ from .util import ( _bslash, _flatten, _escape_regex_range_chars, + make_compressed_re, replaced_by_pep8, ) @@ -203,15 +205,15 @@ def one_of( ) if caseless: - isequal = lambda a, b: a.upper() == b.upper() + is_equal = lambda a, b: a.upper() == b.upper() masks = lambda a, b: b.upper().startswith(a.upper()) - parseElementClass = CaselessKeyword if asKeyword else CaselessLiteral + parse_element_class = CaselessKeyword if asKeyword else CaselessLiteral else: - isequal = lambda a, b: a == b + is_equal = operator.eq masks = lambda a, b: b.startswith(a) - parseElementClass = Keyword if asKeyword else Literal + parse_element_class = Keyword if asKeyword else Literal - symbols: List[str] = [] + symbols: list[str] if isinstance(strs, str_type): strs = typing.cast(str, strs) symbols = strs.split() @@ -224,20 +226,19 @@ def one_of( # reorder given symbols to take care to avoid masking longer choices with shorter ones # (but only if the given symbols are not just single characters) - if any(len(sym) > 1 for sym in symbols): - i = 0 - while i < len(symbols) - 1: - cur = symbols[i] - for j, other in enumerate(symbols[i + 1 :]): - if isequal(other, cur): - del symbols[i + j + 1] - break - if masks(cur, other): - del symbols[i + j + 1] - symbols.insert(i, other) - break - else: - i += 1 + i = 0 + while i < len(symbols) - 1: + cur = symbols[i] + for j, other in enumerate(symbols[i + 1 :]): + if is_equal(other, cur): + del symbols[i + j + 1] + break + if len(other) > len(cur) and masks(cur, other): + del symbols[i + j + 1] + symbols.insert(i, other) + break + else: + i += 1 if useRegex: re_flags: int = re.IGNORECASE if caseless else 0 @@ -269,7 +270,7 @@ def one_of( ) # last resort, just use MatchFirst - return MatchFirst(parseElementClass(sym) for sym in symbols).set_name( + return MatchFirst(parse_element_class(sym) for sym in symbols).set_name( " | ".join(symbols) ) @@ -602,7 +603,7 @@ def _makeTags(tagStr, xml, suppress_LT=Suppress("<"), suppress_GT=Suppress(">")) def make_html_tags( tag_str: Union[str, ParserElement] -) -> Tuple[ParserElement, ParserElement]: +) -> tuple[ParserElement, ParserElement]: """Helper to construct opening and closing tag expressions for HTML, given a tag name. Matches tags in either upper or lower case, attributes with namespaces and with quoted or unquoted values. @@ -629,7 +630,7 @@ def make_html_tags( def make_xml_tags( tag_str: Union[str, ParserElement] -) -> Tuple[ParserElement, ParserElement]: +) -> tuple[ParserElement, ParserElement]: """Helper to construct opening and closing tag expressions for XML, given a tag name. Matches tags only in the given upper/lower case. @@ -645,9 +646,12 @@ any_open_tag, any_close_tag = make_html_tags( ) _htmlEntityMap = {k.rstrip(";"): v for k, v in html.entities.html5.items()} -common_html_entity = Regex("&(?P<entity>" + "|".join(_htmlEntityMap) + ");").set_name( - "common HTML entity" +_most_common_entities = "nbsp lt gt amp quot apos cent pound euro copy".replace( + " ", "|" ) +common_html_entity = Regex( + lambda: f"&(?P<entity>{_most_common_entities}|{make_compressed_re(_htmlEntityMap)});" +).set_name("common HTML entity") def replace_html_entity(s, l, t): @@ -664,16 +668,16 @@ class OpAssoc(Enum): InfixNotationOperatorArgType = Union[ - ParserElement, str, Tuple[Union[ParserElement, str], Union[ParserElement, str]] + ParserElement, str, tuple[Union[ParserElement, str], Union[ParserElement, str]] ] InfixNotationOperatorSpec = Union[ - Tuple[ + tuple[ InfixNotationOperatorArgType, int, OpAssoc, typing.Optional[ParseAction], ], - Tuple[ + tuple[ InfixNotationOperatorArgType, int, OpAssoc, @@ -683,7 +687,7 @@ InfixNotationOperatorSpec = Union[ def infix_notation( base_expr: ParserElement, - op_list: List[InfixNotationOperatorSpec], + op_list: list[InfixNotationOperatorSpec], lpar: Union[str, ParserElement] = Suppress("("), rpar: Union[str, ParserElement] = Suppress(")"), ) -> ParserElement: @@ -1032,7 +1036,7 @@ python_style_comment = Regex(r"#.*").set_name("Python style comment") # build list of built-in expressions, for future reference if a global default value # gets updated -_builtin_exprs: List[ParserElement] = [ +_builtin_exprs: list[ParserElement] = [ v for v in vars().values() if isinstance(v, ParserElement) ] diff --git a/contrib/python/pyparsing/py3/pyparsing/results.py b/contrib/python/pyparsing/py3/pyparsing/results.py index 3bb7c948e0..245847832a 100644 --- a/contrib/python/pyparsing/py3/pyparsing/results.py +++ b/contrib/python/pyparsing/py3/pyparsing/results.py @@ -1,4 +1,7 @@ # results.py +from __future__ import annotations + +import collections from collections.abc import ( MutableMapping, Mapping, @@ -7,21 +10,21 @@ from collections.abc import ( Iterable, ) import pprint -from typing import Tuple, Any, Dict, Set, List +from typing import Any from .util import replaced_by_pep8 -str_type: Tuple[type, ...] = (str, bytes) +str_type: tuple[type, ...] = (str, bytes) _generator_type = type((_ for _ in ())) class _ParseResultsWithOffset: - tup: Tuple["ParseResults", int] + tup: tuple[ParseResults, int] __slots__ = ["tup"] - def __init__(self, p1: "ParseResults", p2: int): - self.tup: Tuple[ParseResults, int] = (p1, p2) + def __init__(self, p1: ParseResults, p2: int): + self.tup: tuple[ParseResults, int] = (p1, p2) def __getitem__(self, i): return self.tup[i] @@ -79,14 +82,14 @@ class ParseResults: - year: '1999' """ - _null_values: Tuple[Any, ...] = (None, [], ()) + _null_values: tuple[Any, ...] = (None, [], ()) _name: str - _parent: "ParseResults" - _all_names: Set[str] + _parent: ParseResults + _all_names: set[str] _modal: bool - _toklist: List[Any] - _tokdict: Dict[str, Any] + _toklist: list[Any] + _tokdict: dict[str, Any] __slots__ = ( "_name", @@ -172,8 +175,8 @@ class ParseResults: # constructor as small and fast as possible def __init__( self, toklist=None, name=None, asList=True, modal=True, isinstance=isinstance - ): - self._tokdict: Dict[str, _ParseResultsWithOffset] + ) -> None: + self._tokdict: dict[str, _ParseResultsWithOffset] self._modal = modal if name is None or name == "": @@ -226,7 +229,7 @@ class ParseResults: self._toklist[k] = v sub = v else: - self._tokdict[k] = self._tokdict.get(k, list()) + [ + self._tokdict[k] = self._tokdict.get(k, []) + [ _ParseResultsWithOffset(v, 0) ] sub = v @@ -443,12 +446,12 @@ class ParseResults: raise AttributeError(name) return "" - def __add__(self, other: "ParseResults") -> "ParseResults": + def __add__(self, other: ParseResults) -> ParseResults: ret = self.copy() ret += other return ret - def __iadd__(self, other: "ParseResults") -> "ParseResults": + def __iadd__(self, other: ParseResults) -> ParseResults: if not other: return self @@ -470,7 +473,7 @@ class ParseResults: self._all_names |= other._all_names return self - def __radd__(self, other) -> "ParseResults": + def __radd__(self, other) -> ParseResults: if isinstance(other, int) and other == 0: # useful for merging many ParseResults using sum() builtin return self.copy() @@ -504,9 +507,10 @@ class ParseResults: out.append(str(item)) return out - def as_list(self) -> list: + def as_list(self, *, flatten: bool = False) -> list: """ Returns the parse results as a nested list of matching tokens, all converted to strings. + If flatten is True, all the nesting levels in the returned list are collapsed. Example:: @@ -519,10 +523,22 @@ class ParseResults: result_list = result.as_list() print(type(result_list), result_list) # -> <class 'list'> ['sldkj', 'lsdkj', 'sldkj'] """ - return [ - res.as_list() if isinstance(res, ParseResults) else res - for res in self._toklist - ] + def flattened(pr): + to_visit = collections.deque([*self]) + while to_visit: + to_do = to_visit.popleft() + if isinstance(to_do, ParseResults): + to_visit.extendleft(to_do[::-1]) + else: + yield to_do + + if flatten: + return [*flattened(self)] + else: + return [ + res.as_list() if isinstance(res, ParseResults) else res + for res in self._toklist + ] def as_dict(self) -> dict: """ @@ -553,7 +569,7 @@ class ParseResults: return dict((k, to_item(v)) for k, v in self.items()) - def copy(self) -> "ParseResults": + def copy(self) -> ParseResults: """ Returns a new shallow copy of a :class:`ParseResults` object. `ParseResults` items contained within the source are shared with the copy. Use @@ -567,7 +583,7 @@ class ParseResults: ret._name = self._name return ret - def deepcopy(self) -> "ParseResults": + def deepcopy(self) -> ParseResults: """ Returns a new deep copy of a :class:`ParseResults` object. """ @@ -584,11 +600,11 @@ class ParseResults: dest[k] = v.deepcopy() if isinstance(v, ParseResults) else v elif isinstance(obj, Iterable): ret._toklist[i] = type(obj)( - v.deepcopy() if isinstance(v, ParseResults) else v for v in obj + v.deepcopy() if isinstance(v, ParseResults) else v for v in obj # type: ignore[call-arg] ) return ret - def get_name(self) -> str: + def get_name(self) -> str | None: r""" Returns the results name for this token expression. Useful when several different expressions might match at a particular location. @@ -616,7 +632,7 @@ class ParseResults: if self._name: return self._name elif self._parent: - par: "ParseResults" = self._parent + par: ParseResults = self._parent parent_tokdict_items = par._tokdict.items() return next( ( @@ -761,7 +777,7 @@ class ParseResults: return dir(type(self)) + list(self.keys()) @classmethod - def from_dict(cls, other, name=None) -> "ParseResults": + def from_dict(cls, other, name=None) -> ParseResults: """ Helper classmethod to construct a ``ParseResults`` from a ``dict``, preserving the name-value relations as results names. If an optional ``name`` argument is diff --git a/contrib/python/pyparsing/py3/pyparsing/testing.py b/contrib/python/pyparsing/py3/pyparsing/testing.py index 5654d47d62..836b2f86fb 100644 --- a/contrib/python/pyparsing/py3/pyparsing/testing.py +++ b/contrib/python/pyparsing/py3/pyparsing/testing.py @@ -257,10 +257,14 @@ class pyparsing_test: eol_mark: str = "|", mark_spaces: typing.Optional[str] = None, mark_control: typing.Optional[str] = None, + *, + indent: typing.Union[str, int] = "", + base_1: bool = True, ) -> str: """ Helpful method for debugging a parser - prints a string with line and column numbers. - (Line and column numbers are 1-based.) + (Line and column numbers are 1-based by default - if debugging a parse action, + pass base_1=False, to correspond to the loc value passed to the parse action.) :param s: tuple(bool, str - string to be printed with line and column numbers :param start_line: int - (optional) starting line number in s to print (default=1) @@ -273,11 +277,18 @@ class pyparsing_test: - "unicode" - replaces control chars with Unicode symbols, such as "␍" and "␊" - any single character string - replace control characters with given string - None (default) - string is displayed as-is + :param indent: str | int - (optional) string to indent with line and column numbers; if an int + is passed, converted to " " * indent + :param base_1: bool - (optional) whether to label string using base 1; if False, string will be + labeled based at 0 (default=True) :return: str - input string with leading line numbers and column number headers """ if expand_tabs: s = s.expandtabs() + if isinstance(indent, int): + indent = " " * indent + indent = indent.expandtabs() if mark_control is not None: mark_control = typing.cast(str, mark_control) if mark_control == "unicode": @@ -300,46 +311,52 @@ class pyparsing_test: else: s = s.replace(" ", mark_spaces) if start_line is None: - start_line = 1 + start_line = 0 if end_line is None: end_line = len(s) end_line = min(end_line, len(s)) - start_line = min(max(1, start_line), end_line) + start_line = min(max(0, start_line), end_line) if mark_control != "unicode": - s_lines = s.splitlines()[start_line - 1 : end_line] + s_lines = s.splitlines()[start_line - base_1 : end_line] else: - s_lines = [line + "␊" for line in s.split("␊")[start_line - 1 : end_line]] + s_lines = [ + line + "␊" for line in s.split("␊")[start_line - base_1 : end_line] + ] if not s_lines: return "" lineno_width = len(str(end_line)) max_line_len = max(len(line) for line in s_lines) - lead = " " * (lineno_width + 1) + lead = indent + " " * (lineno_width + 1) if max_line_len >= 99: header0 = ( lead + + ("" if base_1 else " ") + "".join( f"{' ' * 99}{(i + 1) % 100}" - for i in range(max(max_line_len // 100, 1)) + for i in range(1 if base_1 else 0, max(max_line_len // 100, 1)) ) + "\n" ) else: header0 = "" header1 = ( - header0 + ("" if base_1 else " ") + lead + "".join(f" {(i + 1) % 10}" for i in range(-(-max_line_len // 10))) + "\n" ) - header2 = lead + "1234567890" * (-(-max_line_len // 10)) + "\n" + digits = "1234567890" + header2 = ( + lead + ("" if base_1 else "0") + digits * (-(-max_line_len // 10)) + "\n" + ) return ( header1 + header2 + "\n".join( - f"{i:{lineno_width}d}:{line}{eol_mark}" - for i, line in enumerate(s_lines, start=start_line) + f"{indent}{i:{lineno_width}d}:{line}{eol_mark}" + for i, line in enumerate(s_lines, start=start_line + base_1) ) + "\n" ) diff --git a/contrib/python/pyparsing/py3/pyparsing/unicode.py b/contrib/python/pyparsing/py3/pyparsing/unicode.py index 0e3e06572b..066486c28e 100644 --- a/contrib/python/pyparsing/py3/pyparsing/unicode.py +++ b/contrib/python/pyparsing/py3/pyparsing/unicode.py @@ -2,7 +2,7 @@ import sys from itertools import filterfalse -from typing import List, Tuple, Union +from typing import Union class _lazyclassproperty: @@ -25,7 +25,7 @@ class _lazyclassproperty: return cls._intern[attrname] -UnicodeRangeList = List[Union[Tuple[int, int], Tuple[int]]] +UnicodeRangeList = list[Union[tuple[int, int], tuple[int]]] class unicode_set: @@ -53,9 +53,9 @@ class unicode_set: _ranges: UnicodeRangeList = [] @_lazyclassproperty - def _chars_for_ranges(cls) -> List[str]: - ret: List[int] = [] - for cc in cls.__mro__: + def _chars_for_ranges(cls) -> list[str]: + ret: list[int] = [] + for cc in cls.__mro__: # type: ignore[attr-defined] if cc is unicode_set: break for rr in getattr(cc, "_ranges", ()): diff --git a/contrib/python/pyparsing/py3/pyparsing/util.py b/contrib/python/pyparsing/py3/pyparsing/util.py index 94837fea0f..1487019c27 100644 --- a/contrib/python/pyparsing/py3/pyparsing/util.py +++ b/contrib/python/pyparsing/py3/pyparsing/util.py @@ -1,11 +1,11 @@ # util.py +import contextlib +from functools import lru_cache, wraps import inspect -import warnings -import types -import collections import itertools -from functools import lru_cache, wraps -from typing import Callable, List, Union, Iterable, TypeVar, cast +import types +from typing import Callable, Union, Iterable, TypeVar, cast +import warnings _bslash = chr(92) C = TypeVar("C", bound=Callable) @@ -14,8 +14,8 @@ C = TypeVar("C", bound=Callable) class __config_flags: """Internal class for defining compatibility and debugging flags""" - _all_names: List[str] = [] - _fixed_names: List[str] = [] + _all_names: list[str] = [] + _fixed_names: list[str] = [] _type_desc = "configuration" @classmethod @@ -100,27 +100,24 @@ class _UnboundedCache: class _FifoCache: def __init__(self, size): - self.not_in_cache = not_in_cache = object() cache = {} - keyring = [object()] * size + self.size = size + self.not_in_cache = not_in_cache = object() cache_get = cache.get cache_pop = cache.pop - keyiter = itertools.cycle(range(size)) def get(_, key): return cache_get(key, not_in_cache) def set_(_, key, value): cache[key] = value - i = next(keyiter) - cache_pop(keyring[i], None) - keyring[i] = key + while len(cache) > size: + # pop oldest element in cache by getting the first key + cache_pop(next(iter(cache))) def clear(_): cache.clear() - keyring[:] = [object()] * size - self.size = size self.get = types.MethodType(get, self) self.set = types.MethodType(set_, self) self.clear = types.MethodType(clear, self) @@ -137,13 +134,13 @@ class LRUMemo: def __init__(self, capacity): self._capacity = capacity self._active = {} - self._memory = collections.OrderedDict() + self._memory = {} def __getitem__(self, key): try: return self._active[key] except KeyError: - self._memory.move_to_end(key) + self._memory[key] = self._memory.pop(key) return self._memory[key] def __setitem__(self, key, value): @@ -156,8 +153,9 @@ class LRUMemo: except KeyError: pass else: - while len(self._memory) >= self._capacity: - self._memory.popitem(last=False) + oldest_keys = list(self._memory)[: -(self._capacity + 1)] + for key_to_delete in oldest_keys: + self._memory.pop(key_to_delete) self._memory[key] = value def clear(self): @@ -183,60 +181,182 @@ def _escape_regex_range_chars(s: str) -> str: return str(s) +class _GroupConsecutive: + """ + Used as a callable `key` for itertools.groupby to group + characters that are consecutive: + itertools.groupby("abcdejkmpqrs", key=IsConsecutive()) + yields: + (0, iter(['a', 'b', 'c', 'd', 'e'])) + (1, iter(['j', 'k'])) + (2, iter(['m'])) + (3, iter(['p', 'q', 'r', 's'])) + """ + def __init__(self): + self.prev = 0 + self.counter = itertools.count() + self.value = -1 + + def __call__(self, char: str) -> int: + c_int = ord(char) + self.prev, prev = c_int, self.prev + if c_int - prev > 1: + self.value = next(self.counter) + return self.value + + def _collapse_string_to_ranges( s: Union[str, Iterable[str]], re_escape: bool = True ) -> str: - def is_consecutive(c): - c_int = ord(c) - is_consecutive.prev, prev = c_int, is_consecutive.prev - if c_int - prev > 1: - is_consecutive.value = next(is_consecutive.counter) - return is_consecutive.value + r""" + Take a string or list of single-character strings, and return + a string of the consecutive characters in that string collapsed + into groups, as might be used in a regular expression '[a-z]' + character set: + 'a' -> 'a' -> '[a]' + 'bc' -> 'bc' -> '[bc]' + 'defgh' -> 'd-h' -> '[d-h]' + 'fdgeh' -> 'd-h' -> '[d-h]' + 'jklnpqrtu' -> 'j-lnp-rtu' -> '[j-lnp-rtu]' + Duplicates get collapsed out: + 'aaa' -> 'a' -> '[a]' + 'bcbccb' -> 'bc' -> '[bc]' + 'defghhgf' -> 'd-h' -> '[d-h]' + 'jklnpqrjjjtu' -> 'j-lnp-rtu' -> '[j-lnp-rtu]' + Spaces are preserved: + 'ab c' -> ' a-c' -> '[ a-c]' + Characters that are significant when defining regex ranges + get escaped: + 'acde[]-' -> r'\-\[\]ac-e' -> r'[\-\[\]ac-e]' + """ - is_consecutive.prev = 0 # type: ignore [attr-defined] - is_consecutive.counter = itertools.count() # type: ignore [attr-defined] - is_consecutive.value = -1 # type: ignore [attr-defined] + # Developer notes: + # - Do not optimize this code assuming that the given input string + # or internal lists will be short (such as in loading generators into + # lists to make it easier to find the last element); this method is also + # used to generate regex ranges for character sets in the pyparsing.unicode + # classes, and these can be _very_ long lists of strings - def escape_re_range_char(c): + def escape_re_range_char(c: str) -> str: return "\\" + c if c in r"\^-][" else c - def no_escape_re_range_char(c): + def no_escape_re_range_char(c: str) -> str: return c if not re_escape: escape_re_range_char = no_escape_re_range_char ret = [] - s = "".join(sorted(set(s))) - if len(s) > 3: - for _, chars in itertools.groupby(s, key=is_consecutive): + + # reduce input string to remove duplicates, and put in sorted order + s_chars: list[str] = sorted(set(s)) + + if len(s_chars) > 2: + # find groups of characters that are consecutive (can be collapsed + # down to "<first>-<last>") + for _, chars in itertools.groupby(s_chars, key=_GroupConsecutive()): + # _ is unimportant, is just used to identify groups + # chars is an iterator of one or more consecutive characters + # that comprise the current group first = last = next(chars) - last = collections.deque( - itertools.chain(iter([last]), chars), maxlen=1 - ).pop() + with contextlib.suppress(ValueError): + *_, last = chars + if first == last: + # there was only a single char in this group ret.append(escape_re_range_char(first)) + + elif last == chr(ord(first) + 1): + # there were only 2 characters in this group + # 'a','b' -> 'ab' + ret.append(f"{escape_re_range_char(first)}{escape_re_range_char(last)}") + else: - sep = "" if ord(last) == ord(first) + 1 else "-" + # there were > 2 characters in this group, make into a range + # 'c','d','e' -> 'c-e' ret.append( - f"{escape_re_range_char(first)}{sep}{escape_re_range_char(last)}" + f"{escape_re_range_char(first)}-{escape_re_range_char(last)}" ) else: - ret = [escape_re_range_char(c) for c in s] + # only 1 or 2 chars were given to form into groups + # 'a' -> ['a'] + # 'bc' -> ['b', 'c'] + # 'dg' -> ['d', 'g'] + # no need to list them with "-", just return as a list + # (after escaping) + ret = [escape_re_range_char(c) for c in s_chars] return "".join(ret) -def _flatten(ll: list) -> list: +def _flatten(ll: Iterable) -> list: ret = [] - for i in ll: - if isinstance(i, list): - ret.extend(_flatten(i)) + to_visit = [*ll] + while to_visit: + i = to_visit.pop(0) + if isinstance(i, Iterable) and not isinstance(i, str): + to_visit[:0] = i else: ret.append(i) return ret +def make_compressed_re( + word_list: Iterable[str], max_level: int = 2, _level: int = 1 +) -> str: + """ + Create a regular expression string from a list of words, collapsing by common + prefixes and optional suffixes. + + Calls itself recursively to build nested sublists for each group of suffixes + that have a shared prefix. + """ + + def get_suffixes_from_common_prefixes(namelist: list[str]): + if len(namelist) > 1: + for prefix, suffixes in itertools.groupby(namelist, key=lambda s: s[:1]): + yield prefix, sorted([s[1:] for s in suffixes], key=len, reverse=True) + else: + yield namelist[0][0], [namelist[0][1:]] + + if max_level == 0: + return "|".join(sorted(word_list, key=len, reverse=True)) + + ret = [] + sep = "" + for initial, suffixes in get_suffixes_from_common_prefixes(sorted(word_list)): + ret.append(sep) + sep = "|" + + trailing = "" + if "" in suffixes: + trailing = "?" + suffixes.remove("") + + if len(suffixes) > 1: + if all(len(s) == 1 for s in suffixes): + ret.append(f"{initial}[{''.join(suffixes)}]{trailing}") + else: + if _level < max_level: + suffix_re = make_compressed_re( + sorted(suffixes), max_level, _level + 1 + ) + ret.append(f"{initial}({suffix_re}){trailing}") + else: + suffixes.sort(key=len, reverse=True) + ret.append(f"{initial}({'|'.join(suffixes)}){trailing}") + else: + if suffixes: + suffix = suffixes[0] + if len(suffix) > 1 and trailing: + ret.append(f"{initial}({suffix}){trailing}") + else: + ret.append(f"{initial}{suffix}{trailing}") + else: + ret.append(initial) + return "".join(ret) + + def replaced_by_pep8(compat_name: str, fn: C) -> C: # In a future version, uncomment the code in the internal _inner() functions # to begin emitting DeprecationWarnings. @@ -268,10 +388,10 @@ def replaced_by_pep8(compat_name: str, fn: C) -> C: _inner.__name__ = compat_name _inner.__annotations__ = fn.__annotations__ if isinstance(fn, types.FunctionType): - _inner.__kwdefaults__ = fn.__kwdefaults__ + _inner.__kwdefaults__ = fn.__kwdefaults__ # type: ignore [attr-defined] elif isinstance(fn, type) and hasattr(fn, "__init__"): - _inner.__kwdefaults__ = fn.__init__.__kwdefaults__ + _inner.__kwdefaults__ = fn.__init__.__kwdefaults__ # type: ignore [misc,attr-defined] else: - _inner.__kwdefaults__ = None + _inner.__kwdefaults__ = None # type: ignore [attr-defined] _inner.__qualname__ = fn.__qualname__ return cast(C, _inner) diff --git a/contrib/python/pyparsing/py3/ya.make b/contrib/python/pyparsing/py3/ya.make index c5575db221..1b36194fab 100644 --- a/contrib/python/pyparsing/py3/ya.make +++ b/contrib/python/pyparsing/py3/ya.make @@ -4,7 +4,7 @@ PY3_LIBRARY() PROVIDES(pyparsing) -VERSION(3.1.4) +VERSION(3.2.0) LICENSE(MIT) diff --git a/contrib/python/xmltodict/py3/.dist-info/METADATA b/contrib/python/xmltodict/py3/.dist-info/METADATA index e2aaa3e797..5a038d733d 100644 --- a/contrib/python/xmltodict/py3/.dist-info/METADATA +++ b/contrib/python/xmltodict/py3/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: xmltodict -Version: 0.13.0 +Version: 0.14.2 Summary: Makes working with XML feel like you are working with JSON Home-page: https://github.com/martinblech/xmltodict Author: Martin Blech @@ -12,16 +12,17 @@ Classifier: License :: OSI Approved :: MIT License Classifier: Operating System :: OS Independent Classifier: Programming Language :: Python Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python :: 3.4 -Classifier: Programming Language :: Python :: 3.5 Classifier: Programming Language :: Python :: 3.6 Classifier: Programming Language :: Python :: 3.7 Classifier: Programming Language :: Python :: 3.8 Classifier: Programming Language :: Python :: 3.9 Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 +Classifier: Programming Language :: Python :: 3.13 Classifier: Programming Language :: Python :: Implementation :: PyPy Classifier: Topic :: Text Processing :: Markup :: XML -Requires-Python: >=3.4 +Requires-Python: >=3.6 Description-Content-Type: text/markdown License-File: LICENSE @@ -29,7 +30,7 @@ License-File: LICENSE `xmltodict` is a Python module that makes working with XML feel like you are working with [JSON](http://docs.python.org/library/json.html), as in this ["spec"](http://www.xml.com/pub/a/2006/05/31/converting-between-xml-and-json.html): -[![Build Status](https://travis-ci.com/martinblech/xmltodict.svg?branch=master)](https://travis-ci.com/martinblech/xmltodict) +[![Build Status](https://app.travis-ci.com/martinblech/xmltodict.svg?branch=master)](https://app.travis-ci.com/martinblech/xmltodict) ```python >>> print(json.dumps(xmltodict.parse(""" @@ -231,6 +232,17 @@ You just need to $ pip install xmltodict ``` +### Using conda + +For installing `xmltodict` using Anaconda/Miniconda (*conda*) from the +[conda-forge channel][#xmltodict-conda] all you need to do is: + +[#xmltodict-conda]: https://anaconda.org/conda-forge/xmltodict + +```sh +$ conda install -c conda-forge xmltodict +``` + ### RPM-based distro (Fedora, RHEL, …) There is an [official Fedora package for xmltodict](https://apps.fedoraproject.org/packages/python-xmltodict). @@ -274,5 +286,3 @@ $ zypper in python2-xmltodict # Python3 $ zypper in python3-xmltodict ``` - - diff --git a/contrib/python/xmltodict/py3/README.md b/contrib/python/xmltodict/py3/README.md index ab63401a80..6f776a8b4d 100644 --- a/contrib/python/xmltodict/py3/README.md +++ b/contrib/python/xmltodict/py3/README.md @@ -2,7 +2,7 @@ `xmltodict` is a Python module that makes working with XML feel like you are working with [JSON](http://docs.python.org/library/json.html), as in this ["spec"](http://www.xml.com/pub/a/2006/05/31/converting-between-xml-and-json.html): -[![Build Status](https://travis-ci.com/martinblech/xmltodict.svg?branch=master)](https://travis-ci.com/martinblech/xmltodict) +[![Build Status](https://app.travis-ci.com/martinblech/xmltodict.svg?branch=master)](https://app.travis-ci.com/martinblech/xmltodict) ```python >>> print(json.dumps(xmltodict.parse(""" @@ -204,6 +204,17 @@ You just need to $ pip install xmltodict ``` +### Using conda + +For installing `xmltodict` using Anaconda/Miniconda (*conda*) from the +[conda-forge channel][#xmltodict-conda] all you need to do is: + +[#xmltodict-conda]: https://anaconda.org/conda-forge/xmltodict + +```sh +$ conda install -c conda-forge xmltodict +``` + ### RPM-based distro (Fedora, RHEL, …) There is an [official Fedora package for xmltodict](https://apps.fedoraproject.org/packages/python-xmltodict). diff --git a/contrib/python/xmltodict/py3/tests/__init__.py b/contrib/python/xmltodict/py3/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/contrib/python/xmltodict/py3/tests/__init__.py diff --git a/contrib/python/xmltodict/py3/tests/test_dicttoxml.py b/contrib/python/xmltodict/py3/tests/test_dicttoxml.py index 7fc21718ae..470aca98a1 100644 --- a/contrib/python/xmltodict/py3/tests/test_dicttoxml.py +++ b/contrib/python/xmltodict/py3/tests/test_dicttoxml.py @@ -144,6 +144,26 @@ class DictToXMLTestCase(unittest.TestCase): self.assertEqual(xml, unparse(obj, pretty=True, newl=newl, indent=indent)) + def test_pretty_print_with_int_indent(self): + obj = {'a': OrderedDict(( + ('b', [{'c': [1, 2]}, 3]), + ('x', 'y'), + ))} + newl = '\n' + indent = 2 + xml = dedent('''\ + <?xml version="1.0" encoding="utf-8"?> + <a> + <b> + <c>1</c> + <c>2</c> + </b> + <b>3</b> + <x>y</x> + </a>''') + self.assertEqual(xml, unparse(obj, pretty=True, + newl=newl, indent=indent)) + def test_encoding(self): try: value = unichr(39321) @@ -171,8 +191,6 @@ class DictToXMLTestCase(unittest.TestCase): self.assertEqual('<a attr="1"></a>', _strip(unparse(obj))) def test_short_empty_elements(self): - if sys.version_info[0] < 3: - return obj = {'a': None} self.assertEqual('<a/>', _strip(unparse(obj, short_empty_elements=True))) diff --git a/contrib/python/xmltodict/py3/tests/test_xmltodict.py b/contrib/python/xmltodict/py3/tests/test_xmltodict.py index 04137f9e03..7dd22b53f3 100644 --- a/contrib/python/xmltodict/py3/tests/test_xmltodict.py +++ b/contrib/python/xmltodict/py3/tests/test_xmltodict.py @@ -168,14 +168,14 @@ class XMLToDictTestCase(unittest.TestCase): except NameError: value = chr(39321) self.assertEqual({'a': value}, - parse('<a>%s</a>' % value)) + parse(f'<a>{value}</a>')) def test_encoded_string(self): try: value = unichr(39321) except NameError: value = chr(39321) - xml = '<a>%s</a>' % value + xml = f'<a>{value}</a>' self.assertEqual(parse(xml), parse(xml.encode('utf-8'))) @@ -457,3 +457,21 @@ class XMLToDictTestCase(unittest.TestCase): } } self.assertEqual(parse(xml, process_comments=True), expectedResult) + + def test_streaming_attrs(self): + xml = """ + <a> + <b attr1="value"> + <c>cdata</c> + </b> + </a> + """ + def handler(path, item): + expected = { + '@attr1': 'value', + 'c': 'cdata' + } + self.assertEqual(expected, item) + return True + + parse(xml, item_depth=2, item_callback=handler) diff --git a/contrib/python/xmltodict/py3/xmltodict.py b/contrib/python/xmltodict/py3/xmltodict.py index ca760aa637..098f62762a 100644 --- a/contrib/python/xmltodict/py3/xmltodict.py +++ b/contrib/python/xmltodict/py3/xmltodict.py @@ -1,19 +1,10 @@ #!/usr/bin/env python "Makes working with XML feel like you are working with JSON" -try: - from defusedexpat import pyexpat as expat -except ImportError: - from xml.parsers import expat +from xml.parsers import expat from xml.sax.saxutils import XMLGenerator from xml.sax.xmlreader import AttributesImpl -try: # pragma no cover - from cStringIO import StringIO -except ImportError: # pragma no cover - try: - from StringIO import StringIO - except ImportError: - from io import StringIO +from io import StringIO _dict = dict import platform @@ -22,17 +13,8 @@ if tuple(map(int, platform.python_version_tuple()[:2])) < (3, 7): from inspect import isgenerator -try: # pragma no cover - _basestring = basestring -except NameError: # pragma no cover - _basestring = str -try: # pragma no cover - _unicode = unicode -except NameError: # pragma no cover - _unicode = str - __author__ = 'Martin Blech' -__version__ = '0.13.0' +__version__ = "0.14.2" __license__ = 'MIT' @@ -40,7 +22,7 @@ class ParsingInterrupted(Exception): pass -class _DictSAXHandler(object): +class _DictSAXHandler: def __init__(self, item_depth=0, item_callback=lambda *args: True, @@ -107,7 +89,7 @@ class _DictSAXHandler(object): attrs['xmlns'] = self.namespace_declarations self.namespace_declarations = self.dict_constructor() self.path.append((name, attrs or None)) - if len(self.path) > self.item_depth: + if len(self.path) >= self.item_depth: self.stack.append((self.item, self.data)) if self.xml_attribs: attr_entries = [] @@ -135,7 +117,7 @@ class _DictSAXHandler(object): should_continue = self.item_callback(self.path, item) if not should_continue: - raise ParsingInterrupted() + raise ParsingInterrupted if self.stack: data = (None if not self.data else self.cdata_separator.join(self.data)) @@ -335,9 +317,8 @@ def parse(xml_input, encoding=None, expat=expat, process_namespaces=False, """ handler = _DictSAXHandler(namespace_separator=namespace_separator, **kwargs) - if isinstance(xml_input, _unicode): - if not encoding: - encoding = 'utf-8' + if isinstance(xml_input, str): + encoding = encoding or 'utf-8' xml_input = xml_input.encode(encoding) if not process_namespaces: namespace_separator = None @@ -372,8 +353,8 @@ def parse(xml_input, encoding=None, expat=expat, process_namespaces=False, parser.ParseFile(xml_input) elif isgenerator(xml_input): for chunk in xml_input: - parser.Parse(chunk,False) - parser.Parse(b'',True) + parser.Parse(chunk, False) + parser.Parse(b'', True) else: parser.Parse(xml_input, True) return handler.item @@ -412,9 +393,7 @@ def _emit(key, value, content_handler, if result is None: return key, value = result - if (not hasattr(value, '__iter__') - or isinstance(value, _basestring) - or isinstance(value, dict)): + if not hasattr(value, '__iter__') or isinstance(value, (str, dict)): value = [value] for index, v in enumerate(value): if full_document and depth == 0 and index > 0: @@ -422,16 +401,13 @@ def _emit(key, value, content_handler, if v is None: v = _dict() elif isinstance(v, bool): - if v: - v = _unicode('true') - else: - v = _unicode('false') - elif not isinstance(v, dict): - if expand_iter and hasattr(v, '__iter__') and not isinstance(v, _basestring): + v = 'true' if v else 'false' + elif not isinstance(v, (dict, str)): + if expand_iter and hasattr(v, '__iter__'): v = _dict(((expand_iter, v),)) else: - v = _unicode(v) - if isinstance(v, _basestring): + v = str(v) + if isinstance(v, str): v = _dict(((cdata_key, v),)) cdata = None attrs = _dict() @@ -445,14 +421,16 @@ def _emit(key, value, content_handler, attr_prefix) if ik == '@xmlns' and isinstance(iv, dict): for k, v in iv.items(): - attr = 'xmlns{}'.format(':{}'.format(k) if k else '') - attrs[attr] = _unicode(v) + attr = 'xmlns{}'.format(f':{k}' if k else '') + attrs[attr] = str(v) continue - if not isinstance(iv, _unicode): - iv = _unicode(iv) + if not isinstance(iv, str): + iv = str(iv) attrs[ik[len(attr_prefix):]] = iv continue children.append((ik, iv)) + if isinstance(indent, int): + indent = ' ' * indent if pretty: content_handler.ignorableWhitespace(depth * indent) content_handler.startElement(key, AttributesImpl(attrs)) diff --git a/contrib/python/xmltodict/py3/ya.make b/contrib/python/xmltodict/py3/ya.make index 6872d9affa..7fdc1a14a9 100644 --- a/contrib/python/xmltodict/py3/ya.make +++ b/contrib/python/xmltodict/py3/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(0.13.0) +VERSION(0.14.2) LICENSE(MIT) diff --git a/contrib/tools/protoc/resources.json b/contrib/tools/protoc/resources.json index 2e78d6bf7f..bc895b2859 100644 --- a/contrib/tools/protoc/resources.json +++ b/contrib/tools/protoc/resources.json @@ -1,19 +1,19 @@ { "by_platform": { "darwin": { - "uri": "sbr:7039760192" + "uri": "sbr:7360495216" }, "darwin-arm64": { - "uri": "sbr:7039759830" + "uri": "sbr:7360493349" }, "linux": { - "uri": "sbr:7039760920" + "uri": "sbr:7360499157" }, "linux-aarch64": { - "uri": "sbr:7039760572" + "uri": "sbr:7360497335" }, "win32": { - "uri": "sbr:7039759509" + "uri": "sbr:7360491807" } } } diff --git a/library/cpp/blockcodecs/core/codecs.h b/library/cpp/blockcodecs/core/codecs.h index 9c93c00274..6512abddef 100644 --- a/library/cpp/blockcodecs/core/codecs.h +++ b/library/cpp/blockcodecs/core/codecs.h @@ -25,6 +25,12 @@ namespace NBlockCodecs { : TStringBuf((const char*)t.Data(), t.Size()) { } + + template <> + inline TData(const TString& t) + : TStringBuf((const char*)t.data(), t.size()) + { + } }; struct TCodecError: public yexception { diff --git a/library/cpp/tld/tlds-alpha-by-domain.txt b/library/cpp/tld/tlds-alpha-by-domain.txt index fbf58d0ddb..4964f4e61d 100644 --- a/library/cpp/tld/tlds-alpha-by-domain.txt +++ b/library/cpp/tld/tlds-alpha-by-domain.txt @@ -1,4 +1,4 @@ -# Version 2024102500, Last Updated Fri Oct 25 07:07:02 2024 UTC +# Version 2024102800, Last Updated Mon Oct 28 07:07:02 2024 UTC AAA AARP ABB diff --git a/library/cpp/yt/logging/backends/stream/stream_log_manager.cpp b/library/cpp/yt/logging/backends/stream/stream_log_manager.cpp index 62fa3e91d0..8b46d5679f 100644 --- a/library/cpp/yt/logging/backends/stream/stream_log_manager.cpp +++ b/library/cpp/yt/logging/backends/stream/stream_log_manager.cpp @@ -21,12 +21,10 @@ public: { } void RegisterStaticAnchor( - TLoggingAnchor* anchor, + TLoggingAnchor* /*anchor*/, ::TSourceLocation /*sourceLocation*/, TStringBuf /*anchorMessage*/) override - { - anchor->Registered = true; - } + { } virtual void UpdateAnchor(TLoggingAnchor* /*anchor*/) override { } diff --git a/library/cpp/yt/logging/logger.cpp b/library/cpp/yt/logging/logger.cpp index 58add38429..a6a0010b3b 100644 --- a/library/cpp/yt/logging/logger.cpp +++ b/library/cpp/yt/logging/logger.cpp @@ -195,14 +195,21 @@ bool TLogger::IsEssential() const return Essential_; } -void TLogger::UpdateAnchor(TLoggingAnchor* anchor) const +void TLogger::UpdateStaticAnchor( + TLoggingAnchor* anchor, + std::atomic<bool>* anchorRegistered, + ::TSourceLocation sourceLocation, + TStringBuf message) const { + if (!anchorRegistered->exchange(true)) { + LogManager_->RegisterStaticAnchor(anchor, sourceLocation, message); + } LogManager_->UpdateAnchor(anchor); } -void TLogger::RegisterStaticAnchor(TLoggingAnchor* anchor, ::TSourceLocation sourceLocation, TStringBuf message) const +void TLogger::UpdateDynamicAnchor(TLoggingAnchor* anchor) const { - LogManager_->RegisterStaticAnchor(anchor, sourceLocation, message); + LogManager_->UpdateAnchor(anchor); } void TLogger::Write(TLogEvent&& event) const diff --git a/library/cpp/yt/logging/logger.h b/library/cpp/yt/logging/logger.h index 35aba4eb4c..eff5da078e 100644 --- a/library/cpp/yt/logging/logger.h +++ b/library/cpp/yt/logging/logger.h @@ -206,8 +206,12 @@ public: bool IsEssential() const; bool IsAnchorUpToDate(const TLoggingAnchor& anchor) const; - void UpdateAnchor(TLoggingAnchor* anchor) const; - void RegisterStaticAnchor(TLoggingAnchor* anchor, ::TSourceLocation sourceLocation, TStringBuf message) const; + void UpdateStaticAnchor( + TLoggingAnchor* anchor, + std::atomic<bool>* anchorRegistered, + ::TSourceLocation sourceLocation, + TStringBuf message) const; + void UpdateDynamicAnchor(TLoggingAnchor* anchor) const; void Write(TLogEvent&& event) const; @@ -302,19 +306,12 @@ void LogStructuredEvent( #define YT_LOG_FATAL_UNLESS(condition, ...) if (!Y_LIKELY(condition)) YT_LOG_FATAL(__VA_ARGS__) #define YT_LOG_EVENT(logger, level, ...) \ - YT_LOG_EVENT_WITH_ANCHOR(logger, level, nullptr, __VA_ARGS__) - -#define YT_LOG_EVENT_WITH_ANCHOR(logger, level, anchor, ...) \ do { \ const auto& logger__ = (logger)(); \ auto level__ = (level); \ auto location__ = __LOCATION__; \ - \ - ::NYT::NLogging::TLoggingAnchor* anchor__ = (anchor); \ - [[unlikely]] if (!anchor__) { \ - static ::NYT::TLeakyStorage<::NYT::NLogging::TLoggingAnchor> staticAnchor__; \ - anchor__ = staticAnchor__.Get(); \ - } \ + static ::NYT::TLeakyStorage<::NYT::NLogging::TLoggingAnchor> anchorStorage__; \ + auto* anchor__ = anchorStorage__.Get(); \ \ bool anchorUpToDate__ = logger__.IsAnchorUpToDate(*anchor__); \ [[likely]] if (anchorUpToDate__) { \ @@ -328,7 +325,8 @@ void LogStructuredEvent( auto message__ = ::NYT::NLogging::NDetail::BuildLogMessage(loggingContext__, logger__, __VA_ARGS__); \ \ [[unlikely]] if (!anchorUpToDate__) { \ - logger__.RegisterStaticAnchor(anchor__, location__, message__.Anchor); \ + static std::atomic<bool> anchorRegistered__; \ + logger__.UpdateStaticAnchor(anchor__, &anchorRegistered__, location__, message__.Anchor); \ } \ \ auto effectiveLevel__ = ::NYT::NLogging::TLogger::GetEffectiveLoggingLevel(level__, *anchor__); \ @@ -345,6 +343,35 @@ void LogStructuredEvent( std::move(message__.MessageRef)); \ } while (false) +#define YT_LOG_EVENT_WITH_DYNAMIC_ANCHOR(logger, level, anchor, ...) \ + do { \ + const auto& logger__ = (logger)(); \ + auto level__ = (level); \ + auto location__ = __LOCATION__; \ + auto* anchor__ = (anchor); \ + \ + bool anchorUpToDate__ = logger__.IsAnchorUpToDate(*anchor__); \ + [[unlikely]] if (!anchorUpToDate__) { \ + logger__.UpdateDynamicAnchor(anchor__); \ + } \ + \ + auto effectiveLevel__ = ::NYT::NLogging::TLogger::GetEffectiveLoggingLevel(level__, *anchor__); \ + if (!logger__.IsLevelEnabled(effectiveLevel__)) { \ + break; \ + } \ + \ + auto loggingContext__ = ::NYT::NLogging::GetLoggingContext(); \ + auto message__ = ::NYT::NLogging::NDetail::BuildLogMessage(loggingContext__, logger__, __VA_ARGS__); \ + \ + ::NYT::NLogging::NDetail::LogEventImpl( \ + loggingContext__, \ + logger__, \ + effectiveLevel__, \ + location__, \ + anchor__, \ + std::move(message__.MessageRef)); \ + } while (false) + //////////////////////////////////////////////////////////////////////////////// } // namespace NYT::NLogging diff --git a/library/cpp/yt/yson_string/convert.cpp b/library/cpp/yt/yson_string/convert.cpp index 6d78ea6c9d..68241adb78 100644 --- a/library/cpp/yt/yson_string/convert.cpp +++ b/library/cpp/yt/yson_string/convert.cpp @@ -67,6 +67,12 @@ TYsonString ConvertToYsonString<TString>(const TString& value) return ConvertToYsonString(static_cast<TStringBuf>(value)); } +template <> +TYsonString ConvertToYsonString<std::string>(const std::string& value) +{ + return ConvertToYsonString(static_cast<TStringBuf>(value)); +} + struct TConvertStringToYsonStringTag { }; diff --git a/library/cpp/yt/yson_string/convert.h b/library/cpp/yt/yson_string/convert.h index 06de28d2f9..eedb0939e0 100644 --- a/library/cpp/yt/yson_string/convert.h +++ b/library/cpp/yt/yson_string/convert.h @@ -44,6 +44,8 @@ TYsonString ConvertToYsonString<ui64>(const ui64& value); template <> TYsonString ConvertToYsonString<TString>(const TString& value); template <> +TYsonString ConvertToYsonString<std::string>(const std::string& value); +template <> TYsonString ConvertToYsonString<TStringBuf>(const TStringBuf& value); TYsonString ConvertToYsonString(const char* value); diff --git a/vendor/github.com/golang/protobuf/internal/gengogrpc/ya.make b/vendor/github.com/golang/protobuf/internal/gengogrpc/ya.make index 3c94f9e302..942e632e4a 100644 --- a/vendor/github.com/golang/protobuf/internal/gengogrpc/ya.make +++ b/vendor/github.com/golang/protobuf/internal/gengogrpc/ya.make @@ -2,6 +2,8 @@ GO_LIBRARY() LICENSE(BSD-3-Clause) +VERSION(v1.5.4) + SRCS( grpc.go ) diff --git a/vendor/github.com/golang/protobuf/protoc-gen-go/ya.make b/vendor/github.com/golang/protobuf/protoc-gen-go/ya.make index 71c8487166..db464d8212 100644 --- a/vendor/github.com/golang/protobuf/protoc-gen-go/ya.make +++ b/vendor/github.com/golang/protobuf/protoc-gen-go/ya.make @@ -2,6 +2,8 @@ GO_PROGRAM() LICENSE(BSD-3-Clause) +VERSION(v1.5.4) + SRCS( main.go ) diff --git a/vendor/github.com/spf13/cobra/ya.make b/vendor/github.com/spf13/cobra/ya.make index 4242c95a67..2d15e11c9b 100644 --- a/vendor/github.com/spf13/cobra/ya.make +++ b/vendor/github.com/spf13/cobra/ya.make @@ -2,6 +2,8 @@ GO_LIBRARY() LICENSE(Apache-2.0) +VERSION(v1.8.1) + SRCS( active_help.go args.go diff --git a/vendor/github.com/spf13/pflag/ya.make b/vendor/github.com/spf13/pflag/ya.make index 095b7a2b45..c40c23cce9 100644 --- a/vendor/github.com/spf13/pflag/ya.make +++ b/vendor/github.com/spf13/pflag/ya.make @@ -2,6 +2,8 @@ GO_LIBRARY() LICENSE(BSD-3-Clause) +VERSION(v1.0.6-0.20201009195203-85dd5c8bc61c) + SRCS( bool.go bool_slice.go diff --git a/vendor/google.golang.org/protobuf/cmd/protoc-gen-go/internal_gengo/ya.make b/vendor/google.golang.org/protobuf/cmd/protoc-gen-go/internal_gengo/ya.make index 114c371ff4..765e4b291d 100644 --- a/vendor/google.golang.org/protobuf/cmd/protoc-gen-go/internal_gengo/ya.make +++ b/vendor/google.golang.org/protobuf/cmd/protoc-gen-go/internal_gengo/ya.make @@ -2,6 +2,8 @@ GO_LIBRARY() LICENSE(BSD-3-Clause) +VERSION(v1.34.2) + SRCS( init.go main.go diff --git a/vendor/google.golang.org/protobuf/compiler/protogen/ya.make b/vendor/google.golang.org/protobuf/compiler/protogen/ya.make index 65652dc84a..f2e0e6e3e0 100644 --- a/vendor/google.golang.org/protobuf/compiler/protogen/ya.make +++ b/vendor/google.golang.org/protobuf/compiler/protogen/ya.make @@ -2,6 +2,8 @@ GO_LIBRARY() LICENSE(BSD-3-Clause) +VERSION(v1.34.2) + SRCS( protogen.go ) diff --git a/vendor/google.golang.org/protobuf/encoding/prototext/ya.make b/vendor/google.golang.org/protobuf/encoding/prototext/ya.make index 14ea8e575d..3d35bb8109 100644 --- a/vendor/google.golang.org/protobuf/encoding/prototext/ya.make +++ b/vendor/google.golang.org/protobuf/encoding/prototext/ya.make @@ -2,6 +2,8 @@ GO_LIBRARY() LICENSE(BSD-3-Clause) +VERSION(v1.34.2) + SRCS( decode.go doc.go diff --git a/vendor/google.golang.org/protobuf/encoding/protowire/ya.make b/vendor/google.golang.org/protobuf/encoding/protowire/ya.make index 74a3dc5f92..e944a7face 100644 --- a/vendor/google.golang.org/protobuf/encoding/protowire/ya.make +++ b/vendor/google.golang.org/protobuf/encoding/protowire/ya.make @@ -2,6 +2,8 @@ GO_LIBRARY() LICENSE(BSD-3-Clause) +VERSION(v1.34.2) + SRCS( wire.go ) diff --git a/vendor/google.golang.org/protobuf/internal/descfmt/ya.make b/vendor/google.golang.org/protobuf/internal/descfmt/ya.make index 6f0464b58b..d1c40a75f7 100644 --- a/vendor/google.golang.org/protobuf/internal/descfmt/ya.make +++ b/vendor/google.golang.org/protobuf/internal/descfmt/ya.make @@ -2,6 +2,8 @@ GO_LIBRARY() LICENSE(BSD-3-Clause) +VERSION(v1.34.2) + SRCS( stringer.go ) diff --git a/vendor/google.golang.org/protobuf/internal/descopts/ya.make b/vendor/google.golang.org/protobuf/internal/descopts/ya.make index 26ef6cc24b..b63e12cf89 100644 --- a/vendor/google.golang.org/protobuf/internal/descopts/ya.make +++ b/vendor/google.golang.org/protobuf/internal/descopts/ya.make @@ -2,6 +2,8 @@ GO_LIBRARY() LICENSE(BSD-3-Clause) +VERSION(v1.34.2) + SRCS( options.go ) diff --git a/vendor/google.golang.org/protobuf/internal/detrand/ya.make b/vendor/google.golang.org/protobuf/internal/detrand/ya.make index dd1b8456b9..a0d7ea6c2c 100644 --- a/vendor/google.golang.org/protobuf/internal/detrand/ya.make +++ b/vendor/google.golang.org/protobuf/internal/detrand/ya.make @@ -2,6 +2,8 @@ GO_LIBRARY() LICENSE(BSD-3-Clause) +VERSION(v1.34.2) + SRCS( rand.go ) diff --git a/vendor/google.golang.org/protobuf/internal/editiondefaults/ya.make b/vendor/google.golang.org/protobuf/internal/editiondefaults/ya.make index eff20cca8a..e78ea669be 100644 --- a/vendor/google.golang.org/protobuf/internal/editiondefaults/ya.make +++ b/vendor/google.golang.org/protobuf/internal/editiondefaults/ya.make @@ -2,6 +2,8 @@ GO_LIBRARY() LICENSE(BSD-3-Clause) +VERSION(v1.34.2) + SRCS( defaults.go ) diff --git a/vendor/google.golang.org/protobuf/internal/editionssupport/ya.make b/vendor/google.golang.org/protobuf/internal/editionssupport/ya.make index fde817ec3b..45b82c7de2 100644 --- a/vendor/google.golang.org/protobuf/internal/editionssupport/ya.make +++ b/vendor/google.golang.org/protobuf/internal/editionssupport/ya.make @@ -2,6 +2,8 @@ GO_LIBRARY() LICENSE(BSD-3-Clause) +VERSION(v1.34.2) + SRCS( editions.go ) diff --git a/vendor/google.golang.org/protobuf/internal/encoding/defval/ya.make b/vendor/google.golang.org/protobuf/internal/encoding/defval/ya.make index 9f5b5a3fb7..6ba0c708dc 100644 --- a/vendor/google.golang.org/protobuf/internal/encoding/defval/ya.make +++ b/vendor/google.golang.org/protobuf/internal/encoding/defval/ya.make @@ -2,6 +2,8 @@ GO_LIBRARY() LICENSE(BSD-3-Clause) +VERSION(v1.34.2) + SRCS( default.go ) diff --git a/vendor/google.golang.org/protobuf/internal/encoding/messageset/ya.make b/vendor/google.golang.org/protobuf/internal/encoding/messageset/ya.make index 3681a9a476..946f28d167 100644 --- a/vendor/google.golang.org/protobuf/internal/encoding/messageset/ya.make +++ b/vendor/google.golang.org/protobuf/internal/encoding/messageset/ya.make @@ -2,6 +2,8 @@ GO_LIBRARY() LICENSE(BSD-3-Clause) +VERSION(v1.34.2) + SRCS( messageset.go ) diff --git a/vendor/google.golang.org/protobuf/internal/encoding/tag/ya.make b/vendor/google.golang.org/protobuf/internal/encoding/tag/ya.make index 12d85eb139..3b51433f66 100644 --- a/vendor/google.golang.org/protobuf/internal/encoding/tag/ya.make +++ b/vendor/google.golang.org/protobuf/internal/encoding/tag/ya.make @@ -2,6 +2,8 @@ GO_LIBRARY() LICENSE(BSD-3-Clause) +VERSION(v1.34.2) + SRCS( tag.go ) diff --git a/vendor/google.golang.org/protobuf/internal/encoding/text/ya.make b/vendor/google.golang.org/protobuf/internal/encoding/text/ya.make index 5665f82701..cab9b833b2 100644 --- a/vendor/google.golang.org/protobuf/internal/encoding/text/ya.make +++ b/vendor/google.golang.org/protobuf/internal/encoding/text/ya.make @@ -2,6 +2,8 @@ GO_LIBRARY() LICENSE(BSD-3-Clause) +VERSION(v1.34.2) + SRCS( decode.go decode_number.go diff --git a/vendor/google.golang.org/protobuf/internal/errors/ya.make b/vendor/google.golang.org/protobuf/internal/errors/ya.make index d958a0d019..446386647a 100644 --- a/vendor/google.golang.org/protobuf/internal/errors/ya.make +++ b/vendor/google.golang.org/protobuf/internal/errors/ya.make @@ -2,6 +2,8 @@ GO_LIBRARY() LICENSE(BSD-3-Clause) +VERSION(v1.34.2) + SRCS( errors.go is_go113.go diff --git a/vendor/google.golang.org/protobuf/internal/filedesc/ya.make b/vendor/google.golang.org/protobuf/internal/filedesc/ya.make index df85be08f6..11ee917a02 100644 --- a/vendor/google.golang.org/protobuf/internal/filedesc/ya.make +++ b/vendor/google.golang.org/protobuf/internal/filedesc/ya.make @@ -2,6 +2,8 @@ GO_LIBRARY() LICENSE(BSD-3-Clause) +VERSION(v1.34.2) + SRCS( build.go desc.go diff --git a/vendor/google.golang.org/protobuf/internal/filetype/ya.make b/vendor/google.golang.org/protobuf/internal/filetype/ya.make index e0fc61c11b..9dce0cf0d9 100644 --- a/vendor/google.golang.org/protobuf/internal/filetype/ya.make +++ b/vendor/google.golang.org/protobuf/internal/filetype/ya.make @@ -2,6 +2,8 @@ GO_LIBRARY() LICENSE(BSD-3-Clause) +VERSION(v1.34.2) + SRCS( build.go ) diff --git a/vendor/google.golang.org/protobuf/internal/flags/ya.make b/vendor/google.golang.org/protobuf/internal/flags/ya.make index 0116cd8953..f75da040cc 100644 --- a/vendor/google.golang.org/protobuf/internal/flags/ya.make +++ b/vendor/google.golang.org/protobuf/internal/flags/ya.make @@ -2,6 +2,8 @@ GO_LIBRARY() LICENSE(BSD-3-Clause) +VERSION(v1.34.2) + SRCS( flags.go proto_legacy_disable.go diff --git a/vendor/google.golang.org/protobuf/internal/genid/ya.make b/vendor/google.golang.org/protobuf/internal/genid/ya.make index e257c029a9..3df4ae5b38 100644 --- a/vendor/google.golang.org/protobuf/internal/genid/ya.make +++ b/vendor/google.golang.org/protobuf/internal/genid/ya.make @@ -2,6 +2,8 @@ GO_LIBRARY() LICENSE(BSD-3-Clause) +VERSION(v1.34.2) + SRCS( any_gen.go api_gen.go diff --git a/vendor/google.golang.org/protobuf/internal/impl/ya.make b/vendor/google.golang.org/protobuf/internal/impl/ya.make index 3d5e1a1077..6f477685fc 100644 --- a/vendor/google.golang.org/protobuf/internal/impl/ya.make +++ b/vendor/google.golang.org/protobuf/internal/impl/ya.make @@ -2,6 +2,8 @@ GO_LIBRARY() LICENSE(BSD-3-Clause) +VERSION(v1.34.2) + SRCS( api_export.go checkinit.go diff --git a/vendor/google.golang.org/protobuf/internal/msgfmt/ya.make b/vendor/google.golang.org/protobuf/internal/msgfmt/ya.make index 5ed4b140dc..cd16c939f5 100644 --- a/vendor/google.golang.org/protobuf/internal/msgfmt/ya.make +++ b/vendor/google.golang.org/protobuf/internal/msgfmt/ya.make @@ -2,6 +2,8 @@ GO_LIBRARY() LICENSE(BSD-3-Clause) +VERSION(v1.34.2) + SRCS( format.go ) diff --git a/vendor/google.golang.org/protobuf/internal/order/ya.make b/vendor/google.golang.org/protobuf/internal/order/ya.make index fbc58d1f9b..ed17c14646 100644 --- a/vendor/google.golang.org/protobuf/internal/order/ya.make +++ b/vendor/google.golang.org/protobuf/internal/order/ya.make @@ -2,6 +2,8 @@ GO_LIBRARY() LICENSE(BSD-3-Clause) +VERSION(v1.34.2) + SRCS( order.go range.go diff --git a/vendor/google.golang.org/protobuf/internal/pragma/ya.make b/vendor/google.golang.org/protobuf/internal/pragma/ya.make index 521f22ace6..499f8bb3ed 100644 --- a/vendor/google.golang.org/protobuf/internal/pragma/ya.make +++ b/vendor/google.golang.org/protobuf/internal/pragma/ya.make @@ -2,6 +2,8 @@ GO_LIBRARY() LICENSE(BSD-3-Clause) +VERSION(v1.34.2) + SRCS( pragma.go ) diff --git a/vendor/google.golang.org/protobuf/internal/set/ya.make b/vendor/google.golang.org/protobuf/internal/set/ya.make index ef768eff13..62b43b9e30 100644 --- a/vendor/google.golang.org/protobuf/internal/set/ya.make +++ b/vendor/google.golang.org/protobuf/internal/set/ya.make @@ -2,6 +2,8 @@ GO_LIBRARY() LICENSE(BSD-3-Clause) +VERSION(v1.34.2) + SRCS( ints.go ) diff --git a/vendor/google.golang.org/protobuf/internal/strs/ya.make b/vendor/google.golang.org/protobuf/internal/strs/ya.make index 3e580e1999..5156dc71bd 100644 --- a/vendor/google.golang.org/protobuf/internal/strs/ya.make +++ b/vendor/google.golang.org/protobuf/internal/strs/ya.make @@ -2,6 +2,8 @@ GO_LIBRARY() LICENSE(BSD-3-Clause) +VERSION(v1.34.2) + SRCS( strings.go strings_unsafe_go121.go diff --git a/vendor/google.golang.org/protobuf/internal/version/ya.make b/vendor/google.golang.org/protobuf/internal/version/ya.make index ca3958ecd9..357c70bad1 100644 --- a/vendor/google.golang.org/protobuf/internal/version/ya.make +++ b/vendor/google.golang.org/protobuf/internal/version/ya.make @@ -2,6 +2,8 @@ GO_LIBRARY() LICENSE(BSD-3-Clause) +VERSION(v1.34.2) + SRCS( version.go ) diff --git a/vendor/google.golang.org/protobuf/proto/ya.make b/vendor/google.golang.org/protobuf/proto/ya.make index 6f72cc0ff1..9df48fe669 100644 --- a/vendor/google.golang.org/protobuf/proto/ya.make +++ b/vendor/google.golang.org/protobuf/proto/ya.make @@ -2,6 +2,8 @@ GO_LIBRARY() LICENSE(BSD-3-Clause) +VERSION(v1.34.2) + SRCS( checkinit.go decode.go diff --git a/vendor/google.golang.org/protobuf/reflect/protodesc/ya.make b/vendor/google.golang.org/protobuf/reflect/protodesc/ya.make index cb217be223..1ddb6e4377 100644 --- a/vendor/google.golang.org/protobuf/reflect/protodesc/ya.make +++ b/vendor/google.golang.org/protobuf/reflect/protodesc/ya.make @@ -2,6 +2,8 @@ GO_LIBRARY() LICENSE(BSD-3-Clause) +VERSION(v1.34.2) + SRCS( desc.go desc_init.go diff --git a/vendor/google.golang.org/protobuf/reflect/protopath/ya.make b/vendor/google.golang.org/protobuf/reflect/protopath/ya.make index 5dcf56c0c8..859addc349 100644 --- a/vendor/google.golang.org/protobuf/reflect/protopath/ya.make +++ b/vendor/google.golang.org/protobuf/reflect/protopath/ya.make @@ -2,6 +2,8 @@ GO_LIBRARY() LICENSE(BSD-3-Clause) +VERSION(v1.34.2) + SRCS( path.go step.go diff --git a/vendor/google.golang.org/protobuf/reflect/protorange/ya.make b/vendor/google.golang.org/protobuf/reflect/protorange/ya.make index fc294f2b0e..16bb3bfc5a 100644 --- a/vendor/google.golang.org/protobuf/reflect/protorange/ya.make +++ b/vendor/google.golang.org/protobuf/reflect/protorange/ya.make @@ -2,6 +2,8 @@ GO_LIBRARY() LICENSE(BSD-3-Clause) +VERSION(v1.34.2) + SRCS( range.go ) diff --git a/vendor/google.golang.org/protobuf/reflect/protoreflect/ya.make b/vendor/google.golang.org/protobuf/reflect/protoreflect/ya.make index 7cef01fc14..f4a2f7f4f9 100644 --- a/vendor/google.golang.org/protobuf/reflect/protoreflect/ya.make +++ b/vendor/google.golang.org/protobuf/reflect/protoreflect/ya.make @@ -2,6 +2,8 @@ GO_LIBRARY() LICENSE(BSD-3-Clause) +VERSION(v1.34.2) + SRCS( methods.go proto.go diff --git a/vendor/google.golang.org/protobuf/reflect/protoregistry/ya.make b/vendor/google.golang.org/protobuf/reflect/protoregistry/ya.make index a6fda0d69d..091dd4ba69 100644 --- a/vendor/google.golang.org/protobuf/reflect/protoregistry/ya.make +++ b/vendor/google.golang.org/protobuf/reflect/protoregistry/ya.make @@ -2,6 +2,8 @@ GO_LIBRARY() LICENSE(BSD-3-Clause) +VERSION(v1.34.2) + SRCS( registry.go ) diff --git a/vendor/google.golang.org/protobuf/runtime/protoiface/ya.make b/vendor/google.golang.org/protobuf/runtime/protoiface/ya.make index 76eaf10dee..5b33e7681c 100644 --- a/vendor/google.golang.org/protobuf/runtime/protoiface/ya.make +++ b/vendor/google.golang.org/protobuf/runtime/protoiface/ya.make @@ -2,6 +2,8 @@ GO_LIBRARY() LICENSE(BSD-3-Clause) +VERSION(v1.34.2) + SRCS( legacy.go methods.go diff --git a/vendor/google.golang.org/protobuf/runtime/protoimpl/ya.make b/vendor/google.golang.org/protobuf/runtime/protoimpl/ya.make index 8da76e4e71..41016080d4 100644 --- a/vendor/google.golang.org/protobuf/runtime/protoimpl/ya.make +++ b/vendor/google.golang.org/protobuf/runtime/protoimpl/ya.make @@ -2,6 +2,8 @@ GO_LIBRARY() LICENSE(BSD-3-Clause) +VERSION(v1.34.2) + SRCS( impl.go version.go diff --git a/vendor/google.golang.org/protobuf/types/descriptorpb/ya.make b/vendor/google.golang.org/protobuf/types/descriptorpb/ya.make index a96c8d9d3c..3c3d3a0277 100644 --- a/vendor/google.golang.org/protobuf/types/descriptorpb/ya.make +++ b/vendor/google.golang.org/protobuf/types/descriptorpb/ya.make @@ -2,6 +2,8 @@ GO_LIBRARY() LICENSE(BSD-3-Clause) +VERSION(v1.34.2) + SRCS( descriptor.pb.go ) diff --git a/vendor/google.golang.org/protobuf/types/dynamicpb/ya.make b/vendor/google.golang.org/protobuf/types/dynamicpb/ya.make index ecab55dc87..a11c30ed0e 100644 --- a/vendor/google.golang.org/protobuf/types/dynamicpb/ya.make +++ b/vendor/google.golang.org/protobuf/types/dynamicpb/ya.make @@ -2,6 +2,8 @@ GO_LIBRARY() LICENSE(BSD-3-Clause) +VERSION(v1.34.2) + SRCS( dynamic.go types.go diff --git a/vendor/google.golang.org/protobuf/types/gofeaturespb/ya.make b/vendor/google.golang.org/protobuf/types/gofeaturespb/ya.make index 065e25268b..b2465a613e 100644 --- a/vendor/google.golang.org/protobuf/types/gofeaturespb/ya.make +++ b/vendor/google.golang.org/protobuf/types/gofeaturespb/ya.make @@ -2,6 +2,8 @@ GO_LIBRARY() LICENSE(BSD-3-Clause) +VERSION(v1.34.2) + SRCS( go_features.pb.go ) diff --git a/vendor/google.golang.org/protobuf/types/pluginpb/ya.make b/vendor/google.golang.org/protobuf/types/pluginpb/ya.make index 7b8a798f38..479992020d 100644 --- a/vendor/google.golang.org/protobuf/types/pluginpb/ya.make +++ b/vendor/google.golang.org/protobuf/types/pluginpb/ya.make @@ -2,6 +2,8 @@ GO_LIBRARY() LICENSE(BSD-3-Clause) +VERSION(v1.34.2) + SRCS( plugin.pb.go ) diff --git a/yql/essentials/public/issue/protos/issue_message.proto b/yql/essentials/public/issue/protos/issue_message.proto new file mode 100644 index 0000000000..d581a5d8f0 --- /dev/null +++ b/yql/essentials/public/issue/protos/issue_message.proto @@ -0,0 +1,17 @@ +package NYql.NIssue.NProto; +option java_package = "com.yandex.yql.issue.proto"; + +message IssueMessage { + message Position { + optional uint32 row = 1; + optional uint32 column = 2; + optional string file = 3; + } + + optional Position position = 1; + optional string message = 2; + optional Position end_position = 3; + optional uint32 issue_code = 4; + optional uint32 severity = 5; + repeated IssueMessage issues = 6; +} diff --git a/yql/essentials/public/issue/protos/issue_severity.proto b/yql/essentials/public/issue/protos/issue_severity.proto new file mode 100644 index 0000000000..70fd61f5d9 --- /dev/null +++ b/yql/essentials/public/issue/protos/issue_severity.proto @@ -0,0 +1,14 @@ +syntax = "proto3"; + +package NYql; + +option java_package = "com.yandex.yql.proto"; + +message TSeverityIds { + enum ESeverityId { + S_FATAL = 0; + S_ERROR = 1; + S_WARNING = 2; + S_INFO = 3; + } +} diff --git a/yql/essentials/public/issue/protos/ya.make b/yql/essentials/public/issue/protos/ya.make new file mode 100644 index 0000000000..2451631ea1 --- /dev/null +++ b/yql/essentials/public/issue/protos/ya.make @@ -0,0 +1,10 @@ +PROTO_LIBRARY() + +SRCS( + issue_message.proto + issue_severity.proto +) + +EXCLUDE_TAGS(GO_PROTO) + +END() diff --git a/yql/essentials/public/issue/ut/ya.make b/yql/essentials/public/issue/ut/ya.make new file mode 100644 index 0000000000..46f35941bc --- /dev/null +++ b/yql/essentials/public/issue/ut/ya.make @@ -0,0 +1,16 @@ +UNITTEST_FOR(yql/essentials/public/issue) + +FORK_SUBTESTS() + +SRCS( + yql_issue_ut.cpp + yql_issue_manager_ut.cpp + yql_issue_utils_ut.cpp + yql_warning_ut.cpp +) + +PEERDIR( + library/cpp/unicode/normalization +) + +END() diff --git a/yql/essentials/public/issue/ya.make b/yql/essentials/public/issue/ya.make new file mode 100644 index 0000000000..0179b23bbb --- /dev/null +++ b/yql/essentials/public/issue/ya.make @@ -0,0 +1,26 @@ +LIBRARY() + +SRCS( + yql_issue.cpp + yql_issue_message.cpp + yql_issue_manager.cpp + yql_issue_utils.cpp + yql_warning.cpp +) + +PEERDIR( + contrib/libs/protobuf + library/cpp/colorizer + library/cpp/resource + contrib/ydb/public/api/protos + yql/essentials/public/issue/protos + contrib/ydb/library/yql/utils +) + +GENERATE_ENUM_SERIALIZATION(yql_warning.h) + +END() + +RECURSE_FOR_TESTS( + ut +) diff --git a/yql/essentials/public/issue/yql_issue.cpp b/yql/essentials/public/issue/yql_issue.cpp new file mode 100644 index 0000000000..3fc8e3767c --- /dev/null +++ b/yql/essentials/public/issue/yql_issue.cpp @@ -0,0 +1,310 @@ +#include "yql_issue.h" +#include "yql_issue_id.h" + +#include <contrib/ydb/library/yql/utils/utf8.h> + +#include <library/cpp/colorizer/output.h> + +#include <util/charset/utf8.h> +#include <util/string/ascii.h> +#include <util/string/split.h> +#include <util/string/strip.h> +#include <util/string/subst.h> +#include <util/system/compiler.h> +#include <util/generic/map.h> +#include <util/generic/stack.h> +#include <cstdlib> + + +namespace NYql { + +void SanitizeNonAscii(TString& s) { + if (!NYql::IsUtf8(s)) { + TString escaped; + escaped.reserve(s.size()); + const unsigned char* i = reinterpret_cast<const unsigned char*>(s.data()); + const unsigned char* end = i + s.size(); + while (i < end) { + wchar32 rune; + size_t runeLen; + const RECODE_RESULT result = SafeReadUTF8Char(rune, runeLen, i, end); + if (result == RECODE_OK) { + escaped.insert(escaped.end(), reinterpret_cast<const char*>(i), reinterpret_cast<const char*>(i + runeLen)); + i += runeLen; + } else { + escaped.push_back('?'); + ++i; + } + } + s = escaped; + } +} + +TTextWalker& TTextWalker::Advance(char c) { + if (c == '\n') { + HaveCr = false; + ++LfCount; + return *this; + } + + + if (c == '\r' && !HaveCr) { + HaveCr = true; + return *this; + } + + ui32 charDistance = 1; + if (Utf8Aware && IsUtf8Intermediate(c)) { + charDistance = 0; + } + + // either not '\r' or second '\r' + if (LfCount) { + Position.Row += LfCount; + Position.Column = charDistance; + LfCount = 0; + } else { + Position.Column += charDistance + (HaveCr && c != '\r'); + } + HaveCr = (c == '\r'); + return *this; +} + +void TIssue::PrintTo(IOutputStream& out, bool oneLine) const { + out << Range() << ": " << SeverityToString(GetSeverity()) << ": "; + if (oneLine) { + TString message = StripString(Message); + SubstGlobal(message, '\n', ' '); + out << message; + } else { + out << Message; + } + if (GetCode()) { + out << ", code: " << GetCode(); + } +} + +void WalkThroughIssues(const TIssue& topIssue, bool leafOnly, std::function<void(const TIssue&, ui16 level)> fn, std::function<void(const TIssue&, ui16 level)> afterChildrenFn) { + enum class EFnType { + Main, + AfterChildren, + }; + + const bool hasAfterChildrenFn = bool(afterChildrenFn); + TStack<std::tuple<ui16, const TIssue*, EFnType>> issuesStack; + if (hasAfterChildrenFn) { + issuesStack.push(std::make_tuple(0, &topIssue, EFnType::AfterChildren)); + } + issuesStack.push(std::make_tuple(0, &topIssue, EFnType::Main)); + while (!issuesStack.empty()) { + auto level = std::get<0>(issuesStack.top()); + const auto& curIssue = *std::get<1>(issuesStack.top()); + const EFnType fnType = std::get<2>(issuesStack.top()); + issuesStack.pop(); + if (!leafOnly || curIssue.GetSubIssues().empty()) { + if (fnType == EFnType::Main) { + fn(curIssue, level); + } else { + afterChildrenFn(curIssue, level); + } + } + if (fnType == EFnType::Main) { + level++; + const auto& subIssues = curIssue.GetSubIssues(); + for (int i = subIssues.size() - 1; i >= 0; i--) { + if (hasAfterChildrenFn) { + issuesStack.push(std::make_tuple(level, subIssues[i].Get(), EFnType::AfterChildren)); + } + issuesStack.push(std::make_tuple(level, subIssues[i].Get(), EFnType::Main)); + } + } + } +} + +namespace { + +Y_NO_INLINE void Indent(IOutputStream& out, ui32 indentation) { + char* whitespaces = reinterpret_cast<char*>(alloca(indentation)); + memset(whitespaces, ' ', indentation); + out.Write(whitespaces, indentation); +} + +void ProgramLinesWithErrors( + const TString& programText, + const TVector<TIssue>& errors, + TMap<ui32, TStringBuf>& lines) +{ + TVector<ui32> rows; + for (const auto& topIssue: errors) { + WalkThroughIssues(topIssue, false, [&](const TIssue& issue, ui16 /*level*/) { + for (ui32 row = issue.Position.Row; row <= issue.EndPosition.Row; row++) { + rows.push_back(row); + } + }); + } + std::sort(rows.begin(), rows.end()); + + auto prog = StringSplitter(programText).Split('\n'); + auto progIt = prog.begin(), progEnd = prog.end(); + ui32 progRow = 1; + + for (ui32 row: rows) { + while (progRow < row && progIt != progEnd) { + ++progRow; + ++progIt; + } + if (progIt != progEnd) { + lines[row] = progIt->Token(); + } + } +} + +} // namspace + +void TIssues::PrintTo(IOutputStream& out, bool oneLine) const +{ + if (oneLine) { + bool printWithSpace = false; + if (Issues_.size() > 1) { + printWithSpace = true; + out << "["; + } + for (const auto& topIssue: Issues_) { + WalkThroughIssues(topIssue, false, [&](const TIssue& issue, ui16 level) { + if (level > 0) { + out << " subissue: { "; + } else { + out << (printWithSpace ? " { " : "{ "); + } + issue.PrintTo(out, true); + }, + [&](const TIssue&, ui16) { + out << " }"; + }); + } + if (Issues_.size() > 1) { + out << " ]"; + } + } else { + for (const auto& topIssue: Issues_) { + WalkThroughIssues(topIssue, false, [&](const TIssue& issue, ui16 level) { + auto shift = level * 4; + Indent(out, shift); + out << issue << Endl; + }); + } + } +} + +void TIssues::PrintWithProgramTo( + IOutputStream& out, + const TString& programFilename, + const TString& programText) const +{ + using namespace NColorizer; + + TMap<ui32, TStringBuf> lines; + ProgramLinesWithErrors(programText, Issues_, lines); + + for (const TIssue& topIssue: Issues_) { + WalkThroughIssues(topIssue, false, [&](const TIssue& issue, ui16 level) { + auto shift = level * 4; + Indent(out, shift); + out << DarkGray() << programFilename << Old() << ':'; + out << Purple() << issue.Range() << Old(); + auto color = (issue.GetSeverity() == TSeverityIds::S_WARNING) ? Yellow() : LightRed(); + auto severityName = SeverityToString(issue.GetSeverity()); + out << color << ": "<< severityName << ": " << issue.GetMessage() << Old() << '\n'; + Indent(out, shift); + if (issue.Position.HasValue()) { + out << '\t' << lines[issue.Position.Row] << '\n'; + out << '\t'; + if (issue.Position.Column > 0) { + Indent(out, issue.Position.Column - 1); + } + out << '^'; + } + out << Endl; + }); + } +} + +TIssue ExceptionToIssue(const std::exception& e, const TPosition& pos) { + TStringBuf messageBuf = e.what(); + auto parsedPos = TryParseTerminationMessage(messageBuf); + auto issue = TIssue(parsedPos.GetOrElse(pos), messageBuf); + const TErrorException* errorException = dynamic_cast<const TErrorException*>(&e); + if (errorException) { + issue.SetCode(errorException->GetCode(), ESeverity::TSeverityIds_ESeverityId_S_ERROR); + } else { + issue.SetCode(UNEXPECTED_ERROR, ESeverity::TSeverityIds_ESeverityId_S_FATAL); + } + return issue; +} + +static constexpr TStringBuf TerminationMessageMarker = "Terminate was called, reason("; + +TMaybe<TPosition> TryParseTerminationMessage(TStringBuf& message) { + size_t len = 0; + size_t startPos = message.find(TerminationMessageMarker); + size_t endPos = 0; + if (startPos != TString::npos) { + endPos = message.find(')', startPos + TerminationMessageMarker.size()); + if (endPos != TString::npos) { + TStringBuf lenText = message.Tail(startPos + TerminationMessageMarker.size()) + .Trunc(endPos - startPos - TerminationMessageMarker.size()); + try { + len = FromString<size_t>(lenText); + } catch (const TFromStringException&) { + len = 0; + } + } + } + + if (len) { + message = message.Tail(endPos + 3).Trunc(len); + auto s = message; + TMaybe<TStringBuf> file; + TMaybe<TStringBuf> row; + TMaybe<TStringBuf> column; + GetNext(s, ':', file); + GetNext(s, ':', row); + GetNext(s, ':', column); + ui32 rowValue, columnValue; + if (file && row && column && TryFromString(*row, rowValue) && TryFromString(*column, columnValue)) { + message = StripStringLeft(s); + return TPosition(columnValue, rowValue, TString(*file)); + } + } + + return Nothing(); +} + +} // namspace NYql + +template <> +void Out<NYql::TPosition>(IOutputStream& out, const NYql::TPosition& pos) { + out << (pos.File ? pos.File : "<main>"); + if (pos) { + out << ":" << pos.Row << ':' << pos.Column; + } +} + +template<> +void Out<NYql::TRange>(IOutputStream & out, const NYql::TRange & range) { + if (range.IsRange()) { + out << '[' << range.Position << '-' << range.EndPosition << ']'; + } else { + out << range.Position; + } +} + +template <> +void Out<NYql::TIssue>(IOutputStream& out, const NYql::TIssue& error) { + error.PrintTo(out); +} + +template <> +void Out<NYql::TIssues>(IOutputStream& out, const NYql::TIssues& error) { + error.PrintTo(out); +}
\ No newline at end of file diff --git a/yql/essentials/public/issue/yql_issue.h b/yql/essentials/public/issue/yql_issue.h new file mode 100644 index 0000000000..00c49fb1fc --- /dev/null +++ b/yql/essentials/public/issue/yql_issue.h @@ -0,0 +1,372 @@ +#pragma once + +#include <util/system/types.h> +#include <util/generic/hash.h> +#include <util/generic/maybe.h> +#include <util/generic/vector.h> +#include <util/generic/string.h> +#include <util/generic/strbuf.h> +#include <util/generic/ptr.h> +#include <util/stream/output.h> +#include <util/stream/str.h> +#include <util/digest/numeric.h> +#include <google/protobuf/message.h> + +#include "yql_issue_id.h" + +namespace NYql { + +void SanitizeNonAscii(TString& s); + +/////////////////////////////////////////////////////////////////////////////// +// TPosition +/////////////////////////////////////////////////////////////////////////////// +struct TPosition { + ui32 Column = 0U; + ui32 Row = 0U; + TString File; + + TPosition() = default; + + TPosition(ui32 column, ui32 row, const TString& file = {}) + : Column(column) + , Row(row) + , File(file) + { + SanitizeNonAscii(File); + } + + explicit operator bool() const { + return HasValue(); + } + + inline bool HasValue() const { + return Row | Column; + } + + inline bool operator==(const TPosition& other) const { + return Column == other.Column && Row == other.Row && File == other.File; + } + + inline bool operator<(const TPosition& other) const { + return std::tie(Row, Column, File) < std::tie(other.Row, other.Column, other.File); + } +}; + +class TTextWalker { +public: + TTextWalker(TPosition& position, bool utf8Aware) + : Position(position) + , Utf8Aware(utf8Aware) + , HaveCr(false) + , LfCount(0) + { + } + + static inline bool IsUtf8Intermediate(char c) { + return (c & 0xC0) == 0x80; + } + + template<typename T> + TTextWalker& Advance(const T& buf) { + for (char c : buf) { + Advance(c); + } + return *this; + } + + TTextWalker& Advance(char c); + +private: + TPosition& Position; + const bool Utf8Aware; + bool HaveCr; + ui32 LfCount; +}; + +struct TRange { + TPosition Position; + TPosition EndPosition; + + TRange() = default; + + TRange(TPosition position) + : Position(position) + , EndPosition(position) + { + } + + TRange(TPosition position, TPosition endPosition) + : Position(position) + , EndPosition(endPosition) + { + } + + inline bool IsRange() const { + return !(Position == EndPosition); + } +}; + +/////////////////////////////////////////////////////////////////////////////// +// TIssue +/////////////////////////////////////////////////////////////////////////////// + +class TIssue; +using TIssuePtr = TIntrusivePtr<TIssue>; +class TIssue: public TThrRefBase { + TVector<TIntrusivePtr<TIssue>> Children_; + TString Message; +public: + TPosition Position; + TPosition EndPosition; + TIssueCode IssueCode = 0U; + ESeverity Severity = TSeverityIds::S_ERROR; + + TIssue() = default; + + template <typename T> + explicit TIssue(const T& message) + : Message(message) + , Position(TPosition()) + , EndPosition(TPosition()) + { + SanitizeNonAscii(Message); + } + + template <typename T> + TIssue(TPosition position, const T& message) + : Message(message) + , Position(position) + , EndPosition(position) + { + SanitizeNonAscii(Message); + } + + inline TRange Range() const { + return{ Position, EndPosition }; + } + + template <typename T> + TIssue(TPosition position, TPosition endPosition, const T& message) + : Message(message) + , Position(position) + , EndPosition(endPosition) + { + SanitizeNonAscii(Message); + } + + inline bool operator==(const TIssue& other) const { + return Position == other.Position && Message == other.Message + && IssueCode == other.IssueCode; + } + + ui64 Hash() const noexcept { + return CombineHashes( + CombineHashes( + (size_t)CombineHashes(IntHash(Position.Row), IntHash(Position.Column)), + ComputeHash(Position.File) + ), + (size_t)CombineHashes((size_t)IntHash(static_cast<int>(IssueCode)), ComputeHash(Message))); + } + + TIssue& SetCode(TIssueCode id, ESeverity severity) { + IssueCode = id; + Severity = severity; + return *this; + } + + TIssue& SetMessage(const TString& msg) { + Message = msg; + SanitizeNonAscii(Message); + return *this; + } + + ESeverity GetSeverity() const { + return Severity; + } + + TIssueCode GetCode() const { + return IssueCode; + } + + const TString& GetMessage() const { + return Message; + } + + TIssue& AddSubIssue(TIntrusivePtr<TIssue> issue) { + Severity = (ESeverity)Min((ui32)issue->GetSeverity(), (ui32)Severity); + Children_.push_back(issue); + return *this; + } + + const TVector<TIntrusivePtr<TIssue>>& GetSubIssues() const { + return Children_; + } + + void PrintTo(IOutputStream& out, bool oneLine = false) const; + + TString ToString(bool oneLine = false) const { + TStringStream out; + PrintTo(out, oneLine); + return out.Str(); + } + + // Unsafe method. Doesn't call SanitizeNonAscii(Message) + TString* MutableMessage() { + return &Message; + } + + TIssue& CopyWithoutSubIssues(const TIssue& src) { + Message = src.Message; + IssueCode = src.IssueCode; + Severity = src.Severity; + Position = src.Position; + EndPosition = src.EndPosition; + return *this; + } +}; + +void WalkThroughIssues(const TIssue& topIssue, bool leafOnly, std::function<void(const TIssue&, ui16 level)> fn, std::function<void(const TIssue&, ui16 level)> afterChildrenFn = {}); + +/////////////////////////////////////////////////////////////////////////////// +// TIssues +/////////////////////////////////////////////////////////////////////////////// +class TIssues { +public: + TIssues() = default; + + inline TIssues(const TVector<TIssue>& issues) + : Issues_(issues) + { + } + + inline TIssues(const std::initializer_list<TIssue>& issues) + : TIssues(TVector<TIssue>(issues)) + { + } + + inline TIssues(const TIssues& rhs) + : Issues_(rhs.Issues_) + { + } + + inline TIssues& operator=(const TIssues& rhs) { + Issues_ = rhs.Issues_; + return *this; + } + + inline TIssues(TIssues&& rhs) : Issues_(std::move(rhs.Issues_)) + { + } + + inline TIssues& operator=(TIssues&& rhs) { + Issues_ = std::move(rhs.Issues_); + return *this; + } + + template <typename ... Args> void AddIssue(Args&& ... args) { + Issues_.emplace_back(std::forward<Args>(args)...); + } + + inline void AddIssues(const TIssues& errors) { + Issues_.insert(Issues_.end(), + errors.Issues_.begin(), errors.Issues_.end()); + } + + inline void AddIssues(const TPosition& pos, const TIssues& errors) { + Issues_.reserve(Issues_.size() + errors.Size()); + for (const auto& e: errors) { + TIssue& issue = Issues_.emplace_back(); + *issue.MutableMessage() = e.GetMessage(); // No need to sanitize message, it has already been sanitized. + issue.Position = pos; + issue.SetCode(e.IssueCode, e.Severity); + } + } + + inline const TIssue* begin() const { + return Issues_.begin(); + } + + inline const TIssue* end() const { + return Issues_.end(); + } + + inline TIssue& back() { + return Issues_.back(); + } + + inline const TIssue& back() const { + return Issues_.back(); + } + + inline bool Empty() const { + return Issues_.empty(); + } + + explicit operator bool() const noexcept { + return !Issues_.empty(); + } + + inline size_t Size() const { + return Issues_.size(); + } + + void PrintTo(IOutputStream& out, bool oneLine = false) const; + void PrintWithProgramTo( + IOutputStream& out, + const TString& programFilename, + const TString& programText) const; + + inline TString ToString(bool oneLine = false) const { + TStringStream out; + PrintTo(out, oneLine); + return out.Str(); + } + + TString ToOneLineString() const { + return ToString(true); + } + + inline void Clear() { + Issues_.clear(); + } + + void Reserve(size_t capacity) { + Issues_.reserve(capacity); + } + +private: + TVector<TIssue> Issues_; +}; + +class TErrorException : public yexception { + const TIssueCode Code_; +public: + explicit TErrorException(TIssueCode code) + : Code_(code) + {} + TIssueCode GetCode() const { + return Code_; + } +}; + +TIssue ExceptionToIssue(const std::exception& e, const TPosition& pos = TPosition()); +TMaybe<TPosition> TryParseTerminationMessage(TStringBuf& message); + +} // namespace NYql + +template <> +void Out<NYql::TPosition>(IOutputStream& out, const NYql::TPosition& pos); + +template <> +void Out<NYql::TRange>(IOutputStream& out, const NYql::TRange& pos); + +template <> +void Out<NYql::TIssue>(IOutputStream& out, const NYql::TIssue& error); + +template <> +struct THash<NYql::TIssue> { + inline size_t operator()(const NYql::TIssue& err) const { + return err.Hash(); + } +}; diff --git a/yql/essentials/public/issue/yql_issue_id.h b/yql/essentials/public/issue/yql_issue_id.h new file mode 100644 index 0000000000..b4889319a7 --- /dev/null +++ b/yql/essentials/public/issue/yql_issue_id.h @@ -0,0 +1,128 @@ +#pragma once + +#include <yql/essentials/public/issue/protos/issue_severity.pb.h> + +#include <library/cpp/resource/resource.h> +#include <google/protobuf/descriptor.h> +#include <google/protobuf/repeated_field.h> +#include <google/protobuf/text_format.h> +#include <google/protobuf/message.h> + +#include <util/generic/hash.h> +#include <util/generic/singleton.h> +#include <util/generic/yexception.h> +#include <util/string/subst.h> + +#ifdef _win_ +#ifdef GetMessage +#undef GetMessage +#endif +#endif + +namespace NYql { + +using TIssueCode = ui32; +using ESeverity = NYql::TSeverityIds::ESeverityId; +const TIssueCode DEFAULT_ERROR = 0; +const TIssueCode UNEXPECTED_ERROR = 1; + +inline TString SeverityToString(ESeverity severity) +{ + auto ret = NYql::TSeverityIds::ESeverityId_Name(severity); + return ret.empty() ? "Unknown" : to_title(ret.substr(2)); //remove prefix "S_" +} + +template <typename T> +inline TString IssueCodeToString(TIssueCode id) { + auto ret = T::EIssueCode_Name(static_cast<typename T::EIssueCode>(id)); + if (!ret.empty()) { + SubstGlobal(ret, '_', ' '); + return to_title(ret); + } else { + return "Unknown"; + } +} + +template<typename TProto, const char* ResourceName> +class TIssueId { + TProto ProtoIssues_; + THashMap<TIssueCode, NYql::TSeverityIds::ESeverityId> IssuesMap_; + THashMap<TIssueCode, TString> IssuesFormatMap_; + + const google::protobuf::Descriptor* GetProtoDescriptor() const { + auto ret = ProtoIssues_.GetDescriptor(); + Y_ENSURE(ret != nullptr, "Bad proto file"); + return ret; + } + + bool CheckSeverityNameFormat(const TString& name) const { + if (name.size() > 2 && name.substr(0,2) == "S_") { + return true; + } + return false; + } + +public: + ESeverity GetSeverity(TIssueCode id) const { + auto it = IssuesMap_.find(id); + Y_ENSURE(it != IssuesMap_.end(), "Unknown issue id: " + << id << "(" << IssueCodeToString<TProto>(id) << ")"); + return it->second; + } + + TString GetMessage(TIssueCode id) const { + auto it = IssuesFormatMap_.find(id); + Y_ENSURE(it != IssuesFormatMap_.end(), "Unknown issue id: " + << id << "(" << IssueCodeToString<TProto>(id) << ")"); + return it->second; + } + + TIssueId() { + auto configData = NResource::Find(TStringBuf(ResourceName)); + if (!::google::protobuf::TextFormat::ParseFromString(configData, &ProtoIssues_)) { + Y_ENSURE(false, "Bad format of protobuf data file, resource: " << ResourceName); + } + + auto sDesc = TSeverityIds::ESeverityId_descriptor(); + for (int i = 0; i < sDesc->value_count(); i++) { + const auto& name = sDesc->value(i)->name(); + Y_ENSURE(CheckSeverityNameFormat(name), + "Wrong severity name: " << name << ". Severity must starts with \"S_\" prefix"); + } + + for (const auto& x : ProtoIssues_.ids()) { + auto rv = IssuesMap_.insert(std::make_pair(x.code(), x.severity())); + Y_ENSURE(rv.second, "Duplicate issue code found, code: " + << static_cast<int>(x.code()) + << "(" << IssueCodeToString<TProto>(x.code()) <<")"); + } + + // Check all IssueCodes have mapping to severity + auto eDesc = TProto::EIssueCode_descriptor(); + for (int i = 0; i < eDesc->value_count(); i++) { + auto it = IssuesMap_.find(eDesc->value(i)->number()); + Y_ENSURE(it != IssuesMap_.end(), "IssueCode: " + << eDesc->value(i)->name() + << " is not found in protobuf data file"); + } + + for (const auto& x : ProtoIssues_.ids()) { + auto rv = IssuesFormatMap_.insert(std::make_pair(x.code(), x.format())); + Y_ENSURE(rv.second, "Duplicate issue code found, code: " + << static_cast<int>(x.code()) + << "(" << IssueCodeToString<TProto>(x.code()) <<")"); + } + } +}; + +template<typename TProto, const char* ResourceName> +inline ESeverity GetSeverity(TIssueCode id) { + return Singleton<TIssueId<TProto, ResourceName>>()->GetSeverity(id); +} + +template<typename TProto, const char* ResourceName> +inline TString GetMessage(TIssueCode id) { + return Singleton<TIssueId<TProto, ResourceName>>()->GetMessage(id); +} + +} diff --git a/yql/essentials/public/issue/yql_issue_manager.cpp b/yql/essentials/public/issue/yql_issue_manager.cpp new file mode 100644 index 0000000000..8ffeed9fb0 --- /dev/null +++ b/yql/essentials/public/issue/yql_issue_manager.cpp @@ -0,0 +1,234 @@ +#include "yql_issue_manager.h" + +#include <util/string/cast.h> +#include <util/string/builder.h> + +using namespace NYql; + +void TIssueManager::AddScope(std::function<TIssuePtr()> fn) { + RawIssues_.emplace(std::make_pair(TMaybe<TIssuePtr>(), fn)); +} + +void TIssueManager::LeaveScope() { + Y_ASSERT(HasOpenScopes()); + if (RawIssues_.size() == 1) { + // Last scope, just dump it + auto maybeIssue = RawIssues_.top().first; + if (maybeIssue) { + if ((*maybeIssue)->GetCode() == Max<ui32>()) { + for (const auto& nestedIssue : (*maybeIssue.Get())->GetSubIssues()) { + CompletedIssues_.AddIssue(*nestedIssue); + } + } else { + CompletedIssues_.AddIssue(**maybeIssue); + } + } + RawIssues_.pop(); + return; + } + + if (RawIssues_.top().first.Empty()) { + // No issues in this scope + RawIssues_.pop(); + return; + } + + auto subIssue = *RawIssues_.top().first; + if (subIssue->GetSubIssues().size() == 1) { + auto nestedIssue = subIssue->GetSubIssues().front(); + if (!nestedIssue->GetSubIssues().empty() && nestedIssue->Position == subIssue->Position && nestedIssue->EndPosition == subIssue->EndPosition) { + auto msg = subIssue->GetMessage(); + if (nestedIssue->GetMessage()) { + if (msg) { + msg.append(", "); + } + msg.append(nestedIssue->GetMessage()); + } + subIssue = nestedIssue; + subIssue->SetMessage(msg); + } + } + RawIssues_.pop(); + if (RawIssues_.top().first.Empty()) { + RawIssues_.top().first = RawIssues_.top().second(); + if (!*RawIssues_.top().first) { + RawIssues_.top().first = new TIssue(); + (*RawIssues_.top().first)->SetCode(Max<ui32>(), ESeverity::TSeverityIds_ESeverityId_S_INFO); + } else { + (*RawIssues_.top().first)->Severity = ESeverity::TSeverityIds_ESeverityId_S_INFO; + } + } + + if (subIssue->GetCode() == Max<ui32>()) { + for (const auto& nestedIssue : subIssue->GetSubIssues()) { + RawIssues_.top().first->Get()->AddSubIssue(nestedIssue); + } + } else { + RawIssues_.top().first->Get()->AddSubIssue(subIssue); + } +} + +void TIssueManager::RaiseIssueForEmptyScope() { + if (RawIssues_.top().first.Empty()) { + TIssuePtr materialized = RawIssues_.top().second(); + if (auto p = CheckUniqAndLimit(materialized)) { + RawIssues_.top().first = p; + } + } +} + +void TIssueManager::LeaveAllScopes() { + while (!RawIssues_.empty()) { + LeaveScope(); + } +} + +TIssuePtr TIssueManager::CheckUniqAndLimit(TIssuePtr issue) { + const auto severity = issue->GetSeverity(); + if (OverflowIssues_[severity]) { + return {}; + } + if (UniqueIssues_[severity].contains(issue)) { + return {}; + } + if (IssueLimit_ && UniqueIssues_[severity].size() == IssueLimit_) { + OverflowIssues_[severity] = MakeIntrusive<TIssue>(TStringBuilder() + << "Too many " << SeverityToString(issue->GetSeverity()) << " issues"); + OverflowIssues_[severity]->Severity = severity; + return {}; + } + UniqueIssues_[severity].insert(issue); + return issue; +} + +TIssuePtr TIssueManager::CheckUniqAndLimit(const TIssue& issue) { + const auto severity = issue.GetSeverity(); + if (OverflowIssues_[severity]) { + return {}; + } + return CheckUniqAndLimit(MakeIntrusive<TIssue>(issue)); +} + +void TIssueManager::RaiseIssue(const TIssue& issue) { + TIssuePtr p = CheckUniqAndLimit(issue); + if (!p) { + return; + } + if (RawIssues_.empty()) { + CompletedIssues_.AddIssue(issue); + return; + } + if (RawIssues_.top().first.Empty()) { + RawIssues_.top().first = RawIssues_.top().second(); + if (!*RawIssues_.top().first) { + RawIssues_.top().first = new TIssue(); + (*RawIssues_.top().first)->SetCode(Max<ui32>(), ESeverity::TSeverityIds_ESeverityId_S_INFO); + } else { + (*RawIssues_.top().first)->Severity = ESeverity::TSeverityIds_ESeverityId_S_INFO; + } + } + RawIssues_.top().first->Get()->AddSubIssue(p); +} + +void TIssueManager::RaiseIssues(const TIssues& issues) { + for (const auto& x : issues) { + RaiseIssue(x); + } +} + +bool TIssueManager::RaiseWarning(TIssue issue) { + bool isWarning = true; + if (issue.GetSeverity() == ESeverity::TSeverityIds_ESeverityId_S_WARNING) { + const auto action = WarningPolicy_.GetAction(issue.GetCode()); + switch (action) { + case EWarningAction::DISABLE: + return isWarning; + case EWarningAction::ERROR: + issue.Severity = ESeverity::TSeverityIds_ESeverityId_S_ERROR; + if (WarningToErrorTreatMessage_) { + TIssue newIssue; + newIssue.SetCode(issue.GetCode(), ESeverity::TSeverityIds_ESeverityId_S_ERROR); + newIssue.SetMessage(WarningToErrorTreatMessage_.GetRef()); + newIssue.AddSubIssue(new TIssue(issue)); + issue = newIssue; + } + isWarning = false; + break; + case EWarningAction::DEFAULT: + break; + default: + Y_ENSURE(false, "Unknown action"); + } + } + + RaiseIssue(issue); + return isWarning; +} + +bool TIssueManager::HasOpenScopes() const { + return !RawIssues_.empty(); +} + +TIssues TIssueManager::GetIssues() { + auto tmp = RawIssues_; + LeaveAllScopes(); + auto result = GetCompletedIssues(); + RawIssues_ = tmp; + return result; +} + +TIssues TIssueManager::GetCompletedIssues() const { + TIssues res; + for (auto& p: OverflowIssues_) { + if (p) { + res.AddIssue(*p); + } + } + res.AddIssues(CompletedIssues_); + return res; +} + +void TIssueManager::AddIssues(const TIssues& issues) { + for (auto& issue: issues) { + if (auto p = CheckUniqAndLimit(issue)) { + CompletedIssues_.AddIssue(*p); + } + } +} + +void TIssueManager::AddIssues(const TPosition& pos, const TIssues& issues) { + for (auto& issue: issues) { + if (auto p = CheckUniqAndLimit(TIssue(pos, issue.GetMessage()))) { + CompletedIssues_.AddIssue(*p); + } + } +} + +void TIssueManager::Reset(const TIssues& issues) { + for (auto& p: OverflowIssues_) { + p.Drop(); + } + + for (auto& s: UniqueIssues_) { + s.clear(); + } + CompletedIssues_.Clear(); + + while (!RawIssues_.empty()) { + RawIssues_.pop(); + } + AddIssues(issues); +} + +void TIssueManager::Reset() { + Reset(TIssues()); +} + +void TIssueManager::AddWarningRule(const TWarningRule &rule) +{ + WarningPolicy_.AddRule(rule); +} + +void TIssueManager::SetWarningToErrorTreatMessage(const TString& msg) { + WarningToErrorTreatMessage_ = msg; +} diff --git a/yql/essentials/public/issue/yql_issue_manager.h b/yql/essentials/public/issue/yql_issue_manager.h new file mode 100644 index 0000000000..6b6ec2c719 --- /dev/null +++ b/yql/essentials/public/issue/yql_issue_manager.h @@ -0,0 +1,83 @@ +#pragma once + +#include "yql_issue.h" +#include "yql_warning.h" + +#include <util/generic/maybe.h> +#include <util/generic/stack.h> +#include <util/generic/hash_set.h> + +#include <array> + +namespace NYql { + +class TIssueManager: private TNonCopyable { +public: + void AddScope(std::function<TIssuePtr()> fn); + void LeaveScope(); + void RaiseIssue(const TIssue& issue); + void RaiseIssues(const TIssues& issues); + bool RaiseWarning(TIssue issue); + void AddIssues(const TIssues& errors); + void AddIssues(const TPosition& pos, const TIssues& issues); + + TIssues GetIssues(); + TIssues GetCompletedIssues() const; + + void Reset(const TIssues& issues); + void Reset(); + + void AddWarningRule(const TWarningRule &rule); + void SetWarningToErrorTreatMessage(const TString& msg); + + void SetIssueCountLimit(size_t limit) { + IssueLimit_ = limit; + } + + void RaiseIssueForEmptyScope(); + +private: + TIssuePtr CheckUniqAndLimit(const TIssue& issue); + TIssuePtr CheckUniqAndLimit(TIssuePtr issue); + void LeaveAllScopes(); + bool HasOpenScopes() const; + + struct TIssueHash { + ui64 operator()(const TIssuePtr& p) { + return p->Hash(); + } + }; + struct TIssueEqual { + bool operator()(const TIssuePtr& l, const TIssuePtr& r) { + return *l == *r; + } + }; + TStack<std::pair<TMaybe<TIssuePtr>, std::function<TIssuePtr()>>> RawIssues_; + TIssues CompletedIssues_; + TMaybe<TString> WarningToErrorTreatMessage_; + TWarningPolicy WarningPolicy_; + std::array<TIssuePtr, NYql::TSeverityIds::ESeverityId_ARRAYSIZE> OverflowIssues_; + std::array<THashSet<TIssuePtr, TIssueHash, TIssueEqual>, NYql::TSeverityIds::ESeverityId_ARRAYSIZE> UniqueIssues_; + size_t IssueLimit_ = 0; +}; + +class TIssueScopeGuard: private TNonCopyable { + TIssueManager& Manager_; +public: + TIssueScopeGuard(TIssueManager& manager, std::function<TIssuePtr()> fn) + : Manager_(manager) + { + Manager_.AddScope(fn); + } + + void RaiseIssueForEmptyScope() { + Manager_.RaiseIssueForEmptyScope(); + } + + ~TIssueScopeGuard() + { + Manager_.LeaveScope(); + } +}; + +} diff --git a/yql/essentials/public/issue/yql_issue_manager_ut.cpp b/yql/essentials/public/issue/yql_issue_manager_ut.cpp new file mode 100644 index 0000000000..e3308163c3 --- /dev/null +++ b/yql/essentials/public/issue/yql_issue_manager_ut.cpp @@ -0,0 +1,206 @@ +#include <library/cpp/testing/unittest/registar.h> + +#include "yql_issue_manager.h" + +using namespace NYql; + +static std::function<TIssuePtr()> CreateScopeIssueFunction(TString name, ui32 column, ui32 row) { + return [name, column, row]() { + return new TIssue(TPosition(column, row), name); + }; +} + +Y_UNIT_TEST_SUITE(TIssueManagerTest) { + Y_UNIT_TEST(NoErrorNoLevelTest) { + TIssueManager issueManager; + auto completedIssues = issueManager.GetCompletedIssues(); + UNIT_ASSERT_VALUES_EQUAL(completedIssues.Size(), 0); + auto issues = issueManager.GetIssues(); + UNIT_ASSERT_VALUES_EQUAL(issues.Size(), 0); + } + + Y_UNIT_TEST(NoErrorOneLevelTest) { + TIssueManager issueManager; + issueManager.AddScope(CreateScopeIssueFunction("A", 0, 0)); + auto completedIssues = issueManager.GetCompletedIssues(); + UNIT_ASSERT_VALUES_EQUAL(completedIssues.Size(), 0); + issueManager.LeaveScope(); + auto issues = issueManager.GetIssues(); + UNIT_ASSERT_VALUES_EQUAL(issues.Size(), 0); + } + + Y_UNIT_TEST(NoErrorTwoLevelsTest) { + TIssueManager issueManager; + issueManager.AddScope(CreateScopeIssueFunction("A", 0, 0)); + issueManager.AddScope(CreateScopeIssueFunction("B", 1, 1)); + issueManager.LeaveScope(); + auto completedIssues = issueManager.GetCompletedIssues(); + UNIT_ASSERT_VALUES_EQUAL(completedIssues.Size(), 0); + issueManager.LeaveScope(); + auto issues = issueManager.GetIssues(); + UNIT_ASSERT_VALUES_EQUAL(issues.Size(), 0); + } + + Y_UNIT_TEST(OneErrorOneLevelTest) { + TIssueManager issueManager; + issueManager.AddScope(CreateScopeIssueFunction("A", 0, 0)); + auto completedIssues1 = issueManager.GetCompletedIssues(); + UNIT_ASSERT_VALUES_EQUAL(completedIssues1.Size(), 0); + issueManager.RaiseIssue(TIssue(TPosition(1,2), "IssueOne")); + auto completedIssues2 = issueManager.GetCompletedIssues(); + UNIT_ASSERT_VALUES_EQUAL(completedIssues2.Size(), 0); + issueManager.LeaveScope(); + auto completedIssues3 = issueManager.GetCompletedIssues(); + UNIT_ASSERT_VALUES_EQUAL(completedIssues3.Size(), 1); + + auto issues = issueManager.GetIssues(); + UNIT_ASSERT_VALUES_EQUAL(issues.Size(), 1); + auto scopeIssue = issues.begin(); + UNIT_ASSERT_VALUES_EQUAL(scopeIssue->GetMessage(), "A"); + auto subIssues = scopeIssue->GetSubIssues(); + UNIT_ASSERT_VALUES_EQUAL(subIssues.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(subIssues[0]->GetMessage(), "IssueOne"); + } + + Y_UNIT_TEST(OneErrorTwoLevelsTest) { + TIssueManager issueManager; + issueManager.AddScope(CreateScopeIssueFunction("A", 0, 0)); + issueManager.AddScope(CreateScopeIssueFunction("B", 1, 1)); + issueManager.RaiseIssue(TIssue(TPosition(1,2), "IssueOne")); + issueManager.LeaveScope(); + issueManager.LeaveScope(); + auto issues = issueManager.GetIssues(); + UNIT_ASSERT_VALUES_EQUAL(issues.Size(), 1); + auto scopeIssue = issues.begin(); + UNIT_ASSERT_VALUES_EQUAL(scopeIssue->GetMessage(), "A"); + auto subIssues = scopeIssue->GetSubIssues(); + UNIT_ASSERT_VALUES_EQUAL(subIssues.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(subIssues[0]->GetMessage(), "B"); + + UNIT_ASSERT_VALUES_EQUAL(subIssues[0]->GetSubIssues().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(subIssues[0]->GetSubIssues()[0]->GetMessage(), "IssueOne"); + } + + Y_UNIT_TEST(MultiErrorsMultiLevelsTest) { + TIssueManager issueManager; + issueManager.AddScope(CreateScopeIssueFunction("A", 0, 0)); + issueManager.RaiseIssue(TIssue(TPosition(), "WarningScope1")); + issueManager.AddScope(CreateScopeIssueFunction("B", 1, 1)); + issueManager.AddScope(CreateScopeIssueFunction("C", 2, 2)); + issueManager.RaiseIssue(TIssue(TPosition(), "ErrorScope3")); + issueManager.LeaveScope(); + issueManager.RaiseIssue(TIssue(TPosition(), "ErrorScope2")); + issueManager.LeaveScope(); + issueManager.LeaveScope(); + auto issues = issueManager.GetIssues(); + UNIT_ASSERT_VALUES_EQUAL(issues.Size(), 1); + auto scopeIssue = issues.begin(); + auto subIssues = scopeIssue->GetSubIssues(); + UNIT_ASSERT_VALUES_EQUAL(subIssues.size(), 2); + UNIT_ASSERT_VALUES_EQUAL(subIssues[0]->GetSubIssues().size(), 0); //WarningScope1 + UNIT_ASSERT_VALUES_EQUAL(subIssues[0]->GetMessage(), "WarningScope1"); + UNIT_ASSERT_VALUES_EQUAL(subIssues[1]->GetSubIssues().size(), 2); + UNIT_ASSERT_VALUES_EQUAL(subIssues[1]->GetSubIssues()[0]->GetSubIssues().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(subIssues[1]->GetSubIssues()[0]->GetSubIssues()[0]->GetMessage(), "ErrorScope3"); + UNIT_ASSERT_VALUES_EQUAL(subIssues[1]->GetSubIssues()[1]->GetMessage(), "ErrorScope2"); + auto ref = R"___(<main>: Error: A + <main>: Error: WarningScope1 + <main>:1:1: Error: B + <main>:2:2: Error: C + <main>: Error: ErrorScope3 + <main>: Error: ErrorScope2 +)___"; + UNIT_ASSERT_VALUES_EQUAL(issues.ToString(), ref); + } + + Y_UNIT_TEST(TIssueScopeGuardSimpleTest) { + TIssueManager issueManager; + { + TIssueScopeGuard guard(issueManager, CreateScopeIssueFunction("A", 0, 0)); + issueManager.RaiseIssue(TIssue(TPosition(1,2), "ErrorScope1")); + } + auto issues = issueManager.GetIssues(); + UNIT_ASSERT_VALUES_EQUAL(issues.Size(), 1); + auto scopeIssue = issues.begin(); + UNIT_ASSERT_VALUES_EQUAL(scopeIssue->GetMessage(), "A"); + auto subIssues = scopeIssue->GetSubIssues(); + UNIT_ASSERT_VALUES_EQUAL(subIssues.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(subIssues[0]->GetMessage(), "ErrorScope1"); + } + + Y_UNIT_TEST(FuseScopesTest) { + TIssueManager issueManager; + issueManager.AddScope(CreateScopeIssueFunction("A", 0, 0)); + issueManager.AddScope(CreateScopeIssueFunction("B", 0, 0)); + issueManager.AddScope(CreateScopeIssueFunction("C", 0, 0)); + issueManager.AddScope(CreateScopeIssueFunction("D", 0, 0)); + issueManager.RaiseIssue(TIssue(TPosition(1,2), "IssueOne")); + issueManager.LeaveScope(); + issueManager.RaiseIssue(TIssue(TPosition(1,2), "IssueTwo")); + issueManager.LeaveScope(); + issueManager.LeaveScope(); + issueManager.LeaveScope(); + auto issues = issueManager.GetIssues(); + auto ref = R"___(<main>: Error: A + <main>: Error: B, C + <main>: Error: D + <main>:2:1: Error: IssueOne + <main>:2:1: Error: IssueTwo +)___"; + UNIT_ASSERT_VALUES_EQUAL(issues.ToString(), ref); + } + + Y_UNIT_TEST(UniqIssues) { + TIssueManager issueManager; + issueManager.AddScope(CreateScopeIssueFunction("A", 0, 0)); + issueManager.AddScope(CreateScopeIssueFunction("B", 1, 1)); + issueManager.RaiseIssue(TIssue(TPosition(1,2), "IssueOne")); + issueManager.RaiseIssue(TIssue(TPosition(1,2), "IssueTwo")); + issueManager.RaiseIssue(TIssue(TPosition(2,3), "IssueOne")); + issueManager.RaiseWarning(TIssue(TPosition(2,3), "IssueOne").SetCode(1, ESeverity::TSeverityIds_ESeverityId_S_WARNING)); + issueManager.LeaveScope(); + issueManager.LeaveScope(); + issueManager.RaiseIssue(TIssue(TPosition(1,2), "IssueOne")); + auto issues = issueManager.GetIssues(); + UNIT_ASSERT_VALUES_EQUAL(issues.Size(), 1); + auto ref = R"___(<main>: Error: A + <main>:1:1: Error: B + <main>:2:1: Error: IssueOne + <main>:2:1: Error: IssueTwo + <main>:3:2: Error: IssueOne + <main>:3:2: Warning: IssueOne, code: 1 +)___"; + UNIT_ASSERT_VALUES_EQUAL(issues.ToString(), ref); + } + + Y_UNIT_TEST(Limits) { + TIssueManager issueManager; + issueManager.SetIssueCountLimit(2); + issueManager.AddScope(CreateScopeIssueFunction("A", 0, 0)); + issueManager.AddScope(CreateScopeIssueFunction("B", 1, 1)); + issueManager.RaiseIssue(TIssue(TPosition(1,1), "Issue1")); + issueManager.RaiseIssue(TIssue(TPosition(1,1), "Issue2")); + issueManager.RaiseIssue(TIssue(TPosition(1,1), "Issue3")); + issueManager.RaiseIssue(TIssue(TPosition(1,1), "Issue4").SetCode(1, ESeverity::TSeverityIds_ESeverityId_S_WARNING)); + issueManager.RaiseIssue(TIssue(TPosition(1,1), "Issue5").SetCode(1, ESeverity::TSeverityIds_ESeverityId_S_WARNING)); + issueManager.RaiseIssue(TIssue(TPosition(1,1), "Issue6").SetCode(1, ESeverity::TSeverityIds_ESeverityId_S_WARNING)); + issueManager.RaiseIssue(TIssue(TPosition(1,1), "Issue7").SetCode(2, ESeverity::TSeverityIds_ESeverityId_S_INFO)); + issueManager.RaiseIssue(TIssue(TPosition(1,1), "Issue8").SetCode(2, ESeverity::TSeverityIds_ESeverityId_S_INFO)); + issueManager.LeaveScope(); + issueManager.LeaveScope(); + auto issues = issueManager.GetIssues(); + UNIT_ASSERT_VALUES_EQUAL(issues.Size(), 3); + auto ref = R"___(<main>: Error: Too many Error issues +<main>: Warning: Too many Warning issues +<main>: Error: A + <main>:1:1: Error: B + <main>:1:1: Error: Issue1 + <main>:1:1: Error: Issue2 + <main>:1:1: Warning: Issue4, code: 1 + <main>:1:1: Warning: Issue5, code: 1 + <main>:1:1: Info: Issue7, code: 2 + <main>:1:1: Info: Issue8, code: 2 +)___"; + UNIT_ASSERT_VALUES_EQUAL(issues.ToString(), ref); + } +} diff --git a/yql/essentials/public/issue/yql_issue_message.cpp b/yql/essentials/public/issue/yql_issue_message.cpp new file mode 100644 index 0000000000..757bba3946 --- /dev/null +++ b/yql/essentials/public/issue/yql_issue_message.cpp @@ -0,0 +1,149 @@ +#include "yql_issue_message.h" + +#include <yql/essentials/public/issue/protos/issue_message.pb.h> +#include <contrib/ydb/public/api/protos/ydb_issue_message.pb.h> + +#include <util/generic/deque.h> +#include <util/generic/yexception.h> +#include <util/stream/output.h> +#include <util/string/join.h> + +#include <tuple> + +namespace NYql { + +using namespace NIssue::NProto; + +template<typename TIssueMessage> +TIssue IssueFromMessage(const TIssueMessage& issueMessage) { + TIssue topIssue; + TDeque<std::pair<TIssue*, const TIssueMessage*>> queue; + queue.push_front(std::make_pair(&topIssue, &issueMessage)); + while (!queue.empty()) { + TIssue& issue = *queue.back().first; + const auto& message = *queue.back().second; + queue.pop_back(); + TPosition position(message.position().column(), message.position().row(), message.position().file()); + TPosition endPosition(message.end_position().column(), message.end_position().row()); + if (position.HasValue()) { + if (endPosition.HasValue()) { + issue = TIssue(position, endPosition, message.message()); + } else { + issue = TIssue(position, message.message()); + } + } else { + issue = TIssue(message.message()); + } + + for (const auto& subMessage : message.issues()) { + auto subIssue = new TIssue(); + issue.AddSubIssue(subIssue); + queue.push_front(std::make_pair(subIssue, &subMessage)); + } + + issue.SetCode(message.issue_code(), static_cast<ESeverity>(message.severity())); + } + return topIssue; +} + +template<typename TIssueMessage> +void IssuesFromMessage(const ::google::protobuf::RepeatedPtrField<TIssueMessage> &message, TIssues &issues) { + issues.Clear(); + if (message.size()) { + issues.Reserve(message.size()); + for (auto &x : message) + issues.AddIssue(IssueFromMessage(x)); + } +} + +template<typename TIssueMessage> +void IssueToMessage(const TIssue& topIssue, TIssueMessage* issueMessage) { + TDeque<std::pair<const TIssue*, TIssueMessage*>> queue; + queue.push_front(std::make_pair(&topIssue, issueMessage)); + while (!queue.empty()) { + const TIssue& issue = *queue.back().first; + auto& message = *queue.back().second; + queue.pop_back(); + if (issue.Position) { + auto& position = *message.mutable_position(); + position.set_row(issue.Position.Row); + position.set_column(issue.Position.Column); + position.set_file(issue.Position.File); + } + if (issue.EndPosition) { + auto& endPosition = *message.mutable_end_position(); + endPosition.set_row(issue.EndPosition.Row); + endPosition.set_column(issue.EndPosition.Column); + } + message.set_message(issue.GetMessage()); + message.set_issue_code(issue.GetCode()); + message.set_severity(issue.GetSeverity()); + + for (auto subIssue : issue.GetSubIssues()) { + TIssueMessage* subMessage = message.add_issues(); + queue.push_front(std::make_pair(subIssue.Get(), subMessage)); + } + } +} + +template<typename TIssueMessage> +void IssuesToMessage(const TIssues& issues, ::google::protobuf::RepeatedPtrField<TIssueMessage> *message) { + message->Clear(); + if (!issues) + return; + message->Reserve(issues.Size()); + for (const auto &issue : issues) { + IssueToMessage(issue, message->Add()); + } +} + +template +TIssue IssueFromMessage<Ydb::Issue::IssueMessage>(const Ydb::Issue::IssueMessage& issueMessage); +template +TIssue IssueFromMessage<NYql::NIssue::NProto::IssueMessage>(const NYql::NIssue::NProto::IssueMessage& issueMessage); + +template +void IssuesFromMessage<Ydb::Issue::IssueMessage>(const ::google::protobuf::RepeatedPtrField<Ydb::Issue::IssueMessage>& message, TIssues& issues); +template +void IssuesFromMessage<NYql::NIssue::NProto::IssueMessage>(const ::google::protobuf::RepeatedPtrField<NYql::NIssue::NProto::IssueMessage>& message, TIssues& issues); + +template +void IssueToMessage<Ydb::Issue::IssueMessage>(const TIssue& topIssue, Ydb::Issue::IssueMessage* issueMessage); +template +void IssueToMessage<NYql::NIssue::NProto::IssueMessage>(const TIssue& topIssue, NYql::NIssue::NProto::IssueMessage* issueMessage); + +template +void IssuesToMessage<Ydb::Issue::IssueMessage>(const TIssues& issues, ::google::protobuf::RepeatedPtrField<Ydb::Issue::IssueMessage>* message); +template +void IssuesToMessage<NYql::NIssue::NProto::IssueMessage>(const TIssues& issues, ::google::protobuf::RepeatedPtrField<NYql::NIssue::NProto::IssueMessage>* message); + +NIssue::NProto::IssueMessage IssueToMessage(const TIssue& topIssue) { + NIssue::NProto::IssueMessage issueMessage; + IssueToMessage(topIssue, &issueMessage); + return issueMessage; +} + +TString IssueToBinaryMessage(const TIssue& issue) { + TString result; + Ydb::Issue::IssueMessage protobuf; + IssueToMessage(issue, &protobuf); + Y_PROTOBUF_SUPPRESS_NODISCARD protobuf.SerializeToString(&result); + return result; +} + +TIssue IssueFromBinaryMessage(const TString& binaryMessage) { + Ydb::Issue::IssueMessage protobuf; + if (!protobuf.ParseFromString(binaryMessage)) { + ythrow yexception() << "unable to parse binary string as issue protobuf"; + } + return IssueFromMessage(protobuf); +} + +} + +Y_DECLARE_OUT_SPEC(, google::protobuf::RepeatedPtrField<Ydb::Issue::IssueMessage>, stream, issues) { + stream << JoinSeq("", issues); +} +Y_DECLARE_OUT_SPEC(, google::protobuf::RepeatedPtrField<NYql::NIssue::NProto::IssueMessage>, stream, issues) { + stream << JoinSeq("", issues); +}
\ No newline at end of file diff --git a/yql/essentials/public/issue/yql_issue_message.h b/yql/essentials/public/issue/yql_issue_message.h new file mode 100644 index 0000000000..a3cb63e4f4 --- /dev/null +++ b/yql/essentials/public/issue/yql_issue_message.h @@ -0,0 +1,37 @@ +#pragma once + +#include "yql_issue.h" + +#include <util/generic/ylimits.h> + +namespace NYql { + +namespace NIssue { +namespace NProto { +class IssueMessage; +} +} + +template<typename TIssueMessage> +TIssue IssueFromMessage(const TIssueMessage& issueMessage); +template<typename TIssueMessage> +void IssuesFromMessage(const ::google::protobuf::RepeatedPtrField<TIssueMessage>& message, TIssues& issues); + +template<typename TIssueMessage> +TString IssuesFromMessageAsString(const ::google::protobuf::RepeatedPtrField<TIssueMessage>& message) { + TIssues issues; + IssuesFromMessage(message, issues); + return issues.ToOneLineString(); +} + +NIssue::NProto::IssueMessage IssueToMessage(const TIssue& topIssue); + +template<typename TIssueMessage> +void IssueToMessage(const TIssue& topIssue, TIssueMessage* message); +template<typename TIssueMessage> +void IssuesToMessage(const TIssues& issues, ::google::protobuf::RepeatedPtrField<TIssueMessage>* message); + +TString IssueToBinaryMessage(const TIssue& issue); +TIssue IssueFromBinaryMessage(const TString& binaryMessage); + +} diff --git a/yql/essentials/public/issue/yql_issue_ut.cpp b/yql/essentials/public/issue/yql_issue_ut.cpp new file mode 100644 index 0000000000..02f917fa54 --- /dev/null +++ b/yql/essentials/public/issue/yql_issue_ut.cpp @@ -0,0 +1,249 @@ +#include "yql_issue.h" +#include "yql_issue_message.h" + +#include <library/cpp/testing/unittest/registar.h> +#include <yql/essentials/public/issue/protos/issue_message.pb.h> +#include <yql/essentials/public/issue/yql_issue_message.h> +#include <contrib/ydb/public/api/protos/ydb_issue_message.pb.h> + +#include <library/cpp/unicode/normalization/normalization.h> + +#include <util/charset/utf8.h> +#include <util/charset/wide.h> +#include <util/string/builder.h> + +#include <google/protobuf/message.h> +#include <google/protobuf/descriptor.h> + +using namespace google::protobuf; +using namespace NYql; + +void ensureMessageTypesSame(const Descriptor* a, const Descriptor* b, THashSet<TString>* visitedTypes); +void ensureFieldDescriptorsSame(const FieldDescriptor* a, const FieldDescriptor* b, THashSet<TString>* visitedTypes) { + UNIT_ASSERT(a); + UNIT_ASSERT(b); + + UNIT_ASSERT_VALUES_EQUAL(FieldDescriptor::TypeName(a->type()), FieldDescriptor::TypeName(b->type())); + UNIT_ASSERT_VALUES_EQUAL((int)a->label(), (int)b->label()); + UNIT_ASSERT_VALUES_EQUAL(a->name(), b->name()); + UNIT_ASSERT_VALUES_EQUAL(a->number(), b->number()); + UNIT_ASSERT_VALUES_EQUAL(a->is_repeated(), b->is_repeated()); + UNIT_ASSERT_VALUES_EQUAL(a->is_packed(), b->is_packed()); + UNIT_ASSERT_VALUES_EQUAL(a->index(), b->index()); + if (a->type() == FieldDescriptor::TYPE_MESSAGE || a->type() == FieldDescriptor::TYPE_GROUP) { + ensureMessageTypesSame(a->message_type(), b->message_type(), visitedTypes); + } +} + +void ensureMessageTypesSame(const Descriptor* a, const Descriptor* b, THashSet<TString>* visitedTypes) { + UNIT_ASSERT(a); + UNIT_ASSERT(b); + if (!visitedTypes->insert(a->name()).second) { + return; + } + + UNIT_ASSERT_VALUES_EQUAL(a->name(), b->name()); + UNIT_ASSERT_VALUES_EQUAL(a->field_count(), b->field_count()); + + for (int i = 0; i < a->field_count(); i++) { + ensureFieldDescriptorsSame(a->field(i), b->field(i), visitedTypes); + } +} + +Y_UNIT_TEST_SUITE(IssueTest) { + Y_UNIT_TEST(Ascii) { + TIssue issue1("тест abc"); + UNIT_ASSERT_VALUES_EQUAL(issue1.GetMessage(), "тест abc"); + TIssue issue2("\xFF abc"); + UNIT_ASSERT_VALUES_EQUAL(issue2.GetMessage(), "? abc"); + TIssue issue3(""); + UNIT_ASSERT_VALUES_EQUAL(issue3.GetMessage(), ""); + TIssue issue4("abc"); + UNIT_ASSERT_VALUES_EQUAL(issue4.GetMessage(), "abc"); + } +} + +Y_UNIT_TEST_SUITE(IssueProtoTest) { + Y_UNIT_TEST(KikimrYqlSameLayout) { + Ydb::Issue::IssueMessage yqlIssue; + NYql::NIssue::NProto::IssueMessage kikimrIssue; + THashSet<TString> visitedTypes; + ensureMessageTypesSame(yqlIssue.GetDescriptor(), kikimrIssue.GetDescriptor(), &visitedTypes); + } + + Y_UNIT_TEST(BinarySerialization) { + TIssue issueTo("root_issue"); + TString bin = IssueToBinaryMessage(issueTo); + TIssue issueFrom = IssueFromBinaryMessage(bin); + UNIT_ASSERT_EQUAL(issueTo, issueFrom); + } + + Y_UNIT_TEST(WrongBinStringException) { + UNIT_ASSERT_EXCEPTION(IssueFromBinaryMessage("qqq"), yexception); + } +} + + +Y_UNIT_TEST_SUITE(TextWalkerTest) { + Y_UNIT_TEST(BasicTest) { + TPosition pos; + pos.Row = 1; + + TTextWalker walker(pos, false); + walker.Advance(TStringBuf("a\r\taa")); + + UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(5, 1)); + walker.Advance(TStringBuf("\na")); + UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(1, 2)); + } + + Y_UNIT_TEST(CrLfTest) { + TPosition pos; + pos.Row = 1; + + TTextWalker walker(pos, false); + walker.Advance(TStringBuf("a\raa\r")); + UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(4, 1)); + walker.Advance('\n'); + UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(4, 1)); + walker.Advance(TStringBuf("\r\r\ra")); + UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(4, 2)); + walker.Advance('\r'); + UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(4, 2)); + walker.Advance('\n'); + UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(4, 2)); + walker.Advance('a'); + UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(1, 3)); + } + + Y_UNIT_TEST(UnicodeTest) { + { + TPosition pos; + pos.Row = 1; + + TTextWalker walker(pos, false); + walker.Advance(TStringBuf("привет")); + + UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(12, 1)); + } + + { + TPosition pos; + pos.Row = 1; + + TTextWalker walker(pos, true); + walker.Advance(TStringBuf("привет")); + + UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(6, 1)); + } + } +} + +Y_UNIT_TEST_SUITE(ToOneLineStringTest) { + Y_UNIT_TEST(OneMessageTest) { + TIssues issues; + issues.AddIssue(TPosition(12, 34, "file.abc"), "error"); + UNIT_ASSERT_STRINGS_EQUAL(issues.ToOneLineString(), "{ file.abc:34:12: Error: error }"); + } + + Y_UNIT_TEST(SubIssuesTest) { + TIssue issue(TPosition(12, 34, "file.abc"), "error"); + TIssue subissue("suberror"); + subissue.AddSubIssue(MakeIntrusive<TIssue>("subsuberror")); + issue.AddSubIssue(MakeIntrusive<TIssue>(subissue)); + + TIssues issues; + issues.AddIssue(issue); + UNIT_ASSERT_STRINGS_EQUAL(issues.ToOneLineString(), "{ file.abc:34:12: Error: error subissue: { <main>: Error: suberror subissue: { <main>: Error: subsuberror } } }"); + } + + Y_UNIT_TEST(ManyIssuesTest) { + TIssue issue(TPosition(12, 34, "file.abc"), "error\n"); + issue.AddSubIssue(MakeIntrusive<TIssue>("suberror")); + TIssues issues; + issues.AddIssue(issue); + issues.AddIssue(TPosition(100, 2, "abc.file"), "my\nmessage"); + UNIT_ASSERT_STRINGS_EQUAL(issues.ToOneLineString(), "[ { file.abc:34:12: Error: error subissue: { <main>: Error: suberror } } { abc.file:2:100: Error: my message } ]"); + } +} + +Y_UNIT_TEST_SUITE(ToStreamTest) { + template <typename TIssueMessage> + void CheckSerializationToStream(const TIssues& issues, const TString& expected) { + google::protobuf::RepeatedPtrField<TIssueMessage> protoIssues; + IssuesToMessage(issues, &protoIssues); + TStringBuilder stream; + stream << protoIssues; + UNIT_ASSERT_STRINGS_EQUAL(stream, expected); + }; + + Y_UNIT_TEST(OneMessageTest) { + TIssues issues; + issues.AddIssue("error"); + CheckSerializationToStream<Ydb::Issue::IssueMessage>(issues, "{ message: \"error\" severity: 1 }"); + CheckSerializationToStream<NYql::NIssue::NProto::IssueMessage>(issues, "{ message: \"error\" issue_code: 0 severity: 1 }"); + } + + Y_UNIT_TEST(SubIssuesTest) { + TIssue issue(TPosition(12, 34, "file.abc"), "error"); + TIssue subissue("suberror"); + subissue.AddSubIssue(MakeIntrusive<TIssue>("subsuberror")); + issue.AddSubIssue(MakeIntrusive<TIssue>(subissue)); + TIssues issues; + issues.AddIssue(issue); + CheckSerializationToStream<Ydb::Issue::IssueMessage>(issues, "{ position { row: 34 column: 12 file: \"file.abc\" } message: \"error\" end_position { row: 34 column: 12 } severity: 1 issues { message: \"suberror\" severity: 1 issues { message: \"subsuberror\" severity: 1 } } }"); + CheckSerializationToStream<NYql::NIssue::NProto::IssueMessage>(issues, "{ position { row: 34 column: 12 file: \"file.abc\" } message: \"error\" end_position { row: 34 column: 12 } issue_code: 0 severity: 1 issues { message: \"suberror\" issue_code: 0 severity: 1 issues { message: \"subsuberror\" issue_code: 0 severity: 1 } } }"); + } + + Y_UNIT_TEST(ManyIssuesTest) { + TIssue issue(TPosition(12, 34, "file.abc"), "error"); + issue.AddSubIssue(MakeIntrusive<TIssue>("suberror")); + TIssues issues; + issues.AddIssue(issue); + issues.AddIssue(TPosition(100, 2, "abc.file"), "my message"); + CheckSerializationToStream<Ydb::Issue::IssueMessage>(issues, "{ position { row: 34 column: 12 file: \"file.abc\" } message: \"error\" end_position { row: 34 column: 12 } severity: 1 issues { message: \"suberror\" severity: 1 } }{ position { row: 2 column: 100 file: \"abc.file\" } message: \"my message\" end_position { row: 2 column: 100 } severity: 1 }"); + CheckSerializationToStream<NYql::NIssue::NProto::IssueMessage>(issues, "{ position { row: 34 column: 12 file: \"file.abc\" } message: \"error\" end_position { row: 34 column: 12 } issue_code: 0 severity: 1 issues { message: \"suberror\" issue_code: 0 severity: 1 } }{ position { row: 2 column: 100 file: \"abc.file\" } message: \"my message\" end_position { row: 2 column: 100 } issue_code: 0 severity: 1 }"); + } +} + +Y_UNIT_TEST_SUITE(ToMessage) { + Y_UNIT_TEST(NonUtf8) { + const TString nonUtf8String = "\x7f\xf8\xf7\xff\xf8\x1f\xff\xf2\xaf\xbf\xfe\xfa\xf5\x7f\xfe\xfa\x27\x20\x7d\x20\x5d\x2e"; + UNIT_ASSERT(!IsUtf(nonUtf8String)); + TIssue issue; + issue.SetMessage(nonUtf8String); + + Ydb::Issue::IssueMessage msg; + IssueToMessage(issue, &msg); + TString serialized; + UNIT_ASSERT(msg.SerializeToString(&serialized)); + Ydb::Issue::IssueMessage msg2; + UNIT_ASSERT(msg2.ParseFromString(serialized)); + } +} + +Y_UNIT_TEST_SUITE(EscapeNonUtf8) { + Y_UNIT_TEST(Escape) { + const TString nonUtf8String = "\xfe\xfa\xf5\xc2"; + UNIT_ASSERT(!IsUtf(nonUtf8String)); + + // Check that our escaping correctly processes unicode pairs + const TString toNormalize = "Ёлка"; + const TString nfd = WideToUTF8(Normalize<NUnicode::ENormalization::NFD>(UTF8ToWide(toNormalize))); // dots over 'ё' will be separate unicode symbol + const TString nfc = WideToUTF8(Normalize<NUnicode::ENormalization::NFC>(UTF8ToWide(toNormalize))); // dots over 'ё' will be with with their letter + UNIT_ASSERT_STRINGS_UNEQUAL(nfc, nfd); + std::pair<TString, TString> nonUtf8Messages[] = { + { nonUtf8String, "????" }, + { TStringBuilder() << nonUtf8String << "Failed to parse file " << nonUtf8String << "עברית" << nonUtf8String, "????Failed to parse file ????עברית????" }, + { nfd, nfd }, + { nfc, nfc }, + { TStringBuilder() << nfc << nonUtf8String << nfd, TStringBuilder() << nfc << "????" << nfd }, + { TStringBuilder() << nfd << nonUtf8String << nfc, TStringBuilder() << nfd << "????" << nfc }, + }; + + for (const auto& [src, dst] : nonUtf8Messages) { + TIssue issue(src); + UNIT_ASSERT_STRINGS_EQUAL(issue.GetMessage(), dst); + } + } +} diff --git a/yql/essentials/public/issue/yql_issue_utils.cpp b/yql/essentials/public/issue/yql_issue_utils.cpp new file mode 100644 index 0000000000..2be0b8e57b --- /dev/null +++ b/yql/essentials/public/issue/yql_issue_utils.cpp @@ -0,0 +1,105 @@ +#include "yql_issue_utils.h" + +#include <util/system/yassert.h> + +#include <tuple> +#include <list> +#include <algorithm> +#include <deque> + +namespace NYql { + +TIssue TruncateIssueLevels(const TIssue& topIssue, TTruncateIssueOpts opts) { + // [issue, level, parent, visibleParent] + std::list<std::tuple<const TIssue*, ui32, size_t, size_t>> issueQueue; + // [issue, targetIssue, level, parent, visible, visibleParent, targetSkipIssue] + std::deque<std::tuple<const TIssue*, TIssue*, ui32, size_t, bool, size_t, TIssue*>> issues; + // [depth from bottom, position] + std::list<size_t> leafs; + + const auto depthBeforeLeaf = std::max(opts.KeepTailLevels, ui32(1)) - 1; + const auto maxLevels = std::max(opts.MaxLevels - std::min(opts.MaxLevels, depthBeforeLeaf + 1), ui32(1)); + + issueQueue.emplace_front(&topIssue, 0, 0, 0); + while (!issueQueue.empty()) { + const auto issue = std::get<0>(issueQueue.back()); + const auto level = std::get<1>(issueQueue.back()); + const auto parent = std::get<2>(issueQueue.back()); + const auto visibleParent = std::get<3>(issueQueue.back()); + issueQueue.pop_back(); + + const bool visible = issue->GetSubIssues().empty() || level < maxLevels; + const auto pos = issues.size(); + issues.emplace_back(issue, nullptr, level, parent, visible, visibleParent, nullptr); + if (issue->GetSubIssues().empty()) { + if (level != 0) { + leafs.push_back(pos); + } + } else { + for (auto subIssue : issue->GetSubIssues()) { + issueQueue.emplace_front(subIssue.Get(), level + 1, pos, visible ? pos : visibleParent); + } + } + } + + if (depthBeforeLeaf && !leafs.empty()) { + for (size_t pos: leafs) { + ui32 depth = depthBeforeLeaf; + auto parent = std::get<3>(issues.at(pos)); + while (depth && parent) { + auto& visible = std::get<4>(issues.at(parent)); + auto& visibleParent = std::get<5>(issues.at(pos)); + if (!visible || visibleParent != parent) { + visible = true; + visibleParent = parent; // Update visible parent + --depth; + pos = parent; + parent = std::get<3>(issues.at(parent)); + } else { + break; + } + } + } + } + leafs.clear(); + + TIssue result; + for (auto& i: issues) { + const auto srcIssue = std::get<0>(i); + auto& targetIssue = std::get<1>(i); + const auto level = std::get<2>(i); + const auto parent = std::get<3>(i); + const auto visible = std::get<4>(i); + const auto visibleParent = std::get<5>(i); + + if (0 == level) { + targetIssue = &result; + targetIssue->CopyWithoutSubIssues(*srcIssue); + } else if (visible) { + auto& parentRec = issues.at(visibleParent); + auto& parentTargetIssue = std::get<1>(parentRec); + if (parent != visibleParent) { + auto& parentSkipIssue = std::get<6>(parentRec); + if (!parentSkipIssue) { + const auto parentIssue = std::get<0>(parentRec); + auto newIssue = MakeIntrusive<TIssue>("(skipped levels)"); + newIssue->SetCode(parentIssue->GetCode(), parentIssue->GetSeverity()); + parentTargetIssue->AddSubIssue(newIssue); + parentSkipIssue = newIssue.Get(); + } + auto newIssue = MakeIntrusive<TIssue>(TString{}); + newIssue->CopyWithoutSubIssues(*srcIssue); + parentSkipIssue->AddSubIssue(newIssue); + targetIssue = newIssue.Get(); + } else { + auto newIssue = MakeIntrusive<TIssue>(TString{}); + newIssue->CopyWithoutSubIssues(*srcIssue); + parentTargetIssue->AddSubIssue(newIssue); + targetIssue = newIssue.Get(); + } + } + } + return result; +} + +} // namspace NYql diff --git a/yql/essentials/public/issue/yql_issue_utils.h b/yql/essentials/public/issue/yql_issue_utils.h new file mode 100644 index 0000000000..023ea0edbf --- /dev/null +++ b/yql/essentials/public/issue/yql_issue_utils.h @@ -0,0 +1,31 @@ +#pragma once + +#include "yql_issue.h" + +#include <util/generic/ylimits.h> +#include <util/system/types.h> + +namespace NYql { + +struct TTruncateIssueOpts { + +#define YQL_TRUNC_DECL_FIELD(type, name, def) \ + TTruncateIssueOpts& Set##name(type arg##name)& { \ + name = arg##name; \ + return *this; \ + } \ + TTruncateIssueOpts&& Set##name(type arg##name)&& { \ + name = arg##name; \ + return std::move(*this); \ + } \ + type name = def; + + YQL_TRUNC_DECL_FIELD(ui32, MaxLevels, Max<ui32>()) + YQL_TRUNC_DECL_FIELD(ui32, KeepTailLevels, 1) + +#undef YQL_TRUNC_DECL_FIELD +}; + +TIssue TruncateIssueLevels(const TIssue& topIssue, TTruncateIssueOpts opts = {}); + +} // namespace NYql diff --git a/yql/essentials/public/issue/yql_issue_utils_ut.cpp b/yql/essentials/public/issue/yql_issue_utils_ut.cpp new file mode 100644 index 0000000000..657643a613 --- /dev/null +++ b/yql/essentials/public/issue/yql_issue_utils_ut.cpp @@ -0,0 +1,200 @@ +#include "yql_issue_utils.h" +#include "yql_issue.h" +#include "yql_issue_id.h" + +#include <library/cpp/testing/unittest/registar.h> + +#include <util/string/subst.h> + +using namespace NYql; + +Y_UNIT_TEST_SUITE(TIssueUtilsTest) { + Y_UNIT_TEST(TruncLevels1) { + auto level0 = MakeIntrusive<TIssue>("level0"); + auto level1 = MakeIntrusive<TIssue>("level1"); + auto level2 = MakeIntrusive<TIssue>("level2"); + auto level30 = MakeIntrusive<TIssue>("level30"); + auto level31 = MakeIntrusive<TIssue>("level31"); + auto level40 = MakeIntrusive<TIssue>("level40"); + auto level41 = MakeIntrusive<TIssue>("level41"); + auto level51 = MakeIntrusive<TIssue>("level51"); + + /* + * * 0 + * | + * * 1 + * | + * * 2 -- + * | | + * * 30 * 31 + * | | + * * 40 * 41 + * | + * * 51 + */ + + level0->AddSubIssue(level1); + level1->AddSubIssue(level2); + level2->AddSubIssue(level30); + level2->AddSubIssue(level31); + level30->AddSubIssue(level40); + level31->AddSubIssue(level41); + level41->AddSubIssue(level51); + + { + const auto res = TIssues({TruncateIssueLevels(*level0, TTruncateIssueOpts().SetMaxLevels(4).SetKeepTailLevels(2))}).ToString(); + const auto expected = +R"___(<main>: Error: level0 + <main>: Error: level1 + <main>: Error: (skipped levels) + <main>: Error: level30 + <main>: Error: level40 + <main>: Error: level41 + <main>: Error: level51 +)___"; + UNIT_ASSERT_STRINGS_EQUAL(res, expected); + } + + { + const auto res = TIssues({TruncateIssueLevels(*level0, TTruncateIssueOpts().SetMaxLevels(3).SetKeepTailLevels(1))}).ToString(); + const auto expected = +R"___(<main>: Error: level0 + <main>: Error: level1 + <main>: Error: (skipped levels) + <main>: Error: level40 + <main>: Error: level51 +)___"; + UNIT_ASSERT_STRINGS_EQUAL(res, expected); + } + } + + Y_UNIT_TEST(TruncLevels2) { + auto level0 = MakeIntrusive<TIssue>("level0"); + auto level1 = MakeIntrusive<TIssue>("level1"); + auto level2 = MakeIntrusive<TIssue>("level2"); + auto level3 = MakeIntrusive<TIssue>("level3"); + auto level40 = MakeIntrusive<TIssue>("level40"); + auto level41 = MakeIntrusive<TIssue>("level41"); + + /* + * * 0 + * | + * * 1 + * | + * * 2 + * | + * * 3 -- + * | | + * * 40 * 41 + */ + + level0->AddSubIssue(level1); + level1->AddSubIssue(level2); + level2->AddSubIssue(level3); + level3->AddSubIssue(level40); + level3->AddSubIssue(level41); + + { + const auto res = TIssues({TruncateIssueLevels(*level0, TTruncateIssueOpts().SetMaxLevels(4).SetKeepTailLevels(2))}).ToString(); + const auto expected = +R"___(<main>: Error: level0 + <main>: Error: level1 + <main>: Error: (skipped levels) + <main>: Error: level3 + <main>: Error: level40 + <main>: Error: level41 +)___"; + UNIT_ASSERT_STRINGS_EQUAL(res, expected); + } + } + + Y_UNIT_TEST(TruncLevels3) { + auto level0 = MakeIntrusive<TIssue>("level0"); + auto level1 = MakeIntrusive<TIssue>("level1"); + auto level2 = MakeIntrusive<TIssue>("level2"); + auto level3 = MakeIntrusive<TIssue>("level3"); + auto level40 = MakeIntrusive<TIssue>("level40"); + auto level41 = MakeIntrusive<TIssue>("level41"); + auto level50 = MakeIntrusive<TIssue>("level50"); + + /* + * * 0 + * | + * * 1 + * | + * * 2 + * | + * * 3 -- + * | | + * * 40 | + * | | + * * 50 * 41 + */ + + level0->AddSubIssue(level1); + level1->AddSubIssue(level2); + level2->AddSubIssue(level3); + level3->AddSubIssue(level40); + level3->AddSubIssue(level41); + level40->AddSubIssue(level50); + + { + const auto res = TIssues({TruncateIssueLevels(*level0, TTruncateIssueOpts().SetMaxLevels(4).SetKeepTailLevels(1))}).ToString(); + const auto expected = +R"___(<main>: Error: level0 + <main>: Error: level1 + <main>: Error: level2 + <main>: Error: (skipped levels) + <main>: Error: level41 + <main>: Error: level50 +)___"; + UNIT_ASSERT_STRINGS_EQUAL(res, expected); + } + + { + const auto res = TIssues({TruncateIssueLevels(*level0, TTruncateIssueOpts().SetMaxLevels(4).SetKeepTailLevels(2))}).ToString(); + const auto expected = +R"___(<main>: Error: level0 + <main>: Error: level1 + <main>: Error: (skipped levels) + <main>: Error: level3 + <main>: Error: level41 + <main>: Error: level40 + <main>: Error: level50 +)___"; + UNIT_ASSERT_STRINGS_EQUAL(res, expected); + } + + { + const auto res = TIssues({TruncateIssueLevels(*level0, TTruncateIssueOpts().SetMaxLevels(4).SetKeepTailLevels(3))}).ToString(); + const auto expected = +R"___(<main>: Error: level0 + <main>: Error: (skipped levels) + <main>: Error: level2 + <main>: Error: level3 + <main>: Error: level40 + <main>: Error: level50 + <main>: Error: level41 +)___"; + UNIT_ASSERT_STRINGS_EQUAL(res, expected); + } + } + + Y_UNIT_TEST(KeepSeverity) { + const auto templ = +R"___(<main>: {severity}: level0, code: 1 + <main>: {severity}: level1, code: 1 +)___"; + for (auto severity: {ESeverity::TSeverityIds_ESeverityId_S_INFO, ESeverity::TSeverityIds_ESeverityId_S_WARNING, ESeverity::TSeverityIds_ESeverityId_S_ERROR, ESeverity::TSeverityIds_ESeverityId_S_FATAL}) { + + auto level0 = MakeIntrusive<TIssue>(TIssue("level0").SetCode(1, severity)); + auto level1 = MakeIntrusive<TIssue>(TIssue("level1").SetCode(1, severity)); + + level0->AddSubIssue(level1); + + const auto res = TIssues({TruncateIssueLevels(*level0, TTruncateIssueOpts().SetMaxLevels(15).SetKeepTailLevels(3))}).ToString(); + const auto expected = SubstGlobalCopy<TString, TString>(templ, "{severity}", SeverityToString(severity)); + UNIT_ASSERT_STRINGS_EQUAL(res, expected); + } + } +} diff --git a/yql/essentials/public/issue/yql_warning.cpp b/yql/essentials/public/issue/yql_warning.cpp new file mode 100644 index 0000000000..881e63e95f --- /dev/null +++ b/yql/essentials/public/issue/yql_warning.cpp @@ -0,0 +1,73 @@ +#include "yql_warning.h" + +#include <util/string/cast.h> +#include <util/string/join.h> + + +namespace NYql { + +TWarningRule::EParseResult TWarningRule::ParseFrom(const TString& codePattern, const TString& action, + TWarningRule& result, TString& errorMessage) +{ + errorMessage.clear(); + result.IssueCodePattern.clear(); + result.Action = EWarningAction::DEFAULT; + + if (!TryFromString<EWarningAction>(to_upper(action), result.Action)) { + errorMessage = "unknown warning action '" + action + "', expecting one of " + + Join(", ", EWarningAction::DEFAULT, EWarningAction::ERROR, EWarningAction::DISABLE); + return EParseResult::PARSE_ACTION_FAIL; + } + + if (codePattern != "*") { + ui32 code; + if (!TryFromString(codePattern, code)) { + errorMessage = "unknown warning code '" + codePattern + "', expecting integer or '*'"; + return EParseResult::PARSE_PATTERN_FAIL; + } + } + + result.IssueCodePattern = codePattern; + return EParseResult::PARSE_OK; +} + +void TWarningPolicy::AddRule(const TWarningRule& rule) +{ + TString pattern = rule.GetPattern(); + if (pattern.empty()) { + return; + } + + Rules.push_back(rule); + + EWarningAction action = rule.GetAction(); + if (pattern == "*") { + BaseAction = action; + Overrides.clear(); + return; + } + + TIssueCode code; + Y_ENSURE(TryFromString(pattern, code)); + + if (action == BaseAction) { + Overrides.erase(Overrides.find(code)); + } else { + Overrides[code] = action; + } +} + +EWarningAction TWarningPolicy::GetAction(TIssueCode code) const +{ + auto it = Overrides.find(code); + return (it == Overrides.end()) ? BaseAction : it->second; +} + +void TWarningPolicy::Clear() +{ + BaseAction = EWarningAction::DEFAULT; + Overrides.clear(); + Rules.clear(); +} + +} diff --git a/yql/essentials/public/issue/yql_warning.h b/yql/essentials/public/issue/yql_warning.h new file mode 100644 index 0000000000..d1d6a90922 --- /dev/null +++ b/yql/essentials/public/issue/yql_warning.h @@ -0,0 +1,51 @@ +#pragma once + +#include "yql_issue_id.h" + +#include <util/generic/hash.h> +#include <util/generic/string.h> + +#if defined(_win_) +#undef ERROR +#endif + +namespace NYql { + +enum class EWarningAction { + DISABLE, + ERROR, + DEFAULT, +}; + +class TWarningRule { +public: + const TString& GetPattern() const { return IssueCodePattern; } + EWarningAction GetAction() const { return Action; } + + enum class EParseResult { PARSE_OK, PARSE_PATTERN_FAIL, PARSE_ACTION_FAIL }; + static EParseResult ParseFrom(const TString& codePattern, const TString& action, + TWarningRule& result, TString& errorMessage); +private: + TString IssueCodePattern; + EWarningAction Action = EWarningAction::DEFAULT; +}; + +using TWarningRules = TVector<TWarningRule>; + +class TWarningPolicy { +public: + void AddRule(const TWarningRule& rule); + + EWarningAction GetAction(TIssueCode code) const; + + const TWarningRules& GetRules() const { return Rules; } + + void Clear(); + +private: + TWarningRules Rules; + EWarningAction BaseAction = EWarningAction::DEFAULT; + THashMap<TIssueCode, EWarningAction> Overrides; +}; + +} diff --git a/yql/essentials/public/issue/yql_warning_ut.cpp b/yql/essentials/public/issue/yql_warning_ut.cpp new file mode 100644 index 0000000000..5edf7ed363 --- /dev/null +++ b/yql/essentials/public/issue/yql_warning_ut.cpp @@ -0,0 +1,94 @@ +#include <library/cpp/testing/unittest/registar.h> + +#include "yql_warning.h" + +using namespace NYql; + +Y_UNIT_TEST_SUITE(TWarningRuleTest) { + + Y_UNIT_TEST(AllValidActionsAndPatternsAreParsed) { + TWarningRule rule; + TString errorMessage; + UNIT_ASSERT_VALUES_EQUAL(TWarningRule::ParseFrom("1234", "disable", rule, errorMessage), TWarningRule::EParseResult::PARSE_OK); + UNIT_ASSERT_VALUES_EQUAL(TWarningRule::ParseFrom("*", "error", rule, errorMessage), TWarningRule::EParseResult::PARSE_OK); + UNIT_ASSERT_VALUES_EQUAL(TWarningRule::ParseFrom("0", "default", rule, errorMessage), TWarningRule::EParseResult::PARSE_OK); + } + + Y_UNIT_TEST(InvalidActionIsDetected) { + TWarningRule rule; + TString errorMessage; + UNIT_ASSERT_VALUES_EQUAL(TWarningRule::ParseFrom("1234", "wrong", rule, errorMessage), TWarningRule::EParseResult::PARSE_ACTION_FAIL); + UNIT_ASSERT_VALUES_EQUAL(TWarningRule::ParseFrom("1234", "", rule, errorMessage), TWarningRule::EParseResult::PARSE_ACTION_FAIL); + } + + Y_UNIT_TEST(InvalidPatternIsDetected) { + TWarningRule rule; + TString errorMessage; + UNIT_ASSERT_VALUES_EQUAL(TWarningRule::ParseFrom("", "default", rule, errorMessage), TWarningRule::EParseResult::PARSE_PATTERN_FAIL); + UNIT_ASSERT_VALUES_EQUAL(TWarningRule::ParseFrom("-1", "default", rule, errorMessage), TWarningRule::EParseResult::PARSE_PATTERN_FAIL); + UNIT_ASSERT_VALUES_EQUAL(TWarningRule::ParseFrom("*1", "default", rule, errorMessage), TWarningRule::EParseResult::PARSE_PATTERN_FAIL); + UNIT_ASSERT_VALUES_EQUAL(TWarningRule::ParseFrom("default", "default", rule, errorMessage), TWarningRule::EParseResult::PARSE_PATTERN_FAIL); + } +} + + +void AddRule(TWarningPolicy& policy, const TString& codePattern, const TString& action) +{ + TWarningRule rule; + TString errorMessage; + UNIT_ASSERT_VALUES_EQUAL(TWarningRule::ParseFrom(codePattern, action, rule, errorMessage), TWarningRule::EParseResult::PARSE_OK); + policy.AddRule(rule); +} + + +Y_UNIT_TEST_SUITE(TWarningPolicyTest) { + + Y_UNIT_TEST(BasicIntegerRules) { + TWarningPolicy policy; + AddRule(policy, "123", "error"); + AddRule(policy, "456", "error"); + AddRule(policy, "456", "disable"); + + UNIT_ASSERT_VALUES_EQUAL(policy.GetRules().size(), 3); + UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(123), EWarningAction::ERROR); + UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(456), EWarningAction::DISABLE); + UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(111), EWarningAction::DEFAULT); + } + + + Y_UNIT_TEST(AsteriskRules) { + TWarningPolicy policy; + AddRule(policy, "*", "error"); + AddRule(policy, "456", "disable"); + + UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(999), EWarningAction::ERROR); + UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(456), EWarningAction::DISABLE); + + AddRule(policy, "*", "default"); + + UNIT_ASSERT_VALUES_EQUAL(policy.GetRules().size(), 3); + UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(999), EWarningAction::DEFAULT); + UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(456), EWarningAction::DEFAULT); + UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(123), EWarningAction::DEFAULT); + + } + + Y_UNIT_TEST(ReactionOnPull) { + TWarningPolicy policy; + AddRule(policy, "*", "error"); + AddRule(policy, "456", "default"); + AddRule(policy, "999", "disable"); + + UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(999), EWarningAction::DISABLE); + UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(456), EWarningAction::DEFAULT); + UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(123), EWarningAction::ERROR); + + UNIT_ASSERT_VALUES_EQUAL(policy.GetRules().size(), 3); + policy.Clear(); + UNIT_ASSERT_VALUES_EQUAL(policy.GetRules().size(), 0); + + UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(999), EWarningAction::DEFAULT); + UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(456), EWarningAction::DEFAULT); + UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(123), EWarningAction::DEFAULT); + } +} diff --git a/yql/essentials/public/ya.make b/yql/essentials/public/ya.make new file mode 100644 index 0000000000..14a283e5a9 --- /dev/null +++ b/yql/essentials/public/ya.make @@ -0,0 +1,4 @@ +RECURSE( + issue +) + diff --git a/yql/essentials/ya.make b/yql/essentials/ya.make index 48f46c7cd8..57841ee7e2 100644 --- a/yql/essentials/ya.make +++ b/yql/essentials/ya.make @@ -1,5 +1,6 @@ SUBSCRIBER(g:yql) RECURSE( + public ) diff --git a/yt/cpp/mapreduce/interface/logging/yt_log.cpp b/yt/cpp/mapreduce/interface/logging/yt_log.cpp index 4ff38d4c58..d41bfa559f 100644 --- a/yt/cpp/mapreduce/interface/logging/yt_log.cpp +++ b/yt/cpp/mapreduce/interface/logging/yt_log.cpp @@ -26,10 +26,6 @@ public: ::TSourceLocation sourceLocation, TStringBuf anchorMessage) override { - if (anchor->Registered.exchange(true)) { - return; - } - auto guard = Guard(Mutex_); anchor->SourceLocation = sourceLocation; anchor->AnchorMessage = anchorMessage; diff --git a/yt/cpp/mapreduce/interface/operation-inl.h b/yt/cpp/mapreduce/interface/operation-inl.h index 20526f5215..db900c1438 100644 --- a/yt/cpp/mapreduce/interface/operation-inl.h +++ b/yt/cpp/mapreduce/interface/operation-inl.h @@ -93,6 +93,7 @@ template <typename TDerived> TDerived& TRawOperationIoTableSpec<TDerived>::SetInput(size_t tableIndex, const TRichYPath& path) { NDetail::Assign(Inputs_, tableIndex, path); + return static_cast<TDerived&>(*this); } template <typename TDerived> @@ -106,6 +107,7 @@ template <typename TDerived> TDerived& TRawOperationIoTableSpec<TDerived>::SetOutput(size_t tableIndex, const TRichYPath& path) { NDetail::Assign(Outputs_, tableIndex, path); + return static_cast<TDerived&>(*this); } template <typename TDerived> @@ -133,6 +135,7 @@ template <typename TDerived> TDerived& TRawMapReduceOperationIoSpec<TDerived>::SetMapOutput(size_t tableIndex, const TRichYPath& path) { NDetail::Assign(MapOutputs_, tableIndex, path); + return static_cast<TDerived&>(*this); } template <typename TDerived> diff --git a/yt/yt/client/api/delegating_client.h b/yt/yt/client/api/delegating_client.h index 6f85657139..9cb73f3fb1 100644 --- a/yt/yt/client/api/delegating_client.h +++ b/yt/yt/client/api/delegating_client.h @@ -881,7 +881,7 @@ public: DELEGATE_METHOD(TFuture<IRowBatchWriterPtr>, CreateShuffleWriter, ( const TShuffleHandlePtr& shuffleHandle, - const TString& partitionColumn, + const std::string& partitionColumn, const NTableClient::TTableWriterConfigPtr& config), (shuffleHandle, partitionColumn, config)) diff --git a/yt/yt/client/api/delegating_transaction.cpp b/yt/yt/client/api/delegating_transaction.cpp index 156eb1dde7..8a8ddf96dd 100644 --- a/yt/yt/client/api/delegating_transaction.cpp +++ b/yt/yt/client/api/delegating_transaction.cpp @@ -264,7 +264,7 @@ DELEGATE_METHOD(void, LockRows, ( const NYPath::TYPath& path, NTableClient::TNameTablePtr nameTable, TSharedRange<NTableClient::TLegacyKey> keys, - const std::vector<TString>& locks, + const std::vector<std::string>& locks, NTableClient::ELockType lockType), (path, nameTable, keys, locks, lockType)) diff --git a/yt/yt/client/api/delegating_transaction.h b/yt/yt/client/api/delegating_transaction.h index e0a71dee00..e0263fc799 100644 --- a/yt/yt/client/api/delegating_transaction.h +++ b/yt/yt/client/api/delegating_transaction.h @@ -214,7 +214,7 @@ public: const NYPath::TYPath& path, NTableClient::TNameTablePtr nameTable, TSharedRange<NTableClient::TLegacyKey> keys, - const std::vector<TString>& locks, + const std::vector<std::string>& locks, NTableClient::ELockType lockType) override; void ModifyRows( diff --git a/yt/yt/client/api/dynamic_table_transaction.h b/yt/yt/client/api/dynamic_table_transaction.h index b22e9bcd8f..37db074bdf 100644 --- a/yt/yt/client/api/dynamic_table_transaction.h +++ b/yt/yt/client/api/dynamic_table_transaction.h @@ -106,7 +106,7 @@ struct IDynamicTableTransaction const NYPath::TYPath& path, NTableClient::TNameTablePtr nameTable, TSharedRange<NTableClient::TLegacyKey> keys, - const std::vector<TString>& locks, + const std::vector<std::string>& locks, NTableClient::ELockType lockType = NTableClient::ELockType::SharedStrong) = 0; virtual void ModifyRows( diff --git a/yt/yt/client/api/dynamic_table_transaction_mixin.cpp b/yt/yt/client/api/dynamic_table_transaction_mixin.cpp index 240e4ea548..4f1d2c901f 100644 --- a/yt/yt/client/api/dynamic_table_transaction_mixin.cpp +++ b/yt/yt/client/api/dynamic_table_transaction_mixin.cpp @@ -152,7 +152,7 @@ void TDynamicTableTransactionMixin::LockRows( const NYPath::TYPath& path, TNameTablePtr nameTable, TSharedRange<TLegacyKey> keys, - const std::vector<TString>& locks, + const std::vector<std::string>& locks, ELockType lockType) { const auto& tableMountCache = GetClient()->GetTableMountCache(); diff --git a/yt/yt/client/api/dynamic_table_transaction_mixin.h b/yt/yt/client/api/dynamic_table_transaction_mixin.h index c8492f1bf6..ab94daaa77 100644 --- a/yt/yt/client/api/dynamic_table_transaction_mixin.h +++ b/yt/yt/client/api/dynamic_table_transaction_mixin.h @@ -43,7 +43,7 @@ public: const NYPath::TYPath& path, NTableClient::TNameTablePtr nameTable, TSharedRange<NTableClient::TLegacyKey> keys, - const std::vector<TString>& locks, + const std::vector<std::string>& locks, NTableClient::ELockType lockType = NTableClient::ELockType::SharedStrong) override; }; diff --git a/yt/yt/client/api/query_tracker_client.h b/yt/yt/client/api/query_tracker_client.h index 4e198ed4b3..0e0fc147bf 100644 --- a/yt/yt/client/api/query_tracker_client.h +++ b/yt/yt/client/api/query_tracker_client.h @@ -62,7 +62,7 @@ struct TReadQueryResultOptions : public TTimeoutOptions , public TQueryTrackerOptions { - std::optional<std::vector<TString>> Columns; + std::optional<std::vector<std::string>> Columns; std::optional<i64> LowerRowIndex; std::optional<i64> UpperRowIndex; }; diff --git a/yt/yt/client/api/rpc_proxy/client_impl.cpp b/yt/yt/client/api/rpc_proxy/client_impl.cpp index cc9694ca70..7575886a68 100644 --- a/yt/yt/client/api/rpc_proxy/client_impl.cpp +++ b/yt/yt/client/api/rpc_proxy/client_impl.cpp @@ -10,47 +10,34 @@ #include "timestamp_provider.h" #include "transaction.h" -#include <yt/yt/client/api/distributed_table_session.h> #include <yt/yt/client/api/helpers.h> -#include <yt/yt/client/api/rowset.h> -#include <yt/yt/client/api/transaction.h> #include <yt/yt/client/chaos_client/replication_card_serialization.h> -#include <yt/yt/client/transaction_client/remote_timestamp_provider.h> - #include <yt/yt/client/scheduler/operation_id_or_alias.h> #include <yt/yt/client/table_client/columnar_statistics.h> -#include <yt/yt/client/table_client/name_table.h> -#include <yt/yt/client/table_client/row_base.h> -#include <yt/yt/client/table_client/row_buffer.h> #include <yt/yt/client/table_client/schema.h> #include <yt/yt/client/table_client/unversioned_row.h> #include <yt/yt/client/table_client/wire_protocol.h> -#include <yt/yt/client/tablet_client/table_mount_cache.h> +#include <yt/yt/client/api/distributed_table_session.h> #include <yt/yt/client/ypath/rich.h> #include <yt/yt/library/auth/credentials_injecting_channel.h> -#include <yt/yt/core/net/address.h> - -#include <yt/yt/core/rpc/dynamic_channel_pool.h> #include <yt/yt/core/rpc/retrying_channel.h> #include <yt/yt/core/rpc/stream.h> #include <yt/yt/core/ytree/convert.h> -#include <util/generic/cast.h> - namespace NYT::NApi::NRpcProxy { //////////////////////////////////////////////////////////////////////////////// -using NYT::ToProto; using NYT::FromProto; +using NYT::ToProto; using namespace NAuth; using namespace NChaosClient; @@ -2323,7 +2310,7 @@ TFuture<IUnversionedRowsetPtr> TClient::ReadQueryResult( if (options.Columns) { auto* protoColumns = req->mutable_columns(); for (const auto& column : *options.Columns) { - protoColumns->add_items(column); + protoColumns->add_items(ToProto(column)); } } if (options.LowerRowIndex) { @@ -2733,7 +2720,7 @@ TFuture<IRowBatchReaderPtr> TClient::CreateShuffleReader( TFuture<IRowBatchWriterPtr> TClient::CreateShuffleWriter( const TShuffleHandlePtr& shuffleHandle, - const TString& partitionColumn, + const std::string& partitionColumn, const TTableWriterConfigPtr& config) { auto proxy = CreateApiServiceProxy(); @@ -2741,7 +2728,7 @@ TFuture<IRowBatchWriterPtr> TClient::CreateShuffleWriter( InitStreamingRequest(*req); req->set_shuffle_handle(ConvertToYsonString(shuffleHandle).ToString()); - req->set_partition_column(partitionColumn); + req->set_partition_column(ToProto(partitionColumn)); req->set_writer_config(ConvertToYsonString(config).ToString()); return CreateRpcClientOutputStream(std::move(req)) diff --git a/yt/yt/client/api/rpc_proxy/client_impl.h b/yt/yt/client/api/rpc_proxy/client_impl.h index 16e9400391..3e73bc47f0 100644 --- a/yt/yt/client/api/rpc_proxy/client_impl.h +++ b/yt/yt/client/api/rpc_proxy/client_impl.h @@ -585,7 +585,7 @@ public: TFuture<IRowBatchWriterPtr> CreateShuffleWriter( const TShuffleHandlePtr& shuffleHandle, - const TString& partitionColumn, + const std::string& partitionColumn, const NTableClient::TTableWriterConfigPtr& config) override; private: diff --git a/yt/yt/client/api/rpc_proxy/helpers.cpp b/yt/yt/client/api/rpc_proxy/helpers.cpp index 13c51ed862..d37fc8b2bd 100644 --- a/yt/yt/client/api/rpc_proxy/helpers.cpp +++ b/yt/yt/client/api/rpc_proxy/helpers.cpp @@ -468,13 +468,13 @@ void FromProto( void ToProto(NProto::TColumnSchema* protoSchema, const NTableClient::TColumnSchema& schema) { - protoSchema->set_stable_name(schema.StableName().Underlying()); - protoSchema->set_name(schema.Name()); + protoSchema->set_stable_name(ToProto(schema.StableName())); + protoSchema->set_name(ToProto(schema.Name())); protoSchema->set_type(ToProto(GetPhysicalType(schema.CastToV1Type()))); auto typeV3Yson = ConvertToYsonString(TTypeV3LogicalTypeWrapper{schema.LogicalType()}); protoSchema->set_type_v3(typeV3Yson.ToString()); if (schema.Lock()) { - protoSchema->set_lock(*schema.Lock()); + protoSchema->set_lock(ToProto(*schema.Lock())); } else { protoSchema->clear_lock(); } @@ -489,7 +489,7 @@ void ToProto(NProto::TColumnSchema* protoSchema, const NTableClient::TColumnSche protoSchema->clear_materialized(); } if (schema.Aggregate()) { - protoSchema->set_aggregate(*schema.Aggregate()); + protoSchema->set_aggregate(ToProto(*schema.Aggregate())); } else { protoSchema->clear_aggregate(); } @@ -499,7 +499,7 @@ void ToProto(NProto::TColumnSchema* protoSchema, const NTableClient::TColumnSche protoSchema->clear_sort_order(); } if (schema.Group()) { - protoSchema->set_group(*schema.Group()); + protoSchema->set_group(ToProto(*schema.Group())); } else { protoSchema->clear_group(); } @@ -2158,7 +2158,7 @@ std::vector<TSharedRef> SerializeRowset( // COMPAT(babenko) for (const auto& column : schema.Columns()) { auto* entry = descriptor->add_name_table_entries(); - entry->set_name(column.Name()); + entry->set_name(ToProto(column.Name())); // we save physical type for backward compatibility // COMPAT(babenko) entry->set_type(ToProto(column.GetWireType())); diff --git a/yt/yt/client/api/rpc_proxy/table_reader.cpp b/yt/yt/client/api/rpc_proxy/table_reader.cpp index bb1f571332..6df6bef241 100644 --- a/yt/yt/client/api/rpc_proxy/table_reader.cpp +++ b/yt/yt/client/api/rpc_proxy/table_reader.cpp @@ -22,7 +22,7 @@ public: TTableReader( IAsyncZeroCopyInputStreamPtr underlying, i64 startRowIndex, - const std::vector<TString>& omittedInaccessibleColumns, + const std::vector<std::string>& omittedInaccessibleColumns, TTableSchemaPtr schema, const NProto::TRowsetStatistics& statistics) : TRowBatchReader(std::move(underlying), /*isStreamWithStatistics*/ true) @@ -57,7 +57,7 @@ public: return TableSchema_; } - const std::vector<TString>& GetOmittedInaccessibleColumns() const override + const std::vector<std::string>& GetOmittedInaccessibleColumns() const override { return OmittedInaccessibleColumns_; } @@ -65,7 +65,7 @@ public: private: const i64 StartRowIndex_; const TTableSchemaPtr TableSchema_; - const std::vector<TString> OmittedInaccessibleColumns_; + const std::vector<std::string> OmittedInaccessibleColumns_; NChunkClient::NProto::TDataStatistics DataStatistics_; i64 TotalRowCount_; @@ -85,15 +85,11 @@ TFuture<ITableReaderPtr> CreateTableReader(IAsyncZeroCopyInputStreamPtr inputStr THROW_ERROR_EXCEPTION("Failed to deserialize table reader meta information"); } - i64 startRowIndex = meta.start_row_index(); - auto omittedInaccessibleColumns = FromProto<std::vector<TString>>( - meta.omitted_inaccessible_columns()); - auto schema = NYT::FromProto<TTableSchemaPtr>(meta.schema()); return New<TTableReader>( inputStream, - startRowIndex, - std::move(omittedInaccessibleColumns), - std::move(schema), + meta.start_row_index(), + FromProto<std::vector<std::string>>(meta.omitted_inaccessible_columns()), + NYT::FromProto<TTableSchemaPtr>(meta.schema()), meta.statistics()); })).As<ITableReaderPtr>(); } diff --git a/yt/yt/client/api/security_client.cpp b/yt/yt/client/api/security_client.cpp index 2b0d0f675c..f30a288614 100644 --- a/yt/yt/client/api/security_client.cpp +++ b/yt/yt/client/api/security_client.cpp @@ -9,7 +9,7 @@ using namespace NYTree; TError TCheckPermissionResult::ToError( const std::string& user, EPermission permission, - const std::optional<TString>& column) const + const std::optional<std::string>& column) const { switch (Action) { case NSecurityClient::ESecurityAction::Allow: diff --git a/yt/yt/client/api/security_client.h b/yt/yt/client/api/security_client.h index a606229e49..cb639d678b 100644 --- a/yt/yt/client/api/security_client.h +++ b/yt/yt/client/api/security_client.h @@ -26,7 +26,7 @@ struct TCheckPermissionOptions , public TTransactionalOptions , public TPrerequisiteOptions { - std::optional<std::vector<TString>> Columns; + std::optional<std::vector<std::string>> Columns; std::optional<bool> Vital; }; @@ -35,7 +35,7 @@ struct TCheckPermissionResult TError ToError( const std::string& user, NYTree::EPermission permission, - const std::optional<TString>& columns = {}) const; + const std::optional<std::string>& columns = {}) const; NSecurityClient::ESecurityAction Action; NObjectClient::TObjectId ObjectId; diff --git a/yt/yt/client/api/shuffle_client.h b/yt/yt/client/api/shuffle_client.h index d7972cd680..650096a30c 100644 --- a/yt/yt/client/api/shuffle_client.h +++ b/yt/yt/client/api/shuffle_client.h @@ -53,7 +53,7 @@ struct IShuffleClient virtual TFuture<IRowBatchWriterPtr> CreateShuffleWriter( const TShuffleHandlePtr& shuffleHandle, - const TString& partitionColumn, + const std::string& partitionColumn, const NTableClient::TTableWriterConfigPtr& config = New<NTableClient::TTableWriterConfig>()) = 0; }; diff --git a/yt/yt/client/api/table_reader.h b/yt/yt/client/api/table_reader.h index b26e970657..8a447ea5ed 100644 --- a/yt/yt/client/api/table_reader.h +++ b/yt/yt/client/api/table_reader.h @@ -26,7 +26,7 @@ struct ITableReader //! Returns the names of columns that are not accessible according to columnar ACL //! and were omitted. See #TTableReaderOptions::OmitInaccessibleColumns. - virtual const std::vector<TString>& GetOmittedInaccessibleColumns() const = 0; + virtual const std::vector<std::string>& GetOmittedInaccessibleColumns() const = 0; }; DEFINE_REFCOUNTED_TYPE(ITableReader) diff --git a/yt/yt/client/arrow/arrow_row_stream_encoder.cpp b/yt/yt/client/arrow/arrow_row_stream_encoder.cpp index 23a06a45b7..25a403790c 100644 --- a/yt/yt/client/arrow/arrow_row_stream_encoder.cpp +++ b/yt/yt/client/arrow/arrow_row_stream_encoder.cpp @@ -39,7 +39,7 @@ constexpr i64 ArrowAlignment = 8; flatbuffers::Offset<flatbuffers::String> SerializeString( flatbuffers::FlatBufferBuilder* flatbufBuilder, - const TString& str) + const std::string& str) { return flatbufBuilder->CreateString(str.data(), str.length()); } diff --git a/yt/yt/client/driver/driver.cpp b/yt/yt/client/driver/driver.cpp index 3e57e388df..4de907381d 100644 --- a/yt/yt/client/driver/driver.cpp +++ b/yt/yt/client/driver/driver.cpp @@ -17,25 +17,20 @@ #include "query_commands.h" #include "queue_commands.h" #include "scheduler_commands.h" +#include "shuffle_commands.h" #include "table_commands.h" #include "transaction_commands.h" -#include "shuffle_commands.h" + +#include <yt/yt/library/formats/format.h> #include <yt/yt/client/api/client_cache.h> #include <yt/yt/client/api/connection.h> #include <yt/yt/client/api/sticky_transaction_pool.h> -#include <yt/yt/client/api/transaction.h> - -#include <yt/yt/client/api/rpc_proxy/connection_impl.h> - -#include <yt/yt/client/node_tracker_client/node_directory.h> #include <yt/yt/core/yson/null_consumer.h> #include <yt/yt/core/tracing/trace_context.h> -#include <yt/yt/library/formats/format.h> - #include <yt/yt/library/tvm/tvm_base.h> diff --git a/yt/yt/client/driver/etc_commands.cpp b/yt/yt/client/driver/etc_commands.cpp index 25da308a57..288d42610f 100644 --- a/yt/yt/client/driver/etc_commands.cpp +++ b/yt/yt/client/driver/etc_commands.cpp @@ -109,7 +109,7 @@ void TCheckPermissionCommand::Register(TRegistrar registrar) registrar.Parameter("user", &TThis::User); registrar.Parameter("permission", &TThis::Permission); registrar.Parameter("path", &TThis::Path); - registrar.ParameterWithUniversalAccessor<std::optional<std::vector<TString>>>( + registrar.ParameterWithUniversalAccessor<std::optional<std::vector<std::string>>>( "columns", [] (TThis* command) -> auto& { return command->Options.Columns; diff --git a/yt/yt/client/driver/query_commands.cpp b/yt/yt/client/driver/query_commands.cpp index e3903d004b..ab2d9395e1 100644 --- a/yt/yt/client/driver/query_commands.cpp +++ b/yt/yt/client/driver/query_commands.cpp @@ -153,7 +153,7 @@ void TReadQueryResultCommand::Register(TRegistrar registrar) }) .Optional(/*init*/ false); - registrar.ParameterWithUniversalAccessor<std::optional<std::vector<TString>>>( + registrar.ParameterWithUniversalAccessor<std::optional<std::vector<std::string>>>( "columns", [] (TThis* command) -> auto& { return command->Options.Columns; diff --git a/yt/yt/client/driver/shuffle_commands.cpp b/yt/yt/client/driver/shuffle_commands.cpp index 4e2ee805e8..ae8e5adbcd 100644 --- a/yt/yt/client/driver/shuffle_commands.cpp +++ b/yt/yt/client/driver/shuffle_commands.cpp @@ -1,13 +1,15 @@ #include "shuffle_commands.h" +#include <yt/yt/client/driver/config.h> + +#include <yt/yt/library/formats/format.h> + +#include <yt/yt/client/formats/config.h> + #include <yt/yt/client/table_client/adapters.h> #include <yt/yt/client/table_client/table_output.h> #include <yt/yt/client/table_client/value_consumer.h> -#include <yt/yt/client/driver/config.h> -#include <yt/yt/client/formats/config.h> - -#include <yt/yt/library/formats/format.h> namespace NYT::NDriver { diff --git a/yt/yt/client/driver/shuffle_commands.h b/yt/yt/client/driver/shuffle_commands.h index 7b4394c49a..1eebb8304a 100644 --- a/yt/yt/client/driver/shuffle_commands.h +++ b/yt/yt/client/driver/shuffle_commands.h @@ -2,8 +2,6 @@ #include "command.h" -#include <yt/yt/client/ypath/rich.h> - namespace NYT::NDriver { //////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/client/driver/table_commands.h b/yt/yt/client/driver/table_commands.h index 2e1e8e22d5..fde7e0b66a 100644 --- a/yt/yt/client/driver/table_commands.h +++ b/yt/yt/client/driver/table_commands.h @@ -46,8 +46,8 @@ private: NYPath::TRichYPath Path; NYTree::INodePtr TableReader; - std::optional<TString> PartIndexColumnName; - std::optional<TString> DataColumnName; + std::optional<std::string> PartIndexColumnName; + std::optional<std::string> DataColumnName; i64 StartPartIndex; i64 Offset; @@ -375,7 +375,7 @@ public: private: NYTree::INodePtr TableWriter; NYPath::TRichYPath Path; - std::optional<std::vector<TString>> ColumnNames; + std::optional<std::vector<std::string>> ColumnNames; bool Versioned; NTableClient::TRetentionConfigPtr RetentionConfig; @@ -451,7 +451,7 @@ public: private: NYTree::INodePtr TableWriter; NYPath::TRichYPath Path; - std::vector<TString> Locks; + std::vector<std::string> Locks; NTableClient::ELockType LockType; void DoExecute(ICommandContextPtr context) override; diff --git a/yt/yt/client/federated/client.cpp b/yt/yt/client/federated/client.cpp index f43f76958d..040c19a17c 100644 --- a/yt/yt/client/federated/client.cpp +++ b/yt/yt/client/federated/client.cpp @@ -490,7 +490,7 @@ public: UNIMPLEMENTED_METHOD(TFuture<TShuffleHandlePtr>, StartShuffle, (const TString& , int, const TStartShuffleOptions&)); UNIMPLEMENTED_METHOD(TFuture<void>, FinishShuffle, (const TShuffleHandlePtr&, const TFinishShuffleOptions&)); UNIMPLEMENTED_METHOD(TFuture<IRowBatchReaderPtr>, CreateShuffleReader, (const TShuffleHandlePtr&, int, const NTableClient::TTableReaderConfigPtr&)); - UNIMPLEMENTED_METHOD(TFuture<IRowBatchWriterPtr>, CreateShuffleWriter, (const TShuffleHandlePtr&, const TString&, const NTableClient::TTableWriterConfigPtr&)); + UNIMPLEMENTED_METHOD(TFuture<IRowBatchWriterPtr>, CreateShuffleWriter, (const TShuffleHandlePtr&, const std::string&, const NTableClient::TTableWriterConfigPtr&)); private: friend class TTransaction; diff --git a/yt/yt/client/federated/unittests/client_ut.cpp b/yt/yt/client/federated/unittests/client_ut.cpp index 54ec11d6cb..daf7529e78 100644 --- a/yt/yt/client/federated/unittests/client_ut.cpp +++ b/yt/yt/client/federated/unittests/client_ut.cpp @@ -78,8 +78,8 @@ struct TTestDataStorage } const NYPath::TYPath Path = "/test/table"; - const TString KeyColumn = "key"; - const TString ValueColumn = "value"; + const std::string KeyColumn = "key"; + const std::string ValueColumn = "value"; const NTableClient::TColumnSchema KeyColumnSchema = NTableClient::TColumnSchema(KeyColumn, NTableClient::EValueType::Uint64); const NTableClient::TColumnSchema ValueColumnSchema = NTableClient::TColumnSchema(ValueColumn, NTableClient::EValueType::Uint64); diff --git a/yt/yt/client/formats/config.cpp b/yt/yt/client/formats/config.cpp index f6c6a8c8a6..71c1152061 100644 --- a/yt/yt/client/formats/config.cpp +++ b/yt/yt/client/formats/config.cpp @@ -131,7 +131,7 @@ void TYamredDsvFormatConfig::Register(TRegistrar registrar) }); registrar.Postprocessor([] (TThis* config) { - THashSet<TString> names; + THashSet<std::string> names; for (const auto& name : config->KeyColumnNames) { if (!names.insert(name).second) { @@ -151,7 +151,7 @@ void TYamredDsvFormatConfig::Register(TRegistrar registrar) //////////////////////////////////////////////////////////////////////////////// -const std::vector<TString>& TSchemafulDsvFormatConfig::GetColumnsOrThrow() const +const std::vector<std::string>& TSchemafulDsvFormatConfig::GetColumnsOrThrow() const { if (!Columns) { THROW_ERROR_EXCEPTION("Missing \"columns\" attribute in schemaful DSV format"); @@ -188,7 +188,7 @@ void TSchemafulDsvFormatConfig::Register(TRegistrar registrar) registrar.Postprocessor([] (TThis* config) { if (config->Columns) { - THashSet<TString> names; + THashSet<std::string> names; for (const auto& name : *config->Columns) { if (!names.insert(name).second) { THROW_ERROR_EXCEPTION("Duplicate column name %Qv in schemaful DSV configuration", diff --git a/yt/yt/client/formats/config.h b/yt/yt/client/formats/config.h index a371050ae6..aec3f30db4 100644 --- a/yt/yt/client/formats/config.h +++ b/yt/yt/client/formats/config.h @@ -149,8 +149,7 @@ class TDsvFormatConfig : public TDsvFormatConfigBase { public: - - TString TableIndexColumn; + std::string TableIndexColumn; bool SkipUnsupportedTypes = false; REGISTER_YSON_STRUCT(TDsvFormatConfig); @@ -169,8 +168,8 @@ class TYamredDsvFormatConfig public: char YamrKeysSeparator; - std::vector<TString> KeyColumnNames; - std::vector<TString> SubkeyColumnNames; + std::vector<std::string> KeyColumnNames; + std::vector<std::string> SubkeyColumnNames; bool SkipUnsupportedTypesInValue = false; @@ -193,14 +192,14 @@ class TSchemafulDsvFormatConfig : public TTableFormatConfigBase { public: - std::optional<std::vector<TString>> Columns; + std::optional<std::vector<std::string>> Columns; EMissingSchemafulDsvValueMode MissingValueMode; TString MissingValueSentinel; std::optional<bool> EnableColumnNamesHeader; - const std::vector<TString>& GetColumnsOrThrow() const; + const std::vector<std::string>& GetColumnsOrThrow() const; REGISTER_YSON_STRUCT(TSchemafulDsvFormatConfig); @@ -382,7 +381,7 @@ public: int FieldWeightLimit; int StringWeightLimit; int MaxAllColumnNamesCount; - std::optional<std::vector<TString>> ColumnNames; + std::optional<std::vector<std::string>> ColumnNames; EWebJsonValueFormat ValueFormat; // Intentionally do not reveal following options to user. diff --git a/yt/yt/client/hedging/hedging.cpp b/yt/yt/client/hedging/hedging.cpp index 3fb36f8560..b083f87ff3 100644 --- a/yt/yt/client/hedging/hedging.cpp +++ b/yt/yt/client/hedging/hedging.cpp @@ -231,7 +231,7 @@ public: UNSUPPORTED_METHOD(TFuture<TShuffleHandlePtr>, StartShuffle, (const TString&, int, const TStartShuffleOptions&)); UNSUPPORTED_METHOD(TFuture<void>, FinishShuffle, (const TShuffleHandlePtr&, const TFinishShuffleOptions&)); UNSUPPORTED_METHOD(TFuture<IRowBatchReaderPtr>, CreateShuffleReader, (const TShuffleHandlePtr&, int, const NTableClient::TTableReaderConfigPtr&)); - UNSUPPORTED_METHOD(TFuture<IRowBatchWriterPtr>, CreateShuffleWriter, (const TShuffleHandlePtr&, const TString&, const NTableClient::TTableWriterConfigPtr&)); + UNSUPPORTED_METHOD(TFuture<IRowBatchWriterPtr>, CreateShuffleWriter, (const TShuffleHandlePtr&, const std::string&, const NTableClient::TTableWriterConfigPtr&)); private: const THedgingExecutorPtr Executor_; diff --git a/yt/yt/client/table_client/blob_reader.cpp b/yt/yt/client/table_client/blob_reader.cpp index e354c9008b..fb4fc93a17 100644 --- a/yt/yt/client/table_client/blob_reader.cpp +++ b/yt/yt/client/table_client/blob_reader.cpp @@ -15,8 +15,8 @@ using namespace NConcurrency; //////////////////////////////////////////////////////////////////////////////// -const TString TBlobTableSchema::PartIndexColumn = "part_index"; -const TString TBlobTableSchema::DataColumn = "data"; +const std::string TBlobTableSchema::PartIndexColumn = "part_index"; +const std::string TBlobTableSchema::DataColumn = "data"; TTableSchemaPtr TBlobTableSchema::ToTableSchema() const { @@ -46,8 +46,8 @@ class TBlobTableReader public: TBlobTableReader( ITableReaderPtr reader, - const std::optional<TString>& partIndexColumnName, - const std::optional<TString>& dataColumnName, + const std::optional<std::string>& partIndexColumnName, + const std::optional<std::string>& dataColumnName, i64 startPartIndex, const std::optional<i64>& offset, const std::optional<i64>& partSize) @@ -87,8 +87,8 @@ public: private: const ITableReaderPtr Reader_; - const TString PartIndexColumnName_; - const TString DataColumnName_; + const std::string PartIndexColumnName_; + const std::string DataColumnName_; i64 Offset_; std::optional<i64> PartSize_; @@ -122,7 +122,7 @@ private: TUnversionedValue GetAndValidateValue( TUnversionedRow row, - const TString& name, + const std::string& name, EColumnType columnType, EValueType expectedType) { @@ -197,8 +197,8 @@ private: IAsyncZeroCopyInputStreamPtr CreateBlobTableReader( ITableReaderPtr reader, - const std::optional<TString>& partIndexColumnName, - const std::optional<TString>& dataColumnName, + const std::optional<std::string>& partIndexColumnName, + const std::optional<std::string>& dataColumnName, i64 startPartIndex, const std::optional<i64>& offset, const std::optional<i64>& partSize) diff --git a/yt/yt/client/table_client/blob_reader.h b/yt/yt/client/table_client/blob_reader.h index 71f8810341..0bfe4579a9 100644 --- a/yt/yt/client/table_client/blob_reader.h +++ b/yt/yt/client/table_client/blob_reader.h @@ -11,8 +11,8 @@ namespace NYT::NTableClient { struct TBlobTableSchema { // Names of special blob columns. - static const TString PartIndexColumn; - static const TString DataColumn; + static const std::string PartIndexColumn; + static const std::string DataColumn; // Do not specify anything except name and value // type in all column schemas. @@ -25,8 +25,8 @@ struct TBlobTableSchema NConcurrency::IAsyncZeroCopyInputStreamPtr CreateBlobTableReader( NApi::ITableReaderPtr reader, - const std::optional<TString>& partIndexColumnName, - const std::optional<TString>& dataColumnName, + const std::optional<std::string>& partIndexColumnName, + const std::optional<std::string>& dataColumnName, i64 startPartIndex, const std::optional<i64>& offset = std::nullopt, const std::optional<i64>& partSize = std::nullopt); diff --git a/yt/yt/client/table_client/column_sort_schema.cpp b/yt/yt/client/table_client/column_sort_schema.cpp index 964f7549a4..e574bd6cb2 100644 --- a/yt/yt/client/table_client/column_sort_schema.cpp +++ b/yt/yt/client/table_client/column_sort_schema.cpp @@ -62,7 +62,7 @@ void ValidateSortColumns(const std::vector<TColumnSortSchema>& columns) { ValidateKeyColumnCount(columns.size()); - THashSet<TString> names; + THashSet<std::string> names; for (const auto& column : columns) { if (!names.insert(column.Name).second) { THROW_ERROR_EXCEPTION("Duplicate sort column name %Qv", @@ -77,9 +77,11 @@ void ToProto( NProto::TSortColumnsExt* protoSortColumns, const TSortColumns& sortColumns) { + using NYT::ToProto; + for (const auto& sortColumn : sortColumns) { - protoSortColumns->add_names(sortColumn.Name); - protoSortColumns->add_sort_orders(static_cast<int>(sortColumn.SortOrder)); + protoSortColumns->add_names(ToProto(sortColumn.Name)); + protoSortColumns->add_sort_orders(ToProto<int>(sortColumn.SortOrder)); } } diff --git a/yt/yt/client/table_client/column_sort_schema.h b/yt/yt/client/table_client/column_sort_schema.h index 70a8b62596..21579e98df 100644 --- a/yt/yt/client/table_client/column_sort_schema.h +++ b/yt/yt/client/table_client/column_sort_schema.h @@ -9,7 +9,7 @@ namespace NYT::NTableClient { struct TColumnSortSchema { - TString Name; + std::string Name; ESortOrder SortOrder; bool operator==(const TColumnSortSchema& other) const = default; diff --git a/yt/yt/client/table_client/columnar_statistics.cpp b/yt/yt/client/table_client/columnar_statistics.cpp index 85a223a8a4..c53acdfbf5 100644 --- a/yt/yt/client/table_client/columnar_statistics.cpp +++ b/yt/yt/client/table_client/columnar_statistics.cpp @@ -265,7 +265,7 @@ TLightweightColumnarStatistics TColumnarStatistics::MakeLightweightStatistics() }; } -TNamedColumnarStatistics TColumnarStatistics::MakeNamedStatistics(const std::vector<TString>& names) const +TNamedColumnarStatistics TColumnarStatistics::MakeNamedStatistics(const std::vector<std::string>& names) const { TNamedColumnarStatistics result; result.TimestampTotalWeight = TimestampTotalWeight; diff --git a/yt/yt/client/table_client/columnar_statistics.h b/yt/yt/client/table_client/columnar_statistics.h index 82dbec47c0..0ead116aff 100644 --- a/yt/yt/client/table_client/columnar_statistics.h +++ b/yt/yt/client/table_client/columnar_statistics.h @@ -21,7 +21,7 @@ struct TLightweightColumnarStatistics struct TNamedColumnarStatistics { //! Per-column total data weight for chunks whose meta contains columnar statistics. - THashMap<TString, i64> ColumnDataWeights; + THashMap<std::string, i64> ColumnDataWeights; //! Total weight of all write and delete timestamps. std::optional<i64> TimestampTotalWeight; //! Total data weight of legacy chunks whose meta misses columnar statistics. @@ -97,7 +97,7 @@ struct TColumnarStatistics TLightweightColumnarStatistics MakeLightweightStatistics() const; - TNamedColumnarStatistics MakeNamedStatistics(const std::vector<TString>& names) const; + TNamedColumnarStatistics MakeNamedStatistics(const std::vector<std::string>& names) const; //! Checks if there are minimum, maximum, and non-null value statistics. bool HasValueStatistics() const; diff --git a/yt/yt/client/table_client/logical_type.cpp b/yt/yt/client/table_client/logical_type.cpp index fda90c21ec..679540a999 100644 --- a/yt/yt/client/table_client/logical_type.cpp +++ b/yt/yt/client/table_client/logical_type.cpp @@ -467,19 +467,19 @@ TComplexTypeFieldDescriptor::TComplexTypeFieldDescriptor(const NYT::NTableClient : TComplexTypeFieldDescriptor(column.Name(), column.LogicalType()) { } -TComplexTypeFieldDescriptor::TComplexTypeFieldDescriptor(TString columnName, TLogicalTypePtr type) - : Descriptor_(std::move(columnName)) +TComplexTypeFieldDescriptor::TComplexTypeFieldDescriptor(const std::string& columnName, TLogicalTypePtr type) + : Description_(columnName) , Type_(std::move(type)) { } TComplexTypeFieldDescriptor TComplexTypeFieldDescriptor::OptionalElement() const { - return TComplexTypeFieldDescriptor(Descriptor_ + ".<optional-element>", Type_->AsOptionalTypeRef().GetElement()); + return TComplexTypeFieldDescriptor(Description_ + ".<optional-element>", Type_->AsOptionalTypeRef().GetElement()); } TComplexTypeFieldDescriptor TComplexTypeFieldDescriptor::ListElement() const { - return TComplexTypeFieldDescriptor(Descriptor_ + ".<list-element>", Type_->AsListTypeRef().GetElement()); + return TComplexTypeFieldDescriptor(Description_ + ".<list-element>", Type_->AsListTypeRef().GetElement()); } TComplexTypeFieldDescriptor TComplexTypeFieldDescriptor::Field(size_t i) const @@ -499,7 +499,7 @@ TComplexTypeFieldDescriptor TComplexTypeFieldDescriptor::StructField(size_t i) c const auto& fields = Type_->AsStructTypeRef().GetFields(); YT_VERIFY(i < fields.size()); const auto& field = fields[i]; - return TComplexTypeFieldDescriptor(Descriptor_ + "." + field.Name, field.Type); + return TComplexTypeFieldDescriptor(Description_ + "." + field.Name, field.Type); } TComplexTypeFieldDescriptor TComplexTypeFieldDescriptor::Element(size_t i) const @@ -518,14 +518,14 @@ TComplexTypeFieldDescriptor TComplexTypeFieldDescriptor::TupleElement(size_t i) { const auto& elements = Type_->AsTupleTypeRef().GetElements(); YT_VERIFY(i < elements.size()); - return TComplexTypeFieldDescriptor(Descriptor_ + Format(".<tuple-element-%v>", i), elements[i]); + return TComplexTypeFieldDescriptor(Description_ + Format(".<tuple-element-%v>", i), elements[i]); } TComplexTypeFieldDescriptor TComplexTypeFieldDescriptor::VariantTupleElement(size_t i) const { const auto& elements = Type_->AsVariantTupleTypeRef().GetElements(); YT_VERIFY(i < elements.size()); - return TComplexTypeFieldDescriptor(Descriptor_ + Format(".<variant-element-%v>", i), elements[i]); + return TComplexTypeFieldDescriptor(Description_ + Format(".<variant-element-%v>", i), elements[i]); } TComplexTypeFieldDescriptor TComplexTypeFieldDescriptor::VariantStructField(size_t i) const @@ -533,32 +533,32 @@ TComplexTypeFieldDescriptor TComplexTypeFieldDescriptor::VariantStructField(size const auto& fields = Type_->AsVariantStructTypeRef().GetFields(); YT_VERIFY(i < fields.size()); const auto& field = fields[i]; - return TComplexTypeFieldDescriptor(Descriptor_ + "." + field.Name, field.Type); + return TComplexTypeFieldDescriptor(Description_ + "." + field.Name, field.Type); } TComplexTypeFieldDescriptor TComplexTypeFieldDescriptor::DictKey() const { - return TComplexTypeFieldDescriptor(Descriptor_ + ".<key>", Type_->AsDictTypeRef().GetKey()); + return TComplexTypeFieldDescriptor(Description_ + ".<key>", Type_->AsDictTypeRef().GetKey()); } TComplexTypeFieldDescriptor TComplexTypeFieldDescriptor::DictValue() const { - return TComplexTypeFieldDescriptor(Descriptor_ + ".<value>", Type_->AsDictTypeRef().GetValue()); + return TComplexTypeFieldDescriptor(Description_ + ".<value>", Type_->AsDictTypeRef().GetValue()); } TComplexTypeFieldDescriptor TComplexTypeFieldDescriptor::TaggedElement() const { - return TComplexTypeFieldDescriptor(Descriptor_ + ".<tagged-element>", Type_->AsTaggedTypeRef().GetElement()); + return TComplexTypeFieldDescriptor(Description_ + ".<tagged-element>", Type_->AsTaggedTypeRef().GetElement()); } TComplexTypeFieldDescriptor TComplexTypeFieldDescriptor::Detag() const { - return TComplexTypeFieldDescriptor(Descriptor_, DetagLogicalType(Type_)); + return TComplexTypeFieldDescriptor(Description_, DetagLogicalType(Type_)); } -const TString& TComplexTypeFieldDescriptor::GetDescription() const +const std::string& TComplexTypeFieldDescriptor::GetDescription() const { - return Descriptor_; + return Description_; } const TLogicalTypePtr& TComplexTypeFieldDescriptor::GetType() const @@ -1053,6 +1053,8 @@ void ValidateLogicalType(const TComplexTypeFieldDescriptor& rootDescriptor, std: void ToProto(NProto::TLogicalType* protoLogicalType, const TLogicalTypePtr& logicalType) { + using NYT::ToProto; + switch (logicalType->GetMetatype()) { case ELogicalMetatype::Simple: protoLogicalType->set_simple(ToProto(logicalType->AsSimpleTypeRef().GetElement())); @@ -1071,7 +1073,7 @@ void ToProto(NProto::TLogicalType* protoLogicalType, const TLogicalTypePtr& logi auto protoStruct = protoLogicalType->mutable_struct_(); for (const auto& structField : logicalType->AsStructTypeRef().GetFields()) { auto protoStructField = protoStruct->add_fields(); - protoStructField->set_name(structField.Name); + protoStructField->set_name(ToProto(structField.Name)); ToProto(protoStructField->mutable_type(), structField.Type); } return; @@ -1088,7 +1090,7 @@ void ToProto(NProto::TLogicalType* protoLogicalType, const TLogicalTypePtr& logi auto protoVariantStruct = protoLogicalType->mutable_variant_struct(); for (const auto& field : logicalType->AsVariantStructTypeRef().GetFields()) { auto protoField = protoVariantStruct->add_fields(); - protoField->set_name(field.Name); + protoField->set_name(ToProto(field.Name)); ToProto(protoField->mutable_type(), field.Type); } return; diff --git a/yt/yt/client/table_client/logical_type.h b/yt/yt/client/table_client/logical_type.h index bd482df984..1a0b2dd0b5 100644 --- a/yt/yt/client/table_client/logical_type.h +++ b/yt/yt/client/table_client/logical_type.h @@ -260,7 +260,7 @@ class TComplexTypeFieldDescriptor public: explicit TComplexTypeFieldDescriptor(TLogicalTypePtr type); explicit TComplexTypeFieldDescriptor(const TColumnSchema& column); - TComplexTypeFieldDescriptor(TString columnName, TLogicalTypePtr type); + TComplexTypeFieldDescriptor(const std::string& columnName, TLogicalTypePtr type); TComplexTypeFieldDescriptor OptionalElement() const; TComplexTypeFieldDescriptor ListElement() const; @@ -276,11 +276,11 @@ public: TComplexTypeFieldDescriptor Detag() const; - const TString& GetDescription() const; + const std::string& GetDescription() const; const TLogicalTypePtr& GetType() const; private: - TString Descriptor_; + std::string Description_; TLogicalTypePtr Type_; }; @@ -288,7 +288,7 @@ private: struct TStructField { - TString Name; + std::string Name; TLogicalTypePtr Type; }; @@ -299,8 +299,6 @@ class TStructLogicalTypeBase : public TLogicalType { public: - -public: TStructLogicalTypeBase(ELogicalMetatype metatype, std::vector<TStructField> fields); Y_FORCE_INLINE const std::vector<TStructField>& GetFields() const; diff --git a/yt/yt/client/table_client/name_table.cpp b/yt/yt/client/table_client/name_table.cpp index 2c22700fc7..6747c30539 100644 --- a/yt/yt/client/table_client/name_table.cpp +++ b/yt/yt/client/table_client/name_table.cpp @@ -167,10 +167,10 @@ int TNameTable::DoRegisterNameOrThrow(TStringBuf name) return DoRegisterName(name); } -std::vector<TString> TNameTable::GetNames() const +std::vector<std::string> TNameTable::GetNames() const { auto guard = Guard(SpinLock_); - std::vector<TString> result(IdToName_.begin(), IdToName_.end()); + std::vector<std::string> result(IdToName_.begin(), IdToName_.end()); return result; } @@ -255,7 +255,7 @@ std::optional<int> TNameTableWriter::FindId(TStringBuf name) const auto optionalId = NameTable_->FindId(name); if (optionalId) { - Names_.push_back(TString(name)); + Names_.push_back(std::string(name)); YT_VERIFY(NameToId_.emplace(Names_.back(), *optionalId).second); } return optionalId; @@ -278,7 +278,7 @@ int TNameTableWriter::GetIdOrRegisterName(TStringBuf name) } auto id = NameTable_->GetIdOrRegisterName(name); - Names_.push_back(TString(name)); + Names_.push_back(std::string(name)); YT_VERIFY(NameToId_.emplace(Names_.back(), id).second); return id; } @@ -296,7 +296,7 @@ void FromProto(TNameTablePtr* nameTable, const NProto::TNameTableExt& protoNameT { using NYT::FromProto; - *nameTable = TNameTable::FromKeyColumns(FromProto<std::vector<TString>>(protoNameTable.names())); + *nameTable = TNameTable::FromKeyColumns(FromProto<std::vector<std::string>>(protoNameTable.names())); } //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/client/table_client/name_table.h b/yt/yt/client/table_client/name_table.h index f0e94f3617..4d5b7ae404 100644 --- a/yt/yt/client/table_client/name_table.h +++ b/yt/yt/client/table_client/name_table.h @@ -37,7 +37,7 @@ public: TStringBuf GetName(int id) const; TStringBuf GetNameOrThrow(int id) const; - std::vector<TString> GetNames() const; + std::vector<std::string> GetNames() const; private: YT_DECLARE_SPIN_LOCK(NThreading::TSpinLock, SpinLock_); @@ -46,7 +46,7 @@ private: // String values are owned by IdToName_. // NB: Names may be SSO-strings, using a deque to avoid string view invalidation. - std::deque<TString> IdToName_; + std::deque<std::string> IdToName_; THashMap<TStringBuf, int> NameToId_; i64 ByteSize_ = 0; @@ -97,7 +97,7 @@ private: // String values are owned by Names_ // NB: Names may be SSO-strings, using a deque to avoid string view invalidation - mutable std::deque<TString> Names_; + mutable std::deque<std::string> Names_; mutable THashMap<TStringBuf, int> NameToId_; }; diff --git a/yt/yt/client/table_client/public.cpp b/yt/yt/client/table_client/public.cpp index 4fc79ad447..1008d882c4 100644 --- a/yt/yt/client/table_client/public.cpp +++ b/yt/yt/client/table_client/public.cpp @@ -4,19 +4,20 @@ namespace NYT::NTableClient { //////////////////////////////////////////////////////////////////////////////// -const TString PrimaryLockName("<primary>"); +const std::string PrimaryLockName("<primary>"); -const TString SystemColumnNamePrefix("$"); -const TString TableIndexColumnName = SystemColumnNamePrefix + "table_index"; -const TString RowIndexColumnName = SystemColumnNamePrefix + "row_index"; -const TString RangeIndexColumnName = SystemColumnNamePrefix + "range_index"; -const TString TabletIndexColumnName = SystemColumnNamePrefix + "tablet_index"; -const TString TimestampColumnName = SystemColumnNamePrefix + "timestamp"; -const TString TtlColumnName = SystemColumnNamePrefix + "ttl"; -const TString TimestampColumnPrefix = SystemColumnNamePrefix + "timestamp:"; -const TString CumulativeDataWeightColumnName = SystemColumnNamePrefix + "cumulative_data_weight"; -const TString EmptyValueColumnName = SystemColumnNamePrefix + "empty"; -const TString SequenceNumberColumnName = SystemColumnNamePrefix + "sequence_number"; +const std::string SystemColumnNamePrefix("$"); +const std::string NonexistentColumnName = SystemColumnNamePrefix + "__YT_NONEXISTENT_COLUMN_NAME__"; +const std::string TableIndexColumnName = SystemColumnNamePrefix + "table_index"; +const std::string RowIndexColumnName = SystemColumnNamePrefix + "row_index"; +const std::string RangeIndexColumnName = SystemColumnNamePrefix + "range_index"; +const std::string TabletIndexColumnName = SystemColumnNamePrefix + "tablet_index"; +const std::string TimestampColumnName = SystemColumnNamePrefix + "timestamp"; +const std::string TtlColumnName = SystemColumnNamePrefix + "ttl"; +const std::string TimestampColumnPrefix = SystemColumnNamePrefix + "timestamp:"; +const std::string CumulativeDataWeightColumnName = SystemColumnNamePrefix + "cumulative_data_weight"; +const std::string EmptyValueColumnName = SystemColumnNamePrefix + "empty"; +const std::string SequenceNumberColumnName = SystemColumnNamePrefix + "sequence_number"; //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/client/table_client/public.h b/yt/yt/client/table_client/public.h index e52eaecf48..1c795adb32 100644 --- a/yt/yt/client/table_client/public.h +++ b/yt/yt/client/table_client/public.h @@ -77,7 +77,7 @@ using NTransactionClient::AsyncLastCommittedTimestamp; using NTransactionClient::AllCommittedTimestamp; using NTransactionClient::NotPreparedTimestamp; -using TKeyColumns = std::vector<TString>; +using TKeyColumns = std::vector<std::string>; //////////////////////////////////////////////////////////////////////////////// @@ -119,20 +119,20 @@ constexpr int MaxColumnId = 32 * 1024; constexpr int MaxSchemaTotalTypeComplexity = MaxColumnId; constexpr int MaxSchemaDepth = 32; - -extern const TString PrimaryLockName; - -extern const TString SystemColumnNamePrefix; -extern const TString TableIndexColumnName; -extern const TString RowIndexColumnName; -extern const TString RangeIndexColumnName; -extern const TString TabletIndexColumnName; -extern const TString TimestampColumnName; -extern const TString TtlColumnName; -extern const TString TimestampColumnPrefix; -extern const TString CumulativeDataWeightColumnName; -extern const TString EmptyValueColumnName; -extern const TString SequenceNumberColumnName; +extern const std::string PrimaryLockName; + +extern const std::string SystemColumnNamePrefix; +extern const std::string NonexistentColumnName; +extern const std::string TableIndexColumnName; +extern const std::string RowIndexColumnName; +extern const std::string RangeIndexColumnName; +extern const std::string TabletIndexColumnName; +extern const std::string TimestampColumnName; +extern const std::string TtlColumnName; +extern const std::string TimestampColumnPrefix; +extern const std::string CumulativeDataWeightColumnName; +extern const std::string EmptyValueColumnName; +extern const std::string SequenceNumberColumnName; constexpr int TypicalHunkColumnCount = 8; @@ -308,7 +308,7 @@ class TKeyComparer; struct TColumnRenameDescriptor; using TColumnRenameDescriptors = std::vector<TColumnRenameDescriptor>; -YT_DEFINE_STRONG_TYPEDEF(TColumnStableName, TString); +YT_DEFINE_STRONG_TYPEDEF(TColumnStableName, std::string); class TColumnSchema; diff --git a/yt/yt/client/table_client/schema.cpp b/yt/yt/client/table_client/schema.cpp index 3d5a8b92ce..1acb033cde 100644 --- a/yt/yt/client/table_client/schema.cpp +++ b/yt/yt/client/table_client/schema.cpp @@ -148,33 +148,33 @@ void FromProto(TLockMask* lockMask, const NTabletClient::NProto::TLockMask& prot TColumnSchema::TColumnSchema() : TColumnSchema( - TString(), + std::string(), NullLogicalType(), std::nullopt) { } TColumnSchema::TColumnSchema( - TString name, + const std::string& name, EValueType type, std::optional<ESortOrder> sortOrder) : TColumnSchema( - std::move(name), + name, MakeLogicalType(GetLogicalType(type), /*required*/ false), sortOrder) { } TColumnSchema::TColumnSchema( - TString name, + const std::string& name, ESimpleLogicalValueType type, std::optional<ESortOrder> sortOrder) : TColumnSchema( - std::move(name), + name, MakeLogicalType(type, /*required*/ false), sortOrder) { } TColumnSchema::TColumnSchema( - TString name, + const std::string& name, TLogicalTypePtr type, std::optional<ESortOrder> sortOrder) : StableName_(name) @@ -184,13 +184,13 @@ TColumnSchema::TColumnSchema( SetLogicalType(std::move(type)); } -TColumnSchema& TColumnSchema::SetStableName(TColumnStableName value) +TColumnSchema& TColumnSchema::SetStableName(const TColumnStableName& value) { - StableName_ = std::move(value); + StableName_ = value; return *this; } -TColumnSchema& TColumnSchema::SetName(TString value) +TColumnSchema& TColumnSchema::SetName(const std::string& value) { Name_ = value; return *this; @@ -202,21 +202,21 @@ TColumnSchema& TColumnSchema::SetSortOrder(std::optional<ESortOrder> value) return *this; } -TColumnSchema& TColumnSchema::SetLock(std::optional<TString> value) +TColumnSchema& TColumnSchema::SetLock(const std::optional<std::string>& value) { - Lock_ = std::move(value); + Lock_ = value; return *this; } -TColumnSchema& TColumnSchema::SetGroup(std::optional<TString> value) +TColumnSchema& TColumnSchema::SetGroup(const std::optional<std::string>& value) { - Group_ = std::move(value); + Group_ = value; return *this; } -TColumnSchema& TColumnSchema::SetExpression(std::optional<TString> value) +TColumnSchema& TColumnSchema::SetExpression(const std::optional<TString>& value) { - Expression_ = std::move(value); + Expression_ = value; return *this; } @@ -226,9 +226,9 @@ TColumnSchema& TColumnSchema::SetMaterialized(std::optional<bool> value) return *this; } -TColumnSchema& TColumnSchema::SetAggregate(std::optional<TString> value) +TColumnSchema& TColumnSchema::SetAggregate(const std::optional<std::string>& value) { - Aggregate_ = std::move(value); + Aggregate_ = value; return *this; } @@ -300,7 +300,7 @@ bool TColumnSchema::IsRenamed() const return Name() != StableName().Underlying(); } -TString TColumnSchema::GetDiagnosticNameString() const +std::string TColumnSchema::GetDiagnosticNameString() const { if (IsRenamed()) { return Format("%Qv (stable name %Qv)", Name(), StableName().Underlying()); @@ -389,9 +389,9 @@ void FormatValue(TStringBuilderBase* builder, const TDeletedColumn& schema, TStr void ToProto(NProto::TColumnSchema* protoSchema, const TColumnSchema& schema) { - protoSchema->set_name(schema.Name()); + protoSchema->set_name(ToProto(schema.Name())); if (schema.IsRenamed()) { - protoSchema->set_stable_name(schema.StableName().Underlying()); + protoSchema->set_stable_name(ToProto(schema.StableName())); } protoSchema->set_type(ToProto(GetPhysicalType(schema.CastToV1Type()))); @@ -408,7 +408,7 @@ void ToProto(NProto::TColumnSchema* protoSchema, const TColumnSchema& schema) } ToProto(protoSchema->mutable_logical_type(), schema.LogicalType()); if (schema.Lock()) { - protoSchema->set_lock(*schema.Lock()); + protoSchema->set_lock(ToProto(*schema.Lock())); } else { protoSchema->clear_lock(); } @@ -423,7 +423,7 @@ void ToProto(NProto::TColumnSchema* protoSchema, const TColumnSchema& schema) protoSchema->clear_materialized(); } if (schema.Aggregate()) { - protoSchema->set_aggregate(*schema.Aggregate()); + protoSchema->set_aggregate(ToProto(*schema.Aggregate())); } else { protoSchema->clear_aggregate(); } @@ -433,7 +433,7 @@ void ToProto(NProto::TColumnSchema* protoSchema, const TColumnSchema& schema) protoSchema->clear_sort_order(); } if (schema.Group()) { - protoSchema->set_group(*schema.Group()); + protoSchema->set_group(ToProto(*schema.Group())); } else { protoSchema->clear_group(); } @@ -446,7 +446,7 @@ void ToProto(NProto::TColumnSchema* protoSchema, const TColumnSchema& schema) void ToProto(NProto::TDeletedColumn* protoSchema, const TDeletedColumn& schema) { - protoSchema->set_stable_name(schema.StableName().Underlying()); + protoSchema->set_stable_name(ToProto(schema.StableName())); } void FromProto(TColumnSchema* schema, const NProto::TColumnSchema& protoSchema) @@ -500,7 +500,7 @@ bool TTableSchema::TNameMapping::IsDeleted(const TColumnStableName& stableName) return Schema_.FindDeletedColumn(stableName) != nullptr; } -TString TTableSchema::TNameMapping::StableNameToName(const TColumnStableName& stableName) const +std::string TTableSchema::TNameMapping::StableNameToName(const TColumnStableName& stableName) const { auto* column = Schema_.FindColumnByStableName(stableName); if (!column) { @@ -523,7 +523,7 @@ TColumnStableName TTableSchema::TNameMapping::NameToStableName(TStringBuf name) THROW_ERROR_EXCEPTION("No column with name %Qv in strict schema", name); } } - return TColumnStableName(TString(name)); + return TColumnStableName(std::string(name)); } return column->StableName(); } @@ -705,7 +705,7 @@ TTableSchemaPtr TTableSchema::Filter(const TColumnFilter& columnFilter, bool dis DeletedColumns()); } -TTableSchemaPtr TTableSchema::Filter(const THashSet<TString>& columnNames, bool discardSortOrder) const +TTableSchemaPtr TTableSchema::Filter(const THashSet<std::string>& columnNames, bool discardSortOrder) const { TColumnFilter::TIndexes indexes; for (const auto& column : Columns()) { @@ -716,13 +716,13 @@ TTableSchemaPtr TTableSchema::Filter(const THashSet<TString>& columnNames, bool return Filter(TColumnFilter(std::move(indexes)), discardSortOrder); } -TTableSchemaPtr TTableSchema::Filter(const std::optional<std::vector<TString>>& columnNames, bool discardSortOrder) const +TTableSchemaPtr TTableSchema::Filter(const std::optional<std::vector<std::string>>& columnNames, bool discardSortOrder) const { if (!columnNames) { return Filter(TColumnFilter(), discardSortOrder); } - return Filter(THashSet<TString>(columnNames->begin(), columnNames->end()), discardSortOrder); + return Filter(THashSet<std::string>(columnNames->begin(), columnNames->end()), discardSortOrder); } bool TTableSchema::HasMaterializedComputedColumns() const @@ -835,12 +835,12 @@ int TTableSchema::GetColumnCount() const return ssize(Columns()); } -std::vector<TString> TTableSchema::GetColumnNames() const +std::vector<std::string> TTableSchema::GetColumnNames() const { if (!ColumnInfo_) { - return std::vector<TString>(); + return std::vector<std::string>(); } - std::vector<TString> result; + std::vector<std::string> result; const auto& info = *ColumnInfo_; result.reserve(info.Columns.size()); for (const auto& column : info.Columns) { @@ -870,8 +870,8 @@ const THunkColumnIds& TTableSchema::GetHunkColumnIds() const std::vector<TColumnStableName> MapNamesToStableNames( const TTableSchema& schema, - std::vector<TString> names, - const std::optional<TStringBuf>& missingColumnReplacement) + const std::vector<std::string>& names, + std::optional<TStringBuf> missingColumnReplacement) { std::vector<TColumnStableName> stableNames; stableNames.reserve(names.size()); @@ -882,7 +882,7 @@ std::vector<TColumnStableName> MapNamesToStableNames( } else if (!schema.GetStrict()) { stableNames.push_back(TColumnStableName(name)); } else if (missingColumnReplacement) { - stableNames.push_back(TColumnStableName(TString(*missingColumnReplacement))); + stableNames.push_back(TColumnStableName(std::string(*missingColumnReplacement))); } else { THROW_ERROR_EXCEPTION("Column %Qv is missing in strict schema", name); @@ -914,7 +914,7 @@ TSortColumns TTableSchema::GetSortColumns(const std::optional<TNameMapping>& nam if (column.SortOrder()) { const auto& name = actualNameMapping.StableNameToName(column.StableName()); sortColumns.push_back(TColumnSortSchema{ - .Name = TString(name), + .Name = std::string(name), .SortOrder = *column.SortOrder(), }); } @@ -1483,14 +1483,14 @@ void FormatValue(TStringBuilderBase* builder, const TTableSchemaPtr& schema, TSt } } -TString SerializeToWireProto(const TTableSchemaPtr& schema) +std::string SerializeToWireProto(const TTableSchemaPtr& schema) { NTableClient::NProto::TTableSchemaExt protoSchema; ToProto(&protoSchema, schema); return protoSchema.SerializeAsString(); } -void DeserializeFromWireProto(TTableSchemaPtr* schema, const TString& serializedProto) +void DeserializeFromWireProto(TTableSchemaPtr* schema, const std::string& serializedProto) { NTableClient::NProto::TTableSchemaExt protoSchema; if (!protoSchema.ParseFromString(serializedProto)) { @@ -1626,7 +1626,7 @@ bool IsEqualIgnoringRequiredness(const TTableSchema& lhs, const TTableSchema& rh // Parses description of the following form nested_key.name or nested_value.name or nested_value.name.sum std::optional<TNestedColumn> TryParseNestedAggregate(TStringBuf description) { - if (!description.StartsWith("nested")) { + if (!description.starts_with("nested")) { return std::nullopt; } @@ -1658,7 +1658,7 @@ std::optional<TNestedColumn> TryParseNestedAggregate(TStringBuf description) }; auto parseToken = [&] (TStringBuf token) { - if (TStringBuf(ptr, ptrEnd).StartsWith(token)) { + if (TStringBuf(ptr, ptrEnd).starts_with(token)) { ptr += token.size(); return true; } @@ -1749,7 +1749,7 @@ void ValidateKeyColumns(const TKeyColumns& keyColumns) { ValidateKeyColumnCount(keyColumns.size()); - THashSet<TString> names; + THashSet<std::string> names; for (const auto& name : keyColumns) { if (!names.insert(name).second) { THROW_ERROR_EXCEPTION("Duplicate key column name %Qv", @@ -1776,12 +1776,12 @@ void ValidateSystemColumnSchema( bool allowUnversionedUpdateColumns, bool allowTimestampColumns) { - static const auto allowedSortedTablesSystemColumns = THashMap<TString, ESimpleLogicalValueType>{ + static const auto allowedSortedTablesSystemColumns = THashMap<std::string, ESimpleLogicalValueType>{ {EmptyValueColumnName, ESimpleLogicalValueType::Int64}, {TtlColumnName, ESimpleLogicalValueType::Uint64}, }; - static const auto allowedOrderedTablesSystemColumns = THashMap<TString, ESimpleLogicalValueType>{ + static const auto allowedOrderedTablesSystemColumns = THashMap<std::string, ESimpleLogicalValueType>{ {TimestampColumnName, ESimpleLogicalValueType::Uint64}, {CumulativeDataWeightColumnName, ESimpleLogicalValueType::Int64}, }; @@ -1819,17 +1819,17 @@ void ValidateSystemColumnSchema( if (name == TUnversionedUpdateSchema::ChangeTypeColumnName) { validateType(ESimpleLogicalValueType::Uint64); return; - } else if (name.StartsWith(TUnversionedUpdateSchema::FlagsColumnNamePrefix)) { + } else if (name.starts_with(TUnversionedUpdateSchema::FlagsColumnNamePrefix)) { validateType(ESimpleLogicalValueType::Uint64); return; - } else if (name.StartsWith(TUnversionedUpdateSchema::ValueColumnNamePrefix)) { + } else if (name.starts_with(TUnversionedUpdateSchema::ValueColumnNamePrefix)) { // Value can have any type. return; } } if (allowTimestampColumns) { - if (name.StartsWith(TimestampColumnPrefix)) { + if (name.starts_with(TimestampColumnPrefix)) { validateType(ESimpleLogicalValueType::Uint64); return; } @@ -1840,7 +1840,7 @@ void ValidateSystemColumnSchema( name); } -void ValidateColumnName(const TString& name) +void ValidateColumnName(const std::string& name) { if (name.empty()) { THROW_ERROR_EXCEPTION("Column name cannot be empty"); @@ -1861,7 +1861,7 @@ void ValidateColumnSchema( bool allowUnversionedUpdateColumns, bool allowTimestampColumns) { - static const auto allowedAggregates = THashSet<TString>{ + static const auto allowedAggregates = THashSet<std::string, THash<TStringBuf>, TEqualTo<>>{ "sum", "min", "max", @@ -1872,7 +1872,7 @@ void ValidateColumnSchema( "dict_sum", }; - static const auto allowedNestedAggregates = THashSet<TString>{ + static const auto allowedNestedAggregates = THashSet<std::string, THash<TStringBuf>, TEqualTo<>>{ "sum", "max" }; @@ -1884,7 +1884,7 @@ void ValidateColumnSchema( const auto& name = columnSchema.Name(); ValidateColumnName(name); - if (stableName.Underlying().StartsWith(SystemColumnNamePrefix) || name.StartsWith(SystemColumnNamePrefix)) { + if (stableName.Underlying().starts_with(SystemColumnNamePrefix) || name.starts_with(SystemColumnNamePrefix)) { ValidateSystemColumnSchema( columnSchema, isTableSorted, @@ -2093,7 +2093,7 @@ void ValidatePivotKey( //! Validates that number of locks doesn't exceed #MaxColumnLockCount. void ValidateLocks(const TTableSchema& schema) { - THashSet<TString> lockNames; + THashSet<std::string> lockNames; YT_VERIFY(lockNames.insert(PrimaryLockName).second); for (const auto& column : schema.Columns()) { if (column.Lock()) { @@ -2323,11 +2323,11 @@ void ValidateNoRenamedColumns(const TTableSchema& schema) //////////////////////////////////////////////////////////////////////////////// -THashMap<TString, int> GetLocksMapping( +THashMap<std::string, int> GetLocksMapping( const NTableClient::TTableSchema& schema, bool fullAtomicity, std::vector<int>* columnIndexToLockIndex, - std::vector<TString>* lockIndexToName) + std::vector<std::string>* lockIndexToName) { if (columnIndexToLockIndex) { // Assign dummy lock indexes to key components. @@ -2338,7 +2338,7 @@ THashMap<TString, int> GetLocksMapping( lockIndexToName->push_back(PrimaryLockName); } - THashMap<TString, int> groupToIndex; + THashMap<std::string, int> groupToIndex; if (fullAtomicity) { // Assign lock indexes to data components. for (int index = schema.GetKeyColumnCount(); index < std::ssize(schema.Columns()); ++index) { @@ -2370,7 +2370,7 @@ THashMap<TString, int> GetLocksMapping( TLockMask GetLockMask( const NTableClient::TTableSchema& schema, bool fullAtomicity, - const std::vector<TString>& locks, + const std::vector<std::string>& locks, ELockType lockType) { auto groupToIndex = GetLocksMapping(schema, fullAtomicity); @@ -2444,7 +2444,7 @@ TCellTaggedTableSchemaPtr::TCellTaggedTableSchemaPtr(TTableSchemaPtr tableSchema size_t THash<NYT::NTableClient::TColumnStableName>::operator()(const NYT::NTableClient::TColumnStableName& stableName) const { - return THash<TString>()(stableName.Underlying()); + return THash<std::string>()(stableName.Underlying()); } size_t THash<NYT::NTableClient::TColumnSchema>::operator()(const NYT::NTableClient::TColumnSchema& columnSchema) const diff --git a/yt/yt/client/table_client/schema.h b/yt/yt/client/table_client/schema.h index 4fd3c3f486..7cbbc86048 100644 --- a/yt/yt/client/table_client/schema.h +++ b/yt/yt/client/table_client/schema.h @@ -117,30 +117,30 @@ class TColumnSchema public: // Keep in sync with hasher below. DEFINE_BYREF_RO_PROPERTY(TColumnStableName, StableName); - DEFINE_BYREF_RO_PROPERTY(TString, Name); + DEFINE_BYREF_RO_PROPERTY(std::string, Name); DEFINE_BYREF_RO_PROPERTY(TLogicalTypePtr, LogicalType); DEFINE_BYREF_RO_PROPERTY(std::optional<ESortOrder>, SortOrder); - DEFINE_BYREF_RO_PROPERTY(std::optional<TString>, Lock); + DEFINE_BYREF_RO_PROPERTY(std::optional<std::string>, Lock); DEFINE_BYREF_RO_PROPERTY(std::optional<TString>, Expression); DEFINE_BYREF_RO_PROPERTY(std::optional<bool>, Materialized); - DEFINE_BYREF_RO_PROPERTY(std::optional<TString>, Aggregate); - DEFINE_BYREF_RO_PROPERTY(std::optional<TString>, Group); + DEFINE_BYREF_RO_PROPERTY(std::optional<std::string>, Aggregate); + DEFINE_BYREF_RO_PROPERTY(std::optional<std::string>, Group); DEFINE_BYREF_RO_PROPERTY(bool, Required); DEFINE_BYREF_RO_PROPERTY(std::optional<i64>, MaxInlineHunkSize); public: TColumnSchema(); TColumnSchema( - TString name, + const std::string& name, EValueType type, std::optional<ESortOrder> sortOrder = {}); TColumnSchema( - TString name, + const std::string& name, ESimpleLogicalValueType type, std::optional<ESortOrder> sortOrder = {}); TColumnSchema( - TString name, + const std::string& name, TLogicalTypePtr type, std::optional<ESortOrder> sortOrder = {}); @@ -150,16 +150,16 @@ public: TColumnSchema& operator=(const TColumnSchema&) = default; TColumnSchema& operator=(TColumnSchema&&) = default; - TColumnSchema& SetStableName(TColumnStableName stableName); - TColumnSchema& SetName(TString name); + TColumnSchema& SetStableName(const TColumnStableName& stableName); + TColumnSchema& SetName(const std::string& name); TColumnSchema& SetLogicalType(TLogicalTypePtr valueType); TColumnSchema& SetSimpleLogicalType(ESimpleLogicalValueType type); TColumnSchema& SetSortOrder(std::optional<ESortOrder> value); - TColumnSchema& SetLock(std::optional<TString> value); - TColumnSchema& SetExpression(std::optional<TString> value); + TColumnSchema& SetLock(const std::optional<std::string>& value); + TColumnSchema& SetExpression(const std::optional<TString>& value); TColumnSchema& SetMaterialized(std::optional<bool> value); - TColumnSchema& SetAggregate(std::optional<TString> value); - TColumnSchema& SetGroup(std::optional<TString> value); + TColumnSchema& SetAggregate(const std::optional<std::string>& value); + TColumnSchema& SetGroup(const std::optional<std::string>& value); TColumnSchema& SetRequired(bool value); TColumnSchema& SetMaxInlineHunkSize(std::optional<i64> value); @@ -176,7 +176,7 @@ public: ESimpleLogicalValueType CastToV1Type() const; bool IsRenamed() const; - TString GetDiagnosticNameString() const; + std::string GetDiagnosticNameString() const; private: ESimpleLogicalValueType V1Type_; @@ -221,7 +221,7 @@ public: explicit TNameMapping(const TTableSchema& schema); bool IsDeleted(const TColumnStableName& stableName) const; - TString StableNameToName(const TColumnStableName& stableName) const; + std::string StableNameToName(const TColumnStableName& stableName) const; TColumnStableName NameToStableName(TStringBuf name) const; private: @@ -267,16 +267,16 @@ public: const TColumnSchema* FindColumn(TStringBuf name) const; const TColumnSchema& GetColumn(TStringBuf name) const; const TColumnSchema& GetColumnOrThrow(TStringBuf name) const; - std::vector<TString> GetColumnNames() const; + std::vector<std::string> GetColumnNames() const; TTableSchemaPtr Filter( const TColumnFilter& columnFilter, bool discardSortOrder = false) const; TTableSchemaPtr Filter( - const THashSet<TString>& columnNames, + const THashSet<std::string>& columnNames, bool discardSortOrder = false) const; TTableSchemaPtr Filter( - const std::optional<std::vector<TString>>& columnNames, + const std::optional<std::vector<std::string>>& columnNames, bool discardSortOrder = false) const; bool HasMaterializedComputedColumns() const; @@ -432,9 +432,9 @@ void FormatValue(TStringBuilderBase* builder, const TTableSchema& schema, TStrin void FormatValue(TStringBuilderBase* builder, const TTableSchemaPtr& schema, TStringBuf spec); //! Returns serialized NTableClient.NProto.TTableSchemaExt. -TString SerializeToWireProto(const TTableSchemaPtr& schema); +std::string SerializeToWireProto(const TTableSchemaPtr& schema); -void DeserializeFromWireProto(TTableSchemaPtr* schema, const TString& serializedProto); +void DeserializeFromWireProto(TTableSchemaPtr* schema, const std::string& serializedProto); void Serialize(const TTableSchema& schema, NYson::IYsonConsumer* consumer); void Deserialize(TTableSchema& schema, NYTree::INodePtr node); @@ -473,12 +473,10 @@ bool IsEqualIgnoringRequiredness(const TTableSchema& lhs, const TTableSchema& rh //////////////////////////////////////////////////////////////////////////////// -static constexpr TStringBuf NonexistentColumnName = "$__YT_NONEXISTENT_COLUMN_NAME__"; - std::vector<TColumnStableName> MapNamesToStableNames( const TTableSchema& schema, - std::vector<TString> names, - const std::optional<TStringBuf>& missingColumnReplacement = std::nullopt); + const std::vector<std::string>& names, + std::optional<TStringBuf> missingColumnReplacement = std::nullopt); //////////////////////////////////////////////////////////////////////////////// @@ -499,7 +497,7 @@ void ValidateKeyColumns(const TKeyColumns& keyColumns); void ValidateDynamicTableKeyColumnCount(int count); -void ValidateColumnName(const TString& name); +void ValidateColumnName(const std::string& name); void ValidateColumnSchema( const TColumnSchema& columnSchema, @@ -528,16 +526,16 @@ void ValidatePivotKey( //////////////////////////////////////////////////////////////////////////////// -THashMap<TString, int> GetLocksMapping( +THashMap<std::string, int> GetLocksMapping( const NTableClient::TTableSchema& schema, bool fullAtomicity, std::vector<int>* columnIndexToLockIndex = nullptr, - std::vector<TString>* lockIndexToName = nullptr); + std::vector<std::string>* lockIndexToName = nullptr); TLockMask GetLockMask( const NTableClient::TTableSchema& schema, bool fullAtomicity, - const std::vector<TString>& locks, + const std::vector<std::string>& locks, ELockType lockType = ELockType::SharedWeak); //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/client/table_client/validate_logical_type.cpp b/yt/yt/client/table_client/validate_logical_type.cpp index 3e6b3d1d65..59cb8d2a0b 100644 --- a/yt/yt/client/table_client/validate_logical_type.cpp +++ b/yt/yt/client/table_client/validate_logical_type.cpp @@ -448,7 +448,7 @@ private: Cursor_.Next(); } - TString GetDescription(const TFieldId& fieldId) const + std::string GetDescription(const TFieldId& fieldId) const { return fieldId.GetDescriptor(RootDescriptor_).GetDescription(); } @@ -509,7 +509,7 @@ private: { std::vector<int> path; const auto* current = this; - while (current->Parent_ != nullptr) { + while (current->Parent_) { path.push_back(current->SiblingIndex_); current = current->Parent_; } diff --git a/yt/yt/client/tablet_client/table_mount_cache_detail.cpp b/yt/yt/client/tablet_client/table_mount_cache_detail.cpp index 248e19de7e..ae1d41c975 100644 --- a/yt/yt/client/tablet_client/table_mount_cache_detail.cpp +++ b/yt/yt/client/tablet_client/table_mount_cache_detail.cpp @@ -304,11 +304,14 @@ auto TTableMountCacheBase::TryHandleServantNotActiveError(const TError& error) } } - TabletInfoOwnerCache_.Insert(*tabletId, MakeWeak(clone)); clonedTableInfos.push_back(std::move(clone)); } for (const auto& tableInfo : clonedTableInfos) { + for (const auto& tabletInfo : tableInfo->Tablets) { + TabletInfoOwnerCache_.Insert(tabletInfo->TabletId, MakeWeak(tableInfo)); + } + TAsyncExpiringCache::Set(tableInfo->Path, tableInfo); } diff --git a/yt/yt/client/tablet_client/watermark_runtime_data.h b/yt/yt/client/tablet_client/watermark_runtime_data.h index 54c62c27bd..08f876a3cf 100644 --- a/yt/yt/client/tablet_client/watermark_runtime_data.h +++ b/yt/yt/client/tablet_client/watermark_runtime_data.h @@ -11,7 +11,7 @@ namespace NYT::NTabletClient { struct TWatermarkRuntimeDataConfig : public NYTree::TYsonStructLite { - TString ColumnName; + std::string ColumnName; ui64 Watermark; REGISTER_YSON_STRUCT_LITE(TWatermarkRuntimeDataConfig); diff --git a/yt/yt/client/unittests/mock/client.h b/yt/yt/client/unittests/mock/client.h index 43e8056b1e..fc59a92b9b 100644 --- a/yt/yt/client/unittests/mock/client.h +++ b/yt/yt/client/unittests/mock/client.h @@ -865,7 +865,7 @@ public: MOCK_METHOD(TFuture<IRowBatchWriterPtr>, CreateShuffleWriter, ( const TShuffleHandlePtr& shuffleHandle, - const TString& partitionColumn, + const std::string& partitionColumn, const NTableClient::TTableWriterConfigPtr& config), (override)); diff --git a/yt/yt/client/unittests/mock/table_reader.h b/yt/yt/client/unittests/mock/table_reader.h index be901f4881..a827b7c8b6 100644 --- a/yt/yt/client/unittests/mock/table_reader.h +++ b/yt/yt/client/unittests/mock/table_reader.h @@ -24,7 +24,7 @@ public: MOCK_METHOD(NTableClient::IUnversionedRowBatchPtr, Read, (const NTableClient::TRowBatchReadOptions& options), (override)); - MOCK_METHOD(const std::vector<TString>&, GetOmittedInaccessibleColumns, (), (const, override)); + MOCK_METHOD(const std::vector<std::string>&, GetOmittedInaccessibleColumns, (), (const, override)); const NTableClient::TNameTablePtr& GetNameTable() const override; diff --git a/yt/yt/client/ypath/parser_detail.cpp b/yt/yt/client/ypath/parser_detail.cpp index d1852f526f..cb11a20767 100644 --- a/yt/yt/client/ypath/parser_detail.cpp +++ b/yt/yt/client/ypath/parser_detail.cpp @@ -171,7 +171,7 @@ void ParseColumns(NYson::TTokenizer& tokenizer, IAttributeDictionary* attributes return; } - std::vector<TString> columns; + std::vector<std::string> columns; tokenizer.ParseNext(); while (tokenizer.GetCurrentType() != EndColumnSelectorToken) { diff --git a/yt/yt/client/ypath/rich.cpp b/yt/yt/client/ypath/rich.cpp index ddd386df64..717194f64b 100644 --- a/yt/yt/client/ypath/rich.cpp +++ b/yt/yt/client/ypath/rich.cpp @@ -217,15 +217,15 @@ void TRichYPath::SetReadViaExecNode(bool value) Attributes().Set("read_via_exec_node", value); } -std::optional<std::vector<TString>> TRichYPath::GetColumns() const +std::optional<std::vector<std::string>> TRichYPath::GetColumns() const { if (Attributes().Contains("channel")) { THROW_ERROR_EXCEPTION("Deprecated attribute \"channel\" in YPath"); } - return FindAttribute<std::vector<TString>>(*this, "columns"); + return FindAttribute<std::vector<std::string>>(*this, "columns"); } -void TRichYPath::SetColumns(const std::vector<TString>& columns) +void TRichYPath::SetColumns(const std::vector<std::string>& columns) { Attributes().Set("columns", columns); } diff --git a/yt/yt/client/ypath/rich.h b/yt/yt/client/ypath/rich.h index b6d7d60ccf..75335b7606 100644 --- a/yt/yt/client/ypath/rich.h +++ b/yt/yt/client/ypath/rich.h @@ -67,8 +67,8 @@ public: void SetReadViaExecNode(bool value); // "columns" - std::optional<std::vector<TString>> GetColumns() const; - void SetColumns(const std::vector<TString>& columns); + std::optional<std::vector<std::string>> GetColumns() const; + void SetColumns(const std::vector<std::string>& columns); // "rename_columns" std::optional<NTableClient::TColumnRenameDescriptors> GetColumnRenameDescriptors() const; diff --git a/yt/yt/core/concurrency/unittests/scheduled_executor_ut.cpp b/yt/yt/core/concurrency/unittests/scheduled_executor_ut.cpp index 2d7143297d..f0573463c6 100644 --- a/yt/yt/core/concurrency/unittests/scheduled_executor_ut.cpp +++ b/yt/yt/core/concurrency/unittests/scheduled_executor_ut.cpp @@ -19,7 +19,7 @@ class TScheduledExecutorTest //////////////////////////////////////////////////////////////////////////////// -constexpr auto ErrorMargin = TDuration::MilliSeconds(20); +constexpr auto ErrorMargin = TDuration::MilliSeconds(50); //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/core/logging/log_manager.cpp b/yt/yt/core/logging/log_manager.cpp index d7e15992be..5694698800 100644 --- a/yt/yt/core/logging/log_manager.cpp +++ b/yt/yt/core/logging/log_manager.cpp @@ -519,10 +519,6 @@ public: ::TSourceLocation sourceLocation, TStringBuf message) { - if (anchor->Registered.exchange(true)) { - return; - } - auto guard = Guard(SpinLock_); auto config = Config_.Acquire(); anchor->SourceLocation = sourceLocation; @@ -537,12 +533,13 @@ public: if (auto it = AnchorMap_.find(anchorMessage)) { return it->second; } + auto config = Config_.Acquire(); auto anchor = std::make_unique<TLoggingAnchor>(); - anchor->Registered = true; anchor->AnchorMessage = std::move(anchorMessage); auto* rawAnchor = anchor.get(); DynamicAnchors_.push_back(std::move(anchor)); DoRegisterAnchor(rawAnchor); + DoUpdateAnchor(config, rawAnchor); return rawAnchor; } diff --git a/yt/yt/core/net/address.cpp b/yt/yt/core/net/address.cpp index c08fb94226..6f7a44100f 100644 --- a/yt/yt/core/net/address.cpp +++ b/yt/yt/core/net/address.cpp @@ -110,6 +110,13 @@ TStringBuf GetServiceHostName(TStringBuf address) //////////////////////////////////////////////////////////////////////////////// +TString FormatNetworkAddress(TStringBuf address, int port) +{ + return Format("[%v]:%v", address, port); +} + +//////////////////////////////////////////////////////////////////////////////// + const TNetworkAddress NullNetworkAddress; TNetworkAddress::TNetworkAddress() diff --git a/yt/yt/core/net/address.h b/yt/yt/core/net/address.h index 6f507b2900..99e7869f9e 100644 --- a/yt/yt/core/net/address.h +++ b/yt/yt/core/net/address.h @@ -43,6 +43,9 @@ TStringBuf GetServiceHostName(TStringBuf address); //////////////////////////////////////////////////////////////////////////////// +//! Constructs an address of the form |[address]:port|. +TString FormatNetworkAddress(TStringBuf address, int port); + class TIP6Address; //! An opaque wrapper for |sockaddr| type. diff --git a/yt/yt/core/rpc/balancing_channel.cpp b/yt/yt/core/rpc/balancing_channel.cpp index 7884bba0d2..7e9cd188d0 100644 --- a/yt/yt/core/rpc/balancing_channel.cpp +++ b/yt/yt/core/rpc/balancing_channel.cpp @@ -151,7 +151,10 @@ private: continue; } - auto addresses = AddressesFromEndpointSet(endpointSetOrError.Value()); + auto addresses = AddressesFromEndpointSet( + endpointSetOrError.Value(), + Config_->Endpoints->UseIPv4, + Config_->Endpoints->UseIPv6); allAddresses.insert(allAddresses.end(), addresses.begin(), addresses.end()); } diff --git a/yt/yt/core/rpc/config.cpp b/yt/yt/core/rpc/config.cpp index 50030c0c3b..b9cb445371 100644 --- a/yt/yt/core/rpc/config.cpp +++ b/yt/yt/core/rpc/config.cpp @@ -220,6 +220,10 @@ void TServiceDiscoveryEndpointsConfig::Register(TRegistrar registrar) registrar.Parameter("endpoint_set_id", &TThis::EndpointSetId); registrar.Parameter("update_period", &TThis::UpdatePeriod) .Default(TDuration::Seconds(60)); + registrar.Parameter("use_ipv4", &TThis::UseIPv4) + .Default(false); + registrar.Parameter("use_ipv6", &TThis::UseIPv6) + .Default(false); registrar.Postprocessor([] (TThis* config) { if (config->Cluster.has_value() == !config->Clusters.empty()) { diff --git a/yt/yt/core/rpc/config.h b/yt/yt/core/rpc/config.h index aceeadc725..c3ce1d08d2 100644 --- a/yt/yt/core/rpc/config.h +++ b/yt/yt/core/rpc/config.h @@ -132,7 +132,7 @@ public: std::optional<bool> EnableErrorCodeCounter; std::optional<ERequestTracingMode> TracingMode; TTimeHistogramConfigPtr TimeHistogram; - THashMap<std::string, TMethodConfigPtr> Methods; + THashMap<TString, TMethodConfigPtr> Methods; std::optional<int> AuthenticationQueueSizeLimit; std::optional<TDuration> PendingPayloadsTimeout; std::optional<bool> Pooled; @@ -302,6 +302,11 @@ public: TString EndpointSetId; TDuration UpdatePeriod; + //! Use IPv4 address of endpoint. + bool UseIPv4; + //! Use IPv6 address of endpoint. + bool UseIPv6; + REGISTER_YSON_STRUCT(TServiceDiscoveryEndpointsConfig); static void Register(TRegistrar registrar); diff --git a/yt/yt/core/rpc/helpers.cpp b/yt/yt/core/rpc/helpers.cpp index d9d2a93a89..ca7b6221e5 100644 --- a/yt/yt/core/rpc/helpers.cpp +++ b/yt/yt/core/rpc/helpers.cpp @@ -514,12 +514,21 @@ void SetCurrentAuthenticationIdentity(const IClientRequestPtr& request) SetAuthenticationIdentity(request, GetCurrentAuthenticationIdentity()); } -std::vector<std::string> AddressesFromEndpointSet(const NServiceDiscovery::TEndpointSet& endpointSet) +std::vector<std::string> AddressesFromEndpointSet( + const NServiceDiscovery::TEndpointSet& endpointSet, + bool useIPv4, + bool useIPv6) { std::vector<std::string> addresses; addresses.reserve(endpointSet.Endpoints.size()); for (const auto& endpoint : endpointSet.Endpoints) { - addresses.push_back(NNet::BuildServiceAddress(endpoint.Fqdn, endpoint.Port)); + if (useIPv6 && !endpoint.IP6Address.empty()) { + addresses.push_back(NNet::FormatNetworkAddress(endpoint.IP6Address, endpoint.Port)); + } else if (useIPv4 && !endpoint.IP4Address.empty()) { + addresses.push_back(NNet::FormatNetworkAddress(endpoint.IP4Address, endpoint.Port)); + } else { + addresses.push_back(NNet::BuildServiceAddress(endpoint.Fqdn, endpoint.Port)); + } } return addresses; } diff --git a/yt/yt/core/rpc/helpers.h b/yt/yt/core/rpc/helpers.h index c631684555..0624017613 100644 --- a/yt/yt/core/rpc/helpers.h +++ b/yt/yt/core/rpc/helpers.h @@ -93,7 +93,10 @@ void WriteAuthenticationIdentityToProto(T* proto, const TAuthenticationIdentity& template <class T> TAuthenticationIdentity ParseAuthenticationIdentityFromProto(const T& proto); -std::vector<std::string> AddressesFromEndpointSet(const NServiceDiscovery::TEndpointSet& endpointSet); +std::vector<std::string> AddressesFromEndpointSet( + const NServiceDiscovery::TEndpointSet& endpointSet, + bool useIPv4 = false, + bool useIPv6 = false); //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/core/rpc/server_detail.cpp b/yt/yt/core/rpc/server_detail.cpp index 3deb728c2d..dbbdd0e2bf 100644 --- a/yt/yt/core/rpc/server_detail.cpp +++ b/yt/yt/core/rpc/server_detail.cpp @@ -374,6 +374,9 @@ std::optional<TDuration> TServiceContextBase::GetExecutionDuration() const return std::nullopt; } +void TServiceContextBase::RecordThrottling(TDuration /*throttleDuration*/) +{ } + TTraceContextPtr TServiceContextBase::GetTraceContext() const { return nullptr; @@ -581,6 +584,11 @@ std::optional<TDuration> TServiceContextWrapper::GetExecutionDuration() const return UnderlyingContext_->GetExecutionDuration(); } +void TServiceContextWrapper::RecordThrottling(TDuration throttleDuration) +{ + return UnderlyingContext_->RecordThrottling(throttleDuration); +} + TTraceContextPtr TServiceContextWrapper::GetTraceContext() const { return UnderlyingContext_->GetTraceContext(); diff --git a/yt/yt/core/rpc/server_detail.h b/yt/yt/core/rpc/server_detail.h index 16c56e9eb1..3683ca7477 100644 --- a/yt/yt/core/rpc/server_detail.h +++ b/yt/yt/core/rpc/server_detail.h @@ -45,6 +45,7 @@ public: std::optional<TInstant> GetFinishInstant() const override; std::optional<TDuration> GetWaitDuration() const override; std::optional<TDuration> GetExecutionDuration() const override; + void RecordThrottling(TDuration throttleDuration) override; NTracing::TTraceContextPtr GetTraceContext() const override; std::optional<TDuration> GetTraceContextTime() const override; @@ -208,6 +209,7 @@ public: std::optional<TInstant> GetFinishInstant() const override; std::optional<TDuration> GetWaitDuration() const override; std::optional<TDuration> GetExecutionDuration() const override; + void RecordThrottling(TDuration throttleDuration) override; NTracing::TTraceContextPtr GetTraceContext() const override; std::optional<TDuration> GetTraceContextTime() const override; diff --git a/yt/yt/core/rpc/service.h b/yt/yt/core/rpc/service.h index 8511ad549a..3eb6d06a70 100644 --- a/yt/yt/core/rpc/service.h +++ b/yt/yt/core/rpc/service.h @@ -82,6 +82,9 @@ struct IServiceContext //! Returns time between request execution start and the moment of reply or cancellation (if it already happened). virtual std::optional<TDuration> GetExecutionDuration() const = 0; + //! Substract given throttle duration time from request execution time. + virtual void RecordThrottling(TDuration throttleDuration) = 0; + //! Returns trace context associated with request. virtual NTracing::TTraceContextPtr GetTraceContext() const = 0; diff --git a/yt/yt/core/rpc/service_detail.cpp b/yt/yt/core/rpc/service_detail.cpp index 970552940b..87d3fcd032 100644 --- a/yt/yt/core/rpc/service_detail.cpp +++ b/yt/yt/core/rpc/service_detail.cpp @@ -50,8 +50,6 @@ static const auto InfiniteRequestThrottlerConfig = New<TThroughputThrottlerConfi static const auto DefaultLoggingSuppressionFailedRequestThrottlerConfig = TThroughputThrottlerConfig::Create(1'000); constexpr int MaxUserAgentLength = 200; -constexpr TStringBuf UnknownUserAgent = "unknown"; - constexpr auto ServiceLivenessCheckPeriod = TDuration::MilliSeconds(100); //////////////////////////////////////////////////////////////////////////////// @@ -266,15 +264,15 @@ auto TServiceBase::TMethodDescriptor::SetHandleMethodError(bool value) const -> //////////////////////////////////////////////////////////////////////////////// -TServiceBase::TErrorCodeCounters::TErrorCodeCounters(NProfiling::TProfiler profiler) +TServiceBase::TErrorCodeCounter::TErrorCodeCounter(NProfiling::TProfiler profiler) : Profiler_(std::move(profiler)) { } -NProfiling::TCounter* TServiceBase::TErrorCodeCounters::GetCounter(TErrorCode code) +void TServiceBase::TErrorCodeCounter::Increment(TErrorCode code) { - return CodeToCounter_.FindOrInsert(code, [&] { + CodeToCounter_.FindOrInsert(code, [&] { return Profiler_.WithTag("code", ToString(code)).Counter("/code_count"); - }).first; + }).first->Increment(); } //////////////////////////////////////////////////////////////////////////////// @@ -291,7 +289,7 @@ TServiceBase::TMethodPerformanceCounters::TMethodPerformanceCounters( , RequestMessageAttachmentSizeCounter(profiler.Counter("/request_message_attachment_bytes")) , ResponseMessageBodySizeCounter(profiler.Counter("/response_message_body_bytes")) , ResponseMessageAttachmentSizeCounter(profiler.Counter("/response_message_attachment_bytes")) - , ErrorCodeCounters(profiler) + , ErrorCodeCounter(profiler) { if (timeHistogramConfig && timeHistogramConfig->CustomBounds) { const auto& customBounds = *timeHistogramConfig->CustomBounds; @@ -340,19 +338,6 @@ TRequestQueue* TServiceBase::TRuntimeMethodInfo::GetDefaultRequestQueue() //////////////////////////////////////////////////////////////////////////////// -TServiceBase::TPerformanceCounters::TPerformanceCounters(const NProfiling::TProfiler& profiler) - : Profiler_(profiler.WithHot().WithSparse()) -{ } - -NProfiling::TCounter* TServiceBase::TPerformanceCounters::GetRequestsPerUserAgentCounter(TStringBuf userAgent) -{ - return RequestsPerUserAgent_.FindOrInsert(userAgent, [&] { - return Profiler_.WithRequiredTag("user_agent", TString(userAgent)).Counter("/user_agent"); - }).first; -} - -//////////////////////////////////////////////////////////////////////////////// - class TServiceBase::TServiceContext : public TServiceContextBase { @@ -375,9 +360,11 @@ public: , TraceContext_(std::move(acceptedRequest.TraceContext)) , RequestQueue_(acceptedRequest.RequestQueue) , ThrottledError_(std::move(acceptedRequest.ThrottledError)) - , MethodPerformanceCounters_(acceptedRequest.MethodPerformanceCounters) + , MethodPerformanceCounters_(Service_->GetMethodPerformanceCounters( + RuntimeInfo_, + {GetAuthenticationIdentity().UserTag, RequestQueue_})) , PerformanceCounters_(Service_->GetPerformanceCounters()) - , ArriveInstant_(acceptedRequest.ArriveInstant) + , ArriveInstant_(NProfiling::GetInstant()) { YT_ASSERT(RequestMessage_); YT_ASSERT(ReplyBus_); @@ -600,6 +587,14 @@ public: return ExecutionTime_; } + void RecordThrottling(TDuration throttleDuration) override + { + ThrottlingTime_ = ThrottlingTime_ ? *ThrottlingTime_ + throttleDuration : throttleDuration; + if (ExecutionTime_) { + *ExecutionTime_ -= throttleDuration; + } + } + TTraceContextPtr GetTraceContext() const override { return TraceContext_; @@ -724,6 +719,7 @@ private: std::optional<TInstant> RunInstant_; std::optional<TInstant> ReplyInstant_; std::optional<TInstant> CancelInstant_; + std::optional<TDuration> ThrottlingTime_; std::optional<TDuration> ExecutionTime_; std::optional<TDuration> TotalTime_; @@ -756,6 +752,24 @@ private: void Initialize() { + constexpr TStringBuf UnknownUserAgent = "unknown"; + auto userAgent = RequestHeader_->has_user_agent() + ? TStringBuf(RequestHeader_->user_agent()) + : UnknownUserAgent; + PerformanceCounters_->IncrementRequestsPerUserAgent(userAgent.SubString(0, MaxUserAgentLength)); + + MethodPerformanceCounters_->RequestCounter.Increment(); + MethodPerformanceCounters_->RequestMessageBodySizeCounter.Increment( + GetMessageBodySize(RequestMessage_)); + MethodPerformanceCounters_->RequestMessageAttachmentSizeCounter.Increment( + GetTotalMessageAttachmentSize(RequestMessage_)); + + if (RequestHeader_->has_start_time()) { + auto retryStart = FromProto<TInstant>(RequestHeader_->start_time()); + auto now = NProfiling::GetInstant(); + MethodPerformanceCounters_->RemoteWaitTimeCounter.Record(now - retryStart); + } + // COMPAT(danilalexeev): legacy RPC codecs RequestCodec_ = RequestHeader_->has_request_codec() ? CheckedEnumCast<NCompression::ECodec>(RequestHeader_->request_codec()) @@ -1013,14 +1027,16 @@ private: ReplyInstant_ = NProfiling::GetInstant(); ExecutionTime_ = RunInstant_ ? *ReplyInstant_ - *RunInstant_ : TDuration(); + if (RunInstant_ && ThrottlingTime_) { + *ExecutionTime_ -= *ThrottlingTime_; + } TotalTime_ = *ReplyInstant_ - ArriveInstant_; MethodPerformanceCounters_->ExecutionTimeCounter.Record(*ExecutionTime_); MethodPerformanceCounters_->TotalTimeCounter.Record(*TotalTime_); if (!Error_.IsOK()) { if (Service_->EnableErrorCodeCounter_.load()) { - const auto* counter = MethodPerformanceCounters_->ErrorCodeCounters.GetCounter(Error_.GetNonTrivialCode()); - counter->Increment(); + MethodPerformanceCounters_->ErrorCodeCounter.Increment(Error_.GetNonTrivialCode()); } else { MethodPerformanceCounters_->FailedRequestCounter.Increment(); } @@ -1117,7 +1133,7 @@ private: TraceContext_->AddTag(RequestUser, builder.Flush()); } } - YT_LOG_EVENT_WITH_ANCHOR(Logger, LogLevel_, RuntimeInfo_->RequestLoggingAnchor, logMessage); + YT_LOG_EVENT_WITH_DYNAMIC_ANCHOR(Logger, LogLevel_, RuntimeInfo_->RequestLoggingAnchor, logMessage); } void LogResponse() override @@ -1164,7 +1180,7 @@ private: if (TraceContext_ && TraceContext_->IsRecorded()) { TraceContext_->AddTag(ResponseInfoAnnotation, logMessage); } - YT_LOG_EVENT_WITH_ANCHOR(Logger, LogLevel_, RuntimeInfo_->ResponseLoggingAnchor, logMessage); + YT_LOG_EVENT_WITH_DYNAMIC_ANCHOR(Logger, LogLevel_, RuntimeInfo_->ResponseLoggingAnchor, logMessage); } @@ -1303,7 +1319,7 @@ private: //////////////////////////////////////////////////////////////////////////////// -TRequestQueue::TRequestQueue(const std::string& name, const NProfiling::TProfiler& profiler) +TRequestQueue::TRequestQueue(const std::string& name, NProfiling::TProfiler profiler) : Name_(name) , BytesThrottler_{CreateReconfigurableThroughputThrottler(InfiniteRequestThrottlerConfig, NLogging::TLogger(), @@ -1606,20 +1622,23 @@ void TRequestQueue::SubscribeToThrottlers() //////////////////////////////////////////////////////////////////////////////// -bool TServiceBase::TRuntimeMethodInfo::TPerformanceCountersKeyEquals::operator()( - const TNonowningPerformanceCountersKey& lhs, - const TNonowningPerformanceCountersKey& rhs) const +struct TServiceBase::TRuntimeMethodInfo::TPerformanceCountersKeyEquals { - return lhs == rhs; -} + bool operator()( + const TNonowningPerformanceCountersKey& lhs, + const TNonowningPerformanceCountersKey& rhs) const + { + return lhs == rhs; + } -bool TServiceBase::TRuntimeMethodInfo::TPerformanceCountersKeyEquals::operator()( - const TOwningPerformanceCountersKey& lhs, - const TNonowningPerformanceCountersKey& rhs) const -{ - const auto& [lhsUserTag, lhsRequestQueue] = lhs; - return TNonowningPerformanceCountersKey{lhsUserTag, lhsRequestQueue} == rhs; -} + bool operator()( + const TOwningPerformanceCountersKey& lhs, + const TNonowningPerformanceCountersKey& rhs) const + { + const auto& [lhsUserTag, lhsRequestQueue] = lhs; + return TNonowningPerformanceCountersKey{lhsUserTag, lhsRequestQueue} == rhs; + } +}; //////////////////////////////////////////////////////////////////////////////// @@ -1668,15 +1687,8 @@ void TServiceBase::HandleRequest( { SetActive(); - auto arriveInstant = NProfiling::GetInstant(); - - const auto& method = header->method(); + auto method = FromProto<TString>(header->method()); auto requestId = FromProto<TRequestId>(header->request_id()); - auto userAgent = header->has_user_agent() - ? TStringBuf(header->user_agent()).SubString(0, MaxUserAgentLength) - : UnknownUserAgent; - const auto& user = header->has_user() ? header->user() : RootUserName; - const auto& userTag = header->has_user_tag() ? header->user_tag() : user; auto replyError = [&] (TError error) { ReplyError(std::move(error), *header, replyBus); @@ -1689,6 +1701,11 @@ void TServiceBase::HandleRequest( return; } + if (auto error = DoCheckRequestCompatibility(*header); !error.IsOK()) { + replyError(std::move(error)); + return; + } + auto* runtimeInfo = FindMethodInfo(method); if (!runtimeInfo) { replyError(TError( @@ -1697,29 +1714,8 @@ void TServiceBase::HandleRequest( return; } - auto* requestQueue = GetRequestQueue(runtimeInfo, *header); - - const auto* requestsPerUserAgentCounter = PerformanceCounters_->GetRequestsPerUserAgentCounter(userAgent); - requestsPerUserAgentCounter->Increment(); - - auto* methodPerformanceCounters = GetMethodPerformanceCounters(runtimeInfo, {userTag, requestQueue}); - methodPerformanceCounters->RequestCounter.Increment(); - methodPerformanceCounters->RequestMessageBodySizeCounter.Increment(GetMessageBodySize(message)); - methodPerformanceCounters->RequestMessageAttachmentSizeCounter.Increment(GetTotalMessageAttachmentSize(message)); - - if (header->has_start_time()) { - auto retryStart = FromProto<TInstant>(header->start_time()); - methodPerformanceCounters->RemoteWaitTimeCounter.Record(arriveInstant - retryStart); - } - - if (auto error = DoCheckRequestCompatibility(*header); !error.IsOK()) { - replyError(std::move(error)); - return; - } - auto memoryGuard = TMemoryUsageTrackerGuard::Acquire(MemoryUsageTracker_, TypicalRequestSize); message = TrackMemory(MemoryUsageTracker_, std::move(message)); - if (MemoryUsageTracker_ && MemoryUsageTracker_->IsExceeded()) { return replyError(TError( NRpc::EErrorCode::MemoryPressure, @@ -1730,12 +1726,14 @@ void TServiceBase::HandleRequest( auto traceContext = tracingMode == ERequestTracingMode::Disable ? NTracing::TTraceContextPtr() : GetOrCreateHandlerTraceContext(*header, tracingMode == ERequestTracingMode::Force); - if (traceContext && traceContext->IsRecorded()) { traceContext->AddTag(EndpointAnnotation, replyBus->GetEndpointDescription()); } - auto throttledError = GetThrottledError(*header); + auto* requestQueue = GetRequestQueue(runtimeInfo, *header); + RegisterRequestQueue(runtimeInfo, requestQueue); + + auto maybeThrottled = GetThrottledError(*header); if (requestQueue->IsQueueSizeLimitExceeded()) { runtimeInfo->RequestQueueSizeLimitErrorCounter.Increment(); @@ -1744,7 +1742,7 @@ void TServiceBase::HandleRequest( "Request queue size limit exceeded") << TErrorAttribute("limit", runtimeInfo->QueueSizeLimit.load()) << TErrorAttribute("queue", requestQueue->GetName()) - << throttledError); + << maybeThrottled); return; } @@ -1755,7 +1753,7 @@ void TServiceBase::HandleRequest( "Request queue bytes size limit exceeded") << TErrorAttribute("limit", runtimeInfo->QueueByteSizeLimit.load()) << TErrorAttribute("queue", requestQueue->GetName()) - << throttledError); + << maybeThrottled); return; } @@ -1763,7 +1761,6 @@ void TServiceBase::HandleRequest( // NOTE: Do not use replyError() after this line. TAcceptedRequest acceptedRequest{ - .ArriveInstant = arriveInstant, .RequestId = requestId, .ReplyBus = std::move(replyBus), .RuntimeInfo = std::move(runtimeInfo), @@ -1771,8 +1768,7 @@ void TServiceBase::HandleRequest( .Header = std::move(header), .Message = std::move(message), .RequestQueue = requestQueue, - .MethodPerformanceCounters = methodPerformanceCounters, - .ThrottledError = throttledError, + .ThrottledError = maybeThrottled, .MemoryGuard = std::move(memoryGuard), .MemoryUsageTracker = MemoryUsageTracker_, }; @@ -1925,49 +1921,55 @@ TRequestQueue* TServiceBase::GetRequestQueue( const NRpc::NProto::TRequestHeader& requestHeader) { TRequestQueue* requestQueue = nullptr; - if (const auto& provider = runtimeInfo->Descriptor.RequestQueueProvider) { + if (auto& provider = runtimeInfo->Descriptor.RequestQueueProvider) { requestQueue = provider->GetQueue(requestHeader); } if (!requestQueue) { requestQueue = runtimeInfo->DefaultRequestQueue.Get(); } + return requestQueue; +} - if (requestQueue->Register(this, runtimeInfo)) { - const auto& method = runtimeInfo->Descriptor.Method; - YT_LOG_DEBUG("Request queue registered (Method: %v, Queue: %v)", - method, - requestQueue->GetName()); +void TServiceBase::RegisterRequestQueue( + TRuntimeMethodInfo* runtimeInfo, + TRequestQueue* requestQueue) +{ + if (!requestQueue->Register(this, runtimeInfo)) { + return; + } - auto profiler = runtimeInfo->Profiler.WithSparse(); - if (runtimeInfo->Descriptor.RequestQueueProvider) { - profiler = profiler.WithTag("queue", requestQueue->GetName()); - } - profiler.AddFuncGauge("/request_queue_size", MakeStrong(this), [=] { - return requestQueue->GetQueueSize(); - }); - profiler.AddFuncGauge("/request_queue_byte_size", MakeStrong(this), [=] { - return requestQueue->GetQueueByteSize(); - }); - profiler.AddFuncGauge("/concurrency", MakeStrong(this), [=] { - return requestQueue->GetConcurrency(); - }); - profiler.AddFuncGauge("/concurrency_byte", MakeStrong(this), [=] { - return requestQueue->GetConcurrencyByte(); - }); + const auto& method = runtimeInfo->Descriptor.Method; + YT_LOG_DEBUG("Request queue registered (Method: %v, Queue: %v)", + method, + requestQueue->GetName()); - TMethodConfigPtr methodConfig; - if (auto config = Config_.Acquire()) { - methodConfig = GetOrDefault(config->Methods, method); - } - ConfigureRequestQueue(runtimeInfo, requestQueue, methodConfig); + auto profiler = runtimeInfo->Profiler.WithSparse(); + if (runtimeInfo->Descriptor.RequestQueueProvider) { + profiler = profiler.WithTag("queue", requestQueue->GetName()); + } + profiler.AddFuncGauge("/request_queue_size", MakeStrong(this), [=] { + return requestQueue->GetQueueSize(); + }); + profiler.AddFuncGauge("/request_queue_byte_size", MakeStrong(this), [=] { + return requestQueue->GetQueueByteSize(); + }); + profiler.AddFuncGauge("/concurrency", MakeStrong(this), [=] { + return requestQueue->GetConcurrency(); + }); + profiler.AddFuncGauge("/concurrency_byte", MakeStrong(this), [=] { + return requestQueue->GetConcurrencyByte(); + }); - { - auto guard = Guard(runtimeInfo->RequestQueuesLock); - runtimeInfo->RequestQueues.push_back(requestQueue); - } + TMethodConfigPtr methodConfig; + if (auto config = Config_.Acquire()) { + methodConfig = GetOrDefault(config->Methods, method); } + ConfigureRequestQueue(runtimeInfo, requestQueue, methodConfig); - return requestQueue; + { + auto guard = Guard(runtimeInfo->RequestQueuesLock); + runtimeInfo->RequestQueues.push_back(requestQueue); + } } void TServiceBase::ConfigureRequestQueue( @@ -2694,13 +2696,13 @@ TFuture<void> TServiceBase::Stop() return StopResult_.ToFuture(); } -TServiceBase::TRuntimeMethodInfo* TServiceBase::FindMethodInfo(const std::string& method) +TServiceBase::TRuntimeMethodInfo* TServiceBase::FindMethodInfo(const TString& method) { auto it = MethodMap_.find(method); return it == MethodMap_.end() ? nullptr : it->second.Get(); } -TServiceBase::TRuntimeMethodInfo* TServiceBase::GetMethodInfoOrThrow(const std::string& method) +TServiceBase::TRuntimeMethodInfo* TServiceBase::GetMethodInfoOrThrow(const TString& method) { auto* runtimeInfo = FindMethodInfo(method); if (!runtimeInfo) { diff --git a/yt/yt/core/rpc/service_detail.h b/yt/yt/core/rpc/service_detail.h index 4641c2f98a..ab32bce755 100644 --- a/yt/yt/core/rpc/service_detail.h +++ b/yt/yt/core/rpc/service_detail.h @@ -657,12 +657,11 @@ protected: TMethodDescriptor SetHandleMethodError(bool value) const; }; - class TErrorCodeCounters + struct TErrorCodeCounter { - public: - explicit TErrorCodeCounters(NProfiling::TProfiler profiler); + explicit TErrorCodeCounter(NProfiling::TProfiler profiler); - NProfiling::TCounter* GetCounter(TErrorCode code); + void Increment(TErrorCode code); private: const NProfiling::TProfiler Profiler_; @@ -718,7 +717,7 @@ protected: NProfiling::TCounter ResponseMessageAttachmentSizeCounter; //! Counts the number of errors, per error code. - TErrorCodeCounters ErrorCodeCounters; + TErrorCodeCounter ErrorCodeCounter; }; using TMethodPerformanceCountersPtr = TIntrusivePtr<TMethodPerformanceCounters>; @@ -764,24 +763,13 @@ protected: using TNonowningPerformanceCountersKey = std::tuple<TStringBuf, TRequestQueue*>; using TOwningPerformanceCountersKey = std::tuple<TString, TRequestQueue*>; using TPerformanceCountersKeyHash = THash<TNonowningPerformanceCountersKey>; - - struct TPerformanceCountersKeyEquals - { - bool operator()( - const TNonowningPerformanceCountersKey& lhs, - const TNonowningPerformanceCountersKey& rhs) const; - bool operator()( - const TOwningPerformanceCountersKey& lhs, - const TNonowningPerformanceCountersKey& rhs) const; - }; - + struct TPerformanceCountersKeyEquals; using TPerformanceCountersMap = NConcurrency::TSyncMap< TOwningPerformanceCountersKey, TMethodPerformanceCountersPtr, TPerformanceCountersKeyHash, TPerformanceCountersKeyEquals >; - TPerformanceCountersMap PerformanceCountersMap; TMethodPerformanceCountersPtr BasePerformanceCounters; TMethodPerformanceCountersPtr RootPerformanceCounters; @@ -801,9 +789,16 @@ protected: : public TRefCounted { public: - explicit TPerformanceCounters(const NProfiling::TProfiler& profiler); + explicit TPerformanceCounters(const NProfiling::TProfiler& profiler) + : Profiler_(profiler.WithHot().WithSparse()) + { } - NProfiling::TCounter* GetRequestsPerUserAgentCounter(TStringBuf userAgent); + void IncrementRequestsPerUserAgent(TStringBuf userAgent) + { + RequestsPerUserAgent_.FindOrInsert(userAgent, [&] { + return Profiler_.WithRequiredTag("user_agent", TString(userAgent)).Counter("/user_agent"); + }).first->Increment(); + } private: const NProfiling::TProfiler Profiler_; @@ -851,10 +846,10 @@ protected: //! Returns a (non-owning!) pointer to TRuntimeMethodInfo for a given method's name //! or |nullptr| if no such method is registered. - TRuntimeMethodInfo* FindMethodInfo(const std::string& method); + TRuntimeMethodInfo* FindMethodInfo(const TString& method); //! Similar to #FindMethodInfo but throws if no method is found. - TRuntimeMethodInfo* GetMethodInfoOrThrow(const std::string& method); + TRuntimeMethodInfo* GetMethodInfoOrThrow(const TString& method); //! Returns the default invoker passed during construction. const IInvokerPtr& GetDefaultInvoker() const; @@ -996,7 +991,6 @@ private: struct TAcceptedRequest { - TInstant ArriveInstant; TRequestId RequestId; NYT::NBus::IBusPtr ReplyBus; TRuntimeMethodInfo* RuntimeInfo; @@ -1004,7 +998,6 @@ private: std::unique_ptr<NRpc::NProto::TRequestHeader> Header; TSharedRefArray Message; TRequestQueue* RequestQueue; - TMethodPerformanceCounters* MethodPerformanceCounters; std::optional<TError> ThrottledError; TMemoryUsageTrackerGuard MemoryGuard; IMemoryUsageTrackerPtr MemoryUsageTracker; @@ -1029,6 +1022,9 @@ private: TRequestQueue* GetRequestQueue( TRuntimeMethodInfo* runtimeInfo, const NRpc::NProto::TRequestHeader& requestHeader); + void RegisterRequestQueue( + TRuntimeMethodInfo* runtimeInfo, + TRequestQueue* requestQueue); void ConfigureRequestQueue( TRuntimeMethodInfo* runtimeInfo, TRequestQueue* requestQueue, @@ -1075,7 +1071,6 @@ private: static TString GetDiscoverRequestPayload(const TCtxDiscoverPtr& context); void OnServiceLivenessCheck(); - }; DEFINE_REFCOUNTED_TYPE(TServiceBase) @@ -1086,9 +1081,7 @@ class TRequestQueue : public TRefCounted { public: - TRequestQueue( - const std::string& name, - const NProfiling::TProfiler& profiler); + TRequestQueue(const std::string& name, NProfiling::TProfiler profiler); bool Register(TServiceBase* service, TServiceBase::TRuntimeMethodInfo* runtimeInfo); void Configure(const TMethodConfigPtr& config); diff --git a/yt/yt/library/formats/arrow_writer.cpp b/yt/yt/library/formats/arrow_writer.cpp index 657da97ce5..a3ff967bc6 100644 --- a/yt/yt/library/formats/arrow_writer.cpp +++ b/yt/yt/library/formats/arrow_writer.cpp @@ -49,14 +49,14 @@ const TString AlignmentString(ArrowAlignment, 0); flatbuffers::Offset<flatbuffers::String> SerializeString( flatbuffers::FlatBufferBuilder* flatbufBuilder, - const TString& str) + const std::string& str) { return flatbufBuilder->CreateString(str.data(), str.length()); } std::tuple<org::apache::arrow::flatbuf::Type, flatbuffers::Offset<void>> SerializeColumnType( flatbuffers::FlatBufferBuilder* flatbufBuilder, - TColumnSchema schema) + const TColumnSchema& schema) { auto simpleType = CastToV1Type(schema.LogicalType()).first; switch (simpleType) { diff --git a/yt/yt/library/formats/protobuf.cpp b/yt/yt/library/formats/protobuf.cpp index 3da3f04b48..a710bf9e61 100644 --- a/yt/yt/library/formats/protobuf.cpp +++ b/yt/yt/library/formats/protobuf.cpp @@ -1013,7 +1013,8 @@ void TProtobufFormatDescriptionBase<TType>::InitFromProtobufSchema( field->GetType()); } const auto& enumerationConfig = field->AsMap(); - EnumerationDescriptionMap_.emplace(name, CreateEnumerationMap(TimestampColumnName, enumerationConfig)); + // TODO(babenko): migrate to std::string + EnumerationDescriptionMap_.emplace(name, CreateEnumerationMap(TString(TimestampColumnName), enumerationConfig)); } } diff --git a/yt/yt/library/formats/schemaful_dsv_parser.cpp b/yt/yt/library/formats/schemaful_dsv_parser.cpp index 3149f28851..7137e2672d 100644 --- a/yt/yt/library/formats/schemaful_dsv_parser.cpp +++ b/yt/yt/library/formats/schemaful_dsv_parser.cpp @@ -26,9 +26,9 @@ public: void Finish() override; private: - IYsonConsumer* Consumer_; - TSchemafulDsvFormatConfigPtr Config_; - const std::vector<TString>& Columns_; + IYsonConsumer* const Consumer_; + const TSchemafulDsvFormatConfigPtr Config_; + const std::vector<std::string>& Columns_; TEscapeTable EscapeTable_; @@ -49,8 +49,8 @@ private: //////////////////////////////////////////////////////////////////////////////// TSchemafulDsvParser::TSchemafulDsvParser( - IYsonConsumer* consumer, - TSchemafulDsvFormatConfigPtr config) + IYsonConsumer* consumer, + TSchemafulDsvFormatConfigPtr config) : Consumer_(consumer) , Config_(config) , Columns_(Config_->GetColumnsOrThrow()) diff --git a/yt/yt/library/formats/schemaful_dsv_writer.cpp b/yt/yt/library/formats/schemaful_dsv_writer.cpp index 4d2e7e1e6e..b37486c581 100644 --- a/yt/yt/library/formats/schemaful_dsv_writer.cpp +++ b/yt/yt/library/formats/schemaful_dsv_writer.cpp @@ -266,9 +266,9 @@ private: //////////////////////////////////////////////////////////////////////////////// -void ValidateDuplicateColumns(const std::vector<TString>& columns) +void ValidateDuplicateColumns(const std::vector<std::string>& columns) { - THashSet<TString> names; + THashSet<std::string> names; for (const auto& name : columns) { if (!names.insert(name).second) { THROW_ERROR_EXCEPTION("Duplicate column name %Qv in schemaful DSV config", diff --git a/yt/yt/library/formats/web_json_writer.cpp b/yt/yt/library/formats/web_json_writer.cpp index d717cafb10..8049e134c9 100644 --- a/yt/yt/library/formats/web_json_writer.cpp +++ b/yt/yt/library/formats/web_json_writer.cpp @@ -48,7 +48,7 @@ static constexpr auto ContextBufferCapacity = 1_MB; class TWebJsonColumnFilter { public: - TWebJsonColumnFilter(int maxSelectedColumnCount, std::optional<THashSet<TString>> names) + TWebJsonColumnFilter(int maxSelectedColumnCount, std::optional<THashSet<std::string, THash<TStringBuf>, TEqualTo<>>> names) : MaxSelectedColumnCount_(maxSelectedColumnCount) , Names_(std::move(names)) { } @@ -68,7 +68,7 @@ public: private: const int MaxSelectedColumnCount_; - std::optional<THashSet<TString>> Names_; + const std::optional<THashSet<std::string, THash<TStringBuf>, TEqualTo<>>> Names_; THashSet<ui16> AcceptedColumnIds_; @@ -91,7 +91,7 @@ private: TWebJsonColumnFilter CreateWebJsonColumnFilter(const TWebJsonFormatConfigPtr& webJsonConfig) { - std::optional<THashSet<TString>> columnNames; + std::optional<THashSet<std::string, THash<TStringBuf>, TEqualTo<>>> columnNames; if (webJsonConfig->ColumnNames) { columnNames.emplace(); for (const auto& columnName : *webJsonConfig->ColumnNames) { diff --git a/yt/yt/library/formats/yamred_dsv_parser.cpp b/yt/yt/library/formats/yamred_dsv_parser.cpp index 476e760ea6..79246cb92a 100644 --- a/yt/yt/library/formats/yamred_dsv_parser.cpp +++ b/yt/yt/library/formats/yamred_dsv_parser.cpp @@ -17,39 +17,39 @@ class TYamredDsvParserConsumer public: TYamredDsvParserConsumer(IYsonConsumer* consumer, TYamredDsvFormatConfigPtr config) : TYamrConsumerBase(consumer) - , Config(config) - , DsvParser(CreateParserForDsv(consumer, ConvertTo<TDsvFormatConfigPtr>(Config), /*wrapWithMap*/ false)) + , Config_(config) + , DsvParser_(CreateParserForDsv(consumer, ConvertTo<TDsvFormatConfigPtr>(Config_), /*wrapWithMap*/ false)) { } void ConsumeKey(TStringBuf key) override { Consumer->OnListItem(); Consumer->OnBeginMap(); - ConsumeFields(Config->KeyColumnNames, key); + ConsumeFields(Config_->KeyColumnNames, key); } void ConsumeSubkey(TStringBuf subkey) override { - ConsumeFields(Config->SubkeyColumnNames, subkey); + ConsumeFields(Config_->SubkeyColumnNames, subkey); } void ConsumeValue(TStringBuf value) override { - DsvParser->Read(value); - DsvParser->Finish(); + DsvParser_->Read(value); + DsvParser_->Finish(); Consumer->OnEndMap(); } private: - TYamredDsvFormatConfigPtr Config; - std::unique_ptr<IParser> DsvParser; + const TYamredDsvFormatConfigPtr Config_; + const std::unique_ptr<IParser> DsvParser_; void ConsumeFields( - const std::vector<TString>& fieldNames, + const std::vector<std::string>& fieldNames, TStringBuf wholeField) { static const char* emptyString = ""; - char delimiter = Config->YamrKeysSeparator; + char delimiter = Config_->YamrKeysSeparator; std::vector<TStringBuf> fields; if (wholeField.length() == 0) { diff --git a/yt/yt/library/named_value/named_value.h b/yt/yt/library/named_value/named_value.h index 3a76346589..67592fe9f3 100644 --- a/yt/yt/library/named_value/named_value.h +++ b/yt/yt/library/named_value/named_value.h @@ -38,11 +38,13 @@ NTableClient::TUnversionedOwningRow MakeRow( class TNamedValue { public: - struct TAny { + struct TAny + { TString Value; }; - struct TComposite { + struct TComposite + { TString Value; }; diff --git a/yt/yt/library/numeric/unittests/piecewise_linear_function_ut.cpp b/yt/yt/library/numeric/unittests/piecewise_linear_function_ut.cpp index 7fde320ec0..4ea3e3e815 100644 --- a/yt/yt/library/numeric/unittests/piecewise_linear_function_ut.cpp +++ b/yt/yt/library/numeric/unittests/piecewise_linear_function_ut.cpp @@ -810,7 +810,7 @@ TEST_F(TPiecewiseLinearFunctionTest, TestCompose) { /* Name */ "compositionLinearWithLinearInjection", /* Lhs */ TPiecewiseLinearFunction<double>::Linear(0.1, 17, 1.2, 42), - /* Rhs */ TPiecewiseLinearFunction<double>::Linear(0, 0.3, 100, 0.98), + /* Rhs */ TPiecewiseLinearFunction<double>::Linear(0, 0.3, 100, 0.87), /* Samples */ { { /* Argument */ 0, @@ -819,15 +819,15 @@ TEST_F(TPiecewiseLinearFunctionTest, TestCompose) }, { /* Argument */ 100, - /* ExpectedLeftLimit */ 37, - /* ExpectedRightLimit */ 37, + /* ExpectedLeftLimit */ 34.5, + /* ExpectedRightLimit */ 34.5, } } }, { /* Name */ "compositionLinearWithDiscontinuous", /* Lhs */ TPiecewiseLinearFunction<double>::Linear(0.1, 17, 1.2, 42), - /* Rhs */ BuildFunctionFromPoints<double>({{0, 0.1}, {1, 0.3}, {1, 0.98}, {2, 1.2}}), + /* Rhs */ BuildFunctionFromPoints<double>({{0, 0.1}, {1, 0.3}, {1, 0.87}, {2, 1.2}}), /* Samples */ { { /* Argument */ 0, @@ -837,7 +837,7 @@ TEST_F(TPiecewiseLinearFunctionTest, TestCompose) { /* Argument */ 1, /* ExpectedLeftLimit */ 21.5454545454545454545454545454545454545454545454, - /* ExpectedRightLimit */ 37, + /* ExpectedRightLimit */ 34.5, }, { /* Argument */ 2, diff --git a/yt/yt/library/oom/oom.h b/yt/yt/library/oom/oom.h index 11edd7ca02..33a9206398 100644 --- a/yt/yt/library/oom/oom.h +++ b/yt/yt/library/oom/oom.h @@ -23,6 +23,10 @@ void EnableEarlyOomWatchdog(TOomWatchdogOptions options); struct TTCMallocLimitHandlerOptions { TString HeapDumpDirectory; + + // Files structure would have the following form: + // HeapDumpDirectory/<ActualName>_FilenameSuffix_Timestamp.ext. + TString FilenameSuffix = ""; TDuration Timeout = TDuration::Minutes(5); }; diff --git a/yt/yt/library/oom/tcmalloc_memory_limit_handler.cpp b/yt/yt/library/oom/tcmalloc_memory_limit_handler.cpp index 38e7f267a8..5710629d19 100644 --- a/yt/yt/library/oom/tcmalloc_memory_limit_handler.cpp +++ b/yt/yt/library/oom/tcmalloc_memory_limit_handler.cpp @@ -35,6 +35,13 @@ namespace NYT { //////////////////////////////////////////////////////////////////////////////// +TString MakeIncompletePath(const TString& path) +{ + return NYT::Format("%v_incomplete", path); +} + +//////////////////////////////////////////////////////////////////////////////// + void CollectAndDumpMemoryProfile(const TString& memoryProfilePath, tcmalloc::ProfileType profileType) { auto profile = NYTProf::ReadHeapProfile(profileType); @@ -45,9 +52,12 @@ void CollectAndDumpMemoryProfile(const TString& memoryProfilePath, tcmalloc::Pro }, }); - TFileOutput output(memoryProfilePath); + auto incompletePath = NYT::MakeIncompletePath(memoryProfilePath); + + TFileOutput output(incompletePath); NYTProf::WriteProfile(&output, profile); output.Finish(); + NFs::Rename(incompletePath, memoryProfilePath); } //////////////////////////////////////////////////////////////////////////////// @@ -185,12 +195,22 @@ private: AbortProcess(ToUnderlying(EProcessExitCode::OK)); } + auto MakeSuffixFormatter(const TString& timestamp) const + { + return NYT::MakeFormatterWrapper([this, ×tamp] (TStringBuilderBase* builder) { + if (Options_.FilenameSuffix) { + builder->AppendFormat("%v_", Options_.FilenameSuffix); + } + FormatValue(builder, timestamp, "v"); + }); + } + TString GetHeapDumpPath(const TString& timestamp) const { return Format( "%v/heap_%v.pb.gz", Options_.HeapDumpDirectory, - timestamp); + MakeSuffixFormatter(timestamp)); } TString GetPeakDumpPath(const TString& timestamp) const @@ -198,7 +218,7 @@ private: return Format( "%v/peak_%v.pb.gz", Options_.HeapDumpDirectory, - timestamp); + MakeSuffixFormatter(timestamp)); } TString GetProfilePaths(const TString& timestamp) const @@ -206,7 +226,7 @@ private: return Format( "%v/oom_profile_paths_%v.yson", Options_.HeapDumpDirectory, - timestamp); + MakeSuffixFormatter(timestamp)); } void ExecWaitForChild(int pid) diff --git a/yt/yt/library/profiling/sensors_owner/README.md b/yt/yt/library/profiling/sensors_owner/README.md index 48e9ed83d9..66aa91bd42 100644 --- a/yt/yt/library/profiling/sensors_owner/README.md +++ b/yt/yt/library/profiling/sensors_owner/README.md @@ -8,7 +8,7 @@ TSensorsOwner может владеть другими TSensorsOwner. Имеет * Простейший пример использования: ```cpp -sensorsOwner.Inc(".my_simple_counter", 1); +sensorsOwner.Inc("/my_simple_counter", 1); ``` Когда в конкретном месте нужно проинкрементить всего один счетчик. Объект счетчика в этом случае создатся один раз и будет храниться внутри sensorsOwner. @@ -22,8 +22,8 @@ void DoSmth(/*... , */ const TSensorsOwner& sensorsOwner) struct TSensors { NYT::NProfiling::TProfiler Profiler; - NYT::NProfiling::TCounter TotalCount = Profiler.Counter(".count"); - NYT::NProfiling::TCounter FailedCount = Profiler.Counter(".failed_count"); + NYT::NProfiling::TCounter TotalCount = Profiler.Counter("/total_count"); + NYT::NProfiling::TCounter FailedCount = Profiler.Counter("/failed_count"); }; // Тут одна и та же ссылка на объект метрик при условии, что в функцию передается один и тот же sensorsOwner. // Метод `.Get` достаточно эффективен, но всё же лучше не вызывать лишний раз. @@ -47,7 +47,7 @@ struct THistogramSensors NYT::NProfiling::TProfiler Profiler; int Key; std::vector<TDuration> Buckets; - NYT::NProfiling::TEventTimer Histogram = Profiler.WithTag("tag", ToString(Key)).TimeHistogram(".another_counter", Buckets); + NYT::NProfiling::TEventTimer Histogram = Profiler.WithTag("tag", ToString(Key)).TimeHistogram("/another_counter", Buckets); }; owner.Get<THistogramSensors>(/*Key*/ 132, /*Buckets*/ std::vector<TDuration>{5s, 10min}).Histogram.Record(6s); @@ -60,7 +60,7 @@ struct TChildSensors NYT::NProfiling::TCounter Counter; TChildSensors(const NYT::NProfiling::TProfiler& p) - : Counter(p.Counter(".my_counter_2")) + : Counter(p.Counter("/my_counter_2")) { } }; ``` @@ -70,7 +70,7 @@ struct TChildSensors struct TSharedSensors final { TProfiler Profiler; - TCounter Counter = Profiler.Counter(".under_ptr_counter"); + TCounter Counter = Profiler.Counter("/under_ptr_counter"); }; using TSharedSensorsPtr = NYT::TIntrusivePtr<TSharedSensors>; @@ -79,7 +79,7 @@ owner.Get<TSharedSensorsPtr>()->Counter.Increment(1); * TSensorsOwner мимикрирует под TProfiler в ряде моментов: ```cpp -auto subOwner = owner.WithPrefix("prefix.").WithTags(NYT::NProfiling::TTagSet().WithTag({"key", "value2"})); +auto subOwner = owner.WithPrefix("/prefix").WithTags(NYT::NProfiling::TTagSet().WithTag({"key", "value2"})); ``` ## Когда использовать? diff --git a/yt/yt/library/skiff_ext/parser-inl.h b/yt/yt/library/skiff_ext/parser-inl.h index 0f781646c9..a48498ba25 100644 --- a/yt/yt/library/skiff_ext/parser-inl.h +++ b/yt/yt/library/skiff_ext/parser-inl.h @@ -22,8 +22,8 @@ public: TConsumer* consumer, TSkiffSchemaList skiffSchemaList, const std::vector<TSkiffTableColumnIds>& tablesColumnIds, - const TString& rangeIndexColumnName, - const TString& rowIndexColumnName) + const std::string& rangeIndexColumnName, + const std::string& rowIndexColumnName) : Consumer_(consumer) , SkiffSchemaList_(std::move(skiffSchemaList)) { @@ -183,8 +183,8 @@ TSkiffMultiTableParser<TConsumer>::TSkiffMultiTableParser( TConsumer* consumer, TSkiffSchemaList schemaList, const std::vector<TSkiffTableColumnIds>& tablesColumnIds, - const TString& rangeIndexColumnName, - const TString& rowIndexColumnName) + const std::string& rangeIndexColumnName, + const std::string& rowIndexColumnName) : ParserImpl_(new TImpl(consumer, schemaList, tablesColumnIds, diff --git a/yt/yt/library/skiff_ext/parser.h b/yt/yt/library/skiff_ext/parser.h index d9fb4c43fa..82ad4cea50 100644 --- a/yt/yt/library/skiff_ext/parser.h +++ b/yt/yt/library/skiff_ext/parser.h @@ -19,8 +19,8 @@ public: TConsumer* consumer, NSkiff::TSkiffSchemaList schemaList, const std::vector<TSkiffTableColumnIds>& tablesColumnIds, - const TString& rangeIndexColumnName, - const TString& rowIndexColumnName); + const std::string& rangeIndexColumnName, + const std::string& rowIndexColumnName); ~TSkiffMultiTableParser(); diff --git a/yt/yt/library/skiff_ext/schema_match.cpp b/yt/yt/library/skiff_ext/schema_match.cpp index 7587b8b8e9..390e756798 100644 --- a/yt/yt/library/skiff_ext/schema_match.cpp +++ b/yt/yt/library/skiff_ext/schema_match.cpp @@ -20,7 +20,7 @@ const TString SparseColumnsName = "$sparse_columns"; //////////////////////////////////////////////////////////////////////////////// -static void ThrowInvalidSkiffTypeError(const TString& columnName, std::shared_ptr<TSkiffSchema> expectedType, std::shared_ptr<TSkiffSchema> actualType) +static void ThrowInvalidSkiffTypeError(const std::string& columnName, std::shared_ptr<TSkiffSchema> expectedType, std::shared_ptr<TSkiffSchema> actualType) { THROW_ERROR_EXCEPTION("Column %Qv has unexpected Skiff type: expected %Qv, found type %Qv", columnName, @@ -56,13 +56,13 @@ static ERowRangeIndexMode GetRowRangeIndexMode(const std::shared_ptr<TSkiffSchem static bool IsSkiffSpecialColumn( TStringBuf columnName, - const TString& rangeIndexColumnName, - const TString& rowIndexColumnName) + const std::string& rangeIndexColumnName, + const std::string& rowIndexColumnName) { - static const THashSet<TString> specialColumns = { + static const THashSet<std::string, THash<TStringBuf>, TEqualTo<>> specialColumns{ KeySwitchColumnName, OtherColumnsName, - SparseColumnsName + SparseColumnsName, }; return specialColumns.contains(columnName) || columnName == rangeIndexColumnName || columnName == rowIndexColumnName; } @@ -85,8 +85,8 @@ static std::pair<std::shared_ptr<TSkiffSchema>, bool> DeoptionalizeSchema(std::s static TSkiffTableDescription CreateTableDescription( const std::shared_ptr<TSkiffSchema>& skiffSchema, - const TString& rangeIndexColumnName, - const TString& rowIndexColumnName) + const std::string& rangeIndexColumnName, + const std::string& rowIndexColumnName) { TSkiffTableDescription result; THashSet<TString> topLevelNames; @@ -186,8 +186,8 @@ static TSkiffTableDescription CreateTableDescription( std::vector<TSkiffTableDescription> CreateTableDescriptionList( const std::vector<std::shared_ptr<TSkiffSchema>>& skiffSchemas, - const TString& rangeIndexColumnName, - const TString& rowIndexColumnName) + const std::string& rangeIndexColumnName, + const std::string& rowIndexColumnName) { std::vector<TSkiffTableDescription> result; for (ui16 index = 0; index < skiffSchemas.size(); ++index) { diff --git a/yt/yt/library/skiff_ext/schema_match.h b/yt/yt/library/skiff_ext/schema_match.h index dc5a6c2dbc..d7183c6ce3 100644 --- a/yt/yt/library/skiff_ext/schema_match.h +++ b/yt/yt/library/skiff_ext/schema_match.h @@ -76,8 +76,8 @@ struct TSkiffTableColumnIds std::vector<TSkiffTableDescription> CreateTableDescriptionList( const std::vector<std::shared_ptr<NSkiff::TSkiffSchema>>& skiffSchema, - const TString& rangeIndexColumnName, - const TString& rowIndexColumnName); + const std::string& rangeIndexColumnName, + const std::string& rowIndexColumnName); std::vector<std::shared_ptr<NSkiff::TSkiffSchema>> ParseSkiffSchemas( const NYTree::IMapNodePtr& skiffSchemaRegistry, |