aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorAlexander Smirnov <alex@ydb.tech>2024-10-31 17:11:37 +0000
committerAlexander Smirnov <alex@ydb.tech>2024-10-31 17:11:37 +0000
commita4a4e847216dbe1e32056717eb466537d2128bce (patch)
tree27070fcd1102d57042f56b8eb602789b020a1d04
parente13dea6e57e441f5dc2fe09409a2932cdc4f821c (diff)
parent6c9f2f9532a9812c29ae9f3ed08ad266b93993c0 (diff)
downloadydb-a4a4e847216dbe1e32056717eb466537d2128bce.tar.gz
Merge branch 'rightlib' into mergelibs-241031-1710
-rw-r--r--build/export_generators/cmake/generator.toml4
-rw-r--r--build/export_generators/ide-gradle/build.gradle.kts.jinja56
-rw-r--r--build/export_generators/ide-gradle/build.gradle.kts.proto.jinja28
-rw-r--r--build/export_generators/ide-gradle/generator.toml5
-rw-r--r--build/export_generators/ide-gradle/settings.gradle.kts.jinja2
-rw-r--r--build/external_resources/yexport/public.resources.json6
-rw-r--r--build/external_resources/yexport/resources.json6
-rw-r--r--build/external_resources/ymake/public.resources.json10
-rw-r--r--build/external_resources/ymake/resources.json10
-rw-r--r--build/mapping.conf.json26
-rw-r--r--build/plugins/nots.py4
-rw-r--r--build/scripts/_fake_src.cpp3
-rw-r--r--build/ymake.core.conf7
-rw-r--r--contrib/libs/cxxsupp/libcxxrt/ya.make4
-rw-r--r--contrib/libs/protobuf/patches/dont-use-string-view.patch56
-rw-r--r--contrib/libs/protobuf/patches/dot-generate-string-view-setters.patch21
-rw-r--r--contrib/libs/protobuf/patches/fix-string-from-nullptr.patch19
-rw-r--r--contrib/libs/protobuf/patches/protoc-dont-generate-setters-with-string-view.patch40
-rw-r--r--contrib/libs/protobuf/patches/protoc-resolve-string-method-ambiguity.patch26
-rw-r--r--contrib/libs/protobuf/patches/resolve-string-method-ambiguity.patch30
-rw-r--r--contrib/libs/protobuf/patches/z_05_use_ref_count.patch6
-rw-r--r--contrib/libs/protobuf/src/google/protobuf/arenastring.cc5
-rw-r--r--contrib/libs/protobuf/src/google/protobuf/arenastring.h19
-rw-r--r--contrib/libs/protoc/src/google/protobuf/compiler/cpp/field_generators/string_field.cc18
-rw-r--r--contrib/python/anyio/.dist-info/METADATA5
-rw-r--r--contrib/python/anyio/anyio/_backends/_asyncio.py277
-rw-r--r--contrib/python/anyio/anyio/_backends/_trio.py28
-rw-r--r--contrib/python/anyio/anyio/_core/_fileio.py3
-rw-r--r--contrib/python/anyio/anyio/_core/_signals.py6
-rw-r--r--contrib/python/anyio/anyio/_core/_streams.py4
-rw-r--r--contrib/python/anyio/anyio/_core/_subprocesses.py21
-rw-r--r--contrib/python/anyio/anyio/abc/_eventloop.py8
-rw-r--r--contrib/python/anyio/anyio/abc/_sockets.py8
-rw-r--r--contrib/python/anyio/anyio/from_thread.py16
-rw-r--r--contrib/python/anyio/anyio/pytest_plugin.py4
-rw-r--r--contrib/python/anyio/anyio/streams/tls.py6
-rw-r--r--contrib/python/anyio/ya.make2
-rw-r--r--contrib/python/pyparsing/py3/.dist-info/METADATA7
-rw-r--r--contrib/python/pyparsing/py3/pyparsing/__init__.py13
-rw-r--r--contrib/python/pyparsing/py3/pyparsing/core.py403
-rw-r--r--contrib/python/pyparsing/py3/pyparsing/diagram/__init__.py99
-rw-r--r--contrib/python/pyparsing/py3/pyparsing/exceptions.py85
-rw-r--r--contrib/python/pyparsing/py3/pyparsing/helpers.py62
-rw-r--r--contrib/python/pyparsing/py3/pyparsing/results.py70
-rw-r--r--contrib/python/pyparsing/py3/pyparsing/testing.py39
-rw-r--r--contrib/python/pyparsing/py3/pyparsing/unicode.py10
-rw-r--r--contrib/python/pyparsing/py3/pyparsing/util.py212
-rw-r--r--contrib/python/pyparsing/py3/ya.make2
-rw-r--r--contrib/python/xmltodict/py3/.dist-info/METADATA24
-rw-r--r--contrib/python/xmltodict/py3/README.md13
-rw-r--r--contrib/python/xmltodict/py3/tests/__init__.py0
-rw-r--r--contrib/python/xmltodict/py3/tests/test_dicttoxml.py22
-rw-r--r--contrib/python/xmltodict/py3/tests/test_xmltodict.py22
-rw-r--r--contrib/python/xmltodict/py3/xmltodict.py66
-rw-r--r--contrib/python/xmltodict/py3/ya.make2
-rw-r--r--contrib/tools/protoc/resources.json10
-rw-r--r--library/cpp/blockcodecs/core/codecs.h6
-rw-r--r--library/cpp/tld/tlds-alpha-by-domain.txt2
-rw-r--r--library/cpp/yt/logging/backends/stream/stream_log_manager.cpp6
-rw-r--r--library/cpp/yt/logging/logger.cpp13
-rw-r--r--library/cpp/yt/logging/logger.h51
-rw-r--r--library/cpp/yt/yson_string/convert.cpp6
-rw-r--r--library/cpp/yt/yson_string/convert.h2
-rw-r--r--vendor/github.com/golang/protobuf/internal/gengogrpc/ya.make2
-rw-r--r--vendor/github.com/golang/protobuf/protoc-gen-go/ya.make2
-rw-r--r--vendor/github.com/spf13/cobra/ya.make2
-rw-r--r--vendor/github.com/spf13/pflag/ya.make2
-rw-r--r--vendor/google.golang.org/protobuf/cmd/protoc-gen-go/internal_gengo/ya.make2
-rw-r--r--vendor/google.golang.org/protobuf/compiler/protogen/ya.make2
-rw-r--r--vendor/google.golang.org/protobuf/encoding/prototext/ya.make2
-rw-r--r--vendor/google.golang.org/protobuf/encoding/protowire/ya.make2
-rw-r--r--vendor/google.golang.org/protobuf/internal/descfmt/ya.make2
-rw-r--r--vendor/google.golang.org/protobuf/internal/descopts/ya.make2
-rw-r--r--vendor/google.golang.org/protobuf/internal/detrand/ya.make2
-rw-r--r--vendor/google.golang.org/protobuf/internal/editiondefaults/ya.make2
-rw-r--r--vendor/google.golang.org/protobuf/internal/editionssupport/ya.make2
-rw-r--r--vendor/google.golang.org/protobuf/internal/encoding/defval/ya.make2
-rw-r--r--vendor/google.golang.org/protobuf/internal/encoding/messageset/ya.make2
-rw-r--r--vendor/google.golang.org/protobuf/internal/encoding/tag/ya.make2
-rw-r--r--vendor/google.golang.org/protobuf/internal/encoding/text/ya.make2
-rw-r--r--vendor/google.golang.org/protobuf/internal/errors/ya.make2
-rw-r--r--vendor/google.golang.org/protobuf/internal/filedesc/ya.make2
-rw-r--r--vendor/google.golang.org/protobuf/internal/filetype/ya.make2
-rw-r--r--vendor/google.golang.org/protobuf/internal/flags/ya.make2
-rw-r--r--vendor/google.golang.org/protobuf/internal/genid/ya.make2
-rw-r--r--vendor/google.golang.org/protobuf/internal/impl/ya.make2
-rw-r--r--vendor/google.golang.org/protobuf/internal/msgfmt/ya.make2
-rw-r--r--vendor/google.golang.org/protobuf/internal/order/ya.make2
-rw-r--r--vendor/google.golang.org/protobuf/internal/pragma/ya.make2
-rw-r--r--vendor/google.golang.org/protobuf/internal/set/ya.make2
-rw-r--r--vendor/google.golang.org/protobuf/internal/strs/ya.make2
-rw-r--r--vendor/google.golang.org/protobuf/internal/version/ya.make2
-rw-r--r--vendor/google.golang.org/protobuf/proto/ya.make2
-rw-r--r--vendor/google.golang.org/protobuf/reflect/protodesc/ya.make2
-rw-r--r--vendor/google.golang.org/protobuf/reflect/protopath/ya.make2
-rw-r--r--vendor/google.golang.org/protobuf/reflect/protorange/ya.make2
-rw-r--r--vendor/google.golang.org/protobuf/reflect/protoreflect/ya.make2
-rw-r--r--vendor/google.golang.org/protobuf/reflect/protoregistry/ya.make2
-rw-r--r--vendor/google.golang.org/protobuf/runtime/protoiface/ya.make2
-rw-r--r--vendor/google.golang.org/protobuf/runtime/protoimpl/ya.make2
-rw-r--r--vendor/google.golang.org/protobuf/types/descriptorpb/ya.make2
-rw-r--r--vendor/google.golang.org/protobuf/types/dynamicpb/ya.make2
-rw-r--r--vendor/google.golang.org/protobuf/types/gofeaturespb/ya.make2
-rw-r--r--vendor/google.golang.org/protobuf/types/pluginpb/ya.make2
-rw-r--r--yql/essentials/public/issue/protos/issue_message.proto17
-rw-r--r--yql/essentials/public/issue/protos/issue_severity.proto14
-rw-r--r--yql/essentials/public/issue/protos/ya.make10
-rw-r--r--yql/essentials/public/issue/ut/ya.make16
-rw-r--r--yql/essentials/public/issue/ya.make26
-rw-r--r--yql/essentials/public/issue/yql_issue.cpp310
-rw-r--r--yql/essentials/public/issue/yql_issue.h372
-rw-r--r--yql/essentials/public/issue/yql_issue_id.h128
-rw-r--r--yql/essentials/public/issue/yql_issue_manager.cpp234
-rw-r--r--yql/essentials/public/issue/yql_issue_manager.h83
-rw-r--r--yql/essentials/public/issue/yql_issue_manager_ut.cpp206
-rw-r--r--yql/essentials/public/issue/yql_issue_message.cpp149
-rw-r--r--yql/essentials/public/issue/yql_issue_message.h37
-rw-r--r--yql/essentials/public/issue/yql_issue_ut.cpp249
-rw-r--r--yql/essentials/public/issue/yql_issue_utils.cpp105
-rw-r--r--yql/essentials/public/issue/yql_issue_utils.h31
-rw-r--r--yql/essentials/public/issue/yql_issue_utils_ut.cpp200
-rw-r--r--yql/essentials/public/issue/yql_warning.cpp73
-rw-r--r--yql/essentials/public/issue/yql_warning.h51
-rw-r--r--yql/essentials/public/issue/yql_warning_ut.cpp94
-rw-r--r--yql/essentials/public/ya.make4
-rw-r--r--yql/essentials/ya.make1
-rw-r--r--yt/cpp/mapreduce/interface/logging/yt_log.cpp4
-rw-r--r--yt/cpp/mapreduce/interface/operation-inl.h3
-rw-r--r--yt/yt/client/api/delegating_client.h2
-rw-r--r--yt/yt/client/api/delegating_transaction.cpp2
-rw-r--r--yt/yt/client/api/delegating_transaction.h2
-rw-r--r--yt/yt/client/api/dynamic_table_transaction.h2
-rw-r--r--yt/yt/client/api/dynamic_table_transaction_mixin.cpp2
-rw-r--r--yt/yt/client/api/dynamic_table_transaction_mixin.h2
-rw-r--r--yt/yt/client/api/query_tracker_client.h2
-rw-r--r--yt/yt/client/api/rpc_proxy/client_impl.cpp23
-rw-r--r--yt/yt/client/api/rpc_proxy/client_impl.h2
-rw-r--r--yt/yt/client/api/rpc_proxy/helpers.cpp12
-rw-r--r--yt/yt/client/api/rpc_proxy/table_reader.cpp16
-rw-r--r--yt/yt/client/api/security_client.cpp2
-rw-r--r--yt/yt/client/api/security_client.h4
-rw-r--r--yt/yt/client/api/shuffle_client.h2
-rw-r--r--yt/yt/client/api/table_reader.h2
-rw-r--r--yt/yt/client/arrow/arrow_row_stream_encoder.cpp2
-rw-r--r--yt/yt/client/driver/driver.cpp11
-rw-r--r--yt/yt/client/driver/etc_commands.cpp2
-rw-r--r--yt/yt/client/driver/query_commands.cpp2
-rw-r--r--yt/yt/client/driver/shuffle_commands.cpp10
-rw-r--r--yt/yt/client/driver/shuffle_commands.h2
-rw-r--r--yt/yt/client/driver/table_commands.h8
-rw-r--r--yt/yt/client/federated/client.cpp2
-rw-r--r--yt/yt/client/federated/unittests/client_ut.cpp4
-rw-r--r--yt/yt/client/formats/config.cpp6
-rw-r--r--yt/yt/client/formats/config.h13
-rw-r--r--yt/yt/client/hedging/hedging.cpp2
-rw-r--r--yt/yt/client/table_client/blob_reader.cpp18
-rw-r--r--yt/yt/client/table_client/blob_reader.h8
-rw-r--r--yt/yt/client/table_client/column_sort_schema.cpp8
-rw-r--r--yt/yt/client/table_client/column_sort_schema.h2
-rw-r--r--yt/yt/client/table_client/columnar_statistics.cpp2
-rw-r--r--yt/yt/client/table_client/columnar_statistics.h4
-rw-r--r--yt/yt/client/table_client/logical_type.cpp34
-rw-r--r--yt/yt/client/table_client/logical_type.h10
-rw-r--r--yt/yt/client/table_client/name_table.cpp10
-rw-r--r--yt/yt/client/table_client/name_table.h6
-rw-r--r--yt/yt/client/table_client/public.cpp25
-rw-r--r--yt/yt/client/table_client/public.h32
-rw-r--r--yt/yt/client/table_client/schema.cpp112
-rw-r--r--yt/yt/client/table_client/schema.h54
-rw-r--r--yt/yt/client/table_client/validate_logical_type.cpp4
-rw-r--r--yt/yt/client/tablet_client/table_mount_cache_detail.cpp5
-rw-r--r--yt/yt/client/tablet_client/watermark_runtime_data.h2
-rw-r--r--yt/yt/client/unittests/mock/client.h2
-rw-r--r--yt/yt/client/unittests/mock/table_reader.h2
-rw-r--r--yt/yt/client/ypath/parser_detail.cpp2
-rw-r--r--yt/yt/client/ypath/rich.cpp6
-rw-r--r--yt/yt/client/ypath/rich.h4
-rw-r--r--yt/yt/core/concurrency/unittests/scheduled_executor_ut.cpp2
-rw-r--r--yt/yt/core/logging/log_manager.cpp7
-rw-r--r--yt/yt/core/net/address.cpp7
-rw-r--r--yt/yt/core/net/address.h3
-rw-r--r--yt/yt/core/rpc/balancing_channel.cpp5
-rw-r--r--yt/yt/core/rpc/config.cpp4
-rw-r--r--yt/yt/core/rpc/config.h7
-rw-r--r--yt/yt/core/rpc/helpers.cpp13
-rw-r--r--yt/yt/core/rpc/helpers.h5
-rw-r--r--yt/yt/core/rpc/server_detail.cpp8
-rw-r--r--yt/yt/core/rpc/server_detail.h2
-rw-r--r--yt/yt/core/rpc/service.h3
-rw-r--r--yt/yt/core/rpc/service_detail.cpp220
-rw-r--r--yt/yt/core/rpc/service_detail.h47
-rw-r--r--yt/yt/library/formats/arrow_writer.cpp4
-rw-r--r--yt/yt/library/formats/protobuf.cpp3
-rw-r--r--yt/yt/library/formats/schemaful_dsv_parser.cpp10
-rw-r--r--yt/yt/library/formats/schemaful_dsv_writer.cpp4
-rw-r--r--yt/yt/library/formats/web_json_writer.cpp6
-rw-r--r--yt/yt/library/formats/yamred_dsv_parser.cpp20
-rw-r--r--yt/yt/library/named_value/named_value.h6
-rw-r--r--yt/yt/library/numeric/unittests/piecewise_linear_function_ut.cpp10
-rw-r--r--yt/yt/library/oom/oom.h4
-rw-r--r--yt/yt/library/oom/tcmalloc_memory_limit_handler.cpp28
-rw-r--r--yt/yt/library/profiling/sensors_owner/README.md14
-rw-r--r--yt/yt/library/skiff_ext/parser-inl.h8
-rw-r--r--yt/yt/library/skiff_ext/parser.h4
-rw-r--r--yt/yt/library/skiff_ext/schema_match.cpp18
-rw-r--r--yt/yt/library/skiff_ext/schema_match.h4
206 files changed, 4243 insertions, 1271 deletions
diff --git a/build/export_generators/cmake/generator.toml b/build/export_generators/cmake/generator.toml
index 4fcb521f2f..fd4ab84d8a 100644
--- a/build/export_generators/cmake/generator.toml
+++ b/build/export_generators/cmake/generator.toml
@@ -47,6 +47,10 @@ copy=[
template={ path="dir_cmake_lists.jinja", dest="CMakeLists{PLATFORM}.txt" }
merge_platform_template={ path="merge_platforms_cmake_lists.jinja", dest="CMakeLists.txt" }
+[targets.EXTRA_ONLY]
+template={ path="dir_cmake_lists.jinja", dest="CMakeLists{PLATFORM}.txt" }
+merge_platform_template={ path="merge_platforms_cmake_lists.jinja", dest="CMakeLists.txt" }
+
[targets.add_executable]
template={ path="dir_cmake_lists.jinja", dest="CMakeLists{PLATFORM}.txt" }
merge_platform_template={ path="merge_platforms_cmake_lists.jinja", dest="CMakeLists.txt" }
diff --git a/build/export_generators/ide-gradle/build.gradle.kts.jinja b/build/export_generators/ide-gradle/build.gradle.kts.jinja
index 3f7755b94e..3833801b66 100644
--- a/build/export_generators/ide-gradle/build.gradle.kts.jinja
+++ b/build/export_generators/ide-gradle/build.gradle.kts.jinja
@@ -1,20 +1,26 @@
-{%- set mainClass = target.app_main_class -%}
+val baseBuildDir = "{{ export_root }}/gradle.build/"
+buildDir = file(baseBuildDir + project.path.replaceFirst(":", "/").replace(":", "."))
+subprojects {
+ buildDir = file(baseBuildDir + project.path.replaceFirst(":", "/").replace(":", "."))
+}
+
+{% set mainClass = target.app_main_class -%}
{%- set publish = target.publish -%}
{%- set with_kotlin = target.with_kotlin -%}
{%- set kotlin_version = target.kotlin_version -%}
{%- set hasJunit5Test = extra_targets|selectattr('junit5_test') -%}
{%- set errorprone_plugin_version = "4.0.0" -%}
plugins {
-{# some plugins configuration #}
+{#- some plugins configuration -#}
{%- for library in target.consumer if library.classpath -%}
-{# error prone plugin configuration #}
+{#- error prone plugin configuration -#}
{%- if library.prebuilt and library.jar and (library.type != "contrib" or build_contribs) and "contrib/java/com/google/errorprone/error_prone_annotations/" in library.jar -%}
id("net.ltgt.errorprone") version "{{ errorprone_plugin_version }}"
{%- endif -%}
{%- endfor -%}
-{# lombok configuration #}
+{#- lombok configuration -#}
{%- if "lombok.launch.AnnotationProcessorHider$AnnotationProcessor" in target.annotation_processors %}
id("io.freefair.lombok") version "8.6"
{%- endif -%}
@@ -43,7 +49,7 @@ plugins {
{% endif -%}
{%- endif %}
}
-{# language level #}
+{#- language level -#}
{%- if target.required_jdk is defined and target.required_jdk|length %}
java {
toolchain {
@@ -98,7 +104,7 @@ tasks.withType<Javadoc> {
}
{% endif %}
-{# javac flags #}
+{#- javac flags -#}
{%- if (target.javac.flags is defined) and (target.javac.flags|length) %}
tasks.withType<JavaCompile> {
{%- for javac_flag in target.javac.flags %}
@@ -113,20 +119,18 @@ tasks.withType<JavaCompile> {
val bucketUsername: String by project
val bucketPassword: String by project
repositories {
- repositories {
- maven {
- url = uri("https://bucket.yandex-team.ru/v1/maven/central")
- credentials {
- username = "$bucketUsername"
- password = "$bucketPassword"
+ repositories {
+ maven {
+ url = uri("https://bucket.yandex-team.ru/v1/maven/central")
+ credentials {
+ username = "$bucketUsername"
+ password = "$bucketPassword"
+ }
}
}
}
-
-}
-
-val project_root="{%- if export_root.startswith(arcadia_root + '/') -%}{{ arcadia_root }}{%- else -%}{{ export_root }}/.hic_sunt_dracones{%- endif -%}"
+val project_root = "{{ arcadia_root }}"
{% if mainClass -%}
application {
@@ -269,13 +273,13 @@ tasks.named<Test>("test") {
useJUnitPlatform()
}
-{% endif -%}
+{% endif -%}
-{# run_java_program #}
-{# {% set runs = targets|selectattr("runs") -%} #}
-{% set runs = target.runs -%}
-{% if runs -%}
-{% for run in runs -%}
+{#- run_java_program -#}
+{#- {% set runs = targets|selectattr("runs") -%} -#}
+{%- set runs = target.runs -%}
+{%- if runs -%}
+{%- for run in runs -%}
val runJav{{ loop.index }} = task<JavaExec>("runJavaProgram{{ loop.index }}") {
group = "build"
@@ -287,7 +291,7 @@ val runJav{{ loop.index }} = task<JavaExec>("runJavaProgram{{ loop.index }}") {
{% set real_classpath = classpath|replace('@', '') -%}
{% set real_classpath = real_classpath|replace('.run.cp', '') -%}
{% set real_classpath = real_classpath|replace('.cplst', '') -%}
-{% set real_classpath = real_classpath|replace(export_root, '') -%}
+{% set real_classpath = real_classpath|replace(export_root, '')|replace(arcadia_root, '') -%}
{% set real_gradle_classpath = real_classpath|replace('/', ':') %}
val classPath = "{{ real_gradle_classpath }}"
val classPathParts = classPath.split(":")
@@ -321,15 +325,15 @@ val runJav{{ loop.index }} = task<JavaExec>("runJavaProgram{{ loop.index }}") {
{% for dir in run.out_dir -%}
outputs.dir("{{ dir }}")
{% endfor -%}
-{#
+{#-
Не использованы аттрибуты
run-cwd="str"
run-in_dirs_inputs="list"
run-in_noparse="list"
run-out_dir="list"
run-tool="list"
-#}
-{% endif -%}
+-#}
+{%- endif -%}
}
tasks {
build {
diff --git a/build/export_generators/ide-gradle/build.gradle.kts.proto.jinja b/build/export_generators/ide-gradle/build.gradle.kts.proto.jinja
index 9a930779d4..d81a7ea0b0 100644
--- a/build/export_generators/ide-gradle/build.gradle.kts.proto.jinja
+++ b/build/export_generators/ide-gradle/build.gradle.kts.proto.jinja
@@ -1,6 +1,12 @@
{%- set publish = target.publish -%}
import com.google.protobuf.gradle.*
+val baseBuildDir = "{{ export_root }}/gradle.build/"
+buildDir = file(baseBuildDir + project.path.replaceFirst(":", "/").replace(":", "."))
+subprojects {
+ buildDir = file(baseBuildDir + project.path.replaceFirst(":", "/").replace(":", "."))
+}
+
val buildProtoDir = File("${buildDir}", "__proto__")
plugins {
@@ -9,7 +15,7 @@ plugins {
{%- if publish %}
`maven-publish`
`signing`
-{%- endif -%}
+{%- endif %}
}
{%- if publish %}
@@ -20,16 +26,16 @@ version = {% if target.publish_version -%}"{{ target.publish_version }}"{%- else
val bucketUsername: String by project
val bucketPassword: String by project
repositories {
- repositories {
- maven {
- url = uri("https://bucket.yandex-team.ru/v1/maven/central")
- credentials {
- username = "$bucketUsername"
- password = "$bucketPassword"
+ repositories {
+ maven {
+ url = uri("https://bucket.yandex-team.ru/v1/maven/central")
+ credentials {
+ username = "$bucketUsername"
+ password = "$bucketPassword"
+ }
}
}
}
-}
sourceSets {
main {
@@ -37,7 +43,7 @@ sourceSets {
}
}
-val project_root="{%- if export_root.startswith(arcadia_root + '/') -%}{{ arcadia_root }}{%- else -%}{{ export_root }}{%- endif -%}"
+val project_root = "{{ arcadia_root }}"
java {
withSourcesJar()
@@ -58,10 +64,10 @@ configurations.testImplementation {
{%- if target.jar_source_set is defined -%}
{%- for source_set in target.jar_source_set -%}
-{%- set srcdir_glob = split(source_set, ':') -%}
+{%- set srcdir_glob = split(source_set, ':') %}
sourceSets.main.java.srcDirs += "{{ srcdir_glob[0] }}"
{% endfor -%}
-{%- endif -%}
+{%- endif %}
dependencies {
{%- for library in target.consumer if library.classpath -%}
diff --git a/build/export_generators/ide-gradle/generator.toml b/build/export_generators/ide-gradle/generator.toml
index ea589c40f5..1684f86626 100644
--- a/build/export_generators/ide-gradle/generator.toml
+++ b/build/export_generators/ide-gradle/generator.toml
@@ -8,9 +8,12 @@ copy=[
"gradlew.bat",
"gradle/wrapper/gradle-wrapper.jar",
"gradle/wrapper/gradle-wrapper.properties",
- ".idea/vcs.xml"
+ ".idea/vcs.xml",
]
+[targets.EXTRA_ONLY]
+template="build.gradle.kts.jinja"
+
[targets.jar]
template="build.gradle.kts.jinja"
diff --git a/build/export_generators/ide-gradle/settings.gradle.kts.jinja b/build/export_generators/ide-gradle/settings.gradle.kts.jinja
index da32f75ab1..a71aefe4f4 100644
--- a/build/export_generators/ide-gradle/settings.gradle.kts.jinja
+++ b/build/export_generators/ide-gradle/settings.gradle.kts.jinja
@@ -3,7 +3,7 @@ rootProject.name = "{{ project_name }}"
{% for subdir in subdirs -%}
{%- set classname = subdir | replace("/", ":") -%}
include(":{{ classname }}")
-project(":{{ classname }}").projectDir = file("{{ export_root }}/{{ subdir }}")
+project(":{{ classname }}").projectDir = file("{{ arcadia_root }}/{{ subdir }}")
{% endfor -%}
{%- include "[generator]/debug.jinja" ignore missing -%}
diff --git a/build/external_resources/yexport/public.resources.json b/build/external_resources/yexport/public.resources.json
index f304fcf477..eca2298793 100644
--- a/build/external_resources/yexport/public.resources.json
+++ b/build/external_resources/yexport/public.resources.json
@@ -1,13 +1,13 @@
{
"by_platform": {
"darwin": {
- "uri": "sbr:7196092605"
+ "uri": "sbr:7370212609"
},
"darwin-arm64": {
- "uri": "sbr:7196092155"
+ "uri": "sbr:7370211678"
},
"linux": {
- "uri": "sbr:7196091589"
+ "uri": "sbr:7370210775"
}
}
}
diff --git a/build/external_resources/yexport/resources.json b/build/external_resources/yexport/resources.json
index d2b3ee6f17..2936b9695b 100644
--- a/build/external_resources/yexport/resources.json
+++ b/build/external_resources/yexport/resources.json
@@ -1,13 +1,13 @@
{
"by_platform": {
"darwin": {
- "uri": "sbr:7196077522"
+ "uri": "sbr:7370190051"
},
"darwin-arm64": {
- "uri": "sbr:7196076872"
+ "uri": "sbr:7370188821"
},
"linux": {
- "uri": "sbr:7196076129"
+ "uri": "sbr:7370187741"
}
}
}
diff --git a/build/external_resources/ymake/public.resources.json b/build/external_resources/ymake/public.resources.json
index 3d0dcf2c0c..5c5c163c9b 100644
--- a/build/external_resources/ymake/public.resources.json
+++ b/build/external_resources/ymake/public.resources.json
@@ -1,19 +1,19 @@
{
"by_platform": {
"darwin": {
- "uri": "sbr:7299619998"
+ "uri": "sbr:7367909157"
},
"darwin-arm64": {
- "uri": "sbr:7299619667"
+ "uri": "sbr:7367907802"
},
"linux": {
- "uri": "sbr:7299620690"
+ "uri": "sbr:7367911510"
},
"linux-aarch64": {
- "uri": "sbr:7299619265"
+ "uri": "sbr:7367906163"
},
"win32-clang-cl": {
- "uri": "sbr:7299620278"
+ "uri": "sbr:7367910505"
}
}
}
diff --git a/build/external_resources/ymake/resources.json b/build/external_resources/ymake/resources.json
index 91bc84b2d7..49b22c1888 100644
--- a/build/external_resources/ymake/resources.json
+++ b/build/external_resources/ymake/resources.json
@@ -1,19 +1,19 @@
{
"by_platform": {
"darwin": {
- "uri": "sbr:7299618240"
+ "uri": "sbr:7367869442"
},
"darwin-arm64": {
- "uri": "sbr:7299617682"
+ "uri": "sbr:7367867142"
},
"linux": {
- "uri": "sbr:7299619235"
+ "uri": "sbr:7367873946"
},
"linux-aarch64": {
- "uri": "sbr:7299616994"
+ "uri": "sbr:7367864752"
},
"win32-clang-cl": {
- "uri": "sbr:7299618667"
+ "uri": "sbr:7367871824"
}
}
}
diff --git a/build/mapping.conf.json b/build/mapping.conf.json
index da06c1f807..7d4ff8fec6 100644
--- a/build/mapping.conf.json
+++ b/build/mapping.conf.json
@@ -493,6 +493,7 @@
"6767780570": "https://devtools-registry.s3.yandex.net/6767780570",
"6913307396": "https://devtools-registry.s3.yandex.net/6913307396",
"7196092605": "https://devtools-registry.s3.yandex.net/7196092605",
+ "7370212609": "https://devtools-registry.s3.yandex.net/7370212609",
"5811823398": "https://devtools-registry.s3.yandex.net/5811823398",
"5840611310": "https://devtools-registry.s3.yandex.net/5840611310",
"5860185593": "https://devtools-registry.s3.yandex.net/5860185593",
@@ -515,6 +516,7 @@
"6767778005": "https://devtools-registry.s3.yandex.net/6767778005",
"6913305359": "https://devtools-registry.s3.yandex.net/6913305359",
"7196092155": "https://devtools-registry.s3.yandex.net/7196092155",
+ "7370211678": "https://devtools-registry.s3.yandex.net/7370211678",
"5811822876": "https://devtools-registry.s3.yandex.net/5811822876",
"5840610640": "https://devtools-registry.s3.yandex.net/5840610640",
"5860184285": "https://devtools-registry.s3.yandex.net/5860184285",
@@ -537,6 +539,7 @@
"6767775474": "https://devtools-registry.s3.yandex.net/6767775474",
"6913303010": "https://devtools-registry.s3.yandex.net/6913303010",
"7196091589": "https://devtools-registry.s3.yandex.net/7196091589",
+ "7370210775": "https://devtools-registry.s3.yandex.net/7370210775",
"5766172292": "https://devtools-registry.s3.yandex.net/5766172292",
"5805431504": "https://devtools-registry.s3.yandex.net/5805431504",
"5829027626": "https://devtools-registry.s3.yandex.net/5829027626",
@@ -582,6 +585,8 @@
"7211385890": "https://devtools-registry.s3.yandex.net/7211385890",
"7255221832": "https://devtools-registry.s3.yandex.net/7255221832",
"7299619998": "https://devtools-registry.s3.yandex.net/7299619998",
+ "7343906703": "https://devtools-registry.s3.yandex.net/7343906703",
+ "7367909157": "https://devtools-registry.s3.yandex.net/7367909157",
"5766171800": "https://devtools-registry.s3.yandex.net/5766171800",
"5805430761": "https://devtools-registry.s3.yandex.net/5805430761",
"5829025456": "https://devtools-registry.s3.yandex.net/5829025456",
@@ -627,6 +632,8 @@
"7211384971": "https://devtools-registry.s3.yandex.net/7211384971",
"7255220942": "https://devtools-registry.s3.yandex.net/7255220942",
"7299619667": "https://devtools-registry.s3.yandex.net/7299619667",
+ "7343906236": "https://devtools-registry.s3.yandex.net/7343906236",
+ "7367907802": "https://devtools-registry.s3.yandex.net/7367907802",
"5766173070": "https://devtools-registry.s3.yandex.net/5766173070",
"5805432830": "https://devtools-registry.s3.yandex.net/5805432830",
"5829031598": "https://devtools-registry.s3.yandex.net/5829031598",
@@ -672,6 +679,8 @@
"7211387031": "https://devtools-registry.s3.yandex.net/7211387031",
"7255223649": "https://devtools-registry.s3.yandex.net/7255223649",
"7299620690": "https://devtools-registry.s3.yandex.net/7299620690",
+ "7343907747": "https://devtools-registry.s3.yandex.net/7343907747",
+ "7367911510": "https://devtools-registry.s3.yandex.net/7367911510",
"5766171341": "https://devtools-registry.s3.yandex.net/5766171341",
"5805430188": "https://devtools-registry.s3.yandex.net/5805430188",
"5829023352": "https://devtools-registry.s3.yandex.net/5829023352",
@@ -717,6 +726,8 @@
"7211384106": "https://devtools-registry.s3.yandex.net/7211384106",
"7255220285": "https://devtools-registry.s3.yandex.net/7255220285",
"7299619265": "https://devtools-registry.s3.yandex.net/7299619265",
+ "7343905894": "https://devtools-registry.s3.yandex.net/7343905894",
+ "7367906163": "https://devtools-registry.s3.yandex.net/7367906163",
"5766172695": "https://devtools-registry.s3.yandex.net/5766172695",
"5805432230": "https://devtools-registry.s3.yandex.net/5805432230",
"5829029743": "https://devtools-registry.s3.yandex.net/5829029743",
@@ -762,6 +773,8 @@
"7211386552": "https://devtools-registry.s3.yandex.net/7211386552",
"7255222862": "https://devtools-registry.s3.yandex.net/7255222862",
"7299620278": "https://devtools-registry.s3.yandex.net/7299620278",
+ "7343907143": "https://devtools-registry.s3.yandex.net/7343907143",
+ "7367910505": "https://devtools-registry.s3.yandex.net/7367910505",
"4307890075": "https://devtools-registry.s3.yandex.net/4307890075",
"5517245192": "https://devtools-registry.s3.yandex.net/5517245192",
"4307901240": "https://devtools-registry.s3.yandex.net/4307901240",
@@ -1452,6 +1465,7 @@
"6767780570": "devtools/yexport/bin/yexport for darwin",
"6913307396": "devtools/yexport/bin/yexport for darwin",
"7196092605": "devtools/yexport/bin/yexport for darwin",
+ "7370212609": "devtools/yexport/bin/yexport for darwin",
"5811823398": "devtools/yexport/bin/yexport for darwin-arm64",
"5840611310": "devtools/yexport/bin/yexport for darwin-arm64",
"5860185593": "devtools/yexport/bin/yexport for darwin-arm64",
@@ -1474,6 +1488,7 @@
"6767778005": "devtools/yexport/bin/yexport for darwin-arm64",
"6913305359": "devtools/yexport/bin/yexport for darwin-arm64",
"7196092155": "devtools/yexport/bin/yexport for darwin-arm64",
+ "7370211678": "devtools/yexport/bin/yexport for darwin-arm64",
"5811822876": "devtools/yexport/bin/yexport for linux",
"5840610640": "devtools/yexport/bin/yexport for linux",
"5860184285": "devtools/yexport/bin/yexport for linux",
@@ -1496,6 +1511,7 @@
"6767775474": "devtools/yexport/bin/yexport for linux",
"6913303010": "devtools/yexport/bin/yexport for linux",
"7196091589": "devtools/yexport/bin/yexport for linux",
+ "7370210775": "devtools/yexport/bin/yexport for linux",
"5766172292": "devtools/ymake/bin/ymake for darwin",
"5805431504": "devtools/ymake/bin/ymake for darwin",
"5829027626": "devtools/ymake/bin/ymake for darwin",
@@ -1541,6 +1557,8 @@
"7211385890": "devtools/ymake/bin/ymake for darwin",
"7255221832": "devtools/ymake/bin/ymake for darwin",
"7299619998": "devtools/ymake/bin/ymake for darwin",
+ "7343906703": "devtools/ymake/bin/ymake for darwin",
+ "7367909157": "devtools/ymake/bin/ymake for darwin",
"5766171800": "devtools/ymake/bin/ymake for darwin-arm64",
"5805430761": "devtools/ymake/bin/ymake for darwin-arm64",
"5829025456": "devtools/ymake/bin/ymake for darwin-arm64",
@@ -1586,6 +1604,8 @@
"7211384971": "devtools/ymake/bin/ymake for darwin-arm64",
"7255220942": "devtools/ymake/bin/ymake for darwin-arm64",
"7299619667": "devtools/ymake/bin/ymake for darwin-arm64",
+ "7343906236": "devtools/ymake/bin/ymake for darwin-arm64",
+ "7367907802": "devtools/ymake/bin/ymake for darwin-arm64",
"5766173070": "devtools/ymake/bin/ymake for linux",
"5805432830": "devtools/ymake/bin/ymake for linux",
"5829031598": "devtools/ymake/bin/ymake for linux",
@@ -1631,6 +1651,8 @@
"7211387031": "devtools/ymake/bin/ymake for linux",
"7255223649": "devtools/ymake/bin/ymake for linux",
"7299620690": "devtools/ymake/bin/ymake for linux",
+ "7343907747": "devtools/ymake/bin/ymake for linux",
+ "7367911510": "devtools/ymake/bin/ymake for linux",
"5766171341": "devtools/ymake/bin/ymake for linux-aarch64",
"5805430188": "devtools/ymake/bin/ymake for linux-aarch64",
"5829023352": "devtools/ymake/bin/ymake for linux-aarch64",
@@ -1676,6 +1698,8 @@
"7211384106": "devtools/ymake/bin/ymake for linux-aarch64",
"7255220285": "devtools/ymake/bin/ymake for linux-aarch64",
"7299619265": "devtools/ymake/bin/ymake for linux-aarch64",
+ "7343905894": "devtools/ymake/bin/ymake for linux-aarch64",
+ "7367906163": "devtools/ymake/bin/ymake for linux-aarch64",
"5766172695": "devtools/ymake/bin/ymake for win32-clang-cl",
"5805432230": "devtools/ymake/bin/ymake for win32-clang-cl",
"5829029743": "devtools/ymake/bin/ymake for win32-clang-cl",
@@ -1721,6 +1745,8 @@
"7211386552": "devtools/ymake/bin/ymake for win32-clang-cl",
"7255222862": "devtools/ymake/bin/ymake for win32-clang-cl",
"7299620278": "devtools/ymake/bin/ymake for win32-clang-cl",
+ "7343907143": "devtools/ymake/bin/ymake for win32-clang-cl",
+ "7367910505": "devtools/ymake/bin/ymake for win32-clang-cl",
"4307890075": "flake8_linter for linux",
"5517245192": "flake8_linter for linux",
"4307901240": "flake8_linter for linux-aarch64",
diff --git a/build/plugins/nots.py b/build/plugins/nots.py
index 2f49030375..7615849861 100644
--- a/build/plugins/nots.py
+++ b/build/plugins/nots.py
@@ -828,7 +828,7 @@ def on_node_modules_configure(unit: NotsUnitType) -> None:
ymake.report_configure_error(
"Project is configured to use @yatool/prebuilder. \n"
+ "Some packages in the pnpm-lock.yaml are misconfigured.\n"
- + "Run {COLORS.green}`ya tool nots update-lockfile`{COLORS.reset} to fix lockfile.\n"
+ + f"Run {COLORS.green}`ya tool nots update-lockfile`{COLORS.reset} to fix lockfile.\n"
+ "All packages with `requiresBuild:true` have to be marked with `hasAddons:true/false`.\n"
+ "Misconfigured keys: \n"
+ " - "
@@ -843,7 +843,7 @@ def on_node_modules_configure(unit: NotsUnitType) -> None:
ymake.report_configure_error(
"Project is configured to use @yatool/prebuilder. \n"
+ "Some packages are misconfigured.\n"
- + "Run {COLORS.green}`ya tool nots update-lockfile`{COLORS.reset} to fix pnpm-lock.yaml and package.json.\n"
+ + f"Run {COLORS.green}`ya tool nots update-lockfile`{COLORS.reset} to fix pnpm-lock.yaml and package.json.\n"
+ "Validation details: \n"
+ "\n".join(validation_messages)
)
diff --git a/build/scripts/_fake_src.cpp b/build/scripts/_fake_src.cpp
index 139597f9cb..4f31cbffc7 100644
--- a/build/scripts/_fake_src.cpp
+++ b/build/scripts/_fake_src.cpp
@@ -1,2 +1 @@
-
-
+// DO_NOT_STYLE because this script is sometimes implicilty added as a styling target to STYLE_CPP
diff --git a/build/ymake.core.conf b/build/ymake.core.conf
index 7964201a89..d9c2d7a43d 100644
--- a/build/ymake.core.conf
+++ b/build/ymake.core.conf
@@ -5594,7 +5594,12 @@ macro _SRC("xib", SRC, SRCFLAGS...) {
# tag:src-processing
macro _SRC("msg", SRC, SRCFLAGS...) {
- .CMD=$ROS_CMD($SRC)
+ .CMD=$ROS_MSG_CMD($SRC)
+}
+
+# tag:src-processing
+macro _SRC("srv", SRC, SRCFLAGS...) {
+ .CMD=$ROS_SRV_CMD($SRC)
}
ACTOOL_PATH=$XCODE_TOOLS_ROOT_RESOURCE_GLOBAL/Xcode/Contents/Developer/usr/bin/ibtool
diff --git a/contrib/libs/cxxsupp/libcxxrt/ya.make b/contrib/libs/cxxsupp/libcxxrt/ya.make
index bb32702ba5..568f018f17 100644
--- a/contrib/libs/cxxsupp/libcxxrt/ya.make
+++ b/contrib/libs/cxxsupp/libcxxrt/ya.make
@@ -11,9 +11,9 @@ LICENSE(
LICENSE_TEXTS(.yandex_meta/licenses.list.txt)
-VERSION(2024-09-24)
+VERSION(2024-10-14)
-ORIGINAL_SOURCE(https://github.com/libcxxrt/libcxxrt/archive/40e4fa2049930412a2c43cdf0c39b6b5aa735341.tar.gz)
+ORIGINAL_SOURCE(https://github.com/libcxxrt/libcxxrt/archive/76435c4451aeb5e04e9500b090293347a38cef8d.tar.gz)
ADDINCL(
contrib/libs/cxxsupp/libcxxrt
diff --git a/contrib/libs/protobuf/patches/dont-use-string-view.patch b/contrib/libs/protobuf/patches/dont-use-string-view.patch
deleted file mode 100644
index 084341c464..0000000000
--- a/contrib/libs/protobuf/patches/dont-use-string-view.patch
+++ /dev/null
@@ -1,56 +0,0 @@
-commit 4a6dd4d9aeb3e3f9c033d092c86d5be63ca8cba9
-author: nechda
-date: 2024-08-08T14:02:13+03:00
-
- [PROTOBUF] Dont use string_view
-
---- contrib/libs/protobuf/src/google/protobuf/arenastring.cc (e2819444222a1e9154b9c7b701eff6427206db7c)
-+++ contrib/libs/protobuf/src/google/protobuf/arenastring.cc (4a6dd4d9aeb3e3f9c033d092c86d5be63ca8cba9)
-@@ -115,7 +115,7 @@ TaggedStringPtr CreateArenaString(Arena& arena, absl::string_view s) {
-
- } // namespace
-
--void ArenaStringPtr::Set(absl::string_view value, Arena* arena) {
-+void ArenaStringPtr::Set(const TProtoStringType& value, Arena* arena) {
- ScopedCheckPtrInvariants check(&tagged_ptr_);
- if (IsDefault()) {
- // If we're not on an arena, skip straight to a true string to avoid
---- contrib/libs/protobuf/src/google/protobuf/arenastring.h (e2819444222a1e9154b9c7b701eff6427206db7c)
-+++ contrib/libs/protobuf/src/google/protobuf/arenastring.h (4a6dd4d9aeb3e3f9c033d092c86d5be63ca8cba9)
-@@ -259,7 +259,7 @@ struct PROTOBUF_EXPORT ArenaStringPtr {
- // instance known to not carry any heap allocated value.
- inline void InitAllocated(TProtoStringType* str, Arena* arena);
-
-- void Set(absl::string_view value, Arena* arena);
-+ void Set(const TProtoStringType& value, Arena* arena);
- void Set(TProtoStringType&& value, Arena* arena);
- template <typename... OverloadDisambiguator>
- void Set(const TProtoStringType& value, Arena* arena);
-@@ -410,15 +410,15 @@ inline void ArenaStringPtr::InitAllocated(TProtoStringType* str, Arena* arena) {
- }
-
- inline void ArenaStringPtr::Set(const char* s, Arena* arena) {
-- Set(absl::string_view{s}, arena);
-+ Set(TProtoStringType{s}, arena);
- }
-
- inline void ArenaStringPtr::Set(const char* s, size_t n, Arena* arena) {
-- Set(absl::string_view{s, n}, arena);
-+ Set(TProtoStringType(s, s + n), arena);
- }
-
- inline void ArenaStringPtr::SetBytes(absl::string_view value, Arena* arena) {
-- Set(value, arena);
-+ Set(TProtoStringType(value.data(), value.size()), arena);
- }
-
- template <>
-@@ -439,7 +439,7 @@ inline void ArenaStringPtr::SetBytes(const char* s, Arena* arena) {
- }
-
- inline void ArenaStringPtr::SetBytes(const void* p, size_t n, Arena* arena) {
-- Set(absl::string_view{static_cast<const char*>(p), n}, arena);
-+ Set(TProtoStringType(static_cast<const char*>(p), static_cast<const char*>(p) + n), arena);
- }
-
- // Make sure rhs_arena allocated rhs, and lhs_arena allocated lhs.
diff --git a/contrib/libs/protobuf/patches/dot-generate-string-view-setters.patch b/contrib/libs/protobuf/patches/dot-generate-string-view-setters.patch
deleted file mode 100644
index 88ec297018..0000000000
--- a/contrib/libs/protobuf/patches/dot-generate-string-view-setters.patch
+++ /dev/null
@@ -1,21 +0,0 @@
---- contrib/libs/protobuf/src/google/protobuf/arenastring.h (d40b078e35f492f4ad417c8dd4a0103d4e577049)
-+++ contrib/libs/protobuf/src/google/protobuf/arenastring.h (2babb01d30e285a96044041b98053d9edbb48b2a)
-@@ -266,7 +266,6 @@ struct PROTOBUF_EXPORT ArenaStringPtr {
- void Set(const char* s, Arena* arena);
- void Set(const char* s, size_t n, Arena* arena);
-
-- void SetBytes(absl::string_view value, Arena* arena);
- void SetBytes(TProtoStringType&& value, Arena* arena);
- template <typename... OverloadDisambiguator>
- void SetBytes(const TProtoStringType& value, Arena* arena);
-@@ -417,10 +416,6 @@ inline void ArenaStringPtr::Set(const char* s, size_t n, Arena* arena) {
- Set(TProtoStringType(s, s + n), arena);
- }
-
--inline void ArenaStringPtr::SetBytes(absl::string_view value, Arena* arena) {
-- Set(TProtoStringType(value.data(), value.size()), arena);
--}
--
- template <>
- PROTOBUF_EXPORT void ArenaStringPtr::Set(const TProtoStringType& value,
- Arena* arena);
diff --git a/contrib/libs/protobuf/patches/fix-string-from-nullptr.patch b/contrib/libs/protobuf/patches/fix-string-from-nullptr.patch
new file mode 100644
index 0000000000..27c1110066
--- /dev/null
+++ b/contrib/libs/protobuf/patches/fix-string-from-nullptr.patch
@@ -0,0 +1,19 @@
+commit 556f12d15e9e59a132af2d4605b7444c4e3b623e (wip)
+author: vadim-xd
+date: 2024-09-09T01:14:56+03:00
+
+ Fix ArenaStringPtr::Set(nullptr).
+
+ TString(nullptr) is allowed (at least currently), string_view(nullptr) is UB.
+
+--- contrib/libs/protobuf/src/google/protobuf/arenastring.h (bff10d7f3e550777570ff58e01b9ae291d02490e)
++++ contrib/libs/protobuf/src/google/protobuf/arenastring.h (556f12d15e9e59a132af2d4605b7444c4e3b623e)
+@@ -424,7 +424,7 @@ inline void ArenaStringPtr::InitAllocated(TProtoStringType* str, Arena* arena) {
+ }
+
+ inline void ArenaStringPtr::Set(const char* s, Arena* arena) {
+- Set(absl::string_view{s}, arena);
++ Set(TProtoStringType{s}, arena);
+ }
+
+ inline void ArenaStringPtr::Set(const char* s, size_t n, Arena* arena) {
diff --git a/contrib/libs/protobuf/patches/protoc-dont-generate-setters-with-string-view.patch b/contrib/libs/protobuf/patches/protoc-dont-generate-setters-with-string-view.patch
deleted file mode 100644
index f2e020253a..0000000000
--- a/contrib/libs/protobuf/patches/protoc-dont-generate-setters-with-string-view.patch
+++ /dev/null
@@ -1,40 +0,0 @@
---- a/src/google/protobuf/compiler/cpp/field_generators/string_field.cc (192810303a994f7cbd6bf596a3d56393bfefee6c)
-+++ b/src/google/protobuf/compiler/cpp/field_generators/string_field.cc (2b6f1fdeb6f4d5c82840dbe69e4f0d3149c8fe5a)
-@@ -781,13 +781,11 @@ void RepeatedString::GenerateAccessorDeclarations(io::Printer* p) const {
- $DEPRECATED$ void $set_name$(int index, TProtoStringType&& value);
- $DEPRECATED$ void $set_name$(int index, const char* value);
- $DEPRECATED$ void $set_name$(int index, const $byte$* value, std::size_t size);
-- $DEPRECATED$ void $set_name$(int index, absl::string_view value);
- $DEPRECATED$ TProtoStringType* $add_name$();
- $DEPRECATED$ void $add_name$(const TProtoStringType& value);
- $DEPRECATED$ void $add_name$(TProtoStringType&& value);
- $DEPRECATED$ void $add_name$(const char* value);
- $DEPRECATED$ void $add_name$(const $byte$* value, std::size_t size);
-- $DEPRECATED$ void $add_name$(absl::string_view value);
- $DEPRECATED$ const $pb$::RepeatedPtrField<TProtoStringType>& $name$() const;
- $DEPRECATED$ $pb$::RepeatedPtrField<TProtoStringType>* $mutable_name$();
-
-@@ -852,11 +850,6 @@ void RepeatedString::GenerateInlineAccessorDefinitions(io::Printer* p) const {
- $annotate_set$;
- // @@protoc_insertion_point(field_set_pointer:$pkg.Msg.field$)
- }
-- inline void $Msg$::set_$name$(int index, absl::string_view value) {
-- $field_$.Mutable(index)->assign(value.data(), value.size());
-- $annotate_set$;
-- // @@protoc_insertion_point(field_set_string_piece:$pkg.Msg.field$)
-- }
- inline TProtoStringType* $Msg$::_internal_add_$name$() { return $field_$.Add(); }
- inline void $Msg$::add_$name$(const TProtoStringType& value) {
- $field_$.Add()->assign(value);
-@@ -879,11 +872,6 @@ void RepeatedString::GenerateInlineAccessorDefinitions(io::Printer* p) const {
- $annotate_add$;
- // @@protoc_insertion_point(field_add_pointer:$pkg.Msg.field$)
- }
-- inline void $Msg$::add_$name$(absl::string_view value) {
-- $field_$.Add()->assign(value.data(), value.size());
-- $annotate_add$;
-- // @@protoc_insertion_point(field_add_string_piece:$pkg.Msg.field$)
-- }
- inline const ::$proto_ns$::RepeatedPtrField<TProtoStringType>&
- $Msg$::$name$() const {
- $annotate_list$;
diff --git a/contrib/libs/protobuf/patches/protoc-resolve-string-method-ambiguity.patch b/contrib/libs/protobuf/patches/protoc-resolve-string-method-ambiguity.patch
new file mode 100644
index 0000000000..81df82ffae
--- /dev/null
+++ b/contrib/libs/protobuf/patches/protoc-resolve-string-method-ambiguity.patch
@@ -0,0 +1,26 @@
+commit 3bc481cb6e0013371bdc01ae0aef7d85dc9abe1a (HEAD)
+author: vadim-xd
+date: 2024-10-28T13:19:23+03:00
+
+ Resolve ambiguity for repeated string methods
+
+--- a/src/google/protobuf/compiler/cpp/field_generators/string_field.cc (192810303a994f7cbd6bf596a3d56393bfefee6c)
++++ b/src/google/protobuf/compiler/cpp/field_generators/string_field.cc (3bc481cb6e0013371bdc01ae0aef7d85dc9abe1a)
+@@ -782,12 +782,18 @@ void RepeatedString::GenerateAccessorDeclarations(io::Printer* p) const {
+ $DEPRECATED$ void $set_name$(int index, const char* value);
+ $DEPRECATED$ void $set_name$(int index, const $byte$* value, std::size_t size);
+ $DEPRECATED$ void $set_name$(int index, absl::string_view value);
++ $DEPRECATED$ void $set_name$(int index, std::string&& value) {
++ return $set_name$(index, TProtoStringType(std::move(value)));
++ }
+ $DEPRECATED$ TProtoStringType* $add_name$();
+ $DEPRECATED$ void $add_name$(const TProtoStringType& value);
+ $DEPRECATED$ void $add_name$(TProtoStringType&& value);
+ $DEPRECATED$ void $add_name$(const char* value);
+ $DEPRECATED$ void $add_name$(const $byte$* value, std::size_t size);
+ $DEPRECATED$ void $add_name$(absl::string_view value);
++ $DEPRECATED$ void $add_name$(std::string&& value) {
++ return $add_name$(TProtoStringType(std::move(value)));
++ }
+ $DEPRECATED$ const $pb$::RepeatedPtrField<TProtoStringType>& $name$() const;
+ $DEPRECATED$ $pb$::RepeatedPtrField<TProtoStringType>* $mutable_name$();
diff --git a/contrib/libs/protobuf/patches/resolve-string-method-ambiguity.patch b/contrib/libs/protobuf/patches/resolve-string-method-ambiguity.patch
new file mode 100644
index 0000000000..ad4f12f4e7
--- /dev/null
+++ b/contrib/libs/protobuf/patches/resolve-string-method-ambiguity.patch
@@ -0,0 +1,30 @@
+commit 4a55ea8892270807a046eb33fd5bc336b5fa93cb (HEAD)
+author: vadim-xd
+date: 2024-10-28T12:47:03+03:00
+
+ Resolve ambiguity in ArenaStringPtr methods with std::string&&
+
+--- contrib/libs/protobuf/src/google/protobuf/arenastring.h (e2819444222a1e9154b9c7b701eff6427206db7c)
++++ contrib/libs/protobuf/src/google/protobuf/arenastring.h (4a55ea8892270807a046eb33fd5bc336b5fa93cb)
+@@ -266,6 +266,10 @@ struct PROTOBUF_EXPORT ArenaStringPtr {
+ void Set(const char* s, Arena* arena);
+ void Set(const char* s, size_t n, Arena* arena);
+
++ void Set(std::string&& value, Arena* arena) {
++ return Set(TProtoStringType(std::move(value)), arena);
++ }
++
+ void SetBytes(absl::string_view value, Arena* arena);
+ void SetBytes(TProtoStringType&& value, Arena* arena);
+ template <typename... OverloadDisambiguator>
+@@ -273,6 +277,10 @@ struct PROTOBUF_EXPORT ArenaStringPtr {
+ void SetBytes(const char* s, Arena* arena);
+ void SetBytes(const void* p, size_t n, Arena* arena);
+
++ void SetBytes(std::string&& value, Arena* arena) {
++ return SetBytes(TProtoStringType(std::move(value)), arena);
++ }
++
+ template <typename RefWrappedType>
+ void Set(std::reference_wrapper<RefWrappedType> const_string_ref,
+ ::google::protobuf::Arena* arena) {
diff --git a/contrib/libs/protobuf/patches/z_05_use_ref_count.patch b/contrib/libs/protobuf/patches/z_05_use_ref_count.patch
index 391af0669c..1944eace13 100644
--- a/contrib/libs/protobuf/patches/z_05_use_ref_count.patch
+++ b/contrib/libs/protobuf/patches/z_05_use_ref_count.patch
@@ -1,14 +1,14 @@
--- contrib/libs/protobuf/src/google/protobuf/arenastring.cc (5de6c47c971aa5c751f17043b8ddf45bcc6daee0)
+++ contrib/libs/protobuf/src/google/protobuf/arenastring.cc (c7e1b2fc457f6c68ed50780be734651758d4f548)
-@@ -96,9 +96,9 @@ class ScopedCheckPtrInvariants {
+@@ -96,9 +96,10 @@ class ScopedCheckPtrInvariants {
#endif // NDEBUG || !GOOGLE_PROTOBUF_INTERNAL_DONATE_STEAL
// Creates a heap allocated TProtoStringType value.
-inline TaggedStringPtr CreateString(absl::string_view value) {
-+inline TaggedStringPtr CreateString(const TProtoStringType& value) {
++template <typename TArg>
++inline TaggedStringPtr CreateString(const TArg& value) {
TaggedStringPtr res;
- res.SetAllocated(new TProtoStringType(value.data(), value.length()));
+ res.SetAllocated(new TProtoStringType(value));
return res;
}
-
diff --git a/contrib/libs/protobuf/src/google/protobuf/arenastring.cc b/contrib/libs/protobuf/src/google/protobuf/arenastring.cc
index 4c4eb75da1..f466d542d9 100644
--- a/contrib/libs/protobuf/src/google/protobuf/arenastring.cc
+++ b/contrib/libs/protobuf/src/google/protobuf/arenastring.cc
@@ -96,7 +96,8 @@ class ScopedCheckPtrInvariants {
#endif // NDEBUG || !GOOGLE_PROTOBUF_INTERNAL_DONATE_STEAL
// Creates a heap allocated TProtoStringType value.
-inline TaggedStringPtr CreateString(const TProtoStringType& value) {
+template <typename TArg>
+inline TaggedStringPtr CreateString(const TArg& value) {
TaggedStringPtr res;
res.SetAllocated(new TProtoStringType(value));
return res;
@@ -115,7 +116,7 @@ TaggedStringPtr CreateArenaString(Arena& arena, y_absl::string_view s) {
} // namespace
-void ArenaStringPtr::Set(const TProtoStringType& value, Arena* arena) {
+void ArenaStringPtr::Set(y_absl::string_view value, Arena* arena) {
ScopedCheckPtrInvariants check(&tagged_ptr_);
if (IsDefault()) {
// If we're not on an arena, skip straight to a true string to avoid
diff --git a/contrib/libs/protobuf/src/google/protobuf/arenastring.h b/contrib/libs/protobuf/src/google/protobuf/arenastring.h
index 62a1ae3d4a..edee6ef4cb 100644
--- a/contrib/libs/protobuf/src/google/protobuf/arenastring.h
+++ b/contrib/libs/protobuf/src/google/protobuf/arenastring.h
@@ -259,19 +259,28 @@ struct PROTOBUF_EXPORT ArenaStringPtr {
// instance known to not carry any heap allocated value.
inline void InitAllocated(TProtoStringType* str, Arena* arena);
- void Set(const TProtoStringType& value, Arena* arena);
+ void Set(y_absl::string_view value, Arena* arena);
void Set(TProtoStringType&& value, Arena* arena);
template <typename... OverloadDisambiguator>
void Set(const TProtoStringType& value, Arena* arena);
void Set(const char* s, Arena* arena);
void Set(const char* s, size_t n, Arena* arena);
+ void Set(std::string&& value, Arena* arena) {
+ return Set(TProtoStringType(std::move(value)), arena);
+ }
+
+ void SetBytes(y_absl::string_view value, Arena* arena);
void SetBytes(TProtoStringType&& value, Arena* arena);
template <typename... OverloadDisambiguator>
void SetBytes(const TProtoStringType& value, Arena* arena);
void SetBytes(const char* s, Arena* arena);
void SetBytes(const void* p, size_t n, Arena* arena);
+ void SetBytes(std::string&& value, Arena* arena) {
+ return SetBytes(TProtoStringType(std::move(value)), arena);
+ }
+
template <typename RefWrappedType>
void Set(std::reference_wrapper<RefWrappedType> const_string_ref,
::google::protobuf::Arena* arena) {
@@ -413,7 +422,11 @@ inline void ArenaStringPtr::Set(const char* s, Arena* arena) {
}
inline void ArenaStringPtr::Set(const char* s, size_t n, Arena* arena) {
- Set(TProtoStringType(s, s + n), arena);
+ Set(y_absl::string_view{s, n}, arena);
+}
+
+inline void ArenaStringPtr::SetBytes(y_absl::string_view value, Arena* arena) {
+ Set(value, arena);
}
template <>
@@ -434,7 +447,7 @@ inline void ArenaStringPtr::SetBytes(const char* s, Arena* arena) {
}
inline void ArenaStringPtr::SetBytes(const void* p, size_t n, Arena* arena) {
- Set(TProtoStringType(static_cast<const char*>(p), static_cast<const char*>(p) + n), arena);
+ Set(y_absl::string_view{static_cast<const char*>(p), n}, arena);
}
// Make sure rhs_arena allocated rhs, and lhs_arena allocated lhs.
diff --git a/contrib/libs/protoc/src/google/protobuf/compiler/cpp/field_generators/string_field.cc b/contrib/libs/protoc/src/google/protobuf/compiler/cpp/field_generators/string_field.cc
index 11b5b83ea6..0da0e56609 100644
--- a/contrib/libs/protoc/src/google/protobuf/compiler/cpp/field_generators/string_field.cc
+++ b/contrib/libs/protoc/src/google/protobuf/compiler/cpp/field_generators/string_field.cc
@@ -781,11 +781,19 @@ void RepeatedString::GenerateAccessorDeclarations(io::Printer* p) const {
$DEPRECATED$ void $set_name$(int index, TProtoStringType&& value);
$DEPRECATED$ void $set_name$(int index, const char* value);
$DEPRECATED$ void $set_name$(int index, const $byte$* value, std::size_t size);
+ $DEPRECATED$ void $set_name$(int index, y_absl::string_view value);
+ $DEPRECATED$ void $set_name$(int index, std::string&& value) {
+ return $set_name$(index, TProtoStringType(std::move(value)));
+ }
$DEPRECATED$ TProtoStringType* $add_name$();
$DEPRECATED$ void $add_name$(const TProtoStringType& value);
$DEPRECATED$ void $add_name$(TProtoStringType&& value);
$DEPRECATED$ void $add_name$(const char* value);
$DEPRECATED$ void $add_name$(const $byte$* value, std::size_t size);
+ $DEPRECATED$ void $add_name$(y_absl::string_view value);
+ $DEPRECATED$ void $add_name$(std::string&& value) {
+ return $add_name$(TProtoStringType(std::move(value)));
+ }
$DEPRECATED$ const $pb$::RepeatedPtrField<TProtoStringType>& $name$() const;
$DEPRECATED$ $pb$::RepeatedPtrField<TProtoStringType>* $mutable_name$();
@@ -850,6 +858,11 @@ void RepeatedString::GenerateInlineAccessorDefinitions(io::Printer* p) const {
$annotate_set$;
// @@protoc_insertion_point(field_set_pointer:$pkg.Msg.field$)
}
+ inline void $Msg$::set_$name$(int index, y_absl::string_view value) {
+ $field_$.Mutable(index)->assign(value.data(), value.size());
+ $annotate_set$;
+ // @@protoc_insertion_point(field_set_string_piece:$pkg.Msg.field$)
+ }
inline TProtoStringType* $Msg$::_internal_add_$name$() { return $field_$.Add(); }
inline void $Msg$::add_$name$(const TProtoStringType& value) {
$field_$.Add()->assign(value);
@@ -872,6 +885,11 @@ void RepeatedString::GenerateInlineAccessorDefinitions(io::Printer* p) const {
$annotate_add$;
// @@protoc_insertion_point(field_add_pointer:$pkg.Msg.field$)
}
+ inline void $Msg$::add_$name$(y_absl::string_view value) {
+ $field_$.Add()->assign(value.data(), value.size());
+ $annotate_add$;
+ // @@protoc_insertion_point(field_add_string_piece:$pkg.Msg.field$)
+ }
inline const ::$proto_ns$::RepeatedPtrField<TProtoStringType>&
$Msg$::$name$() const {
$annotate_list$;
diff --git a/contrib/python/anyio/.dist-info/METADATA b/contrib/python/anyio/.dist-info/METADATA
index e28bbd52d0..10d7aafc77 100644
--- a/contrib/python/anyio/.dist-info/METADATA
+++ b/contrib/python/anyio/.dist-info/METADATA
@@ -1,6 +1,6 @@
Metadata-Version: 2.1
Name: anyio
-Version: 4.6.2
+Version: 4.6.2.post1
Summary: High level compatibility layer for multiple asynchronous event loop implementations
Author-email: Alex Grönholm <alex.gronholm@nextday.fi>
License: MIT
@@ -15,13 +15,12 @@ Classifier: Framework :: AnyIO
Classifier: Typing :: Typed
Classifier: Programming Language :: Python
Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.8
Classifier: Programming Language :: Python :: 3.9
Classifier: Programming Language :: Python :: 3.10
Classifier: Programming Language :: Python :: 3.11
Classifier: Programming Language :: Python :: 3.12
Classifier: Programming Language :: Python :: 3.13
-Requires-Python: >=3.8
+Requires-Python: >=3.9
Description-Content-Type: text/x-rst
License-File: LICENSE
Requires-Dist: idna >=2.8
diff --git a/contrib/python/anyio/anyio/_backends/_asyncio.py b/contrib/python/anyio/anyio/_backends/_asyncio.py
index fa5349a8c2..0a69e7ac61 100644
--- a/contrib/python/anyio/anyio/_backends/_asyncio.py
+++ b/contrib/python/anyio/anyio/_backends/_asyncio.py
@@ -20,9 +20,18 @@ from asyncio import (
)
from asyncio.base_events import _run_until_complete_cb # type: ignore[attr-defined]
from collections import OrderedDict, deque
-from collections.abc import AsyncIterator, Iterable
+from collections.abc import (
+ AsyncGenerator,
+ AsyncIterator,
+ Awaitable,
+ Callable,
+ Collection,
+ Coroutine,
+ Iterable,
+ Sequence,
+)
from concurrent.futures import Future
-from contextlib import suppress
+from contextlib import AbstractContextManager, suppress
from contextvars import Context, copy_context
from dataclasses import dataclass
from functools import partial, wraps
@@ -42,15 +51,7 @@ from types import TracebackType
from typing import (
IO,
Any,
- AsyncGenerator,
- Awaitable,
- Callable,
- Collection,
- ContextManager,
- Coroutine,
Optional,
- Sequence,
- Tuple,
TypeVar,
cast,
)
@@ -358,6 +359,14 @@ def _task_started(task: asyncio.Task) -> bool:
#
+def is_anyio_cancellation(exc: CancelledError) -> bool:
+ return (
+ bool(exc.args)
+ and isinstance(exc.args[0], str)
+ and exc.args[0].startswith("Cancelled by cancel scope ")
+ )
+
+
class CancelScope(BaseCancelScope):
def __new__(
cls, *, deadline: float = math.inf, shield: bool = False
@@ -416,6 +425,8 @@ class CancelScope(BaseCancelScope):
exc_val: BaseException | None,
exc_tb: TracebackType | None,
) -> bool | None:
+ del exc_tb
+
if not self._active:
raise RuntimeError("This cancel scope is not active")
if current_task() is not self._host_task:
@@ -432,47 +443,93 @@ class CancelScope(BaseCancelScope):
"current cancel scope"
)
- self._active = False
- if self._timeout_handle:
- self._timeout_handle.cancel()
- self._timeout_handle = None
-
- self._tasks.remove(self._host_task)
- if self._parent_scope is not None:
- self._parent_scope._child_scopes.remove(self)
- self._parent_scope._tasks.add(self._host_task)
+ try:
+ self._active = False
+ if self._timeout_handle:
+ self._timeout_handle.cancel()
+ self._timeout_handle = None
- host_task_state.cancel_scope = self._parent_scope
+ self._tasks.remove(self._host_task)
+ if self._parent_scope is not None:
+ self._parent_scope._child_scopes.remove(self)
+ self._parent_scope._tasks.add(self._host_task)
- # Restart the cancellation effort in the closest directly cancelled parent
- # scope if this one was shielded
- self._restart_cancellation_in_parent()
+ host_task_state.cancel_scope = self._parent_scope
- if self._cancel_called and exc_val is not None:
- for exc in iterate_exceptions(exc_val):
- if isinstance(exc, CancelledError):
- self._cancelled_caught = self._uncancel(exc)
- if self._cancelled_caught:
+ # Undo all cancellations done by this scope
+ if self._cancelling is not None:
+ while self._cancel_calls:
+ self._cancel_calls -= 1
+ if self._host_task.uncancel() <= self._cancelling:
break
- return self._cancelled_caught
+ # We only swallow the exception iff it was an AnyIO CancelledError, either
+ # directly as exc_val or inside an exception group and there are no cancelled
+ # parent cancel scopes visible to us here
+ not_swallowed_exceptions = 0
+ swallow_exception = False
+ if exc_val is not None:
+ for exc in iterate_exceptions(exc_val):
+ if self._cancel_called and isinstance(exc, CancelledError):
+ if not (swallow_exception := self._uncancel(exc)):
+ not_swallowed_exceptions += 1
+ else:
+ not_swallowed_exceptions += 1
- return None
+ # Restart the cancellation effort in the closest visible, cancelled parent
+ # scope if necessary
+ self._restart_cancellation_in_parent()
+ return swallow_exception and not not_swallowed_exceptions
+ finally:
+ self._host_task = None
+ del exc_val
+
+ @property
+ def _effectively_cancelled(self) -> bool:
+ cancel_scope: CancelScope | None = self
+ while cancel_scope is not None:
+ if cancel_scope._cancel_called:
+ return True
+
+ if cancel_scope.shield:
+ return False
+
+ cancel_scope = cancel_scope._parent_scope
+
+ return False
+
+ @property
+ def _parent_cancellation_is_visible_to_us(self) -> bool:
+ return (
+ self._parent_scope is not None
+ and not self.shield
+ and self._parent_scope._effectively_cancelled
+ )
def _uncancel(self, cancelled_exc: CancelledError) -> bool:
- if sys.version_info < (3, 9) or self._host_task is None:
+ if self._host_task is None:
self._cancel_calls = 0
return True
- # Undo all cancellations done by this scope
- if self._cancelling is not None:
- while self._cancel_calls:
- self._cancel_calls -= 1
- if self._host_task.uncancel() <= self._cancelling:
- return True
+ while True:
+ if is_anyio_cancellation(cancelled_exc):
+ # Only swallow the cancellation exception if it's an AnyIO cancel
+ # exception and there are no other cancel scopes down the line pending
+ # cancellation
+ self._cancelled_caught = (
+ self._effectively_cancelled
+ and not self._parent_cancellation_is_visible_to_us
+ )
+ return self._cancelled_caught
- self._cancel_calls = 0
- return f"Cancelled by cancel scope {id(self):x}" in cancelled_exc.args
+ # Sometimes third party frameworks catch a CancelledError and raise a new
+ # one, so as a workaround we have to look at the previous ones in
+ # __context__ too for a matching cancel message
+ if isinstance(cancelled_exc.__context__, CancelledError):
+ cancelled_exc = cancelled_exc.__context__
+ continue
+
+ return False
def _timeout(self) -> None:
if self._deadline != math.inf:
@@ -496,19 +553,17 @@ class CancelScope(BaseCancelScope):
should_retry = False
current = current_task()
for task in self._tasks:
+ should_retry = True
if task._must_cancel: # type: ignore[attr-defined]
continue
# The task is eligible for cancellation if it has started
- should_retry = True
if task is not current and (task is self._host_task or _task_started(task)):
waiter = task._fut_waiter # type: ignore[attr-defined]
if not isinstance(waiter, asyncio.Future) or not waiter.done():
- origin._cancel_calls += 1
- if sys.version_info >= (3, 9):
- task.cancel(f"Cancelled by cancel scope {id(origin):x}")
- else:
- task.cancel()
+ task.cancel(f"Cancelled by cancel scope {id(origin):x}")
+ if task is origin._host_task:
+ origin._cancel_calls += 1
# Deliver cancellation to child scopes that aren't shielded or running their own
# cancellation callbacks
@@ -546,17 +601,6 @@ class CancelScope(BaseCancelScope):
scope = scope._parent_scope
- def _parent_cancelled(self) -> bool:
- # Check whether any parent has been cancelled
- cancel_scope = self._parent_scope
- while cancel_scope is not None and not cancel_scope._shield:
- if cancel_scope._cancel_called:
- return True
- else:
- cancel_scope = cancel_scope._parent_scope
-
- return False
-
def cancel(self) -> None:
if not self._cancel_called:
if self._timeout_handle:
@@ -645,6 +689,26 @@ class _AsyncioTaskStatus(abc.TaskStatus):
_task_states[task].parent_id = self._parent_id
+async def _wait(tasks: Iterable[asyncio.Task[object]]) -> None:
+ tasks = set(tasks)
+ waiter = get_running_loop().create_future()
+
+ def on_completion(task: asyncio.Task[object]) -> None:
+ tasks.discard(task)
+ if not tasks and not waiter.done():
+ waiter.set_result(None)
+
+ for task in tasks:
+ task.add_done_callback(on_completion)
+ del task
+
+ try:
+ await waiter
+ finally:
+ while tasks:
+ tasks.pop().remove_done_callback(on_completion)
+
+
class TaskGroup(abc.TaskGroup):
def __init__(self) -> None:
self.cancel_scope: CancelScope = CancelScope()
@@ -663,38 +727,53 @@ class TaskGroup(abc.TaskGroup):
exc_val: BaseException | None,
exc_tb: TracebackType | None,
) -> bool | None:
- ignore_exception = self.cancel_scope.__exit__(exc_type, exc_val, exc_tb)
- if exc_val is not None:
- self.cancel_scope.cancel()
- if not isinstance(exc_val, CancelledError):
- self._exceptions.append(exc_val)
-
- cancelled_exc_while_waiting_tasks: CancelledError | None = None
- while self._tasks:
- try:
- await asyncio.wait(self._tasks)
- except CancelledError as exc:
- # This task was cancelled natively; reraise the CancelledError later
- # unless this task was already interrupted by another exception
+ try:
+ if exc_val is not None:
self.cancel_scope.cancel()
- if cancelled_exc_while_waiting_tasks is None:
- cancelled_exc_while_waiting_tasks = exc
+ if not isinstance(exc_val, CancelledError):
+ self._exceptions.append(exc_val)
- self._active = False
- if self._exceptions:
- raise BaseExceptionGroup(
- "unhandled errors in a TaskGroup", self._exceptions
- )
+ try:
+ if self._tasks:
+ with CancelScope() as wait_scope:
+ while self._tasks:
+ try:
+ await _wait(self._tasks)
+ except CancelledError as exc:
+ # Shield the scope against further cancellation attempts,
+ # as they're not productive (#695)
+ wait_scope.shield = True
+ self.cancel_scope.cancel()
+
+ # Set exc_val from the cancellation exception if it was
+ # previously unset. However, we should not replace a native
+ # cancellation exception with one raise by a cancel scope.
+ if exc_val is None or (
+ isinstance(exc_val, CancelledError)
+ and not is_anyio_cancellation(exc)
+ ):
+ exc_val = exc
+ else:
+ # If there are no child tasks to wait on, run at least one checkpoint
+ # anyway
+ await AsyncIOBackend.cancel_shielded_checkpoint()
- # Raise the CancelledError received while waiting for child tasks to exit,
- # unless the context manager itself was previously exited with another
- # exception, or if any of the child tasks raised an exception other than
- # CancelledError
- if cancelled_exc_while_waiting_tasks:
- if exc_val is None or ignore_exception:
- raise cancelled_exc_while_waiting_tasks
+ self._active = False
+ if self._exceptions:
+ raise BaseExceptionGroup(
+ "unhandled errors in a TaskGroup", self._exceptions
+ )
+ elif exc_val:
+ raise exc_val
+ except BaseException as exc:
+ if self.cancel_scope.__exit__(type(exc), exc, exc.__traceback__):
+ return True
- return ignore_exception
+ raise
+
+ return self.cancel_scope.__exit__(exc_type, exc_val, exc_tb)
+ finally:
+ del exc_val, exc_tb, self._exceptions
def _spawn(
self,
@@ -730,7 +809,7 @@ class TaskGroup(abc.TaskGroup):
if not isinstance(exc, CancelledError):
self._exceptions.append(exc)
- if not self.cancel_scope._parent_cancelled():
+ if not self.cancel_scope._effectively_cancelled:
self.cancel_scope.cancel()
else:
task_status_future.set_exception(exc)
@@ -806,7 +885,7 @@ class TaskGroup(abc.TaskGroup):
# Threads
#
-_Retval_Queue_Type = Tuple[Optional[T_Retval], Optional[BaseException]]
+_Retval_Queue_Type = tuple[Optional[T_Retval], Optional[BaseException]]
class WorkerThread(Thread):
@@ -955,7 +1034,7 @@ class Process(abc.Process):
_stderr: StreamReaderWrapper | None
async def aclose(self) -> None:
- with CancelScope(shield=True):
+ with CancelScope(shield=True) as scope:
if self._stdin:
await self._stdin.aclose()
if self._stdout:
@@ -963,14 +1042,14 @@ class Process(abc.Process):
if self._stderr:
await self._stderr.aclose()
- try:
- await self.wait()
- except BaseException:
- self.kill()
- with CancelScope(shield=True):
+ scope.shield = False
+ try:
await self.wait()
-
- raise
+ except BaseException:
+ scope.shield = True
+ self.kill()
+ await self.wait()
+ raise
async def wait(self) -> int:
return await self._process.wait()
@@ -2022,9 +2101,7 @@ class AsyncIOTaskInfo(TaskInfo):
if task_state := _task_states.get(task):
if cancel_scope := task_state.cancel_scope:
- return cancel_scope.cancel_called or (
- not cancel_scope.shield and cancel_scope._parent_cancelled()
- )
+ return cancel_scope._effectively_cancelled
return False
@@ -2118,7 +2195,7 @@ class TestRunner(abc.TestRunner):
) -> T_Retval:
if not self._runner_task:
self._send_stream, receive_stream = create_memory_object_stream[
- Tuple[Awaitable[Any], asyncio.Future]
+ tuple[Awaitable[Any], asyncio.Future]
](1)
self._runner_task = self.get_loop().create_task(
self._run_tests_and_fixtures(receive_stream)
@@ -2480,7 +2557,7 @@ class AsyncIOBackend(AsyncBackend):
cls, host: str, port: int, local_address: IPSockAddrType | None = None
) -> abc.SocketStream:
transport, protocol = cast(
- Tuple[asyncio.Transport, StreamProtocol],
+ tuple[asyncio.Transport, StreamProtocol],
await get_running_loop().create_connection(
StreamProtocol, host, port, local_addr=local_address
),
@@ -2659,7 +2736,7 @@ class AsyncIOBackend(AsyncBackend):
@classmethod
def open_signal_receiver(
cls, *signals: Signals
- ) -> ContextManager[AsyncIterator[Signals]]:
+ ) -> AbstractContextManager[AsyncIterator[Signals]]:
return _SignalReceiver(signals)
@classmethod
diff --git a/contrib/python/anyio/anyio/_backends/_trio.py b/contrib/python/anyio/anyio/_backends/_trio.py
index aee974deb6..24dcd74446 100644
--- a/contrib/python/anyio/anyio/_backends/_trio.py
+++ b/contrib/python/anyio/anyio/_backends/_trio.py
@@ -7,8 +7,18 @@ import socket
import sys
import types
import weakref
-from collections.abc import AsyncIterator, Iterable
+from collections.abc import (
+ AsyncGenerator,
+ AsyncIterator,
+ Awaitable,
+ Callable,
+ Collection,
+ Coroutine,
+ Iterable,
+ Sequence,
+)
from concurrent.futures import Future
+from contextlib import AbstractContextManager
from dataclasses import dataclass
from functools import partial
from io import IOBase
@@ -19,15 +29,8 @@ from types import TracebackType
from typing import (
IO,
Any,
- AsyncGenerator,
- Awaitable,
- Callable,
- Collection,
- ContextManager,
- Coroutine,
Generic,
NoReturn,
- Sequence,
TypeVar,
cast,
overload,
@@ -183,13 +186,12 @@ class TaskGroup(abc.TaskGroup):
try:
return await self._nursery_manager.__aexit__(exc_type, exc_val, exc_tb)
except BaseExceptionGroup as exc:
- _, rest = exc.split(trio.Cancelled)
- if not rest:
- cancelled_exc = trio.Cancelled._create()
- raise cancelled_exc from exc
+ if not exc.split(trio.Cancelled)[1]:
+ raise trio.Cancelled._create() from exc
raise
finally:
+ del exc_val, exc_tb
self._active = False
def start_soon(
@@ -1289,7 +1291,7 @@ class TrioBackend(AsyncBackend):
@classmethod
def open_signal_receiver(
cls, *signals: Signals
- ) -> ContextManager[AsyncIterator[Signals]]:
+ ) -> AbstractContextManager[AsyncIterator[Signals]]:
return _SignalReceiver(signals)
@classmethod
diff --git a/contrib/python/anyio/anyio/_core/_fileio.py b/contrib/python/anyio/anyio/_core/_fileio.py
index 214a90bfd8..53d3288c29 100644
--- a/contrib/python/anyio/anyio/_core/_fileio.py
+++ b/contrib/python/anyio/anyio/_core/_fileio.py
@@ -3,7 +3,7 @@ from __future__ import annotations
import os
import pathlib
import sys
-from collections.abc import Callable, Iterable, Iterator, Sequence
+from collections.abc import AsyncIterator, Callable, Iterable, Iterator, Sequence
from dataclasses import dataclass
from functools import partial
from os import PathLike
@@ -12,7 +12,6 @@ from typing import (
TYPE_CHECKING,
Any,
AnyStr,
- AsyncIterator,
Final,
Generic,
overload,
diff --git a/contrib/python/anyio/anyio/_core/_signals.py b/contrib/python/anyio/anyio/_core/_signals.py
index 115c749bd9..f3451d302f 100644
--- a/contrib/python/anyio/anyio/_core/_signals.py
+++ b/contrib/python/anyio/anyio/_core/_signals.py
@@ -1,13 +1,15 @@
from __future__ import annotations
from collections.abc import AsyncIterator
+from contextlib import AbstractContextManager
from signal import Signals
-from typing import ContextManager
from ._eventloop import get_async_backend
-def open_signal_receiver(*signals: Signals) -> ContextManager[AsyncIterator[Signals]]:
+def open_signal_receiver(
+ *signals: Signals,
+) -> AbstractContextManager[AsyncIterator[Signals]]:
"""
Start receiving operating system signals.
diff --git a/contrib/python/anyio/anyio/_core/_streams.py b/contrib/python/anyio/anyio/_core/_streams.py
index aa6b0c222a..6a9814e5a9 100644
--- a/contrib/python/anyio/anyio/_core/_streams.py
+++ b/contrib/python/anyio/anyio/_core/_streams.py
@@ -1,7 +1,7 @@
from __future__ import annotations
import math
-from typing import Tuple, TypeVar
+from typing import TypeVar
from warnings import warn
from ..streams.memory import (
@@ -14,7 +14,7 @@ T_Item = TypeVar("T_Item")
class create_memory_object_stream(
- Tuple[MemoryObjectSendStream[T_Item], MemoryObjectReceiveStream[T_Item]],
+ tuple[MemoryObjectSendStream[T_Item], MemoryObjectReceiveStream[T_Item]],
):
"""
Create a memory object stream.
diff --git a/contrib/python/anyio/anyio/_core/_subprocesses.py b/contrib/python/anyio/anyio/_core/_subprocesses.py
index 1ac2d549df..7ba41a5b03 100644
--- a/contrib/python/anyio/anyio/_core/_subprocesses.py
+++ b/contrib/python/anyio/anyio/_core/_subprocesses.py
@@ -160,38 +160,25 @@ async def open_process(
child process prior to the execution of the subprocess. (POSIX only)
:param pass_fds: sequence of file descriptors to keep open between the parent and
child processes. (POSIX only)
- :param user: effective user to run the process as (Python >= 3.9; POSIX only)
- :param group: effective group to run the process as (Python >= 3.9; POSIX only)
- :param extra_groups: supplementary groups to set in the subprocess (Python >= 3.9;
- POSIX only)
+ :param user: effective user to run the process as (POSIX only)
+ :param group: effective group to run the process as (POSIX only)
+ :param extra_groups: supplementary groups to set in the subprocess (POSIX only)
:param umask: if not negative, this umask is applied in the child process before
- running the given command (Python >= 3.9; POSIX only)
+ running the given command (POSIX only)
:return: an asynchronous process object
"""
kwargs: dict[str, Any] = {}
if user is not None:
- if sys.version_info < (3, 9):
- raise TypeError("the 'user' argument requires Python 3.9 or later")
-
kwargs["user"] = user
if group is not None:
- if sys.version_info < (3, 9):
- raise TypeError("the 'group' argument requires Python 3.9 or later")
-
kwargs["group"] = group
if extra_groups is not None:
- if sys.version_info < (3, 9):
- raise TypeError("the 'extra_groups' argument requires Python 3.9 or later")
-
kwargs["extra_groups"] = group
if umask >= 0:
- if sys.version_info < (3, 9):
- raise TypeError("the 'umask' argument requires Python 3.9 or later")
-
kwargs["umask"] = umask
return await get_async_backend().open_process(
diff --git a/contrib/python/anyio/anyio/abc/_eventloop.py b/contrib/python/anyio/anyio/abc/_eventloop.py
index 2c73bb9ffb..93d0e9d25b 100644
--- a/contrib/python/anyio/anyio/abc/_eventloop.py
+++ b/contrib/python/anyio/anyio/abc/_eventloop.py
@@ -3,7 +3,8 @@ from __future__ import annotations
import math
import sys
from abc import ABCMeta, abstractmethod
-from collections.abc import AsyncIterator, Awaitable
+from collections.abc import AsyncIterator, Awaitable, Callable, Sequence
+from contextlib import AbstractContextManager
from os import PathLike
from signal import Signals
from socket import AddressFamily, SocketKind, socket
@@ -11,9 +12,6 @@ from typing import (
IO,
TYPE_CHECKING,
Any,
- Callable,
- ContextManager,
- Sequence,
TypeVar,
Union,
overload,
@@ -352,7 +350,7 @@ class AsyncBackend(metaclass=ABCMeta):
@abstractmethod
def open_signal_receiver(
cls, *signals: Signals
- ) -> ContextManager[AsyncIterator[Signals]]:
+ ) -> AbstractContextManager[AsyncIterator[Signals]]:
pass
@classmethod
diff --git a/contrib/python/anyio/anyio/abc/_sockets.py b/contrib/python/anyio/anyio/abc/_sockets.py
index b321225a7b..1c6a450cdc 100644
--- a/contrib/python/anyio/anyio/abc/_sockets.py
+++ b/contrib/python/anyio/anyio/abc/_sockets.py
@@ -8,7 +8,7 @@ from io import IOBase
from ipaddress import IPv4Address, IPv6Address
from socket import AddressFamily
from types import TracebackType
-from typing import Any, Tuple, TypeVar, Union
+from typing import Any, TypeVar, Union
from .._core._typedattr import (
TypedAttributeProvider,
@@ -19,10 +19,10 @@ from ._streams import ByteStream, Listener, UnreliableObjectStream
from ._tasks import TaskGroup
IPAddressType = Union[str, IPv4Address, IPv6Address]
-IPSockAddrType = Tuple[str, int]
+IPSockAddrType = tuple[str, int]
SockAddrType = Union[IPSockAddrType, str]
-UDPPacketType = Tuple[bytes, IPSockAddrType]
-UNIXDatagramPacketType = Tuple[bytes, str]
+UDPPacketType = tuple[bytes, IPSockAddrType]
+UNIXDatagramPacketType = tuple[bytes, str]
T_Retval = TypeVar("T_Retval")
diff --git a/contrib/python/anyio/anyio/from_thread.py b/contrib/python/anyio/anyio/from_thread.py
index b8785845ba..93a4cfe8e4 100644
--- a/contrib/python/anyio/anyio/from_thread.py
+++ b/contrib/python/anyio/anyio/from_thread.py
@@ -3,15 +3,17 @@ from __future__ import annotations
import sys
from collections.abc import Awaitable, Callable, Generator
from concurrent.futures import Future
-from contextlib import AbstractContextManager, contextmanager
+from contextlib import (
+ AbstractAsyncContextManager,
+ AbstractContextManager,
+ contextmanager,
+)
from dataclasses import dataclass, field
from inspect import isawaitable
from threading import Lock, Thread, get_ident
from types import TracebackType
from typing import (
Any,
- AsyncContextManager,
- ContextManager,
Generic,
TypeVar,
cast,
@@ -87,7 +89,9 @@ class _BlockingAsyncContextManager(Generic[T_co], AbstractContextManager):
type[BaseException] | None, BaseException | None, TracebackType | None
] = (None, None, None)
- def __init__(self, async_cm: AsyncContextManager[T_co], portal: BlockingPortal):
+ def __init__(
+ self, async_cm: AbstractAsyncContextManager[T_co], portal: BlockingPortal
+ ):
self._async_cm = async_cm
self._portal = portal
@@ -374,8 +378,8 @@ class BlockingPortal:
return f, task_status_future.result()
def wrap_async_context_manager(
- self, cm: AsyncContextManager[T_co]
- ) -> ContextManager[T_co]:
+ self, cm: AbstractAsyncContextManager[T_co]
+ ) -> AbstractContextManager[T_co]:
"""
Wrap an async context manager as a synchronous context manager via this portal.
diff --git a/contrib/python/anyio/anyio/pytest_plugin.py b/contrib/python/anyio/anyio/pytest_plugin.py
index b7d9305614..4a0d59dd06 100644
--- a/contrib/python/anyio/anyio/pytest_plugin.py
+++ b/contrib/python/anyio/anyio/pytest_plugin.py
@@ -4,7 +4,7 @@ import sys
from collections.abc import Generator, Iterator
from contextlib import ExitStack, contextmanager
from inspect import isasyncgenfunction, iscoroutinefunction, ismethod
-from typing import Any, Dict, Tuple, cast
+from typing import Any, cast
import pytest
import sniffio
@@ -28,7 +28,7 @@ def extract_backend_and_options(backend: object) -> tuple[str, dict[str, Any]]:
return backend, {}
elif isinstance(backend, tuple) and len(backend) == 2:
if isinstance(backend[0], str) and isinstance(backend[1], dict):
- return cast(Tuple[str, Dict[str, Any]], backend)
+ return cast(tuple[str, dict[str, Any]], backend)
raise TypeError("anyio_backend must be either a string or tuple of (string, dict)")
diff --git a/contrib/python/anyio/anyio/streams/tls.py b/contrib/python/anyio/anyio/streams/tls.py
index d01c8e6f4c..b6961bee16 100644
--- a/contrib/python/anyio/anyio/streams/tls.py
+++ b/contrib/python/anyio/anyio/streams/tls.py
@@ -7,7 +7,7 @@ import sys
from collections.abc import Callable, Mapping
from dataclasses import dataclass
from functools import wraps
-from typing import Any, Tuple, TypeVar
+from typing import Any, TypeVar
from .. import (
BrokenResourceError,
@@ -25,8 +25,8 @@ else:
T_Retval = TypeVar("T_Retval")
PosArgsT = TypeVarTuple("PosArgsT")
-_PCTRTT = Tuple[Tuple[str, str], ...]
-_PCTRTTT = Tuple[_PCTRTT, ...]
+_PCTRTT = tuple[tuple[str, str], ...]
+_PCTRTTT = tuple[_PCTRTT, ...]
class TLSAttribute(TypedAttributeSet):
diff --git a/contrib/python/anyio/ya.make b/contrib/python/anyio/ya.make
index bb56a53ce5..aadbb5b297 100644
--- a/contrib/python/anyio/ya.make
+++ b/contrib/python/anyio/ya.make
@@ -2,7 +2,7 @@
PY3_LIBRARY()
-VERSION(4.6.2)
+VERSION(4.6.2.post1)
LICENSE(MIT)
diff --git a/contrib/python/pyparsing/py3/.dist-info/METADATA b/contrib/python/pyparsing/py3/.dist-info/METADATA
index 1aa7a1fc04..ff6f9b6227 100644
--- a/contrib/python/pyparsing/py3/.dist-info/METADATA
+++ b/contrib/python/pyparsing/py3/.dist-info/METADATA
@@ -1,9 +1,9 @@
Metadata-Version: 2.1
Name: pyparsing
-Version: 3.1.4
+Version: 3.2.0
Summary: pyparsing module - Classes and methods to define and execute parsing grammars
Author-email: Paul McGuire <ptmcg.gm+pyparsing@gmail.com>
-Requires-Python: >=3.6.8
+Requires-Python: >=3.9
Description-Content-Type: text/x-rst
Classifier: Development Status :: 5 - Production/Stable
Classifier: Intended Audience :: Developers
@@ -12,9 +12,6 @@ Classifier: License :: OSI Approved :: MIT License
Classifier: Operating System :: OS Independent
Classifier: Programming Language :: Python
Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.6
-Classifier: Programming Language :: Python :: 3.7
-Classifier: Programming Language :: Python :: 3.8
Classifier: Programming Language :: Python :: 3.9
Classifier: Programming Language :: Python :: 3.10
Classifier: Programming Language :: Python :: 3.11
diff --git a/contrib/python/pyparsing/py3/pyparsing/__init__.py b/contrib/python/pyparsing/py3/pyparsing/__init__.py
index a440cfbefa..543ceb62bd 100644
--- a/contrib/python/pyparsing/py3/pyparsing/__init__.py
+++ b/contrib/python/pyparsing/py3/pyparsing/__init__.py
@@ -120,8 +120,8 @@ class version_info(NamedTuple):
return f"{__name__}.{type(self).__name__}({', '.join('{}={!r}'.format(*nv) for nv in zip(self._fields, self))})"
-__version_info__ = version_info(3, 1, 4, "final", 1)
-__version_time__ = "25 Aug 2024 14:40 UTC"
+__version_info__ = version_info(3, 2, 0, "final", 1)
+__version_time__ = "13 Oct 2024 09:46 UTC"
__version__ = __version_info__.__version__
__versionTime__ = __version_time__
__author__ = "Paul McGuire <ptmcg.gm+pyparsing@gmail.com>"
@@ -131,9 +131,9 @@ from .exceptions import *
from .actions import *
from .core import __diag__, __compat__
from .results import *
-from .core import * # type: ignore[misc, assignment]
+from .core import *
from .core import _builtin_exprs as core_builtin_exprs
-from .helpers import * # type: ignore[misc, assignment]
+from .helpers import *
from .helpers import _builtin_exprs as helper_builtin_exprs
from .unicode import unicode_set, UnicodeRangeList, pyparsing_unicode as unicode
@@ -147,9 +147,9 @@ from .common import (
if "pyparsing_unicode" not in globals():
pyparsing_unicode = unicode # type: ignore[misc]
if "pyparsing_common" not in globals():
- pyparsing_common = common # type: ignore[misc]
+ pyparsing_common = common
if "pyparsing_test" not in globals():
- pyparsing_test = testing # type: ignore[misc]
+ pyparsing_test = testing
core_builtin_exprs += common_builtin_exprs + helper_builtin_exprs
@@ -208,6 +208,7 @@ __all__ = [
"StringEnd",
"StringStart",
"Suppress",
+ "Tag",
"Token",
"TokenConverter",
"White",
diff --git a/contrib/python/pyparsing/py3/pyparsing/core.py b/contrib/python/pyparsing/py3/pyparsing/core.py
index cbe73c987a..4f43c3bf99 100644
--- a/contrib/python/pyparsing/py3/pyparsing/core.py
+++ b/contrib/python/pyparsing/py3/pyparsing/core.py
@@ -1,7 +1,9 @@
#
# core.py
#
+from __future__ import annotations
+import collections.abc
from collections import deque
import os
import typing
@@ -9,12 +11,9 @@ from typing import (
Any,
Callable,
Generator,
- List,
NamedTuple,
Sequence,
- Set,
TextIO,
- Tuple,
Union,
cast,
)
@@ -51,12 +50,7 @@ from .results import ParseResults, _ParseResultsWithOffset
from .unicode import pyparsing_unicode
_MAX_INT = sys.maxsize
-str_type: Tuple[type, ...] = (str, bytes)
-
-if sys.version_info >= (3, 7):
- _RePattern = re.Pattern
-else:
- _RePattern = typing.Pattern
+str_type: tuple[type, ...] = (str, bytes)
#
# Copyright (c) 2003-2022 Paul T. McGuire
@@ -81,18 +75,7 @@ else:
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
#
-
-if sys.version_info >= (3, 8):
- from functools import cached_property
-else:
-
- class cached_property:
- def __init__(self, func):
- self._func = func
-
- def __get__(self, instance, owner=None):
- ret = instance.__dict__[self._func.__name__] = self._func(instance)
- return ret
+from functools import cached_property
class __compat__(__config_flags):
@@ -230,7 +213,7 @@ _single_arg_builtins = {
# fmt: on
_generatorType = types.GeneratorType
-ParseImplReturnType = Tuple[int, Any]
+ParseImplReturnType = tuple[int, Any]
PostParseReturnType = Union[ParseResults, Sequence[ParseResults]]
ParseAction = Union[
Callable[[], Any],
@@ -260,12 +243,26 @@ hexnums: str = nums + "ABCDEFabcdef"
alphanums: str = alphas + nums
printables: str = "".join([c for c in string.printable if c not in string.whitespace])
+
+class _ParseActionIndexError(Exception):
+ """
+ Internal wrapper around IndexError so that IndexErrors raised inside
+ parse actions aren't misinterpreted as IndexErrors raised inside
+ ParserElement parseImpl methods.
+ """
+
+ def __init__(self, msg: str, exc: BaseException):
+ self.msg: str = msg
+ self.exc: BaseException = exc
+
+
_trim_arity_call_line: traceback.StackSummary = None # type: ignore[assignment]
+pa_call_line_synth = ()
def _trim_arity(func, max_limit=3):
"""decorator to trim function calls to match the arity of the target"""
- global _trim_arity_call_line
+ global _trim_arity_call_line, pa_call_line_synth
if func in _single_arg_builtins:
return lambda s, l, t: func(t)
@@ -280,8 +277,8 @@ def _trim_arity(func, max_limit=3):
LINE_DIFF = 9
# IF ANY CODE CHANGES, EVEN JUST COMMENTS OR BLANK LINES, BETWEEN THE NEXT LINE AND
# THE CALL TO FUNC INSIDE WRAPPER, LINE_DIFF MUST BE MODIFIED!!!!
- _trim_arity_call_line = (_trim_arity_call_line or traceback.extract_stack(limit=2)[-1])
- pa_call_line_synth = (_trim_arity_call_line[0], _trim_arity_call_line[1] + LINE_DIFF)
+ _trim_arity_call_line = _trim_arity_call_line or traceback.extract_stack(limit=2)[-1]
+ pa_call_line_synth = pa_call_line_synth or (_trim_arity_call_line[0], _trim_arity_call_line[1] + LINE_DIFF)
def wrapper(*args):
nonlocal found_arity, limit
@@ -311,6 +308,11 @@ def _trim_arity(func, max_limit=3):
continue
raise
+ except IndexError as ie:
+ # wrap IndexErrors inside a _ParseActionIndexError
+ raise _ParseActionIndexError(
+ "IndexError raised in parse action", ie
+ ).with_traceback(None)
# fmt: on
# copy func name to wrapper for sensible debug output
@@ -351,7 +353,7 @@ def condition_as_parse_action(
def _default_start_debug_action(
- instring: str, loc: int, expr: "ParserElement", cache_hit: bool = False
+ instring: str, loc: int, expr: ParserElement, cache_hit: bool = False
):
cache_hit_str = "*" if cache_hit else ""
print(
@@ -367,7 +369,7 @@ def _default_success_debug_action(
instring: str,
startloc: int,
endloc: int,
- expr: "ParserElement",
+ expr: ParserElement,
toks: ParseResults,
cache_hit: bool = False,
):
@@ -378,7 +380,7 @@ def _default_success_debug_action(
def _default_exception_debug_action(
instring: str,
loc: int,
- expr: "ParserElement",
+ expr: ParserElement,
exc: Exception,
cache_hit: bool = False,
):
@@ -443,7 +445,7 @@ class ParserElement(ABC):
@classmethod
def using_each(cls, seq, **class_kwargs):
"""
- Yields a sequence of class(obj, **class_kwargs) for obj in seq.
+ Yields a sequence of ``class(obj, **class_kwargs)`` for obj in seq.
Example::
@@ -458,7 +460,7 @@ class ParserElement(ABC):
debug_fail: typing.Optional[DebugExceptionAction]
def __init__(self, savelist: bool = False):
- self.parseAction: List[ParseAction] = list()
+ self.parseAction: list[ParseAction] = list()
self.failAction: typing.Optional[ParseFailAction] = None
self.customName: str = None # type: ignore[assignment]
self._defaultName: typing.Optional[str] = None
@@ -470,7 +472,7 @@ class ParserElement(ABC):
# used when checking for left-recursion
self.mayReturnEmpty = False
self.keepTabs = False
- self.ignoreExprs: List["ParserElement"] = list()
+ self.ignoreExprs: list[ParserElement] = list()
self.debug = False
self.streamlined = False
# optimize exception handling for subclasses that don't advance parse index
@@ -483,9 +485,9 @@ class ParserElement(ABC):
# avoid redundant calls to preParse
self.callPreparse = True
self.callDuringTry = False
- self.suppress_warnings_: List[Diagnostics] = []
+ self.suppress_warnings_: list[Diagnostics] = []
- def suppress_warning(self, warning_type: Diagnostics) -> "ParserElement":
+ def suppress_warning(self, warning_type: Diagnostics) -> ParserElement:
"""
Suppress warnings emitted for a particular diagnostic on this expression.
@@ -518,7 +520,7 @@ class ParserElement(ABC):
to_visit.extend(cur.recurse())
yield cur
- def copy(self) -> "ParserElement":
+ def copy(self) -> ParserElement:
"""
Make a copy of this :class:`ParserElement`. Useful for defining
different parse actions for the same parsing pattern, using copies of
@@ -549,7 +551,7 @@ class ParserElement(ABC):
def set_results_name(
self, name: str, list_all_matches: bool = False, *, listAllMatches: bool = False
- ) -> "ParserElement":
+ ) -> ParserElement:
"""
Define name for referencing matching tokens as a nested attribute
of the returned parse results.
@@ -581,7 +583,7 @@ class ParserElement(ABC):
listAllMatches = listAllMatches or list_all_matches
return self._setResultsName(name, listAllMatches)
- def _setResultsName(self, name, list_all_matches=False) -> "ParserElement":
+ def _setResultsName(self, name, list_all_matches=False) -> ParserElement:
if name is None:
return self
newself = self.copy()
@@ -592,7 +594,7 @@ class ParserElement(ABC):
newself.modalResults = not list_all_matches
return newself
- def set_break(self, break_flag: bool = True) -> "ParserElement":
+ def set_break(self, break_flag: bool = True) -> ParserElement:
"""
Method to invoke the Python pdb debugger when this element is
about to be parsed. Set ``break_flag`` to ``True`` to enable, ``False`` to
@@ -602,19 +604,17 @@ class ParserElement(ABC):
_parseMethod = self._parse
def breaker(instring, loc, do_actions=True, callPreParse=True):
- import pdb
-
- # this call to pdb.set_trace() is intentional, not a checkin error
- pdb.set_trace()
+ # this call to breakpoint() is intentional, not a checkin error
+ breakpoint()
return _parseMethod(instring, loc, do_actions, callPreParse)
breaker._originalParseMethod = _parseMethod # type: ignore [attr-defined]
- self._parse = breaker # type: ignore [assignment]
+ self._parse = breaker # type: ignore [method-assign]
elif hasattr(self._parse, "_originalParseMethod"):
- self._parse = self._parse._originalParseMethod # type: ignore [attr-defined, assignment]
+ self._parse = self._parse._originalParseMethod # type: ignore [method-assign]
return self
- def set_parse_action(self, *fns: ParseAction, **kwargs: Any) -> "ParserElement":
+ def set_parse_action(self, *fns: ParseAction, **kwargs: Any) -> ParserElement:
"""
Define one or more actions to perform when successfully matching parse element definition.
@@ -702,7 +702,7 @@ class ParserElement(ABC):
return self
- def add_parse_action(self, *fns: ParseAction, **kwargs: Any) -> "ParserElement":
+ def add_parse_action(self, *fns: ParseAction, **kwargs: Any) -> ParserElement:
"""
Add one or more parse actions to expression's list of parse actions. See :class:`set_parse_action`.
@@ -714,7 +714,7 @@ class ParserElement(ABC):
)
return self
- def add_condition(self, *fns: ParseCondition, **kwargs: Any) -> "ParserElement":
+ def add_condition(self, *fns: ParseCondition, **kwargs: Any) -> ParserElement:
"""Add a boolean predicate function to expression's list of parse actions. See
:class:`set_parse_action` for function call signatures. Unlike ``set_parse_action``,
functions passed to ``add_condition`` need to return boolean success/fail of the condition.
@@ -751,7 +751,7 @@ class ParserElement(ABC):
)
return self
- def set_fail_action(self, fn: ParseFailAction) -> "ParserElement":
+ def set_fail_action(self, fn: ParseFailAction) -> ParserElement:
"""
Define action to perform if parsing fails at this expression.
Fail acton fn is a callable function that takes the arguments
@@ -809,8 +809,7 @@ class ParserElement(ABC):
# @profile
def _parseNoCache(
self, instring, loc, do_actions=True, callPreParse=True
- ) -> Tuple[int, ParseResults]:
- TRY, MATCH, FAIL = 0, 1, 2
+ ) -> tuple[int, ParseResults]:
debugging = self.debug # and do_actions)
len_instring = len(instring)
@@ -934,25 +933,42 @@ class ParserElement(ABC):
# cache for left-recursion in Forward references
recursion_lock = RLock()
- recursion_memos: typing.Dict[
- Tuple[int, "Forward", bool], Tuple[int, Union[ParseResults, Exception]]
+ recursion_memos: collections.abc.MutableMapping[
+ tuple[int, Forward, bool], tuple[int, Union[ParseResults, Exception]]
] = {}
- class _CacheType(dict):
+ class _CacheType(typing.Protocol):
"""
- class to help type checking
+ Class to be used for packrat and left-recursion cacheing of results
+ and exceptions.
"""
not_in_cache: bool
- def get(self, *args): ...
+ def get(self, *args) -> typing.Any: ...
+
+ def set(self, *args) -> None: ...
+
+ def clear(self) -> None: ...
+
+ class NullCache(dict):
+ """
+ A null cache type for initialization of the packrat_cache class variable.
+ If/when enable_packrat() is called, this null cache will be replaced by a
+ proper _CacheType class instance.
+ """
+
+ not_in_cache: bool = True
+
+ def get(self, *args) -> typing.Any: ...
- def set(self, *args): ...
+ def set(self, *args) -> None: ...
- # argument cache for optimizing repeated calls when backtracking through recursive expressions
- packrat_cache = (
- _CacheType()
- ) # set later by enable_packrat(); this is here so that reset_cache() doesn't fail
+ def clear(self) -> None: ...
+
+ # class-level argument cache for optimizing repeated calls when backtracking
+ # through recursive expressions
+ packrat_cache: _CacheType = NullCache()
packrat_cache_lock = RLock()
packrat_cache_stats = [0, 0]
@@ -960,9 +976,8 @@ class ParserElement(ABC):
# we can cache these arguments and save ourselves the trouble of re-parsing the contained expression
def _parseCache(
self, instring, loc, do_actions=True, callPreParse=True
- ) -> Tuple[int, ParseResults]:
+ ) -> tuple[int, ParseResults]:
HIT, MISS = 0, 1
- TRY, MATCH, FAIL = 0, 1, 2
lookup = (self, instring, loc, callPreParse, do_actions)
with ParserElement.packrat_cache_lock:
cache = ParserElement.packrat_cache
@@ -995,7 +1010,7 @@ class ParserElement(ABC):
pass
raise value
- value = cast(Tuple[int, ParseResults, int], value)
+ value = cast(tuple[int, ParseResults, int], value)
loc_, result, endloc = value[0], value[1].copy(), value[2]
if self.debug and self.debugActions.debug_match:
try:
@@ -1075,7 +1090,7 @@ class ParserElement(ABC):
elif ParserElement._packratEnabled:
raise RuntimeError("Packrat and Bounded Recursion are not compatible")
if cache_size_limit is None:
- ParserElement.recursion_memos = _UnboundedMemo() # type: ignore[assignment]
+ ParserElement.recursion_memos = _UnboundedMemo()
elif cache_size_limit > 0:
ParserElement.recursion_memos = _LRUMemo(capacity=cache_size_limit) # type: ignore[assignment]
else:
@@ -1128,7 +1143,7 @@ class ParserElement(ABC):
if cache_size_limit is None:
ParserElement.packrat_cache = _UnboundedCache()
else:
- ParserElement.packrat_cache = _FifoCache(cache_size_limit) # type: ignore[assignment]
+ ParserElement.packrat_cache = _FifoCache(cache_size_limit)
ParserElement._parse = ParserElement._parseCache
def parse_string(
@@ -1191,12 +1206,14 @@ class ParserElement(ABC):
loc = self.preParse(instring, loc)
se = Empty() + StringEnd().set_debug(False)
se._parse(instring, loc)
+ except _ParseActionIndexError as pa_exc:
+ raise pa_exc.exc
except ParseBaseException as exc:
if ParserElement.verbose_stacktrace:
raise
- else:
- # catch and re-raise exception from here, clearing out pyparsing internal stack trace
- raise exc.with_traceback(None)
+
+ # catch and re-raise exception from here, clearing out pyparsing internal stack trace
+ raise exc.with_traceback(None)
else:
return tokens
@@ -1205,10 +1222,11 @@ class ParserElement(ABC):
instring: str,
max_matches: int = _MAX_INT,
overlap: bool = False,
+ always_skip_whitespace=True,
*,
debug: bool = False,
maxMatches: int = _MAX_INT,
- ) -> Generator[Tuple[ParseResults, int, int], None, None]:
+ ) -> Generator[tuple[ParseResults, int, int], None, None]:
"""
Scan the input string for expression matches. Each match will return the
matching tokens, start location, and end location. May be called with optional
@@ -1249,7 +1267,13 @@ class ParserElement(ABC):
instring = str(instring).expandtabs()
instrlen = len(instring)
loc = 0
- preparseFn = self.preParse
+ if always_skip_whitespace:
+ preparser = Empty()
+ preparser.ignoreExprs = self.ignoreExprs
+ preparser.whiteChars = self.whiteChars
+ preparseFn = preparser.preParse
+ else:
+ preparseFn = self.preParse
parseFn = self._parse
ParserElement.resetCache()
matches = 0
@@ -1311,14 +1335,15 @@ class ParserElement(ABC):
Now Is The Winter Of Our Discontent Made Glorious Summer By This Sun Of York.
"""
- out: List[str] = []
+ out: list[str] = []
lastE = 0
# force preservation of <TAB>s, to minimize unwanted transformation of string, and to
# keep string locs straight between transform_string and scan_string
self.keepTabs = True
try:
for t, s, e in self.scan_string(instring, debug=debug):
- out.append(instring[lastE:s])
+ if s > lastE:
+ out.append(instring[lastE:s])
lastE = e
if not t:
@@ -1372,7 +1397,12 @@ class ParserElement(ABC):
maxMatches = min(maxMatches, max_matches)
try:
return ParseResults(
- [t for t, s, e in self.scan_string(instring, maxMatches, debug=debug)]
+ [
+ t
+ for t, s, e in self.scan_string(
+ instring, maxMatches, always_skip_whitespace=False, debug=debug
+ )
+ ]
)
except ParseBaseException as exc:
if ParserElement.verbose_stacktrace:
@@ -1413,7 +1443,7 @@ class ParserElement(ABC):
last = e
yield instring[last:]
- def __add__(self, other) -> "ParserElement":
+ def __add__(self, other) -> ParserElement:
"""
Implementation of ``+`` operator - returns :class:`And`. Adding strings to a :class:`ParserElement`
converts them to :class:`Literal`\\ s by default.
@@ -1449,7 +1479,7 @@ class ParserElement(ABC):
return NotImplemented
return And([self, other])
- def __radd__(self, other) -> "ParserElement":
+ def __radd__(self, other) -> ParserElement:
"""
Implementation of ``+`` operator when left operand is not a :class:`ParserElement`
"""
@@ -1462,7 +1492,7 @@ class ParserElement(ABC):
return NotImplemented
return other + self
- def __sub__(self, other) -> "ParserElement":
+ def __sub__(self, other) -> ParserElement:
"""
Implementation of ``-`` operator, returns :class:`And` with error stop
"""
@@ -1472,7 +1502,7 @@ class ParserElement(ABC):
return NotImplemented
return self + And._ErrorStop() + other
- def __rsub__(self, other) -> "ParserElement":
+ def __rsub__(self, other) -> ParserElement:
"""
Implementation of ``-`` operator when left operand is not a :class:`ParserElement`
"""
@@ -1482,7 +1512,7 @@ class ParserElement(ABC):
return NotImplemented
return other - self
- def __mul__(self, other) -> "ParserElement":
+ def __mul__(self, other) -> ParserElement:
"""
Implementation of ``*`` operator, allows use of ``expr * 3`` in place of
``expr + expr + expr``. Expressions may also be multiplied by a 2-integer
@@ -1562,10 +1592,10 @@ class ParserElement(ABC):
ret = And([self] * minElements)
return ret
- def __rmul__(self, other) -> "ParserElement":
+ def __rmul__(self, other) -> ParserElement:
return self.__mul__(other)
- def __or__(self, other) -> "ParserElement":
+ def __or__(self, other) -> ParserElement:
"""
Implementation of ``|`` operator - returns :class:`MatchFirst`
"""
@@ -1581,7 +1611,7 @@ class ParserElement(ABC):
return NotImplemented
return MatchFirst([self, other])
- def __ror__(self, other) -> "ParserElement":
+ def __ror__(self, other) -> ParserElement:
"""
Implementation of ``|`` operator when left operand is not a :class:`ParserElement`
"""
@@ -1591,7 +1621,7 @@ class ParserElement(ABC):
return NotImplemented
return other | self
- def __xor__(self, other) -> "ParserElement":
+ def __xor__(self, other) -> ParserElement:
"""
Implementation of ``^`` operator - returns :class:`Or`
"""
@@ -1601,7 +1631,7 @@ class ParserElement(ABC):
return NotImplemented
return Or([self, other])
- def __rxor__(self, other) -> "ParserElement":
+ def __rxor__(self, other) -> ParserElement:
"""
Implementation of ``^`` operator when left operand is not a :class:`ParserElement`
"""
@@ -1611,7 +1641,7 @@ class ParserElement(ABC):
return NotImplemented
return other ^ self
- def __and__(self, other) -> "ParserElement":
+ def __and__(self, other) -> ParserElement:
"""
Implementation of ``&`` operator - returns :class:`Each`
"""
@@ -1621,7 +1651,7 @@ class ParserElement(ABC):
return NotImplemented
return Each([self, other])
- def __rand__(self, other) -> "ParserElement":
+ def __rand__(self, other) -> ParserElement:
"""
Implementation of ``&`` operator when left operand is not a :class:`ParserElement`
"""
@@ -1631,7 +1661,7 @@ class ParserElement(ABC):
return NotImplemented
return other & self
- def __invert__(self) -> "ParserElement":
+ def __invert__(self) -> ParserElement:
"""
Implementation of ``~`` operator - returns :class:`NotAny`
"""
@@ -1701,7 +1731,7 @@ class ParserElement(ABC):
return ret
- def __call__(self, name: typing.Optional[str] = None) -> "ParserElement":
+ def __call__(self, name: typing.Optional[str] = None) -> ParserElement:
"""
Shortcut for :class:`set_results_name`, with ``list_all_matches=False``.
@@ -1721,14 +1751,14 @@ class ParserElement(ABC):
return self.copy()
- def suppress(self) -> "ParserElement":
+ def suppress(self) -> ParserElement:
"""
Suppresses the output of this :class:`ParserElement`; useful to keep punctuation from
cluttering up returned output.
"""
return Suppress(self)
- def ignore_whitespace(self, recursive: bool = True) -> "ParserElement":
+ def ignore_whitespace(self, recursive: bool = True) -> ParserElement:
"""
Enables the skipping of whitespace before matching the characters in the
:class:`ParserElement`'s defined pattern.
@@ -1738,7 +1768,7 @@ class ParserElement(ABC):
self.skipWhitespace = True
return self
- def leave_whitespace(self, recursive: bool = True) -> "ParserElement":
+ def leave_whitespace(self, recursive: bool = True) -> ParserElement:
"""
Disables the skipping of whitespace before matching the characters in the
:class:`ParserElement`'s defined pattern. This is normally only used internally by
@@ -1750,8 +1780,8 @@ class ParserElement(ABC):
return self
def set_whitespace_chars(
- self, chars: Union[Set[str], str], copy_defaults: bool = False
- ) -> "ParserElement":
+ self, chars: Union[set[str], str], copy_defaults: bool = False
+ ) -> ParserElement:
"""
Overrides the default whitespace chars
"""
@@ -1760,7 +1790,7 @@ class ParserElement(ABC):
self.copyDefaultWhiteChars = copy_defaults
return self
- def parse_with_tabs(self) -> "ParserElement":
+ def parse_with_tabs(self) -> ParserElement:
"""
Overrides default behavior to expand ``<TAB>`` s to spaces before parsing the input string.
Must be called before ``parse_string`` when the input grammar contains elements that
@@ -1769,7 +1799,7 @@ class ParserElement(ABC):
self.keepTabs = True
return self
- def ignore(self, other: "ParserElement") -> "ParserElement":
+ def ignore(self, other: ParserElement) -> ParserElement:
"""
Define expression to be ignored (e.g., comments) while doing pattern
matching; may be called repeatedly, to define multiple comment or other
@@ -1800,7 +1830,7 @@ class ParserElement(ABC):
start_action: DebugStartAction,
success_action: DebugSuccessAction,
exception_action: DebugExceptionAction,
- ) -> "ParserElement":
+ ) -> ParserElement:
"""
Customize display of debugging messages while doing pattern matching:
@@ -1821,7 +1851,7 @@ class ParserElement(ABC):
self.debug = True
return self
- def set_debug(self, flag: bool = True, recurse: bool = False) -> "ParserElement":
+ def set_debug(self, flag: bool = True, recurse: bool = False) -> ParserElement:
"""
Enable display of debugging messages while doing pattern matching.
Set ``flag`` to ``True`` to enable, ``False`` to disable.
@@ -1886,7 +1916,7 @@ class ParserElement(ABC):
Child classes must define this method, which defines how the ``default_name`` is set.
"""
- def set_name(self, name: typing.Optional[str]) -> "ParserElement":
+ def set_name(self, name: typing.Optional[str]) -> ParserElement:
"""
Define name for this expression, makes debugging and exception messages clearer. If
`__diag__.enable_debug_on_named_expressions` is set to True, setting a name will also
@@ -1903,7 +1933,7 @@ class ParserElement(ABC):
integer.set_name("integer")
integer.parse_string("ABC") # -> Exception: Expected integer (at char 0), (line:1, col:1)
"""
- self.customName = name
+ self.customName = name # type: ignore[assignment]
self.errmsg = f"Expected {str(self)}"
if __diag__.enable_debug_on_named_expressions:
@@ -1926,12 +1956,12 @@ class ParserElement(ABC):
def __repr__(self) -> str:
return str(self)
- def streamline(self) -> "ParserElement":
+ def streamline(self) -> ParserElement:
self.streamlined = True
self._defaultName = None
return self
- def recurse(self) -> List["ParserElement"]:
+ def recurse(self) -> list[ParserElement]:
return []
def _checkRecursion(self, parseElementList):
@@ -2018,9 +2048,9 @@ class ParserElement(ABC):
def run_tests(
self,
- tests: Union[str, List[str]],
+ tests: Union[str, list[str]],
parse_all: bool = True,
- comment: typing.Optional[Union["ParserElement", str]] = "#",
+ comment: typing.Optional[Union[ParserElement, str]] = "#",
full_dump: bool = True,
print_results: bool = True,
failure_tests: bool = False,
@@ -2037,7 +2067,7 @@ class ParserElement(ABC):
postParse: typing.Optional[
Callable[[str, ParseResults], typing.Optional[str]]
] = None,
- ) -> Tuple[bool, List[Tuple[str, Union[ParseResults, Exception]]]]:
+ ) -> tuple[bool, list[tuple[str, Union[ParseResults, Exception]]]]:
"""
Execute the parse expression on a series of test strings, showing each
test, the parsed results or where the parse failed. Quick and easy way to
@@ -2155,8 +2185,8 @@ class ParserElement(ABC):
print_ = file.write
result: Union[ParseResults, Exception]
- allResults: List[Tuple[str, Union[ParseResults, Exception]]] = []
- comments: List[str] = []
+ allResults: list[tuple[str, Union[ParseResults, Exception]]] = []
+ comments: list[str] = []
success = True
NL = Literal(r"\n").add_parse_action(replace_with("\n")).ignore(quoted_string)
BOM = "\ufeff"
@@ -2187,7 +2217,18 @@ class ParserElement(ABC):
success = success and failureTests
result = pe
except Exception as exc:
- out.append(f"FAIL-EXCEPTION: {type(exc).__name__}: {exc}")
+ tag = "FAIL-EXCEPTION"
+
+ # see if this exception was raised in a parse action
+ tb = exc.__traceback__
+ it = iter(traceback.walk_tb(tb))
+ for f, line in it:
+ if (f.f_code.co_filename, line) == pa_call_line_synth:
+ next_f = next(it)[0]
+ tag += f" (raised in parse action {next_f.f_code.co_name!r})"
+ break
+
+ out.append(f"{tag}: {type(exc).__name__}: {exc}")
if ParserElement.verbose_stacktrace:
out.extend(traceback.format_tb(exc.__traceback__))
success = success and failureTests
@@ -2323,7 +2364,7 @@ class _PendingSkip(ParserElement):
def _generateDefaultName(self) -> str:
return str(self.anchor + Empty()).replace("Empty", "...")
- def __add__(self, other) -> "ParserElement":
+ def __add__(self, other) -> ParserElement:
skipper = SkipTo(other).set_name("...")("_skipped*")
if self.must_skip:
@@ -2505,9 +2546,8 @@ class Keyword(Token):
match_string = matchString or match_string
self.match = match_string
self.matchLen = len(match_string)
- try:
- self.firstMatchChar = match_string[0]
- except IndexError:
+ self.firstMatchChar = match_string[:1]
+ if not self.firstMatchChar:
raise ValueError("null string passed to Keyword; use Empty() instead")
self.errmsg = f"Expected {type(self).__name__} {self.name}"
self.mayReturnEmpty = False
@@ -2522,7 +2562,7 @@ class Keyword(Token):
return repr(self.match)
def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
- errmsg = self.errmsg
+ errmsg = self.errmsg or ""
errloc = loc
if self.caseless:
if instring[loc : loc + self.matchLen].upper() == self.caselessmatch:
@@ -2900,7 +2940,7 @@ class Word(Token):
self.re = None # type: ignore[assignment]
else:
self.re_match = self.re.match
- self.parseImpl = self.parseImpl_regex # type: ignore[assignment]
+ self.parseImpl = self.parseImpl_regex # type: ignore[method-assign]
def _generateDefaultName(self) -> str:
def charsAsStr(s):
@@ -3040,48 +3080,65 @@ class Regex(Token):
self._re = None
self.reString = self.pattern = pattern
- self.flags = flags
elif hasattr(pattern, "pattern") and hasattr(pattern, "match"):
self._re = pattern
self.pattern = self.reString = pattern.pattern
- self.flags = flags
+
+ elif callable(pattern):
+ # defer creating this pattern until we really need it
+ self.pattern = pattern
+ self._re = None
else:
raise TypeError(
- "Regex may only be constructed with a string or a compiled RE object"
+ "Regex may only be constructed with a string or a compiled RE object,"
+ " or a callable that takes no arguments and returns a string or a"
+ " compiled RE object"
)
+ self.flags = flags
self.errmsg = f"Expected {self.name}"
self.mayIndexError = False
self.asGroupList = asGroupList
self.asMatch = asMatch
if self.asGroupList:
- self.parseImpl = self.parseImplAsGroupList # type: ignore [assignment]
+ self.parseImpl = self.parseImplAsGroupList # type: ignore [method-assign]
if self.asMatch:
- self.parseImpl = self.parseImplAsMatch # type: ignore [assignment]
+ self.parseImpl = self.parseImplAsMatch # type: ignore [method-assign]
@cached_property
- def re(self) -> _RePattern:
+ def re(self) -> re.Pattern:
if self._re:
return self._re
+ if callable(self.pattern):
+ # replace self.pattern with the string returned by calling self.pattern()
+ self.pattern = cast(Callable[[], str], self.pattern)()
+
+ # see if we got a compiled RE back instead of a str - if so, we're done
+ if hasattr(self.pattern, "pattern") and hasattr(self.pattern, "match"):
+ self._re = cast(re.Pattern[str], self.pattern)
+ self.pattern = self.reString = self._re.pattern
+ return self._re
+
try:
- return re.compile(self.pattern, self.flags)
+ self._re = re.compile(self.pattern, self.flags)
+ return self._re
except re.error:
raise ValueError(f"invalid pattern ({self.pattern!r}) passed to Regex")
@cached_property
- def re_match(self) -> Callable[[str], Any]:
+ def re_match(self) -> Callable[[str, int], Any]:
return self.re.match
@cached_property
- def mayReturnEmpty(self) -> bool:
- return self.re_match("") is not None
+ def mayReturnEmpty(self) -> bool: # type: ignore[override]
+ return self.re_match("", 0) is not None
def _generateDefaultName(self) -> str:
- unescaped = self.pattern.replace("\\\\", "\\")
- return f"Re:({unescaped!r})"
+ unescaped = repr(self.pattern).replace("\\\\", "\\")
+ return f"Re:({unescaped})"
def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
result = self.re_match(instring, loc)
@@ -3243,7 +3300,7 @@ class QuotedString(Token):
# fmt: off
# build up re pattern for the content between the quote delimiters
- inner_pattern: List[str] = []
+ inner_pattern: list[str] = []
if esc_quote:
inner_pattern.append(rf"(?:{re.escape(esc_quote)})")
@@ -3287,6 +3344,7 @@ class QuotedString(Token):
if self.convert_whitespace_escapes:
self.unquote_scan_re = re.compile(
rf"({'|'.join(re.escape(k) for k in self.ws_map)})"
+ rf"|(\\[0-7]{3}|\\0|\\x[0-9a-fA-F]{2}|\\u[0-9a-fA-F]{4})"
rf"|({re.escape(self.esc_char)}.)"
rf"|(\n|.)",
flags=self.re_flags,
@@ -3333,6 +3391,16 @@ class QuotedString(Token):
loc = result.end()
ret = result.group()
+ def convert_escaped_numerics(s: str) -> str:
+ if s == "0":
+ return "\0"
+ if s.isdigit() and len(s) == 3:
+ return chr(int(s, base=8))
+ elif s.startswith(("u", "x")):
+ return chr(int(s[1:], base=16))
+ else:
+ return s
+
if self.unquote_results:
# strip off quotes
ret = ret[self.quote_char_len : -self.end_quote_char_len]
@@ -3346,10 +3414,13 @@ class QuotedString(Token):
ret = "".join(
# match group 1 matches \t, \n, etc.
self.ws_map[match.group(1)] if match.group(1)
- # match group 2 matches escaped characters
- else match.group(2)[-1] if match.group(2)
- # match group 3 matches any character
- else match.group(3)
+ # match group 2 matches escaped octal, null, hex, and Unicode
+ # sequences
+ else convert_escaped_numerics(match.group(2)[1:]) if match.group(2)
+ # match group 3 matches escaped characters
+ else match.group(3)[-1] if match.group(3)
+ # match group 4 matches any character
+ else match.group(4)
for match in self.unquote_scan_re.finditer(ret)
)
else:
@@ -3754,6 +3825,7 @@ class Tag(Token):
['Hello,', 'World', '!']
- enthusiastic: True
"""
+
def __init__(self, tag_name: str, value: Any = True):
super().__init__()
self.mayReturnEmpty = True
@@ -3777,7 +3849,7 @@ class ParseExpression(ParserElement):
def __init__(self, exprs: typing.Iterable[ParserElement], savelist: bool = False):
super().__init__(savelist)
- self.exprs: List[ParserElement]
+ self.exprs: list[ParserElement]
if isinstance(exprs, _generatorType):
exprs = list(exprs)
@@ -3801,7 +3873,7 @@ class ParseExpression(ParserElement):
self.exprs = [exprs]
self.callPreparse = False
- def recurse(self) -> List[ParserElement]:
+ def recurse(self) -> list[ParserElement]:
return self.exprs[:]
def append(self, other) -> ParserElement:
@@ -3943,7 +4015,7 @@ class ParseExpression(ParserElement):
class And(ParseExpression):
"""
- Requires all given :class:`ParseExpression` s to be found in the given order.
+ Requires all given :class:`ParserElement` s to be found in the given order.
Expressions may be separated by whitespace.
May be constructed using the ``'+'`` operator.
May also be constructed using the ``'-'`` operator, which will
@@ -3970,9 +4042,9 @@ class And(ParseExpression):
def __init__(
self, exprs_arg: typing.Iterable[ParserElement], savelist: bool = True
):
- exprs: List[ParserElement] = list(exprs_arg)
+ exprs: list[ParserElement] = list(exprs_arg)
if exprs and Ellipsis in exprs:
- tmp: List[ParserElement] = []
+ tmp: list[ParserElement] = []
for i, expr in enumerate(exprs):
if expr is not Ellipsis:
tmp.append(expr)
@@ -4104,7 +4176,7 @@ class And(ParseExpression):
class Or(ParseExpression):
- """Requires that at least one :class:`ParseExpression` is found. If
+ """Requires that at least one :class:`ParserElement` is found. If
two expressions match, the expression that matches the longest
string will be used. May be constructed using the ``'^'``
operator.
@@ -4144,8 +4216,8 @@ class Or(ParseExpression):
def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
maxExcLoc = -1
maxException = None
- matches: List[Tuple[int, ParserElement]] = []
- fatals: List[ParseFatalException] = []
+ matches: list[tuple[int, ParserElement]] = []
+ fatals: list[ParseFatalException] = []
if all(e.callPreparse for e in self.exprs):
loc = self.preParse(instring, loc)
for e in self.exprs:
@@ -4184,7 +4256,7 @@ class Or(ParseExpression):
best_expr = matches[0][1]
return best_expr._parse(instring, loc, do_actions)
- longest = -1, None
+ longest: tuple[int, typing.Optional[ParseResults]] = -1, None
for loc1, expr1 in matches:
if loc1 <= longest[0]:
# already have a longer match than this one will deliver, we are done
@@ -4219,7 +4291,7 @@ class Or(ParseExpression):
# infer from this check that all alternatives failed at the current position
# so emit this collective error message instead of any single error message
if maxExcLoc == loc:
- maxException.msg = self.errmsg
+ maxException.msg = self.errmsg or ""
raise maxException
raise ParseException(instring, loc, "no defined alternatives to match", self)
@@ -4259,7 +4331,7 @@ class Or(ParseExpression):
class MatchFirst(ParseExpression):
- """Requires that at least one :class:`ParseExpression` is found. If
+ """Requires that at least one :class:`ParserElement` is found. If
more than one expression matches, the first one listed is the one that will
match. May be constructed using the ``'|'`` operator.
@@ -4326,7 +4398,7 @@ class MatchFirst(ParseExpression):
# infer from this check that all alternatives failed at the current position
# so emit this collective error message instead of any individual error message
if maxExcLoc == loc:
- maxException.msg = self.errmsg
+ maxException.msg = self.errmsg or ""
raise maxException
raise ParseException(instring, loc, "no defined alternatives to match", self)
@@ -4366,7 +4438,7 @@ class MatchFirst(ParseExpression):
class Each(ParseExpression):
- """Requires all given :class:`ParseExpression` s to be found, but in
+ """Requires all given :class:`ParserElement` s to be found, but in
any order. Expressions may be separated by whitespace.
May be constructed using the ``'&'`` operator.
@@ -4480,11 +4552,11 @@ class Each(ParseExpression):
tmpReqd = self.required[:]
tmpOpt = self.optionals[:]
multis = self.multioptionals[:]
- matchOrder: List[ParserElement] = []
+ matchOrder: list[ParserElement] = []
keepMatching = True
- failed: List[ParserElement] = []
- fatals: List[ParseFatalException] = []
+ failed: list[ParserElement] = []
+ fatals: list[ParseFatalException] = []
while keepMatching:
tmpExprs = tmpReqd + tmpOpt + multis
failed.clear()
@@ -4567,7 +4639,7 @@ class ParseElementEnhance(ParserElement):
self.callPreparse = expr.callPreparse
self.ignoreExprs.extend(expr.ignoreExprs)
- def recurse(self) -> List[ParserElement]:
+ def recurse(self) -> list[ParserElement]:
return [self.expr] if self.expr is not None else []
def parseImpl(self, instring, loc, do_actions=True):
@@ -4579,7 +4651,10 @@ class ParseElementEnhance(ParserElement):
except ParseSyntaxException:
raise
except ParseBaseException as pbe:
- if not isinstance(self, Forward) or self.customName is not None:
+ pbe.pstr = pbe.pstr or instring
+ pbe.loc = pbe.loc or loc
+ pbe.parser_element = pbe.parser_element or self
+ if not isinstance(self, Forward) and self.customName is not None:
if self.errmsg:
pbe.msg = self.errmsg
raise
@@ -4704,7 +4779,7 @@ class IndentedBlock(ParseElementEnhance):
if self._grouped:
wrapper = Group
else:
- wrapper = lambda expr: expr
+ wrapper = lambda expr: expr # type: ignore[misc, assignment]
return (wrapper(block) + Optional(trailing_undent)).parseImpl(
instring, anchor_loc, do_actions
)
@@ -4830,9 +4905,7 @@ class PrecededBy(ParseElementEnhance):
"""
- def __init__(
- self, expr: Union[ParserElement, str], retreat: typing.Optional[int] = None
- ):
+ def __init__(self, expr: Union[ParserElement, str], retreat: int = 0):
super().__init__(expr)
self.expr = self.expr().leave_whitespace()
self.mayReturnEmpty = True
@@ -4859,7 +4932,7 @@ class PrecededBy(ParseElementEnhance):
def parseImpl(self, instring, loc=0, do_actions=True) -> ParseImplReturnType:
if self.exact:
if loc < self.retreat:
- raise ParseException(instring, loc, self.errmsg)
+ raise ParseException(instring, loc, self.errmsg, self)
start = loc - self.retreat
_, ret = self.expr._parse(instring, start)
return loc, ret
@@ -4867,7 +4940,7 @@ class PrecededBy(ParseElementEnhance):
# retreat specified a maximum lookbehind window, iterate
test_expr = self.expr + StringEnd()
instring_slice = instring[max(0, loc - self.retreat) : loc]
- last_expr = ParseException(instring, loc, self.errmsg)
+ last_expr: ParseBaseException = ParseException(instring, loc, self.errmsg, self)
for offset in range(1, min(loc, self.retreat + 1) + 1):
try:
@@ -5236,7 +5309,9 @@ class Opt(ParseElementEnhance):
def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
self_expr = self.expr
try:
- loc, tokens = self_expr._parse(instring, loc, do_actions, callPreParse=False)
+ loc, tokens = self_expr._parse(
+ instring, loc, do_actions, callPreParse=False
+ )
except (ParseException, IndexError):
default_value = self.defaultValue
if default_value is not self.__optionalNotMatched:
@@ -5244,9 +5319,9 @@ class Opt(ParseElementEnhance):
tokens = ParseResults([default_value])
tokens[self_expr.resultsName] = default_value
else:
- tokens = [default_value]
+ tokens = [default_value] # type: ignore[assignment]
else:
- tokens = []
+ tokens = [] # type: ignore[assignment]
return loc, tokens
def _generateDefaultName(self) -> str:
@@ -5446,7 +5521,7 @@ class Forward(ParseElementEnhance):
super().__init__(other, savelist=False) # type: ignore[arg-type]
self.lshift_line = None
- def __lshift__(self, other) -> "Forward":
+ def __lshift__(self, other) -> Forward:
if hasattr(self, "caller_frame"):
del self.caller_frame
if isinstance(other, str_type):
@@ -5468,13 +5543,13 @@ class Forward(ParseElementEnhance):
self.lshift_line = traceback.extract_stack(limit=2)[-2] # type: ignore[assignment]
return self
- def __ilshift__(self, other) -> "Forward":
+ def __ilshift__(self, other) -> Forward:
if not isinstance(other, ParserElement):
return NotImplemented
return self << other
- def __or__(self, other) -> "ParserElement":
+ def __or__(self, other) -> ParserElement:
caller_line = traceback.extract_stack(limit=2)[-2]
if (
__diag__.warn_on_match_first_with_lshift_operator
@@ -5585,9 +5660,9 @@ class Forward(ParseElementEnhance):
# in case the action did backtrack
prev_loc, prev_result = memo[peek_key] = memo[act_key]
del memo[peek_key], memo[act_key]
- return prev_loc, prev_result.copy()
+ return prev_loc, copy.copy(prev_result)
del memo[peek_key]
- return prev_loc, prev_peek.copy()
+ return prev_loc, copy.copy(prev_peek)
# the match did get better: see if we can improve further
if do_actions:
try:
@@ -5675,7 +5750,7 @@ class Forward(ParseElementEnhance):
class TokenConverter(ParseElementEnhance):
"""
- Abstract subclass of :class:`ParseExpression`, for converting parsed results.
+ Abstract subclass of :class:`ParseElementEnhance`, for converting parsed results.
"""
def __init__(self, expr: Union[ParserElement, str], savelist=False):
@@ -5900,13 +5975,13 @@ class Suppress(TokenConverter):
expr = _PendingSkip(NoMatch())
super().__init__(expr)
- def __add__(self, other) -> "ParserElement":
+ def __add__(self, other) -> ParserElement:
if isinstance(self.expr, _PendingSkip):
return Suppress(SkipTo(other)) + other
return super().__add__(other)
- def __sub__(self, other) -> "ParserElement":
+ def __sub__(self, other) -> ParserElement:
if isinstance(self.expr, _PendingSkip):
return Suppress(SkipTo(other)) - other
@@ -6131,7 +6206,7 @@ punc8bit = srange(r"[\0xa1-\0xbf\0xd7\0xf7]")
# build list of built-in expressions, for future reference if a global default value
# gets updated
-_builtin_exprs: List[ParserElement] = [
+_builtin_exprs: list[ParserElement] = [
v for v in vars().values() if isinstance(v, ParserElement)
]
diff --git a/contrib/python/pyparsing/py3/pyparsing/diagram/__init__.py b/contrib/python/pyparsing/py3/pyparsing/diagram/__init__.py
index 3275adafb6..7926f2c355 100644
--- a/contrib/python/pyparsing/py3/pyparsing/diagram/__init__.py
+++ b/contrib/python/pyparsing/py3/pyparsing/diagram/__init__.py
@@ -1,20 +1,20 @@
# mypy: ignore-errors
+from __future__ import annotations
+
import railroad
import pyparsing
+import dataclasses
import typing
from typing import (
- List,
- NamedTuple,
Generic,
TypeVar,
- Dict,
Callable,
- Set,
Iterable,
)
from jinja2 import Template
from io import StringIO
import inspect
+import re
jinja2_template_source = """\
@@ -55,14 +55,23 @@ jinja2_template_source = """\
template = Template(jinja2_template_source)
-# Note: ideally this would be a dataclass, but we're supporting Python 3.5+ so we can't do this yet
-NamedDiagram = NamedTuple(
- "NamedDiagram",
- [("name", str), ("diagram", typing.Optional[railroad.DiagramItem]), ("index", int)],
-)
-"""
-A simple structure for associating a name with a railroad diagram
-"""
+
+def _collapse_verbose_regex(regex_str: str) -> str:
+ collapsed = pyparsing.Regex(r"#.*").suppress().transform_string(regex_str)
+ collapsed = re.sub(r"\s*\n\s*", "", collapsed)
+ return collapsed
+
+
+@dataclasses.dataclass
+class NamedDiagram:
+ """
+ A simple structure for associating a name with a railroad diagram
+ """
+
+ name: str
+ index: int
+ diagram: railroad.DiagramItem = None
+
T = TypeVar("T")
@@ -108,7 +117,7 @@ class EditablePartial(Generic[T]):
self.kwargs = kwargs
@classmethod
- def from_call(cls, func: Callable[..., T], *args, **kwargs) -> "EditablePartial[T]":
+ def from_call(cls, func: Callable[..., T], *args, **kwargs) -> EditablePartial[T]:
"""
If you call this function in the same way that you would call the constructor, it will store the arguments
as you expect. For example EditablePartial.from_call(Fraction, 1, 3)() == Fraction(1, 3)
@@ -135,7 +144,7 @@ class EditablePartial(Generic[T]):
return self.func(*args, **kwargs)
-def railroad_to_html(diagrams: List[NamedDiagram], embed=False, **kwargs) -> str:
+def railroad_to_html(diagrams: list[NamedDiagram], embed=False, **kwargs) -> str:
"""
Given a list of NamedDiagram, produce a single HTML string that visualises those diagrams
:params kwargs: kwargs to be passed in to the template
@@ -158,7 +167,7 @@ def railroad_to_html(diagrams: List[NamedDiagram], embed=False, **kwargs) -> str
return template.render(diagrams=data, embed=embed, **kwargs)
-def resolve_partial(partial: "EditablePartial[T]") -> T:
+def resolve_partial(partial: EditablePartial[T]) -> T:
"""
Recursively resolves a collection of Partials into whatever type they are
"""
@@ -180,7 +189,7 @@ def to_railroad(
vertical: int = 3,
show_results_names: bool = False,
show_groups: bool = False,
-) -> List[NamedDiagram]:
+) -> list[NamedDiagram]:
"""
Convert a pyparsing element tree into a list of diagrams. This is the recommended entrypoint to diagram
creation if you want to access the Railroad tree before it is converted to HTML
@@ -244,40 +253,31 @@ def _should_vertical(
return len(_visible_exprs(exprs)) >= specification
+@dataclasses.dataclass
class ElementState:
"""
State recorded for an individual pyparsing Element
"""
- # Note: this should be a dataclass, but we have to support Python 3.5
- def __init__(
- self,
- element: pyparsing.ParserElement,
- converted: EditablePartial,
- parent: EditablePartial,
- number: int,
- name: str = None,
- parent_index: typing.Optional[int] = None,
- ):
- #: The pyparsing element that this represents
- self.element: pyparsing.ParserElement = element
- #: The name of the element
- self.name: typing.Optional[str] = name
- #: The output Railroad element in an unconverted state
- self.converted: EditablePartial = converted
- #: The parent Railroad element, which we store so that we can extract this if it's duplicated
- self.parent: EditablePartial = parent
- #: The order in which we found this element, used for sorting diagrams if this is extracted into a diagram
- self.number: int = number
- #: The index of this inside its parent
- self.parent_index: typing.Optional[int] = parent_index
- #: If true, we should extract this out into a subdiagram
- self.extract: bool = False
- #: If true, all of this element's children have been filled out
- self.complete: bool = False
+ #: The pyparsing element that this represents
+ element: pyparsing.ParserElement
+ #: The output Railroad element in an unconverted state
+ converted: EditablePartial
+ #: The parent Railroad element, which we store so that we can extract this if it's duplicated
+ parent: EditablePartial
+ #: The order in which we found this element, used for sorting diagrams if this is extracted into a diagram
+ number: int
+ #: The name of the element
+ name: str = None
+ #: The index of this inside its parent
+ parent_index: typing.Optional[int] = None
+ #: If true, we should extract this out into a subdiagram
+ extract: bool = False
+ #: If true, all of this element's children have been filled out
+ complete: bool = False
def mark_for_extraction(
- self, el_id: int, state: "ConverterState", name: str = None, force: bool = False
+ self, el_id: int, state: ConverterState, name: str = None, force: bool = False
):
"""
Called when this instance has been seen twice, and thus should eventually be extracted into a sub-diagram
@@ -313,16 +313,16 @@ class ConverterState:
def __init__(self, diagram_kwargs: typing.Optional[dict] = None):
#: A dictionary mapping ParserElements to state relating to them
- self._element_diagram_states: Dict[int, ElementState] = {}
+ self._element_diagram_states: dict[int, ElementState] = {}
#: A dictionary mapping ParserElement IDs to subdiagrams generated from them
- self.diagrams: Dict[int, EditablePartial[NamedDiagram]] = {}
+ self.diagrams: dict[int, EditablePartial[NamedDiagram]] = {}
#: The index of the next unnamed element
self.unnamed_index: int = 1
#: The index of the next element. This is used for sorting
self.index: int = 0
#: Shared kwargs that are used to customize the construction of diagrams
self.diagram_kwargs: dict = diagram_kwargs or {}
- self.extracted_diagram_names: Set[str] = set()
+ self.extracted_diagram_names: set[str] = set()
def __setitem__(self, key: int, value: ElementState):
self._element_diagram_states[key] = value
@@ -513,7 +513,7 @@ def _to_diagram_element(
# If the element isn't worth extracting, we always treat it as the first time we say it
if _worth_extracting(element):
- if el_id in lookup:
+ if el_id in lookup and lookup[el_id].name is not None:
# If we've seen this element exactly once before, we are only just now finding out that it's a duplicate,
# so we have to extract it into a new diagram.
looked_up = lookup[el_id]
@@ -618,6 +618,11 @@ def _to_diagram_element(
ret = EditablePartial.from_call(railroad.Sequence, items=[])
elif len(exprs) > 0 and not element_results_name:
ret = EditablePartial.from_call(railroad.Group, item="", label=name)
+ elif isinstance(element, pyparsing.Regex):
+ patt = _collapse_verbose_regex(element.pattern)
+ element.pattern = patt
+ element._defaultName = None
+ ret = EditablePartial.from_call(railroad.Terminal, element.defaultName)
elif len(exprs) > 0:
ret = EditablePartial.from_call(railroad.Sequence, items=[])
else:
diff --git a/contrib/python/pyparsing/py3/pyparsing/exceptions.py b/contrib/python/pyparsing/py3/pyparsing/exceptions.py
index 8db34f195a..57a1579d12 100644
--- a/contrib/python/pyparsing/py3/pyparsing/exceptions.py
+++ b/contrib/python/pyparsing/py3/pyparsing/exceptions.py
@@ -1,17 +1,20 @@
# exceptions.py
+from __future__ import annotations
+import copy
import re
import sys
import typing
+from functools import cached_property
+from .unicode import pyparsing_unicode as ppu
from .util import (
+ _collapse_string_to_ranges,
col,
line,
lineno,
- _collapse_string_to_ranges,
replaced_by_pep8,
)
-from .unicode import pyparsing_unicode as ppu
class _ExceptionWordUnicodeSet(
@@ -31,7 +34,7 @@ class ParseBaseException(Exception):
msg: str
pstr: str
parser_element: typing.Any # "ParserElement"
- args: typing.Tuple[str, int, typing.Optional[str]]
+ args: tuple[str, int, typing.Optional[str]]
__slots__ = (
"loc",
@@ -50,18 +53,17 @@ class ParseBaseException(Exception):
msg: typing.Optional[str] = None,
elem=None,
):
- self.loc = loc
if msg is None:
- self.msg = pstr
- self.pstr = ""
- else:
- self.msg = msg
- self.pstr = pstr
+ msg, pstr = pstr, ""
+
+ self.loc = loc
+ self.msg = msg
+ self.pstr = pstr
self.parser_element = elem
self.args = (pstr, loc, msg)
@staticmethod
- def explain_exception(exc, depth=16):
+ def explain_exception(exc: Exception, depth: int = 16) -> str:
"""
Method to take an exception and translate the Python internal traceback into a list
of the pyparsing expressions that caused the exception to be raised.
@@ -82,17 +84,17 @@ class ParseBaseException(Exception):
if depth is None:
depth = sys.getrecursionlimit()
- ret = []
+ ret: list[str] = []
if isinstance(exc, ParseBaseException):
ret.append(exc.line)
ret.append(f"{' ' * (exc.column - 1)}^")
ret.append(f"{type(exc).__name__}: {exc}")
- if depth <= 0:
+ if depth <= 0 or exc.__traceback__ is None:
return "\n".join(ret)
callers = inspect.getinnerframes(exc.__traceback__, context=depth)
- seen = set()
+ seen: set[int] = set()
for ff in callers[-depth:]:
frm = ff[0]
@@ -125,41 +127,58 @@ class ParseBaseException(Exception):
return "\n".join(ret)
@classmethod
- def _from_exception(cls, pe):
+ def _from_exception(cls, pe) -> ParseBaseException:
"""
internal factory method to simplify creating one type of ParseException
from another - avoids having __init__ signature conflicts among subclasses
"""
return cls(pe.pstr, pe.loc, pe.msg, pe.parser_element)
- @property
+ @cached_property
def line(self) -> str:
"""
Return the line of text where the exception occurred.
"""
return line(self.loc, self.pstr)
- @property
+ @cached_property
def lineno(self) -> int:
"""
Return the 1-based line number of text where the exception occurred.
"""
return lineno(self.loc, self.pstr)
- @property
+ @cached_property
def col(self) -> int:
"""
Return the 1-based column on the line of text where the exception occurred.
"""
return col(self.loc, self.pstr)
- @property
+ @cached_property
def column(self) -> int:
"""
Return the 1-based column on the line of text where the exception occurred.
"""
return col(self.loc, self.pstr)
+ @cached_property
+ def found(self) -> str:
+ if not self.pstr:
+ return ""
+
+ if self.loc >= len(self.pstr):
+ return "end of text"
+
+ # pull out next word at error location
+ found_match = _exception_word_extractor.match(self.pstr, self.loc)
+ if found_match is not None:
+ found_text = found_match.group(0)
+ else:
+ found_text = self.pstr[self.loc : self.loc + 1]
+
+ return repr(found_text).replace(r"\\", "\\")
+
# pre-PEP8 compatibility
@property
def parserElement(self):
@@ -169,21 +188,15 @@ class ParseBaseException(Exception):
def parserElement(self, elem):
self.parser_element = elem
+ def copy(self):
+ return copy.copy(self)
+
+ def formatted_message(self) -> str:
+ found_phrase = f", found {self.found}" if self.found else ""
+ return f"{self.msg}{found_phrase} (at char {self.loc}), (line:{self.lineno}, col:{self.column})"
+
def __str__(self) -> str:
- if self.pstr:
- if self.loc >= len(self.pstr):
- foundstr = ", found end of text"
- else:
- # pull out next word at error location
- found_match = _exception_word_extractor.match(self.pstr, self.loc)
- if found_match is not None:
- found = found_match.group(0)
- else:
- found = self.pstr[self.loc : self.loc + 1]
- foundstr = (", found %r" % found).replace(r"\\", "\\")
- else:
- foundstr = ""
- return f"{self.msg}{foundstr} (at char {self.loc}), (line:{self.lineno}, col:{self.column})"
+ return self.formatted_message()
def __repr__(self):
return str(self)
@@ -199,12 +212,10 @@ class ParseBaseException(Exception):
line_str = self.line
line_column = self.column - 1
if markerString:
- line_str = "".join(
- (line_str[:line_column], markerString, line_str[line_column:])
- )
+ line_str = f"{line_str[:line_column]}{markerString}{line_str[line_column:]}"
return line_str.strip()
- def explain(self, depth=16) -> str:
+ def explain(self, depth: int = 16) -> str:
"""
Method to translate the Python internal traceback into a list
of the pyparsing expressions that caused the exception to be raised.
@@ -292,6 +303,8 @@ class RecursiveGrammarException(Exception):
Exception thrown by :class:`ParserElement.validate` if the
grammar could be left-recursive; parser may need to enable
left recursion using :class:`ParserElement.enable_left_recursion<ParserElement.enable_left_recursion>`
+
+ Deprecated: only used by deprecated method ParserElement.validate.
"""
def __init__(self, parseElementList):
diff --git a/contrib/python/pyparsing/py3/pyparsing/helpers.py b/contrib/python/pyparsing/py3/pyparsing/helpers.py
index d5d14a08d6..d2bd05f3d3 100644
--- a/contrib/python/pyparsing/py3/pyparsing/helpers.py
+++ b/contrib/python/pyparsing/py3/pyparsing/helpers.py
@@ -1,5 +1,6 @@
# helpers.py
import html.entities
+import operator
import re
import sys
import typing
@@ -10,6 +11,7 @@ from .util import (
_bslash,
_flatten,
_escape_regex_range_chars,
+ make_compressed_re,
replaced_by_pep8,
)
@@ -203,15 +205,15 @@ def one_of(
)
if caseless:
- isequal = lambda a, b: a.upper() == b.upper()
+ is_equal = lambda a, b: a.upper() == b.upper()
masks = lambda a, b: b.upper().startswith(a.upper())
- parseElementClass = CaselessKeyword if asKeyword else CaselessLiteral
+ parse_element_class = CaselessKeyword if asKeyword else CaselessLiteral
else:
- isequal = lambda a, b: a == b
+ is_equal = operator.eq
masks = lambda a, b: b.startswith(a)
- parseElementClass = Keyword if asKeyword else Literal
+ parse_element_class = Keyword if asKeyword else Literal
- symbols: List[str] = []
+ symbols: list[str]
if isinstance(strs, str_type):
strs = typing.cast(str, strs)
symbols = strs.split()
@@ -224,20 +226,19 @@ def one_of(
# reorder given symbols to take care to avoid masking longer choices with shorter ones
# (but only if the given symbols are not just single characters)
- if any(len(sym) > 1 for sym in symbols):
- i = 0
- while i < len(symbols) - 1:
- cur = symbols[i]
- for j, other in enumerate(symbols[i + 1 :]):
- if isequal(other, cur):
- del symbols[i + j + 1]
- break
- if masks(cur, other):
- del symbols[i + j + 1]
- symbols.insert(i, other)
- break
- else:
- i += 1
+ i = 0
+ while i < len(symbols) - 1:
+ cur = symbols[i]
+ for j, other in enumerate(symbols[i + 1 :]):
+ if is_equal(other, cur):
+ del symbols[i + j + 1]
+ break
+ if len(other) > len(cur) and masks(cur, other):
+ del symbols[i + j + 1]
+ symbols.insert(i, other)
+ break
+ else:
+ i += 1
if useRegex:
re_flags: int = re.IGNORECASE if caseless else 0
@@ -269,7 +270,7 @@ def one_of(
)
# last resort, just use MatchFirst
- return MatchFirst(parseElementClass(sym) for sym in symbols).set_name(
+ return MatchFirst(parse_element_class(sym) for sym in symbols).set_name(
" | ".join(symbols)
)
@@ -602,7 +603,7 @@ def _makeTags(tagStr, xml, suppress_LT=Suppress("<"), suppress_GT=Suppress(">"))
def make_html_tags(
tag_str: Union[str, ParserElement]
-) -> Tuple[ParserElement, ParserElement]:
+) -> tuple[ParserElement, ParserElement]:
"""Helper to construct opening and closing tag expressions for HTML,
given a tag name. Matches tags in either upper or lower case,
attributes with namespaces and with quoted or unquoted values.
@@ -629,7 +630,7 @@ def make_html_tags(
def make_xml_tags(
tag_str: Union[str, ParserElement]
-) -> Tuple[ParserElement, ParserElement]:
+) -> tuple[ParserElement, ParserElement]:
"""Helper to construct opening and closing tag expressions for XML,
given a tag name. Matches tags only in the given upper/lower case.
@@ -645,9 +646,12 @@ any_open_tag, any_close_tag = make_html_tags(
)
_htmlEntityMap = {k.rstrip(";"): v for k, v in html.entities.html5.items()}
-common_html_entity = Regex("&(?P<entity>" + "|".join(_htmlEntityMap) + ");").set_name(
- "common HTML entity"
+_most_common_entities = "nbsp lt gt amp quot apos cent pound euro copy".replace(
+ " ", "|"
)
+common_html_entity = Regex(
+ lambda: f"&(?P<entity>{_most_common_entities}|{make_compressed_re(_htmlEntityMap)});"
+).set_name("common HTML entity")
def replace_html_entity(s, l, t):
@@ -664,16 +668,16 @@ class OpAssoc(Enum):
InfixNotationOperatorArgType = Union[
- ParserElement, str, Tuple[Union[ParserElement, str], Union[ParserElement, str]]
+ ParserElement, str, tuple[Union[ParserElement, str], Union[ParserElement, str]]
]
InfixNotationOperatorSpec = Union[
- Tuple[
+ tuple[
InfixNotationOperatorArgType,
int,
OpAssoc,
typing.Optional[ParseAction],
],
- Tuple[
+ tuple[
InfixNotationOperatorArgType,
int,
OpAssoc,
@@ -683,7 +687,7 @@ InfixNotationOperatorSpec = Union[
def infix_notation(
base_expr: ParserElement,
- op_list: List[InfixNotationOperatorSpec],
+ op_list: list[InfixNotationOperatorSpec],
lpar: Union[str, ParserElement] = Suppress("("),
rpar: Union[str, ParserElement] = Suppress(")"),
) -> ParserElement:
@@ -1032,7 +1036,7 @@ python_style_comment = Regex(r"#.*").set_name("Python style comment")
# build list of built-in expressions, for future reference if a global default value
# gets updated
-_builtin_exprs: List[ParserElement] = [
+_builtin_exprs: list[ParserElement] = [
v for v in vars().values() if isinstance(v, ParserElement)
]
diff --git a/contrib/python/pyparsing/py3/pyparsing/results.py b/contrib/python/pyparsing/py3/pyparsing/results.py
index 3bb7c948e0..245847832a 100644
--- a/contrib/python/pyparsing/py3/pyparsing/results.py
+++ b/contrib/python/pyparsing/py3/pyparsing/results.py
@@ -1,4 +1,7 @@
# results.py
+from __future__ import annotations
+
+import collections
from collections.abc import (
MutableMapping,
Mapping,
@@ -7,21 +10,21 @@ from collections.abc import (
Iterable,
)
import pprint
-from typing import Tuple, Any, Dict, Set, List
+from typing import Any
from .util import replaced_by_pep8
-str_type: Tuple[type, ...] = (str, bytes)
+str_type: tuple[type, ...] = (str, bytes)
_generator_type = type((_ for _ in ()))
class _ParseResultsWithOffset:
- tup: Tuple["ParseResults", int]
+ tup: tuple[ParseResults, int]
__slots__ = ["tup"]
- def __init__(self, p1: "ParseResults", p2: int):
- self.tup: Tuple[ParseResults, int] = (p1, p2)
+ def __init__(self, p1: ParseResults, p2: int):
+ self.tup: tuple[ParseResults, int] = (p1, p2)
def __getitem__(self, i):
return self.tup[i]
@@ -79,14 +82,14 @@ class ParseResults:
- year: '1999'
"""
- _null_values: Tuple[Any, ...] = (None, [], ())
+ _null_values: tuple[Any, ...] = (None, [], ())
_name: str
- _parent: "ParseResults"
- _all_names: Set[str]
+ _parent: ParseResults
+ _all_names: set[str]
_modal: bool
- _toklist: List[Any]
- _tokdict: Dict[str, Any]
+ _toklist: list[Any]
+ _tokdict: dict[str, Any]
__slots__ = (
"_name",
@@ -172,8 +175,8 @@ class ParseResults:
# constructor as small and fast as possible
def __init__(
self, toklist=None, name=None, asList=True, modal=True, isinstance=isinstance
- ):
- self._tokdict: Dict[str, _ParseResultsWithOffset]
+ ) -> None:
+ self._tokdict: dict[str, _ParseResultsWithOffset]
self._modal = modal
if name is None or name == "":
@@ -226,7 +229,7 @@ class ParseResults:
self._toklist[k] = v
sub = v
else:
- self._tokdict[k] = self._tokdict.get(k, list()) + [
+ self._tokdict[k] = self._tokdict.get(k, []) + [
_ParseResultsWithOffset(v, 0)
]
sub = v
@@ -443,12 +446,12 @@ class ParseResults:
raise AttributeError(name)
return ""
- def __add__(self, other: "ParseResults") -> "ParseResults":
+ def __add__(self, other: ParseResults) -> ParseResults:
ret = self.copy()
ret += other
return ret
- def __iadd__(self, other: "ParseResults") -> "ParseResults":
+ def __iadd__(self, other: ParseResults) -> ParseResults:
if not other:
return self
@@ -470,7 +473,7 @@ class ParseResults:
self._all_names |= other._all_names
return self
- def __radd__(self, other) -> "ParseResults":
+ def __radd__(self, other) -> ParseResults:
if isinstance(other, int) and other == 0:
# useful for merging many ParseResults using sum() builtin
return self.copy()
@@ -504,9 +507,10 @@ class ParseResults:
out.append(str(item))
return out
- def as_list(self) -> list:
+ def as_list(self, *, flatten: bool = False) -> list:
"""
Returns the parse results as a nested list of matching tokens, all converted to strings.
+ If flatten is True, all the nesting levels in the returned list are collapsed.
Example::
@@ -519,10 +523,22 @@ class ParseResults:
result_list = result.as_list()
print(type(result_list), result_list) # -> <class 'list'> ['sldkj', 'lsdkj', 'sldkj']
"""
- return [
- res.as_list() if isinstance(res, ParseResults) else res
- for res in self._toklist
- ]
+ def flattened(pr):
+ to_visit = collections.deque([*self])
+ while to_visit:
+ to_do = to_visit.popleft()
+ if isinstance(to_do, ParseResults):
+ to_visit.extendleft(to_do[::-1])
+ else:
+ yield to_do
+
+ if flatten:
+ return [*flattened(self)]
+ else:
+ return [
+ res.as_list() if isinstance(res, ParseResults) else res
+ for res in self._toklist
+ ]
def as_dict(self) -> dict:
"""
@@ -553,7 +569,7 @@ class ParseResults:
return dict((k, to_item(v)) for k, v in self.items())
- def copy(self) -> "ParseResults":
+ def copy(self) -> ParseResults:
"""
Returns a new shallow copy of a :class:`ParseResults` object. `ParseResults`
items contained within the source are shared with the copy. Use
@@ -567,7 +583,7 @@ class ParseResults:
ret._name = self._name
return ret
- def deepcopy(self) -> "ParseResults":
+ def deepcopy(self) -> ParseResults:
"""
Returns a new deep copy of a :class:`ParseResults` object.
"""
@@ -584,11 +600,11 @@ class ParseResults:
dest[k] = v.deepcopy() if isinstance(v, ParseResults) else v
elif isinstance(obj, Iterable):
ret._toklist[i] = type(obj)(
- v.deepcopy() if isinstance(v, ParseResults) else v for v in obj
+ v.deepcopy() if isinstance(v, ParseResults) else v for v in obj # type: ignore[call-arg]
)
return ret
- def get_name(self) -> str:
+ def get_name(self) -> str | None:
r"""
Returns the results name for this token expression. Useful when several
different expressions might match at a particular location.
@@ -616,7 +632,7 @@ class ParseResults:
if self._name:
return self._name
elif self._parent:
- par: "ParseResults" = self._parent
+ par: ParseResults = self._parent
parent_tokdict_items = par._tokdict.items()
return next(
(
@@ -761,7 +777,7 @@ class ParseResults:
return dir(type(self)) + list(self.keys())
@classmethod
- def from_dict(cls, other, name=None) -> "ParseResults":
+ def from_dict(cls, other, name=None) -> ParseResults:
"""
Helper classmethod to construct a ``ParseResults`` from a ``dict``, preserving the
name-value relations as results names. If an optional ``name`` argument is
diff --git a/contrib/python/pyparsing/py3/pyparsing/testing.py b/contrib/python/pyparsing/py3/pyparsing/testing.py
index 5654d47d62..836b2f86fb 100644
--- a/contrib/python/pyparsing/py3/pyparsing/testing.py
+++ b/contrib/python/pyparsing/py3/pyparsing/testing.py
@@ -257,10 +257,14 @@ class pyparsing_test:
eol_mark: str = "|",
mark_spaces: typing.Optional[str] = None,
mark_control: typing.Optional[str] = None,
+ *,
+ indent: typing.Union[str, int] = "",
+ base_1: bool = True,
) -> str:
"""
Helpful method for debugging a parser - prints a string with line and column numbers.
- (Line and column numbers are 1-based.)
+ (Line and column numbers are 1-based by default - if debugging a parse action,
+ pass base_1=False, to correspond to the loc value passed to the parse action.)
:param s: tuple(bool, str - string to be printed with line and column numbers
:param start_line: int - (optional) starting line number in s to print (default=1)
@@ -273,11 +277,18 @@ class pyparsing_test:
- "unicode" - replaces control chars with Unicode symbols, such as "␍" and "␊"
- any single character string - replace control characters with given string
- None (default) - string is displayed as-is
+ :param indent: str | int - (optional) string to indent with line and column numbers; if an int
+ is passed, converted to " " * indent
+ :param base_1: bool - (optional) whether to label string using base 1; if False, string will be
+ labeled based at 0 (default=True)
:return: str - input string with leading line numbers and column number headers
"""
if expand_tabs:
s = s.expandtabs()
+ if isinstance(indent, int):
+ indent = " " * indent
+ indent = indent.expandtabs()
if mark_control is not None:
mark_control = typing.cast(str, mark_control)
if mark_control == "unicode":
@@ -300,46 +311,52 @@ class pyparsing_test:
else:
s = s.replace(" ", mark_spaces)
if start_line is None:
- start_line = 1
+ start_line = 0
if end_line is None:
end_line = len(s)
end_line = min(end_line, len(s))
- start_line = min(max(1, start_line), end_line)
+ start_line = min(max(0, start_line), end_line)
if mark_control != "unicode":
- s_lines = s.splitlines()[start_line - 1 : end_line]
+ s_lines = s.splitlines()[start_line - base_1 : end_line]
else:
- s_lines = [line + "␊" for line in s.split("␊")[start_line - 1 : end_line]]
+ s_lines = [
+ line + "␊" for line in s.split("␊")[start_line - base_1 : end_line]
+ ]
if not s_lines:
return ""
lineno_width = len(str(end_line))
max_line_len = max(len(line) for line in s_lines)
- lead = " " * (lineno_width + 1)
+ lead = indent + " " * (lineno_width + 1)
if max_line_len >= 99:
header0 = (
lead
+ + ("" if base_1 else " ")
+ "".join(
f"{' ' * 99}{(i + 1) % 100}"
- for i in range(max(max_line_len // 100, 1))
+ for i in range(1 if base_1 else 0, max(max_line_len // 100, 1))
)
+ "\n"
)
else:
header0 = ""
header1 = (
- header0
+ ("" if base_1 else " ")
+ lead
+ "".join(f" {(i + 1) % 10}" for i in range(-(-max_line_len // 10)))
+ "\n"
)
- header2 = lead + "1234567890" * (-(-max_line_len // 10)) + "\n"
+ digits = "1234567890"
+ header2 = (
+ lead + ("" if base_1 else "0") + digits * (-(-max_line_len // 10)) + "\n"
+ )
return (
header1
+ header2
+ "\n".join(
- f"{i:{lineno_width}d}:{line}{eol_mark}"
- for i, line in enumerate(s_lines, start=start_line)
+ f"{indent}{i:{lineno_width}d}:{line}{eol_mark}"
+ for i, line in enumerate(s_lines, start=start_line + base_1)
)
+ "\n"
)
diff --git a/contrib/python/pyparsing/py3/pyparsing/unicode.py b/contrib/python/pyparsing/py3/pyparsing/unicode.py
index 0e3e06572b..066486c28e 100644
--- a/contrib/python/pyparsing/py3/pyparsing/unicode.py
+++ b/contrib/python/pyparsing/py3/pyparsing/unicode.py
@@ -2,7 +2,7 @@
import sys
from itertools import filterfalse
-from typing import List, Tuple, Union
+from typing import Union
class _lazyclassproperty:
@@ -25,7 +25,7 @@ class _lazyclassproperty:
return cls._intern[attrname]
-UnicodeRangeList = List[Union[Tuple[int, int], Tuple[int]]]
+UnicodeRangeList = list[Union[tuple[int, int], tuple[int]]]
class unicode_set:
@@ -53,9 +53,9 @@ class unicode_set:
_ranges: UnicodeRangeList = []
@_lazyclassproperty
- def _chars_for_ranges(cls) -> List[str]:
- ret: List[int] = []
- for cc in cls.__mro__:
+ def _chars_for_ranges(cls) -> list[str]:
+ ret: list[int] = []
+ for cc in cls.__mro__: # type: ignore[attr-defined]
if cc is unicode_set:
break
for rr in getattr(cc, "_ranges", ()):
diff --git a/contrib/python/pyparsing/py3/pyparsing/util.py b/contrib/python/pyparsing/py3/pyparsing/util.py
index 94837fea0f..1487019c27 100644
--- a/contrib/python/pyparsing/py3/pyparsing/util.py
+++ b/contrib/python/pyparsing/py3/pyparsing/util.py
@@ -1,11 +1,11 @@
# util.py
+import contextlib
+from functools import lru_cache, wraps
import inspect
-import warnings
-import types
-import collections
import itertools
-from functools import lru_cache, wraps
-from typing import Callable, List, Union, Iterable, TypeVar, cast
+import types
+from typing import Callable, Union, Iterable, TypeVar, cast
+import warnings
_bslash = chr(92)
C = TypeVar("C", bound=Callable)
@@ -14,8 +14,8 @@ C = TypeVar("C", bound=Callable)
class __config_flags:
"""Internal class for defining compatibility and debugging flags"""
- _all_names: List[str] = []
- _fixed_names: List[str] = []
+ _all_names: list[str] = []
+ _fixed_names: list[str] = []
_type_desc = "configuration"
@classmethod
@@ -100,27 +100,24 @@ class _UnboundedCache:
class _FifoCache:
def __init__(self, size):
- self.not_in_cache = not_in_cache = object()
cache = {}
- keyring = [object()] * size
+ self.size = size
+ self.not_in_cache = not_in_cache = object()
cache_get = cache.get
cache_pop = cache.pop
- keyiter = itertools.cycle(range(size))
def get(_, key):
return cache_get(key, not_in_cache)
def set_(_, key, value):
cache[key] = value
- i = next(keyiter)
- cache_pop(keyring[i], None)
- keyring[i] = key
+ while len(cache) > size:
+ # pop oldest element in cache by getting the first key
+ cache_pop(next(iter(cache)))
def clear(_):
cache.clear()
- keyring[:] = [object()] * size
- self.size = size
self.get = types.MethodType(get, self)
self.set = types.MethodType(set_, self)
self.clear = types.MethodType(clear, self)
@@ -137,13 +134,13 @@ class LRUMemo:
def __init__(self, capacity):
self._capacity = capacity
self._active = {}
- self._memory = collections.OrderedDict()
+ self._memory = {}
def __getitem__(self, key):
try:
return self._active[key]
except KeyError:
- self._memory.move_to_end(key)
+ self._memory[key] = self._memory.pop(key)
return self._memory[key]
def __setitem__(self, key, value):
@@ -156,8 +153,9 @@ class LRUMemo:
except KeyError:
pass
else:
- while len(self._memory) >= self._capacity:
- self._memory.popitem(last=False)
+ oldest_keys = list(self._memory)[: -(self._capacity + 1)]
+ for key_to_delete in oldest_keys:
+ self._memory.pop(key_to_delete)
self._memory[key] = value
def clear(self):
@@ -183,60 +181,182 @@ def _escape_regex_range_chars(s: str) -> str:
return str(s)
+class _GroupConsecutive:
+ """
+ Used as a callable `key` for itertools.groupby to group
+ characters that are consecutive:
+ itertools.groupby("abcdejkmpqrs", key=IsConsecutive())
+ yields:
+ (0, iter(['a', 'b', 'c', 'd', 'e']))
+ (1, iter(['j', 'k']))
+ (2, iter(['m']))
+ (3, iter(['p', 'q', 'r', 's']))
+ """
+ def __init__(self):
+ self.prev = 0
+ self.counter = itertools.count()
+ self.value = -1
+
+ def __call__(self, char: str) -> int:
+ c_int = ord(char)
+ self.prev, prev = c_int, self.prev
+ if c_int - prev > 1:
+ self.value = next(self.counter)
+ return self.value
+
+
def _collapse_string_to_ranges(
s: Union[str, Iterable[str]], re_escape: bool = True
) -> str:
- def is_consecutive(c):
- c_int = ord(c)
- is_consecutive.prev, prev = c_int, is_consecutive.prev
- if c_int - prev > 1:
- is_consecutive.value = next(is_consecutive.counter)
- return is_consecutive.value
+ r"""
+ Take a string or list of single-character strings, and return
+ a string of the consecutive characters in that string collapsed
+ into groups, as might be used in a regular expression '[a-z]'
+ character set:
+ 'a' -> 'a' -> '[a]'
+ 'bc' -> 'bc' -> '[bc]'
+ 'defgh' -> 'd-h' -> '[d-h]'
+ 'fdgeh' -> 'd-h' -> '[d-h]'
+ 'jklnpqrtu' -> 'j-lnp-rtu' -> '[j-lnp-rtu]'
+ Duplicates get collapsed out:
+ 'aaa' -> 'a' -> '[a]'
+ 'bcbccb' -> 'bc' -> '[bc]'
+ 'defghhgf' -> 'd-h' -> '[d-h]'
+ 'jklnpqrjjjtu' -> 'j-lnp-rtu' -> '[j-lnp-rtu]'
+ Spaces are preserved:
+ 'ab c' -> ' a-c' -> '[ a-c]'
+ Characters that are significant when defining regex ranges
+ get escaped:
+ 'acde[]-' -> r'\-\[\]ac-e' -> r'[\-\[\]ac-e]'
+ """
- is_consecutive.prev = 0 # type: ignore [attr-defined]
- is_consecutive.counter = itertools.count() # type: ignore [attr-defined]
- is_consecutive.value = -1 # type: ignore [attr-defined]
+ # Developer notes:
+ # - Do not optimize this code assuming that the given input string
+ # or internal lists will be short (such as in loading generators into
+ # lists to make it easier to find the last element); this method is also
+ # used to generate regex ranges for character sets in the pyparsing.unicode
+ # classes, and these can be _very_ long lists of strings
- def escape_re_range_char(c):
+ def escape_re_range_char(c: str) -> str:
return "\\" + c if c in r"\^-][" else c
- def no_escape_re_range_char(c):
+ def no_escape_re_range_char(c: str) -> str:
return c
if not re_escape:
escape_re_range_char = no_escape_re_range_char
ret = []
- s = "".join(sorted(set(s)))
- if len(s) > 3:
- for _, chars in itertools.groupby(s, key=is_consecutive):
+
+ # reduce input string to remove duplicates, and put in sorted order
+ s_chars: list[str] = sorted(set(s))
+
+ if len(s_chars) > 2:
+ # find groups of characters that are consecutive (can be collapsed
+ # down to "<first>-<last>")
+ for _, chars in itertools.groupby(s_chars, key=_GroupConsecutive()):
+ # _ is unimportant, is just used to identify groups
+ # chars is an iterator of one or more consecutive characters
+ # that comprise the current group
first = last = next(chars)
- last = collections.deque(
- itertools.chain(iter([last]), chars), maxlen=1
- ).pop()
+ with contextlib.suppress(ValueError):
+ *_, last = chars
+
if first == last:
+ # there was only a single char in this group
ret.append(escape_re_range_char(first))
+
+ elif last == chr(ord(first) + 1):
+ # there were only 2 characters in this group
+ # 'a','b' -> 'ab'
+ ret.append(f"{escape_re_range_char(first)}{escape_re_range_char(last)}")
+
else:
- sep = "" if ord(last) == ord(first) + 1 else "-"
+ # there were > 2 characters in this group, make into a range
+ # 'c','d','e' -> 'c-e'
ret.append(
- f"{escape_re_range_char(first)}{sep}{escape_re_range_char(last)}"
+ f"{escape_re_range_char(first)}-{escape_re_range_char(last)}"
)
else:
- ret = [escape_re_range_char(c) for c in s]
+ # only 1 or 2 chars were given to form into groups
+ # 'a' -> ['a']
+ # 'bc' -> ['b', 'c']
+ # 'dg' -> ['d', 'g']
+ # no need to list them with "-", just return as a list
+ # (after escaping)
+ ret = [escape_re_range_char(c) for c in s_chars]
return "".join(ret)
-def _flatten(ll: list) -> list:
+def _flatten(ll: Iterable) -> list:
ret = []
- for i in ll:
- if isinstance(i, list):
- ret.extend(_flatten(i))
+ to_visit = [*ll]
+ while to_visit:
+ i = to_visit.pop(0)
+ if isinstance(i, Iterable) and not isinstance(i, str):
+ to_visit[:0] = i
else:
ret.append(i)
return ret
+def make_compressed_re(
+ word_list: Iterable[str], max_level: int = 2, _level: int = 1
+) -> str:
+ """
+ Create a regular expression string from a list of words, collapsing by common
+ prefixes and optional suffixes.
+
+ Calls itself recursively to build nested sublists for each group of suffixes
+ that have a shared prefix.
+ """
+
+ def get_suffixes_from_common_prefixes(namelist: list[str]):
+ if len(namelist) > 1:
+ for prefix, suffixes in itertools.groupby(namelist, key=lambda s: s[:1]):
+ yield prefix, sorted([s[1:] for s in suffixes], key=len, reverse=True)
+ else:
+ yield namelist[0][0], [namelist[0][1:]]
+
+ if max_level == 0:
+ return "|".join(sorted(word_list, key=len, reverse=True))
+
+ ret = []
+ sep = ""
+ for initial, suffixes in get_suffixes_from_common_prefixes(sorted(word_list)):
+ ret.append(sep)
+ sep = "|"
+
+ trailing = ""
+ if "" in suffixes:
+ trailing = "?"
+ suffixes.remove("")
+
+ if len(suffixes) > 1:
+ if all(len(s) == 1 for s in suffixes):
+ ret.append(f"{initial}[{''.join(suffixes)}]{trailing}")
+ else:
+ if _level < max_level:
+ suffix_re = make_compressed_re(
+ sorted(suffixes), max_level, _level + 1
+ )
+ ret.append(f"{initial}({suffix_re}){trailing}")
+ else:
+ suffixes.sort(key=len, reverse=True)
+ ret.append(f"{initial}({'|'.join(suffixes)}){trailing}")
+ else:
+ if suffixes:
+ suffix = suffixes[0]
+ if len(suffix) > 1 and trailing:
+ ret.append(f"{initial}({suffix}){trailing}")
+ else:
+ ret.append(f"{initial}{suffix}{trailing}")
+ else:
+ ret.append(initial)
+ return "".join(ret)
+
+
def replaced_by_pep8(compat_name: str, fn: C) -> C:
# In a future version, uncomment the code in the internal _inner() functions
# to begin emitting DeprecationWarnings.
@@ -268,10 +388,10 @@ def replaced_by_pep8(compat_name: str, fn: C) -> C:
_inner.__name__ = compat_name
_inner.__annotations__ = fn.__annotations__
if isinstance(fn, types.FunctionType):
- _inner.__kwdefaults__ = fn.__kwdefaults__
+ _inner.__kwdefaults__ = fn.__kwdefaults__ # type: ignore [attr-defined]
elif isinstance(fn, type) and hasattr(fn, "__init__"):
- _inner.__kwdefaults__ = fn.__init__.__kwdefaults__
+ _inner.__kwdefaults__ = fn.__init__.__kwdefaults__ # type: ignore [misc,attr-defined]
else:
- _inner.__kwdefaults__ = None
+ _inner.__kwdefaults__ = None # type: ignore [attr-defined]
_inner.__qualname__ = fn.__qualname__
return cast(C, _inner)
diff --git a/contrib/python/pyparsing/py3/ya.make b/contrib/python/pyparsing/py3/ya.make
index c5575db221..1b36194fab 100644
--- a/contrib/python/pyparsing/py3/ya.make
+++ b/contrib/python/pyparsing/py3/ya.make
@@ -4,7 +4,7 @@ PY3_LIBRARY()
PROVIDES(pyparsing)
-VERSION(3.1.4)
+VERSION(3.2.0)
LICENSE(MIT)
diff --git a/contrib/python/xmltodict/py3/.dist-info/METADATA b/contrib/python/xmltodict/py3/.dist-info/METADATA
index e2aaa3e797..5a038d733d 100644
--- a/contrib/python/xmltodict/py3/.dist-info/METADATA
+++ b/contrib/python/xmltodict/py3/.dist-info/METADATA
@@ -1,6 +1,6 @@
Metadata-Version: 2.1
Name: xmltodict
-Version: 0.13.0
+Version: 0.14.2
Summary: Makes working with XML feel like you are working with JSON
Home-page: https://github.com/martinblech/xmltodict
Author: Martin Blech
@@ -12,16 +12,17 @@ Classifier: License :: OSI Approved :: MIT License
Classifier: Operating System :: OS Independent
Classifier: Programming Language :: Python
Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.4
-Classifier: Programming Language :: Python :: 3.5
Classifier: Programming Language :: Python :: 3.6
Classifier: Programming Language :: Python :: 3.7
Classifier: Programming Language :: Python :: 3.8
Classifier: Programming Language :: Python :: 3.9
Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
Classifier: Programming Language :: Python :: Implementation :: PyPy
Classifier: Topic :: Text Processing :: Markup :: XML
-Requires-Python: >=3.4
+Requires-Python: >=3.6
Description-Content-Type: text/markdown
License-File: LICENSE
@@ -29,7 +30,7 @@ License-File: LICENSE
`xmltodict` is a Python module that makes working with XML feel like you are working with [JSON](http://docs.python.org/library/json.html), as in this ["spec"](http://www.xml.com/pub/a/2006/05/31/converting-between-xml-and-json.html):
-[![Build Status](https://travis-ci.com/martinblech/xmltodict.svg?branch=master)](https://travis-ci.com/martinblech/xmltodict)
+[![Build Status](https://app.travis-ci.com/martinblech/xmltodict.svg?branch=master)](https://app.travis-ci.com/martinblech/xmltodict)
```python
>>> print(json.dumps(xmltodict.parse("""
@@ -231,6 +232,17 @@ You just need to
$ pip install xmltodict
```
+### Using conda
+
+For installing `xmltodict` using Anaconda/Miniconda (*conda*) from the
+[conda-forge channel][#xmltodict-conda] all you need to do is:
+
+[#xmltodict-conda]: https://anaconda.org/conda-forge/xmltodict
+
+```sh
+$ conda install -c conda-forge xmltodict
+```
+
### RPM-based distro (Fedora, RHEL, …)
There is an [official Fedora package for xmltodict](https://apps.fedoraproject.org/packages/python-xmltodict).
@@ -274,5 +286,3 @@ $ zypper in python2-xmltodict
# Python3
$ zypper in python3-xmltodict
```
-
-
diff --git a/contrib/python/xmltodict/py3/README.md b/contrib/python/xmltodict/py3/README.md
index ab63401a80..6f776a8b4d 100644
--- a/contrib/python/xmltodict/py3/README.md
+++ b/contrib/python/xmltodict/py3/README.md
@@ -2,7 +2,7 @@
`xmltodict` is a Python module that makes working with XML feel like you are working with [JSON](http://docs.python.org/library/json.html), as in this ["spec"](http://www.xml.com/pub/a/2006/05/31/converting-between-xml-and-json.html):
-[![Build Status](https://travis-ci.com/martinblech/xmltodict.svg?branch=master)](https://travis-ci.com/martinblech/xmltodict)
+[![Build Status](https://app.travis-ci.com/martinblech/xmltodict.svg?branch=master)](https://app.travis-ci.com/martinblech/xmltodict)
```python
>>> print(json.dumps(xmltodict.parse("""
@@ -204,6 +204,17 @@ You just need to
$ pip install xmltodict
```
+### Using conda
+
+For installing `xmltodict` using Anaconda/Miniconda (*conda*) from the
+[conda-forge channel][#xmltodict-conda] all you need to do is:
+
+[#xmltodict-conda]: https://anaconda.org/conda-forge/xmltodict
+
+```sh
+$ conda install -c conda-forge xmltodict
+```
+
### RPM-based distro (Fedora, RHEL, …)
There is an [official Fedora package for xmltodict](https://apps.fedoraproject.org/packages/python-xmltodict).
diff --git a/contrib/python/xmltodict/py3/tests/__init__.py b/contrib/python/xmltodict/py3/tests/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/contrib/python/xmltodict/py3/tests/__init__.py
diff --git a/contrib/python/xmltodict/py3/tests/test_dicttoxml.py b/contrib/python/xmltodict/py3/tests/test_dicttoxml.py
index 7fc21718ae..470aca98a1 100644
--- a/contrib/python/xmltodict/py3/tests/test_dicttoxml.py
+++ b/contrib/python/xmltodict/py3/tests/test_dicttoxml.py
@@ -144,6 +144,26 @@ class DictToXMLTestCase(unittest.TestCase):
self.assertEqual(xml, unparse(obj, pretty=True,
newl=newl, indent=indent))
+ def test_pretty_print_with_int_indent(self):
+ obj = {'a': OrderedDict((
+ ('b', [{'c': [1, 2]}, 3]),
+ ('x', 'y'),
+ ))}
+ newl = '\n'
+ indent = 2
+ xml = dedent('''\
+ <?xml version="1.0" encoding="utf-8"?>
+ <a>
+ <b>
+ <c>1</c>
+ <c>2</c>
+ </b>
+ <b>3</b>
+ <x>y</x>
+ </a>''')
+ self.assertEqual(xml, unparse(obj, pretty=True,
+ newl=newl, indent=indent))
+
def test_encoding(self):
try:
value = unichr(39321)
@@ -171,8 +191,6 @@ class DictToXMLTestCase(unittest.TestCase):
self.assertEqual('<a attr="1"></a>', _strip(unparse(obj)))
def test_short_empty_elements(self):
- if sys.version_info[0] < 3:
- return
obj = {'a': None}
self.assertEqual('<a/>', _strip(unparse(obj, short_empty_elements=True)))
diff --git a/contrib/python/xmltodict/py3/tests/test_xmltodict.py b/contrib/python/xmltodict/py3/tests/test_xmltodict.py
index 04137f9e03..7dd22b53f3 100644
--- a/contrib/python/xmltodict/py3/tests/test_xmltodict.py
+++ b/contrib/python/xmltodict/py3/tests/test_xmltodict.py
@@ -168,14 +168,14 @@ class XMLToDictTestCase(unittest.TestCase):
except NameError:
value = chr(39321)
self.assertEqual({'a': value},
- parse('<a>%s</a>' % value))
+ parse(f'<a>{value}</a>'))
def test_encoded_string(self):
try:
value = unichr(39321)
except NameError:
value = chr(39321)
- xml = '<a>%s</a>' % value
+ xml = f'<a>{value}</a>'
self.assertEqual(parse(xml),
parse(xml.encode('utf-8')))
@@ -457,3 +457,21 @@ class XMLToDictTestCase(unittest.TestCase):
}
}
self.assertEqual(parse(xml, process_comments=True), expectedResult)
+
+ def test_streaming_attrs(self):
+ xml = """
+ <a>
+ <b attr1="value">
+ <c>cdata</c>
+ </b>
+ </a>
+ """
+ def handler(path, item):
+ expected = {
+ '@attr1': 'value',
+ 'c': 'cdata'
+ }
+ self.assertEqual(expected, item)
+ return True
+
+ parse(xml, item_depth=2, item_callback=handler)
diff --git a/contrib/python/xmltodict/py3/xmltodict.py b/contrib/python/xmltodict/py3/xmltodict.py
index ca760aa637..098f62762a 100644
--- a/contrib/python/xmltodict/py3/xmltodict.py
+++ b/contrib/python/xmltodict/py3/xmltodict.py
@@ -1,19 +1,10 @@
#!/usr/bin/env python
"Makes working with XML feel like you are working with JSON"
-try:
- from defusedexpat import pyexpat as expat
-except ImportError:
- from xml.parsers import expat
+from xml.parsers import expat
from xml.sax.saxutils import XMLGenerator
from xml.sax.xmlreader import AttributesImpl
-try: # pragma no cover
- from cStringIO import StringIO
-except ImportError: # pragma no cover
- try:
- from StringIO import StringIO
- except ImportError:
- from io import StringIO
+from io import StringIO
_dict = dict
import platform
@@ -22,17 +13,8 @@ if tuple(map(int, platform.python_version_tuple()[:2])) < (3, 7):
from inspect import isgenerator
-try: # pragma no cover
- _basestring = basestring
-except NameError: # pragma no cover
- _basestring = str
-try: # pragma no cover
- _unicode = unicode
-except NameError: # pragma no cover
- _unicode = str
-
__author__ = 'Martin Blech'
-__version__ = '0.13.0'
+__version__ = "0.14.2"
__license__ = 'MIT'
@@ -40,7 +22,7 @@ class ParsingInterrupted(Exception):
pass
-class _DictSAXHandler(object):
+class _DictSAXHandler:
def __init__(self,
item_depth=0,
item_callback=lambda *args: True,
@@ -107,7 +89,7 @@ class _DictSAXHandler(object):
attrs['xmlns'] = self.namespace_declarations
self.namespace_declarations = self.dict_constructor()
self.path.append((name, attrs or None))
- if len(self.path) > self.item_depth:
+ if len(self.path) >= self.item_depth:
self.stack.append((self.item, self.data))
if self.xml_attribs:
attr_entries = []
@@ -135,7 +117,7 @@ class _DictSAXHandler(object):
should_continue = self.item_callback(self.path, item)
if not should_continue:
- raise ParsingInterrupted()
+ raise ParsingInterrupted
if self.stack:
data = (None if not self.data
else self.cdata_separator.join(self.data))
@@ -335,9 +317,8 @@ def parse(xml_input, encoding=None, expat=expat, process_namespaces=False,
"""
handler = _DictSAXHandler(namespace_separator=namespace_separator,
**kwargs)
- if isinstance(xml_input, _unicode):
- if not encoding:
- encoding = 'utf-8'
+ if isinstance(xml_input, str):
+ encoding = encoding or 'utf-8'
xml_input = xml_input.encode(encoding)
if not process_namespaces:
namespace_separator = None
@@ -372,8 +353,8 @@ def parse(xml_input, encoding=None, expat=expat, process_namespaces=False,
parser.ParseFile(xml_input)
elif isgenerator(xml_input):
for chunk in xml_input:
- parser.Parse(chunk,False)
- parser.Parse(b'',True)
+ parser.Parse(chunk, False)
+ parser.Parse(b'', True)
else:
parser.Parse(xml_input, True)
return handler.item
@@ -412,9 +393,7 @@ def _emit(key, value, content_handler,
if result is None:
return
key, value = result
- if (not hasattr(value, '__iter__')
- or isinstance(value, _basestring)
- or isinstance(value, dict)):
+ if not hasattr(value, '__iter__') or isinstance(value, (str, dict)):
value = [value]
for index, v in enumerate(value):
if full_document and depth == 0 and index > 0:
@@ -422,16 +401,13 @@ def _emit(key, value, content_handler,
if v is None:
v = _dict()
elif isinstance(v, bool):
- if v:
- v = _unicode('true')
- else:
- v = _unicode('false')
- elif not isinstance(v, dict):
- if expand_iter and hasattr(v, '__iter__') and not isinstance(v, _basestring):
+ v = 'true' if v else 'false'
+ elif not isinstance(v, (dict, str)):
+ if expand_iter and hasattr(v, '__iter__'):
v = _dict(((expand_iter, v),))
else:
- v = _unicode(v)
- if isinstance(v, _basestring):
+ v = str(v)
+ if isinstance(v, str):
v = _dict(((cdata_key, v),))
cdata = None
attrs = _dict()
@@ -445,14 +421,16 @@ def _emit(key, value, content_handler,
attr_prefix)
if ik == '@xmlns' and isinstance(iv, dict):
for k, v in iv.items():
- attr = 'xmlns{}'.format(':{}'.format(k) if k else '')
- attrs[attr] = _unicode(v)
+ attr = 'xmlns{}'.format(f':{k}' if k else '')
+ attrs[attr] = str(v)
continue
- if not isinstance(iv, _unicode):
- iv = _unicode(iv)
+ if not isinstance(iv, str):
+ iv = str(iv)
attrs[ik[len(attr_prefix):]] = iv
continue
children.append((ik, iv))
+ if isinstance(indent, int):
+ indent = ' ' * indent
if pretty:
content_handler.ignorableWhitespace(depth * indent)
content_handler.startElement(key, AttributesImpl(attrs))
diff --git a/contrib/python/xmltodict/py3/ya.make b/contrib/python/xmltodict/py3/ya.make
index 6872d9affa..7fdc1a14a9 100644
--- a/contrib/python/xmltodict/py3/ya.make
+++ b/contrib/python/xmltodict/py3/ya.make
@@ -2,7 +2,7 @@
PY3_LIBRARY()
-VERSION(0.13.0)
+VERSION(0.14.2)
LICENSE(MIT)
diff --git a/contrib/tools/protoc/resources.json b/contrib/tools/protoc/resources.json
index 2e78d6bf7f..bc895b2859 100644
--- a/contrib/tools/protoc/resources.json
+++ b/contrib/tools/protoc/resources.json
@@ -1,19 +1,19 @@
{
"by_platform": {
"darwin": {
- "uri": "sbr:7039760192"
+ "uri": "sbr:7360495216"
},
"darwin-arm64": {
- "uri": "sbr:7039759830"
+ "uri": "sbr:7360493349"
},
"linux": {
- "uri": "sbr:7039760920"
+ "uri": "sbr:7360499157"
},
"linux-aarch64": {
- "uri": "sbr:7039760572"
+ "uri": "sbr:7360497335"
},
"win32": {
- "uri": "sbr:7039759509"
+ "uri": "sbr:7360491807"
}
}
}
diff --git a/library/cpp/blockcodecs/core/codecs.h b/library/cpp/blockcodecs/core/codecs.h
index 9c93c00274..6512abddef 100644
--- a/library/cpp/blockcodecs/core/codecs.h
+++ b/library/cpp/blockcodecs/core/codecs.h
@@ -25,6 +25,12 @@ namespace NBlockCodecs {
: TStringBuf((const char*)t.Data(), t.Size())
{
}
+
+ template <>
+ inline TData(const TString& t)
+ : TStringBuf((const char*)t.data(), t.size())
+ {
+ }
};
struct TCodecError: public yexception {
diff --git a/library/cpp/tld/tlds-alpha-by-domain.txt b/library/cpp/tld/tlds-alpha-by-domain.txt
index fbf58d0ddb..4964f4e61d 100644
--- a/library/cpp/tld/tlds-alpha-by-domain.txt
+++ b/library/cpp/tld/tlds-alpha-by-domain.txt
@@ -1,4 +1,4 @@
-# Version 2024102500, Last Updated Fri Oct 25 07:07:02 2024 UTC
+# Version 2024102800, Last Updated Mon Oct 28 07:07:02 2024 UTC
AAA
AARP
ABB
diff --git a/library/cpp/yt/logging/backends/stream/stream_log_manager.cpp b/library/cpp/yt/logging/backends/stream/stream_log_manager.cpp
index 62fa3e91d0..8b46d5679f 100644
--- a/library/cpp/yt/logging/backends/stream/stream_log_manager.cpp
+++ b/library/cpp/yt/logging/backends/stream/stream_log_manager.cpp
@@ -21,12 +21,10 @@ public:
{ }
void RegisterStaticAnchor(
- TLoggingAnchor* anchor,
+ TLoggingAnchor* /*anchor*/,
::TSourceLocation /*sourceLocation*/,
TStringBuf /*anchorMessage*/) override
- {
- anchor->Registered = true;
- }
+ { }
virtual void UpdateAnchor(TLoggingAnchor* /*anchor*/) override
{ }
diff --git a/library/cpp/yt/logging/logger.cpp b/library/cpp/yt/logging/logger.cpp
index 58add38429..a6a0010b3b 100644
--- a/library/cpp/yt/logging/logger.cpp
+++ b/library/cpp/yt/logging/logger.cpp
@@ -195,14 +195,21 @@ bool TLogger::IsEssential() const
return Essential_;
}
-void TLogger::UpdateAnchor(TLoggingAnchor* anchor) const
+void TLogger::UpdateStaticAnchor(
+ TLoggingAnchor* anchor,
+ std::atomic<bool>* anchorRegistered,
+ ::TSourceLocation sourceLocation,
+ TStringBuf message) const
{
+ if (!anchorRegistered->exchange(true)) {
+ LogManager_->RegisterStaticAnchor(anchor, sourceLocation, message);
+ }
LogManager_->UpdateAnchor(anchor);
}
-void TLogger::RegisterStaticAnchor(TLoggingAnchor* anchor, ::TSourceLocation sourceLocation, TStringBuf message) const
+void TLogger::UpdateDynamicAnchor(TLoggingAnchor* anchor) const
{
- LogManager_->RegisterStaticAnchor(anchor, sourceLocation, message);
+ LogManager_->UpdateAnchor(anchor);
}
void TLogger::Write(TLogEvent&& event) const
diff --git a/library/cpp/yt/logging/logger.h b/library/cpp/yt/logging/logger.h
index 35aba4eb4c..eff5da078e 100644
--- a/library/cpp/yt/logging/logger.h
+++ b/library/cpp/yt/logging/logger.h
@@ -206,8 +206,12 @@ public:
bool IsEssential() const;
bool IsAnchorUpToDate(const TLoggingAnchor& anchor) const;
- void UpdateAnchor(TLoggingAnchor* anchor) const;
- void RegisterStaticAnchor(TLoggingAnchor* anchor, ::TSourceLocation sourceLocation, TStringBuf message) const;
+ void UpdateStaticAnchor(
+ TLoggingAnchor* anchor,
+ std::atomic<bool>* anchorRegistered,
+ ::TSourceLocation sourceLocation,
+ TStringBuf message) const;
+ void UpdateDynamicAnchor(TLoggingAnchor* anchor) const;
void Write(TLogEvent&& event) const;
@@ -302,19 +306,12 @@ void LogStructuredEvent(
#define YT_LOG_FATAL_UNLESS(condition, ...) if (!Y_LIKELY(condition)) YT_LOG_FATAL(__VA_ARGS__)
#define YT_LOG_EVENT(logger, level, ...) \
- YT_LOG_EVENT_WITH_ANCHOR(logger, level, nullptr, __VA_ARGS__)
-
-#define YT_LOG_EVENT_WITH_ANCHOR(logger, level, anchor, ...) \
do { \
const auto& logger__ = (logger)(); \
auto level__ = (level); \
auto location__ = __LOCATION__; \
- \
- ::NYT::NLogging::TLoggingAnchor* anchor__ = (anchor); \
- [[unlikely]] if (!anchor__) { \
- static ::NYT::TLeakyStorage<::NYT::NLogging::TLoggingAnchor> staticAnchor__; \
- anchor__ = staticAnchor__.Get(); \
- } \
+ static ::NYT::TLeakyStorage<::NYT::NLogging::TLoggingAnchor> anchorStorage__; \
+ auto* anchor__ = anchorStorage__.Get(); \
\
bool anchorUpToDate__ = logger__.IsAnchorUpToDate(*anchor__); \
[[likely]] if (anchorUpToDate__) { \
@@ -328,7 +325,8 @@ void LogStructuredEvent(
auto message__ = ::NYT::NLogging::NDetail::BuildLogMessage(loggingContext__, logger__, __VA_ARGS__); \
\
[[unlikely]] if (!anchorUpToDate__) { \
- logger__.RegisterStaticAnchor(anchor__, location__, message__.Anchor); \
+ static std::atomic<bool> anchorRegistered__; \
+ logger__.UpdateStaticAnchor(anchor__, &anchorRegistered__, location__, message__.Anchor); \
} \
\
auto effectiveLevel__ = ::NYT::NLogging::TLogger::GetEffectiveLoggingLevel(level__, *anchor__); \
@@ -345,6 +343,35 @@ void LogStructuredEvent(
std::move(message__.MessageRef)); \
} while (false)
+#define YT_LOG_EVENT_WITH_DYNAMIC_ANCHOR(logger, level, anchor, ...) \
+ do { \
+ const auto& logger__ = (logger)(); \
+ auto level__ = (level); \
+ auto location__ = __LOCATION__; \
+ auto* anchor__ = (anchor); \
+ \
+ bool anchorUpToDate__ = logger__.IsAnchorUpToDate(*anchor__); \
+ [[unlikely]] if (!anchorUpToDate__) { \
+ logger__.UpdateDynamicAnchor(anchor__); \
+ } \
+ \
+ auto effectiveLevel__ = ::NYT::NLogging::TLogger::GetEffectiveLoggingLevel(level__, *anchor__); \
+ if (!logger__.IsLevelEnabled(effectiveLevel__)) { \
+ break; \
+ } \
+ \
+ auto loggingContext__ = ::NYT::NLogging::GetLoggingContext(); \
+ auto message__ = ::NYT::NLogging::NDetail::BuildLogMessage(loggingContext__, logger__, __VA_ARGS__); \
+ \
+ ::NYT::NLogging::NDetail::LogEventImpl( \
+ loggingContext__, \
+ logger__, \
+ effectiveLevel__, \
+ location__, \
+ anchor__, \
+ std::move(message__.MessageRef)); \
+ } while (false)
+
////////////////////////////////////////////////////////////////////////////////
} // namespace NYT::NLogging
diff --git a/library/cpp/yt/yson_string/convert.cpp b/library/cpp/yt/yson_string/convert.cpp
index 6d78ea6c9d..68241adb78 100644
--- a/library/cpp/yt/yson_string/convert.cpp
+++ b/library/cpp/yt/yson_string/convert.cpp
@@ -67,6 +67,12 @@ TYsonString ConvertToYsonString<TString>(const TString& value)
return ConvertToYsonString(static_cast<TStringBuf>(value));
}
+template <>
+TYsonString ConvertToYsonString<std::string>(const std::string& value)
+{
+ return ConvertToYsonString(static_cast<TStringBuf>(value));
+}
+
struct TConvertStringToYsonStringTag
{ };
diff --git a/library/cpp/yt/yson_string/convert.h b/library/cpp/yt/yson_string/convert.h
index 06de28d2f9..eedb0939e0 100644
--- a/library/cpp/yt/yson_string/convert.h
+++ b/library/cpp/yt/yson_string/convert.h
@@ -44,6 +44,8 @@ TYsonString ConvertToYsonString<ui64>(const ui64& value);
template <>
TYsonString ConvertToYsonString<TString>(const TString& value);
template <>
+TYsonString ConvertToYsonString<std::string>(const std::string& value);
+template <>
TYsonString ConvertToYsonString<TStringBuf>(const TStringBuf& value);
TYsonString ConvertToYsonString(const char* value);
diff --git a/vendor/github.com/golang/protobuf/internal/gengogrpc/ya.make b/vendor/github.com/golang/protobuf/internal/gengogrpc/ya.make
index 3c94f9e302..942e632e4a 100644
--- a/vendor/github.com/golang/protobuf/internal/gengogrpc/ya.make
+++ b/vendor/github.com/golang/protobuf/internal/gengogrpc/ya.make
@@ -2,6 +2,8 @@ GO_LIBRARY()
LICENSE(BSD-3-Clause)
+VERSION(v1.5.4)
+
SRCS(
grpc.go
)
diff --git a/vendor/github.com/golang/protobuf/protoc-gen-go/ya.make b/vendor/github.com/golang/protobuf/protoc-gen-go/ya.make
index 71c8487166..db464d8212 100644
--- a/vendor/github.com/golang/protobuf/protoc-gen-go/ya.make
+++ b/vendor/github.com/golang/protobuf/protoc-gen-go/ya.make
@@ -2,6 +2,8 @@ GO_PROGRAM()
LICENSE(BSD-3-Clause)
+VERSION(v1.5.4)
+
SRCS(
main.go
)
diff --git a/vendor/github.com/spf13/cobra/ya.make b/vendor/github.com/spf13/cobra/ya.make
index 4242c95a67..2d15e11c9b 100644
--- a/vendor/github.com/spf13/cobra/ya.make
+++ b/vendor/github.com/spf13/cobra/ya.make
@@ -2,6 +2,8 @@ GO_LIBRARY()
LICENSE(Apache-2.0)
+VERSION(v1.8.1)
+
SRCS(
active_help.go
args.go
diff --git a/vendor/github.com/spf13/pflag/ya.make b/vendor/github.com/spf13/pflag/ya.make
index 095b7a2b45..c40c23cce9 100644
--- a/vendor/github.com/spf13/pflag/ya.make
+++ b/vendor/github.com/spf13/pflag/ya.make
@@ -2,6 +2,8 @@ GO_LIBRARY()
LICENSE(BSD-3-Clause)
+VERSION(v1.0.6-0.20201009195203-85dd5c8bc61c)
+
SRCS(
bool.go
bool_slice.go
diff --git a/vendor/google.golang.org/protobuf/cmd/protoc-gen-go/internal_gengo/ya.make b/vendor/google.golang.org/protobuf/cmd/protoc-gen-go/internal_gengo/ya.make
index 114c371ff4..765e4b291d 100644
--- a/vendor/google.golang.org/protobuf/cmd/protoc-gen-go/internal_gengo/ya.make
+++ b/vendor/google.golang.org/protobuf/cmd/protoc-gen-go/internal_gengo/ya.make
@@ -2,6 +2,8 @@ GO_LIBRARY()
LICENSE(BSD-3-Clause)
+VERSION(v1.34.2)
+
SRCS(
init.go
main.go
diff --git a/vendor/google.golang.org/protobuf/compiler/protogen/ya.make b/vendor/google.golang.org/protobuf/compiler/protogen/ya.make
index 65652dc84a..f2e0e6e3e0 100644
--- a/vendor/google.golang.org/protobuf/compiler/protogen/ya.make
+++ b/vendor/google.golang.org/protobuf/compiler/protogen/ya.make
@@ -2,6 +2,8 @@ GO_LIBRARY()
LICENSE(BSD-3-Clause)
+VERSION(v1.34.2)
+
SRCS(
protogen.go
)
diff --git a/vendor/google.golang.org/protobuf/encoding/prototext/ya.make b/vendor/google.golang.org/protobuf/encoding/prototext/ya.make
index 14ea8e575d..3d35bb8109 100644
--- a/vendor/google.golang.org/protobuf/encoding/prototext/ya.make
+++ b/vendor/google.golang.org/protobuf/encoding/prototext/ya.make
@@ -2,6 +2,8 @@ GO_LIBRARY()
LICENSE(BSD-3-Clause)
+VERSION(v1.34.2)
+
SRCS(
decode.go
doc.go
diff --git a/vendor/google.golang.org/protobuf/encoding/protowire/ya.make b/vendor/google.golang.org/protobuf/encoding/protowire/ya.make
index 74a3dc5f92..e944a7face 100644
--- a/vendor/google.golang.org/protobuf/encoding/protowire/ya.make
+++ b/vendor/google.golang.org/protobuf/encoding/protowire/ya.make
@@ -2,6 +2,8 @@ GO_LIBRARY()
LICENSE(BSD-3-Clause)
+VERSION(v1.34.2)
+
SRCS(
wire.go
)
diff --git a/vendor/google.golang.org/protobuf/internal/descfmt/ya.make b/vendor/google.golang.org/protobuf/internal/descfmt/ya.make
index 6f0464b58b..d1c40a75f7 100644
--- a/vendor/google.golang.org/protobuf/internal/descfmt/ya.make
+++ b/vendor/google.golang.org/protobuf/internal/descfmt/ya.make
@@ -2,6 +2,8 @@ GO_LIBRARY()
LICENSE(BSD-3-Clause)
+VERSION(v1.34.2)
+
SRCS(
stringer.go
)
diff --git a/vendor/google.golang.org/protobuf/internal/descopts/ya.make b/vendor/google.golang.org/protobuf/internal/descopts/ya.make
index 26ef6cc24b..b63e12cf89 100644
--- a/vendor/google.golang.org/protobuf/internal/descopts/ya.make
+++ b/vendor/google.golang.org/protobuf/internal/descopts/ya.make
@@ -2,6 +2,8 @@ GO_LIBRARY()
LICENSE(BSD-3-Clause)
+VERSION(v1.34.2)
+
SRCS(
options.go
)
diff --git a/vendor/google.golang.org/protobuf/internal/detrand/ya.make b/vendor/google.golang.org/protobuf/internal/detrand/ya.make
index dd1b8456b9..a0d7ea6c2c 100644
--- a/vendor/google.golang.org/protobuf/internal/detrand/ya.make
+++ b/vendor/google.golang.org/protobuf/internal/detrand/ya.make
@@ -2,6 +2,8 @@ GO_LIBRARY()
LICENSE(BSD-3-Clause)
+VERSION(v1.34.2)
+
SRCS(
rand.go
)
diff --git a/vendor/google.golang.org/protobuf/internal/editiondefaults/ya.make b/vendor/google.golang.org/protobuf/internal/editiondefaults/ya.make
index eff20cca8a..e78ea669be 100644
--- a/vendor/google.golang.org/protobuf/internal/editiondefaults/ya.make
+++ b/vendor/google.golang.org/protobuf/internal/editiondefaults/ya.make
@@ -2,6 +2,8 @@ GO_LIBRARY()
LICENSE(BSD-3-Clause)
+VERSION(v1.34.2)
+
SRCS(
defaults.go
)
diff --git a/vendor/google.golang.org/protobuf/internal/editionssupport/ya.make b/vendor/google.golang.org/protobuf/internal/editionssupport/ya.make
index fde817ec3b..45b82c7de2 100644
--- a/vendor/google.golang.org/protobuf/internal/editionssupport/ya.make
+++ b/vendor/google.golang.org/protobuf/internal/editionssupport/ya.make
@@ -2,6 +2,8 @@ GO_LIBRARY()
LICENSE(BSD-3-Clause)
+VERSION(v1.34.2)
+
SRCS(
editions.go
)
diff --git a/vendor/google.golang.org/protobuf/internal/encoding/defval/ya.make b/vendor/google.golang.org/protobuf/internal/encoding/defval/ya.make
index 9f5b5a3fb7..6ba0c708dc 100644
--- a/vendor/google.golang.org/protobuf/internal/encoding/defval/ya.make
+++ b/vendor/google.golang.org/protobuf/internal/encoding/defval/ya.make
@@ -2,6 +2,8 @@ GO_LIBRARY()
LICENSE(BSD-3-Clause)
+VERSION(v1.34.2)
+
SRCS(
default.go
)
diff --git a/vendor/google.golang.org/protobuf/internal/encoding/messageset/ya.make b/vendor/google.golang.org/protobuf/internal/encoding/messageset/ya.make
index 3681a9a476..946f28d167 100644
--- a/vendor/google.golang.org/protobuf/internal/encoding/messageset/ya.make
+++ b/vendor/google.golang.org/protobuf/internal/encoding/messageset/ya.make
@@ -2,6 +2,8 @@ GO_LIBRARY()
LICENSE(BSD-3-Clause)
+VERSION(v1.34.2)
+
SRCS(
messageset.go
)
diff --git a/vendor/google.golang.org/protobuf/internal/encoding/tag/ya.make b/vendor/google.golang.org/protobuf/internal/encoding/tag/ya.make
index 12d85eb139..3b51433f66 100644
--- a/vendor/google.golang.org/protobuf/internal/encoding/tag/ya.make
+++ b/vendor/google.golang.org/protobuf/internal/encoding/tag/ya.make
@@ -2,6 +2,8 @@ GO_LIBRARY()
LICENSE(BSD-3-Clause)
+VERSION(v1.34.2)
+
SRCS(
tag.go
)
diff --git a/vendor/google.golang.org/protobuf/internal/encoding/text/ya.make b/vendor/google.golang.org/protobuf/internal/encoding/text/ya.make
index 5665f82701..cab9b833b2 100644
--- a/vendor/google.golang.org/protobuf/internal/encoding/text/ya.make
+++ b/vendor/google.golang.org/protobuf/internal/encoding/text/ya.make
@@ -2,6 +2,8 @@ GO_LIBRARY()
LICENSE(BSD-3-Clause)
+VERSION(v1.34.2)
+
SRCS(
decode.go
decode_number.go
diff --git a/vendor/google.golang.org/protobuf/internal/errors/ya.make b/vendor/google.golang.org/protobuf/internal/errors/ya.make
index d958a0d019..446386647a 100644
--- a/vendor/google.golang.org/protobuf/internal/errors/ya.make
+++ b/vendor/google.golang.org/protobuf/internal/errors/ya.make
@@ -2,6 +2,8 @@ GO_LIBRARY()
LICENSE(BSD-3-Clause)
+VERSION(v1.34.2)
+
SRCS(
errors.go
is_go113.go
diff --git a/vendor/google.golang.org/protobuf/internal/filedesc/ya.make b/vendor/google.golang.org/protobuf/internal/filedesc/ya.make
index df85be08f6..11ee917a02 100644
--- a/vendor/google.golang.org/protobuf/internal/filedesc/ya.make
+++ b/vendor/google.golang.org/protobuf/internal/filedesc/ya.make
@@ -2,6 +2,8 @@ GO_LIBRARY()
LICENSE(BSD-3-Clause)
+VERSION(v1.34.2)
+
SRCS(
build.go
desc.go
diff --git a/vendor/google.golang.org/protobuf/internal/filetype/ya.make b/vendor/google.golang.org/protobuf/internal/filetype/ya.make
index e0fc61c11b..9dce0cf0d9 100644
--- a/vendor/google.golang.org/protobuf/internal/filetype/ya.make
+++ b/vendor/google.golang.org/protobuf/internal/filetype/ya.make
@@ -2,6 +2,8 @@ GO_LIBRARY()
LICENSE(BSD-3-Clause)
+VERSION(v1.34.2)
+
SRCS(
build.go
)
diff --git a/vendor/google.golang.org/protobuf/internal/flags/ya.make b/vendor/google.golang.org/protobuf/internal/flags/ya.make
index 0116cd8953..f75da040cc 100644
--- a/vendor/google.golang.org/protobuf/internal/flags/ya.make
+++ b/vendor/google.golang.org/protobuf/internal/flags/ya.make
@@ -2,6 +2,8 @@ GO_LIBRARY()
LICENSE(BSD-3-Clause)
+VERSION(v1.34.2)
+
SRCS(
flags.go
proto_legacy_disable.go
diff --git a/vendor/google.golang.org/protobuf/internal/genid/ya.make b/vendor/google.golang.org/protobuf/internal/genid/ya.make
index e257c029a9..3df4ae5b38 100644
--- a/vendor/google.golang.org/protobuf/internal/genid/ya.make
+++ b/vendor/google.golang.org/protobuf/internal/genid/ya.make
@@ -2,6 +2,8 @@ GO_LIBRARY()
LICENSE(BSD-3-Clause)
+VERSION(v1.34.2)
+
SRCS(
any_gen.go
api_gen.go
diff --git a/vendor/google.golang.org/protobuf/internal/impl/ya.make b/vendor/google.golang.org/protobuf/internal/impl/ya.make
index 3d5e1a1077..6f477685fc 100644
--- a/vendor/google.golang.org/protobuf/internal/impl/ya.make
+++ b/vendor/google.golang.org/protobuf/internal/impl/ya.make
@@ -2,6 +2,8 @@ GO_LIBRARY()
LICENSE(BSD-3-Clause)
+VERSION(v1.34.2)
+
SRCS(
api_export.go
checkinit.go
diff --git a/vendor/google.golang.org/protobuf/internal/msgfmt/ya.make b/vendor/google.golang.org/protobuf/internal/msgfmt/ya.make
index 5ed4b140dc..cd16c939f5 100644
--- a/vendor/google.golang.org/protobuf/internal/msgfmt/ya.make
+++ b/vendor/google.golang.org/protobuf/internal/msgfmt/ya.make
@@ -2,6 +2,8 @@ GO_LIBRARY()
LICENSE(BSD-3-Clause)
+VERSION(v1.34.2)
+
SRCS(
format.go
)
diff --git a/vendor/google.golang.org/protobuf/internal/order/ya.make b/vendor/google.golang.org/protobuf/internal/order/ya.make
index fbc58d1f9b..ed17c14646 100644
--- a/vendor/google.golang.org/protobuf/internal/order/ya.make
+++ b/vendor/google.golang.org/protobuf/internal/order/ya.make
@@ -2,6 +2,8 @@ GO_LIBRARY()
LICENSE(BSD-3-Clause)
+VERSION(v1.34.2)
+
SRCS(
order.go
range.go
diff --git a/vendor/google.golang.org/protobuf/internal/pragma/ya.make b/vendor/google.golang.org/protobuf/internal/pragma/ya.make
index 521f22ace6..499f8bb3ed 100644
--- a/vendor/google.golang.org/protobuf/internal/pragma/ya.make
+++ b/vendor/google.golang.org/protobuf/internal/pragma/ya.make
@@ -2,6 +2,8 @@ GO_LIBRARY()
LICENSE(BSD-3-Clause)
+VERSION(v1.34.2)
+
SRCS(
pragma.go
)
diff --git a/vendor/google.golang.org/protobuf/internal/set/ya.make b/vendor/google.golang.org/protobuf/internal/set/ya.make
index ef768eff13..62b43b9e30 100644
--- a/vendor/google.golang.org/protobuf/internal/set/ya.make
+++ b/vendor/google.golang.org/protobuf/internal/set/ya.make
@@ -2,6 +2,8 @@ GO_LIBRARY()
LICENSE(BSD-3-Clause)
+VERSION(v1.34.2)
+
SRCS(
ints.go
)
diff --git a/vendor/google.golang.org/protobuf/internal/strs/ya.make b/vendor/google.golang.org/protobuf/internal/strs/ya.make
index 3e580e1999..5156dc71bd 100644
--- a/vendor/google.golang.org/protobuf/internal/strs/ya.make
+++ b/vendor/google.golang.org/protobuf/internal/strs/ya.make
@@ -2,6 +2,8 @@ GO_LIBRARY()
LICENSE(BSD-3-Clause)
+VERSION(v1.34.2)
+
SRCS(
strings.go
strings_unsafe_go121.go
diff --git a/vendor/google.golang.org/protobuf/internal/version/ya.make b/vendor/google.golang.org/protobuf/internal/version/ya.make
index ca3958ecd9..357c70bad1 100644
--- a/vendor/google.golang.org/protobuf/internal/version/ya.make
+++ b/vendor/google.golang.org/protobuf/internal/version/ya.make
@@ -2,6 +2,8 @@ GO_LIBRARY()
LICENSE(BSD-3-Clause)
+VERSION(v1.34.2)
+
SRCS(
version.go
)
diff --git a/vendor/google.golang.org/protobuf/proto/ya.make b/vendor/google.golang.org/protobuf/proto/ya.make
index 6f72cc0ff1..9df48fe669 100644
--- a/vendor/google.golang.org/protobuf/proto/ya.make
+++ b/vendor/google.golang.org/protobuf/proto/ya.make
@@ -2,6 +2,8 @@ GO_LIBRARY()
LICENSE(BSD-3-Clause)
+VERSION(v1.34.2)
+
SRCS(
checkinit.go
decode.go
diff --git a/vendor/google.golang.org/protobuf/reflect/protodesc/ya.make b/vendor/google.golang.org/protobuf/reflect/protodesc/ya.make
index cb217be223..1ddb6e4377 100644
--- a/vendor/google.golang.org/protobuf/reflect/protodesc/ya.make
+++ b/vendor/google.golang.org/protobuf/reflect/protodesc/ya.make
@@ -2,6 +2,8 @@ GO_LIBRARY()
LICENSE(BSD-3-Clause)
+VERSION(v1.34.2)
+
SRCS(
desc.go
desc_init.go
diff --git a/vendor/google.golang.org/protobuf/reflect/protopath/ya.make b/vendor/google.golang.org/protobuf/reflect/protopath/ya.make
index 5dcf56c0c8..859addc349 100644
--- a/vendor/google.golang.org/protobuf/reflect/protopath/ya.make
+++ b/vendor/google.golang.org/protobuf/reflect/protopath/ya.make
@@ -2,6 +2,8 @@ GO_LIBRARY()
LICENSE(BSD-3-Clause)
+VERSION(v1.34.2)
+
SRCS(
path.go
step.go
diff --git a/vendor/google.golang.org/protobuf/reflect/protorange/ya.make b/vendor/google.golang.org/protobuf/reflect/protorange/ya.make
index fc294f2b0e..16bb3bfc5a 100644
--- a/vendor/google.golang.org/protobuf/reflect/protorange/ya.make
+++ b/vendor/google.golang.org/protobuf/reflect/protorange/ya.make
@@ -2,6 +2,8 @@ GO_LIBRARY()
LICENSE(BSD-3-Clause)
+VERSION(v1.34.2)
+
SRCS(
range.go
)
diff --git a/vendor/google.golang.org/protobuf/reflect/protoreflect/ya.make b/vendor/google.golang.org/protobuf/reflect/protoreflect/ya.make
index 7cef01fc14..f4a2f7f4f9 100644
--- a/vendor/google.golang.org/protobuf/reflect/protoreflect/ya.make
+++ b/vendor/google.golang.org/protobuf/reflect/protoreflect/ya.make
@@ -2,6 +2,8 @@ GO_LIBRARY()
LICENSE(BSD-3-Clause)
+VERSION(v1.34.2)
+
SRCS(
methods.go
proto.go
diff --git a/vendor/google.golang.org/protobuf/reflect/protoregistry/ya.make b/vendor/google.golang.org/protobuf/reflect/protoregistry/ya.make
index a6fda0d69d..091dd4ba69 100644
--- a/vendor/google.golang.org/protobuf/reflect/protoregistry/ya.make
+++ b/vendor/google.golang.org/protobuf/reflect/protoregistry/ya.make
@@ -2,6 +2,8 @@ GO_LIBRARY()
LICENSE(BSD-3-Clause)
+VERSION(v1.34.2)
+
SRCS(
registry.go
)
diff --git a/vendor/google.golang.org/protobuf/runtime/protoiface/ya.make b/vendor/google.golang.org/protobuf/runtime/protoiface/ya.make
index 76eaf10dee..5b33e7681c 100644
--- a/vendor/google.golang.org/protobuf/runtime/protoiface/ya.make
+++ b/vendor/google.golang.org/protobuf/runtime/protoiface/ya.make
@@ -2,6 +2,8 @@ GO_LIBRARY()
LICENSE(BSD-3-Clause)
+VERSION(v1.34.2)
+
SRCS(
legacy.go
methods.go
diff --git a/vendor/google.golang.org/protobuf/runtime/protoimpl/ya.make b/vendor/google.golang.org/protobuf/runtime/protoimpl/ya.make
index 8da76e4e71..41016080d4 100644
--- a/vendor/google.golang.org/protobuf/runtime/protoimpl/ya.make
+++ b/vendor/google.golang.org/protobuf/runtime/protoimpl/ya.make
@@ -2,6 +2,8 @@ GO_LIBRARY()
LICENSE(BSD-3-Clause)
+VERSION(v1.34.2)
+
SRCS(
impl.go
version.go
diff --git a/vendor/google.golang.org/protobuf/types/descriptorpb/ya.make b/vendor/google.golang.org/protobuf/types/descriptorpb/ya.make
index a96c8d9d3c..3c3d3a0277 100644
--- a/vendor/google.golang.org/protobuf/types/descriptorpb/ya.make
+++ b/vendor/google.golang.org/protobuf/types/descriptorpb/ya.make
@@ -2,6 +2,8 @@ GO_LIBRARY()
LICENSE(BSD-3-Clause)
+VERSION(v1.34.2)
+
SRCS(
descriptor.pb.go
)
diff --git a/vendor/google.golang.org/protobuf/types/dynamicpb/ya.make b/vendor/google.golang.org/protobuf/types/dynamicpb/ya.make
index ecab55dc87..a11c30ed0e 100644
--- a/vendor/google.golang.org/protobuf/types/dynamicpb/ya.make
+++ b/vendor/google.golang.org/protobuf/types/dynamicpb/ya.make
@@ -2,6 +2,8 @@ GO_LIBRARY()
LICENSE(BSD-3-Clause)
+VERSION(v1.34.2)
+
SRCS(
dynamic.go
types.go
diff --git a/vendor/google.golang.org/protobuf/types/gofeaturespb/ya.make b/vendor/google.golang.org/protobuf/types/gofeaturespb/ya.make
index 065e25268b..b2465a613e 100644
--- a/vendor/google.golang.org/protobuf/types/gofeaturespb/ya.make
+++ b/vendor/google.golang.org/protobuf/types/gofeaturespb/ya.make
@@ -2,6 +2,8 @@ GO_LIBRARY()
LICENSE(BSD-3-Clause)
+VERSION(v1.34.2)
+
SRCS(
go_features.pb.go
)
diff --git a/vendor/google.golang.org/protobuf/types/pluginpb/ya.make b/vendor/google.golang.org/protobuf/types/pluginpb/ya.make
index 7b8a798f38..479992020d 100644
--- a/vendor/google.golang.org/protobuf/types/pluginpb/ya.make
+++ b/vendor/google.golang.org/protobuf/types/pluginpb/ya.make
@@ -2,6 +2,8 @@ GO_LIBRARY()
LICENSE(BSD-3-Clause)
+VERSION(v1.34.2)
+
SRCS(
plugin.pb.go
)
diff --git a/yql/essentials/public/issue/protos/issue_message.proto b/yql/essentials/public/issue/protos/issue_message.proto
new file mode 100644
index 0000000000..d581a5d8f0
--- /dev/null
+++ b/yql/essentials/public/issue/protos/issue_message.proto
@@ -0,0 +1,17 @@
+package NYql.NIssue.NProto;
+option java_package = "com.yandex.yql.issue.proto";
+
+message IssueMessage {
+ message Position {
+ optional uint32 row = 1;
+ optional uint32 column = 2;
+ optional string file = 3;
+ }
+
+ optional Position position = 1;
+ optional string message = 2;
+ optional Position end_position = 3;
+ optional uint32 issue_code = 4;
+ optional uint32 severity = 5;
+ repeated IssueMessage issues = 6;
+}
diff --git a/yql/essentials/public/issue/protos/issue_severity.proto b/yql/essentials/public/issue/protos/issue_severity.proto
new file mode 100644
index 0000000000..70fd61f5d9
--- /dev/null
+++ b/yql/essentials/public/issue/protos/issue_severity.proto
@@ -0,0 +1,14 @@
+syntax = "proto3";
+
+package NYql;
+
+option java_package = "com.yandex.yql.proto";
+
+message TSeverityIds {
+ enum ESeverityId {
+ S_FATAL = 0;
+ S_ERROR = 1;
+ S_WARNING = 2;
+ S_INFO = 3;
+ }
+}
diff --git a/yql/essentials/public/issue/protos/ya.make b/yql/essentials/public/issue/protos/ya.make
new file mode 100644
index 0000000000..2451631ea1
--- /dev/null
+++ b/yql/essentials/public/issue/protos/ya.make
@@ -0,0 +1,10 @@
+PROTO_LIBRARY()
+
+SRCS(
+ issue_message.proto
+ issue_severity.proto
+)
+
+EXCLUDE_TAGS(GO_PROTO)
+
+END()
diff --git a/yql/essentials/public/issue/ut/ya.make b/yql/essentials/public/issue/ut/ya.make
new file mode 100644
index 0000000000..46f35941bc
--- /dev/null
+++ b/yql/essentials/public/issue/ut/ya.make
@@ -0,0 +1,16 @@
+UNITTEST_FOR(yql/essentials/public/issue)
+
+FORK_SUBTESTS()
+
+SRCS(
+ yql_issue_ut.cpp
+ yql_issue_manager_ut.cpp
+ yql_issue_utils_ut.cpp
+ yql_warning_ut.cpp
+)
+
+PEERDIR(
+ library/cpp/unicode/normalization
+)
+
+END()
diff --git a/yql/essentials/public/issue/ya.make b/yql/essentials/public/issue/ya.make
new file mode 100644
index 0000000000..0179b23bbb
--- /dev/null
+++ b/yql/essentials/public/issue/ya.make
@@ -0,0 +1,26 @@
+LIBRARY()
+
+SRCS(
+ yql_issue.cpp
+ yql_issue_message.cpp
+ yql_issue_manager.cpp
+ yql_issue_utils.cpp
+ yql_warning.cpp
+)
+
+PEERDIR(
+ contrib/libs/protobuf
+ library/cpp/colorizer
+ library/cpp/resource
+ contrib/ydb/public/api/protos
+ yql/essentials/public/issue/protos
+ contrib/ydb/library/yql/utils
+)
+
+GENERATE_ENUM_SERIALIZATION(yql_warning.h)
+
+END()
+
+RECURSE_FOR_TESTS(
+ ut
+)
diff --git a/yql/essentials/public/issue/yql_issue.cpp b/yql/essentials/public/issue/yql_issue.cpp
new file mode 100644
index 0000000000..3fc8e3767c
--- /dev/null
+++ b/yql/essentials/public/issue/yql_issue.cpp
@@ -0,0 +1,310 @@
+#include "yql_issue.h"
+#include "yql_issue_id.h"
+
+#include <contrib/ydb/library/yql/utils/utf8.h>
+
+#include <library/cpp/colorizer/output.h>
+
+#include <util/charset/utf8.h>
+#include <util/string/ascii.h>
+#include <util/string/split.h>
+#include <util/string/strip.h>
+#include <util/string/subst.h>
+#include <util/system/compiler.h>
+#include <util/generic/map.h>
+#include <util/generic/stack.h>
+#include <cstdlib>
+
+
+namespace NYql {
+
+void SanitizeNonAscii(TString& s) {
+ if (!NYql::IsUtf8(s)) {
+ TString escaped;
+ escaped.reserve(s.size());
+ const unsigned char* i = reinterpret_cast<const unsigned char*>(s.data());
+ const unsigned char* end = i + s.size();
+ while (i < end) {
+ wchar32 rune;
+ size_t runeLen;
+ const RECODE_RESULT result = SafeReadUTF8Char(rune, runeLen, i, end);
+ if (result == RECODE_OK) {
+ escaped.insert(escaped.end(), reinterpret_cast<const char*>(i), reinterpret_cast<const char*>(i + runeLen));
+ i += runeLen;
+ } else {
+ escaped.push_back('?');
+ ++i;
+ }
+ }
+ s = escaped;
+ }
+}
+
+TTextWalker& TTextWalker::Advance(char c) {
+ if (c == '\n') {
+ HaveCr = false;
+ ++LfCount;
+ return *this;
+ }
+
+
+ if (c == '\r' && !HaveCr) {
+ HaveCr = true;
+ return *this;
+ }
+
+ ui32 charDistance = 1;
+ if (Utf8Aware && IsUtf8Intermediate(c)) {
+ charDistance = 0;
+ }
+
+ // either not '\r' or second '\r'
+ if (LfCount) {
+ Position.Row += LfCount;
+ Position.Column = charDistance;
+ LfCount = 0;
+ } else {
+ Position.Column += charDistance + (HaveCr && c != '\r');
+ }
+ HaveCr = (c == '\r');
+ return *this;
+}
+
+void TIssue::PrintTo(IOutputStream& out, bool oneLine) const {
+ out << Range() << ": " << SeverityToString(GetSeverity()) << ": ";
+ if (oneLine) {
+ TString message = StripString(Message);
+ SubstGlobal(message, '\n', ' ');
+ out << message;
+ } else {
+ out << Message;
+ }
+ if (GetCode()) {
+ out << ", code: " << GetCode();
+ }
+}
+
+void WalkThroughIssues(const TIssue& topIssue, bool leafOnly, std::function<void(const TIssue&, ui16 level)> fn, std::function<void(const TIssue&, ui16 level)> afterChildrenFn) {
+ enum class EFnType {
+ Main,
+ AfterChildren,
+ };
+
+ const bool hasAfterChildrenFn = bool(afterChildrenFn);
+ TStack<std::tuple<ui16, const TIssue*, EFnType>> issuesStack;
+ if (hasAfterChildrenFn) {
+ issuesStack.push(std::make_tuple(0, &topIssue, EFnType::AfterChildren));
+ }
+ issuesStack.push(std::make_tuple(0, &topIssue, EFnType::Main));
+ while (!issuesStack.empty()) {
+ auto level = std::get<0>(issuesStack.top());
+ const auto& curIssue = *std::get<1>(issuesStack.top());
+ const EFnType fnType = std::get<2>(issuesStack.top());
+ issuesStack.pop();
+ if (!leafOnly || curIssue.GetSubIssues().empty()) {
+ if (fnType == EFnType::Main) {
+ fn(curIssue, level);
+ } else {
+ afterChildrenFn(curIssue, level);
+ }
+ }
+ if (fnType == EFnType::Main) {
+ level++;
+ const auto& subIssues = curIssue.GetSubIssues();
+ for (int i = subIssues.size() - 1; i >= 0; i--) {
+ if (hasAfterChildrenFn) {
+ issuesStack.push(std::make_tuple(level, subIssues[i].Get(), EFnType::AfterChildren));
+ }
+ issuesStack.push(std::make_tuple(level, subIssues[i].Get(), EFnType::Main));
+ }
+ }
+ }
+}
+
+namespace {
+
+Y_NO_INLINE void Indent(IOutputStream& out, ui32 indentation) {
+ char* whitespaces = reinterpret_cast<char*>(alloca(indentation));
+ memset(whitespaces, ' ', indentation);
+ out.Write(whitespaces, indentation);
+}
+
+void ProgramLinesWithErrors(
+ const TString& programText,
+ const TVector<TIssue>& errors,
+ TMap<ui32, TStringBuf>& lines)
+{
+ TVector<ui32> rows;
+ for (const auto& topIssue: errors) {
+ WalkThroughIssues(topIssue, false, [&](const TIssue& issue, ui16 /*level*/) {
+ for (ui32 row = issue.Position.Row; row <= issue.EndPosition.Row; row++) {
+ rows.push_back(row);
+ }
+ });
+ }
+ std::sort(rows.begin(), rows.end());
+
+ auto prog = StringSplitter(programText).Split('\n');
+ auto progIt = prog.begin(), progEnd = prog.end();
+ ui32 progRow = 1;
+
+ for (ui32 row: rows) {
+ while (progRow < row && progIt != progEnd) {
+ ++progRow;
+ ++progIt;
+ }
+ if (progIt != progEnd) {
+ lines[row] = progIt->Token();
+ }
+ }
+}
+
+} // namspace
+
+void TIssues::PrintTo(IOutputStream& out, bool oneLine) const
+{
+ if (oneLine) {
+ bool printWithSpace = false;
+ if (Issues_.size() > 1) {
+ printWithSpace = true;
+ out << "[";
+ }
+ for (const auto& topIssue: Issues_) {
+ WalkThroughIssues(topIssue, false, [&](const TIssue& issue, ui16 level) {
+ if (level > 0) {
+ out << " subissue: { ";
+ } else {
+ out << (printWithSpace ? " { " : "{ ");
+ }
+ issue.PrintTo(out, true);
+ },
+ [&](const TIssue&, ui16) {
+ out << " }";
+ });
+ }
+ if (Issues_.size() > 1) {
+ out << " ]";
+ }
+ } else {
+ for (const auto& topIssue: Issues_) {
+ WalkThroughIssues(topIssue, false, [&](const TIssue& issue, ui16 level) {
+ auto shift = level * 4;
+ Indent(out, shift);
+ out << issue << Endl;
+ });
+ }
+ }
+}
+
+void TIssues::PrintWithProgramTo(
+ IOutputStream& out,
+ const TString& programFilename,
+ const TString& programText) const
+{
+ using namespace NColorizer;
+
+ TMap<ui32, TStringBuf> lines;
+ ProgramLinesWithErrors(programText, Issues_, lines);
+
+ for (const TIssue& topIssue: Issues_) {
+ WalkThroughIssues(topIssue, false, [&](const TIssue& issue, ui16 level) {
+ auto shift = level * 4;
+ Indent(out, shift);
+ out << DarkGray() << programFilename << Old() << ':';
+ out << Purple() << issue.Range() << Old();
+ auto color = (issue.GetSeverity() == TSeverityIds::S_WARNING) ? Yellow() : LightRed();
+ auto severityName = SeverityToString(issue.GetSeverity());
+ out << color << ": "<< severityName << ": " << issue.GetMessage() << Old() << '\n';
+ Indent(out, shift);
+ if (issue.Position.HasValue()) {
+ out << '\t' << lines[issue.Position.Row] << '\n';
+ out << '\t';
+ if (issue.Position.Column > 0) {
+ Indent(out, issue.Position.Column - 1);
+ }
+ out << '^';
+ }
+ out << Endl;
+ });
+ }
+}
+
+TIssue ExceptionToIssue(const std::exception& e, const TPosition& pos) {
+ TStringBuf messageBuf = e.what();
+ auto parsedPos = TryParseTerminationMessage(messageBuf);
+ auto issue = TIssue(parsedPos.GetOrElse(pos), messageBuf);
+ const TErrorException* errorException = dynamic_cast<const TErrorException*>(&e);
+ if (errorException) {
+ issue.SetCode(errorException->GetCode(), ESeverity::TSeverityIds_ESeverityId_S_ERROR);
+ } else {
+ issue.SetCode(UNEXPECTED_ERROR, ESeverity::TSeverityIds_ESeverityId_S_FATAL);
+ }
+ return issue;
+}
+
+static constexpr TStringBuf TerminationMessageMarker = "Terminate was called, reason(";
+
+TMaybe<TPosition> TryParseTerminationMessage(TStringBuf& message) {
+ size_t len = 0;
+ size_t startPos = message.find(TerminationMessageMarker);
+ size_t endPos = 0;
+ if (startPos != TString::npos) {
+ endPos = message.find(')', startPos + TerminationMessageMarker.size());
+ if (endPos != TString::npos) {
+ TStringBuf lenText = message.Tail(startPos + TerminationMessageMarker.size())
+ .Trunc(endPos - startPos - TerminationMessageMarker.size());
+ try {
+ len = FromString<size_t>(lenText);
+ } catch (const TFromStringException&) {
+ len = 0;
+ }
+ }
+ }
+
+ if (len) {
+ message = message.Tail(endPos + 3).Trunc(len);
+ auto s = message;
+ TMaybe<TStringBuf> file;
+ TMaybe<TStringBuf> row;
+ TMaybe<TStringBuf> column;
+ GetNext(s, ':', file);
+ GetNext(s, ':', row);
+ GetNext(s, ':', column);
+ ui32 rowValue, columnValue;
+ if (file && row && column && TryFromString(*row, rowValue) && TryFromString(*column, columnValue)) {
+ message = StripStringLeft(s);
+ return TPosition(columnValue, rowValue, TString(*file));
+ }
+ }
+
+ return Nothing();
+}
+
+} // namspace NYql
+
+template <>
+void Out<NYql::TPosition>(IOutputStream& out, const NYql::TPosition& pos) {
+ out << (pos.File ? pos.File : "<main>");
+ if (pos) {
+ out << ":" << pos.Row << ':' << pos.Column;
+ }
+}
+
+template<>
+void Out<NYql::TRange>(IOutputStream & out, const NYql::TRange & range) {
+ if (range.IsRange()) {
+ out << '[' << range.Position << '-' << range.EndPosition << ']';
+ } else {
+ out << range.Position;
+ }
+}
+
+template <>
+void Out<NYql::TIssue>(IOutputStream& out, const NYql::TIssue& error) {
+ error.PrintTo(out);
+}
+
+template <>
+void Out<NYql::TIssues>(IOutputStream& out, const NYql::TIssues& error) {
+ error.PrintTo(out);
+} \ No newline at end of file
diff --git a/yql/essentials/public/issue/yql_issue.h b/yql/essentials/public/issue/yql_issue.h
new file mode 100644
index 0000000000..00c49fb1fc
--- /dev/null
+++ b/yql/essentials/public/issue/yql_issue.h
@@ -0,0 +1,372 @@
+#pragma once
+
+#include <util/system/types.h>
+#include <util/generic/hash.h>
+#include <util/generic/maybe.h>
+#include <util/generic/vector.h>
+#include <util/generic/string.h>
+#include <util/generic/strbuf.h>
+#include <util/generic/ptr.h>
+#include <util/stream/output.h>
+#include <util/stream/str.h>
+#include <util/digest/numeric.h>
+#include <google/protobuf/message.h>
+
+#include "yql_issue_id.h"
+
+namespace NYql {
+
+void SanitizeNonAscii(TString& s);
+
+///////////////////////////////////////////////////////////////////////////////
+// TPosition
+///////////////////////////////////////////////////////////////////////////////
+struct TPosition {
+ ui32 Column = 0U;
+ ui32 Row = 0U;
+ TString File;
+
+ TPosition() = default;
+
+ TPosition(ui32 column, ui32 row, const TString& file = {})
+ : Column(column)
+ , Row(row)
+ , File(file)
+ {
+ SanitizeNonAscii(File);
+ }
+
+ explicit operator bool() const {
+ return HasValue();
+ }
+
+ inline bool HasValue() const {
+ return Row | Column;
+ }
+
+ inline bool operator==(const TPosition& other) const {
+ return Column == other.Column && Row == other.Row && File == other.File;
+ }
+
+ inline bool operator<(const TPosition& other) const {
+ return std::tie(Row, Column, File) < std::tie(other.Row, other.Column, other.File);
+ }
+};
+
+class TTextWalker {
+public:
+ TTextWalker(TPosition& position, bool utf8Aware)
+ : Position(position)
+ , Utf8Aware(utf8Aware)
+ , HaveCr(false)
+ , LfCount(0)
+ {
+ }
+
+ static inline bool IsUtf8Intermediate(char c) {
+ return (c & 0xC0) == 0x80;
+ }
+
+ template<typename T>
+ TTextWalker& Advance(const T& buf) {
+ for (char c : buf) {
+ Advance(c);
+ }
+ return *this;
+ }
+
+ TTextWalker& Advance(char c);
+
+private:
+ TPosition& Position;
+ const bool Utf8Aware;
+ bool HaveCr;
+ ui32 LfCount;
+};
+
+struct TRange {
+ TPosition Position;
+ TPosition EndPosition;
+
+ TRange() = default;
+
+ TRange(TPosition position)
+ : Position(position)
+ , EndPosition(position)
+ {
+ }
+
+ TRange(TPosition position, TPosition endPosition)
+ : Position(position)
+ , EndPosition(endPosition)
+ {
+ }
+
+ inline bool IsRange() const {
+ return !(Position == EndPosition);
+ }
+};
+
+///////////////////////////////////////////////////////////////////////////////
+// TIssue
+///////////////////////////////////////////////////////////////////////////////
+
+class TIssue;
+using TIssuePtr = TIntrusivePtr<TIssue>;
+class TIssue: public TThrRefBase {
+ TVector<TIntrusivePtr<TIssue>> Children_;
+ TString Message;
+public:
+ TPosition Position;
+ TPosition EndPosition;
+ TIssueCode IssueCode = 0U;
+ ESeverity Severity = TSeverityIds::S_ERROR;
+
+ TIssue() = default;
+
+ template <typename T>
+ explicit TIssue(const T& message)
+ : Message(message)
+ , Position(TPosition())
+ , EndPosition(TPosition())
+ {
+ SanitizeNonAscii(Message);
+ }
+
+ template <typename T>
+ TIssue(TPosition position, const T& message)
+ : Message(message)
+ , Position(position)
+ , EndPosition(position)
+ {
+ SanitizeNonAscii(Message);
+ }
+
+ inline TRange Range() const {
+ return{ Position, EndPosition };
+ }
+
+ template <typename T>
+ TIssue(TPosition position, TPosition endPosition, const T& message)
+ : Message(message)
+ , Position(position)
+ , EndPosition(endPosition)
+ {
+ SanitizeNonAscii(Message);
+ }
+
+ inline bool operator==(const TIssue& other) const {
+ return Position == other.Position && Message == other.Message
+ && IssueCode == other.IssueCode;
+ }
+
+ ui64 Hash() const noexcept {
+ return CombineHashes(
+ CombineHashes(
+ (size_t)CombineHashes(IntHash(Position.Row), IntHash(Position.Column)),
+ ComputeHash(Position.File)
+ ),
+ (size_t)CombineHashes((size_t)IntHash(static_cast<int>(IssueCode)), ComputeHash(Message)));
+ }
+
+ TIssue& SetCode(TIssueCode id, ESeverity severity) {
+ IssueCode = id;
+ Severity = severity;
+ return *this;
+ }
+
+ TIssue& SetMessage(const TString& msg) {
+ Message = msg;
+ SanitizeNonAscii(Message);
+ return *this;
+ }
+
+ ESeverity GetSeverity() const {
+ return Severity;
+ }
+
+ TIssueCode GetCode() const {
+ return IssueCode;
+ }
+
+ const TString& GetMessage() const {
+ return Message;
+ }
+
+ TIssue& AddSubIssue(TIntrusivePtr<TIssue> issue) {
+ Severity = (ESeverity)Min((ui32)issue->GetSeverity(), (ui32)Severity);
+ Children_.push_back(issue);
+ return *this;
+ }
+
+ const TVector<TIntrusivePtr<TIssue>>& GetSubIssues() const {
+ return Children_;
+ }
+
+ void PrintTo(IOutputStream& out, bool oneLine = false) const;
+
+ TString ToString(bool oneLine = false) const {
+ TStringStream out;
+ PrintTo(out, oneLine);
+ return out.Str();
+ }
+
+ // Unsafe method. Doesn't call SanitizeNonAscii(Message)
+ TString* MutableMessage() {
+ return &Message;
+ }
+
+ TIssue& CopyWithoutSubIssues(const TIssue& src) {
+ Message = src.Message;
+ IssueCode = src.IssueCode;
+ Severity = src.Severity;
+ Position = src.Position;
+ EndPosition = src.EndPosition;
+ return *this;
+ }
+};
+
+void WalkThroughIssues(const TIssue& topIssue, bool leafOnly, std::function<void(const TIssue&, ui16 level)> fn, std::function<void(const TIssue&, ui16 level)> afterChildrenFn = {});
+
+///////////////////////////////////////////////////////////////////////////////
+// TIssues
+///////////////////////////////////////////////////////////////////////////////
+class TIssues {
+public:
+ TIssues() = default;
+
+ inline TIssues(const TVector<TIssue>& issues)
+ : Issues_(issues)
+ {
+ }
+
+ inline TIssues(const std::initializer_list<TIssue>& issues)
+ : TIssues(TVector<TIssue>(issues))
+ {
+ }
+
+ inline TIssues(const TIssues& rhs)
+ : Issues_(rhs.Issues_)
+ {
+ }
+
+ inline TIssues& operator=(const TIssues& rhs) {
+ Issues_ = rhs.Issues_;
+ return *this;
+ }
+
+ inline TIssues(TIssues&& rhs) : Issues_(std::move(rhs.Issues_))
+ {
+ }
+
+ inline TIssues& operator=(TIssues&& rhs) {
+ Issues_ = std::move(rhs.Issues_);
+ return *this;
+ }
+
+ template <typename ... Args> void AddIssue(Args&& ... args) {
+ Issues_.emplace_back(std::forward<Args>(args)...);
+ }
+
+ inline void AddIssues(const TIssues& errors) {
+ Issues_.insert(Issues_.end(),
+ errors.Issues_.begin(), errors.Issues_.end());
+ }
+
+ inline void AddIssues(const TPosition& pos, const TIssues& errors) {
+ Issues_.reserve(Issues_.size() + errors.Size());
+ for (const auto& e: errors) {
+ TIssue& issue = Issues_.emplace_back();
+ *issue.MutableMessage() = e.GetMessage(); // No need to sanitize message, it has already been sanitized.
+ issue.Position = pos;
+ issue.SetCode(e.IssueCode, e.Severity);
+ }
+ }
+
+ inline const TIssue* begin() const {
+ return Issues_.begin();
+ }
+
+ inline const TIssue* end() const {
+ return Issues_.end();
+ }
+
+ inline TIssue& back() {
+ return Issues_.back();
+ }
+
+ inline const TIssue& back() const {
+ return Issues_.back();
+ }
+
+ inline bool Empty() const {
+ return Issues_.empty();
+ }
+
+ explicit operator bool() const noexcept {
+ return !Issues_.empty();
+ }
+
+ inline size_t Size() const {
+ return Issues_.size();
+ }
+
+ void PrintTo(IOutputStream& out, bool oneLine = false) const;
+ void PrintWithProgramTo(
+ IOutputStream& out,
+ const TString& programFilename,
+ const TString& programText) const;
+
+ inline TString ToString(bool oneLine = false) const {
+ TStringStream out;
+ PrintTo(out, oneLine);
+ return out.Str();
+ }
+
+ TString ToOneLineString() const {
+ return ToString(true);
+ }
+
+ inline void Clear() {
+ Issues_.clear();
+ }
+
+ void Reserve(size_t capacity) {
+ Issues_.reserve(capacity);
+ }
+
+private:
+ TVector<TIssue> Issues_;
+};
+
+class TErrorException : public yexception {
+ const TIssueCode Code_;
+public:
+ explicit TErrorException(TIssueCode code)
+ : Code_(code)
+ {}
+ TIssueCode GetCode() const {
+ return Code_;
+ }
+};
+
+TIssue ExceptionToIssue(const std::exception& e, const TPosition& pos = TPosition());
+TMaybe<TPosition> TryParseTerminationMessage(TStringBuf& message);
+
+} // namespace NYql
+
+template <>
+void Out<NYql::TPosition>(IOutputStream& out, const NYql::TPosition& pos);
+
+template <>
+void Out<NYql::TRange>(IOutputStream& out, const NYql::TRange& pos);
+
+template <>
+void Out<NYql::TIssue>(IOutputStream& out, const NYql::TIssue& error);
+
+template <>
+struct THash<NYql::TIssue> {
+ inline size_t operator()(const NYql::TIssue& err) const {
+ return err.Hash();
+ }
+};
diff --git a/yql/essentials/public/issue/yql_issue_id.h b/yql/essentials/public/issue/yql_issue_id.h
new file mode 100644
index 0000000000..b4889319a7
--- /dev/null
+++ b/yql/essentials/public/issue/yql_issue_id.h
@@ -0,0 +1,128 @@
+#pragma once
+
+#include <yql/essentials/public/issue/protos/issue_severity.pb.h>
+
+#include <library/cpp/resource/resource.h>
+#include <google/protobuf/descriptor.h>
+#include <google/protobuf/repeated_field.h>
+#include <google/protobuf/text_format.h>
+#include <google/protobuf/message.h>
+
+#include <util/generic/hash.h>
+#include <util/generic/singleton.h>
+#include <util/generic/yexception.h>
+#include <util/string/subst.h>
+
+#ifdef _win_
+#ifdef GetMessage
+#undef GetMessage
+#endif
+#endif
+
+namespace NYql {
+
+using TIssueCode = ui32;
+using ESeverity = NYql::TSeverityIds::ESeverityId;
+const TIssueCode DEFAULT_ERROR = 0;
+const TIssueCode UNEXPECTED_ERROR = 1;
+
+inline TString SeverityToString(ESeverity severity)
+{
+ auto ret = NYql::TSeverityIds::ESeverityId_Name(severity);
+ return ret.empty() ? "Unknown" : to_title(ret.substr(2)); //remove prefix "S_"
+}
+
+template <typename T>
+inline TString IssueCodeToString(TIssueCode id) {
+ auto ret = T::EIssueCode_Name(static_cast<typename T::EIssueCode>(id));
+ if (!ret.empty()) {
+ SubstGlobal(ret, '_', ' ');
+ return to_title(ret);
+ } else {
+ return "Unknown";
+ }
+}
+
+template<typename TProto, const char* ResourceName>
+class TIssueId {
+ TProto ProtoIssues_;
+ THashMap<TIssueCode, NYql::TSeverityIds::ESeverityId> IssuesMap_;
+ THashMap<TIssueCode, TString> IssuesFormatMap_;
+
+ const google::protobuf::Descriptor* GetProtoDescriptor() const {
+ auto ret = ProtoIssues_.GetDescriptor();
+ Y_ENSURE(ret != nullptr, "Bad proto file");
+ return ret;
+ }
+
+ bool CheckSeverityNameFormat(const TString& name) const {
+ if (name.size() > 2 && name.substr(0,2) == "S_") {
+ return true;
+ }
+ return false;
+ }
+
+public:
+ ESeverity GetSeverity(TIssueCode id) const {
+ auto it = IssuesMap_.find(id);
+ Y_ENSURE(it != IssuesMap_.end(), "Unknown issue id: "
+ << id << "(" << IssueCodeToString<TProto>(id) << ")");
+ return it->second;
+ }
+
+ TString GetMessage(TIssueCode id) const {
+ auto it = IssuesFormatMap_.find(id);
+ Y_ENSURE(it != IssuesFormatMap_.end(), "Unknown issue id: "
+ << id << "(" << IssueCodeToString<TProto>(id) << ")");
+ return it->second;
+ }
+
+ TIssueId() {
+ auto configData = NResource::Find(TStringBuf(ResourceName));
+ if (!::google::protobuf::TextFormat::ParseFromString(configData, &ProtoIssues_)) {
+ Y_ENSURE(false, "Bad format of protobuf data file, resource: " << ResourceName);
+ }
+
+ auto sDesc = TSeverityIds::ESeverityId_descriptor();
+ for (int i = 0; i < sDesc->value_count(); i++) {
+ const auto& name = sDesc->value(i)->name();
+ Y_ENSURE(CheckSeverityNameFormat(name),
+ "Wrong severity name: " << name << ". Severity must starts with \"S_\" prefix");
+ }
+
+ for (const auto& x : ProtoIssues_.ids()) {
+ auto rv = IssuesMap_.insert(std::make_pair(x.code(), x.severity()));
+ Y_ENSURE(rv.second, "Duplicate issue code found, code: "
+ << static_cast<int>(x.code())
+ << "(" << IssueCodeToString<TProto>(x.code()) <<")");
+ }
+
+ // Check all IssueCodes have mapping to severity
+ auto eDesc = TProto::EIssueCode_descriptor();
+ for (int i = 0; i < eDesc->value_count(); i++) {
+ auto it = IssuesMap_.find(eDesc->value(i)->number());
+ Y_ENSURE(it != IssuesMap_.end(), "IssueCode: "
+ << eDesc->value(i)->name()
+ << " is not found in protobuf data file");
+ }
+
+ for (const auto& x : ProtoIssues_.ids()) {
+ auto rv = IssuesFormatMap_.insert(std::make_pair(x.code(), x.format()));
+ Y_ENSURE(rv.second, "Duplicate issue code found, code: "
+ << static_cast<int>(x.code())
+ << "(" << IssueCodeToString<TProto>(x.code()) <<")");
+ }
+ }
+};
+
+template<typename TProto, const char* ResourceName>
+inline ESeverity GetSeverity(TIssueCode id) {
+ return Singleton<TIssueId<TProto, ResourceName>>()->GetSeverity(id);
+}
+
+template<typename TProto, const char* ResourceName>
+inline TString GetMessage(TIssueCode id) {
+ return Singleton<TIssueId<TProto, ResourceName>>()->GetMessage(id);
+}
+
+}
diff --git a/yql/essentials/public/issue/yql_issue_manager.cpp b/yql/essentials/public/issue/yql_issue_manager.cpp
new file mode 100644
index 0000000000..8ffeed9fb0
--- /dev/null
+++ b/yql/essentials/public/issue/yql_issue_manager.cpp
@@ -0,0 +1,234 @@
+#include "yql_issue_manager.h"
+
+#include <util/string/cast.h>
+#include <util/string/builder.h>
+
+using namespace NYql;
+
+void TIssueManager::AddScope(std::function<TIssuePtr()> fn) {
+ RawIssues_.emplace(std::make_pair(TMaybe<TIssuePtr>(), fn));
+}
+
+void TIssueManager::LeaveScope() {
+ Y_ASSERT(HasOpenScopes());
+ if (RawIssues_.size() == 1) {
+ // Last scope, just dump it
+ auto maybeIssue = RawIssues_.top().first;
+ if (maybeIssue) {
+ if ((*maybeIssue)->GetCode() == Max<ui32>()) {
+ for (const auto& nestedIssue : (*maybeIssue.Get())->GetSubIssues()) {
+ CompletedIssues_.AddIssue(*nestedIssue);
+ }
+ } else {
+ CompletedIssues_.AddIssue(**maybeIssue);
+ }
+ }
+ RawIssues_.pop();
+ return;
+ }
+
+ if (RawIssues_.top().first.Empty()) {
+ // No issues in this scope
+ RawIssues_.pop();
+ return;
+ }
+
+ auto subIssue = *RawIssues_.top().first;
+ if (subIssue->GetSubIssues().size() == 1) {
+ auto nestedIssue = subIssue->GetSubIssues().front();
+ if (!nestedIssue->GetSubIssues().empty() && nestedIssue->Position == subIssue->Position && nestedIssue->EndPosition == subIssue->EndPosition) {
+ auto msg = subIssue->GetMessage();
+ if (nestedIssue->GetMessage()) {
+ if (msg) {
+ msg.append(", ");
+ }
+ msg.append(nestedIssue->GetMessage());
+ }
+ subIssue = nestedIssue;
+ subIssue->SetMessage(msg);
+ }
+ }
+ RawIssues_.pop();
+ if (RawIssues_.top().first.Empty()) {
+ RawIssues_.top().first = RawIssues_.top().second();
+ if (!*RawIssues_.top().first) {
+ RawIssues_.top().first = new TIssue();
+ (*RawIssues_.top().first)->SetCode(Max<ui32>(), ESeverity::TSeverityIds_ESeverityId_S_INFO);
+ } else {
+ (*RawIssues_.top().first)->Severity = ESeverity::TSeverityIds_ESeverityId_S_INFO;
+ }
+ }
+
+ if (subIssue->GetCode() == Max<ui32>()) {
+ for (const auto& nestedIssue : subIssue->GetSubIssues()) {
+ RawIssues_.top().first->Get()->AddSubIssue(nestedIssue);
+ }
+ } else {
+ RawIssues_.top().first->Get()->AddSubIssue(subIssue);
+ }
+}
+
+void TIssueManager::RaiseIssueForEmptyScope() {
+ if (RawIssues_.top().first.Empty()) {
+ TIssuePtr materialized = RawIssues_.top().second();
+ if (auto p = CheckUniqAndLimit(materialized)) {
+ RawIssues_.top().first = p;
+ }
+ }
+}
+
+void TIssueManager::LeaveAllScopes() {
+ while (!RawIssues_.empty()) {
+ LeaveScope();
+ }
+}
+
+TIssuePtr TIssueManager::CheckUniqAndLimit(TIssuePtr issue) {
+ const auto severity = issue->GetSeverity();
+ if (OverflowIssues_[severity]) {
+ return {};
+ }
+ if (UniqueIssues_[severity].contains(issue)) {
+ return {};
+ }
+ if (IssueLimit_ && UniqueIssues_[severity].size() == IssueLimit_) {
+ OverflowIssues_[severity] = MakeIntrusive<TIssue>(TStringBuilder()
+ << "Too many " << SeverityToString(issue->GetSeverity()) << " issues");
+ OverflowIssues_[severity]->Severity = severity;
+ return {};
+ }
+ UniqueIssues_[severity].insert(issue);
+ return issue;
+}
+
+TIssuePtr TIssueManager::CheckUniqAndLimit(const TIssue& issue) {
+ const auto severity = issue.GetSeverity();
+ if (OverflowIssues_[severity]) {
+ return {};
+ }
+ return CheckUniqAndLimit(MakeIntrusive<TIssue>(issue));
+}
+
+void TIssueManager::RaiseIssue(const TIssue& issue) {
+ TIssuePtr p = CheckUniqAndLimit(issue);
+ if (!p) {
+ return;
+ }
+ if (RawIssues_.empty()) {
+ CompletedIssues_.AddIssue(issue);
+ return;
+ }
+ if (RawIssues_.top().first.Empty()) {
+ RawIssues_.top().first = RawIssues_.top().second();
+ if (!*RawIssues_.top().first) {
+ RawIssues_.top().first = new TIssue();
+ (*RawIssues_.top().first)->SetCode(Max<ui32>(), ESeverity::TSeverityIds_ESeverityId_S_INFO);
+ } else {
+ (*RawIssues_.top().first)->Severity = ESeverity::TSeverityIds_ESeverityId_S_INFO;
+ }
+ }
+ RawIssues_.top().first->Get()->AddSubIssue(p);
+}
+
+void TIssueManager::RaiseIssues(const TIssues& issues) {
+ for (const auto& x : issues) {
+ RaiseIssue(x);
+ }
+}
+
+bool TIssueManager::RaiseWarning(TIssue issue) {
+ bool isWarning = true;
+ if (issue.GetSeverity() == ESeverity::TSeverityIds_ESeverityId_S_WARNING) {
+ const auto action = WarningPolicy_.GetAction(issue.GetCode());
+ switch (action) {
+ case EWarningAction::DISABLE:
+ return isWarning;
+ case EWarningAction::ERROR:
+ issue.Severity = ESeverity::TSeverityIds_ESeverityId_S_ERROR;
+ if (WarningToErrorTreatMessage_) {
+ TIssue newIssue;
+ newIssue.SetCode(issue.GetCode(), ESeverity::TSeverityIds_ESeverityId_S_ERROR);
+ newIssue.SetMessage(WarningToErrorTreatMessage_.GetRef());
+ newIssue.AddSubIssue(new TIssue(issue));
+ issue = newIssue;
+ }
+ isWarning = false;
+ break;
+ case EWarningAction::DEFAULT:
+ break;
+ default:
+ Y_ENSURE(false, "Unknown action");
+ }
+ }
+
+ RaiseIssue(issue);
+ return isWarning;
+}
+
+bool TIssueManager::HasOpenScopes() const {
+ return !RawIssues_.empty();
+}
+
+TIssues TIssueManager::GetIssues() {
+ auto tmp = RawIssues_;
+ LeaveAllScopes();
+ auto result = GetCompletedIssues();
+ RawIssues_ = tmp;
+ return result;
+}
+
+TIssues TIssueManager::GetCompletedIssues() const {
+ TIssues res;
+ for (auto& p: OverflowIssues_) {
+ if (p) {
+ res.AddIssue(*p);
+ }
+ }
+ res.AddIssues(CompletedIssues_);
+ return res;
+}
+
+void TIssueManager::AddIssues(const TIssues& issues) {
+ for (auto& issue: issues) {
+ if (auto p = CheckUniqAndLimit(issue)) {
+ CompletedIssues_.AddIssue(*p);
+ }
+ }
+}
+
+void TIssueManager::AddIssues(const TPosition& pos, const TIssues& issues) {
+ for (auto& issue: issues) {
+ if (auto p = CheckUniqAndLimit(TIssue(pos, issue.GetMessage()))) {
+ CompletedIssues_.AddIssue(*p);
+ }
+ }
+}
+
+void TIssueManager::Reset(const TIssues& issues) {
+ for (auto& p: OverflowIssues_) {
+ p.Drop();
+ }
+
+ for (auto& s: UniqueIssues_) {
+ s.clear();
+ }
+ CompletedIssues_.Clear();
+
+ while (!RawIssues_.empty()) {
+ RawIssues_.pop();
+ }
+ AddIssues(issues);
+}
+
+void TIssueManager::Reset() {
+ Reset(TIssues());
+}
+
+void TIssueManager::AddWarningRule(const TWarningRule &rule)
+{
+ WarningPolicy_.AddRule(rule);
+}
+
+void TIssueManager::SetWarningToErrorTreatMessage(const TString& msg) {
+ WarningToErrorTreatMessage_ = msg;
+}
diff --git a/yql/essentials/public/issue/yql_issue_manager.h b/yql/essentials/public/issue/yql_issue_manager.h
new file mode 100644
index 0000000000..6b6ec2c719
--- /dev/null
+++ b/yql/essentials/public/issue/yql_issue_manager.h
@@ -0,0 +1,83 @@
+#pragma once
+
+#include "yql_issue.h"
+#include "yql_warning.h"
+
+#include <util/generic/maybe.h>
+#include <util/generic/stack.h>
+#include <util/generic/hash_set.h>
+
+#include <array>
+
+namespace NYql {
+
+class TIssueManager: private TNonCopyable {
+public:
+ void AddScope(std::function<TIssuePtr()> fn);
+ void LeaveScope();
+ void RaiseIssue(const TIssue& issue);
+ void RaiseIssues(const TIssues& issues);
+ bool RaiseWarning(TIssue issue);
+ void AddIssues(const TIssues& errors);
+ void AddIssues(const TPosition& pos, const TIssues& issues);
+
+ TIssues GetIssues();
+ TIssues GetCompletedIssues() const;
+
+ void Reset(const TIssues& issues);
+ void Reset();
+
+ void AddWarningRule(const TWarningRule &rule);
+ void SetWarningToErrorTreatMessage(const TString& msg);
+
+ void SetIssueCountLimit(size_t limit) {
+ IssueLimit_ = limit;
+ }
+
+ void RaiseIssueForEmptyScope();
+
+private:
+ TIssuePtr CheckUniqAndLimit(const TIssue& issue);
+ TIssuePtr CheckUniqAndLimit(TIssuePtr issue);
+ void LeaveAllScopes();
+ bool HasOpenScopes() const;
+
+ struct TIssueHash {
+ ui64 operator()(const TIssuePtr& p) {
+ return p->Hash();
+ }
+ };
+ struct TIssueEqual {
+ bool operator()(const TIssuePtr& l, const TIssuePtr& r) {
+ return *l == *r;
+ }
+ };
+ TStack<std::pair<TMaybe<TIssuePtr>, std::function<TIssuePtr()>>> RawIssues_;
+ TIssues CompletedIssues_;
+ TMaybe<TString> WarningToErrorTreatMessage_;
+ TWarningPolicy WarningPolicy_;
+ std::array<TIssuePtr, NYql::TSeverityIds::ESeverityId_ARRAYSIZE> OverflowIssues_;
+ std::array<THashSet<TIssuePtr, TIssueHash, TIssueEqual>, NYql::TSeverityIds::ESeverityId_ARRAYSIZE> UniqueIssues_;
+ size_t IssueLimit_ = 0;
+};
+
+class TIssueScopeGuard: private TNonCopyable {
+ TIssueManager& Manager_;
+public:
+ TIssueScopeGuard(TIssueManager& manager, std::function<TIssuePtr()> fn)
+ : Manager_(manager)
+ {
+ Manager_.AddScope(fn);
+ }
+
+ void RaiseIssueForEmptyScope() {
+ Manager_.RaiseIssueForEmptyScope();
+ }
+
+ ~TIssueScopeGuard()
+ {
+ Manager_.LeaveScope();
+ }
+};
+
+}
diff --git a/yql/essentials/public/issue/yql_issue_manager_ut.cpp b/yql/essentials/public/issue/yql_issue_manager_ut.cpp
new file mode 100644
index 0000000000..e3308163c3
--- /dev/null
+++ b/yql/essentials/public/issue/yql_issue_manager_ut.cpp
@@ -0,0 +1,206 @@
+#include <library/cpp/testing/unittest/registar.h>
+
+#include "yql_issue_manager.h"
+
+using namespace NYql;
+
+static std::function<TIssuePtr()> CreateScopeIssueFunction(TString name, ui32 column, ui32 row) {
+ return [name, column, row]() {
+ return new TIssue(TPosition(column, row), name);
+ };
+}
+
+Y_UNIT_TEST_SUITE(TIssueManagerTest) {
+ Y_UNIT_TEST(NoErrorNoLevelTest) {
+ TIssueManager issueManager;
+ auto completedIssues = issueManager.GetCompletedIssues();
+ UNIT_ASSERT_VALUES_EQUAL(completedIssues.Size(), 0);
+ auto issues = issueManager.GetIssues();
+ UNIT_ASSERT_VALUES_EQUAL(issues.Size(), 0);
+ }
+
+ Y_UNIT_TEST(NoErrorOneLevelTest) {
+ TIssueManager issueManager;
+ issueManager.AddScope(CreateScopeIssueFunction("A", 0, 0));
+ auto completedIssues = issueManager.GetCompletedIssues();
+ UNIT_ASSERT_VALUES_EQUAL(completedIssues.Size(), 0);
+ issueManager.LeaveScope();
+ auto issues = issueManager.GetIssues();
+ UNIT_ASSERT_VALUES_EQUAL(issues.Size(), 0);
+ }
+
+ Y_UNIT_TEST(NoErrorTwoLevelsTest) {
+ TIssueManager issueManager;
+ issueManager.AddScope(CreateScopeIssueFunction("A", 0, 0));
+ issueManager.AddScope(CreateScopeIssueFunction("B", 1, 1));
+ issueManager.LeaveScope();
+ auto completedIssues = issueManager.GetCompletedIssues();
+ UNIT_ASSERT_VALUES_EQUAL(completedIssues.Size(), 0);
+ issueManager.LeaveScope();
+ auto issues = issueManager.GetIssues();
+ UNIT_ASSERT_VALUES_EQUAL(issues.Size(), 0);
+ }
+
+ Y_UNIT_TEST(OneErrorOneLevelTest) {
+ TIssueManager issueManager;
+ issueManager.AddScope(CreateScopeIssueFunction("A", 0, 0));
+ auto completedIssues1 = issueManager.GetCompletedIssues();
+ UNIT_ASSERT_VALUES_EQUAL(completedIssues1.Size(), 0);
+ issueManager.RaiseIssue(TIssue(TPosition(1,2), "IssueOne"));
+ auto completedIssues2 = issueManager.GetCompletedIssues();
+ UNIT_ASSERT_VALUES_EQUAL(completedIssues2.Size(), 0);
+ issueManager.LeaveScope();
+ auto completedIssues3 = issueManager.GetCompletedIssues();
+ UNIT_ASSERT_VALUES_EQUAL(completedIssues3.Size(), 1);
+
+ auto issues = issueManager.GetIssues();
+ UNIT_ASSERT_VALUES_EQUAL(issues.Size(), 1);
+ auto scopeIssue = issues.begin();
+ UNIT_ASSERT_VALUES_EQUAL(scopeIssue->GetMessage(), "A");
+ auto subIssues = scopeIssue->GetSubIssues();
+ UNIT_ASSERT_VALUES_EQUAL(subIssues.size(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(subIssues[0]->GetMessage(), "IssueOne");
+ }
+
+ Y_UNIT_TEST(OneErrorTwoLevelsTest) {
+ TIssueManager issueManager;
+ issueManager.AddScope(CreateScopeIssueFunction("A", 0, 0));
+ issueManager.AddScope(CreateScopeIssueFunction("B", 1, 1));
+ issueManager.RaiseIssue(TIssue(TPosition(1,2), "IssueOne"));
+ issueManager.LeaveScope();
+ issueManager.LeaveScope();
+ auto issues = issueManager.GetIssues();
+ UNIT_ASSERT_VALUES_EQUAL(issues.Size(), 1);
+ auto scopeIssue = issues.begin();
+ UNIT_ASSERT_VALUES_EQUAL(scopeIssue->GetMessage(), "A");
+ auto subIssues = scopeIssue->GetSubIssues();
+ UNIT_ASSERT_VALUES_EQUAL(subIssues.size(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(subIssues[0]->GetMessage(), "B");
+
+ UNIT_ASSERT_VALUES_EQUAL(subIssues[0]->GetSubIssues().size(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(subIssues[0]->GetSubIssues()[0]->GetMessage(), "IssueOne");
+ }
+
+ Y_UNIT_TEST(MultiErrorsMultiLevelsTest) {
+ TIssueManager issueManager;
+ issueManager.AddScope(CreateScopeIssueFunction("A", 0, 0));
+ issueManager.RaiseIssue(TIssue(TPosition(), "WarningScope1"));
+ issueManager.AddScope(CreateScopeIssueFunction("B", 1, 1));
+ issueManager.AddScope(CreateScopeIssueFunction("C", 2, 2));
+ issueManager.RaiseIssue(TIssue(TPosition(), "ErrorScope3"));
+ issueManager.LeaveScope();
+ issueManager.RaiseIssue(TIssue(TPosition(), "ErrorScope2"));
+ issueManager.LeaveScope();
+ issueManager.LeaveScope();
+ auto issues = issueManager.GetIssues();
+ UNIT_ASSERT_VALUES_EQUAL(issues.Size(), 1);
+ auto scopeIssue = issues.begin();
+ auto subIssues = scopeIssue->GetSubIssues();
+ UNIT_ASSERT_VALUES_EQUAL(subIssues.size(), 2);
+ UNIT_ASSERT_VALUES_EQUAL(subIssues[0]->GetSubIssues().size(), 0); //WarningScope1
+ UNIT_ASSERT_VALUES_EQUAL(subIssues[0]->GetMessage(), "WarningScope1");
+ UNIT_ASSERT_VALUES_EQUAL(subIssues[1]->GetSubIssues().size(), 2);
+ UNIT_ASSERT_VALUES_EQUAL(subIssues[1]->GetSubIssues()[0]->GetSubIssues().size(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(subIssues[1]->GetSubIssues()[0]->GetSubIssues()[0]->GetMessage(), "ErrorScope3");
+ UNIT_ASSERT_VALUES_EQUAL(subIssues[1]->GetSubIssues()[1]->GetMessage(), "ErrorScope2");
+ auto ref = R"___(<main>: Error: A
+ <main>: Error: WarningScope1
+ <main>:1:1: Error: B
+ <main>:2:2: Error: C
+ <main>: Error: ErrorScope3
+ <main>: Error: ErrorScope2
+)___";
+ UNIT_ASSERT_VALUES_EQUAL(issues.ToString(), ref);
+ }
+
+ Y_UNIT_TEST(TIssueScopeGuardSimpleTest) {
+ TIssueManager issueManager;
+ {
+ TIssueScopeGuard guard(issueManager, CreateScopeIssueFunction("A", 0, 0));
+ issueManager.RaiseIssue(TIssue(TPosition(1,2), "ErrorScope1"));
+ }
+ auto issues = issueManager.GetIssues();
+ UNIT_ASSERT_VALUES_EQUAL(issues.Size(), 1);
+ auto scopeIssue = issues.begin();
+ UNIT_ASSERT_VALUES_EQUAL(scopeIssue->GetMessage(), "A");
+ auto subIssues = scopeIssue->GetSubIssues();
+ UNIT_ASSERT_VALUES_EQUAL(subIssues.size(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(subIssues[0]->GetMessage(), "ErrorScope1");
+ }
+
+ Y_UNIT_TEST(FuseScopesTest) {
+ TIssueManager issueManager;
+ issueManager.AddScope(CreateScopeIssueFunction("A", 0, 0));
+ issueManager.AddScope(CreateScopeIssueFunction("B", 0, 0));
+ issueManager.AddScope(CreateScopeIssueFunction("C", 0, 0));
+ issueManager.AddScope(CreateScopeIssueFunction("D", 0, 0));
+ issueManager.RaiseIssue(TIssue(TPosition(1,2), "IssueOne"));
+ issueManager.LeaveScope();
+ issueManager.RaiseIssue(TIssue(TPosition(1,2), "IssueTwo"));
+ issueManager.LeaveScope();
+ issueManager.LeaveScope();
+ issueManager.LeaveScope();
+ auto issues = issueManager.GetIssues();
+ auto ref = R"___(<main>: Error: A
+ <main>: Error: B, C
+ <main>: Error: D
+ <main>:2:1: Error: IssueOne
+ <main>:2:1: Error: IssueTwo
+)___";
+ UNIT_ASSERT_VALUES_EQUAL(issues.ToString(), ref);
+ }
+
+ Y_UNIT_TEST(UniqIssues) {
+ TIssueManager issueManager;
+ issueManager.AddScope(CreateScopeIssueFunction("A", 0, 0));
+ issueManager.AddScope(CreateScopeIssueFunction("B", 1, 1));
+ issueManager.RaiseIssue(TIssue(TPosition(1,2), "IssueOne"));
+ issueManager.RaiseIssue(TIssue(TPosition(1,2), "IssueTwo"));
+ issueManager.RaiseIssue(TIssue(TPosition(2,3), "IssueOne"));
+ issueManager.RaiseWarning(TIssue(TPosition(2,3), "IssueOne").SetCode(1, ESeverity::TSeverityIds_ESeverityId_S_WARNING));
+ issueManager.LeaveScope();
+ issueManager.LeaveScope();
+ issueManager.RaiseIssue(TIssue(TPosition(1,2), "IssueOne"));
+ auto issues = issueManager.GetIssues();
+ UNIT_ASSERT_VALUES_EQUAL(issues.Size(), 1);
+ auto ref = R"___(<main>: Error: A
+ <main>:1:1: Error: B
+ <main>:2:1: Error: IssueOne
+ <main>:2:1: Error: IssueTwo
+ <main>:3:2: Error: IssueOne
+ <main>:3:2: Warning: IssueOne, code: 1
+)___";
+ UNIT_ASSERT_VALUES_EQUAL(issues.ToString(), ref);
+ }
+
+ Y_UNIT_TEST(Limits) {
+ TIssueManager issueManager;
+ issueManager.SetIssueCountLimit(2);
+ issueManager.AddScope(CreateScopeIssueFunction("A", 0, 0));
+ issueManager.AddScope(CreateScopeIssueFunction("B", 1, 1));
+ issueManager.RaiseIssue(TIssue(TPosition(1,1), "Issue1"));
+ issueManager.RaiseIssue(TIssue(TPosition(1,1), "Issue2"));
+ issueManager.RaiseIssue(TIssue(TPosition(1,1), "Issue3"));
+ issueManager.RaiseIssue(TIssue(TPosition(1,1), "Issue4").SetCode(1, ESeverity::TSeverityIds_ESeverityId_S_WARNING));
+ issueManager.RaiseIssue(TIssue(TPosition(1,1), "Issue5").SetCode(1, ESeverity::TSeverityIds_ESeverityId_S_WARNING));
+ issueManager.RaiseIssue(TIssue(TPosition(1,1), "Issue6").SetCode(1, ESeverity::TSeverityIds_ESeverityId_S_WARNING));
+ issueManager.RaiseIssue(TIssue(TPosition(1,1), "Issue7").SetCode(2, ESeverity::TSeverityIds_ESeverityId_S_INFO));
+ issueManager.RaiseIssue(TIssue(TPosition(1,1), "Issue8").SetCode(2, ESeverity::TSeverityIds_ESeverityId_S_INFO));
+ issueManager.LeaveScope();
+ issueManager.LeaveScope();
+ auto issues = issueManager.GetIssues();
+ UNIT_ASSERT_VALUES_EQUAL(issues.Size(), 3);
+ auto ref = R"___(<main>: Error: Too many Error issues
+<main>: Warning: Too many Warning issues
+<main>: Error: A
+ <main>:1:1: Error: B
+ <main>:1:1: Error: Issue1
+ <main>:1:1: Error: Issue2
+ <main>:1:1: Warning: Issue4, code: 1
+ <main>:1:1: Warning: Issue5, code: 1
+ <main>:1:1: Info: Issue7, code: 2
+ <main>:1:1: Info: Issue8, code: 2
+)___";
+ UNIT_ASSERT_VALUES_EQUAL(issues.ToString(), ref);
+ }
+}
diff --git a/yql/essentials/public/issue/yql_issue_message.cpp b/yql/essentials/public/issue/yql_issue_message.cpp
new file mode 100644
index 0000000000..757bba3946
--- /dev/null
+++ b/yql/essentials/public/issue/yql_issue_message.cpp
@@ -0,0 +1,149 @@
+#include "yql_issue_message.h"
+
+#include <yql/essentials/public/issue/protos/issue_message.pb.h>
+#include <contrib/ydb/public/api/protos/ydb_issue_message.pb.h>
+
+#include <util/generic/deque.h>
+#include <util/generic/yexception.h>
+#include <util/stream/output.h>
+#include <util/string/join.h>
+
+#include <tuple>
+
+namespace NYql {
+
+using namespace NIssue::NProto;
+
+template<typename TIssueMessage>
+TIssue IssueFromMessage(const TIssueMessage& issueMessage) {
+ TIssue topIssue;
+ TDeque<std::pair<TIssue*, const TIssueMessage*>> queue;
+ queue.push_front(std::make_pair(&topIssue, &issueMessage));
+ while (!queue.empty()) {
+ TIssue& issue = *queue.back().first;
+ const auto& message = *queue.back().second;
+ queue.pop_back();
+ TPosition position(message.position().column(), message.position().row(), message.position().file());
+ TPosition endPosition(message.end_position().column(), message.end_position().row());
+ if (position.HasValue()) {
+ if (endPosition.HasValue()) {
+ issue = TIssue(position, endPosition, message.message());
+ } else {
+ issue = TIssue(position, message.message());
+ }
+ } else {
+ issue = TIssue(message.message());
+ }
+
+ for (const auto& subMessage : message.issues()) {
+ auto subIssue = new TIssue();
+ issue.AddSubIssue(subIssue);
+ queue.push_front(std::make_pair(subIssue, &subMessage));
+ }
+
+ issue.SetCode(message.issue_code(), static_cast<ESeverity>(message.severity()));
+ }
+ return topIssue;
+}
+
+template<typename TIssueMessage>
+void IssuesFromMessage(const ::google::protobuf::RepeatedPtrField<TIssueMessage> &message, TIssues &issues) {
+ issues.Clear();
+ if (message.size()) {
+ issues.Reserve(message.size());
+ for (auto &x : message)
+ issues.AddIssue(IssueFromMessage(x));
+ }
+}
+
+template<typename TIssueMessage>
+void IssueToMessage(const TIssue& topIssue, TIssueMessage* issueMessage) {
+ TDeque<std::pair<const TIssue*, TIssueMessage*>> queue;
+ queue.push_front(std::make_pair(&topIssue, issueMessage));
+ while (!queue.empty()) {
+ const TIssue& issue = *queue.back().first;
+ auto& message = *queue.back().second;
+ queue.pop_back();
+ if (issue.Position) {
+ auto& position = *message.mutable_position();
+ position.set_row(issue.Position.Row);
+ position.set_column(issue.Position.Column);
+ position.set_file(issue.Position.File);
+ }
+ if (issue.EndPosition) {
+ auto& endPosition = *message.mutable_end_position();
+ endPosition.set_row(issue.EndPosition.Row);
+ endPosition.set_column(issue.EndPosition.Column);
+ }
+ message.set_message(issue.GetMessage());
+ message.set_issue_code(issue.GetCode());
+ message.set_severity(issue.GetSeverity());
+
+ for (auto subIssue : issue.GetSubIssues()) {
+ TIssueMessage* subMessage = message.add_issues();
+ queue.push_front(std::make_pair(subIssue.Get(), subMessage));
+ }
+ }
+}
+
+template<typename TIssueMessage>
+void IssuesToMessage(const TIssues& issues, ::google::protobuf::RepeatedPtrField<TIssueMessage> *message) {
+ message->Clear();
+ if (!issues)
+ return;
+ message->Reserve(issues.Size());
+ for (const auto &issue : issues) {
+ IssueToMessage(issue, message->Add());
+ }
+}
+
+template
+TIssue IssueFromMessage<Ydb::Issue::IssueMessage>(const Ydb::Issue::IssueMessage& issueMessage);
+template
+TIssue IssueFromMessage<NYql::NIssue::NProto::IssueMessage>(const NYql::NIssue::NProto::IssueMessage& issueMessage);
+
+template
+void IssuesFromMessage<Ydb::Issue::IssueMessage>(const ::google::protobuf::RepeatedPtrField<Ydb::Issue::IssueMessage>& message, TIssues& issues);
+template
+void IssuesFromMessage<NYql::NIssue::NProto::IssueMessage>(const ::google::protobuf::RepeatedPtrField<NYql::NIssue::NProto::IssueMessage>& message, TIssues& issues);
+
+template
+void IssueToMessage<Ydb::Issue::IssueMessage>(const TIssue& topIssue, Ydb::Issue::IssueMessage* issueMessage);
+template
+void IssueToMessage<NYql::NIssue::NProto::IssueMessage>(const TIssue& topIssue, NYql::NIssue::NProto::IssueMessage* issueMessage);
+
+template
+void IssuesToMessage<Ydb::Issue::IssueMessage>(const TIssues& issues, ::google::protobuf::RepeatedPtrField<Ydb::Issue::IssueMessage>* message);
+template
+void IssuesToMessage<NYql::NIssue::NProto::IssueMessage>(const TIssues& issues, ::google::protobuf::RepeatedPtrField<NYql::NIssue::NProto::IssueMessage>* message);
+
+NIssue::NProto::IssueMessage IssueToMessage(const TIssue& topIssue) {
+ NIssue::NProto::IssueMessage issueMessage;
+ IssueToMessage(topIssue, &issueMessage);
+ return issueMessage;
+}
+
+TString IssueToBinaryMessage(const TIssue& issue) {
+ TString result;
+ Ydb::Issue::IssueMessage protobuf;
+ IssueToMessage(issue, &protobuf);
+ Y_PROTOBUF_SUPPRESS_NODISCARD protobuf.SerializeToString(&result);
+ return result;
+}
+
+TIssue IssueFromBinaryMessage(const TString& binaryMessage) {
+ Ydb::Issue::IssueMessage protobuf;
+ if (!protobuf.ParseFromString(binaryMessage)) {
+ ythrow yexception() << "unable to parse binary string as issue protobuf";
+ }
+ return IssueFromMessage(protobuf);
+}
+
+}
+
+Y_DECLARE_OUT_SPEC(, google::protobuf::RepeatedPtrField<Ydb::Issue::IssueMessage>, stream, issues) {
+ stream << JoinSeq("", issues);
+}
+Y_DECLARE_OUT_SPEC(, google::protobuf::RepeatedPtrField<NYql::NIssue::NProto::IssueMessage>, stream, issues) {
+ stream << JoinSeq("", issues);
+} \ No newline at end of file
diff --git a/yql/essentials/public/issue/yql_issue_message.h b/yql/essentials/public/issue/yql_issue_message.h
new file mode 100644
index 0000000000..a3cb63e4f4
--- /dev/null
+++ b/yql/essentials/public/issue/yql_issue_message.h
@@ -0,0 +1,37 @@
+#pragma once
+
+#include "yql_issue.h"
+
+#include <util/generic/ylimits.h>
+
+namespace NYql {
+
+namespace NIssue {
+namespace NProto {
+class IssueMessage;
+}
+}
+
+template<typename TIssueMessage>
+TIssue IssueFromMessage(const TIssueMessage& issueMessage);
+template<typename TIssueMessage>
+void IssuesFromMessage(const ::google::protobuf::RepeatedPtrField<TIssueMessage>& message, TIssues& issues);
+
+template<typename TIssueMessage>
+TString IssuesFromMessageAsString(const ::google::protobuf::RepeatedPtrField<TIssueMessage>& message) {
+ TIssues issues;
+ IssuesFromMessage(message, issues);
+ return issues.ToOneLineString();
+}
+
+NIssue::NProto::IssueMessage IssueToMessage(const TIssue& topIssue);
+
+template<typename TIssueMessage>
+void IssueToMessage(const TIssue& topIssue, TIssueMessage* message);
+template<typename TIssueMessage>
+void IssuesToMessage(const TIssues& issues, ::google::protobuf::RepeatedPtrField<TIssueMessage>* message);
+
+TString IssueToBinaryMessage(const TIssue& issue);
+TIssue IssueFromBinaryMessage(const TString& binaryMessage);
+
+}
diff --git a/yql/essentials/public/issue/yql_issue_ut.cpp b/yql/essentials/public/issue/yql_issue_ut.cpp
new file mode 100644
index 0000000000..02f917fa54
--- /dev/null
+++ b/yql/essentials/public/issue/yql_issue_ut.cpp
@@ -0,0 +1,249 @@
+#include "yql_issue.h"
+#include "yql_issue_message.h"
+
+#include <library/cpp/testing/unittest/registar.h>
+#include <yql/essentials/public/issue/protos/issue_message.pb.h>
+#include <yql/essentials/public/issue/yql_issue_message.h>
+#include <contrib/ydb/public/api/protos/ydb_issue_message.pb.h>
+
+#include <library/cpp/unicode/normalization/normalization.h>
+
+#include <util/charset/utf8.h>
+#include <util/charset/wide.h>
+#include <util/string/builder.h>
+
+#include <google/protobuf/message.h>
+#include <google/protobuf/descriptor.h>
+
+using namespace google::protobuf;
+using namespace NYql;
+
+void ensureMessageTypesSame(const Descriptor* a, const Descriptor* b, THashSet<TString>* visitedTypes);
+void ensureFieldDescriptorsSame(const FieldDescriptor* a, const FieldDescriptor* b, THashSet<TString>* visitedTypes) {
+ UNIT_ASSERT(a);
+ UNIT_ASSERT(b);
+
+ UNIT_ASSERT_VALUES_EQUAL(FieldDescriptor::TypeName(a->type()), FieldDescriptor::TypeName(b->type()));
+ UNIT_ASSERT_VALUES_EQUAL((int)a->label(), (int)b->label());
+ UNIT_ASSERT_VALUES_EQUAL(a->name(), b->name());
+ UNIT_ASSERT_VALUES_EQUAL(a->number(), b->number());
+ UNIT_ASSERT_VALUES_EQUAL(a->is_repeated(), b->is_repeated());
+ UNIT_ASSERT_VALUES_EQUAL(a->is_packed(), b->is_packed());
+ UNIT_ASSERT_VALUES_EQUAL(a->index(), b->index());
+ if (a->type() == FieldDescriptor::TYPE_MESSAGE || a->type() == FieldDescriptor::TYPE_GROUP) {
+ ensureMessageTypesSame(a->message_type(), b->message_type(), visitedTypes);
+ }
+}
+
+void ensureMessageTypesSame(const Descriptor* a, const Descriptor* b, THashSet<TString>* visitedTypes) {
+ UNIT_ASSERT(a);
+ UNIT_ASSERT(b);
+ if (!visitedTypes->insert(a->name()).second) {
+ return;
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(a->name(), b->name());
+ UNIT_ASSERT_VALUES_EQUAL(a->field_count(), b->field_count());
+
+ for (int i = 0; i < a->field_count(); i++) {
+ ensureFieldDescriptorsSame(a->field(i), b->field(i), visitedTypes);
+ }
+}
+
+Y_UNIT_TEST_SUITE(IssueTest) {
+ Y_UNIT_TEST(Ascii) {
+ TIssue issue1("тест abc");
+ UNIT_ASSERT_VALUES_EQUAL(issue1.GetMessage(), "тест abc");
+ TIssue issue2("\xFF abc");
+ UNIT_ASSERT_VALUES_EQUAL(issue2.GetMessage(), "? abc");
+ TIssue issue3("");
+ UNIT_ASSERT_VALUES_EQUAL(issue3.GetMessage(), "");
+ TIssue issue4("abc");
+ UNIT_ASSERT_VALUES_EQUAL(issue4.GetMessage(), "abc");
+ }
+}
+
+Y_UNIT_TEST_SUITE(IssueProtoTest) {
+ Y_UNIT_TEST(KikimrYqlSameLayout) {
+ Ydb::Issue::IssueMessage yqlIssue;
+ NYql::NIssue::NProto::IssueMessage kikimrIssue;
+ THashSet<TString> visitedTypes;
+ ensureMessageTypesSame(yqlIssue.GetDescriptor(), kikimrIssue.GetDescriptor(), &visitedTypes);
+ }
+
+ Y_UNIT_TEST(BinarySerialization) {
+ TIssue issueTo("root_issue");
+ TString bin = IssueToBinaryMessage(issueTo);
+ TIssue issueFrom = IssueFromBinaryMessage(bin);
+ UNIT_ASSERT_EQUAL(issueTo, issueFrom);
+ }
+
+ Y_UNIT_TEST(WrongBinStringException) {
+ UNIT_ASSERT_EXCEPTION(IssueFromBinaryMessage("qqq"), yexception);
+ }
+}
+
+
+Y_UNIT_TEST_SUITE(TextWalkerTest) {
+ Y_UNIT_TEST(BasicTest) {
+ TPosition pos;
+ pos.Row = 1;
+
+ TTextWalker walker(pos, false);
+ walker.Advance(TStringBuf("a\r\taa"));
+
+ UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(5, 1));
+ walker.Advance(TStringBuf("\na"));
+ UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(1, 2));
+ }
+
+ Y_UNIT_TEST(CrLfTest) {
+ TPosition pos;
+ pos.Row = 1;
+
+ TTextWalker walker(pos, false);
+ walker.Advance(TStringBuf("a\raa\r"));
+ UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(4, 1));
+ walker.Advance('\n');
+ UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(4, 1));
+ walker.Advance(TStringBuf("\r\r\ra"));
+ UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(4, 2));
+ walker.Advance('\r');
+ UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(4, 2));
+ walker.Advance('\n');
+ UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(4, 2));
+ walker.Advance('a');
+ UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(1, 3));
+ }
+
+ Y_UNIT_TEST(UnicodeTest) {
+ {
+ TPosition pos;
+ pos.Row = 1;
+
+ TTextWalker walker(pos, false);
+ walker.Advance(TStringBuf("привет"));
+
+ UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(12, 1));
+ }
+
+ {
+ TPosition pos;
+ pos.Row = 1;
+
+ TTextWalker walker(pos, true);
+ walker.Advance(TStringBuf("привет"));
+
+ UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(6, 1));
+ }
+ }
+}
+
+Y_UNIT_TEST_SUITE(ToOneLineStringTest) {
+ Y_UNIT_TEST(OneMessageTest) {
+ TIssues issues;
+ issues.AddIssue(TPosition(12, 34, "file.abc"), "error");
+ UNIT_ASSERT_STRINGS_EQUAL(issues.ToOneLineString(), "{ file.abc:34:12: Error: error }");
+ }
+
+ Y_UNIT_TEST(SubIssuesTest) {
+ TIssue issue(TPosition(12, 34, "file.abc"), "error");
+ TIssue subissue("suberror");
+ subissue.AddSubIssue(MakeIntrusive<TIssue>("subsuberror"));
+ issue.AddSubIssue(MakeIntrusive<TIssue>(subissue));
+
+ TIssues issues;
+ issues.AddIssue(issue);
+ UNIT_ASSERT_STRINGS_EQUAL(issues.ToOneLineString(), "{ file.abc:34:12: Error: error subissue: { <main>: Error: suberror subissue: { <main>: Error: subsuberror } } }");
+ }
+
+ Y_UNIT_TEST(ManyIssuesTest) {
+ TIssue issue(TPosition(12, 34, "file.abc"), "error\n");
+ issue.AddSubIssue(MakeIntrusive<TIssue>("suberror"));
+ TIssues issues;
+ issues.AddIssue(issue);
+ issues.AddIssue(TPosition(100, 2, "abc.file"), "my\nmessage");
+ UNIT_ASSERT_STRINGS_EQUAL(issues.ToOneLineString(), "[ { file.abc:34:12: Error: error subissue: { <main>: Error: suberror } } { abc.file:2:100: Error: my message } ]");
+ }
+}
+
+Y_UNIT_TEST_SUITE(ToStreamTest) {
+ template <typename TIssueMessage>
+ void CheckSerializationToStream(const TIssues& issues, const TString& expected) {
+ google::protobuf::RepeatedPtrField<TIssueMessage> protoIssues;
+ IssuesToMessage(issues, &protoIssues);
+ TStringBuilder stream;
+ stream << protoIssues;
+ UNIT_ASSERT_STRINGS_EQUAL(stream, expected);
+ };
+
+ Y_UNIT_TEST(OneMessageTest) {
+ TIssues issues;
+ issues.AddIssue("error");
+ CheckSerializationToStream<Ydb::Issue::IssueMessage>(issues, "{ message: \"error\" severity: 1 }");
+ CheckSerializationToStream<NYql::NIssue::NProto::IssueMessage>(issues, "{ message: \"error\" issue_code: 0 severity: 1 }");
+ }
+
+ Y_UNIT_TEST(SubIssuesTest) {
+ TIssue issue(TPosition(12, 34, "file.abc"), "error");
+ TIssue subissue("suberror");
+ subissue.AddSubIssue(MakeIntrusive<TIssue>("subsuberror"));
+ issue.AddSubIssue(MakeIntrusive<TIssue>(subissue));
+ TIssues issues;
+ issues.AddIssue(issue);
+ CheckSerializationToStream<Ydb::Issue::IssueMessage>(issues, "{ position { row: 34 column: 12 file: \"file.abc\" } message: \"error\" end_position { row: 34 column: 12 } severity: 1 issues { message: \"suberror\" severity: 1 issues { message: \"subsuberror\" severity: 1 } } }");
+ CheckSerializationToStream<NYql::NIssue::NProto::IssueMessage>(issues, "{ position { row: 34 column: 12 file: \"file.abc\" } message: \"error\" end_position { row: 34 column: 12 } issue_code: 0 severity: 1 issues { message: \"suberror\" issue_code: 0 severity: 1 issues { message: \"subsuberror\" issue_code: 0 severity: 1 } } }");
+ }
+
+ Y_UNIT_TEST(ManyIssuesTest) {
+ TIssue issue(TPosition(12, 34, "file.abc"), "error");
+ issue.AddSubIssue(MakeIntrusive<TIssue>("suberror"));
+ TIssues issues;
+ issues.AddIssue(issue);
+ issues.AddIssue(TPosition(100, 2, "abc.file"), "my message");
+ CheckSerializationToStream<Ydb::Issue::IssueMessage>(issues, "{ position { row: 34 column: 12 file: \"file.abc\" } message: \"error\" end_position { row: 34 column: 12 } severity: 1 issues { message: \"suberror\" severity: 1 } }{ position { row: 2 column: 100 file: \"abc.file\" } message: \"my message\" end_position { row: 2 column: 100 } severity: 1 }");
+ CheckSerializationToStream<NYql::NIssue::NProto::IssueMessage>(issues, "{ position { row: 34 column: 12 file: \"file.abc\" } message: \"error\" end_position { row: 34 column: 12 } issue_code: 0 severity: 1 issues { message: \"suberror\" issue_code: 0 severity: 1 } }{ position { row: 2 column: 100 file: \"abc.file\" } message: \"my message\" end_position { row: 2 column: 100 } issue_code: 0 severity: 1 }");
+ }
+}
+
+Y_UNIT_TEST_SUITE(ToMessage) {
+ Y_UNIT_TEST(NonUtf8) {
+ const TString nonUtf8String = "\x7f\xf8\xf7\xff\xf8\x1f\xff\xf2\xaf\xbf\xfe\xfa\xf5\x7f\xfe\xfa\x27\x20\x7d\x20\x5d\x2e";
+ UNIT_ASSERT(!IsUtf(nonUtf8String));
+ TIssue issue;
+ issue.SetMessage(nonUtf8String);
+
+ Ydb::Issue::IssueMessage msg;
+ IssueToMessage(issue, &msg);
+ TString serialized;
+ UNIT_ASSERT(msg.SerializeToString(&serialized));
+ Ydb::Issue::IssueMessage msg2;
+ UNIT_ASSERT(msg2.ParseFromString(serialized));
+ }
+}
+
+Y_UNIT_TEST_SUITE(EscapeNonUtf8) {
+ Y_UNIT_TEST(Escape) {
+ const TString nonUtf8String = "\xfe\xfa\xf5\xc2";
+ UNIT_ASSERT(!IsUtf(nonUtf8String));
+
+ // Check that our escaping correctly processes unicode pairs
+ const TString toNormalize = "Ёлка";
+ const TString nfd = WideToUTF8(Normalize<NUnicode::ENormalization::NFD>(UTF8ToWide(toNormalize))); // dots over 'ё' will be separate unicode symbol
+ const TString nfc = WideToUTF8(Normalize<NUnicode::ENormalization::NFC>(UTF8ToWide(toNormalize))); // dots over 'ё' will be with with their letter
+ UNIT_ASSERT_STRINGS_UNEQUAL(nfc, nfd);
+ std::pair<TString, TString> nonUtf8Messages[] = {
+ { nonUtf8String, "????" },
+ { TStringBuilder() << nonUtf8String << "Failed to parse file " << nonUtf8String << "עברית" << nonUtf8String, "????Failed to parse file ????עברית????" },
+ { nfd, nfd },
+ { nfc, nfc },
+ { TStringBuilder() << nfc << nonUtf8String << nfd, TStringBuilder() << nfc << "????" << nfd },
+ { TStringBuilder() << nfd << nonUtf8String << nfc, TStringBuilder() << nfd << "????" << nfc },
+ };
+
+ for (const auto& [src, dst] : nonUtf8Messages) {
+ TIssue issue(src);
+ UNIT_ASSERT_STRINGS_EQUAL(issue.GetMessage(), dst);
+ }
+ }
+}
diff --git a/yql/essentials/public/issue/yql_issue_utils.cpp b/yql/essentials/public/issue/yql_issue_utils.cpp
new file mode 100644
index 0000000000..2be0b8e57b
--- /dev/null
+++ b/yql/essentials/public/issue/yql_issue_utils.cpp
@@ -0,0 +1,105 @@
+#include "yql_issue_utils.h"
+
+#include <util/system/yassert.h>
+
+#include <tuple>
+#include <list>
+#include <algorithm>
+#include <deque>
+
+namespace NYql {
+
+TIssue TruncateIssueLevels(const TIssue& topIssue, TTruncateIssueOpts opts) {
+ // [issue, level, parent, visibleParent]
+ std::list<std::tuple<const TIssue*, ui32, size_t, size_t>> issueQueue;
+ // [issue, targetIssue, level, parent, visible, visibleParent, targetSkipIssue]
+ std::deque<std::tuple<const TIssue*, TIssue*, ui32, size_t, bool, size_t, TIssue*>> issues;
+ // [depth from bottom, position]
+ std::list<size_t> leafs;
+
+ const auto depthBeforeLeaf = std::max(opts.KeepTailLevels, ui32(1)) - 1;
+ const auto maxLevels = std::max(opts.MaxLevels - std::min(opts.MaxLevels, depthBeforeLeaf + 1), ui32(1));
+
+ issueQueue.emplace_front(&topIssue, 0, 0, 0);
+ while (!issueQueue.empty()) {
+ const auto issue = std::get<0>(issueQueue.back());
+ const auto level = std::get<1>(issueQueue.back());
+ const auto parent = std::get<2>(issueQueue.back());
+ const auto visibleParent = std::get<3>(issueQueue.back());
+ issueQueue.pop_back();
+
+ const bool visible = issue->GetSubIssues().empty() || level < maxLevels;
+ const auto pos = issues.size();
+ issues.emplace_back(issue, nullptr, level, parent, visible, visibleParent, nullptr);
+ if (issue->GetSubIssues().empty()) {
+ if (level != 0) {
+ leafs.push_back(pos);
+ }
+ } else {
+ for (auto subIssue : issue->GetSubIssues()) {
+ issueQueue.emplace_front(subIssue.Get(), level + 1, pos, visible ? pos : visibleParent);
+ }
+ }
+ }
+
+ if (depthBeforeLeaf && !leafs.empty()) {
+ for (size_t pos: leafs) {
+ ui32 depth = depthBeforeLeaf;
+ auto parent = std::get<3>(issues.at(pos));
+ while (depth && parent) {
+ auto& visible = std::get<4>(issues.at(parent));
+ auto& visibleParent = std::get<5>(issues.at(pos));
+ if (!visible || visibleParent != parent) {
+ visible = true;
+ visibleParent = parent; // Update visible parent
+ --depth;
+ pos = parent;
+ parent = std::get<3>(issues.at(parent));
+ } else {
+ break;
+ }
+ }
+ }
+ }
+ leafs.clear();
+
+ TIssue result;
+ for (auto& i: issues) {
+ const auto srcIssue = std::get<0>(i);
+ auto& targetIssue = std::get<1>(i);
+ const auto level = std::get<2>(i);
+ const auto parent = std::get<3>(i);
+ const auto visible = std::get<4>(i);
+ const auto visibleParent = std::get<5>(i);
+
+ if (0 == level) {
+ targetIssue = &result;
+ targetIssue->CopyWithoutSubIssues(*srcIssue);
+ } else if (visible) {
+ auto& parentRec = issues.at(visibleParent);
+ auto& parentTargetIssue = std::get<1>(parentRec);
+ if (parent != visibleParent) {
+ auto& parentSkipIssue = std::get<6>(parentRec);
+ if (!parentSkipIssue) {
+ const auto parentIssue = std::get<0>(parentRec);
+ auto newIssue = MakeIntrusive<TIssue>("(skipped levels)");
+ newIssue->SetCode(parentIssue->GetCode(), parentIssue->GetSeverity());
+ parentTargetIssue->AddSubIssue(newIssue);
+ parentSkipIssue = newIssue.Get();
+ }
+ auto newIssue = MakeIntrusive<TIssue>(TString{});
+ newIssue->CopyWithoutSubIssues(*srcIssue);
+ parentSkipIssue->AddSubIssue(newIssue);
+ targetIssue = newIssue.Get();
+ } else {
+ auto newIssue = MakeIntrusive<TIssue>(TString{});
+ newIssue->CopyWithoutSubIssues(*srcIssue);
+ parentTargetIssue->AddSubIssue(newIssue);
+ targetIssue = newIssue.Get();
+ }
+ }
+ }
+ return result;
+}
+
+} // namspace NYql
diff --git a/yql/essentials/public/issue/yql_issue_utils.h b/yql/essentials/public/issue/yql_issue_utils.h
new file mode 100644
index 0000000000..023ea0edbf
--- /dev/null
+++ b/yql/essentials/public/issue/yql_issue_utils.h
@@ -0,0 +1,31 @@
+#pragma once
+
+#include "yql_issue.h"
+
+#include <util/generic/ylimits.h>
+#include <util/system/types.h>
+
+namespace NYql {
+
+struct TTruncateIssueOpts {
+
+#define YQL_TRUNC_DECL_FIELD(type, name, def) \
+ TTruncateIssueOpts& Set##name(type arg##name)& { \
+ name = arg##name; \
+ return *this; \
+ } \
+ TTruncateIssueOpts&& Set##name(type arg##name)&& { \
+ name = arg##name; \
+ return std::move(*this); \
+ } \
+ type name = def;
+
+ YQL_TRUNC_DECL_FIELD(ui32, MaxLevels, Max<ui32>())
+ YQL_TRUNC_DECL_FIELD(ui32, KeepTailLevels, 1)
+
+#undef YQL_TRUNC_DECL_FIELD
+};
+
+TIssue TruncateIssueLevels(const TIssue& topIssue, TTruncateIssueOpts opts = {});
+
+} // namespace NYql
diff --git a/yql/essentials/public/issue/yql_issue_utils_ut.cpp b/yql/essentials/public/issue/yql_issue_utils_ut.cpp
new file mode 100644
index 0000000000..657643a613
--- /dev/null
+++ b/yql/essentials/public/issue/yql_issue_utils_ut.cpp
@@ -0,0 +1,200 @@
+#include "yql_issue_utils.h"
+#include "yql_issue.h"
+#include "yql_issue_id.h"
+
+#include <library/cpp/testing/unittest/registar.h>
+
+#include <util/string/subst.h>
+
+using namespace NYql;
+
+Y_UNIT_TEST_SUITE(TIssueUtilsTest) {
+ Y_UNIT_TEST(TruncLevels1) {
+ auto level0 = MakeIntrusive<TIssue>("level0");
+ auto level1 = MakeIntrusive<TIssue>("level1");
+ auto level2 = MakeIntrusive<TIssue>("level2");
+ auto level30 = MakeIntrusive<TIssue>("level30");
+ auto level31 = MakeIntrusive<TIssue>("level31");
+ auto level40 = MakeIntrusive<TIssue>("level40");
+ auto level41 = MakeIntrusive<TIssue>("level41");
+ auto level51 = MakeIntrusive<TIssue>("level51");
+
+ /*
+ * * 0
+ * |
+ * * 1
+ * |
+ * * 2 --
+ * | |
+ * * 30 * 31
+ * | |
+ * * 40 * 41
+ * |
+ * * 51
+ */
+
+ level0->AddSubIssue(level1);
+ level1->AddSubIssue(level2);
+ level2->AddSubIssue(level30);
+ level2->AddSubIssue(level31);
+ level30->AddSubIssue(level40);
+ level31->AddSubIssue(level41);
+ level41->AddSubIssue(level51);
+
+ {
+ const auto res = TIssues({TruncateIssueLevels(*level0, TTruncateIssueOpts().SetMaxLevels(4).SetKeepTailLevels(2))}).ToString();
+ const auto expected =
+R"___(<main>: Error: level0
+ <main>: Error: level1
+ <main>: Error: (skipped levels)
+ <main>: Error: level30
+ <main>: Error: level40
+ <main>: Error: level41
+ <main>: Error: level51
+)___";
+ UNIT_ASSERT_STRINGS_EQUAL(res, expected);
+ }
+
+ {
+ const auto res = TIssues({TruncateIssueLevels(*level0, TTruncateIssueOpts().SetMaxLevels(3).SetKeepTailLevels(1))}).ToString();
+ const auto expected =
+R"___(<main>: Error: level0
+ <main>: Error: level1
+ <main>: Error: (skipped levels)
+ <main>: Error: level40
+ <main>: Error: level51
+)___";
+ UNIT_ASSERT_STRINGS_EQUAL(res, expected);
+ }
+ }
+
+ Y_UNIT_TEST(TruncLevels2) {
+ auto level0 = MakeIntrusive<TIssue>("level0");
+ auto level1 = MakeIntrusive<TIssue>("level1");
+ auto level2 = MakeIntrusive<TIssue>("level2");
+ auto level3 = MakeIntrusive<TIssue>("level3");
+ auto level40 = MakeIntrusive<TIssue>("level40");
+ auto level41 = MakeIntrusive<TIssue>("level41");
+
+ /*
+ * * 0
+ * |
+ * * 1
+ * |
+ * * 2
+ * |
+ * * 3 --
+ * | |
+ * * 40 * 41
+ */
+
+ level0->AddSubIssue(level1);
+ level1->AddSubIssue(level2);
+ level2->AddSubIssue(level3);
+ level3->AddSubIssue(level40);
+ level3->AddSubIssue(level41);
+
+ {
+ const auto res = TIssues({TruncateIssueLevels(*level0, TTruncateIssueOpts().SetMaxLevels(4).SetKeepTailLevels(2))}).ToString();
+ const auto expected =
+R"___(<main>: Error: level0
+ <main>: Error: level1
+ <main>: Error: (skipped levels)
+ <main>: Error: level3
+ <main>: Error: level40
+ <main>: Error: level41
+)___";
+ UNIT_ASSERT_STRINGS_EQUAL(res, expected);
+ }
+ }
+
+ Y_UNIT_TEST(TruncLevels3) {
+ auto level0 = MakeIntrusive<TIssue>("level0");
+ auto level1 = MakeIntrusive<TIssue>("level1");
+ auto level2 = MakeIntrusive<TIssue>("level2");
+ auto level3 = MakeIntrusive<TIssue>("level3");
+ auto level40 = MakeIntrusive<TIssue>("level40");
+ auto level41 = MakeIntrusive<TIssue>("level41");
+ auto level50 = MakeIntrusive<TIssue>("level50");
+
+ /*
+ * * 0
+ * |
+ * * 1
+ * |
+ * * 2
+ * |
+ * * 3 --
+ * | |
+ * * 40 |
+ * | |
+ * * 50 * 41
+ */
+
+ level0->AddSubIssue(level1);
+ level1->AddSubIssue(level2);
+ level2->AddSubIssue(level3);
+ level3->AddSubIssue(level40);
+ level3->AddSubIssue(level41);
+ level40->AddSubIssue(level50);
+
+ {
+ const auto res = TIssues({TruncateIssueLevels(*level0, TTruncateIssueOpts().SetMaxLevels(4).SetKeepTailLevels(1))}).ToString();
+ const auto expected =
+R"___(<main>: Error: level0
+ <main>: Error: level1
+ <main>: Error: level2
+ <main>: Error: (skipped levels)
+ <main>: Error: level41
+ <main>: Error: level50
+)___";
+ UNIT_ASSERT_STRINGS_EQUAL(res, expected);
+ }
+
+ {
+ const auto res = TIssues({TruncateIssueLevels(*level0, TTruncateIssueOpts().SetMaxLevels(4).SetKeepTailLevels(2))}).ToString();
+ const auto expected =
+R"___(<main>: Error: level0
+ <main>: Error: level1
+ <main>: Error: (skipped levels)
+ <main>: Error: level3
+ <main>: Error: level41
+ <main>: Error: level40
+ <main>: Error: level50
+)___";
+ UNIT_ASSERT_STRINGS_EQUAL(res, expected);
+ }
+
+ {
+ const auto res = TIssues({TruncateIssueLevels(*level0, TTruncateIssueOpts().SetMaxLevels(4).SetKeepTailLevels(3))}).ToString();
+ const auto expected =
+R"___(<main>: Error: level0
+ <main>: Error: (skipped levels)
+ <main>: Error: level2
+ <main>: Error: level3
+ <main>: Error: level40
+ <main>: Error: level50
+ <main>: Error: level41
+)___";
+ UNIT_ASSERT_STRINGS_EQUAL(res, expected);
+ }
+ }
+
+ Y_UNIT_TEST(KeepSeverity) {
+ const auto templ =
+R"___(<main>: {severity}: level0, code: 1
+ <main>: {severity}: level1, code: 1
+)___";
+ for (auto severity: {ESeverity::TSeverityIds_ESeverityId_S_INFO, ESeverity::TSeverityIds_ESeverityId_S_WARNING, ESeverity::TSeverityIds_ESeverityId_S_ERROR, ESeverity::TSeverityIds_ESeverityId_S_FATAL}) {
+
+ auto level0 = MakeIntrusive<TIssue>(TIssue("level0").SetCode(1, severity));
+ auto level1 = MakeIntrusive<TIssue>(TIssue("level1").SetCode(1, severity));
+
+ level0->AddSubIssue(level1);
+
+ const auto res = TIssues({TruncateIssueLevels(*level0, TTruncateIssueOpts().SetMaxLevels(15).SetKeepTailLevels(3))}).ToString();
+ const auto expected = SubstGlobalCopy<TString, TString>(templ, "{severity}", SeverityToString(severity));
+ UNIT_ASSERT_STRINGS_EQUAL(res, expected);
+ }
+ }
+}
diff --git a/yql/essentials/public/issue/yql_warning.cpp b/yql/essentials/public/issue/yql_warning.cpp
new file mode 100644
index 0000000000..881e63e95f
--- /dev/null
+++ b/yql/essentials/public/issue/yql_warning.cpp
@@ -0,0 +1,73 @@
+#include "yql_warning.h"
+
+#include <util/string/cast.h>
+#include <util/string/join.h>
+
+
+namespace NYql {
+
+TWarningRule::EParseResult TWarningRule::ParseFrom(const TString& codePattern, const TString& action,
+ TWarningRule& result, TString& errorMessage)
+{
+ errorMessage.clear();
+ result.IssueCodePattern.clear();
+ result.Action = EWarningAction::DEFAULT;
+
+ if (!TryFromString<EWarningAction>(to_upper(action), result.Action)) {
+ errorMessage = "unknown warning action '" + action + "', expecting one of " +
+ Join(", ", EWarningAction::DEFAULT, EWarningAction::ERROR, EWarningAction::DISABLE);
+ return EParseResult::PARSE_ACTION_FAIL;
+ }
+
+ if (codePattern != "*") {
+ ui32 code;
+ if (!TryFromString(codePattern, code)) {
+ errorMessage = "unknown warning code '" + codePattern + "', expecting integer or '*'";
+ return EParseResult::PARSE_PATTERN_FAIL;
+ }
+ }
+
+ result.IssueCodePattern = codePattern;
+ return EParseResult::PARSE_OK;
+}
+
+void TWarningPolicy::AddRule(const TWarningRule& rule)
+{
+ TString pattern = rule.GetPattern();
+ if (pattern.empty()) {
+ return;
+ }
+
+ Rules.push_back(rule);
+
+ EWarningAction action = rule.GetAction();
+ if (pattern == "*") {
+ BaseAction = action;
+ Overrides.clear();
+ return;
+ }
+
+ TIssueCode code;
+ Y_ENSURE(TryFromString(pattern, code));
+
+ if (action == BaseAction) {
+ Overrides.erase(Overrides.find(code));
+ } else {
+ Overrides[code] = action;
+ }
+}
+
+EWarningAction TWarningPolicy::GetAction(TIssueCode code) const
+{
+ auto it = Overrides.find(code);
+ return (it == Overrides.end()) ? BaseAction : it->second;
+}
+
+void TWarningPolicy::Clear()
+{
+ BaseAction = EWarningAction::DEFAULT;
+ Overrides.clear();
+ Rules.clear();
+}
+
+}
diff --git a/yql/essentials/public/issue/yql_warning.h b/yql/essentials/public/issue/yql_warning.h
new file mode 100644
index 0000000000..d1d6a90922
--- /dev/null
+++ b/yql/essentials/public/issue/yql_warning.h
@@ -0,0 +1,51 @@
+#pragma once
+
+#include "yql_issue_id.h"
+
+#include <util/generic/hash.h>
+#include <util/generic/string.h>
+
+#if defined(_win_)
+#undef ERROR
+#endif
+
+namespace NYql {
+
+enum class EWarningAction {
+ DISABLE,
+ ERROR,
+ DEFAULT,
+};
+
+class TWarningRule {
+public:
+ const TString& GetPattern() const { return IssueCodePattern; }
+ EWarningAction GetAction() const { return Action; }
+
+ enum class EParseResult { PARSE_OK, PARSE_PATTERN_FAIL, PARSE_ACTION_FAIL };
+ static EParseResult ParseFrom(const TString& codePattern, const TString& action,
+ TWarningRule& result, TString& errorMessage);
+private:
+ TString IssueCodePattern;
+ EWarningAction Action = EWarningAction::DEFAULT;
+};
+
+using TWarningRules = TVector<TWarningRule>;
+
+class TWarningPolicy {
+public:
+ void AddRule(const TWarningRule& rule);
+
+ EWarningAction GetAction(TIssueCode code) const;
+
+ const TWarningRules& GetRules() const { return Rules; }
+
+ void Clear();
+
+private:
+ TWarningRules Rules;
+ EWarningAction BaseAction = EWarningAction::DEFAULT;
+ THashMap<TIssueCode, EWarningAction> Overrides;
+};
+
+}
diff --git a/yql/essentials/public/issue/yql_warning_ut.cpp b/yql/essentials/public/issue/yql_warning_ut.cpp
new file mode 100644
index 0000000000..5edf7ed363
--- /dev/null
+++ b/yql/essentials/public/issue/yql_warning_ut.cpp
@@ -0,0 +1,94 @@
+#include <library/cpp/testing/unittest/registar.h>
+
+#include "yql_warning.h"
+
+using namespace NYql;
+
+Y_UNIT_TEST_SUITE(TWarningRuleTest) {
+
+ Y_UNIT_TEST(AllValidActionsAndPatternsAreParsed) {
+ TWarningRule rule;
+ TString errorMessage;
+ UNIT_ASSERT_VALUES_EQUAL(TWarningRule::ParseFrom("1234", "disable", rule, errorMessage), TWarningRule::EParseResult::PARSE_OK);
+ UNIT_ASSERT_VALUES_EQUAL(TWarningRule::ParseFrom("*", "error", rule, errorMessage), TWarningRule::EParseResult::PARSE_OK);
+ UNIT_ASSERT_VALUES_EQUAL(TWarningRule::ParseFrom("0", "default", rule, errorMessage), TWarningRule::EParseResult::PARSE_OK);
+ }
+
+ Y_UNIT_TEST(InvalidActionIsDetected) {
+ TWarningRule rule;
+ TString errorMessage;
+ UNIT_ASSERT_VALUES_EQUAL(TWarningRule::ParseFrom("1234", "wrong", rule, errorMessage), TWarningRule::EParseResult::PARSE_ACTION_FAIL);
+ UNIT_ASSERT_VALUES_EQUAL(TWarningRule::ParseFrom("1234", "", rule, errorMessage), TWarningRule::EParseResult::PARSE_ACTION_FAIL);
+ }
+
+ Y_UNIT_TEST(InvalidPatternIsDetected) {
+ TWarningRule rule;
+ TString errorMessage;
+ UNIT_ASSERT_VALUES_EQUAL(TWarningRule::ParseFrom("", "default", rule, errorMessage), TWarningRule::EParseResult::PARSE_PATTERN_FAIL);
+ UNIT_ASSERT_VALUES_EQUAL(TWarningRule::ParseFrom("-1", "default", rule, errorMessage), TWarningRule::EParseResult::PARSE_PATTERN_FAIL);
+ UNIT_ASSERT_VALUES_EQUAL(TWarningRule::ParseFrom("*1", "default", rule, errorMessage), TWarningRule::EParseResult::PARSE_PATTERN_FAIL);
+ UNIT_ASSERT_VALUES_EQUAL(TWarningRule::ParseFrom("default", "default", rule, errorMessage), TWarningRule::EParseResult::PARSE_PATTERN_FAIL);
+ }
+}
+
+
+void AddRule(TWarningPolicy& policy, const TString& codePattern, const TString& action)
+{
+ TWarningRule rule;
+ TString errorMessage;
+ UNIT_ASSERT_VALUES_EQUAL(TWarningRule::ParseFrom(codePattern, action, rule, errorMessage), TWarningRule::EParseResult::PARSE_OK);
+ policy.AddRule(rule);
+}
+
+
+Y_UNIT_TEST_SUITE(TWarningPolicyTest) {
+
+ Y_UNIT_TEST(BasicIntegerRules) {
+ TWarningPolicy policy;
+ AddRule(policy, "123", "error");
+ AddRule(policy, "456", "error");
+ AddRule(policy, "456", "disable");
+
+ UNIT_ASSERT_VALUES_EQUAL(policy.GetRules().size(), 3);
+ UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(123), EWarningAction::ERROR);
+ UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(456), EWarningAction::DISABLE);
+ UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(111), EWarningAction::DEFAULT);
+ }
+
+
+ Y_UNIT_TEST(AsteriskRules) {
+ TWarningPolicy policy;
+ AddRule(policy, "*", "error");
+ AddRule(policy, "456", "disable");
+
+ UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(999), EWarningAction::ERROR);
+ UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(456), EWarningAction::DISABLE);
+
+ AddRule(policy, "*", "default");
+
+ UNIT_ASSERT_VALUES_EQUAL(policy.GetRules().size(), 3);
+ UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(999), EWarningAction::DEFAULT);
+ UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(456), EWarningAction::DEFAULT);
+ UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(123), EWarningAction::DEFAULT);
+
+ }
+
+ Y_UNIT_TEST(ReactionOnPull) {
+ TWarningPolicy policy;
+ AddRule(policy, "*", "error");
+ AddRule(policy, "456", "default");
+ AddRule(policy, "999", "disable");
+
+ UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(999), EWarningAction::DISABLE);
+ UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(456), EWarningAction::DEFAULT);
+ UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(123), EWarningAction::ERROR);
+
+ UNIT_ASSERT_VALUES_EQUAL(policy.GetRules().size(), 3);
+ policy.Clear();
+ UNIT_ASSERT_VALUES_EQUAL(policy.GetRules().size(), 0);
+
+ UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(999), EWarningAction::DEFAULT);
+ UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(456), EWarningAction::DEFAULT);
+ UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(123), EWarningAction::DEFAULT);
+ }
+}
diff --git a/yql/essentials/public/ya.make b/yql/essentials/public/ya.make
new file mode 100644
index 0000000000..14a283e5a9
--- /dev/null
+++ b/yql/essentials/public/ya.make
@@ -0,0 +1,4 @@
+RECURSE(
+ issue
+)
+
diff --git a/yql/essentials/ya.make b/yql/essentials/ya.make
index 48f46c7cd8..57841ee7e2 100644
--- a/yql/essentials/ya.make
+++ b/yql/essentials/ya.make
@@ -1,5 +1,6 @@
SUBSCRIBER(g:yql)
RECURSE(
+ public
)
diff --git a/yt/cpp/mapreduce/interface/logging/yt_log.cpp b/yt/cpp/mapreduce/interface/logging/yt_log.cpp
index 4ff38d4c58..d41bfa559f 100644
--- a/yt/cpp/mapreduce/interface/logging/yt_log.cpp
+++ b/yt/cpp/mapreduce/interface/logging/yt_log.cpp
@@ -26,10 +26,6 @@ public:
::TSourceLocation sourceLocation,
TStringBuf anchorMessage) override
{
- if (anchor->Registered.exchange(true)) {
- return;
- }
-
auto guard = Guard(Mutex_);
anchor->SourceLocation = sourceLocation;
anchor->AnchorMessage = anchorMessage;
diff --git a/yt/cpp/mapreduce/interface/operation-inl.h b/yt/cpp/mapreduce/interface/operation-inl.h
index 20526f5215..db900c1438 100644
--- a/yt/cpp/mapreduce/interface/operation-inl.h
+++ b/yt/cpp/mapreduce/interface/operation-inl.h
@@ -93,6 +93,7 @@ template <typename TDerived>
TDerived& TRawOperationIoTableSpec<TDerived>::SetInput(size_t tableIndex, const TRichYPath& path)
{
NDetail::Assign(Inputs_, tableIndex, path);
+ return static_cast<TDerived&>(*this);
}
template <typename TDerived>
@@ -106,6 +107,7 @@ template <typename TDerived>
TDerived& TRawOperationIoTableSpec<TDerived>::SetOutput(size_t tableIndex, const TRichYPath& path)
{
NDetail::Assign(Outputs_, tableIndex, path);
+ return static_cast<TDerived&>(*this);
}
template <typename TDerived>
@@ -133,6 +135,7 @@ template <typename TDerived>
TDerived& TRawMapReduceOperationIoSpec<TDerived>::SetMapOutput(size_t tableIndex, const TRichYPath& path)
{
NDetail::Assign(MapOutputs_, tableIndex, path);
+ return static_cast<TDerived&>(*this);
}
template <typename TDerived>
diff --git a/yt/yt/client/api/delegating_client.h b/yt/yt/client/api/delegating_client.h
index 6f85657139..9cb73f3fb1 100644
--- a/yt/yt/client/api/delegating_client.h
+++ b/yt/yt/client/api/delegating_client.h
@@ -881,7 +881,7 @@ public:
DELEGATE_METHOD(TFuture<IRowBatchWriterPtr>, CreateShuffleWriter, (
const TShuffleHandlePtr& shuffleHandle,
- const TString& partitionColumn,
+ const std::string& partitionColumn,
const NTableClient::TTableWriterConfigPtr& config),
(shuffleHandle, partitionColumn, config))
diff --git a/yt/yt/client/api/delegating_transaction.cpp b/yt/yt/client/api/delegating_transaction.cpp
index 156eb1dde7..8a8ddf96dd 100644
--- a/yt/yt/client/api/delegating_transaction.cpp
+++ b/yt/yt/client/api/delegating_transaction.cpp
@@ -264,7 +264,7 @@ DELEGATE_METHOD(void, LockRows, (
const NYPath::TYPath& path,
NTableClient::TNameTablePtr nameTable,
TSharedRange<NTableClient::TLegacyKey> keys,
- const std::vector<TString>& locks,
+ const std::vector<std::string>& locks,
NTableClient::ELockType lockType),
(path, nameTable, keys, locks, lockType))
diff --git a/yt/yt/client/api/delegating_transaction.h b/yt/yt/client/api/delegating_transaction.h
index e0a71dee00..e0263fc799 100644
--- a/yt/yt/client/api/delegating_transaction.h
+++ b/yt/yt/client/api/delegating_transaction.h
@@ -214,7 +214,7 @@ public:
const NYPath::TYPath& path,
NTableClient::TNameTablePtr nameTable,
TSharedRange<NTableClient::TLegacyKey> keys,
- const std::vector<TString>& locks,
+ const std::vector<std::string>& locks,
NTableClient::ELockType lockType) override;
void ModifyRows(
diff --git a/yt/yt/client/api/dynamic_table_transaction.h b/yt/yt/client/api/dynamic_table_transaction.h
index b22e9bcd8f..37db074bdf 100644
--- a/yt/yt/client/api/dynamic_table_transaction.h
+++ b/yt/yt/client/api/dynamic_table_transaction.h
@@ -106,7 +106,7 @@ struct IDynamicTableTransaction
const NYPath::TYPath& path,
NTableClient::TNameTablePtr nameTable,
TSharedRange<NTableClient::TLegacyKey> keys,
- const std::vector<TString>& locks,
+ const std::vector<std::string>& locks,
NTableClient::ELockType lockType = NTableClient::ELockType::SharedStrong) = 0;
virtual void ModifyRows(
diff --git a/yt/yt/client/api/dynamic_table_transaction_mixin.cpp b/yt/yt/client/api/dynamic_table_transaction_mixin.cpp
index 240e4ea548..4f1d2c901f 100644
--- a/yt/yt/client/api/dynamic_table_transaction_mixin.cpp
+++ b/yt/yt/client/api/dynamic_table_transaction_mixin.cpp
@@ -152,7 +152,7 @@ void TDynamicTableTransactionMixin::LockRows(
const NYPath::TYPath& path,
TNameTablePtr nameTable,
TSharedRange<TLegacyKey> keys,
- const std::vector<TString>& locks,
+ const std::vector<std::string>& locks,
ELockType lockType)
{
const auto& tableMountCache = GetClient()->GetTableMountCache();
diff --git a/yt/yt/client/api/dynamic_table_transaction_mixin.h b/yt/yt/client/api/dynamic_table_transaction_mixin.h
index c8492f1bf6..ab94daaa77 100644
--- a/yt/yt/client/api/dynamic_table_transaction_mixin.h
+++ b/yt/yt/client/api/dynamic_table_transaction_mixin.h
@@ -43,7 +43,7 @@ public:
const NYPath::TYPath& path,
NTableClient::TNameTablePtr nameTable,
TSharedRange<NTableClient::TLegacyKey> keys,
- const std::vector<TString>& locks,
+ const std::vector<std::string>& locks,
NTableClient::ELockType lockType = NTableClient::ELockType::SharedStrong) override;
};
diff --git a/yt/yt/client/api/query_tracker_client.h b/yt/yt/client/api/query_tracker_client.h
index 4e198ed4b3..0e0fc147bf 100644
--- a/yt/yt/client/api/query_tracker_client.h
+++ b/yt/yt/client/api/query_tracker_client.h
@@ -62,7 +62,7 @@ struct TReadQueryResultOptions
: public TTimeoutOptions
, public TQueryTrackerOptions
{
- std::optional<std::vector<TString>> Columns;
+ std::optional<std::vector<std::string>> Columns;
std::optional<i64> LowerRowIndex;
std::optional<i64> UpperRowIndex;
};
diff --git a/yt/yt/client/api/rpc_proxy/client_impl.cpp b/yt/yt/client/api/rpc_proxy/client_impl.cpp
index cc9694ca70..7575886a68 100644
--- a/yt/yt/client/api/rpc_proxy/client_impl.cpp
+++ b/yt/yt/client/api/rpc_proxy/client_impl.cpp
@@ -10,47 +10,34 @@
#include "timestamp_provider.h"
#include "transaction.h"
-#include <yt/yt/client/api/distributed_table_session.h>
#include <yt/yt/client/api/helpers.h>
-#include <yt/yt/client/api/rowset.h>
-#include <yt/yt/client/api/transaction.h>
#include <yt/yt/client/chaos_client/replication_card_serialization.h>
-#include <yt/yt/client/transaction_client/remote_timestamp_provider.h>
-
#include <yt/yt/client/scheduler/operation_id_or_alias.h>
#include <yt/yt/client/table_client/columnar_statistics.h>
-#include <yt/yt/client/table_client/name_table.h>
-#include <yt/yt/client/table_client/row_base.h>
-#include <yt/yt/client/table_client/row_buffer.h>
#include <yt/yt/client/table_client/schema.h>
#include <yt/yt/client/table_client/unversioned_row.h>
#include <yt/yt/client/table_client/wire_protocol.h>
-#include <yt/yt/client/tablet_client/table_mount_cache.h>
+#include <yt/yt/client/api/distributed_table_session.h>
#include <yt/yt/client/ypath/rich.h>
#include <yt/yt/library/auth/credentials_injecting_channel.h>
-#include <yt/yt/core/net/address.h>
-
-#include <yt/yt/core/rpc/dynamic_channel_pool.h>
#include <yt/yt/core/rpc/retrying_channel.h>
#include <yt/yt/core/rpc/stream.h>
#include <yt/yt/core/ytree/convert.h>
-#include <util/generic/cast.h>
-
namespace NYT::NApi::NRpcProxy {
////////////////////////////////////////////////////////////////////////////////
-using NYT::ToProto;
using NYT::FromProto;
+using NYT::ToProto;
using namespace NAuth;
using namespace NChaosClient;
@@ -2323,7 +2310,7 @@ TFuture<IUnversionedRowsetPtr> TClient::ReadQueryResult(
if (options.Columns) {
auto* protoColumns = req->mutable_columns();
for (const auto& column : *options.Columns) {
- protoColumns->add_items(column);
+ protoColumns->add_items(ToProto(column));
}
}
if (options.LowerRowIndex) {
@@ -2733,7 +2720,7 @@ TFuture<IRowBatchReaderPtr> TClient::CreateShuffleReader(
TFuture<IRowBatchWriterPtr> TClient::CreateShuffleWriter(
const TShuffleHandlePtr& shuffleHandle,
- const TString& partitionColumn,
+ const std::string& partitionColumn,
const TTableWriterConfigPtr& config)
{
auto proxy = CreateApiServiceProxy();
@@ -2741,7 +2728,7 @@ TFuture<IRowBatchWriterPtr> TClient::CreateShuffleWriter(
InitStreamingRequest(*req);
req->set_shuffle_handle(ConvertToYsonString(shuffleHandle).ToString());
- req->set_partition_column(partitionColumn);
+ req->set_partition_column(ToProto(partitionColumn));
req->set_writer_config(ConvertToYsonString(config).ToString());
return CreateRpcClientOutputStream(std::move(req))
diff --git a/yt/yt/client/api/rpc_proxy/client_impl.h b/yt/yt/client/api/rpc_proxy/client_impl.h
index 16e9400391..3e73bc47f0 100644
--- a/yt/yt/client/api/rpc_proxy/client_impl.h
+++ b/yt/yt/client/api/rpc_proxy/client_impl.h
@@ -585,7 +585,7 @@ public:
TFuture<IRowBatchWriterPtr> CreateShuffleWriter(
const TShuffleHandlePtr& shuffleHandle,
- const TString& partitionColumn,
+ const std::string& partitionColumn,
const NTableClient::TTableWriterConfigPtr& config) override;
private:
diff --git a/yt/yt/client/api/rpc_proxy/helpers.cpp b/yt/yt/client/api/rpc_proxy/helpers.cpp
index 13c51ed862..d37fc8b2bd 100644
--- a/yt/yt/client/api/rpc_proxy/helpers.cpp
+++ b/yt/yt/client/api/rpc_proxy/helpers.cpp
@@ -468,13 +468,13 @@ void FromProto(
void ToProto(NProto::TColumnSchema* protoSchema, const NTableClient::TColumnSchema& schema)
{
- protoSchema->set_stable_name(schema.StableName().Underlying());
- protoSchema->set_name(schema.Name());
+ protoSchema->set_stable_name(ToProto(schema.StableName()));
+ protoSchema->set_name(ToProto(schema.Name()));
protoSchema->set_type(ToProto(GetPhysicalType(schema.CastToV1Type())));
auto typeV3Yson = ConvertToYsonString(TTypeV3LogicalTypeWrapper{schema.LogicalType()});
protoSchema->set_type_v3(typeV3Yson.ToString());
if (schema.Lock()) {
- protoSchema->set_lock(*schema.Lock());
+ protoSchema->set_lock(ToProto(*schema.Lock()));
} else {
protoSchema->clear_lock();
}
@@ -489,7 +489,7 @@ void ToProto(NProto::TColumnSchema* protoSchema, const NTableClient::TColumnSche
protoSchema->clear_materialized();
}
if (schema.Aggregate()) {
- protoSchema->set_aggregate(*schema.Aggregate());
+ protoSchema->set_aggregate(ToProto(*schema.Aggregate()));
} else {
protoSchema->clear_aggregate();
}
@@ -499,7 +499,7 @@ void ToProto(NProto::TColumnSchema* protoSchema, const NTableClient::TColumnSche
protoSchema->clear_sort_order();
}
if (schema.Group()) {
- protoSchema->set_group(*schema.Group());
+ protoSchema->set_group(ToProto(*schema.Group()));
} else {
protoSchema->clear_group();
}
@@ -2158,7 +2158,7 @@ std::vector<TSharedRef> SerializeRowset(
// COMPAT(babenko)
for (const auto& column : schema.Columns()) {
auto* entry = descriptor->add_name_table_entries();
- entry->set_name(column.Name());
+ entry->set_name(ToProto(column.Name()));
// we save physical type for backward compatibility
// COMPAT(babenko)
entry->set_type(ToProto(column.GetWireType()));
diff --git a/yt/yt/client/api/rpc_proxy/table_reader.cpp b/yt/yt/client/api/rpc_proxy/table_reader.cpp
index bb1f571332..6df6bef241 100644
--- a/yt/yt/client/api/rpc_proxy/table_reader.cpp
+++ b/yt/yt/client/api/rpc_proxy/table_reader.cpp
@@ -22,7 +22,7 @@ public:
TTableReader(
IAsyncZeroCopyInputStreamPtr underlying,
i64 startRowIndex,
- const std::vector<TString>& omittedInaccessibleColumns,
+ const std::vector<std::string>& omittedInaccessibleColumns,
TTableSchemaPtr schema,
const NProto::TRowsetStatistics& statistics)
: TRowBatchReader(std::move(underlying), /*isStreamWithStatistics*/ true)
@@ -57,7 +57,7 @@ public:
return TableSchema_;
}
- const std::vector<TString>& GetOmittedInaccessibleColumns() const override
+ const std::vector<std::string>& GetOmittedInaccessibleColumns() const override
{
return OmittedInaccessibleColumns_;
}
@@ -65,7 +65,7 @@ public:
private:
const i64 StartRowIndex_;
const TTableSchemaPtr TableSchema_;
- const std::vector<TString> OmittedInaccessibleColumns_;
+ const std::vector<std::string> OmittedInaccessibleColumns_;
NChunkClient::NProto::TDataStatistics DataStatistics_;
i64 TotalRowCount_;
@@ -85,15 +85,11 @@ TFuture<ITableReaderPtr> CreateTableReader(IAsyncZeroCopyInputStreamPtr inputStr
THROW_ERROR_EXCEPTION("Failed to deserialize table reader meta information");
}
- i64 startRowIndex = meta.start_row_index();
- auto omittedInaccessibleColumns = FromProto<std::vector<TString>>(
- meta.omitted_inaccessible_columns());
- auto schema = NYT::FromProto<TTableSchemaPtr>(meta.schema());
return New<TTableReader>(
inputStream,
- startRowIndex,
- std::move(omittedInaccessibleColumns),
- std::move(schema),
+ meta.start_row_index(),
+ FromProto<std::vector<std::string>>(meta.omitted_inaccessible_columns()),
+ NYT::FromProto<TTableSchemaPtr>(meta.schema()),
meta.statistics());
})).As<ITableReaderPtr>();
}
diff --git a/yt/yt/client/api/security_client.cpp b/yt/yt/client/api/security_client.cpp
index 2b0d0f675c..f30a288614 100644
--- a/yt/yt/client/api/security_client.cpp
+++ b/yt/yt/client/api/security_client.cpp
@@ -9,7 +9,7 @@ using namespace NYTree;
TError TCheckPermissionResult::ToError(
const std::string& user,
EPermission permission,
- const std::optional<TString>& column) const
+ const std::optional<std::string>& column) const
{
switch (Action) {
case NSecurityClient::ESecurityAction::Allow:
diff --git a/yt/yt/client/api/security_client.h b/yt/yt/client/api/security_client.h
index a606229e49..cb639d678b 100644
--- a/yt/yt/client/api/security_client.h
+++ b/yt/yt/client/api/security_client.h
@@ -26,7 +26,7 @@ struct TCheckPermissionOptions
, public TTransactionalOptions
, public TPrerequisiteOptions
{
- std::optional<std::vector<TString>> Columns;
+ std::optional<std::vector<std::string>> Columns;
std::optional<bool> Vital;
};
@@ -35,7 +35,7 @@ struct TCheckPermissionResult
TError ToError(
const std::string& user,
NYTree::EPermission permission,
- const std::optional<TString>& columns = {}) const;
+ const std::optional<std::string>& columns = {}) const;
NSecurityClient::ESecurityAction Action;
NObjectClient::TObjectId ObjectId;
diff --git a/yt/yt/client/api/shuffle_client.h b/yt/yt/client/api/shuffle_client.h
index d7972cd680..650096a30c 100644
--- a/yt/yt/client/api/shuffle_client.h
+++ b/yt/yt/client/api/shuffle_client.h
@@ -53,7 +53,7 @@ struct IShuffleClient
virtual TFuture<IRowBatchWriterPtr> CreateShuffleWriter(
const TShuffleHandlePtr& shuffleHandle,
- const TString& partitionColumn,
+ const std::string& partitionColumn,
const NTableClient::TTableWriterConfigPtr& config = New<NTableClient::TTableWriterConfig>()) = 0;
};
diff --git a/yt/yt/client/api/table_reader.h b/yt/yt/client/api/table_reader.h
index b26e970657..8a447ea5ed 100644
--- a/yt/yt/client/api/table_reader.h
+++ b/yt/yt/client/api/table_reader.h
@@ -26,7 +26,7 @@ struct ITableReader
//! Returns the names of columns that are not accessible according to columnar ACL
//! and were omitted. See #TTableReaderOptions::OmitInaccessibleColumns.
- virtual const std::vector<TString>& GetOmittedInaccessibleColumns() const = 0;
+ virtual const std::vector<std::string>& GetOmittedInaccessibleColumns() const = 0;
};
DEFINE_REFCOUNTED_TYPE(ITableReader)
diff --git a/yt/yt/client/arrow/arrow_row_stream_encoder.cpp b/yt/yt/client/arrow/arrow_row_stream_encoder.cpp
index 23a06a45b7..25a403790c 100644
--- a/yt/yt/client/arrow/arrow_row_stream_encoder.cpp
+++ b/yt/yt/client/arrow/arrow_row_stream_encoder.cpp
@@ -39,7 +39,7 @@ constexpr i64 ArrowAlignment = 8;
flatbuffers::Offset<flatbuffers::String> SerializeString(
flatbuffers::FlatBufferBuilder* flatbufBuilder,
- const TString& str)
+ const std::string& str)
{
return flatbufBuilder->CreateString(str.data(), str.length());
}
diff --git a/yt/yt/client/driver/driver.cpp b/yt/yt/client/driver/driver.cpp
index 3e57e388df..4de907381d 100644
--- a/yt/yt/client/driver/driver.cpp
+++ b/yt/yt/client/driver/driver.cpp
@@ -17,25 +17,20 @@
#include "query_commands.h"
#include "queue_commands.h"
#include "scheduler_commands.h"
+#include "shuffle_commands.h"
#include "table_commands.h"
#include "transaction_commands.h"
-#include "shuffle_commands.h"
+
+#include <yt/yt/library/formats/format.h>
#include <yt/yt/client/api/client_cache.h>
#include <yt/yt/client/api/connection.h>
#include <yt/yt/client/api/sticky_transaction_pool.h>
-#include <yt/yt/client/api/transaction.h>
-
-#include <yt/yt/client/api/rpc_proxy/connection_impl.h>
-
-#include <yt/yt/client/node_tracker_client/node_directory.h>
#include <yt/yt/core/yson/null_consumer.h>
#include <yt/yt/core/tracing/trace_context.h>
-#include <yt/yt/library/formats/format.h>
-
#include <yt/yt/library/tvm/tvm_base.h>
diff --git a/yt/yt/client/driver/etc_commands.cpp b/yt/yt/client/driver/etc_commands.cpp
index 25da308a57..288d42610f 100644
--- a/yt/yt/client/driver/etc_commands.cpp
+++ b/yt/yt/client/driver/etc_commands.cpp
@@ -109,7 +109,7 @@ void TCheckPermissionCommand::Register(TRegistrar registrar)
registrar.Parameter("user", &TThis::User);
registrar.Parameter("permission", &TThis::Permission);
registrar.Parameter("path", &TThis::Path);
- registrar.ParameterWithUniversalAccessor<std::optional<std::vector<TString>>>(
+ registrar.ParameterWithUniversalAccessor<std::optional<std::vector<std::string>>>(
"columns",
[] (TThis* command) -> auto& {
return command->Options.Columns;
diff --git a/yt/yt/client/driver/query_commands.cpp b/yt/yt/client/driver/query_commands.cpp
index e3903d004b..ab2d9395e1 100644
--- a/yt/yt/client/driver/query_commands.cpp
+++ b/yt/yt/client/driver/query_commands.cpp
@@ -153,7 +153,7 @@ void TReadQueryResultCommand::Register(TRegistrar registrar)
})
.Optional(/*init*/ false);
- registrar.ParameterWithUniversalAccessor<std::optional<std::vector<TString>>>(
+ registrar.ParameterWithUniversalAccessor<std::optional<std::vector<std::string>>>(
"columns",
[] (TThis* command) -> auto& {
return command->Options.Columns;
diff --git a/yt/yt/client/driver/shuffle_commands.cpp b/yt/yt/client/driver/shuffle_commands.cpp
index 4e2ee805e8..ae8e5adbcd 100644
--- a/yt/yt/client/driver/shuffle_commands.cpp
+++ b/yt/yt/client/driver/shuffle_commands.cpp
@@ -1,13 +1,15 @@
#include "shuffle_commands.h"
+#include <yt/yt/client/driver/config.h>
+
+#include <yt/yt/library/formats/format.h>
+
+#include <yt/yt/client/formats/config.h>
+
#include <yt/yt/client/table_client/adapters.h>
#include <yt/yt/client/table_client/table_output.h>
#include <yt/yt/client/table_client/value_consumer.h>
-#include <yt/yt/client/driver/config.h>
-#include <yt/yt/client/formats/config.h>
-
-#include <yt/yt/library/formats/format.h>
namespace NYT::NDriver {
diff --git a/yt/yt/client/driver/shuffle_commands.h b/yt/yt/client/driver/shuffle_commands.h
index 7b4394c49a..1eebb8304a 100644
--- a/yt/yt/client/driver/shuffle_commands.h
+++ b/yt/yt/client/driver/shuffle_commands.h
@@ -2,8 +2,6 @@
#include "command.h"
-#include <yt/yt/client/ypath/rich.h>
-
namespace NYT::NDriver {
////////////////////////////////////////////////////////////////////////////
diff --git a/yt/yt/client/driver/table_commands.h b/yt/yt/client/driver/table_commands.h
index 2e1e8e22d5..fde7e0b66a 100644
--- a/yt/yt/client/driver/table_commands.h
+++ b/yt/yt/client/driver/table_commands.h
@@ -46,8 +46,8 @@ private:
NYPath::TRichYPath Path;
NYTree::INodePtr TableReader;
- std::optional<TString> PartIndexColumnName;
- std::optional<TString> DataColumnName;
+ std::optional<std::string> PartIndexColumnName;
+ std::optional<std::string> DataColumnName;
i64 StartPartIndex;
i64 Offset;
@@ -375,7 +375,7 @@ public:
private:
NYTree::INodePtr TableWriter;
NYPath::TRichYPath Path;
- std::optional<std::vector<TString>> ColumnNames;
+ std::optional<std::vector<std::string>> ColumnNames;
bool Versioned;
NTableClient::TRetentionConfigPtr RetentionConfig;
@@ -451,7 +451,7 @@ public:
private:
NYTree::INodePtr TableWriter;
NYPath::TRichYPath Path;
- std::vector<TString> Locks;
+ std::vector<std::string> Locks;
NTableClient::ELockType LockType;
void DoExecute(ICommandContextPtr context) override;
diff --git a/yt/yt/client/federated/client.cpp b/yt/yt/client/federated/client.cpp
index f43f76958d..040c19a17c 100644
--- a/yt/yt/client/federated/client.cpp
+++ b/yt/yt/client/federated/client.cpp
@@ -490,7 +490,7 @@ public:
UNIMPLEMENTED_METHOD(TFuture<TShuffleHandlePtr>, StartShuffle, (const TString& , int, const TStartShuffleOptions&));
UNIMPLEMENTED_METHOD(TFuture<void>, FinishShuffle, (const TShuffleHandlePtr&, const TFinishShuffleOptions&));
UNIMPLEMENTED_METHOD(TFuture<IRowBatchReaderPtr>, CreateShuffleReader, (const TShuffleHandlePtr&, int, const NTableClient::TTableReaderConfigPtr&));
- UNIMPLEMENTED_METHOD(TFuture<IRowBatchWriterPtr>, CreateShuffleWriter, (const TShuffleHandlePtr&, const TString&, const NTableClient::TTableWriterConfigPtr&));
+ UNIMPLEMENTED_METHOD(TFuture<IRowBatchWriterPtr>, CreateShuffleWriter, (const TShuffleHandlePtr&, const std::string&, const NTableClient::TTableWriterConfigPtr&));
private:
friend class TTransaction;
diff --git a/yt/yt/client/federated/unittests/client_ut.cpp b/yt/yt/client/federated/unittests/client_ut.cpp
index 54ec11d6cb..daf7529e78 100644
--- a/yt/yt/client/federated/unittests/client_ut.cpp
+++ b/yt/yt/client/federated/unittests/client_ut.cpp
@@ -78,8 +78,8 @@ struct TTestDataStorage
}
const NYPath::TYPath Path = "/test/table";
- const TString KeyColumn = "key";
- const TString ValueColumn = "value";
+ const std::string KeyColumn = "key";
+ const std::string ValueColumn = "value";
const NTableClient::TColumnSchema KeyColumnSchema = NTableClient::TColumnSchema(KeyColumn, NTableClient::EValueType::Uint64);
const NTableClient::TColumnSchema ValueColumnSchema = NTableClient::TColumnSchema(ValueColumn, NTableClient::EValueType::Uint64);
diff --git a/yt/yt/client/formats/config.cpp b/yt/yt/client/formats/config.cpp
index f6c6a8c8a6..71c1152061 100644
--- a/yt/yt/client/formats/config.cpp
+++ b/yt/yt/client/formats/config.cpp
@@ -131,7 +131,7 @@ void TYamredDsvFormatConfig::Register(TRegistrar registrar)
});
registrar.Postprocessor([] (TThis* config) {
- THashSet<TString> names;
+ THashSet<std::string> names;
for (const auto& name : config->KeyColumnNames) {
if (!names.insert(name).second) {
@@ -151,7 +151,7 @@ void TYamredDsvFormatConfig::Register(TRegistrar registrar)
////////////////////////////////////////////////////////////////////////////////
-const std::vector<TString>& TSchemafulDsvFormatConfig::GetColumnsOrThrow() const
+const std::vector<std::string>& TSchemafulDsvFormatConfig::GetColumnsOrThrow() const
{
if (!Columns) {
THROW_ERROR_EXCEPTION("Missing \"columns\" attribute in schemaful DSV format");
@@ -188,7 +188,7 @@ void TSchemafulDsvFormatConfig::Register(TRegistrar registrar)
registrar.Postprocessor([] (TThis* config) {
if (config->Columns) {
- THashSet<TString> names;
+ THashSet<std::string> names;
for (const auto& name : *config->Columns) {
if (!names.insert(name).second) {
THROW_ERROR_EXCEPTION("Duplicate column name %Qv in schemaful DSV configuration",
diff --git a/yt/yt/client/formats/config.h b/yt/yt/client/formats/config.h
index a371050ae6..aec3f30db4 100644
--- a/yt/yt/client/formats/config.h
+++ b/yt/yt/client/formats/config.h
@@ -149,8 +149,7 @@ class TDsvFormatConfig
: public TDsvFormatConfigBase
{
public:
-
- TString TableIndexColumn;
+ std::string TableIndexColumn;
bool SkipUnsupportedTypes = false;
REGISTER_YSON_STRUCT(TDsvFormatConfig);
@@ -169,8 +168,8 @@ class TYamredDsvFormatConfig
public:
char YamrKeysSeparator;
- std::vector<TString> KeyColumnNames;
- std::vector<TString> SubkeyColumnNames;
+ std::vector<std::string> KeyColumnNames;
+ std::vector<std::string> SubkeyColumnNames;
bool SkipUnsupportedTypesInValue = false;
@@ -193,14 +192,14 @@ class TSchemafulDsvFormatConfig
: public TTableFormatConfigBase
{
public:
- std::optional<std::vector<TString>> Columns;
+ std::optional<std::vector<std::string>> Columns;
EMissingSchemafulDsvValueMode MissingValueMode;
TString MissingValueSentinel;
std::optional<bool> EnableColumnNamesHeader;
- const std::vector<TString>& GetColumnsOrThrow() const;
+ const std::vector<std::string>& GetColumnsOrThrow() const;
REGISTER_YSON_STRUCT(TSchemafulDsvFormatConfig);
@@ -382,7 +381,7 @@ public:
int FieldWeightLimit;
int StringWeightLimit;
int MaxAllColumnNamesCount;
- std::optional<std::vector<TString>> ColumnNames;
+ std::optional<std::vector<std::string>> ColumnNames;
EWebJsonValueFormat ValueFormat;
// Intentionally do not reveal following options to user.
diff --git a/yt/yt/client/hedging/hedging.cpp b/yt/yt/client/hedging/hedging.cpp
index 3fb36f8560..b083f87ff3 100644
--- a/yt/yt/client/hedging/hedging.cpp
+++ b/yt/yt/client/hedging/hedging.cpp
@@ -231,7 +231,7 @@ public:
UNSUPPORTED_METHOD(TFuture<TShuffleHandlePtr>, StartShuffle, (const TString&, int, const TStartShuffleOptions&));
UNSUPPORTED_METHOD(TFuture<void>, FinishShuffle, (const TShuffleHandlePtr&, const TFinishShuffleOptions&));
UNSUPPORTED_METHOD(TFuture<IRowBatchReaderPtr>, CreateShuffleReader, (const TShuffleHandlePtr&, int, const NTableClient::TTableReaderConfigPtr&));
- UNSUPPORTED_METHOD(TFuture<IRowBatchWriterPtr>, CreateShuffleWriter, (const TShuffleHandlePtr&, const TString&, const NTableClient::TTableWriterConfigPtr&));
+ UNSUPPORTED_METHOD(TFuture<IRowBatchWriterPtr>, CreateShuffleWriter, (const TShuffleHandlePtr&, const std::string&, const NTableClient::TTableWriterConfigPtr&));
private:
const THedgingExecutorPtr Executor_;
diff --git a/yt/yt/client/table_client/blob_reader.cpp b/yt/yt/client/table_client/blob_reader.cpp
index e354c9008b..fb4fc93a17 100644
--- a/yt/yt/client/table_client/blob_reader.cpp
+++ b/yt/yt/client/table_client/blob_reader.cpp
@@ -15,8 +15,8 @@ using namespace NConcurrency;
////////////////////////////////////////////////////////////////////////////////
-const TString TBlobTableSchema::PartIndexColumn = "part_index";
-const TString TBlobTableSchema::DataColumn = "data";
+const std::string TBlobTableSchema::PartIndexColumn = "part_index";
+const std::string TBlobTableSchema::DataColumn = "data";
TTableSchemaPtr TBlobTableSchema::ToTableSchema() const
{
@@ -46,8 +46,8 @@ class TBlobTableReader
public:
TBlobTableReader(
ITableReaderPtr reader,
- const std::optional<TString>& partIndexColumnName,
- const std::optional<TString>& dataColumnName,
+ const std::optional<std::string>& partIndexColumnName,
+ const std::optional<std::string>& dataColumnName,
i64 startPartIndex,
const std::optional<i64>& offset,
const std::optional<i64>& partSize)
@@ -87,8 +87,8 @@ public:
private:
const ITableReaderPtr Reader_;
- const TString PartIndexColumnName_;
- const TString DataColumnName_;
+ const std::string PartIndexColumnName_;
+ const std::string DataColumnName_;
i64 Offset_;
std::optional<i64> PartSize_;
@@ -122,7 +122,7 @@ private:
TUnversionedValue GetAndValidateValue(
TUnversionedRow row,
- const TString& name,
+ const std::string& name,
EColumnType columnType,
EValueType expectedType)
{
@@ -197,8 +197,8 @@ private:
IAsyncZeroCopyInputStreamPtr CreateBlobTableReader(
ITableReaderPtr reader,
- const std::optional<TString>& partIndexColumnName,
- const std::optional<TString>& dataColumnName,
+ const std::optional<std::string>& partIndexColumnName,
+ const std::optional<std::string>& dataColumnName,
i64 startPartIndex,
const std::optional<i64>& offset,
const std::optional<i64>& partSize)
diff --git a/yt/yt/client/table_client/blob_reader.h b/yt/yt/client/table_client/blob_reader.h
index 71f8810341..0bfe4579a9 100644
--- a/yt/yt/client/table_client/blob_reader.h
+++ b/yt/yt/client/table_client/blob_reader.h
@@ -11,8 +11,8 @@ namespace NYT::NTableClient {
struct TBlobTableSchema
{
// Names of special blob columns.
- static const TString PartIndexColumn;
- static const TString DataColumn;
+ static const std::string PartIndexColumn;
+ static const std::string DataColumn;
// Do not specify anything except name and value
// type in all column schemas.
@@ -25,8 +25,8 @@ struct TBlobTableSchema
NConcurrency::IAsyncZeroCopyInputStreamPtr CreateBlobTableReader(
NApi::ITableReaderPtr reader,
- const std::optional<TString>& partIndexColumnName,
- const std::optional<TString>& dataColumnName,
+ const std::optional<std::string>& partIndexColumnName,
+ const std::optional<std::string>& dataColumnName,
i64 startPartIndex,
const std::optional<i64>& offset = std::nullopt,
const std::optional<i64>& partSize = std::nullopt);
diff --git a/yt/yt/client/table_client/column_sort_schema.cpp b/yt/yt/client/table_client/column_sort_schema.cpp
index 964f7549a4..e574bd6cb2 100644
--- a/yt/yt/client/table_client/column_sort_schema.cpp
+++ b/yt/yt/client/table_client/column_sort_schema.cpp
@@ -62,7 +62,7 @@ void ValidateSortColumns(const std::vector<TColumnSortSchema>& columns)
{
ValidateKeyColumnCount(columns.size());
- THashSet<TString> names;
+ THashSet<std::string> names;
for (const auto& column : columns) {
if (!names.insert(column.Name).second) {
THROW_ERROR_EXCEPTION("Duplicate sort column name %Qv",
@@ -77,9 +77,11 @@ void ToProto(
NProto::TSortColumnsExt* protoSortColumns,
const TSortColumns& sortColumns)
{
+ using NYT::ToProto;
+
for (const auto& sortColumn : sortColumns) {
- protoSortColumns->add_names(sortColumn.Name);
- protoSortColumns->add_sort_orders(static_cast<int>(sortColumn.SortOrder));
+ protoSortColumns->add_names(ToProto(sortColumn.Name));
+ protoSortColumns->add_sort_orders(ToProto<int>(sortColumn.SortOrder));
}
}
diff --git a/yt/yt/client/table_client/column_sort_schema.h b/yt/yt/client/table_client/column_sort_schema.h
index 70a8b62596..21579e98df 100644
--- a/yt/yt/client/table_client/column_sort_schema.h
+++ b/yt/yt/client/table_client/column_sort_schema.h
@@ -9,7 +9,7 @@ namespace NYT::NTableClient {
struct TColumnSortSchema
{
- TString Name;
+ std::string Name;
ESortOrder SortOrder;
bool operator==(const TColumnSortSchema& other) const = default;
diff --git a/yt/yt/client/table_client/columnar_statistics.cpp b/yt/yt/client/table_client/columnar_statistics.cpp
index 85a223a8a4..c53acdfbf5 100644
--- a/yt/yt/client/table_client/columnar_statistics.cpp
+++ b/yt/yt/client/table_client/columnar_statistics.cpp
@@ -265,7 +265,7 @@ TLightweightColumnarStatistics TColumnarStatistics::MakeLightweightStatistics()
};
}
-TNamedColumnarStatistics TColumnarStatistics::MakeNamedStatistics(const std::vector<TString>& names) const
+TNamedColumnarStatistics TColumnarStatistics::MakeNamedStatistics(const std::vector<std::string>& names) const
{
TNamedColumnarStatistics result;
result.TimestampTotalWeight = TimestampTotalWeight;
diff --git a/yt/yt/client/table_client/columnar_statistics.h b/yt/yt/client/table_client/columnar_statistics.h
index 82dbec47c0..0ead116aff 100644
--- a/yt/yt/client/table_client/columnar_statistics.h
+++ b/yt/yt/client/table_client/columnar_statistics.h
@@ -21,7 +21,7 @@ struct TLightweightColumnarStatistics
struct TNamedColumnarStatistics
{
//! Per-column total data weight for chunks whose meta contains columnar statistics.
- THashMap<TString, i64> ColumnDataWeights;
+ THashMap<std::string, i64> ColumnDataWeights;
//! Total weight of all write and delete timestamps.
std::optional<i64> TimestampTotalWeight;
//! Total data weight of legacy chunks whose meta misses columnar statistics.
@@ -97,7 +97,7 @@ struct TColumnarStatistics
TLightweightColumnarStatistics MakeLightweightStatistics() const;
- TNamedColumnarStatistics MakeNamedStatistics(const std::vector<TString>& names) const;
+ TNamedColumnarStatistics MakeNamedStatistics(const std::vector<std::string>& names) const;
//! Checks if there are minimum, maximum, and non-null value statistics.
bool HasValueStatistics() const;
diff --git a/yt/yt/client/table_client/logical_type.cpp b/yt/yt/client/table_client/logical_type.cpp
index fda90c21ec..679540a999 100644
--- a/yt/yt/client/table_client/logical_type.cpp
+++ b/yt/yt/client/table_client/logical_type.cpp
@@ -467,19 +467,19 @@ TComplexTypeFieldDescriptor::TComplexTypeFieldDescriptor(const NYT::NTableClient
: TComplexTypeFieldDescriptor(column.Name(), column.LogicalType())
{ }
-TComplexTypeFieldDescriptor::TComplexTypeFieldDescriptor(TString columnName, TLogicalTypePtr type)
- : Descriptor_(std::move(columnName))
+TComplexTypeFieldDescriptor::TComplexTypeFieldDescriptor(const std::string& columnName, TLogicalTypePtr type)
+ : Description_(columnName)
, Type_(std::move(type))
{ }
TComplexTypeFieldDescriptor TComplexTypeFieldDescriptor::OptionalElement() const
{
- return TComplexTypeFieldDescriptor(Descriptor_ + ".<optional-element>", Type_->AsOptionalTypeRef().GetElement());
+ return TComplexTypeFieldDescriptor(Description_ + ".<optional-element>", Type_->AsOptionalTypeRef().GetElement());
}
TComplexTypeFieldDescriptor TComplexTypeFieldDescriptor::ListElement() const
{
- return TComplexTypeFieldDescriptor(Descriptor_ + ".<list-element>", Type_->AsListTypeRef().GetElement());
+ return TComplexTypeFieldDescriptor(Description_ + ".<list-element>", Type_->AsListTypeRef().GetElement());
}
TComplexTypeFieldDescriptor TComplexTypeFieldDescriptor::Field(size_t i) const
@@ -499,7 +499,7 @@ TComplexTypeFieldDescriptor TComplexTypeFieldDescriptor::StructField(size_t i) c
const auto& fields = Type_->AsStructTypeRef().GetFields();
YT_VERIFY(i < fields.size());
const auto& field = fields[i];
- return TComplexTypeFieldDescriptor(Descriptor_ + "." + field.Name, field.Type);
+ return TComplexTypeFieldDescriptor(Description_ + "." + field.Name, field.Type);
}
TComplexTypeFieldDescriptor TComplexTypeFieldDescriptor::Element(size_t i) const
@@ -518,14 +518,14 @@ TComplexTypeFieldDescriptor TComplexTypeFieldDescriptor::TupleElement(size_t i)
{
const auto& elements = Type_->AsTupleTypeRef().GetElements();
YT_VERIFY(i < elements.size());
- return TComplexTypeFieldDescriptor(Descriptor_ + Format(".<tuple-element-%v>", i), elements[i]);
+ return TComplexTypeFieldDescriptor(Description_ + Format(".<tuple-element-%v>", i), elements[i]);
}
TComplexTypeFieldDescriptor TComplexTypeFieldDescriptor::VariantTupleElement(size_t i) const
{
const auto& elements = Type_->AsVariantTupleTypeRef().GetElements();
YT_VERIFY(i < elements.size());
- return TComplexTypeFieldDescriptor(Descriptor_ + Format(".<variant-element-%v>", i), elements[i]);
+ return TComplexTypeFieldDescriptor(Description_ + Format(".<variant-element-%v>", i), elements[i]);
}
TComplexTypeFieldDescriptor TComplexTypeFieldDescriptor::VariantStructField(size_t i) const
@@ -533,32 +533,32 @@ TComplexTypeFieldDescriptor TComplexTypeFieldDescriptor::VariantStructField(size
const auto& fields = Type_->AsVariantStructTypeRef().GetFields();
YT_VERIFY(i < fields.size());
const auto& field = fields[i];
- return TComplexTypeFieldDescriptor(Descriptor_ + "." + field.Name, field.Type);
+ return TComplexTypeFieldDescriptor(Description_ + "." + field.Name, field.Type);
}
TComplexTypeFieldDescriptor TComplexTypeFieldDescriptor::DictKey() const
{
- return TComplexTypeFieldDescriptor(Descriptor_ + ".<key>", Type_->AsDictTypeRef().GetKey());
+ return TComplexTypeFieldDescriptor(Description_ + ".<key>", Type_->AsDictTypeRef().GetKey());
}
TComplexTypeFieldDescriptor TComplexTypeFieldDescriptor::DictValue() const
{
- return TComplexTypeFieldDescriptor(Descriptor_ + ".<value>", Type_->AsDictTypeRef().GetValue());
+ return TComplexTypeFieldDescriptor(Description_ + ".<value>", Type_->AsDictTypeRef().GetValue());
}
TComplexTypeFieldDescriptor TComplexTypeFieldDescriptor::TaggedElement() const
{
- return TComplexTypeFieldDescriptor(Descriptor_ + ".<tagged-element>", Type_->AsTaggedTypeRef().GetElement());
+ return TComplexTypeFieldDescriptor(Description_ + ".<tagged-element>", Type_->AsTaggedTypeRef().GetElement());
}
TComplexTypeFieldDescriptor TComplexTypeFieldDescriptor::Detag() const
{
- return TComplexTypeFieldDescriptor(Descriptor_, DetagLogicalType(Type_));
+ return TComplexTypeFieldDescriptor(Description_, DetagLogicalType(Type_));
}
-const TString& TComplexTypeFieldDescriptor::GetDescription() const
+const std::string& TComplexTypeFieldDescriptor::GetDescription() const
{
- return Descriptor_;
+ return Description_;
}
const TLogicalTypePtr& TComplexTypeFieldDescriptor::GetType() const
@@ -1053,6 +1053,8 @@ void ValidateLogicalType(const TComplexTypeFieldDescriptor& rootDescriptor, std:
void ToProto(NProto::TLogicalType* protoLogicalType, const TLogicalTypePtr& logicalType)
{
+ using NYT::ToProto;
+
switch (logicalType->GetMetatype()) {
case ELogicalMetatype::Simple:
protoLogicalType->set_simple(ToProto(logicalType->AsSimpleTypeRef().GetElement()));
@@ -1071,7 +1073,7 @@ void ToProto(NProto::TLogicalType* protoLogicalType, const TLogicalTypePtr& logi
auto protoStruct = protoLogicalType->mutable_struct_();
for (const auto& structField : logicalType->AsStructTypeRef().GetFields()) {
auto protoStructField = protoStruct->add_fields();
- protoStructField->set_name(structField.Name);
+ protoStructField->set_name(ToProto(structField.Name));
ToProto(protoStructField->mutable_type(), structField.Type);
}
return;
@@ -1088,7 +1090,7 @@ void ToProto(NProto::TLogicalType* protoLogicalType, const TLogicalTypePtr& logi
auto protoVariantStruct = protoLogicalType->mutable_variant_struct();
for (const auto& field : logicalType->AsVariantStructTypeRef().GetFields()) {
auto protoField = protoVariantStruct->add_fields();
- protoField->set_name(field.Name);
+ protoField->set_name(ToProto(field.Name));
ToProto(protoField->mutable_type(), field.Type);
}
return;
diff --git a/yt/yt/client/table_client/logical_type.h b/yt/yt/client/table_client/logical_type.h
index bd482df984..1a0b2dd0b5 100644
--- a/yt/yt/client/table_client/logical_type.h
+++ b/yt/yt/client/table_client/logical_type.h
@@ -260,7 +260,7 @@ class TComplexTypeFieldDescriptor
public:
explicit TComplexTypeFieldDescriptor(TLogicalTypePtr type);
explicit TComplexTypeFieldDescriptor(const TColumnSchema& column);
- TComplexTypeFieldDescriptor(TString columnName, TLogicalTypePtr type);
+ TComplexTypeFieldDescriptor(const std::string& columnName, TLogicalTypePtr type);
TComplexTypeFieldDescriptor OptionalElement() const;
TComplexTypeFieldDescriptor ListElement() const;
@@ -276,11 +276,11 @@ public:
TComplexTypeFieldDescriptor Detag() const;
- const TString& GetDescription() const;
+ const std::string& GetDescription() const;
const TLogicalTypePtr& GetType() const;
private:
- TString Descriptor_;
+ std::string Description_;
TLogicalTypePtr Type_;
};
@@ -288,7 +288,7 @@ private:
struct TStructField
{
- TString Name;
+ std::string Name;
TLogicalTypePtr Type;
};
@@ -299,8 +299,6 @@ class TStructLogicalTypeBase
: public TLogicalType
{
public:
-
-public:
TStructLogicalTypeBase(ELogicalMetatype metatype, std::vector<TStructField> fields);
Y_FORCE_INLINE const std::vector<TStructField>& GetFields() const;
diff --git a/yt/yt/client/table_client/name_table.cpp b/yt/yt/client/table_client/name_table.cpp
index 2c22700fc7..6747c30539 100644
--- a/yt/yt/client/table_client/name_table.cpp
+++ b/yt/yt/client/table_client/name_table.cpp
@@ -167,10 +167,10 @@ int TNameTable::DoRegisterNameOrThrow(TStringBuf name)
return DoRegisterName(name);
}
-std::vector<TString> TNameTable::GetNames() const
+std::vector<std::string> TNameTable::GetNames() const
{
auto guard = Guard(SpinLock_);
- std::vector<TString> result(IdToName_.begin(), IdToName_.end());
+ std::vector<std::string> result(IdToName_.begin(), IdToName_.end());
return result;
}
@@ -255,7 +255,7 @@ std::optional<int> TNameTableWriter::FindId(TStringBuf name) const
auto optionalId = NameTable_->FindId(name);
if (optionalId) {
- Names_.push_back(TString(name));
+ Names_.push_back(std::string(name));
YT_VERIFY(NameToId_.emplace(Names_.back(), *optionalId).second);
}
return optionalId;
@@ -278,7 +278,7 @@ int TNameTableWriter::GetIdOrRegisterName(TStringBuf name)
}
auto id = NameTable_->GetIdOrRegisterName(name);
- Names_.push_back(TString(name));
+ Names_.push_back(std::string(name));
YT_VERIFY(NameToId_.emplace(Names_.back(), id).second);
return id;
}
@@ -296,7 +296,7 @@ void FromProto(TNameTablePtr* nameTable, const NProto::TNameTableExt& protoNameT
{
using NYT::FromProto;
- *nameTable = TNameTable::FromKeyColumns(FromProto<std::vector<TString>>(protoNameTable.names()));
+ *nameTable = TNameTable::FromKeyColumns(FromProto<std::vector<std::string>>(protoNameTable.names()));
}
////////////////////////////////////////////////////////////////////////////////
diff --git a/yt/yt/client/table_client/name_table.h b/yt/yt/client/table_client/name_table.h
index f0e94f3617..4d5b7ae404 100644
--- a/yt/yt/client/table_client/name_table.h
+++ b/yt/yt/client/table_client/name_table.h
@@ -37,7 +37,7 @@ public:
TStringBuf GetName(int id) const;
TStringBuf GetNameOrThrow(int id) const;
- std::vector<TString> GetNames() const;
+ std::vector<std::string> GetNames() const;
private:
YT_DECLARE_SPIN_LOCK(NThreading::TSpinLock, SpinLock_);
@@ -46,7 +46,7 @@ private:
// String values are owned by IdToName_.
// NB: Names may be SSO-strings, using a deque to avoid string view invalidation.
- std::deque<TString> IdToName_;
+ std::deque<std::string> IdToName_;
THashMap<TStringBuf, int> NameToId_;
i64 ByteSize_ = 0;
@@ -97,7 +97,7 @@ private:
// String values are owned by Names_
// NB: Names may be SSO-strings, using a deque to avoid string view invalidation
- mutable std::deque<TString> Names_;
+ mutable std::deque<std::string> Names_;
mutable THashMap<TStringBuf, int> NameToId_;
};
diff --git a/yt/yt/client/table_client/public.cpp b/yt/yt/client/table_client/public.cpp
index 4fc79ad447..1008d882c4 100644
--- a/yt/yt/client/table_client/public.cpp
+++ b/yt/yt/client/table_client/public.cpp
@@ -4,19 +4,20 @@ namespace NYT::NTableClient {
////////////////////////////////////////////////////////////////////////////////
-const TString PrimaryLockName("<primary>");
+const std::string PrimaryLockName("<primary>");
-const TString SystemColumnNamePrefix("$");
-const TString TableIndexColumnName = SystemColumnNamePrefix + "table_index";
-const TString RowIndexColumnName = SystemColumnNamePrefix + "row_index";
-const TString RangeIndexColumnName = SystemColumnNamePrefix + "range_index";
-const TString TabletIndexColumnName = SystemColumnNamePrefix + "tablet_index";
-const TString TimestampColumnName = SystemColumnNamePrefix + "timestamp";
-const TString TtlColumnName = SystemColumnNamePrefix + "ttl";
-const TString TimestampColumnPrefix = SystemColumnNamePrefix + "timestamp:";
-const TString CumulativeDataWeightColumnName = SystemColumnNamePrefix + "cumulative_data_weight";
-const TString EmptyValueColumnName = SystemColumnNamePrefix + "empty";
-const TString SequenceNumberColumnName = SystemColumnNamePrefix + "sequence_number";
+const std::string SystemColumnNamePrefix("$");
+const std::string NonexistentColumnName = SystemColumnNamePrefix + "__YT_NONEXISTENT_COLUMN_NAME__";
+const std::string TableIndexColumnName = SystemColumnNamePrefix + "table_index";
+const std::string RowIndexColumnName = SystemColumnNamePrefix + "row_index";
+const std::string RangeIndexColumnName = SystemColumnNamePrefix + "range_index";
+const std::string TabletIndexColumnName = SystemColumnNamePrefix + "tablet_index";
+const std::string TimestampColumnName = SystemColumnNamePrefix + "timestamp";
+const std::string TtlColumnName = SystemColumnNamePrefix + "ttl";
+const std::string TimestampColumnPrefix = SystemColumnNamePrefix + "timestamp:";
+const std::string CumulativeDataWeightColumnName = SystemColumnNamePrefix + "cumulative_data_weight";
+const std::string EmptyValueColumnName = SystemColumnNamePrefix + "empty";
+const std::string SequenceNumberColumnName = SystemColumnNamePrefix + "sequence_number";
////////////////////////////////////////////////////////////////////////////////
diff --git a/yt/yt/client/table_client/public.h b/yt/yt/client/table_client/public.h
index e52eaecf48..1c795adb32 100644
--- a/yt/yt/client/table_client/public.h
+++ b/yt/yt/client/table_client/public.h
@@ -77,7 +77,7 @@ using NTransactionClient::AsyncLastCommittedTimestamp;
using NTransactionClient::AllCommittedTimestamp;
using NTransactionClient::NotPreparedTimestamp;
-using TKeyColumns = std::vector<TString>;
+using TKeyColumns = std::vector<std::string>;
////////////////////////////////////////////////////////////////////////////////
@@ -119,20 +119,20 @@ constexpr int MaxColumnId = 32 * 1024;
constexpr int MaxSchemaTotalTypeComplexity = MaxColumnId;
constexpr int MaxSchemaDepth = 32;
-
-extern const TString PrimaryLockName;
-
-extern const TString SystemColumnNamePrefix;
-extern const TString TableIndexColumnName;
-extern const TString RowIndexColumnName;
-extern const TString RangeIndexColumnName;
-extern const TString TabletIndexColumnName;
-extern const TString TimestampColumnName;
-extern const TString TtlColumnName;
-extern const TString TimestampColumnPrefix;
-extern const TString CumulativeDataWeightColumnName;
-extern const TString EmptyValueColumnName;
-extern const TString SequenceNumberColumnName;
+extern const std::string PrimaryLockName;
+
+extern const std::string SystemColumnNamePrefix;
+extern const std::string NonexistentColumnName;
+extern const std::string TableIndexColumnName;
+extern const std::string RowIndexColumnName;
+extern const std::string RangeIndexColumnName;
+extern const std::string TabletIndexColumnName;
+extern const std::string TimestampColumnName;
+extern const std::string TtlColumnName;
+extern const std::string TimestampColumnPrefix;
+extern const std::string CumulativeDataWeightColumnName;
+extern const std::string EmptyValueColumnName;
+extern const std::string SequenceNumberColumnName;
constexpr int TypicalHunkColumnCount = 8;
@@ -308,7 +308,7 @@ class TKeyComparer;
struct TColumnRenameDescriptor;
using TColumnRenameDescriptors = std::vector<TColumnRenameDescriptor>;
-YT_DEFINE_STRONG_TYPEDEF(TColumnStableName, TString);
+YT_DEFINE_STRONG_TYPEDEF(TColumnStableName, std::string);
class TColumnSchema;
diff --git a/yt/yt/client/table_client/schema.cpp b/yt/yt/client/table_client/schema.cpp
index 3d5a8b92ce..1acb033cde 100644
--- a/yt/yt/client/table_client/schema.cpp
+++ b/yt/yt/client/table_client/schema.cpp
@@ -148,33 +148,33 @@ void FromProto(TLockMask* lockMask, const NTabletClient::NProto::TLockMask& prot
TColumnSchema::TColumnSchema()
: TColumnSchema(
- TString(),
+ std::string(),
NullLogicalType(),
std::nullopt)
{ }
TColumnSchema::TColumnSchema(
- TString name,
+ const std::string& name,
EValueType type,
std::optional<ESortOrder> sortOrder)
: TColumnSchema(
- std::move(name),
+ name,
MakeLogicalType(GetLogicalType(type), /*required*/ false),
sortOrder)
{ }
TColumnSchema::TColumnSchema(
- TString name,
+ const std::string& name,
ESimpleLogicalValueType type,
std::optional<ESortOrder> sortOrder)
: TColumnSchema(
- std::move(name),
+ name,
MakeLogicalType(type, /*required*/ false),
sortOrder)
{ }
TColumnSchema::TColumnSchema(
- TString name,
+ const std::string& name,
TLogicalTypePtr type,
std::optional<ESortOrder> sortOrder)
: StableName_(name)
@@ -184,13 +184,13 @@ TColumnSchema::TColumnSchema(
SetLogicalType(std::move(type));
}
-TColumnSchema& TColumnSchema::SetStableName(TColumnStableName value)
+TColumnSchema& TColumnSchema::SetStableName(const TColumnStableName& value)
{
- StableName_ = std::move(value);
+ StableName_ = value;
return *this;
}
-TColumnSchema& TColumnSchema::SetName(TString value)
+TColumnSchema& TColumnSchema::SetName(const std::string& value)
{
Name_ = value;
return *this;
@@ -202,21 +202,21 @@ TColumnSchema& TColumnSchema::SetSortOrder(std::optional<ESortOrder> value)
return *this;
}
-TColumnSchema& TColumnSchema::SetLock(std::optional<TString> value)
+TColumnSchema& TColumnSchema::SetLock(const std::optional<std::string>& value)
{
- Lock_ = std::move(value);
+ Lock_ = value;
return *this;
}
-TColumnSchema& TColumnSchema::SetGroup(std::optional<TString> value)
+TColumnSchema& TColumnSchema::SetGroup(const std::optional<std::string>& value)
{
- Group_ = std::move(value);
+ Group_ = value;
return *this;
}
-TColumnSchema& TColumnSchema::SetExpression(std::optional<TString> value)
+TColumnSchema& TColumnSchema::SetExpression(const std::optional<TString>& value)
{
- Expression_ = std::move(value);
+ Expression_ = value;
return *this;
}
@@ -226,9 +226,9 @@ TColumnSchema& TColumnSchema::SetMaterialized(std::optional<bool> value)
return *this;
}
-TColumnSchema& TColumnSchema::SetAggregate(std::optional<TString> value)
+TColumnSchema& TColumnSchema::SetAggregate(const std::optional<std::string>& value)
{
- Aggregate_ = std::move(value);
+ Aggregate_ = value;
return *this;
}
@@ -300,7 +300,7 @@ bool TColumnSchema::IsRenamed() const
return Name() != StableName().Underlying();
}
-TString TColumnSchema::GetDiagnosticNameString() const
+std::string TColumnSchema::GetDiagnosticNameString() const
{
if (IsRenamed()) {
return Format("%Qv (stable name %Qv)", Name(), StableName().Underlying());
@@ -389,9 +389,9 @@ void FormatValue(TStringBuilderBase* builder, const TDeletedColumn& schema, TStr
void ToProto(NProto::TColumnSchema* protoSchema, const TColumnSchema& schema)
{
- protoSchema->set_name(schema.Name());
+ protoSchema->set_name(ToProto(schema.Name()));
if (schema.IsRenamed()) {
- protoSchema->set_stable_name(schema.StableName().Underlying());
+ protoSchema->set_stable_name(ToProto(schema.StableName()));
}
protoSchema->set_type(ToProto(GetPhysicalType(schema.CastToV1Type())));
@@ -408,7 +408,7 @@ void ToProto(NProto::TColumnSchema* protoSchema, const TColumnSchema& schema)
}
ToProto(protoSchema->mutable_logical_type(), schema.LogicalType());
if (schema.Lock()) {
- protoSchema->set_lock(*schema.Lock());
+ protoSchema->set_lock(ToProto(*schema.Lock()));
} else {
protoSchema->clear_lock();
}
@@ -423,7 +423,7 @@ void ToProto(NProto::TColumnSchema* protoSchema, const TColumnSchema& schema)
protoSchema->clear_materialized();
}
if (schema.Aggregate()) {
- protoSchema->set_aggregate(*schema.Aggregate());
+ protoSchema->set_aggregate(ToProto(*schema.Aggregate()));
} else {
protoSchema->clear_aggregate();
}
@@ -433,7 +433,7 @@ void ToProto(NProto::TColumnSchema* protoSchema, const TColumnSchema& schema)
protoSchema->clear_sort_order();
}
if (schema.Group()) {
- protoSchema->set_group(*schema.Group());
+ protoSchema->set_group(ToProto(*schema.Group()));
} else {
protoSchema->clear_group();
}
@@ -446,7 +446,7 @@ void ToProto(NProto::TColumnSchema* protoSchema, const TColumnSchema& schema)
void ToProto(NProto::TDeletedColumn* protoSchema, const TDeletedColumn& schema)
{
- protoSchema->set_stable_name(schema.StableName().Underlying());
+ protoSchema->set_stable_name(ToProto(schema.StableName()));
}
void FromProto(TColumnSchema* schema, const NProto::TColumnSchema& protoSchema)
@@ -500,7 +500,7 @@ bool TTableSchema::TNameMapping::IsDeleted(const TColumnStableName& stableName)
return Schema_.FindDeletedColumn(stableName) != nullptr;
}
-TString TTableSchema::TNameMapping::StableNameToName(const TColumnStableName& stableName) const
+std::string TTableSchema::TNameMapping::StableNameToName(const TColumnStableName& stableName) const
{
auto* column = Schema_.FindColumnByStableName(stableName);
if (!column) {
@@ -523,7 +523,7 @@ TColumnStableName TTableSchema::TNameMapping::NameToStableName(TStringBuf name)
THROW_ERROR_EXCEPTION("No column with name %Qv in strict schema", name);
}
}
- return TColumnStableName(TString(name));
+ return TColumnStableName(std::string(name));
}
return column->StableName();
}
@@ -705,7 +705,7 @@ TTableSchemaPtr TTableSchema::Filter(const TColumnFilter& columnFilter, bool dis
DeletedColumns());
}
-TTableSchemaPtr TTableSchema::Filter(const THashSet<TString>& columnNames, bool discardSortOrder) const
+TTableSchemaPtr TTableSchema::Filter(const THashSet<std::string>& columnNames, bool discardSortOrder) const
{
TColumnFilter::TIndexes indexes;
for (const auto& column : Columns()) {
@@ -716,13 +716,13 @@ TTableSchemaPtr TTableSchema::Filter(const THashSet<TString>& columnNames, bool
return Filter(TColumnFilter(std::move(indexes)), discardSortOrder);
}
-TTableSchemaPtr TTableSchema::Filter(const std::optional<std::vector<TString>>& columnNames, bool discardSortOrder) const
+TTableSchemaPtr TTableSchema::Filter(const std::optional<std::vector<std::string>>& columnNames, bool discardSortOrder) const
{
if (!columnNames) {
return Filter(TColumnFilter(), discardSortOrder);
}
- return Filter(THashSet<TString>(columnNames->begin(), columnNames->end()), discardSortOrder);
+ return Filter(THashSet<std::string>(columnNames->begin(), columnNames->end()), discardSortOrder);
}
bool TTableSchema::HasMaterializedComputedColumns() const
@@ -835,12 +835,12 @@ int TTableSchema::GetColumnCount() const
return ssize(Columns());
}
-std::vector<TString> TTableSchema::GetColumnNames() const
+std::vector<std::string> TTableSchema::GetColumnNames() const
{
if (!ColumnInfo_) {
- return std::vector<TString>();
+ return std::vector<std::string>();
}
- std::vector<TString> result;
+ std::vector<std::string> result;
const auto& info = *ColumnInfo_;
result.reserve(info.Columns.size());
for (const auto& column : info.Columns) {
@@ -870,8 +870,8 @@ const THunkColumnIds& TTableSchema::GetHunkColumnIds() const
std::vector<TColumnStableName> MapNamesToStableNames(
const TTableSchema& schema,
- std::vector<TString> names,
- const std::optional<TStringBuf>& missingColumnReplacement)
+ const std::vector<std::string>& names,
+ std::optional<TStringBuf> missingColumnReplacement)
{
std::vector<TColumnStableName> stableNames;
stableNames.reserve(names.size());
@@ -882,7 +882,7 @@ std::vector<TColumnStableName> MapNamesToStableNames(
} else if (!schema.GetStrict()) {
stableNames.push_back(TColumnStableName(name));
} else if (missingColumnReplacement) {
- stableNames.push_back(TColumnStableName(TString(*missingColumnReplacement)));
+ stableNames.push_back(TColumnStableName(std::string(*missingColumnReplacement)));
} else {
THROW_ERROR_EXCEPTION("Column %Qv is missing in strict schema",
name);
@@ -914,7 +914,7 @@ TSortColumns TTableSchema::GetSortColumns(const std::optional<TNameMapping>& nam
if (column.SortOrder()) {
const auto& name = actualNameMapping.StableNameToName(column.StableName());
sortColumns.push_back(TColumnSortSchema{
- .Name = TString(name),
+ .Name = std::string(name),
.SortOrder = *column.SortOrder(),
});
}
@@ -1483,14 +1483,14 @@ void FormatValue(TStringBuilderBase* builder, const TTableSchemaPtr& schema, TSt
}
}
-TString SerializeToWireProto(const TTableSchemaPtr& schema)
+std::string SerializeToWireProto(const TTableSchemaPtr& schema)
{
NTableClient::NProto::TTableSchemaExt protoSchema;
ToProto(&protoSchema, schema);
return protoSchema.SerializeAsString();
}
-void DeserializeFromWireProto(TTableSchemaPtr* schema, const TString& serializedProto)
+void DeserializeFromWireProto(TTableSchemaPtr* schema, const std::string& serializedProto)
{
NTableClient::NProto::TTableSchemaExt protoSchema;
if (!protoSchema.ParseFromString(serializedProto)) {
@@ -1626,7 +1626,7 @@ bool IsEqualIgnoringRequiredness(const TTableSchema& lhs, const TTableSchema& rh
// Parses description of the following form nested_key.name or nested_value.name or nested_value.name.sum
std::optional<TNestedColumn> TryParseNestedAggregate(TStringBuf description)
{
- if (!description.StartsWith("nested")) {
+ if (!description.starts_with("nested")) {
return std::nullopt;
}
@@ -1658,7 +1658,7 @@ std::optional<TNestedColumn> TryParseNestedAggregate(TStringBuf description)
};
auto parseToken = [&] (TStringBuf token) {
- if (TStringBuf(ptr, ptrEnd).StartsWith(token)) {
+ if (TStringBuf(ptr, ptrEnd).starts_with(token)) {
ptr += token.size();
return true;
}
@@ -1749,7 +1749,7 @@ void ValidateKeyColumns(const TKeyColumns& keyColumns)
{
ValidateKeyColumnCount(keyColumns.size());
- THashSet<TString> names;
+ THashSet<std::string> names;
for (const auto& name : keyColumns) {
if (!names.insert(name).second) {
THROW_ERROR_EXCEPTION("Duplicate key column name %Qv",
@@ -1776,12 +1776,12 @@ void ValidateSystemColumnSchema(
bool allowUnversionedUpdateColumns,
bool allowTimestampColumns)
{
- static const auto allowedSortedTablesSystemColumns = THashMap<TString, ESimpleLogicalValueType>{
+ static const auto allowedSortedTablesSystemColumns = THashMap<std::string, ESimpleLogicalValueType>{
{EmptyValueColumnName, ESimpleLogicalValueType::Int64},
{TtlColumnName, ESimpleLogicalValueType::Uint64},
};
- static const auto allowedOrderedTablesSystemColumns = THashMap<TString, ESimpleLogicalValueType>{
+ static const auto allowedOrderedTablesSystemColumns = THashMap<std::string, ESimpleLogicalValueType>{
{TimestampColumnName, ESimpleLogicalValueType::Uint64},
{CumulativeDataWeightColumnName, ESimpleLogicalValueType::Int64},
};
@@ -1819,17 +1819,17 @@ void ValidateSystemColumnSchema(
if (name == TUnversionedUpdateSchema::ChangeTypeColumnName) {
validateType(ESimpleLogicalValueType::Uint64);
return;
- } else if (name.StartsWith(TUnversionedUpdateSchema::FlagsColumnNamePrefix)) {
+ } else if (name.starts_with(TUnversionedUpdateSchema::FlagsColumnNamePrefix)) {
validateType(ESimpleLogicalValueType::Uint64);
return;
- } else if (name.StartsWith(TUnversionedUpdateSchema::ValueColumnNamePrefix)) {
+ } else if (name.starts_with(TUnversionedUpdateSchema::ValueColumnNamePrefix)) {
// Value can have any type.
return;
}
}
if (allowTimestampColumns) {
- if (name.StartsWith(TimestampColumnPrefix)) {
+ if (name.starts_with(TimestampColumnPrefix)) {
validateType(ESimpleLogicalValueType::Uint64);
return;
}
@@ -1840,7 +1840,7 @@ void ValidateSystemColumnSchema(
name);
}
-void ValidateColumnName(const TString& name)
+void ValidateColumnName(const std::string& name)
{
if (name.empty()) {
THROW_ERROR_EXCEPTION("Column name cannot be empty");
@@ -1861,7 +1861,7 @@ void ValidateColumnSchema(
bool allowUnversionedUpdateColumns,
bool allowTimestampColumns)
{
- static const auto allowedAggregates = THashSet<TString>{
+ static const auto allowedAggregates = THashSet<std::string, THash<TStringBuf>, TEqualTo<>>{
"sum",
"min",
"max",
@@ -1872,7 +1872,7 @@ void ValidateColumnSchema(
"dict_sum",
};
- static const auto allowedNestedAggregates = THashSet<TString>{
+ static const auto allowedNestedAggregates = THashSet<std::string, THash<TStringBuf>, TEqualTo<>>{
"sum",
"max"
};
@@ -1884,7 +1884,7 @@ void ValidateColumnSchema(
const auto& name = columnSchema.Name();
ValidateColumnName(name);
- if (stableName.Underlying().StartsWith(SystemColumnNamePrefix) || name.StartsWith(SystemColumnNamePrefix)) {
+ if (stableName.Underlying().starts_with(SystemColumnNamePrefix) || name.starts_with(SystemColumnNamePrefix)) {
ValidateSystemColumnSchema(
columnSchema,
isTableSorted,
@@ -2093,7 +2093,7 @@ void ValidatePivotKey(
//! Validates that number of locks doesn't exceed #MaxColumnLockCount.
void ValidateLocks(const TTableSchema& schema)
{
- THashSet<TString> lockNames;
+ THashSet<std::string> lockNames;
YT_VERIFY(lockNames.insert(PrimaryLockName).second);
for (const auto& column : schema.Columns()) {
if (column.Lock()) {
@@ -2323,11 +2323,11 @@ void ValidateNoRenamedColumns(const TTableSchema& schema)
////////////////////////////////////////////////////////////////////////////////
-THashMap<TString, int> GetLocksMapping(
+THashMap<std::string, int> GetLocksMapping(
const NTableClient::TTableSchema& schema,
bool fullAtomicity,
std::vector<int>* columnIndexToLockIndex,
- std::vector<TString>* lockIndexToName)
+ std::vector<std::string>* lockIndexToName)
{
if (columnIndexToLockIndex) {
// Assign dummy lock indexes to key components.
@@ -2338,7 +2338,7 @@ THashMap<TString, int> GetLocksMapping(
lockIndexToName->push_back(PrimaryLockName);
}
- THashMap<TString, int> groupToIndex;
+ THashMap<std::string, int> groupToIndex;
if (fullAtomicity) {
// Assign lock indexes to data components.
for (int index = schema.GetKeyColumnCount(); index < std::ssize(schema.Columns()); ++index) {
@@ -2370,7 +2370,7 @@ THashMap<TString, int> GetLocksMapping(
TLockMask GetLockMask(
const NTableClient::TTableSchema& schema,
bool fullAtomicity,
- const std::vector<TString>& locks,
+ const std::vector<std::string>& locks,
ELockType lockType)
{
auto groupToIndex = GetLocksMapping(schema, fullAtomicity);
@@ -2444,7 +2444,7 @@ TCellTaggedTableSchemaPtr::TCellTaggedTableSchemaPtr(TTableSchemaPtr tableSchema
size_t THash<NYT::NTableClient::TColumnStableName>::operator()(const NYT::NTableClient::TColumnStableName& stableName) const
{
- return THash<TString>()(stableName.Underlying());
+ return THash<std::string>()(stableName.Underlying());
}
size_t THash<NYT::NTableClient::TColumnSchema>::operator()(const NYT::NTableClient::TColumnSchema& columnSchema) const
diff --git a/yt/yt/client/table_client/schema.h b/yt/yt/client/table_client/schema.h
index 4fd3c3f486..7cbbc86048 100644
--- a/yt/yt/client/table_client/schema.h
+++ b/yt/yt/client/table_client/schema.h
@@ -117,30 +117,30 @@ class TColumnSchema
public:
// Keep in sync with hasher below.
DEFINE_BYREF_RO_PROPERTY(TColumnStableName, StableName);
- DEFINE_BYREF_RO_PROPERTY(TString, Name);
+ DEFINE_BYREF_RO_PROPERTY(std::string, Name);
DEFINE_BYREF_RO_PROPERTY(TLogicalTypePtr, LogicalType);
DEFINE_BYREF_RO_PROPERTY(std::optional<ESortOrder>, SortOrder);
- DEFINE_BYREF_RO_PROPERTY(std::optional<TString>, Lock);
+ DEFINE_BYREF_RO_PROPERTY(std::optional<std::string>, Lock);
DEFINE_BYREF_RO_PROPERTY(std::optional<TString>, Expression);
DEFINE_BYREF_RO_PROPERTY(std::optional<bool>, Materialized);
- DEFINE_BYREF_RO_PROPERTY(std::optional<TString>, Aggregate);
- DEFINE_BYREF_RO_PROPERTY(std::optional<TString>, Group);
+ DEFINE_BYREF_RO_PROPERTY(std::optional<std::string>, Aggregate);
+ DEFINE_BYREF_RO_PROPERTY(std::optional<std::string>, Group);
DEFINE_BYREF_RO_PROPERTY(bool, Required);
DEFINE_BYREF_RO_PROPERTY(std::optional<i64>, MaxInlineHunkSize);
public:
TColumnSchema();
TColumnSchema(
- TString name,
+ const std::string& name,
EValueType type,
std::optional<ESortOrder> sortOrder = {});
TColumnSchema(
- TString name,
+ const std::string& name,
ESimpleLogicalValueType type,
std::optional<ESortOrder> sortOrder = {});
TColumnSchema(
- TString name,
+ const std::string& name,
TLogicalTypePtr type,
std::optional<ESortOrder> sortOrder = {});
@@ -150,16 +150,16 @@ public:
TColumnSchema& operator=(const TColumnSchema&) = default;
TColumnSchema& operator=(TColumnSchema&&) = default;
- TColumnSchema& SetStableName(TColumnStableName stableName);
- TColumnSchema& SetName(TString name);
+ TColumnSchema& SetStableName(const TColumnStableName& stableName);
+ TColumnSchema& SetName(const std::string& name);
TColumnSchema& SetLogicalType(TLogicalTypePtr valueType);
TColumnSchema& SetSimpleLogicalType(ESimpleLogicalValueType type);
TColumnSchema& SetSortOrder(std::optional<ESortOrder> value);
- TColumnSchema& SetLock(std::optional<TString> value);
- TColumnSchema& SetExpression(std::optional<TString> value);
+ TColumnSchema& SetLock(const std::optional<std::string>& value);
+ TColumnSchema& SetExpression(const std::optional<TString>& value);
TColumnSchema& SetMaterialized(std::optional<bool> value);
- TColumnSchema& SetAggregate(std::optional<TString> value);
- TColumnSchema& SetGroup(std::optional<TString> value);
+ TColumnSchema& SetAggregate(const std::optional<std::string>& value);
+ TColumnSchema& SetGroup(const std::optional<std::string>& value);
TColumnSchema& SetRequired(bool value);
TColumnSchema& SetMaxInlineHunkSize(std::optional<i64> value);
@@ -176,7 +176,7 @@ public:
ESimpleLogicalValueType CastToV1Type() const;
bool IsRenamed() const;
- TString GetDiagnosticNameString() const;
+ std::string GetDiagnosticNameString() const;
private:
ESimpleLogicalValueType V1Type_;
@@ -221,7 +221,7 @@ public:
explicit TNameMapping(const TTableSchema& schema);
bool IsDeleted(const TColumnStableName& stableName) const;
- TString StableNameToName(const TColumnStableName& stableName) const;
+ std::string StableNameToName(const TColumnStableName& stableName) const;
TColumnStableName NameToStableName(TStringBuf name) const;
private:
@@ -267,16 +267,16 @@ public:
const TColumnSchema* FindColumn(TStringBuf name) const;
const TColumnSchema& GetColumn(TStringBuf name) const;
const TColumnSchema& GetColumnOrThrow(TStringBuf name) const;
- std::vector<TString> GetColumnNames() const;
+ std::vector<std::string> GetColumnNames() const;
TTableSchemaPtr Filter(
const TColumnFilter& columnFilter,
bool discardSortOrder = false) const;
TTableSchemaPtr Filter(
- const THashSet<TString>& columnNames,
+ const THashSet<std::string>& columnNames,
bool discardSortOrder = false) const;
TTableSchemaPtr Filter(
- const std::optional<std::vector<TString>>& columnNames,
+ const std::optional<std::vector<std::string>>& columnNames,
bool discardSortOrder = false) const;
bool HasMaterializedComputedColumns() const;
@@ -432,9 +432,9 @@ void FormatValue(TStringBuilderBase* builder, const TTableSchema& schema, TStrin
void FormatValue(TStringBuilderBase* builder, const TTableSchemaPtr& schema, TStringBuf spec);
//! Returns serialized NTableClient.NProto.TTableSchemaExt.
-TString SerializeToWireProto(const TTableSchemaPtr& schema);
+std::string SerializeToWireProto(const TTableSchemaPtr& schema);
-void DeserializeFromWireProto(TTableSchemaPtr* schema, const TString& serializedProto);
+void DeserializeFromWireProto(TTableSchemaPtr* schema, const std::string& serializedProto);
void Serialize(const TTableSchema& schema, NYson::IYsonConsumer* consumer);
void Deserialize(TTableSchema& schema, NYTree::INodePtr node);
@@ -473,12 +473,10 @@ bool IsEqualIgnoringRequiredness(const TTableSchema& lhs, const TTableSchema& rh
////////////////////////////////////////////////////////////////////////////////
-static constexpr TStringBuf NonexistentColumnName = "$__YT_NONEXISTENT_COLUMN_NAME__";
-
std::vector<TColumnStableName> MapNamesToStableNames(
const TTableSchema& schema,
- std::vector<TString> names,
- const std::optional<TStringBuf>& missingColumnReplacement = std::nullopt);
+ const std::vector<std::string>& names,
+ std::optional<TStringBuf> missingColumnReplacement = std::nullopt);
////////////////////////////////////////////////////////////////////////////////
@@ -499,7 +497,7 @@ void ValidateKeyColumns(const TKeyColumns& keyColumns);
void ValidateDynamicTableKeyColumnCount(int count);
-void ValidateColumnName(const TString& name);
+void ValidateColumnName(const std::string& name);
void ValidateColumnSchema(
const TColumnSchema& columnSchema,
@@ -528,16 +526,16 @@ void ValidatePivotKey(
////////////////////////////////////////////////////////////////////////////////
-THashMap<TString, int> GetLocksMapping(
+THashMap<std::string, int> GetLocksMapping(
const NTableClient::TTableSchema& schema,
bool fullAtomicity,
std::vector<int>* columnIndexToLockIndex = nullptr,
- std::vector<TString>* lockIndexToName = nullptr);
+ std::vector<std::string>* lockIndexToName = nullptr);
TLockMask GetLockMask(
const NTableClient::TTableSchema& schema,
bool fullAtomicity,
- const std::vector<TString>& locks,
+ const std::vector<std::string>& locks,
ELockType lockType = ELockType::SharedWeak);
////////////////////////////////////////////////////////////////////////////////
diff --git a/yt/yt/client/table_client/validate_logical_type.cpp b/yt/yt/client/table_client/validate_logical_type.cpp
index 3e6b3d1d65..59cb8d2a0b 100644
--- a/yt/yt/client/table_client/validate_logical_type.cpp
+++ b/yt/yt/client/table_client/validate_logical_type.cpp
@@ -448,7 +448,7 @@ private:
Cursor_.Next();
}
- TString GetDescription(const TFieldId& fieldId) const
+ std::string GetDescription(const TFieldId& fieldId) const
{
return fieldId.GetDescriptor(RootDescriptor_).GetDescription();
}
@@ -509,7 +509,7 @@ private:
{
std::vector<int> path;
const auto* current = this;
- while (current->Parent_ != nullptr) {
+ while (current->Parent_) {
path.push_back(current->SiblingIndex_);
current = current->Parent_;
}
diff --git a/yt/yt/client/tablet_client/table_mount_cache_detail.cpp b/yt/yt/client/tablet_client/table_mount_cache_detail.cpp
index 248e19de7e..ae1d41c975 100644
--- a/yt/yt/client/tablet_client/table_mount_cache_detail.cpp
+++ b/yt/yt/client/tablet_client/table_mount_cache_detail.cpp
@@ -304,11 +304,14 @@ auto TTableMountCacheBase::TryHandleServantNotActiveError(const TError& error)
}
}
- TabletInfoOwnerCache_.Insert(*tabletId, MakeWeak(clone));
clonedTableInfos.push_back(std::move(clone));
}
for (const auto& tableInfo : clonedTableInfos) {
+ for (const auto& tabletInfo : tableInfo->Tablets) {
+ TabletInfoOwnerCache_.Insert(tabletInfo->TabletId, MakeWeak(tableInfo));
+ }
+
TAsyncExpiringCache::Set(tableInfo->Path, tableInfo);
}
diff --git a/yt/yt/client/tablet_client/watermark_runtime_data.h b/yt/yt/client/tablet_client/watermark_runtime_data.h
index 54c62c27bd..08f876a3cf 100644
--- a/yt/yt/client/tablet_client/watermark_runtime_data.h
+++ b/yt/yt/client/tablet_client/watermark_runtime_data.h
@@ -11,7 +11,7 @@ namespace NYT::NTabletClient {
struct TWatermarkRuntimeDataConfig
: public NYTree::TYsonStructLite
{
- TString ColumnName;
+ std::string ColumnName;
ui64 Watermark;
REGISTER_YSON_STRUCT_LITE(TWatermarkRuntimeDataConfig);
diff --git a/yt/yt/client/unittests/mock/client.h b/yt/yt/client/unittests/mock/client.h
index 43e8056b1e..fc59a92b9b 100644
--- a/yt/yt/client/unittests/mock/client.h
+++ b/yt/yt/client/unittests/mock/client.h
@@ -865,7 +865,7 @@ public:
MOCK_METHOD(TFuture<IRowBatchWriterPtr>, CreateShuffleWriter, (
const TShuffleHandlePtr& shuffleHandle,
- const TString& partitionColumn,
+ const std::string& partitionColumn,
const NTableClient::TTableWriterConfigPtr& config),
(override));
diff --git a/yt/yt/client/unittests/mock/table_reader.h b/yt/yt/client/unittests/mock/table_reader.h
index be901f4881..a827b7c8b6 100644
--- a/yt/yt/client/unittests/mock/table_reader.h
+++ b/yt/yt/client/unittests/mock/table_reader.h
@@ -24,7 +24,7 @@ public:
MOCK_METHOD(NTableClient::IUnversionedRowBatchPtr, Read, (const NTableClient::TRowBatchReadOptions& options), (override));
- MOCK_METHOD(const std::vector<TString>&, GetOmittedInaccessibleColumns, (), (const, override));
+ MOCK_METHOD(const std::vector<std::string>&, GetOmittedInaccessibleColumns, (), (const, override));
const NTableClient::TNameTablePtr& GetNameTable() const override;
diff --git a/yt/yt/client/ypath/parser_detail.cpp b/yt/yt/client/ypath/parser_detail.cpp
index d1852f526f..cb11a20767 100644
--- a/yt/yt/client/ypath/parser_detail.cpp
+++ b/yt/yt/client/ypath/parser_detail.cpp
@@ -171,7 +171,7 @@ void ParseColumns(NYson::TTokenizer& tokenizer, IAttributeDictionary* attributes
return;
}
- std::vector<TString> columns;
+ std::vector<std::string> columns;
tokenizer.ParseNext();
while (tokenizer.GetCurrentType() != EndColumnSelectorToken) {
diff --git a/yt/yt/client/ypath/rich.cpp b/yt/yt/client/ypath/rich.cpp
index ddd386df64..717194f64b 100644
--- a/yt/yt/client/ypath/rich.cpp
+++ b/yt/yt/client/ypath/rich.cpp
@@ -217,15 +217,15 @@ void TRichYPath::SetReadViaExecNode(bool value)
Attributes().Set("read_via_exec_node", value);
}
-std::optional<std::vector<TString>> TRichYPath::GetColumns() const
+std::optional<std::vector<std::string>> TRichYPath::GetColumns() const
{
if (Attributes().Contains("channel")) {
THROW_ERROR_EXCEPTION("Deprecated attribute \"channel\" in YPath");
}
- return FindAttribute<std::vector<TString>>(*this, "columns");
+ return FindAttribute<std::vector<std::string>>(*this, "columns");
}
-void TRichYPath::SetColumns(const std::vector<TString>& columns)
+void TRichYPath::SetColumns(const std::vector<std::string>& columns)
{
Attributes().Set("columns", columns);
}
diff --git a/yt/yt/client/ypath/rich.h b/yt/yt/client/ypath/rich.h
index b6d7d60ccf..75335b7606 100644
--- a/yt/yt/client/ypath/rich.h
+++ b/yt/yt/client/ypath/rich.h
@@ -67,8 +67,8 @@ public:
void SetReadViaExecNode(bool value);
// "columns"
- std::optional<std::vector<TString>> GetColumns() const;
- void SetColumns(const std::vector<TString>& columns);
+ std::optional<std::vector<std::string>> GetColumns() const;
+ void SetColumns(const std::vector<std::string>& columns);
// "rename_columns"
std::optional<NTableClient::TColumnRenameDescriptors> GetColumnRenameDescriptors() const;
diff --git a/yt/yt/core/concurrency/unittests/scheduled_executor_ut.cpp b/yt/yt/core/concurrency/unittests/scheduled_executor_ut.cpp
index 2d7143297d..f0573463c6 100644
--- a/yt/yt/core/concurrency/unittests/scheduled_executor_ut.cpp
+++ b/yt/yt/core/concurrency/unittests/scheduled_executor_ut.cpp
@@ -19,7 +19,7 @@ class TScheduledExecutorTest
////////////////////////////////////////////////////////////////////////////////
-constexpr auto ErrorMargin = TDuration::MilliSeconds(20);
+constexpr auto ErrorMargin = TDuration::MilliSeconds(50);
////////////////////////////////////////////////////////////////////////////////
diff --git a/yt/yt/core/logging/log_manager.cpp b/yt/yt/core/logging/log_manager.cpp
index d7e15992be..5694698800 100644
--- a/yt/yt/core/logging/log_manager.cpp
+++ b/yt/yt/core/logging/log_manager.cpp
@@ -519,10 +519,6 @@ public:
::TSourceLocation sourceLocation,
TStringBuf message)
{
- if (anchor->Registered.exchange(true)) {
- return;
- }
-
auto guard = Guard(SpinLock_);
auto config = Config_.Acquire();
anchor->SourceLocation = sourceLocation;
@@ -537,12 +533,13 @@ public:
if (auto it = AnchorMap_.find(anchorMessage)) {
return it->second;
}
+ auto config = Config_.Acquire();
auto anchor = std::make_unique<TLoggingAnchor>();
- anchor->Registered = true;
anchor->AnchorMessage = std::move(anchorMessage);
auto* rawAnchor = anchor.get();
DynamicAnchors_.push_back(std::move(anchor));
DoRegisterAnchor(rawAnchor);
+ DoUpdateAnchor(config, rawAnchor);
return rawAnchor;
}
diff --git a/yt/yt/core/net/address.cpp b/yt/yt/core/net/address.cpp
index c08fb94226..6f7a44100f 100644
--- a/yt/yt/core/net/address.cpp
+++ b/yt/yt/core/net/address.cpp
@@ -110,6 +110,13 @@ TStringBuf GetServiceHostName(TStringBuf address)
////////////////////////////////////////////////////////////////////////////////
+TString FormatNetworkAddress(TStringBuf address, int port)
+{
+ return Format("[%v]:%v", address, port);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
const TNetworkAddress NullNetworkAddress;
TNetworkAddress::TNetworkAddress()
diff --git a/yt/yt/core/net/address.h b/yt/yt/core/net/address.h
index 6f507b2900..99e7869f9e 100644
--- a/yt/yt/core/net/address.h
+++ b/yt/yt/core/net/address.h
@@ -43,6 +43,9 @@ TStringBuf GetServiceHostName(TStringBuf address);
////////////////////////////////////////////////////////////////////////////////
+//! Constructs an address of the form |[address]:port|.
+TString FormatNetworkAddress(TStringBuf address, int port);
+
class TIP6Address;
//! An opaque wrapper for |sockaddr| type.
diff --git a/yt/yt/core/rpc/balancing_channel.cpp b/yt/yt/core/rpc/balancing_channel.cpp
index 7884bba0d2..7e9cd188d0 100644
--- a/yt/yt/core/rpc/balancing_channel.cpp
+++ b/yt/yt/core/rpc/balancing_channel.cpp
@@ -151,7 +151,10 @@ private:
continue;
}
- auto addresses = AddressesFromEndpointSet(endpointSetOrError.Value());
+ auto addresses = AddressesFromEndpointSet(
+ endpointSetOrError.Value(),
+ Config_->Endpoints->UseIPv4,
+ Config_->Endpoints->UseIPv6);
allAddresses.insert(allAddresses.end(), addresses.begin(), addresses.end());
}
diff --git a/yt/yt/core/rpc/config.cpp b/yt/yt/core/rpc/config.cpp
index 50030c0c3b..b9cb445371 100644
--- a/yt/yt/core/rpc/config.cpp
+++ b/yt/yt/core/rpc/config.cpp
@@ -220,6 +220,10 @@ void TServiceDiscoveryEndpointsConfig::Register(TRegistrar registrar)
registrar.Parameter("endpoint_set_id", &TThis::EndpointSetId);
registrar.Parameter("update_period", &TThis::UpdatePeriod)
.Default(TDuration::Seconds(60));
+ registrar.Parameter("use_ipv4", &TThis::UseIPv4)
+ .Default(false);
+ registrar.Parameter("use_ipv6", &TThis::UseIPv6)
+ .Default(false);
registrar.Postprocessor([] (TThis* config) {
if (config->Cluster.has_value() == !config->Clusters.empty()) {
diff --git a/yt/yt/core/rpc/config.h b/yt/yt/core/rpc/config.h
index aceeadc725..c3ce1d08d2 100644
--- a/yt/yt/core/rpc/config.h
+++ b/yt/yt/core/rpc/config.h
@@ -132,7 +132,7 @@ public:
std::optional<bool> EnableErrorCodeCounter;
std::optional<ERequestTracingMode> TracingMode;
TTimeHistogramConfigPtr TimeHistogram;
- THashMap<std::string, TMethodConfigPtr> Methods;
+ THashMap<TString, TMethodConfigPtr> Methods;
std::optional<int> AuthenticationQueueSizeLimit;
std::optional<TDuration> PendingPayloadsTimeout;
std::optional<bool> Pooled;
@@ -302,6 +302,11 @@ public:
TString EndpointSetId;
TDuration UpdatePeriod;
+ //! Use IPv4 address of endpoint.
+ bool UseIPv4;
+ //! Use IPv6 address of endpoint.
+ bool UseIPv6;
+
REGISTER_YSON_STRUCT(TServiceDiscoveryEndpointsConfig);
static void Register(TRegistrar registrar);
diff --git a/yt/yt/core/rpc/helpers.cpp b/yt/yt/core/rpc/helpers.cpp
index d9d2a93a89..ca7b6221e5 100644
--- a/yt/yt/core/rpc/helpers.cpp
+++ b/yt/yt/core/rpc/helpers.cpp
@@ -514,12 +514,21 @@ void SetCurrentAuthenticationIdentity(const IClientRequestPtr& request)
SetAuthenticationIdentity(request, GetCurrentAuthenticationIdentity());
}
-std::vector<std::string> AddressesFromEndpointSet(const NServiceDiscovery::TEndpointSet& endpointSet)
+std::vector<std::string> AddressesFromEndpointSet(
+ const NServiceDiscovery::TEndpointSet& endpointSet,
+ bool useIPv4,
+ bool useIPv6)
{
std::vector<std::string> addresses;
addresses.reserve(endpointSet.Endpoints.size());
for (const auto& endpoint : endpointSet.Endpoints) {
- addresses.push_back(NNet::BuildServiceAddress(endpoint.Fqdn, endpoint.Port));
+ if (useIPv6 && !endpoint.IP6Address.empty()) {
+ addresses.push_back(NNet::FormatNetworkAddress(endpoint.IP6Address, endpoint.Port));
+ } else if (useIPv4 && !endpoint.IP4Address.empty()) {
+ addresses.push_back(NNet::FormatNetworkAddress(endpoint.IP4Address, endpoint.Port));
+ } else {
+ addresses.push_back(NNet::BuildServiceAddress(endpoint.Fqdn, endpoint.Port));
+ }
}
return addresses;
}
diff --git a/yt/yt/core/rpc/helpers.h b/yt/yt/core/rpc/helpers.h
index c631684555..0624017613 100644
--- a/yt/yt/core/rpc/helpers.h
+++ b/yt/yt/core/rpc/helpers.h
@@ -93,7 +93,10 @@ void WriteAuthenticationIdentityToProto(T* proto, const TAuthenticationIdentity&
template <class T>
TAuthenticationIdentity ParseAuthenticationIdentityFromProto(const T& proto);
-std::vector<std::string> AddressesFromEndpointSet(const NServiceDiscovery::TEndpointSet& endpointSet);
+std::vector<std::string> AddressesFromEndpointSet(
+ const NServiceDiscovery::TEndpointSet& endpointSet,
+ bool useIPv4 = false,
+ bool useIPv6 = false);
////////////////////////////////////////////////////////////////////////////////
diff --git a/yt/yt/core/rpc/server_detail.cpp b/yt/yt/core/rpc/server_detail.cpp
index 3deb728c2d..dbbdd0e2bf 100644
--- a/yt/yt/core/rpc/server_detail.cpp
+++ b/yt/yt/core/rpc/server_detail.cpp
@@ -374,6 +374,9 @@ std::optional<TDuration> TServiceContextBase::GetExecutionDuration() const
return std::nullopt;
}
+void TServiceContextBase::RecordThrottling(TDuration /*throttleDuration*/)
+{ }
+
TTraceContextPtr TServiceContextBase::GetTraceContext() const
{
return nullptr;
@@ -581,6 +584,11 @@ std::optional<TDuration> TServiceContextWrapper::GetExecutionDuration() const
return UnderlyingContext_->GetExecutionDuration();
}
+void TServiceContextWrapper::RecordThrottling(TDuration throttleDuration)
+{
+ return UnderlyingContext_->RecordThrottling(throttleDuration);
+}
+
TTraceContextPtr TServiceContextWrapper::GetTraceContext() const
{
return UnderlyingContext_->GetTraceContext();
diff --git a/yt/yt/core/rpc/server_detail.h b/yt/yt/core/rpc/server_detail.h
index 16c56e9eb1..3683ca7477 100644
--- a/yt/yt/core/rpc/server_detail.h
+++ b/yt/yt/core/rpc/server_detail.h
@@ -45,6 +45,7 @@ public:
std::optional<TInstant> GetFinishInstant() const override;
std::optional<TDuration> GetWaitDuration() const override;
std::optional<TDuration> GetExecutionDuration() const override;
+ void RecordThrottling(TDuration throttleDuration) override;
NTracing::TTraceContextPtr GetTraceContext() const override;
std::optional<TDuration> GetTraceContextTime() const override;
@@ -208,6 +209,7 @@ public:
std::optional<TInstant> GetFinishInstant() const override;
std::optional<TDuration> GetWaitDuration() const override;
std::optional<TDuration> GetExecutionDuration() const override;
+ void RecordThrottling(TDuration throttleDuration) override;
NTracing::TTraceContextPtr GetTraceContext() const override;
std::optional<TDuration> GetTraceContextTime() const override;
diff --git a/yt/yt/core/rpc/service.h b/yt/yt/core/rpc/service.h
index 8511ad549a..3eb6d06a70 100644
--- a/yt/yt/core/rpc/service.h
+++ b/yt/yt/core/rpc/service.h
@@ -82,6 +82,9 @@ struct IServiceContext
//! Returns time between request execution start and the moment of reply or cancellation (if it already happened).
virtual std::optional<TDuration> GetExecutionDuration() const = 0;
+ //! Substract given throttle duration time from request execution time.
+ virtual void RecordThrottling(TDuration throttleDuration) = 0;
+
//! Returns trace context associated with request.
virtual NTracing::TTraceContextPtr GetTraceContext() const = 0;
diff --git a/yt/yt/core/rpc/service_detail.cpp b/yt/yt/core/rpc/service_detail.cpp
index 970552940b..87d3fcd032 100644
--- a/yt/yt/core/rpc/service_detail.cpp
+++ b/yt/yt/core/rpc/service_detail.cpp
@@ -50,8 +50,6 @@ static const auto InfiniteRequestThrottlerConfig = New<TThroughputThrottlerConfi
static const auto DefaultLoggingSuppressionFailedRequestThrottlerConfig = TThroughputThrottlerConfig::Create(1'000);
constexpr int MaxUserAgentLength = 200;
-constexpr TStringBuf UnknownUserAgent = "unknown";
-
constexpr auto ServiceLivenessCheckPeriod = TDuration::MilliSeconds(100);
////////////////////////////////////////////////////////////////////////////////
@@ -266,15 +264,15 @@ auto TServiceBase::TMethodDescriptor::SetHandleMethodError(bool value) const ->
////////////////////////////////////////////////////////////////////////////////
-TServiceBase::TErrorCodeCounters::TErrorCodeCounters(NProfiling::TProfiler profiler)
+TServiceBase::TErrorCodeCounter::TErrorCodeCounter(NProfiling::TProfiler profiler)
: Profiler_(std::move(profiler))
{ }
-NProfiling::TCounter* TServiceBase::TErrorCodeCounters::GetCounter(TErrorCode code)
+void TServiceBase::TErrorCodeCounter::Increment(TErrorCode code)
{
- return CodeToCounter_.FindOrInsert(code, [&] {
+ CodeToCounter_.FindOrInsert(code, [&] {
return Profiler_.WithTag("code", ToString(code)).Counter("/code_count");
- }).first;
+ }).first->Increment();
}
////////////////////////////////////////////////////////////////////////////////
@@ -291,7 +289,7 @@ TServiceBase::TMethodPerformanceCounters::TMethodPerformanceCounters(
, RequestMessageAttachmentSizeCounter(profiler.Counter("/request_message_attachment_bytes"))
, ResponseMessageBodySizeCounter(profiler.Counter("/response_message_body_bytes"))
, ResponseMessageAttachmentSizeCounter(profiler.Counter("/response_message_attachment_bytes"))
- , ErrorCodeCounters(profiler)
+ , ErrorCodeCounter(profiler)
{
if (timeHistogramConfig && timeHistogramConfig->CustomBounds) {
const auto& customBounds = *timeHistogramConfig->CustomBounds;
@@ -340,19 +338,6 @@ TRequestQueue* TServiceBase::TRuntimeMethodInfo::GetDefaultRequestQueue()
////////////////////////////////////////////////////////////////////////////////
-TServiceBase::TPerformanceCounters::TPerformanceCounters(const NProfiling::TProfiler& profiler)
- : Profiler_(profiler.WithHot().WithSparse())
-{ }
-
-NProfiling::TCounter* TServiceBase::TPerformanceCounters::GetRequestsPerUserAgentCounter(TStringBuf userAgent)
-{
- return RequestsPerUserAgent_.FindOrInsert(userAgent, [&] {
- return Profiler_.WithRequiredTag("user_agent", TString(userAgent)).Counter("/user_agent");
- }).first;
-}
-
-////////////////////////////////////////////////////////////////////////////////
-
class TServiceBase::TServiceContext
: public TServiceContextBase
{
@@ -375,9 +360,11 @@ public:
, TraceContext_(std::move(acceptedRequest.TraceContext))
, RequestQueue_(acceptedRequest.RequestQueue)
, ThrottledError_(std::move(acceptedRequest.ThrottledError))
- , MethodPerformanceCounters_(acceptedRequest.MethodPerformanceCounters)
+ , MethodPerformanceCounters_(Service_->GetMethodPerformanceCounters(
+ RuntimeInfo_,
+ {GetAuthenticationIdentity().UserTag, RequestQueue_}))
, PerformanceCounters_(Service_->GetPerformanceCounters())
- , ArriveInstant_(acceptedRequest.ArriveInstant)
+ , ArriveInstant_(NProfiling::GetInstant())
{
YT_ASSERT(RequestMessage_);
YT_ASSERT(ReplyBus_);
@@ -600,6 +587,14 @@ public:
return ExecutionTime_;
}
+ void RecordThrottling(TDuration throttleDuration) override
+ {
+ ThrottlingTime_ = ThrottlingTime_ ? *ThrottlingTime_ + throttleDuration : throttleDuration;
+ if (ExecutionTime_) {
+ *ExecutionTime_ -= throttleDuration;
+ }
+ }
+
TTraceContextPtr GetTraceContext() const override
{
return TraceContext_;
@@ -724,6 +719,7 @@ private:
std::optional<TInstant> RunInstant_;
std::optional<TInstant> ReplyInstant_;
std::optional<TInstant> CancelInstant_;
+ std::optional<TDuration> ThrottlingTime_;
std::optional<TDuration> ExecutionTime_;
std::optional<TDuration> TotalTime_;
@@ -756,6 +752,24 @@ private:
void Initialize()
{
+ constexpr TStringBuf UnknownUserAgent = "unknown";
+ auto userAgent = RequestHeader_->has_user_agent()
+ ? TStringBuf(RequestHeader_->user_agent())
+ : UnknownUserAgent;
+ PerformanceCounters_->IncrementRequestsPerUserAgent(userAgent.SubString(0, MaxUserAgentLength));
+
+ MethodPerformanceCounters_->RequestCounter.Increment();
+ MethodPerformanceCounters_->RequestMessageBodySizeCounter.Increment(
+ GetMessageBodySize(RequestMessage_));
+ MethodPerformanceCounters_->RequestMessageAttachmentSizeCounter.Increment(
+ GetTotalMessageAttachmentSize(RequestMessage_));
+
+ if (RequestHeader_->has_start_time()) {
+ auto retryStart = FromProto<TInstant>(RequestHeader_->start_time());
+ auto now = NProfiling::GetInstant();
+ MethodPerformanceCounters_->RemoteWaitTimeCounter.Record(now - retryStart);
+ }
+
// COMPAT(danilalexeev): legacy RPC codecs
RequestCodec_ = RequestHeader_->has_request_codec()
? CheckedEnumCast<NCompression::ECodec>(RequestHeader_->request_codec())
@@ -1013,14 +1027,16 @@ private:
ReplyInstant_ = NProfiling::GetInstant();
ExecutionTime_ = RunInstant_ ? *ReplyInstant_ - *RunInstant_ : TDuration();
+ if (RunInstant_ && ThrottlingTime_) {
+ *ExecutionTime_ -= *ThrottlingTime_;
+ }
TotalTime_ = *ReplyInstant_ - ArriveInstant_;
MethodPerformanceCounters_->ExecutionTimeCounter.Record(*ExecutionTime_);
MethodPerformanceCounters_->TotalTimeCounter.Record(*TotalTime_);
if (!Error_.IsOK()) {
if (Service_->EnableErrorCodeCounter_.load()) {
- const auto* counter = MethodPerformanceCounters_->ErrorCodeCounters.GetCounter(Error_.GetNonTrivialCode());
- counter->Increment();
+ MethodPerformanceCounters_->ErrorCodeCounter.Increment(Error_.GetNonTrivialCode());
} else {
MethodPerformanceCounters_->FailedRequestCounter.Increment();
}
@@ -1117,7 +1133,7 @@ private:
TraceContext_->AddTag(RequestUser, builder.Flush());
}
}
- YT_LOG_EVENT_WITH_ANCHOR(Logger, LogLevel_, RuntimeInfo_->RequestLoggingAnchor, logMessage);
+ YT_LOG_EVENT_WITH_DYNAMIC_ANCHOR(Logger, LogLevel_, RuntimeInfo_->RequestLoggingAnchor, logMessage);
}
void LogResponse() override
@@ -1164,7 +1180,7 @@ private:
if (TraceContext_ && TraceContext_->IsRecorded()) {
TraceContext_->AddTag(ResponseInfoAnnotation, logMessage);
}
- YT_LOG_EVENT_WITH_ANCHOR(Logger, LogLevel_, RuntimeInfo_->ResponseLoggingAnchor, logMessage);
+ YT_LOG_EVENT_WITH_DYNAMIC_ANCHOR(Logger, LogLevel_, RuntimeInfo_->ResponseLoggingAnchor, logMessage);
}
@@ -1303,7 +1319,7 @@ private:
////////////////////////////////////////////////////////////////////////////////
-TRequestQueue::TRequestQueue(const std::string& name, const NProfiling::TProfiler& profiler)
+TRequestQueue::TRequestQueue(const std::string& name, NProfiling::TProfiler profiler)
: Name_(name)
, BytesThrottler_{CreateReconfigurableThroughputThrottler(InfiniteRequestThrottlerConfig,
NLogging::TLogger(),
@@ -1606,20 +1622,23 @@ void TRequestQueue::SubscribeToThrottlers()
////////////////////////////////////////////////////////////////////////////////
-bool TServiceBase::TRuntimeMethodInfo::TPerformanceCountersKeyEquals::operator()(
- const TNonowningPerformanceCountersKey& lhs,
- const TNonowningPerformanceCountersKey& rhs) const
+struct TServiceBase::TRuntimeMethodInfo::TPerformanceCountersKeyEquals
{
- return lhs == rhs;
-}
+ bool operator()(
+ const TNonowningPerformanceCountersKey& lhs,
+ const TNonowningPerformanceCountersKey& rhs) const
+ {
+ return lhs == rhs;
+ }
-bool TServiceBase::TRuntimeMethodInfo::TPerformanceCountersKeyEquals::operator()(
- const TOwningPerformanceCountersKey& lhs,
- const TNonowningPerformanceCountersKey& rhs) const
-{
- const auto& [lhsUserTag, lhsRequestQueue] = lhs;
- return TNonowningPerformanceCountersKey{lhsUserTag, lhsRequestQueue} == rhs;
-}
+ bool operator()(
+ const TOwningPerformanceCountersKey& lhs,
+ const TNonowningPerformanceCountersKey& rhs) const
+ {
+ const auto& [lhsUserTag, lhsRequestQueue] = lhs;
+ return TNonowningPerformanceCountersKey{lhsUserTag, lhsRequestQueue} == rhs;
+ }
+};
////////////////////////////////////////////////////////////////////////////////
@@ -1668,15 +1687,8 @@ void TServiceBase::HandleRequest(
{
SetActive();
- auto arriveInstant = NProfiling::GetInstant();
-
- const auto& method = header->method();
+ auto method = FromProto<TString>(header->method());
auto requestId = FromProto<TRequestId>(header->request_id());
- auto userAgent = header->has_user_agent()
- ? TStringBuf(header->user_agent()).SubString(0, MaxUserAgentLength)
- : UnknownUserAgent;
- const auto& user = header->has_user() ? header->user() : RootUserName;
- const auto& userTag = header->has_user_tag() ? header->user_tag() : user;
auto replyError = [&] (TError error) {
ReplyError(std::move(error), *header, replyBus);
@@ -1689,6 +1701,11 @@ void TServiceBase::HandleRequest(
return;
}
+ if (auto error = DoCheckRequestCompatibility(*header); !error.IsOK()) {
+ replyError(std::move(error));
+ return;
+ }
+
auto* runtimeInfo = FindMethodInfo(method);
if (!runtimeInfo) {
replyError(TError(
@@ -1697,29 +1714,8 @@ void TServiceBase::HandleRequest(
return;
}
- auto* requestQueue = GetRequestQueue(runtimeInfo, *header);
-
- const auto* requestsPerUserAgentCounter = PerformanceCounters_->GetRequestsPerUserAgentCounter(userAgent);
- requestsPerUserAgentCounter->Increment();
-
- auto* methodPerformanceCounters = GetMethodPerformanceCounters(runtimeInfo, {userTag, requestQueue});
- methodPerformanceCounters->RequestCounter.Increment();
- methodPerformanceCounters->RequestMessageBodySizeCounter.Increment(GetMessageBodySize(message));
- methodPerformanceCounters->RequestMessageAttachmentSizeCounter.Increment(GetTotalMessageAttachmentSize(message));
-
- if (header->has_start_time()) {
- auto retryStart = FromProto<TInstant>(header->start_time());
- methodPerformanceCounters->RemoteWaitTimeCounter.Record(arriveInstant - retryStart);
- }
-
- if (auto error = DoCheckRequestCompatibility(*header); !error.IsOK()) {
- replyError(std::move(error));
- return;
- }
-
auto memoryGuard = TMemoryUsageTrackerGuard::Acquire(MemoryUsageTracker_, TypicalRequestSize);
message = TrackMemory(MemoryUsageTracker_, std::move(message));
-
if (MemoryUsageTracker_ && MemoryUsageTracker_->IsExceeded()) {
return replyError(TError(
NRpc::EErrorCode::MemoryPressure,
@@ -1730,12 +1726,14 @@ void TServiceBase::HandleRequest(
auto traceContext = tracingMode == ERequestTracingMode::Disable
? NTracing::TTraceContextPtr()
: GetOrCreateHandlerTraceContext(*header, tracingMode == ERequestTracingMode::Force);
-
if (traceContext && traceContext->IsRecorded()) {
traceContext->AddTag(EndpointAnnotation, replyBus->GetEndpointDescription());
}
- auto throttledError = GetThrottledError(*header);
+ auto* requestQueue = GetRequestQueue(runtimeInfo, *header);
+ RegisterRequestQueue(runtimeInfo, requestQueue);
+
+ auto maybeThrottled = GetThrottledError(*header);
if (requestQueue->IsQueueSizeLimitExceeded()) {
runtimeInfo->RequestQueueSizeLimitErrorCounter.Increment();
@@ -1744,7 +1742,7 @@ void TServiceBase::HandleRequest(
"Request queue size limit exceeded")
<< TErrorAttribute("limit", runtimeInfo->QueueSizeLimit.load())
<< TErrorAttribute("queue", requestQueue->GetName())
- << throttledError);
+ << maybeThrottled);
return;
}
@@ -1755,7 +1753,7 @@ void TServiceBase::HandleRequest(
"Request queue bytes size limit exceeded")
<< TErrorAttribute("limit", runtimeInfo->QueueByteSizeLimit.load())
<< TErrorAttribute("queue", requestQueue->GetName())
- << throttledError);
+ << maybeThrottled);
return;
}
@@ -1763,7 +1761,6 @@ void TServiceBase::HandleRequest(
// NOTE: Do not use replyError() after this line.
TAcceptedRequest acceptedRequest{
- .ArriveInstant = arriveInstant,
.RequestId = requestId,
.ReplyBus = std::move(replyBus),
.RuntimeInfo = std::move(runtimeInfo),
@@ -1771,8 +1768,7 @@ void TServiceBase::HandleRequest(
.Header = std::move(header),
.Message = std::move(message),
.RequestQueue = requestQueue,
- .MethodPerformanceCounters = methodPerformanceCounters,
- .ThrottledError = throttledError,
+ .ThrottledError = maybeThrottled,
.MemoryGuard = std::move(memoryGuard),
.MemoryUsageTracker = MemoryUsageTracker_,
};
@@ -1925,49 +1921,55 @@ TRequestQueue* TServiceBase::GetRequestQueue(
const NRpc::NProto::TRequestHeader& requestHeader)
{
TRequestQueue* requestQueue = nullptr;
- if (const auto& provider = runtimeInfo->Descriptor.RequestQueueProvider) {
+ if (auto& provider = runtimeInfo->Descriptor.RequestQueueProvider) {
requestQueue = provider->GetQueue(requestHeader);
}
if (!requestQueue) {
requestQueue = runtimeInfo->DefaultRequestQueue.Get();
}
+ return requestQueue;
+}
- if (requestQueue->Register(this, runtimeInfo)) {
- const auto& method = runtimeInfo->Descriptor.Method;
- YT_LOG_DEBUG("Request queue registered (Method: %v, Queue: %v)",
- method,
- requestQueue->GetName());
+void TServiceBase::RegisterRequestQueue(
+ TRuntimeMethodInfo* runtimeInfo,
+ TRequestQueue* requestQueue)
+{
+ if (!requestQueue->Register(this, runtimeInfo)) {
+ return;
+ }
- auto profiler = runtimeInfo->Profiler.WithSparse();
- if (runtimeInfo->Descriptor.RequestQueueProvider) {
- profiler = profiler.WithTag("queue", requestQueue->GetName());
- }
- profiler.AddFuncGauge("/request_queue_size", MakeStrong(this), [=] {
- return requestQueue->GetQueueSize();
- });
- profiler.AddFuncGauge("/request_queue_byte_size", MakeStrong(this), [=] {
- return requestQueue->GetQueueByteSize();
- });
- profiler.AddFuncGauge("/concurrency", MakeStrong(this), [=] {
- return requestQueue->GetConcurrency();
- });
- profiler.AddFuncGauge("/concurrency_byte", MakeStrong(this), [=] {
- return requestQueue->GetConcurrencyByte();
- });
+ const auto& method = runtimeInfo->Descriptor.Method;
+ YT_LOG_DEBUG("Request queue registered (Method: %v, Queue: %v)",
+ method,
+ requestQueue->GetName());
- TMethodConfigPtr methodConfig;
- if (auto config = Config_.Acquire()) {
- methodConfig = GetOrDefault(config->Methods, method);
- }
- ConfigureRequestQueue(runtimeInfo, requestQueue, methodConfig);
+ auto profiler = runtimeInfo->Profiler.WithSparse();
+ if (runtimeInfo->Descriptor.RequestQueueProvider) {
+ profiler = profiler.WithTag("queue", requestQueue->GetName());
+ }
+ profiler.AddFuncGauge("/request_queue_size", MakeStrong(this), [=] {
+ return requestQueue->GetQueueSize();
+ });
+ profiler.AddFuncGauge("/request_queue_byte_size", MakeStrong(this), [=] {
+ return requestQueue->GetQueueByteSize();
+ });
+ profiler.AddFuncGauge("/concurrency", MakeStrong(this), [=] {
+ return requestQueue->GetConcurrency();
+ });
+ profiler.AddFuncGauge("/concurrency_byte", MakeStrong(this), [=] {
+ return requestQueue->GetConcurrencyByte();
+ });
- {
- auto guard = Guard(runtimeInfo->RequestQueuesLock);
- runtimeInfo->RequestQueues.push_back(requestQueue);
- }
+ TMethodConfigPtr methodConfig;
+ if (auto config = Config_.Acquire()) {
+ methodConfig = GetOrDefault(config->Methods, method);
}
+ ConfigureRequestQueue(runtimeInfo, requestQueue, methodConfig);
- return requestQueue;
+ {
+ auto guard = Guard(runtimeInfo->RequestQueuesLock);
+ runtimeInfo->RequestQueues.push_back(requestQueue);
+ }
}
void TServiceBase::ConfigureRequestQueue(
@@ -2694,13 +2696,13 @@ TFuture<void> TServiceBase::Stop()
return StopResult_.ToFuture();
}
-TServiceBase::TRuntimeMethodInfo* TServiceBase::FindMethodInfo(const std::string& method)
+TServiceBase::TRuntimeMethodInfo* TServiceBase::FindMethodInfo(const TString& method)
{
auto it = MethodMap_.find(method);
return it == MethodMap_.end() ? nullptr : it->second.Get();
}
-TServiceBase::TRuntimeMethodInfo* TServiceBase::GetMethodInfoOrThrow(const std::string& method)
+TServiceBase::TRuntimeMethodInfo* TServiceBase::GetMethodInfoOrThrow(const TString& method)
{
auto* runtimeInfo = FindMethodInfo(method);
if (!runtimeInfo) {
diff --git a/yt/yt/core/rpc/service_detail.h b/yt/yt/core/rpc/service_detail.h
index 4641c2f98a..ab32bce755 100644
--- a/yt/yt/core/rpc/service_detail.h
+++ b/yt/yt/core/rpc/service_detail.h
@@ -657,12 +657,11 @@ protected:
TMethodDescriptor SetHandleMethodError(bool value) const;
};
- class TErrorCodeCounters
+ struct TErrorCodeCounter
{
- public:
- explicit TErrorCodeCounters(NProfiling::TProfiler profiler);
+ explicit TErrorCodeCounter(NProfiling::TProfiler profiler);
- NProfiling::TCounter* GetCounter(TErrorCode code);
+ void Increment(TErrorCode code);
private:
const NProfiling::TProfiler Profiler_;
@@ -718,7 +717,7 @@ protected:
NProfiling::TCounter ResponseMessageAttachmentSizeCounter;
//! Counts the number of errors, per error code.
- TErrorCodeCounters ErrorCodeCounters;
+ TErrorCodeCounter ErrorCodeCounter;
};
using TMethodPerformanceCountersPtr = TIntrusivePtr<TMethodPerformanceCounters>;
@@ -764,24 +763,13 @@ protected:
using TNonowningPerformanceCountersKey = std::tuple<TStringBuf, TRequestQueue*>;
using TOwningPerformanceCountersKey = std::tuple<TString, TRequestQueue*>;
using TPerformanceCountersKeyHash = THash<TNonowningPerformanceCountersKey>;
-
- struct TPerformanceCountersKeyEquals
- {
- bool operator()(
- const TNonowningPerformanceCountersKey& lhs,
- const TNonowningPerformanceCountersKey& rhs) const;
- bool operator()(
- const TOwningPerformanceCountersKey& lhs,
- const TNonowningPerformanceCountersKey& rhs) const;
- };
-
+ struct TPerformanceCountersKeyEquals;
using TPerformanceCountersMap = NConcurrency::TSyncMap<
TOwningPerformanceCountersKey,
TMethodPerformanceCountersPtr,
TPerformanceCountersKeyHash,
TPerformanceCountersKeyEquals
>;
-
TPerformanceCountersMap PerformanceCountersMap;
TMethodPerformanceCountersPtr BasePerformanceCounters;
TMethodPerformanceCountersPtr RootPerformanceCounters;
@@ -801,9 +789,16 @@ protected:
: public TRefCounted
{
public:
- explicit TPerformanceCounters(const NProfiling::TProfiler& profiler);
+ explicit TPerformanceCounters(const NProfiling::TProfiler& profiler)
+ : Profiler_(profiler.WithHot().WithSparse())
+ { }
- NProfiling::TCounter* GetRequestsPerUserAgentCounter(TStringBuf userAgent);
+ void IncrementRequestsPerUserAgent(TStringBuf userAgent)
+ {
+ RequestsPerUserAgent_.FindOrInsert(userAgent, [&] {
+ return Profiler_.WithRequiredTag("user_agent", TString(userAgent)).Counter("/user_agent");
+ }).first->Increment();
+ }
private:
const NProfiling::TProfiler Profiler_;
@@ -851,10 +846,10 @@ protected:
//! Returns a (non-owning!) pointer to TRuntimeMethodInfo for a given method's name
//! or |nullptr| if no such method is registered.
- TRuntimeMethodInfo* FindMethodInfo(const std::string& method);
+ TRuntimeMethodInfo* FindMethodInfo(const TString& method);
//! Similar to #FindMethodInfo but throws if no method is found.
- TRuntimeMethodInfo* GetMethodInfoOrThrow(const std::string& method);
+ TRuntimeMethodInfo* GetMethodInfoOrThrow(const TString& method);
//! Returns the default invoker passed during construction.
const IInvokerPtr& GetDefaultInvoker() const;
@@ -996,7 +991,6 @@ private:
struct TAcceptedRequest
{
- TInstant ArriveInstant;
TRequestId RequestId;
NYT::NBus::IBusPtr ReplyBus;
TRuntimeMethodInfo* RuntimeInfo;
@@ -1004,7 +998,6 @@ private:
std::unique_ptr<NRpc::NProto::TRequestHeader> Header;
TSharedRefArray Message;
TRequestQueue* RequestQueue;
- TMethodPerformanceCounters* MethodPerformanceCounters;
std::optional<TError> ThrottledError;
TMemoryUsageTrackerGuard MemoryGuard;
IMemoryUsageTrackerPtr MemoryUsageTracker;
@@ -1029,6 +1022,9 @@ private:
TRequestQueue* GetRequestQueue(
TRuntimeMethodInfo* runtimeInfo,
const NRpc::NProto::TRequestHeader& requestHeader);
+ void RegisterRequestQueue(
+ TRuntimeMethodInfo* runtimeInfo,
+ TRequestQueue* requestQueue);
void ConfigureRequestQueue(
TRuntimeMethodInfo* runtimeInfo,
TRequestQueue* requestQueue,
@@ -1075,7 +1071,6 @@ private:
static TString GetDiscoverRequestPayload(const TCtxDiscoverPtr& context);
void OnServiceLivenessCheck();
-
};
DEFINE_REFCOUNTED_TYPE(TServiceBase)
@@ -1086,9 +1081,7 @@ class TRequestQueue
: public TRefCounted
{
public:
- TRequestQueue(
- const std::string& name,
- const NProfiling::TProfiler& profiler);
+ TRequestQueue(const std::string& name, NProfiling::TProfiler profiler);
bool Register(TServiceBase* service, TServiceBase::TRuntimeMethodInfo* runtimeInfo);
void Configure(const TMethodConfigPtr& config);
diff --git a/yt/yt/library/formats/arrow_writer.cpp b/yt/yt/library/formats/arrow_writer.cpp
index 657da97ce5..a3ff967bc6 100644
--- a/yt/yt/library/formats/arrow_writer.cpp
+++ b/yt/yt/library/formats/arrow_writer.cpp
@@ -49,14 +49,14 @@ const TString AlignmentString(ArrowAlignment, 0);
flatbuffers::Offset<flatbuffers::String> SerializeString(
flatbuffers::FlatBufferBuilder* flatbufBuilder,
- const TString& str)
+ const std::string& str)
{
return flatbufBuilder->CreateString(str.data(), str.length());
}
std::tuple<org::apache::arrow::flatbuf::Type, flatbuffers::Offset<void>> SerializeColumnType(
flatbuffers::FlatBufferBuilder* flatbufBuilder,
- TColumnSchema schema)
+ const TColumnSchema& schema)
{
auto simpleType = CastToV1Type(schema.LogicalType()).first;
switch (simpleType) {
diff --git a/yt/yt/library/formats/protobuf.cpp b/yt/yt/library/formats/protobuf.cpp
index 3da3f04b48..a710bf9e61 100644
--- a/yt/yt/library/formats/protobuf.cpp
+++ b/yt/yt/library/formats/protobuf.cpp
@@ -1013,7 +1013,8 @@ void TProtobufFormatDescriptionBase<TType>::InitFromProtobufSchema(
field->GetType());
}
const auto& enumerationConfig = field->AsMap();
- EnumerationDescriptionMap_.emplace(name, CreateEnumerationMap(TimestampColumnName, enumerationConfig));
+ // TODO(babenko): migrate to std::string
+ EnumerationDescriptionMap_.emplace(name, CreateEnumerationMap(TString(TimestampColumnName), enumerationConfig));
}
}
diff --git a/yt/yt/library/formats/schemaful_dsv_parser.cpp b/yt/yt/library/formats/schemaful_dsv_parser.cpp
index 3149f28851..7137e2672d 100644
--- a/yt/yt/library/formats/schemaful_dsv_parser.cpp
+++ b/yt/yt/library/formats/schemaful_dsv_parser.cpp
@@ -26,9 +26,9 @@ public:
void Finish() override;
private:
- IYsonConsumer* Consumer_;
- TSchemafulDsvFormatConfigPtr Config_;
- const std::vector<TString>& Columns_;
+ IYsonConsumer* const Consumer_;
+ const TSchemafulDsvFormatConfigPtr Config_;
+ const std::vector<std::string>& Columns_;
TEscapeTable EscapeTable_;
@@ -49,8 +49,8 @@ private:
////////////////////////////////////////////////////////////////////////////////
TSchemafulDsvParser::TSchemafulDsvParser(
- IYsonConsumer* consumer,
- TSchemafulDsvFormatConfigPtr config)
+ IYsonConsumer* consumer,
+ TSchemafulDsvFormatConfigPtr config)
: Consumer_(consumer)
, Config_(config)
, Columns_(Config_->GetColumnsOrThrow())
diff --git a/yt/yt/library/formats/schemaful_dsv_writer.cpp b/yt/yt/library/formats/schemaful_dsv_writer.cpp
index 4d2e7e1e6e..b37486c581 100644
--- a/yt/yt/library/formats/schemaful_dsv_writer.cpp
+++ b/yt/yt/library/formats/schemaful_dsv_writer.cpp
@@ -266,9 +266,9 @@ private:
////////////////////////////////////////////////////////////////////////////////
-void ValidateDuplicateColumns(const std::vector<TString>& columns)
+void ValidateDuplicateColumns(const std::vector<std::string>& columns)
{
- THashSet<TString> names;
+ THashSet<std::string> names;
for (const auto& name : columns) {
if (!names.insert(name).second) {
THROW_ERROR_EXCEPTION("Duplicate column name %Qv in schemaful DSV config",
diff --git a/yt/yt/library/formats/web_json_writer.cpp b/yt/yt/library/formats/web_json_writer.cpp
index d717cafb10..8049e134c9 100644
--- a/yt/yt/library/formats/web_json_writer.cpp
+++ b/yt/yt/library/formats/web_json_writer.cpp
@@ -48,7 +48,7 @@ static constexpr auto ContextBufferCapacity = 1_MB;
class TWebJsonColumnFilter
{
public:
- TWebJsonColumnFilter(int maxSelectedColumnCount, std::optional<THashSet<TString>> names)
+ TWebJsonColumnFilter(int maxSelectedColumnCount, std::optional<THashSet<std::string, THash<TStringBuf>, TEqualTo<>>> names)
: MaxSelectedColumnCount_(maxSelectedColumnCount)
, Names_(std::move(names))
{ }
@@ -68,7 +68,7 @@ public:
private:
const int MaxSelectedColumnCount_;
- std::optional<THashSet<TString>> Names_;
+ const std::optional<THashSet<std::string, THash<TStringBuf>, TEqualTo<>>> Names_;
THashSet<ui16> AcceptedColumnIds_;
@@ -91,7 +91,7 @@ private:
TWebJsonColumnFilter CreateWebJsonColumnFilter(const TWebJsonFormatConfigPtr& webJsonConfig)
{
- std::optional<THashSet<TString>> columnNames;
+ std::optional<THashSet<std::string, THash<TStringBuf>, TEqualTo<>>> columnNames;
if (webJsonConfig->ColumnNames) {
columnNames.emplace();
for (const auto& columnName : *webJsonConfig->ColumnNames) {
diff --git a/yt/yt/library/formats/yamred_dsv_parser.cpp b/yt/yt/library/formats/yamred_dsv_parser.cpp
index 476e760ea6..79246cb92a 100644
--- a/yt/yt/library/formats/yamred_dsv_parser.cpp
+++ b/yt/yt/library/formats/yamred_dsv_parser.cpp
@@ -17,39 +17,39 @@ class TYamredDsvParserConsumer
public:
TYamredDsvParserConsumer(IYsonConsumer* consumer, TYamredDsvFormatConfigPtr config)
: TYamrConsumerBase(consumer)
- , Config(config)
- , DsvParser(CreateParserForDsv(consumer, ConvertTo<TDsvFormatConfigPtr>(Config), /*wrapWithMap*/ false))
+ , Config_(config)
+ , DsvParser_(CreateParserForDsv(consumer, ConvertTo<TDsvFormatConfigPtr>(Config_), /*wrapWithMap*/ false))
{ }
void ConsumeKey(TStringBuf key) override
{
Consumer->OnListItem();
Consumer->OnBeginMap();
- ConsumeFields(Config->KeyColumnNames, key);
+ ConsumeFields(Config_->KeyColumnNames, key);
}
void ConsumeSubkey(TStringBuf subkey) override
{
- ConsumeFields(Config->SubkeyColumnNames, subkey);
+ ConsumeFields(Config_->SubkeyColumnNames, subkey);
}
void ConsumeValue(TStringBuf value) override
{
- DsvParser->Read(value);
- DsvParser->Finish();
+ DsvParser_->Read(value);
+ DsvParser_->Finish();
Consumer->OnEndMap();
}
private:
- TYamredDsvFormatConfigPtr Config;
- std::unique_ptr<IParser> DsvParser;
+ const TYamredDsvFormatConfigPtr Config_;
+ const std::unique_ptr<IParser> DsvParser_;
void ConsumeFields(
- const std::vector<TString>& fieldNames,
+ const std::vector<std::string>& fieldNames,
TStringBuf wholeField)
{
static const char* emptyString = "";
- char delimiter = Config->YamrKeysSeparator;
+ char delimiter = Config_->YamrKeysSeparator;
std::vector<TStringBuf> fields;
if (wholeField.length() == 0) {
diff --git a/yt/yt/library/named_value/named_value.h b/yt/yt/library/named_value/named_value.h
index 3a76346589..67592fe9f3 100644
--- a/yt/yt/library/named_value/named_value.h
+++ b/yt/yt/library/named_value/named_value.h
@@ -38,11 +38,13 @@ NTableClient::TUnversionedOwningRow MakeRow(
class TNamedValue
{
public:
- struct TAny {
+ struct TAny
+ {
TString Value;
};
- struct TComposite {
+ struct TComposite
+ {
TString Value;
};
diff --git a/yt/yt/library/numeric/unittests/piecewise_linear_function_ut.cpp b/yt/yt/library/numeric/unittests/piecewise_linear_function_ut.cpp
index 7fde320ec0..4ea3e3e815 100644
--- a/yt/yt/library/numeric/unittests/piecewise_linear_function_ut.cpp
+++ b/yt/yt/library/numeric/unittests/piecewise_linear_function_ut.cpp
@@ -810,7 +810,7 @@ TEST_F(TPiecewiseLinearFunctionTest, TestCompose)
{
/* Name */ "compositionLinearWithLinearInjection",
/* Lhs */ TPiecewiseLinearFunction<double>::Linear(0.1, 17, 1.2, 42),
- /* Rhs */ TPiecewiseLinearFunction<double>::Linear(0, 0.3, 100, 0.98),
+ /* Rhs */ TPiecewiseLinearFunction<double>::Linear(0, 0.3, 100, 0.87),
/* Samples */ {
{
/* Argument */ 0,
@@ -819,15 +819,15 @@ TEST_F(TPiecewiseLinearFunctionTest, TestCompose)
},
{
/* Argument */ 100,
- /* ExpectedLeftLimit */ 37,
- /* ExpectedRightLimit */ 37,
+ /* ExpectedLeftLimit */ 34.5,
+ /* ExpectedRightLimit */ 34.5,
}
}
},
{
/* Name */ "compositionLinearWithDiscontinuous",
/* Lhs */ TPiecewiseLinearFunction<double>::Linear(0.1, 17, 1.2, 42),
- /* Rhs */ BuildFunctionFromPoints<double>({{0, 0.1}, {1, 0.3}, {1, 0.98}, {2, 1.2}}),
+ /* Rhs */ BuildFunctionFromPoints<double>({{0, 0.1}, {1, 0.3}, {1, 0.87}, {2, 1.2}}),
/* Samples */ {
{
/* Argument */ 0,
@@ -837,7 +837,7 @@ TEST_F(TPiecewiseLinearFunctionTest, TestCompose)
{
/* Argument */ 1,
/* ExpectedLeftLimit */ 21.5454545454545454545454545454545454545454545454,
- /* ExpectedRightLimit */ 37,
+ /* ExpectedRightLimit */ 34.5,
},
{
/* Argument */ 2,
diff --git a/yt/yt/library/oom/oom.h b/yt/yt/library/oom/oom.h
index 11edd7ca02..33a9206398 100644
--- a/yt/yt/library/oom/oom.h
+++ b/yt/yt/library/oom/oom.h
@@ -23,6 +23,10 @@ void EnableEarlyOomWatchdog(TOomWatchdogOptions options);
struct TTCMallocLimitHandlerOptions
{
TString HeapDumpDirectory;
+
+ // Files structure would have the following form:
+ // HeapDumpDirectory/<ActualName>_FilenameSuffix_Timestamp.ext.
+ TString FilenameSuffix = "";
TDuration Timeout = TDuration::Minutes(5);
};
diff --git a/yt/yt/library/oom/tcmalloc_memory_limit_handler.cpp b/yt/yt/library/oom/tcmalloc_memory_limit_handler.cpp
index 38e7f267a8..5710629d19 100644
--- a/yt/yt/library/oom/tcmalloc_memory_limit_handler.cpp
+++ b/yt/yt/library/oom/tcmalloc_memory_limit_handler.cpp
@@ -35,6 +35,13 @@ namespace NYT {
////////////////////////////////////////////////////////////////////////////////
+TString MakeIncompletePath(const TString& path)
+{
+ return NYT::Format("%v_incomplete", path);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
void CollectAndDumpMemoryProfile(const TString& memoryProfilePath, tcmalloc::ProfileType profileType)
{
auto profile = NYTProf::ReadHeapProfile(profileType);
@@ -45,9 +52,12 @@ void CollectAndDumpMemoryProfile(const TString& memoryProfilePath, tcmalloc::Pro
},
});
- TFileOutput output(memoryProfilePath);
+ auto incompletePath = NYT::MakeIncompletePath(memoryProfilePath);
+
+ TFileOutput output(incompletePath);
NYTProf::WriteProfile(&output, profile);
output.Finish();
+ NFs::Rename(incompletePath, memoryProfilePath);
}
////////////////////////////////////////////////////////////////////////////////
@@ -185,12 +195,22 @@ private:
AbortProcess(ToUnderlying(EProcessExitCode::OK));
}
+ auto MakeSuffixFormatter(const TString& timestamp) const
+ {
+ return NYT::MakeFormatterWrapper([this, &timestamp] (TStringBuilderBase* builder) {
+ if (Options_.FilenameSuffix) {
+ builder->AppendFormat("%v_", Options_.FilenameSuffix);
+ }
+ FormatValue(builder, timestamp, "v");
+ });
+ }
+
TString GetHeapDumpPath(const TString& timestamp) const
{
return Format(
"%v/heap_%v.pb.gz",
Options_.HeapDumpDirectory,
- timestamp);
+ MakeSuffixFormatter(timestamp));
}
TString GetPeakDumpPath(const TString& timestamp) const
@@ -198,7 +218,7 @@ private:
return Format(
"%v/peak_%v.pb.gz",
Options_.HeapDumpDirectory,
- timestamp);
+ MakeSuffixFormatter(timestamp));
}
TString GetProfilePaths(const TString& timestamp) const
@@ -206,7 +226,7 @@ private:
return Format(
"%v/oom_profile_paths_%v.yson",
Options_.HeapDumpDirectory,
- timestamp);
+ MakeSuffixFormatter(timestamp));
}
void ExecWaitForChild(int pid)
diff --git a/yt/yt/library/profiling/sensors_owner/README.md b/yt/yt/library/profiling/sensors_owner/README.md
index 48e9ed83d9..66aa91bd42 100644
--- a/yt/yt/library/profiling/sensors_owner/README.md
+++ b/yt/yt/library/profiling/sensors_owner/README.md
@@ -8,7 +8,7 @@ TSensorsOwner может владеть другими TSensorsOwner. Имеет
* Простейший пример использования:
```cpp
-sensorsOwner.Inc(".my_simple_counter", 1);
+sensorsOwner.Inc("/my_simple_counter", 1);
```
Когда в конкретном месте нужно проинкрементить всего один счетчик.
Объект счетчика в этом случае создатся один раз и будет храниться внутри sensorsOwner.
@@ -22,8 +22,8 @@ void DoSmth(/*... , */ const TSensorsOwner& sensorsOwner)
struct TSensors
{
NYT::NProfiling::TProfiler Profiler;
- NYT::NProfiling::TCounter TotalCount = Profiler.Counter(".count");
- NYT::NProfiling::TCounter FailedCount = Profiler.Counter(".failed_count");
+ NYT::NProfiling::TCounter TotalCount = Profiler.Counter("/total_count");
+ NYT::NProfiling::TCounter FailedCount = Profiler.Counter("/failed_count");
};
// Тут одна и та же ссылка на объект метрик при условии, что в функцию передается один и тот же sensorsOwner.
// Метод `.Get` достаточно эффективен, но всё же лучше не вызывать лишний раз.
@@ -47,7 +47,7 @@ struct THistogramSensors
NYT::NProfiling::TProfiler Profiler;
int Key;
std::vector<TDuration> Buckets;
- NYT::NProfiling::TEventTimer Histogram = Profiler.WithTag("tag", ToString(Key)).TimeHistogram(".another_counter", Buckets);
+ NYT::NProfiling::TEventTimer Histogram = Profiler.WithTag("tag", ToString(Key)).TimeHistogram("/another_counter", Buckets);
};
owner.Get<THistogramSensors>(/*Key*/ 132, /*Buckets*/ std::vector<TDuration>{5s, 10min}).Histogram.Record(6s);
@@ -60,7 +60,7 @@ struct TChildSensors
NYT::NProfiling::TCounter Counter;
TChildSensors(const NYT::NProfiling::TProfiler& p)
- : Counter(p.Counter(".my_counter_2"))
+ : Counter(p.Counter("/my_counter_2"))
{ }
};
```
@@ -70,7 +70,7 @@ struct TChildSensors
struct TSharedSensors final
{
TProfiler Profiler;
- TCounter Counter = Profiler.Counter(".under_ptr_counter");
+ TCounter Counter = Profiler.Counter("/under_ptr_counter");
};
using TSharedSensorsPtr = NYT::TIntrusivePtr<TSharedSensors>;
@@ -79,7 +79,7 @@ owner.Get<TSharedSensorsPtr>()->Counter.Increment(1);
* TSensorsOwner мимикрирует под TProfiler в ряде моментов:
```cpp
-auto subOwner = owner.WithPrefix("prefix.").WithTags(NYT::NProfiling::TTagSet().WithTag({"key", "value2"}));
+auto subOwner = owner.WithPrefix("/prefix").WithTags(NYT::NProfiling::TTagSet().WithTag({"key", "value2"}));
```
## Когда использовать?
diff --git a/yt/yt/library/skiff_ext/parser-inl.h b/yt/yt/library/skiff_ext/parser-inl.h
index 0f781646c9..a48498ba25 100644
--- a/yt/yt/library/skiff_ext/parser-inl.h
+++ b/yt/yt/library/skiff_ext/parser-inl.h
@@ -22,8 +22,8 @@ public:
TConsumer* consumer,
TSkiffSchemaList skiffSchemaList,
const std::vector<TSkiffTableColumnIds>& tablesColumnIds,
- const TString& rangeIndexColumnName,
- const TString& rowIndexColumnName)
+ const std::string& rangeIndexColumnName,
+ const std::string& rowIndexColumnName)
: Consumer_(consumer)
, SkiffSchemaList_(std::move(skiffSchemaList))
{
@@ -183,8 +183,8 @@ TSkiffMultiTableParser<TConsumer>::TSkiffMultiTableParser(
TConsumer* consumer,
TSkiffSchemaList schemaList,
const std::vector<TSkiffTableColumnIds>& tablesColumnIds,
- const TString& rangeIndexColumnName,
- const TString& rowIndexColumnName)
+ const std::string& rangeIndexColumnName,
+ const std::string& rowIndexColumnName)
: ParserImpl_(new TImpl(consumer,
schemaList,
tablesColumnIds,
diff --git a/yt/yt/library/skiff_ext/parser.h b/yt/yt/library/skiff_ext/parser.h
index d9fb4c43fa..82ad4cea50 100644
--- a/yt/yt/library/skiff_ext/parser.h
+++ b/yt/yt/library/skiff_ext/parser.h
@@ -19,8 +19,8 @@ public:
TConsumer* consumer,
NSkiff::TSkiffSchemaList schemaList,
const std::vector<TSkiffTableColumnIds>& tablesColumnIds,
- const TString& rangeIndexColumnName,
- const TString& rowIndexColumnName);
+ const std::string& rangeIndexColumnName,
+ const std::string& rowIndexColumnName);
~TSkiffMultiTableParser();
diff --git a/yt/yt/library/skiff_ext/schema_match.cpp b/yt/yt/library/skiff_ext/schema_match.cpp
index 7587b8b8e9..390e756798 100644
--- a/yt/yt/library/skiff_ext/schema_match.cpp
+++ b/yt/yt/library/skiff_ext/schema_match.cpp
@@ -20,7 +20,7 @@ const TString SparseColumnsName = "$sparse_columns";
////////////////////////////////////////////////////////////////////////////////
-static void ThrowInvalidSkiffTypeError(const TString& columnName, std::shared_ptr<TSkiffSchema> expectedType, std::shared_ptr<TSkiffSchema> actualType)
+static void ThrowInvalidSkiffTypeError(const std::string& columnName, std::shared_ptr<TSkiffSchema> expectedType, std::shared_ptr<TSkiffSchema> actualType)
{
THROW_ERROR_EXCEPTION("Column %Qv has unexpected Skiff type: expected %Qv, found type %Qv",
columnName,
@@ -56,13 +56,13 @@ static ERowRangeIndexMode GetRowRangeIndexMode(const std::shared_ptr<TSkiffSchem
static bool IsSkiffSpecialColumn(
TStringBuf columnName,
- const TString& rangeIndexColumnName,
- const TString& rowIndexColumnName)
+ const std::string& rangeIndexColumnName,
+ const std::string& rowIndexColumnName)
{
- static const THashSet<TString> specialColumns = {
+ static const THashSet<std::string, THash<TStringBuf>, TEqualTo<>> specialColumns{
KeySwitchColumnName,
OtherColumnsName,
- SparseColumnsName
+ SparseColumnsName,
};
return specialColumns.contains(columnName) || columnName == rangeIndexColumnName || columnName == rowIndexColumnName;
}
@@ -85,8 +85,8 @@ static std::pair<std::shared_ptr<TSkiffSchema>, bool> DeoptionalizeSchema(std::s
static TSkiffTableDescription CreateTableDescription(
const std::shared_ptr<TSkiffSchema>& skiffSchema,
- const TString& rangeIndexColumnName,
- const TString& rowIndexColumnName)
+ const std::string& rangeIndexColumnName,
+ const std::string& rowIndexColumnName)
{
TSkiffTableDescription result;
THashSet<TString> topLevelNames;
@@ -186,8 +186,8 @@ static TSkiffTableDescription CreateTableDescription(
std::vector<TSkiffTableDescription> CreateTableDescriptionList(
const std::vector<std::shared_ptr<TSkiffSchema>>& skiffSchemas,
- const TString& rangeIndexColumnName,
- const TString& rowIndexColumnName)
+ const std::string& rangeIndexColumnName,
+ const std::string& rowIndexColumnName)
{
std::vector<TSkiffTableDescription> result;
for (ui16 index = 0; index < skiffSchemas.size(); ++index) {
diff --git a/yt/yt/library/skiff_ext/schema_match.h b/yt/yt/library/skiff_ext/schema_match.h
index dc5a6c2dbc..d7183c6ce3 100644
--- a/yt/yt/library/skiff_ext/schema_match.h
+++ b/yt/yt/library/skiff_ext/schema_match.h
@@ -76,8 +76,8 @@ struct TSkiffTableColumnIds
std::vector<TSkiffTableDescription> CreateTableDescriptionList(
const std::vector<std::shared_ptr<NSkiff::TSkiffSchema>>& skiffSchema,
- const TString& rangeIndexColumnName,
- const TString& rowIndexColumnName);
+ const std::string& rangeIndexColumnName,
+ const std::string& rowIndexColumnName);
std::vector<std::shared_ptr<NSkiff::TSkiffSchema>> ParseSkiffSchemas(
const NYTree::IMapNodePtr& skiffSchemaRegistry,