aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorAlexander Smirnov <alex@ydb.tech>2025-05-31 13:17:09 +0000
committerAlexander Smirnov <alex@ydb.tech>2025-05-31 13:17:09 +0000
commit2b00537e14e6dc51140ea1abed802b2949530d4c (patch)
tree9da3dba2543e13fda51d15519c0b22a2f9ce570d
parentb545ee9d9fff3931d4a2437f607d31bdae792335 (diff)
parent96dc38494ea248471e034b276c574cda7c8f1c5e (diff)
downloadydb-2b00537e14e6dc51140ea1abed802b2949530d4c.tar.gz
Merge pull request #19096 from ydb-platform/merge-libs-250530-2116
-rw-r--r--.github/config/muted_ya.txt1
-rw-r--r--build/conf/go.conf3
-rw-r--r--build/conf/java.conf8
-rw-r--r--build/external_resources/go_tools/ya.make5
-rw-r--r--build/mapping.conf.json22
-rw-r--r--build/ymake.core.conf2
-rw-r--r--contrib/deprecated/http-parser/.yandex_meta/__init__.py (renamed from contrib/restricted/http-parser/.yandex_meta/__init__.py)2
-rw-r--r--contrib/deprecated/http-parser/.yandex_meta/devtools.copyrights.report (renamed from contrib/restricted/http-parser/.yandex_meta/devtools.copyrights.report)0
-rw-r--r--contrib/deprecated/http-parser/.yandex_meta/devtools.licenses.report (renamed from contrib/restricted/http-parser/.yandex_meta/devtools.licenses.report)0
-rw-r--r--contrib/deprecated/http-parser/.yandex_meta/licenses.list.txt (renamed from contrib/restricted/http-parser/.yandex_meta/licenses.list.txt)0
-rw-r--r--contrib/deprecated/http-parser/.yandex_meta/override.nix (renamed from contrib/restricted/http-parser/.yandex_meta/override.nix)0
-rw-r--r--contrib/deprecated/http-parser/AUTHORS (renamed from contrib/restricted/http-parser/AUTHORS)0
-rw-r--r--contrib/deprecated/http-parser/LICENSE-MIT (renamed from contrib/restricted/http-parser/LICENSE-MIT)0
-rw-r--r--contrib/deprecated/http-parser/README.md (renamed from contrib/restricted/http-parser/README.md)0
-rw-r--r--contrib/deprecated/http-parser/http_parser.c (renamed from contrib/restricted/http-parser/http_parser.c)0
-rw-r--r--contrib/deprecated/http-parser/http_parser.h (renamed from contrib/restricted/http-parser/http_parser.h)0
-rw-r--r--contrib/deprecated/http-parser/patches/field-uint32.patch (renamed from contrib/restricted/http-parser/patches/field-uint32.patch)0
-rw-r--r--contrib/deprecated/http-parser/patches/parser-flags.patch (renamed from contrib/restricted/http-parser/patches/parser-flags.patch)6
-rw-r--r--contrib/deprecated/http-parser/ya.make (renamed from contrib/restricted/http-parser/ya.make)2
-rw-r--r--contrib/deprecated/yajl/.yandex_meta/devtools.copyrights.report (renamed from contrib/libs/yajl/.yandex_meta/devtools.copyrights.report)0
-rw-r--r--contrib/deprecated/yajl/.yandex_meta/devtools.licenses.report (renamed from contrib/libs/yajl/.yandex_meta/devtools.licenses.report)0
-rw-r--r--contrib/deprecated/yajl/.yandex_meta/licenses.list.txt (renamed from contrib/libs/yajl/.yandex_meta/licenses.list.txt)0
-rw-r--r--contrib/deprecated/yajl/patches/fix_memory_leak.patch (renamed from contrib/libs/yajl/patches/fix_memory_leak.patch)4
-rw-r--r--contrib/deprecated/yajl/patches/uint64_support_r767862_review16855.patch (renamed from contrib/libs/yajl/patches/uint64_support_r767862_review16855.patch)0
-rw-r--r--contrib/deprecated/yajl/src/api/yajl_common.h (renamed from contrib/libs/yajl/src/api/yajl_common.h)0
-rw-r--r--contrib/deprecated/yajl/src/api/yajl_gen.h (renamed from contrib/libs/yajl/src/api/yajl_gen.h)0
-rw-r--r--contrib/deprecated/yajl/src/api/yajl_parse.h (renamed from contrib/libs/yajl/src/api/yajl_parse.h)0
-rw-r--r--contrib/deprecated/yajl/src/api/yajl_tree.h (renamed from contrib/libs/yajl/src/api/yajl_tree.h)0
-rw-r--r--contrib/deprecated/yajl/src/api/yajl_version.h (renamed from contrib/libs/yajl/src/api/yajl_version.h)0
-rw-r--r--contrib/deprecated/yajl/src/yajl.c (renamed from contrib/libs/yajl/src/yajl.c)0
-rw-r--r--contrib/deprecated/yajl/src/yajl_alloc.c (renamed from contrib/libs/yajl/src/yajl_alloc.c)0
-rw-r--r--contrib/deprecated/yajl/src/yajl_alloc.h (renamed from contrib/libs/yajl/src/yajl_alloc.h)0
-rw-r--r--contrib/deprecated/yajl/src/yajl_api.h (renamed from contrib/libs/yajl/src/yajl_api.h)0
-rw-r--r--contrib/deprecated/yajl/src/yajl_buf.c (renamed from contrib/libs/yajl/src/yajl_buf.c)0
-rw-r--r--contrib/deprecated/yajl/src/yajl_buf.h (renamed from contrib/libs/yajl/src/yajl_buf.h)0
-rw-r--r--contrib/deprecated/yajl/src/yajl_bytestack.h (renamed from contrib/libs/yajl/src/yajl_bytestack.h)0
-rw-r--r--contrib/deprecated/yajl/src/yajl_encode.c (renamed from contrib/libs/yajl/src/yajl_encode.c)0
-rw-r--r--contrib/deprecated/yajl/src/yajl_encode.h (renamed from contrib/libs/yajl/src/yajl_encode.h)0
-rw-r--r--contrib/deprecated/yajl/src/yajl_gen.c (renamed from contrib/libs/yajl/src/yajl_gen.c)0
-rw-r--r--contrib/deprecated/yajl/src/yajl_lex.c (renamed from contrib/libs/yajl/src/yajl_lex.c)0
-rw-r--r--contrib/deprecated/yajl/src/yajl_lex.h (renamed from contrib/libs/yajl/src/yajl_lex.h)0
-rw-r--r--contrib/deprecated/yajl/src/yajl_parser.c (renamed from contrib/libs/yajl/src/yajl_parser.c)0
-rw-r--r--contrib/deprecated/yajl/src/yajl_parser.h (renamed from contrib/libs/yajl/src/yajl_parser.h)0
-rw-r--r--contrib/deprecated/yajl/src/yajl_tree.c (renamed from contrib/libs/yajl/src/yajl_tree.c)0
-rw-r--r--contrib/deprecated/yajl/src/yajl_version.c (renamed from contrib/libs/yajl/src/yajl_version.c)0
-rw-r--r--contrib/deprecated/yajl/ya.make (renamed from contrib/libs/yajl/ya.make)0
-rw-r--r--contrib/deprecated/yajl/yajl_parser.cpp (renamed from contrib/libs/yajl/yajl_parser.cpp)0
-rw-r--r--library/cpp/threading/chunk_queue/queue.h44
-rw-r--r--util/generic/hash_primes.cpp2
-rw-r--r--util/generic/yexception.cpp1
-rw-r--r--util/stream/str.cpp2
-rw-r--r--util/system/thread.cpp1
-rw-r--r--ydb/ci/rightlib.txt2
-rw-r--r--yql/essentials/core/cbo/cbo_interesting_orderings.cpp409
-rw-r--r--yql/essentials/core/cbo/cbo_interesting_orderings.h88
-rw-r--r--yql/essentials/core/peephole_opt/yql_opt_peephole_physical.cpp29
-rw-r--r--yql/essentials/docs/en/changelog/2025.02.md10
-rw-r--r--yql/essentials/docs/en/changelog/2025.03.md2
-rw-r--r--yql/essentials/docs/en/udf/list/yson.md4
-rw-r--r--yql/essentials/docs/ru/changelog/2025.02.md10
-rw-r--r--yql/essentials/docs/ru/changelog/2025.03.md2
-rw-r--r--yql/essentials/docs/ru/udf/list/yson.md8
-rw-r--r--yql/essentials/minikql/comp_nodes/mkql_udf.cpp14
-rw-r--r--yql/essentials/minikql/comp_nodes/ut/mkql_computation_node_ut.h2
-rw-r--r--yql/essentials/minikql/comp_nodes/ut/mkql_udf_ut.cpp53
-rw-r--r--yql/essentials/providers/common/udf_resolve/yql_udf_resolver_logger.cpp2
-rw-r--r--yt/cpp/mapreduce/http_client/raw_client.cpp22
-rw-r--r--yt/cpp/mapreduce/http_client/rpc_parameters_serialization.cpp56
-rw-r--r--yt/cpp/mapreduce/http_client/rpc_parameters_serialization.h8
-rw-r--r--yt/cpp/mapreduce/interface/client_method_options.h25
-rw-r--r--yt/yql/providers/yt/provider/yql_yt_join_impl.cpp44
-rw-r--r--yt/yql/tests/sql/suites/aggregate/too_wide.sql122
-rw-r--r--yt/yt/client/api/rpc_proxy/transaction_impl.cpp1
-rw-r--r--yt/yt/core/bus/tcp/config.cpp6
-rw-r--r--yt/yt/core/bus/tcp/config.h4
-rw-r--r--yt/yt/core/bus/tcp/connection.cpp139
-rw-r--r--yt/yt/core/bus/tcp/connection.h7
-rw-r--r--yt/yt/core/bus/tcp/ssl_context.cpp322
-rw-r--r--yt/yt/core/bus/tcp/ssl_context.h44
-rw-r--r--yt/yt/core/bus/tcp/ssl_helpers.cpp103
-rw-r--r--yt/yt/core/bus/tcp/ssl_helpers.h34
-rw-r--r--yt/yt/core/bus/unittests/ssl_ut.cpp407
-rw-r--r--yt/yt/core/bus/unittests/testdata/ca.pem31
-rw-r--r--yt/yt/core/bus/unittests/testdata/ca_ec.pem13
-rw-r--r--yt/yt/core/bus/unittests/testdata/ca_with_ip_in_san.pem30
-rw-r--r--yt/yt/core/bus/unittests/testdata/cert.pem31
-rw-r--r--yt/yt/core/bus/unittests/testdata/cert_ec.pem14
-rw-r--r--yt/yt/core/bus/unittests/testdata/cert_with_ip_in_san.pem31
-rw-r--r--yt/yt/core/bus/unittests/testdata/generate.sh37
-rw-r--r--yt/yt/core/bus/unittests/testdata/key.pem52
-rw-r--r--yt/yt/core/bus/unittests/testdata/key_ec.pem5
-rw-r--r--yt/yt/core/bus/unittests/testdata/key_with_ip_in_san.pem52
-rw-r--r--yt/yt/core/bus/unittests/ya.make4
-rw-r--r--yt/yt/core/crypto/config.cpp56
-rw-r--r--yt/yt/core/crypto/config.h41
-rw-r--r--yt/yt/core/crypto/public.h2
-rw-r--r--yt/yt/core/crypto/tls.cpp395
-rw-r--r--yt/yt/core/crypto/tls.h36
-rw-r--r--yt/yt/core/crypto/unittests/tls_ut.cpp5
-rw-r--r--yt/yt/core/http/http.cpp2
-rw-r--r--yt/yt/core/http/stream.h2
-rw-r--r--yt/yt/core/http/unittests/http_ut.cpp8
-rw-r--r--yt/yt/core/http/ya.make2
-rw-r--r--yt/yt/core/https/client.cpp19
-rw-r--r--yt/yt/core/https/config.cpp10
-rw-r--r--yt/yt/core/https/config.h9
-rw-r--r--yt/yt/core/https/server.cpp19
-rw-r--r--yt/yt/core/json/json_parser.cpp2
-rw-r--r--yt/yt/core/json/json_writer.cpp2
-rw-r--r--yt/yt/core/misc/error.cpp4
-rw-r--r--yt/yt/core/net/config.h2
-rw-r--r--yt/yt/core/rpc/unittests/lib/common.h8
-rw-r--r--yt/yt/core/ya.make4
-rw-r--r--yt/yt_proto/yt/client/api/rpc_proxy/proto/api_service.proto1
114 files changed, 1653 insertions, 1360 deletions
diff --git a/.github/config/muted_ya.txt b/.github/config/muted_ya.txt
index 54e169b6d43..2d5a044b17a 100644
--- a/.github/config/muted_ya.txt
+++ b/.github/config/muted_ya.txt
@@ -60,6 +60,7 @@ ydb/library/actors/interconnect/ut_huge_cluster HugeCluster.AllToAll
ydb/library/actors/interconnect/ut_huge_cluster sole chunk chunk
ydb/library/yaml_config/ut_transform test_transform.py.TestYamlConfigTransformations.test_basic[args1-dump_ds_init]
ydb/library/yql/dq/opt/ut sole chunk chunk
+ydb/library/yql/dq/opt/ut InterestingOrderingsShuffle.Join64ChainImitation
ydb/public/sdk/cpp/src/client/topic/ut TxUsage.Sinks_Oltp_WriteToTopicAndTable_6_Query
ydb/public/sdk/cpp/src/client/topic/ut TxUsage.Sinks_Oltp_WriteToTopicAndTable_6_Table
ydb/public/sdk/cpp/src/client/topic/ut TxUsage.Sinks_Oltp_WriteToTopic_3_Query
diff --git a/build/conf/go.conf b/build/conf/go.conf
index cc6770ef611..317e5bbbc05 100644
--- a/build/conf/go.conf
+++ b/build/conf/go.conf
@@ -69,9 +69,6 @@ GOSTD_VERSION=1.23
when ($GOSTD_VERSION == "1.23") {
GOSTD=contrib/go/_std_1.23/src
}
-elsewhen ($GOSTD_VERSION == "1.22") {
- GOSTD=contrib/go/_std_1.22/src
-}
otherwise {
GOSTD=__unsupported_go_std_library_version_[$GOSTD_VERSION]__
}
diff --git a/build/conf/java.conf b/build/conf/java.conf
index 23619f8cd5a..b96a5a483da 100644
--- a/build/conf/java.conf
+++ b/build/conf/java.conf
@@ -1099,7 +1099,7 @@ module JAR_LIBRARY: _COMPILABLE_JAR_BASE {
when ($WITH_KAPT_VALUE == "yes") {
# For Kapt usage see: https://kotlinlang.org/docs/kapt.html#using-in-cli
# See for kapt.kotlin.generated: https://github.com/JetBrains/kotlin/blob/master/plugins/kapt3/kapt3-cli/testData/integration/kotlinFileGeneration/build.txt
- _KAPT_OPTS=-Xplugin=${tool:"contrib/java/org/jetbrains/kotlin/kotlin-annotation-processing/2.1.20"} $KT_KAPT_PLUGIN_OPTS
+ _KAPT_OPTS=-Xplugin=${tool:"contrib/java/org/jetbrains/kotlin/kotlin-annotation-processing/2.1.21"} $KT_KAPT_PLUGIN_OPTS
_RUN_KAPT=${YMAKE_PYTHON} ${input:"build/scripts/with_kapt_args.py"} --ap-classpath ${KT_KAPT_AP_CLASSPATH} -- $COMPILE_KT $_KAPT_OPTS
_APPEND_KAPT_GENERATED_SRCS=$YMAKE_PYTHON3 ${input:"build/scripts/resolve_java_srcs.py"} -d $KT_KAPT_SOURCES_DIR --include-patterns '**/*.java' '**/*.kt' --resolve-kotlin --append -s ${BINDIR}/all-java.srclst -k $KT_SRCLIST -r ${BINDIR}/not-used.txt
@@ -2087,9 +2087,9 @@ macro WITH_KOTLINC_DETEKT(Options...) {
# tag:kotlin-specific
### Also search for _KAPT_OPTS and change version there
### change supported jvmTarget in https://a.yandex-team.ru/arcadia/devtools/ya/jbuild/gen/actions/idea.py?rev=r16258919#L668
-_KOTLIN_VERSION=2.1.20
-KOTLIN_VERSION=2.1.20
-KOTLIN_BOM_FILE=${ARCADIA_ROOT}/contrib/java/org/jetbrains/kotlin/kotlin-bom/2.1.20/ya.dependency_management.inc
+_KOTLIN_VERSION=2.1.21
+KOTLIN_VERSION=2.1.21
+KOTLIN_BOM_FILE=${ARCADIA_ROOT}/contrib/java/org/jetbrains/kotlin/kotlin-bom/2.1.21/ya.dependency_management.inc
DETEKT_VERSION=1.23.7
_KOTLIN_SEM= \
diff --git a/build/external_resources/go_tools/ya.make b/build/external_resources/go_tools/ya.make
index a962a6fe9e8..5b16d56efa0 100644
--- a/build/external_resources/go_tools/ya.make
+++ b/build/external_resources/go_tools/ya.make
@@ -2,10 +2,7 @@ RESOURCES_LIBRARY()
TOOLCHAIN(go)
-IF(GOSTD_VERSION == 1.22)
- VERSION(1.22)
- DECLARE_EXTERNAL_HOST_RESOURCES_BUNDLE_BY_JSON(GO_TOOLS go1.22.json)
-ELSEIF(GOSTD_VERSION == 1.23)
+IF(GOSTD_VERSION == 1.23)
VERSION(1.23)
DECLARE_EXTERNAL_HOST_RESOURCES_BUNDLE_BY_JSON(GO_TOOLS go1.23.json)
ELSE()
diff --git a/build/mapping.conf.json b/build/mapping.conf.json
index ad8dce755c1..a3645ed4ddf 100644
--- a/build/mapping.conf.json
+++ b/build/mapping.conf.json
@@ -70,18 +70,24 @@
"7245028547": "{registry_endpoint}/7245028547",
"8314755214": "{registry_endpoint}/8314755214",
"8314760034": "{registry_endpoint}/8314760034",
+ "8704092379": "{registry_endpoint}/8704092379",
+ "8704092450": "{registry_endpoint}/8704092450",
"6638916300": "{registry_endpoint}/6638916300",
"6638917016": "{registry_endpoint}/6638917016",
"7245028516": "{registry_endpoint}/7245028516",
"7245028641": "{registry_endpoint}/7245028641",
"8314760058": "{registry_endpoint}/8314760058",
"8314762661": "{registry_endpoint}/8314762661",
+ "8704093001": "{registry_endpoint}/8704093001",
+ "8704092368": "{registry_endpoint}/8704092368",
"6638915855": "{registry_endpoint}/6638915855",
"6638914429": "{registry_endpoint}/6638914429",
"7245029625": "{registry_endpoint}/7245029625",
"7245029630": "{registry_endpoint}/7245029630",
"8314758936": "{registry_endpoint}/8314758936",
"8314762236": "{registry_endpoint}/8314762236",
+ "8704093021": "{registry_endpoint}/8704093021",
+ "8704092521": "{registry_endpoint}/8704092521",
"721500304": "{registry_endpoint}/721500304",
"3573990573": "{registry_endpoint}/3573990573",
"3573996018": "{registry_endpoint}/3573996018",
@@ -308,8 +314,12 @@
"7709336951": "{registry_endpoint}/7709336951",
"7709336970": "{registry_endpoint}/7709336970",
"7709337139": "{registry_endpoint}/7709337139",
+ "8825002146": "{registry_endpoint}/8825002146",
+ "8825002149": "{registry_endpoint}/8825002149",
+ "8825002315": "{registry_endpoint}/8825002315",
"6576113106": "{registry_endpoint}/6576113106",
"7709337153": "{registry_endpoint}/7709337153",
+ "8825002211": "{registry_endpoint}/8825002211",
"6148089711": "{registry_endpoint}/6148089711",
"6512097221": "{registry_endpoint}/6512097221",
"6812263131": "{registry_endpoint}/6812263131",
@@ -1139,6 +1149,7 @@
"6639202855": "{registry_endpoint}/6639202855",
"7249973735": "{registry_endpoint}/7249973735",
"8314768876": "{registry_endpoint}/8314768876",
+ "8705239478": "{registry_endpoint}/8705239478",
"4758626187": "{registry_endpoint}/4758626187",
"4758626560": "{registry_endpoint}/4758626560",
"4758626176": "{registry_endpoint}/4758626176",
@@ -1508,18 +1519,24 @@
"7245028547": "JAVA_LIBRARY-none-none-kotlin-stdlib-2.0.21.jar",
"8314755214": "JAVA_LIBRARY-none-none-kotlin-stdlib-2.1.20-sources.jar",
"8314760034": "JAVA_LIBRARY-none-none-kotlin-stdlib-2.1.20.jar",
+ "8704092379": "JAVA_LIBRARY-none-none-kotlin-stdlib-2.1.21-sources.jar",
+ "8704092450": "JAVA_LIBRARY-none-none-kotlin-stdlib-2.1.21.jar",
"6638916300": "JAVA_LIBRARY-none-none-kotlin-stdlib-jdk7-1.9.24-sources.jar",
"6638917016": "JAVA_LIBRARY-none-none-kotlin-stdlib-jdk7-1.9.24.jar",
"7245028516": "JAVA_LIBRARY-none-none-kotlin-stdlib-jdk7-2.0.21-sources.jar",
"7245028641": "JAVA_LIBRARY-none-none-kotlin-stdlib-jdk7-2.0.21.jar",
"8314760058": "JAVA_LIBRARY-none-none-kotlin-stdlib-jdk7-2.1.20-sources.jar",
"8314762661": "JAVA_LIBRARY-none-none-kotlin-stdlib-jdk7-2.1.20.jar",
+ "8704093001": "JAVA_LIBRARY-none-none-kotlin-stdlib-jdk7-2.1.21-sources.jar",
+ "8704092368": "JAVA_LIBRARY-none-none-kotlin-stdlib-jdk7-2.1.21.jar",
"6638915855": "JAVA_LIBRARY-none-none-kotlin-stdlib-jdk8-1.9.24-sources.jar",
"6638914429": "JAVA_LIBRARY-none-none-kotlin-stdlib-jdk8-1.9.24.jar",
"7245029625": "JAVA_LIBRARY-none-none-kotlin-stdlib-jdk8-2.0.21-sources.jar",
"7245029630": "JAVA_LIBRARY-none-none-kotlin-stdlib-jdk8-2.0.21.jar",
"8314758936": "JAVA_LIBRARY-none-none-kotlin-stdlib-jdk8-2.1.20-sources.jar",
"8314762236": "JAVA_LIBRARY-none-none-kotlin-stdlib-jdk8-2.1.20.jar",
+ "8704093021": "JAVA_LIBRARY-none-none-kotlin-stdlib-jdk8-2.1.21-sources.jar",
+ "8704092521": "JAVA_LIBRARY-none-none-kotlin-stdlib-jdk8-2.1.21.jar",
"721500304": "JAVA_LIBRARY-none-none-listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar",
"3573990573": "JAVA_LIBRARY-none-none-proto-google-common-protos-2.9.0-sources.jar",
"3573996018": "JAVA_LIBRARY-none-none-proto-google-common-protos-2.9.0.jar",
@@ -1746,8 +1763,12 @@
"7709336951": "devtools/ktlint_build/package.json",
"7709336970": "devtools/ktlint_build/package.json",
"7709337139": "devtools/ktlint_build/package.json",
+ "8825002146": "devtools/ktlint_build/package.json",
+ "8825002149": "devtools/ktlint_build/package.json",
+ "8825002315": "devtools/ktlint_build/package.json",
"6576113106": "devtools/ktlint_build/package_win.json",
"7709337153": "devtools/ktlint_build/package_win.json",
+ "8825002211": "devtools/ktlint_build/package_win.json",
"6148089711": "devtools/local_cache/toolscache/server/ya-tc for darwin",
"6512097221": "devtools/local_cache/toolscache/server/ya-tc for darwin",
"6812263131": "devtools/local_cache/toolscache/server/ya-tc for darwin",
@@ -2577,6 +2598,7 @@
"6639202855": "kotlin_kompiler_1.9.24 with plugins",
"7249973735": "kotlin_kompiler_2.0.21 with plugins",
"8314768876": "kotlin_kompiler_2.1.20 with plugins",
+ "8705239478": "kotlin_kompiler_2.1.21 with plugins",
"4758626187": "ktlint for darwin (0.50.0)",
"4758626560": "ktlint for darwin-arm64 (0.50.0)",
"4758626176": "ktlint for linux (0.50.0)",
diff --git a/build/ymake.core.conf b/build/ymake.core.conf
index c36714a94cb..cb17f630479 100644
--- a/build/ymake.core.conf
+++ b/build/ymake.core.conf
@@ -14,7 +14,7 @@ GO_FAKEID=11100371
ANDROID_FAKEID=2023-05-17
CLANG_TIDY_FAKEID=2023-06-06
CYTHON_FAKEID=16618405
-JAVA_FAKEID=14386852
+JAVA_FAKEID=8908076
PROTO_FAKEID=0
FBS_FAKEID=2024-03-13
diff --git a/contrib/restricted/http-parser/.yandex_meta/__init__.py b/contrib/deprecated/http-parser/.yandex_meta/__init__.py
index 9947099a0d3..6bb05a2a85f 100644
--- a/contrib/restricted/http-parser/.yandex_meta/__init__.py
+++ b/contrib/deprecated/http-parser/.yandex_meta/__init__.py
@@ -3,7 +3,7 @@ from devtools.yamaker.project import NixProject
http_parser = NixProject(
owners=["g:mds", "g:cpp-contrib"],
- arcdir="contrib/restricted/http-parser",
+ arcdir="contrib/deprecated/http-parser",
nixattr="http-parser",
# By default maximium header size allowed is 80Kb. To remove the effective limit
# on the size of the header, we define the macro to a very large number (0x7fffffff).
diff --git a/contrib/restricted/http-parser/.yandex_meta/devtools.copyrights.report b/contrib/deprecated/http-parser/.yandex_meta/devtools.copyrights.report
index 3c9e1fcf0b0..3c9e1fcf0b0 100644
--- a/contrib/restricted/http-parser/.yandex_meta/devtools.copyrights.report
+++ b/contrib/deprecated/http-parser/.yandex_meta/devtools.copyrights.report
diff --git a/contrib/restricted/http-parser/.yandex_meta/devtools.licenses.report b/contrib/deprecated/http-parser/.yandex_meta/devtools.licenses.report
index 7911cf6e365..7911cf6e365 100644
--- a/contrib/restricted/http-parser/.yandex_meta/devtools.licenses.report
+++ b/contrib/deprecated/http-parser/.yandex_meta/devtools.licenses.report
diff --git a/contrib/restricted/http-parser/.yandex_meta/licenses.list.txt b/contrib/deprecated/http-parser/.yandex_meta/licenses.list.txt
index e8deffa4a52..e8deffa4a52 100644
--- a/contrib/restricted/http-parser/.yandex_meta/licenses.list.txt
+++ b/contrib/deprecated/http-parser/.yandex_meta/licenses.list.txt
diff --git a/contrib/restricted/http-parser/.yandex_meta/override.nix b/contrib/deprecated/http-parser/.yandex_meta/override.nix
index d09d27b7a6e..d09d27b7a6e 100644
--- a/contrib/restricted/http-parser/.yandex_meta/override.nix
+++ b/contrib/deprecated/http-parser/.yandex_meta/override.nix
diff --git a/contrib/restricted/http-parser/AUTHORS b/contrib/deprecated/http-parser/AUTHORS
index 5323b685cae..5323b685cae 100644
--- a/contrib/restricted/http-parser/AUTHORS
+++ b/contrib/deprecated/http-parser/AUTHORS
diff --git a/contrib/restricted/http-parser/LICENSE-MIT b/contrib/deprecated/http-parser/LICENSE-MIT
index 1ec0ab4e174..1ec0ab4e174 100644
--- a/contrib/restricted/http-parser/LICENSE-MIT
+++ b/contrib/deprecated/http-parser/LICENSE-MIT
diff --git a/contrib/restricted/http-parser/README.md b/contrib/deprecated/http-parser/README.md
index b265d71715f..b265d71715f 100644
--- a/contrib/restricted/http-parser/README.md
+++ b/contrib/deprecated/http-parser/README.md
diff --git a/contrib/restricted/http-parser/http_parser.c b/contrib/deprecated/http-parser/http_parser.c
index 726810366a1..726810366a1 100644
--- a/contrib/restricted/http-parser/http_parser.c
+++ b/contrib/deprecated/http-parser/http_parser.c
diff --git a/contrib/restricted/http-parser/http_parser.h b/contrib/deprecated/http-parser/http_parser.h
index 262a0394c83..262a0394c83 100644
--- a/contrib/restricted/http-parser/http_parser.h
+++ b/contrib/deprecated/http-parser/http_parser.h
diff --git a/contrib/restricted/http-parser/patches/field-uint32.patch b/contrib/deprecated/http-parser/patches/field-uint32.patch
index 1d952b3eedf..1d952b3eedf 100644
--- a/contrib/restricted/http-parser/patches/field-uint32.patch
+++ b/contrib/deprecated/http-parser/patches/field-uint32.patch
diff --git a/contrib/restricted/http-parser/patches/parser-flags.patch b/contrib/deprecated/http-parser/patches/parser-flags.patch
index 71297c691bd..8552e93ef28 100644
--- a/contrib/restricted/http-parser/patches/parser-flags.patch
+++ b/contrib/deprecated/http-parser/patches/parser-flags.patch
@@ -1,7 +1,7 @@
-diff --git a/contrib/restricted/http-parser/http_parser.c b/contrib/restricted/http-parser/http_parser.c
+diff --git a/contrib/deprecated/http-parser/http_parser.c b/contrib/deprecated/http-parser/http_parser.c
index 95ff42f783d9..726810366a11 100644
---- a/contrib/restricted/http-parser/http_parser.c
-+++ b/contrib/restricted/http-parser/http_parser.c
+--- a/contrib/deprecated/http-parser/http_parser.c
++++ b/contrib/deprecated/http-parser/http_parser.c
@@ -1342,6 +1342,12 @@ size_t http_parser_execute (http_parser *parser,
} else if (parser->index == sizeof(TRANSFER_ENCODING)-2) {
parser->header_state = h_transfer_encoding;
diff --git a/contrib/restricted/http-parser/ya.make b/contrib/deprecated/http-parser/ya.make
index 134e593b695..9e83bba192d 100644
--- a/contrib/restricted/http-parser/ya.make
+++ b/contrib/deprecated/http-parser/ya.make
@@ -11,7 +11,7 @@ VERSION(2.9.4)
ORIGINAL_SOURCE(https://github.com/nodejs/http-parser/archive/v2.9.4.tar.gz)
ADDINCL(
- contrib/restricted/http-parser
+ contrib/deprecated/http-parser
)
NO_COMPILER_WARNINGS()
diff --git a/contrib/libs/yajl/.yandex_meta/devtools.copyrights.report b/contrib/deprecated/yajl/.yandex_meta/devtools.copyrights.report
index 312635e0f5f..312635e0f5f 100644
--- a/contrib/libs/yajl/.yandex_meta/devtools.copyrights.report
+++ b/contrib/deprecated/yajl/.yandex_meta/devtools.copyrights.report
diff --git a/contrib/libs/yajl/.yandex_meta/devtools.licenses.report b/contrib/deprecated/yajl/.yandex_meta/devtools.licenses.report
index 37b6ae68ea9..37b6ae68ea9 100644
--- a/contrib/libs/yajl/.yandex_meta/devtools.licenses.report
+++ b/contrib/deprecated/yajl/.yandex_meta/devtools.licenses.report
diff --git a/contrib/libs/yajl/.yandex_meta/licenses.list.txt b/contrib/deprecated/yajl/.yandex_meta/licenses.list.txt
index 9517d66999d..9517d66999d 100644
--- a/contrib/libs/yajl/.yandex_meta/licenses.list.txt
+++ b/contrib/deprecated/yajl/.yandex_meta/licenses.list.txt
diff --git a/contrib/libs/yajl/patches/fix_memory_leak.patch b/contrib/deprecated/yajl/patches/fix_memory_leak.patch
index 0ac5f0f3914..5d1e5ea7803 100644
--- a/contrib/libs/yajl/patches/fix_memory_leak.patch
+++ b/contrib/deprecated/yajl/patches/fix_memory_leak.patch
@@ -1,5 +1,5 @@
---- contrib/libs/yajl/yajl_tree.c (3b4a018b6b389390da3ee13f6b4ce0880cb71807)
-+++ contrib/libs/yajl/yajl_tree.c (5fe0776c5f96630ddffb63e4ff0292037e12039e)
+--- contrib/deprecated/yajl/yajl_tree.c (3b4a018b6b389390da3ee13f6b4ce0880cb71807)
++++ contrib/deprecated/yajl/yajl_tree.c (5fe0776c5f96630ddffb63e4ff0292037e12039e)
@@ -143,7 +143,7 @@ static yajl_val context_pop(context_t *ctx)
ctx->stack = stack->next;
diff --git a/contrib/libs/yajl/patches/uint64_support_r767862_review16855.patch b/contrib/deprecated/yajl/patches/uint64_support_r767862_review16855.patch
index 62da2521b0f..62da2521b0f 100644
--- a/contrib/libs/yajl/patches/uint64_support_r767862_review16855.patch
+++ b/contrib/deprecated/yajl/patches/uint64_support_r767862_review16855.patch
diff --git a/contrib/libs/yajl/src/api/yajl_common.h b/contrib/deprecated/yajl/src/api/yajl_common.h
index 2507576a1a6..2507576a1a6 100644
--- a/contrib/libs/yajl/src/api/yajl_common.h
+++ b/contrib/deprecated/yajl/src/api/yajl_common.h
diff --git a/contrib/libs/yajl/src/api/yajl_gen.h b/contrib/deprecated/yajl/src/api/yajl_gen.h
index fb1409df709..fb1409df709 100644
--- a/contrib/libs/yajl/src/api/yajl_gen.h
+++ b/contrib/deprecated/yajl/src/api/yajl_gen.h
diff --git a/contrib/libs/yajl/src/api/yajl_parse.h b/contrib/deprecated/yajl/src/api/yajl_parse.h
index 36a89a52a41..36a89a52a41 100644
--- a/contrib/libs/yajl/src/api/yajl_parse.h
+++ b/contrib/deprecated/yajl/src/api/yajl_parse.h
diff --git a/contrib/libs/yajl/src/api/yajl_tree.h b/contrib/deprecated/yajl/src/api/yajl_tree.h
index 2ec36fd576c..2ec36fd576c 100644
--- a/contrib/libs/yajl/src/api/yajl_tree.h
+++ b/contrib/deprecated/yajl/src/api/yajl_tree.h
diff --git a/contrib/libs/yajl/src/api/yajl_version.h b/contrib/deprecated/yajl/src/api/yajl_version.h
index adc0ad82255..adc0ad82255 100644
--- a/contrib/libs/yajl/src/api/yajl_version.h
+++ b/contrib/deprecated/yajl/src/api/yajl_version.h
diff --git a/contrib/libs/yajl/src/yajl.c b/contrib/deprecated/yajl/src/yajl.c
index 0a97f6ef001..0a97f6ef001 100644
--- a/contrib/libs/yajl/src/yajl.c
+++ b/contrib/deprecated/yajl/src/yajl.c
diff --git a/contrib/libs/yajl/src/yajl_alloc.c b/contrib/deprecated/yajl/src/yajl_alloc.c
index 96ad1d33049..96ad1d33049 100644
--- a/contrib/libs/yajl/src/yajl_alloc.c
+++ b/contrib/deprecated/yajl/src/yajl_alloc.c
diff --git a/contrib/libs/yajl/src/yajl_alloc.h b/contrib/deprecated/yajl/src/yajl_alloc.h
index 203c2f97bd9..203c2f97bd9 100644
--- a/contrib/libs/yajl/src/yajl_alloc.h
+++ b/contrib/deprecated/yajl/src/yajl_alloc.h
diff --git a/contrib/libs/yajl/src/yajl_api.h b/contrib/deprecated/yajl/src/yajl_api.h
index f45fcdf8367..f45fcdf8367 100644
--- a/contrib/libs/yajl/src/yajl_api.h
+++ b/contrib/deprecated/yajl/src/yajl_api.h
diff --git a/contrib/libs/yajl/src/yajl_buf.c b/contrib/deprecated/yajl/src/yajl_buf.c
index 2f055a6e6a3..2f055a6e6a3 100644
--- a/contrib/libs/yajl/src/yajl_buf.c
+++ b/contrib/deprecated/yajl/src/yajl_buf.c
diff --git a/contrib/libs/yajl/src/yajl_buf.h b/contrib/deprecated/yajl/src/yajl_buf.h
index b855a08b8bc..b855a08b8bc 100644
--- a/contrib/libs/yajl/src/yajl_buf.h
+++ b/contrib/deprecated/yajl/src/yajl_buf.h
diff --git a/contrib/libs/yajl/src/yajl_bytestack.h b/contrib/deprecated/yajl/src/yajl_bytestack.h
index 9ea7d151e3c..9ea7d151e3c 100644
--- a/contrib/libs/yajl/src/yajl_bytestack.h
+++ b/contrib/deprecated/yajl/src/yajl_bytestack.h
diff --git a/contrib/libs/yajl/src/yajl_encode.c b/contrib/deprecated/yajl/src/yajl_encode.c
index 0d97cc5290c..0d97cc5290c 100644
--- a/contrib/libs/yajl/src/yajl_encode.c
+++ b/contrib/deprecated/yajl/src/yajl_encode.c
diff --git a/contrib/libs/yajl/src/yajl_encode.h b/contrib/deprecated/yajl/src/yajl_encode.h
index 853a1a701ce..853a1a701ce 100644
--- a/contrib/libs/yajl/src/yajl_encode.h
+++ b/contrib/deprecated/yajl/src/yajl_encode.h
diff --git a/contrib/libs/yajl/src/yajl_gen.c b/contrib/deprecated/yajl/src/yajl_gen.c
index 5b1dd93246e..5b1dd93246e 100644
--- a/contrib/libs/yajl/src/yajl_gen.c
+++ b/contrib/deprecated/yajl/src/yajl_gen.c
diff --git a/contrib/libs/yajl/src/yajl_lex.c b/contrib/deprecated/yajl/src/yajl_lex.c
index 70555c5ea3c..70555c5ea3c 100644
--- a/contrib/libs/yajl/src/yajl_lex.c
+++ b/contrib/deprecated/yajl/src/yajl_lex.c
diff --git a/contrib/libs/yajl/src/yajl_lex.h b/contrib/deprecated/yajl/src/yajl_lex.h
index 309c29c54c6..309c29c54c6 100644
--- a/contrib/libs/yajl/src/yajl_lex.h
+++ b/contrib/deprecated/yajl/src/yajl_lex.h
diff --git a/contrib/libs/yajl/src/yajl_parser.c b/contrib/deprecated/yajl/src/yajl_parser.c
index d1758d67c9f..d1758d67c9f 100644
--- a/contrib/libs/yajl/src/yajl_parser.c
+++ b/contrib/deprecated/yajl/src/yajl_parser.c
diff --git a/contrib/libs/yajl/src/yajl_parser.h b/contrib/deprecated/yajl/src/yajl_parser.h
index 6ecc0f3a5c4..6ecc0f3a5c4 100644
--- a/contrib/libs/yajl/src/yajl_parser.h
+++ b/contrib/deprecated/yajl/src/yajl_parser.h
diff --git a/contrib/libs/yajl/src/yajl_tree.c b/contrib/deprecated/yajl/src/yajl_tree.c
index 6a9d9ffb1b5..6a9d9ffb1b5 100644
--- a/contrib/libs/yajl/src/yajl_tree.c
+++ b/contrib/deprecated/yajl/src/yajl_tree.c
diff --git a/contrib/libs/yajl/src/yajl_version.c b/contrib/deprecated/yajl/src/yajl_version.c
index 618ac1b77ed..618ac1b77ed 100644
--- a/contrib/libs/yajl/src/yajl_version.c
+++ b/contrib/deprecated/yajl/src/yajl_version.c
diff --git a/contrib/libs/yajl/ya.make b/contrib/deprecated/yajl/ya.make
index 3f8ce132621..3f8ce132621 100644
--- a/contrib/libs/yajl/ya.make
+++ b/contrib/deprecated/yajl/ya.make
diff --git a/contrib/libs/yajl/yajl_parser.cpp b/contrib/deprecated/yajl/yajl_parser.cpp
index 8cd1ce373ae..8cd1ce373ae 100644
--- a/contrib/libs/yajl/yajl_parser.cpp
+++ b/contrib/deprecated/yajl/yajl_parser.cpp
diff --git a/library/cpp/threading/chunk_queue/queue.h b/library/cpp/threading/chunk_queue/queue.h
index feee776b872..fdde719e503 100644
--- a/library/cpp/threading/chunk_queue/queue.h
+++ b/library/cpp/threading/chunk_queue/queue.h
@@ -4,13 +4,13 @@
#include <util/generic/noncopyable.h>
#include <util/generic/ptr.h>
#include <util/generic/typetraits.h>
-#include <util/generic/vector.h>
#include <util/generic/ylimits.h>
#include <library/cpp/deprecated/atomic/atomic.h>
#include <util/system/guard.h>
#include <util/system/spinlock.h>
#include <util/system/yassert.h>
+#include <atomic>
#include <type_traits>
#include <utility>
@@ -240,7 +240,7 @@ namespace NThreading {
};
struct TQueueType: public TOneOneQueue<TEntry, ChunkSize> {
- TAtomic WriteLock = 0;
+ TSpinLock WriteLock;
using TOneOneQueue<TEntry, ChunkSize>::PrepareWrite;
using TOneOneQueue<TEntry, ChunkSize>::CompleteWrite;
@@ -250,11 +250,7 @@ namespace NThreading {
};
private:
- union {
- TAtomic WriteTag = 0;
- char Pad[PLATFORM_CACHE_LINE];
- };
-
+ TPadded<std::atomic<ui64>> WriteTag{1};
TQueueType Queues[Concurrency];
public:
@@ -291,20 +287,20 @@ namespace NThreading {
ui64 NextTag() {
// TODO: can we avoid synchronization here? it costs 1.5x performance penalty
// return GetCycleCount();
- return AtomicIncrement(WriteTag);
+ return WriteTag.fetch_add(1);
}
template <typename TT>
bool TryEnqueue(TT&& value, ui64 tag) {
for (size_t i = 0; i < Concurrency; ++i) {
TQueueType& queue = Queues[i];
- if (AtomicTryAndTryLock(&queue.WriteLock)) {
+ if (!queue.WriteLock.IsLocked() && queue.WriteLock.TryAcquire()) {
TEntry* entry = queue.PrepareWrite();
Y_ASSERT(entry);
TTypeHelper::Write(&entry->Value, std::forward<TT>(value));
entry->Tag = tag;
queue.CompleteWrite();
- AtomicUnlock(&queue.WriteLock);
+ queue.WriteLock.Release();
return true;
}
}
@@ -383,7 +379,7 @@ namespace NThreading {
template <typename T, size_t Concurrency = 4, size_t ChunkSize = PLATFORM_PAGE_SIZE>
class TRelaxedManyOneQueue: private TNonCopyable {
struct TQueueType: public TOneOneQueue<T, ChunkSize> {
- TAtomic WriteLock = 0;
+ TSpinLock WriteLock;
};
private:
@@ -429,9 +425,9 @@ namespace NThreading {
size_t writePos = GetCycleCount();
for (size_t i = 0; i < Concurrency; ++i) {
TQueueType& queue = Queues[writePos++ % Concurrency];
- if (AtomicTryAndTryLock(&queue.WriteLock)) {
+ if (!queue.WriteLock.IsLocked() && queue.WriteLock.TryAcquire()) {
queue.Enqueue(std::forward<TT>(value));
- AtomicUnlock(&queue.WriteLock);
+ queue.WriteLock.Release();
return true;
}
}
@@ -446,14 +442,8 @@ namespace NThreading {
template <typename T, size_t Concurrency = 4, size_t ChunkSize = PLATFORM_PAGE_SIZE>
class TRelaxedManyManyQueue: private TNonCopyable {
struct TQueueType: public TOneOneQueue<T, ChunkSize> {
- union {
- TAtomic WriteLock = 0;
- char Pad1[PLATFORM_CACHE_LINE];
- };
- union {
- TAtomic ReadLock = 0;
- char Pad2[PLATFORM_CACHE_LINE];
- };
+ TPadded<TSpinLock> WriteLock;
+ TPadded<TSpinLock> ReadLock;
};
private:
@@ -473,9 +463,9 @@ namespace NThreading {
size_t readPos = GetCycleCount();
for (size_t i = 0; i < Concurrency; ++i) {
TQueueType& queue = Queues[readPos++ % Concurrency];
- if (AtomicTryAndTryLock(&queue.ReadLock)) {
+ if (!queue.ReadLock.IsLocked() && queue.ReadLock.TryAcquire()) {
bool dequeued = queue.Dequeue(value);
- AtomicUnlock(&queue.ReadLock);
+ queue.ReadLock.Release();
if (dequeued) {
return true;
}
@@ -487,9 +477,9 @@ namespace NThreading {
bool IsEmpty() {
for (size_t i = 0; i < Concurrency; ++i) {
TQueueType& queue = Queues[i];
- if (AtomicTryAndTryLock(&queue.ReadLock)) {
+ if (!queue.ReadLock.IsLocked() && queue.ReadLock.TryAcquire()) {
bool empty = queue.IsEmpty();
- AtomicUnlock(&queue.ReadLock);
+ queue.ReadLock.Release();
if (!empty) {
return false;
}
@@ -504,9 +494,9 @@ namespace NThreading {
size_t writePos = GetCycleCount();
for (size_t i = 0; i < Concurrency; ++i) {
TQueueType& queue = Queues[writePos++ % Concurrency];
- if (AtomicTryAndTryLock(&queue.WriteLock)) {
+ if (!queue.WriteLock.IsLocked() && queue.WriteLock.TryAcquire()) {
queue.Enqueue(std::forward<TT>(value));
- AtomicUnlock(&queue.WriteLock);
+ queue.WriteLock.Release();
return true;
}
}
diff --git a/util/generic/hash_primes.cpp b/util/generic/hash_primes.cpp
index 656d31e0464..5b9723d98f2 100644
--- a/util/generic/hash_primes.cpp
+++ b/util/generic/hash_primes.cpp
@@ -2,6 +2,8 @@
#include "array_size.h"
#include "algorithm.h"
+#include <functional>
+
/// Order of fields: reciprocal, reciprocal shift, adjacent hint, divisor
#if defined(_32_)
static constexpr ::NPrivate::THashDivisor PRIME_DIVISORS_HOLDER[]{
diff --git a/util/generic/yexception.cpp b/util/generic/yexception.cpp
index 6611fb51ff9..2e5df6a9ff2 100644
--- a/util/generic/yexception.cpp
+++ b/util/generic/yexception.cpp
@@ -1,5 +1,6 @@
#include "yexception.h"
+#include <util/stream/str.h>
#include <util/system/backtrace.h>
#include <util/system/type_name.h>
diff --git a/util/stream/str.cpp b/util/stream/str.cpp
index a0c3b1c75c7..9111e61ed14 100644
--- a/util/stream/str.cpp
+++ b/util/stream/str.cpp
@@ -1,5 +1,7 @@
#include "str.h"
+#include <util/generic/bitops.h>
+
static constexpr size_t MIN_BUFFER_GROW_SIZE = 16;
TStringInput::~TStringInput() = default;
diff --git a/util/system/thread.cpp b/util/system/thread.cpp
index 7c880a2909b..661e473256c 100644
--- a/util/system/thread.cpp
+++ b/util/system/thread.cpp
@@ -4,6 +4,7 @@
#include "thread.h"
#include "thread.i"
+#include <util/generic/bitops.h>
#include <util/generic/ptr.h>
#include <util/generic/ymath.h>
#include <util/generic/ylimits.h>
diff --git a/ydb/ci/rightlib.txt b/ydb/ci/rightlib.txt
index 2b19faa77b9..717c1562469 100644
--- a/ydb/ci/rightlib.txt
+++ b/ydb/ci/rightlib.txt
@@ -1 +1 @@
-fdbc38349df2ee0ddc678fa2bffe84786f9639a3
+fcc18ade321e3471411a93114f83ca8c339b876e
diff --git a/yql/essentials/core/cbo/cbo_interesting_orderings.cpp b/yql/essentials/core/cbo/cbo_interesting_orderings.cpp
index 82237151690..b34fe05628f 100644
--- a/yql/essentials/core/cbo/cbo_interesting_orderings.cpp
+++ b/yql/essentials/core/cbo/cbo_interesting_orderings.cpp
@@ -2,6 +2,8 @@
#include <library/cpp/disjoint_sets/disjoint_sets.h>
+#include <yql/essentials/utils/log/log.h>
+
#include <util/string/builder.h>
#include <util/string/join.h>
@@ -24,18 +26,25 @@ bool TFunctionalDependency::IsEquivalence() const {
return Type == EType::EEquivalence;
}
-bool TFunctionalDependency::MatchesAntecedentItems(const TOrdering& ordering) const {
- if (ordering.Items.size() < AntecedentItems.size()) {
- return false;
- }
+bool TFunctionalDependency::IsImplication() const {
+ return Type == EType::EImplication;
+}
- for (std::size_t i = 0; i < AntecedentItems.size(); ++i) {
- if (ordering.Items[i] != AntecedentItems[i]) {
- return false;
- }
+bool TFunctionalDependency::IsConstant() const {
+ return AntecedentItems.empty();
+}
+
+TMaybe<std::size_t> TFunctionalDependency::MatchesAntecedentItems(const TOrdering& ordering) const {
+ auto it = std::search(
+ ordering.Items.begin(), ordering.Items.end(),
+ AntecedentItems.begin(), AntecedentItems.end()
+ );
+
+ if (it == ordering.Items.end()) {
+ return Nothing();
}
- return true;
+ return static_cast<i64>(std::distance(ordering.Items.begin(), it));
}
TString TFunctionalDependency::ToString() const {
@@ -49,6 +58,10 @@ TString TFunctionalDependency::ToString() const {
}
ss << ConsequentItem;
+ if (AlwaysActive) {
+ ss << "(AA)";
+ }
+
return ss;
}
@@ -91,6 +104,11 @@ TTableAliasMap::TBaseColumn TTableAliasMap::GetBaseColumnByRename(const TString&
if (auto baseTable = GetBaseTableByAlias(alias)) {
return TBaseColumn(std::move(baseTable), std::move(column));
}
+
+ if (alias.empty() && BaseColumnByRename.contains(column)) {
+ return BaseColumnByRename[column];
+ }
+
return TBaseColumn(std::move(alias), std::move(column));
}
@@ -107,7 +125,7 @@ TString TTableAliasMap::ToString() const {
if (!BaseColumnByRename.empty()) {
result += "Renames: ";
for (const auto& [from, to] : BaseColumnByRename) {
- result += from + "->" + to.Relation + "." + to.Column + " ";
+ result += from + " -> " + to.Relation + "." + to.Column + " ";
}
result.pop_back();
result += ", ";
@@ -115,7 +133,7 @@ TString TTableAliasMap::ToString() const {
result += "TableAliases: ";
for (const auto& [alias, table] : TableByAlias) {
- result += alias + "->" + table + ", ";
+ result += alias + " -> " + table + ", ";
}
result.pop_back();
@@ -132,6 +150,9 @@ void TTableAliasMap::Merge(const TTableAliasMap& other) {
}
TString TTableAliasMap::GetBaseTableByAlias(const TString& alias) {
+ if (!TableByAlias.contains(alias)) {
+ return alias;
+ }
return TableByAlias[alias];
}
@@ -142,8 +163,13 @@ i64 TFDStorage::FindFDIdx(
TTableAliasMap* tableAliases
) {
auto convertedAntecedent = ConvertColumnIntoIndexes({antecedentColumn}, false, tableAliases);
- auto convertedConsequent = ConvertColumnIntoIndexes({consequentColumn}, false, tableAliases).at(0);
+ auto convertedConsequents = ConvertColumnIntoIndexes({consequentColumn}, false, tableAliases);
+
+ if (convertedAntecedent.empty() || convertedConsequents.empty()) {
+ return -1;
+ }
+ auto convertedConsequent = convertedConsequents[0];
for (std::size_t i = 0; i < FDs.size(); ++i) {
auto& fd = FDs[i];
if (
@@ -158,6 +184,36 @@ i64 TFDStorage::FindFDIdx(
return -1;
}
+bool operator==(const TFunctionalDependency& lhs, const TFunctionalDependency& rhs) {
+ if (lhs.IsConstant() && rhs.IsConstant()) {
+ return lhs.ConsequentItem == rhs.ConsequentItem;
+ }
+
+ if (lhs.IsImplication() && rhs.IsImplication()) {
+ return std::tie(lhs.AntecedentItems, lhs.ConsequentItem) == std::tie(rhs.AntecedentItems, rhs.ConsequentItem);
+ }
+
+ if (lhs.IsEquivalence() && rhs.IsEquivalence()) {
+ return
+ rhs.AntecedentItems.size() == 1 && rhs.AntecedentItems[0] == lhs.ConsequentItem ||
+ lhs.AntecedentItems.size() == 1 && lhs.AntecedentItems[0] == rhs.ConsequentItem;
+ }
+
+ return false;
+}
+
+
+std::size_t TFDStorage::AddFDImpl(TFunctionalDependency fd) {
+ for (std::size_t i = 0; i < FDs.size(); ++i) {
+ if (FDs[i] == fd) {
+ return i;
+ }
+ }
+
+ FDs.push_back(std::move(fd));
+ return FDs.size() - 1;
+}
+
std::size_t TFDStorage::AddFD(
const TJoinColumn& antecedentColumn,
const TJoinColumn& consequentColumn,
@@ -172,6 +228,53 @@ std::size_t TFDStorage::AddFD(
.AlwaysActive = alwaysActive
};
+ return AddFDImpl(std::move(fd));
+}
+
+std::size_t TFDStorage::AddConstant(
+ const TJoinColumn& constantColumn,
+ bool alwaysActive,
+ TTableAliasMap* tableAliases
+) {
+ auto fd = TFunctionalDependency{
+ .AntecedentItems = {},
+ .ConsequentItem = GetIdxByColumn(constantColumn, true, tableAliases),
+ .Type = TFunctionalDependency::EImplication,
+ .AlwaysActive = alwaysActive
+ };
+
+ return AddFDImpl(std::move(fd));
+}
+
+std::size_t TFDStorage::AddImplication(
+ const TVector<TJoinColumn>& antecedentColumns,
+ const TJoinColumn& consequentColumn,
+ bool alwaysActive,
+ TTableAliasMap* tableAliases
+) {
+ auto fd = TFunctionalDependency{
+ .AntecedentItems = ConvertColumnIntoIndexes(antecedentColumns, true, tableAliases),
+ .ConsequentItem = GetIdxByColumn(consequentColumn, true, tableAliases),
+ .Type = TFunctionalDependency::EImplication,
+ .AlwaysActive = alwaysActive
+ };
+
+ return AddFDImpl(std::move(fd));
+}
+
+std::size_t TFDStorage::AddEquivalence(
+ const TJoinColumn& lhs,
+ const TJoinColumn& rhs,
+ bool alwaysActive,
+ TTableAliasMap* tableAliases
+) {
+ auto fd = TFunctionalDependency{
+ .AntecedentItems = {GetIdxByColumn(lhs, true, tableAliases)},
+ .ConsequentItem = GetIdxByColumn(rhs, true, tableAliases),
+ .Type = TFunctionalDependency::EEquivalence,
+ .AlwaysActive = alwaysActive
+ };
+
FDs.push_back(std::move(fd));
return FDs.size() - 1;
}
@@ -186,6 +289,19 @@ i64 TFDStorage::FindInterestingOrderingIdx(
}
std::size_t TFDStorage::AddInterestingOrdering(
+ const TVector<TString>& interestingOrdering,
+ TOrdering::EType type,
+ TTableAliasMap* tableAliases
+) {
+ std::vector<TJoinColumn> columns;
+ columns.reserve(interestingOrdering.size());
+ for (const auto& column: interestingOrdering) {
+ columns.push_back(TJoinColumn("", column));
+ }
+ return AddInterestingOrdering(columns, type, tableAliases);
+}
+
+std::size_t TFDStorage::AddInterestingOrdering(
const std::vector<TJoinColumn>& interestingOrdering,
TOrdering::EType type,
TTableAliasMap* tableAliases
@@ -249,6 +365,9 @@ std::pair<std::vector<std::size_t>, i64> TFDStorage::ConvertColumnsAndFindExisti
TTableAliasMap* tableAliases
) {
std::vector<std::size_t> items = ConvertColumnIntoIndexes(interestingOrdering, createIfNotExists, tableAliases);
+ if (items.empty()) {
+ return {{}, -1};
+ }
for (std::size_t i = 0; i < InterestingOrderings.size(); ++i) {
if (items == InterestingOrderings[i].Items && type == InterestingOrderings[i].Type) {
@@ -268,7 +387,11 @@ std::vector<std::size_t> TFDStorage::ConvertColumnIntoIndexes(
items.reserve(ordering.size());
for (const auto& column: ordering) {
- items.push_back(GetIdxByColumn(column, createIfNotExists, tableAliases));
+ if (auto idx = GetIdxByColumn(column, createIfNotExists, tableAliases); idx != Max<size_t>()) {
+ items.push_back(idx);
+ } else {
+ return {};
+ }
}
return items;
@@ -293,21 +416,27 @@ std::size_t TFDStorage::GetIdxByColumn(
}
Y_ENSURE(!baseColumn.AttributeName.empty());
- Y_ENSURE(createIfNotExists, "There's no such column: " + fullPath);
+ if (!createIfNotExists) {
+ return Max<size_t>();
+ }
ColumnByIdx.push_back(baseColumn);
IdxByColumn[fullPath] = IdCounter++;
return IdxByColumn[fullPath];
}
-bool TOrderingsStateMachine::TLogicalOrderings::ContainsShuffle(std::size_t orderingIdx) {
- return DFSM->ContainsMatrix[State][orderingIdx];
+bool TOrderingsStateMachine::TLogicalOrderings::ContainsShuffle(i64 orderingIdx) {
+ return IsInitialized() && HasState() && (orderingIdx >= 0) && DFSM->Nodes[State].InterestingOrderings[orderingIdx];
+}
+
+bool TOrderingsStateMachine::TLogicalOrderings::ContainsSorting(i64 orderingIdx) {
+ return IsInitialized() && HasState() && (orderingIdx >= 0) && DFSM->Nodes[State].InterestingOrderings[orderingIdx];
}
void TOrderingsStateMachine::TLogicalOrderings::InduceNewOrderings(const TFDSet& fds) {
AppliedFDs |= fds;
- if (State == -1) {
+ if (!(HasState() && IsInitialized())) {
return;
}
@@ -327,7 +456,14 @@ void TOrderingsStateMachine::TLogicalOrderings::RemoveState() {
}
void TOrderingsStateMachine::TLogicalOrderings::SetOrdering(i64 orderingIdx) {
- Y_ASSERT(0 <= orderingIdx && orderingIdx < static_cast<i64>(DFSM->InitStateByOrderingIdx.size()));
+ if (orderingIdx < 0 || orderingIdx >= static_cast<i64>(DFSM->InitStateByOrderingIdx.size())) {
+ RemoveState();
+ return;
+ }
+
+ if (!IsInitialized()) {
+ return;
+ }
auto state = DFSM->InitStateByOrderingIdx[orderingIdx];
State = state.StateIdx;
@@ -354,12 +490,20 @@ bool TOrderingsStateMachine::TLogicalOrderings::HasState() const {
return State != -1;
}
-bool TOrderingsStateMachine::TLogicalOrderings::IsSubsetOf(const TLogicalOrderings& logicalOrderings) {
- if (DFSM == nullptr || logicalOrderings.DFSM == nullptr) {
- return false;
- }
+bool TOrderingsStateMachine::TLogicalOrderings::IsInitialized() {
+ return DFSM != nullptr;
+}
- return (DFSM == logicalOrderings.DFSM) && HasState() && logicalOrderings.HasState() && IsSubset(DFSM->Nodes[State].NFSMNodesBitset, logicalOrderings.DFSM->Nodes[logicalOrderings.State].NFSMNodesBitset);
+bool TOrderingsStateMachine::TLogicalOrderings::IsInitialized() const {
+ return DFSM != nullptr;
+}
+
+bool TOrderingsStateMachine::TLogicalOrderings::IsSubsetOf(const TLogicalOrderings& logicalOrderings) {
+ return
+ HasState() && logicalOrderings.HasState() &&
+ IsInitialized() && logicalOrderings.IsInitialized() &&
+ DFSM == logicalOrderings.DFSM &&
+ IsSubset(DFSM->Nodes[State].NFSMNodesBitset, logicalOrderings.DFSM->Nodes[logicalOrderings.State].NFSMNodesBitset);
}
i64 TOrderingsStateMachine::TLogicalOrderings::GetState() const {
@@ -418,6 +562,7 @@ void TOrderingsStateMachine::Build(
) {
std::vector<TFunctionalDependency> processedFDs = PruneFDs(fds, interestingOrderings);
NFSM.Build(processedFDs, interestingOrderings);
+ DFSM = MakeSimpleShared<TDFSM>();
DFSM->Build(NFSM, processedFDs, interestingOrderings);
Built = true;
}
@@ -466,7 +611,7 @@ void TOrderingsStateMachine::TNFSM::Build(
AddNode(interesting[i], TNFSM::TNode::EInteresting, i);
}
- ApplyFDs(fds);
+ ApplyFDs(fds, interesting);
PrefixClosure();
for (std::size_t idx = 0; idx < Edges.size(); ++idx) {
@@ -485,8 +630,18 @@ std::size_t TOrderingsStateMachine::TNFSM::AddNode(const TOrdering& ordering, TN
return Nodes.size() - 1;
}
+bool TOrderingsStateMachine::TNFSM::TEdge::operator==(const TEdge& other) const {
+ return std::tie(srcNodeIdx, dstNodeIdx, fdIdx) == std::tie(other.srcNodeIdx, other.dstNodeIdx, other.fdIdx);
+}
+
void TOrderingsStateMachine::TNFSM::AddEdge(std::size_t srcNodeIdx, std::size_t dstNodeIdx, i64 fdIdx) {
- Edges.emplace_back(srcNodeIdx, dstNodeIdx, fdIdx);
+ auto newEdge = TNFSM::TEdge(srcNodeIdx, dstNodeIdx, fdIdx);
+ for (std::size_t i = 0; i < Edges.size(); ++i) {
+ if (Edges[i] == newEdge) {
+ return;
+ }
+ }
+ Edges.emplace_back(newEdge);
}
void TOrderingsStateMachine::TNFSM::PrefixClosure() {
@@ -504,30 +659,123 @@ void TOrderingsStateMachine::TNFSM::PrefixClosure() {
}
if (k == iItems.size()) {
- AddEdge(i, j, TNFSM::TEdge::EPSILON);
+ if (Nodes[i].Ordering.Type == TOrdering::EShuffle) {
+ AddEdge(i, j, TNFSM::TEdge::EPSILON);
+ }
+
+ if (Nodes[i].Ordering.Type == TOrdering::ESorting) {
+ AddEdge(j, i, TNFSM::TEdge::EPSILON);
+ }
}
}
}
}
-void TOrderingsStateMachine::TNFSM::ApplyFDs(const std::vector<TFunctionalDependency>& fds) {
+void TOrderingsStateMachine::TNFSM::ApplyFDs(
+ const std::vector<TFunctionalDependency>& fds,
+ const std::vector<TOrdering>& interestingOrderings
+) {
+ std::size_t maxInterestingOrderingSize = 0;
+ if (!interestingOrderings.empty()) {
+ maxInterestingOrderingSize =
+ std::max_element(
+ interestingOrderings.begin(),
+ interestingOrderings.end(),
+ [](const TOrdering& a, const TOrdering& b) { return a.Items.size() < b.Items.size(); }
+ )->Items.size();
+ }
+
+
for (std::size_t nodeIdx = 0; nodeIdx < Nodes.size() && Nodes.size() < EMaxNFSMStates; ++nodeIdx) {
for (std::size_t fdIdx = 0; fdIdx < fds.size() && Nodes.size() < EMaxNFSMStates; ++fdIdx) {
- const auto& fd = fds[fdIdx];
+ TFunctionalDependency fd = fds[fdIdx];
- if (!fd.MatchesAntecedentItems(Nodes[nodeIdx].Ordering)) {
- continue;
- }
+ auto applyFD = [this, nodeIdx, maxInterestingOrderingSize](const TFunctionalDependency& fd, std::size_t fdIdx) {
+ if (Nodes.size() >= EMaxNFSMStates) {
+ return;
+ }
- if (fd.IsEquivalence()) {
- std::size_t replacedElement = fd.AntecedentItems[0];
- std::vector<std::size_t> newOrdering = Nodes[nodeIdx].Ordering.Items;
- *std::find(newOrdering.begin(), newOrdering.end(), replacedElement) = fd.ConsequentItem;
+ if (fd.IsConstant() && Nodes[nodeIdx].Ordering.Items.size() > 1) {
+ std::vector<std::size_t> newOrdering = Nodes[nodeIdx].Ordering.Items;
+ auto it = std::find(newOrdering.begin(), newOrdering.end(), fd.ConsequentItem);
+ if (it == newOrdering.end()) {
+ return;
+ }
+ bool isNewOrderingPrefixOfOld = (it == (newOrdering.end() - 1));
+ newOrdering.erase(it);
- std::size_t dstIdx = AddNode(TOrdering(newOrdering, Nodes[nodeIdx].Ordering.Type), TNode::EArtificial);
- AddEdge(nodeIdx, dstIdx, fdIdx);
- AddEdge(dstIdx, nodeIdx, fdIdx);
- continue;
+ std::size_t dstIdx = AddNode(TOrdering(std::move(newOrdering), Nodes[nodeIdx].Ordering.Type), TNode::EArtificial);
+
+ if (!isNewOrderingPrefixOfOld || Nodes[nodeIdx].Ordering.Type == TOrdering::EShuffle) {
+ AddEdge(nodeIdx, dstIdx, fdIdx);
+ }
+
+ if (!isNewOrderingPrefixOfOld || Nodes[nodeIdx].Ordering.Type == TOrdering::ESorting) {
+ AddEdge(dstIdx, nodeIdx, fdIdx);
+ }
+ }
+
+ if (fd.IsConstant()) {
+ return;
+ }
+
+ auto maybeAntecedentItemIdx = fd.MatchesAntecedentItems(Nodes[nodeIdx].Ordering);
+ if (!maybeAntecedentItemIdx) {
+ return;
+ }
+
+ std::size_t antecedentItemIdx = maybeAntecedentItemIdx.GetRef();
+ if (
+ auto it = std::find(Nodes[nodeIdx].Ordering.Items.begin(), Nodes[nodeIdx].Ordering.Items.end(), fd.ConsequentItem);
+ it != Nodes[nodeIdx].Ordering.Items.end()
+ ) {
+ if (fd.IsEquivalence()) { // (a, b) -> (b, a)
+ std::size_t consequentItemIdx = std::distance(Nodes[nodeIdx].Ordering.Items.begin(), it);
+ auto newOrdering = Nodes[nodeIdx].Ordering.Items;
+ std::swap(newOrdering[antecedentItemIdx], newOrdering[consequentItemIdx]);
+ std::size_t dstIdx = AddNode(TOrdering(std::move(newOrdering), Nodes[nodeIdx].Ordering.Type), TNode::EArtificial);
+ AddEdge(nodeIdx, dstIdx, fdIdx);
+ AddEdge(dstIdx, nodeIdx, fdIdx);
+ }
+
+ return;
+ }
+
+ Y_ENSURE(antecedentItemIdx < Nodes[nodeIdx].Ordering.Items.size());
+ if (fd.IsEquivalence()) {
+ Y_ENSURE(fd.AntecedentItems.size() == 1);
+ std::vector<std::size_t> newOrdering = Nodes[nodeIdx].Ordering.Items;
+ newOrdering[antecedentItemIdx] = fd.ConsequentItem;
+
+ std::size_t dstIdx = AddNode(TOrdering(std::move(newOrdering), Nodes[nodeIdx].Ordering.Type), TNode::EArtificial);
+ AddEdge(nodeIdx, dstIdx, fdIdx);
+ AddEdge(dstIdx, nodeIdx, fdIdx);
+ }
+
+ if (
+ Nodes[nodeIdx].Ordering.Type == TOrdering::EShuffle ||
+ Nodes[nodeIdx].Ordering.Items.size() == maxInterestingOrderingSize
+ ) {
+ return;
+ }
+
+ if (fd.IsImplication() || fd.IsEquivalence()) {
+ for (std::size_t i = antecedentItemIdx + fd.AntecedentItems.size(); i <= Nodes[nodeIdx].Ordering.Items.size() && Nodes.size() < EMaxNFSMStates; ++i) {
+ std::vector<std::size_t> newOrdering = Nodes[nodeIdx].Ordering.Items;
+ newOrdering.insert(newOrdering.begin() + i, fd.ConsequentItem);
+
+ std::size_t dstIdx = AddNode(TOrdering(std::move(newOrdering), Nodes[nodeIdx].Ordering.Type), TNode::EArtificial);
+ AddEdge(nodeIdx, dstIdx, fdIdx); // Epsilon edge will be added during PrefixClosure
+ }
+ }
+ };
+
+ applyFD(fd, fdIdx);
+ if (fd.IsEquivalence()) {
+ Y_ENSURE(fd.AntecedentItems.size() == 1);
+ TFunctionalDependency reversedEquiv = fd;
+ std::swap(reversedEquiv.ConsequentItem, reversedEquiv.AntecedentItems[0]);
+ applyFD(reversedEquiv, fdIdx);
}
}
}
@@ -594,7 +842,7 @@ void TOrderingsStateMachine::TDFSM::Build(
for (std::size_t i = 0; i < interestingOrderings.size(); ++i) {
for (std::size_t nfsmNodeIdx = 0; nfsmNodeIdx < nfsm.Nodes.size(); ++nfsmNodeIdx) {
if (nfsm.Nodes[nfsmNodeIdx].Ordering == interestingOrderings[i]) {
- auto nfsmNodes = CollectNodesWithEpsOrFdEdge(nfsm, {i});
+ auto nfsmNodes = CollectNodesWithEpsOrFdEdge(nfsm, {i}, fds);
InitStateByOrderingIdx[i] = TInitState{AddNode(std::move(nfsmNodes)), interestingOrderings[i].Items.size()};
}
}
@@ -613,7 +861,7 @@ void TOrderingsStateMachine::TDFSM::Build(
continue;
}
- std::size_t dstNodeIdx = AddNode(CollectNodesWithEpsOrFdEdge(nfsm, Nodes[nodeIdx].NFSMNodes, fdIdx));
+ std::size_t dstNodeIdx = AddNode(CollectNodesWithEpsOrFdEdge(nfsm, Nodes[nodeIdx].NFSMNodes, fds, fdIdx));
if (nodeIdx == dstNodeIdx) {
continue;
}
@@ -623,7 +871,7 @@ void TOrderingsStateMachine::TDFSM::Build(
}
}
- Precompute(nfsm, fds, interestingOrderings);
+ Precompute(nfsm, fds);
}
std::size_t TOrderingsStateMachine::TDFSM::AddNode(const std::vector<std::size_t>& nfsmNodes) {
@@ -644,12 +892,13 @@ void TOrderingsStateMachine::TDFSM::AddEdge(std::size_t srcNodeIdx, std::size_t
std::vector<std::size_t> TOrderingsStateMachine::TDFSM::CollectNodesWithEpsOrFdEdge(
const TNFSM& nfsm,
const std::vector<std::size_t>& startNFSMNodes,
+ const std::vector<TFunctionalDependency>& fds,
i64 fdIdx
) {
std::set<std::size_t> visited;
std::function<void(std::size_t)> DFS;
- DFS = [&DFS, &visited, fdIdx, &nfsm](std::size_t nodeIdx){
+ DFS = [&DFS, &visited, fdIdx, &nfsm, &fds](std::size_t nodeIdx){
if (visited.contains(nodeIdx)) {
return;
}
@@ -658,7 +907,7 @@ std::vector<std::size_t> TOrderingsStateMachine::TDFSM::CollectNodesWithEpsOrFdE
for (std::size_t edgeIdx: nfsm.Nodes[nodeIdx].OutgoingEdges) {
const TNFSM::TEdge& edge = nfsm.Edges[edgeIdx];
- if (edge.fdIdx == fdIdx || edge.fdIdx == TNFSM::TEdge::EPSILON) {
+ if (edge.fdIdx == fdIdx || edge.fdIdx == TNFSM::TEdge::EPSILON || fds[edge.fdIdx].AlwaysActive) {
DFS(edge.dstNodeIdx);
}
}
@@ -673,21 +922,19 @@ std::vector<std::size_t> TOrderingsStateMachine::TDFSM::CollectNodesWithEpsOrFdE
void TOrderingsStateMachine::TDFSM::Precompute(
const TNFSM& nfsm,
- const std::vector<TFunctionalDependency>& fds,
- const std::vector<TOrdering>& interestingOrderings
+ const std::vector<TFunctionalDependency>& fds
) {
TransitionMatrix = std::vector<std::vector<i64>>(Nodes.size(), std::vector<i64>(fds.size(), -1));
for (const auto& edge: Edges) {
TransitionMatrix[edge.srcNodeIdx][edge.fdIdx] = edge.dstNodeIdx;
}
- ContainsMatrix = std::vector<std::vector<bool>>(Nodes.size(), std::vector<bool>(interestingOrderings.size(), false));
for (std::size_t dfsmNodeIdx = 0; dfsmNodeIdx < Nodes.size(); ++dfsmNodeIdx) {
for (std::size_t nfsmNodeIdx : Nodes[dfsmNodeIdx].NFSMNodes) {
auto interestingOrderIdx = nfsm.Nodes[nfsmNodeIdx].InterestingOrderingIdx;
if (interestingOrderIdx == -1) { continue; }
- ContainsMatrix[dfsmNodeIdx][interestingOrderIdx] = true;
+ Nodes[dfsmNodeIdx].InterestingOrderings[interestingOrderIdx] = 1;
}
}
@@ -702,63 +949,33 @@ std::vector<TFunctionalDependency> TOrderingsStateMachine::PruneFDs(
const std::vector<TFunctionalDependency>& fds,
const std::vector<TOrdering>& interestingOrderings
) {
- std::size_t newIdxCounter = 0;
- std::unordered_map<std::size_t, std::size_t> idxByItem;
-
- for (const auto& ordering: interestingOrderings) {
- for (const auto& item: ordering.Items) {
- if (!idxByItem.contains(item)) {
- idxByItem[item] = newIdxCounter++;
- }
- }
- }
-
- for (const auto& fd: fds) {
- if (fd.IsEquivalence()) {
- if (!idxByItem.contains(fd.ConsequentItem)) {
- idxByItem[fd.ConsequentItem] = newIdxCounter++;
- }
-
- if (!idxByItem.contains(fd.AntecedentItems[0])) {
- idxByItem[fd.AntecedentItems[0]] = newIdxCounter++;
- }
- }
- }
-
- TDisjointSets equivalenceSets(idxByItem.size());
- for (const auto& fd: fds) {
- if (fd.IsEquivalence()) {
- equivalenceSets.UnionSets(idxByItem[fd.ConsequentItem], idxByItem[fd.AntecedentItems[0]]);
- }
- }
-
std::vector<TFunctionalDependency> filteredFds;
filteredFds.reserve(fds.size());
FdMapping.resize(fds.size());
for (std::size_t i = 0; i < fds.size(); ++i) {
- if (fds[i].IsEquivalence()) {
- bool canLeadToInteresting = false;
-
- for (const auto& ordering: interestingOrderings) {
- for (const auto& item: ordering.Items) {
- if (
- equivalenceSets.CanonicSetElement(idxByItem[item]) == equivalenceSets.CanonicSetElement(idxByItem[fds[i].ConsequentItem]) ||
- equivalenceSets.CanonicSetElement(idxByItem[item]) == equivalenceSets.CanonicSetElement(idxByItem[fds[i].AntecedentItems[0]])
- ) {
- canLeadToInteresting = true;
- break;
- }
- }
+ bool canLeadToInteresting = false;
+ for (const auto& ordering: interestingOrderings) {
+ if (std::find(ordering.Items.begin(), ordering.Items.end(), fds[i].ConsequentItem) != ordering.Items.end()) {
+ canLeadToInteresting = true;
+ break;
}
- if (canLeadToInteresting && filteredFds.size() < EMaxFDCount) {
- filteredFds.push_back(std::move(fds[i]));
- FdMapping[i] = filteredFds.size() - 1;
- } else {
- FdMapping[i] = -1;
+ if (
+ fds[i].IsEquivalence() &&
+ std::find(ordering.Items.begin(), ordering.Items.end(), fds[i].AntecedentItems[0]) != ordering.Items.end()
+ ) {
+ canLeadToInteresting = true;
+ break;
}
}
+
+ if (canLeadToInteresting && filteredFds.size() < EMaxFDCount) {
+ filteredFds.push_back(std::move(fds[i]));
+ FdMapping[i] = filteredFds.size() - 1;
+ } else {
+ FdMapping[i] = -1;
+ }
}
return filteredFds;
diff --git a/yql/essentials/core/cbo/cbo_interesting_orderings.h b/yql/essentials/core/cbo/cbo_interesting_orderings.h
index 311059cb407..77a20ef1964 100644
--- a/yql/essentials/core/cbo/cbo_interesting_orderings.h
+++ b/yql/essentials/core/cbo/cbo_interesting_orderings.h
@@ -3,6 +3,7 @@
#include <yql/essentials/core/yql_cost_function.h>
#include <util/generic/hash.h>
+#include <util/generic/algorithm.h>
#include <bitset>
#include <stdint.h>
@@ -20,7 +21,8 @@ namespace NYql::NDq {
struct TOrdering {
enum EType : uint32_t {
- EShuffle
+ EShuffle = 0,
+ ESorting = 1
};
bool operator==(const TOrdering& other) const;
@@ -52,11 +54,18 @@ struct TOrdering {
*/
struct TFunctionalDependency {
enum EType : uint32_t {
- EEquivalence
+ /* default fd: a -> b */
+ EImplication = 0,
+ /* equivalence: a = b */
+ EEquivalence = 1
};
bool IsEquivalence() const;
- bool MatchesAntecedentItems(const TOrdering& ordering) const;
+ bool IsImplication() const;
+ bool IsConstant() const;
+
+ // Returns index of the first matching antecedent item in the ordering, if it matches
+ TMaybe<std::size_t> MatchesAntecedentItems(const TOrdering& ordering) const;
TString ToString() const;
std::vector<std::size_t> AntecedentItems;
@@ -66,6 +75,8 @@ struct TFunctionalDependency {
bool AlwaysActive;
};
+bool operator==(const TFunctionalDependency& lhs, const TFunctionalDependency& rhs);
+
// Map of table aliases to their original table names
struct TTableAliasMap : public TSimpleRefCount<TTableAliasMap> {
public:
@@ -102,6 +113,7 @@ public:
TBaseColumn GetBaseColumnByRename(const NDq::TJoinColumn& renamedColumn);
TString ToString() const;
void Merge(const TTableAliasMap& other);
+ bool Empty() const { return TableByAlias.empty() && BaseColumnByRename.empty(); }
private:
TString GetBaseTableByAlias(const TString& alias);
@@ -123,14 +135,41 @@ public:
TTableAliasMap* tableAliases = nullptr
);
+public:
+ // Deprecated, use the others in this public section instead
std::size_t AddFD(
const TJoinColumn& antecedentColumn,
const TJoinColumn& consequentColumn,
TFunctionalDependency::EType type,
+ bool alwaysActive = false,
+ TTableAliasMap* tableAliases = nullptr
+ );
+
+ std::size_t AddConstant(
+ const TJoinColumn& constantColumn,
+ bool alwaysActive,
+ TTableAliasMap* tableAliases = nullptr
+ );
+
+ std::size_t AddImplication(
+ const TVector<TJoinColumn>& antecedentColumns,
+ const TJoinColumn& consequentColumn,
+ bool alwaysActive,
+ TTableAliasMap* tableAliases = nullptr
+ );
+
+ std::size_t AddEquivalence(
+ const TJoinColumn& lhs,
+ const TJoinColumn& rhs,
bool alwaysActive,
TTableAliasMap* tableAliases = nullptr
);
+private:
+ std::size_t AddFDImpl(TFunctionalDependency fd);
+
+public:
+
i64 FindInterestingOrderingIdx(
const std::vector<TJoinColumn>& interestingOrdering,
TOrdering::EType type,
@@ -143,6 +182,12 @@ public:
TTableAliasMap* tableAliases = nullptr
);
+ std::size_t AddInterestingOrdering(
+ const TVector<TString>& interestingOrdering, // column names
+ TOrdering::EType type,
+ TTableAliasMap* tableAliases = nullptr
+ );
+
TVector<TJoinColumn> GetInterestingOrderingsColumnNamesByIdx(std::size_t interestingOrderingIdx) const;
TString ToString() const;
@@ -190,8 +235,8 @@ private:
class TDFSM;
enum _ : std::uint32_t {
EMaxFDCount = 64,
- EMaxNFSMStates = 512,
- EMaxDFSMStates = 32768,
+ EMaxNFSMStates = 256,
+ EMaxDFSMStates = 512,
};
public:
@@ -211,18 +256,23 @@ public:
{}
public: // API
- bool ContainsShuffle(std::size_t orderingIdx);
+ bool ContainsShuffle(i64 orderingIdx);
+ bool ContainsSorting(i64 orderingIdx);
void InduceNewOrderings(const TFDSet& fds);
void RemoveState();
void SetOrdering(i64 orderingIdx);
i64 GetShuffleHashFuncArgsCount();
void SetShuffleHashFuncArgsCount(std::size_t value);
TFDSet GetFDs();
- bool HasState();
- bool HasState() const;
bool IsSubsetOf(const TLogicalOrderings& logicalOrderings);
i64 GetState() const;
+ public:
+ bool HasState();
+ bool HasState() const;
+ bool IsInitialized();
+ bool IsInitialized() const;
+
private:
bool IsSubset(const std::bitset<EMaxNFSMStates>& lhs, const std::bitset<EMaxNFSMStates>& rhs);
@@ -240,10 +290,13 @@ public:
public:
TOrderingsStateMachine() = default;
- TOrderingsStateMachine(TFDStorage fdStorage)
+ TOrderingsStateMachine(
+ TFDStorage fdStorage,
+ TOrdering::EType machineType = TOrdering::EShuffle
+ )
: FDStorage(std::move(fdStorage))
- , DFSM(MakeSimpleShared<TDFSM>())
{
+ EraseIf(FDStorage.InterestingOrderings, [machineType](const TOrdering& ordering){ return ordering.Type != machineType; });
Build(FDStorage.FDs, FDStorage.InterestingOrderings);
}
@@ -251,7 +304,6 @@ public:
const std::vector<TFunctionalDependency>& fds,
const std::vector<TOrdering>& interestingOrderings
) {
- DFSM = MakeSimpleShared<TDFSM>();
Build(fds, interestingOrderings);
}
@@ -281,7 +333,7 @@ private:
TNode(TOrdering ordering, EType type, i64 interestingOrderingIdx = -1)
: Type(type)
- , Ordering(ordering)
+ , Ordering(std::move(ordering))
, InterestingOrderingIdx(interestingOrderingIdx)
{}
@@ -303,6 +355,8 @@ private:
EPSILON = -1 // eps edges with give us nodes without applying any FDs.
};
+ bool operator==(const TEdge& other) const;
+
TString ToString() const;
};
@@ -317,7 +371,10 @@ private:
std::size_t AddNode(const TOrdering& ordering, TNode::EType type, i64 interestingOrderingIdx = -1);
void AddEdge(std::size_t srcNodeIdx, std::size_t dstNodeIdx, i64 fdIdx);
void PrefixClosure();
- void ApplyFDs(const std::vector<TFunctionalDependency>& fds);
+ void ApplyFDs(
+ const std::vector<TFunctionalDependency>& fds,
+ const std::vector<TOrdering>& interesting
+ );
private:
std::vector<TNode> Nodes;
@@ -333,6 +390,7 @@ private:
std::vector<std::size_t> NFSMNodes;
std::bitset<EMaxFDCount> OutgoingFDs;
std::bitset<EMaxNFSMStates> NFSMNodesBitset;
+ std::bitset<EMaxNFSMStates> InterestingOrderings;
TString ToString() const;
};
@@ -359,12 +417,12 @@ private:
std::vector<std::size_t> CollectNodesWithEpsOrFdEdge(
const TNFSM& nfsm,
const std::vector<std::size_t>& startNFSMNodes,
+ const std::vector<TFunctionalDependency>& fds,
i64 fdIdx = TNFSM::TEdge::EPSILON
);
void Precompute(
const TNFSM& nfsm,
- const std::vector<TFunctionalDependency>& fds,
- const std::vector<TOrdering>& interestingOrderings
+ const std::vector<TFunctionalDependency>& fds
);
private:
diff --git a/yql/essentials/core/peephole_opt/yql_opt_peephole_physical.cpp b/yql/essentials/core/peephole_opt/yql_opt_peephole_physical.cpp
index 2759510b09a..3f62f755562 100644
--- a/yql/essentials/core/peephole_opt/yql_opt_peephole_physical.cpp
+++ b/yql/essentials/core/peephole_opt/yql_opt_peephole_physical.cpp
@@ -4263,12 +4263,15 @@ TExprNode::TPtr OptimizeExpandMap(const TExprNode::TPtr& node, TExprContext& ctx
input.Child(4U)->Tail().IsCallable("Just") && ETypeAnnotationKind::Struct == input.Child(4U)->Tail().Head().GetTypeAnn()->GetKind()) {
if (const auto inItemType = GetSeqItemType(input.Head().GetTypeAnn()); ETypeAnnotationKind::Struct == inItemType->GetKind()) {
if (const auto inStructType = inItemType->Cast<TStructExprType>(); inStructType->GetSize() > 0U) {
- YQL_CLOG(DEBUG, CorePeepHole) << "Swap " << node->Content() << " with " << input.Content();
-
const auto& output = input.Child(4U)->Tail().Head();
const auto structType = output.GetTypeAnn()->Cast<TStructExprType>();
const auto outputWidth = structType->GetSize();
+ if (outputWidth > WideLimit) {
+ return node;
+ }
+
+ YQL_CLOG(DEBUG, CorePeepHole) << "Swap " << node->Content() << " with " << input.Content();
const auto inputWidth = inStructType->GetSize();
TExprNode::TListType inputFilelds, stateFields, outputFields, init, update, finish;
@@ -4660,12 +4663,15 @@ TExprNode::TPtr OptimizeMember(const TExprNode::TPtr& node, TExprContext& ctx) {
TExprNode::TPtr OptimizeCondense1(const TExprNode::TPtr& node, TExprContext& ctx) {
if (node->Head().IsCallable("NarrowMap") &&
ETypeAnnotationKind::Struct == node->Tail().Tail().GetTypeAnn()->GetKind()) {
- YQL_CLOG(DEBUG, CorePeepHole) << "Swap " << node->Content() << " with " << node->Head().Content();
const auto inputWidth = node->Head().Tail().Head().ChildrenSize();
TExprNode::TListType fields, init, update;
const auto outputWidth = CollectStateNodes(*node->Child(1U), node->Tail(), fields, init, update, ctx);
+ if (outputWidth > WideLimit) {
+ return node;
+ }
+ YQL_CLOG(DEBUG, CorePeepHole) << "Swap " << node->Content() << " with " << node->Head().Content();
return ctx.Builder(node->Pos())
.Callable("NarrowMap")
.Callable(0, "WideCondense1")
@@ -4751,14 +4757,16 @@ TExprNode::TPtr OptimizeCondense1(const TExprNode::TPtr& node, TExprContext& ctx
TExprNode::TPtr OptimizeCombineCore(const TExprNode::TPtr& node, TExprContext& ctx) {
if (node->Head().IsCallable("NarrowMap") && node->Child(4U)->Tail().IsCallable("Just")) {
- YQL_CLOG(DEBUG, CorePeepHole) << "Swap " << node->Content() << " with " << node->Head().Content();
-
const auto& output = node->Child(4U)->Tail().Head();
const auto inputWidth = node->Head().Tail().Head().ChildrenSize();
const auto structType = ETypeAnnotationKind::Struct == output.GetTypeAnn()->GetKind() ? output.GetTypeAnn()->Cast<TStructExprType>() : nullptr;
const auto outputWidth = structType ? structType->GetSize() : 1U;
+ if (outputWidth > WideLimit) {
+ return node;
+ }
+ YQL_CLOG(DEBUG, CorePeepHole) << "Swap " << node->Content() << " with " << node->Head().Content();
TExprNode::TListType stateFields, outputFields, init, update, finish;
outputFields.reserve(outputWidth);
finish.reserve(outputWidth);
@@ -5064,12 +5072,17 @@ TExprNode::TPtr OptimizeChopper(const TExprNode::TPtr& node, TExprContext& ctx)
node->Tail().GetTypeAnn()->GetKind() == ETypeAnnotationKind::Flow &&
node->Tail().GetTypeAnn()->Cast<TFlowExprType>()->GetItemType()->GetKind() == ETypeAnnotationKind::Struct &&
node->Tail().GetTypeAnn()->Cast<TFlowExprType>()->GetItemType()->Cast<TStructExprType>()->GetSize() > 0U) {
- YQL_CLOG(DEBUG, CorePeepHole) << "Swap " << node->Content() << " with " << node->Head().Content();
const auto inputWidth = node->Head().Tail().Head().ChildrenSize();
const auto structType = node->Tail().GetTypeAnn()->Cast<TFlowExprType>()->GetItemType()->Cast<TStructExprType>();
const auto outputWidth = structType->GetSize();
+ if (outputWidth > WideLimit) {
+ return node;
+ }
+
+ YQL_CLOG(DEBUG, CorePeepHole) << "Swap " << node->Content() << " with " << node->Head().Content();
+
TExprNode::TListType fields;
fields.reserve(outputWidth);
for (const auto& item : structType->GetItems())
@@ -5665,9 +5678,9 @@ TExprNode::TPtr OptimizeWideCombiner(const TExprNode::TPtr& node, TExprContext&
if (needKeyFlatten.front()) {
const auto flattenSize = *needKeyFlatten.front();
- if (flattenSize > WideLimit) {
+ /*if (flattenSize > WideLimit) { don't emit wide combiner with too many keys
return node;
- }
+ }*/
YQL_CLOG(DEBUG, CorePeepHole) << "Flatten key by tuple for " << node->Content() << " from " << originalKeySize << " to " << flattenSize;
auto children = node->ChildrenList();
diff --git a/yql/essentials/docs/en/changelog/2025.02.md b/yql/essentials/docs/en/changelog/2025.02.md
index 841efc2fb62..4a2edc44063 100644
--- a/yql/essentials/docs/en/changelog/2025.02.md
+++ b/yql/essentials/docs/en/changelog/2025.02.md
@@ -28,8 +28,8 @@ SELECT foo_new, ... WHERE foo = 1 GROUP BY expr AS foo_new
## Changes in the String module {#string-module}
-* Removed String::Reverse function, use Unicode::Reverse.
-* Removed String::HasPrefixIgnoreCase and String::StartsWithIgnoreCase functions, use String::AsciiStartsWithIgnoreCase.
-* Removed String::HasSuffixIgnoreCase and String::EndsWithIgnoreCase functions, use String::AsciiEndsWithIgnoreCase.
-* Added String::AsciiContainsIgnoreCase and String::AsciiEqualsIgnoreCase functions.
-* Added String::ReverseBytes and String::ReverseBits functions.
+* Removed `String::Reverse` function, use `Unicode::Reverse`.
+* Removed `String::HasPrefixIgnoreCase` and `String::StartsWithIgnoreCase` functions, use `String::AsciiStartsWithIgnoreCase`.
+* Removed `String::HasSuffixIgnoreCase` and `String::EndsWithIgnoreCase` functions, use `String::AsciiEndsWithIgnoreCase`.
+* Added `String::AsciiContainsIgnoreCase` and `String::AsciiEqualsIgnoreCase` functions.
+* Added `String::ReverseBytes` and `String::ReverseBits` functions.
diff --git a/yql/essentials/docs/en/changelog/2025.03.md b/yql/essentials/docs/en/changelog/2025.03.md
index e85da1f9c28..6a22743eb74 100644
--- a/yql/essentials/docs/en/changelog/2025.03.md
+++ b/yql/essentials/docs/en/changelog/2025.03.md
@@ -1,3 +1,3 @@
## Changes in RE2 module {#re2-module}
-* Queries are guaranteed to fail when invalid regular expressions are passed
+* Queries are guaranteed to fail when invalid regular expressions are passed.
diff --git a/yql/essentials/docs/en/udf/list/yson.md b/yql/essentials/docs/en/udf/list/yson.md
index 3d4dd2d7cba..756ea377800 100644
--- a/yql/essentials/docs/en/udf/list/yson.md
+++ b/yql/essentials/docs/en/udf/list/yson.md
@@ -291,11 +291,11 @@ $yson = @@{y = true; x = 5.5}@@y;
SELECT Yson::LookupBool($yson, "z"); --- null
SELECT Yson::LookupBool($yson, "y"); --- true
-SELECT Yson::LookupInt64($yson, "x"); --- Error
+-- SELECT Yson::LookupInt64($yson, "x"); --- Error
SELECT Yson::LookupInt64($yson, "x", Yson::Options(false as Strict)); --- null
SELECT Yson::LookupInt64($yson, "x", Yson::Options(true as AutoConvert)); --- 5
-SELECT Yson::ConvertToBoolDict($yson); --- Error
+-- SELECT Yson::ConvertToBoolDict($yson); --- Error
SELECT Yson::ConvertToBoolDict($yson, Yson::Options(false as Strict)); --- { "y": true }
SELECT Yson::ConvertToDoubleDict($yson, Yson::Options(false as Strict)); --- { "x": 5.5 }
```
diff --git a/yql/essentials/docs/ru/changelog/2025.02.md b/yql/essentials/docs/ru/changelog/2025.02.md
index e24b5646f5c..7904d4b893d 100644
--- a/yql/essentials/docs/ru/changelog/2025.02.md
+++ b/yql/essentials/docs/ru/changelog/2025.02.md
@@ -28,8 +28,8 @@ SELECT foo_new, ... WHERE foo = 1 GROUP BY expr AS foo_new
## Изменения в модуле String {#string-module}
-* Удалена функция String::Reverse, используйте Unicode::Reverse.
-* Удалены функции String::HasPrefixIgnoreCase и String::StartsWithIgnoreCase, используйте String::AsciiStartsWithIgnoreCase.
-* Удалены функции String::HasSuffixIgnoreCase и String::EndsWithIgnoreCase, используйте String::AsciiEndsWithIgnoreCase.
-* Добавлены функции String::AsciiContainsIgnoreCase и String::AsciiEqualsIgnoreCase.
-* Добавлены функции String::ReverseBytes и String::ReverseBits.
+* Удалена функция `String::Reverse`, используйте `Unicode::Reverse`.
+* Удалены функции `String::HasPrefixIgnoreCase` и `String::StartsWithIgnoreCase`, используйте `String::AsciiStartsWithIgnoreCase`.
+* Удалены функции `String::HasSuffixIgnoreCase` и `String::EndsWithIgnoreCase`, используйте `String::AsciiEndsWithIgnoreCase`.
+* Добавлены функции `String::AsciiContainsIgnoreCase` и `String::AsciiEqualsIgnoreCase`.
+* Добавлены функции `String::ReverseBytes` и `String::ReverseBits`.
diff --git a/yql/essentials/docs/ru/changelog/2025.03.md b/yql/essentials/docs/ru/changelog/2025.03.md
index 0a25c97b15b..2f87f374da0 100644
--- a/yql/essentials/docs/ru/changelog/2025.03.md
+++ b/yql/essentials/docs/ru/changelog/2025.03.md
@@ -1,3 +1,3 @@
## Изменения в модуле RE2 {#re2-module}
-* Гарантируется падение запросов при передаче невалидных регулярных выражений
+* Гарантируется падение запросов при передаче невалидных регулярных выражений.
diff --git a/yql/essentials/docs/ru/udf/list/yson.md b/yql/essentials/docs/ru/udf/list/yson.md
index 0f5d6094e83..2b5f86d9864 100644
--- a/yql/essentials/docs/ru/udf/list/yson.md
+++ b/yql/essentials/docs/ru/udf/list/yson.md
@@ -29,7 +29,7 @@ YSON — разработанный в Яндексе формат данных,
* `Yson::YPath***` — получение одного элемента дерева документа по указанному относительному пути с опциональным преобразованием в нужный тип данных;
* `Yson::Serialize***` — получить из ресурса копию его данных, сериализованную в одном из форматов;
-* Для удобства при передаче сериализованного Yson и Json в функции, ожидающие на входе ресурс с DOM-объектом, неявное преобразование через `Yson::Parse` или `Yson::ParseJson` происходит автоматически. Также в SQL синтаксисе оператор точки или квадратных скобок автоматически добавляет вызов `Yson::Lookup`. Для сериализации ресурса по-прежнему нужно вызывать `Yson::ConvertTo***` или `Yson::Serialize***`. Таким образом, например, получение элемента "foo" словаря из колонки mycolumn типа Yson в виде строки может выглядеть так: `SELECT Yson::ConvertToString(mycolumn["foo"]) FROM mytable;` или `SELECT Yson::ConvertToString(mycolumn.foo) FROM mytable;`. В варианте с точкой можно экранировать спецсимволы по [общим правилам для индентификаторов](../../syntax/expressions.md#escape).
+* Для удобства при передаче сериализованного Yson и Json в функции, ожидающие на входе ресурс с DOM-объектом, неявное преобразование через `Yson::Parse` или `Yson::ParseJson` происходит автоматически. Также в SQL синтаксисе оператор точки или квадратных скобок автоматически добавляет вызов `Yson::Lookup`. Для сериализации ресурса по-прежнему нужно вызывать `Yson::ConvertTo***` или `Yson::Serialize***`. Таким образом, например, получение элемента "foo" словаря из колонки mycolumn типа Yson в виде строки может выглядеть так: `SELECT Yson::ConvertToString(mycolumn["foo"]) FROM mytable;` или `SELECT Yson::ConvertToString(mycolumn.foo) FROM mytable;`. В варианте с точкой можно экранировать спецсимволы по [общим правилам для идентификаторов](../../syntax/expressions.md#escape).
Функции модуля стоит рассматривать как «кубики», из которых можно собирать разные конструкции, например:
@@ -82,7 +82,7 @@ Yson::ParseJson(String{Flags:AutoMap}) -> Resource<'Yson2.Node'>?
Yson::ParseJsonDecodeUtf8(String{Flags:AutoMap}) -> Resource<'Yson2.Node'>?
```
-Результат всех трёх функций является несериализуемым: его можно только передать на вход другой функции из библиотеки Yson, но нельзя сохранить в таблицу или вернуть на клиент в результате операции — попытка так сделать приведет к ошибке типизации. Также запрещено возвращать его за пределы [подзапросов](../../syntax/select/index.md): если это требуется, то надо вызвать [Yson::Serialize](#ysonserialize), а оптимизатор уберёт лишнюю сериализию и десериализацию, если материализация в конечном счёте не потребуется.
+Результат всех трёх функций является несериализуемым: его можно только передать на вход другой функции из библиотеки Yson, но нельзя сохранить в таблицу или вернуть на клиент в результате операции — попытка так сделать приведет к ошибке типизации. Также запрещено возвращать его за пределы [подзапросов](../../syntax/select/index.md): если это требуется, то надо вызвать [Yson::Serialize](#ysonserialize), а оптимизатор уберёт лишнюю сериализацию и десериализацию, если материализация в конечном счёте не потребуется.
{% note info %}
@@ -292,11 +292,11 @@ $yson = @@{y = true; x = 5.5}@@y;
SELECT Yson::LookupBool($yson, "z"); --- null
SELECT Yson::LookupBool($yson, "y"); --- true
-SELECT Yson::LookupInt64($yson, "x"); --- Ошибка
+-- SELECT Yson::LookupInt64($yson, "x"); --- Ошибка
SELECT Yson::LookupInt64($yson, "x", Yson::Options(false as Strict)); --- null
SELECT Yson::LookupInt64($yson, "x", Yson::Options(true as AutoConvert)); --- 5
-SELECT Yson::ConvertToBoolDict($yson); --- Ошибка
+-- SELECT Yson::ConvertToBoolDict($yson); --- Ошибка
SELECT Yson::ConvertToBoolDict($yson, Yson::Options(false as Strict)); --- { "y": true }
SELECT Yson::ConvertToDoubleDict($yson, Yson::Options(false as Strict)); --- { "x": 5.5 }
```
diff --git a/yql/essentials/minikql/comp_nodes/mkql_udf.cpp b/yql/essentials/minikql/comp_nodes/mkql_udf.cpp
index 8230a759d6b..44c45888c0a 100644
--- a/yql/essentials/minikql/comp_nodes/mkql_udf.cpp
+++ b/yql/essentials/minikql/comp_nodes/mkql_udf.cpp
@@ -180,7 +180,7 @@ public:
NUdf::TSourcePosition pos,
IComputationNode* runConfigNode,
ui32 runConfigArgs,
- const TCallableType* functionType,
+ const TCallableType* callableType,
TType* userType)
: TBaseComputation(mutables, EValueRepresentation::Boxed)
, FunctionName(std::move(functionName))
@@ -188,7 +188,7 @@ public:
, Pos(pos)
, RunConfigNode(runConfigNode)
, RunConfigArgs(runConfigArgs)
- , FunctionType(functionType)
+ , CallableType(callableType)
, UserType(userType)
, UdfIndex(mutables.CurValueIndex++)
{
@@ -280,7 +280,7 @@ private:
}
void Wrap(NUdf::TUnboxedValue& callable) const {
- TValidate<TValidatePolicy,TValidateMode>::WrapCallable(FunctionType, callable, TStringBuilder() << "FunctionWithConfig<" << FunctionName << ">");
+ TValidate<TValidatePolicy,TValidateMode>::WrapCallable(CallableType, callable, TStringBuilder() << "FunctionWithConfig<" << FunctionName << ">");
}
void RegisterDependencies() const final {
@@ -292,7 +292,7 @@ private:
const NUdf::TSourcePosition Pos;
IComputationNode* const RunConfigNode;
const ui32 RunConfigArgs;
- const TCallableType* FunctionType;
+ const TCallableType* CallableType;
TType* const UserType;
const ui32 UdfIndex;
};
@@ -421,7 +421,7 @@ IComputationNode* WrapUdf(TCallable& callable, const TComputationNodeFactoryCont
const auto closureNodeType = runConfigNodeType->IsVoid()
? AS_TYPE(TCallableType, callableNodeType)->GetReturnType()
: callableNodeType;
- if (!closureNodeType->IsSameType(*closureFuncType)) {
+ if (!closureNodeType->IsConvertableTo(*closureFuncType)) {
TString diff = TStringBuilder()
<< "type mismatch, expected return type: "
<< PrintNode(closureNodeType, true)
@@ -438,7 +438,7 @@ IComputationNode* WrapUdf(TCallable& callable, const TComputationNodeFactoryCont
const auto runConfigArgs = funcInfo.FunctionType->GetArgumentsCount();
return runConfigNodeType->IsVoid()
? CreateUdfWrapper<true>(ctx, std::move(funcName), std::move(typeConfig), pos, callableNodeType, callableFuncType, userType)
- : CreateUdfWrapper<false>(ctx, std::move(funcName), std::move(typeConfig), pos, runConfigCompNode, runConfigArgs, callableFuncType, userType);
+ : CreateUdfWrapper<false>(ctx, std::move(funcName), std::move(typeConfig), pos, runConfigCompNode, runConfigArgs, callableNodeType, userType);
}
if (!callableFuncType->IsConvertableTo(*callableNodeType, true)) {
@@ -462,7 +462,7 @@ IComputationNode* WrapUdf(TCallable& callable, const TComputationNodeFactoryCont
}
const auto runCfgCompNode = LocateNode(ctx.NodeLocator, *runCfgNode.GetNode());
- return CreateUdfWrapper<false>(ctx, std::move(funcName), std::move(typeConfig), pos, runCfgCompNode, 1U, callableFuncType, userType);
+ return CreateUdfWrapper<false>(ctx, std::move(funcName), std::move(typeConfig), pos, runCfgCompNode, 1U, callableNodeType, userType);
}
IComputationNode* WrapScriptUdf(TCallable& callable, const TComputationNodeFactoryContext& ctx) {
diff --git a/yql/essentials/minikql/comp_nodes/ut/mkql_computation_node_ut.h b/yql/essentials/minikql/comp_nodes/ut/mkql_computation_node_ut.h
index a118540e625..0d783f677d1 100644
--- a/yql/essentials/minikql/comp_nodes/ut/mkql_computation_node_ut.h
+++ b/yql/essentials/minikql/comp_nodes/ut/mkql_computation_node_ut.h
@@ -118,7 +118,7 @@ struct TSetup {
Reset();
Explorer.Walk(pgm.GetNode(), Env->GetNodeStack());
TComputationPatternOpts opts(Alloc.Ref(), *Env, NodeFactory,
- FunctionRegistry.Get(), NUdf::EValidateMode::None, NUdf::EValidatePolicy::Exception,
+ FunctionRegistry.Get(), NUdf::EValidateMode::Greedy, NUdf::EValidatePolicy::Exception,
UseLLVM ? "" : "OFF", graphPerProcess, StatsRegistry.Get(), nullptr, nullptr);
Pattern = MakeComputationPattern(Explorer, pgm, entryPoints, opts);
auto graph = Pattern->Clone(opts.ToComputationOptions(*RandomProvider, *TimeProvider));
diff --git a/yql/essentials/minikql/comp_nodes/ut/mkql_udf_ut.cpp b/yql/essentials/minikql/comp_nodes/ut/mkql_udf_ut.cpp
index 265ad3bd059..fe7e32e189d 100644
--- a/yql/essentials/minikql/comp_nodes/ut/mkql_udf_ut.cpp
+++ b/yql/essentials/minikql/comp_nodes/ut/mkql_udf_ut.cpp
@@ -1,10 +1,55 @@
#include "mkql_computation_node_ut.h"
#include <yql/essentials/public/udf/udf_helpers.h>
#include <yql/essentials/minikql/mkql_node_serialization.h>
+#include <yql/essentials/minikql/mkql_node_cast.h>
namespace NKikimr {
namespace NMiniKQL {
+// XXX: Emulate type transformations similar to the one made by
+// type annotation and compilation phases. As a result, the name
+// (i.e. "UDF") of callable type is lost. Hence, the type resolved
+// at the type annotation phase (and, ergo, used for bytecode
+// compilation) differs from the type, resolved for the underline
+// function at runtime.
+template<typename TUdf>
+static TType* TweakUdfType(const NYql::NUdf::TStringRef& name, TType* userType,
+ const TTypeEnvironment& env)
+{
+ TFunctionTypeInfoBuilder typeInfoBuilder(NYql::UnknownLangVersion, env,
+ new TTypeInfoHelper(),
+ "", nullptr, {});
+
+ // Obtain the callable type of the particular UDF.
+ TFunctionTypeInfo funcInfo;
+ UNIT_ASSERT(TUdf::DeclareSignature(name, userType, typeInfoBuilder, true));
+ typeInfoBuilder.Build(&funcInfo);
+
+ // Create the new MiniKQL type to emulate two conversions:
+ // * Convert the given MiniKQL type to the expression type.
+ // See <NYql::NCommon::ConvertMiniKQLType>.
+ // * Convert the expression type back to the MiniKQL one.
+ // See <NYql::NCommon::TMkqlBuildContext::BuildType>.
+ // The aforementioned conversions are made by the pipeline in
+ // scope of type annotation and compilation phases.
+ // As a result of the first conversion, the name of the
+ // callable type is lost, so the new MiniKQL type has to be
+ // the same as the resolved one, but the name is omitted.
+ const auto funcType = AS_TYPE(TCallableType, funcInfo.FunctionType);
+ TVector<TType*> argsTypes;
+ for (size_t i = 0; i < funcType->GetArgumentsCount(); i++) {
+ argsTypes.push_back(funcType->GetArgumentType(i));
+ }
+ const auto nodeType = TCallableType::Create("", /* Name has to be empty. */
+ funcType->GetReturnType(),
+ funcType->GetArgumentsCount(),
+ argsTypes.data(),
+ funcType->GetPayload(),
+ env);
+ nodeType->SetOptionalArgumentsCount(funcType->GetOptionalArgumentsCount());
+ return nodeType;
+};
+
class TImpl : public NYql::NUdf::TBoxedValue {
public:
explicit TImpl(NYql::NUdf::TSourcePosition pos,
@@ -219,7 +264,9 @@ Y_UNIT_TEST_SUITE(TMiniKQLUdfTest) {
pb.NewTupleType({strType}),
pb.NewEmptyStructType(),
pb.NewEmptyTupleType()});
- const auto udf = pb.Udf("TestModule.Test", upvalue, userType);
+
+ const auto udfType = TweakUdfType<TRunConfig>("Test", userType, *compileSetup.Env);
+ const auto udf = pb.TypedUdf("TestModule.Test", udfType, upvalue, userType);
const auto list = pb.NewList(strType, {value});
const auto pgmReturn = pb.Map(list, [&pb, udf](const TRuntimeNode item) {
@@ -268,7 +315,9 @@ Y_UNIT_TEST_SUITE(TMiniKQLUdfTest) {
pb.NewTupleType({strType}),
pb.NewEmptyStructType(),
pb.NewEmptyTupleType()});
- const auto udf = pb.Udf("TestModule.Test", pb.NewVoid(), userType);
+
+ const auto udfType = TweakUdfType<TCurrying>("Test", userType, *compileSetup.Env);
+ const auto udf = pb.TypedUdf("TestModule.Test", udfType, pb.NewVoid(), userType);
const auto closure = pb.Apply(udf, {upvalue, optional});
const auto list = pb.NewList(strType, {value});
diff --git a/yql/essentials/providers/common/udf_resolve/yql_udf_resolver_logger.cpp b/yql/essentials/providers/common/udf_resolve/yql_udf_resolver_logger.cpp
index 5cb14c7619c..d09d7cfafce 100644
--- a/yql/essentials/providers/common/udf_resolve/yql_udf_resolver_logger.cpp
+++ b/yql/essentials/providers/common/udf_resolve/yql_udf_resolver_logger.cpp
@@ -12,7 +12,7 @@ using namespace NYql;
class TUdfResolverWithLoggerDecorator : public IUdfResolver {
public:
TUdfResolverWithLoggerDecorator(IUdfResolver::TPtr underlying, const TString& path, const TString& sessionId)
- : Underlying_(underlying), Out_(TFile(path, WrOnly | ForAppend)), SessionId_(sessionId) {}
+ : Underlying_(underlying), Out_(TFile(path, WrOnly | ForAppend | OpenAlways)), SessionId_(sessionId) {}
TMaybe<TFilePathWithMd5> GetSystemModulePath(const TStringBuf& moduleName) const override {
return Underlying_->GetSystemModulePath(moduleName);
diff --git a/yt/cpp/mapreduce/http_client/raw_client.cpp b/yt/cpp/mapreduce/http_client/raw_client.cpp
index cb5a852b176..f50b239c2aa 100644
--- a/yt/cpp/mapreduce/http_client/raw_client.cpp
+++ b/yt/cpp/mapreduce/http_client/raw_client.cpp
@@ -569,10 +569,10 @@ std::unique_ptr<IInputStream> THttpRawClient::ReadFile(
{
TMutationId mutationId;
THttpHeader header("GET", GetReadFileCommand(Context_.Config->ApiVersion));
- header.AddTransactionId(transactionId);
header.SetOutputFormat(TMaybe<TFormat>()); // Binary format
- header.MergeParameters(FormIORequestParameters(path, options));
header.SetResponseCompression(ToString(Context_.Config->AcceptEncoding));
+ header.MergeParameters(NRawClient::SerializeParamsForReadFile(transactionId, options));
+ header.MergeParameters(FormIORequestParameters(path, options));
TRequestConfig config;
config.IsHeavy = true;
@@ -740,23 +740,7 @@ TNode::TListType THttpRawClient::SelectRows(
THttpHeader header("GET", "select_rows");
header.SetInputFormat(TFormat::YsonBinary());
header.SetOutputFormat(TFormat::YsonBinary());
-
- header.MergeParameters(BuildYsonNodeFluently().BeginMap()
- .Item("query").Value(query)
- .DoIf(options.Timeout_.Defined(), [&] (TFluentMap fluent) {
- fluent.Item("timeout").Value(static_cast<i64>(options.Timeout_->MilliSeconds()));
- })
- .DoIf(options.InputRowLimit_.Defined(), [&] (TFluentMap fluent) {
- fluent.Item("input_row_limit").Value(*options.InputRowLimit_);
- })
- .DoIf(options.OutputRowLimit_.Defined(), [&] (TFluentMap fluent) {
- fluent.Item("output_row_limit").Value(*options.OutputRowLimit_);
- })
- .Item("range_expansion_limit").Value(options.RangeExpansionLimit_)
- .Item("fail_on_incomplete_result").Value(options.FailOnIncompleteResult_)
- .Item("verbose_logging").Value(options.VerboseLogging_)
- .Item("enable_code_cache").Value(options.EnableCodeCache_)
- .EndMap());
+ header.MergeParameters(NRawClient::SerializeParamsForSelectRows(query, options));
TRequestConfig config;
config.IsHeavy = true;
diff --git a/yt/cpp/mapreduce/http_client/rpc_parameters_serialization.cpp b/yt/cpp/mapreduce/http_client/rpc_parameters_serialization.cpp
index 14ab0e1db85..7a8ce040ac0 100644
--- a/yt/cpp/mapreduce/http_client/rpc_parameters_serialization.cpp
+++ b/yt/cpp/mapreduce/http_client/rpc_parameters_serialization.cpp
@@ -71,13 +71,24 @@ static void SetFirstLastTabletIndex(TNode* node, const TOptions& options)
}
template <typename T>
-void SerializeMasterReadOptions(TNode* node, const TMasterReadOptions<T>& options)
+static void SerializeMasterReadOptions(TNode* node, const TMasterReadOptions<T>& options)
{
if (options.ReadFrom_) {
(*node)["read_from"] = ToString(*options.ReadFrom_);
}
}
+template <typename T>
+static void SerializeSuppressableAccessTrackingOptions(TNode* node, const TSuppressableAccessTrackingOptions<T>& options)
+{
+ if (options.SuppressAccessTracking_) {
+ (*node)["suppress_access_tracking"] = options.SuppressAccessTracking_;
+ }
+ if (options.SuppressModificationTracking_) {
+ (*node)["suppress_modification_tracking"] = options.SuppressAccessTracking_;
+ }
+}
+
////////////////////////////////////////////////////////////////////////////////
TNode SerializeParamsForCreate(
@@ -124,6 +135,7 @@ TNode SerializeParamsForExists(
SetTransactionIdParam(&result, transactionId);
SetPathParam(&result, pathPrefix, path);
SerializeMasterReadOptions(&result, options);
+ SerializeSuppressableAccessTrackingOptions(&result, options);
return result;
}
@@ -137,6 +149,7 @@ TNode SerializeParamsForGet(
SetTransactionIdParam(&result, transactionId);
SetPathParam(&result, pathPrefix, path);
SerializeMasterReadOptions(&result, options);
+ SerializeSuppressableAccessTrackingOptions(&result, options);
if (options.AttributeFilter_) {
result["attributes"] = SerializeAttributeFilter(*options.AttributeFilter_);
}
@@ -155,6 +168,7 @@ TNode SerializeParamsForSet(
TNode result;
SetTransactionIdParam(&result, transactionId);
SetPathParam(&result, pathPrefix, path);
+ SerializeSuppressableAccessTrackingOptions(&result, options);
result["recursive"] = options.Recursive_;
if (options.Force_) {
result["force"] = *options.Force_;
@@ -171,6 +185,7 @@ TNode SerializeParamsForMultisetAttributes(
TNode result;
SetTransactionIdParam(&result, transactionId);
SetPathParam(&result, pathPrefix, path);
+ SerializeSuppressableAccessTrackingOptions(&result, options);
if (options.Force_) {
result["force"] = *options.Force_;
}
@@ -187,6 +202,7 @@ TNode SerializeParamsForList(
SetTransactionIdParam(&result, transactionId);
SetPathParam(&result, pathPrefix, path);
SerializeMasterReadOptions(&result, options);
+ SerializeSuppressableAccessTrackingOptions(&result, options);
if (options.MaxSize_) {
result["max_size"] = *options.MaxSize_;
}
@@ -598,6 +614,29 @@ TNode SerializeParamsForListJobs(
return result;
}
+TNode SerializeParamsForSelectRows(
+ const TString& query,
+ const TSelectRowsOptions& options)
+{
+ TNode result;
+ result["query"] = query;
+ SerializeSuppressableAccessTrackingOptions(&result, options);
+ if (options.Timeout_) {
+ result["timeout"] = static_cast<i64>(options.Timeout_->MilliSeconds());
+ }
+ if (options.InputRowLimit_) {
+ result["input_row_limit"] = *options.InputRowLimit_;
+ }
+ if (options.OutputRowLimit_) {
+ result["output_row_limit"] = *options.OutputRowLimit_;
+ }
+ result["range_expansion_limit"] = options.RangeExpansionLimit_;
+ result["fail_on_incomplete_result"] = options.FailOnIncompleteResult_;
+ result["verbose_logging"] = options.VerboseLogging_;
+ result["enable_code_cache"] = options.EnableCodeCache_;
+ return result;
+}
+
TNode SerializeParametersForInsertRows(
const TString& pathPrefix,
const TYPath& path,
@@ -658,6 +697,7 @@ TNode SerializeParamsForReadTable(
{
TNode result;
SetTransactionIdParam(&result, transactionId);
+ SerializeSuppressableAccessTrackingOptions(&result, options);
result["control_attributes"] = BuildYsonNodeFluently()
.BeginMap()
.Item("enable_row_index").Value(options.ControlAttributes_.EnableRowIndex_)
@@ -666,10 +706,11 @@ TNode SerializeParamsForReadTable(
return result;
}
-TNode SerializeParamsForReadTablePartition(const TString& cookie, const TTablePartitionReaderOptions& /*options*/)
+TNode SerializeParamsForReadTablePartition(const TString& cookie, const TTablePartitionReaderOptions& options)
{
TNode node;
node["cookie"] = cookie;
+ SerializeSuppressableAccessTrackingOptions(&node, options);
return node;
}
@@ -691,6 +732,7 @@ TNode SerializeParamsForReadBlobTable(
.UpperLimit(TReadLimit().Key(upperLimitKey))));
SetTransactionIdParam(&result, transactionId);
+ SerializeSuppressableAccessTrackingOptions(&result, options);
result["start_part_index"] = options.StartPartIndex_;
result["offset"] = options.Offset_;
@@ -824,6 +866,16 @@ TNode SerializeParamsForGetTablePartitions(
return result;
}
+TNode SerializeParamsForReadFile(
+ const TTransactionId& transactionId,
+ const TFileReaderOptions& options)
+{
+ TNode result;
+ SetTransactionIdParam(&result, transactionId);
+ SerializeSuppressableAccessTrackingOptions(&result, options);
+ return result;
+}
+
TNode SerializeParamsForGetFileFromCache(
const TTransactionId& transactionId,
const TString& md5Signature,
diff --git a/yt/cpp/mapreduce/http_client/rpc_parameters_serialization.h b/yt/cpp/mapreduce/http_client/rpc_parameters_serialization.h
index 90c8dd8e247..560d3263732 100644
--- a/yt/cpp/mapreduce/http_client/rpc_parameters_serialization.h
+++ b/yt/cpp/mapreduce/http_client/rpc_parameters_serialization.h
@@ -140,6 +140,10 @@ TNode SerializeParamsForGetJobTrace(
const TOperationId& operationId,
const TGetJobTraceOptions& options);
+TNode SerializeParamsForSelectRows(
+ const TString& query,
+ const TSelectRowsOptions& options);
+
TNode SerializeParametersForInsertRows(
const TString& pathPrefix,
const TYPath& path,
@@ -208,6 +212,10 @@ TNode SerializeParamsForGetTablePartitions(
const TVector<TRichYPath>& paths,
const TGetTablePartitionsOptions& options);
+TNode SerializeParamsForReadFile(
+ const TTransactionId& transactionId,
+ const TFileReaderOptions& options);
+
TNode SerializeParamsForGetFileFromCache(
const TTransactionId& transactionId,
const TString& md5Signature,
diff --git a/yt/cpp/mapreduce/interface/client_method_options.h b/yt/cpp/mapreduce/interface/client_method_options.h
index 10c200715df..0fc8c8d00fa 100644
--- a/yt/cpp/mapreduce/interface/client_method_options.h
+++ b/yt/cpp/mapreduce/interface/client_method_options.h
@@ -50,6 +50,21 @@ enum class EComplexTypeMode : int
Positional /* "positional" */,
};
+/// Base class for options dealing with access tracking suppression.
+template <typename TDerived>
+struct TSuppressableAccessTrackingOptions
+{
+ /// @cond Doxygen_Suppress
+ using TSelf = TDerived;
+ /// @endcond
+
+ /// @brief Whether to disable update of access_time on read/write operations.
+ FLUENT_FIELD_DEFAULT(bool, SuppressAccessTracking, false);
+
+ /// @brief Whether to disable update of modification on read/write operations.
+ FLUENT_FIELD_DEFAULT(bool, SuppressModificationTracking, false);
+};
+
///
/// @brief Options for @ref NYT::ICypressClient::Create
///
@@ -118,6 +133,7 @@ struct TMasterReadOptions
/// @see https://ytsaurus.tech/docs/en/api/commands.html#exists
struct TExistsOptions
: public TMasterReadOptions<TExistsOptions>
+ , public TSuppressableAccessTrackingOptions<TExistsOptions>
{
};
@@ -127,6 +143,7 @@ struct TExistsOptions
/// @see https://ytsaurus.tech/docs/en/api/commands.html#get
struct TGetOptions
: public TMasterReadOptions<TGetOptions>
+ , public TSuppressableAccessTrackingOptions<TGetOptions>
{
/// @brief Attributes that should be fetched with each node.
FLUENT_FIELD_OPTION(TAttributeFilter, AttributeFilter);
@@ -140,6 +157,7 @@ struct TGetOptions
///
/// @see https://ytsaurus.tech/docs/en/api/commands.html#set
struct TSetOptions
+ : public TSuppressableAccessTrackingOptions<TSetOptions>
{
/// @cond Doxygen_Suppress
using TSelf = TSetOptions;
@@ -157,6 +175,7 @@ struct TSetOptions
///
/// @see https://ytsaurus.tech/docs/en/api/commands.html#multiset_attributes
struct TMultisetAttributesOptions
+ : public TSuppressableAccessTrackingOptions<TMultisetAttributesOptions>
{
/// @cond Doxygen_Suppress
using TSelf = TMultisetAttributesOptions;
@@ -171,6 +190,7 @@ struct TMultisetAttributesOptions
/// @see https://ytsaurus.tech/docs/en/api/commands.html#list
struct TListOptions
: public TMasterReadOptions<TListOptions>
+ , public TSuppressableAccessTrackingOptions<TListOptions>
{
/// @cond Doxygen_Suppress
using TSelf = TListOptions;
@@ -277,6 +297,7 @@ struct TConcatenateOptions
///
/// @see https://ytsaurus.tech/docs/en/api/commands.html#read_blob_table
struct TBlobTableReaderOptions
+ : public TSuppressableAccessTrackingOptions<TBlobTableReaderOptions>
{
/// @cond Doxygen_Suppress
using TSelf = TBlobTableReaderOptions;
@@ -477,6 +498,7 @@ struct TIOOptions
/// @brief Options for reading file from YT.
struct TFileReaderOptions
: public TIOOptions<TFileReaderOptions>
+ , public TSuppressableAccessTrackingOptions<TFileReaderOptions>
{
///
/// @brief Offset to start reading from.
@@ -670,6 +692,7 @@ public:
/// Options for @ref NYT::IClient::CreateTableReader
struct TTableReaderOptions
: public TIOOptions<TTableReaderOptions>
+ , public TSuppressableAccessTrackingOptions<TTableReaderOptions>
{
/// @deprecated Size of internal client buffer.
FLUENT_FIELD_DEFAULT(size_t, SizeLimit, 4 << 20);
@@ -688,6 +711,7 @@ struct TTableReaderOptions
/// Options for @ref NYT::IClient::CreatePartitionTableReader
struct TTablePartitionReaderOptions
+ : public TSuppressableAccessTrackingOptions<TTablePartitionReaderOptions>
{
/// @cond Doxygen_Suppress
using TSelf = TTablePartitionReaderOptions;
@@ -1027,6 +1051,7 @@ struct TLookupRowsOptions
///
/// @see https://ytsaurus.tech/docs/en/api/commands#select_rows
struct TSelectRowsOptions
+ : public TSuppressableAccessTrackingOptions<TSelectRowsOptions>
{
/// @cond Doxygen_Suppress
using TSelf = TSelectRowsOptions;
diff --git a/yt/yql/providers/yt/provider/yql_yt_join_impl.cpp b/yt/yql/providers/yt/provider/yql_yt_join_impl.cpp
index a5a6d119d58..04ae3a8943d 100644
--- a/yt/yql/providers/yt/provider/yql_yt_join_impl.cpp
+++ b/yt/yql/providers/yt/provider/yql_yt_join_impl.cpp
@@ -1724,18 +1724,6 @@ bool RewriteYtMapJoin(TYtEquiJoin equiJoin, const TJoinLabels& labels, bool isLo
YQL_CLOG(INFO, ProviderYt) << strategyName << " assumes unique keys for the small table";
}
- ui64 partCount = 1;
- ui64 partRows = settings.RightRows;
- if ((settings.RightSize > 0) && useShards) {
- partCount = (settings.RightMemSize + settings.MapJoinLimit - 1) / settings.MapJoinLimit;
- partRows = (settings.RightRows + partCount - 1) / partCount;
- }
-
- if (partCount > 1) {
- YQL_ENSURE(!isLookupJoin);
- YQL_CLOG(INFO, ProviderYt) << strategyName << " sharded into " << partCount << " parts, each " << partRows << " rows";
- }
-
auto leftKeyColumns = settings.SwapTables ? op.RightLabel : op.LeftLabel;
auto rightKeyColumns = settings.SwapTables ? op.LeftLabel : op.RightLabel;
auto joinTree = ctx.NewList(pos, {
@@ -1776,6 +1764,28 @@ bool RewriteYtMapJoin(TYtEquiJoin equiJoin, const TJoinLabels& labels, bool isLo
}
}
+ if (useBlocks) {
+ for (auto& [_, columnType] : columnTypes) {
+ if (!IsSupportedAsBlockType(pos, *columnType, ctx, *state->Types)) {
+ useBlocks = false;
+ YQL_CLOG(INFO, ProviderYt) << "Block mapjoin won't be used because of unsupported type: " << *columnType;
+ break;
+ }
+ }
+ }
+
+ ui64 partCount = 1;
+ ui64 partRows = settings.RightRows;
+ if ((settings.RightSize > 0) && useShards) {
+ partCount = std::min(((useBlocks ? settings.RightMemSizeUsingBlocks : settings.RightMemSize) + settings.MapJoinLimit - 1) / settings.MapJoinLimit, settings.RightRows);
+ partRows = (settings.RightRows + partCount - 1) / partCount;
+ }
+
+ if (partCount > 1) {
+ YQL_ENSURE(!isLookupJoin);
+ YQL_CLOG(INFO, ProviderYt) << strategyName << " sharded into " << partCount << " parts, each " << partRows << " rows";
+ }
+
auto mainPaths = MakeUnorderedSection(leftLeaf.Section, ctx).Paths();
auto mainSettings = NYql::RemoveSettings(leftLeaf.Section.Settings().Ref(), EYtSettingType::JoinLabel | EYtSettingType::StatColumns, ctx);
auto smallPaths = MakeUnorderedSection(rightLeaf.Section, ctx).Paths();
@@ -2117,16 +2127,6 @@ bool RewriteYtMapJoin(TYtEquiJoin equiJoin, const TJoinLabels& labels, bool isLo
}
}
- if (useBlocks) {
- for (auto& [_, columnType] : columnTypes) {
- if (!IsSupportedAsBlockType(pos, *columnType, ctx, *state->Types)) {
- useBlocks = false;
- YQL_CLOG(INFO, ProviderYt) << "Block mapjoin won't be used because of unsupported type: " << *columnType;
- break;
- }
- }
- }
-
TExprNode::TPtr joined;
if (useBlocks) {
TExprNode::TListType leftKeyColumnNodes;
diff --git a/yt/yql/tests/sql/suites/aggregate/too_wide.sql b/yt/yql/tests/sql/suites/aggregate/too_wide.sql
new file mode 100644
index 00000000000..71b4319224a
--- /dev/null
+++ b/yt/yql/tests/sql/suites/aggregate/too_wide.sql
@@ -0,0 +1,122 @@
+use plato;
+
+select * from (select cast(value as Int32) as value from Input) group by
+value+0,
+value+1,
+value+2,
+value+3,
+value+4,
+value+5,
+value+6,
+value+7,
+value+8,
+value+9,
+value+10,
+value+11,
+value+12,
+value+13,
+value+14,
+value+15,
+value+16,
+value+17,
+value+18,
+value+19,
+value+20,
+value+21,
+value+22,
+value+23,
+value+24,
+value+25,
+value+26,
+value+27,
+value+28,
+value+29,
+value+30,
+value+31,
+value+32,
+value+33,
+value+34,
+value+35,
+value+36,
+value+37,
+value+38,
+value+39,
+value+40,
+value+41,
+value+42,
+value+43,
+value+44,
+value+45,
+value+46,
+value+47,
+value+48,
+value+49,
+value+50,
+value+51,
+value+52,
+value+53,
+value+54,
+value+55,
+value+56,
+value+57,
+value+58,
+value+59,
+value+60,
+value+61,
+value+62,
+value+63,
+value+64,
+value+65,
+value+66,
+value+67,
+value+68,
+value+69,
+value+70,
+value+71,
+value+72,
+value+73,
+value+74,
+value+75,
+value+76,
+value+77,
+value+78,
+value+79,
+value+80,
+value+81,
+value+82,
+value+83,
+value+84,
+value+85,
+value+86,
+value+87,
+value+88,
+value+89,
+value+90,
+value+91,
+value+92,
+value+93,
+value+94,
+value+95,
+value+96,
+value+97,
+value+98,
+value+99,
+value+100,
+value+101,
+value+102,
+value+103,
+value+104,
+value+105,
+value+106,
+value+107,
+value+108,
+value+109,
+value+110,
+value+111,
+value+112,
+value+113,
+value+114,
+value+115,
+value+116,
+value+117,
+value+118
diff --git a/yt/yt/client/api/rpc_proxy/transaction_impl.cpp b/yt/yt/client/api/rpc_proxy/transaction_impl.cpp
index 164c5234049..bfd6d96b391 100644
--- a/yt/yt/client/api/rpc_proxy/transaction_impl.cpp
+++ b/yt/yt/client/api/rpc_proxy/transaction_impl.cpp
@@ -320,6 +320,7 @@ TFuture<TTransactionCommitResult> TTransaction::Commit(const TTransactionCommitO
ToProto(req->mutable_transaction_id(), GetId());
ToProto(req->mutable_additional_participant_cell_ids(), AdditionalParticipantCellIds_);
ToProto(req->mutable_prerequisite_options(), options);
+ req->set_max_allowed_commit_timestamp(options.MaxAllowedCommitTimestamp);
return req->Invoke();
}))
.Apply(
diff --git a/yt/yt/core/bus/tcp/config.cpp b/yt/yt/core/bus/tcp/config.cpp
index ed4aea2f75c..e88699accd7 100644
--- a/yt/yt/core/bus/tcp/config.cpp
+++ b/yt/yt/core/bus/tcp/config.cpp
@@ -157,12 +157,6 @@ void TBusConfig::Register(TRegistrar registrar)
.Default(EEncryptionMode::Optional);
registrar.Parameter("verification_mode", &TThis::VerificationMode)
.Default(EVerificationMode::None);
- registrar.Parameter("ca", &TThis::CA)
- .Default();
- registrar.Parameter("cert_chain", &TThis::CertificateChain)
- .Default();
- registrar.Parameter("private_key", &TThis::PrivateKey)
- .Default();
registrar.Parameter("cipher_list", &TThis::CipherList)
.Default();
registrar.Parameter("load_certs_from_bus_certs_directory", &TThis::LoadCertsFromBusCertsDirectory)
diff --git a/yt/yt/core/bus/tcp/config.h b/yt/yt/core/bus/tcp/config.h
index 1596590a678..84a8b174afd 100644
--- a/yt/yt/core/bus/tcp/config.h
+++ b/yt/yt/core/bus/tcp/config.h
@@ -90,6 +90,7 @@ DEFINE_REFCOUNTED_TYPE(TTcpDispatcherDynamicConfig)
struct TBusConfig
: public NNet::TDialerConfig
+ , public NCrypto::TSslContextConfig
{
bool EnableQuickAck;
@@ -110,9 +111,6 @@ struct TBusConfig
// Ssl options.
EEncryptionMode EncryptionMode;
EVerificationMode VerificationMode;
- NCrypto::TPemBlobConfigPtr CA;
- NCrypto::TPemBlobConfigPtr CertificateChain;
- NCrypto::TPemBlobConfigPtr PrivateKey;
std::optional<TString> CipherList;
bool LoadCertsFromBusCertsDirectory;
std::optional<TString> PeerAlternativeHostName;
diff --git a/yt/yt/core/bus/tcp/connection.cpp b/yt/yt/core/bus/tcp/connection.cpp
index 52ec0dfd1c2..9bd094e940a 100644
--- a/yt/yt/core/bus/tcp/connection.cpp
+++ b/yt/yt/core/bus/tcp/connection.cpp
@@ -3,8 +3,6 @@
#include "config.h"
#include "local_bypass.h"
#include "dispatcher_impl.h"
-#include "ssl_context.h"
-#include "ssl_helpers.h"
#include <yt/yt/core/misc/fs.h>
#include <yt/yt/core/misc/memory_usage_tracker.h>
@@ -40,6 +38,7 @@
namespace NYT::NBus {
using namespace NConcurrency;
+using namespace NCrypto;
using namespace NFS;
using namespace NNet;
using namespace NYTree;
@@ -1969,8 +1968,7 @@ bool TTcpConnection::CheckSslReadError(ssize_t result)
[[fallthrough]];
default:
UpdateBusCounter(&TBusNetworkBandCounters::ReadErrors, 1);
- Abort(TError(NBus::EErrorCode::SslError, "TLS/SSL read error")
- << TErrorAttribute("ssl_error", GetLastSslErrorString())
+ Abort(GetLastSslError("TLS/SSL read error")
<< TErrorAttribute("sys_error", TError::FromSystem(LastSystemError())));
break;
}
@@ -1997,8 +1995,7 @@ bool TTcpConnection::CheckSslWriteError(ssize_t result)
[[fallthrough]];
default:
UpdateBusCounter(&TBusNetworkBandCounters::WriteErrors, 1);
- Abort(TError(NBus::EErrorCode::SslError, "TLS/SSL write error")
- << TErrorAttribute("ssl_error", GetLastSslErrorString())
+ Abort(GetLastSslError("TLS/SSL write error")
<< TErrorAttribute("sys_error", TError::FromSystem(LastSystemError())));
break;
}
@@ -2055,8 +2052,7 @@ bool TTcpConnection::DoSslHandshake()
break;
}
- Abort(TError(NBus::EErrorCode::SslError, "Failed to establish TLS/SSL session")
- << TErrorAttribute("ssl_error", GetLastSslErrorString())
+ Abort(GetLastSslError("Failed to establish TLS/SSL session")
<< TErrorAttribute("sys_error", TError::FromSystem(LastSystemError())));
return false;
}
@@ -2094,78 +2090,45 @@ void TTcpConnection::TryEstablishSslSession()
YT_LOG_DEBUG("Starting TLS/SSL connection");
- if (Config_->LoadCertsFromBusCertsDirectory && !TTcpDispatcher::TImpl::Get()->GetBusCertsDirectoryPath()) {
- Abort(TError(NBus::EErrorCode::SslError, "bus_certs_directory_path is not set in tcp_dispatcher config"));
- return;
+ NCrypto::TCertificatePathResolver pathResolver;
+ if (Config_->LoadCertsFromBusCertsDirectory) {
+ // FIXME(khlebnikov): Move this and config into proper place.
+ if (!TTcpDispatcher::TImpl::Get()->GetBusCertsDirectoryPath()) {
+ Abort(TError(NBus::EErrorCode::SslError, "bus_certs_directory_path is not set in tcp_dispatcher config"));
+ return;
+ }
+ pathResolver = [&](const TString fileName) {
+ return JoinPaths(*TTcpDispatcher::TImpl::Get()->GetBusCertsDirectoryPath(), fileName);
+ };
}
- Ssl_.reset(SSL_new(TSslContext::Get()->GetSslCtx()));
- if (!Ssl_) {
- Abort(TError(NBus::EErrorCode::SslError, "Failed to create a new SSL structure: %v", GetLastSslErrorString()));
+ if (VerificationMode_ != EVerificationMode::None && !Config_->CertificateAuthority) {
+ Abort(TError(NBus::EErrorCode::SslError, "CA file is not set in bus config"));
return;
}
- if (SSL_set_fd(Ssl_.get(), Socket_) != 1) {
- Abort(TError(NBus::EErrorCode::SslError, "Failed to bind socket to SSL handle: %v", GetLastSslErrorString()));
+ if (ConnectionType_ == EConnectionType::Server && !Config_->CertificateChain) {
+ Abort(TError(NBus::EErrorCode::SslError, "Certificate chain file is not set in bus config"));
return;
}
- if (Config_->CipherList) {
- if (SSL_set_cipher_list(Ssl_.get(), Config_->CipherList->data()) != 1) {
- Abort(TError(NBus::EErrorCode::SslError, "Failed to set cipher list: %v", GetLastSslErrorString()));
- return;
- }
+ if (ConnectionType_ == EConnectionType::Server && !Config_->PrivateKey) {
+ Abort(TError(NBus::EErrorCode::SslError, "The private key file is not set in bus config"));
+ return;
}
-#define GET_CERT_FILE_PATH(file) \
- (Config_->LoadCertsFromBusCertsDirectory ? JoinPaths(*TTcpDispatcher::TImpl::Get()->GetBusCertsDirectoryPath(), (file)) : (file))
+ // FIXME(khlebnikov): Stop constructing SSL context from scratch for each connection.
+ auto sslContext = New<TSslContext>();
- if (ConnectionType_ == EConnectionType::Server) {
- SSL_set_accept_state(Ssl_.get());
+ sslContext->ApplyConfig(Config_, pathResolver);
- if (!Config_->CertificateChain) {
- Abort(TError(NBus::EErrorCode::SslError, "Certificate chain file is not set in bus config"));
- return;
- }
-
- if (Config_->CertificateChain->FileName) {
- const auto& certChainFile = GET_CERT_FILE_PATH(*Config_->CertificateChain->FileName);
- if (SSL_use_certificate_chain_file(Ssl_.get(), certChainFile.data()) != 1) {
- Abort(TError(NBus::EErrorCode::SslError, "Failed to load certificate chain file: %v", GetLastSslErrorString()));
- return;
- }
- } else {
- if (!UseCertificateChain(*Config_->CertificateChain->Value, Ssl_.get())) {
- Abort(TError(NBus::EErrorCode::SslError, "Failed to load certificate chain: %v", GetLastSslErrorString()));
- return;
- }
- }
-
- if (!Config_->PrivateKey) {
- Abort(TError(NBus::EErrorCode::SslError, "The private key file is not set in bus config"));
- return;
- }
+ if (Config_->CipherList) {
+ sslContext->SetCipherList(*Config_->CipherList);
+ }
- if (Config_->PrivateKey->FileName) {
- const auto& privateKeyFile = GET_CERT_FILE_PATH(*Config_->PrivateKey->FileName);
- if (SSL_use_PrivateKey_file(Ssl_.get(), privateKeyFile.data(), SSL_FILETYPE_PEM) != 1) {
- Abort(TError(NBus::EErrorCode::SslError, "Failed to load private key file: %v", GetLastSslErrorString()));
- return;
- }
- } else {
- if (!UsePrivateKey(*Config_->PrivateKey->Value, Ssl_.get())) {
- Abort(TError(NBus::EErrorCode::SslError, "Failed to load private key: %v", GetLastSslErrorString()));
- return;
- }
- }
+ sslContext->Commit();
- if (SSL_check_private_key(Ssl_.get()) != 1) {
- Abort(TError(NBus::EErrorCode::SslError, "Failed to check the consistency of a private key with the corresponding certificate: %v", GetLastSslErrorString()));
- return;
- }
- } else {
- SSL_set_connect_state(Ssl_.get());
- }
+ Ssl_ = std::move(sslContext->NewSsl());
switch (VerificationMode_) {
case EVerificationMode::Full:
@@ -2174,43 +2137,35 @@ void TTcpConnection::TryEstablishSslSession()
if (Config_->PeerAlternativeHostName) {
// Set hostname for peer certificate verification.
if (SSL_set1_host(Ssl_.get(), EndpointHostName_.c_str()) != 1) {
- Abort(TError(NBus::EErrorCode::SslError, "Failed to set hostname %v for peer certificate verification", EndpointHostName_));
+ Abort(GetLastSslError("Failed to set hostname for peer certificate verification")
+ << TErrorAttribute("ssl_peer_hostname", EndpointHostName_));
return;
}
// Add alternative hostname for peer certificate verification.
if (SSL_add1_host(Ssl_.get(), Config_->PeerAlternativeHostName->c_str()) != 1) {
- Abort(TError(NBus::EErrorCode::SslError, "Failed to add alternative hostname %v for peer certificate verification", *Config_->PeerAlternativeHostName));
+ Abort(GetLastSslError("Failed to add alternative hostname for peer certificate verification")
+ << TErrorAttribute("ssl_peer_hostname", *Config_->PeerAlternativeHostName));
return;
}
} else if (auto networkAddress = TNetworkAddress::TryParse(EndpointHostName_); networkAddress.IsOK() && networkAddress.Value().IsIP()) {
// Set IP address for peer certificate verification.
auto address = ToString(networkAddress.Value(), {.IncludePort = false, .IncludeTcpProtocol = false});
if (X509_VERIFY_PARAM_set1_ip_asc(SSL_get0_param(Ssl_.get()), address.c_str()) != 1) {
- Abort(TError(NBus::EErrorCode::SslError, "Failed to set IP address %v for peer certificate verification", address));
+ Abort(GetLastSslError("Failed to set IP address for peer certificate verification")
+ << TErrorAttribute("ssl_peer_address", address));
return;
}
} else {
// Set hostname for peer certificate verification.
if (SSL_set1_host(Ssl_.get(), EndpointHostName_.c_str()) != 1) {
- Abort(TError(NBus::EErrorCode::SslError, "Failed to set hostname %v for peer certificate verification", EndpointHostName_));
+ Abort(GetLastSslError("Failed to set hostname for peer certificate verification")
+ << TErrorAttribute("ssl_peer_hostname", EndpointHostName_));
return;
}
}
[[fallthrough]];
case EVerificationMode::Ca: {
- if (!Config_->CA) {
- Abort(TError(NBus::EErrorCode::SslError, "CA file is not set in bus config"));
- return;
- }
-
- if (Config_->CA->FileName) {
- const auto& caFile = GET_CERT_FILE_PATH(*Config_->CA->FileName);
- TSslContext::Get()->LoadCAFileIfNotLoaded(caFile);
- } else {
- TSslContext::Get()->UseCAIfNotUsed(*Config_->CA->Value);
- }
-
// Enable verification of the peer's certificate with the CA.
SSL_set_verify(Ssl_.get(), SSL_VERIFY_PEER, /*callback*/ nullptr);
break;
@@ -2221,6 +2176,24 @@ void TTcpConnection::TryEstablishSslSession()
YT_ABORT();
}
+ SSL_set_mode(Ssl_.get(), SSL_MODE_ENABLE_PARTIAL_WRITE | SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER);
+
+ if (SSL_set_fd(Ssl_.get(), Socket_) != 1) {
+ Abort(GetLastSslError("Failed to bind socket to SSL handle"));
+ return;
+ }
+
+ if (ConnectionType_ == EConnectionType::Server) {
+ SSL_set_accept_state(Ssl_.get());
+
+ if (SSL_check_private_key(Ssl_.get()) != 1) {
+ Abort(GetLastSslError("Failed to check the consistency of a private key with the corresponding certificate"));
+ return;
+ }
+ } else {
+ SSL_set_connect_state(Ssl_.get());
+ }
+
PendingSslHandshake_ = DoSslHandshake();
// Check that connection hasn't been aborted in DoSslHandshake().
@@ -2230,8 +2203,6 @@ void TTcpConnection::TryEstablishSslSession()
NetworkCounters_.Exchange(TTcpDispatcher::TImpl::Get()->GetCounters(NetworkName_, IsEncrypted()));
UpdateConnectionCount(1);
}
-
-#undef GET_CERT_FILE_PATH
}
bool TTcpConnection::OnSslAckPacketReceived()
diff --git a/yt/yt/core/bus/tcp/connection.h b/yt/yt/core/bus/tcp/connection.h
index ff8c4eb52fb..43c489e4db2 100644
--- a/yt/yt/core/bus/tcp/connection.h
+++ b/yt/yt/core/bus/tcp/connection.h
@@ -2,7 +2,6 @@
#include "packet.h"
#include "dispatcher_impl.h"
-#include "ssl_helpers.h"
#include <yt/yt/core/bus/private.h>
#include <yt/yt/core/bus/bus.h>
@@ -11,6 +10,8 @@
#include <yt/yt/core/actions/future.h>
+#include <yt/yt/core/crypto/tls.h>
+
#include <yt/yt/core/logging/log.h>
#include <yt/yt/core/net/address.h>
@@ -31,8 +32,6 @@
#include <util/network/init.h>
-#include <openssl/ssl.h>
-
#ifdef _win_
#include <winsock2.h>
@@ -274,7 +273,7 @@ private:
bool SslAckEnqueued_ = false;
bool SslAckReceived_ = false;
bool SslAckSent_ = false;
- std::unique_ptr<SSL, TDeleter> Ssl_;
+ NCrypto::TSslPtr Ssl_;
std::atomic<ESslState> SslState_ = ESslState::None;
const EEncryptionMode EncryptionMode_;
diff --git a/yt/yt/core/bus/tcp/ssl_context.cpp b/yt/yt/core/bus/tcp/ssl_context.cpp
deleted file mode 100644
index 8084b641c8c..00000000000
--- a/yt/yt/core/bus/tcp/ssl_context.cpp
+++ /dev/null
@@ -1,322 +0,0 @@
-#include "ssl_context.h"
-#include "ssl_helpers.h"
-
-#include <yt/yt/core/misc/error.h>
-
-#include <library/cpp/openssl/init/init.h>
-
-#include <library/cpp/yt/threading/spin_lock.h>
-
-#include <library/cpp/yt/memory/leaky_singleton.h>
-
-#include <util/system/mutex.h>
-
-#include <openssl/err.h>
-#include <openssl/ssl.h>
-
-#include <memory>
-
-namespace NYT::NBus {
-
-////////////////////////////////////////////////////////////////////////////////
-
-class TSslContext::TImpl
-{
-public:
- TImpl()
- {
- InitOpenSSL();
-
- SslCtx_.reset(SSL_CTX_new(TLS_method()));
- if (!SslCtx_) {
- THROW_ERROR_EXCEPTION("Failed to create TLS/SSL context: %v", GetLastSslErrorString());
- }
-
- if (SSL_CTX_set_min_proto_version(SslCtx_.get(), TLS1_2_VERSION) != 1) {
- THROW_ERROR_EXCEPTION("Failed to set min protocol version: %v", GetLastSslErrorString());
- }
-
- if (SSL_CTX_set_max_proto_version(SslCtx_.get(), TLS1_2_VERSION) != 1) {
- THROW_ERROR_EXCEPTION("Failed to set max protocol version: %v", GetLastSslErrorString());
- }
-
- SSL_CTX_set_mode(SslCtx_.get(), SSL_MODE_ENABLE_PARTIAL_WRITE | SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER);
- }
-
- SSL_CTX* GetCtx()
- {
- return SslCtx_.get();
- }
-
- //! This function is for testing purposes.
- void LoadCAFile(const TString& filePath)
- {
- auto guard = Guard(SpinLock_);
-
- LoadCAFileUnlocked(filePath);
-
- CAIsLoaded_ = true;
- }
-
- void LoadCAFileIfNotLoaded(const TString& filePath)
- {
- if (CAIsLoaded_) {
- return;
- }
-
- auto guard = Guard(SpinLock_);
-
- if (CAIsLoaded_) {
- return;
- }
-
- LoadCAFileUnlocked(filePath);
-
- CAIsLoaded_ = true;
- }
-
- void LoadCertificateChain(const TString& filePath)
- {
- auto ret = SSL_CTX_use_certificate_chain_file(SslCtx_.get(), filePath.data());
- if (ret != 1) {
- THROW_ERROR_EXCEPTION("Failed to load certificate chain: %v", GetLastSslErrorString());
- }
- }
-
- void LoadPrivateKey(const TString& filePath)
- {
- auto ret = SSL_CTX_use_RSAPrivateKey_file(SslCtx_.get(), filePath.data(), SSL_FILETYPE_PEM);
- if (ret != 1) {
- THROW_ERROR_EXCEPTION("Failed to load private key: %v", GetLastSslErrorString());
- }
- }
-
- //! This function is for testing purposes.
- void UseCA(const TString& ca)
- {
- auto guard = Guard(SpinLock_);
-
- UseCAUnlocked(ca);
-
- CAIsLoaded_ = true;
- }
-
- void UseCAIfNotUsed(const TString& ca)
- {
- if (CAIsLoaded_) {
- return;
- }
-
- auto guard = Guard(SpinLock_);
-
- if (CAIsLoaded_) {
- return;
- }
-
- UseCAUnlocked(ca);
-
- CAIsLoaded_ = true;
- }
-
- void UseCertificateChain(const TString& certificate)
- {
- std::unique_ptr<BIO, TDeleter> bio(BIO_new_mem_buf(certificate.data(), certificate.size()));
- if (!bio) {
- THROW_ERROR_EXCEPTION("Failed to allocate memory buffer for certificate: %v", GetLastSslErrorString());
- }
-
- // The first certificate in the chain is expected to be a leaf.
- std::unique_ptr<X509, TDeleter> cert(PEM_read_bio_X509(bio.get(), nullptr, nullptr, nullptr));
- if (!cert) {
- THROW_ERROR_EXCEPTION("Failed to read certificate from memory buffer: %v", GetLastSslErrorString());
- }
-
- if (SSL_CTX_use_certificate(SslCtx_.get(), cert.get()) != 1) {
- THROW_ERROR_EXCEPTION("Failed to use cert in ssl: %v", GetLastSslErrorString());
- }
-
- // Load additional certificates in the chain.
- while (auto cert = PEM_read_bio_X509(bio.get(), nullptr, nullptr, nullptr)) {
- if (SSL_CTX_add0_chain_cert(SslCtx_.get(), cert) != 1) {
- THROW_ERROR_EXCEPTION("Failed to add cert to ssl: %v", GetLastSslErrorString());
- }
- // Do not X509_free() if certificate was added by SSL_CTX_add0_chain_cert().
- }
- }
-
- void UsePrivateKey(const TString& privateKey)
- {
- std::unique_ptr<BIO, TDeleter> bio(BIO_new_mem_buf(privateKey.data(), privateKey.size()));
- if (!bio) {
- THROW_ERROR_EXCEPTION("Failed to allocate memory buffer for private key: %v", GetLastSslErrorString());
- }
-
- std::unique_ptr<RSA, TDeleter> pkey(PEM_read_bio_RSAPrivateKey(bio.get(), nullptr, nullptr, nullptr));
- if (!pkey) {
- THROW_ERROR_EXCEPTION("Failed to read private key from memory buffer: %v", GetLastSslErrorString());
- }
-
- if (SSL_CTX_use_RSAPrivateKey(SslCtx_.get(), pkey.get()) != 1) {
- THROW_ERROR_EXCEPTION("Failed to add the private RSA key to ctx: %v", GetLastSslErrorString());
- }
- }
-
- void SetCipherListIfUnset(const TString& cipherList)
- {
- if (CipherListIsSet_) {
- return;
- }
-
- auto guard = Guard(SpinLock_);
-
- if (CipherListIsSet_) {
- return;
- }
-
- SetCipherListUnlocked(cipherList);
-
- CipherListIsSet_ = true;
- }
-
- //! This function is for testing purposes.
- void SetCipherList(const TString& cipherList)
- {
- auto guard = Guard(SpinLock_);
-
- SetCipherListUnlocked(cipherList);
-
- CipherListIsSet_ = true;
- }
-
- //! Check the consistency of a private key with the corresponding certificate.
- //! A private key and the corresponding certificate have to be loaded into ctx.
- void CheckPrivateKeyWithCertificate()
- {
- if (SSL_CTX_check_private_key(SslCtx_.get()) != 1) {
- THROW_ERROR_EXCEPTION("Failed to check the consistency of a private key with the corresponding certificate: %v", GetLastSslErrorString());
- }
- }
-
-private:
- void LoadCAFileUnlocked(const TString& filePath)
- {
- auto ret = SSL_CTX_load_verify_locations(SslCtx_.get(), filePath.data(), nullptr);
- if (ret != 1) {
- THROW_ERROR_EXCEPTION("Failed to load CA file: %v", GetLastSslErrorString());
- }
- }
-
- void SetCipherListUnlocked(const TString& cipherList)
- {
- if (SSL_CTX_set_cipher_list(SslCtx_.get(), cipherList.data()) != 1) {
- THROW_ERROR_EXCEPTION("Failed to set cipher list: %v", GetLastSslErrorString());
- }
- }
-
- void UseCAUnlocked(const TString& ca)
- {
- std::unique_ptr<BIO, TDeleter> bio(BIO_new_mem_buf(ca.data(), ca.size()));
- if (!bio) {
- THROW_ERROR_EXCEPTION("Failed to allocate memory buffer for CA certificate: %v", GetLastSslErrorString());
- }
-
- auto store = SSL_CTX_get_cert_store(SslCtx_.get());
-
- // Load certificate chain.
- while (auto cert = PEM_read_bio_X509(bio.get(), nullptr, nullptr, nullptr)) {
- if (X509_STORE_add_cert(store, cert) != 1) {
- THROW_ERROR_EXCEPTION("Failed to add cert to store: %v", GetLastSslErrorString());
- }
- X509_free(cert);
- }
- }
-
-private:
- YT_DECLARE_SPIN_LOCK(NThreading::TSpinLock, SpinLock_);
- std::atomic<bool> CAIsLoaded_ = false;
- std::atomic<bool> CipherListIsSet_ = false;
- std::unique_ptr<SSL_CTX, TDeleter> SslCtx_;
-};
-
-////////////////////////////////////////////////////////////////////////////////
-
-TSslContext::TSslContext()
-{
- Reset();
-}
-
-TSslContext::~TSslContext() = default;
-
-SSL_CTX* TSslContext::GetSslCtx()
-{
- return Impl_->GetCtx();
-}
-
-TSslContext* TSslContext::Get()
-{
- return LeakySingleton<TSslContext>();
-}
-
-void TSslContext::LoadCAFile(const TString& filePath)
-{
- return Impl_->LoadCAFile(filePath);
-}
-
-void TSslContext::LoadCAFileIfNotLoaded(const TString& filePath)
-{
- return Impl_->LoadCAFileIfNotLoaded(filePath);
-}
-
-void TSslContext::LoadCertificateChain(const TString& filePath)
-{
- return Impl_->LoadCertificateChain(filePath);
-}
-
-void TSslContext::LoadPrivateKey(const TString& filePath)
-{
- return Impl_->LoadPrivateKey(filePath);
-}
-
-void TSslContext::UseCA(const TString& ca)
-{
- return Impl_->UseCA(ca);
-}
-
-void TSslContext::UseCAIfNotUsed(const TString& ca)
-{
- return Impl_->UseCAIfNotUsed(ca);
-}
-
-void TSslContext::UseCertificateChain(const TString& certificate)
-{
- return Impl_->UseCertificateChain(certificate);
-}
-
-void TSslContext::UsePrivateKey(const TString& privateKey)
-{
- return Impl_->UsePrivateKey(privateKey);
-}
-
-void TSslContext::SetCipherList(const TString& cipherList)
-{
- return Impl_->SetCipherList(cipherList);
-}
-
-void TSslContext::SetCipherListIfUnset(const TString& cipherList)
-{
- return Impl_->SetCipherListIfUnset(cipherList);
-}
-
-void TSslContext::CheckPrivateKeyWithCertificate()
-{
- return Impl_->CheckPrivateKeyWithCertificate();
-}
-
-void TSslContext::Reset()
-{
- Impl_ = std::make_unique<TImpl>();
-}
-
-////////////////////////////////////////////////////////////////////////////////
-
-} // namespace NYT::NBus
diff --git a/yt/yt/core/bus/tcp/ssl_context.h b/yt/yt/core/bus/tcp/ssl_context.h
deleted file mode 100644
index 21d89d55624..00000000000
--- a/yt/yt/core/bus/tcp/ssl_context.h
+++ /dev/null
@@ -1,44 +0,0 @@
-#pragma once
-
-#include "public.h"
-
-#include <openssl/ssl.h>
-
-#include <memory>
-
-namespace NYT::NBus {
-
-////////////////////////////////////////////////////////////////////////////////
-
-class TSslContext
-{
-public:
- TSslContext();
- ~TSslContext();
-
- SSL_CTX* GetSslCtx();
- static TSslContext* Get();
-
- void LoadCAFileIfNotLoaded(const TString& filePath);
- void LoadCertificateChain(const TString& filePath);
- void LoadPrivateKey(const TString& filePath);
- void UseCAIfNotUsed(const TString& ca);
- void UseCertificateChain(const TString& certificate);
- void UsePrivateKey(const TString& privateKey);
- void SetCipherListIfUnset(const TString& cipherList);
- void CheckPrivateKeyWithCertificate();
-
- // For testing purposes.
- void LoadCAFile(const TString& filePath);
- void Reset();
- void SetCipherList(const TString& cipherList);
- void UseCA(const TString& ca);
-
-private:
- class TImpl;
- std::unique_ptr<TImpl> Impl_;
-};
-
-////////////////////////////////////////////////////////////////////////////////
-
-} // namespace NYT::NBus
diff --git a/yt/yt/core/bus/tcp/ssl_helpers.cpp b/yt/yt/core/bus/tcp/ssl_helpers.cpp
deleted file mode 100644
index 0a115002332..00000000000
--- a/yt/yt/core/bus/tcp/ssl_helpers.cpp
+++ /dev/null
@@ -1,103 +0,0 @@
-#include "ssl_helpers.h"
-
-#include <yt/yt/core/misc/error.h>
-
-#include <openssl/ssl.h>
-#include <openssl/err.h>
-
-#include <util/generic/string.h>
-
-#include <memory>
-
-namespace NYT::NBus {
-
-////////////////////////////////////////////////////////////////////////////////
-
-void TDeleter::operator()(SSL_CTX* ctx) const
-{
- SSL_CTX_free(ctx);
-}
-
-void TDeleter::operator()(SSL* ctx) const
-{
- SSL_free(ctx);
-}
-
-void TDeleter::operator()(BIO* bio) const
-{
- BIO_free(bio);
-}
-
-void TDeleter::operator()(RSA* rsa) const
-{
- RSA_free(rsa);
-}
-
-void TDeleter::operator()(X509* x509) const
-{
- X509_free(x509);
-}
-
-////////////////////////////////////////////////////////////////////////////////
-
-TString GetLastSslErrorString()
-{
- char errorStr[256];
- ERR_error_string_n(ERR_get_error(), errorStr, sizeof(errorStr));
- return errorStr;
-}
-
-////////////////////////////////////////////////////////////////////////////////
-
-bool UseCertificateChain(const TString& certificate, SSL* ssl)
-{
- std::unique_ptr<BIO, TDeleter> bio(BIO_new_mem_buf(certificate.data(), certificate.size()));
- if (!bio) {
- return false;
- }
-
- // The first certificate in the chain is expected to be a leaf.
- std::unique_ptr<X509, TDeleter> cert(PEM_read_bio_X509(bio.get(), nullptr, nullptr, nullptr));
- if (!cert) {
- return false;
- }
-
- if (SSL_use_certificate(ssl, cert.get()) != 1) {
- return false;
- }
-
- // Load additional certificates in the chain.
- while (auto cert = PEM_read_bio_X509(bio.get(), nullptr, nullptr, nullptr)) {
- if (SSL_add0_chain_cert(ssl, cert) != 1) {
- return false;
- }
- // Do not X509_free() if certificate was added by SSL_CTX_add0_chain_cert().
- }
-
- return true;
-}
-
-////////////////////////////////////////////////////////////////////////////////
-
-bool UsePrivateKey(const TString& privateKey, SSL* ssl)
-{
- std::unique_ptr<BIO, TDeleter> bio(BIO_new_mem_buf(privateKey.data(), privateKey.size()));
- if (!bio) {
- return false;
- }
-
- std::unique_ptr<RSA, TDeleter> pkey(PEM_read_bio_RSAPrivateKey(bio.get(), nullptr, nullptr, nullptr));
- if (!pkey) {
- return false;
- }
-
- if (SSL_use_RSAPrivateKey(ssl, pkey.get()) != 1) {
- return false;
- }
-
- return true;
-}
-
-////////////////////////////////////////////////////////////////////////////////
-
-} // namespace NYT::NBus
diff --git a/yt/yt/core/bus/tcp/ssl_helpers.h b/yt/yt/core/bus/tcp/ssl_helpers.h
deleted file mode 100644
index e99f826cb80..00000000000
--- a/yt/yt/core/bus/tcp/ssl_helpers.h
+++ /dev/null
@@ -1,34 +0,0 @@
-#pragma once
-
-#include <openssl/ssl.h>
-
-#include <util/generic/string.h>
-
-namespace NYT::NBus {
-
-////////////////////////////////////////////////////////////////////////////////
-
-struct TDeleter
-{
- void operator()(SSL_CTX* ctx) const;
- void operator()(SSL* ctx) const;
- void operator()(BIO* bio) const;
- void operator()(RSA* rsa) const;
- void operator()(X509* x509) const;
-};
-
-////////////////////////////////////////////////////////////////////////////////
-
-TString GetLastSslErrorString();
-
-////////////////////////////////////////////////////////////////////////////////
-
-bool UseCertificateChain(const TString& certificate, SSL* ssl);
-
-////////////////////////////////////////////////////////////////////////////////
-
-bool UsePrivateKey(const TString& privateKey, SSL* ssl);
-
-////////////////////////////////////////////////////////////////////////////////
-
-} // namespace NYT::NBus
diff --git a/yt/yt/core/bus/unittests/ssl_ut.cpp b/yt/yt/core/bus/unittests/ssl_ut.cpp
index 6f65b24c8e1..e99bf169821 100644
--- a/yt/yt/core/bus/unittests/ssl_ut.cpp
+++ b/yt/yt/core/bus/unittests/ssl_ut.cpp
@@ -7,13 +7,15 @@
#include <yt/yt/core/bus/tcp/config.h>
#include <yt/yt/core/bus/tcp/client.h>
#include <yt/yt/core/bus/tcp/server.h>
-#include <yt/yt/core/bus/tcp/ssl_context.h>
+#include <library/cpp/testing/common/env.h>
#include <library/cpp/testing/common/network.h>
namespace NYT::NBus {
namespace {
+using namespace NCrypto;
+
////////////////////////////////////////////////////////////////////////////////
TSharedRefArray CreateMessage(int numParts, int partSize = 1)
@@ -54,238 +56,15 @@ public:
TString AddressWithIpV4;
TString AddressWithIpV6;
- const char* CA = R"foo(-----BEGIN CERTIFICATE-----
-MIIFWjCCA0KgAwIBAgIBATANBgkqhkiG9w0BAQsFADBGMQswCQYDVQQGEwJSVTEP
-MA0GA1UECAwGTW9zY293MQ8wDQYDVQQKDAZZYW5kZXgxFTATBgNVBAMMDENBQGxv
-Y2FsaG9zdDAeFw0yMzA3MTMxMTUxMTFaFw0zMzA3MTAxMTUxMTFaMEYxCzAJBgNV
-BAYTAlJVMQ8wDQYDVQQIDAZNb3Njb3cxDzANBgNVBAoMBllhbmRleDEVMBMGA1UE
-AwwMQ0FAbG9jYWxob3N0MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA
-mxWRSfbYAJcBgGhYagnQMJOiW60B82Ok94n8muIaa2gRo0ZHKmtY7CMiFIIN0GhI
-Blw+G6HPN8SPYK0+bTlAIHlb0u6/ty2AVyveiH31p7Ld/Ib5S/MSKuMiGt5S/Ci9
-mSdRAmD46twIKfm9bY/8SZmNsTCJIrKnJaTbjTnbz9O3FVAYjiuc8yNGb1LnZNA6
-B4ZrF3fkYr9kn4nKSOd6mypPFIOZAxKGQ95X9nCblajEMAPzHfr+1EArnM+PauAO
-cMcxfKT+OlMPGR4ZtZpl90qX6ZVZqD9zd8gp2I/2SM1vVS7AT96t89T7Uag8OjH2
-c8jMCD2z1fk1oDD4K53pGkBucTwDolCvxIbN77gcwkutdor/dbHLqIvGV/M4Z/iA
-GzqCD6pCpi5gT8SXn2IQrlvSdF01k9YZS093Y8bIQm3dCo1lKDaz6oHytk9Ro+fu
-b+dLOhSjopNfa/1Thw6fgta/mRsdgubWI/IHn+IyMpW65vGbnrMD4oQl1AanRJj5
-iBS73ZXIs/Y9LuMtSNk/I9u1o2fc+Sg0zb1AchC70h8M3sAYaGbqg1churOaDuTO
-Yom5W+bAQO8uLUvmlDcpqxMBYqeE8VJGJWRLtQnhM18iYAa03w7m6uMzrlkoX1Q7
-AHnX4899WSUcz/BJc5JBAtHQXCzEzyudT+8xw5MwqRkCAwEAAaNTMFEwHQYDVR0O
-BBYEFFuWdWxzQ6TF/yH3sQnlk3EEr1VjMB8GA1UdIwQYMBaAFFuWdWxzQ6TF/yH3
-sQnlk3EEr1VjMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggIBAHqS
-W4FIjH/YKkcj/wuh7sQQTIBAx+LsjosyHb9K/1QbgLNVpxawl/YlOiArfVmgTGud
-9B6xdVacxqI6saMqHNPPgYm7OPnozSnprRKh9yGb2TS9j+2G7hCQLnM+xYroP8XE
-8fL9tyq14lHb/BFYnPsFFxqeA6lXzzm73hlm/hH58CWpL/3eR/TDjC+oiEBBV4VF
-Dm4X3L73MFcniDKkCb7Mw3wdi6zqx231F4+9Cqgq+RqAucnmLXTG7yR7wOKP2u8E
-Ye1Jrt3nnMlD1tqiXyRJywPSK9mMiBTGbzGLLiTcvecBIHG8oRuvn1DnkZa9L/48
-+1aTC8QhbRslaeDyXj5IY/7scW7ZkEw18qYCT+9sI4/LngS4U8+taKqLf5S8A5sR
-O8OCP0nMk/l5f84cDwe1dp4GXVQdkbnfT1sd93BLww7Szbw2iMt9sLMmyY8qIe7a
-Ss3OEfP6eyNCBu6KtL8oufdj1AqAQdmYYTlGwgaFZTAomDiJU662EGSt3uXK7V32
-EzgJbpxSSjWh5JpCDA/jnqzlkFkSaw92/HZwO5l1lCdqK+F1UWYQyU0NveOaSEuX
-344PIi8m1YWwUfukUB97D+C6B4y5vgdkC7OYLHb4W4+N0ZvYtbDDaBeTpn70CiI7
-JFWcF3ghP7uPmbONWLiTFwxsSJHT0svVQZgq1aZz
------END CERTIFICATE-----)foo";
-
- const char* CertChain = R"foo(-----BEGIN CERTIFICATE-----
-MIIFUTCCAzmgAwIBAgIBATANBgkqhkiG9w0BAQsFADBGMQswCQYDVQQGEwJSVTEP
-MA0GA1UECAwGTW9zY293MQ8wDQYDVQQKDAZZYW5kZXgxFTATBgNVBAMMDENBQGxv
-Y2FsaG9zdDAeFw0yMzA3MTMxMTUxMTdaFw0zMzA3MTAxMTUxMTdaMEMxCzAJBgNV
-BAYTAlJVMQ8wDQYDVQQIDAZNb3Njb3cxDzANBgNVBAoMBllhbmRleDESMBAGA1UE
-AwwJbG9jYWxob3N0MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA1ZrD
-+kcfmsi52zMZ7cJxWl8pMI1kX1uQ6KkOJRVQVuweJuhp35rJGK0vHkWvrpihO+Ue
-h1jncReH8PaNdejt3gPsYJksUeRxCaZ8O+awtyOq6TK8L9pDzbQyXme5k2Ea/+yl
-XSHrGGzZUuHaZRzBigyQXzb04ZdQMzqGGKUz52ed5elZnOoAMUL7/wZ7O8n5dlXh
-DwxEQ4SQ3a39SLpPWv3hWXiIDrlR5xhkqQjZnb0WEVlSwN8lpyjA6XwCs/IX7PVr
-khoBKq7664rsR3B7qhGA5cmSy4+KrSoYKYHGCvl6OakO7fn4FcJwHHFNQzMq8oa5
-i5LbYsaCy6IgJt07nBeNXxaEY+4WIImHfWR0yvQgqtrFQZuIQ4N7YeZpLvTtoO3I
-UJyjMsih5GX4LPY0k9nE8rd437lV3PdiBYAktkwmgCTY9uJFhCBPFsMhE30nL79t
-X0gm5FugW1aZ3CFvgQywkcYQIjBjJQ42zcpqnLjj4lIwfDO5YEAiAOPqQ3VV7FKm
-bSWpeXyRAL/uqkBVCDKTsKJkEjd8gc30honRIKNHQkr/RoV1M1W449uRkvfVfbfg
-kdcRcYMIynTwZMSiKDoxXkQyrcyY6qVjYQNUKwRvtN2m6aGftYhfgo4j01UHTrK7
-BHRHKjW5LFXgNJUXnycmO69B+w1eTRgnVE7ws6sCAwEAAaNNMEswCQYDVR0TBAIw
-ADAdBgNVHQ4EFgQUNvB1ghYixkG53VZoSvUPnCCN7hwwHwYDVR0jBBgwFoAUW5Z1
-bHNDpMX/IfexCeWTcQSvVWMwDQYJKoZIhvcNAQELBQADggIBAHW/+xxign61axqT
-hO338rJGuidhXzraCQtJT/J6OwUaI+P6C71wUoBKEk3SrIVNG1Q8mrlsj93EzQuv
-M9OvBenIomF7A2ydaFL/6NJhZ8W6PmZwYF3VvInYJ2p7zqCMjqECg2C92NIC7+Z6
-fdmL+xoj3XKYypqA1x6xvSnCtXyXGRCxta3Es163wbqffq+4jjBUFOyUr9vk2N7X
-vy3/x8LWHIXffzNSJLtnXiznSNBSubmedac8JQ+XE9RgK+R0kUrj1lqnkSg+tPWD
-jP52kG84J9URV18BZfLFpcUiYloWXREfwNXRhMAQ6DyucupLW1Skl+Nf9K+41C3h
-f4mDn4Axn6toBzav9NLdFelGVAp4R3Yjoiv2LvpwYxGnMs/cPJMGm/NqoAbuyOMY
-ZKZPWvwsbxeaG8u9GRGvTSpawnGWJqIgxknKpQT5QztjNwtI9iT0/f4n3CPEGdAi
-6bHw0q+jiCKmXZMZPHyJ/tSJ74H7tdeWYYjhthJWnrAz4BziZ+bFBLkcYq95VV4L
-pOeMUr0PUi8fpSK06wIVTES9AWgcfuXL6i7AQ+hadEa3Ve1BGRsu0KOXW2XZZFeo
-3Pczm/o+jwMiLELcgrM5Ngy8dcCKr6v84F+fi9Y+C8+RZ7g37aLJM0kqbaoN8owL
-mP88f9xDGRAmesvuYlHo+57VTyzU
------END CERTIFICATE-----)foo";
-
- const char* PrivateKey = R"foo(-----BEGIN PRIVATE KEY-----
-MIIJQQIBADANBgkqhkiG9w0BAQEFAASCCSswggknAgEAAoICAQDVmsP6Rx+ayLnb
-MxntwnFaXykwjWRfW5DoqQ4lFVBW7B4m6GnfmskYrS8eRa+umKE75R6HWOdxF4fw
-9o116O3eA+xgmSxR5HEJpnw75rC3I6rpMrwv2kPNtDJeZ7mTYRr/7KVdIesYbNlS
-4dplHMGKDJBfNvThl1AzOoYYpTPnZ53l6Vmc6gAxQvv/Bns7yfl2VeEPDERDhJDd
-rf1Iuk9a/eFZeIgOuVHnGGSpCNmdvRYRWVLA3yWnKMDpfAKz8hfs9WuSGgEqrvrr
-iuxHcHuqEYDlyZLLj4qtKhgpgcYK+Xo5qQ7t+fgVwnAccU1DMyryhrmLkttixoLL
-oiAm3TucF41fFoRj7hYgiYd9ZHTK9CCq2sVBm4hDg3th5mku9O2g7chQnKMyyKHk
-Zfgs9jST2cTyt3jfuVXc92IFgCS2TCaAJNj24kWEIE8WwyETfScvv21fSCbkW6Bb
-VpncIW+BDLCRxhAiMGMlDjbNymqcuOPiUjB8M7lgQCIA4+pDdVXsUqZtJal5fJEA
-v+6qQFUIMpOwomQSN3yBzfSGidEgo0dCSv9GhXUzVbjj25GS99V9t+CR1xFxgwjK
-dPBkxKIoOjFeRDKtzJjqpWNhA1QrBG+03abpoZ+1iF+CjiPTVQdOsrsEdEcqNbks
-VeA0lRefJyY7r0H7DV5NGCdUTvCzqwIDAQABAoICAEvCv0TLKiH/lK/y4XzrTMIF
-Y3oVhCawNubWYy5y71JNH+qj3z1QTIgEkOQ3Sjbuaq1wN9JAjaIWewBTqlvKOGfY
-02N1oHsRP6hxFLo4ObBTJcDdXlLIoujYQ08pke/8bpOcDxDHwXch0Djt40SenOSG
-TUSAHP3QacEpvjsKiSzHmwDbMY4Oju/p9q/+0AGmQuUeU5s/Og0KfUkq911uu0um
-JWHS9srmHu8Mv1MW0Px5/tQ7brb6zoOJ2FZXxiulr6e7aiJhN824T0XwuZojArGQ
-0LtvsbGiYUjG19gM772fu6Ks3B863Ct3kcT8yK8PfGmVsESZW1ee2fA4uheeuw+c
-/3D71+dnjvw0/LFQEWyiH9NhjvafALBGxtlbfTTVtw5bHVaoalYpEggKnJrRabPa
-HmdPkEAK0vJzBvTjPW1lEvlxUkol1SsYq8xpBkatieZzf4+SWk/ugP7rvoRVRbB1
-GmHc9CK9jCDTcN7ii5pTOpc2VOK5cvn+K5L4P7Qw4kK37pyO56jhVbTAavNxotSc
-+mZa8OqZaK0jvD9sUPCSuY44x5X6WhXjILE+R3QXXXtGkyjlgJxGmfkYcu19GV4B
-ziU7hVCjqtNOQ51ywEtWUA93lchj3fD5ryo9dCv5a5Xco4O5E24VeGjW3MBqeBNW
-wNIUintgwVt20P1I+xZRAoIBAQD9UzjlhvrDgNZsaJu82+PoOEJM2pa5F+fDGSeq
-XTYCCeqioMUujJWDEkBEzfTvu4r7Pr7uExOlkyUXdSELfm+ABkt+eZguPB15Ua6V
-Y59Z4yrpqp0JIwpxXGxNOsSUnVwGg3OTlUGWYVyxyGHYQnSGJNe2/NhSlQZmvxx2
-PxqTH+g9dLn7FkQk/FNQI4l4LnphgGvNBVyXZoHcw16y86MAKLyrcZBmbHnuWMC6
-zjIu61uXXd8GU7IU8BWJbUQKvz0Y5susWf921U/7Qa8NZ2vFY4Q7oGlJVBVhMNmB
-WLL8/WUeWXu9HvL89QWpb31l5V5L+lF+GSq5ZQxFAjLEVl77AoIBAQDX3CwirHtf
-MI7z+zjwLDWgVG7RFVY4Pv5TWDWYHiQT9VPBU16K0+fVS1iXxYvoiGJkrj38Re8y
-rkTZ+qvKLmlDftqx7bImbzs01QJRbUH2gxZFMDQIF7uTSFynycywaQk+xpFEPTHl
-CXMIyGAMAsd0B2OKPAxZNoHTd0P10FYBIENwxadG9w7Ocm41J/d4Qx6wQsV10hGF
-0OgtIioplSDz1Ean/IT8XmwLK7tIiShJzZidE+0sY4depy3JdAHWv50uXMs3bIcu
-xvVBA/e/jf+HssfLjzzrYtwp2JBzSOTox/eCUJ/i9gatC5ox+ewjwOaFXV3bGEib
-mK+I3xiQ3B8RAoIBAE0Jc/IJHFU75vlMzp+eVy6VfUQV7WQYavifu7pJYlU4YsxW
-C+DeC9GySS0jXOtSoy9Io5OO5ZiiqNL7YbM3Hf1W7Lpni+nzihsMxgTUKO+S78fj
-hKH0sAZNTvoldwai3At3CjzFVQ7ASQofn/G+M+VfauJQ/hAPFcVFNQiYpCI9v8iA
-qNY8rTh6K3Pherq7l6fy/9V3XfMEz1UtbK0K/nTb7pRMktczAdmD0Ah/EC/Ijy/2
-8g3ggfVwFXyXZ+vEwHXEKggdzlx6/jmwfeWbn+CFJP9lBt+v3FiUHHEDYlshTBDw
-sXqP4OEgOjqOlxnXqNd+Ji4sxRtgKV0LEBk5EuUCggEAIlBbq79jdURQ1TQQXw2I
-EM6bNx1/MT3CTBlvm5je/1U2VTsdglAhQGTT1nyOuw5DJeIU9G9hkNrnEweoG2G5
-VgNqXHJ+qWFxNfrOfYcyvy8jcSgyfT7YkJcmM33+zeRElfgWy5Q2xEP2R2Ui74XZ
-kvZBuo3FIMFrbeQ9p2vQ4Cjyz5B8AOnxLpw+LLEHw9RXoolavloAcxc8cUBHF4kf
-TeNmv/mCYmPYJQZ0pRk4kFLgecfbIf1IXaGRw75vNGYNZHtXyp2z95mlDwrEbWzz
-O+0NmaxRcNGsUfKdM9ZYnTB8hfivEfMuKH/5qQwjn6Nggb7P1q5LjIB/FvDwBMcZ
-IQKCAQBfPoddROG6bTkzn7kL8rDwuk2H2sLh5njRKo3u29dVlwU93lmKLoP4blcf
-tAbXDVrlGpZbIo73baE57Rv5ehcV64oS/G63Lcw8nsqmBvs9982tpYPNX9EOvXV5
-1iK+hs8ZUUVCVkTOXsZb5M90VKQiHrnddO+08lE5YI/lzM5laqcytPmYcLkVWPPz
-qrpW/AReSwhvwVugcMFUgMXaDx/3SAY75B808wX1tizv76omWZAQ774FeGQGyP4C
-8f4t3LIV9h/q2Hj8geMjil9ZGogtWJ5uDspp7As5OyMF0ZMXTMwSnFsXB/L4YIk+
-rPl77gAcribJm3TzBVHm2m6jBGtb
------END PRIVATE KEY-----)foo";
-
- const char* CAWithIpInSAN = R"foo(-----BEGIN CERTIFICATE-----
-MIIFHzCCAwegAwIBAgIUQEt4xnHWGulMGzqad434c4Mw+cAwDQYJKoZIhvcNAQEL
-BQAwJjERMA8GA1UECgwIWVRzYXVydXMxETAPBgNVBAMMCFlUc2F1cnVzMB4XDTI0
-MDIyOTEwMTIzM1oXDTM0MDIyNjEwMTIzM1owJjERMA8GA1UECgwIWVRzYXVydXMx
-ETAPBgNVBAMMCFlUc2F1cnVzMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKC
-AgEAwUS9deIwfMNeicFqw7/fDslSb7sSiPbYfYUxwuELBR8nVomaL/a7IpQx5dOw
-nB2CSUiTDsLczNaBvQyNkjJ8VIX4fEPtgfHCuoUaCg4NwcaHfY0TSssbCKh8U92z
-fM/iSCjjkMdp9pTzZL93yam72dqfFbHmBvO6XiGjYeR9BL6AqoZMBnIjmxsU9JCW
-SM7B0l+Sf8dsH8DmnzqtLUXKIZJyVK6LOVb7MtSw0iAYatrIt3t7IytqZMOe+dEZ
-V0C0YzTecOgCWKS+rA8X1h7zHW5vXLDYU6RR+qw3gFsRK0oHmECuo+4Ui2nMzkx1
-jf2nK8L0Z2MHP0YFI9cGCBfSZMv6eocbYZI1DmhNKZNFhSLL8lmfzv4uR7WPyNL6
-Ml+2xgp03r1QFx93cRNW5bjjFytzYQkEVwbvLsABHDFm056PpNXKh6eoXGkUfaWf
-iXxUWhAFJRxggkTrPawCr+YQtVbBFe/YcdrTkdvFengFLm4wE20RiJUZWvG4oXmV
-JdKvDMsR1RFoQQfSqKL3mERMvVBBO1CXqymdPW6K+S2r367ryXGCG6kRDGpfmu7R
-vYZcA9BJqNvGcowf5fBe2VZf3cqZwdZKjCYOR3dCVDbXClraeStzy0VXKgxukHyT
-g3XfPqptaUKtpy413b/AVG2YupEn+SoOGMP0NKMyj9XAr20CAwEAAaNFMEMwDgYD
-VR0PAQH/BAQDAgLkMBIGA1UdEwEB/wQIMAYBAf8CAQEwHQYDVR0OBBYEFAYGwcRb
-cWnfIkC1Mozkzm0CDcFbMA0GCSqGSIb3DQEBCwUAA4ICAQCztlZxgJNdhebicTkT
-B4iXXntoOzlnusua1lCBKRUowHAocw2ylXb32Ahh/fENmHyt/jYTlcFu/qK6Q1T8
-/pN14hnyF2pB+ZJjyAr3vNnZtHGJ2xz3wPqFRzD00YUqvvLi9xD2nhsfn4xex/OK
-KUx/dbV6FkJ60Fpg97zeOmO+5Kh9srIEGEczPb+y9meiMGB91tm+ZUcd9cGShTRH
-krFkuCySJuNgrAZCxqvdsUPvPDd12lPyqmfmuVjauON9ENYUYwNoxQc8MxEx0x6j
-QolYDmc2A2VP29rTLQhyjf8TvwXfq8z+zNoeQZCNCON5jg7zH67XarXxK1AZOePc
-ZqHtjfTLTqmmKkkG7IRt1h9dtGsROMaRfXf0t4M9mvqx/1Cx6abA6LfcOkp7OG8S
-0tx0IzIRnQh2iN4zR1MihS3hn3s9ayviiaopIPVyCKEKRZpsL3QzhzydnQM/Wb+r
-UTe546vZV3q2irHH/x4SZFWoFhDwepAyUMI4qo1REd+cM/MakLP5x4nFzzDmPjyf
-FiuyqHTlkMtjveytSblzpzWE1/Sum3RcMh4s9ECq8XaUl/8FerUYvIJfRDq1j79J
-w/cDyD142joRYwQG0HQkmE4ph4mYFwKhmYOv11Wik9zvEt156VPFaExu6rkjmLia
-nTkPBMUXiU3GIb4H7k78sEjv1g==
------END CERTIFICATE-----)foo";
-
- const char* CertChainWithIpInSAN = R"foo(-----BEGIN CERTIFICATE-----
-MIIFVTCCAz2gAwIBAgIBATANBgkqhkiG9w0BAQsFADAmMREwDwYDVQQKDAhZVHNh
-dXJ1czERMA8GA1UEAwwIWVRzYXVydXMwHhcNMjQwMjI5MTAxMjMzWhcNMzQwMjI2
-MTAxMjMzWjATMREwDwYDVQQKDAhZVHNhdXJ1czCCAiIwDQYJKoZIhvcNAQEBBQAD
-ggIPADCCAgoCggIBAJ4BhT+WpWpkEEgaqcbZoV5/JOeAkFj2iZL08nIjkDRVhDBh
-dQoop47vB1lpLD/1i5+2YEackPrfeJawbNu32s/belvxHpTc1ljndCJlilMiR5B+
-MccP00fUZRsFhkzznANz++31PI4wamHJ1kdJOgD/4civ0mpWgd5s1hityqsypXI3
-RFbi689+mnBp2sGCD6l3QVbMj+8ORvXOVC1h1W3tExiivabjFILgXwb6WG6ZNgg8
-T20zdH4uEc4d2v6XKY4nz4AqYDHax9oqs3XTOTo0Bld6m7oipjGjToMqqpJD32pb
-nxSNT/XECpsNqZ/UMtcQf3HoA2LEOZOg/Knf1mosEww2svb0CqMGfsxxHehCaFJT
-CWkdd8GTQ/3t85xtrd5Ccdqb78o+5039H4GkfkcxAIZQe7siNLbUJ+dR6wRMzJw2
-GMcAHEIozOYVDgwyOT/Q3gaKMg4A3Ki2x7tkCie1KmmAcnECWDRhzG/Hg/RdnYqA
-L6g/m7Z3bw93rUbwXNudVj1ls5MPoyYtTACtZAXnv/PBBd+KR+RZ//T6dD7B9zzC
-MiIsIHemPzE3XsWB6I1AqXv2B8THELgJ8foN6Rho4YQ0t0wjhcM0higfSaiHwqhf
-Cgoc5xxOGa8PlEETWQvcT3ORAukpjM22QbZmA3uLN/MR1DJbwNHqiYPiFZ1zAgMB
-AAGjgaAwgZ0wDgYDVR0PAQH/BAQDAgWgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggr
-BgEFBQcDAjAsBgNVHREEJTAjgglsb2NhbGhvc3SHBH8AAAGHEAAAAAAAAAAAAAAA
-AAAAAAEwHQYDVR0OBBYEFDyAfXqSAoKxK/sCyFs3d2yUpSntMB8GA1UdIwQYMBaA
-FAYGwcRbcWnfIkC1Mozkzm0CDcFbMA0GCSqGSIb3DQEBCwUAA4ICAQCHRuHI5XaS
-PIK3JKzA6l01z7YDHROGn5Xf9hqo5QKHv/aUD01UUhxCbx5z/YZMFDc6iRvu4xQw
-HMl8BCvTo3gHSS8JRcgRn1ov5m50NZMl0Ws9xvvy1j5rZWd3HkBTQSpzr4eSHAZp
-dUWXN2JEW7QIaIxtoiLwMZVjo3Lfr2Qv4uIn9lbIR0F7s39+qFlzvDx+6JbstJsv
-5wPjTS9og3WfpBDaOecHM/nP8v1H9ilx8/EW0nM3jlS0q0Gj9whkha0Pcl156Bga
-biLDoQk7uTccO8Wiyddwfq6tlYy1OAIMqDy0vmoz4L/3FHJUqrzO/fdI9VQLlug+
-M2G6qTJHKzmDkvmtxPfTjRFMu+g7L3QEdYCBogfIHS+VoB9a9K/XmoWBg85cWIPw
-Kfjjf7OouqksfOQopxY2+PCQ66nnkN7y13RjoU2heAme8Fexiowkjhzc1lq0Zn0Q
-XPlnvCHAQMNRNmvBLwNEkW+KN4no0TCImOOTuInBrlKGTaBkinUNS39AF9lZWwAE
-hd1kK5zzF6XvZnKXdVIn4MjcW81hcbrnulq5GHz7XY+lwmORYumYo3Gykjj/+G93
-K9HRlSRV1+BNXmPYtI8hvbAYw05+AWKCk0J5r1GQtPx+Tx3sug/2qks26oURgEHc
-ySl4OPJLp2lhKCUkKVP24Tzg/iS1xT/uHQ==
------END CERTIFICATE-----)foo";
-
- const char* PrivateKeyWithIpInSAN = R"foo(-----BEGIN PRIVATE KEY-----
-MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQCeAYU/lqVqZBBI
-GqnG2aFefyTngJBY9omS9PJyI5A0VYQwYXUKKKeO7wdZaSw/9YuftmBGnJD633iW
-sGzbt9rP23pb8R6U3NZY53QiZYpTIkeQfjHHD9NH1GUbBYZM85wDc/vt9TyOMGph
-ydZHSToA/+HIr9JqVoHebNYYrcqrMqVyN0RW4uvPfppwadrBgg+pd0FWzI/vDkb1
-zlQtYdVt7RMYor2m4xSC4F8G+lhumTYIPE9tM3R+LhHOHdr+lymOJ8+AKmAx2sfa
-KrN10zk6NAZXepu6IqYxo06DKqqSQ99qW58UjU/1xAqbDamf1DLXEH9x6ANixDmT
-oPyp39ZqLBMMNrL29AqjBn7McR3oQmhSUwlpHXfBk0P97fOcba3eQnHam+/KPudN
-/R+BpH5HMQCGUHu7IjS21CfnUesETMycNhjHABxCKMzmFQ4MMjk/0N4GijIOANyo
-tse7ZAontSppgHJxAlg0Ycxvx4P0XZ2KgC+oP5u2d28Pd61G8FzbnVY9ZbOTD6Mm
-LUwArWQF57/zwQXfikfkWf/0+nQ+wfc8wjIiLCB3pj8xN17FgeiNQKl79gfExxC4
-CfH6DekYaOGENLdMI4XDNIYoH0moh8KoXwoKHOccThmvD5RBE1kL3E9zkQLpKYzN
-tkG2ZgN7izfzEdQyW8DR6omD4hWdcwIDAQABAoICADzT/QZD6p6QsyvvB9lDwznr
-3Ls65Vc6YjAvGH8UbdmX6nHtsu8cQ5VlNAEZ2i0tTHlJ7rqAX9gU3Am3FdFocFaA
-+hQXOVy5v9MuF6l+SchDdCWOT3+A+ie2/s1uTQum5TL3Hc+4D3316Z6H43RCHpBv
-8e4esfS6JPkKEUoi7dkGgGb+G9MPPRT+elo4hjzk4z6saH0P94Fij7LlocZu2Yme
-MTHUxQpQdX8E/dBj5FN/rCtzfGhf3MMO3U/qcnp8m0Tc0qdWqP3IahP1SG1dybQ8
-fwyCaR05ZZ3KbtlUPaJdes8pQo7Y8CV/OU4D7n9XY9MjyMyDM3p8bGYHHf4P7C0M
-qdTYAVXZ9NvKlf87J8zMJ4YDiKTMLdgFjgnj/tYJ9PkUZHgfsk1KSr5IT6wpLBAx
-jEs0+WsbQRmtlJ93ZvZsDJyGZbkeCRIAxyz/eN2Rf56YGrWZcPcn8yrAsnt4nYqz
-ewjM9wu6jB8bfAgDS0y3kPsl1www+AkRDnNQ9s+Vw5x0nccjbV7fdezkwf04cA6N
-QL+0WSALsU0iGGPKrLA5L/kC3W650xSeOgINdvCnZsSEPAF6HQ/E5udd9HD5+7Ma
-hVl2S7FbEVxKgXpXPd1GFBKZKnUTCB5erMDyaN7jAiaVBtRE8qc0hQRhHCtE7qNT
-HvWkztmzBuVGTpOkBQuRAoIBAQDDIsOMkYo/svV/lsUJM4fVMTVJ354ThhxvFdJs
-rlaUEX2mZIsbzjI76yuOsu6nlGKLuiYVfAAV3HNEhNrSnp6+3NPXMgSsE33TFRPR
-Y6P1oT/ZE8YrBkndSioU1ad9Z0ClRNhDlP2c9xCBGpXUSQLt/4C2Nk/wgG/wy7gG
-+hJ2ybn133DPpwcx1KQCpslF+VaECPLrpjxWCFEGInOV8gVvxhEb7hrD8qmUPlAR
-3WYFT6/RJGCJ7BeOvXWYa8AfWRysLfLwnzuoUxl45O7X7eZqPufrhAJyDNgN3c6J
-VgVxNPSSc6mjsY0NyItQZEOXtzRJqnRlmGD2Zd5BJFG7PmIVAoIBAQDPSgFqnFe1
-RM7AtC5wRBLTU/LQo8XV3WPsvQpYaa2+YyPG1bjSMWmdejsy4k+lbUx4VzD+veX7
-ZdSmukYmeGT0pp6Yy7FYYAcZ8Tu+CSAxGH8Bvab+U1oy2Y3++lXdAc0OVjuezVSy
-RbUE97S9MjeFMTKxIg2RkJc4p+Ir0sFn8rC9a0ZqNdcV1BCzJkMB5Y0PbG+afopL
-wS4xIMIy2Jp3Y0CanOW/dl6LnWYdEtWMt0Pi5wFDVVn0/DgGNx0SDG5fKj4rJE4F
-6mezK3kyAf7qI35nf8wm/rL7wL262F4iGZp+OKNFQnaEuKmrUsz/l2GZZkfIIcbI
-pPVaqKbAQEtnAoIBAQCj9V/NirRY1WuFqw8frhahwVj/G09dJEBb7kACZXIFs7SZ
-zL09vcFjqzPMEPiKAhnTQbOiNbB0reiEWATtF65WvIGavUJDu5TreThPpaMsTjKx
-mPXXTM9fimNVYjf7HHiq5O+5yURXURijAc2Gs1os05Q4heYhNCnab7HO2uwMt27y
-8q19LODUs9CjEbTogJp7EnHaIrFrsE00FFp+UP7Ubd4OU8BViF2IW9s3R4njSJN3
-7VLYUHFy1CosycyCCoQW//yyxXiA9GHgvKsa75+9AeIod6D+Z2BaNlbF+mtUNaSS
-MXEGQ7c7L5gvEi/hGGRsyTZH7wL5xZo7reKmq8IJAoIBAHIiaHdAEFcBxOl8DFnK
-UadEgNz6X/LqzJtMV0bpIT5ELi3L/dDWXjXUWIYi8AHBFarpL1QEUX5Dynvm8rs5
-7TR8DbVJ6qMjdKWHGwL+2VfPChd2Sl2cnXyEJ1gulFp1JGfxeTBuFGV4Vjye+0h1
-Pva6aRP5EQmGWI1cev7wM4e9rC0PxRyz+nLNakiKF7kSoMHOTgD+Db26Z2mrhOIk
-O6Di6G55V1M9pL8w8kmt1iF9wwZLdXmSpE5tFZfufrYyXA9QHhz5B3DgaSrRFBFB
-4g8fbfkk9868zOYrcQxRGDukZ1l6bAO1nbZkSx/HHpLY0md5VqrOVjqiAWpilDYk
-8J8CggEAVN7WQIMWzuGBc9Hj6H539eXLnoWgccvYN+GwqclQGKQJdUQrM7Ci3kP/
-Zj00T//jew/vHtxx6U1XUfCIbI5SjujqhbkCkikUMAhozsjlmUlAxKz8689/XFXK
-9bPRRvmKw6p14drk507w6t3uD4E/O3PHeEtrdEeDmY78e01T5zkg90jVq10szUMo
-sxw0PoOG4ktYtCsmwXXPHq9gRtzfOqZT/UNHgnBsLALmWcec2gZLa++M1OnVaqI2
-hTyvWDnxD5oKa7hDSBXTOorcfQVRSaC05HPGMhX+HkfHwfXJBxAE1UC38UMD7x58
-AbE/BnHl1tAmZXLMrHq/4r0wYUjBsA==
------END PRIVATE KEY-----)foo";
+ TPemBlobConfigPtr CACert;
+ TPemBlobConfigPtr PrivateKey;
+ TPemBlobConfigPtr CertificateChain;
+ TPemBlobConfigPtr CACertWithIPInSAN;
+ TPemBlobConfigPtr PrivateKeyWithIpInSAN;
+ TPemBlobConfigPtr CertificateChainWithIpInSAN;
+ TPemBlobConfigPtr CACertEC;
+ TPemBlobConfigPtr PrivateKeyEC;
+ TPemBlobConfigPtr CertificateChainEC;
TSslTest()
{
@@ -293,6 +72,17 @@ AbE/BnHl1tAmZXLMrHq/4r0wYUjBsA==
AddressWithHostName = Format("localhost:%v", Port);
AddressWithIpV4 = Format("127.0.0.1:%v", Port);
AddressWithIpV6 = Format("[::1]:%v", Port);
+
+ CACert = TPemBlobConfig::CreateFileReference(SRC_("testdata/ca.pem"));
+ PrivateKey = TPemBlobConfig::CreateFileReference(SRC_("testdata/key.pem"));
+ CertificateChain = TPemBlobConfig::CreateFileReference(SRC_("testdata/cert.pem"));
+ CACertWithIPInSAN = TPemBlobConfig::CreateFileReference(SRC_("testdata/ca_with_ip_in_san.pem"));
+ PrivateKeyWithIpInSAN = TPemBlobConfig::CreateFileReference(SRC_("testdata/key_with_ip_in_san.pem"));
+ CertificateChainWithIpInSAN = TPemBlobConfig::CreateFileReference(SRC_("testdata/cert_with_ip_in_san.pem"));
+
+ CACertEC = TPemBlobConfig::CreateFileReference(SRC_("testdata/ca_ec.pem"));
+ PrivateKeyEC = TPemBlobConfig::CreateFileReference(SRC_("testdata/key_ec.pem"));
+ CertificateChainEC = TPemBlobConfig::CreateFileReference(SRC_("testdata/cert_ec.pem"));
}
};
@@ -302,10 +92,8 @@ TEST_F(TSslTest, RequiredAndRequiredEncryptionMode)
{
auto serverConfig = TBusServerConfig::CreateTcp(Port);
serverConfig->EncryptionMode = EEncryptionMode::Required;
- serverConfig->CertificateChain = New<NCrypto::TPemBlobConfig>();
- serverConfig->CertificateChain->Value = CertChain;
- serverConfig->PrivateKey = New<NCrypto::TPemBlobConfig>();
- serverConfig->PrivateKey->Value = PrivateKey;
+ serverConfig->CertificateChain = CertificateChain;
+ serverConfig->PrivateKey = PrivateKey;
auto server = CreateBusServer(serverConfig);
server->Start(New<TEmptyBusHandler>());
@@ -330,10 +118,8 @@ TEST_F(TSslTest, RequiredAndOptionalEncryptionMode)
{
auto serverConfig = TBusServerConfig::CreateTcp(Port);
serverConfig->EncryptionMode = EEncryptionMode::Required;
- serverConfig->CertificateChain = New<NCrypto::TPemBlobConfig>();
- serverConfig->CertificateChain->Value = CertChain;
- serverConfig->PrivateKey = New<NCrypto::TPemBlobConfig>();
- serverConfig->PrivateKey->Value = PrivateKey;
+ serverConfig->CertificateChain = CertificateChain;
+ serverConfig->PrivateKey = PrivateKey;
auto server = CreateBusServer(serverConfig);
server->Start(New<TEmptyBusHandler>());
@@ -358,10 +144,8 @@ TEST_F(TSslTest, OptionalAndRequiredEncryptionMode)
{
auto serverConfig = TBusServerConfig::CreateTcp(Port);
serverConfig->EncryptionMode = EEncryptionMode::Optional;
- serverConfig->CertificateChain = New<NCrypto::TPemBlobConfig>();
- serverConfig->CertificateChain->Value = CertChain;
- serverConfig->PrivateKey = New<NCrypto::TPemBlobConfig>();
- serverConfig->PrivateKey->Value = PrivateKey;
+ serverConfig->CertificateChain = CertificateChain;
+ serverConfig->PrivateKey = PrivateKey;
auto server = CreateBusServer(serverConfig);
server->Start(New<TEmptyBusHandler>());
@@ -386,10 +170,8 @@ TEST_F(TSslTest, OptionalAndOptionalEncryptionMode)
{
auto serverConfig = TBusServerConfig::CreateTcp(Port);
serverConfig->EncryptionMode = EEncryptionMode::Optional;
- serverConfig->CertificateChain = New<NCrypto::TPemBlobConfig>();
- serverConfig->CertificateChain->Value = CertChain;
- serverConfig->PrivateKey = New<NCrypto::TPemBlobConfig>();
- serverConfig->PrivateKey->Value = PrivateKey;
+ serverConfig->CertificateChain = CertificateChain;
+ serverConfig->PrivateKey = PrivateKey;
auto server = CreateBusServer(serverConfig);
server->Start(New<TEmptyBusHandler>());
@@ -438,10 +220,8 @@ TEST_F(TSslTest, RequiredAndDisabledEncryptionMode)
{
auto serverConfig = TBusServerConfig::CreateTcp(Port);
serverConfig->EncryptionMode = EEncryptionMode::Required;
- serverConfig->CertificateChain = New<NCrypto::TPemBlobConfig>();
- serverConfig->CertificateChain->Value = CertChain;
- serverConfig->PrivateKey = New<NCrypto::TPemBlobConfig>();
- serverConfig->PrivateKey->Value = PrivateKey;
+ serverConfig->CertificateChain = CertificateChain;
+ serverConfig->PrivateKey = PrivateKey;
auto server = CreateBusServer(serverConfig);
server->Start(New<TEmptyBusHandler>());
@@ -530,16 +310,11 @@ TEST_F(TSslTest, OptionalAndDisabledEncryptionMode)
TEST_F(TSslTest, CAVerificationModeFailure)
{
- // Reset ctx in order to unload possibly loaded CA.
- TSslContext::Get()->Reset();
-
auto serverConfig = TBusServerConfig::CreateTcp(Port);
serverConfig->EncryptionMode = EEncryptionMode::Required;
serverConfig->VerificationMode = EVerificationMode::None;
- serverConfig->CertificateChain = New<NCrypto::TPemBlobConfig>();
- serverConfig->CertificateChain->Value = CertChain;
- serverConfig->PrivateKey = New<NCrypto::TPemBlobConfig>();
- serverConfig->PrivateKey->Value = PrivateKey;
+ serverConfig->CertificateChain = CertificateChain;
+ serverConfig->PrivateKey = PrivateKey;
auto server = CreateBusServer(serverConfig);
server->Start(New<TEmptyBusHandler>());
@@ -560,22 +335,16 @@ TEST_F(TSslTest, CAVerificationModeFailure)
TEST_F(TSslTest, CAVerificationModeSuccess)
{
- // Reset ctx in order to unload possibly loaded CA.
- TSslContext::Get()->Reset();
-
auto serverConfig = TBusServerConfig::CreateTcp(Port);
serverConfig->EncryptionMode = EEncryptionMode::Required;
serverConfig->VerificationMode = EVerificationMode::None;
- serverConfig->CertificateChain = New<NCrypto::TPemBlobConfig>();
- serverConfig->CertificateChain->Value = CertChain;
- serverConfig->PrivateKey = New<NCrypto::TPemBlobConfig>();
- serverConfig->PrivateKey->Value = PrivateKey;
+ serverConfig->CertificateChain = CertificateChain;
+ serverConfig->PrivateKey = PrivateKey;
auto server = CreateBusServer(serverConfig);
server->Start(New<TEmptyBusHandler>());
auto clientConfig = TBusClientConfig::CreateTcp(AddressWithHostName);
- clientConfig->CA = New<NCrypto::TPemBlobConfig>();
- clientConfig->CA->Value = CA;
+ clientConfig->CertificateAuthority = CACert;
clientConfig->EncryptionMode = EEncryptionMode::Required;
clientConfig->VerificationMode = EVerificationMode::Ca;
auto client = CreateBusClient(clientConfig);
@@ -598,24 +367,18 @@ TEST_F(TSslTest, CAVerificationModeSuccess)
TEST_F(TSslTest, FullVerificationModeByHostName)
{
- // Reset ctx in order to unload possibly loaded CA.
- TSslContext::Get()->Reset();
-
auto serverConfig = TBusServerConfig::CreateTcp(Port);
serverConfig->EncryptionMode = EEncryptionMode::Required;
serverConfig->VerificationMode = EVerificationMode::None;
- serverConfig->CertificateChain = New<NCrypto::TPemBlobConfig>();
- serverConfig->CertificateChain->Value = CertChain;
- serverConfig->PrivateKey = New<NCrypto::TPemBlobConfig>();
- serverConfig->PrivateKey->Value = PrivateKey;
+ serverConfig->CertificateChain = CertificateChain;
+ serverConfig->PrivateKey = PrivateKey;
auto server = CreateBusServer(serverConfig);
server->Start(New<TEmptyBusHandler>());
auto clientConfig = TBusClientConfig::CreateTcp(AddressWithHostName);
clientConfig->EncryptionMode = EEncryptionMode::Required;
clientConfig->VerificationMode = EVerificationMode::Full;
- clientConfig->CA = New<NCrypto::TPemBlobConfig>();
- clientConfig->CA->Value = CA;
+ clientConfig->CertificateAuthority = CACert;
auto client = CreateBusClient(clientConfig);
auto bus = client->CreateBus(New<TEmptyBusHandler>());
@@ -634,26 +397,20 @@ TEST_F(TSslTest, FullVerificationModeByHostName)
TEST_F(TSslTest, FullVerificationModeByIpAddress)
{
- // Reset ctx in order to unload possibly loaded CA.
- TSslContext::Get()->Reset();
-
// Connect via ipv4 and ipv6 addresses.
for (const auto& address : {AddressWithIpV4, AddressWithIpV6}) {
auto serverConfig = TBusServerConfig::CreateTcp(Port);
serverConfig->EncryptionMode = EEncryptionMode::Required;
serverConfig->VerificationMode = EVerificationMode::None;
- serverConfig->CertificateChain = New<NCrypto::TPemBlobConfig>();
- serverConfig->CertificateChain->Value = CertChainWithIpInSAN;
- serverConfig->PrivateKey = New<NCrypto::TPemBlobConfig>();
- serverConfig->PrivateKey->Value = PrivateKeyWithIpInSAN;
+ serverConfig->CertificateChain = CertificateChainWithIpInSAN;
+ serverConfig->PrivateKey = PrivateKeyWithIpInSAN;
auto server = CreateBusServer(serverConfig);
server->Start(New<TEmptyBusHandler>());
auto clientConfig = TBusClientConfig::CreateTcp(address);
clientConfig->EncryptionMode = EEncryptionMode::Required;
clientConfig->VerificationMode = EVerificationMode::Full;
- clientConfig->CA = New<NCrypto::TPemBlobConfig>();
- clientConfig->CA->Value = CAWithIpInSAN;
+ clientConfig->CertificateAuthority = CACertWithIPInSAN;
auto client = CreateBusClient(clientConfig);
auto bus = client->CreateBus(New<TEmptyBusHandler>());
@@ -673,17 +430,12 @@ TEST_F(TSslTest, FullVerificationModeByIpAddress)
TEST_F(TSslTest, FullVerificationByAlternativeHostName)
{
- // Reset ctx in order to unload possibly loaded CA.
- TSslContext::Get()->Reset();
-
for (const auto& address : {AddressWithIpV4, AddressWithIpV6}) {
auto serverConfig = TBusServerConfig::CreateTcp(Port);
serverConfig->EncryptionMode = EEncryptionMode::Required;
serverConfig->VerificationMode = EVerificationMode::None;
- serverConfig->CertificateChain = New<NCrypto::TPemBlobConfig>();
- serverConfig->CertificateChain->Value = CertChain;
- serverConfig->PrivateKey = New<NCrypto::TPemBlobConfig>();
- serverConfig->PrivateKey->Value = PrivateKey;
+ serverConfig->CertificateChain = CertificateChain;
+ serverConfig->PrivateKey = PrivateKey;
auto server = CreateBusServer(serverConfig);
server->Start(New<TEmptyBusHandler>());
@@ -691,8 +443,7 @@ TEST_F(TSslTest, FullVerificationByAlternativeHostName)
auto clientConfig = TBusClientConfig::CreateTcp(address);
clientConfig->EncryptionMode = EEncryptionMode::Required;
clientConfig->VerificationMode = EVerificationMode::Full;
- clientConfig->CA = New<NCrypto::TPemBlobConfig>();
- clientConfig->CA->Value = CA;
+ clientConfig->CertificateAuthority = CACert;
{
auto client = CreateBusClient(clientConfig);
@@ -728,10 +479,8 @@ TEST_F(TSslTest, ServerCipherList)
auto serverConfig = TBusServerConfig::CreateTcp(Port);
serverConfig->EncryptionMode = EEncryptionMode::Required;
serverConfig->VerificationMode = EVerificationMode::None;
- serverConfig->CertificateChain = New<NCrypto::TPemBlobConfig>();
- serverConfig->CertificateChain->Value = CertChain;
- serverConfig->PrivateKey = New<NCrypto::TPemBlobConfig>();
- serverConfig->PrivateKey->Value = PrivateKey;
+ serverConfig->CertificateChain = CertificateChain;
+ serverConfig->PrivateKey = PrivateKey;
serverConfig->CipherList = "AES128-GCM-SHA256:PSK-AES128-GCM-SHA256";
auto server = CreateBusServer(serverConfig);
server->Start(New<TEmptyBusHandler>());
@@ -759,21 +508,27 @@ TEST_F(TSslTest, ServerCipherList)
TEST_F(TSslTest, DifferentCipherLists)
{
+ auto cipherStringAES128 = TSslContextCommand::Create("CipherString", "AES128-GCM-SHA256");
+ auto cipherSuitesAES128 = TSslContextCommand::Create("CipherSuites", "TLS_AES_128_GCM_SHA256");
+ auto cipherStringAES256 = TSslContextCommand::Create("CipherString", "AES256-GCM-SHA384");
+ auto cipherSuitesAES256 = TSslContextCommand::Create("CipherSuites", "TLS_AES_256_GCM_SHA384");
+ auto maxProtocolTLS1_2 = TSslContextCommand::Create("MaxProtocol", "TLSv1.2");
+
auto serverConfig = TBusServerConfig::CreateTcp(Port);
serverConfig->EncryptionMode = EEncryptionMode::Required;
serverConfig->VerificationMode = EVerificationMode::None;
- serverConfig->CertificateChain = New<NCrypto::TPemBlobConfig>();
- serverConfig->CertificateChain->Value = CertChain;
- serverConfig->PrivateKey = New<NCrypto::TPemBlobConfig>();
- serverConfig->PrivateKey->Value = PrivateKey;
- serverConfig->CipherList = "PSK-AES128-GCM-SHA256";
+ serverConfig->CertificateChain = CertificateChain;
+ serverConfig->PrivateKey = PrivateKey;
+ serverConfig->SslConfigurationCommands.push_back(cipherStringAES256);
+ serverConfig->SslConfigurationCommands.push_back(cipherSuitesAES256);
auto server = CreateBusServer(serverConfig);
server->Start(New<TEmptyBusHandler>());
auto clientConfig = TBusClientConfig::CreateTcp(AddressWithHostName);
clientConfig->EncryptionMode = EEncryptionMode::Required;
clientConfig->VerificationMode = EVerificationMode::None;
- clientConfig->CipherList = "AES128-GCM-SHA256";
+ clientConfig->SslConfigurationCommands.push_back(cipherStringAES128);
+ clientConfig->SslConfigurationCommands.push_back(cipherSuitesAES128);
auto client = CreateBusClient(clientConfig);
auto bus = client->CreateBus(New<TEmptyBusHandler>());
@@ -781,11 +536,51 @@ TEST_F(TSslTest, DifferentCipherLists)
EXPECT_FALSE(error.IsOK());
EXPECT_EQ(error.GetCode(), EErrorCode::SslError);
+ {
+ clientConfig->SslConfigurationCommands.push_back(maxProtocolTLS1_2);
+ auto client = CreateBusClient(clientConfig);
+ auto bus = client->CreateBus(New<TEmptyBusHandler>());
+ EXPECT_FALSE(bus->GetReadyFuture().Get().IsOK());
+ }
+
server->Stop()
.Get()
.ThrowOnError();
}
+TEST_F(TSslTest, FullVerificationWithEllipticCurve)
+{
+ // Connect via localhost, ipv4 and ipv6 addresses.
+ for (const auto& address : {AddressWithHostName, AddressWithIpV4, AddressWithIpV6}) {
+ auto serverConfig = TBusServerConfig::CreateTcp(Port);
+ serverConfig->EncryptionMode = EEncryptionMode::Required;
+ serverConfig->VerificationMode = EVerificationMode::None;
+ serverConfig->CertificateChain = CertificateChainEC;
+ serverConfig->PrivateKey = PrivateKeyEC;
+ auto server = CreateBusServer(serverConfig);
+ server->Start(New<TEmptyBusHandler>());
+
+ auto clientConfig = TBusClientConfig::CreateTcp(address);
+ clientConfig->EncryptionMode = EEncryptionMode::Required;
+ clientConfig->VerificationMode = EVerificationMode::Full;
+ clientConfig->CertificateAuthority = CACertEC;
+ auto client = CreateBusClient(clientConfig);
+
+ auto bus = client->CreateBus(New<TEmptyBusHandler>());
+ // This test should pass since (localhost | 127.0.0.1 | [::1]) is in SAN.
+ EXPECT_TRUE(bus->GetReadyFuture().Get().IsOK());
+ EXPECT_TRUE(bus->IsEncrypted());
+
+ auto message = CreateMessage(1);
+ auto sendFuture = bus->Send(message, {.TrackingLevel = EDeliveryTrackingLevel::Full});
+ EXPECT_TRUE(sendFuture.Get().IsOK());
+
+ server->Stop()
+ .Get()
+ .ThrowOnError();
+ }
+}
+
////////////////////////////////////////////////////////////////////////////////
} // namespace
diff --git a/yt/yt/core/bus/unittests/testdata/ca.pem b/yt/yt/core/bus/unittests/testdata/ca.pem
new file mode 100644
index 00000000000..6a91723fe4f
--- /dev/null
+++ b/yt/yt/core/bus/unittests/testdata/ca.pem
@@ -0,0 +1,31 @@
+-----BEGIN CERTIFICATE-----
+MIIFWjCCA0KgAwIBAgIBATANBgkqhkiG9w0BAQsFADBGMQswCQYDVQQGEwJSVTEP
+MA0GA1UECAwGTW9zY293MQ8wDQYDVQQKDAZZYW5kZXgxFTATBgNVBAMMDENBQGxv
+Y2FsaG9zdDAeFw0yMzA3MTMxMTUxMTFaFw0zMzA3MTAxMTUxMTFaMEYxCzAJBgNV
+BAYTAlJVMQ8wDQYDVQQIDAZNb3Njb3cxDzANBgNVBAoMBllhbmRleDEVMBMGA1UE
+AwwMQ0FAbG9jYWxob3N0MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA
+mxWRSfbYAJcBgGhYagnQMJOiW60B82Ok94n8muIaa2gRo0ZHKmtY7CMiFIIN0GhI
+Blw+G6HPN8SPYK0+bTlAIHlb0u6/ty2AVyveiH31p7Ld/Ib5S/MSKuMiGt5S/Ci9
+mSdRAmD46twIKfm9bY/8SZmNsTCJIrKnJaTbjTnbz9O3FVAYjiuc8yNGb1LnZNA6
+B4ZrF3fkYr9kn4nKSOd6mypPFIOZAxKGQ95X9nCblajEMAPzHfr+1EArnM+PauAO
+cMcxfKT+OlMPGR4ZtZpl90qX6ZVZqD9zd8gp2I/2SM1vVS7AT96t89T7Uag8OjH2
+c8jMCD2z1fk1oDD4K53pGkBucTwDolCvxIbN77gcwkutdor/dbHLqIvGV/M4Z/iA
+GzqCD6pCpi5gT8SXn2IQrlvSdF01k9YZS093Y8bIQm3dCo1lKDaz6oHytk9Ro+fu
+b+dLOhSjopNfa/1Thw6fgta/mRsdgubWI/IHn+IyMpW65vGbnrMD4oQl1AanRJj5
+iBS73ZXIs/Y9LuMtSNk/I9u1o2fc+Sg0zb1AchC70h8M3sAYaGbqg1churOaDuTO
+Yom5W+bAQO8uLUvmlDcpqxMBYqeE8VJGJWRLtQnhM18iYAa03w7m6uMzrlkoX1Q7
+AHnX4899WSUcz/BJc5JBAtHQXCzEzyudT+8xw5MwqRkCAwEAAaNTMFEwHQYDVR0O
+BBYEFFuWdWxzQ6TF/yH3sQnlk3EEr1VjMB8GA1UdIwQYMBaAFFuWdWxzQ6TF/yH3
+sQnlk3EEr1VjMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggIBAHqS
+W4FIjH/YKkcj/wuh7sQQTIBAx+LsjosyHb9K/1QbgLNVpxawl/YlOiArfVmgTGud
+9B6xdVacxqI6saMqHNPPgYm7OPnozSnprRKh9yGb2TS9j+2G7hCQLnM+xYroP8XE
+8fL9tyq14lHb/BFYnPsFFxqeA6lXzzm73hlm/hH58CWpL/3eR/TDjC+oiEBBV4VF
+Dm4X3L73MFcniDKkCb7Mw3wdi6zqx231F4+9Cqgq+RqAucnmLXTG7yR7wOKP2u8E
+Ye1Jrt3nnMlD1tqiXyRJywPSK9mMiBTGbzGLLiTcvecBIHG8oRuvn1DnkZa9L/48
++1aTC8QhbRslaeDyXj5IY/7scW7ZkEw18qYCT+9sI4/LngS4U8+taKqLf5S8A5sR
+O8OCP0nMk/l5f84cDwe1dp4GXVQdkbnfT1sd93BLww7Szbw2iMt9sLMmyY8qIe7a
+Ss3OEfP6eyNCBu6KtL8oufdj1AqAQdmYYTlGwgaFZTAomDiJU662EGSt3uXK7V32
+EzgJbpxSSjWh5JpCDA/jnqzlkFkSaw92/HZwO5l1lCdqK+F1UWYQyU0NveOaSEuX
+344PIi8m1YWwUfukUB97D+C6B4y5vgdkC7OYLHb4W4+N0ZvYtbDDaBeTpn70CiI7
+JFWcF3ghP7uPmbONWLiTFwxsSJHT0svVQZgq1aZz
+-----END CERTIFICATE-----
diff --git a/yt/yt/core/bus/unittests/testdata/ca_ec.pem b/yt/yt/core/bus/unittests/testdata/ca_ec.pem
new file mode 100644
index 00000000000..73fbb2884d0
--- /dev/null
+++ b/yt/yt/core/bus/unittests/testdata/ca_ec.pem
@@ -0,0 +1,13 @@
+-----BEGIN CERTIFICATE-----
+MIIB8TCCAZegAwIBAgIUW7+MbzA1uAX+WMOEf0RPu8LtFiswCgYIKoZIzj0EAwIw
+RjELMAkGA1UEBhMCUlUxDzANBgNVBAcMBk1vc2NvdzEPMA0GA1UECgwGWWFuZGV4
+MRUwEwYDVQQDDAxMb2NhbGhvc3QgQ0EwHhcNMjUwNTEyMTQxOTEzWhcNMzUwNTEw
+MTQxOTEzWjBGMQswCQYDVQQGEwJSVTEPMA0GA1UEBwwGTW9zY293MQ8wDQYDVQQK
+DAZZYW5kZXgxFTATBgNVBAMMDExvY2FsaG9zdCBDQTBZMBMGByqGSM49AgEGCCqG
+SM49AwEHA0IABP5RdA2cxb9QfFA5qpP3R49HFRWnWk0obvp2laNMJSfO4rV372lB
+jp3nzZQCRO20oYfjNTbW3YX28IlbV4KiYeijYzBhMB0GA1UdDgQWBBTF9gjbblH+
+zdrnV6d15/WHVsvXMzAfBgNVHSMEGDAWgBTF9gjbblH+zdrnV6d15/WHVsvXMzAS
+BgNVHRMBAf8ECDAGAQH/AgEAMAsGA1UdDwQEAwICBDAKBggqhkjOPQQDAgNIADBF
+AiAkB3dhKbgCWQ5uoo94FY4IqtY/uaf4+HVMXKzpwK3pAwIhAMjTWGA5wh4RFaYp
++qMvt7QTKc+irhL9nHrNhJUqplnF
+-----END CERTIFICATE-----
diff --git a/yt/yt/core/bus/unittests/testdata/ca_with_ip_in_san.pem b/yt/yt/core/bus/unittests/testdata/ca_with_ip_in_san.pem
new file mode 100644
index 00000000000..4fbccb3c3bc
--- /dev/null
+++ b/yt/yt/core/bus/unittests/testdata/ca_with_ip_in_san.pem
@@ -0,0 +1,30 @@
+-----BEGIN CERTIFICATE-----
+MIIFHzCCAwegAwIBAgIUQEt4xnHWGulMGzqad434c4Mw+cAwDQYJKoZIhvcNAQEL
+BQAwJjERMA8GA1UECgwIWVRzYXVydXMxETAPBgNVBAMMCFlUc2F1cnVzMB4XDTI0
+MDIyOTEwMTIzM1oXDTM0MDIyNjEwMTIzM1owJjERMA8GA1UECgwIWVRzYXVydXMx
+ETAPBgNVBAMMCFlUc2F1cnVzMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKC
+AgEAwUS9deIwfMNeicFqw7/fDslSb7sSiPbYfYUxwuELBR8nVomaL/a7IpQx5dOw
+nB2CSUiTDsLczNaBvQyNkjJ8VIX4fEPtgfHCuoUaCg4NwcaHfY0TSssbCKh8U92z
+fM/iSCjjkMdp9pTzZL93yam72dqfFbHmBvO6XiGjYeR9BL6AqoZMBnIjmxsU9JCW
+SM7B0l+Sf8dsH8DmnzqtLUXKIZJyVK6LOVb7MtSw0iAYatrIt3t7IytqZMOe+dEZ
+V0C0YzTecOgCWKS+rA8X1h7zHW5vXLDYU6RR+qw3gFsRK0oHmECuo+4Ui2nMzkx1
+jf2nK8L0Z2MHP0YFI9cGCBfSZMv6eocbYZI1DmhNKZNFhSLL8lmfzv4uR7WPyNL6
+Ml+2xgp03r1QFx93cRNW5bjjFytzYQkEVwbvLsABHDFm056PpNXKh6eoXGkUfaWf
+iXxUWhAFJRxggkTrPawCr+YQtVbBFe/YcdrTkdvFengFLm4wE20RiJUZWvG4oXmV
+JdKvDMsR1RFoQQfSqKL3mERMvVBBO1CXqymdPW6K+S2r367ryXGCG6kRDGpfmu7R
+vYZcA9BJqNvGcowf5fBe2VZf3cqZwdZKjCYOR3dCVDbXClraeStzy0VXKgxukHyT
+g3XfPqptaUKtpy413b/AVG2YupEn+SoOGMP0NKMyj9XAr20CAwEAAaNFMEMwDgYD
+VR0PAQH/BAQDAgLkMBIGA1UdEwEB/wQIMAYBAf8CAQEwHQYDVR0OBBYEFAYGwcRb
+cWnfIkC1Mozkzm0CDcFbMA0GCSqGSIb3DQEBCwUAA4ICAQCztlZxgJNdhebicTkT
+B4iXXntoOzlnusua1lCBKRUowHAocw2ylXb32Ahh/fENmHyt/jYTlcFu/qK6Q1T8
+/pN14hnyF2pB+ZJjyAr3vNnZtHGJ2xz3wPqFRzD00YUqvvLi9xD2nhsfn4xex/OK
+KUx/dbV6FkJ60Fpg97zeOmO+5Kh9srIEGEczPb+y9meiMGB91tm+ZUcd9cGShTRH
+krFkuCySJuNgrAZCxqvdsUPvPDd12lPyqmfmuVjauON9ENYUYwNoxQc8MxEx0x6j
+QolYDmc2A2VP29rTLQhyjf8TvwXfq8z+zNoeQZCNCON5jg7zH67XarXxK1AZOePc
+ZqHtjfTLTqmmKkkG7IRt1h9dtGsROMaRfXf0t4M9mvqx/1Cx6abA6LfcOkp7OG8S
+0tx0IzIRnQh2iN4zR1MihS3hn3s9ayviiaopIPVyCKEKRZpsL3QzhzydnQM/Wb+r
+UTe546vZV3q2irHH/x4SZFWoFhDwepAyUMI4qo1REd+cM/MakLP5x4nFzzDmPjyf
+FiuyqHTlkMtjveytSblzpzWE1/Sum3RcMh4s9ECq8XaUl/8FerUYvIJfRDq1j79J
+w/cDyD142joRYwQG0HQkmE4ph4mYFwKhmYOv11Wik9zvEt156VPFaExu6rkjmLia
+nTkPBMUXiU3GIb4H7k78sEjv1g==
+-----END CERTIFICATE-----
diff --git a/yt/yt/core/bus/unittests/testdata/cert.pem b/yt/yt/core/bus/unittests/testdata/cert.pem
new file mode 100644
index 00000000000..ffcb0da681b
--- /dev/null
+++ b/yt/yt/core/bus/unittests/testdata/cert.pem
@@ -0,0 +1,31 @@
+-----BEGIN CERTIFICATE-----
+MIIFUTCCAzmgAwIBAgIBATANBgkqhkiG9w0BAQsFADBGMQswCQYDVQQGEwJSVTEP
+MA0GA1UECAwGTW9zY293MQ8wDQYDVQQKDAZZYW5kZXgxFTATBgNVBAMMDENBQGxv
+Y2FsaG9zdDAeFw0yMzA3MTMxMTUxMTdaFw0zMzA3MTAxMTUxMTdaMEMxCzAJBgNV
+BAYTAlJVMQ8wDQYDVQQIDAZNb3Njb3cxDzANBgNVBAoMBllhbmRleDESMBAGA1UE
+AwwJbG9jYWxob3N0MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA1ZrD
++kcfmsi52zMZ7cJxWl8pMI1kX1uQ6KkOJRVQVuweJuhp35rJGK0vHkWvrpihO+Ue
+h1jncReH8PaNdejt3gPsYJksUeRxCaZ8O+awtyOq6TK8L9pDzbQyXme5k2Ea/+yl
+XSHrGGzZUuHaZRzBigyQXzb04ZdQMzqGGKUz52ed5elZnOoAMUL7/wZ7O8n5dlXh
+DwxEQ4SQ3a39SLpPWv3hWXiIDrlR5xhkqQjZnb0WEVlSwN8lpyjA6XwCs/IX7PVr
+khoBKq7664rsR3B7qhGA5cmSy4+KrSoYKYHGCvl6OakO7fn4FcJwHHFNQzMq8oa5
+i5LbYsaCy6IgJt07nBeNXxaEY+4WIImHfWR0yvQgqtrFQZuIQ4N7YeZpLvTtoO3I
+UJyjMsih5GX4LPY0k9nE8rd437lV3PdiBYAktkwmgCTY9uJFhCBPFsMhE30nL79t
+X0gm5FugW1aZ3CFvgQywkcYQIjBjJQ42zcpqnLjj4lIwfDO5YEAiAOPqQ3VV7FKm
+bSWpeXyRAL/uqkBVCDKTsKJkEjd8gc30honRIKNHQkr/RoV1M1W449uRkvfVfbfg
+kdcRcYMIynTwZMSiKDoxXkQyrcyY6qVjYQNUKwRvtN2m6aGftYhfgo4j01UHTrK7
+BHRHKjW5LFXgNJUXnycmO69B+w1eTRgnVE7ws6sCAwEAAaNNMEswCQYDVR0TBAIw
+ADAdBgNVHQ4EFgQUNvB1ghYixkG53VZoSvUPnCCN7hwwHwYDVR0jBBgwFoAUW5Z1
+bHNDpMX/IfexCeWTcQSvVWMwDQYJKoZIhvcNAQELBQADggIBAHW/+xxign61axqT
+hO338rJGuidhXzraCQtJT/J6OwUaI+P6C71wUoBKEk3SrIVNG1Q8mrlsj93EzQuv
+M9OvBenIomF7A2ydaFL/6NJhZ8W6PmZwYF3VvInYJ2p7zqCMjqECg2C92NIC7+Z6
+fdmL+xoj3XKYypqA1x6xvSnCtXyXGRCxta3Es163wbqffq+4jjBUFOyUr9vk2N7X
+vy3/x8LWHIXffzNSJLtnXiznSNBSubmedac8JQ+XE9RgK+R0kUrj1lqnkSg+tPWD
+jP52kG84J9URV18BZfLFpcUiYloWXREfwNXRhMAQ6DyucupLW1Skl+Nf9K+41C3h
+f4mDn4Axn6toBzav9NLdFelGVAp4R3Yjoiv2LvpwYxGnMs/cPJMGm/NqoAbuyOMY
+ZKZPWvwsbxeaG8u9GRGvTSpawnGWJqIgxknKpQT5QztjNwtI9iT0/f4n3CPEGdAi
+6bHw0q+jiCKmXZMZPHyJ/tSJ74H7tdeWYYjhthJWnrAz4BziZ+bFBLkcYq95VV4L
+pOeMUr0PUi8fpSK06wIVTES9AWgcfuXL6i7AQ+hadEa3Ve1BGRsu0KOXW2XZZFeo
+3Pczm/o+jwMiLELcgrM5Ngy8dcCKr6v84F+fi9Y+C8+RZ7g37aLJM0kqbaoN8owL
+mP88f9xDGRAmesvuYlHo+57VTyzU
+-----END CERTIFICATE-----
diff --git a/yt/yt/core/bus/unittests/testdata/cert_ec.pem b/yt/yt/core/bus/unittests/testdata/cert_ec.pem
new file mode 100644
index 00000000000..27e2b46b02e
--- /dev/null
+++ b/yt/yt/core/bus/unittests/testdata/cert_ec.pem
@@ -0,0 +1,14 @@
+-----BEGIN CERTIFICATE-----
+MIICODCCAd2gAwIBAgIUMizwaqRBfZMqZLAfqOon9LxVUwswCgYIKoZIzj0EAwIw
+RjELMAkGA1UEBhMCUlUxDzANBgNVBAcMBk1vc2NvdzEPMA0GA1UECgwGWWFuZGV4
+MRUwEwYDVQQDDAxMb2NhbGhvc3QgQ0EwHhcNMjUwNTEyMTQxOTEzWhcNMzUwNTEw
+MTQxOTEzWjBDMQswCQYDVQQGEwJSVTEPMA0GA1UEBwwGTW9zY293MQ8wDQYDVQQK
+DAZZYW5kZXgxEjAQBgNVBAMMCWxvY2FsaG9zdDBZMBMGByqGSM49AgEGCCqGSM49
+AwEHA0IABOM5UtoWuYTndtECRIdexT/5o5M7Dj5sBe4mPcydByC8AY6sSUghpzTD
+HU/gNnfja8unXt1gukZEe9h03uoH37mjgaswgagwHQYDVR0OBBYEFD3bqNr1fG99
+dKCFAsKkTHwNHkF+MB8GA1UdIwQYMBaAFMX2CNtuUf7N2udXp3Xn9YdWy9czMCwG
+A1UdEQQlMCOCCWxvY2FsaG9zdIcEfwAAAYcQAAAAAAAAAAAAAAAAAAAAATAMBgNV
+HRMBAf8EAjAAMAsGA1UdDwQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYB
+BQUHAwIwCgYIKoZIzj0EAwIDSQAwRgIhANIfRVOvxINkIBKeW60zD7BkrYBXWczl
+FmSagrLHbVpDAiEA+HkxWw0d1mFaF5ZQ03DEbeb0gFQQDQCrqk/hS6ofCbs=
+-----END CERTIFICATE-----
diff --git a/yt/yt/core/bus/unittests/testdata/cert_with_ip_in_san.pem b/yt/yt/core/bus/unittests/testdata/cert_with_ip_in_san.pem
new file mode 100644
index 00000000000..0c3b3cb4ae1
--- /dev/null
+++ b/yt/yt/core/bus/unittests/testdata/cert_with_ip_in_san.pem
@@ -0,0 +1,31 @@
+-----BEGIN CERTIFICATE-----
+MIIFVTCCAz2gAwIBAgIBATANBgkqhkiG9w0BAQsFADAmMREwDwYDVQQKDAhZVHNh
+dXJ1czERMA8GA1UEAwwIWVRzYXVydXMwHhcNMjQwMjI5MTAxMjMzWhcNMzQwMjI2
+MTAxMjMzWjATMREwDwYDVQQKDAhZVHNhdXJ1czCCAiIwDQYJKoZIhvcNAQEBBQAD
+ggIPADCCAgoCggIBAJ4BhT+WpWpkEEgaqcbZoV5/JOeAkFj2iZL08nIjkDRVhDBh
+dQoop47vB1lpLD/1i5+2YEackPrfeJawbNu32s/belvxHpTc1ljndCJlilMiR5B+
+MccP00fUZRsFhkzznANz++31PI4wamHJ1kdJOgD/4civ0mpWgd5s1hityqsypXI3
+RFbi689+mnBp2sGCD6l3QVbMj+8ORvXOVC1h1W3tExiivabjFILgXwb6WG6ZNgg8
+T20zdH4uEc4d2v6XKY4nz4AqYDHax9oqs3XTOTo0Bld6m7oipjGjToMqqpJD32pb
+nxSNT/XECpsNqZ/UMtcQf3HoA2LEOZOg/Knf1mosEww2svb0CqMGfsxxHehCaFJT
+CWkdd8GTQ/3t85xtrd5Ccdqb78o+5039H4GkfkcxAIZQe7siNLbUJ+dR6wRMzJw2
+GMcAHEIozOYVDgwyOT/Q3gaKMg4A3Ki2x7tkCie1KmmAcnECWDRhzG/Hg/RdnYqA
+L6g/m7Z3bw93rUbwXNudVj1ls5MPoyYtTACtZAXnv/PBBd+KR+RZ//T6dD7B9zzC
+MiIsIHemPzE3XsWB6I1AqXv2B8THELgJ8foN6Rho4YQ0t0wjhcM0higfSaiHwqhf
+Cgoc5xxOGa8PlEETWQvcT3ORAukpjM22QbZmA3uLN/MR1DJbwNHqiYPiFZ1zAgMB
+AAGjgaAwgZ0wDgYDVR0PAQH/BAQDAgWgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggr
+BgEFBQcDAjAsBgNVHREEJTAjgglsb2NhbGhvc3SHBH8AAAGHEAAAAAAAAAAAAAAA
+AAAAAAEwHQYDVR0OBBYEFDyAfXqSAoKxK/sCyFs3d2yUpSntMB8GA1UdIwQYMBaA
+FAYGwcRbcWnfIkC1Mozkzm0CDcFbMA0GCSqGSIb3DQEBCwUAA4ICAQCHRuHI5XaS
+PIK3JKzA6l01z7YDHROGn5Xf9hqo5QKHv/aUD01UUhxCbx5z/YZMFDc6iRvu4xQw
+HMl8BCvTo3gHSS8JRcgRn1ov5m50NZMl0Ws9xvvy1j5rZWd3HkBTQSpzr4eSHAZp
+dUWXN2JEW7QIaIxtoiLwMZVjo3Lfr2Qv4uIn9lbIR0F7s39+qFlzvDx+6JbstJsv
+5wPjTS9og3WfpBDaOecHM/nP8v1H9ilx8/EW0nM3jlS0q0Gj9whkha0Pcl156Bga
+biLDoQk7uTccO8Wiyddwfq6tlYy1OAIMqDy0vmoz4L/3FHJUqrzO/fdI9VQLlug+
+M2G6qTJHKzmDkvmtxPfTjRFMu+g7L3QEdYCBogfIHS+VoB9a9K/XmoWBg85cWIPw
+Kfjjf7OouqksfOQopxY2+PCQ66nnkN7y13RjoU2heAme8Fexiowkjhzc1lq0Zn0Q
+XPlnvCHAQMNRNmvBLwNEkW+KN4no0TCImOOTuInBrlKGTaBkinUNS39AF9lZWwAE
+hd1kK5zzF6XvZnKXdVIn4MjcW81hcbrnulq5GHz7XY+lwmORYumYo3Gykjj/+G93
+K9HRlSRV1+BNXmPYtI8hvbAYw05+AWKCk0J5r1GQtPx+Tx3sug/2qks26oURgEHc
+ySl4OPJLp2lhKCUkKVP24Tzg/iS1xT/uHQ==
+-----END CERTIFICATE-----
diff --git a/yt/yt/core/bus/unittests/testdata/generate.sh b/yt/yt/core/bus/unittests/testdata/generate.sh
new file mode 100644
index 00000000000..816edc383cf
--- /dev/null
+++ b/yt/yt/core/bus/unittests/testdata/generate.sh
@@ -0,0 +1,37 @@
+openssl req -x509 -noenc -newkey rsa:4096 -keyout ca_key.pem -out ca.pem \
+ -days 3650 -subj "/C=RU/L=Moscow/O=Yandex/CN=Localhost CA" \
+ -addext "basicConstraints=critical,CA:TRUE,pathlen:0" \
+ -addext "keyUsage=keyCertSign"
+
+openssl req -new -x509 -noenc -newkey rsa:4096 -keyout key.pem -out cert.pem \
+ -CA ca.pem -CAkey ca_key.pem \
+ -days 3650 -subj "/C=RU/L=Moscow/O=Yandex/CN=localhost" \
+ -addext "basicConstraints=critical,CA:FALSE" \
+ -addext "keyUsage=digitalSignature,keyEncipherment" \
+ -addext "extendedKeyUsage=serverAuth,clientAuth"
+
+openssl req -x509 -noenc -newkey rsa:4096 -keyout ca_key_with_ip_in_san.pem -out ca_with_ip_in_san.pem \
+ -days 3650 -subj "/C=RU/L=Moscow/O=Yandex/CN=Localhost CA" \
+ -addext "basicConstraints=critical,CA:TRUE,pathlen:0" \
+ -addext "keyUsage=keyCertSign"
+
+openssl req -new -x509 -noenc -newkey rsa:4096 -keyout key_with_ip_in_san.pem -out cert_with_ip_in_san.pem \
+ -CA ca_with_ip_in_san.pem -CAkey ca_key_with_ip_in_san.pem \
+ -days 3650 -subj "/C=RU/L=Moscow/O=Yandex/CN=localhost" \
+ -addext "subjectAltName=DNS:localhost,IP:127.0.0.1,IP:::1" \
+ -addext "basicConstraints=critical,CA:FALSE" \
+ -addext "keyUsage=digitalSignature,keyEncipherment" \
+ -addext "extendedKeyUsage=serverAuth,clientAuth"
+
+openssl req -x509 -noenc -newkey ec:<(openssl ecparam -name prime256v1) -keyout ca_key_ec.pem -out ca_ec.pem \
+ -days 3650 -subj "/C=RU/L=Moscow/O=Yandex/CN=Localhost CA" \
+ -addext "basicConstraints=critical,CA:TRUE,pathlen:0" \
+ -addext "keyUsage=keyCertSign"
+
+openssl req -new -x509 -noenc -newkey ec:<(openssl ecparam -name prime256v1) -keyout key_ec.pem -out cert_ec.pem \
+ -CA ca_ec.pem -CAkey ca_key_ec.pem \
+ -days 3650 -subj "/C=RU/L=Moscow/O=Yandex/CN=localhost" \
+ -addext "subjectAltName=DNS:localhost,IP:127.0.0.1,IP:::1" \
+ -addext "basicConstraints=critical,CA:FALSE" \
+ -addext "keyUsage=digitalSignature,keyEncipherment" \
+ -addext "extendedKeyUsage=serverAuth,clientAuth"
diff --git a/yt/yt/core/bus/unittests/testdata/key.pem b/yt/yt/core/bus/unittests/testdata/key.pem
new file mode 100644
index 00000000000..2192f1c96f6
--- /dev/null
+++ b/yt/yt/core/bus/unittests/testdata/key.pem
@@ -0,0 +1,52 @@
+-----BEGIN PRIVATE KEY-----
+MIIJQQIBADANBgkqhkiG9w0BAQEFAASCCSswggknAgEAAoICAQDVmsP6Rx+ayLnb
+MxntwnFaXykwjWRfW5DoqQ4lFVBW7B4m6GnfmskYrS8eRa+umKE75R6HWOdxF4fw
+9o116O3eA+xgmSxR5HEJpnw75rC3I6rpMrwv2kPNtDJeZ7mTYRr/7KVdIesYbNlS
+4dplHMGKDJBfNvThl1AzOoYYpTPnZ53l6Vmc6gAxQvv/Bns7yfl2VeEPDERDhJDd
+rf1Iuk9a/eFZeIgOuVHnGGSpCNmdvRYRWVLA3yWnKMDpfAKz8hfs9WuSGgEqrvrr
+iuxHcHuqEYDlyZLLj4qtKhgpgcYK+Xo5qQ7t+fgVwnAccU1DMyryhrmLkttixoLL
+oiAm3TucF41fFoRj7hYgiYd9ZHTK9CCq2sVBm4hDg3th5mku9O2g7chQnKMyyKHk
+Zfgs9jST2cTyt3jfuVXc92IFgCS2TCaAJNj24kWEIE8WwyETfScvv21fSCbkW6Bb
+VpncIW+BDLCRxhAiMGMlDjbNymqcuOPiUjB8M7lgQCIA4+pDdVXsUqZtJal5fJEA
+v+6qQFUIMpOwomQSN3yBzfSGidEgo0dCSv9GhXUzVbjj25GS99V9t+CR1xFxgwjK
+dPBkxKIoOjFeRDKtzJjqpWNhA1QrBG+03abpoZ+1iF+CjiPTVQdOsrsEdEcqNbks
+VeA0lRefJyY7r0H7DV5NGCdUTvCzqwIDAQABAoICAEvCv0TLKiH/lK/y4XzrTMIF
+Y3oVhCawNubWYy5y71JNH+qj3z1QTIgEkOQ3Sjbuaq1wN9JAjaIWewBTqlvKOGfY
+02N1oHsRP6hxFLo4ObBTJcDdXlLIoujYQ08pke/8bpOcDxDHwXch0Djt40SenOSG
+TUSAHP3QacEpvjsKiSzHmwDbMY4Oju/p9q/+0AGmQuUeU5s/Og0KfUkq911uu0um
+JWHS9srmHu8Mv1MW0Px5/tQ7brb6zoOJ2FZXxiulr6e7aiJhN824T0XwuZojArGQ
+0LtvsbGiYUjG19gM772fu6Ks3B863Ct3kcT8yK8PfGmVsESZW1ee2fA4uheeuw+c
+/3D71+dnjvw0/LFQEWyiH9NhjvafALBGxtlbfTTVtw5bHVaoalYpEggKnJrRabPa
+HmdPkEAK0vJzBvTjPW1lEvlxUkol1SsYq8xpBkatieZzf4+SWk/ugP7rvoRVRbB1
+GmHc9CK9jCDTcN7ii5pTOpc2VOK5cvn+K5L4P7Qw4kK37pyO56jhVbTAavNxotSc
++mZa8OqZaK0jvD9sUPCSuY44x5X6WhXjILE+R3QXXXtGkyjlgJxGmfkYcu19GV4B
+ziU7hVCjqtNOQ51ywEtWUA93lchj3fD5ryo9dCv5a5Xco4O5E24VeGjW3MBqeBNW
+wNIUintgwVt20P1I+xZRAoIBAQD9UzjlhvrDgNZsaJu82+PoOEJM2pa5F+fDGSeq
+XTYCCeqioMUujJWDEkBEzfTvu4r7Pr7uExOlkyUXdSELfm+ABkt+eZguPB15Ua6V
+Y59Z4yrpqp0JIwpxXGxNOsSUnVwGg3OTlUGWYVyxyGHYQnSGJNe2/NhSlQZmvxx2
+PxqTH+g9dLn7FkQk/FNQI4l4LnphgGvNBVyXZoHcw16y86MAKLyrcZBmbHnuWMC6
+zjIu61uXXd8GU7IU8BWJbUQKvz0Y5susWf921U/7Qa8NZ2vFY4Q7oGlJVBVhMNmB
+WLL8/WUeWXu9HvL89QWpb31l5V5L+lF+GSq5ZQxFAjLEVl77AoIBAQDX3CwirHtf
+MI7z+zjwLDWgVG7RFVY4Pv5TWDWYHiQT9VPBU16K0+fVS1iXxYvoiGJkrj38Re8y
+rkTZ+qvKLmlDftqx7bImbzs01QJRbUH2gxZFMDQIF7uTSFynycywaQk+xpFEPTHl
+CXMIyGAMAsd0B2OKPAxZNoHTd0P10FYBIENwxadG9w7Ocm41J/d4Qx6wQsV10hGF
+0OgtIioplSDz1Ean/IT8XmwLK7tIiShJzZidE+0sY4depy3JdAHWv50uXMs3bIcu
+xvVBA/e/jf+HssfLjzzrYtwp2JBzSOTox/eCUJ/i9gatC5ox+ewjwOaFXV3bGEib
+mK+I3xiQ3B8RAoIBAE0Jc/IJHFU75vlMzp+eVy6VfUQV7WQYavifu7pJYlU4YsxW
+C+DeC9GySS0jXOtSoy9Io5OO5ZiiqNL7YbM3Hf1W7Lpni+nzihsMxgTUKO+S78fj
+hKH0sAZNTvoldwai3At3CjzFVQ7ASQofn/G+M+VfauJQ/hAPFcVFNQiYpCI9v8iA
+qNY8rTh6K3Pherq7l6fy/9V3XfMEz1UtbK0K/nTb7pRMktczAdmD0Ah/EC/Ijy/2
+8g3ggfVwFXyXZ+vEwHXEKggdzlx6/jmwfeWbn+CFJP9lBt+v3FiUHHEDYlshTBDw
+sXqP4OEgOjqOlxnXqNd+Ji4sxRtgKV0LEBk5EuUCggEAIlBbq79jdURQ1TQQXw2I
+EM6bNx1/MT3CTBlvm5je/1U2VTsdglAhQGTT1nyOuw5DJeIU9G9hkNrnEweoG2G5
+VgNqXHJ+qWFxNfrOfYcyvy8jcSgyfT7YkJcmM33+zeRElfgWy5Q2xEP2R2Ui74XZ
+kvZBuo3FIMFrbeQ9p2vQ4Cjyz5B8AOnxLpw+LLEHw9RXoolavloAcxc8cUBHF4kf
+TeNmv/mCYmPYJQZ0pRk4kFLgecfbIf1IXaGRw75vNGYNZHtXyp2z95mlDwrEbWzz
+O+0NmaxRcNGsUfKdM9ZYnTB8hfivEfMuKH/5qQwjn6Nggb7P1q5LjIB/FvDwBMcZ
+IQKCAQBfPoddROG6bTkzn7kL8rDwuk2H2sLh5njRKo3u29dVlwU93lmKLoP4blcf
+tAbXDVrlGpZbIo73baE57Rv5ehcV64oS/G63Lcw8nsqmBvs9982tpYPNX9EOvXV5
+1iK+hs8ZUUVCVkTOXsZb5M90VKQiHrnddO+08lE5YI/lzM5laqcytPmYcLkVWPPz
+qrpW/AReSwhvwVugcMFUgMXaDx/3SAY75B808wX1tizv76omWZAQ774FeGQGyP4C
+8f4t3LIV9h/q2Hj8geMjil9ZGogtWJ5uDspp7As5OyMF0ZMXTMwSnFsXB/L4YIk+
+rPl77gAcribJm3TzBVHm2m6jBGtb
+-----END PRIVATE KEY-----
diff --git a/yt/yt/core/bus/unittests/testdata/key_ec.pem b/yt/yt/core/bus/unittests/testdata/key_ec.pem
new file mode 100644
index 00000000000..8000d1435c4
--- /dev/null
+++ b/yt/yt/core/bus/unittests/testdata/key_ec.pem
@@ -0,0 +1,5 @@
+-----BEGIN PRIVATE KEY-----
+MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgYzqbMURDpLQaUBEI
+sR9pNuyopS4/ACCkDkxjlUX3aquhRANCAATjOVLaFrmE53bRAkSHXsU/+aOTOw4+
+bAXuJj3MnQcgvAGOrElIIac0wx1P4DZ342vLp17dYLpGRHvYdN7qB9+5
+-----END PRIVATE KEY-----
diff --git a/yt/yt/core/bus/unittests/testdata/key_with_ip_in_san.pem b/yt/yt/core/bus/unittests/testdata/key_with_ip_in_san.pem
new file mode 100644
index 00000000000..2e314b8f7b4
--- /dev/null
+++ b/yt/yt/core/bus/unittests/testdata/key_with_ip_in_san.pem
@@ -0,0 +1,52 @@
+-----BEGIN PRIVATE KEY-----
+MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQCeAYU/lqVqZBBI
+GqnG2aFefyTngJBY9omS9PJyI5A0VYQwYXUKKKeO7wdZaSw/9YuftmBGnJD633iW
+sGzbt9rP23pb8R6U3NZY53QiZYpTIkeQfjHHD9NH1GUbBYZM85wDc/vt9TyOMGph
+ydZHSToA/+HIr9JqVoHebNYYrcqrMqVyN0RW4uvPfppwadrBgg+pd0FWzI/vDkb1
+zlQtYdVt7RMYor2m4xSC4F8G+lhumTYIPE9tM3R+LhHOHdr+lymOJ8+AKmAx2sfa
+KrN10zk6NAZXepu6IqYxo06DKqqSQ99qW58UjU/1xAqbDamf1DLXEH9x6ANixDmT
+oPyp39ZqLBMMNrL29AqjBn7McR3oQmhSUwlpHXfBk0P97fOcba3eQnHam+/KPudN
+/R+BpH5HMQCGUHu7IjS21CfnUesETMycNhjHABxCKMzmFQ4MMjk/0N4GijIOANyo
+tse7ZAontSppgHJxAlg0Ycxvx4P0XZ2KgC+oP5u2d28Pd61G8FzbnVY9ZbOTD6Mm
+LUwArWQF57/zwQXfikfkWf/0+nQ+wfc8wjIiLCB3pj8xN17FgeiNQKl79gfExxC4
+CfH6DekYaOGENLdMI4XDNIYoH0moh8KoXwoKHOccThmvD5RBE1kL3E9zkQLpKYzN
+tkG2ZgN7izfzEdQyW8DR6omD4hWdcwIDAQABAoICADzT/QZD6p6QsyvvB9lDwznr
+3Ls65Vc6YjAvGH8UbdmX6nHtsu8cQ5VlNAEZ2i0tTHlJ7rqAX9gU3Am3FdFocFaA
++hQXOVy5v9MuF6l+SchDdCWOT3+A+ie2/s1uTQum5TL3Hc+4D3316Z6H43RCHpBv
+8e4esfS6JPkKEUoi7dkGgGb+G9MPPRT+elo4hjzk4z6saH0P94Fij7LlocZu2Yme
+MTHUxQpQdX8E/dBj5FN/rCtzfGhf3MMO3U/qcnp8m0Tc0qdWqP3IahP1SG1dybQ8
+fwyCaR05ZZ3KbtlUPaJdes8pQo7Y8CV/OU4D7n9XY9MjyMyDM3p8bGYHHf4P7C0M
+qdTYAVXZ9NvKlf87J8zMJ4YDiKTMLdgFjgnj/tYJ9PkUZHgfsk1KSr5IT6wpLBAx
+jEs0+WsbQRmtlJ93ZvZsDJyGZbkeCRIAxyz/eN2Rf56YGrWZcPcn8yrAsnt4nYqz
+ewjM9wu6jB8bfAgDS0y3kPsl1www+AkRDnNQ9s+Vw5x0nccjbV7fdezkwf04cA6N
+QL+0WSALsU0iGGPKrLA5L/kC3W650xSeOgINdvCnZsSEPAF6HQ/E5udd9HD5+7Ma
+hVl2S7FbEVxKgXpXPd1GFBKZKnUTCB5erMDyaN7jAiaVBtRE8qc0hQRhHCtE7qNT
+HvWkztmzBuVGTpOkBQuRAoIBAQDDIsOMkYo/svV/lsUJM4fVMTVJ354ThhxvFdJs
+rlaUEX2mZIsbzjI76yuOsu6nlGKLuiYVfAAV3HNEhNrSnp6+3NPXMgSsE33TFRPR
+Y6P1oT/ZE8YrBkndSioU1ad9Z0ClRNhDlP2c9xCBGpXUSQLt/4C2Nk/wgG/wy7gG
++hJ2ybn133DPpwcx1KQCpslF+VaECPLrpjxWCFEGInOV8gVvxhEb7hrD8qmUPlAR
+3WYFT6/RJGCJ7BeOvXWYa8AfWRysLfLwnzuoUxl45O7X7eZqPufrhAJyDNgN3c6J
+VgVxNPSSc6mjsY0NyItQZEOXtzRJqnRlmGD2Zd5BJFG7PmIVAoIBAQDPSgFqnFe1
+RM7AtC5wRBLTU/LQo8XV3WPsvQpYaa2+YyPG1bjSMWmdejsy4k+lbUx4VzD+veX7
+ZdSmukYmeGT0pp6Yy7FYYAcZ8Tu+CSAxGH8Bvab+U1oy2Y3++lXdAc0OVjuezVSy
+RbUE97S9MjeFMTKxIg2RkJc4p+Ir0sFn8rC9a0ZqNdcV1BCzJkMB5Y0PbG+afopL
+wS4xIMIy2Jp3Y0CanOW/dl6LnWYdEtWMt0Pi5wFDVVn0/DgGNx0SDG5fKj4rJE4F
+6mezK3kyAf7qI35nf8wm/rL7wL262F4iGZp+OKNFQnaEuKmrUsz/l2GZZkfIIcbI
+pPVaqKbAQEtnAoIBAQCj9V/NirRY1WuFqw8frhahwVj/G09dJEBb7kACZXIFs7SZ
+zL09vcFjqzPMEPiKAhnTQbOiNbB0reiEWATtF65WvIGavUJDu5TreThPpaMsTjKx
+mPXXTM9fimNVYjf7HHiq5O+5yURXURijAc2Gs1os05Q4heYhNCnab7HO2uwMt27y
+8q19LODUs9CjEbTogJp7EnHaIrFrsE00FFp+UP7Ubd4OU8BViF2IW9s3R4njSJN3
+7VLYUHFy1CosycyCCoQW//yyxXiA9GHgvKsa75+9AeIod6D+Z2BaNlbF+mtUNaSS
+MXEGQ7c7L5gvEi/hGGRsyTZH7wL5xZo7reKmq8IJAoIBAHIiaHdAEFcBxOl8DFnK
+UadEgNz6X/LqzJtMV0bpIT5ELi3L/dDWXjXUWIYi8AHBFarpL1QEUX5Dynvm8rs5
+7TR8DbVJ6qMjdKWHGwL+2VfPChd2Sl2cnXyEJ1gulFp1JGfxeTBuFGV4Vjye+0h1
+Pva6aRP5EQmGWI1cev7wM4e9rC0PxRyz+nLNakiKF7kSoMHOTgD+Db26Z2mrhOIk
+O6Di6G55V1M9pL8w8kmt1iF9wwZLdXmSpE5tFZfufrYyXA9QHhz5B3DgaSrRFBFB
+4g8fbfkk9868zOYrcQxRGDukZ1l6bAO1nbZkSx/HHpLY0md5VqrOVjqiAWpilDYk
+8J8CggEAVN7WQIMWzuGBc9Hj6H539eXLnoWgccvYN+GwqclQGKQJdUQrM7Ci3kP/
+Zj00T//jew/vHtxx6U1XUfCIbI5SjujqhbkCkikUMAhozsjlmUlAxKz8689/XFXK
+9bPRRvmKw6p14drk507w6t3uD4E/O3PHeEtrdEeDmY78e01T5zkg90jVq10szUMo
+sxw0PoOG4ktYtCsmwXXPHq9gRtzfOqZT/UNHgnBsLALmWcec2gZLa++M1OnVaqI2
+hTyvWDnxD5oKa7hDSBXTOorcfQVRSaC05HPGMhX+HkfHwfXJBxAE1UC38UMD7x58
+AbE/BnHl1tAmZXLMrHq/4r0wYUjBsA==
+-----END PRIVATE KEY-----
diff --git a/yt/yt/core/bus/unittests/ya.make b/yt/yt/core/bus/unittests/ya.make
index a36ce3f0374..75643918d97 100644
--- a/yt/yt/core/bus/unittests/ya.make
+++ b/yt/yt/core/bus/unittests/ya.make
@@ -17,6 +17,10 @@ PEERDIR(
library/cpp/testing/common
)
+EXPLICIT_DATA()
+
+DATA(arcadia/yt/yt/core/bus/unittests/testdata)
+
REQUIREMENTS(
cpu:4
ram:4
diff --git a/yt/yt/core/crypto/config.cpp b/yt/yt/core/crypto/config.cpp
index 1face17aca6..0981b952b9b 100644
--- a/yt/yt/core/crypto/config.cpp
+++ b/yt/yt/core/crypto/config.cpp
@@ -1,38 +1,80 @@
#include "config.h"
+#include <util/system/env.h>
+
namespace NYT::NCrypto {
////////////////////////////////////////////////////////////////////////////////
void TPemBlobConfig::Register(TRegistrar registrar)
{
+ registrar.Parameter("environment_variable", &TThis::EnvironmentVariable)
+ .Optional();
registrar.Parameter("file_name", &TThis::FileName)
.Optional();
registrar.Parameter("value", &TThis::Value)
.Optional();
registrar.Postprocessor([] (TThis* config) {
- if (config->FileName && config->Value) {
- THROW_ERROR_EXCEPTION("Cannot specify both \"file_name\" and \"value\"");
- }
- if (!config->FileName && !config->Value) {
- THROW_ERROR_EXCEPTION("Must specify either \"file_name\" or \"value\"");
+ if (!!config->EnvironmentVariable + !!config->FileName + !!config->Value != 1) {
+ THROW_ERROR_EXCEPTION("Must specify one of \"environment_variable\", \"file_name\", or \"value\"");
}
});
}
+TPemBlobConfigPtr TPemBlobConfig::CreateFileReference(const TString& fileName)
+{
+ auto config = New<TPemBlobConfig>();
+ config->FileName = fileName;
+ return config;
+}
+
TString TPemBlobConfig::LoadBlob() const
{
- if (FileName) {
+ if (EnvironmentVariable) {
+ return GetEnv(*EnvironmentVariable);
+ } else if (FileName) {
return TFileInput(*FileName).ReadAll();
} else if (Value) {
return *Value;
} else {
- THROW_ERROR_EXCEPTION("Neither \"file_name\" nor \"value\" is given");
+ THROW_ERROR_EXCEPTION("Neither \"environment_variable\" nor \"file_name\" nor \"value\" is given");
}
}
////////////////////////////////////////////////////////////////////////////////
+void TSslContextCommand::Register(TRegistrar registrar)
+{
+ registrar.Parameter("name", &TThis::Name)
+ .NonEmpty();
+ registrar.Parameter("value", &TThis::Value);
+}
+
+
+TSslContextCommandPtr TSslContextCommand::Create(std::string name, std::string value)
+{
+ auto command = New<TSslContextCommand>();
+ command->Name = std::move(name);
+ command->Value = std::move(value);
+ return command;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+void TSslContextConfig::Register(TRegistrar registrar)
+{
+ registrar.Parameter("ca", &TThis::CertificateAuthority)
+ .Default();
+ registrar.Parameter("cert_chain", &TThis::CertificateChain)
+ .Optional();
+ registrar.Parameter("private_key", &TThis::PrivateKey)
+ .Optional();
+ registrar.Parameter("ssl_configuration_commands", &TThis::SslConfigurationCommands)
+ .Optional();
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
} // namespace NYT::NCrypto
diff --git a/yt/yt/core/crypto/config.h b/yt/yt/core/crypto/config.h
index 4bd7058140a..d5bc6e8cf41 100644
--- a/yt/yt/core/crypto/config.h
+++ b/yt/yt/core/crypto/config.h
@@ -8,15 +8,18 @@ namespace NYT::NCrypto {
////////////////////////////////////////////////////////////////////////////////
-//! Either an inlined value or a file reference.
+//! Either an inlined value, environment variable, or a file reference.
struct TPemBlobConfig
: public NYTree::TYsonStruct
{
+ std::optional<TString> EnvironmentVariable;
std::optional<TString> FileName;
std::optional<TString> Value;
TString LoadBlob() const;
+ static TPemBlobConfigPtr CreateFileReference(const TString& fileName);
+
REGISTER_YSON_STRUCT(TPemBlobConfig);
static void Register(TRegistrar registrar);
@@ -26,4 +29,40 @@ DEFINE_REFCOUNTED_TYPE(TPemBlobConfig)
////////////////////////////////////////////////////////////////////////////////
+struct TSslContextCommand
+ : public NYTree::TYsonStruct
+{
+ std::string Name;
+ std::string Value;
+
+ static TSslContextCommandPtr Create(std::string name, std::string value);
+
+ REGISTER_YSON_STRUCT(TSslContextCommand);
+
+ static void Register(TRegistrar registrar);
+};
+
+DEFINE_REFCOUNTED_TYPE(TSslContextCommand)
+
+////////////////////////////////////////////////////////////////////////////////
+
+struct TSslContextConfig
+ : public virtual NYTree::TYsonStruct
+{
+ NCrypto::TPemBlobConfigPtr CertificateAuthority;
+ NCrypto::TPemBlobConfigPtr CertificateChain;
+ NCrypto::TPemBlobConfigPtr PrivateKey;
+
+ //! Commands for SSL context configuration handled by SSL_CONF_cmd.
+ std::vector<TSslContextCommandPtr> SslConfigurationCommands;
+
+ REGISTER_YSON_STRUCT(TSslContextConfig);
+
+ static void Register(TRegistrar registrar);
+};
+
+DEFINE_REFCOUNTED_TYPE(TSslContextConfig)
+
+////////////////////////////////////////////////////////////////////////////////
+
} // namespace NYT::NCrypto
diff --git a/yt/yt/core/crypto/public.h b/yt/yt/core/crypto/public.h
index 9c04e5eba04..c71a42df5f6 100644
--- a/yt/yt/core/crypto/public.h
+++ b/yt/yt/core/crypto/public.h
@@ -9,6 +9,8 @@ namespace NYT::NCrypto {
////////////////////////////////////////////////////////////////////////////////
DECLARE_REFCOUNTED_STRUCT(TPemBlobConfig)
+DECLARE_REFCOUNTED_STRUCT(TSslContextCommand)
+DECLARE_REFCOUNTED_STRUCT(TSslContextConfig)
DECLARE_REFCOUNTED_CLASS(TSslContext)
////////////////////////////////////////////////////////////////////////////////
diff --git a/yt/yt/core/crypto/tls.cpp b/yt/yt/core/crypto/tls.cpp
index a2dcb50d124..5ac5e062659 100644
--- a/yt/yt/core/crypto/tls.cpp
+++ b/yt/yt/core/crypto/tls.cpp
@@ -1,4 +1,5 @@
#include "tls.h"
+#include "config.h"
#include <yt/yt/core/misc/error.h>
#include <yt/yt/core/misc/finally.h>
@@ -12,6 +13,8 @@
#include <yt/yt/core/logging/log.h>
+#include <yt/yt/core/rpc/public.h> // For EErrorCode::SslError
+
#include <library/cpp/yt/memory/ref.h>
#include <library/cpp/openssl/io/stream.h>
@@ -36,31 +39,62 @@ static YT_DEFINE_GLOBAL(const NLogging::TLogger, Logger, "Tls");
namespace {
-TErrorAttribute GetSslErrors()
+constexpr auto TlsBufferSize = 1_MB;
+
+} // namespace
+
+////////////////////////////////////////////////////////////////////////////////
+
+// Get all saved SSL errors for current thread.
+TError GetLastSslError(TString message)
{
- TString errorStr;
+ auto lastSslError = ERR_peek_last_error();
+ TStringBuilder errorStr;
ERR_print_errors_cb([] (const char* str, size_t len, void* ctx) {
- auto& out = *reinterpret_cast<TString*>(ctx);
- if (!out.empty()) {
- out += ", ";
+ auto& out = *reinterpret_cast<TStringBuilder*>(ctx);
+ if (!out.GetLength()) {
+ out.AppendString(", ");
}
- out.append(str, len);
+ out.AppendString(TStringBuf(str, len));
return 1;
}, &errorStr);
- return TErrorAttribute("ssl_error", errorStr);
+ return TError(NRpc::EErrorCode::SslError, std::move(message), TError::DisableFormat)
+ << TErrorAttribute("ssl_last_error_code", lastSslError)
+ << TErrorAttribute("ssl_error", errorStr.Flush());
}
-constexpr auto TlsBufferSize = 1_MB;
+////////////////////////////////////////////////////////////////////////////////
-} // namespace
+void TSslDeleter::operator()(BIO* bio) const noexcept
+{
+ BIO_free(bio);
+}
+
+void TSslDeleter::operator()(X509* x509) const noexcept
+{
+ X509_free(x509);
+}
+
+void TSslDeleter::operator()(EVP_PKEY* key) const noexcept
+{
+ EVP_PKEY_free(key);
+}
+
+void TSslDeleter::operator()(SSL_CTX* ctx) const noexcept
+{
+ SSL_CTX_free(ctx);
+}
+
+void TSslDeleter::operator()(SSL* ssl) const noexcept
+{
+ SSL_free(ssl);
+}
////////////////////////////////////////////////////////////////////////////////
struct TSslContextImpl
: public TRefCounted
{
- SSL_CTX* Context = nullptr;
-
TSslContextImpl()
{
Reset();
@@ -68,56 +102,44 @@ struct TSslContextImpl
~TSslContextImpl()
{
- if (Context) {
- SSL_CTX_free(Context);
- }
- if (ActiveContext_) {
- SSL_CTX_free(ActiveContext_);
- }
+ }
+
+ SSL_CTX* GetContext() const
+ {
+ return Context_.get();
}
void Reset()
{
- if (Context) {
- SSL_CTX_free(Context);
- }
#if OPENSSL_VERSION_NUMBER >= 0x10100000L
- Context = SSL_CTX_new(TLS_method());
- if (!Context) {
- THROW_ERROR_EXCEPTION("SSL_CTX_new(TLS_method()) failed")
- << GetSslErrors();
+ Context_.reset(SSL_CTX_new(TLS_method()));
+ if (!Context_) {
+ THROW_ERROR GetLastSslError("SSL_CTX_new(TLS_method()) failed");
}
- if (SSL_CTX_set_min_proto_version(Context, TLS1_2_VERSION) == 0) {
- THROW_ERROR_EXCEPTION("SSL_CTX_set_min_proto_version failed")
- << GetSslErrors();
+ if (SSL_CTX_set_min_proto_version(GetContext(), TLS1_2_VERSION) == 0) {
+ THROW_ERROR GetLastSslError("SSL_CTX_set_min_proto_version failed");
}
- if (SSL_CTX_set_max_proto_version(Context, TLS1_2_VERSION) == 0) {
- THROW_ERROR_EXCEPTION("SSL_CTX_set_max_proto_version failed")
- << GetSslErrors();
+ if (SSL_CTX_set_max_proto_version(GetContext(), TLS1_3_VERSION) == 0) {
+ THROW_ERROR GetLastSslError("SSL_CTX_set_max_proto_version failed");
}
#else
- Ctx = SSL_CTX_new(TLSv1_2_method());
+ Context.reset(SSL_CTX_new(TLSv1_2_method()));
if (!Context) {
- THROW_ERROR_EXCEPTION("SSL_CTX_new(TLSv1_2_method()) failed")
- << GetSslErrors();
+ THROW_ERROR GetLastSslError("SSL_CTX_new(TLSv1_2_method()) failed");
}
#endif
}
void Commit(TInstant time)
{
- SSL_CTX* oldContext;
- YT_ASSERT(Context);
+ TSslCtxPtr oldContext;
+ YT_ASSERT(Context_);
{
auto guard = WriterGuard(Lock_);
- oldContext = ActiveContext_;
- ActiveContext_ = Context;
- Context = nullptr;
+ oldContext.swap(ActiveContext_);
+ ActiveContext_.swap(Context_);
CommitTime_ = time;
}
- if (oldContext) {
- SSL_CTX_free(oldContext);
- }
}
TInstant GetCommitTime() const
@@ -126,22 +148,30 @@ struct TSslContextImpl
return CommitTime_;
}
- SSL* NewSsl()
+ TSslPtr NewSsl()
{
- auto guard = ReaderGuard(Lock_);
- YT_ASSERT(ActiveContext_);
- return SSL_new(ActiveContext_);
+ SSL* ssl;
+ {
+ auto guard = ReaderGuard(Lock_);
+ YT_ASSERT(ActiveContext_);
+ ssl = SSL_new(ActiveContext_.get());
+ }
+ if (!ssl) {
+ THROW_ERROR GetLastSslError("SSL_new failed");
+ }
+ return TSslPtr(ssl);
}
bool IsActive(const SSL* ssl)
{
auto guard = ReaderGuard(Lock_);
- return SSL_get_SSL_CTX(ssl) == ActiveContext_;
+ return SSL_get_SSL_CTX(ssl) == ActiveContext_.get();
}
private:
YT_DECLARE_SPIN_LOCK(NThreading::TReaderWriterSpinLock, Lock_);
- SSL_CTX* ActiveContext_ = nullptr;
+ TSslCtxPtr Context_;
+ TSslCtxPtr ActiveContext_;
TInstant CommitTime_;
};
@@ -165,10 +195,6 @@ public:
, Underlying_(std::move(connection))
{
Ssl_ = Context_->NewSsl();
- if (!Ssl_) {
- THROW_ERROR_EXCEPTION("SSL_new failed")
- << GetSslErrors();
- }
InputBIO_ = BIO_new(BIO_s_mem());
YT_VERIFY(InputBIO_);
@@ -178,7 +204,7 @@ public:
OutputBIO_ = BIO_new(BIO_s_mem());
YT_VERIFY(OutputBIO_);
- SSL_set_bio(Ssl_, InputBIO_, OutputBIO_);
+ SSL_set_bio(Ssl_.get(), InputBIO_, OutputBIO_);
InputBuffer_ = TSharedMutableRef::Allocate<TTlsBufferTag>(TlsBufferSize);
OutputBuffer_ = TSharedMutableRef::Allocate<TTlsBufferTag>(TlsBufferSize);
@@ -186,19 +212,18 @@ public:
void SetHost(const TString& host)
{
- SSL_set_tlsext_host_name(Ssl_, host.c_str());
+ SSL_set_tlsext_host_name(Ssl_.get(), host.c_str());
}
~TTlsConnection()
{
- SSL_free(Ssl_);
}
void StartClient()
{
- SSL_set_connect_state(Ssl_);
- auto sslResult = SSL_do_handshake(Ssl_);
- sslResult = SSL_get_error(Ssl_, sslResult);
+ SSL_set_connect_state(Ssl_.get());
+ auto sslResult = SSL_do_handshake(Ssl_.get());
+ sslResult = SSL_get_error(Ssl_.get(), sslResult);
YT_VERIFY(sslResult == SSL_ERROR_WANT_READ);
Invoker_->Invoke(BIND(&TTlsConnection::DoRun, MakeWeak(this)));
@@ -206,7 +231,7 @@ public:
void StartServer()
{
- SSL_set_accept_state(Ssl_);
+ SSL_set_accept_state(Ssl_.get());
Invoker_->Invoke(BIND(&TTlsConnection::DoRun, MakeWeak(this)));
}
@@ -372,7 +397,7 @@ private:
const IInvokerPtr Invoker_;
const IConnectionPtr Underlying_;
- SSL* Ssl_ = nullptr;
+ TSslPtr Ssl_;
BIO* InputBIO_ = nullptr;
BIO* OutputBIO_ = nullptr;
@@ -493,23 +518,22 @@ private:
CheckError();
if (CloseRequested_ && !HandshakeInProgress_) {
- SSL_shutdown(Ssl_);
+ SSL_shutdown(Ssl_.get());
MaybeStartUnderlyingIO(false);
}
// NB: We should check for an error here, because Underylying_ might have failed already, and then
// we will loop on SSL_ERROR_WANT_READ forever.
if (HandshakeInProgress_ && Error_.IsOK()) {
- int sslResult = SSL_do_handshake(Ssl_);
+ int sslResult = SSL_do_handshake(Ssl_.get());
if (sslResult == 1) {
HandshakeInProgress_ = false;
} else {
- int sslError = SSL_get_error(Ssl_, sslResult);
+ int sslError = SSL_get_error(Ssl_.get(), sslResult);
if (sslError == SSL_ERROR_WANT_READ) {
MaybeStartUnderlyingIO(true);
} else {
- Error_ = TError("SSL_do_handshake failed")
- << GetSslErrors();
+ Error_ = GetLastSslError("SSL_do_handshake failed");
YT_LOG_DEBUG(Error_, "TLS handshake failed");
CheckError();
return;
@@ -524,11 +548,10 @@ private:
// Second condition acts as a poor-man backpressure.
if (WriteActive_ && !UnderlyingWriteActive_) {
for (const auto& ref : WriteBuffer_) {
- int count = SSL_write(Ssl_, ref.Begin(), ref.Size());
+ int count = SSL_write(Ssl_.get(), ref.Begin(), ref.Size());
if (count < 0) {
- Error_ = TError("SSL_write failed")
- << GetSslErrors();
+ Error_ = GetLastSslError("SSL_write failed");
YT_LOG_DEBUG(Error_, "TLS write failed");
CheckError();
return;
@@ -547,7 +570,7 @@ private:
}
if (ReadActive_) {
- int count = SSL_read(Ssl_, ReadBuffer_.Begin(), ReadBuffer_.Size());
+ int count = SSL_read(Ssl_.get(), ReadBuffer_.Begin(), ReadBuffer_.Size());
if (count >= 0) {
ReadActive_ = false;
ReadBuffer_.Reset();
@@ -555,12 +578,11 @@ private:
ReadPromise_.Set(count);
ReadPromise_.Reset();
} else {
- int sslError = SSL_get_error(Ssl_, count);
+ int sslError = SSL_get_error(Ssl_.get(), count);
if (sslError == SSL_ERROR_WANT_READ) {
MaybeStartUnderlyingIO(true);
} else {
- Error_ = TError("SSL_read failed")
- << GetSslErrors();
+ Error_ = GetLastSslError("SSL_read failed");
YT_LOG_DEBUG(Error_, "TLS read failed");
CheckError();
return;
@@ -668,72 +690,109 @@ TInstant TSslContext::GetCommitTime() const
return Impl_->GetCommitTime();
}
+void TSslContext::ApplyConfig(const TSslContextConfigPtr& config, TCertificatePathResolver pathResolver)
+{
+ if (!config) {
+ return;
+ }
+
+ const auto& commands = config->SslConfigurationCommands;
+ if (!commands.empty()) {
+ std::unique_ptr<SSL_CONF_CTX, decltype(&SSL_CONF_CTX_free)> configContext(SSL_CONF_CTX_new(), SSL_CONF_CTX_free);
+ if (!configContext) {
+ THROW_ERROR GetLastSslError("SSL_CONF_CTX_new failed");
+ }
+
+ SSL_CONF_CTX_set_flags(configContext.get(),
+ SSL_CONF_FLAG_FILE |
+ SSL_CONF_FLAG_CLIENT |
+ SSL_CONF_FLAG_SERVER |
+ SSL_CONF_FLAG_CERTIFICATE |
+ SSL_CONF_FLAG_SHOW_ERRORS);
+
+ SSL_CONF_CTX_set_ssl_ctx(configContext.get(), Impl_->GetContext());
+
+ for (const auto& command: commands) {
+ if (SSL_CONF_cmd(configContext.get(), command->Name.c_str(), command->Value.c_str()) <= 0) {
+ THROW_ERROR GetLastSslError("SSL_CONF_cmd failed")
+ << TErrorAttribute("ssl_config_command_name", command->Name);
+ }
+ }
+
+ if (SSL_CONF_CTX_finish(configContext.get()) != 1) {
+ THROW_ERROR GetLastSslError("SSL_CONF_CTX_finish failed");
+ }
+ }
+
+ AddCertificateAuthority(config->CertificateAuthority, pathResolver);
+ AddCertificateChain(config->CertificateChain, pathResolver);
+ AddPrivateKey(config->PrivateKey, pathResolver);
+}
+
void TSslContext::UseBuiltinOpenSslX509Store()
{
- SSL_CTX_set_cert_store(Impl_->Context, GetBuiltinOpenSslX509Store().Release());
+ SSL_CTX_set_cert_store(Impl_->GetContext(), GetBuiltinOpenSslX509Store().Release());
}
void TSslContext::SetCipherList(const TString& list)
{
- if (SSL_CTX_set_cipher_list(Impl_->Context, list.data()) == 0) {
- THROW_ERROR_EXCEPTION("SSL_CTX_set_cipher_list failed")
- << TErrorAttribute("cipher_list", list)
- << GetSslErrors();
+ if (SSL_CTX_set_cipher_list(Impl_->GetContext(), list.data()) == 0) {
+ THROW_ERROR GetLastSslError("SSL_CTX_set_cipher_list failed")
+ << TErrorAttribute("cipher_list", list);
+ }
+}
+
+void TSslContext::AddCertificateAuthorityFromFile(const TString& path)
+{
+ if (SSL_CTX_load_verify_locations(Impl_->GetContext(), path.c_str(), nullptr) != 1) {
+ THROW_ERROR GetLastSslError("SSL_CTX_load_verify_locations failed")
+ << TErrorAttribute("path", path);
}
}
void TSslContext::AddCertificateFromFile(const TString& path)
{
- if (SSL_CTX_use_certificate_file(Impl_->Context, path.c_str(), SSL_FILETYPE_PEM) != 1) {
- THROW_ERROR_EXCEPTION("SSL_CTX_use_certificate_file failed")
- << TErrorAttribute("path", path)
- << GetSslErrors();
+ if (SSL_CTX_use_certificate_file(Impl_->GetContext(), path.c_str(), SSL_FILETYPE_PEM) != 1) {
+ THROW_ERROR GetLastSslError("SSL_CTX_use_certificate_file failed")
+ << TErrorAttribute("path", path);
}
}
void TSslContext::AddCertificateChainFromFile(const TString& path)
{
- if (SSL_CTX_use_certificate_chain_file(Impl_->Context, path.c_str()) != 1) {
- THROW_ERROR_EXCEPTION("SSL_CTX_use_certificate_chain_file failed")
- << TErrorAttribute("path", path)
- << GetSslErrors();
+ if (SSL_CTX_use_certificate_chain_file(Impl_->GetContext(), path.c_str()) != 1) {
+ THROW_ERROR GetLastSslError("SSL_CTX_use_certificate_chain_file failed")
+ << TErrorAttribute("path", path);
}
}
void TSslContext::AddPrivateKeyFromFile(const TString& path)
{
- if (SSL_CTX_use_PrivateKey_file(Impl_->Context, path.c_str(), SSL_FILETYPE_PEM) != 1) {
- THROW_ERROR_EXCEPTION("SSL_CTX_use_PrivateKey_file failed")
- << TErrorAttribute("path", path)
- << GetSslErrors();
+ if (SSL_CTX_use_PrivateKey_file(Impl_->GetContext(), path.c_str(), SSL_FILETYPE_PEM) != 1) {
+ THROW_ERROR GetLastSslError("SSL_CTX_use_PrivateKey_file failed")
+ << TErrorAttribute("path", path);
}
}
void TSslContext::AddCertificateChain(const TString& certificateChain)
{
- auto bio = BIO_new_mem_buf(certificateChain.c_str(), certificateChain.size());
- YT_VERIFY(bio);
- auto freeBio = Finally([&] {
- BIO_free(bio);
- });
+ TBioPtr bio(BIO_new_mem_buf(certificateChain.c_str(), certificateChain.size()));
+ if (!bio) {
+ THROW_ERROR GetLastSslError("Failed to allocate memory buffer for certificate chain");
+ }
- auto certificateObject = PEM_read_bio_X509_AUX(bio, nullptr, nullptr, nullptr);
+ TX509Ptr certificateObject(PEM_read_bio_X509_AUX(bio.get(), nullptr, nullptr, nullptr));
if (!certificateObject) {
- THROW_ERROR_EXCEPTION("PEM_read_bio_X509_AUX failed")
- << GetSslErrors();
+ THROW_ERROR GetLastSslError("PEM_read_bio_X509_AUX failed");
}
- auto freeCertificate = Finally([&] {
- X509_free(certificateObject);
- });
- if (SSL_CTX_use_certificate(Impl_->Context, certificateObject) != 1) {
- THROW_ERROR_EXCEPTION("SSL_CTX_use_certificate failed")
- << GetSslErrors();
+ if (SSL_CTX_use_certificate(Impl_->GetContext(), certificateObject.get()) != 1) {
+ THROW_ERROR GetLastSslError("SSL_CTX_use_certificate failed");
}
- SSL_CTX_clear_chain_certs(Impl_->Context);
+ SSL_CTX_clear_chain_certs(Impl_->GetContext());
while (true) {
- auto chainCertificateObject = PEM_read_bio_X509(bio, nullptr, nullptr, nullptr);
+ TX509Ptr chainCertificateObject(PEM_read_bio_X509(bio.get(), nullptr, nullptr, nullptr));
if (!chainCertificateObject) {
int err = ERR_peek_last_error();
if (ERR_GET_LIB(err) == ERR_LIB_PEM && ERR_GET_REASON(err) == PEM_R_NO_START_LINE) {
@@ -741,65 +800,123 @@ void TSslContext::AddCertificateChain(const TString& certificateChain)
break;
}
- THROW_ERROR_EXCEPTION("PEM_read_bio_X509")
- << GetSslErrors();
+ THROW_ERROR GetLastSslError("PEM_read_bio_X509");
}
- int result = SSL_CTX_add0_chain_cert(Impl_->Context, chainCertificateObject);
- if (!result) {
- X509_free(chainCertificateObject);
- THROW_ERROR_EXCEPTION("SSL_CTX_add0_chain_cert")
- << GetSslErrors();
+ if (SSL_CTX_add0_chain_cert(Impl_->GetContext(), chainCertificateObject.get()) != 1) {
+ THROW_ERROR GetLastSslError("SSL_CTX_add0_chain_cert");
+ }
+
+ // Do not X509_free() if certificate was added by SSL_CTX_add0_chain_cert().
+ Y_UNUSED(chainCertificateObject.release());
+ }
+}
+
+void TSslContext::AddCertificateAuthority(const TString& ca)
+{
+ TBioPtr bio(BIO_new_mem_buf(ca.data(), ca.size()));
+ if (!bio) {
+ THROW_ERROR GetLastSslError("Failed to allocate memory buffer for CA certificate");
+ }
+
+ auto store = SSL_CTX_get_cert_store(Impl_->GetContext());
+
+ // Load certificate chain.
+ TX509Ptr cert;
+ while (cert.reset(PEM_read_bio_X509(bio.get(), nullptr, nullptr, nullptr)), cert) {
+ if (X509_STORE_add_cert(store, cert.get()) != 1) {
+ THROW_ERROR GetLastSslError("Failed to add cert to store");
}
}
}
void TSslContext::AddCertificate(const TString& certificate)
{
- auto bio = BIO_new_mem_buf(certificate.c_str(), certificate.size());
- YT_VERIFY(bio);
- auto freeBio = Finally([&] {
- BIO_free(bio);
- });
+ TBioPtr bio(BIO_new_mem_buf(certificate.c_str(), certificate.size()));
+ if (!bio) {
+ THROW_ERROR GetLastSslError("Failed to allocate memory buffer for certificate");
+ }
- auto certificateObject = PEM_read_bio_X509_AUX(bio, nullptr, nullptr, nullptr);
+ TX509Ptr certificateObject(PEM_read_bio_X509_AUX(bio.get(), nullptr, nullptr, nullptr));
if (!certificateObject) {
- THROW_ERROR_EXCEPTION("PEM_read_bio_X509_AUX")
- << GetSslErrors();
+ THROW_ERROR GetLastSslError("PEM_read_bio_X509_AUX");
}
- auto freeCertificate = Finally([&] {
- X509_free(certificateObject);
- });
- if (SSL_CTX_use_certificate(Impl_->Context, certificateObject) != 1) {
- THROW_ERROR_EXCEPTION("SSL_CTX_use_certificate failed")
- << GetSslErrors();
+ if (SSL_CTX_use_certificate(Impl_->GetContext(), certificateObject.get()) != 1) {
+ THROW_ERROR GetLastSslError("SSL_CTX_use_certificate failed");
}
}
void TSslContext::AddPrivateKey(const TString& privateKey)
{
- auto bio = BIO_new_mem_buf(privateKey.c_str(), privateKey.size());
- YT_VERIFY(bio);
- auto freeBio = Finally([&] {
- BIO_free(bio);
- });
+ TBioPtr bio(BIO_new_mem_buf(privateKey.c_str(), privateKey.size()));
+ if (!bio) {
+ THROW_ERROR GetLastSslError("Failed to allocate memory buffer for private key");
+ }
- auto privateKeyObject = PEM_read_bio_PrivateKey(bio, nullptr, nullptr, nullptr);
+ TEvpPKeyPtr privateKeyObject(PEM_read_bio_PrivateKey(bio.get(), nullptr, nullptr, nullptr));
if (!privateKeyObject) {
- THROW_ERROR_EXCEPTION("PEM_read_bio_PrivateKey failed")
- << GetSslErrors();
+ THROW_ERROR GetLastSslError("PEM_read_bio_PrivateKey failed");
}
- auto freePrivateKey = Finally([&] {
- EVP_PKEY_free(privateKeyObject);
- });
- if (SSL_CTX_use_PrivateKey(Impl_->Context, privateKeyObject) != 1) {
- THROW_ERROR_EXCEPTION("SSL_CTX_use_PrivateKey failed")
- << GetSslErrors();
+ if (SSL_CTX_use_PrivateKey(Impl_->GetContext(), privateKeyObject.get()) != 1) {
+ THROW_ERROR GetLastSslError("SSL_CTX_use_PrivateKey failed");
}
}
+void TSslContext::AddCertificateAuthority(const TPemBlobConfigPtr& pem, TCertificatePathResolver resolver)
+{
+ if (pem) {
+ if (pem->FileName) {
+ auto filePath = resolver ? resolver(*pem->FileName) : *pem->FileName;
+ AddCertificateAuthorityFromFile(filePath);
+ } else {
+ AddCertificateAuthority(pem->LoadBlob());
+ }
+ }
+}
+
+void TSslContext::AddCertificate(const TPemBlobConfigPtr& pem, TCertificatePathResolver resolver)
+{
+ if (pem) {
+ if (pem->FileName) {
+ auto filePath = resolver ? resolver(*pem->FileName) : *pem->FileName;
+ AddCertificateFromFile(filePath);
+ } else {
+ AddCertificate(pem->LoadBlob());
+ }
+ }
+}
+
+void TSslContext::AddCertificateChain(const TPemBlobConfigPtr& pem, TCertificatePathResolver resolver)
+{
+ if (pem) {
+ if (pem->FileName) {
+ auto filePath = resolver ? resolver(*pem->FileName) : *pem->FileName;
+ AddCertificateChainFromFile(filePath);
+ } else {
+ AddCertificateChain(pem->LoadBlob());
+ }
+ }
+}
+
+void TSslContext::AddPrivateKey(const TPemBlobConfigPtr& pem, TCertificatePathResolver resolver)
+{
+ if (pem) {
+ if (pem->FileName) {
+ auto filePath = resolver ? resolver(*pem->FileName) : *pem->FileName;
+ AddPrivateKeyFromFile(filePath);
+ } else {
+ AddPrivateKey(pem->LoadBlob());
+ }
+ }
+}
+
+TSslPtr TSslContext::NewSsl()
+{
+ return Impl_->NewSsl();
+}
+
IDialerPtr TSslContext::CreateDialer(
const TDialerConfigPtr& config,
const IPollerPtr& poller,
diff --git a/yt/yt/core/crypto/tls.h b/yt/yt/core/crypto/tls.h
index 7903cb35701..e40a41f2f3f 100644
--- a/yt/yt/core/crypto/tls.h
+++ b/yt/yt/core/crypto/tls.h
@@ -8,12 +8,37 @@
#include <yt/yt/core/concurrency/public.h>
+#include <openssl/ossl_typ.h>
+
namespace NYT::NCrypto {
////////////////////////////////////////////////////////////////////////////////
+TError GetLastSslError(TString message);
+
+////////////////////////////////////////////////////////////////////////////////
+
+struct TSslDeleter
+{
+ void operator() (BIO*) const noexcept;
+ void operator() (X509*) const noexcept;
+ void operator() (EVP_PKEY*) const noexcept;
+ void operator() (SSL_CTX*) const noexcept;
+ void operator() (SSL*) const noexcept;
+};
+
+using TBioPtr = std::unique_ptr<BIO, TSslDeleter>;
+using TX509Ptr = std::unique_ptr<X509, TSslDeleter>;
+using TEvpPKeyPtr = std::unique_ptr<EVP_PKEY, TSslDeleter>;
+using TSslCtxPtr = std::unique_ptr<SSL_CTX, TSslDeleter>;
+using TSslPtr = std::unique_ptr<SSL, TSslDeleter>;
+
+////////////////////////////////////////////////////////////////////////////////
+
DECLARE_REFCOUNTED_STRUCT(TSslContextImpl)
+using TCertificatePathResolver = std::function<TString(const TString&)>;
+
class TSslContext
: public TRefCounted
{
@@ -24,18 +49,29 @@ public:
void Commit(TInstant time = TInstant::Zero());
TInstant GetCommitTime() const;
+ void ApplyConfig(const TSslContextConfigPtr& config, TCertificatePathResolver pathResolver = nullptr);
+
void UseBuiltinOpenSslX509Store();
void SetCipherList(const TString& list);
+ void AddCertificateAuthorityFromFile(const TString& path);
void AddCertificateFromFile(const TString& path);
void AddCertificateChainFromFile(const TString& path);
void AddPrivateKeyFromFile(const TString& path);
+ void AddCertificateAuthority(const TString& ca);
void AddCertificate(const TString& certificate);
void AddCertificateChain(const TString& certificateChain);
void AddPrivateKey(const TString& privateKey);
+ void AddCertificateAuthority(const TPemBlobConfigPtr& pem, TCertificatePathResolver resolver = nullptr);
+ void AddCertificate(const TPemBlobConfigPtr& pem, TCertificatePathResolver resolver = nullptr);
+ void AddCertificateChain(const TPemBlobConfigPtr& pem, TCertificatePathResolver resolver = nullptr);
+ void AddPrivateKey(const TPemBlobConfigPtr& pem, TCertificatePathResolver resolver = nullptr);
+
+ TSslPtr NewSsl();
+
NNet::IDialerPtr CreateDialer(
const NNet::TDialerConfigPtr& config,
const NConcurrency::IPollerPtr& poller,
diff --git a/yt/yt/core/crypto/unittests/tls_ut.cpp b/yt/yt/core/crypto/unittests/tls_ut.cpp
index e756b2c8084..6281d96b439 100644
--- a/yt/yt/core/crypto/unittests/tls_ut.cpp
+++ b/yt/yt/core/crypto/unittests/tls_ut.cpp
@@ -57,6 +57,11 @@ TEST_F(TTlsTest, CreateContext)
// Not that trivial as it seems!
}
+TEST_F(TTlsTest, CreateSslObject)
+{
+ auto ssl = Context->NewSsl();
+}
+
TEST_F(TTlsTest, CreateListener)
{
auto localhost = TNetworkAddress::CreateIPv6Loopback(0);
diff --git a/yt/yt/core/http/http.cpp b/yt/yt/core/http/http.cpp
index 702ebefef65..376b7d91e4f 100644
--- a/yt/yt/core/http/http.cpp
+++ b/yt/yt/core/http/http.cpp
@@ -1,6 +1,6 @@
#include "http.h"
-#include <contrib/restricted/http-parser/http_parser.h>
+#include <contrib/deprecated/http-parser/http_parser.h>
namespace NYT::NHttp {
diff --git a/yt/yt/core/http/stream.h b/yt/yt/core/http/stream.h
index a45ab15f0ff..00236665bed 100644
--- a/yt/yt/core/http/stream.h
+++ b/yt/yt/core/http/stream.h
@@ -7,7 +7,7 @@
#include <yt/yt/core/net/connection.h>
#include <yt/yt/core/net/address.h>
-#include <contrib/restricted/http-parser/http_parser.h>
+#include <contrib/deprecated/http-parser/http_parser.h>
#include <util/stream/buffer.h>
diff --git a/yt/yt/core/http/unittests/http_ut.cpp b/yt/yt/core/http/unittests/http_ut.cpp
index 1ff0db8034a..0dea66e0419 100644
--- a/yt/yt/core/http/unittests/http_ut.cpp
+++ b/yt/yt/core/http/unittests/http_ut.cpp
@@ -641,8 +641,8 @@ private:
serverConfig->Credentials = New<NHttps::TServerCredentialsConfig>();
serverConfig->Credentials->PrivateKey = New<TPemBlobConfig>();
serverConfig->Credentials->PrivateKey->Value = TestCertificate;
- serverConfig->Credentials->CertChain = New<TPemBlobConfig>();
- serverConfig->Credentials->CertChain->Value = TestCertificate;
+ serverConfig->Credentials->CertificateChain = New<TPemBlobConfig>();
+ serverConfig->Credentials->CertificateChain->Value = TestCertificate;
SetupServer(serverConfig);
ServerConfig = serverConfig;
Server = NHttps::CreateServer(serverConfig, Poller);
@@ -651,8 +651,8 @@ private:
clientConfig->Credentials = New<NHttps::TClientCredentialsConfig>();
clientConfig->Credentials->PrivateKey = New<TPemBlobConfig>();
clientConfig->Credentials->PrivateKey->Value = TestCertificate;
- clientConfig->Credentials->CertChain = New<TPemBlobConfig>();
- clientConfig->Credentials->CertChain->Value = TestCertificate;
+ clientConfig->Credentials->CertificateChain = New<TPemBlobConfig>();
+ clientConfig->Credentials->CertificateChain->Value = TestCertificate;
SetupClient(clientConfig);
Client = NHttps::CreateClient(clientConfig, Poller);
}
diff --git a/yt/yt/core/http/ya.make b/yt/yt/core/http/ya.make
index 4ca1860a54b..53376347555 100644
--- a/yt/yt/core/http/ya.make
+++ b/yt/yt/core/http/ya.make
@@ -27,7 +27,7 @@ ENDIF()
PEERDIR(
yt/yt/core
- contrib/restricted/http-parser
+ contrib/deprecated/http-parser
)
END()
diff --git a/yt/yt/core/https/client.cpp b/yt/yt/core/https/client.cpp
index 7d5fbc70e57..4c9acaee9c5 100644
--- a/yt/yt/core/https/client.cpp
+++ b/yt/yt/core/https/client.cpp
@@ -108,24 +108,7 @@ IClientPtr CreateClient(
{
auto sslContext = New<TSslContext>();
if (config->Credentials) {
- if (config->Credentials->CertChain) {
- if (config->Credentials->CertChain->FileName) {
- sslContext->AddCertificateChainFromFile(*config->Credentials->CertChain->FileName);
- } else if (config->Credentials->CertChain->Value) {
- sslContext->AddCertificateChain(*config->Credentials->CertChain->Value);
- } else {
- THROW_ERROR_EXCEPTION("Neither \"file_name\" nor \"value\" is given for client certificate chain");
- }
- }
- if (config->Credentials->PrivateKey) {
- if (config->Credentials->PrivateKey->FileName) {
- sslContext->AddPrivateKeyFromFile(*config->Credentials->PrivateKey->FileName);
- } else if (config->Credentials->PrivateKey->Value) {
- sslContext->AddPrivateKey(*config->Credentials->PrivateKey->Value);
- } else {
- THROW_ERROR_EXCEPTION("Neither \"file_name\" nor \"value\" is given for client private key");
- }
- }
+ sslContext->ApplyConfig(config->Credentials);
} else {
sslContext->UseBuiltinOpenSslX509Store();
}
diff --git a/yt/yt/core/https/config.cpp b/yt/yt/core/https/config.cpp
index c41fde96241..dcc6c4bca15 100644
--- a/yt/yt/core/https/config.cpp
+++ b/yt/yt/core/https/config.cpp
@@ -6,10 +6,6 @@ namespace NYT::NHttps {
void TServerCredentialsConfig::Register(TRegistrar registrar)
{
- registrar.Parameter("private_key", &TThis::PrivateKey)
- .Optional();
- registrar.Parameter("cert_chain", &TThis::CertChain)
- .Optional();
registrar.Parameter("update_period", &TThis::UpdatePeriod)
.Optional();
}
@@ -23,12 +19,8 @@ void TServerConfig::Register(TRegistrar registrar)
////////////////////////////////////////////////////////////////////////////////
-void TClientCredentialsConfig::Register(TRegistrar registrar)
+void TClientCredentialsConfig::Register(TRegistrar /*registrar*/)
{
- registrar.Parameter("private_key", &TThis::PrivateKey)
- .Optional();
- registrar.Parameter("cert_chain", &TThis::CertChain)
- .Optional();
}
////////////////////////////////////////////////////////////////////////////////
diff --git a/yt/yt/core/https/config.h b/yt/yt/core/https/config.h
index ad740cc0fe7..8d9c381c678 100644
--- a/yt/yt/core/https/config.h
+++ b/yt/yt/core/https/config.h
@@ -11,10 +11,8 @@ namespace NYT::NHttps {
////////////////////////////////////////////////////////////////////////////////
struct TServerCredentialsConfig
- : public NYTree::TYsonStruct
+ : public NCrypto::TSslContextConfig
{
- NCrypto::TPemBlobConfigPtr PrivateKey;
- NCrypto::TPemBlobConfigPtr CertChain;
TDuration UpdatePeriod;
REGISTER_YSON_STRUCT(TServerCredentialsConfig);
@@ -41,11 +39,8 @@ DEFINE_REFCOUNTED_TYPE(TServerConfig)
////////////////////////////////////////////////////////////////////////////////
struct TClientCredentialsConfig
- : public NYTree::TYsonStruct
+ : public NCrypto::TSslContextConfig
{
- NCrypto::TPemBlobConfigPtr PrivateKey;
- NCrypto::TPemBlobConfigPtr CertChain;
-
REGISTER_YSON_STRUCT(TClientCredentialsConfig);
static void Register(TRegistrar registrar);
diff --git a/yt/yt/core/https/server.cpp b/yt/yt/core/https/server.cpp
index 6c6f997cd4c..f8ef027dc38 100644
--- a/yt/yt/core/https/server.cpp
+++ b/yt/yt/core/https/server.cpp
@@ -92,20 +92,7 @@ private:
static void ApplySslConfig(const TSslContextPtr& sslContext, const TServerCredentialsConfigPtr& sslConfig)
{
- if (sslConfig->CertChain->FileName) {
- sslContext->AddCertificateChainFromFile(*sslConfig->CertChain->FileName);
- } else if (sslConfig->CertChain->Value) {
- sslContext->AddCertificateChain(*sslConfig->CertChain->Value);
- } else {
- YT_ABORT();
- }
- if (sslConfig->PrivateKey->FileName) {
- sslContext->AddPrivateKeyFromFile(*sslConfig->PrivateKey->FileName);
- } else if (sslConfig->PrivateKey->Value) {
- sslContext->AddPrivateKey(*sslConfig->PrivateKey->Value);
- } else {
- YT_ABORT();
- }
+ sslContext->ApplyConfig(sslConfig);
}
IServerPtr CreateServer(
@@ -121,7 +108,7 @@ IServerPtr CreateServer(
auto sslConfig = config->Credentials;
TPeriodicExecutorPtr certificateUpdater;
if (sslConfig->UpdatePeriod &&
- sslConfig->CertChain->FileName &&
+ sslConfig->CertificateChain->FileName &&
sslConfig->PrivateKey->FileName)
{
YT_VERIFY(controlInvoker);
@@ -130,7 +117,7 @@ IServerPtr CreateServer(
BIND([=] {
try {
auto modificationTime = Max(
- NFS::GetPathStatistics(*sslConfig->CertChain->FileName).ModificationTime,
+ NFS::GetPathStatistics(*sslConfig->CertificateChain->FileName).ModificationTime,
NFS::GetPathStatistics(*sslConfig->PrivateKey->FileName).ModificationTime);
// Detect fresh and stable updates.
diff --git a/yt/yt/core/json/json_parser.cpp b/yt/yt/core/json/json_parser.cpp
index c8bfcf77b46..269dd0eaa8d 100644
--- a/yt/yt/core/json/json_parser.cpp
+++ b/yt/yt/core/json/json_parser.cpp
@@ -6,7 +6,7 @@
#include <array>
-#include <contrib/libs/yajl/src/api/yajl_parse.h>
+#include <contrib/deprecated/yajl/src/api/yajl_parse.h>
namespace NYT::NJson {
diff --git a/yt/yt/core/json/json_writer.cpp b/yt/yt/core/json/json_writer.cpp
index d8b1118044d..3986f8f8175 100644
--- a/yt/yt/core/json/json_writer.cpp
+++ b/yt/yt/core/json/json_writer.cpp
@@ -4,7 +4,7 @@
#include <yt/yt/core/misc/utf8_decoder.h>
-#include <contrib/libs/yajl/src/api/yajl_gen.h>
+#include <contrib/deprecated/yajl/src/api/yajl_gen.h>
#include <cmath>
diff --git a/yt/yt/core/misc/error.cpp b/yt/yt/core/misc/error.cpp
index 7d20101a8ff..9fc1a7fa150 100644
--- a/yt/yt/core/misc/error.cpp
+++ b/yt/yt/core/misc/error.cpp
@@ -757,13 +757,13 @@ void TErrorCodicils::Set(std::string key, TGetter getter)
if (getter) {
Getters_.insert_or_assign(std::move(key), std::move(getter));
} else {
- Getters_.erase(std::move(key));
+ Getters_.erase(key);
}
}
auto TErrorCodicils::Get(const std::string& key) const -> TGetter
{
- return GetOrDefault(Getters_, std::move(key));
+ return GetOrDefault(Getters_, key);
}
////////////////////////////////////////////////////////////////////////////////
diff --git a/yt/yt/core/net/config.h b/yt/yt/core/net/config.h
index eb65aee470b..a6b596d0156 100644
--- a/yt/yt/core/net/config.h
+++ b/yt/yt/core/net/config.h
@@ -13,7 +13,7 @@ namespace NYT::NNet {
////////////////////////////////////////////////////////////////////////////////
struct TDialerConfig
- : public NYTree::TYsonStruct
+ : public virtual NYTree::TYsonStruct
{
bool EnableNoDelay;
bool EnableAggressiveReconnect;
diff --git a/yt/yt/core/rpc/unittests/lib/common.h b/yt/yt/core/rpc/unittests/lib/common.h
index cb3b0ab6b06..050159f078a 100644
--- a/yt/yt/core/rpc/unittests/lib/common.h
+++ b/yt/yt/core/rpc/unittests/lib/common.h
@@ -501,8 +501,8 @@ public:
auto credentials = New<NHttps::TClientCredentialsConfig>();
credentials->PrivateKey = New<NCrypto::TPemBlobConfig>();
credentials->PrivateKey->Value = ClientKey;
- credentials->CertChain = New<NCrypto::TPemBlobConfig>();
- credentials->CertChain->Value = ClientCert;
+ credentials->CertificateChain = New<NCrypto::TPemBlobConfig>();
+ credentials->CertificateChain->Value = ClientCert;
return NHttp::CreateHttpChannel(address, poller, EnableSsl, credentials);
}
@@ -521,8 +521,8 @@ public:
config->Credentials = New<NHttps::TServerCredentialsConfig>();
config->Credentials->PrivateKey = New<NCrypto::TPemBlobConfig>();
config->Credentials->PrivateKey->Value = ServerKey;
- config->Credentials->CertChain = New<NCrypto::TPemBlobConfig>();
- config->Credentials->CertChain->Value = ServerCert;
+ config->Credentials->CertificateChain = New<NCrypto::TPemBlobConfig>();
+ config->Credentials->CertificateChain->Value = ServerCert;
httpServer = NYT::NHttps::CreateServer(config, 4);
} else {
httpServer = NYT::NHttp::CreateServer(config, 4);
diff --git a/yt/yt/core/ya.make b/yt/yt/core/ya.make
index 88951dd698a..9d46e7f6970 100644
--- a/yt/yt/core/ya.make
+++ b/yt/yt/core/ya.make
@@ -32,8 +32,6 @@ SRCS(
bus/tcp/client.cpp
bus/tcp/local_bypass.cpp
bus/tcp/server.cpp
- bus/tcp/ssl_context.cpp
- bus/tcp/ssl_helpers.cpp
compression/brotli.cpp
compression/bzip2.cpp
@@ -333,7 +331,7 @@ PEERDIR(
contrib/libs/libbz2
contrib/libs/c-ares
contrib/libs/farmhash
- contrib/libs/yajl
+ contrib/deprecated/yajl
contrib/libs/lz4
contrib/libs/openssl
diff --git a/yt/yt_proto/yt/client/api/rpc_proxy/proto/api_service.proto b/yt/yt_proto/yt/client/api/rpc_proxy/proto/api_service.proto
index fe5cd1da967..b7115ac5438 100644
--- a/yt/yt_proto/yt/client/api/rpc_proxy/proto/api_service.proto
+++ b/yt/yt_proto/yt/client/api/rpc_proxy/proto/api_service.proto
@@ -443,6 +443,7 @@ message TReqCommitTransaction
required NYT.NProto.TGuid transaction_id = 1;
reserved 6;
repeated NYT.NProto.TGuid additional_participant_cell_ids = 7;
+ optional uint64 max_allowed_commit_timestamp = 8;
optional TPrerequisiteOptions prerequisite_options = 101;
optional TMutatingOptions mutating_options = 103;