diff options
author | Alexander Smirnov <alex@ydb.tech> | 2025-05-31 13:17:09 +0000 |
---|---|---|
committer | Alexander Smirnov <alex@ydb.tech> | 2025-05-31 13:17:09 +0000 |
commit | 2b00537e14e6dc51140ea1abed802b2949530d4c (patch) | |
tree | 9da3dba2543e13fda51d15519c0b22a2f9ce570d | |
parent | b545ee9d9fff3931d4a2437f607d31bdae792335 (diff) | |
parent | 96dc38494ea248471e034b276c574cda7c8f1c5e (diff) | |
download | ydb-2b00537e14e6dc51140ea1abed802b2949530d4c.tar.gz |
Merge pull request #19096 from ydb-platform/merge-libs-250530-2116
114 files changed, 1653 insertions, 1360 deletions
diff --git a/.github/config/muted_ya.txt b/.github/config/muted_ya.txt index 54e169b6d43..2d5a044b17a 100644 --- a/.github/config/muted_ya.txt +++ b/.github/config/muted_ya.txt @@ -60,6 +60,7 @@ ydb/library/actors/interconnect/ut_huge_cluster HugeCluster.AllToAll ydb/library/actors/interconnect/ut_huge_cluster sole chunk chunk ydb/library/yaml_config/ut_transform test_transform.py.TestYamlConfigTransformations.test_basic[args1-dump_ds_init] ydb/library/yql/dq/opt/ut sole chunk chunk +ydb/library/yql/dq/opt/ut InterestingOrderingsShuffle.Join64ChainImitation ydb/public/sdk/cpp/src/client/topic/ut TxUsage.Sinks_Oltp_WriteToTopicAndTable_6_Query ydb/public/sdk/cpp/src/client/topic/ut TxUsage.Sinks_Oltp_WriteToTopicAndTable_6_Table ydb/public/sdk/cpp/src/client/topic/ut TxUsage.Sinks_Oltp_WriteToTopic_3_Query diff --git a/build/conf/go.conf b/build/conf/go.conf index cc6770ef611..317e5bbbc05 100644 --- a/build/conf/go.conf +++ b/build/conf/go.conf @@ -69,9 +69,6 @@ GOSTD_VERSION=1.23 when ($GOSTD_VERSION == "1.23") { GOSTD=contrib/go/_std_1.23/src } -elsewhen ($GOSTD_VERSION == "1.22") { - GOSTD=contrib/go/_std_1.22/src -} otherwise { GOSTD=__unsupported_go_std_library_version_[$GOSTD_VERSION]__ } diff --git a/build/conf/java.conf b/build/conf/java.conf index 23619f8cd5a..b96a5a483da 100644 --- a/build/conf/java.conf +++ b/build/conf/java.conf @@ -1099,7 +1099,7 @@ module JAR_LIBRARY: _COMPILABLE_JAR_BASE { when ($WITH_KAPT_VALUE == "yes") { # For Kapt usage see: https://kotlinlang.org/docs/kapt.html#using-in-cli # See for kapt.kotlin.generated: https://github.com/JetBrains/kotlin/blob/master/plugins/kapt3/kapt3-cli/testData/integration/kotlinFileGeneration/build.txt - _KAPT_OPTS=-Xplugin=${tool:"contrib/java/org/jetbrains/kotlin/kotlin-annotation-processing/2.1.20"} $KT_KAPT_PLUGIN_OPTS + _KAPT_OPTS=-Xplugin=${tool:"contrib/java/org/jetbrains/kotlin/kotlin-annotation-processing/2.1.21"} $KT_KAPT_PLUGIN_OPTS _RUN_KAPT=${YMAKE_PYTHON} ${input:"build/scripts/with_kapt_args.py"} --ap-classpath ${KT_KAPT_AP_CLASSPATH} -- $COMPILE_KT $_KAPT_OPTS _APPEND_KAPT_GENERATED_SRCS=$YMAKE_PYTHON3 ${input:"build/scripts/resolve_java_srcs.py"} -d $KT_KAPT_SOURCES_DIR --include-patterns '**/*.java' '**/*.kt' --resolve-kotlin --append -s ${BINDIR}/all-java.srclst -k $KT_SRCLIST -r ${BINDIR}/not-used.txt @@ -2087,9 +2087,9 @@ macro WITH_KOTLINC_DETEKT(Options...) { # tag:kotlin-specific ### Also search for _KAPT_OPTS and change version there ### change supported jvmTarget in https://a.yandex-team.ru/arcadia/devtools/ya/jbuild/gen/actions/idea.py?rev=r16258919#L668 -_KOTLIN_VERSION=2.1.20 -KOTLIN_VERSION=2.1.20 -KOTLIN_BOM_FILE=${ARCADIA_ROOT}/contrib/java/org/jetbrains/kotlin/kotlin-bom/2.1.20/ya.dependency_management.inc +_KOTLIN_VERSION=2.1.21 +KOTLIN_VERSION=2.1.21 +KOTLIN_BOM_FILE=${ARCADIA_ROOT}/contrib/java/org/jetbrains/kotlin/kotlin-bom/2.1.21/ya.dependency_management.inc DETEKT_VERSION=1.23.7 _KOTLIN_SEM= \ diff --git a/build/external_resources/go_tools/ya.make b/build/external_resources/go_tools/ya.make index a962a6fe9e8..5b16d56efa0 100644 --- a/build/external_resources/go_tools/ya.make +++ b/build/external_resources/go_tools/ya.make @@ -2,10 +2,7 @@ RESOURCES_LIBRARY() TOOLCHAIN(go) -IF(GOSTD_VERSION == 1.22) - VERSION(1.22) - DECLARE_EXTERNAL_HOST_RESOURCES_BUNDLE_BY_JSON(GO_TOOLS go1.22.json) -ELSEIF(GOSTD_VERSION == 1.23) +IF(GOSTD_VERSION == 1.23) VERSION(1.23) DECLARE_EXTERNAL_HOST_RESOURCES_BUNDLE_BY_JSON(GO_TOOLS go1.23.json) ELSE() diff --git a/build/mapping.conf.json b/build/mapping.conf.json index ad8dce755c1..a3645ed4ddf 100644 --- a/build/mapping.conf.json +++ b/build/mapping.conf.json @@ -70,18 +70,24 @@ "7245028547": "{registry_endpoint}/7245028547", "8314755214": "{registry_endpoint}/8314755214", "8314760034": "{registry_endpoint}/8314760034", + "8704092379": "{registry_endpoint}/8704092379", + "8704092450": "{registry_endpoint}/8704092450", "6638916300": "{registry_endpoint}/6638916300", "6638917016": "{registry_endpoint}/6638917016", "7245028516": "{registry_endpoint}/7245028516", "7245028641": "{registry_endpoint}/7245028641", "8314760058": "{registry_endpoint}/8314760058", "8314762661": "{registry_endpoint}/8314762661", + "8704093001": "{registry_endpoint}/8704093001", + "8704092368": "{registry_endpoint}/8704092368", "6638915855": "{registry_endpoint}/6638915855", "6638914429": "{registry_endpoint}/6638914429", "7245029625": "{registry_endpoint}/7245029625", "7245029630": "{registry_endpoint}/7245029630", "8314758936": "{registry_endpoint}/8314758936", "8314762236": "{registry_endpoint}/8314762236", + "8704093021": "{registry_endpoint}/8704093021", + "8704092521": "{registry_endpoint}/8704092521", "721500304": "{registry_endpoint}/721500304", "3573990573": "{registry_endpoint}/3573990573", "3573996018": "{registry_endpoint}/3573996018", @@ -308,8 +314,12 @@ "7709336951": "{registry_endpoint}/7709336951", "7709336970": "{registry_endpoint}/7709336970", "7709337139": "{registry_endpoint}/7709337139", + "8825002146": "{registry_endpoint}/8825002146", + "8825002149": "{registry_endpoint}/8825002149", + "8825002315": "{registry_endpoint}/8825002315", "6576113106": "{registry_endpoint}/6576113106", "7709337153": "{registry_endpoint}/7709337153", + "8825002211": "{registry_endpoint}/8825002211", "6148089711": "{registry_endpoint}/6148089711", "6512097221": "{registry_endpoint}/6512097221", "6812263131": "{registry_endpoint}/6812263131", @@ -1139,6 +1149,7 @@ "6639202855": "{registry_endpoint}/6639202855", "7249973735": "{registry_endpoint}/7249973735", "8314768876": "{registry_endpoint}/8314768876", + "8705239478": "{registry_endpoint}/8705239478", "4758626187": "{registry_endpoint}/4758626187", "4758626560": "{registry_endpoint}/4758626560", "4758626176": "{registry_endpoint}/4758626176", @@ -1508,18 +1519,24 @@ "7245028547": "JAVA_LIBRARY-none-none-kotlin-stdlib-2.0.21.jar", "8314755214": "JAVA_LIBRARY-none-none-kotlin-stdlib-2.1.20-sources.jar", "8314760034": "JAVA_LIBRARY-none-none-kotlin-stdlib-2.1.20.jar", + "8704092379": "JAVA_LIBRARY-none-none-kotlin-stdlib-2.1.21-sources.jar", + "8704092450": "JAVA_LIBRARY-none-none-kotlin-stdlib-2.1.21.jar", "6638916300": "JAVA_LIBRARY-none-none-kotlin-stdlib-jdk7-1.9.24-sources.jar", "6638917016": "JAVA_LIBRARY-none-none-kotlin-stdlib-jdk7-1.9.24.jar", "7245028516": "JAVA_LIBRARY-none-none-kotlin-stdlib-jdk7-2.0.21-sources.jar", "7245028641": "JAVA_LIBRARY-none-none-kotlin-stdlib-jdk7-2.0.21.jar", "8314760058": "JAVA_LIBRARY-none-none-kotlin-stdlib-jdk7-2.1.20-sources.jar", "8314762661": "JAVA_LIBRARY-none-none-kotlin-stdlib-jdk7-2.1.20.jar", + "8704093001": "JAVA_LIBRARY-none-none-kotlin-stdlib-jdk7-2.1.21-sources.jar", + "8704092368": "JAVA_LIBRARY-none-none-kotlin-stdlib-jdk7-2.1.21.jar", "6638915855": "JAVA_LIBRARY-none-none-kotlin-stdlib-jdk8-1.9.24-sources.jar", "6638914429": "JAVA_LIBRARY-none-none-kotlin-stdlib-jdk8-1.9.24.jar", "7245029625": "JAVA_LIBRARY-none-none-kotlin-stdlib-jdk8-2.0.21-sources.jar", "7245029630": "JAVA_LIBRARY-none-none-kotlin-stdlib-jdk8-2.0.21.jar", "8314758936": "JAVA_LIBRARY-none-none-kotlin-stdlib-jdk8-2.1.20-sources.jar", "8314762236": "JAVA_LIBRARY-none-none-kotlin-stdlib-jdk8-2.1.20.jar", + "8704093021": "JAVA_LIBRARY-none-none-kotlin-stdlib-jdk8-2.1.21-sources.jar", + "8704092521": "JAVA_LIBRARY-none-none-kotlin-stdlib-jdk8-2.1.21.jar", "721500304": "JAVA_LIBRARY-none-none-listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar", "3573990573": "JAVA_LIBRARY-none-none-proto-google-common-protos-2.9.0-sources.jar", "3573996018": "JAVA_LIBRARY-none-none-proto-google-common-protos-2.9.0.jar", @@ -1746,8 +1763,12 @@ "7709336951": "devtools/ktlint_build/package.json", "7709336970": "devtools/ktlint_build/package.json", "7709337139": "devtools/ktlint_build/package.json", + "8825002146": "devtools/ktlint_build/package.json", + "8825002149": "devtools/ktlint_build/package.json", + "8825002315": "devtools/ktlint_build/package.json", "6576113106": "devtools/ktlint_build/package_win.json", "7709337153": "devtools/ktlint_build/package_win.json", + "8825002211": "devtools/ktlint_build/package_win.json", "6148089711": "devtools/local_cache/toolscache/server/ya-tc for darwin", "6512097221": "devtools/local_cache/toolscache/server/ya-tc for darwin", "6812263131": "devtools/local_cache/toolscache/server/ya-tc for darwin", @@ -2577,6 +2598,7 @@ "6639202855": "kotlin_kompiler_1.9.24 with plugins", "7249973735": "kotlin_kompiler_2.0.21 with plugins", "8314768876": "kotlin_kompiler_2.1.20 with plugins", + "8705239478": "kotlin_kompiler_2.1.21 with plugins", "4758626187": "ktlint for darwin (0.50.0)", "4758626560": "ktlint for darwin-arm64 (0.50.0)", "4758626176": "ktlint for linux (0.50.0)", diff --git a/build/ymake.core.conf b/build/ymake.core.conf index c36714a94cb..cb17f630479 100644 --- a/build/ymake.core.conf +++ b/build/ymake.core.conf @@ -14,7 +14,7 @@ GO_FAKEID=11100371 ANDROID_FAKEID=2023-05-17 CLANG_TIDY_FAKEID=2023-06-06 CYTHON_FAKEID=16618405 -JAVA_FAKEID=14386852 +JAVA_FAKEID=8908076 PROTO_FAKEID=0 FBS_FAKEID=2024-03-13 diff --git a/contrib/restricted/http-parser/.yandex_meta/__init__.py b/contrib/deprecated/http-parser/.yandex_meta/__init__.py index 9947099a0d3..6bb05a2a85f 100644 --- a/contrib/restricted/http-parser/.yandex_meta/__init__.py +++ b/contrib/deprecated/http-parser/.yandex_meta/__init__.py @@ -3,7 +3,7 @@ from devtools.yamaker.project import NixProject http_parser = NixProject( owners=["g:mds", "g:cpp-contrib"], - arcdir="contrib/restricted/http-parser", + arcdir="contrib/deprecated/http-parser", nixattr="http-parser", # By default maximium header size allowed is 80Kb. To remove the effective limit # on the size of the header, we define the macro to a very large number (0x7fffffff). diff --git a/contrib/restricted/http-parser/.yandex_meta/devtools.copyrights.report b/contrib/deprecated/http-parser/.yandex_meta/devtools.copyrights.report index 3c9e1fcf0b0..3c9e1fcf0b0 100644 --- a/contrib/restricted/http-parser/.yandex_meta/devtools.copyrights.report +++ b/contrib/deprecated/http-parser/.yandex_meta/devtools.copyrights.report diff --git a/contrib/restricted/http-parser/.yandex_meta/devtools.licenses.report b/contrib/deprecated/http-parser/.yandex_meta/devtools.licenses.report index 7911cf6e365..7911cf6e365 100644 --- a/contrib/restricted/http-parser/.yandex_meta/devtools.licenses.report +++ b/contrib/deprecated/http-parser/.yandex_meta/devtools.licenses.report diff --git a/contrib/restricted/http-parser/.yandex_meta/licenses.list.txt b/contrib/deprecated/http-parser/.yandex_meta/licenses.list.txt index e8deffa4a52..e8deffa4a52 100644 --- a/contrib/restricted/http-parser/.yandex_meta/licenses.list.txt +++ b/contrib/deprecated/http-parser/.yandex_meta/licenses.list.txt diff --git a/contrib/restricted/http-parser/.yandex_meta/override.nix b/contrib/deprecated/http-parser/.yandex_meta/override.nix index d09d27b7a6e..d09d27b7a6e 100644 --- a/contrib/restricted/http-parser/.yandex_meta/override.nix +++ b/contrib/deprecated/http-parser/.yandex_meta/override.nix diff --git a/contrib/restricted/http-parser/AUTHORS b/contrib/deprecated/http-parser/AUTHORS index 5323b685cae..5323b685cae 100644 --- a/contrib/restricted/http-parser/AUTHORS +++ b/contrib/deprecated/http-parser/AUTHORS diff --git a/contrib/restricted/http-parser/LICENSE-MIT b/contrib/deprecated/http-parser/LICENSE-MIT index 1ec0ab4e174..1ec0ab4e174 100644 --- a/contrib/restricted/http-parser/LICENSE-MIT +++ b/contrib/deprecated/http-parser/LICENSE-MIT diff --git a/contrib/restricted/http-parser/README.md b/contrib/deprecated/http-parser/README.md index b265d71715f..b265d71715f 100644 --- a/contrib/restricted/http-parser/README.md +++ b/contrib/deprecated/http-parser/README.md diff --git a/contrib/restricted/http-parser/http_parser.c b/contrib/deprecated/http-parser/http_parser.c index 726810366a1..726810366a1 100644 --- a/contrib/restricted/http-parser/http_parser.c +++ b/contrib/deprecated/http-parser/http_parser.c diff --git a/contrib/restricted/http-parser/http_parser.h b/contrib/deprecated/http-parser/http_parser.h index 262a0394c83..262a0394c83 100644 --- a/contrib/restricted/http-parser/http_parser.h +++ b/contrib/deprecated/http-parser/http_parser.h diff --git a/contrib/restricted/http-parser/patches/field-uint32.patch b/contrib/deprecated/http-parser/patches/field-uint32.patch index 1d952b3eedf..1d952b3eedf 100644 --- a/contrib/restricted/http-parser/patches/field-uint32.patch +++ b/contrib/deprecated/http-parser/patches/field-uint32.patch diff --git a/contrib/restricted/http-parser/patches/parser-flags.patch b/contrib/deprecated/http-parser/patches/parser-flags.patch index 71297c691bd..8552e93ef28 100644 --- a/contrib/restricted/http-parser/patches/parser-flags.patch +++ b/contrib/deprecated/http-parser/patches/parser-flags.patch @@ -1,7 +1,7 @@ -diff --git a/contrib/restricted/http-parser/http_parser.c b/contrib/restricted/http-parser/http_parser.c +diff --git a/contrib/deprecated/http-parser/http_parser.c b/contrib/deprecated/http-parser/http_parser.c index 95ff42f783d9..726810366a11 100644 ---- a/contrib/restricted/http-parser/http_parser.c -+++ b/contrib/restricted/http-parser/http_parser.c +--- a/contrib/deprecated/http-parser/http_parser.c ++++ b/contrib/deprecated/http-parser/http_parser.c @@ -1342,6 +1342,12 @@ size_t http_parser_execute (http_parser *parser, } else if (parser->index == sizeof(TRANSFER_ENCODING)-2) { parser->header_state = h_transfer_encoding; diff --git a/contrib/restricted/http-parser/ya.make b/contrib/deprecated/http-parser/ya.make index 134e593b695..9e83bba192d 100644 --- a/contrib/restricted/http-parser/ya.make +++ b/contrib/deprecated/http-parser/ya.make @@ -11,7 +11,7 @@ VERSION(2.9.4) ORIGINAL_SOURCE(https://github.com/nodejs/http-parser/archive/v2.9.4.tar.gz) ADDINCL( - contrib/restricted/http-parser + contrib/deprecated/http-parser ) NO_COMPILER_WARNINGS() diff --git a/contrib/libs/yajl/.yandex_meta/devtools.copyrights.report b/contrib/deprecated/yajl/.yandex_meta/devtools.copyrights.report index 312635e0f5f..312635e0f5f 100644 --- a/contrib/libs/yajl/.yandex_meta/devtools.copyrights.report +++ b/contrib/deprecated/yajl/.yandex_meta/devtools.copyrights.report diff --git a/contrib/libs/yajl/.yandex_meta/devtools.licenses.report b/contrib/deprecated/yajl/.yandex_meta/devtools.licenses.report index 37b6ae68ea9..37b6ae68ea9 100644 --- a/contrib/libs/yajl/.yandex_meta/devtools.licenses.report +++ b/contrib/deprecated/yajl/.yandex_meta/devtools.licenses.report diff --git a/contrib/libs/yajl/.yandex_meta/licenses.list.txt b/contrib/deprecated/yajl/.yandex_meta/licenses.list.txt index 9517d66999d..9517d66999d 100644 --- a/contrib/libs/yajl/.yandex_meta/licenses.list.txt +++ b/contrib/deprecated/yajl/.yandex_meta/licenses.list.txt diff --git a/contrib/libs/yajl/patches/fix_memory_leak.patch b/contrib/deprecated/yajl/patches/fix_memory_leak.patch index 0ac5f0f3914..5d1e5ea7803 100644 --- a/contrib/libs/yajl/patches/fix_memory_leak.patch +++ b/contrib/deprecated/yajl/patches/fix_memory_leak.patch @@ -1,5 +1,5 @@ ---- contrib/libs/yajl/yajl_tree.c (3b4a018b6b389390da3ee13f6b4ce0880cb71807) -+++ contrib/libs/yajl/yajl_tree.c (5fe0776c5f96630ddffb63e4ff0292037e12039e) +--- contrib/deprecated/yajl/yajl_tree.c (3b4a018b6b389390da3ee13f6b4ce0880cb71807) ++++ contrib/deprecated/yajl/yajl_tree.c (5fe0776c5f96630ddffb63e4ff0292037e12039e) @@ -143,7 +143,7 @@ static yajl_val context_pop(context_t *ctx) ctx->stack = stack->next; diff --git a/contrib/libs/yajl/patches/uint64_support_r767862_review16855.patch b/contrib/deprecated/yajl/patches/uint64_support_r767862_review16855.patch index 62da2521b0f..62da2521b0f 100644 --- a/contrib/libs/yajl/patches/uint64_support_r767862_review16855.patch +++ b/contrib/deprecated/yajl/patches/uint64_support_r767862_review16855.patch diff --git a/contrib/libs/yajl/src/api/yajl_common.h b/contrib/deprecated/yajl/src/api/yajl_common.h index 2507576a1a6..2507576a1a6 100644 --- a/contrib/libs/yajl/src/api/yajl_common.h +++ b/contrib/deprecated/yajl/src/api/yajl_common.h diff --git a/contrib/libs/yajl/src/api/yajl_gen.h b/contrib/deprecated/yajl/src/api/yajl_gen.h index fb1409df709..fb1409df709 100644 --- a/contrib/libs/yajl/src/api/yajl_gen.h +++ b/contrib/deprecated/yajl/src/api/yajl_gen.h diff --git a/contrib/libs/yajl/src/api/yajl_parse.h b/contrib/deprecated/yajl/src/api/yajl_parse.h index 36a89a52a41..36a89a52a41 100644 --- a/contrib/libs/yajl/src/api/yajl_parse.h +++ b/contrib/deprecated/yajl/src/api/yajl_parse.h diff --git a/contrib/libs/yajl/src/api/yajl_tree.h b/contrib/deprecated/yajl/src/api/yajl_tree.h index 2ec36fd576c..2ec36fd576c 100644 --- a/contrib/libs/yajl/src/api/yajl_tree.h +++ b/contrib/deprecated/yajl/src/api/yajl_tree.h diff --git a/contrib/libs/yajl/src/api/yajl_version.h b/contrib/deprecated/yajl/src/api/yajl_version.h index adc0ad82255..adc0ad82255 100644 --- a/contrib/libs/yajl/src/api/yajl_version.h +++ b/contrib/deprecated/yajl/src/api/yajl_version.h diff --git a/contrib/libs/yajl/src/yajl.c b/contrib/deprecated/yajl/src/yajl.c index 0a97f6ef001..0a97f6ef001 100644 --- a/contrib/libs/yajl/src/yajl.c +++ b/contrib/deprecated/yajl/src/yajl.c diff --git a/contrib/libs/yajl/src/yajl_alloc.c b/contrib/deprecated/yajl/src/yajl_alloc.c index 96ad1d33049..96ad1d33049 100644 --- a/contrib/libs/yajl/src/yajl_alloc.c +++ b/contrib/deprecated/yajl/src/yajl_alloc.c diff --git a/contrib/libs/yajl/src/yajl_alloc.h b/contrib/deprecated/yajl/src/yajl_alloc.h index 203c2f97bd9..203c2f97bd9 100644 --- a/contrib/libs/yajl/src/yajl_alloc.h +++ b/contrib/deprecated/yajl/src/yajl_alloc.h diff --git a/contrib/libs/yajl/src/yajl_api.h b/contrib/deprecated/yajl/src/yajl_api.h index f45fcdf8367..f45fcdf8367 100644 --- a/contrib/libs/yajl/src/yajl_api.h +++ b/contrib/deprecated/yajl/src/yajl_api.h diff --git a/contrib/libs/yajl/src/yajl_buf.c b/contrib/deprecated/yajl/src/yajl_buf.c index 2f055a6e6a3..2f055a6e6a3 100644 --- a/contrib/libs/yajl/src/yajl_buf.c +++ b/contrib/deprecated/yajl/src/yajl_buf.c diff --git a/contrib/libs/yajl/src/yajl_buf.h b/contrib/deprecated/yajl/src/yajl_buf.h index b855a08b8bc..b855a08b8bc 100644 --- a/contrib/libs/yajl/src/yajl_buf.h +++ b/contrib/deprecated/yajl/src/yajl_buf.h diff --git a/contrib/libs/yajl/src/yajl_bytestack.h b/contrib/deprecated/yajl/src/yajl_bytestack.h index 9ea7d151e3c..9ea7d151e3c 100644 --- a/contrib/libs/yajl/src/yajl_bytestack.h +++ b/contrib/deprecated/yajl/src/yajl_bytestack.h diff --git a/contrib/libs/yajl/src/yajl_encode.c b/contrib/deprecated/yajl/src/yajl_encode.c index 0d97cc5290c..0d97cc5290c 100644 --- a/contrib/libs/yajl/src/yajl_encode.c +++ b/contrib/deprecated/yajl/src/yajl_encode.c diff --git a/contrib/libs/yajl/src/yajl_encode.h b/contrib/deprecated/yajl/src/yajl_encode.h index 853a1a701ce..853a1a701ce 100644 --- a/contrib/libs/yajl/src/yajl_encode.h +++ b/contrib/deprecated/yajl/src/yajl_encode.h diff --git a/contrib/libs/yajl/src/yajl_gen.c b/contrib/deprecated/yajl/src/yajl_gen.c index 5b1dd93246e..5b1dd93246e 100644 --- a/contrib/libs/yajl/src/yajl_gen.c +++ b/contrib/deprecated/yajl/src/yajl_gen.c diff --git a/contrib/libs/yajl/src/yajl_lex.c b/contrib/deprecated/yajl/src/yajl_lex.c index 70555c5ea3c..70555c5ea3c 100644 --- a/contrib/libs/yajl/src/yajl_lex.c +++ b/contrib/deprecated/yajl/src/yajl_lex.c diff --git a/contrib/libs/yajl/src/yajl_lex.h b/contrib/deprecated/yajl/src/yajl_lex.h index 309c29c54c6..309c29c54c6 100644 --- a/contrib/libs/yajl/src/yajl_lex.h +++ b/contrib/deprecated/yajl/src/yajl_lex.h diff --git a/contrib/libs/yajl/src/yajl_parser.c b/contrib/deprecated/yajl/src/yajl_parser.c index d1758d67c9f..d1758d67c9f 100644 --- a/contrib/libs/yajl/src/yajl_parser.c +++ b/contrib/deprecated/yajl/src/yajl_parser.c diff --git a/contrib/libs/yajl/src/yajl_parser.h b/contrib/deprecated/yajl/src/yajl_parser.h index 6ecc0f3a5c4..6ecc0f3a5c4 100644 --- a/contrib/libs/yajl/src/yajl_parser.h +++ b/contrib/deprecated/yajl/src/yajl_parser.h diff --git a/contrib/libs/yajl/src/yajl_tree.c b/contrib/deprecated/yajl/src/yajl_tree.c index 6a9d9ffb1b5..6a9d9ffb1b5 100644 --- a/contrib/libs/yajl/src/yajl_tree.c +++ b/contrib/deprecated/yajl/src/yajl_tree.c diff --git a/contrib/libs/yajl/src/yajl_version.c b/contrib/deprecated/yajl/src/yajl_version.c index 618ac1b77ed..618ac1b77ed 100644 --- a/contrib/libs/yajl/src/yajl_version.c +++ b/contrib/deprecated/yajl/src/yajl_version.c diff --git a/contrib/libs/yajl/ya.make b/contrib/deprecated/yajl/ya.make index 3f8ce132621..3f8ce132621 100644 --- a/contrib/libs/yajl/ya.make +++ b/contrib/deprecated/yajl/ya.make diff --git a/contrib/libs/yajl/yajl_parser.cpp b/contrib/deprecated/yajl/yajl_parser.cpp index 8cd1ce373ae..8cd1ce373ae 100644 --- a/contrib/libs/yajl/yajl_parser.cpp +++ b/contrib/deprecated/yajl/yajl_parser.cpp diff --git a/library/cpp/threading/chunk_queue/queue.h b/library/cpp/threading/chunk_queue/queue.h index feee776b872..fdde719e503 100644 --- a/library/cpp/threading/chunk_queue/queue.h +++ b/library/cpp/threading/chunk_queue/queue.h @@ -4,13 +4,13 @@ #include <util/generic/noncopyable.h> #include <util/generic/ptr.h> #include <util/generic/typetraits.h> -#include <util/generic/vector.h> #include <util/generic/ylimits.h> #include <library/cpp/deprecated/atomic/atomic.h> #include <util/system/guard.h> #include <util/system/spinlock.h> #include <util/system/yassert.h> +#include <atomic> #include <type_traits> #include <utility> @@ -240,7 +240,7 @@ namespace NThreading { }; struct TQueueType: public TOneOneQueue<TEntry, ChunkSize> { - TAtomic WriteLock = 0; + TSpinLock WriteLock; using TOneOneQueue<TEntry, ChunkSize>::PrepareWrite; using TOneOneQueue<TEntry, ChunkSize>::CompleteWrite; @@ -250,11 +250,7 @@ namespace NThreading { }; private: - union { - TAtomic WriteTag = 0; - char Pad[PLATFORM_CACHE_LINE]; - }; - + TPadded<std::atomic<ui64>> WriteTag{1}; TQueueType Queues[Concurrency]; public: @@ -291,20 +287,20 @@ namespace NThreading { ui64 NextTag() { // TODO: can we avoid synchronization here? it costs 1.5x performance penalty // return GetCycleCount(); - return AtomicIncrement(WriteTag); + return WriteTag.fetch_add(1); } template <typename TT> bool TryEnqueue(TT&& value, ui64 tag) { for (size_t i = 0; i < Concurrency; ++i) { TQueueType& queue = Queues[i]; - if (AtomicTryAndTryLock(&queue.WriteLock)) { + if (!queue.WriteLock.IsLocked() && queue.WriteLock.TryAcquire()) { TEntry* entry = queue.PrepareWrite(); Y_ASSERT(entry); TTypeHelper::Write(&entry->Value, std::forward<TT>(value)); entry->Tag = tag; queue.CompleteWrite(); - AtomicUnlock(&queue.WriteLock); + queue.WriteLock.Release(); return true; } } @@ -383,7 +379,7 @@ namespace NThreading { template <typename T, size_t Concurrency = 4, size_t ChunkSize = PLATFORM_PAGE_SIZE> class TRelaxedManyOneQueue: private TNonCopyable { struct TQueueType: public TOneOneQueue<T, ChunkSize> { - TAtomic WriteLock = 0; + TSpinLock WriteLock; }; private: @@ -429,9 +425,9 @@ namespace NThreading { size_t writePos = GetCycleCount(); for (size_t i = 0; i < Concurrency; ++i) { TQueueType& queue = Queues[writePos++ % Concurrency]; - if (AtomicTryAndTryLock(&queue.WriteLock)) { + if (!queue.WriteLock.IsLocked() && queue.WriteLock.TryAcquire()) { queue.Enqueue(std::forward<TT>(value)); - AtomicUnlock(&queue.WriteLock); + queue.WriteLock.Release(); return true; } } @@ -446,14 +442,8 @@ namespace NThreading { template <typename T, size_t Concurrency = 4, size_t ChunkSize = PLATFORM_PAGE_SIZE> class TRelaxedManyManyQueue: private TNonCopyable { struct TQueueType: public TOneOneQueue<T, ChunkSize> { - union { - TAtomic WriteLock = 0; - char Pad1[PLATFORM_CACHE_LINE]; - }; - union { - TAtomic ReadLock = 0; - char Pad2[PLATFORM_CACHE_LINE]; - }; + TPadded<TSpinLock> WriteLock; + TPadded<TSpinLock> ReadLock; }; private: @@ -473,9 +463,9 @@ namespace NThreading { size_t readPos = GetCycleCount(); for (size_t i = 0; i < Concurrency; ++i) { TQueueType& queue = Queues[readPos++ % Concurrency]; - if (AtomicTryAndTryLock(&queue.ReadLock)) { + if (!queue.ReadLock.IsLocked() && queue.ReadLock.TryAcquire()) { bool dequeued = queue.Dequeue(value); - AtomicUnlock(&queue.ReadLock); + queue.ReadLock.Release(); if (dequeued) { return true; } @@ -487,9 +477,9 @@ namespace NThreading { bool IsEmpty() { for (size_t i = 0; i < Concurrency; ++i) { TQueueType& queue = Queues[i]; - if (AtomicTryAndTryLock(&queue.ReadLock)) { + if (!queue.ReadLock.IsLocked() && queue.ReadLock.TryAcquire()) { bool empty = queue.IsEmpty(); - AtomicUnlock(&queue.ReadLock); + queue.ReadLock.Release(); if (!empty) { return false; } @@ -504,9 +494,9 @@ namespace NThreading { size_t writePos = GetCycleCount(); for (size_t i = 0; i < Concurrency; ++i) { TQueueType& queue = Queues[writePos++ % Concurrency]; - if (AtomicTryAndTryLock(&queue.WriteLock)) { + if (!queue.WriteLock.IsLocked() && queue.WriteLock.TryAcquire()) { queue.Enqueue(std::forward<TT>(value)); - AtomicUnlock(&queue.WriteLock); + queue.WriteLock.Release(); return true; } } diff --git a/util/generic/hash_primes.cpp b/util/generic/hash_primes.cpp index 656d31e0464..5b9723d98f2 100644 --- a/util/generic/hash_primes.cpp +++ b/util/generic/hash_primes.cpp @@ -2,6 +2,8 @@ #include "array_size.h" #include "algorithm.h" +#include <functional> + /// Order of fields: reciprocal, reciprocal shift, adjacent hint, divisor #if defined(_32_) static constexpr ::NPrivate::THashDivisor PRIME_DIVISORS_HOLDER[]{ diff --git a/util/generic/yexception.cpp b/util/generic/yexception.cpp index 6611fb51ff9..2e5df6a9ff2 100644 --- a/util/generic/yexception.cpp +++ b/util/generic/yexception.cpp @@ -1,5 +1,6 @@ #include "yexception.h" +#include <util/stream/str.h> #include <util/system/backtrace.h> #include <util/system/type_name.h> diff --git a/util/stream/str.cpp b/util/stream/str.cpp index a0c3b1c75c7..9111e61ed14 100644 --- a/util/stream/str.cpp +++ b/util/stream/str.cpp @@ -1,5 +1,7 @@ #include "str.h" +#include <util/generic/bitops.h> + static constexpr size_t MIN_BUFFER_GROW_SIZE = 16; TStringInput::~TStringInput() = default; diff --git a/util/system/thread.cpp b/util/system/thread.cpp index 7c880a2909b..661e473256c 100644 --- a/util/system/thread.cpp +++ b/util/system/thread.cpp @@ -4,6 +4,7 @@ #include "thread.h" #include "thread.i" +#include <util/generic/bitops.h> #include <util/generic/ptr.h> #include <util/generic/ymath.h> #include <util/generic/ylimits.h> diff --git a/ydb/ci/rightlib.txt b/ydb/ci/rightlib.txt index 2b19faa77b9..717c1562469 100644 --- a/ydb/ci/rightlib.txt +++ b/ydb/ci/rightlib.txt @@ -1 +1 @@ -fdbc38349df2ee0ddc678fa2bffe84786f9639a3 +fcc18ade321e3471411a93114f83ca8c339b876e diff --git a/yql/essentials/core/cbo/cbo_interesting_orderings.cpp b/yql/essentials/core/cbo/cbo_interesting_orderings.cpp index 82237151690..b34fe05628f 100644 --- a/yql/essentials/core/cbo/cbo_interesting_orderings.cpp +++ b/yql/essentials/core/cbo/cbo_interesting_orderings.cpp @@ -2,6 +2,8 @@ #include <library/cpp/disjoint_sets/disjoint_sets.h> +#include <yql/essentials/utils/log/log.h> + #include <util/string/builder.h> #include <util/string/join.h> @@ -24,18 +26,25 @@ bool TFunctionalDependency::IsEquivalence() const { return Type == EType::EEquivalence; } -bool TFunctionalDependency::MatchesAntecedentItems(const TOrdering& ordering) const { - if (ordering.Items.size() < AntecedentItems.size()) { - return false; - } +bool TFunctionalDependency::IsImplication() const { + return Type == EType::EImplication; +} - for (std::size_t i = 0; i < AntecedentItems.size(); ++i) { - if (ordering.Items[i] != AntecedentItems[i]) { - return false; - } +bool TFunctionalDependency::IsConstant() const { + return AntecedentItems.empty(); +} + +TMaybe<std::size_t> TFunctionalDependency::MatchesAntecedentItems(const TOrdering& ordering) const { + auto it = std::search( + ordering.Items.begin(), ordering.Items.end(), + AntecedentItems.begin(), AntecedentItems.end() + ); + + if (it == ordering.Items.end()) { + return Nothing(); } - return true; + return static_cast<i64>(std::distance(ordering.Items.begin(), it)); } TString TFunctionalDependency::ToString() const { @@ -49,6 +58,10 @@ TString TFunctionalDependency::ToString() const { } ss << ConsequentItem; + if (AlwaysActive) { + ss << "(AA)"; + } + return ss; } @@ -91,6 +104,11 @@ TTableAliasMap::TBaseColumn TTableAliasMap::GetBaseColumnByRename(const TString& if (auto baseTable = GetBaseTableByAlias(alias)) { return TBaseColumn(std::move(baseTable), std::move(column)); } + + if (alias.empty() && BaseColumnByRename.contains(column)) { + return BaseColumnByRename[column]; + } + return TBaseColumn(std::move(alias), std::move(column)); } @@ -107,7 +125,7 @@ TString TTableAliasMap::ToString() const { if (!BaseColumnByRename.empty()) { result += "Renames: "; for (const auto& [from, to] : BaseColumnByRename) { - result += from + "->" + to.Relation + "." + to.Column + " "; + result += from + " -> " + to.Relation + "." + to.Column + " "; } result.pop_back(); result += ", "; @@ -115,7 +133,7 @@ TString TTableAliasMap::ToString() const { result += "TableAliases: "; for (const auto& [alias, table] : TableByAlias) { - result += alias + "->" + table + ", "; + result += alias + " -> " + table + ", "; } result.pop_back(); @@ -132,6 +150,9 @@ void TTableAliasMap::Merge(const TTableAliasMap& other) { } TString TTableAliasMap::GetBaseTableByAlias(const TString& alias) { + if (!TableByAlias.contains(alias)) { + return alias; + } return TableByAlias[alias]; } @@ -142,8 +163,13 @@ i64 TFDStorage::FindFDIdx( TTableAliasMap* tableAliases ) { auto convertedAntecedent = ConvertColumnIntoIndexes({antecedentColumn}, false, tableAliases); - auto convertedConsequent = ConvertColumnIntoIndexes({consequentColumn}, false, tableAliases).at(0); + auto convertedConsequents = ConvertColumnIntoIndexes({consequentColumn}, false, tableAliases); + + if (convertedAntecedent.empty() || convertedConsequents.empty()) { + return -1; + } + auto convertedConsequent = convertedConsequents[0]; for (std::size_t i = 0; i < FDs.size(); ++i) { auto& fd = FDs[i]; if ( @@ -158,6 +184,36 @@ i64 TFDStorage::FindFDIdx( return -1; } +bool operator==(const TFunctionalDependency& lhs, const TFunctionalDependency& rhs) { + if (lhs.IsConstant() && rhs.IsConstant()) { + return lhs.ConsequentItem == rhs.ConsequentItem; + } + + if (lhs.IsImplication() && rhs.IsImplication()) { + return std::tie(lhs.AntecedentItems, lhs.ConsequentItem) == std::tie(rhs.AntecedentItems, rhs.ConsequentItem); + } + + if (lhs.IsEquivalence() && rhs.IsEquivalence()) { + return + rhs.AntecedentItems.size() == 1 && rhs.AntecedentItems[0] == lhs.ConsequentItem || + lhs.AntecedentItems.size() == 1 && lhs.AntecedentItems[0] == rhs.ConsequentItem; + } + + return false; +} + + +std::size_t TFDStorage::AddFDImpl(TFunctionalDependency fd) { + for (std::size_t i = 0; i < FDs.size(); ++i) { + if (FDs[i] == fd) { + return i; + } + } + + FDs.push_back(std::move(fd)); + return FDs.size() - 1; +} + std::size_t TFDStorage::AddFD( const TJoinColumn& antecedentColumn, const TJoinColumn& consequentColumn, @@ -172,6 +228,53 @@ std::size_t TFDStorage::AddFD( .AlwaysActive = alwaysActive }; + return AddFDImpl(std::move(fd)); +} + +std::size_t TFDStorage::AddConstant( + const TJoinColumn& constantColumn, + bool alwaysActive, + TTableAliasMap* tableAliases +) { + auto fd = TFunctionalDependency{ + .AntecedentItems = {}, + .ConsequentItem = GetIdxByColumn(constantColumn, true, tableAliases), + .Type = TFunctionalDependency::EImplication, + .AlwaysActive = alwaysActive + }; + + return AddFDImpl(std::move(fd)); +} + +std::size_t TFDStorage::AddImplication( + const TVector<TJoinColumn>& antecedentColumns, + const TJoinColumn& consequentColumn, + bool alwaysActive, + TTableAliasMap* tableAliases +) { + auto fd = TFunctionalDependency{ + .AntecedentItems = ConvertColumnIntoIndexes(antecedentColumns, true, tableAliases), + .ConsequentItem = GetIdxByColumn(consequentColumn, true, tableAliases), + .Type = TFunctionalDependency::EImplication, + .AlwaysActive = alwaysActive + }; + + return AddFDImpl(std::move(fd)); +} + +std::size_t TFDStorage::AddEquivalence( + const TJoinColumn& lhs, + const TJoinColumn& rhs, + bool alwaysActive, + TTableAliasMap* tableAliases +) { + auto fd = TFunctionalDependency{ + .AntecedentItems = {GetIdxByColumn(lhs, true, tableAliases)}, + .ConsequentItem = GetIdxByColumn(rhs, true, tableAliases), + .Type = TFunctionalDependency::EEquivalence, + .AlwaysActive = alwaysActive + }; + FDs.push_back(std::move(fd)); return FDs.size() - 1; } @@ -186,6 +289,19 @@ i64 TFDStorage::FindInterestingOrderingIdx( } std::size_t TFDStorage::AddInterestingOrdering( + const TVector<TString>& interestingOrdering, + TOrdering::EType type, + TTableAliasMap* tableAliases +) { + std::vector<TJoinColumn> columns; + columns.reserve(interestingOrdering.size()); + for (const auto& column: interestingOrdering) { + columns.push_back(TJoinColumn("", column)); + } + return AddInterestingOrdering(columns, type, tableAliases); +} + +std::size_t TFDStorage::AddInterestingOrdering( const std::vector<TJoinColumn>& interestingOrdering, TOrdering::EType type, TTableAliasMap* tableAliases @@ -249,6 +365,9 @@ std::pair<std::vector<std::size_t>, i64> TFDStorage::ConvertColumnsAndFindExisti TTableAliasMap* tableAliases ) { std::vector<std::size_t> items = ConvertColumnIntoIndexes(interestingOrdering, createIfNotExists, tableAliases); + if (items.empty()) { + return {{}, -1}; + } for (std::size_t i = 0; i < InterestingOrderings.size(); ++i) { if (items == InterestingOrderings[i].Items && type == InterestingOrderings[i].Type) { @@ -268,7 +387,11 @@ std::vector<std::size_t> TFDStorage::ConvertColumnIntoIndexes( items.reserve(ordering.size()); for (const auto& column: ordering) { - items.push_back(GetIdxByColumn(column, createIfNotExists, tableAliases)); + if (auto idx = GetIdxByColumn(column, createIfNotExists, tableAliases); idx != Max<size_t>()) { + items.push_back(idx); + } else { + return {}; + } } return items; @@ -293,21 +416,27 @@ std::size_t TFDStorage::GetIdxByColumn( } Y_ENSURE(!baseColumn.AttributeName.empty()); - Y_ENSURE(createIfNotExists, "There's no such column: " + fullPath); + if (!createIfNotExists) { + return Max<size_t>(); + } ColumnByIdx.push_back(baseColumn); IdxByColumn[fullPath] = IdCounter++; return IdxByColumn[fullPath]; } -bool TOrderingsStateMachine::TLogicalOrderings::ContainsShuffle(std::size_t orderingIdx) { - return DFSM->ContainsMatrix[State][orderingIdx]; +bool TOrderingsStateMachine::TLogicalOrderings::ContainsShuffle(i64 orderingIdx) { + return IsInitialized() && HasState() && (orderingIdx >= 0) && DFSM->Nodes[State].InterestingOrderings[orderingIdx]; +} + +bool TOrderingsStateMachine::TLogicalOrderings::ContainsSorting(i64 orderingIdx) { + return IsInitialized() && HasState() && (orderingIdx >= 0) && DFSM->Nodes[State].InterestingOrderings[orderingIdx]; } void TOrderingsStateMachine::TLogicalOrderings::InduceNewOrderings(const TFDSet& fds) { AppliedFDs |= fds; - if (State == -1) { + if (!(HasState() && IsInitialized())) { return; } @@ -327,7 +456,14 @@ void TOrderingsStateMachine::TLogicalOrderings::RemoveState() { } void TOrderingsStateMachine::TLogicalOrderings::SetOrdering(i64 orderingIdx) { - Y_ASSERT(0 <= orderingIdx && orderingIdx < static_cast<i64>(DFSM->InitStateByOrderingIdx.size())); + if (orderingIdx < 0 || orderingIdx >= static_cast<i64>(DFSM->InitStateByOrderingIdx.size())) { + RemoveState(); + return; + } + + if (!IsInitialized()) { + return; + } auto state = DFSM->InitStateByOrderingIdx[orderingIdx]; State = state.StateIdx; @@ -354,12 +490,20 @@ bool TOrderingsStateMachine::TLogicalOrderings::HasState() const { return State != -1; } -bool TOrderingsStateMachine::TLogicalOrderings::IsSubsetOf(const TLogicalOrderings& logicalOrderings) { - if (DFSM == nullptr || logicalOrderings.DFSM == nullptr) { - return false; - } +bool TOrderingsStateMachine::TLogicalOrderings::IsInitialized() { + return DFSM != nullptr; +} - return (DFSM == logicalOrderings.DFSM) && HasState() && logicalOrderings.HasState() && IsSubset(DFSM->Nodes[State].NFSMNodesBitset, logicalOrderings.DFSM->Nodes[logicalOrderings.State].NFSMNodesBitset); +bool TOrderingsStateMachine::TLogicalOrderings::IsInitialized() const { + return DFSM != nullptr; +} + +bool TOrderingsStateMachine::TLogicalOrderings::IsSubsetOf(const TLogicalOrderings& logicalOrderings) { + return + HasState() && logicalOrderings.HasState() && + IsInitialized() && logicalOrderings.IsInitialized() && + DFSM == logicalOrderings.DFSM && + IsSubset(DFSM->Nodes[State].NFSMNodesBitset, logicalOrderings.DFSM->Nodes[logicalOrderings.State].NFSMNodesBitset); } i64 TOrderingsStateMachine::TLogicalOrderings::GetState() const { @@ -418,6 +562,7 @@ void TOrderingsStateMachine::Build( ) { std::vector<TFunctionalDependency> processedFDs = PruneFDs(fds, interestingOrderings); NFSM.Build(processedFDs, interestingOrderings); + DFSM = MakeSimpleShared<TDFSM>(); DFSM->Build(NFSM, processedFDs, interestingOrderings); Built = true; } @@ -466,7 +611,7 @@ void TOrderingsStateMachine::TNFSM::Build( AddNode(interesting[i], TNFSM::TNode::EInteresting, i); } - ApplyFDs(fds); + ApplyFDs(fds, interesting); PrefixClosure(); for (std::size_t idx = 0; idx < Edges.size(); ++idx) { @@ -485,8 +630,18 @@ std::size_t TOrderingsStateMachine::TNFSM::AddNode(const TOrdering& ordering, TN return Nodes.size() - 1; } +bool TOrderingsStateMachine::TNFSM::TEdge::operator==(const TEdge& other) const { + return std::tie(srcNodeIdx, dstNodeIdx, fdIdx) == std::tie(other.srcNodeIdx, other.dstNodeIdx, other.fdIdx); +} + void TOrderingsStateMachine::TNFSM::AddEdge(std::size_t srcNodeIdx, std::size_t dstNodeIdx, i64 fdIdx) { - Edges.emplace_back(srcNodeIdx, dstNodeIdx, fdIdx); + auto newEdge = TNFSM::TEdge(srcNodeIdx, dstNodeIdx, fdIdx); + for (std::size_t i = 0; i < Edges.size(); ++i) { + if (Edges[i] == newEdge) { + return; + } + } + Edges.emplace_back(newEdge); } void TOrderingsStateMachine::TNFSM::PrefixClosure() { @@ -504,30 +659,123 @@ void TOrderingsStateMachine::TNFSM::PrefixClosure() { } if (k == iItems.size()) { - AddEdge(i, j, TNFSM::TEdge::EPSILON); + if (Nodes[i].Ordering.Type == TOrdering::EShuffle) { + AddEdge(i, j, TNFSM::TEdge::EPSILON); + } + + if (Nodes[i].Ordering.Type == TOrdering::ESorting) { + AddEdge(j, i, TNFSM::TEdge::EPSILON); + } } } } } -void TOrderingsStateMachine::TNFSM::ApplyFDs(const std::vector<TFunctionalDependency>& fds) { +void TOrderingsStateMachine::TNFSM::ApplyFDs( + const std::vector<TFunctionalDependency>& fds, + const std::vector<TOrdering>& interestingOrderings +) { + std::size_t maxInterestingOrderingSize = 0; + if (!interestingOrderings.empty()) { + maxInterestingOrderingSize = + std::max_element( + interestingOrderings.begin(), + interestingOrderings.end(), + [](const TOrdering& a, const TOrdering& b) { return a.Items.size() < b.Items.size(); } + )->Items.size(); + } + + for (std::size_t nodeIdx = 0; nodeIdx < Nodes.size() && Nodes.size() < EMaxNFSMStates; ++nodeIdx) { for (std::size_t fdIdx = 0; fdIdx < fds.size() && Nodes.size() < EMaxNFSMStates; ++fdIdx) { - const auto& fd = fds[fdIdx]; + TFunctionalDependency fd = fds[fdIdx]; - if (!fd.MatchesAntecedentItems(Nodes[nodeIdx].Ordering)) { - continue; - } + auto applyFD = [this, nodeIdx, maxInterestingOrderingSize](const TFunctionalDependency& fd, std::size_t fdIdx) { + if (Nodes.size() >= EMaxNFSMStates) { + return; + } - if (fd.IsEquivalence()) { - std::size_t replacedElement = fd.AntecedentItems[0]; - std::vector<std::size_t> newOrdering = Nodes[nodeIdx].Ordering.Items; - *std::find(newOrdering.begin(), newOrdering.end(), replacedElement) = fd.ConsequentItem; + if (fd.IsConstant() && Nodes[nodeIdx].Ordering.Items.size() > 1) { + std::vector<std::size_t> newOrdering = Nodes[nodeIdx].Ordering.Items; + auto it = std::find(newOrdering.begin(), newOrdering.end(), fd.ConsequentItem); + if (it == newOrdering.end()) { + return; + } + bool isNewOrderingPrefixOfOld = (it == (newOrdering.end() - 1)); + newOrdering.erase(it); - std::size_t dstIdx = AddNode(TOrdering(newOrdering, Nodes[nodeIdx].Ordering.Type), TNode::EArtificial); - AddEdge(nodeIdx, dstIdx, fdIdx); - AddEdge(dstIdx, nodeIdx, fdIdx); - continue; + std::size_t dstIdx = AddNode(TOrdering(std::move(newOrdering), Nodes[nodeIdx].Ordering.Type), TNode::EArtificial); + + if (!isNewOrderingPrefixOfOld || Nodes[nodeIdx].Ordering.Type == TOrdering::EShuffle) { + AddEdge(nodeIdx, dstIdx, fdIdx); + } + + if (!isNewOrderingPrefixOfOld || Nodes[nodeIdx].Ordering.Type == TOrdering::ESorting) { + AddEdge(dstIdx, nodeIdx, fdIdx); + } + } + + if (fd.IsConstant()) { + return; + } + + auto maybeAntecedentItemIdx = fd.MatchesAntecedentItems(Nodes[nodeIdx].Ordering); + if (!maybeAntecedentItemIdx) { + return; + } + + std::size_t antecedentItemIdx = maybeAntecedentItemIdx.GetRef(); + if ( + auto it = std::find(Nodes[nodeIdx].Ordering.Items.begin(), Nodes[nodeIdx].Ordering.Items.end(), fd.ConsequentItem); + it != Nodes[nodeIdx].Ordering.Items.end() + ) { + if (fd.IsEquivalence()) { // (a, b) -> (b, a) + std::size_t consequentItemIdx = std::distance(Nodes[nodeIdx].Ordering.Items.begin(), it); + auto newOrdering = Nodes[nodeIdx].Ordering.Items; + std::swap(newOrdering[antecedentItemIdx], newOrdering[consequentItemIdx]); + std::size_t dstIdx = AddNode(TOrdering(std::move(newOrdering), Nodes[nodeIdx].Ordering.Type), TNode::EArtificial); + AddEdge(nodeIdx, dstIdx, fdIdx); + AddEdge(dstIdx, nodeIdx, fdIdx); + } + + return; + } + + Y_ENSURE(antecedentItemIdx < Nodes[nodeIdx].Ordering.Items.size()); + if (fd.IsEquivalence()) { + Y_ENSURE(fd.AntecedentItems.size() == 1); + std::vector<std::size_t> newOrdering = Nodes[nodeIdx].Ordering.Items; + newOrdering[antecedentItemIdx] = fd.ConsequentItem; + + std::size_t dstIdx = AddNode(TOrdering(std::move(newOrdering), Nodes[nodeIdx].Ordering.Type), TNode::EArtificial); + AddEdge(nodeIdx, dstIdx, fdIdx); + AddEdge(dstIdx, nodeIdx, fdIdx); + } + + if ( + Nodes[nodeIdx].Ordering.Type == TOrdering::EShuffle || + Nodes[nodeIdx].Ordering.Items.size() == maxInterestingOrderingSize + ) { + return; + } + + if (fd.IsImplication() || fd.IsEquivalence()) { + for (std::size_t i = antecedentItemIdx + fd.AntecedentItems.size(); i <= Nodes[nodeIdx].Ordering.Items.size() && Nodes.size() < EMaxNFSMStates; ++i) { + std::vector<std::size_t> newOrdering = Nodes[nodeIdx].Ordering.Items; + newOrdering.insert(newOrdering.begin() + i, fd.ConsequentItem); + + std::size_t dstIdx = AddNode(TOrdering(std::move(newOrdering), Nodes[nodeIdx].Ordering.Type), TNode::EArtificial); + AddEdge(nodeIdx, dstIdx, fdIdx); // Epsilon edge will be added during PrefixClosure + } + } + }; + + applyFD(fd, fdIdx); + if (fd.IsEquivalence()) { + Y_ENSURE(fd.AntecedentItems.size() == 1); + TFunctionalDependency reversedEquiv = fd; + std::swap(reversedEquiv.ConsequentItem, reversedEquiv.AntecedentItems[0]); + applyFD(reversedEquiv, fdIdx); } } } @@ -594,7 +842,7 @@ void TOrderingsStateMachine::TDFSM::Build( for (std::size_t i = 0; i < interestingOrderings.size(); ++i) { for (std::size_t nfsmNodeIdx = 0; nfsmNodeIdx < nfsm.Nodes.size(); ++nfsmNodeIdx) { if (nfsm.Nodes[nfsmNodeIdx].Ordering == interestingOrderings[i]) { - auto nfsmNodes = CollectNodesWithEpsOrFdEdge(nfsm, {i}); + auto nfsmNodes = CollectNodesWithEpsOrFdEdge(nfsm, {i}, fds); InitStateByOrderingIdx[i] = TInitState{AddNode(std::move(nfsmNodes)), interestingOrderings[i].Items.size()}; } } @@ -613,7 +861,7 @@ void TOrderingsStateMachine::TDFSM::Build( continue; } - std::size_t dstNodeIdx = AddNode(CollectNodesWithEpsOrFdEdge(nfsm, Nodes[nodeIdx].NFSMNodes, fdIdx)); + std::size_t dstNodeIdx = AddNode(CollectNodesWithEpsOrFdEdge(nfsm, Nodes[nodeIdx].NFSMNodes, fds, fdIdx)); if (nodeIdx == dstNodeIdx) { continue; } @@ -623,7 +871,7 @@ void TOrderingsStateMachine::TDFSM::Build( } } - Precompute(nfsm, fds, interestingOrderings); + Precompute(nfsm, fds); } std::size_t TOrderingsStateMachine::TDFSM::AddNode(const std::vector<std::size_t>& nfsmNodes) { @@ -644,12 +892,13 @@ void TOrderingsStateMachine::TDFSM::AddEdge(std::size_t srcNodeIdx, std::size_t std::vector<std::size_t> TOrderingsStateMachine::TDFSM::CollectNodesWithEpsOrFdEdge( const TNFSM& nfsm, const std::vector<std::size_t>& startNFSMNodes, + const std::vector<TFunctionalDependency>& fds, i64 fdIdx ) { std::set<std::size_t> visited; std::function<void(std::size_t)> DFS; - DFS = [&DFS, &visited, fdIdx, &nfsm](std::size_t nodeIdx){ + DFS = [&DFS, &visited, fdIdx, &nfsm, &fds](std::size_t nodeIdx){ if (visited.contains(nodeIdx)) { return; } @@ -658,7 +907,7 @@ std::vector<std::size_t> TOrderingsStateMachine::TDFSM::CollectNodesWithEpsOrFdE for (std::size_t edgeIdx: nfsm.Nodes[nodeIdx].OutgoingEdges) { const TNFSM::TEdge& edge = nfsm.Edges[edgeIdx]; - if (edge.fdIdx == fdIdx || edge.fdIdx == TNFSM::TEdge::EPSILON) { + if (edge.fdIdx == fdIdx || edge.fdIdx == TNFSM::TEdge::EPSILON || fds[edge.fdIdx].AlwaysActive) { DFS(edge.dstNodeIdx); } } @@ -673,21 +922,19 @@ std::vector<std::size_t> TOrderingsStateMachine::TDFSM::CollectNodesWithEpsOrFdE void TOrderingsStateMachine::TDFSM::Precompute( const TNFSM& nfsm, - const std::vector<TFunctionalDependency>& fds, - const std::vector<TOrdering>& interestingOrderings + const std::vector<TFunctionalDependency>& fds ) { TransitionMatrix = std::vector<std::vector<i64>>(Nodes.size(), std::vector<i64>(fds.size(), -1)); for (const auto& edge: Edges) { TransitionMatrix[edge.srcNodeIdx][edge.fdIdx] = edge.dstNodeIdx; } - ContainsMatrix = std::vector<std::vector<bool>>(Nodes.size(), std::vector<bool>(interestingOrderings.size(), false)); for (std::size_t dfsmNodeIdx = 0; dfsmNodeIdx < Nodes.size(); ++dfsmNodeIdx) { for (std::size_t nfsmNodeIdx : Nodes[dfsmNodeIdx].NFSMNodes) { auto interestingOrderIdx = nfsm.Nodes[nfsmNodeIdx].InterestingOrderingIdx; if (interestingOrderIdx == -1) { continue; } - ContainsMatrix[dfsmNodeIdx][interestingOrderIdx] = true; + Nodes[dfsmNodeIdx].InterestingOrderings[interestingOrderIdx] = 1; } } @@ -702,63 +949,33 @@ std::vector<TFunctionalDependency> TOrderingsStateMachine::PruneFDs( const std::vector<TFunctionalDependency>& fds, const std::vector<TOrdering>& interestingOrderings ) { - std::size_t newIdxCounter = 0; - std::unordered_map<std::size_t, std::size_t> idxByItem; - - for (const auto& ordering: interestingOrderings) { - for (const auto& item: ordering.Items) { - if (!idxByItem.contains(item)) { - idxByItem[item] = newIdxCounter++; - } - } - } - - for (const auto& fd: fds) { - if (fd.IsEquivalence()) { - if (!idxByItem.contains(fd.ConsequentItem)) { - idxByItem[fd.ConsequentItem] = newIdxCounter++; - } - - if (!idxByItem.contains(fd.AntecedentItems[0])) { - idxByItem[fd.AntecedentItems[0]] = newIdxCounter++; - } - } - } - - TDisjointSets equivalenceSets(idxByItem.size()); - for (const auto& fd: fds) { - if (fd.IsEquivalence()) { - equivalenceSets.UnionSets(idxByItem[fd.ConsequentItem], idxByItem[fd.AntecedentItems[0]]); - } - } - std::vector<TFunctionalDependency> filteredFds; filteredFds.reserve(fds.size()); FdMapping.resize(fds.size()); for (std::size_t i = 0; i < fds.size(); ++i) { - if (fds[i].IsEquivalence()) { - bool canLeadToInteresting = false; - - for (const auto& ordering: interestingOrderings) { - for (const auto& item: ordering.Items) { - if ( - equivalenceSets.CanonicSetElement(idxByItem[item]) == equivalenceSets.CanonicSetElement(idxByItem[fds[i].ConsequentItem]) || - equivalenceSets.CanonicSetElement(idxByItem[item]) == equivalenceSets.CanonicSetElement(idxByItem[fds[i].AntecedentItems[0]]) - ) { - canLeadToInteresting = true; - break; - } - } + bool canLeadToInteresting = false; + for (const auto& ordering: interestingOrderings) { + if (std::find(ordering.Items.begin(), ordering.Items.end(), fds[i].ConsequentItem) != ordering.Items.end()) { + canLeadToInteresting = true; + break; } - if (canLeadToInteresting && filteredFds.size() < EMaxFDCount) { - filteredFds.push_back(std::move(fds[i])); - FdMapping[i] = filteredFds.size() - 1; - } else { - FdMapping[i] = -1; + if ( + fds[i].IsEquivalence() && + std::find(ordering.Items.begin(), ordering.Items.end(), fds[i].AntecedentItems[0]) != ordering.Items.end() + ) { + canLeadToInteresting = true; + break; } } + + if (canLeadToInteresting && filteredFds.size() < EMaxFDCount) { + filteredFds.push_back(std::move(fds[i])); + FdMapping[i] = filteredFds.size() - 1; + } else { + FdMapping[i] = -1; + } } return filteredFds; diff --git a/yql/essentials/core/cbo/cbo_interesting_orderings.h b/yql/essentials/core/cbo/cbo_interesting_orderings.h index 311059cb407..77a20ef1964 100644 --- a/yql/essentials/core/cbo/cbo_interesting_orderings.h +++ b/yql/essentials/core/cbo/cbo_interesting_orderings.h @@ -3,6 +3,7 @@ #include <yql/essentials/core/yql_cost_function.h> #include <util/generic/hash.h> +#include <util/generic/algorithm.h> #include <bitset> #include <stdint.h> @@ -20,7 +21,8 @@ namespace NYql::NDq { struct TOrdering { enum EType : uint32_t { - EShuffle + EShuffle = 0, + ESorting = 1 }; bool operator==(const TOrdering& other) const; @@ -52,11 +54,18 @@ struct TOrdering { */ struct TFunctionalDependency { enum EType : uint32_t { - EEquivalence + /* default fd: a -> b */ + EImplication = 0, + /* equivalence: a = b */ + EEquivalence = 1 }; bool IsEquivalence() const; - bool MatchesAntecedentItems(const TOrdering& ordering) const; + bool IsImplication() const; + bool IsConstant() const; + + // Returns index of the first matching antecedent item in the ordering, if it matches + TMaybe<std::size_t> MatchesAntecedentItems(const TOrdering& ordering) const; TString ToString() const; std::vector<std::size_t> AntecedentItems; @@ -66,6 +75,8 @@ struct TFunctionalDependency { bool AlwaysActive; }; +bool operator==(const TFunctionalDependency& lhs, const TFunctionalDependency& rhs); + // Map of table aliases to their original table names struct TTableAliasMap : public TSimpleRefCount<TTableAliasMap> { public: @@ -102,6 +113,7 @@ public: TBaseColumn GetBaseColumnByRename(const NDq::TJoinColumn& renamedColumn); TString ToString() const; void Merge(const TTableAliasMap& other); + bool Empty() const { return TableByAlias.empty() && BaseColumnByRename.empty(); } private: TString GetBaseTableByAlias(const TString& alias); @@ -123,14 +135,41 @@ public: TTableAliasMap* tableAliases = nullptr ); +public: + // Deprecated, use the others in this public section instead std::size_t AddFD( const TJoinColumn& antecedentColumn, const TJoinColumn& consequentColumn, TFunctionalDependency::EType type, + bool alwaysActive = false, + TTableAliasMap* tableAliases = nullptr + ); + + std::size_t AddConstant( + const TJoinColumn& constantColumn, + bool alwaysActive, + TTableAliasMap* tableAliases = nullptr + ); + + std::size_t AddImplication( + const TVector<TJoinColumn>& antecedentColumns, + const TJoinColumn& consequentColumn, + bool alwaysActive, + TTableAliasMap* tableAliases = nullptr + ); + + std::size_t AddEquivalence( + const TJoinColumn& lhs, + const TJoinColumn& rhs, bool alwaysActive, TTableAliasMap* tableAliases = nullptr ); +private: + std::size_t AddFDImpl(TFunctionalDependency fd); + +public: + i64 FindInterestingOrderingIdx( const std::vector<TJoinColumn>& interestingOrdering, TOrdering::EType type, @@ -143,6 +182,12 @@ public: TTableAliasMap* tableAliases = nullptr ); + std::size_t AddInterestingOrdering( + const TVector<TString>& interestingOrdering, // column names + TOrdering::EType type, + TTableAliasMap* tableAliases = nullptr + ); + TVector<TJoinColumn> GetInterestingOrderingsColumnNamesByIdx(std::size_t interestingOrderingIdx) const; TString ToString() const; @@ -190,8 +235,8 @@ private: class TDFSM; enum _ : std::uint32_t { EMaxFDCount = 64, - EMaxNFSMStates = 512, - EMaxDFSMStates = 32768, + EMaxNFSMStates = 256, + EMaxDFSMStates = 512, }; public: @@ -211,18 +256,23 @@ public: {} public: // API - bool ContainsShuffle(std::size_t orderingIdx); + bool ContainsShuffle(i64 orderingIdx); + bool ContainsSorting(i64 orderingIdx); void InduceNewOrderings(const TFDSet& fds); void RemoveState(); void SetOrdering(i64 orderingIdx); i64 GetShuffleHashFuncArgsCount(); void SetShuffleHashFuncArgsCount(std::size_t value); TFDSet GetFDs(); - bool HasState(); - bool HasState() const; bool IsSubsetOf(const TLogicalOrderings& logicalOrderings); i64 GetState() const; + public: + bool HasState(); + bool HasState() const; + bool IsInitialized(); + bool IsInitialized() const; + private: bool IsSubset(const std::bitset<EMaxNFSMStates>& lhs, const std::bitset<EMaxNFSMStates>& rhs); @@ -240,10 +290,13 @@ public: public: TOrderingsStateMachine() = default; - TOrderingsStateMachine(TFDStorage fdStorage) + TOrderingsStateMachine( + TFDStorage fdStorage, + TOrdering::EType machineType = TOrdering::EShuffle + ) : FDStorage(std::move(fdStorage)) - , DFSM(MakeSimpleShared<TDFSM>()) { + EraseIf(FDStorage.InterestingOrderings, [machineType](const TOrdering& ordering){ return ordering.Type != machineType; }); Build(FDStorage.FDs, FDStorage.InterestingOrderings); } @@ -251,7 +304,6 @@ public: const std::vector<TFunctionalDependency>& fds, const std::vector<TOrdering>& interestingOrderings ) { - DFSM = MakeSimpleShared<TDFSM>(); Build(fds, interestingOrderings); } @@ -281,7 +333,7 @@ private: TNode(TOrdering ordering, EType type, i64 interestingOrderingIdx = -1) : Type(type) - , Ordering(ordering) + , Ordering(std::move(ordering)) , InterestingOrderingIdx(interestingOrderingIdx) {} @@ -303,6 +355,8 @@ private: EPSILON = -1 // eps edges with give us nodes without applying any FDs. }; + bool operator==(const TEdge& other) const; + TString ToString() const; }; @@ -317,7 +371,10 @@ private: std::size_t AddNode(const TOrdering& ordering, TNode::EType type, i64 interestingOrderingIdx = -1); void AddEdge(std::size_t srcNodeIdx, std::size_t dstNodeIdx, i64 fdIdx); void PrefixClosure(); - void ApplyFDs(const std::vector<TFunctionalDependency>& fds); + void ApplyFDs( + const std::vector<TFunctionalDependency>& fds, + const std::vector<TOrdering>& interesting + ); private: std::vector<TNode> Nodes; @@ -333,6 +390,7 @@ private: std::vector<std::size_t> NFSMNodes; std::bitset<EMaxFDCount> OutgoingFDs; std::bitset<EMaxNFSMStates> NFSMNodesBitset; + std::bitset<EMaxNFSMStates> InterestingOrderings; TString ToString() const; }; @@ -359,12 +417,12 @@ private: std::vector<std::size_t> CollectNodesWithEpsOrFdEdge( const TNFSM& nfsm, const std::vector<std::size_t>& startNFSMNodes, + const std::vector<TFunctionalDependency>& fds, i64 fdIdx = TNFSM::TEdge::EPSILON ); void Precompute( const TNFSM& nfsm, - const std::vector<TFunctionalDependency>& fds, - const std::vector<TOrdering>& interestingOrderings + const std::vector<TFunctionalDependency>& fds ); private: diff --git a/yql/essentials/core/peephole_opt/yql_opt_peephole_physical.cpp b/yql/essentials/core/peephole_opt/yql_opt_peephole_physical.cpp index 2759510b09a..3f62f755562 100644 --- a/yql/essentials/core/peephole_opt/yql_opt_peephole_physical.cpp +++ b/yql/essentials/core/peephole_opt/yql_opt_peephole_physical.cpp @@ -4263,12 +4263,15 @@ TExprNode::TPtr OptimizeExpandMap(const TExprNode::TPtr& node, TExprContext& ctx input.Child(4U)->Tail().IsCallable("Just") && ETypeAnnotationKind::Struct == input.Child(4U)->Tail().Head().GetTypeAnn()->GetKind()) { if (const auto inItemType = GetSeqItemType(input.Head().GetTypeAnn()); ETypeAnnotationKind::Struct == inItemType->GetKind()) { if (const auto inStructType = inItemType->Cast<TStructExprType>(); inStructType->GetSize() > 0U) { - YQL_CLOG(DEBUG, CorePeepHole) << "Swap " << node->Content() << " with " << input.Content(); - const auto& output = input.Child(4U)->Tail().Head(); const auto structType = output.GetTypeAnn()->Cast<TStructExprType>(); const auto outputWidth = structType->GetSize(); + if (outputWidth > WideLimit) { + return node; + } + + YQL_CLOG(DEBUG, CorePeepHole) << "Swap " << node->Content() << " with " << input.Content(); const auto inputWidth = inStructType->GetSize(); TExprNode::TListType inputFilelds, stateFields, outputFields, init, update, finish; @@ -4660,12 +4663,15 @@ TExprNode::TPtr OptimizeMember(const TExprNode::TPtr& node, TExprContext& ctx) { TExprNode::TPtr OptimizeCondense1(const TExprNode::TPtr& node, TExprContext& ctx) { if (node->Head().IsCallable("NarrowMap") && ETypeAnnotationKind::Struct == node->Tail().Tail().GetTypeAnn()->GetKind()) { - YQL_CLOG(DEBUG, CorePeepHole) << "Swap " << node->Content() << " with " << node->Head().Content(); const auto inputWidth = node->Head().Tail().Head().ChildrenSize(); TExprNode::TListType fields, init, update; const auto outputWidth = CollectStateNodes(*node->Child(1U), node->Tail(), fields, init, update, ctx); + if (outputWidth > WideLimit) { + return node; + } + YQL_CLOG(DEBUG, CorePeepHole) << "Swap " << node->Content() << " with " << node->Head().Content(); return ctx.Builder(node->Pos()) .Callable("NarrowMap") .Callable(0, "WideCondense1") @@ -4751,14 +4757,16 @@ TExprNode::TPtr OptimizeCondense1(const TExprNode::TPtr& node, TExprContext& ctx TExprNode::TPtr OptimizeCombineCore(const TExprNode::TPtr& node, TExprContext& ctx) { if (node->Head().IsCallable("NarrowMap") && node->Child(4U)->Tail().IsCallable("Just")) { - YQL_CLOG(DEBUG, CorePeepHole) << "Swap " << node->Content() << " with " << node->Head().Content(); - const auto& output = node->Child(4U)->Tail().Head(); const auto inputWidth = node->Head().Tail().Head().ChildrenSize(); const auto structType = ETypeAnnotationKind::Struct == output.GetTypeAnn()->GetKind() ? output.GetTypeAnn()->Cast<TStructExprType>() : nullptr; const auto outputWidth = structType ? structType->GetSize() : 1U; + if (outputWidth > WideLimit) { + return node; + } + YQL_CLOG(DEBUG, CorePeepHole) << "Swap " << node->Content() << " with " << node->Head().Content(); TExprNode::TListType stateFields, outputFields, init, update, finish; outputFields.reserve(outputWidth); finish.reserve(outputWidth); @@ -5064,12 +5072,17 @@ TExprNode::TPtr OptimizeChopper(const TExprNode::TPtr& node, TExprContext& ctx) node->Tail().GetTypeAnn()->GetKind() == ETypeAnnotationKind::Flow && node->Tail().GetTypeAnn()->Cast<TFlowExprType>()->GetItemType()->GetKind() == ETypeAnnotationKind::Struct && node->Tail().GetTypeAnn()->Cast<TFlowExprType>()->GetItemType()->Cast<TStructExprType>()->GetSize() > 0U) { - YQL_CLOG(DEBUG, CorePeepHole) << "Swap " << node->Content() << " with " << node->Head().Content(); const auto inputWidth = node->Head().Tail().Head().ChildrenSize(); const auto structType = node->Tail().GetTypeAnn()->Cast<TFlowExprType>()->GetItemType()->Cast<TStructExprType>(); const auto outputWidth = structType->GetSize(); + if (outputWidth > WideLimit) { + return node; + } + + YQL_CLOG(DEBUG, CorePeepHole) << "Swap " << node->Content() << " with " << node->Head().Content(); + TExprNode::TListType fields; fields.reserve(outputWidth); for (const auto& item : structType->GetItems()) @@ -5665,9 +5678,9 @@ TExprNode::TPtr OptimizeWideCombiner(const TExprNode::TPtr& node, TExprContext& if (needKeyFlatten.front()) { const auto flattenSize = *needKeyFlatten.front(); - if (flattenSize > WideLimit) { + /*if (flattenSize > WideLimit) { don't emit wide combiner with too many keys return node; - } + }*/ YQL_CLOG(DEBUG, CorePeepHole) << "Flatten key by tuple for " << node->Content() << " from " << originalKeySize << " to " << flattenSize; auto children = node->ChildrenList(); diff --git a/yql/essentials/docs/en/changelog/2025.02.md b/yql/essentials/docs/en/changelog/2025.02.md index 841efc2fb62..4a2edc44063 100644 --- a/yql/essentials/docs/en/changelog/2025.02.md +++ b/yql/essentials/docs/en/changelog/2025.02.md @@ -28,8 +28,8 @@ SELECT foo_new, ... WHERE foo = 1 GROUP BY expr AS foo_new ## Changes in the String module {#string-module} -* Removed String::Reverse function, use Unicode::Reverse. -* Removed String::HasPrefixIgnoreCase and String::StartsWithIgnoreCase functions, use String::AsciiStartsWithIgnoreCase. -* Removed String::HasSuffixIgnoreCase and String::EndsWithIgnoreCase functions, use String::AsciiEndsWithIgnoreCase. -* Added String::AsciiContainsIgnoreCase and String::AsciiEqualsIgnoreCase functions. -* Added String::ReverseBytes and String::ReverseBits functions. +* Removed `String::Reverse` function, use `Unicode::Reverse`. +* Removed `String::HasPrefixIgnoreCase` and `String::StartsWithIgnoreCase` functions, use `String::AsciiStartsWithIgnoreCase`. +* Removed `String::HasSuffixIgnoreCase` and `String::EndsWithIgnoreCase` functions, use `String::AsciiEndsWithIgnoreCase`. +* Added `String::AsciiContainsIgnoreCase` and `String::AsciiEqualsIgnoreCase` functions. +* Added `String::ReverseBytes` and `String::ReverseBits` functions. diff --git a/yql/essentials/docs/en/changelog/2025.03.md b/yql/essentials/docs/en/changelog/2025.03.md index e85da1f9c28..6a22743eb74 100644 --- a/yql/essentials/docs/en/changelog/2025.03.md +++ b/yql/essentials/docs/en/changelog/2025.03.md @@ -1,3 +1,3 @@ ## Changes in RE2 module {#re2-module} -* Queries are guaranteed to fail when invalid regular expressions are passed +* Queries are guaranteed to fail when invalid regular expressions are passed. diff --git a/yql/essentials/docs/en/udf/list/yson.md b/yql/essentials/docs/en/udf/list/yson.md index 3d4dd2d7cba..756ea377800 100644 --- a/yql/essentials/docs/en/udf/list/yson.md +++ b/yql/essentials/docs/en/udf/list/yson.md @@ -291,11 +291,11 @@ $yson = @@{y = true; x = 5.5}@@y; SELECT Yson::LookupBool($yson, "z"); --- null SELECT Yson::LookupBool($yson, "y"); --- true -SELECT Yson::LookupInt64($yson, "x"); --- Error +-- SELECT Yson::LookupInt64($yson, "x"); --- Error SELECT Yson::LookupInt64($yson, "x", Yson::Options(false as Strict)); --- null SELECT Yson::LookupInt64($yson, "x", Yson::Options(true as AutoConvert)); --- 5 -SELECT Yson::ConvertToBoolDict($yson); --- Error +-- SELECT Yson::ConvertToBoolDict($yson); --- Error SELECT Yson::ConvertToBoolDict($yson, Yson::Options(false as Strict)); --- { "y": true } SELECT Yson::ConvertToDoubleDict($yson, Yson::Options(false as Strict)); --- { "x": 5.5 } ``` diff --git a/yql/essentials/docs/ru/changelog/2025.02.md b/yql/essentials/docs/ru/changelog/2025.02.md index e24b5646f5c..7904d4b893d 100644 --- a/yql/essentials/docs/ru/changelog/2025.02.md +++ b/yql/essentials/docs/ru/changelog/2025.02.md @@ -28,8 +28,8 @@ SELECT foo_new, ... WHERE foo = 1 GROUP BY expr AS foo_new ## Изменения в модуле String {#string-module} -* Удалена функция String::Reverse, используйте Unicode::Reverse. -* Удалены функции String::HasPrefixIgnoreCase и String::StartsWithIgnoreCase, используйте String::AsciiStartsWithIgnoreCase. -* Удалены функции String::HasSuffixIgnoreCase и String::EndsWithIgnoreCase, используйте String::AsciiEndsWithIgnoreCase. -* Добавлены функции String::AsciiContainsIgnoreCase и String::AsciiEqualsIgnoreCase. -* Добавлены функции String::ReverseBytes и String::ReverseBits. +* Удалена функция `String::Reverse`, используйте `Unicode::Reverse`. +* Удалены функции `String::HasPrefixIgnoreCase` и `String::StartsWithIgnoreCase`, используйте `String::AsciiStartsWithIgnoreCase`. +* Удалены функции `String::HasSuffixIgnoreCase` и `String::EndsWithIgnoreCase`, используйте `String::AsciiEndsWithIgnoreCase`. +* Добавлены функции `String::AsciiContainsIgnoreCase` и `String::AsciiEqualsIgnoreCase`. +* Добавлены функции `String::ReverseBytes` и `String::ReverseBits`. diff --git a/yql/essentials/docs/ru/changelog/2025.03.md b/yql/essentials/docs/ru/changelog/2025.03.md index 0a25c97b15b..2f87f374da0 100644 --- a/yql/essentials/docs/ru/changelog/2025.03.md +++ b/yql/essentials/docs/ru/changelog/2025.03.md @@ -1,3 +1,3 @@ ## Изменения в модуле RE2 {#re2-module} -* Гарантируется падение запросов при передаче невалидных регулярных выражений +* Гарантируется падение запросов при передаче невалидных регулярных выражений. diff --git a/yql/essentials/docs/ru/udf/list/yson.md b/yql/essentials/docs/ru/udf/list/yson.md index 0f5d6094e83..2b5f86d9864 100644 --- a/yql/essentials/docs/ru/udf/list/yson.md +++ b/yql/essentials/docs/ru/udf/list/yson.md @@ -29,7 +29,7 @@ YSON — разработанный в Яндексе формат данных, * `Yson::YPath***` — получение одного элемента дерева документа по указанному относительному пути с опциональным преобразованием в нужный тип данных; * `Yson::Serialize***` — получить из ресурса копию его данных, сериализованную в одном из форматов; -* Для удобства при передаче сериализованного Yson и Json в функции, ожидающие на входе ресурс с DOM-объектом, неявное преобразование через `Yson::Parse` или `Yson::ParseJson` происходит автоматически. Также в SQL синтаксисе оператор точки или квадратных скобок автоматически добавляет вызов `Yson::Lookup`. Для сериализации ресурса по-прежнему нужно вызывать `Yson::ConvertTo***` или `Yson::Serialize***`. Таким образом, например, получение элемента "foo" словаря из колонки mycolumn типа Yson в виде строки может выглядеть так: `SELECT Yson::ConvertToString(mycolumn["foo"]) FROM mytable;` или `SELECT Yson::ConvertToString(mycolumn.foo) FROM mytable;`. В варианте с точкой можно экранировать спецсимволы по [общим правилам для индентификаторов](../../syntax/expressions.md#escape). +* Для удобства при передаче сериализованного Yson и Json в функции, ожидающие на входе ресурс с DOM-объектом, неявное преобразование через `Yson::Parse` или `Yson::ParseJson` происходит автоматически. Также в SQL синтаксисе оператор точки или квадратных скобок автоматически добавляет вызов `Yson::Lookup`. Для сериализации ресурса по-прежнему нужно вызывать `Yson::ConvertTo***` или `Yson::Serialize***`. Таким образом, например, получение элемента "foo" словаря из колонки mycolumn типа Yson в виде строки может выглядеть так: `SELECT Yson::ConvertToString(mycolumn["foo"]) FROM mytable;` или `SELECT Yson::ConvertToString(mycolumn.foo) FROM mytable;`. В варианте с точкой можно экранировать спецсимволы по [общим правилам для идентификаторов](../../syntax/expressions.md#escape). Функции модуля стоит рассматривать как «кубики», из которых можно собирать разные конструкции, например: @@ -82,7 +82,7 @@ Yson::ParseJson(String{Flags:AutoMap}) -> Resource<'Yson2.Node'>? Yson::ParseJsonDecodeUtf8(String{Flags:AutoMap}) -> Resource<'Yson2.Node'>? ``` -Результат всех трёх функций является несериализуемым: его можно только передать на вход другой функции из библиотеки Yson, но нельзя сохранить в таблицу или вернуть на клиент в результате операции — попытка так сделать приведет к ошибке типизации. Также запрещено возвращать его за пределы [подзапросов](../../syntax/select/index.md): если это требуется, то надо вызвать [Yson::Serialize](#ysonserialize), а оптимизатор уберёт лишнюю сериализию и десериализацию, если материализация в конечном счёте не потребуется. +Результат всех трёх функций является несериализуемым: его можно только передать на вход другой функции из библиотеки Yson, но нельзя сохранить в таблицу или вернуть на клиент в результате операции — попытка так сделать приведет к ошибке типизации. Также запрещено возвращать его за пределы [подзапросов](../../syntax/select/index.md): если это требуется, то надо вызвать [Yson::Serialize](#ysonserialize), а оптимизатор уберёт лишнюю сериализацию и десериализацию, если материализация в конечном счёте не потребуется. {% note info %} @@ -292,11 +292,11 @@ $yson = @@{y = true; x = 5.5}@@y; SELECT Yson::LookupBool($yson, "z"); --- null SELECT Yson::LookupBool($yson, "y"); --- true -SELECT Yson::LookupInt64($yson, "x"); --- Ошибка +-- SELECT Yson::LookupInt64($yson, "x"); --- Ошибка SELECT Yson::LookupInt64($yson, "x", Yson::Options(false as Strict)); --- null SELECT Yson::LookupInt64($yson, "x", Yson::Options(true as AutoConvert)); --- 5 -SELECT Yson::ConvertToBoolDict($yson); --- Ошибка +-- SELECT Yson::ConvertToBoolDict($yson); --- Ошибка SELECT Yson::ConvertToBoolDict($yson, Yson::Options(false as Strict)); --- { "y": true } SELECT Yson::ConvertToDoubleDict($yson, Yson::Options(false as Strict)); --- { "x": 5.5 } ``` diff --git a/yql/essentials/minikql/comp_nodes/mkql_udf.cpp b/yql/essentials/minikql/comp_nodes/mkql_udf.cpp index 8230a759d6b..44c45888c0a 100644 --- a/yql/essentials/minikql/comp_nodes/mkql_udf.cpp +++ b/yql/essentials/minikql/comp_nodes/mkql_udf.cpp @@ -180,7 +180,7 @@ public: NUdf::TSourcePosition pos, IComputationNode* runConfigNode, ui32 runConfigArgs, - const TCallableType* functionType, + const TCallableType* callableType, TType* userType) : TBaseComputation(mutables, EValueRepresentation::Boxed) , FunctionName(std::move(functionName)) @@ -188,7 +188,7 @@ public: , Pos(pos) , RunConfigNode(runConfigNode) , RunConfigArgs(runConfigArgs) - , FunctionType(functionType) + , CallableType(callableType) , UserType(userType) , UdfIndex(mutables.CurValueIndex++) { @@ -280,7 +280,7 @@ private: } void Wrap(NUdf::TUnboxedValue& callable) const { - TValidate<TValidatePolicy,TValidateMode>::WrapCallable(FunctionType, callable, TStringBuilder() << "FunctionWithConfig<" << FunctionName << ">"); + TValidate<TValidatePolicy,TValidateMode>::WrapCallable(CallableType, callable, TStringBuilder() << "FunctionWithConfig<" << FunctionName << ">"); } void RegisterDependencies() const final { @@ -292,7 +292,7 @@ private: const NUdf::TSourcePosition Pos; IComputationNode* const RunConfigNode; const ui32 RunConfigArgs; - const TCallableType* FunctionType; + const TCallableType* CallableType; TType* const UserType; const ui32 UdfIndex; }; @@ -421,7 +421,7 @@ IComputationNode* WrapUdf(TCallable& callable, const TComputationNodeFactoryCont const auto closureNodeType = runConfigNodeType->IsVoid() ? AS_TYPE(TCallableType, callableNodeType)->GetReturnType() : callableNodeType; - if (!closureNodeType->IsSameType(*closureFuncType)) { + if (!closureNodeType->IsConvertableTo(*closureFuncType)) { TString diff = TStringBuilder() << "type mismatch, expected return type: " << PrintNode(closureNodeType, true) @@ -438,7 +438,7 @@ IComputationNode* WrapUdf(TCallable& callable, const TComputationNodeFactoryCont const auto runConfigArgs = funcInfo.FunctionType->GetArgumentsCount(); return runConfigNodeType->IsVoid() ? CreateUdfWrapper<true>(ctx, std::move(funcName), std::move(typeConfig), pos, callableNodeType, callableFuncType, userType) - : CreateUdfWrapper<false>(ctx, std::move(funcName), std::move(typeConfig), pos, runConfigCompNode, runConfigArgs, callableFuncType, userType); + : CreateUdfWrapper<false>(ctx, std::move(funcName), std::move(typeConfig), pos, runConfigCompNode, runConfigArgs, callableNodeType, userType); } if (!callableFuncType->IsConvertableTo(*callableNodeType, true)) { @@ -462,7 +462,7 @@ IComputationNode* WrapUdf(TCallable& callable, const TComputationNodeFactoryCont } const auto runCfgCompNode = LocateNode(ctx.NodeLocator, *runCfgNode.GetNode()); - return CreateUdfWrapper<false>(ctx, std::move(funcName), std::move(typeConfig), pos, runCfgCompNode, 1U, callableFuncType, userType); + return CreateUdfWrapper<false>(ctx, std::move(funcName), std::move(typeConfig), pos, runCfgCompNode, 1U, callableNodeType, userType); } IComputationNode* WrapScriptUdf(TCallable& callable, const TComputationNodeFactoryContext& ctx) { diff --git a/yql/essentials/minikql/comp_nodes/ut/mkql_computation_node_ut.h b/yql/essentials/minikql/comp_nodes/ut/mkql_computation_node_ut.h index a118540e625..0d783f677d1 100644 --- a/yql/essentials/minikql/comp_nodes/ut/mkql_computation_node_ut.h +++ b/yql/essentials/minikql/comp_nodes/ut/mkql_computation_node_ut.h @@ -118,7 +118,7 @@ struct TSetup { Reset(); Explorer.Walk(pgm.GetNode(), Env->GetNodeStack()); TComputationPatternOpts opts(Alloc.Ref(), *Env, NodeFactory, - FunctionRegistry.Get(), NUdf::EValidateMode::None, NUdf::EValidatePolicy::Exception, + FunctionRegistry.Get(), NUdf::EValidateMode::Greedy, NUdf::EValidatePolicy::Exception, UseLLVM ? "" : "OFF", graphPerProcess, StatsRegistry.Get(), nullptr, nullptr); Pattern = MakeComputationPattern(Explorer, pgm, entryPoints, opts); auto graph = Pattern->Clone(opts.ToComputationOptions(*RandomProvider, *TimeProvider)); diff --git a/yql/essentials/minikql/comp_nodes/ut/mkql_udf_ut.cpp b/yql/essentials/minikql/comp_nodes/ut/mkql_udf_ut.cpp index 265ad3bd059..fe7e32e189d 100644 --- a/yql/essentials/minikql/comp_nodes/ut/mkql_udf_ut.cpp +++ b/yql/essentials/minikql/comp_nodes/ut/mkql_udf_ut.cpp @@ -1,10 +1,55 @@ #include "mkql_computation_node_ut.h" #include <yql/essentials/public/udf/udf_helpers.h> #include <yql/essentials/minikql/mkql_node_serialization.h> +#include <yql/essentials/minikql/mkql_node_cast.h> namespace NKikimr { namespace NMiniKQL { +// XXX: Emulate type transformations similar to the one made by +// type annotation and compilation phases. As a result, the name +// (i.e. "UDF") of callable type is lost. Hence, the type resolved +// at the type annotation phase (and, ergo, used for bytecode +// compilation) differs from the type, resolved for the underline +// function at runtime. +template<typename TUdf> +static TType* TweakUdfType(const NYql::NUdf::TStringRef& name, TType* userType, + const TTypeEnvironment& env) +{ + TFunctionTypeInfoBuilder typeInfoBuilder(NYql::UnknownLangVersion, env, + new TTypeInfoHelper(), + "", nullptr, {}); + + // Obtain the callable type of the particular UDF. + TFunctionTypeInfo funcInfo; + UNIT_ASSERT(TUdf::DeclareSignature(name, userType, typeInfoBuilder, true)); + typeInfoBuilder.Build(&funcInfo); + + // Create the new MiniKQL type to emulate two conversions: + // * Convert the given MiniKQL type to the expression type. + // See <NYql::NCommon::ConvertMiniKQLType>. + // * Convert the expression type back to the MiniKQL one. + // See <NYql::NCommon::TMkqlBuildContext::BuildType>. + // The aforementioned conversions are made by the pipeline in + // scope of type annotation and compilation phases. + // As a result of the first conversion, the name of the + // callable type is lost, so the new MiniKQL type has to be + // the same as the resolved one, but the name is omitted. + const auto funcType = AS_TYPE(TCallableType, funcInfo.FunctionType); + TVector<TType*> argsTypes; + for (size_t i = 0; i < funcType->GetArgumentsCount(); i++) { + argsTypes.push_back(funcType->GetArgumentType(i)); + } + const auto nodeType = TCallableType::Create("", /* Name has to be empty. */ + funcType->GetReturnType(), + funcType->GetArgumentsCount(), + argsTypes.data(), + funcType->GetPayload(), + env); + nodeType->SetOptionalArgumentsCount(funcType->GetOptionalArgumentsCount()); + return nodeType; +}; + class TImpl : public NYql::NUdf::TBoxedValue { public: explicit TImpl(NYql::NUdf::TSourcePosition pos, @@ -219,7 +264,9 @@ Y_UNIT_TEST_SUITE(TMiniKQLUdfTest) { pb.NewTupleType({strType}), pb.NewEmptyStructType(), pb.NewEmptyTupleType()}); - const auto udf = pb.Udf("TestModule.Test", upvalue, userType); + + const auto udfType = TweakUdfType<TRunConfig>("Test", userType, *compileSetup.Env); + const auto udf = pb.TypedUdf("TestModule.Test", udfType, upvalue, userType); const auto list = pb.NewList(strType, {value}); const auto pgmReturn = pb.Map(list, [&pb, udf](const TRuntimeNode item) { @@ -268,7 +315,9 @@ Y_UNIT_TEST_SUITE(TMiniKQLUdfTest) { pb.NewTupleType({strType}), pb.NewEmptyStructType(), pb.NewEmptyTupleType()}); - const auto udf = pb.Udf("TestModule.Test", pb.NewVoid(), userType); + + const auto udfType = TweakUdfType<TCurrying>("Test", userType, *compileSetup.Env); + const auto udf = pb.TypedUdf("TestModule.Test", udfType, pb.NewVoid(), userType); const auto closure = pb.Apply(udf, {upvalue, optional}); const auto list = pb.NewList(strType, {value}); diff --git a/yql/essentials/providers/common/udf_resolve/yql_udf_resolver_logger.cpp b/yql/essentials/providers/common/udf_resolve/yql_udf_resolver_logger.cpp index 5cb14c7619c..d09d7cfafce 100644 --- a/yql/essentials/providers/common/udf_resolve/yql_udf_resolver_logger.cpp +++ b/yql/essentials/providers/common/udf_resolve/yql_udf_resolver_logger.cpp @@ -12,7 +12,7 @@ using namespace NYql; class TUdfResolverWithLoggerDecorator : public IUdfResolver { public: TUdfResolverWithLoggerDecorator(IUdfResolver::TPtr underlying, const TString& path, const TString& sessionId) - : Underlying_(underlying), Out_(TFile(path, WrOnly | ForAppend)), SessionId_(sessionId) {} + : Underlying_(underlying), Out_(TFile(path, WrOnly | ForAppend | OpenAlways)), SessionId_(sessionId) {} TMaybe<TFilePathWithMd5> GetSystemModulePath(const TStringBuf& moduleName) const override { return Underlying_->GetSystemModulePath(moduleName); diff --git a/yt/cpp/mapreduce/http_client/raw_client.cpp b/yt/cpp/mapreduce/http_client/raw_client.cpp index cb5a852b176..f50b239c2aa 100644 --- a/yt/cpp/mapreduce/http_client/raw_client.cpp +++ b/yt/cpp/mapreduce/http_client/raw_client.cpp @@ -569,10 +569,10 @@ std::unique_ptr<IInputStream> THttpRawClient::ReadFile( { TMutationId mutationId; THttpHeader header("GET", GetReadFileCommand(Context_.Config->ApiVersion)); - header.AddTransactionId(transactionId); header.SetOutputFormat(TMaybe<TFormat>()); // Binary format - header.MergeParameters(FormIORequestParameters(path, options)); header.SetResponseCompression(ToString(Context_.Config->AcceptEncoding)); + header.MergeParameters(NRawClient::SerializeParamsForReadFile(transactionId, options)); + header.MergeParameters(FormIORequestParameters(path, options)); TRequestConfig config; config.IsHeavy = true; @@ -740,23 +740,7 @@ TNode::TListType THttpRawClient::SelectRows( THttpHeader header("GET", "select_rows"); header.SetInputFormat(TFormat::YsonBinary()); header.SetOutputFormat(TFormat::YsonBinary()); - - header.MergeParameters(BuildYsonNodeFluently().BeginMap() - .Item("query").Value(query) - .DoIf(options.Timeout_.Defined(), [&] (TFluentMap fluent) { - fluent.Item("timeout").Value(static_cast<i64>(options.Timeout_->MilliSeconds())); - }) - .DoIf(options.InputRowLimit_.Defined(), [&] (TFluentMap fluent) { - fluent.Item("input_row_limit").Value(*options.InputRowLimit_); - }) - .DoIf(options.OutputRowLimit_.Defined(), [&] (TFluentMap fluent) { - fluent.Item("output_row_limit").Value(*options.OutputRowLimit_); - }) - .Item("range_expansion_limit").Value(options.RangeExpansionLimit_) - .Item("fail_on_incomplete_result").Value(options.FailOnIncompleteResult_) - .Item("verbose_logging").Value(options.VerboseLogging_) - .Item("enable_code_cache").Value(options.EnableCodeCache_) - .EndMap()); + header.MergeParameters(NRawClient::SerializeParamsForSelectRows(query, options)); TRequestConfig config; config.IsHeavy = true; diff --git a/yt/cpp/mapreduce/http_client/rpc_parameters_serialization.cpp b/yt/cpp/mapreduce/http_client/rpc_parameters_serialization.cpp index 14ab0e1db85..7a8ce040ac0 100644 --- a/yt/cpp/mapreduce/http_client/rpc_parameters_serialization.cpp +++ b/yt/cpp/mapreduce/http_client/rpc_parameters_serialization.cpp @@ -71,13 +71,24 @@ static void SetFirstLastTabletIndex(TNode* node, const TOptions& options) } template <typename T> -void SerializeMasterReadOptions(TNode* node, const TMasterReadOptions<T>& options) +static void SerializeMasterReadOptions(TNode* node, const TMasterReadOptions<T>& options) { if (options.ReadFrom_) { (*node)["read_from"] = ToString(*options.ReadFrom_); } } +template <typename T> +static void SerializeSuppressableAccessTrackingOptions(TNode* node, const TSuppressableAccessTrackingOptions<T>& options) +{ + if (options.SuppressAccessTracking_) { + (*node)["suppress_access_tracking"] = options.SuppressAccessTracking_; + } + if (options.SuppressModificationTracking_) { + (*node)["suppress_modification_tracking"] = options.SuppressAccessTracking_; + } +} + //////////////////////////////////////////////////////////////////////////////// TNode SerializeParamsForCreate( @@ -124,6 +135,7 @@ TNode SerializeParamsForExists( SetTransactionIdParam(&result, transactionId); SetPathParam(&result, pathPrefix, path); SerializeMasterReadOptions(&result, options); + SerializeSuppressableAccessTrackingOptions(&result, options); return result; } @@ -137,6 +149,7 @@ TNode SerializeParamsForGet( SetTransactionIdParam(&result, transactionId); SetPathParam(&result, pathPrefix, path); SerializeMasterReadOptions(&result, options); + SerializeSuppressableAccessTrackingOptions(&result, options); if (options.AttributeFilter_) { result["attributes"] = SerializeAttributeFilter(*options.AttributeFilter_); } @@ -155,6 +168,7 @@ TNode SerializeParamsForSet( TNode result; SetTransactionIdParam(&result, transactionId); SetPathParam(&result, pathPrefix, path); + SerializeSuppressableAccessTrackingOptions(&result, options); result["recursive"] = options.Recursive_; if (options.Force_) { result["force"] = *options.Force_; @@ -171,6 +185,7 @@ TNode SerializeParamsForMultisetAttributes( TNode result; SetTransactionIdParam(&result, transactionId); SetPathParam(&result, pathPrefix, path); + SerializeSuppressableAccessTrackingOptions(&result, options); if (options.Force_) { result["force"] = *options.Force_; } @@ -187,6 +202,7 @@ TNode SerializeParamsForList( SetTransactionIdParam(&result, transactionId); SetPathParam(&result, pathPrefix, path); SerializeMasterReadOptions(&result, options); + SerializeSuppressableAccessTrackingOptions(&result, options); if (options.MaxSize_) { result["max_size"] = *options.MaxSize_; } @@ -598,6 +614,29 @@ TNode SerializeParamsForListJobs( return result; } +TNode SerializeParamsForSelectRows( + const TString& query, + const TSelectRowsOptions& options) +{ + TNode result; + result["query"] = query; + SerializeSuppressableAccessTrackingOptions(&result, options); + if (options.Timeout_) { + result["timeout"] = static_cast<i64>(options.Timeout_->MilliSeconds()); + } + if (options.InputRowLimit_) { + result["input_row_limit"] = *options.InputRowLimit_; + } + if (options.OutputRowLimit_) { + result["output_row_limit"] = *options.OutputRowLimit_; + } + result["range_expansion_limit"] = options.RangeExpansionLimit_; + result["fail_on_incomplete_result"] = options.FailOnIncompleteResult_; + result["verbose_logging"] = options.VerboseLogging_; + result["enable_code_cache"] = options.EnableCodeCache_; + return result; +} + TNode SerializeParametersForInsertRows( const TString& pathPrefix, const TYPath& path, @@ -658,6 +697,7 @@ TNode SerializeParamsForReadTable( { TNode result; SetTransactionIdParam(&result, transactionId); + SerializeSuppressableAccessTrackingOptions(&result, options); result["control_attributes"] = BuildYsonNodeFluently() .BeginMap() .Item("enable_row_index").Value(options.ControlAttributes_.EnableRowIndex_) @@ -666,10 +706,11 @@ TNode SerializeParamsForReadTable( return result; } -TNode SerializeParamsForReadTablePartition(const TString& cookie, const TTablePartitionReaderOptions& /*options*/) +TNode SerializeParamsForReadTablePartition(const TString& cookie, const TTablePartitionReaderOptions& options) { TNode node; node["cookie"] = cookie; + SerializeSuppressableAccessTrackingOptions(&node, options); return node; } @@ -691,6 +732,7 @@ TNode SerializeParamsForReadBlobTable( .UpperLimit(TReadLimit().Key(upperLimitKey)))); SetTransactionIdParam(&result, transactionId); + SerializeSuppressableAccessTrackingOptions(&result, options); result["start_part_index"] = options.StartPartIndex_; result["offset"] = options.Offset_; @@ -824,6 +866,16 @@ TNode SerializeParamsForGetTablePartitions( return result; } +TNode SerializeParamsForReadFile( + const TTransactionId& transactionId, + const TFileReaderOptions& options) +{ + TNode result; + SetTransactionIdParam(&result, transactionId); + SerializeSuppressableAccessTrackingOptions(&result, options); + return result; +} + TNode SerializeParamsForGetFileFromCache( const TTransactionId& transactionId, const TString& md5Signature, diff --git a/yt/cpp/mapreduce/http_client/rpc_parameters_serialization.h b/yt/cpp/mapreduce/http_client/rpc_parameters_serialization.h index 90c8dd8e247..560d3263732 100644 --- a/yt/cpp/mapreduce/http_client/rpc_parameters_serialization.h +++ b/yt/cpp/mapreduce/http_client/rpc_parameters_serialization.h @@ -140,6 +140,10 @@ TNode SerializeParamsForGetJobTrace( const TOperationId& operationId, const TGetJobTraceOptions& options); +TNode SerializeParamsForSelectRows( + const TString& query, + const TSelectRowsOptions& options); + TNode SerializeParametersForInsertRows( const TString& pathPrefix, const TYPath& path, @@ -208,6 +212,10 @@ TNode SerializeParamsForGetTablePartitions( const TVector<TRichYPath>& paths, const TGetTablePartitionsOptions& options); +TNode SerializeParamsForReadFile( + const TTransactionId& transactionId, + const TFileReaderOptions& options); + TNode SerializeParamsForGetFileFromCache( const TTransactionId& transactionId, const TString& md5Signature, diff --git a/yt/cpp/mapreduce/interface/client_method_options.h b/yt/cpp/mapreduce/interface/client_method_options.h index 10c200715df..0fc8c8d00fa 100644 --- a/yt/cpp/mapreduce/interface/client_method_options.h +++ b/yt/cpp/mapreduce/interface/client_method_options.h @@ -50,6 +50,21 @@ enum class EComplexTypeMode : int Positional /* "positional" */, }; +/// Base class for options dealing with access tracking suppression. +template <typename TDerived> +struct TSuppressableAccessTrackingOptions +{ + /// @cond Doxygen_Suppress + using TSelf = TDerived; + /// @endcond + + /// @brief Whether to disable update of access_time on read/write operations. + FLUENT_FIELD_DEFAULT(bool, SuppressAccessTracking, false); + + /// @brief Whether to disable update of modification on read/write operations. + FLUENT_FIELD_DEFAULT(bool, SuppressModificationTracking, false); +}; + /// /// @brief Options for @ref NYT::ICypressClient::Create /// @@ -118,6 +133,7 @@ struct TMasterReadOptions /// @see https://ytsaurus.tech/docs/en/api/commands.html#exists struct TExistsOptions : public TMasterReadOptions<TExistsOptions> + , public TSuppressableAccessTrackingOptions<TExistsOptions> { }; @@ -127,6 +143,7 @@ struct TExistsOptions /// @see https://ytsaurus.tech/docs/en/api/commands.html#get struct TGetOptions : public TMasterReadOptions<TGetOptions> + , public TSuppressableAccessTrackingOptions<TGetOptions> { /// @brief Attributes that should be fetched with each node. FLUENT_FIELD_OPTION(TAttributeFilter, AttributeFilter); @@ -140,6 +157,7 @@ struct TGetOptions /// /// @see https://ytsaurus.tech/docs/en/api/commands.html#set struct TSetOptions + : public TSuppressableAccessTrackingOptions<TSetOptions> { /// @cond Doxygen_Suppress using TSelf = TSetOptions; @@ -157,6 +175,7 @@ struct TSetOptions /// /// @see https://ytsaurus.tech/docs/en/api/commands.html#multiset_attributes struct TMultisetAttributesOptions + : public TSuppressableAccessTrackingOptions<TMultisetAttributesOptions> { /// @cond Doxygen_Suppress using TSelf = TMultisetAttributesOptions; @@ -171,6 +190,7 @@ struct TMultisetAttributesOptions /// @see https://ytsaurus.tech/docs/en/api/commands.html#list struct TListOptions : public TMasterReadOptions<TListOptions> + , public TSuppressableAccessTrackingOptions<TListOptions> { /// @cond Doxygen_Suppress using TSelf = TListOptions; @@ -277,6 +297,7 @@ struct TConcatenateOptions /// /// @see https://ytsaurus.tech/docs/en/api/commands.html#read_blob_table struct TBlobTableReaderOptions + : public TSuppressableAccessTrackingOptions<TBlobTableReaderOptions> { /// @cond Doxygen_Suppress using TSelf = TBlobTableReaderOptions; @@ -477,6 +498,7 @@ struct TIOOptions /// @brief Options for reading file from YT. struct TFileReaderOptions : public TIOOptions<TFileReaderOptions> + , public TSuppressableAccessTrackingOptions<TFileReaderOptions> { /// /// @brief Offset to start reading from. @@ -670,6 +692,7 @@ public: /// Options for @ref NYT::IClient::CreateTableReader struct TTableReaderOptions : public TIOOptions<TTableReaderOptions> + , public TSuppressableAccessTrackingOptions<TTableReaderOptions> { /// @deprecated Size of internal client buffer. FLUENT_FIELD_DEFAULT(size_t, SizeLimit, 4 << 20); @@ -688,6 +711,7 @@ struct TTableReaderOptions /// Options for @ref NYT::IClient::CreatePartitionTableReader struct TTablePartitionReaderOptions + : public TSuppressableAccessTrackingOptions<TTablePartitionReaderOptions> { /// @cond Doxygen_Suppress using TSelf = TTablePartitionReaderOptions; @@ -1027,6 +1051,7 @@ struct TLookupRowsOptions /// /// @see https://ytsaurus.tech/docs/en/api/commands#select_rows struct TSelectRowsOptions + : public TSuppressableAccessTrackingOptions<TSelectRowsOptions> { /// @cond Doxygen_Suppress using TSelf = TSelectRowsOptions; diff --git a/yt/yql/providers/yt/provider/yql_yt_join_impl.cpp b/yt/yql/providers/yt/provider/yql_yt_join_impl.cpp index a5a6d119d58..04ae3a8943d 100644 --- a/yt/yql/providers/yt/provider/yql_yt_join_impl.cpp +++ b/yt/yql/providers/yt/provider/yql_yt_join_impl.cpp @@ -1724,18 +1724,6 @@ bool RewriteYtMapJoin(TYtEquiJoin equiJoin, const TJoinLabels& labels, bool isLo YQL_CLOG(INFO, ProviderYt) << strategyName << " assumes unique keys for the small table"; } - ui64 partCount = 1; - ui64 partRows = settings.RightRows; - if ((settings.RightSize > 0) && useShards) { - partCount = (settings.RightMemSize + settings.MapJoinLimit - 1) / settings.MapJoinLimit; - partRows = (settings.RightRows + partCount - 1) / partCount; - } - - if (partCount > 1) { - YQL_ENSURE(!isLookupJoin); - YQL_CLOG(INFO, ProviderYt) << strategyName << " sharded into " << partCount << " parts, each " << partRows << " rows"; - } - auto leftKeyColumns = settings.SwapTables ? op.RightLabel : op.LeftLabel; auto rightKeyColumns = settings.SwapTables ? op.LeftLabel : op.RightLabel; auto joinTree = ctx.NewList(pos, { @@ -1776,6 +1764,28 @@ bool RewriteYtMapJoin(TYtEquiJoin equiJoin, const TJoinLabels& labels, bool isLo } } + if (useBlocks) { + for (auto& [_, columnType] : columnTypes) { + if (!IsSupportedAsBlockType(pos, *columnType, ctx, *state->Types)) { + useBlocks = false; + YQL_CLOG(INFO, ProviderYt) << "Block mapjoin won't be used because of unsupported type: " << *columnType; + break; + } + } + } + + ui64 partCount = 1; + ui64 partRows = settings.RightRows; + if ((settings.RightSize > 0) && useShards) { + partCount = std::min(((useBlocks ? settings.RightMemSizeUsingBlocks : settings.RightMemSize) + settings.MapJoinLimit - 1) / settings.MapJoinLimit, settings.RightRows); + partRows = (settings.RightRows + partCount - 1) / partCount; + } + + if (partCount > 1) { + YQL_ENSURE(!isLookupJoin); + YQL_CLOG(INFO, ProviderYt) << strategyName << " sharded into " << partCount << " parts, each " << partRows << " rows"; + } + auto mainPaths = MakeUnorderedSection(leftLeaf.Section, ctx).Paths(); auto mainSettings = NYql::RemoveSettings(leftLeaf.Section.Settings().Ref(), EYtSettingType::JoinLabel | EYtSettingType::StatColumns, ctx); auto smallPaths = MakeUnorderedSection(rightLeaf.Section, ctx).Paths(); @@ -2117,16 +2127,6 @@ bool RewriteYtMapJoin(TYtEquiJoin equiJoin, const TJoinLabels& labels, bool isLo } } - if (useBlocks) { - for (auto& [_, columnType] : columnTypes) { - if (!IsSupportedAsBlockType(pos, *columnType, ctx, *state->Types)) { - useBlocks = false; - YQL_CLOG(INFO, ProviderYt) << "Block mapjoin won't be used because of unsupported type: " << *columnType; - break; - } - } - } - TExprNode::TPtr joined; if (useBlocks) { TExprNode::TListType leftKeyColumnNodes; diff --git a/yt/yql/tests/sql/suites/aggregate/too_wide.sql b/yt/yql/tests/sql/suites/aggregate/too_wide.sql new file mode 100644 index 00000000000..71b4319224a --- /dev/null +++ b/yt/yql/tests/sql/suites/aggregate/too_wide.sql @@ -0,0 +1,122 @@ +use plato; + +select * from (select cast(value as Int32) as value from Input) group by +value+0, +value+1, +value+2, +value+3, +value+4, +value+5, +value+6, +value+7, +value+8, +value+9, +value+10, +value+11, +value+12, +value+13, +value+14, +value+15, +value+16, +value+17, +value+18, +value+19, +value+20, +value+21, +value+22, +value+23, +value+24, +value+25, +value+26, +value+27, +value+28, +value+29, +value+30, +value+31, +value+32, +value+33, +value+34, +value+35, +value+36, +value+37, +value+38, +value+39, +value+40, +value+41, +value+42, +value+43, +value+44, +value+45, +value+46, +value+47, +value+48, +value+49, +value+50, +value+51, +value+52, +value+53, +value+54, +value+55, +value+56, +value+57, +value+58, +value+59, +value+60, +value+61, +value+62, +value+63, +value+64, +value+65, +value+66, +value+67, +value+68, +value+69, +value+70, +value+71, +value+72, +value+73, +value+74, +value+75, +value+76, +value+77, +value+78, +value+79, +value+80, +value+81, +value+82, +value+83, +value+84, +value+85, +value+86, +value+87, +value+88, +value+89, +value+90, +value+91, +value+92, +value+93, +value+94, +value+95, +value+96, +value+97, +value+98, +value+99, +value+100, +value+101, +value+102, +value+103, +value+104, +value+105, +value+106, +value+107, +value+108, +value+109, +value+110, +value+111, +value+112, +value+113, +value+114, +value+115, +value+116, +value+117, +value+118 diff --git a/yt/yt/client/api/rpc_proxy/transaction_impl.cpp b/yt/yt/client/api/rpc_proxy/transaction_impl.cpp index 164c5234049..bfd6d96b391 100644 --- a/yt/yt/client/api/rpc_proxy/transaction_impl.cpp +++ b/yt/yt/client/api/rpc_proxy/transaction_impl.cpp @@ -320,6 +320,7 @@ TFuture<TTransactionCommitResult> TTransaction::Commit(const TTransactionCommitO ToProto(req->mutable_transaction_id(), GetId()); ToProto(req->mutable_additional_participant_cell_ids(), AdditionalParticipantCellIds_); ToProto(req->mutable_prerequisite_options(), options); + req->set_max_allowed_commit_timestamp(options.MaxAllowedCommitTimestamp); return req->Invoke(); })) .Apply( diff --git a/yt/yt/core/bus/tcp/config.cpp b/yt/yt/core/bus/tcp/config.cpp index ed4aea2f75c..e88699accd7 100644 --- a/yt/yt/core/bus/tcp/config.cpp +++ b/yt/yt/core/bus/tcp/config.cpp @@ -157,12 +157,6 @@ void TBusConfig::Register(TRegistrar registrar) .Default(EEncryptionMode::Optional); registrar.Parameter("verification_mode", &TThis::VerificationMode) .Default(EVerificationMode::None); - registrar.Parameter("ca", &TThis::CA) - .Default(); - registrar.Parameter("cert_chain", &TThis::CertificateChain) - .Default(); - registrar.Parameter("private_key", &TThis::PrivateKey) - .Default(); registrar.Parameter("cipher_list", &TThis::CipherList) .Default(); registrar.Parameter("load_certs_from_bus_certs_directory", &TThis::LoadCertsFromBusCertsDirectory) diff --git a/yt/yt/core/bus/tcp/config.h b/yt/yt/core/bus/tcp/config.h index 1596590a678..84a8b174afd 100644 --- a/yt/yt/core/bus/tcp/config.h +++ b/yt/yt/core/bus/tcp/config.h @@ -90,6 +90,7 @@ DEFINE_REFCOUNTED_TYPE(TTcpDispatcherDynamicConfig) struct TBusConfig : public NNet::TDialerConfig + , public NCrypto::TSslContextConfig { bool EnableQuickAck; @@ -110,9 +111,6 @@ struct TBusConfig // Ssl options. EEncryptionMode EncryptionMode; EVerificationMode VerificationMode; - NCrypto::TPemBlobConfigPtr CA; - NCrypto::TPemBlobConfigPtr CertificateChain; - NCrypto::TPemBlobConfigPtr PrivateKey; std::optional<TString> CipherList; bool LoadCertsFromBusCertsDirectory; std::optional<TString> PeerAlternativeHostName; diff --git a/yt/yt/core/bus/tcp/connection.cpp b/yt/yt/core/bus/tcp/connection.cpp index 52ec0dfd1c2..9bd094e940a 100644 --- a/yt/yt/core/bus/tcp/connection.cpp +++ b/yt/yt/core/bus/tcp/connection.cpp @@ -3,8 +3,6 @@ #include "config.h" #include "local_bypass.h" #include "dispatcher_impl.h" -#include "ssl_context.h" -#include "ssl_helpers.h" #include <yt/yt/core/misc/fs.h> #include <yt/yt/core/misc/memory_usage_tracker.h> @@ -40,6 +38,7 @@ namespace NYT::NBus { using namespace NConcurrency; +using namespace NCrypto; using namespace NFS; using namespace NNet; using namespace NYTree; @@ -1969,8 +1968,7 @@ bool TTcpConnection::CheckSslReadError(ssize_t result) [[fallthrough]]; default: UpdateBusCounter(&TBusNetworkBandCounters::ReadErrors, 1); - Abort(TError(NBus::EErrorCode::SslError, "TLS/SSL read error") - << TErrorAttribute("ssl_error", GetLastSslErrorString()) + Abort(GetLastSslError("TLS/SSL read error") << TErrorAttribute("sys_error", TError::FromSystem(LastSystemError()))); break; } @@ -1997,8 +1995,7 @@ bool TTcpConnection::CheckSslWriteError(ssize_t result) [[fallthrough]]; default: UpdateBusCounter(&TBusNetworkBandCounters::WriteErrors, 1); - Abort(TError(NBus::EErrorCode::SslError, "TLS/SSL write error") - << TErrorAttribute("ssl_error", GetLastSslErrorString()) + Abort(GetLastSslError("TLS/SSL write error") << TErrorAttribute("sys_error", TError::FromSystem(LastSystemError()))); break; } @@ -2055,8 +2052,7 @@ bool TTcpConnection::DoSslHandshake() break; } - Abort(TError(NBus::EErrorCode::SslError, "Failed to establish TLS/SSL session") - << TErrorAttribute("ssl_error", GetLastSslErrorString()) + Abort(GetLastSslError("Failed to establish TLS/SSL session") << TErrorAttribute("sys_error", TError::FromSystem(LastSystemError()))); return false; } @@ -2094,78 +2090,45 @@ void TTcpConnection::TryEstablishSslSession() YT_LOG_DEBUG("Starting TLS/SSL connection"); - if (Config_->LoadCertsFromBusCertsDirectory && !TTcpDispatcher::TImpl::Get()->GetBusCertsDirectoryPath()) { - Abort(TError(NBus::EErrorCode::SslError, "bus_certs_directory_path is not set in tcp_dispatcher config")); - return; + NCrypto::TCertificatePathResolver pathResolver; + if (Config_->LoadCertsFromBusCertsDirectory) { + // FIXME(khlebnikov): Move this and config into proper place. + if (!TTcpDispatcher::TImpl::Get()->GetBusCertsDirectoryPath()) { + Abort(TError(NBus::EErrorCode::SslError, "bus_certs_directory_path is not set in tcp_dispatcher config")); + return; + } + pathResolver = [&](const TString fileName) { + return JoinPaths(*TTcpDispatcher::TImpl::Get()->GetBusCertsDirectoryPath(), fileName); + }; } - Ssl_.reset(SSL_new(TSslContext::Get()->GetSslCtx())); - if (!Ssl_) { - Abort(TError(NBus::EErrorCode::SslError, "Failed to create a new SSL structure: %v", GetLastSslErrorString())); + if (VerificationMode_ != EVerificationMode::None && !Config_->CertificateAuthority) { + Abort(TError(NBus::EErrorCode::SslError, "CA file is not set in bus config")); return; } - if (SSL_set_fd(Ssl_.get(), Socket_) != 1) { - Abort(TError(NBus::EErrorCode::SslError, "Failed to bind socket to SSL handle: %v", GetLastSslErrorString())); + if (ConnectionType_ == EConnectionType::Server && !Config_->CertificateChain) { + Abort(TError(NBus::EErrorCode::SslError, "Certificate chain file is not set in bus config")); return; } - if (Config_->CipherList) { - if (SSL_set_cipher_list(Ssl_.get(), Config_->CipherList->data()) != 1) { - Abort(TError(NBus::EErrorCode::SslError, "Failed to set cipher list: %v", GetLastSslErrorString())); - return; - } + if (ConnectionType_ == EConnectionType::Server && !Config_->PrivateKey) { + Abort(TError(NBus::EErrorCode::SslError, "The private key file is not set in bus config")); + return; } -#define GET_CERT_FILE_PATH(file) \ - (Config_->LoadCertsFromBusCertsDirectory ? JoinPaths(*TTcpDispatcher::TImpl::Get()->GetBusCertsDirectoryPath(), (file)) : (file)) + // FIXME(khlebnikov): Stop constructing SSL context from scratch for each connection. + auto sslContext = New<TSslContext>(); - if (ConnectionType_ == EConnectionType::Server) { - SSL_set_accept_state(Ssl_.get()); + sslContext->ApplyConfig(Config_, pathResolver); - if (!Config_->CertificateChain) { - Abort(TError(NBus::EErrorCode::SslError, "Certificate chain file is not set in bus config")); - return; - } - - if (Config_->CertificateChain->FileName) { - const auto& certChainFile = GET_CERT_FILE_PATH(*Config_->CertificateChain->FileName); - if (SSL_use_certificate_chain_file(Ssl_.get(), certChainFile.data()) != 1) { - Abort(TError(NBus::EErrorCode::SslError, "Failed to load certificate chain file: %v", GetLastSslErrorString())); - return; - } - } else { - if (!UseCertificateChain(*Config_->CertificateChain->Value, Ssl_.get())) { - Abort(TError(NBus::EErrorCode::SslError, "Failed to load certificate chain: %v", GetLastSslErrorString())); - return; - } - } - - if (!Config_->PrivateKey) { - Abort(TError(NBus::EErrorCode::SslError, "The private key file is not set in bus config")); - return; - } + if (Config_->CipherList) { + sslContext->SetCipherList(*Config_->CipherList); + } - if (Config_->PrivateKey->FileName) { - const auto& privateKeyFile = GET_CERT_FILE_PATH(*Config_->PrivateKey->FileName); - if (SSL_use_PrivateKey_file(Ssl_.get(), privateKeyFile.data(), SSL_FILETYPE_PEM) != 1) { - Abort(TError(NBus::EErrorCode::SslError, "Failed to load private key file: %v", GetLastSslErrorString())); - return; - } - } else { - if (!UsePrivateKey(*Config_->PrivateKey->Value, Ssl_.get())) { - Abort(TError(NBus::EErrorCode::SslError, "Failed to load private key: %v", GetLastSslErrorString())); - return; - } - } + sslContext->Commit(); - if (SSL_check_private_key(Ssl_.get()) != 1) { - Abort(TError(NBus::EErrorCode::SslError, "Failed to check the consistency of a private key with the corresponding certificate: %v", GetLastSslErrorString())); - return; - } - } else { - SSL_set_connect_state(Ssl_.get()); - } + Ssl_ = std::move(sslContext->NewSsl()); switch (VerificationMode_) { case EVerificationMode::Full: @@ -2174,43 +2137,35 @@ void TTcpConnection::TryEstablishSslSession() if (Config_->PeerAlternativeHostName) { // Set hostname for peer certificate verification. if (SSL_set1_host(Ssl_.get(), EndpointHostName_.c_str()) != 1) { - Abort(TError(NBus::EErrorCode::SslError, "Failed to set hostname %v for peer certificate verification", EndpointHostName_)); + Abort(GetLastSslError("Failed to set hostname for peer certificate verification") + << TErrorAttribute("ssl_peer_hostname", EndpointHostName_)); return; } // Add alternative hostname for peer certificate verification. if (SSL_add1_host(Ssl_.get(), Config_->PeerAlternativeHostName->c_str()) != 1) { - Abort(TError(NBus::EErrorCode::SslError, "Failed to add alternative hostname %v for peer certificate verification", *Config_->PeerAlternativeHostName)); + Abort(GetLastSslError("Failed to add alternative hostname for peer certificate verification") + << TErrorAttribute("ssl_peer_hostname", *Config_->PeerAlternativeHostName)); return; } } else if (auto networkAddress = TNetworkAddress::TryParse(EndpointHostName_); networkAddress.IsOK() && networkAddress.Value().IsIP()) { // Set IP address for peer certificate verification. auto address = ToString(networkAddress.Value(), {.IncludePort = false, .IncludeTcpProtocol = false}); if (X509_VERIFY_PARAM_set1_ip_asc(SSL_get0_param(Ssl_.get()), address.c_str()) != 1) { - Abort(TError(NBus::EErrorCode::SslError, "Failed to set IP address %v for peer certificate verification", address)); + Abort(GetLastSslError("Failed to set IP address for peer certificate verification") + << TErrorAttribute("ssl_peer_address", address)); return; } } else { // Set hostname for peer certificate verification. if (SSL_set1_host(Ssl_.get(), EndpointHostName_.c_str()) != 1) { - Abort(TError(NBus::EErrorCode::SslError, "Failed to set hostname %v for peer certificate verification", EndpointHostName_)); + Abort(GetLastSslError("Failed to set hostname for peer certificate verification") + << TErrorAttribute("ssl_peer_hostname", EndpointHostName_)); return; } } [[fallthrough]]; case EVerificationMode::Ca: { - if (!Config_->CA) { - Abort(TError(NBus::EErrorCode::SslError, "CA file is not set in bus config")); - return; - } - - if (Config_->CA->FileName) { - const auto& caFile = GET_CERT_FILE_PATH(*Config_->CA->FileName); - TSslContext::Get()->LoadCAFileIfNotLoaded(caFile); - } else { - TSslContext::Get()->UseCAIfNotUsed(*Config_->CA->Value); - } - // Enable verification of the peer's certificate with the CA. SSL_set_verify(Ssl_.get(), SSL_VERIFY_PEER, /*callback*/ nullptr); break; @@ -2221,6 +2176,24 @@ void TTcpConnection::TryEstablishSslSession() YT_ABORT(); } + SSL_set_mode(Ssl_.get(), SSL_MODE_ENABLE_PARTIAL_WRITE | SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER); + + if (SSL_set_fd(Ssl_.get(), Socket_) != 1) { + Abort(GetLastSslError("Failed to bind socket to SSL handle")); + return; + } + + if (ConnectionType_ == EConnectionType::Server) { + SSL_set_accept_state(Ssl_.get()); + + if (SSL_check_private_key(Ssl_.get()) != 1) { + Abort(GetLastSslError("Failed to check the consistency of a private key with the corresponding certificate")); + return; + } + } else { + SSL_set_connect_state(Ssl_.get()); + } + PendingSslHandshake_ = DoSslHandshake(); // Check that connection hasn't been aborted in DoSslHandshake(). @@ -2230,8 +2203,6 @@ void TTcpConnection::TryEstablishSslSession() NetworkCounters_.Exchange(TTcpDispatcher::TImpl::Get()->GetCounters(NetworkName_, IsEncrypted())); UpdateConnectionCount(1); } - -#undef GET_CERT_FILE_PATH } bool TTcpConnection::OnSslAckPacketReceived() diff --git a/yt/yt/core/bus/tcp/connection.h b/yt/yt/core/bus/tcp/connection.h index ff8c4eb52fb..43c489e4db2 100644 --- a/yt/yt/core/bus/tcp/connection.h +++ b/yt/yt/core/bus/tcp/connection.h @@ -2,7 +2,6 @@ #include "packet.h" #include "dispatcher_impl.h" -#include "ssl_helpers.h" #include <yt/yt/core/bus/private.h> #include <yt/yt/core/bus/bus.h> @@ -11,6 +10,8 @@ #include <yt/yt/core/actions/future.h> +#include <yt/yt/core/crypto/tls.h> + #include <yt/yt/core/logging/log.h> #include <yt/yt/core/net/address.h> @@ -31,8 +32,6 @@ #include <util/network/init.h> -#include <openssl/ssl.h> - #ifdef _win_ #include <winsock2.h> @@ -274,7 +273,7 @@ private: bool SslAckEnqueued_ = false; bool SslAckReceived_ = false; bool SslAckSent_ = false; - std::unique_ptr<SSL, TDeleter> Ssl_; + NCrypto::TSslPtr Ssl_; std::atomic<ESslState> SslState_ = ESslState::None; const EEncryptionMode EncryptionMode_; diff --git a/yt/yt/core/bus/tcp/ssl_context.cpp b/yt/yt/core/bus/tcp/ssl_context.cpp deleted file mode 100644 index 8084b641c8c..00000000000 --- a/yt/yt/core/bus/tcp/ssl_context.cpp +++ /dev/null @@ -1,322 +0,0 @@ -#include "ssl_context.h" -#include "ssl_helpers.h" - -#include <yt/yt/core/misc/error.h> - -#include <library/cpp/openssl/init/init.h> - -#include <library/cpp/yt/threading/spin_lock.h> - -#include <library/cpp/yt/memory/leaky_singleton.h> - -#include <util/system/mutex.h> - -#include <openssl/err.h> -#include <openssl/ssl.h> - -#include <memory> - -namespace NYT::NBus { - -//////////////////////////////////////////////////////////////////////////////// - -class TSslContext::TImpl -{ -public: - TImpl() - { - InitOpenSSL(); - - SslCtx_.reset(SSL_CTX_new(TLS_method())); - if (!SslCtx_) { - THROW_ERROR_EXCEPTION("Failed to create TLS/SSL context: %v", GetLastSslErrorString()); - } - - if (SSL_CTX_set_min_proto_version(SslCtx_.get(), TLS1_2_VERSION) != 1) { - THROW_ERROR_EXCEPTION("Failed to set min protocol version: %v", GetLastSslErrorString()); - } - - if (SSL_CTX_set_max_proto_version(SslCtx_.get(), TLS1_2_VERSION) != 1) { - THROW_ERROR_EXCEPTION("Failed to set max protocol version: %v", GetLastSslErrorString()); - } - - SSL_CTX_set_mode(SslCtx_.get(), SSL_MODE_ENABLE_PARTIAL_WRITE | SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER); - } - - SSL_CTX* GetCtx() - { - return SslCtx_.get(); - } - - //! This function is for testing purposes. - void LoadCAFile(const TString& filePath) - { - auto guard = Guard(SpinLock_); - - LoadCAFileUnlocked(filePath); - - CAIsLoaded_ = true; - } - - void LoadCAFileIfNotLoaded(const TString& filePath) - { - if (CAIsLoaded_) { - return; - } - - auto guard = Guard(SpinLock_); - - if (CAIsLoaded_) { - return; - } - - LoadCAFileUnlocked(filePath); - - CAIsLoaded_ = true; - } - - void LoadCertificateChain(const TString& filePath) - { - auto ret = SSL_CTX_use_certificate_chain_file(SslCtx_.get(), filePath.data()); - if (ret != 1) { - THROW_ERROR_EXCEPTION("Failed to load certificate chain: %v", GetLastSslErrorString()); - } - } - - void LoadPrivateKey(const TString& filePath) - { - auto ret = SSL_CTX_use_RSAPrivateKey_file(SslCtx_.get(), filePath.data(), SSL_FILETYPE_PEM); - if (ret != 1) { - THROW_ERROR_EXCEPTION("Failed to load private key: %v", GetLastSslErrorString()); - } - } - - //! This function is for testing purposes. - void UseCA(const TString& ca) - { - auto guard = Guard(SpinLock_); - - UseCAUnlocked(ca); - - CAIsLoaded_ = true; - } - - void UseCAIfNotUsed(const TString& ca) - { - if (CAIsLoaded_) { - return; - } - - auto guard = Guard(SpinLock_); - - if (CAIsLoaded_) { - return; - } - - UseCAUnlocked(ca); - - CAIsLoaded_ = true; - } - - void UseCertificateChain(const TString& certificate) - { - std::unique_ptr<BIO, TDeleter> bio(BIO_new_mem_buf(certificate.data(), certificate.size())); - if (!bio) { - THROW_ERROR_EXCEPTION("Failed to allocate memory buffer for certificate: %v", GetLastSslErrorString()); - } - - // The first certificate in the chain is expected to be a leaf. - std::unique_ptr<X509, TDeleter> cert(PEM_read_bio_X509(bio.get(), nullptr, nullptr, nullptr)); - if (!cert) { - THROW_ERROR_EXCEPTION("Failed to read certificate from memory buffer: %v", GetLastSslErrorString()); - } - - if (SSL_CTX_use_certificate(SslCtx_.get(), cert.get()) != 1) { - THROW_ERROR_EXCEPTION("Failed to use cert in ssl: %v", GetLastSslErrorString()); - } - - // Load additional certificates in the chain. - while (auto cert = PEM_read_bio_X509(bio.get(), nullptr, nullptr, nullptr)) { - if (SSL_CTX_add0_chain_cert(SslCtx_.get(), cert) != 1) { - THROW_ERROR_EXCEPTION("Failed to add cert to ssl: %v", GetLastSslErrorString()); - } - // Do not X509_free() if certificate was added by SSL_CTX_add0_chain_cert(). - } - } - - void UsePrivateKey(const TString& privateKey) - { - std::unique_ptr<BIO, TDeleter> bio(BIO_new_mem_buf(privateKey.data(), privateKey.size())); - if (!bio) { - THROW_ERROR_EXCEPTION("Failed to allocate memory buffer for private key: %v", GetLastSslErrorString()); - } - - std::unique_ptr<RSA, TDeleter> pkey(PEM_read_bio_RSAPrivateKey(bio.get(), nullptr, nullptr, nullptr)); - if (!pkey) { - THROW_ERROR_EXCEPTION("Failed to read private key from memory buffer: %v", GetLastSslErrorString()); - } - - if (SSL_CTX_use_RSAPrivateKey(SslCtx_.get(), pkey.get()) != 1) { - THROW_ERROR_EXCEPTION("Failed to add the private RSA key to ctx: %v", GetLastSslErrorString()); - } - } - - void SetCipherListIfUnset(const TString& cipherList) - { - if (CipherListIsSet_) { - return; - } - - auto guard = Guard(SpinLock_); - - if (CipherListIsSet_) { - return; - } - - SetCipherListUnlocked(cipherList); - - CipherListIsSet_ = true; - } - - //! This function is for testing purposes. - void SetCipherList(const TString& cipherList) - { - auto guard = Guard(SpinLock_); - - SetCipherListUnlocked(cipherList); - - CipherListIsSet_ = true; - } - - //! Check the consistency of a private key with the corresponding certificate. - //! A private key and the corresponding certificate have to be loaded into ctx. - void CheckPrivateKeyWithCertificate() - { - if (SSL_CTX_check_private_key(SslCtx_.get()) != 1) { - THROW_ERROR_EXCEPTION("Failed to check the consistency of a private key with the corresponding certificate: %v", GetLastSslErrorString()); - } - } - -private: - void LoadCAFileUnlocked(const TString& filePath) - { - auto ret = SSL_CTX_load_verify_locations(SslCtx_.get(), filePath.data(), nullptr); - if (ret != 1) { - THROW_ERROR_EXCEPTION("Failed to load CA file: %v", GetLastSslErrorString()); - } - } - - void SetCipherListUnlocked(const TString& cipherList) - { - if (SSL_CTX_set_cipher_list(SslCtx_.get(), cipherList.data()) != 1) { - THROW_ERROR_EXCEPTION("Failed to set cipher list: %v", GetLastSslErrorString()); - } - } - - void UseCAUnlocked(const TString& ca) - { - std::unique_ptr<BIO, TDeleter> bio(BIO_new_mem_buf(ca.data(), ca.size())); - if (!bio) { - THROW_ERROR_EXCEPTION("Failed to allocate memory buffer for CA certificate: %v", GetLastSslErrorString()); - } - - auto store = SSL_CTX_get_cert_store(SslCtx_.get()); - - // Load certificate chain. - while (auto cert = PEM_read_bio_X509(bio.get(), nullptr, nullptr, nullptr)) { - if (X509_STORE_add_cert(store, cert) != 1) { - THROW_ERROR_EXCEPTION("Failed to add cert to store: %v", GetLastSslErrorString()); - } - X509_free(cert); - } - } - -private: - YT_DECLARE_SPIN_LOCK(NThreading::TSpinLock, SpinLock_); - std::atomic<bool> CAIsLoaded_ = false; - std::atomic<bool> CipherListIsSet_ = false; - std::unique_ptr<SSL_CTX, TDeleter> SslCtx_; -}; - -//////////////////////////////////////////////////////////////////////////////// - -TSslContext::TSslContext() -{ - Reset(); -} - -TSslContext::~TSslContext() = default; - -SSL_CTX* TSslContext::GetSslCtx() -{ - return Impl_->GetCtx(); -} - -TSslContext* TSslContext::Get() -{ - return LeakySingleton<TSslContext>(); -} - -void TSslContext::LoadCAFile(const TString& filePath) -{ - return Impl_->LoadCAFile(filePath); -} - -void TSslContext::LoadCAFileIfNotLoaded(const TString& filePath) -{ - return Impl_->LoadCAFileIfNotLoaded(filePath); -} - -void TSslContext::LoadCertificateChain(const TString& filePath) -{ - return Impl_->LoadCertificateChain(filePath); -} - -void TSslContext::LoadPrivateKey(const TString& filePath) -{ - return Impl_->LoadPrivateKey(filePath); -} - -void TSslContext::UseCA(const TString& ca) -{ - return Impl_->UseCA(ca); -} - -void TSslContext::UseCAIfNotUsed(const TString& ca) -{ - return Impl_->UseCAIfNotUsed(ca); -} - -void TSslContext::UseCertificateChain(const TString& certificate) -{ - return Impl_->UseCertificateChain(certificate); -} - -void TSslContext::UsePrivateKey(const TString& privateKey) -{ - return Impl_->UsePrivateKey(privateKey); -} - -void TSslContext::SetCipherList(const TString& cipherList) -{ - return Impl_->SetCipherList(cipherList); -} - -void TSslContext::SetCipherListIfUnset(const TString& cipherList) -{ - return Impl_->SetCipherListIfUnset(cipherList); -} - -void TSslContext::CheckPrivateKeyWithCertificate() -{ - return Impl_->CheckPrivateKeyWithCertificate(); -} - -void TSslContext::Reset() -{ - Impl_ = std::make_unique<TImpl>(); -} - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NBus diff --git a/yt/yt/core/bus/tcp/ssl_context.h b/yt/yt/core/bus/tcp/ssl_context.h deleted file mode 100644 index 21d89d55624..00000000000 --- a/yt/yt/core/bus/tcp/ssl_context.h +++ /dev/null @@ -1,44 +0,0 @@ -#pragma once - -#include "public.h" - -#include <openssl/ssl.h> - -#include <memory> - -namespace NYT::NBus { - -//////////////////////////////////////////////////////////////////////////////// - -class TSslContext -{ -public: - TSslContext(); - ~TSslContext(); - - SSL_CTX* GetSslCtx(); - static TSslContext* Get(); - - void LoadCAFileIfNotLoaded(const TString& filePath); - void LoadCertificateChain(const TString& filePath); - void LoadPrivateKey(const TString& filePath); - void UseCAIfNotUsed(const TString& ca); - void UseCertificateChain(const TString& certificate); - void UsePrivateKey(const TString& privateKey); - void SetCipherListIfUnset(const TString& cipherList); - void CheckPrivateKeyWithCertificate(); - - // For testing purposes. - void LoadCAFile(const TString& filePath); - void Reset(); - void SetCipherList(const TString& cipherList); - void UseCA(const TString& ca); - -private: - class TImpl; - std::unique_ptr<TImpl> Impl_; -}; - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NBus diff --git a/yt/yt/core/bus/tcp/ssl_helpers.cpp b/yt/yt/core/bus/tcp/ssl_helpers.cpp deleted file mode 100644 index 0a115002332..00000000000 --- a/yt/yt/core/bus/tcp/ssl_helpers.cpp +++ /dev/null @@ -1,103 +0,0 @@ -#include "ssl_helpers.h" - -#include <yt/yt/core/misc/error.h> - -#include <openssl/ssl.h> -#include <openssl/err.h> - -#include <util/generic/string.h> - -#include <memory> - -namespace NYT::NBus { - -//////////////////////////////////////////////////////////////////////////////// - -void TDeleter::operator()(SSL_CTX* ctx) const -{ - SSL_CTX_free(ctx); -} - -void TDeleter::operator()(SSL* ctx) const -{ - SSL_free(ctx); -} - -void TDeleter::operator()(BIO* bio) const -{ - BIO_free(bio); -} - -void TDeleter::operator()(RSA* rsa) const -{ - RSA_free(rsa); -} - -void TDeleter::operator()(X509* x509) const -{ - X509_free(x509); -} - -//////////////////////////////////////////////////////////////////////////////// - -TString GetLastSslErrorString() -{ - char errorStr[256]; - ERR_error_string_n(ERR_get_error(), errorStr, sizeof(errorStr)); - return errorStr; -} - -//////////////////////////////////////////////////////////////////////////////// - -bool UseCertificateChain(const TString& certificate, SSL* ssl) -{ - std::unique_ptr<BIO, TDeleter> bio(BIO_new_mem_buf(certificate.data(), certificate.size())); - if (!bio) { - return false; - } - - // The first certificate in the chain is expected to be a leaf. - std::unique_ptr<X509, TDeleter> cert(PEM_read_bio_X509(bio.get(), nullptr, nullptr, nullptr)); - if (!cert) { - return false; - } - - if (SSL_use_certificate(ssl, cert.get()) != 1) { - return false; - } - - // Load additional certificates in the chain. - while (auto cert = PEM_read_bio_X509(bio.get(), nullptr, nullptr, nullptr)) { - if (SSL_add0_chain_cert(ssl, cert) != 1) { - return false; - } - // Do not X509_free() if certificate was added by SSL_CTX_add0_chain_cert(). - } - - return true; -} - -//////////////////////////////////////////////////////////////////////////////// - -bool UsePrivateKey(const TString& privateKey, SSL* ssl) -{ - std::unique_ptr<BIO, TDeleter> bio(BIO_new_mem_buf(privateKey.data(), privateKey.size())); - if (!bio) { - return false; - } - - std::unique_ptr<RSA, TDeleter> pkey(PEM_read_bio_RSAPrivateKey(bio.get(), nullptr, nullptr, nullptr)); - if (!pkey) { - return false; - } - - if (SSL_use_RSAPrivateKey(ssl, pkey.get()) != 1) { - return false; - } - - return true; -} - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NBus diff --git a/yt/yt/core/bus/tcp/ssl_helpers.h b/yt/yt/core/bus/tcp/ssl_helpers.h deleted file mode 100644 index e99f826cb80..00000000000 --- a/yt/yt/core/bus/tcp/ssl_helpers.h +++ /dev/null @@ -1,34 +0,0 @@ -#pragma once - -#include <openssl/ssl.h> - -#include <util/generic/string.h> - -namespace NYT::NBus { - -//////////////////////////////////////////////////////////////////////////////// - -struct TDeleter -{ - void operator()(SSL_CTX* ctx) const; - void operator()(SSL* ctx) const; - void operator()(BIO* bio) const; - void operator()(RSA* rsa) const; - void operator()(X509* x509) const; -}; - -//////////////////////////////////////////////////////////////////////////////// - -TString GetLastSslErrorString(); - -//////////////////////////////////////////////////////////////////////////////// - -bool UseCertificateChain(const TString& certificate, SSL* ssl); - -//////////////////////////////////////////////////////////////////////////////// - -bool UsePrivateKey(const TString& privateKey, SSL* ssl); - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NBus diff --git a/yt/yt/core/bus/unittests/ssl_ut.cpp b/yt/yt/core/bus/unittests/ssl_ut.cpp index 6f65b24c8e1..e99bf169821 100644 --- a/yt/yt/core/bus/unittests/ssl_ut.cpp +++ b/yt/yt/core/bus/unittests/ssl_ut.cpp @@ -7,13 +7,15 @@ #include <yt/yt/core/bus/tcp/config.h> #include <yt/yt/core/bus/tcp/client.h> #include <yt/yt/core/bus/tcp/server.h> -#include <yt/yt/core/bus/tcp/ssl_context.h> +#include <library/cpp/testing/common/env.h> #include <library/cpp/testing/common/network.h> namespace NYT::NBus { namespace { +using namespace NCrypto; + //////////////////////////////////////////////////////////////////////////////// TSharedRefArray CreateMessage(int numParts, int partSize = 1) @@ -54,238 +56,15 @@ public: TString AddressWithIpV4; TString AddressWithIpV6; - const char* CA = R"foo(-----BEGIN CERTIFICATE----- -MIIFWjCCA0KgAwIBAgIBATANBgkqhkiG9w0BAQsFADBGMQswCQYDVQQGEwJSVTEP -MA0GA1UECAwGTW9zY293MQ8wDQYDVQQKDAZZYW5kZXgxFTATBgNVBAMMDENBQGxv -Y2FsaG9zdDAeFw0yMzA3MTMxMTUxMTFaFw0zMzA3MTAxMTUxMTFaMEYxCzAJBgNV -BAYTAlJVMQ8wDQYDVQQIDAZNb3Njb3cxDzANBgNVBAoMBllhbmRleDEVMBMGA1UE -AwwMQ0FAbG9jYWxob3N0MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA -mxWRSfbYAJcBgGhYagnQMJOiW60B82Ok94n8muIaa2gRo0ZHKmtY7CMiFIIN0GhI -Blw+G6HPN8SPYK0+bTlAIHlb0u6/ty2AVyveiH31p7Ld/Ib5S/MSKuMiGt5S/Ci9 -mSdRAmD46twIKfm9bY/8SZmNsTCJIrKnJaTbjTnbz9O3FVAYjiuc8yNGb1LnZNA6 -B4ZrF3fkYr9kn4nKSOd6mypPFIOZAxKGQ95X9nCblajEMAPzHfr+1EArnM+PauAO -cMcxfKT+OlMPGR4ZtZpl90qX6ZVZqD9zd8gp2I/2SM1vVS7AT96t89T7Uag8OjH2 -c8jMCD2z1fk1oDD4K53pGkBucTwDolCvxIbN77gcwkutdor/dbHLqIvGV/M4Z/iA -GzqCD6pCpi5gT8SXn2IQrlvSdF01k9YZS093Y8bIQm3dCo1lKDaz6oHytk9Ro+fu -b+dLOhSjopNfa/1Thw6fgta/mRsdgubWI/IHn+IyMpW65vGbnrMD4oQl1AanRJj5 -iBS73ZXIs/Y9LuMtSNk/I9u1o2fc+Sg0zb1AchC70h8M3sAYaGbqg1churOaDuTO -Yom5W+bAQO8uLUvmlDcpqxMBYqeE8VJGJWRLtQnhM18iYAa03w7m6uMzrlkoX1Q7 -AHnX4899WSUcz/BJc5JBAtHQXCzEzyudT+8xw5MwqRkCAwEAAaNTMFEwHQYDVR0O -BBYEFFuWdWxzQ6TF/yH3sQnlk3EEr1VjMB8GA1UdIwQYMBaAFFuWdWxzQ6TF/yH3 -sQnlk3EEr1VjMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggIBAHqS -W4FIjH/YKkcj/wuh7sQQTIBAx+LsjosyHb9K/1QbgLNVpxawl/YlOiArfVmgTGud -9B6xdVacxqI6saMqHNPPgYm7OPnozSnprRKh9yGb2TS9j+2G7hCQLnM+xYroP8XE -8fL9tyq14lHb/BFYnPsFFxqeA6lXzzm73hlm/hH58CWpL/3eR/TDjC+oiEBBV4VF -Dm4X3L73MFcniDKkCb7Mw3wdi6zqx231F4+9Cqgq+RqAucnmLXTG7yR7wOKP2u8E -Ye1Jrt3nnMlD1tqiXyRJywPSK9mMiBTGbzGLLiTcvecBIHG8oRuvn1DnkZa9L/48 -+1aTC8QhbRslaeDyXj5IY/7scW7ZkEw18qYCT+9sI4/LngS4U8+taKqLf5S8A5sR -O8OCP0nMk/l5f84cDwe1dp4GXVQdkbnfT1sd93BLww7Szbw2iMt9sLMmyY8qIe7a -Ss3OEfP6eyNCBu6KtL8oufdj1AqAQdmYYTlGwgaFZTAomDiJU662EGSt3uXK7V32 -EzgJbpxSSjWh5JpCDA/jnqzlkFkSaw92/HZwO5l1lCdqK+F1UWYQyU0NveOaSEuX -344PIi8m1YWwUfukUB97D+C6B4y5vgdkC7OYLHb4W4+N0ZvYtbDDaBeTpn70CiI7 -JFWcF3ghP7uPmbONWLiTFwxsSJHT0svVQZgq1aZz ------END CERTIFICATE-----)foo"; - - const char* CertChain = R"foo(-----BEGIN CERTIFICATE----- -MIIFUTCCAzmgAwIBAgIBATANBgkqhkiG9w0BAQsFADBGMQswCQYDVQQGEwJSVTEP -MA0GA1UECAwGTW9zY293MQ8wDQYDVQQKDAZZYW5kZXgxFTATBgNVBAMMDENBQGxv -Y2FsaG9zdDAeFw0yMzA3MTMxMTUxMTdaFw0zMzA3MTAxMTUxMTdaMEMxCzAJBgNV -BAYTAlJVMQ8wDQYDVQQIDAZNb3Njb3cxDzANBgNVBAoMBllhbmRleDESMBAGA1UE -AwwJbG9jYWxob3N0MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA1ZrD -+kcfmsi52zMZ7cJxWl8pMI1kX1uQ6KkOJRVQVuweJuhp35rJGK0vHkWvrpihO+Ue -h1jncReH8PaNdejt3gPsYJksUeRxCaZ8O+awtyOq6TK8L9pDzbQyXme5k2Ea/+yl -XSHrGGzZUuHaZRzBigyQXzb04ZdQMzqGGKUz52ed5elZnOoAMUL7/wZ7O8n5dlXh -DwxEQ4SQ3a39SLpPWv3hWXiIDrlR5xhkqQjZnb0WEVlSwN8lpyjA6XwCs/IX7PVr -khoBKq7664rsR3B7qhGA5cmSy4+KrSoYKYHGCvl6OakO7fn4FcJwHHFNQzMq8oa5 -i5LbYsaCy6IgJt07nBeNXxaEY+4WIImHfWR0yvQgqtrFQZuIQ4N7YeZpLvTtoO3I -UJyjMsih5GX4LPY0k9nE8rd437lV3PdiBYAktkwmgCTY9uJFhCBPFsMhE30nL79t -X0gm5FugW1aZ3CFvgQywkcYQIjBjJQ42zcpqnLjj4lIwfDO5YEAiAOPqQ3VV7FKm -bSWpeXyRAL/uqkBVCDKTsKJkEjd8gc30honRIKNHQkr/RoV1M1W449uRkvfVfbfg -kdcRcYMIynTwZMSiKDoxXkQyrcyY6qVjYQNUKwRvtN2m6aGftYhfgo4j01UHTrK7 -BHRHKjW5LFXgNJUXnycmO69B+w1eTRgnVE7ws6sCAwEAAaNNMEswCQYDVR0TBAIw -ADAdBgNVHQ4EFgQUNvB1ghYixkG53VZoSvUPnCCN7hwwHwYDVR0jBBgwFoAUW5Z1 -bHNDpMX/IfexCeWTcQSvVWMwDQYJKoZIhvcNAQELBQADggIBAHW/+xxign61axqT -hO338rJGuidhXzraCQtJT/J6OwUaI+P6C71wUoBKEk3SrIVNG1Q8mrlsj93EzQuv -M9OvBenIomF7A2ydaFL/6NJhZ8W6PmZwYF3VvInYJ2p7zqCMjqECg2C92NIC7+Z6 -fdmL+xoj3XKYypqA1x6xvSnCtXyXGRCxta3Es163wbqffq+4jjBUFOyUr9vk2N7X -vy3/x8LWHIXffzNSJLtnXiznSNBSubmedac8JQ+XE9RgK+R0kUrj1lqnkSg+tPWD -jP52kG84J9URV18BZfLFpcUiYloWXREfwNXRhMAQ6DyucupLW1Skl+Nf9K+41C3h -f4mDn4Axn6toBzav9NLdFelGVAp4R3Yjoiv2LvpwYxGnMs/cPJMGm/NqoAbuyOMY -ZKZPWvwsbxeaG8u9GRGvTSpawnGWJqIgxknKpQT5QztjNwtI9iT0/f4n3CPEGdAi -6bHw0q+jiCKmXZMZPHyJ/tSJ74H7tdeWYYjhthJWnrAz4BziZ+bFBLkcYq95VV4L -pOeMUr0PUi8fpSK06wIVTES9AWgcfuXL6i7AQ+hadEa3Ve1BGRsu0KOXW2XZZFeo -3Pczm/o+jwMiLELcgrM5Ngy8dcCKr6v84F+fi9Y+C8+RZ7g37aLJM0kqbaoN8owL -mP88f9xDGRAmesvuYlHo+57VTyzU ------END CERTIFICATE-----)foo"; - - const char* PrivateKey = R"foo(-----BEGIN PRIVATE KEY----- -MIIJQQIBADANBgkqhkiG9w0BAQEFAASCCSswggknAgEAAoICAQDVmsP6Rx+ayLnb -MxntwnFaXykwjWRfW5DoqQ4lFVBW7B4m6GnfmskYrS8eRa+umKE75R6HWOdxF4fw -9o116O3eA+xgmSxR5HEJpnw75rC3I6rpMrwv2kPNtDJeZ7mTYRr/7KVdIesYbNlS -4dplHMGKDJBfNvThl1AzOoYYpTPnZ53l6Vmc6gAxQvv/Bns7yfl2VeEPDERDhJDd -rf1Iuk9a/eFZeIgOuVHnGGSpCNmdvRYRWVLA3yWnKMDpfAKz8hfs9WuSGgEqrvrr -iuxHcHuqEYDlyZLLj4qtKhgpgcYK+Xo5qQ7t+fgVwnAccU1DMyryhrmLkttixoLL -oiAm3TucF41fFoRj7hYgiYd9ZHTK9CCq2sVBm4hDg3th5mku9O2g7chQnKMyyKHk -Zfgs9jST2cTyt3jfuVXc92IFgCS2TCaAJNj24kWEIE8WwyETfScvv21fSCbkW6Bb -VpncIW+BDLCRxhAiMGMlDjbNymqcuOPiUjB8M7lgQCIA4+pDdVXsUqZtJal5fJEA -v+6qQFUIMpOwomQSN3yBzfSGidEgo0dCSv9GhXUzVbjj25GS99V9t+CR1xFxgwjK -dPBkxKIoOjFeRDKtzJjqpWNhA1QrBG+03abpoZ+1iF+CjiPTVQdOsrsEdEcqNbks -VeA0lRefJyY7r0H7DV5NGCdUTvCzqwIDAQABAoICAEvCv0TLKiH/lK/y4XzrTMIF -Y3oVhCawNubWYy5y71JNH+qj3z1QTIgEkOQ3Sjbuaq1wN9JAjaIWewBTqlvKOGfY -02N1oHsRP6hxFLo4ObBTJcDdXlLIoujYQ08pke/8bpOcDxDHwXch0Djt40SenOSG -TUSAHP3QacEpvjsKiSzHmwDbMY4Oju/p9q/+0AGmQuUeU5s/Og0KfUkq911uu0um -JWHS9srmHu8Mv1MW0Px5/tQ7brb6zoOJ2FZXxiulr6e7aiJhN824T0XwuZojArGQ -0LtvsbGiYUjG19gM772fu6Ks3B863Ct3kcT8yK8PfGmVsESZW1ee2fA4uheeuw+c -/3D71+dnjvw0/LFQEWyiH9NhjvafALBGxtlbfTTVtw5bHVaoalYpEggKnJrRabPa -HmdPkEAK0vJzBvTjPW1lEvlxUkol1SsYq8xpBkatieZzf4+SWk/ugP7rvoRVRbB1 -GmHc9CK9jCDTcN7ii5pTOpc2VOK5cvn+K5L4P7Qw4kK37pyO56jhVbTAavNxotSc -+mZa8OqZaK0jvD9sUPCSuY44x5X6WhXjILE+R3QXXXtGkyjlgJxGmfkYcu19GV4B -ziU7hVCjqtNOQ51ywEtWUA93lchj3fD5ryo9dCv5a5Xco4O5E24VeGjW3MBqeBNW -wNIUintgwVt20P1I+xZRAoIBAQD9UzjlhvrDgNZsaJu82+PoOEJM2pa5F+fDGSeq -XTYCCeqioMUujJWDEkBEzfTvu4r7Pr7uExOlkyUXdSELfm+ABkt+eZguPB15Ua6V -Y59Z4yrpqp0JIwpxXGxNOsSUnVwGg3OTlUGWYVyxyGHYQnSGJNe2/NhSlQZmvxx2 -PxqTH+g9dLn7FkQk/FNQI4l4LnphgGvNBVyXZoHcw16y86MAKLyrcZBmbHnuWMC6 -zjIu61uXXd8GU7IU8BWJbUQKvz0Y5susWf921U/7Qa8NZ2vFY4Q7oGlJVBVhMNmB -WLL8/WUeWXu9HvL89QWpb31l5V5L+lF+GSq5ZQxFAjLEVl77AoIBAQDX3CwirHtf -MI7z+zjwLDWgVG7RFVY4Pv5TWDWYHiQT9VPBU16K0+fVS1iXxYvoiGJkrj38Re8y -rkTZ+qvKLmlDftqx7bImbzs01QJRbUH2gxZFMDQIF7uTSFynycywaQk+xpFEPTHl -CXMIyGAMAsd0B2OKPAxZNoHTd0P10FYBIENwxadG9w7Ocm41J/d4Qx6wQsV10hGF -0OgtIioplSDz1Ean/IT8XmwLK7tIiShJzZidE+0sY4depy3JdAHWv50uXMs3bIcu -xvVBA/e/jf+HssfLjzzrYtwp2JBzSOTox/eCUJ/i9gatC5ox+ewjwOaFXV3bGEib -mK+I3xiQ3B8RAoIBAE0Jc/IJHFU75vlMzp+eVy6VfUQV7WQYavifu7pJYlU4YsxW -C+DeC9GySS0jXOtSoy9Io5OO5ZiiqNL7YbM3Hf1W7Lpni+nzihsMxgTUKO+S78fj -hKH0sAZNTvoldwai3At3CjzFVQ7ASQofn/G+M+VfauJQ/hAPFcVFNQiYpCI9v8iA -qNY8rTh6K3Pherq7l6fy/9V3XfMEz1UtbK0K/nTb7pRMktczAdmD0Ah/EC/Ijy/2 -8g3ggfVwFXyXZ+vEwHXEKggdzlx6/jmwfeWbn+CFJP9lBt+v3FiUHHEDYlshTBDw -sXqP4OEgOjqOlxnXqNd+Ji4sxRtgKV0LEBk5EuUCggEAIlBbq79jdURQ1TQQXw2I -EM6bNx1/MT3CTBlvm5je/1U2VTsdglAhQGTT1nyOuw5DJeIU9G9hkNrnEweoG2G5 -VgNqXHJ+qWFxNfrOfYcyvy8jcSgyfT7YkJcmM33+zeRElfgWy5Q2xEP2R2Ui74XZ -kvZBuo3FIMFrbeQ9p2vQ4Cjyz5B8AOnxLpw+LLEHw9RXoolavloAcxc8cUBHF4kf -TeNmv/mCYmPYJQZ0pRk4kFLgecfbIf1IXaGRw75vNGYNZHtXyp2z95mlDwrEbWzz -O+0NmaxRcNGsUfKdM9ZYnTB8hfivEfMuKH/5qQwjn6Nggb7P1q5LjIB/FvDwBMcZ -IQKCAQBfPoddROG6bTkzn7kL8rDwuk2H2sLh5njRKo3u29dVlwU93lmKLoP4blcf -tAbXDVrlGpZbIo73baE57Rv5ehcV64oS/G63Lcw8nsqmBvs9982tpYPNX9EOvXV5 -1iK+hs8ZUUVCVkTOXsZb5M90VKQiHrnddO+08lE5YI/lzM5laqcytPmYcLkVWPPz -qrpW/AReSwhvwVugcMFUgMXaDx/3SAY75B808wX1tizv76omWZAQ774FeGQGyP4C -8f4t3LIV9h/q2Hj8geMjil9ZGogtWJ5uDspp7As5OyMF0ZMXTMwSnFsXB/L4YIk+ -rPl77gAcribJm3TzBVHm2m6jBGtb ------END PRIVATE KEY-----)foo"; - - const char* CAWithIpInSAN = R"foo(-----BEGIN CERTIFICATE----- -MIIFHzCCAwegAwIBAgIUQEt4xnHWGulMGzqad434c4Mw+cAwDQYJKoZIhvcNAQEL -BQAwJjERMA8GA1UECgwIWVRzYXVydXMxETAPBgNVBAMMCFlUc2F1cnVzMB4XDTI0 -MDIyOTEwMTIzM1oXDTM0MDIyNjEwMTIzM1owJjERMA8GA1UECgwIWVRzYXVydXMx -ETAPBgNVBAMMCFlUc2F1cnVzMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKC -AgEAwUS9deIwfMNeicFqw7/fDslSb7sSiPbYfYUxwuELBR8nVomaL/a7IpQx5dOw -nB2CSUiTDsLczNaBvQyNkjJ8VIX4fEPtgfHCuoUaCg4NwcaHfY0TSssbCKh8U92z -fM/iSCjjkMdp9pTzZL93yam72dqfFbHmBvO6XiGjYeR9BL6AqoZMBnIjmxsU9JCW -SM7B0l+Sf8dsH8DmnzqtLUXKIZJyVK6LOVb7MtSw0iAYatrIt3t7IytqZMOe+dEZ -V0C0YzTecOgCWKS+rA8X1h7zHW5vXLDYU6RR+qw3gFsRK0oHmECuo+4Ui2nMzkx1 -jf2nK8L0Z2MHP0YFI9cGCBfSZMv6eocbYZI1DmhNKZNFhSLL8lmfzv4uR7WPyNL6 -Ml+2xgp03r1QFx93cRNW5bjjFytzYQkEVwbvLsABHDFm056PpNXKh6eoXGkUfaWf -iXxUWhAFJRxggkTrPawCr+YQtVbBFe/YcdrTkdvFengFLm4wE20RiJUZWvG4oXmV -JdKvDMsR1RFoQQfSqKL3mERMvVBBO1CXqymdPW6K+S2r367ryXGCG6kRDGpfmu7R -vYZcA9BJqNvGcowf5fBe2VZf3cqZwdZKjCYOR3dCVDbXClraeStzy0VXKgxukHyT -g3XfPqptaUKtpy413b/AVG2YupEn+SoOGMP0NKMyj9XAr20CAwEAAaNFMEMwDgYD -VR0PAQH/BAQDAgLkMBIGA1UdEwEB/wQIMAYBAf8CAQEwHQYDVR0OBBYEFAYGwcRb -cWnfIkC1Mozkzm0CDcFbMA0GCSqGSIb3DQEBCwUAA4ICAQCztlZxgJNdhebicTkT -B4iXXntoOzlnusua1lCBKRUowHAocw2ylXb32Ahh/fENmHyt/jYTlcFu/qK6Q1T8 -/pN14hnyF2pB+ZJjyAr3vNnZtHGJ2xz3wPqFRzD00YUqvvLi9xD2nhsfn4xex/OK -KUx/dbV6FkJ60Fpg97zeOmO+5Kh9srIEGEczPb+y9meiMGB91tm+ZUcd9cGShTRH -krFkuCySJuNgrAZCxqvdsUPvPDd12lPyqmfmuVjauON9ENYUYwNoxQc8MxEx0x6j -QolYDmc2A2VP29rTLQhyjf8TvwXfq8z+zNoeQZCNCON5jg7zH67XarXxK1AZOePc -ZqHtjfTLTqmmKkkG7IRt1h9dtGsROMaRfXf0t4M9mvqx/1Cx6abA6LfcOkp7OG8S -0tx0IzIRnQh2iN4zR1MihS3hn3s9ayviiaopIPVyCKEKRZpsL3QzhzydnQM/Wb+r -UTe546vZV3q2irHH/x4SZFWoFhDwepAyUMI4qo1REd+cM/MakLP5x4nFzzDmPjyf -FiuyqHTlkMtjveytSblzpzWE1/Sum3RcMh4s9ECq8XaUl/8FerUYvIJfRDq1j79J -w/cDyD142joRYwQG0HQkmE4ph4mYFwKhmYOv11Wik9zvEt156VPFaExu6rkjmLia -nTkPBMUXiU3GIb4H7k78sEjv1g== ------END CERTIFICATE-----)foo"; - - const char* CertChainWithIpInSAN = R"foo(-----BEGIN CERTIFICATE----- -MIIFVTCCAz2gAwIBAgIBATANBgkqhkiG9w0BAQsFADAmMREwDwYDVQQKDAhZVHNh -dXJ1czERMA8GA1UEAwwIWVRzYXVydXMwHhcNMjQwMjI5MTAxMjMzWhcNMzQwMjI2 -MTAxMjMzWjATMREwDwYDVQQKDAhZVHNhdXJ1czCCAiIwDQYJKoZIhvcNAQEBBQAD -ggIPADCCAgoCggIBAJ4BhT+WpWpkEEgaqcbZoV5/JOeAkFj2iZL08nIjkDRVhDBh -dQoop47vB1lpLD/1i5+2YEackPrfeJawbNu32s/belvxHpTc1ljndCJlilMiR5B+ -MccP00fUZRsFhkzznANz++31PI4wamHJ1kdJOgD/4civ0mpWgd5s1hityqsypXI3 -RFbi689+mnBp2sGCD6l3QVbMj+8ORvXOVC1h1W3tExiivabjFILgXwb6WG6ZNgg8 -T20zdH4uEc4d2v6XKY4nz4AqYDHax9oqs3XTOTo0Bld6m7oipjGjToMqqpJD32pb -nxSNT/XECpsNqZ/UMtcQf3HoA2LEOZOg/Knf1mosEww2svb0CqMGfsxxHehCaFJT -CWkdd8GTQ/3t85xtrd5Ccdqb78o+5039H4GkfkcxAIZQe7siNLbUJ+dR6wRMzJw2 -GMcAHEIozOYVDgwyOT/Q3gaKMg4A3Ki2x7tkCie1KmmAcnECWDRhzG/Hg/RdnYqA -L6g/m7Z3bw93rUbwXNudVj1ls5MPoyYtTACtZAXnv/PBBd+KR+RZ//T6dD7B9zzC -MiIsIHemPzE3XsWB6I1AqXv2B8THELgJ8foN6Rho4YQ0t0wjhcM0higfSaiHwqhf -Cgoc5xxOGa8PlEETWQvcT3ORAukpjM22QbZmA3uLN/MR1DJbwNHqiYPiFZ1zAgMB -AAGjgaAwgZ0wDgYDVR0PAQH/BAQDAgWgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggr -BgEFBQcDAjAsBgNVHREEJTAjgglsb2NhbGhvc3SHBH8AAAGHEAAAAAAAAAAAAAAA -AAAAAAEwHQYDVR0OBBYEFDyAfXqSAoKxK/sCyFs3d2yUpSntMB8GA1UdIwQYMBaA -FAYGwcRbcWnfIkC1Mozkzm0CDcFbMA0GCSqGSIb3DQEBCwUAA4ICAQCHRuHI5XaS -PIK3JKzA6l01z7YDHROGn5Xf9hqo5QKHv/aUD01UUhxCbx5z/YZMFDc6iRvu4xQw -HMl8BCvTo3gHSS8JRcgRn1ov5m50NZMl0Ws9xvvy1j5rZWd3HkBTQSpzr4eSHAZp -dUWXN2JEW7QIaIxtoiLwMZVjo3Lfr2Qv4uIn9lbIR0F7s39+qFlzvDx+6JbstJsv -5wPjTS9og3WfpBDaOecHM/nP8v1H9ilx8/EW0nM3jlS0q0Gj9whkha0Pcl156Bga -biLDoQk7uTccO8Wiyddwfq6tlYy1OAIMqDy0vmoz4L/3FHJUqrzO/fdI9VQLlug+ -M2G6qTJHKzmDkvmtxPfTjRFMu+g7L3QEdYCBogfIHS+VoB9a9K/XmoWBg85cWIPw -Kfjjf7OouqksfOQopxY2+PCQ66nnkN7y13RjoU2heAme8Fexiowkjhzc1lq0Zn0Q -XPlnvCHAQMNRNmvBLwNEkW+KN4no0TCImOOTuInBrlKGTaBkinUNS39AF9lZWwAE -hd1kK5zzF6XvZnKXdVIn4MjcW81hcbrnulq5GHz7XY+lwmORYumYo3Gykjj/+G93 -K9HRlSRV1+BNXmPYtI8hvbAYw05+AWKCk0J5r1GQtPx+Tx3sug/2qks26oURgEHc -ySl4OPJLp2lhKCUkKVP24Tzg/iS1xT/uHQ== ------END CERTIFICATE-----)foo"; - - const char* PrivateKeyWithIpInSAN = R"foo(-----BEGIN PRIVATE KEY----- -MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQCeAYU/lqVqZBBI -GqnG2aFefyTngJBY9omS9PJyI5A0VYQwYXUKKKeO7wdZaSw/9YuftmBGnJD633iW -sGzbt9rP23pb8R6U3NZY53QiZYpTIkeQfjHHD9NH1GUbBYZM85wDc/vt9TyOMGph -ydZHSToA/+HIr9JqVoHebNYYrcqrMqVyN0RW4uvPfppwadrBgg+pd0FWzI/vDkb1 -zlQtYdVt7RMYor2m4xSC4F8G+lhumTYIPE9tM3R+LhHOHdr+lymOJ8+AKmAx2sfa -KrN10zk6NAZXepu6IqYxo06DKqqSQ99qW58UjU/1xAqbDamf1DLXEH9x6ANixDmT -oPyp39ZqLBMMNrL29AqjBn7McR3oQmhSUwlpHXfBk0P97fOcba3eQnHam+/KPudN -/R+BpH5HMQCGUHu7IjS21CfnUesETMycNhjHABxCKMzmFQ4MMjk/0N4GijIOANyo -tse7ZAontSppgHJxAlg0Ycxvx4P0XZ2KgC+oP5u2d28Pd61G8FzbnVY9ZbOTD6Mm -LUwArWQF57/zwQXfikfkWf/0+nQ+wfc8wjIiLCB3pj8xN17FgeiNQKl79gfExxC4 -CfH6DekYaOGENLdMI4XDNIYoH0moh8KoXwoKHOccThmvD5RBE1kL3E9zkQLpKYzN -tkG2ZgN7izfzEdQyW8DR6omD4hWdcwIDAQABAoICADzT/QZD6p6QsyvvB9lDwznr -3Ls65Vc6YjAvGH8UbdmX6nHtsu8cQ5VlNAEZ2i0tTHlJ7rqAX9gU3Am3FdFocFaA -+hQXOVy5v9MuF6l+SchDdCWOT3+A+ie2/s1uTQum5TL3Hc+4D3316Z6H43RCHpBv -8e4esfS6JPkKEUoi7dkGgGb+G9MPPRT+elo4hjzk4z6saH0P94Fij7LlocZu2Yme -MTHUxQpQdX8E/dBj5FN/rCtzfGhf3MMO3U/qcnp8m0Tc0qdWqP3IahP1SG1dybQ8 -fwyCaR05ZZ3KbtlUPaJdes8pQo7Y8CV/OU4D7n9XY9MjyMyDM3p8bGYHHf4P7C0M -qdTYAVXZ9NvKlf87J8zMJ4YDiKTMLdgFjgnj/tYJ9PkUZHgfsk1KSr5IT6wpLBAx -jEs0+WsbQRmtlJ93ZvZsDJyGZbkeCRIAxyz/eN2Rf56YGrWZcPcn8yrAsnt4nYqz -ewjM9wu6jB8bfAgDS0y3kPsl1www+AkRDnNQ9s+Vw5x0nccjbV7fdezkwf04cA6N -QL+0WSALsU0iGGPKrLA5L/kC3W650xSeOgINdvCnZsSEPAF6HQ/E5udd9HD5+7Ma -hVl2S7FbEVxKgXpXPd1GFBKZKnUTCB5erMDyaN7jAiaVBtRE8qc0hQRhHCtE7qNT -HvWkztmzBuVGTpOkBQuRAoIBAQDDIsOMkYo/svV/lsUJM4fVMTVJ354ThhxvFdJs -rlaUEX2mZIsbzjI76yuOsu6nlGKLuiYVfAAV3HNEhNrSnp6+3NPXMgSsE33TFRPR -Y6P1oT/ZE8YrBkndSioU1ad9Z0ClRNhDlP2c9xCBGpXUSQLt/4C2Nk/wgG/wy7gG -+hJ2ybn133DPpwcx1KQCpslF+VaECPLrpjxWCFEGInOV8gVvxhEb7hrD8qmUPlAR -3WYFT6/RJGCJ7BeOvXWYa8AfWRysLfLwnzuoUxl45O7X7eZqPufrhAJyDNgN3c6J -VgVxNPSSc6mjsY0NyItQZEOXtzRJqnRlmGD2Zd5BJFG7PmIVAoIBAQDPSgFqnFe1 -RM7AtC5wRBLTU/LQo8XV3WPsvQpYaa2+YyPG1bjSMWmdejsy4k+lbUx4VzD+veX7 -ZdSmukYmeGT0pp6Yy7FYYAcZ8Tu+CSAxGH8Bvab+U1oy2Y3++lXdAc0OVjuezVSy -RbUE97S9MjeFMTKxIg2RkJc4p+Ir0sFn8rC9a0ZqNdcV1BCzJkMB5Y0PbG+afopL -wS4xIMIy2Jp3Y0CanOW/dl6LnWYdEtWMt0Pi5wFDVVn0/DgGNx0SDG5fKj4rJE4F -6mezK3kyAf7qI35nf8wm/rL7wL262F4iGZp+OKNFQnaEuKmrUsz/l2GZZkfIIcbI -pPVaqKbAQEtnAoIBAQCj9V/NirRY1WuFqw8frhahwVj/G09dJEBb7kACZXIFs7SZ -zL09vcFjqzPMEPiKAhnTQbOiNbB0reiEWATtF65WvIGavUJDu5TreThPpaMsTjKx -mPXXTM9fimNVYjf7HHiq5O+5yURXURijAc2Gs1os05Q4heYhNCnab7HO2uwMt27y -8q19LODUs9CjEbTogJp7EnHaIrFrsE00FFp+UP7Ubd4OU8BViF2IW9s3R4njSJN3 -7VLYUHFy1CosycyCCoQW//yyxXiA9GHgvKsa75+9AeIod6D+Z2BaNlbF+mtUNaSS -MXEGQ7c7L5gvEi/hGGRsyTZH7wL5xZo7reKmq8IJAoIBAHIiaHdAEFcBxOl8DFnK -UadEgNz6X/LqzJtMV0bpIT5ELi3L/dDWXjXUWIYi8AHBFarpL1QEUX5Dynvm8rs5 -7TR8DbVJ6qMjdKWHGwL+2VfPChd2Sl2cnXyEJ1gulFp1JGfxeTBuFGV4Vjye+0h1 -Pva6aRP5EQmGWI1cev7wM4e9rC0PxRyz+nLNakiKF7kSoMHOTgD+Db26Z2mrhOIk -O6Di6G55V1M9pL8w8kmt1iF9wwZLdXmSpE5tFZfufrYyXA9QHhz5B3DgaSrRFBFB -4g8fbfkk9868zOYrcQxRGDukZ1l6bAO1nbZkSx/HHpLY0md5VqrOVjqiAWpilDYk -8J8CggEAVN7WQIMWzuGBc9Hj6H539eXLnoWgccvYN+GwqclQGKQJdUQrM7Ci3kP/ -Zj00T//jew/vHtxx6U1XUfCIbI5SjujqhbkCkikUMAhozsjlmUlAxKz8689/XFXK -9bPRRvmKw6p14drk507w6t3uD4E/O3PHeEtrdEeDmY78e01T5zkg90jVq10szUMo -sxw0PoOG4ktYtCsmwXXPHq9gRtzfOqZT/UNHgnBsLALmWcec2gZLa++M1OnVaqI2 -hTyvWDnxD5oKa7hDSBXTOorcfQVRSaC05HPGMhX+HkfHwfXJBxAE1UC38UMD7x58 -AbE/BnHl1tAmZXLMrHq/4r0wYUjBsA== ------END PRIVATE KEY-----)foo"; + TPemBlobConfigPtr CACert; + TPemBlobConfigPtr PrivateKey; + TPemBlobConfigPtr CertificateChain; + TPemBlobConfigPtr CACertWithIPInSAN; + TPemBlobConfigPtr PrivateKeyWithIpInSAN; + TPemBlobConfigPtr CertificateChainWithIpInSAN; + TPemBlobConfigPtr CACertEC; + TPemBlobConfigPtr PrivateKeyEC; + TPemBlobConfigPtr CertificateChainEC; TSslTest() { @@ -293,6 +72,17 @@ AbE/BnHl1tAmZXLMrHq/4r0wYUjBsA== AddressWithHostName = Format("localhost:%v", Port); AddressWithIpV4 = Format("127.0.0.1:%v", Port); AddressWithIpV6 = Format("[::1]:%v", Port); + + CACert = TPemBlobConfig::CreateFileReference(SRC_("testdata/ca.pem")); + PrivateKey = TPemBlobConfig::CreateFileReference(SRC_("testdata/key.pem")); + CertificateChain = TPemBlobConfig::CreateFileReference(SRC_("testdata/cert.pem")); + CACertWithIPInSAN = TPemBlobConfig::CreateFileReference(SRC_("testdata/ca_with_ip_in_san.pem")); + PrivateKeyWithIpInSAN = TPemBlobConfig::CreateFileReference(SRC_("testdata/key_with_ip_in_san.pem")); + CertificateChainWithIpInSAN = TPemBlobConfig::CreateFileReference(SRC_("testdata/cert_with_ip_in_san.pem")); + + CACertEC = TPemBlobConfig::CreateFileReference(SRC_("testdata/ca_ec.pem")); + PrivateKeyEC = TPemBlobConfig::CreateFileReference(SRC_("testdata/key_ec.pem")); + CertificateChainEC = TPemBlobConfig::CreateFileReference(SRC_("testdata/cert_ec.pem")); } }; @@ -302,10 +92,8 @@ TEST_F(TSslTest, RequiredAndRequiredEncryptionMode) { auto serverConfig = TBusServerConfig::CreateTcp(Port); serverConfig->EncryptionMode = EEncryptionMode::Required; - serverConfig->CertificateChain = New<NCrypto::TPemBlobConfig>(); - serverConfig->CertificateChain->Value = CertChain; - serverConfig->PrivateKey = New<NCrypto::TPemBlobConfig>(); - serverConfig->PrivateKey->Value = PrivateKey; + serverConfig->CertificateChain = CertificateChain; + serverConfig->PrivateKey = PrivateKey; auto server = CreateBusServer(serverConfig); server->Start(New<TEmptyBusHandler>()); @@ -330,10 +118,8 @@ TEST_F(TSslTest, RequiredAndOptionalEncryptionMode) { auto serverConfig = TBusServerConfig::CreateTcp(Port); serverConfig->EncryptionMode = EEncryptionMode::Required; - serverConfig->CertificateChain = New<NCrypto::TPemBlobConfig>(); - serverConfig->CertificateChain->Value = CertChain; - serverConfig->PrivateKey = New<NCrypto::TPemBlobConfig>(); - serverConfig->PrivateKey->Value = PrivateKey; + serverConfig->CertificateChain = CertificateChain; + serverConfig->PrivateKey = PrivateKey; auto server = CreateBusServer(serverConfig); server->Start(New<TEmptyBusHandler>()); @@ -358,10 +144,8 @@ TEST_F(TSslTest, OptionalAndRequiredEncryptionMode) { auto serverConfig = TBusServerConfig::CreateTcp(Port); serverConfig->EncryptionMode = EEncryptionMode::Optional; - serverConfig->CertificateChain = New<NCrypto::TPemBlobConfig>(); - serverConfig->CertificateChain->Value = CertChain; - serverConfig->PrivateKey = New<NCrypto::TPemBlobConfig>(); - serverConfig->PrivateKey->Value = PrivateKey; + serverConfig->CertificateChain = CertificateChain; + serverConfig->PrivateKey = PrivateKey; auto server = CreateBusServer(serverConfig); server->Start(New<TEmptyBusHandler>()); @@ -386,10 +170,8 @@ TEST_F(TSslTest, OptionalAndOptionalEncryptionMode) { auto serverConfig = TBusServerConfig::CreateTcp(Port); serverConfig->EncryptionMode = EEncryptionMode::Optional; - serverConfig->CertificateChain = New<NCrypto::TPemBlobConfig>(); - serverConfig->CertificateChain->Value = CertChain; - serverConfig->PrivateKey = New<NCrypto::TPemBlobConfig>(); - serverConfig->PrivateKey->Value = PrivateKey; + serverConfig->CertificateChain = CertificateChain; + serverConfig->PrivateKey = PrivateKey; auto server = CreateBusServer(serverConfig); server->Start(New<TEmptyBusHandler>()); @@ -438,10 +220,8 @@ TEST_F(TSslTest, RequiredAndDisabledEncryptionMode) { auto serverConfig = TBusServerConfig::CreateTcp(Port); serverConfig->EncryptionMode = EEncryptionMode::Required; - serverConfig->CertificateChain = New<NCrypto::TPemBlobConfig>(); - serverConfig->CertificateChain->Value = CertChain; - serverConfig->PrivateKey = New<NCrypto::TPemBlobConfig>(); - serverConfig->PrivateKey->Value = PrivateKey; + serverConfig->CertificateChain = CertificateChain; + serverConfig->PrivateKey = PrivateKey; auto server = CreateBusServer(serverConfig); server->Start(New<TEmptyBusHandler>()); @@ -530,16 +310,11 @@ TEST_F(TSslTest, OptionalAndDisabledEncryptionMode) TEST_F(TSslTest, CAVerificationModeFailure) { - // Reset ctx in order to unload possibly loaded CA. - TSslContext::Get()->Reset(); - auto serverConfig = TBusServerConfig::CreateTcp(Port); serverConfig->EncryptionMode = EEncryptionMode::Required; serverConfig->VerificationMode = EVerificationMode::None; - serverConfig->CertificateChain = New<NCrypto::TPemBlobConfig>(); - serverConfig->CertificateChain->Value = CertChain; - serverConfig->PrivateKey = New<NCrypto::TPemBlobConfig>(); - serverConfig->PrivateKey->Value = PrivateKey; + serverConfig->CertificateChain = CertificateChain; + serverConfig->PrivateKey = PrivateKey; auto server = CreateBusServer(serverConfig); server->Start(New<TEmptyBusHandler>()); @@ -560,22 +335,16 @@ TEST_F(TSslTest, CAVerificationModeFailure) TEST_F(TSslTest, CAVerificationModeSuccess) { - // Reset ctx in order to unload possibly loaded CA. - TSslContext::Get()->Reset(); - auto serverConfig = TBusServerConfig::CreateTcp(Port); serverConfig->EncryptionMode = EEncryptionMode::Required; serverConfig->VerificationMode = EVerificationMode::None; - serverConfig->CertificateChain = New<NCrypto::TPemBlobConfig>(); - serverConfig->CertificateChain->Value = CertChain; - serverConfig->PrivateKey = New<NCrypto::TPemBlobConfig>(); - serverConfig->PrivateKey->Value = PrivateKey; + serverConfig->CertificateChain = CertificateChain; + serverConfig->PrivateKey = PrivateKey; auto server = CreateBusServer(serverConfig); server->Start(New<TEmptyBusHandler>()); auto clientConfig = TBusClientConfig::CreateTcp(AddressWithHostName); - clientConfig->CA = New<NCrypto::TPemBlobConfig>(); - clientConfig->CA->Value = CA; + clientConfig->CertificateAuthority = CACert; clientConfig->EncryptionMode = EEncryptionMode::Required; clientConfig->VerificationMode = EVerificationMode::Ca; auto client = CreateBusClient(clientConfig); @@ -598,24 +367,18 @@ TEST_F(TSslTest, CAVerificationModeSuccess) TEST_F(TSslTest, FullVerificationModeByHostName) { - // Reset ctx in order to unload possibly loaded CA. - TSslContext::Get()->Reset(); - auto serverConfig = TBusServerConfig::CreateTcp(Port); serverConfig->EncryptionMode = EEncryptionMode::Required; serverConfig->VerificationMode = EVerificationMode::None; - serverConfig->CertificateChain = New<NCrypto::TPemBlobConfig>(); - serverConfig->CertificateChain->Value = CertChain; - serverConfig->PrivateKey = New<NCrypto::TPemBlobConfig>(); - serverConfig->PrivateKey->Value = PrivateKey; + serverConfig->CertificateChain = CertificateChain; + serverConfig->PrivateKey = PrivateKey; auto server = CreateBusServer(serverConfig); server->Start(New<TEmptyBusHandler>()); auto clientConfig = TBusClientConfig::CreateTcp(AddressWithHostName); clientConfig->EncryptionMode = EEncryptionMode::Required; clientConfig->VerificationMode = EVerificationMode::Full; - clientConfig->CA = New<NCrypto::TPemBlobConfig>(); - clientConfig->CA->Value = CA; + clientConfig->CertificateAuthority = CACert; auto client = CreateBusClient(clientConfig); auto bus = client->CreateBus(New<TEmptyBusHandler>()); @@ -634,26 +397,20 @@ TEST_F(TSslTest, FullVerificationModeByHostName) TEST_F(TSslTest, FullVerificationModeByIpAddress) { - // Reset ctx in order to unload possibly loaded CA. - TSslContext::Get()->Reset(); - // Connect via ipv4 and ipv6 addresses. for (const auto& address : {AddressWithIpV4, AddressWithIpV6}) { auto serverConfig = TBusServerConfig::CreateTcp(Port); serverConfig->EncryptionMode = EEncryptionMode::Required; serverConfig->VerificationMode = EVerificationMode::None; - serverConfig->CertificateChain = New<NCrypto::TPemBlobConfig>(); - serverConfig->CertificateChain->Value = CertChainWithIpInSAN; - serverConfig->PrivateKey = New<NCrypto::TPemBlobConfig>(); - serverConfig->PrivateKey->Value = PrivateKeyWithIpInSAN; + serverConfig->CertificateChain = CertificateChainWithIpInSAN; + serverConfig->PrivateKey = PrivateKeyWithIpInSAN; auto server = CreateBusServer(serverConfig); server->Start(New<TEmptyBusHandler>()); auto clientConfig = TBusClientConfig::CreateTcp(address); clientConfig->EncryptionMode = EEncryptionMode::Required; clientConfig->VerificationMode = EVerificationMode::Full; - clientConfig->CA = New<NCrypto::TPemBlobConfig>(); - clientConfig->CA->Value = CAWithIpInSAN; + clientConfig->CertificateAuthority = CACertWithIPInSAN; auto client = CreateBusClient(clientConfig); auto bus = client->CreateBus(New<TEmptyBusHandler>()); @@ -673,17 +430,12 @@ TEST_F(TSslTest, FullVerificationModeByIpAddress) TEST_F(TSslTest, FullVerificationByAlternativeHostName) { - // Reset ctx in order to unload possibly loaded CA. - TSslContext::Get()->Reset(); - for (const auto& address : {AddressWithIpV4, AddressWithIpV6}) { auto serverConfig = TBusServerConfig::CreateTcp(Port); serverConfig->EncryptionMode = EEncryptionMode::Required; serverConfig->VerificationMode = EVerificationMode::None; - serverConfig->CertificateChain = New<NCrypto::TPemBlobConfig>(); - serverConfig->CertificateChain->Value = CertChain; - serverConfig->PrivateKey = New<NCrypto::TPemBlobConfig>(); - serverConfig->PrivateKey->Value = PrivateKey; + serverConfig->CertificateChain = CertificateChain; + serverConfig->PrivateKey = PrivateKey; auto server = CreateBusServer(serverConfig); server->Start(New<TEmptyBusHandler>()); @@ -691,8 +443,7 @@ TEST_F(TSslTest, FullVerificationByAlternativeHostName) auto clientConfig = TBusClientConfig::CreateTcp(address); clientConfig->EncryptionMode = EEncryptionMode::Required; clientConfig->VerificationMode = EVerificationMode::Full; - clientConfig->CA = New<NCrypto::TPemBlobConfig>(); - clientConfig->CA->Value = CA; + clientConfig->CertificateAuthority = CACert; { auto client = CreateBusClient(clientConfig); @@ -728,10 +479,8 @@ TEST_F(TSslTest, ServerCipherList) auto serverConfig = TBusServerConfig::CreateTcp(Port); serverConfig->EncryptionMode = EEncryptionMode::Required; serverConfig->VerificationMode = EVerificationMode::None; - serverConfig->CertificateChain = New<NCrypto::TPemBlobConfig>(); - serverConfig->CertificateChain->Value = CertChain; - serverConfig->PrivateKey = New<NCrypto::TPemBlobConfig>(); - serverConfig->PrivateKey->Value = PrivateKey; + serverConfig->CertificateChain = CertificateChain; + serverConfig->PrivateKey = PrivateKey; serverConfig->CipherList = "AES128-GCM-SHA256:PSK-AES128-GCM-SHA256"; auto server = CreateBusServer(serverConfig); server->Start(New<TEmptyBusHandler>()); @@ -759,21 +508,27 @@ TEST_F(TSslTest, ServerCipherList) TEST_F(TSslTest, DifferentCipherLists) { + auto cipherStringAES128 = TSslContextCommand::Create("CipherString", "AES128-GCM-SHA256"); + auto cipherSuitesAES128 = TSslContextCommand::Create("CipherSuites", "TLS_AES_128_GCM_SHA256"); + auto cipherStringAES256 = TSslContextCommand::Create("CipherString", "AES256-GCM-SHA384"); + auto cipherSuitesAES256 = TSslContextCommand::Create("CipherSuites", "TLS_AES_256_GCM_SHA384"); + auto maxProtocolTLS1_2 = TSslContextCommand::Create("MaxProtocol", "TLSv1.2"); + auto serverConfig = TBusServerConfig::CreateTcp(Port); serverConfig->EncryptionMode = EEncryptionMode::Required; serverConfig->VerificationMode = EVerificationMode::None; - serverConfig->CertificateChain = New<NCrypto::TPemBlobConfig>(); - serverConfig->CertificateChain->Value = CertChain; - serverConfig->PrivateKey = New<NCrypto::TPemBlobConfig>(); - serverConfig->PrivateKey->Value = PrivateKey; - serverConfig->CipherList = "PSK-AES128-GCM-SHA256"; + serverConfig->CertificateChain = CertificateChain; + serverConfig->PrivateKey = PrivateKey; + serverConfig->SslConfigurationCommands.push_back(cipherStringAES256); + serverConfig->SslConfigurationCommands.push_back(cipherSuitesAES256); auto server = CreateBusServer(serverConfig); server->Start(New<TEmptyBusHandler>()); auto clientConfig = TBusClientConfig::CreateTcp(AddressWithHostName); clientConfig->EncryptionMode = EEncryptionMode::Required; clientConfig->VerificationMode = EVerificationMode::None; - clientConfig->CipherList = "AES128-GCM-SHA256"; + clientConfig->SslConfigurationCommands.push_back(cipherStringAES128); + clientConfig->SslConfigurationCommands.push_back(cipherSuitesAES128); auto client = CreateBusClient(clientConfig); auto bus = client->CreateBus(New<TEmptyBusHandler>()); @@ -781,11 +536,51 @@ TEST_F(TSslTest, DifferentCipherLists) EXPECT_FALSE(error.IsOK()); EXPECT_EQ(error.GetCode(), EErrorCode::SslError); + { + clientConfig->SslConfigurationCommands.push_back(maxProtocolTLS1_2); + auto client = CreateBusClient(clientConfig); + auto bus = client->CreateBus(New<TEmptyBusHandler>()); + EXPECT_FALSE(bus->GetReadyFuture().Get().IsOK()); + } + server->Stop() .Get() .ThrowOnError(); } +TEST_F(TSslTest, FullVerificationWithEllipticCurve) +{ + // Connect via localhost, ipv4 and ipv6 addresses. + for (const auto& address : {AddressWithHostName, AddressWithIpV4, AddressWithIpV6}) { + auto serverConfig = TBusServerConfig::CreateTcp(Port); + serverConfig->EncryptionMode = EEncryptionMode::Required; + serverConfig->VerificationMode = EVerificationMode::None; + serverConfig->CertificateChain = CertificateChainEC; + serverConfig->PrivateKey = PrivateKeyEC; + auto server = CreateBusServer(serverConfig); + server->Start(New<TEmptyBusHandler>()); + + auto clientConfig = TBusClientConfig::CreateTcp(address); + clientConfig->EncryptionMode = EEncryptionMode::Required; + clientConfig->VerificationMode = EVerificationMode::Full; + clientConfig->CertificateAuthority = CACertEC; + auto client = CreateBusClient(clientConfig); + + auto bus = client->CreateBus(New<TEmptyBusHandler>()); + // This test should pass since (localhost | 127.0.0.1 | [::1]) is in SAN. + EXPECT_TRUE(bus->GetReadyFuture().Get().IsOK()); + EXPECT_TRUE(bus->IsEncrypted()); + + auto message = CreateMessage(1); + auto sendFuture = bus->Send(message, {.TrackingLevel = EDeliveryTrackingLevel::Full}); + EXPECT_TRUE(sendFuture.Get().IsOK()); + + server->Stop() + .Get() + .ThrowOnError(); + } +} + //////////////////////////////////////////////////////////////////////////////// } // namespace diff --git a/yt/yt/core/bus/unittests/testdata/ca.pem b/yt/yt/core/bus/unittests/testdata/ca.pem new file mode 100644 index 00000000000..6a91723fe4f --- /dev/null +++ b/yt/yt/core/bus/unittests/testdata/ca.pem @@ -0,0 +1,31 @@ +-----BEGIN CERTIFICATE----- +MIIFWjCCA0KgAwIBAgIBATANBgkqhkiG9w0BAQsFADBGMQswCQYDVQQGEwJSVTEP +MA0GA1UECAwGTW9zY293MQ8wDQYDVQQKDAZZYW5kZXgxFTATBgNVBAMMDENBQGxv +Y2FsaG9zdDAeFw0yMzA3MTMxMTUxMTFaFw0zMzA3MTAxMTUxMTFaMEYxCzAJBgNV +BAYTAlJVMQ8wDQYDVQQIDAZNb3Njb3cxDzANBgNVBAoMBllhbmRleDEVMBMGA1UE +AwwMQ0FAbG9jYWxob3N0MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA +mxWRSfbYAJcBgGhYagnQMJOiW60B82Ok94n8muIaa2gRo0ZHKmtY7CMiFIIN0GhI +Blw+G6HPN8SPYK0+bTlAIHlb0u6/ty2AVyveiH31p7Ld/Ib5S/MSKuMiGt5S/Ci9 +mSdRAmD46twIKfm9bY/8SZmNsTCJIrKnJaTbjTnbz9O3FVAYjiuc8yNGb1LnZNA6 +B4ZrF3fkYr9kn4nKSOd6mypPFIOZAxKGQ95X9nCblajEMAPzHfr+1EArnM+PauAO +cMcxfKT+OlMPGR4ZtZpl90qX6ZVZqD9zd8gp2I/2SM1vVS7AT96t89T7Uag8OjH2 +c8jMCD2z1fk1oDD4K53pGkBucTwDolCvxIbN77gcwkutdor/dbHLqIvGV/M4Z/iA +GzqCD6pCpi5gT8SXn2IQrlvSdF01k9YZS093Y8bIQm3dCo1lKDaz6oHytk9Ro+fu +b+dLOhSjopNfa/1Thw6fgta/mRsdgubWI/IHn+IyMpW65vGbnrMD4oQl1AanRJj5 +iBS73ZXIs/Y9LuMtSNk/I9u1o2fc+Sg0zb1AchC70h8M3sAYaGbqg1churOaDuTO +Yom5W+bAQO8uLUvmlDcpqxMBYqeE8VJGJWRLtQnhM18iYAa03w7m6uMzrlkoX1Q7 +AHnX4899WSUcz/BJc5JBAtHQXCzEzyudT+8xw5MwqRkCAwEAAaNTMFEwHQYDVR0O +BBYEFFuWdWxzQ6TF/yH3sQnlk3EEr1VjMB8GA1UdIwQYMBaAFFuWdWxzQ6TF/yH3 +sQnlk3EEr1VjMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggIBAHqS +W4FIjH/YKkcj/wuh7sQQTIBAx+LsjosyHb9K/1QbgLNVpxawl/YlOiArfVmgTGud +9B6xdVacxqI6saMqHNPPgYm7OPnozSnprRKh9yGb2TS9j+2G7hCQLnM+xYroP8XE +8fL9tyq14lHb/BFYnPsFFxqeA6lXzzm73hlm/hH58CWpL/3eR/TDjC+oiEBBV4VF +Dm4X3L73MFcniDKkCb7Mw3wdi6zqx231F4+9Cqgq+RqAucnmLXTG7yR7wOKP2u8E +Ye1Jrt3nnMlD1tqiXyRJywPSK9mMiBTGbzGLLiTcvecBIHG8oRuvn1DnkZa9L/48 ++1aTC8QhbRslaeDyXj5IY/7scW7ZkEw18qYCT+9sI4/LngS4U8+taKqLf5S8A5sR +O8OCP0nMk/l5f84cDwe1dp4GXVQdkbnfT1sd93BLww7Szbw2iMt9sLMmyY8qIe7a +Ss3OEfP6eyNCBu6KtL8oufdj1AqAQdmYYTlGwgaFZTAomDiJU662EGSt3uXK7V32 +EzgJbpxSSjWh5JpCDA/jnqzlkFkSaw92/HZwO5l1lCdqK+F1UWYQyU0NveOaSEuX +344PIi8m1YWwUfukUB97D+C6B4y5vgdkC7OYLHb4W4+N0ZvYtbDDaBeTpn70CiI7 +JFWcF3ghP7uPmbONWLiTFwxsSJHT0svVQZgq1aZz +-----END CERTIFICATE----- diff --git a/yt/yt/core/bus/unittests/testdata/ca_ec.pem b/yt/yt/core/bus/unittests/testdata/ca_ec.pem new file mode 100644 index 00000000000..73fbb2884d0 --- /dev/null +++ b/yt/yt/core/bus/unittests/testdata/ca_ec.pem @@ -0,0 +1,13 @@ +-----BEGIN CERTIFICATE----- +MIIB8TCCAZegAwIBAgIUW7+MbzA1uAX+WMOEf0RPu8LtFiswCgYIKoZIzj0EAwIw +RjELMAkGA1UEBhMCUlUxDzANBgNVBAcMBk1vc2NvdzEPMA0GA1UECgwGWWFuZGV4 +MRUwEwYDVQQDDAxMb2NhbGhvc3QgQ0EwHhcNMjUwNTEyMTQxOTEzWhcNMzUwNTEw +MTQxOTEzWjBGMQswCQYDVQQGEwJSVTEPMA0GA1UEBwwGTW9zY293MQ8wDQYDVQQK +DAZZYW5kZXgxFTATBgNVBAMMDExvY2FsaG9zdCBDQTBZMBMGByqGSM49AgEGCCqG +SM49AwEHA0IABP5RdA2cxb9QfFA5qpP3R49HFRWnWk0obvp2laNMJSfO4rV372lB +jp3nzZQCRO20oYfjNTbW3YX28IlbV4KiYeijYzBhMB0GA1UdDgQWBBTF9gjbblH+ +zdrnV6d15/WHVsvXMzAfBgNVHSMEGDAWgBTF9gjbblH+zdrnV6d15/WHVsvXMzAS +BgNVHRMBAf8ECDAGAQH/AgEAMAsGA1UdDwQEAwICBDAKBggqhkjOPQQDAgNIADBF +AiAkB3dhKbgCWQ5uoo94FY4IqtY/uaf4+HVMXKzpwK3pAwIhAMjTWGA5wh4RFaYp ++qMvt7QTKc+irhL9nHrNhJUqplnF +-----END CERTIFICATE----- diff --git a/yt/yt/core/bus/unittests/testdata/ca_with_ip_in_san.pem b/yt/yt/core/bus/unittests/testdata/ca_with_ip_in_san.pem new file mode 100644 index 00000000000..4fbccb3c3bc --- /dev/null +++ b/yt/yt/core/bus/unittests/testdata/ca_with_ip_in_san.pem @@ -0,0 +1,30 @@ +-----BEGIN CERTIFICATE----- +MIIFHzCCAwegAwIBAgIUQEt4xnHWGulMGzqad434c4Mw+cAwDQYJKoZIhvcNAQEL +BQAwJjERMA8GA1UECgwIWVRzYXVydXMxETAPBgNVBAMMCFlUc2F1cnVzMB4XDTI0 +MDIyOTEwMTIzM1oXDTM0MDIyNjEwMTIzM1owJjERMA8GA1UECgwIWVRzYXVydXMx +ETAPBgNVBAMMCFlUc2F1cnVzMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKC +AgEAwUS9deIwfMNeicFqw7/fDslSb7sSiPbYfYUxwuELBR8nVomaL/a7IpQx5dOw +nB2CSUiTDsLczNaBvQyNkjJ8VIX4fEPtgfHCuoUaCg4NwcaHfY0TSssbCKh8U92z +fM/iSCjjkMdp9pTzZL93yam72dqfFbHmBvO6XiGjYeR9BL6AqoZMBnIjmxsU9JCW +SM7B0l+Sf8dsH8DmnzqtLUXKIZJyVK6LOVb7MtSw0iAYatrIt3t7IytqZMOe+dEZ +V0C0YzTecOgCWKS+rA8X1h7zHW5vXLDYU6RR+qw3gFsRK0oHmECuo+4Ui2nMzkx1 +jf2nK8L0Z2MHP0YFI9cGCBfSZMv6eocbYZI1DmhNKZNFhSLL8lmfzv4uR7WPyNL6 +Ml+2xgp03r1QFx93cRNW5bjjFytzYQkEVwbvLsABHDFm056PpNXKh6eoXGkUfaWf +iXxUWhAFJRxggkTrPawCr+YQtVbBFe/YcdrTkdvFengFLm4wE20RiJUZWvG4oXmV +JdKvDMsR1RFoQQfSqKL3mERMvVBBO1CXqymdPW6K+S2r367ryXGCG6kRDGpfmu7R +vYZcA9BJqNvGcowf5fBe2VZf3cqZwdZKjCYOR3dCVDbXClraeStzy0VXKgxukHyT +g3XfPqptaUKtpy413b/AVG2YupEn+SoOGMP0NKMyj9XAr20CAwEAAaNFMEMwDgYD +VR0PAQH/BAQDAgLkMBIGA1UdEwEB/wQIMAYBAf8CAQEwHQYDVR0OBBYEFAYGwcRb +cWnfIkC1Mozkzm0CDcFbMA0GCSqGSIb3DQEBCwUAA4ICAQCztlZxgJNdhebicTkT +B4iXXntoOzlnusua1lCBKRUowHAocw2ylXb32Ahh/fENmHyt/jYTlcFu/qK6Q1T8 +/pN14hnyF2pB+ZJjyAr3vNnZtHGJ2xz3wPqFRzD00YUqvvLi9xD2nhsfn4xex/OK +KUx/dbV6FkJ60Fpg97zeOmO+5Kh9srIEGEczPb+y9meiMGB91tm+ZUcd9cGShTRH +krFkuCySJuNgrAZCxqvdsUPvPDd12lPyqmfmuVjauON9ENYUYwNoxQc8MxEx0x6j +QolYDmc2A2VP29rTLQhyjf8TvwXfq8z+zNoeQZCNCON5jg7zH67XarXxK1AZOePc +ZqHtjfTLTqmmKkkG7IRt1h9dtGsROMaRfXf0t4M9mvqx/1Cx6abA6LfcOkp7OG8S +0tx0IzIRnQh2iN4zR1MihS3hn3s9ayviiaopIPVyCKEKRZpsL3QzhzydnQM/Wb+r +UTe546vZV3q2irHH/x4SZFWoFhDwepAyUMI4qo1REd+cM/MakLP5x4nFzzDmPjyf +FiuyqHTlkMtjveytSblzpzWE1/Sum3RcMh4s9ECq8XaUl/8FerUYvIJfRDq1j79J +w/cDyD142joRYwQG0HQkmE4ph4mYFwKhmYOv11Wik9zvEt156VPFaExu6rkjmLia +nTkPBMUXiU3GIb4H7k78sEjv1g== +-----END CERTIFICATE----- diff --git a/yt/yt/core/bus/unittests/testdata/cert.pem b/yt/yt/core/bus/unittests/testdata/cert.pem new file mode 100644 index 00000000000..ffcb0da681b --- /dev/null +++ b/yt/yt/core/bus/unittests/testdata/cert.pem @@ -0,0 +1,31 @@ +-----BEGIN CERTIFICATE----- +MIIFUTCCAzmgAwIBAgIBATANBgkqhkiG9w0BAQsFADBGMQswCQYDVQQGEwJSVTEP +MA0GA1UECAwGTW9zY293MQ8wDQYDVQQKDAZZYW5kZXgxFTATBgNVBAMMDENBQGxv +Y2FsaG9zdDAeFw0yMzA3MTMxMTUxMTdaFw0zMzA3MTAxMTUxMTdaMEMxCzAJBgNV +BAYTAlJVMQ8wDQYDVQQIDAZNb3Njb3cxDzANBgNVBAoMBllhbmRleDESMBAGA1UE +AwwJbG9jYWxob3N0MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA1ZrD ++kcfmsi52zMZ7cJxWl8pMI1kX1uQ6KkOJRVQVuweJuhp35rJGK0vHkWvrpihO+Ue +h1jncReH8PaNdejt3gPsYJksUeRxCaZ8O+awtyOq6TK8L9pDzbQyXme5k2Ea/+yl +XSHrGGzZUuHaZRzBigyQXzb04ZdQMzqGGKUz52ed5elZnOoAMUL7/wZ7O8n5dlXh +DwxEQ4SQ3a39SLpPWv3hWXiIDrlR5xhkqQjZnb0WEVlSwN8lpyjA6XwCs/IX7PVr +khoBKq7664rsR3B7qhGA5cmSy4+KrSoYKYHGCvl6OakO7fn4FcJwHHFNQzMq8oa5 +i5LbYsaCy6IgJt07nBeNXxaEY+4WIImHfWR0yvQgqtrFQZuIQ4N7YeZpLvTtoO3I +UJyjMsih5GX4LPY0k9nE8rd437lV3PdiBYAktkwmgCTY9uJFhCBPFsMhE30nL79t +X0gm5FugW1aZ3CFvgQywkcYQIjBjJQ42zcpqnLjj4lIwfDO5YEAiAOPqQ3VV7FKm +bSWpeXyRAL/uqkBVCDKTsKJkEjd8gc30honRIKNHQkr/RoV1M1W449uRkvfVfbfg +kdcRcYMIynTwZMSiKDoxXkQyrcyY6qVjYQNUKwRvtN2m6aGftYhfgo4j01UHTrK7 +BHRHKjW5LFXgNJUXnycmO69B+w1eTRgnVE7ws6sCAwEAAaNNMEswCQYDVR0TBAIw +ADAdBgNVHQ4EFgQUNvB1ghYixkG53VZoSvUPnCCN7hwwHwYDVR0jBBgwFoAUW5Z1 +bHNDpMX/IfexCeWTcQSvVWMwDQYJKoZIhvcNAQELBQADggIBAHW/+xxign61axqT +hO338rJGuidhXzraCQtJT/J6OwUaI+P6C71wUoBKEk3SrIVNG1Q8mrlsj93EzQuv +M9OvBenIomF7A2ydaFL/6NJhZ8W6PmZwYF3VvInYJ2p7zqCMjqECg2C92NIC7+Z6 +fdmL+xoj3XKYypqA1x6xvSnCtXyXGRCxta3Es163wbqffq+4jjBUFOyUr9vk2N7X +vy3/x8LWHIXffzNSJLtnXiznSNBSubmedac8JQ+XE9RgK+R0kUrj1lqnkSg+tPWD +jP52kG84J9URV18BZfLFpcUiYloWXREfwNXRhMAQ6DyucupLW1Skl+Nf9K+41C3h +f4mDn4Axn6toBzav9NLdFelGVAp4R3Yjoiv2LvpwYxGnMs/cPJMGm/NqoAbuyOMY +ZKZPWvwsbxeaG8u9GRGvTSpawnGWJqIgxknKpQT5QztjNwtI9iT0/f4n3CPEGdAi +6bHw0q+jiCKmXZMZPHyJ/tSJ74H7tdeWYYjhthJWnrAz4BziZ+bFBLkcYq95VV4L +pOeMUr0PUi8fpSK06wIVTES9AWgcfuXL6i7AQ+hadEa3Ve1BGRsu0KOXW2XZZFeo +3Pczm/o+jwMiLELcgrM5Ngy8dcCKr6v84F+fi9Y+C8+RZ7g37aLJM0kqbaoN8owL +mP88f9xDGRAmesvuYlHo+57VTyzU +-----END CERTIFICATE----- diff --git a/yt/yt/core/bus/unittests/testdata/cert_ec.pem b/yt/yt/core/bus/unittests/testdata/cert_ec.pem new file mode 100644 index 00000000000..27e2b46b02e --- /dev/null +++ b/yt/yt/core/bus/unittests/testdata/cert_ec.pem @@ -0,0 +1,14 @@ +-----BEGIN CERTIFICATE----- +MIICODCCAd2gAwIBAgIUMizwaqRBfZMqZLAfqOon9LxVUwswCgYIKoZIzj0EAwIw +RjELMAkGA1UEBhMCUlUxDzANBgNVBAcMBk1vc2NvdzEPMA0GA1UECgwGWWFuZGV4 +MRUwEwYDVQQDDAxMb2NhbGhvc3QgQ0EwHhcNMjUwNTEyMTQxOTEzWhcNMzUwNTEw +MTQxOTEzWjBDMQswCQYDVQQGEwJSVTEPMA0GA1UEBwwGTW9zY293MQ8wDQYDVQQK +DAZZYW5kZXgxEjAQBgNVBAMMCWxvY2FsaG9zdDBZMBMGByqGSM49AgEGCCqGSM49 +AwEHA0IABOM5UtoWuYTndtECRIdexT/5o5M7Dj5sBe4mPcydByC8AY6sSUghpzTD +HU/gNnfja8unXt1gukZEe9h03uoH37mjgaswgagwHQYDVR0OBBYEFD3bqNr1fG99 +dKCFAsKkTHwNHkF+MB8GA1UdIwQYMBaAFMX2CNtuUf7N2udXp3Xn9YdWy9czMCwG +A1UdEQQlMCOCCWxvY2FsaG9zdIcEfwAAAYcQAAAAAAAAAAAAAAAAAAAAATAMBgNV +HRMBAf8EAjAAMAsGA1UdDwQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYB +BQUHAwIwCgYIKoZIzj0EAwIDSQAwRgIhANIfRVOvxINkIBKeW60zD7BkrYBXWczl +FmSagrLHbVpDAiEA+HkxWw0d1mFaF5ZQ03DEbeb0gFQQDQCrqk/hS6ofCbs= +-----END CERTIFICATE----- diff --git a/yt/yt/core/bus/unittests/testdata/cert_with_ip_in_san.pem b/yt/yt/core/bus/unittests/testdata/cert_with_ip_in_san.pem new file mode 100644 index 00000000000..0c3b3cb4ae1 --- /dev/null +++ b/yt/yt/core/bus/unittests/testdata/cert_with_ip_in_san.pem @@ -0,0 +1,31 @@ +-----BEGIN CERTIFICATE----- +MIIFVTCCAz2gAwIBAgIBATANBgkqhkiG9w0BAQsFADAmMREwDwYDVQQKDAhZVHNh +dXJ1czERMA8GA1UEAwwIWVRzYXVydXMwHhcNMjQwMjI5MTAxMjMzWhcNMzQwMjI2 +MTAxMjMzWjATMREwDwYDVQQKDAhZVHNhdXJ1czCCAiIwDQYJKoZIhvcNAQEBBQAD +ggIPADCCAgoCggIBAJ4BhT+WpWpkEEgaqcbZoV5/JOeAkFj2iZL08nIjkDRVhDBh +dQoop47vB1lpLD/1i5+2YEackPrfeJawbNu32s/belvxHpTc1ljndCJlilMiR5B+ +MccP00fUZRsFhkzznANz++31PI4wamHJ1kdJOgD/4civ0mpWgd5s1hityqsypXI3 +RFbi689+mnBp2sGCD6l3QVbMj+8ORvXOVC1h1W3tExiivabjFILgXwb6WG6ZNgg8 +T20zdH4uEc4d2v6XKY4nz4AqYDHax9oqs3XTOTo0Bld6m7oipjGjToMqqpJD32pb +nxSNT/XECpsNqZ/UMtcQf3HoA2LEOZOg/Knf1mosEww2svb0CqMGfsxxHehCaFJT +CWkdd8GTQ/3t85xtrd5Ccdqb78o+5039H4GkfkcxAIZQe7siNLbUJ+dR6wRMzJw2 +GMcAHEIozOYVDgwyOT/Q3gaKMg4A3Ki2x7tkCie1KmmAcnECWDRhzG/Hg/RdnYqA +L6g/m7Z3bw93rUbwXNudVj1ls5MPoyYtTACtZAXnv/PBBd+KR+RZ//T6dD7B9zzC +MiIsIHemPzE3XsWB6I1AqXv2B8THELgJ8foN6Rho4YQ0t0wjhcM0higfSaiHwqhf +Cgoc5xxOGa8PlEETWQvcT3ORAukpjM22QbZmA3uLN/MR1DJbwNHqiYPiFZ1zAgMB +AAGjgaAwgZ0wDgYDVR0PAQH/BAQDAgWgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggr +BgEFBQcDAjAsBgNVHREEJTAjgglsb2NhbGhvc3SHBH8AAAGHEAAAAAAAAAAAAAAA +AAAAAAEwHQYDVR0OBBYEFDyAfXqSAoKxK/sCyFs3d2yUpSntMB8GA1UdIwQYMBaA +FAYGwcRbcWnfIkC1Mozkzm0CDcFbMA0GCSqGSIb3DQEBCwUAA4ICAQCHRuHI5XaS +PIK3JKzA6l01z7YDHROGn5Xf9hqo5QKHv/aUD01UUhxCbx5z/YZMFDc6iRvu4xQw +HMl8BCvTo3gHSS8JRcgRn1ov5m50NZMl0Ws9xvvy1j5rZWd3HkBTQSpzr4eSHAZp +dUWXN2JEW7QIaIxtoiLwMZVjo3Lfr2Qv4uIn9lbIR0F7s39+qFlzvDx+6JbstJsv +5wPjTS9og3WfpBDaOecHM/nP8v1H9ilx8/EW0nM3jlS0q0Gj9whkha0Pcl156Bga +biLDoQk7uTccO8Wiyddwfq6tlYy1OAIMqDy0vmoz4L/3FHJUqrzO/fdI9VQLlug+ +M2G6qTJHKzmDkvmtxPfTjRFMu+g7L3QEdYCBogfIHS+VoB9a9K/XmoWBg85cWIPw +Kfjjf7OouqksfOQopxY2+PCQ66nnkN7y13RjoU2heAme8Fexiowkjhzc1lq0Zn0Q +XPlnvCHAQMNRNmvBLwNEkW+KN4no0TCImOOTuInBrlKGTaBkinUNS39AF9lZWwAE +hd1kK5zzF6XvZnKXdVIn4MjcW81hcbrnulq5GHz7XY+lwmORYumYo3Gykjj/+G93 +K9HRlSRV1+BNXmPYtI8hvbAYw05+AWKCk0J5r1GQtPx+Tx3sug/2qks26oURgEHc +ySl4OPJLp2lhKCUkKVP24Tzg/iS1xT/uHQ== +-----END CERTIFICATE----- diff --git a/yt/yt/core/bus/unittests/testdata/generate.sh b/yt/yt/core/bus/unittests/testdata/generate.sh new file mode 100644 index 00000000000..816edc383cf --- /dev/null +++ b/yt/yt/core/bus/unittests/testdata/generate.sh @@ -0,0 +1,37 @@ +openssl req -x509 -noenc -newkey rsa:4096 -keyout ca_key.pem -out ca.pem \ + -days 3650 -subj "/C=RU/L=Moscow/O=Yandex/CN=Localhost CA" \ + -addext "basicConstraints=critical,CA:TRUE,pathlen:0" \ + -addext "keyUsage=keyCertSign" + +openssl req -new -x509 -noenc -newkey rsa:4096 -keyout key.pem -out cert.pem \ + -CA ca.pem -CAkey ca_key.pem \ + -days 3650 -subj "/C=RU/L=Moscow/O=Yandex/CN=localhost" \ + -addext "basicConstraints=critical,CA:FALSE" \ + -addext "keyUsage=digitalSignature,keyEncipherment" \ + -addext "extendedKeyUsage=serverAuth,clientAuth" + +openssl req -x509 -noenc -newkey rsa:4096 -keyout ca_key_with_ip_in_san.pem -out ca_with_ip_in_san.pem \ + -days 3650 -subj "/C=RU/L=Moscow/O=Yandex/CN=Localhost CA" \ + -addext "basicConstraints=critical,CA:TRUE,pathlen:0" \ + -addext "keyUsage=keyCertSign" + +openssl req -new -x509 -noenc -newkey rsa:4096 -keyout key_with_ip_in_san.pem -out cert_with_ip_in_san.pem \ + -CA ca_with_ip_in_san.pem -CAkey ca_key_with_ip_in_san.pem \ + -days 3650 -subj "/C=RU/L=Moscow/O=Yandex/CN=localhost" \ + -addext "subjectAltName=DNS:localhost,IP:127.0.0.1,IP:::1" \ + -addext "basicConstraints=critical,CA:FALSE" \ + -addext "keyUsage=digitalSignature,keyEncipherment" \ + -addext "extendedKeyUsage=serverAuth,clientAuth" + +openssl req -x509 -noenc -newkey ec:<(openssl ecparam -name prime256v1) -keyout ca_key_ec.pem -out ca_ec.pem \ + -days 3650 -subj "/C=RU/L=Moscow/O=Yandex/CN=Localhost CA" \ + -addext "basicConstraints=critical,CA:TRUE,pathlen:0" \ + -addext "keyUsage=keyCertSign" + +openssl req -new -x509 -noenc -newkey ec:<(openssl ecparam -name prime256v1) -keyout key_ec.pem -out cert_ec.pem \ + -CA ca_ec.pem -CAkey ca_key_ec.pem \ + -days 3650 -subj "/C=RU/L=Moscow/O=Yandex/CN=localhost" \ + -addext "subjectAltName=DNS:localhost,IP:127.0.0.1,IP:::1" \ + -addext "basicConstraints=critical,CA:FALSE" \ + -addext "keyUsage=digitalSignature,keyEncipherment" \ + -addext "extendedKeyUsage=serverAuth,clientAuth" diff --git a/yt/yt/core/bus/unittests/testdata/key.pem b/yt/yt/core/bus/unittests/testdata/key.pem new file mode 100644 index 00000000000..2192f1c96f6 --- /dev/null +++ b/yt/yt/core/bus/unittests/testdata/key.pem @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQQIBADANBgkqhkiG9w0BAQEFAASCCSswggknAgEAAoICAQDVmsP6Rx+ayLnb +MxntwnFaXykwjWRfW5DoqQ4lFVBW7B4m6GnfmskYrS8eRa+umKE75R6HWOdxF4fw +9o116O3eA+xgmSxR5HEJpnw75rC3I6rpMrwv2kPNtDJeZ7mTYRr/7KVdIesYbNlS +4dplHMGKDJBfNvThl1AzOoYYpTPnZ53l6Vmc6gAxQvv/Bns7yfl2VeEPDERDhJDd +rf1Iuk9a/eFZeIgOuVHnGGSpCNmdvRYRWVLA3yWnKMDpfAKz8hfs9WuSGgEqrvrr +iuxHcHuqEYDlyZLLj4qtKhgpgcYK+Xo5qQ7t+fgVwnAccU1DMyryhrmLkttixoLL +oiAm3TucF41fFoRj7hYgiYd9ZHTK9CCq2sVBm4hDg3th5mku9O2g7chQnKMyyKHk +Zfgs9jST2cTyt3jfuVXc92IFgCS2TCaAJNj24kWEIE8WwyETfScvv21fSCbkW6Bb +VpncIW+BDLCRxhAiMGMlDjbNymqcuOPiUjB8M7lgQCIA4+pDdVXsUqZtJal5fJEA +v+6qQFUIMpOwomQSN3yBzfSGidEgo0dCSv9GhXUzVbjj25GS99V9t+CR1xFxgwjK +dPBkxKIoOjFeRDKtzJjqpWNhA1QrBG+03abpoZ+1iF+CjiPTVQdOsrsEdEcqNbks +VeA0lRefJyY7r0H7DV5NGCdUTvCzqwIDAQABAoICAEvCv0TLKiH/lK/y4XzrTMIF +Y3oVhCawNubWYy5y71JNH+qj3z1QTIgEkOQ3Sjbuaq1wN9JAjaIWewBTqlvKOGfY +02N1oHsRP6hxFLo4ObBTJcDdXlLIoujYQ08pke/8bpOcDxDHwXch0Djt40SenOSG +TUSAHP3QacEpvjsKiSzHmwDbMY4Oju/p9q/+0AGmQuUeU5s/Og0KfUkq911uu0um +JWHS9srmHu8Mv1MW0Px5/tQ7brb6zoOJ2FZXxiulr6e7aiJhN824T0XwuZojArGQ +0LtvsbGiYUjG19gM772fu6Ks3B863Ct3kcT8yK8PfGmVsESZW1ee2fA4uheeuw+c +/3D71+dnjvw0/LFQEWyiH9NhjvafALBGxtlbfTTVtw5bHVaoalYpEggKnJrRabPa +HmdPkEAK0vJzBvTjPW1lEvlxUkol1SsYq8xpBkatieZzf4+SWk/ugP7rvoRVRbB1 +GmHc9CK9jCDTcN7ii5pTOpc2VOK5cvn+K5L4P7Qw4kK37pyO56jhVbTAavNxotSc ++mZa8OqZaK0jvD9sUPCSuY44x5X6WhXjILE+R3QXXXtGkyjlgJxGmfkYcu19GV4B +ziU7hVCjqtNOQ51ywEtWUA93lchj3fD5ryo9dCv5a5Xco4O5E24VeGjW3MBqeBNW +wNIUintgwVt20P1I+xZRAoIBAQD9UzjlhvrDgNZsaJu82+PoOEJM2pa5F+fDGSeq +XTYCCeqioMUujJWDEkBEzfTvu4r7Pr7uExOlkyUXdSELfm+ABkt+eZguPB15Ua6V +Y59Z4yrpqp0JIwpxXGxNOsSUnVwGg3OTlUGWYVyxyGHYQnSGJNe2/NhSlQZmvxx2 +PxqTH+g9dLn7FkQk/FNQI4l4LnphgGvNBVyXZoHcw16y86MAKLyrcZBmbHnuWMC6 +zjIu61uXXd8GU7IU8BWJbUQKvz0Y5susWf921U/7Qa8NZ2vFY4Q7oGlJVBVhMNmB +WLL8/WUeWXu9HvL89QWpb31l5V5L+lF+GSq5ZQxFAjLEVl77AoIBAQDX3CwirHtf +MI7z+zjwLDWgVG7RFVY4Pv5TWDWYHiQT9VPBU16K0+fVS1iXxYvoiGJkrj38Re8y +rkTZ+qvKLmlDftqx7bImbzs01QJRbUH2gxZFMDQIF7uTSFynycywaQk+xpFEPTHl +CXMIyGAMAsd0B2OKPAxZNoHTd0P10FYBIENwxadG9w7Ocm41J/d4Qx6wQsV10hGF +0OgtIioplSDz1Ean/IT8XmwLK7tIiShJzZidE+0sY4depy3JdAHWv50uXMs3bIcu +xvVBA/e/jf+HssfLjzzrYtwp2JBzSOTox/eCUJ/i9gatC5ox+ewjwOaFXV3bGEib +mK+I3xiQ3B8RAoIBAE0Jc/IJHFU75vlMzp+eVy6VfUQV7WQYavifu7pJYlU4YsxW +C+DeC9GySS0jXOtSoy9Io5OO5ZiiqNL7YbM3Hf1W7Lpni+nzihsMxgTUKO+S78fj +hKH0sAZNTvoldwai3At3CjzFVQ7ASQofn/G+M+VfauJQ/hAPFcVFNQiYpCI9v8iA +qNY8rTh6K3Pherq7l6fy/9V3XfMEz1UtbK0K/nTb7pRMktczAdmD0Ah/EC/Ijy/2 +8g3ggfVwFXyXZ+vEwHXEKggdzlx6/jmwfeWbn+CFJP9lBt+v3FiUHHEDYlshTBDw +sXqP4OEgOjqOlxnXqNd+Ji4sxRtgKV0LEBk5EuUCggEAIlBbq79jdURQ1TQQXw2I +EM6bNx1/MT3CTBlvm5je/1U2VTsdglAhQGTT1nyOuw5DJeIU9G9hkNrnEweoG2G5 +VgNqXHJ+qWFxNfrOfYcyvy8jcSgyfT7YkJcmM33+zeRElfgWy5Q2xEP2R2Ui74XZ +kvZBuo3FIMFrbeQ9p2vQ4Cjyz5B8AOnxLpw+LLEHw9RXoolavloAcxc8cUBHF4kf +TeNmv/mCYmPYJQZ0pRk4kFLgecfbIf1IXaGRw75vNGYNZHtXyp2z95mlDwrEbWzz +O+0NmaxRcNGsUfKdM9ZYnTB8hfivEfMuKH/5qQwjn6Nggb7P1q5LjIB/FvDwBMcZ +IQKCAQBfPoddROG6bTkzn7kL8rDwuk2H2sLh5njRKo3u29dVlwU93lmKLoP4blcf +tAbXDVrlGpZbIo73baE57Rv5ehcV64oS/G63Lcw8nsqmBvs9982tpYPNX9EOvXV5 +1iK+hs8ZUUVCVkTOXsZb5M90VKQiHrnddO+08lE5YI/lzM5laqcytPmYcLkVWPPz +qrpW/AReSwhvwVugcMFUgMXaDx/3SAY75B808wX1tizv76omWZAQ774FeGQGyP4C +8f4t3LIV9h/q2Hj8geMjil9ZGogtWJ5uDspp7As5OyMF0ZMXTMwSnFsXB/L4YIk+ +rPl77gAcribJm3TzBVHm2m6jBGtb +-----END PRIVATE KEY----- diff --git a/yt/yt/core/bus/unittests/testdata/key_ec.pem b/yt/yt/core/bus/unittests/testdata/key_ec.pem new file mode 100644 index 00000000000..8000d1435c4 --- /dev/null +++ b/yt/yt/core/bus/unittests/testdata/key_ec.pem @@ -0,0 +1,5 @@ +-----BEGIN PRIVATE KEY----- +MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgYzqbMURDpLQaUBEI +sR9pNuyopS4/ACCkDkxjlUX3aquhRANCAATjOVLaFrmE53bRAkSHXsU/+aOTOw4+ +bAXuJj3MnQcgvAGOrElIIac0wx1P4DZ342vLp17dYLpGRHvYdN7qB9+5 +-----END PRIVATE KEY----- diff --git a/yt/yt/core/bus/unittests/testdata/key_with_ip_in_san.pem b/yt/yt/core/bus/unittests/testdata/key_with_ip_in_san.pem new file mode 100644 index 00000000000..2e314b8f7b4 --- /dev/null +++ b/yt/yt/core/bus/unittests/testdata/key_with_ip_in_san.pem @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQCeAYU/lqVqZBBI +GqnG2aFefyTngJBY9omS9PJyI5A0VYQwYXUKKKeO7wdZaSw/9YuftmBGnJD633iW +sGzbt9rP23pb8R6U3NZY53QiZYpTIkeQfjHHD9NH1GUbBYZM85wDc/vt9TyOMGph +ydZHSToA/+HIr9JqVoHebNYYrcqrMqVyN0RW4uvPfppwadrBgg+pd0FWzI/vDkb1 +zlQtYdVt7RMYor2m4xSC4F8G+lhumTYIPE9tM3R+LhHOHdr+lymOJ8+AKmAx2sfa +KrN10zk6NAZXepu6IqYxo06DKqqSQ99qW58UjU/1xAqbDamf1DLXEH9x6ANixDmT +oPyp39ZqLBMMNrL29AqjBn7McR3oQmhSUwlpHXfBk0P97fOcba3eQnHam+/KPudN +/R+BpH5HMQCGUHu7IjS21CfnUesETMycNhjHABxCKMzmFQ4MMjk/0N4GijIOANyo +tse7ZAontSppgHJxAlg0Ycxvx4P0XZ2KgC+oP5u2d28Pd61G8FzbnVY9ZbOTD6Mm +LUwArWQF57/zwQXfikfkWf/0+nQ+wfc8wjIiLCB3pj8xN17FgeiNQKl79gfExxC4 +CfH6DekYaOGENLdMI4XDNIYoH0moh8KoXwoKHOccThmvD5RBE1kL3E9zkQLpKYzN +tkG2ZgN7izfzEdQyW8DR6omD4hWdcwIDAQABAoICADzT/QZD6p6QsyvvB9lDwznr +3Ls65Vc6YjAvGH8UbdmX6nHtsu8cQ5VlNAEZ2i0tTHlJ7rqAX9gU3Am3FdFocFaA ++hQXOVy5v9MuF6l+SchDdCWOT3+A+ie2/s1uTQum5TL3Hc+4D3316Z6H43RCHpBv +8e4esfS6JPkKEUoi7dkGgGb+G9MPPRT+elo4hjzk4z6saH0P94Fij7LlocZu2Yme +MTHUxQpQdX8E/dBj5FN/rCtzfGhf3MMO3U/qcnp8m0Tc0qdWqP3IahP1SG1dybQ8 +fwyCaR05ZZ3KbtlUPaJdes8pQo7Y8CV/OU4D7n9XY9MjyMyDM3p8bGYHHf4P7C0M +qdTYAVXZ9NvKlf87J8zMJ4YDiKTMLdgFjgnj/tYJ9PkUZHgfsk1KSr5IT6wpLBAx +jEs0+WsbQRmtlJ93ZvZsDJyGZbkeCRIAxyz/eN2Rf56YGrWZcPcn8yrAsnt4nYqz +ewjM9wu6jB8bfAgDS0y3kPsl1www+AkRDnNQ9s+Vw5x0nccjbV7fdezkwf04cA6N +QL+0WSALsU0iGGPKrLA5L/kC3W650xSeOgINdvCnZsSEPAF6HQ/E5udd9HD5+7Ma +hVl2S7FbEVxKgXpXPd1GFBKZKnUTCB5erMDyaN7jAiaVBtRE8qc0hQRhHCtE7qNT +HvWkztmzBuVGTpOkBQuRAoIBAQDDIsOMkYo/svV/lsUJM4fVMTVJ354ThhxvFdJs +rlaUEX2mZIsbzjI76yuOsu6nlGKLuiYVfAAV3HNEhNrSnp6+3NPXMgSsE33TFRPR +Y6P1oT/ZE8YrBkndSioU1ad9Z0ClRNhDlP2c9xCBGpXUSQLt/4C2Nk/wgG/wy7gG ++hJ2ybn133DPpwcx1KQCpslF+VaECPLrpjxWCFEGInOV8gVvxhEb7hrD8qmUPlAR +3WYFT6/RJGCJ7BeOvXWYa8AfWRysLfLwnzuoUxl45O7X7eZqPufrhAJyDNgN3c6J +VgVxNPSSc6mjsY0NyItQZEOXtzRJqnRlmGD2Zd5BJFG7PmIVAoIBAQDPSgFqnFe1 +RM7AtC5wRBLTU/LQo8XV3WPsvQpYaa2+YyPG1bjSMWmdejsy4k+lbUx4VzD+veX7 +ZdSmukYmeGT0pp6Yy7FYYAcZ8Tu+CSAxGH8Bvab+U1oy2Y3++lXdAc0OVjuezVSy +RbUE97S9MjeFMTKxIg2RkJc4p+Ir0sFn8rC9a0ZqNdcV1BCzJkMB5Y0PbG+afopL +wS4xIMIy2Jp3Y0CanOW/dl6LnWYdEtWMt0Pi5wFDVVn0/DgGNx0SDG5fKj4rJE4F +6mezK3kyAf7qI35nf8wm/rL7wL262F4iGZp+OKNFQnaEuKmrUsz/l2GZZkfIIcbI +pPVaqKbAQEtnAoIBAQCj9V/NirRY1WuFqw8frhahwVj/G09dJEBb7kACZXIFs7SZ +zL09vcFjqzPMEPiKAhnTQbOiNbB0reiEWATtF65WvIGavUJDu5TreThPpaMsTjKx +mPXXTM9fimNVYjf7HHiq5O+5yURXURijAc2Gs1os05Q4heYhNCnab7HO2uwMt27y +8q19LODUs9CjEbTogJp7EnHaIrFrsE00FFp+UP7Ubd4OU8BViF2IW9s3R4njSJN3 +7VLYUHFy1CosycyCCoQW//yyxXiA9GHgvKsa75+9AeIod6D+Z2BaNlbF+mtUNaSS +MXEGQ7c7L5gvEi/hGGRsyTZH7wL5xZo7reKmq8IJAoIBAHIiaHdAEFcBxOl8DFnK +UadEgNz6X/LqzJtMV0bpIT5ELi3L/dDWXjXUWIYi8AHBFarpL1QEUX5Dynvm8rs5 +7TR8DbVJ6qMjdKWHGwL+2VfPChd2Sl2cnXyEJ1gulFp1JGfxeTBuFGV4Vjye+0h1 +Pva6aRP5EQmGWI1cev7wM4e9rC0PxRyz+nLNakiKF7kSoMHOTgD+Db26Z2mrhOIk +O6Di6G55V1M9pL8w8kmt1iF9wwZLdXmSpE5tFZfufrYyXA9QHhz5B3DgaSrRFBFB +4g8fbfkk9868zOYrcQxRGDukZ1l6bAO1nbZkSx/HHpLY0md5VqrOVjqiAWpilDYk +8J8CggEAVN7WQIMWzuGBc9Hj6H539eXLnoWgccvYN+GwqclQGKQJdUQrM7Ci3kP/ +Zj00T//jew/vHtxx6U1XUfCIbI5SjujqhbkCkikUMAhozsjlmUlAxKz8689/XFXK +9bPRRvmKw6p14drk507w6t3uD4E/O3PHeEtrdEeDmY78e01T5zkg90jVq10szUMo +sxw0PoOG4ktYtCsmwXXPHq9gRtzfOqZT/UNHgnBsLALmWcec2gZLa++M1OnVaqI2 +hTyvWDnxD5oKa7hDSBXTOorcfQVRSaC05HPGMhX+HkfHwfXJBxAE1UC38UMD7x58 +AbE/BnHl1tAmZXLMrHq/4r0wYUjBsA== +-----END PRIVATE KEY----- diff --git a/yt/yt/core/bus/unittests/ya.make b/yt/yt/core/bus/unittests/ya.make index a36ce3f0374..75643918d97 100644 --- a/yt/yt/core/bus/unittests/ya.make +++ b/yt/yt/core/bus/unittests/ya.make @@ -17,6 +17,10 @@ PEERDIR( library/cpp/testing/common ) +EXPLICIT_DATA() + +DATA(arcadia/yt/yt/core/bus/unittests/testdata) + REQUIREMENTS( cpu:4 ram:4 diff --git a/yt/yt/core/crypto/config.cpp b/yt/yt/core/crypto/config.cpp index 1face17aca6..0981b952b9b 100644 --- a/yt/yt/core/crypto/config.cpp +++ b/yt/yt/core/crypto/config.cpp @@ -1,38 +1,80 @@ #include "config.h" +#include <util/system/env.h> + namespace NYT::NCrypto { //////////////////////////////////////////////////////////////////////////////// void TPemBlobConfig::Register(TRegistrar registrar) { + registrar.Parameter("environment_variable", &TThis::EnvironmentVariable) + .Optional(); registrar.Parameter("file_name", &TThis::FileName) .Optional(); registrar.Parameter("value", &TThis::Value) .Optional(); registrar.Postprocessor([] (TThis* config) { - if (config->FileName && config->Value) { - THROW_ERROR_EXCEPTION("Cannot specify both \"file_name\" and \"value\""); - } - if (!config->FileName && !config->Value) { - THROW_ERROR_EXCEPTION("Must specify either \"file_name\" or \"value\""); + if (!!config->EnvironmentVariable + !!config->FileName + !!config->Value != 1) { + THROW_ERROR_EXCEPTION("Must specify one of \"environment_variable\", \"file_name\", or \"value\""); } }); } +TPemBlobConfigPtr TPemBlobConfig::CreateFileReference(const TString& fileName) +{ + auto config = New<TPemBlobConfig>(); + config->FileName = fileName; + return config; +} + TString TPemBlobConfig::LoadBlob() const { - if (FileName) { + if (EnvironmentVariable) { + return GetEnv(*EnvironmentVariable); + } else if (FileName) { return TFileInput(*FileName).ReadAll(); } else if (Value) { return *Value; } else { - THROW_ERROR_EXCEPTION("Neither \"file_name\" nor \"value\" is given"); + THROW_ERROR_EXCEPTION("Neither \"environment_variable\" nor \"file_name\" nor \"value\" is given"); } } //////////////////////////////////////////////////////////////////////////////// +void TSslContextCommand::Register(TRegistrar registrar) +{ + registrar.Parameter("name", &TThis::Name) + .NonEmpty(); + registrar.Parameter("value", &TThis::Value); +} + + +TSslContextCommandPtr TSslContextCommand::Create(std::string name, std::string value) +{ + auto command = New<TSslContextCommand>(); + command->Name = std::move(name); + command->Value = std::move(value); + return command; +} + +//////////////////////////////////////////////////////////////////////////////// + +void TSslContextConfig::Register(TRegistrar registrar) +{ + registrar.Parameter("ca", &TThis::CertificateAuthority) + .Default(); + registrar.Parameter("cert_chain", &TThis::CertificateChain) + .Optional(); + registrar.Parameter("private_key", &TThis::PrivateKey) + .Optional(); + registrar.Parameter("ssl_configuration_commands", &TThis::SslConfigurationCommands) + .Optional(); +} + +//////////////////////////////////////////////////////////////////////////////// + } // namespace NYT::NCrypto diff --git a/yt/yt/core/crypto/config.h b/yt/yt/core/crypto/config.h index 4bd7058140a..d5bc6e8cf41 100644 --- a/yt/yt/core/crypto/config.h +++ b/yt/yt/core/crypto/config.h @@ -8,15 +8,18 @@ namespace NYT::NCrypto { //////////////////////////////////////////////////////////////////////////////// -//! Either an inlined value or a file reference. +//! Either an inlined value, environment variable, or a file reference. struct TPemBlobConfig : public NYTree::TYsonStruct { + std::optional<TString> EnvironmentVariable; std::optional<TString> FileName; std::optional<TString> Value; TString LoadBlob() const; + static TPemBlobConfigPtr CreateFileReference(const TString& fileName); + REGISTER_YSON_STRUCT(TPemBlobConfig); static void Register(TRegistrar registrar); @@ -26,4 +29,40 @@ DEFINE_REFCOUNTED_TYPE(TPemBlobConfig) //////////////////////////////////////////////////////////////////////////////// +struct TSslContextCommand + : public NYTree::TYsonStruct +{ + std::string Name; + std::string Value; + + static TSslContextCommandPtr Create(std::string name, std::string value); + + REGISTER_YSON_STRUCT(TSslContextCommand); + + static void Register(TRegistrar registrar); +}; + +DEFINE_REFCOUNTED_TYPE(TSslContextCommand) + +//////////////////////////////////////////////////////////////////////////////// + +struct TSslContextConfig + : public virtual NYTree::TYsonStruct +{ + NCrypto::TPemBlobConfigPtr CertificateAuthority; + NCrypto::TPemBlobConfigPtr CertificateChain; + NCrypto::TPemBlobConfigPtr PrivateKey; + + //! Commands for SSL context configuration handled by SSL_CONF_cmd. + std::vector<TSslContextCommandPtr> SslConfigurationCommands; + + REGISTER_YSON_STRUCT(TSslContextConfig); + + static void Register(TRegistrar registrar); +}; + +DEFINE_REFCOUNTED_TYPE(TSslContextConfig) + +//////////////////////////////////////////////////////////////////////////////// + } // namespace NYT::NCrypto diff --git a/yt/yt/core/crypto/public.h b/yt/yt/core/crypto/public.h index 9c04e5eba04..c71a42df5f6 100644 --- a/yt/yt/core/crypto/public.h +++ b/yt/yt/core/crypto/public.h @@ -9,6 +9,8 @@ namespace NYT::NCrypto { //////////////////////////////////////////////////////////////////////////////// DECLARE_REFCOUNTED_STRUCT(TPemBlobConfig) +DECLARE_REFCOUNTED_STRUCT(TSslContextCommand) +DECLARE_REFCOUNTED_STRUCT(TSslContextConfig) DECLARE_REFCOUNTED_CLASS(TSslContext) //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/core/crypto/tls.cpp b/yt/yt/core/crypto/tls.cpp index a2dcb50d124..5ac5e062659 100644 --- a/yt/yt/core/crypto/tls.cpp +++ b/yt/yt/core/crypto/tls.cpp @@ -1,4 +1,5 @@ #include "tls.h" +#include "config.h" #include <yt/yt/core/misc/error.h> #include <yt/yt/core/misc/finally.h> @@ -12,6 +13,8 @@ #include <yt/yt/core/logging/log.h> +#include <yt/yt/core/rpc/public.h> // For EErrorCode::SslError + #include <library/cpp/yt/memory/ref.h> #include <library/cpp/openssl/io/stream.h> @@ -36,31 +39,62 @@ static YT_DEFINE_GLOBAL(const NLogging::TLogger, Logger, "Tls"); namespace { -TErrorAttribute GetSslErrors() +constexpr auto TlsBufferSize = 1_MB; + +} // namespace + +//////////////////////////////////////////////////////////////////////////////// + +// Get all saved SSL errors for current thread. +TError GetLastSslError(TString message) { - TString errorStr; + auto lastSslError = ERR_peek_last_error(); + TStringBuilder errorStr; ERR_print_errors_cb([] (const char* str, size_t len, void* ctx) { - auto& out = *reinterpret_cast<TString*>(ctx); - if (!out.empty()) { - out += ", "; + auto& out = *reinterpret_cast<TStringBuilder*>(ctx); + if (!out.GetLength()) { + out.AppendString(", "); } - out.append(str, len); + out.AppendString(TStringBuf(str, len)); return 1; }, &errorStr); - return TErrorAttribute("ssl_error", errorStr); + return TError(NRpc::EErrorCode::SslError, std::move(message), TError::DisableFormat) + << TErrorAttribute("ssl_last_error_code", lastSslError) + << TErrorAttribute("ssl_error", errorStr.Flush()); } -constexpr auto TlsBufferSize = 1_MB; +//////////////////////////////////////////////////////////////////////////////// -} // namespace +void TSslDeleter::operator()(BIO* bio) const noexcept +{ + BIO_free(bio); +} + +void TSslDeleter::operator()(X509* x509) const noexcept +{ + X509_free(x509); +} + +void TSslDeleter::operator()(EVP_PKEY* key) const noexcept +{ + EVP_PKEY_free(key); +} + +void TSslDeleter::operator()(SSL_CTX* ctx) const noexcept +{ + SSL_CTX_free(ctx); +} + +void TSslDeleter::operator()(SSL* ssl) const noexcept +{ + SSL_free(ssl); +} //////////////////////////////////////////////////////////////////////////////// struct TSslContextImpl : public TRefCounted { - SSL_CTX* Context = nullptr; - TSslContextImpl() { Reset(); @@ -68,56 +102,44 @@ struct TSslContextImpl ~TSslContextImpl() { - if (Context) { - SSL_CTX_free(Context); - } - if (ActiveContext_) { - SSL_CTX_free(ActiveContext_); - } + } + + SSL_CTX* GetContext() const + { + return Context_.get(); } void Reset() { - if (Context) { - SSL_CTX_free(Context); - } #if OPENSSL_VERSION_NUMBER >= 0x10100000L - Context = SSL_CTX_new(TLS_method()); - if (!Context) { - THROW_ERROR_EXCEPTION("SSL_CTX_new(TLS_method()) failed") - << GetSslErrors(); + Context_.reset(SSL_CTX_new(TLS_method())); + if (!Context_) { + THROW_ERROR GetLastSslError("SSL_CTX_new(TLS_method()) failed"); } - if (SSL_CTX_set_min_proto_version(Context, TLS1_2_VERSION) == 0) { - THROW_ERROR_EXCEPTION("SSL_CTX_set_min_proto_version failed") - << GetSslErrors(); + if (SSL_CTX_set_min_proto_version(GetContext(), TLS1_2_VERSION) == 0) { + THROW_ERROR GetLastSslError("SSL_CTX_set_min_proto_version failed"); } - if (SSL_CTX_set_max_proto_version(Context, TLS1_2_VERSION) == 0) { - THROW_ERROR_EXCEPTION("SSL_CTX_set_max_proto_version failed") - << GetSslErrors(); + if (SSL_CTX_set_max_proto_version(GetContext(), TLS1_3_VERSION) == 0) { + THROW_ERROR GetLastSslError("SSL_CTX_set_max_proto_version failed"); } #else - Ctx = SSL_CTX_new(TLSv1_2_method()); + Context.reset(SSL_CTX_new(TLSv1_2_method())); if (!Context) { - THROW_ERROR_EXCEPTION("SSL_CTX_new(TLSv1_2_method()) failed") - << GetSslErrors(); + THROW_ERROR GetLastSslError("SSL_CTX_new(TLSv1_2_method()) failed"); } #endif } void Commit(TInstant time) { - SSL_CTX* oldContext; - YT_ASSERT(Context); + TSslCtxPtr oldContext; + YT_ASSERT(Context_); { auto guard = WriterGuard(Lock_); - oldContext = ActiveContext_; - ActiveContext_ = Context; - Context = nullptr; + oldContext.swap(ActiveContext_); + ActiveContext_.swap(Context_); CommitTime_ = time; } - if (oldContext) { - SSL_CTX_free(oldContext); - } } TInstant GetCommitTime() const @@ -126,22 +148,30 @@ struct TSslContextImpl return CommitTime_; } - SSL* NewSsl() + TSslPtr NewSsl() { - auto guard = ReaderGuard(Lock_); - YT_ASSERT(ActiveContext_); - return SSL_new(ActiveContext_); + SSL* ssl; + { + auto guard = ReaderGuard(Lock_); + YT_ASSERT(ActiveContext_); + ssl = SSL_new(ActiveContext_.get()); + } + if (!ssl) { + THROW_ERROR GetLastSslError("SSL_new failed"); + } + return TSslPtr(ssl); } bool IsActive(const SSL* ssl) { auto guard = ReaderGuard(Lock_); - return SSL_get_SSL_CTX(ssl) == ActiveContext_; + return SSL_get_SSL_CTX(ssl) == ActiveContext_.get(); } private: YT_DECLARE_SPIN_LOCK(NThreading::TReaderWriterSpinLock, Lock_); - SSL_CTX* ActiveContext_ = nullptr; + TSslCtxPtr Context_; + TSslCtxPtr ActiveContext_; TInstant CommitTime_; }; @@ -165,10 +195,6 @@ public: , Underlying_(std::move(connection)) { Ssl_ = Context_->NewSsl(); - if (!Ssl_) { - THROW_ERROR_EXCEPTION("SSL_new failed") - << GetSslErrors(); - } InputBIO_ = BIO_new(BIO_s_mem()); YT_VERIFY(InputBIO_); @@ -178,7 +204,7 @@ public: OutputBIO_ = BIO_new(BIO_s_mem()); YT_VERIFY(OutputBIO_); - SSL_set_bio(Ssl_, InputBIO_, OutputBIO_); + SSL_set_bio(Ssl_.get(), InputBIO_, OutputBIO_); InputBuffer_ = TSharedMutableRef::Allocate<TTlsBufferTag>(TlsBufferSize); OutputBuffer_ = TSharedMutableRef::Allocate<TTlsBufferTag>(TlsBufferSize); @@ -186,19 +212,18 @@ public: void SetHost(const TString& host) { - SSL_set_tlsext_host_name(Ssl_, host.c_str()); + SSL_set_tlsext_host_name(Ssl_.get(), host.c_str()); } ~TTlsConnection() { - SSL_free(Ssl_); } void StartClient() { - SSL_set_connect_state(Ssl_); - auto sslResult = SSL_do_handshake(Ssl_); - sslResult = SSL_get_error(Ssl_, sslResult); + SSL_set_connect_state(Ssl_.get()); + auto sslResult = SSL_do_handshake(Ssl_.get()); + sslResult = SSL_get_error(Ssl_.get(), sslResult); YT_VERIFY(sslResult == SSL_ERROR_WANT_READ); Invoker_->Invoke(BIND(&TTlsConnection::DoRun, MakeWeak(this))); @@ -206,7 +231,7 @@ public: void StartServer() { - SSL_set_accept_state(Ssl_); + SSL_set_accept_state(Ssl_.get()); Invoker_->Invoke(BIND(&TTlsConnection::DoRun, MakeWeak(this))); } @@ -372,7 +397,7 @@ private: const IInvokerPtr Invoker_; const IConnectionPtr Underlying_; - SSL* Ssl_ = nullptr; + TSslPtr Ssl_; BIO* InputBIO_ = nullptr; BIO* OutputBIO_ = nullptr; @@ -493,23 +518,22 @@ private: CheckError(); if (CloseRequested_ && !HandshakeInProgress_) { - SSL_shutdown(Ssl_); + SSL_shutdown(Ssl_.get()); MaybeStartUnderlyingIO(false); } // NB: We should check for an error here, because Underylying_ might have failed already, and then // we will loop on SSL_ERROR_WANT_READ forever. if (HandshakeInProgress_ && Error_.IsOK()) { - int sslResult = SSL_do_handshake(Ssl_); + int sslResult = SSL_do_handshake(Ssl_.get()); if (sslResult == 1) { HandshakeInProgress_ = false; } else { - int sslError = SSL_get_error(Ssl_, sslResult); + int sslError = SSL_get_error(Ssl_.get(), sslResult); if (sslError == SSL_ERROR_WANT_READ) { MaybeStartUnderlyingIO(true); } else { - Error_ = TError("SSL_do_handshake failed") - << GetSslErrors(); + Error_ = GetLastSslError("SSL_do_handshake failed"); YT_LOG_DEBUG(Error_, "TLS handshake failed"); CheckError(); return; @@ -524,11 +548,10 @@ private: // Second condition acts as a poor-man backpressure. if (WriteActive_ && !UnderlyingWriteActive_) { for (const auto& ref : WriteBuffer_) { - int count = SSL_write(Ssl_, ref.Begin(), ref.Size()); + int count = SSL_write(Ssl_.get(), ref.Begin(), ref.Size()); if (count < 0) { - Error_ = TError("SSL_write failed") - << GetSslErrors(); + Error_ = GetLastSslError("SSL_write failed"); YT_LOG_DEBUG(Error_, "TLS write failed"); CheckError(); return; @@ -547,7 +570,7 @@ private: } if (ReadActive_) { - int count = SSL_read(Ssl_, ReadBuffer_.Begin(), ReadBuffer_.Size()); + int count = SSL_read(Ssl_.get(), ReadBuffer_.Begin(), ReadBuffer_.Size()); if (count >= 0) { ReadActive_ = false; ReadBuffer_.Reset(); @@ -555,12 +578,11 @@ private: ReadPromise_.Set(count); ReadPromise_.Reset(); } else { - int sslError = SSL_get_error(Ssl_, count); + int sslError = SSL_get_error(Ssl_.get(), count); if (sslError == SSL_ERROR_WANT_READ) { MaybeStartUnderlyingIO(true); } else { - Error_ = TError("SSL_read failed") - << GetSslErrors(); + Error_ = GetLastSslError("SSL_read failed"); YT_LOG_DEBUG(Error_, "TLS read failed"); CheckError(); return; @@ -668,72 +690,109 @@ TInstant TSslContext::GetCommitTime() const return Impl_->GetCommitTime(); } +void TSslContext::ApplyConfig(const TSslContextConfigPtr& config, TCertificatePathResolver pathResolver) +{ + if (!config) { + return; + } + + const auto& commands = config->SslConfigurationCommands; + if (!commands.empty()) { + std::unique_ptr<SSL_CONF_CTX, decltype(&SSL_CONF_CTX_free)> configContext(SSL_CONF_CTX_new(), SSL_CONF_CTX_free); + if (!configContext) { + THROW_ERROR GetLastSslError("SSL_CONF_CTX_new failed"); + } + + SSL_CONF_CTX_set_flags(configContext.get(), + SSL_CONF_FLAG_FILE | + SSL_CONF_FLAG_CLIENT | + SSL_CONF_FLAG_SERVER | + SSL_CONF_FLAG_CERTIFICATE | + SSL_CONF_FLAG_SHOW_ERRORS); + + SSL_CONF_CTX_set_ssl_ctx(configContext.get(), Impl_->GetContext()); + + for (const auto& command: commands) { + if (SSL_CONF_cmd(configContext.get(), command->Name.c_str(), command->Value.c_str()) <= 0) { + THROW_ERROR GetLastSslError("SSL_CONF_cmd failed") + << TErrorAttribute("ssl_config_command_name", command->Name); + } + } + + if (SSL_CONF_CTX_finish(configContext.get()) != 1) { + THROW_ERROR GetLastSslError("SSL_CONF_CTX_finish failed"); + } + } + + AddCertificateAuthority(config->CertificateAuthority, pathResolver); + AddCertificateChain(config->CertificateChain, pathResolver); + AddPrivateKey(config->PrivateKey, pathResolver); +} + void TSslContext::UseBuiltinOpenSslX509Store() { - SSL_CTX_set_cert_store(Impl_->Context, GetBuiltinOpenSslX509Store().Release()); + SSL_CTX_set_cert_store(Impl_->GetContext(), GetBuiltinOpenSslX509Store().Release()); } void TSslContext::SetCipherList(const TString& list) { - if (SSL_CTX_set_cipher_list(Impl_->Context, list.data()) == 0) { - THROW_ERROR_EXCEPTION("SSL_CTX_set_cipher_list failed") - << TErrorAttribute("cipher_list", list) - << GetSslErrors(); + if (SSL_CTX_set_cipher_list(Impl_->GetContext(), list.data()) == 0) { + THROW_ERROR GetLastSslError("SSL_CTX_set_cipher_list failed") + << TErrorAttribute("cipher_list", list); + } +} + +void TSslContext::AddCertificateAuthorityFromFile(const TString& path) +{ + if (SSL_CTX_load_verify_locations(Impl_->GetContext(), path.c_str(), nullptr) != 1) { + THROW_ERROR GetLastSslError("SSL_CTX_load_verify_locations failed") + << TErrorAttribute("path", path); } } void TSslContext::AddCertificateFromFile(const TString& path) { - if (SSL_CTX_use_certificate_file(Impl_->Context, path.c_str(), SSL_FILETYPE_PEM) != 1) { - THROW_ERROR_EXCEPTION("SSL_CTX_use_certificate_file failed") - << TErrorAttribute("path", path) - << GetSslErrors(); + if (SSL_CTX_use_certificate_file(Impl_->GetContext(), path.c_str(), SSL_FILETYPE_PEM) != 1) { + THROW_ERROR GetLastSslError("SSL_CTX_use_certificate_file failed") + << TErrorAttribute("path", path); } } void TSslContext::AddCertificateChainFromFile(const TString& path) { - if (SSL_CTX_use_certificate_chain_file(Impl_->Context, path.c_str()) != 1) { - THROW_ERROR_EXCEPTION("SSL_CTX_use_certificate_chain_file failed") - << TErrorAttribute("path", path) - << GetSslErrors(); + if (SSL_CTX_use_certificate_chain_file(Impl_->GetContext(), path.c_str()) != 1) { + THROW_ERROR GetLastSslError("SSL_CTX_use_certificate_chain_file failed") + << TErrorAttribute("path", path); } } void TSslContext::AddPrivateKeyFromFile(const TString& path) { - if (SSL_CTX_use_PrivateKey_file(Impl_->Context, path.c_str(), SSL_FILETYPE_PEM) != 1) { - THROW_ERROR_EXCEPTION("SSL_CTX_use_PrivateKey_file failed") - << TErrorAttribute("path", path) - << GetSslErrors(); + if (SSL_CTX_use_PrivateKey_file(Impl_->GetContext(), path.c_str(), SSL_FILETYPE_PEM) != 1) { + THROW_ERROR GetLastSslError("SSL_CTX_use_PrivateKey_file failed") + << TErrorAttribute("path", path); } } void TSslContext::AddCertificateChain(const TString& certificateChain) { - auto bio = BIO_new_mem_buf(certificateChain.c_str(), certificateChain.size()); - YT_VERIFY(bio); - auto freeBio = Finally([&] { - BIO_free(bio); - }); + TBioPtr bio(BIO_new_mem_buf(certificateChain.c_str(), certificateChain.size())); + if (!bio) { + THROW_ERROR GetLastSslError("Failed to allocate memory buffer for certificate chain"); + } - auto certificateObject = PEM_read_bio_X509_AUX(bio, nullptr, nullptr, nullptr); + TX509Ptr certificateObject(PEM_read_bio_X509_AUX(bio.get(), nullptr, nullptr, nullptr)); if (!certificateObject) { - THROW_ERROR_EXCEPTION("PEM_read_bio_X509_AUX failed") - << GetSslErrors(); + THROW_ERROR GetLastSslError("PEM_read_bio_X509_AUX failed"); } - auto freeCertificate = Finally([&] { - X509_free(certificateObject); - }); - if (SSL_CTX_use_certificate(Impl_->Context, certificateObject) != 1) { - THROW_ERROR_EXCEPTION("SSL_CTX_use_certificate failed") - << GetSslErrors(); + if (SSL_CTX_use_certificate(Impl_->GetContext(), certificateObject.get()) != 1) { + THROW_ERROR GetLastSslError("SSL_CTX_use_certificate failed"); } - SSL_CTX_clear_chain_certs(Impl_->Context); + SSL_CTX_clear_chain_certs(Impl_->GetContext()); while (true) { - auto chainCertificateObject = PEM_read_bio_X509(bio, nullptr, nullptr, nullptr); + TX509Ptr chainCertificateObject(PEM_read_bio_X509(bio.get(), nullptr, nullptr, nullptr)); if (!chainCertificateObject) { int err = ERR_peek_last_error(); if (ERR_GET_LIB(err) == ERR_LIB_PEM && ERR_GET_REASON(err) == PEM_R_NO_START_LINE) { @@ -741,65 +800,123 @@ void TSslContext::AddCertificateChain(const TString& certificateChain) break; } - THROW_ERROR_EXCEPTION("PEM_read_bio_X509") - << GetSslErrors(); + THROW_ERROR GetLastSslError("PEM_read_bio_X509"); } - int result = SSL_CTX_add0_chain_cert(Impl_->Context, chainCertificateObject); - if (!result) { - X509_free(chainCertificateObject); - THROW_ERROR_EXCEPTION("SSL_CTX_add0_chain_cert") - << GetSslErrors(); + if (SSL_CTX_add0_chain_cert(Impl_->GetContext(), chainCertificateObject.get()) != 1) { + THROW_ERROR GetLastSslError("SSL_CTX_add0_chain_cert"); + } + + // Do not X509_free() if certificate was added by SSL_CTX_add0_chain_cert(). + Y_UNUSED(chainCertificateObject.release()); + } +} + +void TSslContext::AddCertificateAuthority(const TString& ca) +{ + TBioPtr bio(BIO_new_mem_buf(ca.data(), ca.size())); + if (!bio) { + THROW_ERROR GetLastSslError("Failed to allocate memory buffer for CA certificate"); + } + + auto store = SSL_CTX_get_cert_store(Impl_->GetContext()); + + // Load certificate chain. + TX509Ptr cert; + while (cert.reset(PEM_read_bio_X509(bio.get(), nullptr, nullptr, nullptr)), cert) { + if (X509_STORE_add_cert(store, cert.get()) != 1) { + THROW_ERROR GetLastSslError("Failed to add cert to store"); } } } void TSslContext::AddCertificate(const TString& certificate) { - auto bio = BIO_new_mem_buf(certificate.c_str(), certificate.size()); - YT_VERIFY(bio); - auto freeBio = Finally([&] { - BIO_free(bio); - }); + TBioPtr bio(BIO_new_mem_buf(certificate.c_str(), certificate.size())); + if (!bio) { + THROW_ERROR GetLastSslError("Failed to allocate memory buffer for certificate"); + } - auto certificateObject = PEM_read_bio_X509_AUX(bio, nullptr, nullptr, nullptr); + TX509Ptr certificateObject(PEM_read_bio_X509_AUX(bio.get(), nullptr, nullptr, nullptr)); if (!certificateObject) { - THROW_ERROR_EXCEPTION("PEM_read_bio_X509_AUX") - << GetSslErrors(); + THROW_ERROR GetLastSslError("PEM_read_bio_X509_AUX"); } - auto freeCertificate = Finally([&] { - X509_free(certificateObject); - }); - if (SSL_CTX_use_certificate(Impl_->Context, certificateObject) != 1) { - THROW_ERROR_EXCEPTION("SSL_CTX_use_certificate failed") - << GetSslErrors(); + if (SSL_CTX_use_certificate(Impl_->GetContext(), certificateObject.get()) != 1) { + THROW_ERROR GetLastSslError("SSL_CTX_use_certificate failed"); } } void TSslContext::AddPrivateKey(const TString& privateKey) { - auto bio = BIO_new_mem_buf(privateKey.c_str(), privateKey.size()); - YT_VERIFY(bio); - auto freeBio = Finally([&] { - BIO_free(bio); - }); + TBioPtr bio(BIO_new_mem_buf(privateKey.c_str(), privateKey.size())); + if (!bio) { + THROW_ERROR GetLastSslError("Failed to allocate memory buffer for private key"); + } - auto privateKeyObject = PEM_read_bio_PrivateKey(bio, nullptr, nullptr, nullptr); + TEvpPKeyPtr privateKeyObject(PEM_read_bio_PrivateKey(bio.get(), nullptr, nullptr, nullptr)); if (!privateKeyObject) { - THROW_ERROR_EXCEPTION("PEM_read_bio_PrivateKey failed") - << GetSslErrors(); + THROW_ERROR GetLastSslError("PEM_read_bio_PrivateKey failed"); } - auto freePrivateKey = Finally([&] { - EVP_PKEY_free(privateKeyObject); - }); - if (SSL_CTX_use_PrivateKey(Impl_->Context, privateKeyObject) != 1) { - THROW_ERROR_EXCEPTION("SSL_CTX_use_PrivateKey failed") - << GetSslErrors(); + if (SSL_CTX_use_PrivateKey(Impl_->GetContext(), privateKeyObject.get()) != 1) { + THROW_ERROR GetLastSslError("SSL_CTX_use_PrivateKey failed"); } } +void TSslContext::AddCertificateAuthority(const TPemBlobConfigPtr& pem, TCertificatePathResolver resolver) +{ + if (pem) { + if (pem->FileName) { + auto filePath = resolver ? resolver(*pem->FileName) : *pem->FileName; + AddCertificateAuthorityFromFile(filePath); + } else { + AddCertificateAuthority(pem->LoadBlob()); + } + } +} + +void TSslContext::AddCertificate(const TPemBlobConfigPtr& pem, TCertificatePathResolver resolver) +{ + if (pem) { + if (pem->FileName) { + auto filePath = resolver ? resolver(*pem->FileName) : *pem->FileName; + AddCertificateFromFile(filePath); + } else { + AddCertificate(pem->LoadBlob()); + } + } +} + +void TSslContext::AddCertificateChain(const TPemBlobConfigPtr& pem, TCertificatePathResolver resolver) +{ + if (pem) { + if (pem->FileName) { + auto filePath = resolver ? resolver(*pem->FileName) : *pem->FileName; + AddCertificateChainFromFile(filePath); + } else { + AddCertificateChain(pem->LoadBlob()); + } + } +} + +void TSslContext::AddPrivateKey(const TPemBlobConfigPtr& pem, TCertificatePathResolver resolver) +{ + if (pem) { + if (pem->FileName) { + auto filePath = resolver ? resolver(*pem->FileName) : *pem->FileName; + AddPrivateKeyFromFile(filePath); + } else { + AddPrivateKey(pem->LoadBlob()); + } + } +} + +TSslPtr TSslContext::NewSsl() +{ + return Impl_->NewSsl(); +} + IDialerPtr TSslContext::CreateDialer( const TDialerConfigPtr& config, const IPollerPtr& poller, diff --git a/yt/yt/core/crypto/tls.h b/yt/yt/core/crypto/tls.h index 7903cb35701..e40a41f2f3f 100644 --- a/yt/yt/core/crypto/tls.h +++ b/yt/yt/core/crypto/tls.h @@ -8,12 +8,37 @@ #include <yt/yt/core/concurrency/public.h> +#include <openssl/ossl_typ.h> + namespace NYT::NCrypto { //////////////////////////////////////////////////////////////////////////////// +TError GetLastSslError(TString message); + +//////////////////////////////////////////////////////////////////////////////// + +struct TSslDeleter +{ + void operator() (BIO*) const noexcept; + void operator() (X509*) const noexcept; + void operator() (EVP_PKEY*) const noexcept; + void operator() (SSL_CTX*) const noexcept; + void operator() (SSL*) const noexcept; +}; + +using TBioPtr = std::unique_ptr<BIO, TSslDeleter>; +using TX509Ptr = std::unique_ptr<X509, TSslDeleter>; +using TEvpPKeyPtr = std::unique_ptr<EVP_PKEY, TSslDeleter>; +using TSslCtxPtr = std::unique_ptr<SSL_CTX, TSslDeleter>; +using TSslPtr = std::unique_ptr<SSL, TSslDeleter>; + +//////////////////////////////////////////////////////////////////////////////// + DECLARE_REFCOUNTED_STRUCT(TSslContextImpl) +using TCertificatePathResolver = std::function<TString(const TString&)>; + class TSslContext : public TRefCounted { @@ -24,18 +49,29 @@ public: void Commit(TInstant time = TInstant::Zero()); TInstant GetCommitTime() const; + void ApplyConfig(const TSslContextConfigPtr& config, TCertificatePathResolver pathResolver = nullptr); + void UseBuiltinOpenSslX509Store(); void SetCipherList(const TString& list); + void AddCertificateAuthorityFromFile(const TString& path); void AddCertificateFromFile(const TString& path); void AddCertificateChainFromFile(const TString& path); void AddPrivateKeyFromFile(const TString& path); + void AddCertificateAuthority(const TString& ca); void AddCertificate(const TString& certificate); void AddCertificateChain(const TString& certificateChain); void AddPrivateKey(const TString& privateKey); + void AddCertificateAuthority(const TPemBlobConfigPtr& pem, TCertificatePathResolver resolver = nullptr); + void AddCertificate(const TPemBlobConfigPtr& pem, TCertificatePathResolver resolver = nullptr); + void AddCertificateChain(const TPemBlobConfigPtr& pem, TCertificatePathResolver resolver = nullptr); + void AddPrivateKey(const TPemBlobConfigPtr& pem, TCertificatePathResolver resolver = nullptr); + + TSslPtr NewSsl(); + NNet::IDialerPtr CreateDialer( const NNet::TDialerConfigPtr& config, const NConcurrency::IPollerPtr& poller, diff --git a/yt/yt/core/crypto/unittests/tls_ut.cpp b/yt/yt/core/crypto/unittests/tls_ut.cpp index e756b2c8084..6281d96b439 100644 --- a/yt/yt/core/crypto/unittests/tls_ut.cpp +++ b/yt/yt/core/crypto/unittests/tls_ut.cpp @@ -57,6 +57,11 @@ TEST_F(TTlsTest, CreateContext) // Not that trivial as it seems! } +TEST_F(TTlsTest, CreateSslObject) +{ + auto ssl = Context->NewSsl(); +} + TEST_F(TTlsTest, CreateListener) { auto localhost = TNetworkAddress::CreateIPv6Loopback(0); diff --git a/yt/yt/core/http/http.cpp b/yt/yt/core/http/http.cpp index 702ebefef65..376b7d91e4f 100644 --- a/yt/yt/core/http/http.cpp +++ b/yt/yt/core/http/http.cpp @@ -1,6 +1,6 @@ #include "http.h" -#include <contrib/restricted/http-parser/http_parser.h> +#include <contrib/deprecated/http-parser/http_parser.h> namespace NYT::NHttp { diff --git a/yt/yt/core/http/stream.h b/yt/yt/core/http/stream.h index a45ab15f0ff..00236665bed 100644 --- a/yt/yt/core/http/stream.h +++ b/yt/yt/core/http/stream.h @@ -7,7 +7,7 @@ #include <yt/yt/core/net/connection.h> #include <yt/yt/core/net/address.h> -#include <contrib/restricted/http-parser/http_parser.h> +#include <contrib/deprecated/http-parser/http_parser.h> #include <util/stream/buffer.h> diff --git a/yt/yt/core/http/unittests/http_ut.cpp b/yt/yt/core/http/unittests/http_ut.cpp index 1ff0db8034a..0dea66e0419 100644 --- a/yt/yt/core/http/unittests/http_ut.cpp +++ b/yt/yt/core/http/unittests/http_ut.cpp @@ -641,8 +641,8 @@ private: serverConfig->Credentials = New<NHttps::TServerCredentialsConfig>(); serverConfig->Credentials->PrivateKey = New<TPemBlobConfig>(); serverConfig->Credentials->PrivateKey->Value = TestCertificate; - serverConfig->Credentials->CertChain = New<TPemBlobConfig>(); - serverConfig->Credentials->CertChain->Value = TestCertificate; + serverConfig->Credentials->CertificateChain = New<TPemBlobConfig>(); + serverConfig->Credentials->CertificateChain->Value = TestCertificate; SetupServer(serverConfig); ServerConfig = serverConfig; Server = NHttps::CreateServer(serverConfig, Poller); @@ -651,8 +651,8 @@ private: clientConfig->Credentials = New<NHttps::TClientCredentialsConfig>(); clientConfig->Credentials->PrivateKey = New<TPemBlobConfig>(); clientConfig->Credentials->PrivateKey->Value = TestCertificate; - clientConfig->Credentials->CertChain = New<TPemBlobConfig>(); - clientConfig->Credentials->CertChain->Value = TestCertificate; + clientConfig->Credentials->CertificateChain = New<TPemBlobConfig>(); + clientConfig->Credentials->CertificateChain->Value = TestCertificate; SetupClient(clientConfig); Client = NHttps::CreateClient(clientConfig, Poller); } diff --git a/yt/yt/core/http/ya.make b/yt/yt/core/http/ya.make index 4ca1860a54b..53376347555 100644 --- a/yt/yt/core/http/ya.make +++ b/yt/yt/core/http/ya.make @@ -27,7 +27,7 @@ ENDIF() PEERDIR( yt/yt/core - contrib/restricted/http-parser + contrib/deprecated/http-parser ) END() diff --git a/yt/yt/core/https/client.cpp b/yt/yt/core/https/client.cpp index 7d5fbc70e57..4c9acaee9c5 100644 --- a/yt/yt/core/https/client.cpp +++ b/yt/yt/core/https/client.cpp @@ -108,24 +108,7 @@ IClientPtr CreateClient( { auto sslContext = New<TSslContext>(); if (config->Credentials) { - if (config->Credentials->CertChain) { - if (config->Credentials->CertChain->FileName) { - sslContext->AddCertificateChainFromFile(*config->Credentials->CertChain->FileName); - } else if (config->Credentials->CertChain->Value) { - sslContext->AddCertificateChain(*config->Credentials->CertChain->Value); - } else { - THROW_ERROR_EXCEPTION("Neither \"file_name\" nor \"value\" is given for client certificate chain"); - } - } - if (config->Credentials->PrivateKey) { - if (config->Credentials->PrivateKey->FileName) { - sslContext->AddPrivateKeyFromFile(*config->Credentials->PrivateKey->FileName); - } else if (config->Credentials->PrivateKey->Value) { - sslContext->AddPrivateKey(*config->Credentials->PrivateKey->Value); - } else { - THROW_ERROR_EXCEPTION("Neither \"file_name\" nor \"value\" is given for client private key"); - } - } + sslContext->ApplyConfig(config->Credentials); } else { sslContext->UseBuiltinOpenSslX509Store(); } diff --git a/yt/yt/core/https/config.cpp b/yt/yt/core/https/config.cpp index c41fde96241..dcc6c4bca15 100644 --- a/yt/yt/core/https/config.cpp +++ b/yt/yt/core/https/config.cpp @@ -6,10 +6,6 @@ namespace NYT::NHttps { void TServerCredentialsConfig::Register(TRegistrar registrar) { - registrar.Parameter("private_key", &TThis::PrivateKey) - .Optional(); - registrar.Parameter("cert_chain", &TThis::CertChain) - .Optional(); registrar.Parameter("update_period", &TThis::UpdatePeriod) .Optional(); } @@ -23,12 +19,8 @@ void TServerConfig::Register(TRegistrar registrar) //////////////////////////////////////////////////////////////////////////////// -void TClientCredentialsConfig::Register(TRegistrar registrar) +void TClientCredentialsConfig::Register(TRegistrar /*registrar*/) { - registrar.Parameter("private_key", &TThis::PrivateKey) - .Optional(); - registrar.Parameter("cert_chain", &TThis::CertChain) - .Optional(); } //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/core/https/config.h b/yt/yt/core/https/config.h index ad740cc0fe7..8d9c381c678 100644 --- a/yt/yt/core/https/config.h +++ b/yt/yt/core/https/config.h @@ -11,10 +11,8 @@ namespace NYT::NHttps { //////////////////////////////////////////////////////////////////////////////// struct TServerCredentialsConfig - : public NYTree::TYsonStruct + : public NCrypto::TSslContextConfig { - NCrypto::TPemBlobConfigPtr PrivateKey; - NCrypto::TPemBlobConfigPtr CertChain; TDuration UpdatePeriod; REGISTER_YSON_STRUCT(TServerCredentialsConfig); @@ -41,11 +39,8 @@ DEFINE_REFCOUNTED_TYPE(TServerConfig) //////////////////////////////////////////////////////////////////////////////// struct TClientCredentialsConfig - : public NYTree::TYsonStruct + : public NCrypto::TSslContextConfig { - NCrypto::TPemBlobConfigPtr PrivateKey; - NCrypto::TPemBlobConfigPtr CertChain; - REGISTER_YSON_STRUCT(TClientCredentialsConfig); static void Register(TRegistrar registrar); diff --git a/yt/yt/core/https/server.cpp b/yt/yt/core/https/server.cpp index 6c6f997cd4c..f8ef027dc38 100644 --- a/yt/yt/core/https/server.cpp +++ b/yt/yt/core/https/server.cpp @@ -92,20 +92,7 @@ private: static void ApplySslConfig(const TSslContextPtr& sslContext, const TServerCredentialsConfigPtr& sslConfig) { - if (sslConfig->CertChain->FileName) { - sslContext->AddCertificateChainFromFile(*sslConfig->CertChain->FileName); - } else if (sslConfig->CertChain->Value) { - sslContext->AddCertificateChain(*sslConfig->CertChain->Value); - } else { - YT_ABORT(); - } - if (sslConfig->PrivateKey->FileName) { - sslContext->AddPrivateKeyFromFile(*sslConfig->PrivateKey->FileName); - } else if (sslConfig->PrivateKey->Value) { - sslContext->AddPrivateKey(*sslConfig->PrivateKey->Value); - } else { - YT_ABORT(); - } + sslContext->ApplyConfig(sslConfig); } IServerPtr CreateServer( @@ -121,7 +108,7 @@ IServerPtr CreateServer( auto sslConfig = config->Credentials; TPeriodicExecutorPtr certificateUpdater; if (sslConfig->UpdatePeriod && - sslConfig->CertChain->FileName && + sslConfig->CertificateChain->FileName && sslConfig->PrivateKey->FileName) { YT_VERIFY(controlInvoker); @@ -130,7 +117,7 @@ IServerPtr CreateServer( BIND([=] { try { auto modificationTime = Max( - NFS::GetPathStatistics(*sslConfig->CertChain->FileName).ModificationTime, + NFS::GetPathStatistics(*sslConfig->CertificateChain->FileName).ModificationTime, NFS::GetPathStatistics(*sslConfig->PrivateKey->FileName).ModificationTime); // Detect fresh and stable updates. diff --git a/yt/yt/core/json/json_parser.cpp b/yt/yt/core/json/json_parser.cpp index c8bfcf77b46..269dd0eaa8d 100644 --- a/yt/yt/core/json/json_parser.cpp +++ b/yt/yt/core/json/json_parser.cpp @@ -6,7 +6,7 @@ #include <array> -#include <contrib/libs/yajl/src/api/yajl_parse.h> +#include <contrib/deprecated/yajl/src/api/yajl_parse.h> namespace NYT::NJson { diff --git a/yt/yt/core/json/json_writer.cpp b/yt/yt/core/json/json_writer.cpp index d8b1118044d..3986f8f8175 100644 --- a/yt/yt/core/json/json_writer.cpp +++ b/yt/yt/core/json/json_writer.cpp @@ -4,7 +4,7 @@ #include <yt/yt/core/misc/utf8_decoder.h> -#include <contrib/libs/yajl/src/api/yajl_gen.h> +#include <contrib/deprecated/yajl/src/api/yajl_gen.h> #include <cmath> diff --git a/yt/yt/core/misc/error.cpp b/yt/yt/core/misc/error.cpp index 7d20101a8ff..9fc1a7fa150 100644 --- a/yt/yt/core/misc/error.cpp +++ b/yt/yt/core/misc/error.cpp @@ -757,13 +757,13 @@ void TErrorCodicils::Set(std::string key, TGetter getter) if (getter) { Getters_.insert_or_assign(std::move(key), std::move(getter)); } else { - Getters_.erase(std::move(key)); + Getters_.erase(key); } } auto TErrorCodicils::Get(const std::string& key) const -> TGetter { - return GetOrDefault(Getters_, std::move(key)); + return GetOrDefault(Getters_, key); } //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/core/net/config.h b/yt/yt/core/net/config.h index eb65aee470b..a6b596d0156 100644 --- a/yt/yt/core/net/config.h +++ b/yt/yt/core/net/config.h @@ -13,7 +13,7 @@ namespace NYT::NNet { //////////////////////////////////////////////////////////////////////////////// struct TDialerConfig - : public NYTree::TYsonStruct + : public virtual NYTree::TYsonStruct { bool EnableNoDelay; bool EnableAggressiveReconnect; diff --git a/yt/yt/core/rpc/unittests/lib/common.h b/yt/yt/core/rpc/unittests/lib/common.h index cb3b0ab6b06..050159f078a 100644 --- a/yt/yt/core/rpc/unittests/lib/common.h +++ b/yt/yt/core/rpc/unittests/lib/common.h @@ -501,8 +501,8 @@ public: auto credentials = New<NHttps::TClientCredentialsConfig>(); credentials->PrivateKey = New<NCrypto::TPemBlobConfig>(); credentials->PrivateKey->Value = ClientKey; - credentials->CertChain = New<NCrypto::TPemBlobConfig>(); - credentials->CertChain->Value = ClientCert; + credentials->CertificateChain = New<NCrypto::TPemBlobConfig>(); + credentials->CertificateChain->Value = ClientCert; return NHttp::CreateHttpChannel(address, poller, EnableSsl, credentials); } @@ -521,8 +521,8 @@ public: config->Credentials = New<NHttps::TServerCredentialsConfig>(); config->Credentials->PrivateKey = New<NCrypto::TPemBlobConfig>(); config->Credentials->PrivateKey->Value = ServerKey; - config->Credentials->CertChain = New<NCrypto::TPemBlobConfig>(); - config->Credentials->CertChain->Value = ServerCert; + config->Credentials->CertificateChain = New<NCrypto::TPemBlobConfig>(); + config->Credentials->CertificateChain->Value = ServerCert; httpServer = NYT::NHttps::CreateServer(config, 4); } else { httpServer = NYT::NHttp::CreateServer(config, 4); diff --git a/yt/yt/core/ya.make b/yt/yt/core/ya.make index 88951dd698a..9d46e7f6970 100644 --- a/yt/yt/core/ya.make +++ b/yt/yt/core/ya.make @@ -32,8 +32,6 @@ SRCS( bus/tcp/client.cpp bus/tcp/local_bypass.cpp bus/tcp/server.cpp - bus/tcp/ssl_context.cpp - bus/tcp/ssl_helpers.cpp compression/brotli.cpp compression/bzip2.cpp @@ -333,7 +331,7 @@ PEERDIR( contrib/libs/libbz2 contrib/libs/c-ares contrib/libs/farmhash - contrib/libs/yajl + contrib/deprecated/yajl contrib/libs/lz4 contrib/libs/openssl diff --git a/yt/yt_proto/yt/client/api/rpc_proxy/proto/api_service.proto b/yt/yt_proto/yt/client/api/rpc_proxy/proto/api_service.proto index fe5cd1da967..b7115ac5438 100644 --- a/yt/yt_proto/yt/client/api/rpc_proxy/proto/api_service.proto +++ b/yt/yt_proto/yt/client/api/rpc_proxy/proto/api_service.proto @@ -443,6 +443,7 @@ message TReqCommitTransaction required NYT.NProto.TGuid transaction_id = 1; reserved 6; repeated NYT.NProto.TGuid additional_participant_cell_ids = 7; + optional uint64 max_allowed_commit_timestamp = 8; optional TPrerequisiteOptions prerequisite_options = 101; optional TMutatingOptions mutating_options = 103; |