diff options
| author | Alexander Smirnov <alex@ydb.tech> | 2024-09-03 11:05:27 +0000 |
|---|---|---|
| committer | Alexander Smirnov <alex@ydb.tech> | 2024-09-03 11:05:27 +0000 |
| commit | 8f71d7ed87007ace129f647b242a09d01773d3c5 (patch) | |
| tree | 2c46ca9d89eb0ce5eea79ba1febb79e56efedb0f | |
| parent | 78242bd5894abd6548e45731b464822da55a0796 (diff) | |
| parent | 3da5a68ec3c329240e89bd0ed8c1c39e4359a693 (diff) | |
| download | ydb-8f71d7ed87007ace129f647b242a09d01773d3c5.tar.gz | |
Merge branch 'rightlib' into mergelibs-240903-1104
153 files changed, 3134 insertions, 1820 deletions
diff --git a/build/conf/java.conf b/build/conf/java.conf index 762eb5de4a..8c39aed61d 100644 --- a/build/conf/java.conf +++ b/build/conf/java.conf @@ -1229,9 +1229,6 @@ elsewhen ($JDK_VERSION == "20") { elsewhen ($JDK_VERSION == "17") { JDK_REAL_VERSION=17 } -elsewhen ($JDK_VERSION == "15") { - JDK_REAL_VERSION=15 -} elsewhen ($JDK_VERSION == "11") { JDK_REAL_VERSION=11 } @@ -1261,9 +1258,6 @@ otherwise { # need jdk11 for spare parts in "host" platform JDK_RESOURCE_PEERDIR=build/platform/java/jdk/jdk17 build/platform/java/jdk/jdk11 } - when ($JDK_REAL_VERSION == "15") { - JDK_RESOURCE_PEERDIR=build/platform/java/jdk/jdk15 build/platform/java/jdk/jdk17 - } when ($JDK_REAL_VERSION == "11") { JDK_RESOURCE_PEERDIR=build/platform/java/jdk/jdk11 build/platform/java/jdk/jdk17 } @@ -1282,9 +1276,6 @@ when ($JDK_REAL_VERSION == "20") { when ($JDK_REAL_VERSION == "17") { UBERJAR_RESOURCE_PEERDIR=build/platform/java/uberjar/uberjar17 } -when ($JDK_REAL_VERSION == "15") { - UBERJAR_RESOURCE_PEERDIR=build/platform/java/uberjar/uberjar15 -} when ($JDK_REAL_VERSION == "11") { UBERJAR_RESOURCE_PEERDIR=build/platform/java/uberjar/uberjar11 } @@ -1307,9 +1298,6 @@ otherwise { when ($JDK_REAL_VERSION == "17") { JDK_RESOURCE=$JDK17_RESOURCE_GLOBAL } - when ($JDK_REAL_VERSION == "15") { - JDK_RESOURCE=$JDK15_RESOURCE_GLOBAL - } when ($JDK_REAL_VERSION == "11") { JDK_RESOURCE=$JDK11_RESOURCE_GLOBAL } @@ -1318,7 +1306,7 @@ otherwise { # tag:java-specific when (!$USE_SYSTEM_ERROR_PRONE) { # Still not done: DTCC-667 - when ($JDK_REAL_VERSION == "11" || $JDK_REAL_VERSION == "15") { + when ($JDK_REAL_VERSION == "11") { ERROR_PRONE_VERSION=2.7.1 ERROR_PRONE_PEERDIR=build/platform/java/error_prone/2.7.1 ERROR_PRONE_RESOURCE=$ERROR_PRONE_2_7_1_RESOURCE_GLOBAL @@ -1372,9 +1360,6 @@ otherwise { when ($JDK_REAL_VERSION == "17") { UBERJAR_RESOURCE=$UBERJAR17_RESOURCE_GLOBAL } - when ($JDK_REAL_VERSION == "15") { - UBERJAR_RESOURCE=$UBERJAR15_RESOURCE_GLOBAL - } when ($JDK_REAL_VERSION == "11") { UBERJAR_RESOURCE=$UBERJAR11_RESOURCE_GLOBAL } @@ -1393,9 +1378,6 @@ when ($JDK_REAL_VERSION == "20") { when ($JDK_REAL_VERSION == "17") { WITH_JDK_RESOURCE=$WITH_JDK17_RESOURCE_GLOBAL } -when ($JDK_REAL_VERSION == "15") { - WITH_JDK_RESOURCE=$WITH_JDK15_RESOURCE_GLOBAL -} when ($JDK_REAL_VERSION == "11") { WITH_JDK_RESOURCE=$WITH_JDK11_RESOURCE_GLOBAL } @@ -1745,7 +1727,6 @@ macro ADD_DLLS_TO_JAR() { # tag:java-specific MANAGED_PEERS= MANAGED_PEERS_CLOSURE= -RUN_JAVA_PROGRAM_MANAGED= MANAGEABLE_PEERS_ROOTS=contrib/java HAS_MANAGEABLE_PEERS=no PROPAGATES_MANAGEABLE_PEERS=no diff --git a/build/conf/licenses.json b/build/conf/licenses.json index 182bcef239..abfed06c8d 100644 --- a/build/conf/licenses.json +++ b/build/conf/licenses.json @@ -365,7 +365,6 @@ "default": [ "Service-Default-License", "Service-Dll-Harness", - "Service-Prebuilt-Tool", "Service-Py23-Proxy", "Service-Unknown-License-Java" ] diff --git a/build/config/tests/cpp_style/config.clang-format b/build/config/tests/cpp_style/config.clang-format index b31758e14c..9b762202f5 100644 --- a/build/config/tests/cpp_style/config.clang-format +++ b/build/config/tests/cpp_style/config.clang-format @@ -84,7 +84,7 @@ SortIncludes: false IndentPPDirectives: BeforeHash SpaceBeforeInheritanceColon: false -NamespaceMacros: [Y_UNIT_TEST_SUITE, Y_UNIT_TEST] +NamespaceMacros: [Y_UNIT_TEST_SUITE] AttributeMacros: [Y_PRINTF_FORMAT, Y_NO_SANITIZE, Y_FORCE_INLINE, Y_NO_INLINE, Y_WARN_UNUSED_RESULT, Y_HIDDEN, Y_PUBLIC, Y_PURE_FUNCTION] IndentExternBlock: Indent TypenameMacros: [Y_THREAD, Y_STATIC_THREAD, Y_POD_THREAD, Y_POD_STATIC_THREAD] diff --git a/build/external_resources/ymake/public.resources.json b/build/external_resources/ymake/public.resources.json index 4620950c30..8a80afe094 100644 --- a/build/external_resources/ymake/public.resources.json +++ b/build/external_resources/ymake/public.resources.json @@ -1,19 +1,19 @@ { "by_platform": { "darwin": { - "uri": "sbr:6959220365" + "uri": "sbr:6976467840" }, "darwin-arm64": { - "uri": "sbr:6959217204" + "uri": "sbr:6976467045" }, "linux": { - "uri": "sbr:6959225691" + "uri": "sbr:6976469309" }, "linux-aarch64": { - "uri": "sbr:6959214449" + "uri": "sbr:6976466498" }, "win32-clang-cl": { - "uri": "sbr:6959223374" + "uri": "sbr:6976468552" } } } diff --git a/build/external_resources/ymake/resources.json b/build/external_resources/ymake/resources.json index f0d2712d39..c84e945018 100644 --- a/build/external_resources/ymake/resources.json +++ b/build/external_resources/ymake/resources.json @@ -1,19 +1,19 @@ { "by_platform": { "darwin": { - "uri": "sbr:6959209660" + "uri": "sbr:6976470248" }, "darwin-arm64": { - "uri": "sbr:6959207343" + "uri": "sbr:6976469418" }, "linux": { - "uri": "sbr:6960956636" + "uri": "sbr:6976471430" }, "linux-aarch64": { - "uri": "sbr:6959205082" + "uri": "sbr:6976468560" }, "win32-clang-cl": { - "uri": "sbr:6959211457" + "uri": "sbr:6976470807" } } } diff --git a/build/mapping.conf.json b/build/mapping.conf.json index 1c71408dff..72ffad598e 100644 --- a/build/mapping.conf.json +++ b/build/mapping.conf.json @@ -525,6 +525,7 @@ "6888066924": "https://devtools-registry.s3.yandex.net/6888066924", "6937600791": "https://devtools-registry.s3.yandex.net/6937600791", "6959220365": "https://devtools-registry.s3.yandex.net/6959220365", + "6976467840": "https://devtools-registry.s3.yandex.net/6976467840", "5766171800": "https://devtools-registry.s3.yandex.net/5766171800", "5805430761": "https://devtools-registry.s3.yandex.net/5805430761", "5829025456": "https://devtools-registry.s3.yandex.net/5829025456", @@ -559,6 +560,7 @@ "6888066560": "https://devtools-registry.s3.yandex.net/6888066560", "6937600701": "https://devtools-registry.s3.yandex.net/6937600701", "6959217204": "https://devtools-registry.s3.yandex.net/6959217204", + "6976467045": "https://devtools-registry.s3.yandex.net/6976467045", "5766173070": "https://devtools-registry.s3.yandex.net/5766173070", "5805432830": "https://devtools-registry.s3.yandex.net/5805432830", "5829031598": "https://devtools-registry.s3.yandex.net/5829031598", @@ -593,6 +595,7 @@ "6888067594": "https://devtools-registry.s3.yandex.net/6888067594", "6937601032": "https://devtools-registry.s3.yandex.net/6937601032", "6959225691": "https://devtools-registry.s3.yandex.net/6959225691", + "6976469309": "https://devtools-registry.s3.yandex.net/6976469309", "5766171341": "https://devtools-registry.s3.yandex.net/5766171341", "5805430188": "https://devtools-registry.s3.yandex.net/5805430188", "5829023352": "https://devtools-registry.s3.yandex.net/5829023352", @@ -627,6 +630,7 @@ "6888066108": "https://devtools-registry.s3.yandex.net/6888066108", "6937600548": "https://devtools-registry.s3.yandex.net/6937600548", "6959214449": "https://devtools-registry.s3.yandex.net/6959214449", + "6976466498": "https://devtools-registry.s3.yandex.net/6976466498", "5766172695": "https://devtools-registry.s3.yandex.net/5766172695", "5805432230": "https://devtools-registry.s3.yandex.net/5805432230", "5829029743": "https://devtools-registry.s3.yandex.net/5829029743", @@ -661,6 +665,7 @@ "6888067214": "https://devtools-registry.s3.yandex.net/6888067214", "6937600896": "https://devtools-registry.s3.yandex.net/6937600896", "6959223374": "https://devtools-registry.s3.yandex.net/6959223374", + "6976468552": "https://devtools-registry.s3.yandex.net/6976468552", "4307890075": "https://devtools-registry.s3.yandex.net/4307890075", "5517245192": "https://devtools-registry.s3.yandex.net/5517245192", "4307901240": "https://devtools-registry.s3.yandex.net/4307901240", @@ -734,6 +739,9 @@ "6956610092": "https://devtools-registry.s3.yandex.net/6956610092", "6957903888": "https://devtools-registry.s3.yandex.net/6957903888", "6391354461": "https://devtools-registry.s3.yandex.net/6391354461", + "6990868751": "https://devtools-registry.s3.yandex.net/6990868751", + "6990860705": "https://devtools-registry.s3.yandex.net/6990860705", + "6990881789": "https://devtools-registry.s3.yandex.net/6990881789", "3167009386": "https://devtools-registry.s3.yandex.net/3167009386", "3050798466": "https://devtools-registry.s3.yandex.net/3050798466", "3064614561": "https://devtools-registry.s3.yandex.net/3064614561", @@ -1352,6 +1360,7 @@ "6888066924": "devtools/ymake/bin/ymake for darwin", "6937600791": "devtools/ymake/bin/ymake for darwin", "6959220365": "devtools/ymake/bin/ymake for darwin", + "6976467840": "devtools/ymake/bin/ymake for darwin", "5766171800": "devtools/ymake/bin/ymake for darwin-arm64", "5805430761": "devtools/ymake/bin/ymake for darwin-arm64", "5829025456": "devtools/ymake/bin/ymake for darwin-arm64", @@ -1386,6 +1395,7 @@ "6888066560": "devtools/ymake/bin/ymake for darwin-arm64", "6937600701": "devtools/ymake/bin/ymake for darwin-arm64", "6959217204": "devtools/ymake/bin/ymake for darwin-arm64", + "6976467045": "devtools/ymake/bin/ymake for darwin-arm64", "5766173070": "devtools/ymake/bin/ymake for linux", "5805432830": "devtools/ymake/bin/ymake for linux", "5829031598": "devtools/ymake/bin/ymake for linux", @@ -1420,6 +1430,7 @@ "6888067594": "devtools/ymake/bin/ymake for linux", "6937601032": "devtools/ymake/bin/ymake for linux", "6959225691": "devtools/ymake/bin/ymake for linux", + "6976469309": "devtools/ymake/bin/ymake for linux", "5766171341": "devtools/ymake/bin/ymake for linux-aarch64", "5805430188": "devtools/ymake/bin/ymake for linux-aarch64", "5829023352": "devtools/ymake/bin/ymake for linux-aarch64", @@ -1454,6 +1465,7 @@ "6888066108": "devtools/ymake/bin/ymake for linux-aarch64", "6937600548": "devtools/ymake/bin/ymake for linux-aarch64", "6959214449": "devtools/ymake/bin/ymake for linux-aarch64", + "6976466498": "devtools/ymake/bin/ymake for linux-aarch64", "5766172695": "devtools/ymake/bin/ymake for win32-clang-cl", "5805432230": "devtools/ymake/bin/ymake for win32-clang-cl", "5829029743": "devtools/ymake/bin/ymake for win32-clang-cl", @@ -1488,6 +1500,7 @@ "6888067214": "devtools/ymake/bin/ymake for win32-clang-cl", "6937600896": "devtools/ymake/bin/ymake for win32-clang-cl", "6959223374": "devtools/ymake/bin/ymake for win32-clang-cl", + "6976468552": "devtools/ymake/bin/ymake for win32-clang-cl", "4307890075": "flake8_linter for linux", "5517245192": "flake8_linter for linux", "4307901240": "flake8_linter for linux-aarch64", @@ -1561,6 +1574,9 @@ "6956610092": "none-none-none-result_resources/jdk-windows-amd64.yandex.tgz", "6957903888": "none-none-none-result_resources/jdk-windows-amd64.yandex.tgz", "6391354461": "none-none-none-result_resources/protoc-linux-x86_64.tgz", + "6990868751": "none-none-none-sandbox/backup/3527d100-e2d0-4b0e-bb7a-905010853d98/yfm-docs.tar", + "6990860705": "none-none-none-sandbox/backup/d386643e-58f8-43e1-8760-341d73801df8/yfm-docs.tar", + "6990881789": "none-none-none-sandbox/backup/efc428e5-52a5-4a6f-8f0c-53f1d255efea/yfm-docs.tar", "3167009386": "openjdk 11.0.15 vanilla for darwin", "3050798466": "openjdk 11.0.15 vanilla for darwin-arm64", "3064614561": "openjdk 11.0.15 vanilla for linux", diff --git a/build/platform/java/jdk/jdk15/jdk.json b/build/platform/java/jdk/jdk15/jdk.json deleted file mode 100644 index 5d2bc7ae0a..0000000000 --- a/build/platform/java/jdk/jdk15/jdk.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "by_platform": { - "darwin-arm64": { - "uri": "sbr:2107376046" - }, - "darwin-x86_64": { - "uri": "sbr:1901471162" - }, - "linux-x86_64": { - "uri": "sbr:1901452000" - }, - "win32-x86_64": { - "uri": "sbr:1901563630" - } - } -} - diff --git a/build/platform/java/jdk/jdk15/ya.make b/build/platform/java/jdk/jdk15/ya.make deleted file mode 100644 index cad2f09f35..0000000000 --- a/build/platform/java/jdk/jdk15/ya.make +++ /dev/null @@ -1,10 +0,0 @@ -RESOURCES_LIBRARY() - -DECLARE_EXTERNAL_HOST_RESOURCES_BUNDLE_BY_JSON(JDK15 jdk.json) -SET_RESOURCE_URI_FROM_JSON(WITH_JDK15_URI jdk.json) - -IF (WITH_JDK15_URI) - DECLARE_EXTERNAL_RESOURCE(WITH_JDK15 ${WITH_JDK15_URI}) -ENDIF() - -END() diff --git a/build/platform/java/jdk/ya.make b/build/platform/java/jdk/ya.make index b696766faf..983dfbba15 100644 --- a/build/platform/java/jdk/ya.make +++ b/build/platform/java/jdk/ya.make @@ -15,9 +15,6 @@ ELSEIF(JDK_REAL_VERSION == "20") ELSEIF(JDK_REAL_VERSION == "17") DECLARE_EXTERNAL_HOST_RESOURCES_BUNDLE_BY_JSON(JDK_DEFAULT jdk17/jdk.json) SET_RESOURCE_URI_FROM_JSON(WITH_JDK_URI jdk17/jdk.json) -ELSEIF(JDK_REAL_VERSION == "15") - DECLARE_EXTERNAL_HOST_RESOURCES_BUNDLE_BY_JSON(JDK_DEFAULT jdk15/jdk.json) - SET_RESOURCE_URI_FROM_JSON(WITH_JDK_URI jdk15/jdk.json) ELSEIF(JDK_REAL_VERSION == "11") DECLARE_EXTERNAL_HOST_RESOURCES_BUNDLE_BY_JSON(JDK_DEFAULT jdk11/jdk.json) SET_RESOURCE_URI_FROM_JSON(WITH_JDK_URI jdk11/jdk.json) @@ -33,7 +30,6 @@ END() RECURSE( jdk11 - jdk15 jdk17 jdk20 jdk21 diff --git a/build/platform/lld/ya.make b/build/platform/lld/ya.make index 670dd4e4cd..7ea02fc2ec 100644 --- a/build/platform/lld/ya.make +++ b/build/platform/lld/ya.make @@ -1,9 +1,10 @@ RESOURCES_LIBRARY() -LICENSE(Service-Prebuilt-Tool) - DEFAULT(LLD_VERSION ${CLANG_VER}) +TOOLCHAIN(lld) +VERSION(${LLD_VERSION}) + IF (LLD_VERSION == 14) DECLARE_EXTERNAL_HOST_RESOURCES_BUNDLE_BY_JSON(LLD_ROOT lld14.json) ELSE() diff --git a/build/platform/yfm/ya.make b/build/platform/yfm/ya.make index 0d6635dbc8..a348bb1f1e 100644 --- a/build/platform/yfm/ya.make +++ b/build/platform/yfm/ya.make @@ -6,10 +6,10 @@ ENDIF() DECLARE_EXTERNAL_HOST_RESOURCES_BUNDLE( YFM_TOOL - sbr:6662972834 FOR DARWIN-ARM64 - sbr:6662972834 FOR DARWIN - sbr:6662965433 FOR LINUX - sbr:6662980150 FOR WIN32 + sbr:6990868751 FOR DARWIN-ARM64 + sbr:6990868751 FOR DARWIN + sbr:6990860705 FOR LINUX + sbr:6990881789 FOR WIN32 ) END() diff --git a/build/plugins/_dart_fields.py b/build/plugins/_dart_fields.py index 3fe696d675..4676499399 100644 --- a/build/plugins/_dart_fields.py +++ b/build/plugins/_dart_fields.py @@ -993,7 +993,6 @@ class TestFiles: @classmethod def test_srcs(cls, unit, flat_args, spec_args): test_files = get_values_list(unit, 'TEST_SRCS_VALUE') - test_files = _resolve_module_files(unit, unit.get("MODDIR"), test_files) return {cls.KEY: serialize_list(test_files)} @classmethod diff --git a/build/plugins/java.py b/build/plugins/java.py index 571de73260..c0771722ec 100644 --- a/build/plugins/java.py +++ b/build/plugins/java.py @@ -75,7 +75,7 @@ def onjava_module(unit, *args): 'ANNOTATION_PROCESSOR': extract_macro_calls(unit, 'ANNOTATION_PROCESSOR_VALUE', args_delim), 'EXTERNAL_JAR': extract_macro_calls(unit, 'EXTERNAL_JAR_VALUE', args_delim), 'RUN_JAVA_PROGRAM': [], - 'RUN_JAVA_PROGRAM_MANAGED': '${RUN_JAVA_PROGRAM_MANAGED}', + 'RUN_JAVA_PROGRAM_MANAGED': '', 'MAVEN_GROUP_ID': extract_macro_calls(unit, 'MAVEN_GROUP_ID_VALUE', args_delim), 'JAR_INCLUDE_FILTER': extract_macro_calls(unit, 'JAR_INCLUDE_FILTER_VALUE', args_delim), 'JAR_EXCLUDE_FILTER': extract_macro_calls(unit, 'JAR_EXCLUDE_FILTER_VALUE', args_delim), @@ -100,7 +100,6 @@ def onjava_module(unit, *args): 'JDK_RESOURCE': 'JDK' + (unit.get('JDK_VERSION') or unit.get('JDK_REAL_VERSION') or '_DEFAULT'), } if unit.get('ENABLE_PREVIEW_VALUE') == 'yes' and (unit.get('JDK_VERSION') or unit.get('JDK_REAL_VERSION')) in ( - '15', '17', '20', '21', @@ -372,7 +371,6 @@ def on_jdk_version_macro_check(unit, *args): jdk_version = args[0] available_versions = ( '11', - '15', '17', '20', '21', diff --git a/build/plugins/lib/nots/package_manager/base/lockfile.py b/build/plugins/lib/nots/package_manager/base/lockfile.py index 0dcccb40ef..5ff0cbf449 100644 --- a/build/plugins/lib/nots/package_manager/base/lockfile.py +++ b/build/plugins/lib/nots/package_manager/base/lockfile.py @@ -4,6 +4,17 @@ from abc import ABCMeta, abstractmethod from six import add_metaclass +class LockfilePackageMetaInvalidError(RuntimeError): + pass + + +def is_tarball_url_valid(tarball_url): + if not tarball_url.startswith("https://") and not tarball_url.startswith("http://"): + return True + + return tarball_url.startswith("https://npm.yandex-team.ru/") or tarball_url.startswith("http://npm.yandex-team.ru/") + + class LockfilePackageMeta(object): """ Basic struct representing package meta from lockfile. @@ -16,6 +27,11 @@ class LockfilePackageMeta(object): return LockfilePackageMeta(*s.strip().split(" ")) def __init__(self, key, tarball_url, sky_id, integrity, integrity_algorithm): + if not is_tarball_url_valid(tarball_url): + raise LockfilePackageMetaInvalidError( + "tarball can only point to npm.yandex-team.ru, got {}".format(tarball_url) + ) + # http://npm.yandex-team.ru/@scope%2fname/-/name-0.0.1.tgz parts = tarball_url.split("/") @@ -37,10 +53,6 @@ class LockfilePackageMeta(object): return pkg_uri -class LockfilePackageMetaInvalidError(RuntimeError): - pass - - @add_metaclass(ABCMeta) class BaseLockfile(object): @classmethod diff --git a/build/scripts/compile_cuda.py b/build/scripts/compile_cuda.py index cd37a95c60..9660300069 100644 --- a/build/scripts/compile_cuda.py +++ b/build/scripts/compile_cuda.py @@ -82,6 +82,7 @@ def main(): '-flto', '-faligned-allocation', '-fsized-deallocation', + '-fexperimental-library', # While it might be reasonable to compile host part of .cu sources with these optimizations enabled, # nvcc passes these options down towards cicc which lacks x86_64 extensions support. '-msse2', @@ -95,7 +96,7 @@ def main(): skip_list.append('-nostdinc++') for flag in skip_list: - if flag in cflags: + while flag in cflags: cflags.remove(flag) skip_prefix_list = [ diff --git a/build/scripts/cpp_proto_wrapper.py b/build/scripts/cpp_proto_wrapper.py index a70ba21e57..c29ca53a1d 100644 --- a/build/scripts/cpp_proto_wrapper.py +++ b/build/scripts/cpp_proto_wrapper.py @@ -25,7 +25,7 @@ def main(namespace: argparse.Namespace) -> int: subprocess.check_output(namespace.subcommand, stdin=None, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: sys.stderr.write( - '{} returned non-zero exit code {}.\n{}\n'.format(' '.join(e.cmd), e.returncode, e.output.decode('utf-8')) + '{} returned non-zero exit code {}.\n{}\n'.format(' '.join(e.cmd), e.returncode, e.output.decode('utf-8', errors='ignore')) ) return e.returncode diff --git a/build/scripts/go_proto_wrapper.py b/build/scripts/go_proto_wrapper.py index 1973d6e381..8f2a6130e0 100644 --- a/build/scripts/go_proto_wrapper.py +++ b/build/scripts/go_proto_wrapper.py @@ -67,7 +67,7 @@ def main(args): subprocess.check_output(args, stdin=None, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: sys.stderr.write( - '{} returned non-zero exit code {}.\n{}\n'.format(' '.join(e.cmd), e.returncode, e.output.decode('utf-8')) + '{} returned non-zero exit code {}.\n{}\n'.format(' '.join(e.cmd), e.returncode, e.output.decode('utf-8', errors='ignore')) ) return e.returncode diff --git a/contrib/libs/cxxsupp/libcxxmsvc/include/__support/win32/atomic_win32.h b/contrib/libs/cxxsupp/libcxxmsvc/include/__support/win32/atomic_win32.h index e32bcf9073..1862997f17 100644 --- a/contrib/libs/cxxsupp/libcxxmsvc/include/__support/win32/atomic_win32.h +++ b/contrib/libs/cxxsupp/libcxxmsvc/include/__support/win32/atomic_win32.h @@ -86,14 +86,14 @@ void __msvc_unlock(void* p); template<class _Out, class _Tp> static inline _Out __msvc_cast(_Tp __val) { - _Out __result; + alignas(_Out) char __result[sizeof(_Out)]; volatile char* to = reinterpret_cast<volatile char*>(&__result); volatile char* end = to + sizeof(_Tp); char* from = reinterpret_cast<char*>(&__val); while (to != end) { *to++ = *from++; } - return __result; + return *reinterpret_cast<_Out*>(&__result); } @@ -368,21 +368,20 @@ static inline __int64 __msvc_atomic_load64(volatile __int64* __a, memory_order _ template<typename _Tp> static inline _Tp __c11_atomic_load(volatile _Atomic(_Tp)* __a, int __order) { - _Tp __result; if (sizeof(_Tp) == 1) { - __result = __msvc_cast<_Tp>(__msvc_atomic_load8((volatile char*)__a, (memory_order)__order)); + return __msvc_cast<_Tp>(__msvc_atomic_load8((volatile char*)__a, (memory_order)__order)); } else if (sizeof(_Tp) == 2 && alignof(_Tp) % 2 == 0) { - __result = __msvc_cast<_Tp>(__msvc_atomic_load16((volatile short*)__a, (memory_order)__order)); + return __msvc_cast<_Tp>(__msvc_atomic_load16((volatile short*)__a, (memory_order)__order)); } else if (sizeof(_Tp) == 4 && alignof(_Tp) % 4 == 0) { - __result = __msvc_cast<_Tp>(__msvc_atomic_load32((volatile long*)__a, (memory_order)__order)); + return __msvc_cast<_Tp>(__msvc_atomic_load32((volatile long*)__a, (memory_order)__order)); } else if (sizeof(_Tp) == 8 && alignof(_Tp) % 8 == 0) { - __result = __msvc_cast<_Tp>(__msvc_atomic_load64((volatile __int64*)__a, (memory_order)__order)); + return __msvc_cast<_Tp>(__msvc_atomic_load64((volatile __int64*)__a, (memory_order)__order)); } else { __msvc_lock((void*)__a); - __result = *(_Atomic(_Tp)*)__a; + _Tp __result = *(_Atomic(_Tp)*)__a; __msvc_unlock((void*)__a); + return __result; } - return __result; } template<typename _Tp> diff --git a/contrib/python/Automat/py3/.dist-info/METADATA b/contrib/python/Automat/py3/.dist-info/METADATA index 1df3dba6c8..b86e9ab8e6 100644 --- a/contrib/python/Automat/py3/.dist-info/METADATA +++ b/contrib/python/Automat/py3/.dist-info/METADATA @@ -1,27 +1,199 @@ Metadata-Version: 2.1 Name: Automat -Version: 22.10.0 +Version: 24.8.1 Summary: Self-service finite-state machines for the programmer on the go. -Home-page: https://github.com/glyph/Automat -Author: Glyph -Author-email: glyph@twistedmatrix.com -License: MIT -Keywords: fsm finite state machine automata +Author-email: Glyph <code@glyph.im> +License: Copyright (c) 2014 + Rackspace + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to + permit persons to whom the Software is furnished to do so, subject to + the following conditions: + + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +Project-URL: Documentation, https://automat.readthedocs.io/ +Project-URL: Source, https://github.com/glyph/automat/ +Keywords: fsm,state machine,automata Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: MIT License Classifier: Operating System :: OS Independent Classifier: Programming Language :: Python -Classifier: Programming Language :: Python :: 2 -Classifier: Programming Language :: Python :: 2.7 Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python :: 3.5 -Classifier: Programming Language :: Python :: 3.6 -Classifier: Programming Language :: Python :: 3.7 Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 +Requires-Python: >=3.8 +Description-Content-Type: text/markdown License-File: LICENSE -Requires-Dist: attrs (>=19.2.0) -Requires-Dist: six +Requires-Dist: typing-extensions; python_version < "3.10" Provides-Extra: visualize -Requires-Dist: graphviz (>0.5.1) ; extra == 'visualize' -Requires-Dist: Twisted (>=16.1.1) ; extra == 'visualize' +Requires-Dist: graphviz>0.5.1; extra == "visualize" +Requires-Dist: Twisted>=16.1.1; extra == "visualize" +# Automat # + +[](http://automat.readthedocs.io/en/latest/) +[](https://github.com/glyph/automat/actions/workflows/ci.yml?query=branch%3Atrunk) +[](http://codecov.io/github/glyph/automat?branch=trunk) + +## Self-service finite-state machines for the programmer on the go. ## + +Automat is a library for concise, idiomatic Python expression of finite-state +automata (particularly deterministic finite-state transducers). + +Read more here, or on [Read the Docs](https://automat.readthedocs.io/), or watch the following videos for an overview and presentation + +### Why use state machines? ### + +Sometimes you have to create an object whose behavior varies with its state, +but still wishes to present a consistent interface to its callers. + +For example, let's say you're writing the software for a coffee machine. It +has a lid that can be opened or closed, a chamber for water, a chamber for +coffee beans, and a button for "brew". + +There are a number of possible states for the coffee machine. It might or +might not have water. It might or might not have beans. The lid might be open +or closed. The "brew" button should only actually attempt to brew coffee in +one of these configurations, and the "open lid" button should only work if the +coffee is not, in fact, brewing. + +With diligence and attention to detail, you can implement this correctly using +a collection of attributes on an object; `hasWater`, `hasBeans`, `isLidOpen` +and so on. However, you have to keep all these attributes consistent. As the +coffee maker becomes more complex - perhaps you add an additional chamber for +flavorings so you can make hazelnut coffee, for example - you have to keep +adding more and more checks and more and more reasoning about which +combinations of states are allowed. + +Rather than adding tedious `if` checks to every single method to make sure that +each of these flags are exactly what you expect, you can use a state machine to +ensure that if your code runs at all, it will be run with all the required +values initialized, because they have to be called in the order you declare +them. + +You can read about state machines and their advantages for Python programmers +in more detail [in this excellent article by Jean-Paul +Calderone](https://web.archive.org/web/20160507053658/https://clusterhq.com/2013/12/05/what-is-a-state-machine/). + +### What makes Automat different? ### + +There are +[dozens of libraries on PyPI implementing state machines](https://pypi.org/search/?q=finite+state+machine). +So it behooves me to say why yet another one would be a good idea. + +Automat is designed around this principle: while organizing your code around +state machines is a good idea, your callers don't, and shouldn't have to, care +that you've done so. In Python, the "input" to a stateful system is a method +call; the "output" may be a method call, if you need to invoke a side effect, +or a return value, if you are just performing a computation in memory. Most +other state-machine libraries require you to explicitly create an input object, +provide that object to a generic "input" method, and then receive results, +sometimes in terms of that library's interfaces and sometimes in terms of +classes you define yourself. + +For example, a snippet of the coffee-machine example above might be implemented +as follows in naive Python: + +```python +class CoffeeMachine(object): + def brewButton(self) -> None: + if self.hasWater and self.hasBeans and not self.isLidOpen: + self.heatTheHeatingElement() + # ... +``` + +With Automat, you'd begin with a `typing.Protocol` that describes all of your +inputs: + +```python +from typing import Protocol + +class CoffeeBrewer(Protocol): + def brewButton(self) -> None: + "The user pressed the 'brew' button." + def putInBeans(self) -> None: + "The user put in some beans." +``` + +We'll then need a concrete class to contain the shared core of state shared +among the different states: + +```python +from dataclasses import dataclass + +@dataclass +class BrewerCore: + heatingElement: HeatingElement +``` + +Next, we need to describe our state machine, including all of our states. For +simplicity's sake let's say that the only two states are `noBeans` and +`haveBeans`: + +```python +from automat import TypeMachineBuilder + +builder = TypeMachineBuilder(CoffeeBrewer, BrewerCore) +noBeans = builder.state("noBeans") +haveBeans = builder.state("haveBeans") +``` + +Next we can describe a simple transition; when we put in beans, we move to the +`haveBeans` state, with no other behavior. + +```python +# When we don't have beans, upon putting in beans, we will then have beans +noBeans.upon(CoffeeBrewer.putInBeans).to(haveBeans).returns(None) +``` + +And then another transition that we describe with a decorator, one that *does* +have some behavior, that needs to heat up the heating element to brew the +coffee: + +```python +@haveBeans.upon(CoffeeBrewer.brewButton).to(noBeans) +def heatUp(inputs: CoffeeBrewer, core: BrewerCore) -> None: + """ + When we have beans, upon pressing the brew button, we will then not have + beans any more (as they have been entered into the brewing chamber) and + our output will be heating the heating element. + """ + print("Brewing the coffee...") + core.heatingElement.turnOn() +``` + +Then we finalize the state machine by building it, which gives us a callable +that takes a `BrewerCore` and returns a synthetic `CoffeeBrewer` + +```python +newCoffeeMachine = builder.build() +``` + +```python +>>> coffee = newCoffeeMachine(BrewerCore(HeatingElement())) +>>> machine.putInBeans() +>>> machine.brewButton() +Brewing the coffee... +``` + +All of the *inputs* are provided by calling them like methods, all of the +*output behaviors* are automatically invoked when they are produced according +to the outputs specified to `upon` and all of the states are simply opaque +tokens. diff --git a/contrib/python/Automat/py3/README.md b/contrib/python/Automat/py3/README.md index 488a6c4043..a95e125345 100644 --- a/contrib/python/Automat/py3/README.md +++ b/contrib/python/Automat/py3/README.md @@ -1,8 +1,8 @@ # Automat # [](http://automat.readthedocs.io/en/latest/) -[](https://travis-ci.org/glyph/automat) -[](https://coveralls.io/r/glyph/automat) +[](https://github.com/glyph/automat/actions/workflows/ci.yml?query=branch%3Atrunk) +[](http://codecov.io/github/glyph/automat?branch=trunk) ## Self-service finite-state machines for the programmer on the go. ## @@ -11,12 +11,6 @@ automata (particularly deterministic finite-state transducers). Read more here, or on [Read the Docs](https://automat.readthedocs.io/), or watch the following videos for an overview and presentation -Overview and presentation by **Glyph Lefkowitz** at the first talk of the first Pyninsula meetup, on February 21st, 2017: -[](https://www.youtube.com/watch?v=0wOZBpD1VVk) - -Presentation by **Clinton Roy** at PyCon Australia, on August 6th 2017: -[](https://www.youtube.com/watch?v=TedUKXhu9kE) - ### Why use state machines? ### Sometimes you have to create an object whose behavior varies with its state, @@ -33,14 +27,14 @@ one of these configurations, and the "open lid" button should only work if the coffee is not, in fact, brewing. With diligence and attention to detail, you can implement this correctly using -a collection of attributes on an object; `has_water`, `has_beans`, -`is_lid_open` and so on. However, you have to keep all these attributes -consistent. As the coffee maker becomes more complex - perhaps you add an -additional chamber for flavorings so you can make hazelnut coffee, for -example - you have to keep adding more and more checks and more and more -reasoning about which combinations of states are allowed. - -Rather than adding tedious 'if' checks to every single method to make sure that +a collection of attributes on an object; `hasWater`, `hasBeans`, `isLidOpen` +and so on. However, you have to keep all these attributes consistent. As the +coffee maker becomes more complex - perhaps you add an additional chamber for +flavorings so you can make hazelnut coffee, for example - you have to keep +adding more and more checks and more and more reasoning about which +combinations of states are allowed. + +Rather than adding tedious `if` checks to every single method to make sure that each of these flags are exactly what you expect, you can use a state machine to ensure that if your code runs at all, it will be run with all the required values initialized, because they have to be called in the order you declare @@ -71,360 +65,87 @@ as follows in naive Python: ```python class CoffeeMachine(object): - def brew_button(self): - if self.has_water and self.has_beans and not self.is_lid_open: - self.heat_the_heating_element() + def brewButton(self) -> None: + if self.hasWater and self.hasBeans and not self.isLidOpen: + self.heatTheHeatingElement() # ... ``` -With Automat, you'd create a class with a `MethodicalMachine` attribute: +With Automat, you'd begin with a `typing.Protocol` that describes all of your +inputs: ```python -from automat import MethodicalMachine - -class CoffeeBrewer(object): - _machine = MethodicalMachine() -``` - -and then you would break the above logic into two pieces - the `brew_button` -*input*, declared like so: +from typing import Protocol -```python - @_machine.input() - def brew_button(self): +class CoffeeBrewer(Protocol): + def brewButton(self) -> None: "The user pressed the 'brew' button." -``` - -It wouldn't do any good to declare a method *body* on this, however, because -input methods don't actually execute their bodies when called; doing actual -work is the *output*'s job: - -```python - @_machine.output() - def _heat_the_heating_element(self): - "Heat up the heating element, which should cause coffee to happen." - self._heating_element.turn_on() -``` - -As well as a couple of *states* - and for simplicity's sake let's say that the -only two states are `have_beans` and `dont_have_beans`: - -```python - @_machine.state() - def have_beans(self): - "In this state, you have some beans." - @_machine.state(initial=True) - def dont_have_beans(self): - "In this state, you don't have any beans." -``` - -`dont_have_beans` is the `initial` state because `CoffeeBrewer` starts without beans -in it. - -(And another input to put some beans in:) - -```python - @_machine.input() - def put_in_beans(self): - "The user put in some beans." -``` - -Finally, you hook everything together with the `upon` method of the functions -decorated with `_machine.state`: - -```python - - # When we don't have beans, upon putting in beans, we will then have beans - # (and produce no output) - dont_have_beans.upon(put_in_beans, enter=have_beans, outputs=[]) - - # When we have beans, upon pressing the brew button, we will then not have - # beans any more (as they have been entered into the brewing chamber) and - # our output will be heating the heating element. - have_beans.upon(brew_button, enter=dont_have_beans, - outputs=[_heat_the_heating_element]) -``` - -To *users* of this coffee machine class though, it still looks like a POPO -(Plain Old Python Object): - -```python ->>> coffee_machine = CoffeeMachine() ->>> coffee_machine.put_in_beans() ->>> coffee_machine.brew_button() -``` - -All of the *inputs* are provided by calling them like methods, all of the -*outputs* are automatically invoked when they are produced according to the -outputs specified to `upon` and all of the states are simply opaque tokens - -although the fact that they're defined as methods like inputs and outputs -allows you to put docstrings on them easily to document them. - -## How do I get the current state of a state machine? - -Don't do that. - -One major reason for having a state machine is that you want the callers of the -state machine to just provide the appropriate input to the machine at the -appropriate time, and *not have to check themselves* what state the machine is -in. So if you are tempted to write some code like this: - -```python -if connection_state_machine.state == "CONNECTED": - connection_state_machine.send_message() -else: - print("not connected") -``` - -Instead, just make your calling code do this: - -```python -connection_state_machine.send_message() -``` - -and then change your state machine to look like this: - -```python - @_machine.state() - def connected(self): - "connected" - @_machine.state() - def not_connected(self): - "not connected" - @_machine.input() - def send_message(self): - "send a message" - @_machine.output() - def _actually_send_message(self): - self._transport.send(b"message") - @_machine.output() - def _report_sending_failure(self): - print("not connected") - connected.upon(send_message, enter=connected, [_actually_send_message]) - not_connected.upon(send_message, enter=not_connected, [_report_sending_failure]) -``` - -so that the responsibility for knowing which state the state machine is in -remains within the state machine itself. - -## Input for Inputs and Output for Outputs - -Quite often you want to be able to pass parameters to your methods, as well as -inspecting their results. For example, when you brew the coffee, you might -expect a cup of coffee to result, and you would like to see what kind of coffee -it is. And if you were to put delicious hand-roasted small-batch artisanal -beans into the machine, you would expect a *better* cup of coffee than if you -were to use mass-produced beans. You would do this in plain old Python by -adding a parameter, so that's how you do it in Automat as well. - -```python - @_machine.input() - def put_in_beans(self, beans): + def putInBeans(self) -> None: "The user put in some beans." ``` -However, one important difference here is that *we can't add any -implementation code to the input method*. Inputs are purely a declaration of -the interface; the behavior must all come from outputs. Therefore, the change -in the state of the coffee machine must be represented as an output. We can -add an output method like this: - -```python - @_machine.output() - def _save_beans(self, beans): - "The beans are now in the machine; save them." - self._beans = beans -``` - -and then connect it to the `put_in_beans` by changing the transition from -`dont_have_beans` to `have_beans` like so: - -```python - dont_have_beans.upon(put_in_beans, enter=have_beans, - outputs=[_save_beans]) -``` - -Now, when you call: - -```python -coffee_machine.put_in_beans("real good beans") -``` - -the machine will remember the beans for later. - -So how do we get the beans back out again? One of our outputs needs to have a -return value. It would make sense if our `brew_button` method returned the cup -of coffee that it made, so we should add an output. So, in addition to heating -the heating element, let's add a return value that describes the coffee. First -a new output: - -```python - @_machine.output() - def _describe_coffee(self): - return "A cup of coffee made with {}.".format(self._beans) -``` - -Note that we don't need to check first whether `self._beans` exists or not, -because we can only reach this output method if the state machine says we've -gone through a set of states that sets this attribute. - -Now, we need to hook up `_describe_coffee` to the process of brewing, so change -the brewing transition to: - -```python - have_beans.upon(brew_button, enter=dont_have_beans, - outputs=[_heat_the_heating_element, - _describe_coffee]) -``` - -Now, we can call it: - -```python ->>> coffee_machine.brew_button() -[None, 'A cup of coffee made with real good beans.'] -``` - -Except... wait a second, what's that `None` doing there? - -Since every input can produce multiple outputs, in automat, the default return -value from every input invocation is a `list`. In this case, we have both -`_heat_the_heating_element` and `_describe_coffee` outputs, so we're seeing -both of their return values. However, this can be customized, with the -`collector` argument to `upon`; the `collector` is a callable which takes an -iterable of all the outputs' return values and "collects" a single return value -to return to the caller of the state machine. - -In this case, we only care about the last output, so we can adjust the call to -`upon` like this: +We'll then need a concrete class to contain the shared core of state shared +among the different states: ```python - have_beans.upon(brew_button, enter=dont_have_beans, - outputs=[_heat_the_heating_element, - _describe_coffee], - collector=lambda iterable: list(iterable)[-1] - ) -``` +from dataclasses import dataclass -And now, we'll get just the return value we want: - -```python ->>> coffee_machine.brew_button() -'A cup of coffee made with real good beans.' +@dataclass +class BrewerCore: + heatingElement: HeatingElement ``` -## If I can't get the state of the state machine, how can I save it to (a database, an API response, a file on disk...) - -There are APIs for serializing the state machine. - -First, you have to decide on a persistent representation of each state, via the -`serialized=` argument to the `MethodicalMachine.state()` decorator. - -Let's take this very simple "light switch" state machine, which can be on or -off, and flipped to reverse its state: +Next, we need to describe our state machine, including all of our states. For +simplicity's sake let's say that the only two states are `noBeans` and +`haveBeans`: ```python -class LightSwitch(object): - _machine = MethodicalMachine() - @_machine.state(serialized="on") - def on_state(self): - "the switch is on" - @_machine.state(serialized="off", initial=True) - def off_state(self): - "the switch is off" - @_machine.input() - def flip(self): - "flip the switch" - on_state.upon(flip, enter=off_state, outputs=[]) - off_state.upon(flip, enter=on_state, outputs=[]) -``` +from automat import TypeMachineBuilder -In this case, we've chosen a serialized representation for each state via the -`serialized` argument. The on state is represented by the string `"on"`, and -the off state is represented by the string `"off"`. - -Now, let's just add an input that lets us tell if the switch is on or not. - -```python - @_machine.input() - def query_power(self): - "return True if powered, False otherwise" - @_machine.output() - def _is_powered(self): - return True - @_machine.output() - def _not_powered(self): - return False - on_state.upon(query_power, enter=on_state, outputs=[_is_powered], - collector=next) - off_state.upon(query_power, enter=off_state, outputs=[_not_powered], - collector=next) +builder = TypeMachineBuilder(CoffeeBrewer, BrewerCore) +noBeans = builder.state("noBeans") +haveBeans = builder.state("haveBeans") ``` -To save the state, we have the `MethodicalMachine.serializer()` method. A -method decorated with `@serializer()` gets an extra argument injected at the -beginning of its argument list: the serialized identifier for the state. In -this case, either `"on"` or `"off"`. Since state machine output methods can -also affect other state on the object, a serializer method is expected to -return *all* relevant state for serialization. - -For our simple light switch, such a method might look like this: +Next we can describe a simple transition; when we put in beans, we move to the +`haveBeans` state, with no other behavior. ```python - @_machine.serializer() - def save(self, state): - return {"is-it-on": state} +# When we don't have beans, upon putting in beans, we will then have beans +noBeans.upon(CoffeeBrewer.putInBeans).to(haveBeans).returns(None) ``` -Serializers can be public methods, and they can return whatever you like. If -necessary, you can have different serializers - just multiple methods decorated -with `@_machine.serializer()` - for different formats; return one data-structure -for JSON, one for XML, one for a database row, and so on. - -When it comes time to unserialize, though, you generally want a private method, -because an unserializer has to take a not-fully-initialized instance and -populate it with state. It is expected to *return* the serialized machine -state token that was passed to the serializer, but it can take whatever -arguments you like. Of course, in order to return that, it probably has to -take it somewhere in its arguments, so it will generally take whatever a paired -serializer has returned as an argument. - -So our unserializer would look like this: +And then another transition that we describe with a decorator, one that *does* +have some behavior, that needs to heat up the heating element to brew the +coffee: ```python - @_machine.unserializer() - def _restore(self, blob): - return blob["is-it-on"] +@haveBeans.upon(CoffeeBrewer.brewButton).to(noBeans) +def heatUp(inputs: CoffeeBrewer, core: BrewerCore) -> None: + """ + When we have beans, upon pressing the brew button, we will then not have + beans any more (as they have been entered into the brewing chamber) and + our output will be heating the heating element. + """ + print("Brewing the coffee...") + core.heatingElement.turnOn() ``` -Generally you will want a classmethod deserialization constructor which you -write yourself to call this, so that you know how to create an instance of your -own object, like so: +Then we finalize the state machine by building it, which gives us a callable +that takes a `BrewerCore` and returns a synthetic `CoffeeBrewer` ```python - @classmethod - def from_blob(cls, blob): - self = cls() - self._restore(blob) - return self +newCoffeeMachine = builder.build() ``` -Saving and loading our `LightSwitch` along with its state-machine state can now -be accomplished as follows: - ```python ->>> switch1 = LightSwitch() ->>> switch1.query_power() -False ->>> switch1.flip() -[] ->>> switch1.query_power() -True ->>> blob = switch1.save() ->>> switch2 = LightSwitch.from_blob(blob) ->>> switch2.query_power() -True +>>> coffee = newCoffeeMachine(BrewerCore(HeatingElement())) +>>> machine.putInBeans() +>>> machine.brewButton() +Brewing the coffee... ``` -More comprehensive (tested, working) examples are present in `docs/examples`. - -Go forth and machine all the state! +All of the *inputs* are provided by calling them like methods, all of the +*output behaviors* are automatically invoked when they are produced according +to the outputs specified to `upon` and all of the states are simply opaque +tokens. diff --git a/contrib/python/Automat/py3/automat/__init__.py b/contrib/python/Automat/py3/automat/__init__.py index 570b84f995..c4b34e565a 100644 --- a/contrib/python/Automat/py3/automat/__init__.py +++ b/contrib/python/Automat/py3/automat/__init__.py @@ -1,8 +1,16 @@ # -*- test-case-name: automat -*- -from ._methodical import MethodicalMachine +""" +State-machines. +""" +from ._typed import TypeMachineBuilder, pep614, AlreadyBuiltError, TypeMachine from ._core import NoTransition +from ._methodical import MethodicalMachine __all__ = [ - 'MethodicalMachine', - 'NoTransition', + "TypeMachineBuilder", + "TypeMachine", + "NoTransition", + "AlreadyBuiltError", + "pep614", + "MethodicalMachine", ] diff --git a/contrib/python/Automat/py3/automat/_core.py b/contrib/python/Automat/py3/automat/_core.py index 4118a4b070..fc637b3a0f 100644 --- a/contrib/python/Automat/py3/automat/_core.py +++ b/contrib/python/Automat/py3/automat/_core.py @@ -5,23 +5,41 @@ A core state-machine abstraction. Perhaps something that could be replaced with or integrated into machinist. """ +from __future__ import annotations +import sys from itertools import chain +from typing import Callable, Generic, Optional, Sequence, TypeVar, Hashable + +if sys.version_info >= (3, 10): + from typing import TypeAlias +else: + from typing_extensions import TypeAlias _NO_STATE = "<no state>" +State = TypeVar("State", bound=Hashable) +Input = TypeVar("Input", bound=Hashable) +Output = TypeVar("Output", bound=Hashable) -class NoTransition(Exception): +class NoTransition(Exception, Generic[State, Input]): """ A finite state machine in C{state} has no transition for C{symbol}. - @param state: the finite state machine's state at the time of the - illegal transition. + @ivar state: See C{state} init parameter. - @param symbol: the input symbol for which no transition exists. + @ivar symbol: See C{symbol} init parameter. """ - def __init__(self, state, symbol): + def __init__(self, state: State, symbol: Input): + """ + Construct a L{NoTransition}. + + @param state: the finite state machine's state at the time of the + illegal transition. + + @param symbol: the input symbol for which no transition exists. + """ self.state = state self.symbol = symbol super(Exception, self).__init__( @@ -29,31 +47,33 @@ class NoTransition(Exception): ) -class Automaton(object): +class Automaton(Generic[State, Input, Output]): """ A declaration of a finite state machine. Note that this is not the machine itself; it is immutable. """ - def __init__(self): + def __init__(self, initial: State | None = None) -> None: """ Initialize the set of transitions and the initial state. """ - self._initialState = _NO_STATE - self._transitions = set() - + if initial is None: + initial = _NO_STATE # type:ignore[assignment] + assert initial is not None + self._initialState: State = initial + self._transitions: set[tuple[State, Input, State, Sequence[Output]]] = set() + self._unhandledTransition: Optional[tuple[State, Sequence[Output]]] = None @property - def initialState(self): + def initialState(self) -> State: """ Return this automaton's initial state. """ return self._initialState - @initialState.setter - def initialState(self, state): + def initialState(self, state: State) -> None: """ Set this automaton's initial state. Raises a ValueError if this automaton already has an initial state. @@ -61,12 +81,18 @@ class Automaton(object): if self._initialState is not _NO_STATE: raise ValueError( - "initial state already set to {}".format(self._initialState)) + "initial state already set to {}".format(self._initialState) + ) self._initialState = state - - def addTransition(self, inState, inputSymbol, outState, outputSymbols): + def addTransition( + self, + inState: State, + inputSymbol: Input, + outState: State, + outputSymbols: tuple[Output, ...], + ): """ Add the given transition to the outputSymbol. Raise ValueError if there is already a transition with the same inState and inputSymbol. @@ -74,44 +100,51 @@ class Automaton(object): # keeping self._transitions in a flat list makes addTransition # O(n^2), but state machines don't tend to have hundreds of # transitions. - for (anInState, anInputSymbol, anOutState, _) in self._transitions: - if (anInState == inState and anInputSymbol == inputSymbol): + for anInState, anInputSymbol, anOutState, _ in self._transitions: + if anInState == inState and anInputSymbol == inputSymbol: raise ValueError( - "already have transition from {} via {}".format(inState, inputSymbol)) - self._transitions.add( - (inState, inputSymbol, outState, tuple(outputSymbols)) - ) + "already have transition from {} to {} via {}".format( + inState, anOutState, inputSymbol + ) + ) + self._transitions.add((inState, inputSymbol, outState, tuple(outputSymbols))) + def unhandledTransition( + self, outState: State, outputSymbols: Sequence[Output] + ) -> None: + """ + All unhandled transitions will be handled by transitioning to the given + error state and error-handling output symbols. + """ + self._unhandledTransition = (outState, tuple(outputSymbols)) - def allTransitions(self): + def allTransitions(self) -> frozenset[tuple[State, Input, State, Sequence[Output]]]: """ All transitions. """ return frozenset(self._transitions) - - def inputAlphabet(self): + def inputAlphabet(self) -> set[Input]: """ The full set of symbols acceptable to this automaton. """ - return {inputSymbol for (inState, inputSymbol, outState, - outputSymbol) in self._transitions} + return { + inputSymbol + for (inState, inputSymbol, outState, outputSymbol) in self._transitions + } - - def outputAlphabet(self): + def outputAlphabet(self) -> set[Output]: """ The full set of symbols which can be produced by this automaton. """ return set( chain.from_iterable( - outputSymbols for - (inState, inputSymbol, outState, outputSymbols) - in self._transitions + outputSymbols + for (inState, inputSymbol, outState, outputSymbols) in self._transitions ) ) - - def states(self): + def states(self) -> frozenset[State]: """ All valid states; "Q" in the mathematical description of a state machine. @@ -119,47 +152,52 @@ class Automaton(object): return frozenset( chain.from_iterable( (inState, outState) - for - (inState, inputSymbol, outState, outputSymbol) - in self._transitions + for (inState, inputSymbol, outState, outputSymbol) in self._transitions ) ) - - def outputForInput(self, inState, inputSymbol): + def outputForInput( + self, inState: State, inputSymbol: Input + ) -> tuple[State, Sequence[Output]]: """ A 2-tuple of (outState, outputSymbols) for inputSymbol. """ - for (anInState, anInputSymbol, - outState, outputSymbols) in self._transitions: + for anInState, anInputSymbol, outState, outputSymbols in self._transitions: if (inState, inputSymbol) == (anInState, anInputSymbol): return (outState, list(outputSymbols)) - raise NoTransition(state=inState, symbol=inputSymbol) + if self._unhandledTransition is None: + raise NoTransition(state=inState, symbol=inputSymbol) + return self._unhandledTransition + +OutputTracer = Callable[[Output], None] +Tracer: TypeAlias = "Callable[[State, Input, State], OutputTracer[Output] | None]" -class Transitioner(object): + +class Transitioner(Generic[State, Input, Output]): """ The combination of a current state and an L{Automaton}. """ - def __init__(self, automaton, initialState): - self._automaton = automaton - self._state = initialState - self._tracer = None + def __init__(self, automaton: Automaton[State, Input, Output], initialState: State): + self._automaton: Automaton[State, Input, Output] = automaton + self._state: State = initialState + self._tracer: Tracer[State, Input, Output] | None = None - def setTrace(self, tracer): + def setTrace(self, tracer: Tracer[State, Input, Output] | None) -> None: self._tracer = tracer - def transition(self, inputSymbol): + def transition( + self, inputSymbol: Input + ) -> tuple[Sequence[Output], OutputTracer[Output] | None]: """ Transition between states, returning any outputs. """ - outState, outputSymbols = self._automaton.outputForInput(self._state, - inputSymbol) + outState, outputSymbols = self._automaton.outputForInput( + self._state, inputSymbol + ) outTracer = None if self._tracer: - outTracer = self._tracer(self._state._name(), - inputSymbol._name(), - outState._name()) + outTracer = self._tracer(self._state, inputSymbol, outState) self._state = outState return (outputSymbols, outTracer) diff --git a/contrib/python/Automat/py3/automat/_discover.py b/contrib/python/Automat/py3/automat/_discover.py index c0d88baea4..ae92f82fc0 100644 --- a/contrib/python/Automat/py3/automat/_discover.py +++ b/contrib/python/Automat/py3/automat/_discover.py @@ -1,10 +1,17 @@ +from __future__ import annotations + import collections import inspect +from typing import Any, Iterator + +from twisted.python.modules import PythonAttribute, PythonModule, getModule + from automat import MethodicalMachine -from twisted.python.modules import PythonModule, getModule +from ._typed import TypeMachine, InputProtocol, Core -def isOriginalLocation(attr): + +def isOriginalLocation(attr: PythonAttribute | PythonModule) -> bool: """ Attempt to discover if this appearance of a PythonAttribute representing a class refers to the module where that class was @@ -21,7 +28,9 @@ def isOriginalLocation(attr): return currentModule.name == sourceModule.__name__ -def findMachinesViaWrapper(within): +def findMachinesViaWrapper( + within: PythonModule | PythonAttribute, +) -> Iterator[tuple[str, MethodicalMachine | TypeMachine[InputProtocol, Core]]]: """ Recursively yield L{MethodicalMachine}s and their FQPNs within a L{PythonModule} or a L{twisted.python.modules.PythonAttribute} @@ -40,17 +49,25 @@ def findMachinesViaWrapper(within): @return: a generator which yields FQPN, L{MethodicalMachine} pairs. """ queue = collections.deque([within]) - visited = set() + visited: set[ + PythonModule + | PythonAttribute + | MethodicalMachine + | TypeMachine[InputProtocol, Core] + | type[Any] + ] = set() while queue: attr = queue.pop() value = attr.load() - - if isinstance(value, MethodicalMachine) and value not in visited: + if ( + isinstance(value, MethodicalMachine) or isinstance(value, TypeMachine) + ) and value not in visited: visited.add(value) yield attr.name, value - elif (inspect.isclass(value) and isOriginalLocation(attr) and - value not in visited): + elif ( + inspect.isclass(value) and isOriginalLocation(attr) and value not in visited + ): visited.add(value) queue.extendleft(attr.iterAttributes()) elif isinstance(attr, PythonModule) and value not in visited: @@ -77,7 +94,7 @@ class NoObject(InvalidFQPN): """ -def wrapFQPN(fqpn): +def wrapFQPN(fqpn: str) -> PythonModule | PythonAttribute: """ Given an FQPN, retrieve the object via the global Python module namespace and wrap it with a L{PythonModule} or a @@ -88,12 +105,13 @@ def wrapFQPN(fqpn): if not fqpn: raise InvalidFQPN("FQPN was empty") - components = collections.deque(fqpn.split('.')) + components = collections.deque(fqpn.split(".")) - if '' in components: + if "" in components: raise InvalidFQPN( "name must be a string giving a '.'-separated list of Python " - "identifiers, not %r" % (fqpn,)) + "identifiers, not %r" % (fqpn,) + ) component = components.popleft() try: @@ -118,27 +136,33 @@ def wrapFQPN(fqpn): attribute = module for component in components: try: - attribute = next(child for child in attribute.iterAttributes() - if child.name.rsplit('.', 1)[-1] == component) + attribute = next( + child + for child in attribute.iterAttributes() + if child.name.rsplit(".", 1)[-1] == component + ) except StopIteration: - raise NoObject('{}.{}'.format(attribute.name, component)) + raise NoObject("{}.{}".format(attribute.name, component)) return attribute -def findMachines(fqpn): +def findMachines( + fqpn: str, +) -> Iterator[tuple[str, MethodicalMachine | TypeMachine[InputProtocol, Core]]]: """ - Recursively yield L{MethodicalMachine}s and their FQPNs in and - under the a Python object specified by an FQPN. + Recursively yield L{MethodicalMachine}s and their FQPNs in and under the a + Python object specified by an FQPN. - The discovery heuristic considers L{MethodicalMachine} instances - that are module-level attributes or class-level attributes - accessible from module scope. Machines inside nested classes will - be discovered, but those returned from functions or methods will not be. + The discovery heuristic considers L{MethodicalMachine} instances that are + module-level attributes or class-level attributes accessible from module + scope. Machines inside nested classes will be discovered, but those + returned from functions or methods will not be. - @type within: an FQPN - @param within: Where to start the search. + @param fqpn: a fully-qualified Python identifier (i.e. the dotted + identifier of an object defined at module or class scope, including the + package and modele names); where to start the search. - @return: a generator which yields FQPN, L{MethodicalMachine} pairs. + @return: a generator which yields (C{FQPN}, L{MethodicalMachine}) pairs. """ return findMachinesViaWrapper(wrapFQPN(fqpn)) diff --git a/contrib/python/Automat/py3/automat/_introspection.py b/contrib/python/Automat/py3/automat/_introspection.py index 403cddb15e..e433b2f9f9 100644 --- a/contrib/python/Automat/py3/automat/_introspection.py +++ b/contrib/python/Automat/py3/automat/_introspection.py @@ -7,31 +7,41 @@ from types import CodeType as code, FunctionType as function def copycode(template, changes): if hasattr(code, "replace"): - return template.replace(**{"co_" + k : v for k, v in changes.items()}) + return template.replace(**{"co_" + k: v for k, v in changes.items()}) names = [ - "argcount", "nlocals", "stacksize", "flags", "code", "consts", - "names", "varnames", "filename", "name", "firstlineno", "lnotab", - "freevars", "cellvars" + "argcount", + "nlocals", + "stacksize", + "flags", + "code", + "consts", + "names", + "varnames", + "filename", + "name", + "firstlineno", + "lnotab", + "freevars", + "cellvars", ] if hasattr(code, "co_kwonlyargcount"): names.insert(1, "kwonlyargcount") if hasattr(code, "co_posonlyargcount"): # PEP 570 added "positional only arguments" names.insert(1, "posonlyargcount") - values = [ - changes.get(name, getattr(template, "co_" + name)) - for name in names - ] + values = [changes.get(name, getattr(template, "co_" + name)) for name in names] return code(*values) def copyfunction(template, funcchanges, codechanges): names = [ - "globals", "name", "defaults", "closure", + "globals", + "name", + "defaults", + "closure", ] values = [ - funcchanges.get(name, getattr(template, "__" + name + "__")) - for name in names + funcchanges.get(name, getattr(template, "__" + name + "__")) for name in names ] return function(copycode(template.__code__, codechanges), *values) @@ -40,7 +50,8 @@ def preserveName(f): """ Preserve the name of the given function on the decorated function. """ + def decorator(decorated): - return copyfunction(decorated, - dict(name=f.__name__), dict(name=f.__name__)) + return copyfunction(decorated, dict(name=f.__name__), dict(name=f.__name__)) + return decorator diff --git a/contrib/python/Automat/py3/automat/_methodical.py b/contrib/python/Automat/py3/automat/_methodical.py index 6c9060cbb0..6c46c11e89 100644 --- a/contrib/python/Automat/py3/automat/_methodical.py +++ b/contrib/python/Automat/py3/automat/_methodical.py @@ -1,20 +1,34 @@ # -*- test-case-name: automat._test.test_methodical -*- +from __future__ import annotations import collections +import sys +from dataclasses import dataclass, field from functools import wraps -from itertools import count - from inspect import getfullargspec as getArgsSpec +from itertools import count +from typing import Any, Callable, Hashable, Iterable, TypeVar -import attr +if sys.version_info < (3, 10): + from typing_extensions import TypeAlias +else: + from typing import TypeAlias -from ._core import Transitioner, Automaton +from ._core import Automaton, OutputTracer, Tracer, Transitioner from ._introspection import preserveName - -ArgSpec = collections.namedtuple('ArgSpec', ['args', 'varargs', 'varkw', - 'defaults', 'kwonlyargs', - 'kwonlydefaults', 'annotations']) +ArgSpec = collections.namedtuple( + "ArgSpec", + [ + "args", + "varargs", + "varkw", + "defaults", + "kwonlyargs", + "kwonlydefaults", + "annotations", + ], +) def _getArgSpec(func): @@ -34,8 +48,7 @@ def _getArgSpec(func): defaults=spec.defaults if spec.defaults else (), kwonlyargs=tuple(spec.kwonlyargs), kwonlydefaults=( - tuple(spec.kwonlydefaults.items()) - if spec.kwonlydefaults else () + tuple(spec.kwonlydefaults.items()) if spec.kwonlydefaults else () ), annotations=tuple(spec.annotations.items()), ) @@ -54,8 +67,8 @@ def _getArgNames(spec): return set( spec.args + spec.kwonlyargs - + (('*args',) if spec.varargs else ()) - + (('**kwargs',) if spec.varkw else ()) + + (("*args",) if spec.varargs else ()) + + (("**kwargs",) if spec.varkw else ()) + spec.annotations ) @@ -70,39 +83,51 @@ def _keywords_only(f): Only works for methods right now. """ + @wraps(f) def g(self, **kw): return f(self, **kw) + return g -@attr.s(frozen=True) +@dataclass(frozen=True) class MethodicalState(object): """ A state for a L{MethodicalMachine}. """ - machine = attr.ib(repr=False) - method = attr.ib() - serialized = attr.ib(repr=False) - def upon(self, input, enter=None, outputs=None, collector=list): + machine: MethodicalMachine = field(repr=False) + method: Callable[..., Any] = field() + serialized: bool = field(repr=False) + + def upon( + self, + input: MethodicalInput, + enter: MethodicalState | None = None, + outputs: Iterable[MethodicalOutput] | None = None, + collector: Callable[[Iterable[T]], object] = list, + ) -> None: """ - Declare a state transition within the :class:`automat.MethodicalMachine` - associated with this :class:`automat.MethodicalState`: - upon the receipt of the `input`, enter the `state`, - emitting each output in `outputs`. - - :param MethodicalInput input: The input triggering a state transition. - :param MethodicalState enter: The resulting state. - :param Iterable[MethodicalOutput] outputs: The outputs to be triggered - as a result of the declared state transition. - :param Callable collector: The function to be used when collecting - output return values. - - :raises TypeError: if any of the `outputs` signatures do not match - the `inputs` signature. - :raises ValueError: if the state transition from `self` via `input` - has already been defined. + Declare a state transition within the L{MethodicalMachine} associated + with this L{MethodicalState}: upon the receipt of the `input`, enter + the `state`, emitting each output in `outputs`. + + @param input: The input triggering a state transition. + + @param enter: The resulting state. + + @param outputs: The outputs to be triggered as a result of the declared + state transition. + + @param collector: The function to be used when collecting output return + values. + + @raises TypeError: if any of the `outputs` signatures do not match the + `inputs` signature. + + @raises ValueError: if the state transition from `self` via `input` has + already been defined. """ if enter is None: enter = self @@ -120,14 +145,19 @@ class MethodicalState(object): output=output.method.__name__, inputSignature=getArgsSpec(input.method), outputSignature=getArgsSpec(output.method), - )) + ) + ) self.machine._oneTransition(self, input, enter, outputs, collector) - def _name(self): + def _name(self) -> str: return self.method.__name__ -def _transitionerFromInstance(oself, symbol, automaton): +def _transitionerFromInstance( + oself: object, + symbol: str, + automaton: Automaton[MethodicalState, MethodicalInput, MethodicalOutput], +) -> Transitioner[MethodicalState, MethodicalInput, MethodicalOutput]: """ Get a L{Transitioner} """ @@ -144,10 +174,12 @@ def _transitionerFromInstance(oself, symbol, automaton): def _empty(): pass + def _docstring(): """docstring""" -def assertNoCode(inst, attribute, f): + +def assertNoCode(f: Callable[..., Any]) -> None: # The function body must be empty, i.e. "pass" or "return None", which # both yield the same bytecode: LOAD_CONST (None), RETURN_VALUE. We also # accept functions with only a docstring, which yields slightly different @@ -159,8 +191,7 @@ def assertNoCode(inst, attribute, f): # checking that would require us to parse the bytecode, find the index # being returned, then making sure the table has a None at that index. - if f.__code__.co_code not in (_empty.__code__.co_code, - _docstring.__code__.co_code): + if f.__code__.co_code not in (_empty.__code__.co_code, _docstring.__code__.co_code): raise ValueError("function body must be empty") @@ -198,68 +229,80 @@ def _filterArgs(args, kwargs, inputSpec, outputSpec): else: # Filter out names that the output method does not accept. all_accepted_names = outputSpec.args[1:] + outputSpec.kwonlyargs - return_kwargs = {n: v for n, v in full_kwargs.items() - if n in all_accepted_names} + return_kwargs = { + n: v for n, v in full_kwargs.items() if n in all_accepted_names + } return return_args, return_kwargs -@attr.s(eq=False, hash=False) +T = TypeVar("T") +R = TypeVar("R") + + +@dataclass(eq=False) class MethodicalInput(object): """ An input for a L{MethodicalMachine}. """ - automaton = attr.ib(repr=False) - method = attr.ib(validator=assertNoCode) - symbol = attr.ib(repr=False) - collectors = attr.ib(default=attr.Factory(dict), repr=False) - argSpec = attr.ib(init=False, repr=False) - @argSpec.default - def _buildArgSpec(self): - return _getArgSpec(self.method) + automaton: Automaton[MethodicalState, MethodicalInput, MethodicalOutput] = field( + repr=False + ) + method: Callable[..., Any] = field() + symbol: str = field(repr=False) + collectors: dict[MethodicalState, Callable[[Iterable[T]], R]] = field( + default_factory=dict, repr=False + ) - def __get__(self, oself, type=None): + argSpec: ArgSpec = field(init=False, repr=False) + + def __post_init__(self) -> None: + self.argSpec = _getArgSpec(self.method) + assertNoCode(self.method) + + def __get__(self, oself: object, type: None = None) -> object: """ Return a function that takes no arguments and returns values returned by output functions produced by the given L{MethodicalInput} in C{oself}'s current state. """ - transitioner = _transitionerFromInstance(oself, self.symbol, - self.automaton) + transitioner = _transitionerFromInstance(oself, self.symbol, self.automaton) + @preserveName(self.method) @wraps(self.method) - def doInput(*args, **kwargs): + def doInput(*args: object, **kwargs: object) -> object: self.method(oself, *args, **kwargs) previousState = transitioner._state (outputs, outTracer) = transitioner.transition(self) collector = self.collectors[previousState] values = [] for output in outputs: - if outTracer: - outTracer(output._name()) + if outTracer is not None: + outTracer(output) a, k = _filterArgs(args, kwargs, self.argSpec, output.argSpec) value = output(oself, *a, **k) values.append(value) return collector(values) + return doInput - def _name(self): + def _name(self) -> str: return self.method.__name__ -@attr.s(frozen=True) +@dataclass(frozen=True) class MethodicalOutput(object): """ An output for a L{MethodicalMachine}. """ - machine = attr.ib(repr=False) - method = attr.ib() - argSpec = attr.ib(init=False, repr=False) - @argSpec.default - def _buildArgSpec(self): - return _getArgSpec(self.method) + machine: MethodicalMachine = field(repr=False) + method: Callable[..., Any] + argSpec: ArgSpec = field(init=False, repr=False, compare=False) + + def __post_init__(self) -> None: + self.__dict__["argSpec"] = _getArgSpec(self.method) def __get__(self, oself, type=None): """ @@ -268,37 +311,64 @@ class MethodicalOutput(object): raise AttributeError( "{cls}.{method} is a state-machine output method; " "to produce this output, call an input method instead.".format( - cls=type.__name__, - method=self.method.__name__ + cls=type.__name__, method=self.method.__name__ ) ) - def __call__(self, oself, *args, **kwargs): """ Call the underlying method. """ return self.method(oself, *args, **kwargs) - def _name(self): + def _name(self) -> str: return self.method.__name__ -@attr.s(eq=False, hash=False) + +StringOutputTracer = Callable[[str], None] +StringTracer: TypeAlias = "Callable[[str, str, str], StringOutputTracer | None]" + + +def wrapTracer( + wrapped: StringTracer | None, +) -> Tracer[MethodicalState, MethodicalInput, MethodicalOutput] | None: + if wrapped is None: + return None + + def tracer( + state: MethodicalState, + input: MethodicalInput, + output: MethodicalState, + ) -> OutputTracer[MethodicalOutput] | None: + result = wrapped(state._name(), input._name(), output._name()) + if result is not None: + return lambda out: result(out._name()) + return None + + return tracer + + +@dataclass(eq=False) class MethodicalTracer(object): - automaton = attr.ib(repr=False) - symbol = attr.ib(repr=False) + automaton: Automaton[MethodicalState, MethodicalInput, MethodicalOutput] = field( + repr=False + ) + symbol: str = field(repr=False) + def __get__( + self, oself: object, type: object = None + ) -> Callable[[StringTracer], None]: + transitioner = _transitionerFromInstance(oself, self.symbol, self.automaton) - def __get__(self, oself, type=None): - transitioner = _transitionerFromInstance(oself, self.symbol, - self.automaton) - def setTrace(tracer): - transitioner.setTrace(tracer) - return setTrace + def setTrace(tracer: StringTracer | None) -> None: + transitioner.setTrace(wrapTracer(tracer)) + return setTrace counter = count() + + def gensym(): """ Create a unique Python identifier. @@ -306,11 +376,10 @@ def gensym(): return "_symbol_" + str(next(counter)) - class MethodicalMachine(object): """ - A :class:`MethodicalMachine` is an interface to an `Automaton` - that uses methods on a class. + A L{MethodicalMachine} is an interface to an L{Automaton} that uses methods + on a class. """ def __init__(self): @@ -318,7 +387,6 @@ class MethodicalMachine(object): self._reducers = {} self._symbol = gensym() - def __get__(self, oself, type=None): """ L{MethodicalMachine} is an implementation detail for setting up @@ -326,42 +394,41 @@ class MethodicalMachine(object): instance. """ if oself is not None: - raise AttributeError( - "MethodicalMachine is an implementation detail.") + raise AttributeError("MethodicalMachine is an implementation detail.") return self - @_keywords_only - def state(self, initial=False, terminal=False, - serialized=None): + def state( + self, initial: bool = False, terminal: bool = False, serialized: Hashable = None + ): """ Declare a state, possibly an initial state or a terminal state. This is a decorator for methods, but it will modify the method so as not to be callable any more. - :param bool initial: is this state the initial state? - Only one state on this :class:`automat.MethodicalMachine` - may be an initial state; more than one is an error. + @param initial: is this state the initial state? Only one state on + this L{automat.MethodicalMachine} may be an initial state; more + than one is an error. - :param bool terminal: Is this state a terminal state? - i.e. a state that the machine can end up in? - (This is purely informational at this point.) + @param terminal: Is this state a terminal state? i.e. a state that the + machine can end up in? (This is purely informational at this + point.) - :param Hashable serialized: a serializable value - to be used to represent this state to external systems. - This value should be hashable; - :py:func:`unicode` is a good type to use. + @param serialized: a serializable value to be used to represent this + state to external systems. This value should be hashable; L{str} + is a good type to use. """ + def decorator(stateMethod): - state = MethodicalState(machine=self, - method=stateMethod, - serialized=serialized) + state = MethodicalState( + machine=self, method=stateMethod, serialized=serialized + ) if initial: self._automaton.initialState = state return state - return decorator + return decorator @_keywords_only def input(self): @@ -370,12 +437,13 @@ class MethodicalMachine(object): This is a decorator for methods. """ + def decorator(inputMethod): - return MethodicalInput(automaton=self._automaton, - method=inputMethod, - symbol=self._symbol) - return decorator + return MethodicalInput( + automaton=self._automaton, method=inputMethod, symbol=self._symbol + ) + return decorator @_keywords_only def output(self): @@ -387,13 +455,13 @@ class MethodicalMachine(object): This method will be called when the state machine transitions to this state as specified in the decorated `output` method. """ + def decorator(outputMethod): return MethodicalOutput(machine=self, method=outputMethod) - return decorator + return decorator - def _oneTransition(self, startState, inputToken, endState, outputTokens, - collector): + def _oneTransition(self, startState, inputToken, endState, outputTokens, collector): """ See L{MethodicalState.upon}. """ @@ -411,30 +479,31 @@ class MethodicalMachine(object): # if not isinstance(endState, MethodicalState): # raise NotImplementedError("output state {} isn't a state" # .format(endState)) - self._automaton.addTransition(startState, inputToken, endState, - tuple(outputTokens)) + self._automaton.addTransition( + startState, inputToken, endState, tuple(outputTokens) + ) inputToken.collectors[startState] = collector - @_keywords_only def serializer(self): - """ + """ """ - """ def decorator(decoratee): @wraps(decoratee) def serialize(oself): - transitioner = _transitionerFromInstance(oself, self._symbol, - self._automaton) + transitioner = _transitionerFromInstance( + oself, self._symbol, self._automaton + ) return decoratee(oself, transitioner._state.serialized) + return serialize + return decorator @_keywords_only def unserializer(self): - """ + """ """ - """ def decorator(decoratee): @wraps(decoratee) def unserialize(oself, *args, **kwargs): @@ -443,14 +512,17 @@ class MethodicalMachine(object): for eachState in self._automaton.states(): mapping[eachState.serialized] = eachState transitioner = _transitionerFromInstance( - oself, self._symbol, self._automaton) + oself, self._symbol, self._automaton + ) transitioner._state = mapping[state] - return None # it's on purpose + return None # it's on purpose + return unserialize + return decorator @property - def _setTrace(self): + def _setTrace(self) -> MethodicalTracer: return MethodicalTracer(self._automaton, self._symbol) def asDigraph(self): @@ -464,6 +536,7 @@ class MethodicalMachine(object): """ from ._visualize import makeDigraph + return makeDigraph( self._automaton, stateAsString=lambda state: state.method.__name__, diff --git a/contrib/python/Automat/py3/automat/_runtimeproto.py b/contrib/python/Automat/py3/automat/_runtimeproto.py new file mode 100644 index 0000000000..c9c7409f5a --- /dev/null +++ b/contrib/python/Automat/py3/automat/_runtimeproto.py @@ -0,0 +1,62 @@ +""" +Workaround for U{the lack of TypeForm +<https://github.com/python/mypy/issues/9773>}. +""" + +from __future__ import annotations + +import sys + +from typing import TYPE_CHECKING, Callable, Protocol, TypeVar + +from inspect import signature, Signature + +T = TypeVar("T") + +ProtocolAtRuntime = Callable[[], T] + + +def runtime_name(x: ProtocolAtRuntime[T]) -> str: + return x.__name__ + + +from inspect import getmembers, isfunction + +emptyProtocolMethods: frozenset[str] +if not TYPE_CHECKING: + emptyProtocolMethods = frozenset( + name + for name, each in getmembers(type("Example", tuple([Protocol]), {}), isfunction) + ) + + +def actuallyDefinedProtocolMethods(protocol: object) -> frozenset[str]: + """ + Attempt to ignore implementation details, and get all the methods that the + protocol actually defines. + + that includes locally defined methods and also those defined in inherited + superclasses. + """ + return ( + frozenset(name for name, each in getmembers(protocol, isfunction)) + - emptyProtocolMethods + ) + + +def _fixAnnotation(method: Callable[..., object], it: object, ann: str) -> None: + annotation = getattr(it, ann) + if isinstance(annotation, str): + setattr(it, ann, eval(annotation, method.__globals__)) + + +def _liveSignature(method: Callable[..., object]) -> Signature: + """ + Get a signature with evaluated annotations. + """ + # TODO: could this be replaced with get_type_hints? + result = signature(method) + for param in result.parameters.values(): + _fixAnnotation(method, param, "_annotation") + _fixAnnotation(method, result, "_return_annotation") + return result diff --git a/contrib/python/Automat/py3/automat/_typed.py b/contrib/python/Automat/py3/automat/_typed.py new file mode 100644 index 0000000000..c5d9e128ed --- /dev/null +++ b/contrib/python/Automat/py3/automat/_typed.py @@ -0,0 +1,736 @@ +# -*- test-case-name: automat._test.test_type_based -*- +from __future__ import annotations + +import sys +from dataclasses import dataclass, field +from typing import ( + TYPE_CHECKING, + get_origin, + Any, + Callable, + Generic, + Iterable, + Literal, + Protocol, + TypeVar, + overload, +) + +if TYPE_CHECKING: + from graphviz import Digraph +try: + from zope.interface.interface import InterfaceClass # type:ignore[import-untyped] +except ImportError: + hasInterface = False +else: + hasInterface = True + +if sys.version_info < (3, 10): + from typing_extensions import Concatenate, ParamSpec, TypeAlias +else: + from typing import Concatenate, ParamSpec, TypeAlias + +from ._core import Automaton, Transitioner +from ._runtimeproto import ( + ProtocolAtRuntime, + _liveSignature, + actuallyDefinedProtocolMethods, + runtime_name, +) + + +class AlreadyBuiltError(Exception): + """ + The L{TypeMachine} is already built, and thus can no longer be + modified. + """ + + +InputProtocol = TypeVar("InputProtocol") +Core = TypeVar("Core") +Data = TypeVar("Data") +P = ParamSpec("P") +P1 = ParamSpec("P1") +R = TypeVar("R") +OtherData = TypeVar("OtherData") +Decorator = Callable[[Callable[P, R]], Callable[P, R]] +FactoryParams = ParamSpec("FactoryParams") +OtherFactoryParams = ParamSpec("OtherFactoryParams") + + +def pep614(t: R) -> R: + """ + This is a workaround for Python 3.8, which has U{some restrictions on its + grammar for decorators <https://peps.python.org/pep-0614/>}, and makes + C{@state.to(other).upon(Protocol.input)} invalid syntax; for code that + needs to run on these older Python versions, you can do + C{@pep614(state.to(other).upon(Protocol.input))} instead. + """ + return t + + +@dataclass() +class TransitionRegistrar(Generic[P, P1, R]): + """ + This is a record of a transition that need finalizing; it is the result of + calling L{TypeMachineBuilder.state} and then ``.upon(input).to(state)`` on + the result of that. + + It can be used as a decorator, like:: + + registrar = state.upon(Proto.input).to(state2) + @registrar + def inputImplementation(proto: Proto, core: Core) -> Result: ... + + Or, it can be used used to implement a constant return value with + L{TransitionRegistrar.returns}, like:: + + registrar = state.upon(Proto.input).to(state2) + registrar.returns(value) + + Type parameter P: the precise signature of the decorated implementation + callable. + + Type parameter P1: the precise signature of the input method from the + outward-facing state-machine protocol. + + Type parameter R: the return type of both the protocol method and the input + method. + """ + + _signature: Callable[P1, R] + _old: AnyState + _new: AnyState + _nodata: bool = False + _callback: Callable[P, R] | None = None + + def __post_init__(self) -> None: + self._old.builder._registrars.append(self) + + def __call__(self, impl: Callable[P, R]) -> Callable[P, R]: + """ + Finalize it with C{__call__} to indicate that there is an + implementation to the transition, which can be treated as an output. + """ + if self._callback is not None: + raise AlreadyBuiltError( + f"already registered transition from {self._old.name!r} to {self._new.name!r}" + ) + self._callback = impl + builder = self._old.builder + assert builder is self._new.builder, "states must be from the same builder" + builder._automaton.addTransition( + self._old, + self._signature.__name__, + self._new, + tuple(self._new._produceOutputs(impl, self._old, self._nodata)), + ) + return impl + + def returns(self, result: R) -> None: + """ + Finalize it with C{.returns(constant)} to indicate that there is no + method body, and the given result can just be yielded each time after + the state transition. The only output generated in this case would be + the data-construction factory for the target state. + """ + + def constant(*args: object, **kwargs: object) -> R: + return result + + constant.__name__ = f"returns({result})" + self(constant) + + def _checkComplete(self) -> None: + """ + Raise an exception if the user forgot to decorate a method + implementation or supply a return value for this transition. + """ + # TODO: point at the line where `.to`/`.loop`/`.upon` are called so the + # user can more immediately see the incomplete transition + if not self._callback: + raise ValueError( + f"incomplete transition from {self._old.name} to " + f"{self._new.name} upon {self._signature.__qualname__}: " + "remember to use the transition as a decorator or call " + "`.returns` on it." + ) + + +@dataclass +class UponFromNo(Generic[InputProtocol, Core, P, R]): + """ + Type parameter P: the signature of the input method. + """ + + old: TypedState[InputProtocol, Core] | TypedDataState[InputProtocol, Core, Any, ...] + input: Callable[Concatenate[InputProtocol, P], R] + + @overload + def to( + self, state: TypedState[InputProtocol, Core] + ) -> TransitionRegistrar[Concatenate[InputProtocol, Core, P], P, R]: ... + @overload + def to( + self, + state: TypedDataState[InputProtocol, Core, OtherData, P], + ) -> TransitionRegistrar[ + Concatenate[InputProtocol, Core, P], + Concatenate[InputProtocol, P], + R, + ]: ... + def to( + self, + state: ( + TypedState[InputProtocol, Core] + | TypedDataState[InputProtocol, Core, Any, P] + ), + ) -> ( + TransitionRegistrar[Concatenate[InputProtocol, Core, P], P, R] + | TransitionRegistrar[ + Concatenate[InputProtocol, Core, P], + Concatenate[InputProtocol, P], + R, + ] + ): + """ + Declare a state transition to a new state. + """ + return TransitionRegistrar(self.input, self.old, state, True) + + def loop(self) -> TransitionRegistrar[ + Concatenate[InputProtocol, Core, P], + Concatenate[InputProtocol, P], + R, + ]: + """ + Register a transition back to the same state. + """ + return TransitionRegistrar(self.input, self.old, self.old, True) + + +@dataclass +class UponFromData(Generic[InputProtocol, Core, P, R, Data]): + """ + Type parameter P: the signature of the input method. + """ + + old: TypedDataState[InputProtocol, Core, Data, ...] + input: Callable[Concatenate[InputProtocol, P], R] + + @overload + def to( + self, state: TypedState[InputProtocol, Core] + ) -> TransitionRegistrar[ + Concatenate[InputProtocol, Core, Data, P], Concatenate[InputProtocol, P], R + ]: ... + @overload + def to( + self, + state: TypedDataState[InputProtocol, Core, OtherData, P], + ) -> TransitionRegistrar[ + Concatenate[InputProtocol, Core, Data, P], + Concatenate[InputProtocol, P], + R, + ]: ... + def to( + self, + state: ( + TypedState[InputProtocol, Core] + | TypedDataState[InputProtocol, Core, Any, P] + ), + ) -> ( + TransitionRegistrar[Concatenate[InputProtocol, Core, P], P, R] + | TransitionRegistrar[ + Concatenate[InputProtocol, Core, Data, P], + Concatenate[InputProtocol, P], + R, + ] + ): + """ + Declare a state transition to a new state. + """ + return TransitionRegistrar(self.input, self.old, state) + + def loop(self) -> TransitionRegistrar[ + Concatenate[InputProtocol, Core, Data, P], + Concatenate[InputProtocol, P], + R, + ]: + """ + Register a transition back to the same state. + """ + return TransitionRegistrar(self.input, self.old, self.old) + + +@dataclass(frozen=True) +class TypedState(Generic[InputProtocol, Core]): + """ + The result of L{.state() <automat.TypeMachineBuilder.state>}. + """ + + name: str + builder: TypeMachineBuilder[InputProtocol, Core] = field(repr=False) + + def upon( + self, input: Callable[Concatenate[InputProtocol, P], R] + ) -> UponFromNo[InputProtocol, Core, P, R]: + ".upon()" + self.builder._checkMembership(input) + return UponFromNo(self, input) + + def _produceOutputs( + self, + impl: Callable[..., object], + old: ( + TypedDataState[InputProtocol, Core, OtherData, OtherFactoryParams] + | TypedState[InputProtocol, Core] + ), + nodata: bool = False, + ) -> Iterable[SomeOutput]: + yield MethodOutput._fromImpl(impl, isinstance(old, TypedDataState)) + + +@dataclass(frozen=True) +class TypedDataState(Generic[InputProtocol, Core, Data, FactoryParams]): + name: str + builder: TypeMachineBuilder[InputProtocol, Core] = field(repr=False) + factory: Callable[Concatenate[InputProtocol, Core, FactoryParams], Data] + + @overload + def upon( + self, input: Callable[Concatenate[InputProtocol, P], R] + ) -> UponFromData[InputProtocol, Core, P, R, Data]: ... + @overload + def upon( + self, input: Callable[Concatenate[InputProtocol, P], R], nodata: Literal[False] + ) -> UponFromData[InputProtocol, Core, P, R, Data]: ... + @overload + def upon( + self, input: Callable[Concatenate[InputProtocol, P], R], nodata: Literal[True] + ) -> UponFromNo[InputProtocol, Core, P, R]: ... + def upon( + self, + input: Callable[Concatenate[InputProtocol, P], R], + nodata: bool = False, + ) -> ( + UponFromData[InputProtocol, Core, P, R, Data] + | UponFromNo[InputProtocol, Core, P, R] + ): + self.builder._checkMembership(input) + if nodata: + return UponFromNo(self, input) + else: + return UponFromData(self, input) + + def _produceOutputs( + self, + impl: Callable[..., object], + old: ( + TypedDataState[InputProtocol, Core, OtherData, OtherFactoryParams] + | TypedState[InputProtocol, Core] + ), + nodata: bool, + ) -> Iterable[SomeOutput]: + if self is not old: + yield DataOutput(self.factory) + yield MethodOutput._fromImpl( + impl, isinstance(old, TypedDataState) and not nodata + ) + + +AnyState: TypeAlias = "TypedState[Any, Any] | TypedDataState[Any, Any, Any, Any]" + + +@dataclass +class TypedInput: + name: str + + +class SomeOutput(Protocol): + """ + A state machine output. + """ + + @property + def name(self) -> str: + "read-only name property" + + def __call__(*args: Any, **kwargs: Any) -> Any: ... + + def __hash__(self) -> int: + "must be hashable" + + +@dataclass +class InputImplementer(Generic[InputProtocol, Core]): + """ + An L{InputImplementer} implements an input protocol in terms of a + state machine. + + When the factory returned from L{TypeMachine} + """ + + __automat_core__: Core + __automat_transitioner__: Transitioner[ + TypedState[InputProtocol, Core] + | TypedDataState[InputProtocol, Core, object, ...], + str, + SomeOutput, + ] + __automat_data__: object | None = None + __automat_postponed__: list[Callable[[], None]] | None = None + + +def implementMethod( + method: Callable[..., object], +) -> Callable[..., object]: + """ + Construct a function for populating in the synthetic provider of the Input + Protocol to a L{TypeMachineBuilder}. It should have a signature matching that + of the C{method} parameter, a function from that protocol. + """ + methodInput = method.__name__ + # side-effects can be re-ordered until later. If you need to compute a + # value in your method, then obviously it can't be invoked reentrantly. + returnAnnotation = _liveSignature(method).return_annotation + returnsNone = returnAnnotation is None + + def implementation( + self: InputImplementer[InputProtocol, Core], *args: object, **kwargs: object + ) -> object: + transitioner = self.__automat_transitioner__ + dataAtStart = self.__automat_data__ + if self.__automat_postponed__ is not None: + if not returnsNone: + raise RuntimeError( + f"attempting to reentrantly run {method.__qualname__} " + f"but it wants to return {returnAnnotation!r} not None" + ) + + def rerunme() -> None: + implementation(self, *args, **kwargs) + + self.__automat_postponed__.append(rerunme) + return None + postponed = self.__automat_postponed__ = [] + try: + [outputs, tracer] = transitioner.transition(methodInput) + result: Any = None + for output in outputs: + # here's the idea: there will be a state-setup output and a + # state-teardown output. state-setup outputs are added to the + # *beginning* of any entry into a state, so that by the time you + # are running the *implementation* of a method that has entered + # that state, the protocol is in a self-consistent state and can + # run reentrant outputs. not clear that state-teardown outputs are + # necessary + result = output(self, dataAtStart, *args, **kwargs) + finally: + self.__automat_postponed__ = None + while postponed: + postponed.pop(0)() + return result + + implementation.__qualname__ = implementation.__name__ = ( + f"<implementation for {method}>" + ) + return implementation + + +@dataclass(frozen=True) +class MethodOutput(Generic[Core]): + """ + This is the thing that goes into the automaton's outputs list, and thus + (per the implementation of L{implementMethod}) takes the 'self' of the + InputImplementer instance (i.e. the synthetic protocol implementation) and the + previous result computed by the former output, which will be None + initially. + """ + + method: Callable[..., Any] + requiresData: bool + _assertion: Callable[[object], None] + + @classmethod + def _fromImpl( + cls: type[MethodOutput[Core]], method: Callable[..., Any], requiresData: bool + ) -> MethodOutput[Core]: + parameter = None + annotation: type[object] = object + + def assertion(data: object) -> None: + """ + No assertion about the data. + """ + + # Do our best to compute the declared signature, so that we caan verify + # it's the right type. We can't always do that. + try: + sig = _liveSignature(method) + except NameError: + ... + # An inner function may refer to type aliases that only appear as + # local variables, and those are just lost here; give up. + else: + if requiresData: + # 0: self, 1: self.__automat_core__, 2: self.__automat_data__ + declaredParams = list(sig.parameters.values()) + if len(declaredParams) >= 3: + parameter = declaredParams[2] + annotation = parameter.annotation + origin = get_origin(annotation) + if origin is not None: + annotation = origin + if hasInterface and isinstance(annotation, InterfaceClass): + + def assertion(data: object) -> None: + assert annotation.providedBy(data), ( + f"expected {parameter} to provide {annotation} " + f"but got {type(data)} instead" + ) + + else: + + def assertion(data: object) -> None: + assert isinstance(data, annotation), ( + f"expected {parameter} to be {annotation} " + f"but got {type(data)} instead" + ) + + return cls(method, requiresData, assertion) + + @property + def name(self) -> str: + return f"{self.method.__name__}" + + def __call__( + self, + machine: InputImplementer[InputProtocol, Core], + dataAtStart: Data, + /, + *args: object, + **kwargs: object, + ) -> object: + extraArgs = [machine, machine.__automat_core__] + if self.requiresData: + self._assertion(dataAtStart) + extraArgs += [dataAtStart] + # if anything is invoked reentrantly here, then we can't possibly have + # set __automat_data__ and the data argument to the reentrant method + # will be wrong. we *need* to split out the construction / state-enter + # hook, because it needs to run separately. + return self.method(*extraArgs, *args, **kwargs) + + +@dataclass(frozen=True) +class DataOutput(Generic[Data]): + """ + Construct an output for the given data objects. + """ + + dataFactory: Callable[..., Data] + + @property + def name(self) -> str: + return f"data:{self.dataFactory.__name__}" + + def __call__( + realself, + self: InputImplementer[InputProtocol, Core], + dataAtStart: object, + *args: object, + **kwargs: object, + ) -> Data: + newData = realself.dataFactory(self, self.__automat_core__, *args, **kwargs) + self.__automat_data__ = newData + return newData + + +INVALID_WHILE_DESERIALIZING: TypedState[Any, Any] = TypedState( + "automat:invalid-while-deserializing", + None, # type:ignore[arg-type] +) + + +@dataclass(frozen=True) +class TypeMachine(Generic[InputProtocol, Core]): + """ + A L{TypeMachine} is a factory for instances of C{InputProtocol}. + """ + + __automat_type__: type[InputImplementer[InputProtocol, Core]] + __automat_automaton__: Automaton[ + TypedState[InputProtocol, Core] | TypedDataState[InputProtocol, Core, Any, ...], + str, + SomeOutput, + ] + + @overload + def __call__(self, core: Core) -> InputProtocol: ... + @overload + def __call__( + self, core: Core, state: TypedState[InputProtocol, Core] + ) -> InputProtocol: ... + @overload + def __call__( + self, + core: Core, + state: TypedDataState[InputProtocol, Core, OtherData, ...], + dataFactory: Callable[[InputProtocol, Core], OtherData], + ) -> InputProtocol: ... + + def __call__( + self, + core: Core, + state: ( + TypedState[InputProtocol, Core] + | TypedDataState[InputProtocol, Core, OtherData, ...] + | None + ) = None, + dataFactory: Callable[[InputProtocol, Core], OtherData] | None = None, + ) -> InputProtocol: + """ + Construct an instance of C{InputProtocol} from an instance of the + C{Core} protocol. + """ + if state is None: + state = initial = self.__automat_automaton__.initialState + elif isinstance(state, TypedDataState): + assert dataFactory is not None, "data state requires a data factory" + # Ensure that the machine is in a state with *no* transitions while + # we are doing the initial construction of its state-specific data. + initial = INVALID_WHILE_DESERIALIZING + else: + initial = state + + internals: InputImplementer[InputProtocol, Core] = self.__automat_type__( + core, txnr := Transitioner(self.__automat_automaton__, initial) + ) + result: InputProtocol = internals # type:ignore[assignment] + + if dataFactory is not None: + internals.__automat_data__ = dataFactory(result, core) + txnr._state = state + return result + + def asDigraph(self) -> Digraph: + from ._visualize import makeDigraph + + return makeDigraph( + self.__automat_automaton__, + stateAsString=lambda state: state.name, + inputAsString=lambda input: input, + outputAsString=lambda output: output.name, + ) + + +@dataclass(eq=False) +class TypeMachineBuilder(Generic[InputProtocol, Core]): + """ + The main entry-point into Automat, used to construct a factory for + instances of C{InputProtocol} that take an instance of C{Core}. + + Describe the machine with L{TypeMachineBuilder.state} L{.upon + <automat._typed.TypedState.upon>} L{.to + <automat._typed.UponFromNo.to>}, then build it with + L{TypeMachineBuilder.build}, like so:: + + from typing import Protocol + class Inputs(Protocol): + def method(self) -> None: ... + class Core: ... + + from automat import TypeMachineBuilder + builder = TypeMachineBuilder(Inputs, Core) + state = builder.state("state") + state.upon(Inputs.method).loop().returns(None) + Machine = builder.build() + + machine = Machine(Core()) + machine.method() + """ + + # Public constructor parameters. + inputProtocol: ProtocolAtRuntime[InputProtocol] + coreType: type[Core] + + # Internal state, not in the constructor. + _automaton: Automaton[ + TypedState[InputProtocol, Core] | TypedDataState[InputProtocol, Core, Any, ...], + str, + SomeOutput, + ] = field(default_factory=Automaton, repr=False, init=False) + _initial: bool = field(default=True, init=False) + _registrars: list[TransitionRegistrar[..., ..., Any]] = field( + default_factory=list, init=False + ) + _built: bool = field(default=False, init=False) + + @overload + def state(self, name: str) -> TypedState[InputProtocol, Core]: ... + @overload + def state( + self, + name: str, + dataFactory: Callable[Concatenate[InputProtocol, Core, P], Data], + ) -> TypedDataState[InputProtocol, Core, Data, P]: ... + def state( + self, + name: str, + dataFactory: Callable[Concatenate[InputProtocol, Core, P], Data] | None = None, + ) -> TypedState[InputProtocol, Core] | TypedDataState[InputProtocol, Core, Data, P]: + """ + Construct a state. + """ + if self._built: + raise AlreadyBuiltError( + "Cannot add states to an already-built state machine." + ) + if dataFactory is None: + state = TypedState(name, self) + if self._initial: + self._initial = False + self._automaton.initialState = state + return state + else: + assert not self._initial, "initial state cannot require state-specific data" + return TypedDataState(name, self, dataFactory) + + def build(self) -> TypeMachine[InputProtocol, Core]: + """ + Create a L{TypeMachine}, and prevent further modification to the state + machine being built. + """ + # incompleteness check + if self._built: + raise AlreadyBuiltError("Cannot build a state machine twice.") + self._built = True + + for registrar in self._registrars: + registrar._checkComplete() + + # We were only hanging on to these for error-checking purposes, so we + # can drop them now. + del self._registrars[:] + + runtimeType: type[InputImplementer[InputProtocol, Core]] = type( + f"Typed<{runtime_name(self.inputProtocol)}>", + tuple([InputImplementer]), + { + method_name: implementMethod(getattr(self.inputProtocol, method_name)) + for method_name in actuallyDefinedProtocolMethods(self.inputProtocol) + }, + ) + + return TypeMachine(runtimeType, self._automaton) + + def _checkMembership(self, input: Callable[..., object]) -> None: + """ + Ensure that ``input`` is a valid member function of the input protocol, + not just a function that happens to take the right first argument. + """ + if (checked := getattr(self.inputProtocol, input.__name__, None)) is not input: + raise ValueError( + f"{input.__qualname__} is not a member of {self.inputProtocol.__module__}.{self.inputProtocol.__name__}" + ) diff --git a/contrib/python/Automat/py3/automat/_visualize.py b/contrib/python/Automat/py3/automat/_visualize.py index 7a9c8c6eb5..a2b35e5726 100644 --- a/contrib/python/Automat/py3/automat/_visualize.py +++ b/contrib/python/Automat/py3/automat/_visualize.py @@ -1,56 +1,66 @@ -from __future__ import print_function +from __future__ import annotations + import argparse import sys +from functools import wraps +from typing import Callable, Iterator import graphviz +from ._core import Automaton, Input, Output, State from ._discover import findMachines +from ._methodical import MethodicalMachine +from ._typed import TypeMachine, InputProtocol, Core -def _gvquote(s): - return '"{}"'.format(s.replace('"', r'\"')) +def _gvquote(s: str) -> str: + return '"{}"'.format(s.replace('"', r"\"")) -def _gvhtml(s): - return '<{}>'.format(s) +def _gvhtml(s: str) -> str: + return "<{}>".format(s) -def elementMaker(name, *children, **attrs): +def elementMaker(name: str, *children: str, **attrs: str) -> str: """ Construct a string from the HTML element description. """ - formattedAttrs = ' '.join('{}={}'.format(key, _gvquote(str(value))) - for key, value in sorted(attrs.items())) - formattedChildren = ''.join(children) - return u'<{name} {attrs}>{children}</{name}>'.format( - name=name, - attrs=formattedAttrs, - children=formattedChildren) - - -def tableMaker(inputLabel, outputLabels, port, _E=elementMaker): + formattedAttrs = " ".join( + "{}={}".format(key, _gvquote(str(value))) + for key, value in sorted(attrs.items()) + ) + formattedChildren = "".join(children) + return "<{name} {attrs}>{children}</{name}>".format( + name=name, attrs=formattedAttrs, children=formattedChildren + ) + + +def tableMaker( + inputLabel: str, + outputLabels: list[str], + port: str, + _E: Callable[..., str] = elementMaker, +) -> str: """ Construct an HTML table to label a state transition. """ colspan = {} if outputLabels: - colspan['colspan'] = str(len(outputLabels)) + colspan["colspan"] = str(len(outputLabels)) - inputLabelCell = _E("td", - _E("font", - inputLabel, - face="menlo-italic"), - color="purple", - port=port, - **colspan) + inputLabelCell = _E( + "td", + _E("font", inputLabel, face="menlo-italic"), + color="purple", + port=port, + **colspan, + ) pointSize = {"point-size": "9"} - outputLabelCells = [_E("td", - _E("font", - outputLabel, - **pointSize), - color="pink") - for outputLabel in outputLabels] + outputLabelCells = [ + _E("td", _E("font", outputLabel, **pointSize), color="pink") + for outputLabel in outputLabels + ] rows = [_E("tr", inputLabelCell)] @@ -60,16 +70,33 @@ def tableMaker(inputLabel, outputLabels, port, _E=elementMaker): return _E("table", *rows) -def makeDigraph(automaton, inputAsString=repr, - outputAsString=repr, - stateAsString=repr): +def escapify(x: Callable[[State], str]) -> Callable[[State], str]: + @wraps(x) + def impl(t: State) -> str: + return x(t).replace("<", "<").replace(">", ">") + + return impl + + +def makeDigraph( + automaton: Automaton[State, Input, Output], + inputAsString: Callable[[Input], str] = repr, + outputAsString: Callable[[Output], str] = repr, + stateAsString: Callable[[State], str] = repr, +) -> graphviz.Digraph: """ Produce a L{graphviz.Digraph} object from an automaton. """ - digraph = graphviz.Digraph(graph_attr={'pack': 'true', - 'dpi': '100'}, - node_attr={'fontname': 'Menlo'}, - edge_attr={'fontname': 'Menlo'}) + + inputAsString = escapify(inputAsString) + outputAsString = escapify(outputAsString) + stateAsString = escapify(stateAsString) + + digraph = graphviz.Digraph( + graph_attr={"pack": "true", "dpi": "100"}, + node_attr={"fontname": "Menlo"}, + edge_attr={"fontname": "Menlo"}, + ) for state in automaton.states(): if state is automaton.initialState: @@ -78,38 +105,47 @@ def makeDigraph(automaton, inputAsString=repr, else: stateShape = "" fontName = "Menlo" - digraph.node(stateAsString(state), - fontame=fontName, - shape="ellipse", - style=stateShape, - color="blue") + digraph.node( + stateAsString(state), + fontame=fontName, + shape="ellipse", + style=stateShape, + color="blue", + ) for n, eachTransition in enumerate(automaton.allTransitions()): inState, inputSymbol, outState, outputSymbols = eachTransition thisTransition = "t{}".format(n) inputLabel = inputAsString(inputSymbol) port = "tableport" - table = tableMaker(inputLabel, [outputAsString(outputSymbol) - for outputSymbol in outputSymbols], - port=port) + table = tableMaker( + inputLabel, + [outputAsString(outputSymbol) for outputSymbol in outputSymbols], + port=port, + ) - digraph.node(thisTransition, - label=_gvhtml(table), margin="0.2", shape="none") + digraph.node(thisTransition, label=_gvhtml(table), margin="0.2", shape="none") - digraph.edge(stateAsString(inState), - '{}:{}:w'.format(thisTransition, port), - arrowhead="none") - digraph.edge('{}:{}:e'.format(thisTransition, port), - stateAsString(outState)) + digraph.edge( + stateAsString(inState), + "{}:{}:w".format(thisTransition, port), + arrowhead="none", + ) + digraph.edge("{}:{}:e".format(thisTransition, port), stateAsString(outState)) return digraph -def tool(_progname=sys.argv[0], - _argv=sys.argv[1:], - _syspath=sys.path, - _findMachines=findMachines, - _print=print): +def tool( + _progname: str = sys.argv[0], + _argv: list[str] = sys.argv[1:], + _syspath: list[str] = sys.path, + _findMachines: Callable[ + [str], + Iterator[tuple[str, MethodicalMachine | TypeMachine[InputProtocol, Core]]], + ] = findMachines, + _print: Callable[..., None] = print, +) -> None: """ Entry point for command line utility. """ @@ -122,59 +158,71 @@ def tool(_progname=sys.argv[0], http://www.graphviz.org for more information. """ if _syspath[0]: - _syspath.insert(0, '') + _syspath.insert(0, "") argumentParser = argparse.ArgumentParser( - prog=_progname, - description=DESCRIPTION, - epilog=EPILOG) - argumentParser.add_argument('fqpn', - help="A Fully Qualified Path name" - " representing where to find machines.") - argumentParser.add_argument('--quiet', '-q', - help="suppress output", - default=False, - action="store_true") - argumentParser.add_argument('--dot-directory', '-d', - help="Where to write out .dot files.", - default=".automat_visualize") - argumentParser.add_argument('--image-directory', '-i', - help="Where to write out image files.", - default=".automat_visualize") - argumentParser.add_argument('--image-type', '-t', - help="The image format.", - choices=graphviz.FORMATS, - default='png') - argumentParser.add_argument('--view', '-v', - help="View rendered graphs with" - " default image viewer", - default=False, - action="store_true") + prog=_progname, description=DESCRIPTION, epilog=EPILOG + ) + argumentParser.add_argument( + "fqpn", + help="A Fully Qualified Path name" " representing where to find machines.", + ) + argumentParser.add_argument( + "--quiet", "-q", help="suppress output", default=False, action="store_true" + ) + argumentParser.add_argument( + "--dot-directory", + "-d", + help="Where to write out .dot files.", + default=".automat_visualize", + ) + argumentParser.add_argument( + "--image-directory", + "-i", + help="Where to write out image files.", + default=".automat_visualize", + ) + argumentParser.add_argument( + "--image-type", + "-t", + help="The image format.", + choices=graphviz.FORMATS, + default="png", + ) + argumentParser.add_argument( + "--view", + "-v", + help="View rendered graphs with" " default image viewer", + default=False, + action="store_true", + ) args = argumentParser.parse_args(_argv) - explicitlySaveDot = (args.dot_directory - and (not args.image_directory - or args.image_directory != args.dot_directory)) + explicitlySaveDot = args.dot_directory and ( + not args.image_directory or args.image_directory != args.dot_directory + ) if args.quiet: + def _print(*args): pass for fqpn, machine in _findMachines(args.fqpn): - _print(fqpn, '...discovered') + _print(fqpn, "...discovered") digraph = machine.asDigraph() if explicitlySaveDot: - digraph.save(filename="{}.dot".format(fqpn), - directory=args.dot_directory) + digraph.save(filename="{}.dot".format(fqpn), directory=args.dot_directory) _print(fqpn, "...wrote dot into", args.dot_directory) if args.image_directory: deleteDot = not args.dot_directory or explicitlySaveDot digraph.format = args.image_type - digraph.render(filename="{}.dot".format(fqpn), - directory=args.image_directory, - view=args.view, - cleanup=deleteDot) + digraph.render( + filename="{}.dot".format(fqpn), + directory=args.image_directory, + view=args.view, + cleanup=deleteDot, + ) if deleteDot: msg = "...wrote image into" else: diff --git a/contrib/python/Automat/py3/automat/py.typed b/contrib/python/Automat/py3/automat/py.typed new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/contrib/python/Automat/py3/automat/py.typed diff --git a/contrib/python/Automat/py3/ya.make b/contrib/python/Automat/py3/ya.make index 03a66263d7..89ecf81193 100644 --- a/contrib/python/Automat/py3/ya.make +++ b/contrib/python/Automat/py3/ya.make @@ -2,15 +2,10 @@ PY3_LIBRARY() -VERSION(22.10.0) +VERSION(24.8.1) LICENSE(MIT) -PEERDIR( - contrib/python/attrs - contrib/python/six -) - NO_LINT() NO_CHECK_IMPORTS( @@ -25,6 +20,8 @@ PY_SRCS( automat/_discover.py automat/_introspection.py automat/_methodical.py + automat/_runtimeproto.py + automat/_typed.py automat/_visualize.py ) @@ -33,6 +30,7 @@ RESOURCE_FILES( .dist-info/METADATA .dist-info/entry_points.txt .dist-info/top_level.txt + automat/py.typed ) END() diff --git a/contrib/python/cachetools/py3/.dist-info/METADATA b/contrib/python/cachetools/py3/.dist-info/METADATA index 6b6a201551..b3ff29d899 100644 --- a/contrib/python/cachetools/py3/.dist-info/METADATA +++ b/contrib/python/cachetools/py3/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: cachetools -Version: 5.4.0 +Version: 5.5.0 Summary: Extensible memoizing collections and decorators Home-page: https://github.com/tkem/cachetools/ Author: Thomas Kemmer diff --git a/contrib/python/cachetools/py3/cachetools/__init__.py b/contrib/python/cachetools/py3/cachetools/__init__.py index 5a9a042cbd..2d2e2cf4ae 100644 --- a/contrib/python/cachetools/py3/cachetools/__init__.py +++ b/contrib/python/cachetools/py3/cachetools/__init__.py @@ -13,7 +13,7 @@ __all__ = ( "cachedmethod", ) -__version__ = "5.4.0" +__version__ = "5.5.0" import collections import collections.abc @@ -26,7 +26,6 @@ from . import keys class _DefaultSize: - __slots__ = () def __getitem__(self, _): @@ -378,7 +377,6 @@ class TTLCache(_TimedCache): """LRU Cache implementation with per-item time-to-live (TTL) value.""" class _Link: - __slots__ = ("key", "expires", "next", "prev") def __init__(self, key=None, expires=None): @@ -469,19 +467,26 @@ class TTLCache(_TimedCache): return self.__ttl def expire(self, time=None): - """Remove expired items from the cache.""" + """Remove expired items from the cache and return an iterable of the + expired `(key, value)` pairs. + + """ if time is None: time = self.timer() root = self.__root curr = root.next links = self.__links + expired = [] cache_delitem = Cache.__delitem__ + cache_getitem = Cache.__getitem__ while curr is not root and not (time < curr.expires): + expired.append((curr.key, cache_getitem(self, curr.key))) cache_delitem(self, curr.key) del links[curr.key] next = curr.next curr.unlink() curr = next + return expired def popitem(self): """Remove and return the `(key, value)` pair least recently used that @@ -508,7 +513,6 @@ class TLRUCache(_TimedCache): @functools.total_ordering class _Item: - __slots__ = ("key", "expires", "removed") def __init__(self, key=None, expires=None): @@ -583,7 +587,10 @@ class TLRUCache(_TimedCache): return self.__ttu def expire(self, time=None): - """Remove expired items from the cache.""" + """Remove expired items from the cache and return an iterable of the + expired `(key, value)` pairs. + + """ if time is None: time = self.timer() items = self.__items @@ -592,12 +599,16 @@ class TLRUCache(_TimedCache): if len(order) > len(items) * 2: self.__order = order = [item for item in order if not item.removed] heapq.heapify(order) + expired = [] cache_delitem = Cache.__delitem__ + cache_getitem = Cache.__getitem__ while order and (order[0].removed or not (time < order[0].expires)): item = heapq.heappop(order) if not item.removed: + expired.append((item.key, cache_getitem(self, item.key))) cache_delitem(self, item.key) del items[item.key] + return expired def popitem(self): """Remove and return the `(key, value)` pair least recently used that diff --git a/contrib/python/cachetools/py3/ya.make b/contrib/python/cachetools/py3/ya.make index ac118e7e26..16c48a0f31 100644 --- a/contrib/python/cachetools/py3/ya.make +++ b/contrib/python/cachetools/py3/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(5.4.0) +VERSION(5.5.0) LICENSE(MIT) diff --git a/contrib/python/google-auth/py3/.dist-info/METADATA b/contrib/python/google-auth/py3/.dist-info/METADATA index cdbc683396..26b8a4974a 100644 --- a/contrib/python/google-auth/py3/.dist-info/METADATA +++ b/contrib/python/google-auth/py3/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: google-auth -Version: 2.33.0 +Version: 2.34.0 Summary: Google Authentication Library Home-page: https://github.com/googleapis/google-auth-library-python Author: Google Cloud Platform @@ -31,8 +31,8 @@ Provides-Extra: aiohttp Requires-Dist: aiohttp <4.0.0.dev0,>=3.6.2 ; extra == 'aiohttp' Requires-Dist: requests <3.0.0.dev0,>=2.20.0 ; extra == 'aiohttp' Provides-Extra: enterprise_cert -Requires-Dist: cryptography ==36.0.2 ; extra == 'enterprise_cert' -Requires-Dist: pyopenssl ==22.0.0 ; extra == 'enterprise_cert' +Requires-Dist: cryptography ; extra == 'enterprise_cert' +Requires-Dist: pyopenssl ; extra == 'enterprise_cert' Provides-Extra: pyopenssl Requires-Dist: pyopenssl >=20.0.0 ; extra == 'pyopenssl' Requires-Dist: cryptography >=38.0.3 ; extra == 'pyopenssl' diff --git a/contrib/python/google-auth/py3/google/auth/compute_engine/_metadata.py b/contrib/python/google-auth/py3/google/auth/compute_engine/_metadata.py index 69b7b52458..b66d9f9b37 100644 --- a/contrib/python/google-auth/py3/google/auth/compute_engine/_metadata.py +++ b/contrib/python/google-auth/py3/google/auth/compute_engine/_metadata.py @@ -28,6 +28,7 @@ from google.auth import _helpers from google.auth import environment_vars from google.auth import exceptions from google.auth import metrics +from google.auth import transport from google.auth._exponential_backoff import ExponentialBackoff _LOGGER = logging.getLogger(__name__) @@ -204,7 +205,17 @@ def get( for attempt in backoff: try: response = request(url=url, method="GET", headers=headers_to_use) - break + if response.status in transport.DEFAULT_RETRYABLE_STATUS_CODES: + _LOGGER.warning( + "Compute Engine Metadata server unavailable on " + "attempt %s of %s. Response status: %s", + attempt, + retry_count, + response.status, + ) + continue + else: + break except exceptions.TransportError as e: _LOGGER.warning( diff --git a/contrib/python/google-auth/py3/google/auth/transport/_mtls_helper.py b/contrib/python/google-auth/py3/google/auth/transport/_mtls_helper.py index 6299e2bdea..68568dd603 100644 --- a/contrib/python/google-auth/py3/google/auth/transport/_mtls_helper.py +++ b/contrib/python/google-auth/py3/google/auth/transport/_mtls_helper.py @@ -23,7 +23,7 @@ import subprocess from google.auth import exceptions CONTEXT_AWARE_METADATA_PATH = "~/.secureConnect/context_aware_metadata.json" -_CERTIFICATE_CONFIGURATION_DEFAULT_PATH = "~/.config/gcloud/certificate_config.json" +CERTIFICATE_CONFIGURATION_DEFAULT_PATH = "~/.config/gcloud/certificate_config.json" _CERTIFICATE_CONFIGURATION_ENV = "GOOGLE_API_CERTIFICATE_CONFIG" _CERT_PROVIDER_COMMAND = "cert_provider_command" _CERT_REGEX = re.compile( @@ -48,21 +48,21 @@ _PASSPHRASE_REGEX = re.compile( ) -def _check_dca_metadata_path(metadata_path): - """Checks for context aware metadata. If it exists, returns the absolute path; +def _check_config_path(config_path): + """Checks for config file path. If it exists, returns the absolute path with user expansion; otherwise returns None. Args: - metadata_path (str): context aware metadata path. + config_path (str): The config file path for either context_aware_metadata.json or certificate_config.json for example Returns: str: absolute path if exists and None otherwise. """ - metadata_path = path.expanduser(metadata_path) - if not path.exists(metadata_path): - _LOGGER.debug("%s is not found, skip client SSL authentication.", metadata_path) + config_path = path.expanduser(config_path) + if not path.exists(config_path): + _LOGGER.debug("%s is not found.", config_path) return None - return metadata_path + return config_path def _load_json_file(path): @@ -136,7 +136,7 @@ def _get_cert_config_path(certificate_config_path=None): if env_path is not None and env_path != "": certificate_config_path = env_path else: - certificate_config_path = _CERTIFICATE_CONFIGURATION_DEFAULT_PATH + certificate_config_path = CERTIFICATE_CONFIGURATION_DEFAULT_PATH certificate_config_path = path.expanduser(certificate_config_path) if not path.exists(certificate_config_path): @@ -279,14 +279,22 @@ def _run_cert_provider_command(command, expect_encrypted_key=False): def get_client_ssl_credentials( generate_encrypted_key=False, context_aware_metadata_path=CONTEXT_AWARE_METADATA_PATH, + certificate_config_path=CERTIFICATE_CONFIGURATION_DEFAULT_PATH, ): """Returns the client side certificate, private key and passphrase. + We look for certificates and keys with the following order of priority: + 1. Certificate and key specified by certificate_config.json. + Currently, only X.509 workload certificates are supported. + 2. Certificate and key specified by context aware metadata (i.e. SecureConnect). + Args: generate_encrypted_key (bool): If set to True, encrypted private key and passphrase will be generated; otherwise, unencrypted private key - will be generated and passphrase will be None. + will be generated and passphrase will be None. This option only + affects keys obtained via context_aware_metadata.json. context_aware_metadata_path (str): The context_aware_metadata.json file path. + certificate_config_path (str): The certificate_config.json file path. Returns: Tuple[bool, bytes, bytes, bytes]: @@ -297,7 +305,17 @@ def get_client_ssl_credentials( google.auth.exceptions.ClientCertError: if problems occurs when getting the cert, key and passphrase. """ - metadata_path = _check_dca_metadata_path(context_aware_metadata_path) + + # 1. Check for certificate config json. + cert_config_path = _check_config_path(certificate_config_path) + if cert_config_path: + # Attempt to retrieve X.509 Workload cert and key. + cert, key = _get_workload_cert_and_key(cert_config_path) + if cert and key: + return True, cert, key, None + + # 2. Check for context aware metadata json + metadata_path = _check_config_path(context_aware_metadata_path) if metadata_path: metadata_json = _load_json_file(metadata_path) diff --git a/contrib/python/google-auth/py3/google/auth/transport/grpc.py b/contrib/python/google-auth/py3/google/auth/transport/grpc.py index 9a817976d7..1ebe137957 100644 --- a/contrib/python/google-auth/py3/google/auth/transport/grpc.py +++ b/contrib/python/google-auth/py3/google/auth/transport/grpc.py @@ -302,7 +302,7 @@ class SslCredentials: self._is_mtls = False else: # Load client SSL credentials. - metadata_path = _mtls_helper._check_dca_metadata_path( + metadata_path = _mtls_helper._check_config_path( _mtls_helper.CONTEXT_AWARE_METADATA_PATH ) self._is_mtls = metadata_path is not None diff --git a/contrib/python/google-auth/py3/google/auth/transport/mtls.py b/contrib/python/google-auth/py3/google/auth/transport/mtls.py index c5707617ff..e7a7304f60 100644 --- a/contrib/python/google-auth/py3/google/auth/transport/mtls.py +++ b/contrib/python/google-auth/py3/google/auth/transport/mtls.py @@ -24,10 +24,19 @@ def has_default_client_cert_source(): Returns: bool: indicating if the default client cert source exists. """ - metadata_path = _mtls_helper._check_dca_metadata_path( - _mtls_helper.CONTEXT_AWARE_METADATA_PATH - ) - return metadata_path is not None + if ( + _mtls_helper._check_config_path(_mtls_helper.CONTEXT_AWARE_METADATA_PATH) + is not None + ): + return True + if ( + _mtls_helper._check_config_path( + _mtls_helper.CERTIFICATE_CONFIGURATION_DEFAULT_PATH + ) + is not None + ): + return True + return False def default_client_cert_source(): diff --git a/contrib/python/google-auth/py3/google/auth/version.py b/contrib/python/google-auth/py3/google/auth/version.py index c41f877658..297e18a45f 100644 --- a/contrib/python/google-auth/py3/google/auth/version.py +++ b/contrib/python/google-auth/py3/google/auth/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.33.0" +__version__ = "2.34.0" diff --git a/contrib/python/google-auth/py3/tests/compute_engine/test__metadata.py b/contrib/python/google-auth/py3/tests/compute_engine/test__metadata.py index 352342f150..a06dc4fa19 100644 --- a/contrib/python/google-auth/py3/tests/compute_engine/test__metadata.py +++ b/contrib/python/google-auth/py3/tests/compute_engine/test__metadata.py @@ -433,6 +433,74 @@ def test_get_universe_domain_not_found(): assert universe_domain == "googleapis.com" +def test_get_universe_domain_retryable_error_failure(): + # Test that if the universe domain endpoint returns a retryable error + # we should retry. + # + # In this case, the error persists, and we still fail after retrying. + request = make_request("too many requests", status=http_client.TOO_MANY_REQUESTS) + + with pytest.raises(exceptions.TransportError) as excinfo: + _metadata.get_universe_domain(request) + + assert excinfo.match(r"Compute Engine Metadata server unavailable") + + request.assert_called_with( + method="GET", + url=_metadata._METADATA_ROOT + "universe/universe_domain", + headers=_metadata._METADATA_HEADERS, + ) + assert request.call_count == 5 + + +def test_get_universe_domain_retryable_error_success(): + # Test that if the universe domain endpoint returns a retryable error + # we should retry. + # + # In this case, the error is temporary, and we succeed after retrying. + request_error = make_request( + "too many requests", status=http_client.TOO_MANY_REQUESTS + ) + request_ok = make_request( + "fake_universe_domain", headers={"content-type": "text/plain"} + ) + + class _RequestErrorOnce: + """This class forwards the request parameters to `request_error` once. + + All subsequent calls are forwarded to `request_ok`. + """ + + def __init__(self, request_error, request_ok): + self._request_error = request_error + self._request_ok = request_ok + self._call_index = 0 + + def request(self, *args, **kwargs): + if self._call_index == 0: + self._call_index += 1 + return self._request_error(*args, **kwargs) + + return self._request_ok(*args, **kwargs) + + request = _RequestErrorOnce(request_error, request_ok).request + + universe_domain = _metadata.get_universe_domain(request) + + request_error.assert_called_once_with( + method="GET", + url=_metadata._METADATA_ROOT + "universe/universe_domain", + headers=_metadata._METADATA_HEADERS, + ) + request_ok.assert_called_once_with( + method="GET", + url=_metadata._METADATA_ROOT + "universe/universe_domain", + headers=_metadata._METADATA_HEADERS, + ) + + assert universe_domain == "fake_universe_domain" + + def test_get_universe_domain_other_error(): # Test that if the universe domain endpoint returns an error other than 404 # we should throw the error diff --git a/contrib/python/google-auth/py3/tests/transport/test__mtls_helper.py b/contrib/python/google-auth/py3/tests/transport/test__mtls_helper.py index b195616dd5..f6e20b726a 100644 --- a/contrib/python/google-auth/py3/tests/transport/test__mtls_helper.py +++ b/contrib/python/google-auth/py3/tests/transport/test__mtls_helper.py @@ -111,15 +111,15 @@ class TestCertAndKeyRegex(object): ) -class TestCheckaMetadataPath(object): +class TestCheckConfigPath(object): def test_success(self): metadata_path = os.path.join(pytest.data_dir, "context_aware_metadata.json") - returned_path = _mtls_helper._check_dca_metadata_path(metadata_path) + returned_path = _mtls_helper._check_config_path(metadata_path) assert returned_path is not None def test_failure(self): metadata_path = os.path.join(pytest.data_dir, "not_exists.json") - returned_path = _mtls_helper._check_dca_metadata_path(metadata_path) + returned_path = _mtls_helper._check_config_path(metadata_path) assert returned_path is None @@ -275,21 +275,24 @@ class TestRunCertProviderCommand(object): class TestGetClientSslCredentials(object): @mock.patch( - "google.auth.transport._mtls_helper._run_cert_provider_command", autospec=True + "google.auth.transport._mtls_helper._get_workload_cert_and_key", autospec=True ) - @mock.patch("google.auth.transport._mtls_helper._load_json_file", autospec=True) @mock.patch( - "google.auth.transport._mtls_helper._check_dca_metadata_path", autospec=True + "google.auth.transport._mtls_helper._run_cert_provider_command", autospec=True ) - def test_success( + @mock.patch("google.auth.transport._mtls_helper._load_json_file", autospec=True) + @mock.patch("google.auth.transport._mtls_helper._check_config_path", autospec=True) + def test_success_with_context_aware_metadata( self, - mock_check_dca_metadata_path, + mock_check_config_path, mock_load_json_file, mock_run_cert_provider_command, + mock_get_workload_cert_and_key, ): - mock_check_dca_metadata_path.return_value = True + mock_check_config_path.return_value = "/path/to/config" mock_load_json_file.return_value = {"cert_provider_command": ["command"]} mock_run_cert_provider_command.return_value = (b"cert", b"key", None) + mock_get_workload_cert_and_key.return_value = (None, None) has_cert, cert, key, passphrase = _mtls_helper.get_client_ssl_credentials() assert has_cert assert cert == b"cert" @@ -297,10 +300,42 @@ class TestGetClientSslCredentials(object): assert passphrase is None @mock.patch( - "google.auth.transport._mtls_helper._check_dca_metadata_path", autospec=True + "google.auth.transport._mtls_helper._read_cert_and_key_files", autospec=True ) - def test_success_without_metadata(self, mock_check_dca_metadata_path): - mock_check_dca_metadata_path.return_value = False + @mock.patch( + "google.auth.transport._mtls_helper._get_cert_config_path", autospec=True + ) + @mock.patch("google.auth.transport._mtls_helper._load_json_file", autospec=True) + @mock.patch("google.auth.transport._mtls_helper._check_config_path", autospec=True) + def test_success_with_certificate_config( + self, + mock_check_config_path, + mock_load_json_file, + mock_get_cert_config_path, + mock_read_cert_and_key_files, + ): + cert_config_path = "/path/to/config" + mock_check_config_path.return_value = cert_config_path + mock_load_json_file.return_value = { + "cert_configs": { + "workload": {"cert_path": "cert/path", "key_path": "key/path"} + } + } + mock_get_cert_config_path.return_value = cert_config_path + mock_read_cert_and_key_files.return_value = ( + pytest.public_cert_bytes, + pytest.private_key_bytes, + ) + + has_cert, cert, key, passphrase = _mtls_helper.get_client_ssl_credentials() + assert has_cert + assert cert == pytest.public_cert_bytes + assert key == pytest.private_key_bytes + assert passphrase is None + + @mock.patch("google.auth.transport._mtls_helper._check_config_path", autospec=True) + def test_success_without_metadata(self, mock_check_config_path): + mock_check_config_path.return_value = False has_cert, cert, key, passphrase = _mtls_helper.get_client_ssl_credentials() assert not has_cert assert cert is None @@ -308,21 +343,24 @@ class TestGetClientSslCredentials(object): assert passphrase is None @mock.patch( - "google.auth.transport._mtls_helper._run_cert_provider_command", autospec=True + "google.auth.transport._mtls_helper._get_workload_cert_and_key", autospec=True ) - @mock.patch("google.auth.transport._mtls_helper._load_json_file", autospec=True) @mock.patch( - "google.auth.transport._mtls_helper._check_dca_metadata_path", autospec=True + "google.auth.transport._mtls_helper._run_cert_provider_command", autospec=True ) + @mock.patch("google.auth.transport._mtls_helper._load_json_file", autospec=True) + @mock.patch("google.auth.transport._mtls_helper._check_config_path", autospec=True) def test_success_with_encrypted_key( self, - mock_check_dca_metadata_path, + mock_check_config_path, mock_load_json_file, mock_run_cert_provider_command, + mock_get_workload_cert_and_key, ): - mock_check_dca_metadata_path.return_value = True + mock_check_config_path.return_value = "/path/to/config" mock_load_json_file.return_value = {"cert_provider_command": ["command"]} mock_run_cert_provider_command.return_value = (b"cert", b"key", b"passphrase") + mock_get_workload_cert_and_key.return_value = (None, None) has_cert, cert, key, passphrase = _mtls_helper.get_client_ssl_credentials( generate_encrypted_key=True ) @@ -334,15 +372,20 @@ class TestGetClientSslCredentials(object): ["command", "--with_passphrase"], expect_encrypted_key=True ) - @mock.patch("google.auth.transport._mtls_helper._load_json_file", autospec=True) @mock.patch( - "google.auth.transport._mtls_helper._check_dca_metadata_path", autospec=True + "google.auth.transport._mtls_helper._get_workload_cert_and_key", autospec=True ) + @mock.patch("google.auth.transport._mtls_helper._load_json_file", autospec=True) + @mock.patch("google.auth.transport._mtls_helper._check_config_path", autospec=True) def test_missing_cert_command( - self, mock_check_dca_metadata_path, mock_load_json_file + self, + mock_check_config_path, + mock_load_json_file, + mock_get_workload_cert_and_key, ): - mock_check_dca_metadata_path.return_value = True + mock_check_config_path.return_value = "/path/to/config" mock_load_json_file.return_value = {} + mock_get_workload_cert_and_key.return_value = (None, None) with pytest.raises(exceptions.ClientCertError): _mtls_helper.get_client_ssl_credentials() @@ -350,17 +393,15 @@ class TestGetClientSslCredentials(object): "google.auth.transport._mtls_helper._run_cert_provider_command", autospec=True ) @mock.patch("google.auth.transport._mtls_helper._load_json_file", autospec=True) - @mock.patch( - "google.auth.transport._mtls_helper._check_dca_metadata_path", autospec=True - ) + @mock.patch("google.auth.transport._mtls_helper._check_config_path", autospec=True) def test_customize_context_aware_metadata_path( self, - mock_check_dca_metadata_path, + mock_check_config_path, mock_load_json_file, mock_run_cert_provider_command, ): context_aware_metadata_path = "/path/to/metata/data" - mock_check_dca_metadata_path.return_value = context_aware_metadata_path + mock_check_config_path.return_value = context_aware_metadata_path mock_load_json_file.return_value = {"cert_provider_command": ["command"]} mock_run_cert_provider_command.return_value = (b"cert", b"key", None) @@ -372,7 +413,7 @@ class TestGetClientSslCredentials(object): assert cert == b"cert" assert key == b"key" assert passphrase is None - mock_check_dca_metadata_path.assert_called_with(context_aware_metadata_path) + mock_check_config_path.assert_called_with(context_aware_metadata_path) mock_load_json_file.assert_called_with(context_aware_metadata_path) @@ -520,7 +561,7 @@ class TestGetCertConfigPath(object): mock_path_exists.return_value = True returned_path = _mtls_helper._get_cert_config_path() expected_path = os.path.expanduser( - _mtls_helper._CERTIFICATE_CONFIGURATION_DEFAULT_PATH + _mtls_helper.CERTIFICATE_CONFIGURATION_DEFAULT_PATH ) assert returned_path == expected_path diff --git a/contrib/python/google-auth/py3/tests/transport/test_grpc.py b/contrib/python/google-auth/py3/tests/transport/test_grpc.py index 9badb59b28..80b24e86d4 100644 --- a/contrib/python/google-auth/py3/tests/transport/test_grpc.py +++ b/contrib/python/google-auth/py3/tests/transport/test_grpc.py @@ -143,12 +143,10 @@ class TestAuthMetadataPlugin(object): @mock.patch("grpc.secure_channel", autospec=True) class TestSecureAuthorizedChannel(object): @mock.patch("google.auth.transport._mtls_helper._load_json_file", autospec=True) - @mock.patch( - "google.auth.transport._mtls_helper._check_dca_metadata_path", autospec=True - ) + @mock.patch("google.auth.transport._mtls_helper._check_config_path", autospec=True) def test_secure_authorized_channel_adc( self, - check_dca_metadata_path, + check_config_path, load_json_file, secure_channel, ssl_channel_credentials, @@ -162,7 +160,7 @@ class TestSecureAuthorizedChannel(object): # Mock the context aware metadata and client cert/key so mTLS SSL channel # will be used. - check_dca_metadata_path.return_value = METADATA_PATH + check_config_path.return_value = METADATA_PATH load_json_file.return_value = {"cert_provider_command": ["some command"]} get_client_ssl_credentials.return_value = ( True, @@ -332,12 +330,10 @@ class TestSecureAuthorizedChannel(object): ) @mock.patch("google.auth.transport._mtls_helper._load_json_file", autospec=True) - @mock.patch( - "google.auth.transport._mtls_helper._check_dca_metadata_path", autospec=True - ) + @mock.patch("google.auth.transport._mtls_helper._check_config_path", autospec=True) def test_secure_authorized_channel_with_client_cert_callback_failure( self, - check_dca_metadata_path, + check_config_path, load_json_file, secure_channel, ssl_channel_credentials, @@ -401,19 +397,17 @@ class TestSecureAuthorizedChannel(object): "google.auth.transport._mtls_helper.get_client_ssl_credentials", autospec=True ) @mock.patch("google.auth.transport._mtls_helper._load_json_file", autospec=True) -@mock.patch( - "google.auth.transport._mtls_helper._check_dca_metadata_path", autospec=True -) +@mock.patch("google.auth.transport._mtls_helper._check_config_path", autospec=True) class TestSslCredentials(object): def test_no_context_aware_metadata( self, - mock_check_dca_metadata_path, + mock_check_config_path, mock_load_json_file, mock_get_client_ssl_credentials, mock_ssl_channel_credentials, ): # Mock that the metadata file doesn't exist. - mock_check_dca_metadata_path.return_value = None + mock_check_config_path.return_value = None with mock.patch.dict( os.environ, {environment_vars.GOOGLE_API_USE_CLIENT_CERTIFICATE: "true"} @@ -430,12 +424,12 @@ class TestSslCredentials(object): def test_get_client_ssl_credentials_failure( self, - mock_check_dca_metadata_path, + mock_check_config_path, mock_load_json_file, mock_get_client_ssl_credentials, mock_ssl_channel_credentials, ): - mock_check_dca_metadata_path.return_value = METADATA_PATH + mock_check_config_path.return_value = METADATA_PATH mock_load_json_file.return_value = {"cert_provider_command": ["some command"]} # Mock that client cert and key are not loaded and exception is raised. @@ -449,12 +443,12 @@ class TestSslCredentials(object): def test_get_client_ssl_credentials_success( self, - mock_check_dca_metadata_path, + mock_check_config_path, mock_load_json_file, mock_get_client_ssl_credentials, mock_ssl_channel_credentials, ): - mock_check_dca_metadata_path.return_value = METADATA_PATH + mock_check_config_path.return_value = METADATA_PATH mock_load_json_file.return_value = {"cert_provider_command": ["some command"]} mock_get_client_ssl_credentials.return_value = ( True, @@ -477,7 +471,7 @@ class TestSslCredentials(object): def test_get_client_ssl_credentials_without_client_cert_env( self, - mock_check_dca_metadata_path, + mock_check_config_path, mock_load_json_file, mock_get_client_ssl_credentials, mock_ssl_channel_credentials, @@ -487,7 +481,7 @@ class TestSslCredentials(object): assert ssl_credentials.ssl_credentials is not None assert not ssl_credentials.is_mtls - mock_check_dca_metadata_path.assert_not_called() + mock_check_config_path.assert_not_called() mock_load_json_file.assert_not_called() mock_get_client_ssl_credentials.assert_not_called() mock_ssl_channel_credentials.assert_called_once() diff --git a/contrib/python/google-auth/py3/tests/transport/test_mtls.py b/contrib/python/google-auth/py3/tests/transport/test_mtls.py index b62063e479..ea549ae142 100644 --- a/contrib/python/google-auth/py3/tests/transport/test_mtls.py +++ b/contrib/python/google-auth/py3/tests/transport/test_mtls.py @@ -16,17 +16,30 @@ import mock import pytest # type: ignore from google.auth import exceptions +from google.auth.transport import _mtls_helper from google.auth.transport import mtls -@mock.patch( - "google.auth.transport._mtls_helper._check_dca_metadata_path", autospec=True -) -def test_has_default_client_cert_source(check_dca_metadata_path): - check_dca_metadata_path.return_value = mock.Mock() +@mock.patch("google.auth.transport._mtls_helper._check_config_path", autospec=True) +def test_has_default_client_cert_source(check_config_path): + def return_path_for_metadata(path): + return mock.Mock() if path == _mtls_helper.CONTEXT_AWARE_METADATA_PATH else None + + check_config_path.side_effect = return_path_for_metadata + assert mtls.has_default_client_cert_source() + + def return_path_for_cert_config(path): + return ( + mock.Mock() + if path == _mtls_helper.CERTIFICATE_CONFIGURATION_DEFAULT_PATH + else None + ) + + check_config_path.side_effect = return_path_for_cert_config assert mtls.has_default_client_cert_source() - check_dca_metadata_path.return_value = None + check_config_path.side_effect = None + check_config_path.return_value = None assert not mtls.has_default_client_cert_source() diff --git a/contrib/python/google-auth/py3/ya.make b/contrib/python/google-auth/py3/ya.make index caefae5db6..4f5c4e4ad8 100644 --- a/contrib/python/google-auth/py3/ya.make +++ b/contrib/python/google-auth/py3/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(2.33.0) +VERSION(2.34.0) LICENSE(Apache-2.0) diff --git a/contrib/python/importlib-metadata/py3/.dist-info/METADATA b/contrib/python/importlib-metadata/py3/.dist-info/METADATA index edd2ab879a..d9b8f53e25 100644 --- a/contrib/python/importlib-metadata/py3/.dist-info/METADATA +++ b/contrib/python/importlib-metadata/py3/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: importlib_metadata -Version: 8.2.0 +Version: 8.4.0 Summary: Read metadata from Python packages Author-email: "Jason R. Coombs" <jaraco@jaraco.com> Project-URL: Source, https://github.com/python/importlib_metadata diff --git a/contrib/python/importlib-metadata/py3/importlib_metadata/__init__.py b/contrib/python/importlib-metadata/py3/importlib_metadata/__init__.py index 655bdf80ae..7dc38f73d0 100644 --- a/contrib/python/importlib-metadata/py3/importlib_metadata/__init__.py +++ b/contrib/python/importlib-metadata/py3/importlib_metadata/__init__.py @@ -7,7 +7,6 @@ import sys import json import email import types -import inspect import pathlib import operator import textwrap @@ -232,9 +231,26 @@ class EntryPoint: >>> ep.matches(attr='bong') True """ + self._disallow_dist(params) attrs = (getattr(self, param) for param in params) return all(map(operator.eq, params.values(), attrs)) + @staticmethod + def _disallow_dist(params): + """ + Querying by dist is not allowed (dist objects are not comparable). + >>> EntryPoint(name='fan', value='fav', group='fag').matches(dist='foo') + Traceback (most recent call last): + ... + ValueError: "dist" is not suitable for matching... + """ + if "dist" in params: + raise ValueError( + '"dist" is not suitable for matching. ' + "Instead, use Distribution.entry_points.select() on a " + "located distribution." + ) + def _key(self): return self.name, self.value, self.group @@ -378,6 +394,17 @@ class Distribution(metaclass=abc.ABCMeta): """ Given a path to a file in this distribution, return a SimplePath to it. + + This method is used by callers of ``Distribution.files()`` to + locate files within the distribution. If it's possible for a + Distribution to represent files in the distribution as + ``SimplePath`` objects, it should implement this method + to resolve such objects. + + Some Distribution providers may elect not to resolve SimplePath + objects within the distribution by raising a + NotImplementedError, but consumers of such a Distribution would + be unable to invoke ``Distribution.files()``. """ @classmethod @@ -1136,11 +1163,10 @@ def _get_toplevel_name(name: PackagePath) -> str: >>> _get_toplevel_name(PackagePath('foo.dist-info')) 'foo.dist-info' """ - return _topmost(name) or ( - # python/typeshed#10328 - inspect.getmodulename(name) # type: ignore - or str(name) - ) + # Defer import of inspect for performance (python/cpython#118761) + import inspect + + return _topmost(name) or (inspect.getmodulename(name) or str(name)) def _top_level_inferred(dist): diff --git a/contrib/python/importlib-metadata/py3/ya.make b/contrib/python/importlib-metadata/py3/ya.make index 705317eb97..57e5ec6b3b 100644 --- a/contrib/python/importlib-metadata/py3/ya.make +++ b/contrib/python/importlib-metadata/py3/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(8.2.0) +VERSION(8.4.0) LICENSE(Apache-2.0) diff --git a/contrib/restricted/abseil-cpp-tstring/y_absl/container/internal/btree.h b/contrib/restricted/abseil-cpp-tstring/y_absl/container/internal/btree.h index 64f7a068b1..c30e30149d 100644 --- a/contrib/restricted/abseil-cpp-tstring/y_absl/container/internal/btree.h +++ b/contrib/restricted/abseil-cpp-tstring/y_absl/container/internal/btree.h @@ -436,10 +436,11 @@ struct common_params : common_policy_traits<SlotPolicy> { // This is an integral type large enough to hold as many slots as will fit a // node of TargetNodeSize bytes. + static constexpr bool fit_cond = + kNodeSlotSpace / sizeof(slot_type) > + std::numeric_limits<uint8_t>::max(); using node_count_type = - y_absl::conditional_t<(kNodeSlotSpace / sizeof(slot_type) > - (std::numeric_limits<uint8_t>::max)()), - uint16_t, uint8_t>; // NOLINT + y_absl::conditional_t<fit_cond, uint16_t, uint8_t>; // NOLINT }; // An adapter class that converts a lower-bound compare into an upper-bound diff --git a/library/cpp/tld/tlds-alpha-by-domain.txt b/library/cpp/tld/tlds-alpha-by-domain.txt index a39adf61b5..e92220d7fc 100644 --- a/library/cpp/tld/tlds-alpha-by-domain.txt +++ b/library/cpp/tld/tlds-alpha-by-domain.txt @@ -1,4 +1,4 @@ -# Version 2024082500, Last Updated Sun Aug 25 07:07:01 2024 UTC +# Version 2024083101, Last Updated Sun Sep 1 07:07:02 2024 UTC AAA AARP ABB diff --git a/library/cpp/yt/misc/unaligned-inl.h b/library/cpp/yt/misc/unaligned-inl.h new file mode 100644 index 0000000000..68e1c9b499 --- /dev/null +++ b/library/cpp/yt/misc/unaligned-inl.h @@ -0,0 +1,31 @@ +#ifndef UNALIGNED_INL_H_ +#error "Direct inclusion of this file is not allowed, include unaligned.h" +// For the sake of sane code completion. +#include "unaligned.h" +#endif + +#include <cstring> + +namespace NYT { + +//////////////////////////////////////////////////////////////////////////////// + +template <class T> + requires std::is_trivial_v<T> +T UnalignedLoad(const T* ptr) +{ + T value; + std::memcpy(&value, ptr, sizeof(T)); + return value; +} + +template <class T> + requires std::is_trivial_v<T> +void UnalignedStore(T* ptr, const T& value) +{ + std::memcpy(ptr, &value, sizeof(T)); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT diff --git a/library/cpp/yt/misc/unaligned.h b/library/cpp/yt/misc/unaligned.h new file mode 100644 index 0000000000..68c124183f --- /dev/null +++ b/library/cpp/yt/misc/unaligned.h @@ -0,0 +1,23 @@ +#pragma once + +#include <type_traits> + +namespace NYT { + +//////////////////////////////////////////////////////////////////////////////// + +template <class T> + requires std::is_trivial_v<T> +T UnalignedLoad(const T* ptr); + +template <class T> + requires std::is_trivial_v<T> +void UnalignedStore(T* ptr, const T& value); + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT + +#define UNALIGNED_INL_H_ +#include "unaligned-inl.h" +#undef UNALIGNED_INL_H_ diff --git a/library/cpp/yt/small_containers/compact_flat_map-inl.h b/library/cpp/yt/small_containers/compact_flat_map-inl.h index 459d72fa2c..e781569d9d 100644 --- a/library/cpp/yt/small_containers/compact_flat_map-inl.h +++ b/library/cpp/yt/small_containers/compact_flat_map-inl.h @@ -8,137 +8,140 @@ namespace NYT { /////////////////////////////////////////////////////////////////////////////// -template <class K, class V, size_t N> +template <class TKey, class TValue, size_t N, class TKeyCompare> template <class TInputIterator> -TCompactFlatMap<K, V, N>::TCompactFlatMap(TInputIterator begin, TInputIterator end) +TCompactFlatMap<TKey, TValue, N, TKeyCompare>::TCompactFlatMap(TInputIterator begin, TInputIterator end) { insert(begin, end); } -template <class K, class V, size_t N> -TCompactFlatMap<K, V, N>::TCompactFlatMap(std::initializer_list<value_type> values) - : TCompactFlatMap<K, V, N>(values.begin(), values.end()) +template <class TKey, class TValue, size_t N, class TKeyCompare> +TCompactFlatMap<TKey, TValue, N, TKeyCompare>::TCompactFlatMap(std::initializer_list<value_type> values) + : TCompactFlatMap<TKey, TValue, N, TKeyCompare>(values.begin(), values.end()) { } -template <class K, class V, size_t N> -bool TCompactFlatMap<K, V, N>::operator==(const TCompactFlatMap& rhs) const +template <class TKey, class TValue, size_t N, class TKeyCompare> +bool TCompactFlatMap<TKey, TValue, N, TKeyCompare>::operator==(const TCompactFlatMap& rhs) const { return Storage_ == rhs.Storage_; } -template <class K, class V, size_t N> -bool TCompactFlatMap<K, V, N>::operator!=(const TCompactFlatMap& rhs) const +template <class TKey, class TValue, size_t N, class TKeyCompare> +bool TCompactFlatMap<TKey, TValue, N, TKeyCompare>::operator!=(const TCompactFlatMap& rhs) const { return !(*this == rhs); } -template <class K, class V, size_t N> -typename TCompactFlatMap<K, V, N>::iterator TCompactFlatMap<K, V, N>::begin() +template <class TKey, class TValue, size_t N, class TKeyCompare> +typename TCompactFlatMap<TKey, TValue, N, TKeyCompare>::iterator TCompactFlatMap<TKey, TValue, N, TKeyCompare>::begin() { return Storage_.begin(); } -template <class K, class V, size_t N> -typename TCompactFlatMap<K, V, N>::const_iterator TCompactFlatMap<K, V, N>::begin() const +template <class TKey, class TValue, size_t N, class TKeyCompare> +typename TCompactFlatMap<TKey, TValue, N, TKeyCompare>::const_iterator TCompactFlatMap<TKey, TValue, N, TKeyCompare>::begin() const { return Storage_.begin(); } -template <class K, class V, size_t N> -typename TCompactFlatMap<K, V, N>::const_iterator TCompactFlatMap<K, V, N>::cbegin() const +template <class TKey, class TValue, size_t N, class TKeyCompare> +typename TCompactFlatMap<TKey, TValue, N, TKeyCompare>::const_iterator TCompactFlatMap<TKey, TValue, N, TKeyCompare>::cbegin() const { return Storage_.begin(); } -template <class K, class V, size_t N> -typename TCompactFlatMap<K, V, N>::iterator TCompactFlatMap<K, V, N>::end() +template <class TKey, class TValue, size_t N, class TKeyCompare> +typename TCompactFlatMap<TKey, TValue, N, TKeyCompare>::iterator TCompactFlatMap<TKey, TValue, N, TKeyCompare>::end() { return Storage_.end(); } -template <class K, class V, size_t N> -typename TCompactFlatMap<K, V, N>::const_iterator TCompactFlatMap<K, V, N>::end() const +template <class TKey, class TValue, size_t N, class TKeyCompare> +typename TCompactFlatMap<TKey, TValue, N, TKeyCompare>::const_iterator TCompactFlatMap<TKey, TValue, N, TKeyCompare>::end() const { return Storage_.end(); } -template <class K, class V, size_t N> -typename TCompactFlatMap<K, V, N>::const_iterator TCompactFlatMap<K, V, N>::cend() const +template <class TKey, class TValue, size_t N, class TKeyCompare> +typename TCompactFlatMap<TKey, TValue, N, TKeyCompare>::const_iterator TCompactFlatMap<TKey, TValue, N, TKeyCompare>::cend() const { return Storage_.end(); } -template <class K, class V, size_t N> -void TCompactFlatMap<K, V, N>::reserve(size_type n) +template <class TKey, class TValue, size_t N, class TKeyCompare> +void TCompactFlatMap<TKey, TValue, N, TKeyCompare>::reserve(size_type n) { Storage_.reserve(n); } -template <class K, class V, size_t N> -typename TCompactFlatMap<K, V, N>::size_type TCompactFlatMap<K, V, N>::size() const +template <class TKey, class TValue, size_t N, class TKeyCompare> +typename TCompactFlatMap<TKey, TValue, N, TKeyCompare>::size_type TCompactFlatMap<TKey, TValue, N, TKeyCompare>::size() const { return Storage_.size(); } -template <class K, class V, size_t N> -int TCompactFlatMap<K, V, N>::ssize() const +template <class TKey, class TValue, size_t N, class TKeyCompare> +int TCompactFlatMap<TKey, TValue, N, TKeyCompare>::ssize() const { return static_cast<int>(Storage_.size()); } -template <class K, class V, size_t N> -bool TCompactFlatMap<K, V, N>::empty() const +template <class TKey, class TValue, size_t N, class TKeyCompare> +bool TCompactFlatMap<TKey, TValue, N, TKeyCompare>::empty() const { return Storage_.empty(); } -template <class K, class V, size_t N> -void TCompactFlatMap<K, V, N>::clear() +template <class TKey, class TValue, size_t N, class TKeyCompare> +void TCompactFlatMap<TKey, TValue, N, TKeyCompare>::clear() { Storage_.clear(); } -template <class K, class V, size_t N> -void TCompactFlatMap<K, V, N>::shrink_to_small() +template <class TKey, class TValue, size_t N, class TKeyCompare> +void TCompactFlatMap<TKey, TValue, N, TKeyCompare>::shrink_to_small() { Storage_.shrink_to_small(); } -template <class K, class V, size_t N> -typename TCompactFlatMap<K, V, N>::iterator TCompactFlatMap<K, V, N>::find(const K& k) +template <class TKey, class TValue, size_t N, class TKeyCompare> +template <NDetail::CComparisonAllowed<TKey, TKeyCompare> TOtherKey> +typename TCompactFlatMap<TKey, TValue, N, TKeyCompare>::iterator TCompactFlatMap<TKey, TValue, N, TKeyCompare>::find(const TOtherKey& k) { auto [rangeBegin, rangeEnd] = equal_range(k); return rangeBegin == rangeEnd ? end() : rangeBegin; } -template <class K, class V, size_t N> -typename TCompactFlatMap<K, V, N>::const_iterator TCompactFlatMap<K, V, N>::find(const K& k) const +template <class TKey, class TValue, size_t N, class TKeyCompare> +template <NDetail::CComparisonAllowed<TKey, TKeyCompare> TOtherKey> +typename TCompactFlatMap<TKey, TValue, N, TKeyCompare>::const_iterator TCompactFlatMap<TKey, TValue, N, TKeyCompare>::find(const TOtherKey& k) const { auto [rangeBegin, rangeEnd] = equal_range(k); return rangeBegin == rangeEnd ? end() : rangeBegin; } -template <class K, class V, size_t N> -bool TCompactFlatMap<K, V, N>::contains(const K& k) const +template <class TKey, class TValue, size_t N, class TKeyCompare> +template <NDetail::CComparisonAllowed<TKey, TKeyCompare> TOtherKey> +bool TCompactFlatMap<TKey, TValue, N, TKeyCompare>::contains(const TOtherKey& k) const { return find(k) != end(); } -template <class K, class V, size_t N> -auto TCompactFlatMap<K, V, N>::insert(const value_type& value) -> std::pair<iterator, bool> +template <class TKey, class TValue, size_t N, class TKeyCompare> +auto TCompactFlatMap<TKey, TValue, N, TKeyCompare>::insert(const value_type& value) -> std::pair<iterator, bool> { return do_insert(value); } -template <class K, class V, size_t N> -auto TCompactFlatMap<K, V, N>::insert(value_type&& value) -> std::pair<iterator, bool> +template <class TKey, class TValue, size_t N, class TKeyCompare> +auto TCompactFlatMap<TKey, TValue, N, TKeyCompare>::insert(value_type&& value) -> std::pair<iterator, bool> { return do_insert(std::move(value)); } -template <class K, class V, size_t N> +template <class TKey, class TValue, size_t N, class TKeyCompare> template <class TArg> -auto TCompactFlatMap<K, V, N>::do_insert(TArg&& value) -> std::pair<iterator, bool> +auto TCompactFlatMap<TKey, TValue, N, TKeyCompare>::do_insert(TArg&& value) -> std::pair<iterator, bool> { auto [rangeBegin, rangeEnd] = equal_range(value.first); if (rangeBegin != rangeEnd) { @@ -149,38 +152,38 @@ auto TCompactFlatMap<K, V, N>::do_insert(TArg&& value) -> std::pair<iterator, bo } } -template <class K, class V, size_t N> +template <class TKey, class TValue, size_t N, class TKeyCompare> template <class TInputIterator> -void TCompactFlatMap<K, V, N>::insert(TInputIterator begin, TInputIterator end) +void TCompactFlatMap<TKey, TValue, N, TKeyCompare>::insert(TInputIterator begin, TInputIterator end) { for (auto it = begin; it != end; ++it) { insert(*it); } } -template <class K, class V, size_t N> +template <class TKey, class TValue, size_t N, class TKeyCompare> template <class... TArgs> -auto TCompactFlatMap<K, V, N>::emplace(TArgs&&... args) -> std::pair<iterator, bool> +auto TCompactFlatMap<TKey, TValue, N, TKeyCompare>::emplace(TArgs&&... args) -> std::pair<iterator, bool> { return insert(value_type(std::forward<TArgs>(args)...)); } -template <class K, class V, size_t N> -V& TCompactFlatMap<K, V, N>::operator[](const K& k) +template <class TKey, class TValue, size_t N, class TKeyCompare> +TValue& TCompactFlatMap<TKey, TValue, N, TKeyCompare>::operator[](const TKey& k) { - auto [it, inserted] = insert({k, V()}); + auto [it, inserted] = insert({k, TValue()}); return it->second; } -template <class K, class V, size_t N> -void TCompactFlatMap<K, V, N>::erase(const K& k) +template <class TKey, class TValue, size_t N, class TKeyCompare> +void TCompactFlatMap<TKey, TValue, N, TKeyCompare>::erase(const TKey& k) { auto [rangeBegin, rangeEnd] = equal_range(k); erase(rangeBegin, rangeEnd); } -template <class K, class V, size_t N> -void TCompactFlatMap<K, V, N>::erase(iterator pos) +template <class TKey, class TValue, size_t N, class TKeyCompare> +void TCompactFlatMap<TKey, TValue, N, TKeyCompare>::erase(iterator pos) { Storage_.erase(pos); @@ -188,8 +191,8 @@ void TCompactFlatMap<K, V, N>::erase(iterator pos) Storage_.shrink_to_small(); } -template <class K, class V, size_t N> -void TCompactFlatMap<K, V, N>::erase(iterator b, iterator e) +template <class TKey, class TValue, size_t N, class TKeyCompare> +void TCompactFlatMap<TKey, TValue, N, TKeyCompare>::erase(iterator b, iterator e) { Storage_.erase(b, e); @@ -197,46 +200,52 @@ void TCompactFlatMap<K, V, N>::erase(iterator b, iterator e) Storage_.shrink_to_small(); } -template <class K, class V, size_t N> -std::pair<typename TCompactFlatMap<K, V, N>::iterator, typename TCompactFlatMap<K, V, N>::iterator> -TCompactFlatMap<K, V, N>::equal_range(const K& k) +template <class TKey, class TValue, size_t N, class TKeyCompare> +template <NDetail::CComparisonAllowed<TKey, TKeyCompare> TOtherKey> +std::pair<typename TCompactFlatMap<TKey, TValue, N, TKeyCompare>::iterator, typename TCompactFlatMap<TKey, TValue, N, TKeyCompare>::iterator> +TCompactFlatMap<TKey, TValue, N, TKeyCompare>::equal_range(const TOtherKey& k) { - auto result = std::equal_range(Storage_.begin(), Storage_.end(), k, TKeyComparer()); - YT_ASSERT(std::distance(result.first, result.second) <= 1); + auto result = std::ranges::equal_range(Storage_, k, {}, &value_type::first); + YT_ASSERT(result.size() <= 1); return result; } -template <class K, class V, size_t N> -std::pair<typename TCompactFlatMap<K, V, N>::const_iterator, typename TCompactFlatMap<K, V, N>::const_iterator> -TCompactFlatMap<K, V, N>::equal_range(const K& k) const +template <class TKey, class TValue, size_t N, class TKeyCompare> +template <NDetail::CComparisonAllowed<TKey, TKeyCompare> TOtherKey> +std::pair<typename TCompactFlatMap<TKey, TValue, N, TKeyCompare>::const_iterator, typename TCompactFlatMap<TKey, TValue, N, TKeyCompare>::const_iterator> +TCompactFlatMap<TKey, TValue, N, TKeyCompare>::equal_range(const TOtherKey& k) const { - auto result = std::equal_range(Storage_.begin(), Storage_.end(), k, TKeyComparer()); - YT_ASSERT(std::distance(result.first, result.second) <= 1); + auto result = std::ranges::equal_range(Storage_, k, {}, &value_type::first); + YT_ASSERT(result.size() <= 1); return result; } -template <class K, class V, size_t N> -typename TCompactFlatMap<K, V, N>::const_iterator TCompactFlatMap<K, V, N>::lower_bound(const K& k) const +template <class TKey, class TValue, size_t N, class TKeyCompare> +template <NDetail::CComparisonAllowed<TKey, TKeyCompare> TOtherKey> +typename TCompactFlatMap<TKey, TValue, N, TKeyCompare>::const_iterator TCompactFlatMap<TKey, TValue, N, TKeyCompare>::lower_bound(const TOtherKey& k) const { - return std::lower_bound(Storage_.begin(), Storage_.end(), k, TKeyComparer()); + return std::ranges::lower_bound(Storage_, k, {}, &value_type::first); } -template <class K, class V, size_t N> -typename TCompactFlatMap<K, V, N>::iterator TCompactFlatMap<K, V, N>::lower_bound(const K& k) +template <class TKey, class TValue, size_t N, class TKeyCompare> +template <NDetail::CComparisonAllowed<TKey, TKeyCompare> TOtherKey> +typename TCompactFlatMap<TKey, TValue, N, TKeyCompare>::iterator TCompactFlatMap<TKey, TValue, N, TKeyCompare>::lower_bound(const TOtherKey& k) { - return std::lower_bound(Storage_.begin(), Storage_.end(), k, TKeyComparer()); + return std::ranges::lower_bound(Storage_, k, {}, &value_type::first); } -template <class K, class V, size_t N> -typename TCompactFlatMap<K, V, N>::const_iterator TCompactFlatMap<K, V, N>::upper_bound(const K& k) const +template <class TKey, class TValue, size_t N, class TKeyCompare> +template <NDetail::CComparisonAllowed<TKey, TKeyCompare> TOtherKey> +typename TCompactFlatMap<TKey, TValue, N, TKeyCompare>::const_iterator TCompactFlatMap<TKey, TValue, N, TKeyCompare>::upper_bound(const TOtherKey& k) const { - return std::upper_bound(Storage_.begin(), Storage_.end(), k, TKeyComparer()); + return std::ranges::upper_bound(Storage_, k, {}, &value_type::first); } -template <class K, class V, size_t N> -typename TCompactFlatMap<K, V, N>::iterator TCompactFlatMap<K, V, N>::upper_bound(const K& k) +template <class TKey, class TValue, size_t N, class TKeyCompare> +template <NDetail::CComparisonAllowed<TKey, TKeyCompare> TOtherKey> +typename TCompactFlatMap<TKey, TValue, N, TKeyCompare>::iterator TCompactFlatMap<TKey, TValue, N, TKeyCompare>::upper_bound(const TOtherKey& k) { - return std::upper_bound(Storage_.begin(), Storage_.end(), k, TKeyComparer()); + return std::ranges::upper_bound(Storage_, k, {}, &value_type::first); } //////////////////////////////////////////////////////////////////////////////// diff --git a/library/cpp/yt/small_containers/compact_flat_map.h b/library/cpp/yt/small_containers/compact_flat_map.h index b598a34731..120c2141b9 100644 --- a/library/cpp/yt/small_containers/compact_flat_map.h +++ b/library/cpp/yt/small_containers/compact_flat_map.h @@ -4,9 +4,21 @@ namespace NYT { +namespace NDetail { + +template <typename T> +concept CHasIsTransparentFlag = requires { + typename T::is_transparent; +}; + +template <typename T, typename U, typename TCompare> +concept CComparisonAllowed = std::same_as<T, U> || CHasIsTransparentFlag<TCompare>; + +} // namespace NDetail + /////////////////////////////////////////////////////////////////////////////// -//! A flat map implementation over TCompactVector that tries to keep data inline. +//! A flat map implementation over TCompactTValueector that tries to keep data inline. /*! * Similarly to SmallSet, this is implemented via binary search over a sorted * vector. Unlike SmallSet, however, this one never falls back to std::map (or @@ -21,32 +33,20 @@ namespace NYT { * Because of the latter, one should be very careful with iterators: virtually * any call to insert or erase may potentially invalidate all iterators. */ -template <class K, class V, size_t N> +template <class TKey, class TValue, size_t N, class TKeyCompare = std::ranges::less> class TCompactFlatMap { public: - // NB: can't make this pair<const K, V> as TCompactVector requires its type + // NB: can't make this pair<const TKey, TValue> as TCompactTValueector requires its type // parameter to be copy-assignable. - using value_type = std::pair<K, V>; - using key_type = K; - using mapped_type = V; + using value_type = std::pair<TKey, TValue>; + using key_type = TKey; + using mapped_type = TValue; + using key_compare = TKeyCompare; private: using TStorage = TCompactVector<value_type, N>; - struct TKeyComparer - { - bool operator()(const K& lhs, const value_type& rhs) - { - return lhs < rhs.first; - } - - bool operator()(const value_type& lhs, const K& rhs) - { - return lhs.first < rhs; - } - }; - public: using iterator = typename TStorage::iterator; using const_iterator = typename TStorage::const_iterator; @@ -80,17 +80,26 @@ public: void shrink_to_small(); - iterator find(const K& k); - const_iterator find(const K& k) const; - - iterator lower_bound(const K& k); - const_iterator lower_bound(const K& k) const; - iterator upper_bound(const K& k); - const_iterator upper_bound(const K& k) const; - std::pair<iterator, iterator> equal_range(const K& k); - std::pair<const_iterator, const_iterator> equal_range(const K& k) const; - - bool contains(const K& k) const; + template <NDetail::CComparisonAllowed<TKey, TKeyCompare> TOtherKey> + iterator find(const TOtherKey& k); + template <NDetail::CComparisonAllowed<TKey, TKeyCompare> TOtherKey> + const_iterator find(const TOtherKey& k) const; + + template <NDetail::CComparisonAllowed<TKey, TKeyCompare> TOtherKey> + iterator lower_bound(const TOtherKey& k); + template <NDetail::CComparisonAllowed<TKey, TKeyCompare> TOtherKey> + const_iterator lower_bound(const TOtherKey& k) const; + template <NDetail::CComparisonAllowed<TKey, TKeyCompare> TOtherKey> + iterator upper_bound(const TOtherKey& k); + template <NDetail::CComparisonAllowed<TKey, TKeyCompare> TOtherKey> + const_iterator upper_bound(const TOtherKey& k) const; + template <NDetail::CComparisonAllowed<TKey, TKeyCompare> TOtherKey> + std::pair<iterator, iterator> equal_range(const TOtherKey& k); + template <NDetail::CComparisonAllowed<TKey, TKeyCompare> TOtherKey> + std::pair<const_iterator, const_iterator> equal_range(const TOtherKey& k) const; + + template <NDetail::CComparisonAllowed<TKey, TKeyCompare> TOtherKey> + bool contains(const TOtherKey& k) const; std::pair<iterator, bool> insert(const value_type& value); std::pair<iterator, bool> insert(value_type&& value); @@ -101,9 +110,9 @@ public: template <class... TArgs> std::pair<iterator, bool> emplace(TArgs&&... args); - V& operator[](const K& k); + TValue& operator[](const TKey& k); - void erase(const K& k); + void erase(const TKey& k); void erase(iterator pos); void erase(iterator b, iterator e); diff --git a/library/cpp/yt/small_containers/compact_set-inl.h b/library/cpp/yt/small_containers/compact_set-inl.h index e93d11f126..20736f5083 100644 --- a/library/cpp/yt/small_containers/compact_set-inl.h +++ b/library/cpp/yt/small_containers/compact_set-inl.h @@ -216,19 +216,19 @@ bool TCompactSet<T, N, C, A>::empty() const template <typename T, size_t N, typename C, typename A> typename TCompactSet<T, N, C, A>::size_type TCompactSet<T, N, C, A>::size() const { - return is_small() ? Vector_.size() : Set_.size(); + return IsSmall() ? Vector_.size() : Set_.size(); } template <typename T, size_t N, typename C, typename A> const T& TCompactSet<T, N, C, A>::front() const { - return is_small() ? Vector_.front() : *Set_.begin(); + return IsSmall() ? Vector_.front() : *Set_.begin(); } template <typename T, size_t N, typename C, typename A> typename TCompactSet<T, N, C, A>::size_type TCompactSet<T, N, C, A>::count(const T& v) const { - if (is_small()) { + if (IsSmall()) { return std::binary_search(Vector_.begin(), Vector_.end(), v, C()) ? 1 : 0; } else { return Set_.count(v); @@ -244,7 +244,7 @@ bool TCompactSet<T, N, C, A>::contains(const T& v) const template <typename T, size_t N, typename C, typename A> std::pair<typename TCompactSet<T, N, C, A>::const_iterator, bool> TCompactSet<T, N, C, A>::insert(const T& v) { - if (!is_small()) { + if (!IsSmall()) { auto [it, inserted] = Set_.insert(v); return {const_iterator(std::move(it)), inserted}; } @@ -279,7 +279,7 @@ void TCompactSet<T, N, C, A>::insert(TIter i, TIter e) template <typename T, size_t N, typename C, typename A> bool TCompactSet<T, N, C, A>::erase(const T& v) { - if (!is_small()) { + if (!IsSmall()) { return Set_.erase(v); } @@ -302,7 +302,7 @@ void TCompactSet<T, N, C, A>::clear() template <typename T, size_t N, typename C, typename A> typename TCompactSet<T, N, C, A>::const_iterator TCompactSet<T, N, C, A>::begin() const { - return is_small() ? const_iterator(Vector_.begin()) : const_iterator(Set_.begin()); + return IsSmall() ? const_iterator(Vector_.begin()) : const_iterator(Set_.begin()); } template <typename T, size_t N, typename C, typename A> @@ -314,7 +314,7 @@ typename TCompactSet<T, N, C, A>::const_iterator TCompactSet<T, N, C, A>::cbegin template <typename T, size_t N, typename C, typename A> typename TCompactSet<T, N, C, A>::const_iterator TCompactSet<T, N, C, A>::end() const { - return is_small() ? const_iterator(Vector_.end()) : const_iterator(Set_.end()); + return IsSmall() ? const_iterator(Vector_.end()) : const_iterator(Set_.end()); } template <typename T, size_t N, typename C, typename A> @@ -324,7 +324,7 @@ typename TCompactSet<T, N, C, A>::const_iterator TCompactSet<T, N, C, A>::cend() } template <typename T, size_t N, typename C, typename A> -bool TCompactSet<T, N, C, A>::is_small() const +bool TCompactSet<T, N, C, A>::IsSmall() const { return Set_.empty(); } diff --git a/library/cpp/yt/small_containers/compact_set.h b/library/cpp/yt/small_containers/compact_set.h index ca5e3c6efd..354fad195c 100644 --- a/library/cpp/yt/small_containers/compact_set.h +++ b/library/cpp/yt/small_containers/compact_set.h @@ -80,7 +80,7 @@ private: using TSetConstIterator = typename std::set<T, C, A>::const_iterator; using TVectorConstIterator = typename TCompactVector<T, N>::const_iterator; - bool is_small() const; + bool IsSmall() const; }; /////////////////////////////////////////////////////////////////////////////// @@ -90,4 +90,3 @@ private: #define COMPACT_SET_INL_H_ #include "compact_set-inl.h" #undef COMPACT_SET_INL_H_ - diff --git a/util/datetime/base_ut.cpp b/util/datetime/base_ut.cpp index 3482feafac..3c03626787 100644 --- a/util/datetime/base_ut.cpp +++ b/util/datetime/base_ut.cpp @@ -534,7 +534,6 @@ Y_UNIT_TEST_SUITE(DateTimeTest) { } Y_UNIT_TEST(TestTDurationConstructorFromStdChronoDuration) { - UNIT_ASSERT_VALUES_EQUAL(TDuration::Zero(), TDuration(0ms)); UNIT_ASSERT_VALUES_EQUAL(TDuration::MicroSeconds(42), TDuration(42us)); UNIT_ASSERT_VALUES_EQUAL(TDuration::MicroSeconds(42000000000000L), TDuration(42000000000000us)); diff --git a/util/folder/path_ut.cpp b/util/folder/path_ut.cpp index bb0656ddbc..76dd692ec6 100644 --- a/util/folder/path_ut.cpp +++ b/util/folder/path_ut.cpp @@ -259,7 +259,7 @@ Y_UNIT_TEST_SUITE(TFsPathTests) { // mkdir(2) places umask(2) on mode argument. const int mask = Umask(0); Umask(mask); - UNIT_ASSERT_VALUES_EQUAL(stat.Mode& MODE0777, mode & ~mask); + UNIT_ASSERT_VALUES_EQUAL(stat.Mode & MODE0777, mode & ~mask); } #endif diff --git a/util/generic/flags_ut.cpp b/util/generic/flags_ut.cpp index fa2d4464c9..fc2f73db92 100644 --- a/util/generic/flags_ut.cpp +++ b/util/generic/flags_ut.cpp @@ -114,9 +114,9 @@ Y_UNIT_TEST_SUITE(TFlagsTest) { Y_UNIT_TEST(TestBaseType) { ui16 goodValue = 7; auto goodFlags = ETest1::FromBaseType(goodValue); - UNIT_ASSERT(goodFlags& ETestFlag1::Test1); - UNIT_ASSERT(goodFlags& ETestFlag1::Test2); - UNIT_ASSERT(goodFlags& ETestFlag1::Test4); + UNIT_ASSERT(goodFlags & ETestFlag1::Test1); + UNIT_ASSERT(goodFlags & ETestFlag1::Test2); + UNIT_ASSERT(goodFlags & ETestFlag1::Test4); UNIT_ASSERT_VALUES_EQUAL(goodValue, goodFlags.ToBaseType()); // Passed value is not checked, but preserved as is diff --git a/util/generic/string_transparent_hash_ut.cpp b/util/generic/string_transparent_hash_ut.cpp index b87fa2843e..a03fc7e6d7 100644 --- a/util/generic/string_transparent_hash_ut.cpp +++ b/util/generic/string_transparent_hash_ut.cpp @@ -3,15 +3,27 @@ #include "strbuf.h" #include <library/cpp/testing/unittest/registar.h> -#include <library/cpp/containers/absl_flat_hash/flat_hash_set.h> #include <util/str_stl.h> +#ifdef __cpp_lib_generic_unordered_lookup + #include <unordered_set> + +template <class T, class THasher, class TPred> +using THashSetType = std::unordered_set<T, THasher, TPred>; +#else + // Using Abseil hash set because `std::unordered_set` is transparent only from libstdc++11. + // Meanwhile clang-linux-x86_64-release-stl-system autocheck sets OS_SDK=ubuntu-20, + // that support libstdc++10 by default. + #include <library/cpp/containers/absl_flat_hash/flat_hash_set.h> + +template <class T, class THasher, class TPred> +using THashSetType = absl::flat_hash_set<T, THasher, TPred>; +#endif + Y_UNIT_TEST_SUITE(StringHashFunctorTests) { Y_UNIT_TEST(TestTransparencyWithUnorderedSet) { - // Using Abseil hash set because `std::unordered_set` is transparent only from C++20 (while - // we stuck with C++17 right now). - absl::flat_hash_set<TString, THash<TString>, TEqualTo<TString>> s = {"foo"}; + THashSetType<TString, THash<TString>, TEqualTo<TString>> s = {"foo"}; // If either `THash` or `TEqualTo` is not transparent compilation will fail. UNIT_ASSERT_UNEQUAL(s.find(TStringBuf("foo")), s.end()); UNIT_ASSERT_EQUAL(s.find(TStringBuf("bar")), s.end()); diff --git a/util/generic/ut/ya.make b/util/generic/ut/ya.make index 35bdb72bd4..38978e129d 100644 --- a/util/generic/ut/ya.make +++ b/util/generic/ut/ya.make @@ -41,6 +41,7 @@ SRCS( generic/stack_ut.cpp generic/store_policy_ut.cpp generic/strbuf_ut.cpp + generic/string_transparent_hash_ut.cpp generic/string_ut.cpp generic/typelist_ut.cpp generic/typetraits_ut.cpp @@ -56,16 +57,8 @@ SRCS( INCLUDE(${ARCADIA_ROOT}/util/tests/ya_util_tests.inc) -IF (NOT OS_IOS AND NOT ARCH_PPC64LE) - # Abseil fails to build (with linkage error) on ios and with compilation error on PowerPC - # (somewhere in unscaledcycleclock.cc). - PEERDIR( - library/cpp/containers/absl_flat_hash - ) - - SRCS( - generic/string_transparent_hash_ut.cpp - ) -ENDIF() +PEERDIR( + library/cpp/containers/absl_flat_hash +) END() diff --git a/util/str_stl.h b/util/str_stl.h index 5b0254bee9..b9d16a53b9 100644 --- a/util/str_stl.h +++ b/util/str_stl.h @@ -37,6 +37,8 @@ namespace std { namespace NHashPrivate { template <class T, bool needNumericHashing> struct THashHelper { + using is_default_implementation = std::true_type; + inline size_t operator()(const T& t) const noexcept { return (size_t)t; // If you have a compilation error here, look at explanation below: // Probably error is caused by undefined template specialization of THash<T> diff --git a/util/system/atexit_ut.cpp b/util/system/atexit_ut.cpp index f093c2b86b..190f6dd0c3 100644 --- a/util/system/atexit_ut.cpp +++ b/util/system/atexit_ut.cpp @@ -88,7 +88,6 @@ UNIT_TEST_SUITE_REGISTRATION(TAtExitTest); Y_UNIT_TEST_SUITE(TestAtExit) { Y_UNIT_TEST(CreateUponDestruction) { - struct T1 { }; @@ -99,6 +98,5 @@ Y_UNIT_TEST_SUITE(TestAtExit) { }; Singleton<T2>(); - } } diff --git a/yt/cpp/mapreduce/interface/config.cpp b/yt/cpp/mapreduce/interface/config.cpp index 1ee5a77af7..879bf79561 100644 --- a/yt/cpp/mapreduce/interface/config.cpp +++ b/yt/cpp/mapreduce/interface/config.cpp @@ -225,6 +225,7 @@ void TConfig::Reset() "//tmp/yt_wrapper/table_storage"); RemoteTempTablesDirectory = GetEnv("YT_TEMP_DIR", RemoteTempTablesDirectory); + KeepTempTables = GetBool("YT_KEEP_TEMP_TABLES"); InferTableSchema = false; diff --git a/yt/cpp/mapreduce/interface/config.h b/yt/cpp/mapreduce/interface/config.h index 27a31281ba..a8baac8d4f 100644 --- a/yt/cpp/mapreduce/interface/config.h +++ b/yt/cpp/mapreduce/interface/config.h @@ -126,6 +126,9 @@ struct TConfig TString RemoteTempFilesDirectory; TString RemoteTempTablesDirectory; + // @brief Keep temp tables produced by TTempTable (despite their name). Should not be used in user programs, + // but may be useful for setting via environment variable for debugging purposes. + bool KeepTempTables = false; // // Infer schemas for nonexstent tables from typed rows (e.g. protobuf) diff --git a/yt/yt/client/api/delegating_client.h b/yt/yt/client/api/delegating_client.h index 3369b70c20..f977dc06c0 100644 --- a/yt/yt/client/api/delegating_client.h +++ b/yt/yt/client/api/delegating_client.h @@ -510,7 +510,7 @@ public: const TGetJobSpecOptions& options), (jobId, options)) - DELEGATE_METHOD(TFuture<TSharedRef>, GetJobStderr, ( + DELEGATE_METHOD(TFuture<TGetJobStderrResponse>, GetJobStderr, ( const NScheduler::TOperationIdOrAlias& operationIdOrAlias, NJobTrackerClient::TJobId jobId, const TGetJobStderrOptions& options), diff --git a/yt/yt/client/api/operation_client.cpp b/yt/yt/client/api/operation_client.cpp index acc0dfe7bb..4e1816a6e7 100644 --- a/yt/yt/client/api/operation_client.cpp +++ b/yt/yt/client/api/operation_client.cpp @@ -227,5 +227,52 @@ void TListOperationsAccessFilter::Register(TRegistrar registrar) //////////////////////////////////////////////////////////////////////////////// +TGetJobStderrResponse TGetJobStderrResponse::MakeJobStderr(const TSharedRef& data, const TGetJobStderrOptions& options) +{ + auto totalSize = std::ssize(data); + auto endOffset = totalSize; + auto offset = options.Offset.value_or(0); + auto limit = options.Limit.value_or(0); + + if (!offset && !limit) { + return { + .Data = data, + .TotalSize = totalSize, + .EndOffset = endOffset, + }; + }; + + size_t firstPos = 0; + if (offset > 0) { + firstPos = offset; + } + + if (firstPos >= data.size()) { + return { + .Data = TSharedRef{}, + .TotalSize = totalSize, + .EndOffset = 0, + }; + } else { + auto lastPos = firstPos; + if (limit > 0) { + lastPos += limit; + } else { + lastPos += data.size(); + } + if (lastPos > data.size()) { + lastPos = data.size(); + } + const auto dataCut = data.Slice(firstPos, lastPos); + return { + .Data = dataCut, + .TotalSize = totalSize, + .EndOffset = limit ? static_cast<i64>(firstPos + dataCut.size()) : endOffset, + }; + } +} + +//////////////////////////////////////////////////////////////////////////////// + } // namespace NYT::NApi diff --git a/yt/yt/client/api/operation_client.h b/yt/yt/client/api/operation_client.h index 08fed371f1..48687497da 100644 --- a/yt/yt/client/api/operation_client.h +++ b/yt/yt/client/api/operation_client.h @@ -84,7 +84,10 @@ struct TGetJobSpecOptions struct TGetJobStderrOptions : public TTimeoutOptions , public TMasterReadOptions -{ }; +{ + std::optional<i64> Limit; + std::optional<i64> Offset; +}; struct TGetJobFailContextOptions : public TTimeoutOptions @@ -361,6 +364,27 @@ struct TListJobsResult std::vector<TError> Errors; }; +struct TGetJobStderrResponse +{ + // 0 + // |<- stderr full log ->| + // [ [<- Data ->] ] + // |<- request.Offset + // |<- request.Limit ->| + // |<- EndOffset + // |<- TotalSize + + TSharedRef Data; + + // Total current stderr size. + i64 TotalSize = 0; + + // Index of the last byte of the result in the full stderr. + i64 EndOffset = 0; + + static TGetJobStderrResponse MakeJobStderr(const TSharedRef& data, const TGetJobStderrOptions& options = {}); +}; + //////////////////////////////////////////////////////////////////////////////// struct IOperationClient @@ -414,7 +438,7 @@ struct IOperationClient NJobTrackerClient::TJobId jobId, const TGetJobSpecOptions& options = {}) = 0; - virtual TFuture<TSharedRef> GetJobStderr( + virtual TFuture<TGetJobStderrResponse> GetJobStderr( const NScheduler::TOperationIdOrAlias& operationIdOrAlias, NJobTrackerClient::TJobId jobId, const TGetJobStderrOptions& options = {}) = 0; diff --git a/yt/yt/client/api/rpc_proxy/client_impl.cpp b/yt/yt/client/api/rpc_proxy/client_impl.cpp index 5f3ee6a1eb..454d6fd16b 100644 --- a/yt/yt/client/api/rpc_proxy/client_impl.cpp +++ b/yt/yt/client/api/rpc_proxy/client_impl.cpp @@ -1294,7 +1294,7 @@ TFuture<TYsonString> TClient::GetJobSpec( })); } -TFuture<TSharedRef> TClient::GetJobStderr( +TFuture<TGetJobStderrResponse> TClient::GetJobStderr( const TOperationIdOrAlias& operationIdOrAlias, NJobTrackerClient::TJobId jobId, const TGetJobStderrOptions& options) @@ -1306,10 +1306,17 @@ TFuture<TSharedRef> TClient::GetJobStderr( NScheduler::ToProto(req, operationIdOrAlias); ToProto(req->mutable_job_id(), jobId); + if (options.Limit) { + req->set_limit(*options.Limit); + } + if (options.Offset) { + req->set_offset(*options.Offset); + } - return req->Invoke().Apply(BIND([] (const TApiServiceProxy::TRspGetJobStderrPtr& rsp) { + return req->Invoke().Apply(BIND([req = req](const TApiServiceProxy::TRspGetJobStderrPtr& rsp) { YT_VERIFY(rsp->Attachments().size() == 1); - return rsp->Attachments().front(); + TGetJobStderrOptions options{.Limit = req->limit(), .Offset = req->offset()}; + return TGetJobStderrResponse::MakeJobStderr(rsp->Attachments().front(), options); })); } diff --git a/yt/yt/client/api/rpc_proxy/client_impl.h b/yt/yt/client/api/rpc_proxy/client_impl.h index b868d7493f..9619084ec9 100644 --- a/yt/yt/client/api/rpc_proxy/client_impl.h +++ b/yt/yt/client/api/rpc_proxy/client_impl.h @@ -273,7 +273,7 @@ public: NJobTrackerClient::TJobId jobId, const NApi::TGetJobSpecOptions& options) override; - TFuture<TSharedRef> GetJobStderr( + TFuture<TGetJobStderrResponse> GetJobStderr( const NScheduler::TOperationIdOrAlias& operationIdOrAlias, NJobTrackerClient::TJobId jobId, const NApi::TGetJobStderrOptions& options) override; diff --git a/yt/yt/client/api/rpc_proxy/connection_impl.cpp b/yt/yt/client/api/rpc_proxy/connection_impl.cpp index 3e84e0ef5d..5c2f4a7540 100644 --- a/yt/yt/client/api/rpc_proxy/connection_impl.cpp +++ b/yt/yt/client/api/rpc_proxy/connection_impl.cpp @@ -164,7 +164,7 @@ public: , EndpointAttributes_(MakeEndpointAttributes(config, connectionId)) { } - const TString& GetEndpointDescription() const override + const std::string& GetEndpointDescription() const override { return EndpointDescription_; } @@ -205,7 +205,7 @@ private: const bool Sticky_; const TGuid ConnectionId_; - const TString EndpointDescription_; + const std::string EndpointDescription_; const IAttributeDictionaryPtr EndpointAttributes_; YT_DECLARE_SPIN_LOCK(NThreading::TSpinLock, SpinLock_); diff --git a/yt/yt/client/cache/cache.cpp b/yt/yt/client/cache/cache.cpp index e7b75b25fb..99137c4e46 100644 --- a/yt/yt/client/cache/cache.cpp +++ b/yt/yt/client/cache/cache.cpp @@ -1,48 +1,71 @@ #include "cache_base.h" #include "rpc.h" +#include "config.h" #include <yt/yt/client/api/options.h> #include <yt/yt/core/net/address.h> -#include <yt/yt_proto/yt/client/cache/proto/config.pb.h> - #include <util/stream/str.h> namespace NYT::NClient::NCache { + +using namespace NNet; +using NApi::NRpcProxy::TConnectionConfig; +using NApi::NRpcProxy::TConnectionConfigPtr; + namespace { //////////////////////////////////////////////////////////////////////////////// +template<class T> +TIntrusivePtr<T> CopyConfig(const TIntrusivePtr<T>& config) +{ + auto newConfig = New<T>(); + newConfig->Load( + ConvertToNode(config), + /*postprocess*/ false, + /*setDefaults*/ false, + /*path*/ ""); + return newConfig; +} + TStringBuf GetNormalClusterName(TStringBuf clusterName) { return NNet::InferYTClusterFromClusterUrlRaw(clusterName).value_or(clusterName); } -TClustersConfig GetClustersConfigWithNormalClusterName(const TClustersConfig& config) +// TODO(ignat): move this logic to ads/bsyeti/libs/ytex/client/ +TClientsCacheConfigPtr GetClustersConfigWithNormalClusterName(const TClientsCacheConfigPtr& config) { - TClustersConfig newConfig(config); - newConfig.ClearClusterConfigs(); - for (auto& [clusterName, clusterConfig] : config.GetClusterConfigs()) { - (*newConfig.MutableClusterConfigs())[ToString(GetNormalClusterName(clusterName))] = clusterConfig; + auto newConfig = New<TClientsCacheConfig>(); + + newConfig->DefaultConfig = CopyConfig(config->DefaultConfig); + for (const auto& [clusterName, clusterConfig] : config->ClusterConfigs) { + newConfig->ClusterConfigs[ToString(GetNormalClusterName(clusterName))] = CopyConfig(clusterConfig); } return newConfig; } +//////////////////////////////////////////////////////////////////////////////// + } // namespace -TConfig MakeClusterConfig( - const TClustersConfig& clustersConfig, +TConnectionConfigPtr MakeClusterConfig( + const TClientsCacheConfigPtr& clustersConfig, TStringBuf clusterUrl) { auto [cluster, proxyRole] = ExtractClusterAndProxyRole(clusterUrl); - auto it = clustersConfig.GetClusterConfigs().find(GetNormalClusterName(cluster)); - auto config = (it != clustersConfig.GetClusterConfigs().end()) ? it->second : clustersConfig.GetDefaultConfig(); - config.SetClusterName(ToString(cluster)); + auto it = clustersConfig->ClusterConfigs.find(GetNormalClusterName(cluster)); + auto config = (it != clustersConfig->ClusterConfigs.end()) ? it->second : clustersConfig->DefaultConfig; + + auto newConfig = CopyConfig(config); + newConfig->ClusterUrl = ToString(cluster); + newConfig->ClusterName = InferYTClusterFromClusterUrl(*newConfig->ClusterUrl); if (!proxyRole.empty()) { - config.SetProxyRole(ToString(proxyRole)); + newConfig->ProxyRole = ToString(proxyRole); } - return config; + return newConfig; } //////////////////////////////////////////////////////////////////////////////// @@ -55,7 +78,7 @@ class TClientsCache : public TClientsCacheBase { public: - TClientsCache(const TClustersConfig& config, const NApi::TClientOptions& options) + TClientsCache(const TClientsCacheConfigPtr& config, const NApi::TClientOptions& options) : ClustersConfig_(GetClustersConfigWithNormalClusterName(config)) , Options_(options) { } @@ -67,7 +90,7 @@ protected: } private: - const TClustersConfig ClustersConfig_; + const TClientsCacheConfigPtr ClustersConfig_; const NApi::TClientOptions Options_; }; @@ -77,26 +100,30 @@ private: //////////////////////////////////////////////////////////////////////////////// -IClientsCachePtr CreateClientsCache(const TClustersConfig& config, const NApi::TClientOptions& options) +IClientsCachePtr CreateClientsCache(const TClientsCacheConfigPtr& config, const NApi::TClientOptions& options) { return New<TClientsCache>(config, options); } -IClientsCachePtr CreateClientsCache(const TConfig& config, const NApi::TClientOptions& options) +IClientsCachePtr CreateClientsCache( + const TConnectionConfigPtr& config, + const NApi::TClientOptions& options) { - TClustersConfig clustersConfig; - *clustersConfig.MutableDefaultConfig() = config; + auto clustersConfig = New<TClientsCacheConfig>(); + clustersConfig->DefaultConfig = CopyConfig(config); return CreateClientsCache(clustersConfig, options); } -IClientsCachePtr CreateClientsCache(const TConfig& config) +IClientsCachePtr CreateClientsCache(const TConnectionConfigPtr& config) { return CreateClientsCache(config, NApi::GetClientOpsFromEnvStatic()); } IClientsCachePtr CreateClientsCache(const NApi::TClientOptions& options) { - return CreateClientsCache(TClustersConfig(), options); + auto config = New<TClientsCacheConfig>(); + config->SetDefaults(); + return CreateClientsCache(config, options); } IClientsCachePtr CreateClientsCache() diff --git a/yt/yt/client/cache/cache.h b/yt/yt/client/cache/cache.h index d3e1de30b3..aa7a9d5118 100644 --- a/yt/yt/client/cache/cache.h +++ b/yt/yt/client/cache/cache.h @@ -24,13 +24,15 @@ DEFINE_REFCOUNTED_TYPE(IClientsCache) //////////////////////////////////////////////////////////////////////////////// //! Creates clients cache which explicitly given config. Server name is always overwritten with requested. -IClientsCachePtr CreateClientsCache(const TClustersConfig& config, const NApi::TClientOptions& options); +IClientsCachePtr CreateClientsCache(const TClientsCacheConfigPtr& config, const NApi::TClientOptions& options); //! Creates clients cache which shares same config (except server name). -IClientsCachePtr CreateClientsCache(const TConfig& config, const NApi::TClientOptions& options); +IClientsCachePtr CreateClientsCache( + const NApi::NRpcProxy::TConnectionConfigPtr& config, + const NApi::TClientOptions& options); //! Shortcut to use client options from env. -IClientsCachePtr CreateClientsCache(const TConfig& config); +IClientsCachePtr CreateClientsCache(const NApi::NRpcProxy::TConnectionConfigPtr& config); //! Shortcut to create cache with custom options and proxy role. IClientsCachePtr CreateClientsCache(const NApi::TClientOptions& options); @@ -39,7 +41,7 @@ IClientsCachePtr CreateClientsCache(const NApi::TClientOptions& options); IClientsCachePtr CreateClientsCache(); //! Helper function to create one cluster config from cluster URL and clusters config. -TConfig MakeClusterConfig(const TClustersConfig& config, TStringBuf clusterUrl); +NApi::NRpcProxy::TConnectionConfigPtr MakeClusterConfig(const TClientsCacheConfigPtr& config, TStringBuf clusterUrl); //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/client/cache/config.cpp b/yt/yt/client/cache/config.cpp new file mode 100644 index 0000000000..ad79cda00f --- /dev/null +++ b/yt/yt/client/cache/config.cpp @@ -0,0 +1,17 @@ +#include "config.h" + +namespace NYT::NClient::NCache { + +//////////////////////////////////////////////////////////////////////////////// + +void TClientsCacheConfig::Register(TRegistrar registrar) +{ + registrar.Parameter("default_config", &TThis::DefaultConfig) + .DefaultNew(); + registrar.Parameter("cluster_configs", &TThis::ClusterConfigs) + .Default(); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NClient::NHedging::NRpc diff --git a/yt/yt/client/cache/config.h b/yt/yt/client/cache/config.h new file mode 100644 index 0000000000..f05fcdfd96 --- /dev/null +++ b/yt/yt/client/cache/config.h @@ -0,0 +1,31 @@ +#pragma once + +#include "public.h" + +#include <yt/yt/client/api/rpc_proxy/config.h> + +#include <yt/yt/core/ytree/yson_struct.h> + +#include <util/generic/vector.h> + +namespace NYT::NClient::NCache { + +//////////////////////////////////////////////////////////////////////////////// + +struct TClientsCacheConfig + : public virtual NYTree::TYsonStruct +{ + NApi::NRpcProxy::TConnectionConfigPtr DefaultConfig; + + THashMap<TString, NApi::NRpcProxy::TConnectionConfigPtr> ClusterConfigs; + + REGISTER_YSON_STRUCT(TClientsCacheConfig); + + static void Register(TRegistrar registrar); +}; + +DEFINE_REFCOUNTED_TYPE(TClientsCacheConfig) + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NClient::NHedging::NRpc diff --git a/yt/yt/client/cache/public.h b/yt/yt/client/cache/public.h index 0da8de623c..96667e2da8 100644 --- a/yt/yt/client/cache/public.h +++ b/yt/yt/client/cache/public.h @@ -7,9 +7,7 @@ namespace NYT::NClient::NCache { //////////////////////////////////////////////////////////////////////////////// DECLARE_REFCOUNTED_STRUCT(IClientsCache) - -class TConfig; -class TClustersConfig; +DECLARE_REFCOUNTED_STRUCT(TClientsCacheConfig) //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/client/cache/rpc.cpp b/yt/yt/client/cache/rpc.cpp index e1b71c27f2..da65f90cba 100644 --- a/yt/yt/client/cache/rpc.cpp +++ b/yt/yt/client/cache/rpc.cpp @@ -1,7 +1,5 @@ #include "rpc.h" -#include <yt/yt_proto/yt/client/cache/proto/config.pb.h> - #include <yt/yt/client/api/client.h> #include <yt/yt/client/api/options.h> @@ -15,95 +13,7 @@ namespace NYT::NClient::NCache { -//////////////////////////////////////////////////////////////////////////////// - -NCompression::ECodec GetCompressionCodecFromProto(ECompressionCodec protoCodec) -{ - switch (protoCodec) { - case ECompressionCodec::None: - return NCompression::ECodec::None; - case ECompressionCodec::Lz4: - return NCompression::ECodec::Lz4; - } - YT_ABORT(); -} - -NApi::NRpcProxy::TConnectionConfigPtr GetConnectionConfig(const TConfig& config) -{ - auto connectionConfig = New<NApi::NRpcProxy::TConnectionConfig>(); - connectionConfig->SetDefaults(); - - connectionConfig->ClusterUrl = config.GetClusterName(); - if (!config.GetProxyRole().empty()) { - connectionConfig->ProxyRole = config.GetProxyRole(); - } - if (config.GetChannelPoolSize() != 0) { - connectionConfig->DynamicChannelPool->MaxPeerCount = config.GetChannelPoolSize(); - } - if (config.GetChannelPoolRebalanceIntervalSeconds() != 0) { - connectionConfig->DynamicChannelPool->RandomPeerEvictionPeriod = TDuration::Seconds(config.GetChannelPoolRebalanceIntervalSeconds()); - } - if (config.HasEnablePowerOfTwoChoicesStrategy()) { - connectionConfig->DynamicChannelPool->EnablePowerOfTwoChoicesStrategy = config.GetEnablePowerOfTwoChoicesStrategy(); - } - if (config.GetModifyRowsBatchCapacity() != 0) { - connectionConfig->ModifyRowsBatchCapacity = config.GetModifyRowsBatchCapacity(); - } - if (config.HasEnableProxyDiscovery()) { - connectionConfig->EnableProxyDiscovery = config.GetEnableProxyDiscovery(); - } - if (!config.GetProxyAddresses().empty()) { - connectionConfig->ProxyAddresses = FromProto<std::vector<std::string>>(config.GetProxyAddresses()); - } - -#define SET_TIMEOUT_OPTION(name) \ - if (config.Get##name() != 0) connectionConfig->name = TDuration::MilliSeconds(config.Get ## name()) - - SET_TIMEOUT_OPTION(DefaultTransactionTimeout); - SET_TIMEOUT_OPTION(DefaultSelectRowsTimeout); - SET_TIMEOUT_OPTION(DefaultLookupRowsTimeout); - SET_TIMEOUT_OPTION(DefaultTotalStreamingTimeout); - SET_TIMEOUT_OPTION(DefaultStreamingStallTimeout); - SET_TIMEOUT_OPTION(DefaultPingPeriod); - -#undef SET_TIMEOUT_OPTION - - connectionConfig->RequestCodec = GetCompressionCodecFromProto(config.GetRequestCodec()); - connectionConfig->ResponseCodec = GetCompressionCodecFromProto(config.GetResponseCodec()); - connectionConfig->EnableRetries = config.GetEnableRetries(); - - if (config.HasEnableLegacyRpcCodecs()) { - connectionConfig->EnableLegacyRpcCodecs = config.GetEnableLegacyRpcCodecs(); - } - if (config.HasEnableSelectQueryTracingTag()) { - connectionConfig->EnableSelectQueryTracingTag = config.GetEnableSelectQueryTracingTag(); - } - if (config.HasRetryBackoffTime()) { - connectionConfig->RetryingChannel->RetryBackoffTime = TDuration::MilliSeconds(config.GetRetryBackoffTime()); - } - if (config.HasRetryAttempts()) { - connectionConfig->RetryingChannel->RetryAttempts = config.GetRetryAttempts(); - } - if (config.HasRetryTimeout()) { - connectionConfig->RetryingChannel->RetryTimeout = TDuration::MilliSeconds(config.GetRetryTimeout()); - } - - if (config.HasClusterTag()) { - connectionConfig->ClusterTag = NApi::TClusterTag(config.GetClusterTag()); - } - - if (config.HasClockClusterTag()) { - connectionConfig->ClockClusterTag = NObjectClient::TCellTag(config.GetClockClusterTag()); - } - - if (config.HasUdfRegistryPath()) { - connectionConfig->UdfRegistryPath = config.GetUdfRegistryPath(); - } - - connectionConfig->Postprocess(); - - return connectionConfig; -} +using namespace NNet; //////////////////////////////////////////////////////////////////////////////// @@ -130,42 +40,29 @@ void SetClusterUrl(const NApi::NRpcProxy::TConnectionConfigPtr& config, TStringB { auto [cluster, proxyRole] = ExtractClusterAndProxyRole(clusterUrl); if (!proxyRole.empty()) { + // TODO(ignat): avoid using Y_ENSURE Y_ENSURE(!config->ProxyRole || config->ProxyRole.value().empty(), "ProxyRole specified in both: config and url"); config->ProxyRole = ToString(proxyRole); } config->ClusterUrl = ToString(cluster); -} - -void SetClusterUrl(TConfig& config, TStringBuf clusterUrl) -{ - auto [cluster, proxyRole] = ExtractClusterAndProxyRole(clusterUrl); - if (!proxyRole.empty()) { - Y_ENSURE(config.GetProxyRole().empty(), "ProxyRole specified in both: config and url"); - config.SetProxyRole(ToString(proxyRole)); - } - config.SetClusterName(ToString(cluster)); + config->ClusterName = InferYTClusterFromClusterUrl(*config->ClusterUrl); } NApi::IClientPtr CreateClient(const NApi::NRpcProxy::TConnectionConfigPtr& config, const NApi::TClientOptions& options) { + if (config->ClusterName && *(config->ClusterName) == "" && config->ClusterUrl != "") { + THROW_ERROR_EXCEPTION("Connection config has empty cluster name but non-empty cluster URL, it usually means misconfiguration") + << TErrorAttribute("cluster_name", config->ClusterName) + << TErrorAttribute("cluster_url", config->ClusterUrl); + } return NApi::NRpcProxy::CreateConnection(config)->CreateClient(options); } -NApi::IClientPtr CreateClient(const TConfig& config, const NApi::TClientOptions& options) -{ - return CreateClient(GetConnectionConfig(config), options); -} - NApi::IClientPtr CreateClient(const NApi::NRpcProxy::TConnectionConfigPtr& config) { return CreateClient(config, NApi::GetClientOpsFromEnvStatic()); } -NApi::IClientPtr CreateClient(const TConfig& config) -{ - return CreateClient(GetConnectionConfig(config)); -} - NApi::IClientPtr CreateClient(TStringBuf clusterUrl) { return CreateClient(clusterUrl, NApi::GetClientOpsFromEnvStatic()); @@ -178,6 +75,7 @@ NApi::IClientPtr CreateClient(TStringBuf cluster, TStringBuf proxyRole) if (!proxyRole.empty()) { config->ProxyRole = ToString(proxyRole); } + config->Postprocess(); return CreateClient(config); } @@ -188,9 +86,12 @@ NApi::IClientPtr CreateClient() NApi::IClientPtr CreateClient(TStringBuf clusterUrl, const NApi::TClientOptions& options) { - TConfig config; - SetClusterUrl(config, clusterUrl); - return CreateClient(config, options); + auto connectionConfig = New<NApi::NRpcProxy::TConnectionConfig>(); + connectionConfig->SetDefaults(); + + SetClusterUrl(connectionConfig, clusterUrl); + + return CreateClient(connectionConfig, options); } //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/client/cache/rpc.h b/yt/yt/client/cache/rpc.h index 9916b5d7c1..c2a844bb50 100644 --- a/yt/yt/client/cache/rpc.h +++ b/yt/yt/client/cache/rpc.h @@ -3,33 +3,23 @@ #include <yt/yt/client/api/public.h> #include <yt/yt/client/api/rpc_proxy/config.h> -#include <yt/yt_proto/yt/client/cache/proto/config.pb.h> - #include <util/generic/strbuf.h> - namespace NYT::NClient::NCache { //////////////////////////////////////////////////////////////////////////////// -NApi::NRpcProxy::TConnectionConfigPtr GetConnectionConfig(const TConfig& config); - -//////////////////////////////////////////////////////////////////////////////// - //! Helper function to extract ClusterName and ProxyRole from clusterUrl. std::pair<TStringBuf, TStringBuf> ExtractClusterAndProxyRole(TStringBuf clusterUrl); //! Helper function to properly set ClusterName and ProxyRole from clusterUrl. // Expected that clusterUrl will be in format cluster[/proxyRole]. void SetClusterUrl(const NApi::NRpcProxy::TConnectionConfigPtr& config, TStringBuf clusterUrl); -void SetClusterUrl(TConfig& config, TStringBuf clusterUrl); NApi::IClientPtr CreateClient(const NApi::NRpcProxy::TConnectionConfigPtr& config, const NApi::TClientOptions& options); -NApi::IClientPtr CreateClient(const TConfig& config, const NApi::TClientOptions& options); -//! Shortcut to create rpc client with options from env variables +//! Shortcut to create rpc client with options from env variables. NApi::IClientPtr CreateClient(const NApi::NRpcProxy::TConnectionConfigPtr& config); -NApi::IClientPtr CreateClient(const TConfig& config); //! Shortcut to create client to specified cluster. // `clusterUrl` may have format cluster[/proxyRole], diff --git a/yt/yt/client/cache/unittests/cache_ut.cpp b/yt/yt/client/cache/unittests/cache_ut.cpp index 8ec217d5dd..0759d8e636 100644 --- a/yt/yt/client/cache/unittests/cache_ut.cpp +++ b/yt/yt/client/cache/unittests/cache_ut.cpp @@ -1,5 +1,5 @@ #include <yt/yt/client/cache/cache.h> -#include <yt/yt_proto/yt/client/cache/proto/config.pb.h> +#include <yt/yt/client/cache/config.h> #include <library/cpp/testing/gtest/gtest.h> @@ -88,45 +88,50 @@ TEST(TClientsCacheTest, MultiThreads) } } -TEST(TClientsCacheTest, MakeClusterConfig) { - TClustersConfig clustersCfg; - clustersCfg.MutableDefaultConfig()->SetClusterName("seneca-nan"); // will be ignored - clustersCfg.MutableDefaultConfig()->SetProxyRole("default_role"); // can be overwritten - clustersCfg.MutableDefaultConfig()->SetChannelPoolSize(42u); - auto& senecaVlaCfg = (*clustersCfg.MutableClusterConfigs())["seneca-vla"]; - senecaVlaCfg.SetClusterName(""); // will be ignored - senecaVlaCfg.SetProxyRole("seneca_vla_role"); // can be overwritten - senecaVlaCfg.SetChannelPoolSize(43u); +//////////////////////////////////////////////////////////////////////////////// + +TEST(TClientsCacheTest, MakeClusterConfig) +{ + auto clientsCacheConfig = New<TClientsCacheConfig>(); + clientsCacheConfig->DefaultConfig->ClusterUrl = "seneca-nan"; // will be ignored + clientsCacheConfig->DefaultConfig->ProxyRole = "default_role"; // can be overwritten + clientsCacheConfig->DefaultConfig->DynamicChannelPool->MaxPeerCount = 42; + + auto senecaVlaConfig = New<NApi::NRpcProxy::TConnectionConfig>(); + senecaVlaConfig->ClusterUrl = ""; // will be ignored + senecaVlaConfig->ProxyRole = "seneca_vla_role"; // can be overwritten + senecaVlaConfig->DynamicChannelPool->MaxPeerCount = 43; + clientsCacheConfig->ClusterConfigs["seneca-vla"] = senecaVlaConfig; { - auto cfg = MakeClusterConfig(clustersCfg, "seneca-man"); - EXPECT_EQ(cfg.GetClusterName(), "seneca-man"); - EXPECT_EQ(cfg.GetProxyRole(), "default_role"); - EXPECT_EQ(cfg.GetChannelPoolSize(), 42u); + auto config = MakeClusterConfig(clientsCacheConfig, "seneca-man"); + EXPECT_EQ(config->ClusterUrl, "seneca-man"); + EXPECT_EQ(config->ProxyRole, "default_role"); + EXPECT_EQ(config->DynamicChannelPool->MaxPeerCount, 42); } { - auto cfg = MakeClusterConfig(clustersCfg, "seneca-man/overwriting_role"); - EXPECT_EQ(cfg.GetClusterName(), "seneca-man"); - EXPECT_EQ(cfg.GetProxyRole(), "overwriting_role"); - EXPECT_EQ(cfg.GetChannelPoolSize(), 42u); + auto config = MakeClusterConfig(clientsCacheConfig, "seneca-man/overwriting_role"); + EXPECT_EQ(config->ClusterUrl, "seneca-man"); + EXPECT_EQ(config->ProxyRole, "overwriting_role"); + EXPECT_EQ(config->DynamicChannelPool->MaxPeerCount, 42); } { - auto cfg = MakeClusterConfig(clustersCfg, "seneca-vla"); - EXPECT_EQ(cfg.GetClusterName(), "seneca-vla"); - EXPECT_EQ(cfg.GetProxyRole(), "seneca_vla_role"); - EXPECT_EQ(cfg.GetChannelPoolSize(), 43u); + auto config = MakeClusterConfig(clientsCacheConfig, "seneca-vla"); + EXPECT_EQ(config->ClusterUrl, "seneca-vla"); + EXPECT_EQ(config->ProxyRole, "seneca_vla_role"); + EXPECT_EQ(config->DynamicChannelPool->MaxPeerCount, 43); } { - auto cfg = MakeClusterConfig(clustersCfg, "seneca-vla/overwriting_role"); - EXPECT_EQ(cfg.GetClusterName(), "seneca-vla"); - EXPECT_EQ(cfg.GetProxyRole(), "overwriting_role"); - EXPECT_EQ(cfg.GetChannelPoolSize(), 43u); + auto config = MakeClusterConfig(clientsCacheConfig, "seneca-vla/overwriting_role"); + EXPECT_EQ(config->ClusterUrl, "seneca-vla"); + EXPECT_EQ(config->ProxyRole, "overwriting_role"); + EXPECT_EQ(config->DynamicChannelPool->MaxPeerCount, 43); } { - auto cfg = MakeClusterConfig(clustersCfg, "seneca-vla.yt.yandex.net"); - EXPECT_EQ(cfg.GetClusterName(), "seneca-vla.yt.yandex.net"); - EXPECT_EQ(cfg.GetProxyRole(), "seneca_vla_role"); - EXPECT_EQ(cfg.GetChannelPoolSize(), 43u); + auto config = MakeClusterConfig(clientsCacheConfig, "seneca-vla.yt.yandex.net"); + EXPECT_EQ(config->ClusterUrl, "seneca-vla.yt.yandex.net"); + EXPECT_EQ(config->ProxyRole, "seneca_vla_role"); + EXPECT_EQ(config->DynamicChannelPool->MaxPeerCount, 43); } } diff --git a/yt/yt/client/cache/unittests/options_ut.cpp b/yt/yt/client/cache/unittests/options_ut.cpp index 9e6541a515..fd87cacc11 100644 --- a/yt/yt/client/cache/unittests/options_ut.cpp +++ b/yt/yt/client/cache/unittests/options_ut.cpp @@ -14,6 +14,8 @@ namespace NYT::NClient::NCache { //////////////////////////////////////////////////////////////////////////////// +// TODO(ignat): move these tests to yt/yt/client/api/options.h + TEST(TClientOptionsTest, TokenFromFile) { TTempDir tmpDir; diff --git a/yt/yt/client/cache/unittests/rpc_ut.cpp b/yt/yt/client/cache/unittests/rpc_ut.cpp deleted file mode 100644 index 6037d26d7a..0000000000 --- a/yt/yt/client/cache/unittests/rpc_ut.cpp +++ /dev/null @@ -1,50 +0,0 @@ -#include <yt/yt/client/cache/rpc.h> - -#include <library/cpp/testing/gtest/gtest.h> - -namespace NYT::NClient::NCache { - -//////////////////////////////////////////////////////////////////////////////// - -TEST(RpcClientTest, SetClusterUrlWithoutProxy) -{ - TConfig config; - SetClusterUrl(config, "markov"); - EXPECT_EQ("markov", config.GetClusterName()); - EXPECT_EQ("", config.GetProxyRole()); -} - -TEST(RpcClientTest, SetClusterUrlWithProxy) -{ - TConfig config; - SetClusterUrl(config, "markov/bigb"); - EXPECT_EQ("markov", config.GetClusterName()); - EXPECT_EQ("bigb", config.GetProxyRole()); -} - -TEST(RpcClientTest, SetClusterUrlFqdnWithoutProxy) -{ - TConfig config; - SetClusterUrl(config, "https://markov.yt.yandex.net:443"); - EXPECT_EQ("https://markov.yt.yandex.net:443", config.GetClusterName()); - EXPECT_EQ("", config.GetProxyRole()); -} - -TEST(RpcClientTest, SetClusterUrlFqdnWithProxy) -{ - TConfig config; - SetClusterUrl(config, "https://markov.yt.yandex.net:443/bigb"); - EXPECT_EQ("https://markov.yt.yandex.net:443", config.GetClusterName()); - EXPECT_EQ("bigb", config.GetProxyRole()); -} - -TEST(RpcClientTest, ProxyRoleOverride) -{ - TConfig config; - config.SetProxyRole("role"); - EXPECT_THROW(SetClusterUrl(config, "markov/bigb"), std::exception); -} - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NClient::NCache diff --git a/yt/yt/client/cache/unittests/ya.make b/yt/yt/client/cache/unittests/ya.make index c19c4edc92..e74573a87c 100644 --- a/yt/yt/client/cache/unittests/ya.make +++ b/yt/yt/client/cache/unittests/ya.make @@ -5,7 +5,6 @@ INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc) SRCS( cache_ut.cpp options_ut.cpp - rpc_ut.cpp ) INCLUDE(${ARCADIA_ROOT}/yt/opensource.inc) diff --git a/yt/yt/client/cache/ya.make b/yt/yt/client/cache/ya.make index 05b1df8f99..acd9143df8 100644 --- a/yt/yt/client/cache/ya.make +++ b/yt/yt/client/cache/ya.make @@ -3,13 +3,13 @@ LIBRARY() SRCS( cache.cpp cache_base.cpp + config.cpp rpc.cpp ) PEERDIR( yt/yt/core yt/yt/client - yt/yt_proto/yt/client/cache ) END() diff --git a/yt/yt/client/driver/scheduler_commands.cpp b/yt/yt/client/driver/scheduler_commands.cpp index be52ea9770..9cb7396725 100644 --- a/yt/yt/client/driver/scheduler_commands.cpp +++ b/yt/yt/client/driver/scheduler_commands.cpp @@ -136,6 +136,25 @@ void TGetJobSpecCommand::DoExecute(ICommandContextPtr context) void TGetJobStderrCommand::Register(TRegistrar registrar) { registrar.Parameter("job_id", &TThis::JobId); + + registrar.ParameterWithUniversalAccessor<std::optional<i64>>( + "limit", + [] (TThis* command) -> auto& { + return command->Options.Limit; + }) + .Optional(/*init*/ true); + + registrar.ParameterWithUniversalAccessor<std::optional<i64>>( + "offset", + [] (TThis* command) -> auto& { + return command->Options.Offset; + }) + .Optional(/*init*/ true); +} + +bool TGetJobStderrCommand::HasResponseParameters() const +{ + return true; } void TGetJobStderrCommand::DoExecute(ICommandContextPtr context) @@ -143,8 +162,13 @@ void TGetJobStderrCommand::DoExecute(ICommandContextPtr context) auto result = WaitFor(context->GetClient()->GetJobStderr(OperationIdOrAlias, JobId, Options)) .ValueOrThrow(); + ProduceResponseParameters(context, [&] (NYson::IYsonConsumer* consumer) { + BuildYsonMapFragmentFluently(consumer) + .Item("total_size").Value(result.TotalSize) + .Item("end_offset").Value(result.EndOffset); + }); auto output = context->Request().OutputStream; - WaitFor(output->Write(result)) + WaitFor(output->Write(result.Data)) .ThrowOnError(); } diff --git a/yt/yt/client/driver/scheduler_commands.h b/yt/yt/client/driver/scheduler_commands.h index 349b16d750..280d55e331 100644 --- a/yt/yt/client/driver/scheduler_commands.h +++ b/yt/yt/client/driver/scheduler_commands.h @@ -132,6 +132,7 @@ private: NJobTrackerClient::TJobId JobId; void DoExecute(ICommandContextPtr context) override; + bool HasResponseParameters() const override; }; //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/client/federated/cache.cpp b/yt/yt/client/federated/cache.cpp index d686f01767..1cf3347b84 100644 --- a/yt/yt/client/federated/cache.cpp +++ b/yt/yt/client/federated/cache.cpp @@ -10,35 +10,52 @@ namespace NYT::NClient::NFederated { +using namespace NYT::NClient::NCache; + namespace { //////////////////////////////////////////////////////////////////////////////// +// TODO(ignat): move this function to yt/yt/core/ +template<class T> +TIntrusivePtr<T> CopyConfig(const TIntrusivePtr<T>& config) +{ + auto newConfig = New<T>(); + newConfig->Load( + ConvertToNode(config), + /*postprocess*/ false, + /*setDefaults*/ false, + /*path*/ ""); + return newConfig; +} + +//////////////////////////////////////////////////////////////////////////////// + class TClientsCache : public NCache::TClientsCacheBase { public: TClientsCache( - TClustersConfig clustersConfig, + TClientsCacheConfigPtr clientsCacheConfig, NApi::TClientOptions options, TConnectionConfigPtr federationConfig, TString clusterSeparator) - : ClustersConfig_(std::move(clustersConfig)) + : ClientsCacheConfig_(std::move(clientsCacheConfig)) , Options_(std::move(options)) , FederationConfig_(std::move(federationConfig)) , ClusterSeparator_(std::move(clusterSeparator)) - {} + { } protected: NApi::IClientPtr CreateClient(TStringBuf clusterUrl) override { std::vector<std::string> clusters; - NYT::NApi::IClientPtr client; + NApi::IClientPtr client; StringSplitter(clusterUrl).SplitByString(ClusterSeparator_).SkipEmpty().Collect(&clusters); switch (clusters.size()) { case 0: THROW_ERROR_EXCEPTION("Cannot create client without cluster"); case 1: - return NCache::CreateClient(NCache::MakeClusterConfig(ClustersConfig_, clusterUrl), Options_); + return NCache::CreateClient(NCache::MakeClusterConfig(ClientsCacheConfig_, clusterUrl), Options_); default: return CreateFederatedClient(clusters); } @@ -69,14 +86,14 @@ private: if (!FederatedConnection_) { // TODO(ashishkin): use proper invoker here? - NYT::NApi::NRpcProxy::TConnectionOptions options; + NApi::NRpcProxy::TConnectionOptions options; FederatedConnection_ = CreateConnection(FederationConfig_, std::move(options)); } return FederatedConnection_->CreateClient(Options_); } private: - const TClustersConfig ClustersConfig_; + const TClientsCacheConfigPtr ClientsCacheConfig_; const NApi::TClientOptions Options_; const NFederated::TConnectionConfigPtr FederationConfig_; const TString ClusterSeparator_; @@ -89,12 +106,12 @@ private: IClientsCachePtr CreateFederatedClientsCache( TConnectionConfigPtr federatedConfig, - const TClustersConfig& clustersConfig, - const NYT::NApi::TClientOptions& options, + const TClientsCacheConfigPtr& clientsCacheConfig, + const NApi::TClientOptions& options, TString clusterSeparator) { return NYT::New<TClientsCache>( - clustersConfig, + clientsCacheConfig, options, std::move(federatedConfig), std::move(clusterSeparator)); @@ -102,15 +119,15 @@ IClientsCachePtr CreateFederatedClientsCache( IClientsCachePtr CreateFederatedClientsCache( TConnectionConfigPtr federatedConfig, - const TConfig& config, - const NYT::NApi::TClientOptions& options, + const NApi::NRpcProxy::TConnectionConfigPtr& cacheConfig, + const NApi::TClientOptions& options, TString clusterSeparator) { - TClustersConfig clustersConfig; - *clustersConfig.MutableDefaultConfig() = config; + auto clientsCacheConfig = New<TClientsCacheConfig>(); + clientsCacheConfig->DefaultConfig = CopyConfig(cacheConfig); return NYT::New<TClientsCache>( - std::move(clustersConfig), + std::move(clientsCacheConfig), options, std::move(federatedConfig), std::move(clusterSeparator)); diff --git a/yt/yt/client/federated/cache.h b/yt/yt/client/federated/cache.h index ab244a4a1f..db6110b0f5 100644 --- a/yt/yt/client/federated/cache.h +++ b/yt/yt/client/federated/cache.h @@ -3,14 +3,11 @@ #include "config.h" #include <yt/yt/client/cache/cache.h> +#include <yt/yt/client/cache/config.h> namespace NYT::NClient::NFederated { -using NCache::IClientsCachePtr; -using NCache::TClustersConfig; -using NCache::TConfig; - //////////////////////////////////////////////////////////////////////////////// //! Creates clients cache for generic and federated clients. @@ -18,9 +15,9 @@ using NCache::TConfig; //! Which client to create decided by cluster url: if several clusters concatenated by clusterSeparator requested then //! federated client it is, generic client otherwise. //! For example, for "markov" generic client will be created, and for "seneca-sas+seneca-vla" federated one. -IClientsCachePtr CreateFederatedClientsCache( +NCache::IClientsCachePtr CreateFederatedClientsCache( TConnectionConfigPtr federatedConfig, - const TClustersConfig& clustersConfig, + const NCache::TClientsCacheConfigPtr& clientsCacheConfig, const NYT::NApi::TClientOptions& options, TString clusterSeparator = "+"); @@ -29,9 +26,9 @@ IClientsCachePtr CreateFederatedClientsCache( //! Which client to create decided by cluster url: if several clusters concatenated by clusterSeparator requested then //! federated client it is, generic client otherwise. //! For example, for "markov" generic client will be created, and for "seneca-sas+seneca-vla" federated one. -IClientsCachePtr CreateFederatedClientsCache( +NCache::IClientsCachePtr CreateFederatedClientsCache( TConnectionConfigPtr federatedConfig, - const TConfig& config, + const NApi::NRpcProxy::TConnectionConfigPtr& cacheConfig, const NYT::NApi::TClientOptions& options, TString clusterSeparator = "+"); diff --git a/yt/yt/client/federated/client.cpp b/yt/yt/client/federated/client.cpp index 4f221deedb..fe1992162b 100644 --- a/yt/yt/client/federated/client.cpp +++ b/yt/yt/client/federated/client.cpp @@ -406,7 +406,7 @@ public: UNIMPLEMENTED_METHOD(TFuture<NConcurrency::IAsyncZeroCopyInputStreamPtr>, GetJobInput, (NJobTrackerClient::TJobId, const TGetJobInputOptions&)); UNIMPLEMENTED_METHOD(TFuture<NYson::TYsonString>, GetJobInputPaths, (NJobTrackerClient::TJobId, const TGetJobInputPathsOptions&)); UNIMPLEMENTED_METHOD(TFuture<NYson::TYsonString>, GetJobSpec, (NJobTrackerClient::TJobId, const TGetJobSpecOptions&)); - UNIMPLEMENTED_METHOD(TFuture<TSharedRef>, GetJobStderr, (const NScheduler::TOperationIdOrAlias&, NJobTrackerClient::TJobId, const TGetJobStderrOptions&)); + UNIMPLEMENTED_METHOD(TFuture<TGetJobStderrResponse>, GetJobStderr, (const NScheduler::TOperationIdOrAlias&, NJobTrackerClient::TJobId, const TGetJobStderrOptions&)); UNIMPLEMENTED_METHOD(TFuture<TSharedRef>, GetJobFailContext, (const NScheduler::TOperationIdOrAlias&, NJobTrackerClient::TJobId, const TGetJobFailContextOptions&)); UNIMPLEMENTED_METHOD(TFuture<TListOperationsResult>, ListOperations, (const TListOperationsOptions&)); UNIMPLEMENTED_METHOD(TFuture<TListJobsResult>, ListJobs, (const NScheduler::TOperationIdOrAlias&, const TListJobsOptions&)); diff --git a/yt/yt/client/federated/unittests/cache_ut.cpp b/yt/yt/client/federated/unittests/cache_ut.cpp index 5f5fdf1a91..f66d41a383 100644 --- a/yt/yt/client/federated/unittests/cache_ut.cpp +++ b/yt/yt/client/federated/unittests/cache_ut.cpp @@ -5,8 +5,6 @@ #include <yt/yt/core/misc/error.h> -#include <yt/yt_proto/yt/client/cache/proto/config.pb.h> - #include <library/cpp/testing/gtest/gtest.h> #include <util/system/env.h> @@ -14,6 +12,7 @@ namespace NYT::NClient::NFederated { using namespace NYT::NApi; +using namespace NYT::NClient::NCache; //////////////////////////////////////////////////////////////////////////////// @@ -22,7 +21,7 @@ TEST(TFederatedClientsCacheTest, GetSameClient) SetEnv("YT_TOKEN", "AAAA-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"); auto ytClientsCache = CreateFederatedClientsCache( New<TConnectionConfig>(), - TClustersConfig{}, + New<TClientsCacheConfig>(), NApi::GetClientOpsFromEnvStatic()); auto client1 = ytClientsCache->GetClient("localhost"); @@ -41,7 +40,7 @@ TEST(TFederatedClientsCacheTest, GetFederatedWithEmptyConfig) SetEnv("YT_TOKEN", "AAAA-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"); auto ytClientsCache = CreateFederatedClientsCache( New<TConnectionConfig>(), - TClustersConfig{}, + New<TClientsCacheConfig>(), NApi::GetClientOpsFromEnvStatic()); EXPECT_THROW( @@ -61,7 +60,7 @@ TEST(TFederatedClientsCacheTest, ConfigurationAndClusterUrlMismatch1) auto ytClientsCache = CreateFederatedClientsCache( connectionConfig, - TClustersConfig{}, + New<TClientsCacheConfig>(), NApi::GetClientOpsFromEnvStatic()); EXPECT_THROW( @@ -83,7 +82,7 @@ TEST(TFederatedClientsCacheTest, ConfigurationAndClusterUrlMismatch2) auto ytClientsCache = CreateFederatedClientsCache( connectionConfig, - TClustersConfig{}, + New<TClientsCacheConfig>(), NApi::GetClientOpsFromEnvStatic()); EXPECT_THROW( @@ -103,7 +102,7 @@ TEST(TFederatedClientsCacheTest, ConfigurationMissingCluster) auto ytClientsCache = CreateFederatedClientsCache( connectionConfig, - TClustersConfig{}, + New<TClientsCacheConfig>(), NApi::GetClientOpsFromEnvStatic()); EXPECT_THROW( diff --git a/yt/yt/client/hedging/cache.h b/yt/yt/client/hedging/cache.h index 8ec59fb2ae..51719f786b 100644 --- a/yt/yt/client/hedging/cache.h +++ b/yt/yt/client/hedging/cache.h @@ -13,8 +13,6 @@ namespace NYT::NClient::NHedging::NRpc { //////////////////////////////////////////////////////////////////////////////// -using NCache::TConfig; -using NCache::TClustersConfig; using NCache::CreateClientsCache; //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/client/hedging/config.cpp b/yt/yt/client/hedging/config.cpp index 715b6599e0..083e2d72bc 100644 --- a/yt/yt/client/hedging/config.cpp +++ b/yt/yt/client/hedging/config.cpp @@ -24,4 +24,24 @@ void THedgingClientOptions::Register(TRegistrar registrar) //////////////////////////////////////////////////////////////////////////////// +void TReplicationLagPenaltyProviderOptions::Register(TRegistrar registrar) +{ + registrar.Parameter("replica_clusters", &TThis::ReplicaClusters) + .NonEmpty(); + registrar.Parameter("table_path", &TThis::TablePath); + registrar.Parameter("lag_penalty", &TThis::LagPenalty) + .Default(TDuration::MilliSeconds(10)); + registrar.Parameter("max_tablet_lag", &TThis::MaxTabletLag) + .Default(TDuration::Minutes(5)); + registrar.Parameter("max_tablets_with_lag_fraction", &TThis::MaxTabletsWithLagFraction) + .Default(0.05) + .InRange(0.0, 1.0); + registrar.Parameter("check_period", &TThis::CheckPeriod) + .Default(TDuration::Minutes(1)); + registrar.Parameter("clear_penalties_on_errors", &TThis::ClearPenaltiesOnErrors) + .Default(false); +} + +//////////////////////////////////////////////////////////////////////////////// + } // namespace NYT::NClient::NHedging::NRpc diff --git a/yt/yt/client/hedging/config.h b/yt/yt/client/hedging/config.h index 23c11d1ba3..982dbc8cf7 100644 --- a/yt/yt/client/hedging/config.h +++ b/yt/yt/client/hedging/config.h @@ -22,6 +22,7 @@ struct TConnectionWithPenaltyConfig static void Register(TRegistrar registrar); }; +DEFINE_REFCOUNTED_TYPE(TConnectionWithPenaltyConfig) //////////////////////////////////////////////////////////////////////////////// @@ -44,4 +45,36 @@ DEFINE_REFCOUNTED_TYPE(THedgingClientOptions) //////////////////////////////////////////////////////////////////////////////// +struct TReplicationLagPenaltyProviderOptions + : public virtual NYTree::TYsonStruct +{ + // Clusters that need checks for replication lag. + std::vector<TString> ReplicaClusters; + + // Table that needs checks for replication lag. + TString TablePath; + + // Same as BanPenalty in hedging client. + TDuration LagPenalty; + // Tablet is considered "lagged" if CurrentTimestamp - TabletLastReplicationTimestamp >= MaxTabletLag. + TDuration MaxTabletLag; + + // Real value from 0.0 to 1.0. Replica cluster receives LagPenalty if NumberOfTabletsWithLag >= MaxTabletsWithLagFraction * TotalNumberOfTablets. + double MaxTabletsWithLagFraction; + + // Replication lag check period. + TDuration CheckPeriod; + + // In case of any errors from master client - clear all penalties. + bool ClearPenaltiesOnErrors; + + REGISTER_YSON_STRUCT(TReplicationLagPenaltyProviderOptions); + + static void Register(TRegistrar registrar); +}; + +DEFINE_REFCOUNTED_TYPE(TReplicationLagPenaltyProviderOptions) + +//////////////////////////////////////////////////////////////////////////////// + } // namespace NYT::NClient::NHedging::NRpc diff --git a/yt/yt/client/hedging/counter.cpp b/yt/yt/client/hedging/counter.cpp index 2b96cc366d..8ea65e8a34 100644 --- a/yt/yt/client/hedging/counter.cpp +++ b/yt/yt/client/hedging/counter.cpp @@ -28,17 +28,17 @@ TCounter::TCounter(const NProfiling::TTagSet& tagSet) //////////////////////////////////////////////////////////////////////////////// -TLagPenaltyProviderCounters::TLagPenaltyProviderCounters(const NProfiling::TRegistry& registry, const TVector<TString>& clusters) +TLagPenaltyProviderCounters::TLagPenaltyProviderCounters(const NProfiling::TRegistry& registry, const std::vector<TString>& clusters) : SuccessRequestCount(registry.Counter("/update_success")) , ErrorRequestCount(registry.Counter("/update_error")) - , TotalTabletsCount(registry.Gauge("/tablets_total")) + , TotalTabletCount(registry.Gauge("/tablets_total")) { for (const auto& cluster : clusters) { - LagTabletsCount.emplace(cluster, registry.WithTag("yt_cluster", cluster).Gauge("/tablets_with_lag")); + TabletWithLagCountPerReplica.emplace(cluster, registry.WithTag("yt_cluster", cluster).Gauge("/tablets_with_lag")); } } -TLagPenaltyProviderCounters::TLagPenaltyProviderCounters(const TString& tablePath, const TVector<TString>& clusterNames) +TLagPenaltyProviderCounters::TLagPenaltyProviderCounters(const TString& tablePath, const std::vector<TString>& clusterNames) : TLagPenaltyProviderCounters(LagPenaltyProviderProfiler.WithTag("table", tablePath), clusterNames) { } diff --git a/yt/yt/client/hedging/counter.h b/yt/yt/client/hedging/counter.h index b0633cf73a..adc01bd536 100644 --- a/yt/yt/client/hedging/counter.h +++ b/yt/yt/client/hedging/counter.h @@ -34,13 +34,14 @@ DEFINE_REFCOUNTED_TYPE(TCounter) // ! Counters for TReplicationLagPenaltyProvider. struct TLagPenaltyProviderCounters final { - TLagPenaltyProviderCounters(const NProfiling::TRegistry& registry, const TVector<TString>& clusters); - TLagPenaltyProviderCounters(const TString& tablePath, const TVector<TString>& replicaClusters); + TLagPenaltyProviderCounters(const NProfiling::TRegistry& registry, const std::vector<TString>& clusters); + TLagPenaltyProviderCounters(const TString& tablePath, const std::vector<TString>& replicaClusters); NProfiling::TCounter SuccessRequestCount; NProfiling::TCounter ErrorRequestCount; - THashMap<TString, NProfiling::TGauge> LagTabletsCount; // cluster -> # of tablets - NProfiling::TGauge TotalTabletsCount; + // Mapping from cluster replica to number of tablets with lag. + THashMap<TString, NProfiling::TGauge> TabletWithLagCountPerReplica; + NProfiling::TGauge TotalTabletCount; }; DEFINE_REFCOUNTED_TYPE(TLagPenaltyProviderCounters) diff --git a/yt/yt/client/hedging/hedging.cpp b/yt/yt/client/hedging/hedging.cpp index ca5dee6917..7c24690fa4 100644 --- a/yt/yt/client/hedging/hedging.cpp +++ b/yt/yt/client/hedging/hedging.cpp @@ -21,8 +21,6 @@ #include <yt/yt/core/rpc/dispatcher.h> -#include <yt/yt_proto/yt/client/hedging/proto/config.pb.h> - #include <util/datetime/base.h> #include <util/generic/va_args.h> @@ -165,7 +163,7 @@ public: UNSUPPORTED_METHOD(TFuture<NConcurrency::IAsyncZeroCopyInputStreamPtr>, GetJobInput, (NJobTrackerClient::TJobId, const TGetJobInputOptions&)); UNSUPPORTED_METHOD(TFuture<NYson::TYsonString>, GetJobInputPaths, (NJobTrackerClient::TJobId, const TGetJobInputPathsOptions&)); UNSUPPORTED_METHOD(TFuture<NYson::TYsonString>, GetJobSpec, (NJobTrackerClient::TJobId, const TGetJobSpecOptions&)); - UNSUPPORTED_METHOD(TFuture<TSharedRef>, GetJobStderr, (const NScheduler::TOperationIdOrAlias&, NJobTrackerClient::TJobId, const TGetJobStderrOptions&)); + UNSUPPORTED_METHOD(TFuture<TGetJobStderrResponse>, GetJobStderr, (const NScheduler::TOperationIdOrAlias&, NJobTrackerClient::TJobId, const TGetJobStderrOptions&)); UNSUPPORTED_METHOD(TFuture<TSharedRef>, GetJobFailContext, (const NScheduler::TOperationIdOrAlias&, NJobTrackerClient::TJobId, const TGetJobFailContextOptions&)); UNSUPPORTED_METHOD(TFuture<TListOperationsResult>, ListOperations, (const TListOperationsOptions&)); UNSUPPORTED_METHOD(TFuture<TListJobsResult>, ListJobs, (const NScheduler::TOperationIdOrAlias&, const TListJobsOptions&)); @@ -308,46 +306,4 @@ NApi::IClientPtr CreateHedgingClient(const THedgingClientOptionsPtr& config, con //////////////////////////////////////////////////////////////////////////////// -THedgingClientOptionsPtr GetHedgingClientConfig(const THedgingClientConfig& protoConfig) -{ - auto config = New<THedgingClientOptions>(); - config->BanPenalty = TDuration::MilliSeconds(protoConfig.GetBanPenalty()); - config->BanDuration = TDuration::MilliSeconds(protoConfig.GetBanDuration()); - - NProfiling::TTagSet counterTagSet; - - for (const auto& [tagName, tagValue] : protoConfig.GetTags()) { - config->Tags.emplace(tagName, tagValue); - } - - config->Connections.reserve(protoConfig.GetClients().size()); - for (const auto& client : protoConfig.GetClients()) { - auto connectionConfig = ConvertTo<NYT::TIntrusivePtr<TConnectionWithPenaltyConfig>>( - GetConnectionConfig(client.GetClientConfig())); - connectionConfig->InitialPenalty = TDuration::MilliSeconds(client.GetInitialPenalty()); - config->Connections.push_back(std::move(connectionConfig)); - } - return config; -} - -NApi::IClientPtr CreateHedgingClient(const THedgingClientConfig& config) -{ - return CreateHedgingClient(GetHedgingClientConfig(config)); -} - -NApi::IClientPtr CreateHedgingClient(const THedgingClientConfig& config, const IClientsCachePtr& clientsCache) -{ - return CreateHedgingClient(GetHedgingClientConfig(config), clientsCache); -} - -NApi::IClientPtr CreateHedgingClient( - const THedgingClientConfig& config, - const IClientsCachePtr& clientsCache, - const IPenaltyProviderPtr& penaltyProvider) -{ - return CreateHedgingClient(GetHedgingClientConfig(config), clientsCache, penaltyProvider); -} - -//////////////////////////////////////////////////////////////////////////////// - } // namespace NYT::NClient::NHedging::NRpc diff --git a/yt/yt/client/hedging/hedging.h b/yt/yt/client/hedging/hedging.h index 23c127e040..53ffc3633d 100644 --- a/yt/yt/client/hedging/hedging.h +++ b/yt/yt/client/hedging/hedging.h @@ -33,11 +33,9 @@ namespace NYT::NClient::NHedging::NRpc { //////////////////////////////////////////////////////////////////////////////// +//TODO(ignat): drop this using using NCache::IClientsCachePtr; -// from config.proto -class THedgingClientConfig; - //////////////////////////////////////////////////////////////////////////////// NApi::IClientPtr CreateHedgingClient(const THedgingExecutorPtr& hedgingExecutor); @@ -58,20 +56,6 @@ NApi::IClientPtr CreateHedgingClient( const IClientsCachePtr& clientsCache, const IPenaltyProviderPtr& penaltyProvider); -// The following methods should be moved to `ads/bsyeti/libs/ytex/client`. - -//! Method for creating HedgingClient options from given config and preinitialized clients. -THedgingClientOptionsPtr GetHedgingClientConfig(const THedgingClientConfig& config); - -//! Method for creating HedgingClient with given rpc clients config. -NApi::IClientPtr CreateHedgingClient(const THedgingClientConfig& config); - -//! Method for creating HedgingClient with given rpc clients config and preinitialized clients. -NApi::IClientPtr CreateHedgingClient(const THedgingClientConfig& config, const IClientsCachePtr& clientsCache); - -//! Method for creating HedgingClient with given rpc clients config, preinitialized clients and PenaltyProvider. -NApi::IClientPtr CreateHedgingClient(const THedgingClientConfig& config, const IClientsCachePtr& clientsCache, const IPenaltyProviderPtr& penaltyProvider); - //////////////////////////////////////////////////////////////////////////////// } // namespace NYT::NClient::NHedging::NRpc diff --git a/yt/yt/client/hedging/penalty_provider.cpp b/yt/yt/client/hedging/penalty_provider.cpp index f4b717fffb..4ef88ac8bc 100644 --- a/yt/yt/client/hedging/penalty_provider.cpp +++ b/yt/yt/client/hedging/penalty_provider.cpp @@ -4,8 +4,6 @@ #include "private.h" #include "public.h" -#include <yt/yt_proto/yt/client/hedging/proto/config.pb.h> - #include <yt/yt/client/transaction_client/helpers.h> #include <yt/yt/core/concurrency/periodic_executor.h> @@ -21,6 +19,9 @@ namespace NYT::NClient::NHedging::NRpc { +using namespace NConcurrency; +using namespace NYTree; + //////////////////////////////////////////////////////////////////////////////// namespace { @@ -43,30 +44,28 @@ class TLagPenaltyProvider : public IPenaltyProvider { public: - TLagPenaltyProvider(const TReplicationLagPenaltyProviderConfig& config, NApi::IClientPtr client) - : TablePath_(config.GetTablePath()) - , MaxTabletLag_(TDuration::Seconds(config.GetMaxTabletLag())) - , LagPenalty_(TDuration::MilliSeconds(config.GetLagPenalty())) - , MaxTabletsWithLagFraction_(config.GetMaxTabletsWithLagFraction()) + TLagPenaltyProvider( + const TReplicationLagPenaltyProviderOptionsPtr& config, + NApi::IClientPtr client) + : Config_(config) , Client_(client) - , ClearPenaltiesOnErrors_(config.GetClearPenaltiesOnErrors()) - , Counters_(New<TLagPenaltyProviderCounters>(TablePath_, - TVector<TString>{config.GetReplicaClusters().begin(), config.GetReplicaClusters().end()})) - , Executor_(New<NConcurrency::TPeriodicExecutor>( + , Counters_(New<TLagPenaltyProviderCounters>(Config_->TablePath, Config_->ReplicaClusters)) + , Executor_(New<TPeriodicExecutor>( NYT::NRpc::TDispatcher::Get()->GetLightInvoker(), BIND(&TLagPenaltyProvider::UpdateCurrentLagPenalty, MakeWeak(this)), - TDuration::Seconds(config.GetCheckPeriod()))) + Config_->CheckPeriod)) { YT_VERIFY(Executor_); YT_VERIFY(Client_); - for (const auto& cluster : config.GetReplicaClusters()) { + for (const auto& cluster : Config_->ReplicaClusters) { auto [_, inserted] = ReplicaClusters_.try_emplace(cluster); - THROW_ERROR_EXCEPTION_UNLESS(inserted, "Replica cluster %v is listed twice", cluster); + THROW_ERROR_EXCEPTION_UNLESS(inserted, "Duplicate replica cluster %v", cluster); } GetNodeOptions_.Timeout = TDuration::Seconds(5); GetNodeOptions_.ReadFrom = NApi::EMasterChannelKind::Cache; + Executor_->Start(); } @@ -84,31 +83,37 @@ public: // Fills ReplicaIds in ReplicaClusters_. void UpdateReplicaIds() { - auto replicasNode = NYTree::ConvertToNode(NConcurrency::WaitFor(Client_->GetNode(TablePath_ + "/@replicas", GetNodeOptions_)).ValueOrThrow())->AsMap(); + auto replicasYson = WaitFor(Client_->GetNode(Config_->TablePath + "/@replicas", GetNodeOptions_)) + .ValueOrThrow(); + auto replicasNode = ConvertToNode(replicasYson); - for (const auto& row : replicasNode->GetChildren()) { - auto cluster = row.second->AsMap()->GetChildOrThrow("cluster_name")->AsString()->GetValue(); + for (const auto& [key, child] : replicasNode->AsMap()->GetChildren()) { + auto cluster = child->AsMap()->GetChildOrThrow("cluster_name")->AsString()->GetValue(); if (auto* info = ReplicaClusters_.FindPtr(cluster)) { - info->ReplicaId = NTabletClient::TTableReplicaId::FromString(row.first); + info->ReplicaId = NTabletClient::TTableReplicaId::FromString(key); YT_LOG_INFO("Found replica (ReplicaId: %v, Cluster: %v, Table: %v)", info->ReplicaId, cluster, - TablePath_); + Config_->TablePath); }; } - CheckAllReplicaIdsPresent().ThrowOnError(); + CheckAllReplicaIdsPresent() + .ThrowOnError(); } - ui64 GetTotalNumberOfTablets() + int GetTotalNumberOfTablets() { - return NYTree::ConvertTo<ui64>(NConcurrency::WaitFor(Client_->GetNode(TablePath_ + "/@tablet_count", GetNodeOptions_)).ValueOrThrow()); + auto tabletCountNode = WaitFor(Client_->GetNode(Config_->TablePath + "/@tablet_count", GetNodeOptions_)) + .ValueOrThrow(); + return ConvertTo<int>(tabletCountNode); } // Returns a map: ReplicaId -> # of tablets. - THashMap<NTabletClient::TTableReplicaId, ui64> CalculateNumbersOfTabletsWithLag(const ui64 tabletsCount) + THashMap<NTabletClient::TTableReplicaId, ui64> CalculateTabletWithLagCounts(int tabletCount) { - auto tabletsRange = xrange(tabletsCount); - auto tabletsInfo = NConcurrency::WaitFor(Client_->GetTabletInfos(TablePath_, {tabletsRange.begin(), tabletsRange.end()})).ValueOrThrow(); + auto tabletsRange = xrange(tabletCount); + auto tabletsInfo = WaitFor(Client_->GetTabletInfos(Config_->TablePath, {tabletsRange.begin(), tabletsRange.end()})) + .ValueOrThrow(); const auto now = TInstant::Now(); THashMap<NTabletClient::TTableReplicaId, ui64> tabletsWithLag; @@ -120,7 +125,7 @@ public: for (const auto& replicaInfo : *tabletInfo.TableReplicaInfos) { auto lastReplicationTimestamp = TInstant::Seconds(NTransactionClient::UnixTimeFromTimestamp(replicaInfo.LastReplicationTimestamp)); - if (now - lastReplicationTimestamp > MaxTabletLag_) { + if (now - lastReplicationTimestamp > Config_->MaxTabletLag) { ++tabletsWithLag[replicaInfo.ReplicaId]; } } @@ -129,53 +134,55 @@ public: return tabletsWithLag; } - TDuration CalculateLagPenalty(const ui64 tabletsCount, const ui64 tabletsWithLag) + TDuration CalculateLagPenalty(int tabletCount, int tabletWithLagCount) { - return tabletsWithLag >= tabletsCount * MaxTabletsWithLagFraction_ ? LagPenalty_ : TDuration::Zero(); + return tabletWithLagCount >= Config_->MaxTabletsWithLagFraction * tabletCount + ? Config_->LagPenalty + : TDuration::Zero(); } void UpdateCurrentLagPenalty() { try { YT_LOG_INFO("Start penalty updater check (Table: %v)", - TablePath_); + Config_->TablePath); if (!CheckAllReplicaIdsPresent().IsOK()) { UpdateReplicaIds(); } - auto tabletsCount = GetTotalNumberOfTablets(); - auto tabletsWithLag = CalculateNumbersOfTabletsWithLag(tabletsCount); + auto tabletCount = GetTotalNumberOfTablets(); + auto tabletWithLagCountPerReplica = CalculateTabletWithLagCounts(tabletCount); - Counters_->TotalTabletsCount.Update(tabletsCount); + Counters_->TotalTabletCount.Update(tabletCount); for (auto& [cluster, info] : ReplicaClusters_) { Y_ASSERT(info.ReplicaId); - auto curTabletsWithLag = tabletsWithLag.Value(info.ReplicaId, 0); - auto newLagPenalty = CalculateLagPenalty(tabletsCount, curTabletsWithLag); + auto tabletWithLagCount = tabletWithLagCountPerReplica.Value(info.ReplicaId, 0); + auto newLagPenalty = CalculateLagPenalty(tabletCount, tabletWithLagCount); info.CurrentLagPenalty.store(newLagPenalty.GetValue(), std::memory_order::relaxed); - Counters_->LagTabletsCount.at(cluster).Update(curTabletsWithLag); - YT_LOG_INFO("Finish penalty updater check (Cluster: %v: Table: %v, TabletsWithLag: %v/%v, Penalty: %v)", + GetOrCrash(Counters_->TabletWithLagCountPerReplica, cluster).Update(tabletWithLagCount); + YT_LOG_INFO("Lag penalty for cluster replica updated (Cluster: %v, Table: %v, TabletWithLagCount: %v/%v, Penalty: %v)", cluster, - TablePath_, - curTabletsWithLag, - tabletsCount, + Config_->TablePath, + tabletWithLagCount, + tabletCount, newLagPenalty); } Counters_->SuccessRequestCount.Increment(); - } catch (const std::exception& err) { + } catch (const std::exception& ex) { Counters_->ErrorRequestCount.Increment(); - YT_LOG_ERROR(err, "Cannot calculate lag (Table: %v)", - TablePath_); + YT_LOG_ERROR(ex, "Failed to update lag penalty (Table: %v)", + Config_->TablePath); - if (ClearPenaltiesOnErrors_) { + if (Config_->ClearPenaltiesOnErrors) { for (auto& [cluster, info] : ReplicaClusters_) { info.CurrentLagPenalty.store(0, std::memory_order::relaxed); - YT_LOG_INFO("Clearing penalty (Cluster: %v, Table: %v)", + YT_LOG_INFO("Clear lag penalty for cluster replica (Cluster: %v, Table: %v)", cluster, - TablePath_); + Config_->TablePath); } } } @@ -201,17 +208,14 @@ private: std::atomic<ui64> CurrentLagPenalty = 0; }; - const TString TablePath_; + + TReplicationLagPenaltyProviderOptionsPtr Config_; THashMap<std::string, TReplicaInfo> ReplicaClusters_; - const TDuration MaxTabletLag_; - const TDuration LagPenalty_; - const float MaxTabletsWithLagFraction_; NApi::IClientPtr Client_; - const bool ClearPenaltiesOnErrors_; TLagPenaltyProviderCountersPtr Counters_; NApi::TGetNodeOptions GetNodeOptions_; - NConcurrency::TPeriodicExecutorPtr Executor_; + TPeriodicExecutorPtr Executor_; }; //////////////////////////////////////////////////////////////////////////////// @@ -226,7 +230,7 @@ IPenaltyProviderPtr CreateDummyPenaltyProvider() } IPenaltyProviderPtr CreateReplicationLagPenaltyProvider( - TReplicationLagPenaltyProviderConfig config, + TReplicationLagPenaltyProviderOptionsPtr config, NApi::IClientPtr client) { return New<TLagPenaltyProvider>(std::move(config), std::move(client)); diff --git a/yt/yt/client/hedging/penalty_provider.h b/yt/yt/client/hedging/penalty_provider.h index 32071c908b..dd36dc6567 100644 --- a/yt/yt/client/hedging/penalty_provider.h +++ b/yt/yt/client/hedging/penalty_provider.h @@ -1,6 +1,7 @@ #pragma once #include "public.h" +#include "config.h" #include <yt/yt/client/api/client.h> @@ -25,14 +26,11 @@ DEFINE_REFCOUNTED_TYPE(IPenaltyProvider) // @brief DummyPenaltyProvider - always returns 0. IPenaltyProviderPtr CreateDummyPenaltyProvider(); -// From config.proto. -class TReplicationLagPenaltyProviderConfig; - // @brief ReplicationLagPenaltyProvider - periodically checks replication lag for given table AND replica cluster. // Based on values from TReplicationLagPenaltyProviderConfig add current number of tablets with lag, it either returns 0 or LagPenalty value. // Master client - main cluster with replicated table. ReplicaCluster + TablePath specifies concrete replica for table from main cluster. IPenaltyProviderPtr CreateReplicationLagPenaltyProvider( - TReplicationLagPenaltyProviderConfig config, + TReplicationLagPenaltyProviderOptionsPtr config, NApi::IClientPtr client); //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/client/hedging/public.h b/yt/yt/client/hedging/public.h index ee569cc5bd..120837641b 100644 --- a/yt/yt/client/hedging/public.h +++ b/yt/yt/client/hedging/public.h @@ -15,6 +15,8 @@ DECLARE_REFCOUNTED_STRUCT(TLagPenaltyProviderCounters) // TODO(bulatman) Rename to THedgingClientConfig. DECLARE_REFCOUNTED_STRUCT(THedgingClientOptions) +DECLARE_REFCOUNTED_STRUCT(TConnectionWithPenaltyConfig) +DECLARE_REFCOUNTED_STRUCT(TReplicationLagPenaltyProviderOptions) DECLARE_REFCOUNTED_STRUCT(IPenaltyProvider) diff --git a/yt/yt/client/hedging/rpc.h b/yt/yt/client/hedging/rpc.h index 19f3541fe5..89b99ba6d3 100644 --- a/yt/yt/client/hedging/rpc.h +++ b/yt/yt/client/hedging/rpc.h @@ -4,10 +4,6 @@ namespace NYT::NClient::NHedging::NRpc { -using NCache::TConfig; -using NCache::TClustersConfig; -using NCache::ECompressionCodec; -using NCache::GetConnectionConfig; using NCache::ExtractClusterAndProxyRole; using NCache::SetClusterUrl; using NCache::CreateClient; diff --git a/yt/yt/client/hedging/unittests/hedging_ut.cpp b/yt/yt/client/hedging/unittests/hedging_ut.cpp index e2e49d46b2..56309405d8 100644 --- a/yt/yt/client/hedging/unittests/hedging_ut.cpp +++ b/yt/yt/client/hedging/unittests/hedging_ut.cpp @@ -3,7 +3,6 @@ #include <yt/yt/client/hedging/cache.h> #include <yt/yt/client/hedging/counter.h> #include <yt/yt/client/hedging/hedging.h> -#include <yt/yt_proto/yt/client/hedging/proto/config.pb.h> #include <yt/yt/client/transaction_client/helpers.h> @@ -65,15 +64,15 @@ IPenaltyProviderPtr CreateReplicationLagPenaltyProvider( const bool clearPenaltiesOnErrors = false, const TDuration checkPeriod = CheckPeriod) { - TReplicationLagPenaltyProviderConfig config; + auto config = New<TReplicationLagPenaltyProviderOptions>(); - config.SetTablePath(path); - config.AddReplicaClusters(TProtobufString(cluster)); - config.SetMaxTabletsWithLagFraction(0.5); - config.SetMaxTabletLag(maxTabletLag.Seconds()); - config.SetCheckPeriod(checkPeriod.Seconds()); - config.SetLagPenalty(lagPenalty.MilliSeconds()); - config.SetClearPenaltiesOnErrors(clearPenaltiesOnErrors); + config->TablePath = path; + config->ReplicaClusters.push_back(TString(cluster)); + config->MaxTabletsWithLagFraction = 0.5; + config->MaxTabletLag = maxTabletLag; + config->CheckPeriod = checkPeriod; + config->LagPenalty = lagPenalty; + config->ClearPenaltiesOnErrors = clearPenaltiesOnErrors; return CreateReplicationLagPenaltyProvider(config, client); } @@ -442,12 +441,15 @@ TEST(THedgingClientTest, CreatingHedgingClientWithPreinitializedClients) auto mockClientsCache = New<StrictMock<TMockClientsCache>>(); EXPECT_CALL(*mockClientsCache, GetClient(clusterName)).WillOnce(Return(mockClient)); - THedgingClientConfig hedgingClientConfig; - hedgingClientConfig.SetBanDuration(100); - hedgingClientConfig.SetBanPenalty(200); - auto clientOptions = hedgingClientConfig.AddClients(); - clientOptions->SetInitialPenalty(0); - clientOptions->MutableClientConfig()->SetClusterName(clusterName); + auto hedgingClientConfig = New<THedgingClientOptions>(); + hedgingClientConfig->BanDuration = TDuration::MilliSeconds(100); + hedgingClientConfig->BanPenalty = TDuration::MilliSeconds(200); + + auto connectionConfig = New<TConnectionWithPenaltyConfig>(); + connectionConfig->InitialPenalty = TDuration(); + connectionConfig->ClusterUrl = clusterName; + + hedgingClientConfig->Connections.push_back(connectionConfig); auto hedgingClient = CreateHedgingClient(hedgingClientConfig, mockClientsCache); diff --git a/yt/yt/client/hedging/unittests/penalty_provider_ut.cpp b/yt/yt/client/hedging/unittests/penalty_provider_ut.cpp index 9a0e6a21d2..d2318896e7 100644 --- a/yt/yt/client/hedging/unittests/penalty_provider_ut.cpp +++ b/yt/yt/client/hedging/unittests/penalty_provider_ut.cpp @@ -1,5 +1,4 @@ #include <yt/yt/client/hedging/penalty_provider.h> -#include <yt/yt_proto/yt/client/hedging/proto/config.pb.h> #include <yt/yt/client/transaction_client/helpers.h> @@ -22,21 +21,21 @@ using TStrictMockClient = StrictMock<NApi::TMockClient>; namespace { const auto CheckPeriod = TDuration::Seconds(1); - TReplicationLagPenaltyProviderConfig GenerateReplicationLagPenaltyProviderConfig( - const NYPath::TYPath& path, - const std::string& cluster, - const ui32 maxLagInSeconds = 10, - const bool clearPenaltiesOnErrors = false, - const TDuration checkPeriod = CheckPeriod) + TReplicationLagPenaltyProviderOptionsPtr GenerateReplicationLagPenaltyProviderConfig( + const NYPath::TYPath& path, + const std::string& cluster, + const TDuration maxLagInSeconds = TDuration::Seconds(10), + const bool clearPenaltiesOnErrors = false, + const TDuration checkPeriod = CheckPeriod) { - TReplicationLagPenaltyProviderConfig config; + auto config = New<TReplicationLagPenaltyProviderOptions>(); - config.SetTablePath(path); - config.AddReplicaClusters(TProtobufString(cluster)); - config.SetMaxTabletsWithLagFraction(0.5); - config.SetMaxTabletLag(maxLagInSeconds); - config.SetCheckPeriod(checkPeriod.Seconds()); - config.SetClearPenaltiesOnErrors(clearPenaltiesOnErrors); + config->TablePath = path; + config->ReplicaClusters.push_back(TString(cluster)); + config->MaxTabletsWithLagFraction = 0.5; + config->MaxTabletLag = maxLagInSeconds; + config->CheckPeriod = checkPeriod; + config->ClearPenaltiesOnErrors = clearPenaltiesOnErrors; return config; } @@ -52,8 +51,8 @@ TEST(TLagPenaltyProviderTest, UpdateExternalPenaltyWhenReplicaHasLag) NYson::TYsonString replicasResult(TStringBuf("{\"575f-131-40502c5-201b420f\" = {\"cluster_name\" = \"seneca-vla\"}}")); NYson::TYsonString tabletCountResult(TStringBuf("1")); - ui32 maxLagInSeconds = 10; - TReplicationLagPenaltyProviderConfig config = GenerateReplicationLagPenaltyProviderConfig(path, cluster, maxLagInSeconds); + auto maxLagInSeconds = TDuration::Seconds(10); + auto config = GenerateReplicationLagPenaltyProviderConfig(path, cluster, maxLagInSeconds); auto client = New<TStrictMockClient>(); @@ -67,7 +66,8 @@ TEST(TLagPenaltyProviderTest, UpdateExternalPenaltyWhenReplicaHasLag) tabletInfos[0].TableReplicaInfos = std::make_optional(std::vector<NApi::TTabletInfo::TTableReplicaInfo>()); auto& replicaTabletsInfo = tabletInfos[0].TableReplicaInfos->emplace_back(); replicaTabletsInfo.ReplicaId = NTabletClient::TTableReplicaId::FromString("575f-131-40502c5-201b420f"); - replicaTabletsInfo.LastReplicationTimestamp = NTransactionClient::TimestampFromUnixTime(TInstant::Now().Seconds() - 2 * maxLagInSeconds); + replicaTabletsInfo.LastReplicationTimestamp = NTransactionClient::TimestampFromUnixTime( + (TInstant::Now() - 2 * maxLagInSeconds).Seconds()); EXPECT_CALL(*client, GetTabletInfos(path, _, _)) .WillRepeatedly(Return(MakeFuture(tabletInfos))); @@ -75,7 +75,7 @@ TEST(TLagPenaltyProviderTest, UpdateExternalPenaltyWhenReplicaHasLag) auto PenaltyProviderPtr = CreateReplicationLagPenaltyProvider(config, client); Sleep(2 * CheckPeriod); - EXPECT_EQ(PenaltyProviderPtr->Get(cluster), TDuration::MilliSeconds(config.GetLagPenalty())); + EXPECT_EQ(PenaltyProviderPtr->Get(cluster), config->LagPenalty); } TEST(TLagPenaltyProviderTest, DoNotUpdatePenaltyWhenReplicaHasNoLag) @@ -86,7 +86,7 @@ TEST(TLagPenaltyProviderTest, DoNotUpdatePenaltyWhenReplicaHasNoLag) NYson::TYsonString replicasResult(TStringBuf("{\"575f-131-40502c5-201b420f\" = {\"cluster_name\" = \"seneca-vla\"}}")); NYson::TYsonString tabletCountResult(TStringBuf("1")); - TReplicationLagPenaltyProviderConfig config = GenerateReplicationLagPenaltyProviderConfig(path, cluster); + auto config = GenerateReplicationLagPenaltyProviderConfig(path, cluster); auto client = New<TStrictMockClient>(); @@ -116,7 +116,7 @@ TEST(TLagPenaltyProviderTest, DoNotUpdatePenaltyWhenGetReplicaIdFailed) NYPath::TYPath path = "/test/1234"; TString cluster = "seneca-vla"; - NClient::NHedging::NRpc::TReplicationLagPenaltyProviderConfig config = GenerateReplicationLagPenaltyProviderConfig(path, cluster); + auto config = GenerateReplicationLagPenaltyProviderConfig(path, cluster); auto client = New<TStrictMockClient>(); @@ -136,7 +136,7 @@ TEST(TLagPenaltyProviderTest, DoNotUpdatePenaltyWhenGetTabletsCountFailed) NYson::TYsonString replicasResult(TStringBuf("{\"575f-131-40502c5-201b420f\" = {\"cluster_name\" = \"seneca-vla\"}}")); - TReplicationLagPenaltyProviderConfig config = GenerateReplicationLagPenaltyProviderConfig(path, cluster); + auto config = GenerateReplicationLagPenaltyProviderConfig(path, cluster); auto client = New<TStrictMockClient>(); @@ -160,8 +160,7 @@ TEST(TLagPenaltyProviderTest, DoNotUpdatePenaltyWhenGetTabletsInfoFailed) NYson::TYsonString replicasResult(TStringBuf("{\"575f-131-40502c5-201b420f\" = {\"cluster_name\" = \"seneca-vla\"}}")); NYson::TYsonString tabletCountResult(TStringBuf("1")); - NClient::NHedging::NRpc::TReplicationLagPenaltyProviderConfig config = - GenerateReplicationLagPenaltyProviderConfig(path, cluster); + auto config = GenerateReplicationLagPenaltyProviderConfig(path, cluster); auto client = New<TStrictMockClient>(); @@ -188,7 +187,7 @@ TEST(TLagPenaltyProviderTest, ClearPenaltiesAfterError) NYson::TYsonString replicasResult(TStringBuf("{\"575f-131-40502c5-201b420f\" = {\"cluster_name\" = \"seneca-vla\"}}")); NYson::TYsonString tabletCountResult(TStringBuf("1")); - ui32 maxLagInSeconds = 10; + TDuration maxLagInSeconds = TDuration::Seconds(10); auto config = GenerateReplicationLagPenaltyProviderConfig(path, cluster, maxLagInSeconds, true, 2 * CheckPeriod); @@ -205,7 +204,8 @@ TEST(TLagPenaltyProviderTest, ClearPenaltiesAfterError) tabletInfos[0].TableReplicaInfos = std::make_optional(std::vector<NApi::TTabletInfo::TTableReplicaInfo>()); auto& replicaTabletsInfo = tabletInfos[0].TableReplicaInfos->emplace_back(); replicaTabletsInfo.ReplicaId = NTabletClient::TTableReplicaId::FromString("575f-131-40502c5-201b420f"); - replicaTabletsInfo.LastReplicationTimestamp = NTransactionClient::TimestampFromUnixTime(TInstant::Now().Seconds() - 2 * maxLagInSeconds); + replicaTabletsInfo.LastReplicationTimestamp = NTransactionClient::TimestampFromUnixTime( + (TInstant::Now() - 2 * maxLagInSeconds).Seconds()); EXPECT_CALL(*client, GetTabletInfos(path, _, _)) .WillRepeatedly(Return(MakeFuture(tabletInfos))); @@ -213,7 +213,7 @@ TEST(TLagPenaltyProviderTest, ClearPenaltiesAfterError) auto PenaltyProviderPtr = CreateReplicationLagPenaltyProvider(config, client); Sleep(CheckPeriod); - EXPECT_EQ(PenaltyProviderPtr->Get(cluster), TDuration::MilliSeconds(config.GetLagPenalty())); + EXPECT_EQ(PenaltyProviderPtr->Get(cluster), config->LagPenalty); Sleep(2 * CheckPeriod); EXPECT_EQ(PenaltyProviderPtr->Get(cluster), TDuration::Zero()); diff --git a/yt/yt/client/hedging/ya.make b/yt/yt/client/hedging/ya.make index ec442d3f47..9572e465fd 100644 --- a/yt/yt/client/hedging/ya.make +++ b/yt/yt/client/hedging/ya.make @@ -16,7 +16,6 @@ PEERDIR( yt/yt/client yt/yt/client/cache yt/yt/library/profiling - yt/yt_proto/yt/client/hedging ) END() diff --git a/yt/yt/client/table_client/adapters.cpp b/yt/yt/client/table_client/adapters.cpp index 0e78c2f861..0309bbbc48 100644 --- a/yt/yt/client/table_client/adapters.cpp +++ b/yt/yt/client/table_client/adapters.cpp @@ -21,7 +21,7 @@ using NProfiling::TWallTimer; //////////////////////////////////////////////////////////////////////////////// -const NLogging::TLogger Logger("TableClientAdapters"); +YT_DEFINE_GLOBAL(const NLogging::TLogger, Logger, "TableClientAdapters"); //////////////////////////////////////////////////////////////////////////////// @@ -237,7 +237,7 @@ void PipeReaderToWriterByBatches( } catch (const std::exception& ex) { YT_LOG_ERROR(ex, "Failed to transfer batches from reader to writer"); - THROW_ERROR_EXCEPTION(ex); + throw; } } diff --git a/yt/yt/client/transaction_client/remote_timestamp_provider.cpp b/yt/yt/client/transaction_client/remote_timestamp_provider.cpp index d523aaf188..f856c462d3 100644 --- a/yt/yt/client/transaction_client/remote_timestamp_provider.cpp +++ b/yt/yt/client/transaction_client/remote_timestamp_provider.cpp @@ -38,7 +38,7 @@ IChannelPtr CreateTimestampProviderChannel( auto channel = CreateBalancingChannel( config, std::move(channelFactory), - std::move(endpointDescription), + endpointDescription, std::move(endpointAttributes)); channel = CreateRetryingChannel( config, diff --git a/yt/yt/client/unittests/mock/client.h b/yt/yt/client/unittests/mock/client.h index f6290bded7..b2b1dae28e 100644 --- a/yt/yt/client/unittests/mock/client.h +++ b/yt/yt/client/unittests/mock/client.h @@ -620,7 +620,7 @@ public: const TGetJobSpecOptions& options), (override)); - MOCK_METHOD(TFuture<TSharedRef>, GetJobStderr, ( + MOCK_METHOD(TFuture<TGetJobStderrResponse>, GetJobStderr, ( const NScheduler::TOperationIdOrAlias& operationIdOrAlias, NJobTrackerClient::TJobId jobId, const TGetJobStderrOptions& options), diff --git a/yt/yt/core/bus/bus.h b/yt/yt/core/bus/bus.h index 5140baf33f..6c0ded55a4 100644 --- a/yt/yt/core/bus/bus.h +++ b/yt/yt/core/bus/bus.h @@ -70,7 +70,7 @@ struct IBus { //! Returns a textual representation of the bus' endpoint. //! Typically used for logging. - virtual const TString& GetEndpointDescription() const = 0; + virtual const std::string& GetEndpointDescription() const = 0; //! Returns the bus' endpoint attributes. //! Typically used for constructing errors. diff --git a/yt/yt/core/bus/client.h b/yt/yt/core/bus/client.h index ea91efd822..8c23346b95 100644 --- a/yt/yt/core/bus/client.h +++ b/yt/yt/core/bus/client.h @@ -22,7 +22,7 @@ struct IBusClient { //! Returns a textual representation of the bus' endpoint. //! Typically used for logging. - virtual const TString& GetEndpointDescription() const = 0; + virtual const std::string& GetEndpointDescription() const = 0; //! Returns the bus' endpoint attributes. //! Typically used for constructing errors. diff --git a/yt/yt/core/bus/tcp/client.cpp b/yt/yt/core/bus/tcp/client.cpp index 3fc419d7d9..f871d42c68 100644 --- a/yt/yt/core/bus/tcp/client.cpp +++ b/yt/yt/core/bus/tcp/client.cpp @@ -51,7 +51,7 @@ public: Connection_->Terminate(TError(NBus::EErrorCode::TransportError, "Bus terminated")); } - const TString& GetEndpointDescription() const override + const std::string& GetEndpointDescription() const override { VERIFY_THREAD_AFFINITY_ANY(); return Connection_->GetEndpointDescription(); @@ -161,7 +161,7 @@ public: .EndMap()); } - const TString& GetEndpointDescription() const override + const std::string& GetEndpointDescription() const override { return EndpointDescription_; } @@ -220,7 +220,7 @@ private: const IMemoryUsageTrackerPtr MemoryUsageTracker_; - TString EndpointDescription_; + std::string EndpointDescription_; IAttributeDictionaryPtr EndpointAttributes_; }; diff --git a/yt/yt/core/bus/tcp/connection.cpp b/yt/yt/core/bus/tcp/connection.cpp index 871d039ae9..b71fbb9363 100644 --- a/yt/yt/core/bus/tcp/connection.cpp +++ b/yt/yt/core/bus/tcp/connection.cpp @@ -106,7 +106,7 @@ TTcpConnection::TTcpConnection( TConnectionId id, SOCKET socket, EMultiplexingBand multiplexingBand, - const TString& endpointDescription, + const std::string& endpointDescription, const IAttributeDictionary& endpointAttributes, const TNetworkAddress& endpointNetworkAddress, const std::optional<std::string>& endpointAddress, @@ -654,7 +654,7 @@ void TTcpConnection::OnDialerFinished(const TErrorOr<SOCKET>& socketOrError) } } -const TString& TTcpConnection::GetEndpointDescription() const +const std::string& TTcpConnection::GetEndpointDescription() const { return EndpointDescription_; } diff --git a/yt/yt/core/bus/tcp/connection.h b/yt/yt/core/bus/tcp/connection.h index 0559282f6c..d5d631f3b2 100644 --- a/yt/yt/core/bus/tcp/connection.h +++ b/yt/yt/core/bus/tcp/connection.h @@ -80,7 +80,7 @@ public: TConnectionId id, SOCKET socket, EMultiplexingBand multiplexingBand, - const TString& endpointDescription, + const std::string& endpointDescription, const NYTree::IAttributeDictionary& endpointAttributes, const NNet::TNetworkAddress& endpointNetworkAddress, const std::optional<std::string>& endpointAddress, @@ -104,7 +104,7 @@ public: void OnShutdown() override; // IBus implementation. - const TString& GetEndpointDescription() const override; + const std::string& GetEndpointDescription() const override; const NYTree::IAttributeDictionary& GetEndpointAttributes() const override; const std::string& GetEndpointAddress() const override; const NNet::TNetworkAddress& GetEndpointNetworkAddress() const override; @@ -187,7 +187,7 @@ private: const TBusConfigPtr Config_; const EConnectionType ConnectionType_; const TConnectionId Id_; - const TString EndpointDescription_; + const std::string EndpointDescription_; const NYTree::IAttributeDictionaryPtr EndpointAttributes_; const NNet::TNetworkAddress EndpointNetworkAddress_; const std::optional<TString> EndpointAddress_; diff --git a/yt/yt/core/bus/tcp/packet.cpp b/yt/yt/core/bus/tcp/packet.cpp index 039c005e8a..0d673b66bd 100644 --- a/yt/yt/core/bus/tcp/packet.cpp +++ b/yt/yt/core/bus/tcp/packet.cpp @@ -6,6 +6,8 @@ #include <library/cpp/yt/string/guid.h> +#include <library/cpp/yt/misc/unaligned.h> + namespace NYT::NBus { //////////////////////////////////////////////////////////////////////////////// @@ -82,8 +84,6 @@ protected: TCompactVector<char, TypicalVariableHeaderSize> VariableHeader_; size_t VariableHeaderSize_; - ui32* PartSizes_; - ui64* PartChecksums_; int PartIndex_ = -1; TSharedRefArray Message_; @@ -148,6 +148,32 @@ protected: { return static_cast<TDerived*>(this); } + + + ui32 GetPartSize(int index) const + { + return UnalignedLoad(PartSizes_ + index); + } + + void SetPartSize(int index, ui32 size) + { + UnalignedStore(PartSizes_ + index, size); + } + + + ui64 GetPartChecksum(int index) const + { + return UnalignedLoad(PartChecksums_ + index); + } + + void SetPartChecksum(int index, ui64 checksum) + { + UnalignedStore(PartChecksums_ + index, checksum); + } + +private: + ui32* PartSizes_; + ui64* PartChecksums_; }; //////////////////////////////////////////////////////////////////////////////// @@ -283,7 +309,7 @@ private: bool EndVariableHeaderPhase() { if (VerifyChecksum_) { - auto expectedChecksum = PartChecksums_[FixedHeader_.PartCount]; + auto expectedChecksum = GetPartChecksum(FixedHeader_.PartCount); if (expectedChecksum != NullChecksum) { auto actualChecksum = GetVariableChecksum(); if (expectedChecksum != actualChecksum) { @@ -295,7 +321,7 @@ private: } for (int index = 0; index < static_cast<int>(FixedHeader_.PartCount); ++index) { - ui32 partSize = PartSizes_[index]; + ui32 partSize = GetPartSize(index); if (partSize != NullPacketPartSize && partSize > MaxMessagePartSize) { YT_LOG_ERROR("Invalid packet part size (PacketId: %v, PartIndex: %v, PartSize: %v)", FixedHeader_.PacketId, @@ -312,7 +338,7 @@ private: bool EndMessagePartPhase() { if (VerifyChecksum_) { - auto expectedChecksum = PartChecksums_[PartIndex_]; + auto expectedChecksum = GetPartChecksum(PartIndex_); if (expectedChecksum != NullChecksum) { auto actualChecksum = GetChecksum(Parts_[PartIndex_]); if (expectedChecksum != actualChecksum) { @@ -337,7 +363,7 @@ private: break; } - ui32 partSize = PartSizes_[PartIndex_]; + ui32 partSize = GetPartSize(PartIndex_); if (partSize == NullPacketPartSize) { Parts_.push_back(TSharedRef()); } else if (partSize == 0) { @@ -411,19 +437,18 @@ public: AllocateVariableHeader(); for (int index = 0; index < static_cast<int>(Message_.Size()); ++index) { - const auto& part = Message_[index]; - if (part) { - PartSizes_[index] = part.Size(); - PartChecksums_[index] = generateChecksums && index < checksummedPartCount - ? GetChecksum(part) - : NullChecksum; + if (const auto& part = Message_[index]) { + SetPartSize(index, part.Size()); + SetPartChecksum( + index, + generateChecksums && index < checksummedPartCount ? GetChecksum(part) : NullChecksum); } else { - PartSizes_[index] = NullPacketPartSize; - PartChecksums_[index] = NullChecksum; + SetPartSize(index, NullPacketPartSize); + SetPartChecksum(index, NullChecksum); } } - PartChecksums_[Message_.Size()] = generateChecksums ? GetVariableChecksum() : NullChecksum; + SetPartChecksum(Message_.Size(), generateChecksums ? GetVariableChecksum() : NullChecksum); } BeginPhase(EPacketPhase::FixedHeader, &FixedHeader_, sizeof (TPacketHeader)); diff --git a/yt/yt/core/concurrency/unittests/profiled_fair_share_invoker_pool_ut.cpp b/yt/yt/core/concurrency/unittests/profiled_fair_share_invoker_pool_ut.cpp index d1b3ab09d5..6aab403a3e 100644 --- a/yt/yt/core/concurrency/unittests/profiled_fair_share_invoker_pool_ut.cpp +++ b/yt/yt/core/concurrency/unittests/profiled_fair_share_invoker_pool_ut.cpp @@ -14,6 +14,8 @@ #include <yt/yt/library/profiling/solomon/exporter.h> +#include <library/cpp/json/yson/json2yson.h> + #include <util/datetime/base.h> #include <algorithm> @@ -666,15 +668,7 @@ public: auto GetSensors(TString json) { - for (auto& c : json) { - if (c == ':') { - c = '='; - } else if (c == ',') { - c = ';'; - } - } - - auto yson = NYson::TYsonString(json); + auto yson = NYson::TYsonString(NJson2Yson::SerializeJsonValueAsYson(NJson::ReadJsonFastTree(json))); auto list = NYTree::ConvertToNode(yson)->AsMap()->FindChild("sensors"); @@ -730,7 +724,7 @@ public: THashMap<TString, int> invokerNameToDequeued = invokerNameToEnqueued; - for (const auto& entry : GetSensors(json)) { + for (const auto& entry : GetSensors(std::move(json))) { auto mapEntry = entry->AsMap(); auto labels = mapEntry->FindChild("labels")->AsMap(); diff --git a/yt/yt/core/concurrency/unittests/ya.make b/yt/yt/core/concurrency/unittests/ya.make index ae8a5c6f99..b8b94dcfc9 100644 --- a/yt/yt/core/concurrency/unittests/ya.make +++ b/yt/yt/core/concurrency/unittests/ya.make @@ -48,6 +48,8 @@ INCLUDE(${ARCADIA_ROOT}/yt/opensource.inc) PEERDIR( yt/yt/core yt/yt/core/test_framework + + library/cpp/json/yson ) REQUIREMENTS( diff --git a/yt/yt/core/http/server.cpp b/yt/yt/core/http/server.cpp index 48462c7f8d..7464803307 100644 --- a/yt/yt/core/http/server.cpp +++ b/yt/yt/core/http/server.cpp @@ -12,6 +12,8 @@ #include <yt/yt/core/concurrency/thread_pool_poller.h> #include <yt/yt/core/misc/finally.h> +#include <yt/yt/core/misc/memory_usage_tracker.h> +#include <yt/yt/core/misc/public.h> #include <yt/yt/core/ytree/convert.h> @@ -56,20 +58,22 @@ class TServer { public: TServer( - const TServerConfigPtr& config, - const IListenerPtr& listener, - const IPollerPtr& poller, - const IPollerPtr& acceptor, - const IInvokerPtr& invoker, - const IRequestPathMatcherPtr& requestPathMatcher, + TServerConfigPtr config, + IListenerPtr listener, + IPollerPtr poller, + IPollerPtr acceptor, + IInvokerPtr invoker, + IMemoryUsageTrackerPtr memoryUsageTracker, + IRequestPathMatcherPtr requestPathMatcher, bool ownPoller = false) - : Config_(config) - , Listener_(listener) - , Poller_(poller) - , Acceptor_(acceptor) - , Invoker_(invoker) - , RequestPathMatcher_(requestPathMatcher) + : Config_(std::move(config)) + , Listener_(std::move(listener)) + , Poller_(std::move(poller)) + , Acceptor_(std::move(acceptor)) + , Invoker_(std::move(invoker)) + , MemoryUsageTracker_(std::move(memoryUsageTracker)) , OwnPoller_(ownPoller) + , RequestPathMatcher_(std::move(requestPathMatcher)) { } void AddHandler(const TString& path, const IHttpHandlerPtr& handler) override @@ -122,9 +126,10 @@ private: const IPollerPtr Poller_; const IPollerPtr Acceptor_; const IInvokerPtr Invoker_; - IRequestPathMatcherPtr RequestPathMatcher_; + const IMemoryUsageTrackerPtr MemoryUsageTracker_; const bool OwnPoller_ = false; + IRequestPathMatcherPtr RequestPathMatcher_; bool Started_ = false; std::atomic<bool> Stopped_ = false; @@ -220,6 +225,14 @@ private: SetRequestId(response, request->GetRequestId()); + if (MemoryUsageTracker_ && MemoryUsageTracker_->IsExceeded()) { + THROW_ERROR_EXCEPTION( + EStatusCode::TooManyRequests, + "Request is dropped due to high memory pressure") + << TErrorAttribute("total_memory_limit", MemoryUsageTracker_->GetLimit()) + << TErrorAttribute("memory_usage", MemoryUsageTracker_->GetUsed()); + } + handler->HandleRequest(request, response); NTracing::FlushCurrentTraceContextElapsedTime(); @@ -379,36 +392,46 @@ private: //////////////////////////////////////////////////////////////////////////////// IServerPtr CreateServer( - const TServerConfigPtr& config, - const IListenerPtr& listener, - const IPollerPtr& poller, - const IPollerPtr& acceptor, - const IInvokerPtr& invoker, + TServerConfigPtr config, + IListenerPtr listener, + IPollerPtr poller, + IPollerPtr acceptor, + IInvokerPtr invoker, + IMemoryUsageTrackerPtr memoryUsageTracker, bool ownPoller) { auto handlers = New<TRequestPathMatcher>(); return New<TServer>( - config, - listener, - poller, - acceptor, - invoker, - handlers, + std::move(config), + std::move(listener), + std::move(poller), + std::move(acceptor), + std::move(invoker), + std::move(memoryUsageTracker), + std::move(handlers), ownPoller); } IServerPtr CreateServer( - const TServerConfigPtr& config, - const IPollerPtr& poller, - const IPollerPtr& acceptor, - const IInvokerPtr& invoker, + TServerConfigPtr config, + IPollerPtr poller, + IPollerPtr acceptor, + IInvokerPtr invoker, + IMemoryUsageTrackerPtr memoryUsageTracker, bool ownPoller) { auto address = TNetworkAddress::CreateIPv6Any(config->Port); for (int i = 0;; ++i) { try { auto listener = CreateListener(address, poller, acceptor, config->MaxBacklogSize); - return CreateServer(config, listener, poller, acceptor, invoker, ownPoller); + return CreateServer( + std::move(config), + std::move(listener), + std::move(poller), + std::move(acceptor), + std::move(invoker), + std::move(memoryUsageTracker), + ownPoller); } catch (const std::exception& ex) { if (i + 1 == config->BindRetryCount) { throw; @@ -425,80 +448,98 @@ IServerPtr CreateServer( //////////////////////////////////////////////////////////////////////////////// IServerPtr CreateServer( - const TServerConfigPtr& config, - const IListenerPtr& listener, - const IPollerPtr& poller) + TServerConfigPtr config, + IListenerPtr listener, + IPollerPtr poller) { + auto acceptor = poller; + auto invoker = poller->GetInvoker(); return CreateServer( - config, - listener, - poller, - poller, - poller->GetInvoker(), + std::move(config), + std::move(listener), + std::move(poller), + std::move(acceptor), + std::move(invoker), + /*memoryUsageTracker*/ GetNullMemoryUsageTracker(), /*ownPoller*/ false); } IServerPtr CreateServer( - const TServerConfigPtr& config, - const IListenerPtr& listener, - const IPollerPtr& poller, - const IPollerPtr& acceptor) + TServerConfigPtr config, + IListenerPtr listener, + IPollerPtr poller, + IPollerPtr acceptor, + IMemoryUsageTrackerPtr memoryUsageTracker) { + auto invoker = poller->GetInvoker(); return CreateServer( - config, - listener, - poller, - acceptor, - poller->GetInvoker(), + std::move(config), + std::move(listener), + std::move(poller), + std::move(acceptor), + std::move(invoker), + std::move(memoryUsageTracker), /*ownPoller*/ false); } -IServerPtr CreateServer(const TServerConfigPtr& config, const IPollerPtr& poller, const IPollerPtr& acceptor) +IServerPtr CreateServer( + TServerConfigPtr config, + IPollerPtr poller, + IPollerPtr acceptor, + IMemoryUsageTrackerPtr memoryUsageTracker) { + auto invoker = poller->GetInvoker(); return CreateServer( - config, - poller, - acceptor, - poller->GetInvoker(), + std::move(config), + std::move(poller), + std::move(acceptor), + std::move(invoker), + std::move(memoryUsageTracker), /*ownPoller*/ false); } -IServerPtr CreateServer(const TServerConfigPtr& config, const IPollerPtr& poller) +IServerPtr CreateServer(TServerConfigPtr config, IPollerPtr poller) { + auto acceptor = poller; return CreateServer( - config, - poller, - poller); + std::move(config), + std::move(poller), + std::move(acceptor)); } -IServerPtr CreateServer(int port, const IPollerPtr& poller) +IServerPtr CreateServer(int port, IPollerPtr poller) { auto config = New<TServerConfig>(); config->Port = port; - return CreateServer(config, poller); + return CreateServer(std::move(config), std::move(poller)); } -IServerPtr CreateServer(const TServerConfigPtr& config, int pollerThreadCount) +IServerPtr CreateServer(TServerConfigPtr config, int pollerThreadCount) { auto poller = CreateThreadPoolPoller(pollerThreadCount, config->ServerName); + auto acceptor = poller; + auto invoker = poller->GetInvoker(); return CreateServer( - config, - poller, - poller, - poller->GetInvoker(), + std::move(config), + std::move(poller), + std::move(acceptor), + std::move(invoker), + /*memoryUsageTracker*/ GetNullMemoryUsageTracker(), /*ownPoller*/ true); } IServerPtr CreateServer( - const TServerConfigPtr& config, - const NConcurrency::IPollerPtr& poller, - const IInvokerPtr& invoker) + TServerConfigPtr config, + NConcurrency::IPollerPtr poller, + IInvokerPtr invoker) { + auto acceptor = poller; return CreateServer( - config, - poller, - poller, - invoker, + std::move(config), + std::move(poller), + std::move(acceptor), + std::move(invoker), + /*memoryUsageTracker*/ GetNullMemoryUsageTracker(), /*ownPoller*/ false); } diff --git a/yt/yt/core/http/server.h b/yt/yt/core/http/server.h index 3006f0333f..171fb70399 100644 --- a/yt/yt/core/http/server.h +++ b/yt/yt/core/http/server.h @@ -9,6 +9,8 @@ #include <yt/yt/core/actions/future.h> +#include <yt/yt/core/misc/public.h> + #include <library/cpp/yt/memory/ref.h> namespace NYT::NHttp { @@ -80,31 +82,33 @@ DEFINE_REFCOUNTED_TYPE(IServer) //////////////////////////////////////////////////////////////////////////////// IServerPtr CreateServer( - const TServerConfigPtr& config, - const NNet::IListenerPtr& listener, - const NConcurrency::IPollerPtr& poller); + TServerConfigPtr config, + NNet::IListenerPtr listener, + NConcurrency::IPollerPtr poller); IServerPtr CreateServer( - const TServerConfigPtr& config, - const NNet::IListenerPtr& listener, - const NConcurrency::IPollerPtr& poller, - const NConcurrency::IPollerPtr& acceptor); + TServerConfigPtr config, + NNet::IListenerPtr listener, + NConcurrency::IPollerPtr poller, + NConcurrency::IPollerPtr acceptor, + IMemoryUsageTrackerPtr memoryTracker = GetNullMemoryUsageTracker()); IServerPtr CreateServer( - const TServerConfigPtr& config, - const NConcurrency::IPollerPtr& poller); + TServerConfigPtr config, + NConcurrency::IPollerPtr poller); IServerPtr CreateServer( - const TServerConfigPtr& config, - const NConcurrency::IPollerPtr& poller, - const NConcurrency::IPollerPtr& acceptor); + TServerConfigPtr config, + NConcurrency::IPollerPtr poller, + NConcurrency::IPollerPtr acceptor, + IMemoryUsageTrackerPtr memoryTracker = GetNullMemoryUsageTracker()); IServerPtr CreateServer( int port, - const NConcurrency::IPollerPtr& poller); + NConcurrency::IPollerPtr poller); IServerPtr CreateServer( - const TServerConfigPtr& config, + TServerConfigPtr config, int pollerThreadCount = 1); IServerPtr CreateServer( - const TServerConfigPtr& config, - const NConcurrency::IPollerPtr& poller, - const IInvokerPtr& invoker); + TServerConfigPtr config, + NConcurrency::IPollerPtr poller, + IInvokerPtr invoker); //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/core/https/server.cpp b/yt/yt/core/https/server.cpp index 06a34a7176..b97cb801e8 100644 --- a/yt/yt/core/https/server.cpp +++ b/yt/yt/core/https/server.cpp @@ -112,7 +112,8 @@ IServerPtr CreateServer( const TServerConfigPtr& config, const IPollerPtr& poller, const IPollerPtr& acceptor, - const IInvokerPtr& controlInvoker) + const IInvokerPtr& controlInvoker, + const IMemoryUsageTrackerPtr& memoryTracker) { auto sslContext = New<TSslContext>(); ApplySslConfig(sslContext, config->Credentials); @@ -160,7 +161,12 @@ IServerPtr CreateServer( auto configCopy = CloneYsonStruct(config); configCopy->IsHttps = true; - auto httpServer = NHttp::CreateServer(configCopy, tlsListener, poller, acceptor); + auto httpServer = NHttp::CreateServer( + configCopy, + tlsListener, + poller, + acceptor, + memoryTracker); return New<TServer>(std::move(httpServer), std::move(certificateUpdater)); } diff --git a/yt/yt/core/https/server.h b/yt/yt/core/https/server.h index c6c40eeec3..46994e8ea4 100644 --- a/yt/yt/core/https/server.h +++ b/yt/yt/core/https/server.h @@ -8,6 +8,8 @@ #include <yt/yt/core/http/public.h> +#include <yt/yt/core/misc/memory_usage_tracker.h> + namespace NYT::NHttps { //////////////////////////////////////////////////////////////////////////////// @@ -26,7 +28,8 @@ NHttp::IServerPtr CreateServer( const TServerConfigPtr& config, const NConcurrency::IPollerPtr& poller, const NConcurrency::IPollerPtr& acceptor, - const IInvokerPtr& controlInvoker); + const IInvokerPtr& controlInvoker, + const IMemoryUsageTrackerPtr& memoryTracker = GetNullMemoryUsageTracker()); //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/core/misc/memory_usage_tracker.cpp b/yt/yt/core/misc/memory_usage_tracker.cpp index 3f24adc4a8..269632e4fe 100644 --- a/yt/yt/core/misc/memory_usage_tracker.cpp +++ b/yt/yt/core/misc/memory_usage_tracker.cpp @@ -233,11 +233,16 @@ TError TMemoryUsageTrackerGuard::SetSizeImpl(i64 size, auto acquirer) return {}; } -void TMemoryUsageTrackerGuard::IncrementSize(i64 sizeDelta) +void TMemoryUsageTrackerGuard::IncreaseSize(i64 sizeDelta) { SetSize(Size_ + sizeDelta); } +void TMemoryUsageTrackerGuard::DecreaseSize(i64 sizeDelta) +{ + SetSize(Size_ - sizeDelta); +} + TMemoryUsageTrackerGuard TMemoryUsageTrackerGuard::TransferMemory(i64 size) { YT_VERIFY(Size_ >= size); diff --git a/yt/yt/core/misc/memory_usage_tracker.h b/yt/yt/core/misc/memory_usage_tracker.h index c5574ffd4e..c3266ebc29 100644 --- a/yt/yt/core/misc/memory_usage_tracker.h +++ b/yt/yt/core/misc/memory_usage_tracker.h @@ -93,7 +93,8 @@ public: i64 GetSize() const; void SetSize(i64 size); TError TrySetSize(i64 size); - void IncrementSize(i64 sizeDelta); + void IncreaseSize(i64 sizeDelta); + void DecreaseSize(i64 sizeDelta); TMemoryUsageTrackerGuard TransferMemory(i64 size); private: diff --git a/yt/yt/core/rpc/balancing_channel.cpp b/yt/yt/core/rpc/balancing_channel.cpp index f5dbc43ec3..7884bba0d2 100644 --- a/yt/yt/core/rpc/balancing_channel.cpp +++ b/yt/yt/core/rpc/balancing_channel.cpp @@ -42,7 +42,7 @@ public: TBalancingChannelSubprovider( TBalancingChannelConfigPtr config, IChannelFactoryPtr channelFactory, - TString endpointDescription, + const std::string& endpointDescription, IAttributeDictionaryPtr endpointAttributes, std::string serviceName, IPeerDiscoveryPtr peerDiscovery) @@ -96,7 +96,7 @@ public: private: const TBalancingChannelConfigPtr Config_; - const TString EndpointDescription_; + const std::string EndpointDescription_; const IAttributeDictionaryPtr EndpointAttributes_; const TString ServiceName_; @@ -176,7 +176,7 @@ public: TBalancingChannelProvider( TBalancingChannelConfigPtr config, IChannelFactoryPtr channelFactory, - TString endpointDescription, + const std::string& endpointDescription, IAttributeDictionaryPtr endpointAttributes, IPeerDiscoveryPtr peerDiscovery) : Config_(std::move(config)) @@ -192,7 +192,7 @@ public: , PeerDiscovery_(std::move(peerDiscovery)) { } - const TString& GetEndpointDescription() const override + const std::string& GetEndpointDescription() const override { return EndpointDescription_; } @@ -243,7 +243,7 @@ private: const TBalancingChannelConfigPtr Config_; const IChannelFactoryPtr ChannelFactory_; - const TString EndpointDescription_; + const std::string EndpointDescription_; const IAttributeDictionaryPtr EndpointAttributes_; const IPeerDiscoveryPtr PeerDiscovery_; @@ -288,14 +288,14 @@ DEFINE_REFCOUNTED_TYPE(TBalancingChannelProvider) IChannelPtr CreateBalancingChannel( TBalancingChannelConfigPtr config, IChannelFactoryPtr channelFactory, - TString endpointDescription, + const std::string& endpointDescription, IAttributeDictionaryPtr endpointAttributes, IPeerDiscoveryPtr peerDiscovery) { auto channelProvider = CreateBalancingChannelProvider( std::move(config), std::move(channelFactory), - std::move(endpointDescription), + endpointDescription, std::move(endpointAttributes), std::move(peerDiscovery)); @@ -305,7 +305,7 @@ IChannelPtr CreateBalancingChannel( IRoamingChannelProviderPtr CreateBalancingChannelProvider( TBalancingChannelConfigPtr config, IChannelFactoryPtr channelFactory, - TString endpointDescription, + const std::string& endpointDescription, IAttributeDictionaryPtr endpointAttributes, IPeerDiscoveryPtr peerDiscovery) { @@ -316,7 +316,7 @@ IRoamingChannelProviderPtr CreateBalancingChannelProvider( return New<TBalancingChannelProvider>( std::move(config), std::move(channelFactory), - std::move(endpointDescription), + endpointDescription, std::move(endpointAttributes), std::move(peerDiscovery)); } diff --git a/yt/yt/core/rpc/balancing_channel.h b/yt/yt/core/rpc/balancing_channel.h index 2d448dd7cc..452b5e496c 100644 --- a/yt/yt/core/rpc/balancing_channel.h +++ b/yt/yt/core/rpc/balancing_channel.h @@ -16,14 +16,14 @@ namespace NYT::NRpc { IChannelPtr CreateBalancingChannel( TBalancingChannelConfigPtr config, IChannelFactoryPtr channelFactory, - TString endpointDescription, + const std::string& endpointDescription, NYTree::IAttributeDictionaryPtr endpointAttributes, IPeerDiscoveryPtr peerDiscovery = CreateDefaultPeerDiscovery()); IRoamingChannelProviderPtr CreateBalancingChannelProvider( TBalancingChannelConfigPtr config, IChannelFactoryPtr channelFactory, - TString endpointDescription, + const std::string& endpointDescription, NYTree::IAttributeDictionaryPtr endpointAttributes, IPeerDiscoveryPtr peerDiscovery = CreateDefaultPeerDiscovery()); diff --git a/yt/yt/core/rpc/bus/channel.cpp b/yt/yt/core/rpc/bus/channel.cpp index c10204f12d..f3959aed7b 100644 --- a/yt/yt/core/rpc/bus/channel.cpp +++ b/yt/yt/core/rpc/bus/channel.cpp @@ -64,7 +64,7 @@ public: YT_VERIFY(MemoryUsageTracker_); } - const TString& GetEndpointDescription() const override + const std::string& GetEndpointDescription() const override { return Client_->GetEndpointDescription(); } diff --git a/yt/yt/core/rpc/channel.h b/yt/yt/core/rpc/channel.h index 1a815d8878..cd19d408b1 100644 --- a/yt/yt/core/rpc/channel.h +++ b/yt/yt/core/rpc/channel.h @@ -92,7 +92,7 @@ struct IChannel { //! Returns a textual representation of the channel's endpoint. //! Typically used for logging. - virtual const TString& GetEndpointDescription() const = 0; + virtual const std::string& GetEndpointDescription() const = 0; //! Returns the channel's endpoint attributes. //! Typically used for constructing errors. diff --git a/yt/yt/core/rpc/channel_detail.cpp b/yt/yt/core/rpc/channel_detail.cpp index 8b68f3d754..89750b2e6f 100644 --- a/yt/yt/core/rpc/channel_detail.cpp +++ b/yt/yt/core/rpc/channel_detail.cpp @@ -21,7 +21,7 @@ TChannelWrapper::TChannelWrapper(IChannelPtr underlyingChannel) YT_ASSERT(UnderlyingChannel_); } -const TString& TChannelWrapper::GetEndpointDescription() const +const std::string& TChannelWrapper::GetEndpointDescription() const { return UnderlyingChannel_->GetEndpointDescription(); } diff --git a/yt/yt/core/rpc/channel_detail.h b/yt/yt/core/rpc/channel_detail.h index a601ab19c2..63fc79d879 100644 --- a/yt/yt/core/rpc/channel_detail.h +++ b/yt/yt/core/rpc/channel_detail.h @@ -14,7 +14,7 @@ class TChannelWrapper public: explicit TChannelWrapper(IChannelPtr underlyingChannel); - const TString& GetEndpointDescription() const override; + const std::string& GetEndpointDescription() const override; const NYTree::IAttributeDictionary& GetEndpointAttributes() const override; IClientRequestControlPtr Send( diff --git a/yt/yt/core/rpc/dynamic_channel_pool.cpp b/yt/yt/core/rpc/dynamic_channel_pool.cpp index a469175212..ef9c4b80eb 100644 --- a/yt/yt/core/rpc/dynamic_channel_pool.cpp +++ b/yt/yt/core/rpc/dynamic_channel_pool.cpp @@ -43,13 +43,13 @@ public: TImpl( TDynamicChannelPoolConfigPtr config, IChannelFactoryPtr channelFactory, - TString endpointDescription, + const std::string& endpointDescription, IAttributeDictionaryPtr endpointAttributes, std::string serviceName, IPeerDiscoveryPtr peerDiscovery) : Config_(std::move(config)) , ChannelFactory_(std::move(channelFactory)) - , EndpointDescription_(std::move(endpointDescription)) + , EndpointDescription_(endpointDescription) , EndpointAttributes_(ConvertToAttributes(BuildYsonStringFluently() .BeginMap() .Items(*endpointAttributes) @@ -188,7 +188,7 @@ private: const TDynamicChannelPoolConfigPtr Config_; const IChannelFactoryPtr ChannelFactory_; - const TString EndpointDescription_; + const std::string EndpointDescription_; const IAttributeDictionaryPtr EndpointAttributes_; const std::string ServiceName_; IPeerDiscoveryPtr PeerDiscovery_; @@ -900,14 +900,14 @@ private: TDynamicChannelPool::TDynamicChannelPool( TDynamicChannelPoolConfigPtr config, IChannelFactoryPtr channelFactory, - TString endpointDescription, + const std::string& endpointDescription, NYTree::IAttributeDictionaryPtr endpointAttributes, std::string serviceName, IPeerDiscoveryPtr peerDiscovery) : Impl_(New<TImpl>( std::move(config), std::move(channelFactory), - std::move(endpointDescription), + endpointDescription, std::move(endpointAttributes), std::move(serviceName), std::move(peerDiscovery))) diff --git a/yt/yt/core/rpc/dynamic_channel_pool.h b/yt/yt/core/rpc/dynamic_channel_pool.h index 7ac5f2c233..9bd33b9bb1 100644 --- a/yt/yt/core/rpc/dynamic_channel_pool.h +++ b/yt/yt/core/rpc/dynamic_channel_pool.h @@ -28,7 +28,7 @@ public: TDynamicChannelPool( TDynamicChannelPoolConfigPtr config, IChannelFactoryPtr channelFactory, - TString endpointDescription, + const std::string& endpointDescription, NYTree::IAttributeDictionaryPtr endpointAttributes, std::string serviceName, IPeerDiscoveryPtr peerDiscovery); diff --git a/yt/yt/core/rpc/grpc/channel.cpp b/yt/yt/core/rpc/grpc/channel.cpp index 55f803519d..81a17d4d01 100644 --- a/yt/yt/core/rpc/grpc/channel.cpp +++ b/yt/yt/core/rpc/grpc/channel.cpp @@ -149,7 +149,7 @@ public: } // IChannel implementation. - const TString& GetEndpointDescription() const override + const std::string& GetEndpointDescription() const override { return EndpointAddress_; } diff --git a/yt/yt/core/rpc/grpc/server.cpp b/yt/yt/core/rpc/grpc/server.cpp index e19bc7a47d..bb693aea89 100644 --- a/yt/yt/core/rpc/grpc/server.cpp +++ b/yt/yt/core/rpc/grpc/server.cpp @@ -238,7 +238,7 @@ private: { } // IBus overrides. - const TString& GetEndpointDescription() const override + const std::string& GetEndpointDescription() const override { return PeerAddressString_; } diff --git a/yt/yt/core/rpc/hedging_channel.cpp b/yt/yt/core/rpc/hedging_channel.cpp index 0b9fbd3c6f..74d92eb777 100644 --- a/yt/yt/core/rpc/hedging_channel.cpp +++ b/yt/yt/core/rpc/hedging_channel.cpp @@ -345,7 +345,7 @@ public: .EndMap())) { } - const TString& GetEndpointDescription() const override + const std::string& GetEndpointDescription() const override { return EndpointDescription_; } @@ -403,7 +403,7 @@ private: const THedgingChannelOptions Options_; - const TString EndpointDescription_; + const std::string EndpointDescription_; const IAttributeDictionaryPtr EndpointAttributes_; }; diff --git a/yt/yt/core/rpc/http/channel.cpp b/yt/yt/core/rpc/http/channel.cpp index 8b868d9d71..97fea41b23 100644 --- a/yt/yt/core/rpc/http/channel.cpp +++ b/yt/yt/core/rpc/http/channel.cpp @@ -45,7 +45,7 @@ public: } // IChannel implementation. - const TString& GetEndpointDescription() const override + const std::string& GetEndpointDescription() const override { return EndpointAddress_; } diff --git a/yt/yt/core/rpc/http/server.cpp b/yt/yt/core/rpc/http/server.cpp index eb6545015f..cce02b7779 100644 --- a/yt/yt/core/rpc/http/server.cpp +++ b/yt/yt/core/rpc/http/server.cpp @@ -54,7 +54,7 @@ public: , Logger(logger) { } - const TString& GetEndpointDescription() const override + const std::string& GetEndpointDescription() const override { return EndpointAddress_; } diff --git a/yt/yt/core/rpc/local_channel.cpp b/yt/yt/core/rpc/local_channel.cpp index cbe73574cc..9a45500054 100644 --- a/yt/yt/core/rpc/local_channel.cpp +++ b/yt/yt/core/rpc/local_channel.cpp @@ -34,7 +34,7 @@ static constexpr auto& Logger = RpcClientLogger; //////////////////////////////////////////////////////////////////////////////// -static const TString EndpointDescription = "<local>"; +static const std::string EndpointDescription = "<local>"; static const IAttributeDictionaryPtr EndpointAttributes = ConvertToAttributes(BuildYsonStringFluently() .BeginMap() @@ -51,7 +51,7 @@ public: : Server_(std::move(server)) { } - const TString& GetEndpointDescription() const override + const std::string& GetEndpointDescription() const override { return EndpointDescription; } @@ -165,7 +165,7 @@ private: } } - const TString& GetEndpointDescription() const override + const std::string& GetEndpointDescription() const override { return EndpointDescription; } diff --git a/yt/yt/core/rpc/null_channel.cpp b/yt/yt/core/rpc/null_channel.cpp index 9e7b831925..a381ab88d1 100644 --- a/yt/yt/core/rpc/null_channel.cpp +++ b/yt/yt/core/rpc/null_channel.cpp @@ -21,7 +21,7 @@ public: : Address_(address) { } - const TString& GetEndpointDescription() const override + const std::string& GetEndpointDescription() const override { return Address_; } diff --git a/yt/yt/core/rpc/roaming_channel.cpp b/yt/yt/core/rpc/roaming_channel.cpp index f0c011cc15..93717ea4f1 100644 --- a/yt/yt/core/rpc/roaming_channel.cpp +++ b/yt/yt/core/rpc/roaming_channel.cpp @@ -124,7 +124,7 @@ public: : Provider_(std::move(provider)) { } - const TString& GetEndpointDescription() const override + const std::string& GetEndpointDescription() const override { return Provider_->GetEndpointDescription(); } diff --git a/yt/yt/core/rpc/roaming_channel.h b/yt/yt/core/rpc/roaming_channel.h index b0e4524907..2f2874fc17 100644 --- a/yt/yt/core/rpc/roaming_channel.h +++ b/yt/yt/core/rpc/roaming_channel.h @@ -14,7 +14,7 @@ struct IRoamingChannelProvider : public virtual TRefCounted { //! Cf. IChannel::GetEndpointDescription. - virtual const TString& GetEndpointDescription() const = 0; + virtual const std::string& GetEndpointDescription() const = 0; //! Cf. IChannel::GetEndpointAttributes. virtual const NYTree::IAttributeDictionary& GetEndpointAttributes() const = 0; diff --git a/yt/yt/core/rpc/server_detail.cpp b/yt/yt/core/rpc/server_detail.cpp index 7dc3ae052f..252f64d12c 100644 --- a/yt/yt/core/rpc/server_detail.cpp +++ b/yt/yt/core/rpc/server_detail.cpp @@ -328,7 +328,7 @@ const IAttributeDictionary& TServiceContextBase::GetEndpointAttributes() const return EmptyAttributes(); } -const TString& TServiceContextBase::GetEndpointDescription() const +const std::string& TServiceContextBase::GetEndpointDescription() const { static const TString EmptyEndpointDescription; return EmptyEndpointDescription; @@ -530,7 +530,7 @@ const NYTree::IAttributeDictionary& TServiceContextWrapper::GetEndpointAttribute return UnderlyingContext_->GetEndpointAttributes(); } -const TString& TServiceContextWrapper::GetEndpointDescription() const +const std::string& TServiceContextWrapper::GetEndpointDescription() const { return UnderlyingContext_->GetEndpointDescription(); } diff --git a/yt/yt/core/rpc/server_detail.h b/yt/yt/core/rpc/server_detail.h index f1976b091a..16c56e9eb1 100644 --- a/yt/yt/core/rpc/server_detail.h +++ b/yt/yt/core/rpc/server_detail.h @@ -34,7 +34,7 @@ public: TRequestId GetRequestId() const override; NYT::NBus::TBusNetworkStatistics GetBusNetworkStatistics() const override; const NYTree::IAttributeDictionary& GetEndpointAttributes() const override; - const TString& GetEndpointDescription() const override; + const std::string& GetEndpointDescription() const override; i64 GetTotalSize() const override; @@ -199,7 +199,7 @@ public: NRpc::TRequestId GetRequestId() const override; NYT::NBus::TBusNetworkStatistics GetBusNetworkStatistics() const override; const NYTree::IAttributeDictionary& GetEndpointAttributes() const override; - const TString& GetEndpointDescription() const override; + const std::string& GetEndpointDescription() const override; std::optional<TInstant> GetStartTime() const override; std::optional<TDuration> GetTimeout() const override; diff --git a/yt/yt/core/rpc/service.h b/yt/yt/core/rpc/service.h index cab2e99b67..8511ad549a 100644 --- a/yt/yt/core/rpc/service.h +++ b/yt/yt/core/rpc/service.h @@ -59,7 +59,7 @@ struct IServiceContext virtual const NYTree::IAttributeDictionary& GetEndpointAttributes() const = 0; //! Returns the description of the connected endpoint. - virtual const TString& GetEndpointDescription() const = 0; + virtual const std::string& GetEndpointDescription() const = 0; //! Returns the instant when the current retry of request was issued by the client, if known. virtual std::optional<TInstant> GetStartTime() const = 0; diff --git a/yt/yt/core/rpc/service_detail.cpp b/yt/yt/core/rpc/service_detail.cpp index 16175e8a88..aac129a53a 100644 --- a/yt/yt/core/rpc/service_detail.cpp +++ b/yt/yt/core/rpc/service_detail.cpp @@ -413,7 +413,7 @@ public: return ReplyBus_->GetEndpointAttributes(); } - const TString& GetEndpointDescription() const override + const std::string& GetEndpointDescription() const override { return ReplyBus_->GetEndpointDescription(); } diff --git a/yt/yt/core/rpc/unittests/roaming_channel_ut.cpp b/yt/yt/core/rpc/unittests/roaming_channel_ut.cpp index 3c1d214ab9..b527bde12d 100644 --- a/yt/yt/core/rpc/unittests/roaming_channel_ut.cpp +++ b/yt/yt/core/rpc/unittests/roaming_channel_ut.cpp @@ -17,7 +17,7 @@ public: : Channel_(std::move(channel)) { } - const TString& GetEndpointDescription() const override + const std::string& GetEndpointDescription() const override { YT_UNIMPLEMENTED(); } @@ -61,7 +61,7 @@ public: Channel_.Set(std::move(channel)); } - const TString& GetEndpointDescription() const override + const std::string& GetEndpointDescription() const override { YT_UNIMPLEMENTED(); } @@ -99,7 +99,7 @@ class TNeverProvider : public IRoamingChannelProvider { public: - const TString& GetEndpointDescription() const override + const std::string& GetEndpointDescription() const override { YT_UNIMPLEMENTED(); } diff --git a/yt/yt/core/rpc/unittests/viable_peer_registry_ut.cpp b/yt/yt/core/rpc/unittests/viable_peer_registry_ut.cpp index 563ff3c3cf..cad9357c47 100644 --- a/yt/yt/core/rpc/unittests/viable_peer_registry_ut.cpp +++ b/yt/yt/core/rpc/unittests/viable_peer_registry_ut.cpp @@ -95,7 +95,7 @@ public: } } - const TString& GetEndpointDescription() const override + const std::string& GetEndpointDescription() const override { return EndpointDescription_; } @@ -139,7 +139,7 @@ public: private: const std::string Address_; - const TString EndpointDescription_; + const std::string EndpointDescription_; const IMemoryUsageTrackerPtr MemoryUsageTracker_ = GetNullMemoryUsageTracker(); THashSet<std::string>* ChannelRegistry_; @@ -366,7 +366,7 @@ TEST_P(TParametrizedViablePeerRegistryTest, GetRandomChannel) EXPECT_TRUE(viablePeerRegistry->RegisterPeer(Format("address-%v", i))); } - THashSet<TString> retrievedAddresses; + THashSet<std::string> retrievedAddresses; auto req = CreateRequest(); for (int iter = 0; iter < 100; ++iter) { @@ -388,7 +388,7 @@ TEST_P(TParametrizedViablePeerRegistryTest, GetStickyChannel) EXPECT_TRUE(viablePeerRegistry->RegisterPeer(Format("address-%v", i))); } - THashSet<TString> retrievedAddresses; + THashSet<std::string> retrievedAddresses; auto req = CreateRequest(/*enableStickiness*/ true); for (int iter = 0; iter < 100; ++iter) { @@ -544,7 +544,7 @@ TEST(TPreferLocalViablePeerRegistryTest, MinPeerCountForPriorityAwareness) EXPECT_TRUE(viablePeerRegistry->RegisterPeer(Format("non-local-%v.sas.yp-c.yandex.net", i))); } - THashSet<TString> retrievedAddresses; + THashSet<std::string> retrievedAddresses; auto req = CreateRequest(); for (int iter = 0; iter < 100; ++iter) { diff --git a/yt/yt/core/test_framework/test_proxy_service.cpp b/yt/yt/core/test_framework/test_proxy_service.cpp index 56c6df767b..a42aed35d1 100644 --- a/yt/yt/core/test_framework/test_proxy_service.cpp +++ b/yt/yt/core/test_framework/test_proxy_service.cpp @@ -37,7 +37,7 @@ TTestChannel::TTestChannel( .EndMap())) { } -const TString& TTestChannel::GetEndpointDescription() const +const std::string& TTestChannel::GetEndpointDescription() const { return Address_; } @@ -184,7 +184,7 @@ TTestBus::TTestBus(const std::string& address) , NetworkAddress_(NNet::TNetworkAddress()) { } -const TString& TTestBus::GetEndpointDescription() const +const std::string& TTestBus::GetEndpointDescription() const { return EndpointDescription_; } diff --git a/yt/yt/core/test_framework/test_proxy_service.h b/yt/yt/core/test_framework/test_proxy_service.h index e92739e33f..3f2c99415e 100644 --- a/yt/yt/core/test_framework/test_proxy_service.h +++ b/yt/yt/core/test_framework/test_proxy_service.h @@ -45,7 +45,7 @@ class TTestBus public: explicit TTestBus(const std::string& address); - const TString& GetEndpointDescription() const override; + const std::string& GetEndpointDescription() const override; const NYTree::IAttributeDictionary& GetEndpointAttributes() const override; @@ -75,7 +75,7 @@ public: private: const std::string Address_; - const TString EndpointDescription_; + const std::string EndpointDescription_; const NYTree::IAttributeDictionaryPtr Attributes_; const NNet::TNetworkAddress NetworkAddress_; @@ -98,7 +98,7 @@ public: TRealmIdServiceMap defaultServices, const std::string& address); - const TString& GetEndpointDescription() const override; + const std::string& GetEndpointDescription() const override; const NYTree::IAttributeDictionary& GetEndpointAttributes() const override; diff --git a/yt/yt/core/ytree/polymorphic_yson_struct-inl.h b/yt/yt/core/ytree/polymorphic_yson_struct-inl.h index ef8625df47..de5b3ab7ef 100644 --- a/yt/yt/core/ytree/polymorphic_yson_struct-inl.h +++ b/yt/yt/core/ytree/polymorphic_yson_struct-inl.h @@ -10,42 +10,33 @@ namespace NYT::NYTree { namespace NDetail { -template <class TEnum> - requires TEnumTraits<TEnum>::IsEnum -template <class TBase, class TDerived> -TIntrusivePtr<TBase> TEnumTraitsExt<TEnum>::ConcreteFactory() +template <class TEnum, TEnum Value, class TBase, class TDerived> +TIntrusivePtr<TBase> TMappingLeaf<TEnum, Value, TBase, TDerived>::CreateInstance() { return New<TDerived>(); } -template <class TEnum> - requires TEnumTraits<TEnum>::IsEnum -template <class TBase, class... TDerived> -TInstanceFactory<TEnum, TBase> TEnumTraitsExt<TEnum>::MakeFactory() +template <class TEnum, TEnum BaseValue, CYsonStructDerived TBase, TEnum... Values, class... TDerived> + requires (CHierarchy<TBase, TDerived...>) +TIntrusivePtr<TBase> +TPolymorphicMapping<TEnum, TLeafTag<BaseValue, TBase>, TLeafTag<Values, TDerived>...>:: +CreateInstance(TEnum value) { - static constexpr auto keys = TTraits::GetDomainValues(); - using TTuple = std::tuple<TBase, TDerived...>; + if (value == BaseValue) { + return TLeaf<BaseValue, TBase>::CreateInstance(); + } - TInstanceFactory<TEnum, TBase> mapping; + TIntrusivePtr<TBase> ret; - [&] <size_t... Idx> (std::index_sequence<Idx...>) { - ([&] { - mapping[keys[Idx]] = &TEnumTraitsExt<TEnum>::ConcreteFactory<TBase, std::tuple_element_t<Idx, TTuple>>; - } (), ...); - } (std::make_index_sequence<sizeof...(TDerived) + 1>()); + ([&ret, value] { + if (value == Values) { + ret = TLeaf<Values, TDerived>::CreateInstance(); + return false; + } + return true; + } () && ...); - return mapping; -} - -//////////////////////////////////////////////////////////////////////////////// - -template <class TEnum, class TB, class... TD> -TIntrusivePtr<TB> TPolymorphicEnumMapping<TEnum, TB, TD...>::MakeInstance(TEnum e) -{ - static auto factory = - TEnumTraitsExt<TEnum>::template MakeFactory<TB, TD...>(); - - return factory[e](); + return ret; } } // namespace NDetail @@ -53,6 +44,11 @@ TIntrusivePtr<TB> TPolymorphicEnumMapping<TEnum, TB, TD...>::MakeInstance(TEnum //////////////////////////////////////////////////////////////////////////////// template <CPolymorphicEnumMapping TMapping> +TPolymorphicYsonStruct<TMapping>::TPolymorphicYsonStruct(TKey key) + : TPolymorphicYsonStruct(key, TMapping::CreateInstance(key)) +{ } + +template <CPolymorphicEnumMapping TMapping> TPolymorphicYsonStruct<TMapping>::TPolymorphicYsonStruct(TKey key, TIntrusivePtr<TBase> ptr) noexcept : Storage_(std::move(ptr)) , HeldType_(key) @@ -77,6 +73,11 @@ void TPolymorphicYsonStruct<TMapping>::Load( // parse (unless we want to slice which we don't). IMapNodePtr map = TTraits::AsNode(source)->AsMap(); + if (!map || map->GetChildCount() == 0) { + // Empty struct. + return; + } + auto key = map->FindChildValue<TKey>("type"); THROW_ERROR_EXCEPTION_UNLESS( key.has_value(), @@ -86,7 +87,7 @@ void TPolymorphicYsonStruct<TMapping>::Load( if (!Storage_ || HeldType_ != type) { // NB: We will try to merge configs if types match. HeldType_ = type; - Storage_ = TMapping::MakeInstance(HeldType_); + Storage_ = TMapping::CreateInstance(HeldType_); } if (recursiveUnrecognizedStrategy) { @@ -97,6 +98,10 @@ void TPolymorphicYsonStruct<TMapping>::Load( // therefore we must delete it prior to |Load| call. map->RemoveChild("type"); Storage_->Load(map, postprocess, setDefaults, path); + + // NB(arkady-e1ppa): We must not actually remove contents of the node as a postcondition + // since it mutates serialized data which might be used for config validation. + map->AddChild("type", ConvertToNode(HeldType_)); } template <CPolymorphicEnumMapping TMapping> @@ -104,26 +109,51 @@ void TPolymorphicYsonStruct<TMapping>::Save(NYson::IYsonConsumer* consumer) cons { consumer->OnBeginMap(); - consumer->OnKeyedItem("type"); - consumer->OnStringScalar(FormatEnum(HeldType_)); + if (Storage_) { + consumer->OnKeyedItem("type"); + consumer->OnStringScalar(FormatEnum(HeldType_)); + + Storage_->SaveAsMapFragment(consumer); + } - Storage_->SaveAsMapFragment(consumer); consumer->OnEndMap(); } template <CPolymorphicEnumMapping TMapping> -template <std::derived_from<typename TMapping::TBase> TConcrete> +template <std::derived_from<typename TMapping::TBaseClass> TConcrete> TIntrusivePtr<TConcrete> TPolymorphicYsonStruct<TMapping>::TryGetConcrete() const { return DynamicPointerCast<TConcrete>(Storage_); } template <CPolymorphicEnumMapping TMapping> +template <typename TMapping::TKey Value> +TIntrusivePtr<typename TMapping::template TDerivedToEnum<Value>> TPolymorphicYsonStruct<TMapping>::TryGetConcrete() const +{ + if (Value != HeldType_) { + return {}; + } + return TryGetConcrete<typename TMapping::template TDerivedToEnum<Value>>(); +} + +template <CPolymorphicEnumMapping TMapping> typename TPolymorphicYsonStruct<TMapping>::TKey TPolymorphicYsonStruct<TMapping>::GetCurrentType() const { return HeldType_; } +template <CPolymorphicEnumMapping TMapping> +typename TPolymorphicYsonStruct<TMapping>::TBase* TPolymorphicYsonStruct<TMapping>::operator->() +{ + return Storage_.Get(); +} + +template <CPolymorphicEnumMapping TMapping> +const typename TPolymorphicYsonStruct<TMapping>::TBase* TPolymorphicYsonStruct<TMapping>::operator->() const +{ + return Storage_.Get(); +} + //////////////////////////////////////////////////////////////////////////////// template <CPolymorphicEnumMapping TMapping> @@ -141,6 +171,7 @@ void Deserialize(TPolymorphicYsonStruct<TMapping>& value, TSource source) //////////////////////////////////////////////////////////////////////////////// #undef DEFINE_POLYMORPHIC_YSON_STRUCT +#undef DEFINE_POLYMORPHIC_YSON_STRUCT_FOR_ENUM #define POLYMORPHIC_YSON_STRUCT_IMPL__GET_ENUM_SEQ_ELEM(item) \ PP_LEFT_PARENTHESIS PP_ELEMENT(item, 0) PP_RIGHT_PARENTHESIS @@ -151,23 +182,37 @@ void Deserialize(TPolymorphicYsonStruct<TMapping>& value, TSource source) #define POLYMORPHIC_YSON_STRUCT_IMPL__ENUM_NAME(Struct) \ E##Struct##Type -#define POLYMORPHIC_YSON_STRUCT_IMPL__MAKE_MAPPING_ENUM(Struct, seq) \ +#define POLYMORPHIC_YSON_STRUCT_IMPL__MAKE_MAPPING_ENUM(seq) \ DEFINE_ENUM(EType, POLYMORPHIC_YSON_STRUCT_IMPL__GET_ENUM_SEQ(seq)) +#define POLYMORPHIC_YSON_STRUCT_IMPL__MAKE_MAPPING_ENUM_ALIAS(EnumName) \ + using EType = EnumName; + #define POLYMORPHIC_YSON_STRUCT_IMPL__GET_CLASS_ELEM(item) \ PP_COMMA() PP_ELEMENT(item, 1) +#define POLYMORPHIC_YSON_STRUCT_IMPL__MAKE_LEAF_FROM_ETYPE(item) \ + PP_COMMA() ::NYT::NYTree::NDetail::TLeafTag<EType:: PP_ELEMENT(item, 0), PP_ELEMENT(item, 1)> + #define POLYMORPHIC_YSON_STRUCT_IMPL__MAKE_MAPPING_CLASS(Struct, seq) \ - using TMapping = TPolymorphicEnumMapping<EType PP_FOR_EACH(POLYMORPHIC_YSON_STRUCT_IMPL__GET_CLASS_ELEM, seq)> + using TMapping = ::NYT::NYTree::TPolymorphicEnumMapping<EType PP_FOR_EACH(POLYMORPHIC_YSON_STRUCT_IMPL__MAKE_LEAF_FROM_ETYPE, seq)> #define DEFINE_POLYMORPHIC_YSON_STRUCT(name, seq) \ namespace NPolymorphicYsonStructFor##name { \ \ - POLYMORPHIC_YSON_STRUCT_IMPL__MAKE_MAPPING_ENUM(name, seq); \ + POLYMORPHIC_YSON_STRUCT_IMPL__MAKE_MAPPING_ENUM(seq); \ POLYMORPHIC_YSON_STRUCT_IMPL__MAKE_MAPPING_CLASS(name, seq); \ } /*NPolymorphicYsonStructFor##name*/ \ using POLYMORPHIC_YSON_STRUCT_IMPL__ENUM_NAME(name) = NPolymorphicYsonStructFor##name::EType; \ -using T##name = TPolymorphicYsonStruct<NPolymorphicYsonStructFor##name::TMapping>; \ +using T##name = ::NYT::NYTree::TPolymorphicYsonStruct<NPolymorphicYsonStructFor##name::TMapping>; \ +static_assert(true) + +#define DEFINE_POLYMORPHIC_YSON_STRUCT_FOR_ENUM(name, enum, seq) \ +namespace NPolymorphicYsonStructFor##name { \ + POLYMORPHIC_YSON_STRUCT_IMPL__MAKE_MAPPING_ENUM_ALIAS(enum); \ + POLYMORPHIC_YSON_STRUCT_IMPL__MAKE_MAPPING_CLASS(name, seq); \ +} /*NPolymorphicYsonStructFor##name*/ \ +using T##name = ::NYT::NYTree::TPolymorphicYsonStruct<NPolymorphicYsonStructFor##name::TMapping>; \ static_assert(true) //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/core/ytree/polymorphic_yson_struct.h b/yt/yt/core/ytree/polymorphic_yson_struct.h index 0f347abb4b..64566fce4a 100644 --- a/yt/yt/core/ytree/polymorphic_yson_struct.h +++ b/yt/yt/core/ytree/polymorphic_yson_struct.h @@ -42,47 +42,49 @@ constexpr bool CHierarchy = //////////////////////////////////////////////////////////////////////////////// -template <CYsonStructDerived TBase> -using TInstanceFactoryLeaf = - TIntrusivePtr<TBase>(*)(); +template <auto Value> +struct TFactoryTag +{ }; -template <class TEnum, CYsonStructDerived TBase> -using TInstanceFactory = TEnumIndexedArray<TEnum, TInstanceFactoryLeaf<TBase>>; +template <auto Value, class TDerived> +struct TLeafTag +{ }; -template <class TEnum> - requires TEnumTraits<TEnum>::IsEnum -struct TEnumTraitsExt +template <class TEnum, TEnum Value, class TBase, class TDerived> +struct TMappingLeaf { - using TTraits = TEnumTraits<TEnum>; + static TIntrusivePtr<TBase> CreateInstance(); - template <class TBase, class... TDerived> - static constexpr bool CompatibleHierarchy = - CHierarchy<TBase, TDerived...> && - (TTraits::GetDomainSize() == sizeof...(TDerived) + 1); - - template <class TBase, class TDerived> - static TIntrusivePtr<TBase> ConcreteFactory(); - - template <class TBase, class... TDerived> - static TInstanceFactory<TEnum, TBase> MakeFactory(); + // This is intentionally undefined to be used in invoke_result. + friend TDerived EnumToDerivedMethod(TFactoryTag<Value>, TMappingLeaf*); }; -//////////////////////////////////////////////////////////////////////////////// +template <class TEnum, class... TLeafTags> +struct TPolymorphicMapping; -template <class TEnum, class TB, class... TD> -struct TPolymorphicEnumMapping +template <class TEnum, TEnum BaseValue, CYsonStructDerived TBase, TEnum... Values, class... TDerived> + requires (CHierarchy<TBase, TDerived...>) +struct TPolymorphicMapping<TEnum, TLeafTag<BaseValue, TBase>, TLeafTag<Values, TDerived>...> + : public TMappingLeaf<TEnum, BaseValue, TBase, TBase> + , public TMappingLeaf<TEnum, Values, TBase, TDerived>... { + template <TEnum Value, class TConcrete> + using TLeaf = TMappingLeaf<TEnum, Value, TBase, TConcrete>; + + template <TEnum Value> + using TDerivedToEnum = decltype(EnumToDerivedMethod(TFactoryTag<Value>{}, std::declval<TPolymorphicMapping*>())); + using TKey = TEnum; - using TBase = TB; - using TDerived = std::tuple<TD...>; - using THierarchy = std::tuple<TB, TD...>; + using TBaseClass = TBase; - static TIntrusivePtr<TB> MakeInstance(TEnum e); + static TIntrusivePtr<TBase> CreateInstance(TEnum value); }; +//////////////////////////////////////////////////////////////////////////////// + template <class T> constexpr bool IsMapping = requires (T t) { - [] <class E, class B, class... D> (TPolymorphicEnumMapping<E, B, D...>) { + [] <class TEnum, class... TLeafTags> (TPolymorphicMapping<TEnum, TLeafTags...>) { } (t); }; @@ -95,9 +97,8 @@ constexpr bool IsMapping = requires (T t) { template <class TBase, class... TDerived> concept CHierarchy = NDetail::CHierarchy<TBase, TDerived...>; -template <class TEnum, CYsonStructDerived TBase, class... TDerived> - requires CHierarchy<TBase, TDerived...> -using TPolymorphicEnumMapping = NDetail::TPolymorphicEnumMapping<TEnum, TBase, TDerived...>; +template <class TEnum, class... TLeafTags> +using TPolymorphicEnumMapping = NDetail::TPolymorphicMapping<TEnum, TLeafTags...>; template <class T> concept CPolymorphicEnumMapping = NDetail::IsMapping<T>; @@ -137,13 +138,14 @@ class TPolymorphicYsonStruct // TODO(arkady-e1ppa): Support ctor from anyone from the // hierarchy. using TKey = typename TMapping::TKey; - using TBase = typename TMapping::TBase; + using TBase = typename TMapping::TBaseClass; public: using TImplementsYsonStructField = void; TPolymorphicYsonStruct() = default; + explicit TPolymorphicYsonStruct(TKey key); TPolymorphicYsonStruct(TKey key, TIntrusivePtr<TBase> ptr) noexcept; template <CYsonStructSource TSource> @@ -160,8 +162,14 @@ public: template <std::derived_from<TBase> TConcrete> TIntrusivePtr<TConcrete> TryGetConcrete() const; + template <TKey Value> + TIntrusivePtr<typename TMapping::template TDerivedToEnum<Value>> TryGetConcrete() const; + TKey GetCurrentType() const; + TBase* operator->(); + const TBase* operator->() const; + private: TIntrusivePtr<TBase> Storage_; TKey HeldType_; @@ -196,6 +204,29 @@ void Deserialize(TPolymorphicYsonStruct<TMapping>& value, TSource source); */ #define DEFINE_POLYMORPHIC_YSON_STRUCT(name, seq) +//! Usage: +/* + DEFINE_ENUM(EMyEnum + (Pear) + (Apple) + ); + + DEFINE_POLYMORPHIC_YSON_STRUCT_FOR_ENUM(Struct, EMyEnum, + ((Pear) (TPearClass)) + ((Apple) (TAppleClass)) + ) + + // NB(arkady-e1ppa): enum names in the list must be unqualified! E.g. + + DEFINE_POLYMORPHIC_YSON_STRUCT_FOR_ENUM(Struct, EMyEnum, + ((EMyEnum::Pear) (TPearClass)) + ((EMyEnum::Apple) (TAppleClass)) + ) + + Will not compile +*/ +#define DEFINE_POLYMORPHIC_YSON_STRUCT_FOR_ENUM(name, enum, seq) + //////////////////////////////////////////////////////////////////////////////// } // namespace NYT::NYTree diff --git a/yt/yt/core/ytree/unittests/yson_struct_ut.cpp b/yt/yt/core/ytree/unittests/yson_struct_ut.cpp index dd8b43d002..95ec07ab8c 100644 --- a/yt/yt/core/ytree/unittests/yson_struct_ut.cpp +++ b/yt/yt/core/ytree/unittests/yson_struct_ut.cpp @@ -2574,6 +2574,86 @@ TEST(TYsonStructTest, OuterYsonStructWithValidation) EXPECT_EQ(deserialized->Inner->MyInt, 42); } +struct TWithYsonString + : public TYsonStructLite +{ + NYson::TYsonString MyString; + + REGISTER_YSON_STRUCT_LITE(TWithYsonString); + + static void Register(TRegistrar registrar) + { + registrar.Parameter("my_string", &TThis::MyString) + .Default(); + } +}; + +TEST(TYsonStructTest, TYsonStringFieldSimple) +{ + TWithYsonString value; + + { + auto node = BuildYsonNodeFluently() + .BeginMap() + .Item("my_string").Value(ConvertToYsonString(42)) + .EndMap(); + + Deserialize(value, node->AsMap()); + EXPECT_TRUE(value.MyString); + EXPECT_EQ(ConvertTo<i32>(value.MyString), 42); + } + + { + std::string message{"Hi mom!"}; + + auto node = BuildYsonNodeFluently() + .BeginMap() + .Item("my_string").Value(ConvertToYsonString(message)) + .EndMap(); + + Deserialize(value, node->AsMap()); + EXPECT_TRUE(value.MyString); + EXPECT_EQ(ConvertTo<std::string>(value.MyString), message); + } + + { + auto config = New<TTestConfig>(); + config->MyString = "Hello, world!"; + + auto node = BuildYsonNodeFluently() + .BeginMap() + .Item("my_string").Value(ConvertToYsonString(config)) + .EndMap(); + + Deserialize(value, node->AsMap()); + EXPECT_TRUE(value.MyString); + + auto extracted = ConvertTo<TTestConfigPtr>(value.MyString); + EXPECT_EQ(extracted->NullableInt, config->NullableInt); + EXPECT_EQ(extracted->MyString, extracted->MyString); + } +} + +TEST(TYsonStructTest, TYsonStringFieldCompound) +{ + TWithYsonString value; + + auto config = New<TTestConfig>(); + config->MyString = "Hello, world!"; + + auto node = BuildYsonNodeFluently() + .BeginMap() + .Item("my_string").Value(ConvertToYsonString(config)) + .EndMap(); + + Deserialize(value, node->AsMap()); + EXPECT_TRUE(value.MyString); + + auto extracted = ConvertTo<TTestConfigPtr>(value.MyString); + EXPECT_EQ(extracted->NullableInt, config->NullableInt); + EXPECT_EQ(extracted->MyString, extracted->MyString); +} + //////////////////////////////////////////////////////////////////////////////// struct TPolyBase @@ -2797,6 +2877,15 @@ struct TPolyHolder } }; +TEST(TYsonStructTest, TestPolymorphicYsonStructSerializeEmpty) +{ + TPolyHolder holder; + + auto node = ConvertToNode(holder); + + Deserialize(holder, node->AsMap()); +} + TEST(TYsonStructTest, TestPolymorphicYsonStructAsField) { TPolyHolder holder; diff --git a/yt/yt/core/ytree/yson_struct_detail-inl.h b/yt/yt/core/ytree/yson_struct_detail-inl.h index 1deebab6ca..c355e34e13 100644 --- a/yt/yt/core/ytree/yson_struct_detail-inl.h +++ b/yt/yt/core/ytree/yson_struct_detail-inl.h @@ -10,6 +10,8 @@ #include <yt/yt/core/yson/token_writer.h> +#include <library/cpp/yt/yson_string/string.h> + #include <library/cpp/yt/misc/wrapper_traits.h> namespace NYT::NYTree { @@ -225,6 +227,24 @@ void LoadFromSource( } } +// TYsonString +template <CYsonStructSource TSource> +void LoadFromSource( + ::NYT::NYson::TYsonString& parameter, + TSource source, + const NYPath::TYPath& path, + std::optional<EUnrecognizedStrategy> /*ignored*/) +{ + using TTraits = TYsonSourceTraits<TSource>; + + try { + parameter = NYson::ConvertToYsonString(TTraits::AsNode(source)); + } catch (const std::exception& ex) { + THROW_ERROR_EXCEPTION("Error loading parameter %v", path) + << ex; + } +} + // INodePtr template <CYsonStructSource TSource> void LoadFromSource( diff --git a/yt/yt_proto/yt/client/api/rpc_proxy/proto/api_service.proto b/yt/yt_proto/yt/client/api/rpc_proxy/proto/api_service.proto index 7b41d98386..f9606ee6bd 100644 --- a/yt/yt_proto/yt/client/api/rpc_proxy/proto/api_service.proto +++ b/yt/yt_proto/yt/client/api/rpc_proxy/proto/api_service.proto @@ -2467,10 +2467,14 @@ message TReqGetJobStderr string operation_alias = 3; } required NYT.NProto.TGuid job_id = 2; + optional int64 limit = 4; + optional int64 offset = 5; } message TRspGetJobStderr { + optional int64 end_offset = 1; + optional int64 total_size = 2; } //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt_proto/yt/client/cache/proto/config.proto b/yt/yt_proto/yt/client/cache/proto/config.proto deleted file mode 100644 index 172329c919..0000000000 --- a/yt/yt_proto/yt/client/cache/proto/config.proto +++ /dev/null @@ -1,62 +0,0 @@ -package NYT.NClient.NCache; - -import "yt_proto/yt/core/yson/proto/protobuf_interop.proto"; - -option (NYT.NYson.NProto.derive_underscore_case_names) = true; -option go_package = "a.yandex-team.ru/yt/go/proto/client/cache"; - -enum ECompressionCodec -{ - None = 0; - Lz4 = 1; -} - -// Connection options. -message TConfig -{ - optional string ClusterName = 1; - optional string ProxyRole = 2; - optional uint32 ChannelPoolSize = 3; - optional uint32 ChannelPoolRebalanceIntervalSeconds = 4; - - // All timeouts in milliseconds. - optional uint32 DefaultTransactionTimeout = 5; - optional uint32 DefaultSelectRowsTimeout = 6; - optional uint32 DefaultLookupRowsTimeout = 7; - optional uint32 DefaultTotalStreamingTimeout = 8; - optional uint32 DefaultStreamingStallTimeout = 9; - optional uint32 DefaultPingPeriod = 10; - optional uint32 ModifyRowsBatchCapacity = 11; - - optional ECompressionCodec RequestCodec = 17 [default = None]; - optional ECompressionCodec ResponseCodec = 12 [default = None]; - // Should set EnableLegacyRpcCodecs=False, to enable RequestCodec & ResponseCodec: https://nda.ya.ru/t/iXCfYZjS6yNEwg - optional bool EnableLegacyRpcCodecs = 20; - - optional bool EnableRetries = 13; - optional uint32 RetryBackoffTime = 14; - optional uint32 RetryAttempts = 15; - optional uint32 RetryTimeout = 16; - - repeated string ProxyAddresses = 18; - optional bool EnableProxyDiscovery = 19; - optional bool EnablePowerOfTwoChoicesStrategy = 21; - optional bool EnableSelectQueryTracingTag = 22; - - optional uint32 ClusterTag = 23; - optional uint32 ClockClusterTag = 24; - optional string UdfRegistryPath = 25; -} - -message TClustersConfig -{ - // In DefaultConfig and ClusterConfigs field .ClusterName is ignored - // and field .ProxyRole can be overwritten with explicitly provided in cluster url one. - - // DefaultConfig is used for clusters not mentioned in ClusterConfigs. - optional TConfig DefaultConfig = 1; - // Per-cluster configs. - map<string, TConfig> ClusterConfigs = 2 [ - (NYT.NYson.NProto.yson_map) = true - ]; -} diff --git a/yt/yt_proto/yt/client/cache/ya.make b/yt/yt_proto/yt/client/cache/ya.make deleted file mode 100644 index 77903f0861..0000000000 --- a/yt/yt_proto/yt/client/cache/ya.make +++ /dev/null @@ -1,14 +0,0 @@ -PROTO_LIBRARY(yt-client-cache-proto) - -PROTO_NAMESPACE(yt) -PY_NAMESPACE(yt_proto.yt.client.cache) - -SRCS( - proto/config.proto -) - -PEERDIR(yt/yt_proto/yt/core) - -EXCLUDE_TAGS(GO_PROTO) - -END() diff --git a/yt/yt_proto/yt/client/hedging/proto/config.proto b/yt/yt_proto/yt/client/hedging/proto/config.proto deleted file mode 100644 index 4f088c37cb..0000000000 --- a/yt/yt_proto/yt/client/hedging/proto/config.proto +++ /dev/null @@ -1,34 +0,0 @@ -package NYT.NClient.NHedging.NRpc; - -import public "yt_proto/yt/client/cache/proto/config.proto"; -import "yt_proto/yt/core/yson/proto/protobuf_interop.proto"; - -option (NYT.NYson.NProto.derive_underscore_case_names) = true; - - -message THedgingClientConfig -{ - // All timeouts also in milliseconds. - message TClientOptions { - optional NYT.NClient.NCache.TConfig ClientConfig = 1; - optional uint32 InitialPenalty = 2; - }; - - repeated TClientOptions Clients = 1; - optional uint32 BanPenalty = 2 [default = 1]; // The penalty increment during ban. - optional uint32 BanDuration = 3 [default = 50]; // How long need to ban cllent after error. - map<string, string> Tags = 4 [ - (NYT.NYson.NProto.yson_map) = true - ]; // Tags for profiling. -} - -message TReplicationLagPenaltyProviderConfig -{ - repeated string ReplicaClusters = 1; // Clusters that need checks for replication lag. - optional string TablePath = 2; // Table that needs checks for replication lag. - optional uint32 LagPenalty = 3 [default = 10]; // penalty in milliseconds, same as BanPenalty in THedgingClientConfig. - optional uint32 MaxTabletLag = 4 [default = 300]; // Tablet is considered "lagged" if CurrentTimestamp - TabletLastReplicationTimestamp >= MaxTabletLag (in seconds). - optional float MaxTabletsWithLagFraction = 5 [default = 0.05]; // Real value from 0.0 to 1.0. Replica cluster receives LagPenalty if NumberOfTabletsWithLag >= MaxTabletsWithLagFraction * TotalNumberOfTablets. - optional uint32 CheckPeriod = 6 [default = 60]; // Replication lag check period (in seconds). - optional bool ClearPenaltiesOnErrors = 7 [default = false]; // In case of any errors from master client - clear all penalties. -} diff --git a/yt/yt_proto/yt/client/hedging/ya.make b/yt/yt_proto/yt/client/hedging/ya.make deleted file mode 100644 index 11f73bf7b6..0000000000 --- a/yt/yt_proto/yt/client/hedging/ya.make +++ /dev/null @@ -1,14 +0,0 @@ -PROTO_LIBRARY(yt-client-hedging-proto) - -SRCS( - proto/config.proto -) - -PEERDIR( - yt/yt_proto/yt/core - yt/yt_proto/yt/client/cache -) - -EXCLUDE_TAGS(GO_PROTO) - -END() |