diff options
| author | robot-piglet <robot-piglet@yandex-team.com> | 2023-11-14 19:18:07 +0300 |
|---|---|---|
| committer | robot-piglet <robot-piglet@yandex-team.com> | 2023-11-14 20:20:53 +0300 |
| commit | 874ef51d3d3edfa25f5a505ec6ab50e172965d1e (patch) | |
| tree | 620fb5e02063d23509d3aa3df2215c099ccde0b7 | |
| parent | e356b34d3b0399e2f170881af15c91e4db9e3d11 (diff) | |
| download | ydb-874ef51d3d3edfa25f5a505ec6ab50e172965d1e.tar.gz | |
Intermediate changes
263 files changed, 1860 insertions, 14528 deletions
diff --git a/.mapping.json b/.mapping.json index 146d1f262d..f455b84fd1 100644 --- a/.mapping.json +++ b/.mapping.json @@ -2626,16 +2626,6 @@ "library/cpp/pop_count/CMakeLists.linux-x86_64.txt":"", "library/cpp/pop_count/CMakeLists.txt":"", "library/cpp/pop_count/CMakeLists.windows-x86_64.txt":"", - "library/cpp/porto/CMakeLists.darwin-x86_64.txt":"", - "library/cpp/porto/CMakeLists.linux-aarch64.txt":"", - "library/cpp/porto/CMakeLists.linux-x86_64.txt":"", - "library/cpp/porto/CMakeLists.txt":"", - "library/cpp/porto/CMakeLists.windows-x86_64.txt":"", - "library/cpp/porto/proto/CMakeLists.darwin-x86_64.txt":"", - "library/cpp/porto/proto/CMakeLists.linux-aarch64.txt":"", - "library/cpp/porto/proto/CMakeLists.linux-x86_64.txt":"", - "library/cpp/porto/proto/CMakeLists.txt":"", - "library/cpp/porto/proto/CMakeLists.windows-x86_64.txt":"", "library/cpp/presort/CMakeLists.darwin-x86_64.txt":"", "library/cpp/presort/CMakeLists.linux-aarch64.txt":"", "library/cpp/presort/CMakeLists.linux-x86_64.txt":"", @@ -5591,6 +5581,11 @@ "ydb/core/tx/coordinator/ut/CMakeLists.linux-x86_64.txt":"", "ydb/core/tx/coordinator/ut/CMakeLists.txt":"", "ydb/core/tx/coordinator/ut/CMakeLists.windows-x86_64.txt":"", + "ydb/core/tx/data_events/CMakeLists.darwin-x86_64.txt":"", + "ydb/core/tx/data_events/CMakeLists.linux-aarch64.txt":"", + "ydb/core/tx/data_events/CMakeLists.linux-x86_64.txt":"", + "ydb/core/tx/data_events/CMakeLists.txt":"", + "ydb/core/tx/data_events/CMakeLists.windows-x86_64.txt":"", "ydb/core/tx/datashard/CMakeLists.darwin-x86_64.txt":"", "ydb/core/tx/datashard/CMakeLists.linux-aarch64.txt":"", "ydb/core/tx/datashard/CMakeLists.linux-x86_64.txt":"", @@ -5746,11 +5741,6 @@ "ydb/core/tx/datashard/ut_volatile/CMakeLists.linux-x86_64.txt":"", "ydb/core/tx/datashard/ut_volatile/CMakeLists.txt":"", "ydb/core/tx/datashard/ut_volatile/CMakeLists.windows-x86_64.txt":"", - "ydb/core/tx/ev_write/CMakeLists.darwin-x86_64.txt":"", - "ydb/core/tx/ev_write/CMakeLists.linux-aarch64.txt":"", - "ydb/core/tx/ev_write/CMakeLists.linux-x86_64.txt":"", - "ydb/core/tx/ev_write/CMakeLists.txt":"", - "ydb/core/tx/ev_write/CMakeLists.windows-x86_64.txt":"", "ydb/core/tx/long_tx_service/CMakeLists.darwin-x86_64.txt":"", "ydb/core/tx/long_tx_service/CMakeLists.linux-aarch64.txt":"", "ydb/core/tx/long_tx_service/CMakeLists.linux-x86_64.txt":"", @@ -10042,11 +10032,6 @@ "yt/yt/library/backtrace_introspector/http/CMakeLists.linux-aarch64.txt":"", "yt/yt/library/backtrace_introspector/http/CMakeLists.linux-x86_64.txt":"", "yt/yt/library/backtrace_introspector/http/CMakeLists.txt":"", - "yt/yt/library/containers/CMakeLists.darwin-x86_64.txt":"", - "yt/yt/library/containers/CMakeLists.linux-aarch64.txt":"", - "yt/yt/library/containers/CMakeLists.linux-x86_64.txt":"", - "yt/yt/library/containers/CMakeLists.txt":"", - "yt/yt/library/containers/CMakeLists.windows-x86_64.txt":"", "yt/yt/library/decimal/CMakeLists.darwin-x86_64.txt":"", "yt/yt/library/decimal/CMakeLists.linux-aarch64.txt":"", "yt/yt/library/decimal/CMakeLists.linux-x86_64.txt":"", @@ -10064,11 +10049,9 @@ "yt/yt/library/numeric/CMakeLists.linux-aarch64.txt":"", "yt/yt/library/numeric/CMakeLists.linux-x86_64.txt":"", "yt/yt/library/numeric/CMakeLists.txt":"", - "yt/yt/library/process/CMakeLists.darwin-x86_64.txt":"", "yt/yt/library/process/CMakeLists.linux-aarch64.txt":"", "yt/yt/library/process/CMakeLists.linux-x86_64.txt":"", "yt/yt/library/process/CMakeLists.txt":"", - "yt/yt/library/process/CMakeLists.windows-x86_64.txt":"", "yt/yt/library/profiling/CMakeLists.darwin-x86_64.txt":"", "yt/yt/library/profiling/CMakeLists.linux-aarch64.txt":"", "yt/yt/library/profiling/CMakeLists.linux-x86_64.txt":"", diff --git a/build/conf/license.conf b/build/conf/license.conf index 5197299979..3a3e0c54fe 100644 --- a/build/conf/license.conf +++ b/build/conf/license.conf @@ -376,6 +376,7 @@ LICENSES_REQUIRE_CITATION= \ Unicode-DFS-2016 \ Unicode-Icu-58 \ Unicode-Mappings \ + Unittest-Permissive-License \ Unlicense \ Unrar \ UPL-1.0 \ diff --git a/build/conf/project_specific/yql_udf.conf b/build/conf/project_specific/yql_udf.conf index 8a75dc6788..c2cc5218fa 100644 --- a/build/conf/project_specific/yql_udf.conf +++ b/build/conf/project_specific/yql_udf.conf @@ -40,12 +40,14 @@ module YQL_UDF_TEST: PY3TEST_BIN { DEPENDS(contrib/ydb/library/yql/tools/astdiff) DEPENDS(contrib/ydb/library/yql/tools/yqlrun) + DEPENDS(contrib/ydb/library/yql/tools/udf_resolver) DATA(arcadia/contrib/ydb/library/yql/mount) DATA(arcadia/contrib/ydb/library/yql/cfg/udf_test) ENV(YQL_ASTDIFF_PATH="contrib/ydb/library/yql/tools/astdiff/astdiff") ENV(YQL_CONFIG_DIR="contrib/ydb/library/yql/cfg/udf_test") ENV(YQL_YQLRUN_PATH="contrib/ydb/library/yql/tools/yqlrun/yqlrun") ENV(YQL_SQL2YQL_PATH="contrib/ydb/library/yql/tools/sql2yql/sql2yql") + ENV(YQL_UDFRESOLVER_PATH="contrib/ydb/library/yql/tools/udf_resolver/udf_resolver") } module YQL_UDF_YDB_TEST: PY3TEST_BIN { @@ -55,6 +57,7 @@ module YQL_UDF_YDB_TEST: PY3TEST_BIN { DEPENDS(ydb/library/yql/tools/astdiff) DEPENDS(ydb/library/yql/tools/yqlrun) + DEPENDS(ydb/library/yql/tools/udf_resolver) DATA(arcadia/ydb/library/yql/mount) DATA(arcadia/ydb/library/yql/cfg/udf_test) } @@ -66,12 +69,14 @@ module YQL_UDF_TEST_CONTRIB: PY3TEST_BIN { DEPENDS(contrib/ydb/library/yql/tools/astdiff) DEPENDS(contrib/ydb/library/yql/tools/yqlrun) + DEPENDS(contrib/ydb/library/yql/tools/udf_resolver) DATA(arcadia/contrib/ydb/library/yql/mount) DATA(arcadia/contrib/ydb/library/yql/cfg/udf_test) ENV(YQL_ASTDIFF_PATH="contrib/ydb/library/yql/tools/astdiff/astdiff") ENV(YQL_CONFIG_DIR="contrib/ydb/library/yql/cfg/udf_test") ENV(YQL_YQLRUN_PATH="contrib/ydb/library/yql/tools/yqlrun/yqlrun") ENV(YQL_SQL2YQL_PATH="contrib/ydb/library/yql/tools/sql2yql/sql2yql") + ENV(YQL_UDFRESOLVER_PATH="contrib/ydb/library/yql/tools/udf_resolver/udf_resolver") } ### @usage: _ADD_YQL_UDF_DEPS() diff --git a/build/mapping.conf.json b/build/mapping.conf.json index 5d9d9a801f..95b78d2ac3 100644 --- a/build/mapping.conf.json +++ b/build/mapping.conf.json @@ -63,8 +63,8 @@ "3968796477": "http://devtools-registry.s3.yandex.net/3968796477", "3968796981": "http://devtools-registry.s3.yandex.net/3968796981", "3968797636": "http://devtools-registry.s3.yandex.net/3968797636", - "5386809135": "http://devtools-registry.s3.yandex.net/5386809135", - "5386759014": "http://devtools-registry.s3.yandex.net/5386759014", + "5360700919": "http://devtools-registry.s3.yandex.net/5360700919", + "5360699619": "http://devtools-registry.s3.yandex.net/5360699619", "4307900052": "http://devtools-registry.s3.yandex.net/4307900052", "4307900057": "http://devtools-registry.s3.yandex.net/4307900057", "4307890075": "http://devtools-registry.s3.yandex.net/4307890075", @@ -162,8 +162,8 @@ "3968796477": "devtools/ya/test/programs/flake8/py3/flake8 for linux-aarch64", "3968796981": "devtools/ya/test/programs/flake8/py3/flake8 for linux-ppc64le", "3968797636": "devtools/ya/test/programs/flake8/py3/flake8 for win32", - "5386809135": "devtools/ya/test/programs/test_tool/bin/test_tool for linux", - "5386759014": "devtools/ya/test/programs/test_tool/bin3/test_tool3 for linux", + "5360700919": "devtools/ya/test/programs/test_tool/bin/test_tool for linux", + "5360699619": "devtools/ya/test/programs/test_tool/bin3/test_tool3 for linux", "4307900052": "flake8_linter for darwin", "4307900057": "flake8_linter for darwin-arm64", "4307890075": "flake8_linter for linux", diff --git a/build/platform/test_tool/host.ya.make.inc b/build/platform/test_tool/host.ya.make.inc index c519f0f357..41813a9459 100644 --- a/build/platform/test_tool/host.ya.make.inc +++ b/build/platform/test_tool/host.ya.make.inc @@ -1,17 +1,17 @@ IF (HOST_OS_DARWIN AND HOST_ARCH_X86_64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:5386806256) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL3_HOST sbr:5386754279) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:5360699648) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL3_HOST sbr:5360698288) ELSEIF (HOST_OS_DARWIN AND HOST_ARCH_ARM64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:5386805280) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL3_HOST sbr:5386752133) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:5360699112) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL3_HOST sbr:5360697535) ELSEIF (HOST_OS_LINUX AND HOST_ARCH_X86_64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:5386809135) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL3_HOST sbr:5386759014) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:5360700919) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL3_HOST sbr:5360699619) ELSEIF (HOST_OS_LINUX AND HOST_ARCH_AARCH64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:5386803981) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL3_HOST sbr:5386750030) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:5360698627) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL3_HOST sbr:5360696765) ELSEIF (HOST_OS_WINDOWS AND HOST_ARCH_X86_64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:5386807476) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL3_HOST sbr:5386756543) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:5360700342) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL3_HOST sbr:5360698968) ENDIF() diff --git a/contrib/python/backcall/.dist-info/METADATA b/contrib/python/backcall/.dist-info/METADATA deleted file mode 100644 index df0f708d2d..0000000000 --- a/contrib/python/backcall/.dist-info/METADATA +++ /dev/null @@ -1,55 +0,0 @@ -Metadata-Version: 2.1 -Name: backcall -Version: 0.2.0 -Summary: Specifications for callback functions passed in to an API -Home-page: https://github.com/takluyver/backcall -License: UNKNOWN -Author: Thomas Kluyver -Author-email: thomas@kluyver.me.uk -Description-Content-Type: text/x-rst -Classifier: License :: OSI Approved :: BSD License -Classifier: Programming Language :: Python :: 2 -Classifier: Programming Language :: Python :: 2.7 -Classifier: Programming Language :: Python :: 3 - -======== -backcall -======== - -.. image:: https://travis-ci.org/takluyver/backcall.png?branch=master - :target: https://travis-ci.org/takluyver/backcall - -Specifications for callback functions passed in to an API - -If your code lets other people supply callback functions, it's important to -specify the function signature you expect, and check that functions support that. -Adding extra parameters later would break other peoples code unless you're careful. - -backcall provides a way of specifying the callback signature using a prototype -function:: - - from backcall import callback_prototype - - @callback_prototype - def handle_ping(sender, delay=None): - # Specify positional parameters without a default, and keyword - # parameters with a default. - pass - - def register_ping_handler(callback): - # This checks and adapts the function passed in: - callback = handle_ping.adapt(callback) - ping_callbacks.append(callback) - -If the callback takes fewer parameters than your prototype, *backcall* will wrap -it in a function that discards the extra arguments. If the callback expects -more arguments, a TypeError is thrown when it is registered. - -For more details, see the `docs <http://backcall.readthedocs.org/en/latest/>`_ or -the `Demo notebook <http://nbviewer.ipython.org/github/takluyver/backcall/blob/master/Demo.ipynb>`_. - -The tests are run with `pytest <http://pytest.org/latest/>`_. In the root directory, -execute:: - - py.test - diff --git a/contrib/python/backcall/.dist-info/top_level.txt b/contrib/python/backcall/.dist-info/top_level.txt deleted file mode 100644 index e46b94d67c..0000000000 --- a/contrib/python/backcall/.dist-info/top_level.txt +++ /dev/null @@ -1 +0,0 @@ -backcall diff --git a/contrib/python/backcall/LICENSE b/contrib/python/backcall/LICENSE deleted file mode 100644 index 75f097ee3a..0000000000 --- a/contrib/python/backcall/LICENSE +++ /dev/null @@ -1,27 +0,0 @@ -Copyright (c) 2014, Thomas Kluyver -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, this - list of conditions and the following disclaimer in the documentation and/or - other materials provided with the distribution. - -* Neither the name of the {organization} nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON -ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/contrib/python/backcall/README.rst b/contrib/python/backcall/README.rst deleted file mode 100644 index dd82168a5d..0000000000 --- a/contrib/python/backcall/README.rst +++ /dev/null @@ -1,40 +0,0 @@ -======== -backcall -======== - -.. image:: https://travis-ci.org/takluyver/backcall.png?branch=master - :target: https://travis-ci.org/takluyver/backcall - -Specifications for callback functions passed in to an API - -If your code lets other people supply callback functions, it's important to -specify the function signature you expect, and check that functions support that. -Adding extra parameters later would break other peoples code unless you're careful. - -backcall provides a way of specifying the callback signature using a prototype -function:: - - from backcall import callback_prototype - - @callback_prototype - def handle_ping(sender, delay=None): - # Specify positional parameters without a default, and keyword - # parameters with a default. - pass - - def register_ping_handler(callback): - # This checks and adapts the function passed in: - callback = handle_ping.adapt(callback) - ping_callbacks.append(callback) - -If the callback takes fewer parameters than your prototype, *backcall* will wrap -it in a function that discards the extra arguments. If the callback expects -more arguments, a TypeError is thrown when it is registered. - -For more details, see the `docs <http://backcall.readthedocs.org/en/latest/>`_ or -the `Demo notebook <http://nbviewer.ipython.org/github/takluyver/backcall/blob/master/Demo.ipynb>`_. - -The tests are run with `pytest <http://pytest.org/latest/>`_. In the root directory, -execute:: - - py.test diff --git a/contrib/python/backcall/backcall/__init__.py b/contrib/python/backcall/backcall/__init__.py deleted file mode 100644 index 7a5c36673b..0000000000 --- a/contrib/python/backcall/backcall/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -# -*- coding: utf-8 -*- -"""Specifications for callback functions passed in to an API""" -from .backcall import callback_prototype - -__author__ = 'Thomas Kluyver' -__email__ = 'takowl@gmail.com' -__version__ = '0.2.0' diff --git a/contrib/python/backcall/backcall/_signatures.py b/contrib/python/backcall/backcall/_signatures.py deleted file mode 100644 index f37b31c00e..0000000000 --- a/contrib/python/backcall/backcall/_signatures.py +++ /dev/null @@ -1,819 +0,0 @@ -"""Function signature objects for callables - -Back port of Python 3.3's function signature tools from the inspect module, -modified to be compatible with Python 2.6, 2.7 and 3.2+. -""" - -#----------------------------------------------------------------------------- -# Python 3.3 stdlib inspect.py is public domain -# -# Backports Copyright (C) 2013 Aaron Iles -# Used under Apache License Version 2.0 -# -# Further Changes are Copyright (C) 2013 The IPython Development Team -# -# Distributed under the terms of the BSD License. The full license is in -# the file COPYING, distributed as part of this software. -#----------------------------------------------------------------------------- - -from __future__ import absolute_import, division, print_function -import itertools -import functools -import re -import types - - -# patch for single-file -# we don't support 2.6, so we can just import OrderedDict -from collections import OrderedDict - -__version__ = '0.3' -# end patch - -__all__ = ['BoundArguments', 'Parameter', 'Signature', 'signature'] - - -_WrapperDescriptor = type(type.__call__) -_MethodWrapper = type(all.__call__) - -_NonUserDefinedCallables = (_WrapperDescriptor, - _MethodWrapper, - types.BuiltinFunctionType) - - -def formatannotation(annotation, base_module=None): - if isinstance(annotation, type): - if annotation.__module__ in ('builtins', '__builtin__', base_module): - return annotation.__name__ - return annotation.__module__+'.'+annotation.__name__ - return repr(annotation) - - -def _get_user_defined_method(cls, method_name, *nested): - try: - if cls is type: - return - meth = getattr(cls, method_name) - for name in nested: - meth = getattr(meth, name, meth) - except AttributeError: - return - else: - if not isinstance(meth, _NonUserDefinedCallables): - # Once '__signature__' will be added to 'C'-level - # callables, this check won't be necessary - return meth - - -def signature(obj): - '''Get a signature object for the passed callable.''' - - if not callable(obj): - raise TypeError('{0!r} is not a callable object'.format(obj)) - - if isinstance(obj, types.MethodType): - # In this case we skip the first parameter of the underlying - # function (usually `self` or `cls`). - sig = signature(obj.__func__) - return sig.replace(parameters=tuple(sig.parameters.values())[1:]) - - try: - sig = obj.__signature__ - except AttributeError: - pass - else: - if sig is not None: - return sig - - try: - # Was this function wrapped by a decorator? - wrapped = obj.__wrapped__ - except AttributeError: - pass - else: - return signature(wrapped) - - if isinstance(obj, types.FunctionType): - return Signature.from_function(obj) - - if isinstance(obj, functools.partial): - sig = signature(obj.func) - - new_params = OrderedDict(sig.parameters.items()) - - partial_args = obj.args or () - partial_keywords = obj.keywords or {} - try: - ba = sig.bind_partial(*partial_args, **partial_keywords) - except TypeError as ex: - msg = 'partial object {0!r} has incorrect arguments'.format(obj) - raise ValueError(msg) - - for arg_name, arg_value in ba.arguments.items(): - param = new_params[arg_name] - if arg_name in partial_keywords: - # We set a new default value, because the following code - # is correct: - # - # >>> def foo(a): print(a) - # >>> print(partial(partial(foo, a=10), a=20)()) - # 20 - # >>> print(partial(partial(foo, a=10), a=20)(a=30)) - # 30 - # - # So, with 'partial' objects, passing a keyword argument is - # like setting a new default value for the corresponding - # parameter - # - # We also mark this parameter with '_partial_kwarg' - # flag. Later, in '_bind', the 'default' value of this - # parameter will be added to 'kwargs', to simulate - # the 'functools.partial' real call. - new_params[arg_name] = param.replace(default=arg_value, - _partial_kwarg=True) - - elif (param.kind not in (_VAR_KEYWORD, _VAR_POSITIONAL) and - not param._partial_kwarg): - new_params.pop(arg_name) - - return sig.replace(parameters=new_params.values()) - - sig = None - if isinstance(obj, type): - # obj is a class or a metaclass - - # First, let's see if it has an overloaded __call__ defined - # in its metaclass - call = _get_user_defined_method(type(obj), '__call__') - if call is not None: - sig = signature(call) - else: - # Now we check if the 'obj' class has a '__new__' method - new = _get_user_defined_method(obj, '__new__') - if new is not None: - sig = signature(new) - else: - # Finally, we should have at least __init__ implemented - init = _get_user_defined_method(obj, '__init__') - if init is not None: - sig = signature(init) - elif not isinstance(obj, _NonUserDefinedCallables): - # An object with __call__ - # We also check that the 'obj' is not an instance of - # _WrapperDescriptor or _MethodWrapper to avoid - # infinite recursion (and even potential segfault) - call = _get_user_defined_method(type(obj), '__call__', 'im_func') - if call is not None: - sig = signature(call) - - if sig is not None: - return sig - - if isinstance(obj, types.BuiltinFunctionType): - # Raise a nicer error message for builtins - msg = 'no signature found for builtin function {0!r}'.format(obj) - raise ValueError(msg) - - raise ValueError('callable {0!r} is not supported by signature'.format(obj)) - - -class _void(object): - '''A private marker - used in Parameter & Signature''' - - -class _empty(object): - pass - - -class _ParameterKind(int): - def __new__(self, *args, **kwargs): - obj = int.__new__(self, *args) - obj._name = kwargs['name'] - return obj - - def __str__(self): - return self._name - - def __repr__(self): - return '<_ParameterKind: {0!r}>'.format(self._name) - - -_POSITIONAL_ONLY = _ParameterKind(0, name='POSITIONAL_ONLY') -_POSITIONAL_OR_KEYWORD = _ParameterKind(1, name='POSITIONAL_OR_KEYWORD') -_VAR_POSITIONAL = _ParameterKind(2, name='VAR_POSITIONAL') -_KEYWORD_ONLY = _ParameterKind(3, name='KEYWORD_ONLY') -_VAR_KEYWORD = _ParameterKind(4, name='VAR_KEYWORD') - - -class Parameter(object): - '''Represents a parameter in a function signature. - - Has the following public attributes: - - * name : str - The name of the parameter as a string. - * default : object - The default value for the parameter if specified. If the - parameter has no default value, this attribute is not set. - * annotation - The annotation for the parameter if specified. If the - parameter has no annotation, this attribute is not set. - * kind : str - Describes how argument values are bound to the parameter. - Possible values: `Parameter.POSITIONAL_ONLY`, - `Parameter.POSITIONAL_OR_KEYWORD`, `Parameter.VAR_POSITIONAL`, - `Parameter.KEYWORD_ONLY`, `Parameter.VAR_KEYWORD`. - ''' - - __slots__ = ('_name', '_kind', '_default', '_annotation', '_partial_kwarg') - - POSITIONAL_ONLY = _POSITIONAL_ONLY - POSITIONAL_OR_KEYWORD = _POSITIONAL_OR_KEYWORD - VAR_POSITIONAL = _VAR_POSITIONAL - KEYWORD_ONLY = _KEYWORD_ONLY - VAR_KEYWORD = _VAR_KEYWORD - - empty = _empty - - def __init__(self, name, kind, default=_empty, annotation=_empty, - _partial_kwarg=False): - - if kind not in (_POSITIONAL_ONLY, _POSITIONAL_OR_KEYWORD, - _VAR_POSITIONAL, _KEYWORD_ONLY, _VAR_KEYWORD): - raise ValueError("invalid value for 'Parameter.kind' attribute") - self._kind = kind - - if default is not _empty: - if kind in (_VAR_POSITIONAL, _VAR_KEYWORD): - msg = '{0} parameters cannot have default values'.format(kind) - raise ValueError(msg) - self._default = default - self._annotation = annotation - - if name is None: - if kind != _POSITIONAL_ONLY: - raise ValueError("None is not a valid name for a " - "non-positional-only parameter") - self._name = name - else: - name = str(name) - if kind != _POSITIONAL_ONLY and not re.match(r'[a-z_]\w*$', name, re.I): - msg = '{0!r} is not a valid parameter name'.format(name) - raise ValueError(msg) - self._name = name - - self._partial_kwarg = _partial_kwarg - - @property - def name(self): - return self._name - - @property - def default(self): - return self._default - - @property - def annotation(self): - return self._annotation - - @property - def kind(self): - return self._kind - - def replace(self, name=_void, kind=_void, annotation=_void, - default=_void, _partial_kwarg=_void): - '''Creates a customized copy of the Parameter.''' - - if name is _void: - name = self._name - - if kind is _void: - kind = self._kind - - if annotation is _void: - annotation = self._annotation - - if default is _void: - default = self._default - - if _partial_kwarg is _void: - _partial_kwarg = self._partial_kwarg - - return type(self)(name, kind, default=default, annotation=annotation, - _partial_kwarg=_partial_kwarg) - - def __str__(self): - kind = self.kind - - formatted = self._name - if kind == _POSITIONAL_ONLY: - if formatted is None: - formatted = '' - formatted = '<{0}>'.format(formatted) - - # Add annotation and default value - if self._annotation is not _empty: - formatted = '{0}:{1}'.format(formatted, - formatannotation(self._annotation)) - - if self._default is not _empty: - formatted = '{0}={1}'.format(formatted, repr(self._default)) - - if kind == _VAR_POSITIONAL: - formatted = '*' + formatted - elif kind == _VAR_KEYWORD: - formatted = '**' + formatted - - return formatted - - def __repr__(self): - return '<{0} at {1:#x} {2!r}>'.format(self.__class__.__name__, - id(self), self.name) - - def __hash__(self): - msg = "unhashable type: '{0}'".format(self.__class__.__name__) - raise TypeError(msg) - - def __eq__(self, other): - return (issubclass(other.__class__, Parameter) and - self._name == other._name and - self._kind == other._kind and - self._default == other._default and - self._annotation == other._annotation) - - def __ne__(self, other): - return not self.__eq__(other) - - -class BoundArguments(object): - '''Result of `Signature.bind` call. Holds the mapping of arguments - to the function's parameters. - - Has the following public attributes: - - * arguments : OrderedDict - An ordered mutable mapping of parameters' names to arguments' values. - Does not contain arguments' default values. - * signature : Signature - The Signature object that created this instance. - * args : tuple - Tuple of positional arguments values. - * kwargs : dict - Dict of keyword arguments values. - ''' - - def __init__(self, signature, arguments): - self.arguments = arguments - self._signature = signature - - @property - def signature(self): - return self._signature - - @property - def args(self): - args = [] - for param_name, param in self._signature.parameters.items(): - if (param.kind in (_VAR_KEYWORD, _KEYWORD_ONLY) or - param._partial_kwarg): - # Keyword arguments mapped by 'functools.partial' - # (Parameter._partial_kwarg is True) are mapped - # in 'BoundArguments.kwargs', along with VAR_KEYWORD & - # KEYWORD_ONLY - break - - try: - arg = self.arguments[param_name] - except KeyError: - # We're done here. Other arguments - # will be mapped in 'BoundArguments.kwargs' - break - else: - if param.kind == _VAR_POSITIONAL: - # *args - args.extend(arg) - else: - # plain argument - args.append(arg) - - return tuple(args) - - @property - def kwargs(self): - kwargs = {} - kwargs_started = False - for param_name, param in self._signature.parameters.items(): - if not kwargs_started: - if (param.kind in (_VAR_KEYWORD, _KEYWORD_ONLY) or - param._partial_kwarg): - kwargs_started = True - else: - if param_name not in self.arguments: - kwargs_started = True - continue - - if not kwargs_started: - continue - - try: - arg = self.arguments[param_name] - except KeyError: - pass - else: - if param.kind == _VAR_KEYWORD: - # **kwargs - kwargs.update(arg) - else: - # plain keyword argument - kwargs[param_name] = arg - - return kwargs - - def __hash__(self): - msg = "unhashable type: '{0}'".format(self.__class__.__name__) - raise TypeError(msg) - - def __eq__(self, other): - return (issubclass(other.__class__, BoundArguments) and - self.signature == other.signature and - self.arguments == other.arguments) - - def __ne__(self, other): - return not self.__eq__(other) - - -class Signature(object): - '''A Signature object represents the overall signature of a function. - It stores a Parameter object for each parameter accepted by the - function, as well as information specific to the function itself. - - A Signature object has the following public attributes and methods: - - * parameters : OrderedDict - An ordered mapping of parameters' names to the corresponding - Parameter objects (keyword-only arguments are in the same order - as listed in `code.co_varnames`). - * return_annotation : object - The annotation for the return type of the function if specified. - If the function has no annotation for its return type, this - attribute is not set. - * bind(*args, **kwargs) -> BoundArguments - Creates a mapping from positional and keyword arguments to - parameters. - * bind_partial(*args, **kwargs) -> BoundArguments - Creates a partial mapping from positional and keyword arguments - to parameters (simulating 'functools.partial' behavior.) - ''' - - __slots__ = ('_return_annotation', '_parameters') - - _parameter_cls = Parameter - _bound_arguments_cls = BoundArguments - - empty = _empty - - def __init__(self, parameters=None, return_annotation=_empty, - __validate_parameters__=True): - '''Constructs Signature from the given list of Parameter - objects and 'return_annotation'. All arguments are optional. - ''' - - if parameters is None: - params = OrderedDict() - else: - if __validate_parameters__: - params = OrderedDict() - top_kind = _POSITIONAL_ONLY - - for idx, param in enumerate(parameters): - kind = param.kind - if kind < top_kind: - msg = 'wrong parameter order: {0} before {1}' - msg = msg.format(top_kind, param.kind) - raise ValueError(msg) - else: - top_kind = kind - - name = param.name - if name is None: - name = str(idx) - param = param.replace(name=name) - - if name in params: - msg = 'duplicate parameter name: {0!r}'.format(name) - raise ValueError(msg) - params[name] = param - else: - params = OrderedDict(((param.name, param) - for param in parameters)) - - self._parameters = params - self._return_annotation = return_annotation - - @classmethod - def from_function(cls, func): - '''Constructs Signature for the given python function''' - - if not isinstance(func, types.FunctionType): - raise TypeError('{0!r} is not a Python function'.format(func)) - - Parameter = cls._parameter_cls - - # Parameter information. - func_code = func.__code__ - pos_count = func_code.co_argcount - arg_names = func_code.co_varnames - positional = tuple(arg_names[:pos_count]) - keyword_only_count = getattr(func_code, 'co_kwonlyargcount', 0) - keyword_only = arg_names[pos_count:(pos_count + keyword_only_count)] - annotations = getattr(func, '__annotations__', {}) - defaults = func.__defaults__ - kwdefaults = getattr(func, '__kwdefaults__', None) - - if defaults: - pos_default_count = len(defaults) - else: - pos_default_count = 0 - - parameters = [] - - # Non-keyword-only parameters w/o defaults. - non_default_count = pos_count - pos_default_count - for name in positional[:non_default_count]: - annotation = annotations.get(name, _empty) - parameters.append(Parameter(name, annotation=annotation, - kind=_POSITIONAL_OR_KEYWORD)) - - # ... w/ defaults. - for offset, name in enumerate(positional[non_default_count:]): - annotation = annotations.get(name, _empty) - parameters.append(Parameter(name, annotation=annotation, - kind=_POSITIONAL_OR_KEYWORD, - default=defaults[offset])) - - # *args - if func_code.co_flags & 0x04: - name = arg_names[pos_count + keyword_only_count] - annotation = annotations.get(name, _empty) - parameters.append(Parameter(name, annotation=annotation, - kind=_VAR_POSITIONAL)) - - # Keyword-only parameters. - for name in keyword_only: - default = _empty - if kwdefaults is not None: - default = kwdefaults.get(name, _empty) - - annotation = annotations.get(name, _empty) - parameters.append(Parameter(name, annotation=annotation, - kind=_KEYWORD_ONLY, - default=default)) - # **kwargs - if func_code.co_flags & 0x08: - index = pos_count + keyword_only_count - if func_code.co_flags & 0x04: - index += 1 - - name = arg_names[index] - annotation = annotations.get(name, _empty) - parameters.append(Parameter(name, annotation=annotation, - kind=_VAR_KEYWORD)) - - return cls(parameters, - return_annotation=annotations.get('return', _empty), - __validate_parameters__=False) - - @property - def parameters(self): - try: - return types.MappingProxyType(self._parameters) - except AttributeError: - return OrderedDict(self._parameters.items()) - - @property - def return_annotation(self): - return self._return_annotation - - def replace(self, parameters=_void, return_annotation=_void): - '''Creates a customized copy of the Signature. - Pass 'parameters' and/or 'return_annotation' arguments - to override them in the new copy. - ''' - - if parameters is _void: - parameters = self.parameters.values() - - if return_annotation is _void: - return_annotation = self._return_annotation - - return type(self)(parameters, - return_annotation=return_annotation) - - def __hash__(self): - msg = "unhashable type: '{0}'".format(self.__class__.__name__) - raise TypeError(msg) - - def __eq__(self, other): - if (not issubclass(type(other), Signature) or - self.return_annotation != other.return_annotation or - len(self.parameters) != len(other.parameters)): - return False - - other_positions = dict((param, idx) - for idx, param in enumerate(other.parameters.keys())) - - for idx, (param_name, param) in enumerate(self.parameters.items()): - if param.kind == _KEYWORD_ONLY: - try: - other_param = other.parameters[param_name] - except KeyError: - return False - else: - if param != other_param: - return False - else: - try: - other_idx = other_positions[param_name] - except KeyError: - return False - else: - if (idx != other_idx or - param != other.parameters[param_name]): - return False - - return True - - def __ne__(self, other): - return not self.__eq__(other) - - def _bind(self, args, kwargs, partial=False): - '''Private method. Don't use directly.''' - - arguments = OrderedDict() - - parameters = iter(self.parameters.values()) - parameters_ex = () - arg_vals = iter(args) - - if partial: - # Support for binding arguments to 'functools.partial' objects. - # See 'functools.partial' case in 'signature()' implementation - # for details. - for param_name, param in self.parameters.items(): - if (param._partial_kwarg and param_name not in kwargs): - # Simulating 'functools.partial' behavior - kwargs[param_name] = param.default - - while True: - # Let's iterate through the positional arguments and corresponding - # parameters - try: - arg_val = next(arg_vals) - except StopIteration: - # No more positional arguments - try: - param = next(parameters) - except StopIteration: - # No more parameters. That's it. Just need to check that - # we have no `kwargs` after this while loop - break - else: - if param.kind == _VAR_POSITIONAL: - # That's OK, just empty *args. Let's start parsing - # kwargs - break - elif param.name in kwargs: - if param.kind == _POSITIONAL_ONLY: - msg = '{arg!r} parameter is positional only, ' \ - 'but was passed as a keyword' - msg = msg.format(arg=param.name) - raise TypeError(msg) - parameters_ex = (param,) - break - elif (param.kind == _VAR_KEYWORD or - param.default is not _empty): - # That's fine too - we have a default value for this - # parameter. So, lets start parsing `kwargs`, starting - # with the current parameter - parameters_ex = (param,) - break - else: - if partial: - parameters_ex = (param,) - break - else: - msg = '{arg!r} parameter lacking default value' - msg = msg.format(arg=param.name) - raise TypeError(msg) - else: - # We have a positional argument to process - try: - param = next(parameters) - except StopIteration: - raise TypeError('too many positional arguments') - else: - if param.kind in (_VAR_KEYWORD, _KEYWORD_ONLY): - # Looks like we have no parameter for this positional - # argument - raise TypeError('too many positional arguments') - - if param.kind == _VAR_POSITIONAL: - # We have an '*args'-like argument, let's fill it with - # all positional arguments we have left and move on to - # the next phase - values = [arg_val] - values.extend(arg_vals) - arguments[param.name] = tuple(values) - break - - if param.name in kwargs: - raise TypeError('multiple values for argument ' - '{arg!r}'.format(arg=param.name)) - - arguments[param.name] = arg_val - - # Now, we iterate through the remaining parameters to process - # keyword arguments - kwargs_param = None - for param in itertools.chain(parameters_ex, parameters): - if param.kind == _POSITIONAL_ONLY: - # This should never happen in case of a properly built - # Signature object (but let's have this check here - # to ensure correct behaviour just in case) - raise TypeError('{arg!r} parameter is positional only, ' - 'but was passed as a keyword'. \ - format(arg=param.name)) - - if param.kind == _VAR_KEYWORD: - # Memorize that we have a '**kwargs'-like parameter - kwargs_param = param - continue - - param_name = param.name - try: - arg_val = kwargs.pop(param_name) - except KeyError: - # We have no value for this parameter. It's fine though, - # if it has a default value, or it is an '*args'-like - # parameter, left alone by the processing of positional - # arguments. - if (not partial and param.kind != _VAR_POSITIONAL and - param.default is _empty): - raise TypeError('{arg!r} parameter lacking default value'. \ - format(arg=param_name)) - - else: - arguments[param_name] = arg_val - - if kwargs: - if kwargs_param is not None: - # Process our '**kwargs'-like parameter - arguments[kwargs_param.name] = kwargs - else: - raise TypeError('too many keyword arguments') - - return self._bound_arguments_cls(self, arguments) - - def bind(self, *args, **kwargs): - '''Get a BoundArguments object, that maps the passed `args` - and `kwargs` to the function's signature. Raises `TypeError` - if the passed arguments can not be bound. - ''' - return self._bind(args, kwargs) - - def bind_partial(self, *args, **kwargs): - '''Get a BoundArguments object, that partially maps the - passed `args` and `kwargs` to the function's signature. - Raises `TypeError` if the passed arguments can not be bound. - ''' - return self._bind(args, kwargs, partial=True) - - def __str__(self): - result = [] - render_kw_only_separator = True - for idx, param in enumerate(self.parameters.values()): - formatted = str(param) - - kind = param.kind - if kind == _VAR_POSITIONAL: - # OK, we have an '*args'-like parameter, so we won't need - # a '*' to separate keyword-only arguments - render_kw_only_separator = False - elif kind == _KEYWORD_ONLY and render_kw_only_separator: - # We have a keyword-only parameter to render and we haven't - # rendered an '*args'-like parameter before, so add a '*' - # separator to the parameters list ("foo(arg1, *, arg2)" case) - result.append('*') - # This condition should be only triggered once, so - # reset the flag - render_kw_only_separator = False - - result.append(formatted) - - rendered = '({0})'.format(', '.join(result)) - - if self.return_annotation is not _empty: - anno = formatannotation(self.return_annotation) - rendered += ' -> {0}'.format(anno) - - return rendered diff --git a/contrib/python/backcall/backcall/backcall.py b/contrib/python/backcall/backcall/backcall.py deleted file mode 100644 index fe1fdb5470..0000000000 --- a/contrib/python/backcall/backcall/backcall.py +++ /dev/null @@ -1,109 +0,0 @@ -# -*- coding: utf-8 -*- -""" -Created on Mon Jan 13 18:17:15 2014 - -@author: takluyver -""" -import sys -PY3 = (sys.version_info[0] >= 3) - -try: - from inspect import signature, Parameter # Python >= 3.3 -except ImportError: - from ._signatures import signature, Parameter - -if PY3: - from functools import wraps -else: - from functools import wraps as _wraps - def wraps(f): - def dec(func): - _wraps(f)(func) - func.__wrapped__ = f - return func - - return dec - -def callback_prototype(prototype): - """Decorator to process a callback prototype. - - A callback prototype is a function whose signature includes all the values - that will be passed by the callback API in question. - - The original function will be returned, with a ``prototype.adapt`` attribute - which can be used to prepare third party callbacks. - """ - protosig = signature(prototype) - positional, keyword = [], [] - for name, param in protosig.parameters.items(): - if param.kind in (Parameter.VAR_POSITIONAL, Parameter.VAR_KEYWORD): - raise TypeError("*args/**kwargs not supported in prototypes") - - if (param.default is not Parameter.empty) \ - or (param.kind == Parameter.KEYWORD_ONLY): - keyword.append(name) - else: - positional.append(name) - - kwargs = dict.fromkeys(keyword) - def adapt(callback): - """Introspect and prepare a third party callback.""" - sig = signature(callback) - try: - # XXX: callback can have extra optional parameters - OK? - sig.bind(*positional, **kwargs) - return callback - except TypeError: - pass - - # Match up arguments - unmatched_pos = positional[:] - unmatched_kw = kwargs.copy() - unrecognised = [] - # TODO: unrecognised parameters with default values - OK? - for name, param in sig.parameters.items(): - # print(name, param.kind) #DBG - if param.kind == Parameter.POSITIONAL_ONLY: - if len(unmatched_pos) > 0: - unmatched_pos.pop(0) - else: - unrecognised.append(name) - elif param.kind == Parameter.POSITIONAL_OR_KEYWORD: - if (param.default is not Parameter.empty) and (name in unmatched_kw): - unmatched_kw.pop(name) - elif len(unmatched_pos) > 0: - unmatched_pos.pop(0) - else: - unrecognised.append(name) - elif param.kind == Parameter.VAR_POSITIONAL: - unmatched_pos = [] - elif param.kind == Parameter.KEYWORD_ONLY: - if name in unmatched_kw: - unmatched_kw.pop(name) - else: - unrecognised.append(name) - else: # VAR_KEYWORD - unmatched_kw = {} - - # print(unmatched_pos, unmatched_kw, unrecognised) #DBG - - if unrecognised: - raise TypeError("Function {!r} had unmatched arguments: {}".format(callback, unrecognised)) - - n_positional = len(positional) - len(unmatched_pos) - - @wraps(callback) - def adapted(*args, **kwargs): - """Wrapper for third party callbacks that discards excess arguments""" -# print(args, kwargs) - args = args[:n_positional] - for name in unmatched_kw: - # XXX: Could name not be in kwargs? - kwargs.pop(name) -# print(args, kwargs, unmatched_pos, cut_positional, unmatched_kw) - return callback(*args, **kwargs) - - return adapted - - prototype.adapt = adapt - return prototype
\ No newline at end of file diff --git a/contrib/python/backcall/tests/__init__.py b/contrib/python/backcall/tests/__init__.py deleted file mode 100644 index faa18be5bb..0000000000 --- a/contrib/python/backcall/tests/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- diff --git a/contrib/python/backcall/tests/test_callback_prototypes.py b/contrib/python/backcall/tests/test_callback_prototypes.py deleted file mode 100644 index 09ac150cf8..0000000000 --- a/contrib/python/backcall/tests/test_callback_prototypes.py +++ /dev/null @@ -1,38 +0,0 @@ -import sys -from backcall import callback_prototype - -@callback_prototype -def msg_callback(a, b, c, d=None, e=None, f=None): - pass - -def test_all_args(): - @msg_callback.adapt - def thingy1(q, w, s, d, e, f): - return q, w, s, d, e, f - - assert not getattr(thingy1, '__wrapped__', None) - assert thingy1('A', 'B', 'C', d='D', e='E', f='F') == tuple('ABCDEF') - -if sys.version_info[0] >= 3: - exec("@msg_callback.adapt\n" - "def thingy2(t, *, d=None):\n" - " return t, d") - def test_some_args_kwonly(): - assert getattr(thingy2, '__wrapped__', None) - assert thingy2('A', 'B', 'C', d='D', e='E', f='F') == ('A', 'D') - -def test_some_args_defaults(): - @msg_callback.adapt - def thingy2b(t, d=None): - return t, d - - assert getattr(thingy2b, '__wrapped__', None) - assert thingy2b('A', 'B', 'C', d='D', e='E', f='F') == ('A', 'D') - -def test_no_args(): - @msg_callback.adapt - def thingy3(): - return 'Success' - - assert getattr(thingy3, '__wrapped__', None) - assert thingy3('A', 'B', 'C', d='D', e='E', f='F') == 'Success' diff --git a/contrib/python/backcall/tests/ya.make b/contrib/python/backcall/tests/ya.make deleted file mode 100644 index 4d010f5583..0000000000 --- a/contrib/python/backcall/tests/ya.make +++ /dev/null @@ -1,13 +0,0 @@ -PY3TEST() - -PEERDIR( - contrib/python/backcall -) - -TEST_SRCS( - test_callback_prototypes.py -) - -NO_LINT() - -END() diff --git a/contrib/python/backcall/ya.make b/contrib/python/backcall/ya.make deleted file mode 100644 index a3a97831b8..0000000000 --- a/contrib/python/backcall/ya.make +++ /dev/null @@ -1,28 +0,0 @@ -# Generated by devtools/yamaker (pypi). - -PY3_LIBRARY() - -VERSION(0.2.0) - -LICENSE(BSD-3-Clause) - -NO_LINT() - -PY_SRCS( - TOP_LEVEL - backcall/__init__.py - backcall/_signatures.py - backcall/backcall.py -) - -RESOURCE_FILES( - PREFIX contrib/python/backcall/ - .dist-info/METADATA - .dist-info/top_level.txt -) - -END() - -RECURSE_FOR_TESTS( - tests -) diff --git a/contrib/python/ipython/py3/.dist-info/METADATA b/contrib/python/ipython/py3/.dist-info/METADATA index a3e7a39c41..829b1c4f27 100644 --- a/contrib/python/ipython/py3/.dist-info/METADATA +++ b/contrib/python/ipython/py3/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: ipython -Version: 8.16.1 +Version: 8.17.1 Summary: IPython: Productive Interactive Computing Home-page: https://ipython.org Author: The IPython Development Team @@ -26,11 +26,9 @@ Classifier: Topic :: System :: Shells Requires-Python: >=3.9 Description-Content-Type: text/x-rst License-File: LICENSE -Requires-Dist: backcall Requires-Dist: decorator Requires-Dist: jedi >=0.13 Requires-Dist: matplotlib-inline -Requires-Dist: pickleshare Requires-Dist: prompt-toolkit !=3.0.37,<3.1.0,>=3.0.30 Requires-Dist: pygments >=2.4.0 Requires-Dist: stack-data @@ -55,6 +53,7 @@ Requires-Dist: exceptiongroup ; extra == 'all' Requires-Dist: pytest <7.1 ; extra == 'all' Requires-Dist: pytest-asyncio ; extra == 'all' Requires-Dist: testpath ; extra == 'all' +Requires-Dist: pickleshare ; extra == 'all' Requires-Dist: nbconvert ; extra == 'all' Requires-Dist: nbformat ; extra == 'all' Requires-Dist: ipywidgets ; extra == 'all' @@ -63,7 +62,7 @@ Requires-Dist: ipyparallel ; extra == 'all' Requires-Dist: qtconsole ; extra == 'all' Requires-Dist: curio ; extra == 'all' Requires-Dist: matplotlib !=3.2.0 ; extra == 'all' -Requires-Dist: numpy >=1.21 ; extra == 'all' +Requires-Dist: numpy >=1.22 ; extra == 'all' Requires-Dist: pandas ; extra == 'all' Requires-Dist: trio ; extra == 'all' Provides-Extra: black @@ -82,6 +81,7 @@ Requires-Dist: exceptiongroup ; extra == 'doc' Requires-Dist: pytest <7.1 ; extra == 'doc' Requires-Dist: pytest-asyncio ; extra == 'doc' Requires-Dist: testpath ; extra == 'doc' +Requires-Dist: pickleshare ; extra == 'doc' Provides-Extra: kernel Requires-Dist: ipykernel ; extra == 'kernel' Provides-Extra: nbconvert @@ -100,14 +100,16 @@ Provides-Extra: test Requires-Dist: pytest <7.1 ; extra == 'test' Requires-Dist: pytest-asyncio ; extra == 'test' Requires-Dist: testpath ; extra == 'test' +Requires-Dist: pickleshare ; extra == 'test' Provides-Extra: test_extra Requires-Dist: pytest <7.1 ; extra == 'test_extra' Requires-Dist: pytest-asyncio ; extra == 'test_extra' Requires-Dist: testpath ; extra == 'test_extra' +Requires-Dist: pickleshare ; extra == 'test_extra' Requires-Dist: curio ; extra == 'test_extra' Requires-Dist: matplotlib !=3.2.0 ; extra == 'test_extra' Requires-Dist: nbformat ; extra == 'test_extra' -Requires-Dist: numpy >=1.21 ; extra == 'test_extra' +Requires-Dist: numpy >=1.22 ; extra == 'test_extra' Requires-Dist: pandas ; extra == 'test_extra' Requires-Dist: trio ; extra == 'test_extra' diff --git a/contrib/python/ipython/py3/IPython/__main__.py b/contrib/python/ipython/py3/IPython/__main__.py index 8e9f989a82..3b4605601a 100644 --- a/contrib/python/ipython/py3/IPython/__main__.py +++ b/contrib/python/ipython/py3/IPython/__main__.py @@ -2,13 +2,13 @@ # encoding: utf-8 """Terminal-based IPython entry point. """ -#----------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- # Copyright (c) 2012, IPython Development Team. # # Distributed under the terms of the Modified BSD License. # # The full license is in the file COPYING.txt, distributed with this software. -#----------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- from IPython import start_ipython diff --git a/contrib/python/ipython/py3/IPython/core/completerlib.py b/contrib/python/ipython/py3/IPython/core/completerlib.py index 65efa42254..74252dcfad 100644 --- a/contrib/python/ipython/py3/IPython/core/completerlib.py +++ b/contrib/python/ipython/py3/IPython/core/completerlib.py @@ -238,6 +238,7 @@ def try_import(mod: str, only_modules=False) -> List[str]: completions.extend(m_all) if m_is_init: + file_ = m.__file__ completions.extend(arcadia_module_list(mod)) completions_set = {c for c in completions if isinstance(c, str)} completions_set.discard('__init__') diff --git a/contrib/python/ipython/py3/IPython/core/display.py b/contrib/python/ipython/py3/IPython/core/display.py index ffa6e185c4..20e2e34b8f 100644 --- a/contrib/python/ipython/py3/IPython/core/display.py +++ b/contrib/python/ipython/py3/IPython/core/display.py @@ -286,7 +286,7 @@ class DisplayObject(object): in the frontend. The MIME type of the data should match the subclasses used, so the Png subclass should be used for 'image/png' data. If the data is a URL, the data will first be downloaded - and then displayed. If + and then displayed. Parameters ---------- diff --git a/contrib/python/ipython/py3/IPython/core/events.py b/contrib/python/ipython/py3/IPython/core/events.py index 3a66e75e5a..90ff8f746d 100644 --- a/contrib/python/ipython/py3/IPython/core/events.py +++ b/contrib/python/ipython/py3/IPython/core/events.py @@ -13,8 +13,6 @@ events and the arguments which will be passed to them. This API is experimental in IPython 2.0, and may be revised in future versions. """ -from backcall import callback_prototype - class EventManager(object): """Manage a collection of events and a sequence of callbacks for each. @@ -63,23 +61,14 @@ class EventManager(object): """ if not callable(function): raise TypeError('Need a callable, got %r' % function) - callback_proto = available_events.get(event) if function not in self.callbacks[event]: - self.callbacks[event].append(callback_proto.adapt(function)) + self.callbacks[event].append(function) def unregister(self, event, function): """Remove a callback from the given event.""" if function in self.callbacks[event]: return self.callbacks[event].remove(function) - # Remove callback in case ``function`` was adapted by `backcall`. - for callback in self.callbacks[event]: - try: - if callback.__wrapped__ is function: - return self.callbacks[event].remove(callback) - except AttributeError: - pass - raise ValueError('Function {!r} is not registered as a {} callback'.format(function, event)) def trigger(self, event, *args, **kwargs): @@ -100,9 +89,8 @@ class EventManager(object): available_events = {} def _define_event(callback_function): - callback_proto = callback_prototype(callback_function) - available_events[callback_function.__name__] = callback_proto - return callback_proto + available_events[callback_function.__name__] = callback_function + return callback_function # ------------------------------------------------------------------------------ # Callback prototypes diff --git a/contrib/python/ipython/py3/IPython/core/guarded_eval.py b/contrib/python/ipython/py3/IPython/core/guarded_eval.py index 5d405b2208..a304affc35 100644 --- a/contrib/python/ipython/py3/IPython/core/guarded_eval.py +++ b/contrib/python/ipython/py3/IPython/core/guarded_eval.py @@ -1,3 +1,4 @@ +from inspect import signature, Signature from typing import ( Any, Callable, @@ -335,6 +336,7 @@ class _IdentitySubscript: IDENTITY_SUBSCRIPT = _IdentitySubscript() SUBSCRIPT_MARKER = "__SUBSCRIPT_SENTINEL__" +UNKNOWN_SIGNATURE = Signature() class GuardRejection(Exception): @@ -415,6 +417,10 @@ UNARY_OP_DUNDERS: Dict[Type[ast.unaryop], Tuple[str, ...]] = { } +class Duck: + """A dummy class used to create objects of other classes without calling their ``__init__``""" + + def _find_dunder(node_op, dunders) -> Union[Tuple[str, ...], None]: dunder = None for op, candidate_dunder in dunders.items(): @@ -584,6 +590,27 @@ def eval_node(node: Union[ast.AST, None], context: EvaluationContext): if policy.can_call(func) and not node.keywords: args = [eval_node(arg, context) for arg in node.args] return func(*args) + try: + sig = signature(func) + except ValueError: + sig = UNKNOWN_SIGNATURE + # if annotation was not stringized, or it was stringized + # but resolved by signature call we know the return type + not_empty = sig.return_annotation is not Signature.empty + not_stringized = not isinstance(sig.return_annotation, str) + if not_empty and not_stringized: + duck = Duck() + # if allow-listed builtin is on type annotation, instantiate it + if policy.can_call(sig.return_annotation) and not node.keywords: + args = [eval_node(arg, context) for arg in node.args] + return sig.return_annotation(*args) + try: + # if custom class is in type annotation, mock it; + # this only works for heap types, not builtins + duck.__class__ = sig.return_annotation + return duck + except TypeError: + pass raise GuardRejection( "Call for", func, # not joined to avoid calling `repr` diff --git a/contrib/python/ipython/py3/IPython/core/inputsplitter.py b/contrib/python/ipython/py3/IPython/core/inputsplitter.py index a4401184bd..33ed563221 100644 --- a/contrib/python/ipython/py3/IPython/core/inputsplitter.py +++ b/contrib/python/ipython/py3/IPython/core/inputsplitter.py @@ -31,7 +31,8 @@ import sys import tokenize import warnings -from typing import List +from typing import List, Tuple, Union, Optional +from types import CodeType from IPython.core.inputtransformer import (leading_indent, classic_prompt, @@ -91,7 +92,13 @@ def num_ini_spaces(s): ------- n : int """ - + warnings.warn( + "`num_ini_spaces` is Pending Deprecation since IPython 8.17." + "It is considered fro removal in in future version. " + "Please open an issue if you believe it should be kept.", + stacklevel=2, + category=PendingDeprecationWarning, + ) ini_spaces = ini_spaces_re.match(s) if ini_spaces: return ini_spaces.end() @@ -144,7 +151,7 @@ def partial_tokens(s): else: raise -def find_next_indent(code): +def find_next_indent(code) -> int: """Find the number of spaces for the next line of indentation""" tokens = list(partial_tokens(code)) if tokens[-1].type == tokenize.ENDMARKER: @@ -318,7 +325,7 @@ class InputSplitter(object): # If self.source matches the first value, the second value is a valid # current indentation. Otherwise, the cache is invalid and the indentation # must be recalculated. - _indent_spaces_cache = None, None + _indent_spaces_cache: Union[Tuple[None, None], Tuple[str, int]] = None, None # String, indicating the default input encoding. It is computed by default # at initialization time via get_input_encoding(), but it can be reset by a # client with specific knowledge of the encoding. @@ -326,11 +333,11 @@ class InputSplitter(object): # String where the current full source input is stored, properly encoded. # Reading this attribute is the normal way of querying the currently pushed # source code, that has been properly encoded. - source = '' + source: str = "" # Code object corresponding to the current source. It is automatically # synced to the source, so it can be queried at any time to obtain the code # object; it will be None if the source doesn't compile to valid Python. - code = None + code: Optional[CodeType] = None # Private attributes @@ -339,9 +346,9 @@ class InputSplitter(object): # Command compiler _compile: codeop.CommandCompiler # Boolean indicating whether the current block is complete - _is_complete = None + _is_complete: Optional[bool] = None # Boolean indicating whether the current block has an unrecoverable syntax error - _is_invalid = False + _is_invalid: bool = False def __init__(self) -> None: """Create a new InputSplitter instance.""" @@ -511,9 +518,10 @@ class InputSplitter(object): # General fallback - accept more code return True - def get_indent_spaces(self): + def get_indent_spaces(self) -> int: sourcefor, n = self._indent_spaces_cache if sourcefor == self.source: + assert n is not None return n # self.source always has a trailing newline @@ -562,7 +570,7 @@ class IPythonInputSplitter(InputSplitter): # Private attributes # List with lines of raw input accumulated so far. - _buffer_raw = None + _buffer_raw: List[str] def __init__(self, line_input_checker=True, physical_line_transforms=None, logical_line_transforms=None, python_line_transforms=None): @@ -652,8 +660,8 @@ class IPythonInputSplitter(InputSplitter): tmp = transform.reset() if tmp is not None: yield tmp - - out = [] + + out: List[str] = [] for t in self.transforms_in_use: out = _flush(t, out) diff --git a/contrib/python/ipython/py3/IPython/core/inputtransformer.py b/contrib/python/ipython/py3/IPython/core/inputtransformer.py index 81cd1fa08c..bb1061e8dc 100644 --- a/contrib/python/ipython/py3/IPython/core/inputtransformer.py +++ b/contrib/python/ipython/py3/IPython/core/inputtransformer.py @@ -71,8 +71,8 @@ class InputTransformer(metaclass=abc.ABCMeta): """ @functools.wraps(func) def transformer_factory(**kwargs): - return cls(func, **kwargs) - + return cls(func, **kwargs) # type: ignore [call-arg] + return transformer_factory class StatelessInputTransformer(InputTransformer): @@ -194,7 +194,7 @@ def assemble_logical_lines(): line = ''.join(parts) # Utilities -def _make_help_call(target, esc, lspace): +def _make_help_call(target: str, esc: str, lspace: str) -> str: """Prepares a pinfo(2)/psearch call from a target name and the escape (i.e. ? or ??)""" method = 'pinfo2' if esc == '??' \ @@ -212,17 +212,19 @@ def _make_help_call(target, esc, lspace): # These define the transformations for the different escape characters. -def _tr_system(line_info): +def _tr_system(line_info: LineInfo): "Translate lines escaped with: !" cmd = line_info.line.lstrip().lstrip(ESC_SHELL) return '%sget_ipython().system(%r)' % (line_info.pre, cmd) -def _tr_system2(line_info): + +def _tr_system2(line_info: LineInfo): "Translate lines escaped with: !!" cmd = line_info.line.lstrip()[2:] return '%sget_ipython().getoutput(%r)' % (line_info.pre, cmd) -def _tr_help(line_info): + +def _tr_help(line_info: LineInfo): "Translate lines escaped with: ?/??" # A naked help line should just fire the intro help screen if not line_info.line[1:]: @@ -230,7 +232,8 @@ def _tr_help(line_info): return _make_help_call(line_info.ifun, line_info.esc, line_info.pre) -def _tr_magic(line_info): + +def _tr_magic(line_info: LineInfo): "Translate lines escaped with: %" tpl = '%sget_ipython().run_line_magic(%r, %r)' if line_info.line.startswith(ESC_MAGIC2): @@ -241,17 +244,20 @@ def _tr_magic(line_info): t_magic_name = t_magic_name.lstrip(ESC_MAGIC) return tpl % (line_info.pre, t_magic_name, t_magic_arg_s) -def _tr_quote(line_info): + +def _tr_quote(line_info: LineInfo): "Translate lines escaped with: ," return '%s%s("%s")' % (line_info.pre, line_info.ifun, '", "'.join(line_info.the_rest.split()) ) -def _tr_quote2(line_info): + +def _tr_quote2(line_info: LineInfo): "Translate lines escaped with: ;" return '%s%s("%s")' % (line_info.pre, line_info.ifun, line_info.the_rest) -def _tr_paren(line_info): + +def _tr_paren(line_info: LineInfo): "Translate lines escaped with: /" return '%s%s(%s)' % (line_info.pre, line_info.ifun, ", ".join(line_info.the_rest.split())) @@ -266,9 +272,8 @@ tr = { ESC_SHELL : _tr_system, ESC_PAREN : _tr_paren } @StatelessInputTransformer.wrap -def escaped_commands(line): - """Transform escaped commands - %magic, !system, ?help + various autocalls. - """ +def escaped_commands(line: str): + """Transform escaped commands - %magic, !system, ?help + various autocalls.""" if not line or line.isspace(): return line lineinf = LineInfo(line) @@ -342,20 +347,22 @@ def ends_in_comment_or_string(src): @StatelessInputTransformer.wrap -def help_end(line): +def help_end(line: str): """Translate lines with ?/?? at the end""" m = _help_end_re.search(line) if m is None or ends_in_comment_or_string(line): return line target = m.group(1) esc = m.group(3) - lspace = _initial_space_re.match(line).group(0) + match = _initial_space_re.match(line) + assert match is not None + lspace = match.group(0) return _make_help_call(target, esc, lspace) @CoroutineInputTransformer.wrap -def cellmagic(end_on_blank_line=False): +def cellmagic(end_on_blank_line: bool = False): """Captures & transforms cell magics. After a cell magic is started, this stores up any lines it gets until it is diff --git a/contrib/python/ipython/py3/IPython/core/interactiveshell.py b/contrib/python/ipython/py3/IPython/core/interactiveshell.py index 894403f98b..0a528c51f2 100644 --- a/contrib/python/ipython/py3/IPython/core/interactiveshell.py +++ b/contrib/python/ipython/py3/IPython/core/interactiveshell.py @@ -21,6 +21,7 @@ import inspect import os import re import runpy +import shutil import subprocess import sys import tempfile @@ -36,7 +37,28 @@ from typing import List as ListType, Dict as DictType, Any as AnyType from typing import Optional, Sequence, Tuple from warnings import warn -from pickleshare import PickleShareDB +try: + from pickleshare import PickleShareDB +except ModuleNotFoundError: + + class PickleShareDB: # type: ignore [no-redef] + def __init__(self, path): + pass + + def get(self, key, default): + warn( + f"using {key} requires you to install the `pickleshare` library.", + stacklevel=2, + ) + return default + + def __setitem__(self, key, value): + warn( + f"using {key} requires you to install the `pickleshare` library.", + stacklevel=2, + ) + + from tempfile import TemporaryDirectory from traitlets import ( Any, @@ -3902,7 +3924,7 @@ class InteractiveShell(SingletonConfigurable): del self.tempfiles for tdir in self.tempdirs: try: - tdir.rmdir() + shutil.rmtree(tdir) self.tempdirs.remove(tdir) except FileNotFoundError: pass diff --git a/contrib/python/ipython/py3/IPython/core/magics/packaging.py b/contrib/python/ipython/py3/IPython/core/magics/packaging.py index 2f7652c169..093b0a2ec1 100644 --- a/contrib/python/ipython/py3/IPython/core/magics/packaging.py +++ b/contrib/python/ipython/py3/IPython/core/magics/packaging.py @@ -8,6 +8,7 @@ # The full license is in the file COPYING.txt, distributed with this software. #----------------------------------------------------------------------------- +import functools import re import shlex import sys @@ -16,33 +17,49 @@ from pathlib import Path from IPython.core.magic import Magics, magics_class, line_magic -def _is_conda_environment(): - """Return True if the current Python executable is in a conda env""" - # TODO: does this need to change on windows? - return Path(sys.prefix, "conda-meta", "history").exists() +def is_conda_environment(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + """Return True if the current Python executable is in a conda env""" + # TODO: does this need to change on windows? + if not Path(sys.prefix, "conda-meta", "history").exists(): + raise ValueError( + "The python kernel does not appear to be a conda environment. " + "Please use ``%pip install`` instead." + ) + return func(*args, **kwargs) + return wrapper -def _get_conda_executable(): - """Find the path to the conda executable""" + +def _get_conda_like_executable(command): + """Find the path to the given executable + + Parameters + ---------- + + executable: string + Value should be: conda, mamba or micromamba + """ # Check if there is a conda executable in the same directory as the Python executable. # This is the case within conda's root environment. - conda = Path(sys.executable).parent / "conda" - if conda.is_file(): - return str(conda) + executable = Path(sys.executable).parent / command + if executable.is_file(): + return str(executable) # Otherwise, attempt to extract the executable from conda history. # This applies in any conda environment. history = Path(sys.prefix, "conda-meta", "history").read_text(encoding="utf-8") match = re.search( - r"^#\s*cmd:\s*(?P<command>.*conda)\s[create|install]", + rf"^#\s*cmd:\s*(?P<command>.*{executable})\s[create|install]", history, flags=re.MULTILINE, ) if match: return match.groupdict()["command"] - # Fallback: assume conda is available on the system path. - return "conda" + # Fallback: assume the executable is available on the system path. + return command CONDA_COMMANDS_REQUIRING_PREFIX = { @@ -76,18 +93,7 @@ class PackagingMagics(Magics): print("Note: you may need to restart the kernel to use updated packages.") - @line_magic - def conda(self, line): - """Run the conda package manager within the current kernel. - - Usage: - %conda install [pkgs] - """ - if not _is_conda_environment(): - raise ValueError("The python kernel does not appear to be a conda environment. " - "Please use ``%pip install`` instead.") - - conda = _get_conda_executable() + def _run_command(self, cmd, line): args = shlex.split(line) command = args[0] if len(args) > 0 else "" args = args[1:] if len(args) > 1 else [""] @@ -108,5 +114,38 @@ class PackagingMagics(Magics): if needs_prefix and not has_prefix: extra_args.extend(["--prefix", sys.prefix]) - self.shell.system(' '.join([conda, command] + extra_args + args)) + self.shell.system(" ".join([cmd, command] + extra_args + args)) print("\nNote: you may need to restart the kernel to use updated packages.") + + @line_magic + @is_conda_environment + def conda(self, line): + """Run the conda package manager within the current kernel. + + Usage: + %conda install [pkgs] + """ + conda = _get_conda_like_executable("conda") + self._run_command(conda, line) + + @line_magic + @is_conda_environment + def mamba(self, line): + """Run the mamba package manager within the current kernel. + + Usage: + %mamba install [pkgs] + """ + mamba = _get_conda_like_executable("mamba") + self._run_command(mamba, line) + + @line_magic + @is_conda_environment + def micromamba(self, line): + """Run the conda package manager within the current kernel. + + Usage: + %micromamba install [pkgs] + """ + micromamba = _get_conda_like_executable("micromamba") + self._run_command(micromamba, line) diff --git a/contrib/python/ipython/py3/IPython/core/oinspect.py b/contrib/python/ipython/py3/IPython/core/oinspect.py index ef6a0d02d7..d77a732267 100644 --- a/contrib/python/ipython/py3/IPython/core/oinspect.py +++ b/contrib/python/ipython/py3/IPython/core/oinspect.py @@ -416,6 +416,8 @@ class Inspector(Colorable): If any exception is generated, None is returned instead and the exception is suppressed.""" + if not callable(obj): + return None try: return _render_signature(signature(obj), oname) except: diff --git a/contrib/python/ipython/py3/IPython/core/release.py b/contrib/python/ipython/py3/IPython/core/release.py index 152ebcb87b..ee0042d081 100644 --- a/contrib/python/ipython/py3/IPython/core/release.py +++ b/contrib/python/ipython/py3/IPython/core/release.py @@ -16,7 +16,7 @@ # release. 'dev' as a _version_extra string means this is a development # version _version_major = 8 -_version_minor = 16 +_version_minor = 17 _version_patch = 1 _version_extra = ".dev" # _version_extra = "rc1" diff --git a/contrib/python/ipython/py3/IPython/lib/pretty.py b/contrib/python/ipython/py3/IPython/lib/pretty.py index 3486450786..2575c3b0a9 100644 --- a/contrib/python/ipython/py3/IPython/lib/pretty.py +++ b/contrib/python/ipython/py3/IPython/lib/pretty.py @@ -841,17 +841,8 @@ _env_type = type(os.environ) if _env_type is not dict: _type_pprinters[_env_type] = _dict_pprinter_factory('environ{', '}') -try: - # In PyPy, types.DictProxyType is dict, setting the dictproxy printer - # using dict.setdefault avoids overwriting the dict printer - _type_pprinters.setdefault(types.DictProxyType, - _dict_pprinter_factory('dict_proxy({', '})')) - _type_pprinters[types.ClassType] = _type_pprint - _type_pprinters[types.SliceType] = _repr_pprint -except AttributeError: # Python 3 - _type_pprinters[types.MappingProxyType] = \ - _dict_pprinter_factory('mappingproxy({', '})') - _type_pprinters[slice] = _repr_pprint +_type_pprinters[types.MappingProxyType] = _dict_pprinter_factory("mappingproxy({", "})") +_type_pprinters[slice] = _repr_pprint _type_pprinters[range] = _repr_pprint _type_pprinters[bytes] = _repr_pprint diff --git a/contrib/python/ipython/py3/IPython/utils/_sysinfo.py b/contrib/python/ipython/py3/IPython/utils/_sysinfo.py index 5e2eeadcf3..8d771a843f 100644 --- a/contrib/python/ipython/py3/IPython/utils/_sysinfo.py +++ b/contrib/python/ipython/py3/IPython/utils/_sysinfo.py @@ -1,2 +1,2 @@ # GENERATED BY setup.py -commit = "a2af83ddf" +commit = "0cfced36e" diff --git a/contrib/python/ipython/py3/IPython/utils/py3compat.py b/contrib/python/ipython/py3/IPython/utils/py3compat.py index 34af4c58f4..66ef337248 100644 --- a/contrib/python/ipython/py3/IPython/utils/py3compat.py +++ b/contrib/python/ipython/py3/IPython/utils/py3compat.py @@ -57,6 +57,7 @@ def execfile(fname, glob, loc=None, compiler=None): PYPY = platform.python_implementation() == "PyPy" + # Cython still rely on that as a Dec 28 2019 # See https://github.com/cython/cython/pull/3291 and # https://github.com/ipython/ipython/issues/12068 diff --git a/contrib/python/ipython/py3/IPython/utils/sysinfo.py b/contrib/python/ipython/py3/IPython/utils/sysinfo.py index 857f0cf2d8..347123fd53 100644 --- a/contrib/python/ipython/py3/IPython/utils/sysinfo.py +++ b/contrib/python/ipython/py3/IPython/utils/sysinfo.py @@ -20,6 +20,8 @@ import pprint import sys import subprocess +from pathlib import Path + from IPython.core import release from IPython.utils import _sysinfo, encoding @@ -27,7 +29,7 @@ from IPython.utils import _sysinfo, encoding # Code #----------------------------------------------------------------------------- -def pkg_commit_hash(pkg_path): +def pkg_commit_hash(pkg_path: str) -> tuple[str, str]: """Get short form of commit hash given directory `pkg_path` We get the commit hash from (in order of preference): @@ -65,7 +67,7 @@ def pkg_commit_hash(pkg_path): return '(none found)', '<not found>' -def pkg_info(pkg_path): +def pkg_info(pkg_path: str) -> dict: """Return dict describing the context of this package Parameters @@ -92,11 +94,10 @@ def pkg_info(pkg_path): default_encoding=encoding.DEFAULT_ENCODING, ) -def get_sys_info(): +def get_sys_info() -> dict: """Return useful information about IPython and the system, as a dict.""" - p = os.path - path = p.realpath(p.dirname(p.abspath(p.join(__file__, '..')))) - return pkg_info(path) + path = Path(__file__, "..").resolve().parent + return pkg_info(str(path)) def sys_info(): """Return useful information about IPython and the system, as a string. diff --git a/contrib/python/ipython/py3/IPython/utils/text.py b/contrib/python/ipython/py3/IPython/utils/text.py index 74bccddf68..9b653dcee0 100644 --- a/contrib/python/ipython/py3/IPython/utils/text.py +++ b/contrib/python/ipython/py3/IPython/utils/text.py @@ -13,15 +13,12 @@ import re import string import sys import textwrap +import warnings from string import Formatter from pathlib import Path +from typing import List, Union, Optional, Dict, Tuple -# datetime.strftime date format for ipython -if sys.platform == 'win32': - date_format = "%B %d, %Y" -else: - date_format = "%B %-d, %Y" class LSString(str): """String derivative with a special access attributes. @@ -336,7 +333,13 @@ ini_spaces_re = re.compile(r'^(\s+)') def num_ini_spaces(strng): """Return the number of initial spaces in a string""" - + warnings.warn( + "`num_ini_spaces` is Pending Deprecation since IPython 8.17." + "It is considered fro removal in in future version. " + "Please open an issue if you believe it should be kept.", + stacklevel=2, + category=PendingDeprecationWarning, + ) ini_spaces = ini_spaces_re.match(strng) if ini_spaces: return ini_spaces.end() @@ -391,6 +394,13 @@ def wrap_paragraphs(text, ncols=80): ------- list of complete paragraphs, wrapped to fill `ncols` columns. """ + warnings.warn( + "`wrap_paragraphs` is Pending Deprecation since IPython 8.17." + "It is considered fro removal in in future version. " + "Please open an issue if you believe it should be kept.", + stacklevel=2, + category=PendingDeprecationWarning, + ) paragraph_re = re.compile(r'\n(\s*\n)+', re.MULTILINE) text = dedent(text).strip() paragraphs = paragraph_re.split(text)[::2] # every other entry is space @@ -465,6 +475,14 @@ def strip_ansi(source): source : str Source to remove the ansi from """ + warnings.warn( + "`strip_ansi` is Pending Deprecation since IPython 8.17." + "It is considered fro removal in in future version. " + "Please open an issue if you believe it should be kept.", + stacklevel=2, + category=PendingDeprecationWarning, + ) + return re.sub(r'\033\[(\d|;)+?m', '', source) @@ -611,7 +629,7 @@ def _col_chunks(l, max_rows, row_first=False): yield l[i:(i + max_rows)] -def _find_optimal(rlist, row_first=False, separator_size=2, displaywidth=80): +def _find_optimal(rlist, row_first: bool, separator_size: int, displaywidth: int): """Calculate optimal info to columnize a list of string""" for max_rows in range(1, len(rlist) + 1): col_widths = list(map(max, _col_chunks(rlist, max_rows, row_first))) @@ -634,7 +652,9 @@ def _get_or_default(mylist, i, default=None): return mylist[i] -def compute_item_matrix(items, row_first=False, empty=None, *args, **kwargs) : +def compute_item_matrix( + items, row_first: bool = False, empty=None, *, separator_size=2, displaywidth=80 +) -> Tuple[List[List[int]], Dict[str, int]]: """Returns a nested list, and info to columnize items Parameters @@ -682,8 +702,20 @@ def compute_item_matrix(items, row_first=False, empty=None, *args, **kwargs) : In [5]: all((info[k] == ideal[k] for k in ideal.keys())) Out[5]: True """ - info = _find_optimal(list(map(len, items)), row_first, *args, **kwargs) - nrow, ncol = info['max_rows'], info['num_columns'] + warnings.warn( + "`compute_item_matrix` is Pending Deprecation since IPython 8.17." + "It is considered fro removal in in future version. " + "Please open an issue if you believe it should be kept.", + stacklevel=2, + category=PendingDeprecationWarning, + ) + info = _find_optimal( + list(map(len, items)), + row_first, + separator_size=separator_size, + displaywidth=displaywidth, + ) + nrow, ncol = info["max_rows"], info["num_columns"] if row_first: return ([[_get_or_default(items, r * ncol + c, default=empty) for c in range(ncol)] for r in range(nrow)], info) else: @@ -709,14 +741,29 @@ def columnize(items, row_first=False, separator=" ", displaywidth=80, spread=Fa ------- The formatted string. """ + warnings.warn( + "`columnize` is Pending Deprecation since IPython 8.17." + "It is considered fro removal in in future version. " + "Please open an issue if you believe it should be kept.", + stacklevel=2, + category=PendingDeprecationWarning, + ) if not items: - return '\n' - matrix, info = compute_item_matrix(items, row_first=row_first, separator_size=len(separator), displaywidth=displaywidth) + return "\n" + matrix: List[List[int]] + matrix, info = compute_item_matrix( + items, + row_first=row_first, + separator_size=len(separator), + displaywidth=displaywidth, + ) if spread: - separator = separator.ljust(int(info['optimal_separator_width'])) - fmatrix = [filter(None, x) for x in matrix] - sjoin = lambda x : separator.join([ y.ljust(w, ' ') for y, w in zip(x, info['column_widths'])]) - return '\n'.join(map(sjoin, fmatrix))+'\n' + separator = separator.ljust(int(info["optimal_separator_width"])) + fmatrix: List[filter[int]] = [filter(None, x) for x in matrix] + sjoin = lambda x: separator.join( + [y.ljust(w, " ") for y, w in zip(x, info["column_widths"])] + ) + return "\n".join(map(sjoin, fmatrix)) + "\n" def get_text_list(list_, last_sep=' and ', sep=", ", wrap_item_with=""): diff --git a/contrib/python/ipython/py3/IPython/utils/timing.py b/contrib/python/ipython/py3/IPython/utils/timing.py index 3a181ae728..d87e5fa8ef 100644 --- a/contrib/python/ipython/py3/IPython/utils/timing.py +++ b/contrib/python/ipython/py3/IPython/utils/timing.py @@ -23,8 +23,8 @@ import time # If possible (Unix), use the resource module instead of time.clock() try: import resource -except ImportError: - resource = None +except ModuleNotFoundError: + resource = None # type: ignore [assignment] # Some implementations (like jyputerlite) don't have getrusage if resource is not None and hasattr(resource, "getrusage"): diff --git a/contrib/python/ipython/py3/IPython/utils/tokenutil.py b/contrib/python/ipython/py3/IPython/utils/tokenutil.py index 5fd8a1fbe1..6b99a58ae2 100644 --- a/contrib/python/ipython/py3/IPython/utils/tokenutil.py +++ b/contrib/python/ipython/py3/IPython/utils/tokenutil.py @@ -8,12 +8,14 @@ from io import StringIO from keyword import iskeyword import tokenize +from tokenize import TokenInfo +from typing import List, Optional Token = namedtuple('Token', ['token', 'text', 'start', 'end', 'line']) def generate_tokens(readline): - """wrap generate_tokens to catch EOF errors""" + """wrap generate_tkens to catch EOF errors""" try: for token in tokenize.generate_tokens(readline): yield token @@ -22,7 +24,9 @@ def generate_tokens(readline): return -def generate_tokens_catch_errors(readline, extra_errors_to_catch=None): +def generate_tokens_catch_errors( + readline, extra_errors_to_catch: Optional[List[str]] = None +): default_errors_to_catch = [ "unterminated string literal", "invalid non-printable character", @@ -31,7 +35,7 @@ def generate_tokens_catch_errors(readline, extra_errors_to_catch=None): assert extra_errors_to_catch is None or isinstance(extra_errors_to_catch, list) errors_to_catch = default_errors_to_catch + (extra_errors_to_catch or []) - tokens = [] + tokens: List[TokenInfo] = [] try: for token in tokenize.generate_tokens(readline): tokens.append(token) @@ -84,7 +88,8 @@ def line_at_cursor(cell, cursor_pos=0): line = "" return (line, offset) -def token_at_cursor(cell, cursor_pos=0): + +def token_at_cursor(cell: str, cursor_pos: int = 0): """Get the token at a given cursor Used for introspection. @@ -94,13 +99,13 @@ def token_at_cursor(cell, cursor_pos=0): Parameters ---------- - cell : unicode + cell : str A block of Python code cursor_pos : int The location of the cursor in the block where the token should be found """ - names = [] - tokens = [] + names: List[str] = [] + tokens: List[Token] = [] call_names = [] offsets = {1: 0} # lines start at 1 diff --git a/contrib/python/ipython/py3/README.rst b/contrib/python/ipython/py3/README.rst index b004792e0e..2b179ab280 100644 --- a/contrib/python/ipython/py3/README.rst +++ b/contrib/python/ipython/py3/README.rst @@ -10,8 +10,8 @@ .. image:: https://www.codetriage.com/ipython/ipython/badges/users.svg :target: https://www.codetriage.com/ipython/ipython/ -.. image:: https://raster.shields.io/badge/Follows-NEP29-brightgreen.png - :target: https://numpy.org/neps/nep-0029-deprecation_policy.html +.. image:: https://raster.shields.io/badge/Follows-SPEC--0000-brightgreen.png + :target: https://scientific-python.org/specs/spec-0000/ .. image:: https://tidelift.com/badges/package/pypi/ipython?style=flat :target: https://tidelift.com/subscription/pkg/pypi-ipython @@ -31,6 +31,8 @@ IPython (Interactive Python) is a command shell for interactive computing in mul **IPython versions and Python Support** +Starting after IPython 8.16, we will progressively transition to `Spec-0000 <https://scientific-python.org/specs/spec-0000/>`_. + Starting with IPython 7.10, IPython follows `NEP 29 <https://numpy.org/neps/nep-0029-deprecation_policy.html>`_ **IPython 7.17+** requires Python version 3.7 and above. diff --git a/contrib/python/ipython/py3/ya.make b/contrib/python/ipython/py3/ya.make index 14754c3091..870b9e3579 100644 --- a/contrib/python/ipython/py3/ya.make +++ b/contrib/python/ipython/py3/ya.make @@ -2,17 +2,15 @@ PY3_LIBRARY() -VERSION(8.16.1) +VERSION(8.17.1) LICENSE(BSD-3-Clause) PEERDIR( contrib/python/Pygments - contrib/python/backcall contrib/python/decorator contrib/python/jedi contrib/python/matplotlib-inline - contrib/python/pickleshare contrib/python/prompt-toolkit contrib/python/stack-data contrib/python/traitlets diff --git a/library/cpp/CMakeLists.darwin-x86_64.txt b/library/cpp/CMakeLists.darwin-x86_64.txt index 22b108ee05..0f393b2039 100644 --- a/library/cpp/CMakeLists.darwin-x86_64.txt +++ b/library/cpp/CMakeLists.darwin-x86_64.txt @@ -63,7 +63,6 @@ add_subdirectory(openssl) add_subdirectory(packedtypes) add_subdirectory(packers) add_subdirectory(pop_count) -add_subdirectory(porto) add_subdirectory(presort) add_subdirectory(protobuf) add_subdirectory(random_provider) diff --git a/library/cpp/CMakeLists.linux-aarch64.txt b/library/cpp/CMakeLists.linux-aarch64.txt index b42033fd82..cf47314f07 100644 --- a/library/cpp/CMakeLists.linux-aarch64.txt +++ b/library/cpp/CMakeLists.linux-aarch64.txt @@ -62,7 +62,6 @@ add_subdirectory(openssl) add_subdirectory(packedtypes) add_subdirectory(packers) add_subdirectory(pop_count) -add_subdirectory(porto) add_subdirectory(presort) add_subdirectory(protobuf) add_subdirectory(random_provider) diff --git a/library/cpp/CMakeLists.linux-x86_64.txt b/library/cpp/CMakeLists.linux-x86_64.txt index 22b108ee05..0f393b2039 100644 --- a/library/cpp/CMakeLists.linux-x86_64.txt +++ b/library/cpp/CMakeLists.linux-x86_64.txt @@ -63,7 +63,6 @@ add_subdirectory(openssl) add_subdirectory(packedtypes) add_subdirectory(packers) add_subdirectory(pop_count) -add_subdirectory(porto) add_subdirectory(presort) add_subdirectory(protobuf) add_subdirectory(random_provider) diff --git a/library/cpp/CMakeLists.windows-x86_64.txt b/library/cpp/CMakeLists.windows-x86_64.txt index 8925d1f2bf..772027a342 100644 --- a/library/cpp/CMakeLists.windows-x86_64.txt +++ b/library/cpp/CMakeLists.windows-x86_64.txt @@ -62,7 +62,6 @@ add_subdirectory(openssl) add_subdirectory(packedtypes) add_subdirectory(packers) add_subdirectory(pop_count) -add_subdirectory(porto) add_subdirectory(presort) add_subdirectory(protobuf) add_subdirectory(random_provider) diff --git a/library/cpp/getopt/small/completer_command.cpp b/library/cpp/getopt/small/completer_command.cpp index 5e593eec7e..132fdfc4b0 100644 --- a/library/cpp/getopt/small/completer_command.cpp +++ b/library/cpp/getopt/small/completer_command.cpp @@ -64,7 +64,7 @@ namespace NLastGetopt { "completions to `~/.zfunc`:" "\n" "\n" - " $ mkdir -p ~/.zfunc" + " $ mkdir -m755 -p ~/.zfunc" "\n" " $ {command} {completion} zsh > ~/.zfunc/_{command}" "\n" diff --git a/library/cpp/porto/CMakeLists.darwin-x86_64.txt b/library/cpp/porto/CMakeLists.darwin-x86_64.txt deleted file mode 100644 index 499930c4b0..0000000000 --- a/library/cpp/porto/CMakeLists.darwin-x86_64.txt +++ /dev/null @@ -1,9 +0,0 @@ - -# This file was generated by the build system used internally in the Yandex monorepo. -# Only simple modifications are allowed (adding source-files to targets, adding simple properties -# like target_include_directories). These modifications will be ported to original -# ya.make files by maintainers. Any complex modifications which can't be ported back to the -# original buildsystem will not be accepted. - - -add_subdirectory(proto) diff --git a/library/cpp/porto/CMakeLists.linux-aarch64.txt b/library/cpp/porto/CMakeLists.linux-aarch64.txt deleted file mode 100644 index f61df9eb93..0000000000 --- a/library/cpp/porto/CMakeLists.linux-aarch64.txt +++ /dev/null @@ -1,22 +0,0 @@ - -# This file was generated by the build system used internally in the Yandex monorepo. -# Only simple modifications are allowed (adding source-files to targets, adding simple properties -# like target_include_directories). These modifications will be ported to original -# ya.make files by maintainers. Any complex modifications which can't be ported back to the -# original buildsystem will not be accepted. - - -add_subdirectory(proto) - -add_library(library-cpp-porto) -target_link_libraries(library-cpp-porto PUBLIC - contrib-libs-linux-headers - contrib-libs-cxxsupp - yutil - cpp-porto-proto - contrib-libs-protobuf -) -target_sources(library-cpp-porto PRIVATE - ${CMAKE_SOURCE_DIR}/library/cpp/porto/libporto.cpp - ${CMAKE_SOURCE_DIR}/library/cpp/porto/metrics.cpp -) diff --git a/library/cpp/porto/CMakeLists.linux-x86_64.txt b/library/cpp/porto/CMakeLists.linux-x86_64.txt deleted file mode 100644 index f61df9eb93..0000000000 --- a/library/cpp/porto/CMakeLists.linux-x86_64.txt +++ /dev/null @@ -1,22 +0,0 @@ - -# This file was generated by the build system used internally in the Yandex monorepo. -# Only simple modifications are allowed (adding source-files to targets, adding simple properties -# like target_include_directories). These modifications will be ported to original -# ya.make files by maintainers. Any complex modifications which can't be ported back to the -# original buildsystem will not be accepted. - - -add_subdirectory(proto) - -add_library(library-cpp-porto) -target_link_libraries(library-cpp-porto PUBLIC - contrib-libs-linux-headers - contrib-libs-cxxsupp - yutil - cpp-porto-proto - contrib-libs-protobuf -) -target_sources(library-cpp-porto PRIVATE - ${CMAKE_SOURCE_DIR}/library/cpp/porto/libporto.cpp - ${CMAKE_SOURCE_DIR}/library/cpp/porto/metrics.cpp -) diff --git a/library/cpp/porto/CMakeLists.windows-x86_64.txt b/library/cpp/porto/CMakeLists.windows-x86_64.txt deleted file mode 100644 index 499930c4b0..0000000000 --- a/library/cpp/porto/CMakeLists.windows-x86_64.txt +++ /dev/null @@ -1,9 +0,0 @@ - -# This file was generated by the build system used internally in the Yandex monorepo. -# Only simple modifications are allowed (adding source-files to targets, adding simple properties -# like target_include_directories). These modifications will be ported to original -# ya.make files by maintainers. Any complex modifications which can't be ported back to the -# original buildsystem will not be accepted. - - -add_subdirectory(proto) diff --git a/library/cpp/porto/libporto.cpp b/library/cpp/porto/libporto.cpp deleted file mode 100644 index 8fd8924300..0000000000 --- a/library/cpp/porto/libporto.cpp +++ /dev/null @@ -1,1547 +0,0 @@ -#include "libporto.hpp" -#include "metrics.hpp" - -#include <google/protobuf/text_format.h> -#include <google/protobuf/io/zero_copy_stream_impl.h> -#include <google/protobuf/io/coded_stream.h> - -extern "C" { -#include <errno.h> -#include <time.h> -#include <unistd.h> -#include <sys/socket.h> -#include <sys/un.h> - -#ifndef __linux__ -#include <fcntl.h> -#else -#include <sys/epoll.h> -#endif -} - -namespace Porto { - -TPortoApi::~TPortoApi() { - Disconnect(); -} - -EError TPortoApi::SetError(const TString &prefix, int _errno) { - LastErrorMsg = prefix + ": " + strerror(_errno); - - switch (_errno) { - case ENOENT: - LastError = EError::SocketUnavailable; - break; - case EAGAIN: - LastErrorMsg = prefix + ": Timeout exceeded. Timeout value: " + std::to_string(Timeout); - LastError = EError::SocketTimeout; - break; - case EIO: - case EPIPE: - LastError = EError::SocketError; - break; - default: - LastError = EError::Unknown; - break; - } - - Disconnect(); - return LastError; -} - -TString TPortoApi::GetLastError() const { - return EError_Name(LastError) + ":(" + LastErrorMsg + ")"; -} - -EError TPortoApi::Connect(const char *socket_path) { - struct sockaddr_un peer_addr; - socklen_t peer_addr_size; - - Disconnect(); - -#ifdef __linux__ - Fd = socket(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC, 0); - if (Fd < 0) - return SetError("socket", errno); -#else - Fd = socket(AF_UNIX, SOCK_STREAM, 0); - if (Fd < 0) - return SetError("socket", errno); - if (fcntl(Fd, F_SETFD, FD_CLOEXEC) < 0) - return SetError("fcntl FD_CLOEXEC", errno); -#endif - - if (Timeout > 0 && SetSocketTimeout(3, Timeout)) - return LastError; - - memset(&peer_addr, 0, sizeof(struct sockaddr_un)); - peer_addr.sun_family = AF_UNIX; - strncpy(peer_addr.sun_path, socket_path, strlen(socket_path)); - - peer_addr_size = sizeof(struct sockaddr_un); - if (connect(Fd, (struct sockaddr *) &peer_addr, peer_addr_size) < 0) - return SetError("connect", errno); - - /* Restore async wait state */ - if (!AsyncWaitNames.empty()) { - for (auto &name: AsyncWaitNames) - Req.mutable_asyncwait()->add_name(name); - for (auto &label: AsyncWaitLabels) - Req.mutable_asyncwait()->add_label(label); - if (AsyncWaitTimeout >= 0) - Req.mutable_asyncwait()->set_timeout_ms(AsyncWaitTimeout * 1000); - return Call(); - } - - return EError::Success; -} - -void TPortoApi::Disconnect() { - if (Fd >= 0) - close(Fd); - Fd = -1; -} - -EError TPortoApi::SetSocketTimeout(int direction, int timeout) { - struct timeval tv; - - if (Fd < 0) - return EError::Success; - - tv.tv_sec = timeout > 0 ? timeout : 0; - tv.tv_usec = 0; - - if ((direction & 1) && setsockopt(Fd, SOL_SOCKET, SO_SNDTIMEO, &tv, sizeof tv)) - return SetError("setsockopt SO_SNDTIMEO", errno); - - if ((direction & 2) && setsockopt(Fd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof tv)) - return SetError("setsockopt SO_RCVTIMEO", errno); - - return EError::Success; -} - -EError TPortoApi::SetTimeout(int timeout) { - Timeout = timeout ? timeout : DEFAULT_TIMEOUT; - return SetSocketTimeout(3, Timeout); -} - -EError TPortoApi::SetDiskTimeout(int timeout) { - DiskTimeout = timeout ? timeout : DEFAULT_DISK_TIMEOUT; - return EError::Success; -} - -EError TPortoApi::Send(const TPortoRequest &req) { - google::protobuf::io::FileOutputStream raw(Fd); - - if (!req.IsInitialized()) { - LastError = EError::InvalidMethod; - LastErrorMsg = "Request is not initialized"; - return EError::InvalidMethod; - } - - { - google::protobuf::io::CodedOutputStream output(&raw); - - output.WriteVarint32(req.ByteSize()); - req.SerializeWithCachedSizes(&output); - } - - raw.Flush(); - - int err = raw.GetErrno(); - if (err) - return SetError("send", err); - - return EError::Success; -} - -EError TPortoApi::Recv(TPortoResponse &rsp) { - google::protobuf::io::FileInputStream raw(Fd); - google::protobuf::io::CodedInputStream input(&raw); - - while (true) { - uint32_t size; - - if (!input.ReadVarint32(&size)) - return SetError("recv", raw.GetErrno() ?: EIO); - - auto prev_limit = input.PushLimit(size); - - rsp.Clear(); - - if (!rsp.ParseFromCodedStream(&input)) - return SetError("recv", raw.GetErrno() ?: EIO); - - input.PopLimit(prev_limit); - - if (rsp.has_asyncwait()) { - if (AsyncWaitCallback) - AsyncWaitCallback(rsp.asyncwait()); - - if (AsyncWaitOneShot) - return EError::Success; - - continue; - } - - return EError::Success; - } -} - -EError TPortoApi::Call(const TPortoRequest &req, - TPortoResponse &rsp, - int extra_timeout) { - bool reconnect = AutoReconnect; - EError err = EError::Success; - - if (Fd < 0) { - if (!reconnect) - return SetError("Not connected", EIO); - err = Connect(); - reconnect = false; - } - - if (!err) { - err = Send(req); - if (err == EError::SocketError && reconnect) { - err = Connect(); - if (!err) - err = Send(req); - } - } - - if (!err && extra_timeout && Timeout > 0) - err = SetSocketTimeout(2, extra_timeout > 0 ? (extra_timeout + Timeout) : -1); - - if (!err) - err = Recv(rsp); - - if (extra_timeout && Timeout > 0) { - EError err = SetSocketTimeout(2, Timeout); - (void)err; - } - - if (!err) { - err = LastError = rsp.error(); - LastErrorMsg = rsp.errormsg(); - } - - return err; -} - -EError TPortoApi::Call(int extra_timeout) { - return Call(Req, Rsp, extra_timeout); -} - -EError TPortoApi::Call(const TString &req, - TString &rsp, - int extra_timeout) { - Req.Clear(); - if (!google::protobuf::TextFormat::ParseFromString(req, &Req)) { - LastError = EError::InvalidMethod; - LastErrorMsg = "Cannot parse request"; - rsp = ""; - return EError::InvalidMethod; - } - - EError err = Call(Req, Rsp, extra_timeout); - - rsp = Rsp.DebugString(); - - return err; -} - -EError TPortoApi::GetVersion(TString &tag, TString &revision) { - Req.Clear(); - Req.mutable_version(); - - if (!Call()) { - tag = Rsp.version().tag(); - revision = Rsp.version().revision(); - } - - return LastError; -} - -const TGetSystemResponse *TPortoApi::GetSystem() { - Req.Clear(); - Req.mutable_getsystem(); - if (!Call()) - return &Rsp.getsystem(); - return nullptr; -} - -EError TPortoApi::SetSystem(const TString &key, const TString &val) { - TString rsp; - return Call("SetSystem {" + key + ":" + val + "}", rsp); -} - -/* Container */ - -EError TPortoApi::Create(const TString &name) { - Req.Clear(); - auto req = Req.mutable_create(); - req->set_name(name); - return Call(); -} - -EError TPortoApi::CreateWeakContainer(const TString &name) { - Req.Clear(); - auto req = Req.mutable_createweak(); - req->set_name(name); - return Call(); -} - -EError TPortoApi::Destroy(const TString &name) { - Req.Clear(); - auto req = Req.mutable_destroy(); - req->set_name(name); - return Call(); -} - -const TListResponse *TPortoApi::List(const TString &mask) { - Req.Clear(); - auto req = Req.mutable_list(); - - if(!mask.empty()) - req->set_mask(mask); - - if (!Call()) - return &Rsp.list(); - - return nullptr; -} - -EError TPortoApi::List(TVector<TString> &list, const TString &mask) { - Req.Clear(); - auto req = Req.mutable_list(); - if(!mask.empty()) - req->set_mask(mask); - if (!Call()) - list = TVector<TString>(std::begin(Rsp.list().name()), - std::end(Rsp.list().name())); - return LastError; -} - -const TListPropertiesResponse *TPortoApi::ListProperties() { - Req.Clear(); - Req.mutable_listproperties(); - - if (Call()) - return nullptr; - - bool has_data = false; - for (const auto &prop: Rsp.listproperties().list()) { - if (prop.read_only()) { - has_data = true; - break; - } - } - - if (!has_data) { - TPortoRequest req; - TPortoResponse rsp; - - req.mutable_listdataproperties(); - if (!Call(req, rsp)) { - for (const auto &data: rsp.listdataproperties().list()) { - auto d = Rsp.mutable_listproperties()->add_list(); - d->set_name(data.name()); - d->set_desc(data.desc()); - d->set_read_only(true); - } - } - } - - return &Rsp.listproperties(); -} - -EError TPortoApi::ListProperties(TVector<TString> &properties) { - properties.clear(); - auto rsp = ListProperties(); - if (rsp) { - for (auto &prop: rsp->list()) - properties.push_back(prop.name()); - } - return LastError; -} - -const TGetResponse *TPortoApi::Get(const TVector<TString> &names, - const TVector<TString> &vars, - int flags) { - Req.Clear(); - auto get = Req.mutable_get(); - - for (const auto &n : names) - get->add_name(n); - - for (const auto &v : vars) - get->add_variable(v); - - if (flags & GET_NONBLOCK) - get->set_nonblock(true); - if (flags & GET_SYNC) - get->set_sync(true); - if (flags & GET_REAL) - get->set_real(true); - - if (!Call()) - return &Rsp.get(); - - return nullptr; -} - -EError TPortoApi::GetContainerSpec(const TString &name, TContainer &container) { - Req.Clear(); - TListContainersRequest req; - auto filter = req.add_filters(); - filter->set_name(name); - - TVector<TContainer> containers; - - auto ret = ListContainersBy(req, containers); - if (containers.empty()) - return EError::ContainerDoesNotExist; - - if (!ret) - container = containers[0]; - - return ret; -} - -EError TPortoApi::ListContainersBy(const TListContainersRequest &listContainersRequest, TVector<TContainer> &containers) { - Req.Clear(); - auto req = Req.mutable_listcontainersby(); - *req = listContainersRequest; - - auto ret = Call(); - if (ret) - return ret; - - for (auto &ct : Rsp.listcontainersby().containers()) - containers.push_back(ct); - - return EError::Success; -} - -EError TPortoApi::CreateFromSpec(const TContainerSpec &container, TVector<TVolumeSpec> volumes, bool start) { - Req.Clear(); - auto req = Req.mutable_createfromspec(); - - auto ct = req->mutable_container(); - *ct = container; - - for (auto &volume : volumes) { - auto v = req->add_volumes(); - *v = volume; - } - - req->set_start(start); - - return Call(); -} - -EError TPortoApi::UpdateFromSpec(const TContainerSpec &container) { - Req.Clear(); - auto req = Req.mutable_updatefromspec(); - - auto ct = req->mutable_container(); - *ct = container; - - return Call(); -} - -EError TPortoApi::GetProperty(const TString &name, - const TString &property, - TString &value, - int flags) { - Req.Clear(); - auto req = Req.mutable_getproperty(); - - req->set_name(name); - req->set_property(property); - if (flags & GET_SYNC) - req->set_sync(true); - if (flags & GET_REAL) - req->set_real(true); - - if (!Call()) - value = Rsp.getproperty().value(); - - return LastError; -} - -EError TPortoApi::SetProperty(const TString &name, - const TString &property, - const TString &value) { - Req.Clear(); - auto req = Req.mutable_setproperty(); - - req->set_name(name); - req->set_property(property); - req->set_value(value); - - return Call(); -} - -EError TPortoApi::GetInt(const TString &name, - const TString &property, - const TString &index, - uint64_t &value) { - TString key = property, str; - if (index.size()) - key = property + "[" + index + "]"; - if (!GetProperty(name, key, str)) { - const char *ptr = str.c_str(); - char *end; - errno = 0; - value = strtoull(ptr, &end, 10); - if (errno || end == ptr || *end) { - LastError = EError::InvalidValue; - LastErrorMsg = " value: " + str; - } - } - return LastError; -} - -EError TPortoApi::SetInt(const TString &name, - const TString &property, - const TString &index, - uint64_t value) { - TString key = property; - if (index.size()) - key = property + "[" + index + "]"; - return SetProperty(name, key, ToString(value)); -} - -EError TPortoApi::GetProcMetric(const TVector<TString> &names, - const TString &metric, - TMap<TString, uint64_t> &values) { - auto it = ProcMetrics.find(metric); - - if (it == ProcMetrics.end()) { - LastError = EError::InvalidValue; - LastErrorMsg = " Unknown metric: " + metric; - return LastError; - } - - LastError = it->second->GetValues(names, values, *this); - - if (LastError) - LastErrorMsg = "Unknown error on Get() method"; - - return LastError; -} - -EError TPortoApi::SetLabel(const TString &name, - const TString &label, - const TString &value, - const TString &prev_value) { - Req.Clear(); - auto req = Req.mutable_setlabel(); - - req->set_name(name); - req->set_label(label); - req->set_value(value); - if (prev_value != " ") - req->set_prev_value(prev_value); - - return Call(); -} - -EError TPortoApi::IncLabel(const TString &name, - const TString &label, - int64_t add, - int64_t &result) { - Req.Clear(); - auto req = Req.mutable_inclabel(); - - req->set_name(name); - req->set_label(label); - req->set_add(add); - - EError err = Call(); - - if (Rsp.has_inclabel()) - result = Rsp.inclabel().result(); - - return err; -} - -EError TPortoApi::Start(const TString &name) { - Req.Clear(); - auto req = Req.mutable_start(); - - req->set_name(name); - - return Call(); -} - -EError TPortoApi::Stop(const TString &name, int stop_timeout) { - Req.Clear(); - auto req = Req.mutable_stop(); - - req->set_name(name); - if (stop_timeout >= 0) - req->set_timeout_ms(stop_timeout * 1000); - - return Call(stop_timeout > 0 ? stop_timeout : 0); -} - -EError TPortoApi::Kill(const TString &name, int sig) { - Req.Clear(); - auto req = Req.mutable_kill(); - - req->set_name(name); - req->set_sig(sig); - - return Call(); -} - -EError TPortoApi::Pause(const TString &name) { - Req.Clear(); - auto req = Req.mutable_pause(); - - req->set_name(name); - - return Call(); -} - -EError TPortoApi::Resume(const TString &name) { - Req.Clear(); - auto req = Req.mutable_resume(); - - req->set_name(name); - - return Call(); -} - -EError TPortoApi::Respawn(const TString &name) { - Req.Clear(); - auto req = Req.mutable_respawn(); - - req->set_name(name); - - return Call(); -} - -EError TPortoApi::CallWait(TString &result_state, int wait_timeout) { - time_t deadline = 0; - time_t last_retry = 0; - - if (wait_timeout >= 0) { - deadline = time(nullptr) + wait_timeout; - Req.mutable_wait()->set_timeout_ms(wait_timeout * 1000); - } - -retry: - if (!Call(wait_timeout)) { - if (Rsp.wait().has_state()) - result_state = Rsp.wait().state(); - else if (Rsp.wait().name() == "") - result_state = "timeout"; - else - result_state = "dead"; - } else if (LastError == EError::SocketError && AutoReconnect) { - time_t now = time(nullptr); - - if (wait_timeout < 0 || now < deadline) { - if (wait_timeout >= 0) { - wait_timeout = deadline - now; - Req.mutable_wait()->set_timeout_ms(wait_timeout * 1000); - } - if (last_retry == now) - sleep(1); - last_retry = now; - goto retry; - } - - result_state = "timeout"; - } else - result_state = "unknown"; - - return LastError; -} - -EError TPortoApi::WaitContainer(const TString &name, - TString &result_state, - int wait_timeout) { - Req.Clear(); - auto req = Req.mutable_wait(); - - req->add_name(name); - - return CallWait(result_state, wait_timeout); -} - -EError TPortoApi::WaitContainers(const TVector<TString> &names, - TString &result_name, - TString &result_state, - int wait_timeout) { - Req.Clear(); - auto req = Req.mutable_wait(); - - for (auto &c : names) - req->add_name(c); - - EError err = CallWait(result_state, wait_timeout); - - result_name = Rsp.wait().name(); - - return err; -} - -const TWaitResponse *TPortoApi::Wait(const TVector<TString> &names, - const TVector<TString> &labels, - int wait_timeout) { - Req.Clear(); - auto req = Req.mutable_wait(); - TString result_state; - - for (auto &c : names) - req->add_name(c); - for (auto &label: labels) - req->add_label(label); - - EError err = CallWait(result_state, wait_timeout); - (void)err; - - if (Rsp.has_wait()) - return &Rsp.wait(); - - return nullptr; -} - -EError TPortoApi::AsyncWait(const TVector<TString> &names, - const TVector<TString> &labels, - TWaitCallback callback, - int wait_timeout, - const TString &targetState) { - Req.Clear(); - auto req = Req.mutable_asyncwait(); - - AsyncWaitNames.clear(); - AsyncWaitLabels.clear(); - AsyncWaitTimeout = wait_timeout; - AsyncWaitCallback = callback; - - for (auto &name: names) - req->add_name(name); - for (auto &label: labels) - req->add_label(label); - if (wait_timeout >= 0) - req->set_timeout_ms(wait_timeout * 1000); - if (!targetState.empty()) { - req->set_target_state(targetState); - AsyncWaitOneShot = true; - } else - AsyncWaitOneShot = false; - - if (Call()) { - AsyncWaitCallback = nullptr; - } else { - AsyncWaitNames = names; - AsyncWaitLabels = labels; - } - - return LastError; -} - -EError TPortoApi::StopAsyncWait(const TVector<TString> &names, - const TVector<TString> &labels, - const TString &targetState) { - Req.Clear(); - auto req = Req.mutable_stopasyncwait(); - - AsyncWaitNames.clear(); - AsyncWaitLabels.clear(); - - for (auto &name: names) - req->add_name(name); - for (auto &label: labels) - req->add_label(label); - if (!targetState.empty()) { - req->set_target_state(targetState); - } - - return Call(); -} - -EError TPortoApi::ConvertPath(const TString &path, - const TString &src, - const TString &dest, - TString &res) { - Req.Clear(); - auto req = Req.mutable_convertpath(); - - req->set_path(path); - req->set_source(src); - req->set_destination(dest); - - if (!Call()) - res = Rsp.convertpath().path(); - - return LastError; -} - -EError TPortoApi::AttachProcess(const TString &name, int pid, - const TString &comm) { - Req.Clear(); - auto req = Req.mutable_attachprocess(); - - req->set_name(name); - req->set_pid(pid); - req->set_comm(comm); - - return Call(); -} - -EError TPortoApi::AttachThread(const TString &name, int pid, - const TString &comm) { - Req.Clear(); - auto req = Req.mutable_attachthread(); - - req->set_name(name); - req->set_pid(pid); - req->set_comm(comm); - - return Call(); -} - -EError TPortoApi::LocateProcess(int pid, const TString &comm, - TString &name) { - Req.Clear(); - auto req = Req.mutable_locateprocess(); - - req->set_pid(pid); - req->set_comm(comm); - - if (!Call()) - name = Rsp.locateprocess().name(); - - return LastError; -} - -/* Volume */ - -const TListVolumePropertiesResponse *TPortoApi::ListVolumeProperties() { - Req.Clear(); - Req.mutable_listvolumeproperties(); - - if (!Call()) - return &Rsp.listvolumeproperties(); - - return nullptr; -} - -EError TPortoApi::ListVolumeProperties(TVector<TString> &properties) { - properties.clear(); - auto rsp = ListVolumeProperties(); - if (rsp) { - for (auto &prop: rsp->list()) - properties.push_back(prop.name()); - } - return LastError; -} - -EError TPortoApi::CreateVolume(TString &path, - const TMap<TString, TString> &config) { - Req.Clear(); - auto req = Req.mutable_createvolume(); - - req->set_path(path); - - *(req->mutable_properties()) = - google::protobuf::Map<TString, TString>(config.begin(), config.end()); - - if (!Call(DiskTimeout) && path.empty()) - path = Rsp.createvolume().path(); - - return LastError; -} - -EError TPortoApi::TuneVolume(const TString &path, - const TMap<TString, TString> &config) { - Req.Clear(); - auto req = Req.mutable_tunevolume(); - - req->set_path(path); - - *(req->mutable_properties()) = - google::protobuf::Map<TString, TString>(config.begin(), config.end()); - - return Call(DiskTimeout); -} - -EError TPortoApi::LinkVolume(const TString &path, - const TString &container, - const TString &target, - bool read_only, - bool required) { - Req.Clear(); - auto req = (target.empty() && !required) ? Req.mutable_linkvolume() : - Req.mutable_linkvolumetarget(); - - req->set_path(path); - if (!container.empty()) - req->set_container(container); - if (target != "") - req->set_target(target); - if (read_only) - req->set_read_only(read_only); - if (required) - req->set_required(required); - - return Call(); -} - -EError TPortoApi::UnlinkVolume(const TString &path, - const TString &container, - const TString &target, - bool strict) { - Req.Clear(); - auto req = (target == "***") ? Req.mutable_unlinkvolume() : - Req.mutable_unlinkvolumetarget(); - - req->set_path(path); - if (!container.empty()) - req->set_container(container); - if (target != "***") - req->set_target(target); - if (strict) - req->set_strict(strict); - - return Call(DiskTimeout); -} - -const TListVolumesResponse * -TPortoApi::ListVolumes(const TString &path, - const TString &container) { - Req.Clear(); - auto req = Req.mutable_listvolumes(); - - if (!path.empty()) - req->set_path(path); - - if (!container.empty()) - req->set_container(container); - - if (Call()) - return nullptr; - - auto list = Rsp.mutable_listvolumes(); - - /* compat */ - for (auto v: *list->mutable_volumes()) { - if (v.links().size()) - break; - for (auto &ct: v.containers()) - v.add_links()->set_container(ct); - } - - return list; -} - -EError TPortoApi::ListVolumes(TVector<TString> &paths) { - Req.Clear(); - auto rsp = ListVolumes(); - paths.clear(); - if (rsp) { - for (auto &v : rsp->volumes()) - paths.push_back(v.path()); - } - return LastError; -} - -const TVolumeDescription *TPortoApi::GetVolumeDesc(const TString &path) { - Req.Clear(); - auto rsp = ListVolumes(path); - - if (rsp && rsp->volumes().size()) - return &rsp->volumes(0); - - return nullptr; -} - -const TVolumeSpec *TPortoApi::GetVolume(const TString &path) { - Req.Clear(); - auto req = Req.mutable_getvolume(); - - req->add_path(path); - - if (!Call() && Rsp.getvolume().volume().size()) - return &Rsp.getvolume().volume(0); - - return nullptr; -} - -const TGetVolumeResponse *TPortoApi::GetVolumes(uint64_t changed_since) { - Req.Clear(); - auto req = Req.mutable_getvolume(); - - if (changed_since) - req->set_changed_since(changed_since); - - if (!Call() && Rsp.has_getvolume()) - return &Rsp.getvolume(); - - return nullptr; -} - - -EError TPortoApi::ListVolumesBy(const TGetVolumeRequest &getVolumeRequest, TVector<TVolumeSpec> &volumes) { - Req.Clear(); - auto req = Req.mutable_getvolume(); - *req = getVolumeRequest; - - auto ret = Call(); - if (ret) - return ret; - - for (auto volume : Rsp.getvolume().volume()) - volumes.push_back(volume); - return EError::Success; -} - -EError TPortoApi::CreateVolumeFromSpec(const TVolumeSpec &volume, TVolumeSpec &resultSpec) { - Req.Clear(); - auto req = Req.mutable_newvolume(); - auto vol = req->mutable_volume(); - *vol = volume; - - auto ret = Call(); - if (ret) - return ret; - - resultSpec = Rsp.newvolume().volume(); - - return ret; -} - -/* Layer */ - -EError TPortoApi::ImportLayer(const TString &layer, - const TString &tarball, - bool merge, - const TString &place, - const TString &private_value, - bool verboseError) { - Req.Clear(); - auto req = Req.mutable_importlayer(); - - req->set_layer(layer); - req->set_tarball(tarball); - req->set_merge(merge); - req->set_verbose_error(verboseError); - if (place.size()) - req->set_place(place); - if (private_value.size()) - req->set_private_value(private_value); - - return Call(DiskTimeout); -} - -EError TPortoApi::ExportLayer(const TString &volume, - const TString &tarball, - const TString &compress) { - Req.Clear(); - auto req = Req.mutable_exportlayer(); - - req->set_volume(volume); - req->set_tarball(tarball); - if (compress.size()) - req->set_compress(compress); - - return Call(DiskTimeout); -} - -EError TPortoApi::ReExportLayer(const TString &layer, - const TString &tarball, - const TString &compress) { - Req.Clear(); - auto req = Req.mutable_exportlayer(); - - req->set_volume(""); - req->set_layer(layer); - req->set_tarball(tarball); - if (compress.size()) - req->set_compress(compress); - - return Call(DiskTimeout); -} - -EError TPortoApi::RemoveLayer(const TString &layer, - const TString &place, - bool async) { - Req.Clear(); - auto req = Req.mutable_removelayer(); - - req->set_layer(layer); - req->set_async(async); - if (place.size()) - req->set_place(place); - - return Call(DiskTimeout); -} - -const TListLayersResponse *TPortoApi::ListLayers(const TString &place, - const TString &mask) { - Req.Clear(); - auto req = Req.mutable_listlayers(); - - if (place.size()) - req->set_place(place); - if (mask.size()) - req->set_mask(mask); - - if (Call()) - return nullptr; - - auto list = Rsp.mutable_listlayers(); - - /* compat conversion */ - if (!list->layers().size() && list->layer().size()) { - for (auto &name: list->layer()) { - auto l = list->add_layers(); - l->set_name(name); - l->set_owner_user(""); - l->set_owner_group(""); - l->set_last_usage(0); - l->set_private_value(""); - } - } - - return list; -} - -EError TPortoApi::ListLayers(TVector<TString> &layers, - const TString &place, - const TString &mask) { - Req.Clear(); - auto req = Req.mutable_listlayers(); - - if (place.size()) - req->set_place(place); - if (mask.size()) - req->set_mask(mask); - - if (!Call()) - layers = TVector<TString>(std::begin(Rsp.listlayers().layer()), - std::end(Rsp.listlayers().layer())); - - return LastError; -} - -EError TPortoApi::GetLayerPrivate(TString &private_value, - const TString &layer, - const TString &place) { - Req.Clear(); - auto req = Req.mutable_getlayerprivate(); - - req->set_layer(layer); - if (place.size()) - req->set_place(place); - - if (!Call()) - private_value = Rsp.getlayerprivate().private_value(); - - return LastError; -} - -EError TPortoApi::SetLayerPrivate(const TString &private_value, - const TString &layer, - const TString &place) { - Req.Clear(); - auto req = Req.mutable_setlayerprivate(); - - req->set_layer(layer); - req->set_private_value(private_value); - if (place.size()) - req->set_place(place); - - return Call(); -} - -/* Docker images */ - -DockerImage::DockerImage(const TDockerImage &i) { - Id = i.id(); - for (const auto &tag: i.tags()) - Tags.emplace_back(tag); - for (const auto &digest: i.digests()) - Digests.emplace_back(digest); - for (const auto &layer: i.layers()) - Layers.emplace_back(layer); - if (i.has_size()) - Size = i.size(); - if (i.has_config()) { - auto &cfg = i.config(); - for (const auto &cmd: cfg.cmd()) - Config.Cmd.emplace_back(cmd); - for (const auto &env: cfg.env()) - Config.Env.emplace_back(env); - } -} - -EError TPortoApi::DockerImageStatus(DockerImage &image, - const TString &name, - const TString &place) { - auto req = Req.mutable_dockerimagestatus(); - req->set_name(name); - if (!place.empty()) - req->set_place(place); - EError ret = Call(); - if (!ret && Rsp.dockerimagestatus().has_image()) - image = DockerImage(Rsp.dockerimagestatus().image()); - return ret; -} - -EError TPortoApi::ListDockerImages(std::vector<DockerImage> &images, - const TString &place, - const TString &mask) { - auto req = Req.mutable_listdockerimages(); - if (place.size()) - req->set_place(place); - if (mask.size()) - req->set_mask(mask); - EError ret = Call(); - if (!ret) { - for (const auto &i: Rsp.listdockerimages().images()) - images.emplace_back(i); - } - return ret; -} - -EError TPortoApi::PullDockerImage(DockerImage &image, - const TString &name, - const TString &place, - const TString &auth_token, - const TString &auth_path, - const TString &auth_service) { - auto req = Req.mutable_pulldockerimage(); - req->set_name(name); - if (place.size()) - req->set_place(place); - if (auth_token.size()) - req->set_auth_token(auth_token); - if (auth_path.size()) - req->set_auth_path(auth_path); - if (auth_service.size()) - req->set_auth_service(auth_service); - EError ret = Call(); - if (!ret && Rsp.pulldockerimage().has_image()) - image = DockerImage(Rsp.pulldockerimage().image()); - return ret; -} - -EError TPortoApi::RemoveDockerImage(const TString &name, - const TString &place) { - auto req = Req.mutable_removedockerimage(); - req->set_name(name); - if (place.size()) - req->set_place(place); - return Call(); -} - -/* Storage */ - -const TListStoragesResponse *TPortoApi::ListStorages(const TString &place, - const TString &mask) { - Req.Clear(); - auto req = Req.mutable_liststorages(); - - if (place.size()) - req->set_place(place); - if (mask.size()) - req->set_mask(mask); - - if (Call()) - return nullptr; - - return &Rsp.liststorages(); -} - -EError TPortoApi::ListStorages(TVector<TString> &storages, - const TString &place, - const TString &mask) { - Req.Clear(); - auto req = Req.mutable_liststorages(); - - if (place.size()) - req->set_place(place); - if (mask.size()) - req->set_mask(mask); - - if (!Call()) { - storages.clear(); - for (auto &storage: Rsp.liststorages().storages()) - storages.push_back(storage.name()); - } - - return LastError; -} - -EError TPortoApi::RemoveStorage(const TString &storage, - const TString &place) { - Req.Clear(); - auto req = Req.mutable_removestorage(); - - req->set_name(storage); - if (place.size()) - req->set_place(place); - - return Call(DiskTimeout); -} - -EError TPortoApi::ImportStorage(const TString &storage, - const TString &archive, - const TString &place, - const TString &compression, - const TString &private_value) { - Req.Clear(); - auto req = Req.mutable_importstorage(); - - req->set_name(storage); - req->set_tarball(archive); - if (place.size()) - req->set_place(place); - if (compression.size()) - req->set_compress(compression); - if (private_value.size()) - req->set_private_value(private_value); - - return Call(DiskTimeout); -} - -EError TPortoApi::ExportStorage(const TString &storage, - const TString &archive, - const TString &place, - const TString &compression) { - Req.Clear(); - auto req = Req.mutable_exportstorage(); - - req->set_name(storage); - req->set_tarball(archive); - if (place.size()) - req->set_place(place); - if (compression.size()) - req->set_compress(compression); - - return Call(DiskTimeout); -} - -#ifdef __linux__ -void TAsyncWaiter::MainCallback(const TWaitResponse &event) { - CallbacksCount++; - - auto it = AsyncCallbacks.find(event.name()); - if (it != AsyncCallbacks.end() && it->second.State == event.state()) { - it->second.Callback(event); - AsyncCallbacks.erase(it); - } -} - -int TAsyncWaiter::Repair() { - for (const auto &it : AsyncCallbacks) { - int ret = Api.AsyncWait({it.first}, {}, GetMainCallback(), -1, it.second.State); - if (ret) - return ret; - } - return 0; -} - -void TAsyncWaiter::WatchDog() { - int ret; - auto apiFd = Api.Fd; - - while (true) { - struct epoll_event events[2]; - int nfds = epoll_wait(EpollFd, events, 2, -1); - - if (nfds < 0) { - if (errno == EINTR) - continue; - - Fatal("Can not make epoll_wait", errno); - return; - } - - for (int n = 0; n < nfds; ++n) { - if (events[n].data.fd == apiFd) { - TPortoResponse rsp; - ret = Api.Recv(rsp); - // portod reloaded - async_wait must be repaired - if (ret == EError::SocketError) { - ret = Api.Connect(); - if (ret) { - Fatal("Can not connect to porto api", ret); - return; - } - - ret = Repair(); - if (ret) { - Fatal("Can not repair", ret); - return; - } - - apiFd = Api.Fd; - - struct epoll_event portoEv; - portoEv.events = EPOLLIN; - portoEv.data.fd = apiFd; - if (epoll_ctl(EpollFd, EPOLL_CTL_ADD, apiFd, &portoEv)) { - Fatal("Can not epoll_ctl", errno); - return; - } - } - } else if (events[n].data.fd == Sock) { - ERequestType requestType = static_cast<ERequestType>(RecvInt(Sock)); - - switch (requestType) { - case ERequestType::Add: - HandleAddRequest(); - break; - case ERequestType::Del: - HandleDelRequest(); - break; - case ERequestType::Stop: - return; - case ERequestType::None: - default: - Fatal("Unknown request", static_cast<int>(requestType)); - } - } - } - } -} - -void TAsyncWaiter::SendInt(int fd, int value) { - int ret = write(fd, &value, sizeof(value)); - if (ret != sizeof(value)) - Fatal("Can not send int", errno); -} - -int TAsyncWaiter::RecvInt(int fd) { - int value; - int ret = read(fd, &value, sizeof(value)); - if (ret != sizeof(value)) - Fatal("Can not recv int", errno); - - return value; -} - -void TAsyncWaiter::HandleAddRequest() { - int ret = 0; - - auto it = AsyncCallbacks.find(ReqCt); - if (it != AsyncCallbacks.end()) { - ret = Api.StopAsyncWait({ReqCt}, {}, it->second.State); - AsyncCallbacks.erase(it); - } - - AsyncCallbacks.insert(std::make_pair(ReqCt, TCallbackData({ReqCallback, ReqState}))); - - ret = Api.AsyncWait({ReqCt}, {}, GetMainCallback(), -1, ReqState); - SendInt(Sock, ret); -} - -void TAsyncWaiter::HandleDelRequest() { - int ret = 0; - - auto it = AsyncCallbacks.find(ReqCt); - if (it != AsyncCallbacks.end()) { - ret = Api.StopAsyncWait({ReqCt}, {}, it->second.State); - AsyncCallbacks.erase(it); - } - - SendInt(Sock, ret); -} - -TAsyncWaiter::TAsyncWaiter(std::function<void(const TString &error, int ret)> fatalCallback) - : CallbacksCount(0ul) - , FatalCallback(fatalCallback) -{ - int socketPair[2]; - int ret = socketpair(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC, 0, socketPair); - if (ret) { - Fatal("Can not make socketpair", ret); - return; - } - - MasterSock = socketPair[0]; - Sock = socketPair[1]; - - ret = Api.Connect(); - if (ret) { - Fatal("Can not connect to porto api", ret); - return; - } - - auto apiFd = Api.Fd; - - EpollFd = epoll_create1(EPOLL_CLOEXEC); - - if (EpollFd == -1) { - Fatal("Can not epoll_create", errno); - return; - } - - struct epoll_event pairEv; - pairEv.events = EPOLLIN; - pairEv.data.fd = Sock; - - struct epoll_event portoEv; - portoEv.events = EPOLLIN; - portoEv.data.fd = apiFd; - - if (epoll_ctl(EpollFd, EPOLL_CTL_ADD, Sock, &pairEv)) { - Fatal("Can not epoll_ctl", errno); - return; - } - - if (epoll_ctl(EpollFd, EPOLL_CTL_ADD, apiFd, &portoEv)) { - Fatal("Can not epoll_ctl", errno); - return; - } - - WatchDogThread = std::unique_ptr<std::thread>(new std::thread(&TAsyncWaiter::WatchDog, this)); -} - -TAsyncWaiter::~TAsyncWaiter() { - SendInt(MasterSock, static_cast<int>(ERequestType::Stop)); - WatchDogThread->join(); - - // pedantic check, that porto api is watching by epoll - if (epoll_ctl(EpollFd, EPOLL_CTL_DEL, Api.Fd, nullptr) || epoll_ctl(EpollFd, EPOLL_CTL_DEL, Sock, nullptr)) { - Fatal("Can not epoll_ctl_del", errno); - } - - close(EpollFd); - close(Sock); - close(MasterSock); -} - -int TAsyncWaiter::Add(const TString &ct, const TString &state, TWaitCallback callback) { - if (FatalError) - return -1; - - ReqCt = ct; - ReqState = state; - ReqCallback = callback; - - SendInt(MasterSock, static_cast<int>(ERequestType::Add)); - return RecvInt(MasterSock); -} - -int TAsyncWaiter::Remove(const TString &ct) { - if (FatalError) - return -1; - - ReqCt = ct; - - SendInt(MasterSock, static_cast<int>(ERequestType::Del)); - return RecvInt(MasterSock); -} -#endif - -} /* namespace Porto */ diff --git a/library/cpp/porto/libporto.hpp b/library/cpp/porto/libporto.hpp deleted file mode 100644 index e30f22a41e..0000000000 --- a/library/cpp/porto/libporto.hpp +++ /dev/null @@ -1,492 +0,0 @@ -#pragma once - -#include <atomic> -#include <thread> - -#include <util/string/cast.h> -#include <util/generic/hash.h> -#include <util/generic/map.h> -#include <util/generic/vector.h> - -#include <library/cpp/porto/proto/rpc.pb.h> - -namespace Porto { - -constexpr int INFINITE_TIMEOUT = -1; -constexpr int DEFAULT_TIMEOUT = 300; // 5min -constexpr int DEFAULT_DISK_TIMEOUT = 900; // 15min - -constexpr char SOCKET_PATH[] = "/run/portod.socket"; - -typedef std::function<void(const TWaitResponse &event)> TWaitCallback; - -enum { - GET_NONBLOCK = 1, // try lock container state - GET_SYNC = 2, // refresh cached values, cache ttl 5s - GET_REAL = 4, // no faked or inherited values -}; - -struct DockerImage { - std::string Id; - std::vector<std::string> Tags; - std::vector<std::string> Digests; - std::vector<std::string> Layers; - uint64_t Size; - struct Config { - std::vector<std::string> Cmd; - std::vector<std::string> Env; - } Config; - - DockerImage() = default; - DockerImage(const TDockerImage &i); - - DockerImage(const DockerImage &i) = default; - DockerImage(DockerImage &&i) = default; - - DockerImage& operator=(const DockerImage &i) = default; - DockerImage& operator=(DockerImage &&i) = default; -}; - -class TPortoApi { -#ifdef __linux__ - friend class TAsyncWaiter; -#endif -private: - int Fd = -1; - int Timeout = DEFAULT_TIMEOUT; - int DiskTimeout = DEFAULT_DISK_TIMEOUT; - bool AutoReconnect = true; - - EError LastError = EError::Success; - TString LastErrorMsg; - - /* - * These keep last request and response. Method might return - * pointers to Rsp innards -> pointers valid until next call. - */ - TPortoRequest Req; - TPortoResponse Rsp; - - std::vector<TString> AsyncWaitNames; - std::vector<TString> AsyncWaitLabels; - int AsyncWaitTimeout = INFINITE_TIMEOUT; - TWaitCallback AsyncWaitCallback; - bool AsyncWaitOneShot = false; - - EError SetError(const TString &prefix, int _errno) Y_WARN_UNUSED_RESULT; - - EError SetSocketTimeout(int direction, int timeout) Y_WARN_UNUSED_RESULT; - - EError Send(const TPortoRequest &req) Y_WARN_UNUSED_RESULT; - - EError Recv(TPortoResponse &rsp) Y_WARN_UNUSED_RESULT; - - EError Call(int extra_timeout = 0) Y_WARN_UNUSED_RESULT; - - EError CallWait(TString &result_state, int wait_timeout) Y_WARN_UNUSED_RESULT; - -public: - TPortoApi() { } - ~TPortoApi(); - - int GetFd() const { - return Fd; - } - - bool Connected() const { - return Fd >= 0; - } - - EError Connect(const char *socket_path = SOCKET_PATH) Y_WARN_UNUSED_RESULT; - void Disconnect(); - - /* Requires signal(SIGPIPE, SIG_IGN) */ - void SetAutoReconnect(bool auto_reconnect) { - AutoReconnect = auto_reconnect; - } - - /* Request and response timeout in seconds */ - int GetTimeout() const { - return Timeout; - } - EError SetTimeout(int timeout); - - /* Extra timeout for disk operations in seconds */ - int GetDiskTimeout() const { - return DiskTimeout; - } - EError SetDiskTimeout(int timeout); - - EError Error() const Y_WARN_UNUSED_RESULT { - return LastError; - } - - EError GetLastError(TString &msg) const Y_WARN_UNUSED_RESULT { - msg = LastErrorMsg; - return LastError; - } - - /* Returns "LastError:(LastErrorMsg)" */ - TString GetLastError() const Y_WARN_UNUSED_RESULT; - - /* Returns text protobuf */ - TString GetLastRequest() const { - return Req.DebugString(); - } - TString GetLastResponse() const { - return Rsp.DebugString(); - } - - /* To be used for next changed_since */ - uint64_t ResponseTimestamp() const Y_WARN_UNUSED_RESULT { - return Rsp.timestamp(); - } - - // extra_timeout: 0 - none, -1 - infinite - EError Call(const TPortoRequest &req, - TPortoResponse &rsp, - int extra_timeout = 0) Y_WARN_UNUSED_RESULT; - - EError Call(const TString &req, - TString &rsp, - int extra_timeout = 0) Y_WARN_UNUSED_RESULT; - - /* System */ - - EError GetVersion(TString &tag, TString &revision) Y_WARN_UNUSED_RESULT; - - const TGetSystemResponse *GetSystem(); - - EError SetSystem(const TString &key, const TString &val) Y_WARN_UNUSED_RESULT; - - /* Container */ - - const TListPropertiesResponse *ListProperties(); - - EError ListProperties(TVector<TString> &properties) Y_WARN_UNUSED_RESULT; - - const TListResponse *List(const TString &mask = ""); - - EError List(TVector<TString> &names, const TString &mask = "") Y_WARN_UNUSED_RESULT; - - EError Create(const TString &name) Y_WARN_UNUSED_RESULT; - - EError CreateWeakContainer(const TString &name) Y_WARN_UNUSED_RESULT; - - EError Destroy(const TString &name) Y_WARN_UNUSED_RESULT; - - EError Start(const TString &name)Y_WARN_UNUSED_RESULT; - - // stop_timeout: time between SIGTERM and SIGKILL, -1 - default - EError Stop(const TString &name, int stop_timeout = -1) Y_WARN_UNUSED_RESULT; - - EError Kill(const TString &name, int sig = 9) Y_WARN_UNUSED_RESULT; - - EError Pause(const TString &name) Y_WARN_UNUSED_RESULT; - - EError Resume(const TString &name) Y_WARN_UNUSED_RESULT; - - EError Respawn(const TString &name) Y_WARN_UNUSED_RESULT; - - // wait_timeout: 0 - nonblock, -1 - infinite - EError WaitContainer(const TString &name, - TString &result_state, - int wait_timeout = INFINITE_TIMEOUT) Y_WARN_UNUSED_RESULT; - - EError WaitContainers(const TVector<TString> &names, - TString &result_name, - TString &result_state, - int wait_timeout = INFINITE_TIMEOUT) Y_WARN_UNUSED_RESULT; - - const TWaitResponse *Wait(const TVector<TString> &names, - const TVector<TString> &labels, - int wait_timeout = INFINITE_TIMEOUT) Y_WARN_UNUSED_RESULT; - - EError AsyncWait(const TVector<TString> &names, - const TVector<TString> &labels, - TWaitCallback callbacks, - int wait_timeout = INFINITE_TIMEOUT, - const TString &targetState = "") Y_WARN_UNUSED_RESULT; - - EError StopAsyncWait(const TVector<TString> &names, - const TVector<TString> &labels, - const TString &targetState = "") Y_WARN_UNUSED_RESULT; - - const TGetResponse *Get(const TVector<TString> &names, - const TVector<TString> &properties, - int flags = 0) Y_WARN_UNUSED_RESULT; - - /* Porto v5 api */ - EError GetContainerSpec(const TString &name, TContainer &container) Y_WARN_UNUSED_RESULT ; - EError ListContainersBy(const TListContainersRequest &listContainersRequest, TVector<TContainer> &containers) Y_WARN_UNUSED_RESULT; - EError CreateFromSpec(const TContainerSpec &container, TVector<TVolumeSpec> volumes, bool start = false) Y_WARN_UNUSED_RESULT; - EError UpdateFromSpec(const TContainerSpec &container) Y_WARN_UNUSED_RESULT; - - EError GetProperty(const TString &name, - const TString &property, - TString &value, - int flags = 0) Y_WARN_UNUSED_RESULT; - - EError GetProperty(const TString &name, - const TString &property, - const TString &index, - TString &value, - int flags = 0) Y_WARN_UNUSED_RESULT { - return GetProperty(name, property + "[" + index + "]", value, flags); - } - - EError SetProperty(const TString &name, - const TString &property, - const TString &value) Y_WARN_UNUSED_RESULT; - - EError SetProperty(const TString &name, - const TString &property, - const TString &index, - const TString &value) Y_WARN_UNUSED_RESULT { - return SetProperty(name, property + "[" + index + "]", value); - } - - EError GetInt(const TString &name, - const TString &property, - const TString &index, - uint64_t &value) Y_WARN_UNUSED_RESULT; - - EError GetInt(const TString &name, - const TString &property, - uint64_t &value) Y_WARN_UNUSED_RESULT { - return GetInt(name, property, "", value); - } - - EError SetInt(const TString &name, - const TString &property, - const TString &index, - uint64_t value) Y_WARN_UNUSED_RESULT; - - EError SetInt(const TString &name, - const TString &property, - uint64_t value) Y_WARN_UNUSED_RESULT { - return SetInt(name, property, "", value); - } - - EError GetProcMetric(const TVector<TString> &names, - const TString &metric, - TMap<TString, uint64_t> &values); - - EError GetLabel(const TString &name, - const TString &label, - TString &value) Y_WARN_UNUSED_RESULT { - return GetProperty(name, "labels", label, value); - } - - EError SetLabel(const TString &name, - const TString &label, - const TString &value, - const TString &prev_value = " ") Y_WARN_UNUSED_RESULT; - - EError IncLabel(const TString &name, - const TString &label, - int64_t add, - int64_t &result) Y_WARN_UNUSED_RESULT; - - EError IncLabel(const TString &name, - const TString &label, - int64_t add = 1) Y_WARN_UNUSED_RESULT { - int64_t result; - return IncLabel(name, label, add, result); - } - - EError ConvertPath(const TString &path, - const TString &src_name, - const TString &dst_name, - TString &result_path) Y_WARN_UNUSED_RESULT; - - EError AttachProcess(const TString &name, int pid, - const TString &comm = "") Y_WARN_UNUSED_RESULT; - - EError AttachThread(const TString &name, int pid, - const TString &comm = "") Y_WARN_UNUSED_RESULT; - - EError LocateProcess(int pid, - const TString &comm /* = "" */, - TString &name) Y_WARN_UNUSED_RESULT; - - /* Volume */ - - const TListVolumePropertiesResponse *ListVolumeProperties(); - - EError ListVolumeProperties(TVector<TString> &properties) Y_WARN_UNUSED_RESULT; - - const TListVolumesResponse *ListVolumes(const TString &path = "", - const TString &container = ""); - - EError ListVolumes(TVector<TString> &paths) Y_WARN_UNUSED_RESULT; - - const TVolumeDescription *GetVolumeDesc(const TString &path); - - /* Porto v5 api */ - EError ListVolumesBy(const TGetVolumeRequest &getVolumeRequest, TVector<TVolumeSpec> &volumes) Y_WARN_UNUSED_RESULT; - EError CreateVolumeFromSpec(const TVolumeSpec &volume, TVolumeSpec &resultSpec) Y_WARN_UNUSED_RESULT; - - const TVolumeSpec *GetVolume(const TString &path); - - const TGetVolumeResponse *GetVolumes(uint64_t changed_since = 0); - - EError CreateVolume(TString &path, - const TMap<TString, TString> &config) Y_WARN_UNUSED_RESULT; - - EError LinkVolume(const TString &path, - const TString &container = "", - const TString &target = "", - bool read_only = false, - bool required = false) Y_WARN_UNUSED_RESULT; - - EError UnlinkVolume(const TString &path, - const TString &container = "", - const TString &target = "***", - bool strict = false) Y_WARN_UNUSED_RESULT; - - EError TuneVolume(const TString &path, - const TMap<TString, TString> &config) Y_WARN_UNUSED_RESULT; - - /* Layer */ - - const TListLayersResponse *ListLayers(const TString &place = "", - const TString &mask = ""); - - EError ListLayers(TVector<TString> &layers, - const TString &place = "", - const TString &mask = "") Y_WARN_UNUSED_RESULT; - - EError ImportLayer(const TString &layer, - const TString &tarball, - bool merge = false, - const TString &place = "", - const TString &private_value = "", - bool verboseError = false) Y_WARN_UNUSED_RESULT; - - EError ExportLayer(const TString &volume, - const TString &tarball, - const TString &compress = "") Y_WARN_UNUSED_RESULT; - - EError ReExportLayer(const TString &layer, - const TString &tarball, - const TString &compress = "") Y_WARN_UNUSED_RESULT; - - EError RemoveLayer(const TString &layer, - const TString &place = "", - bool async = false) Y_WARN_UNUSED_RESULT; - - EError GetLayerPrivate(TString &private_value, - const TString &layer, - const TString &place = "") Y_WARN_UNUSED_RESULT; - - EError SetLayerPrivate(const TString &private_value, - const TString &layer, - const TString &place = "") Y_WARN_UNUSED_RESULT; - - /* Docker images */ - - EError DockerImageStatus(DockerImage &image, - const TString &name, - const TString &place = "") Y_WARN_UNUSED_RESULT; - - EError ListDockerImages(std::vector<DockerImage> &images, - const TString &place = "", - const TString &mask = "") Y_WARN_UNUSED_RESULT; - - EError PullDockerImage(DockerImage &image, - const TString &name, - const TString &place = "", - const TString &auth_token = "", - const TString &auth_host = "", - const TString &auth_service = "") Y_WARN_UNUSED_RESULT; - - EError RemoveDockerImage(const TString &name, - const TString &place = ""); - - /* Storage */ - - const TListStoragesResponse *ListStorages(const TString &place = "", - const TString &mask = ""); - - EError ListStorages(TVector<TString> &storages, - const TString &place = "", - const TString &mask = "") Y_WARN_UNUSED_RESULT; - - EError RemoveStorage(const TString &storage, - const TString &place = "") Y_WARN_UNUSED_RESULT; - - EError ImportStorage(const TString &storage, - const TString &archive, - const TString &place = "", - const TString &compression = "", - const TString &private_value = "") Y_WARN_UNUSED_RESULT; - - EError ExportStorage(const TString &storage, - const TString &archive, - const TString &place = "", - const TString &compression = "") Y_WARN_UNUSED_RESULT; -}; - -#ifdef __linux__ -class TAsyncWaiter { - struct TCallbackData { - const TWaitCallback Callback; - const TString State; - }; - - enum class ERequestType { - None, - Add, - Del, - Stop, - }; - - THashMap<TString, TCallbackData> AsyncCallbacks; - std::unique_ptr<std::thread> WatchDogThread; - std::atomic<uint64_t> CallbacksCount; - int EpollFd = -1; - TPortoApi Api; - - int Sock, MasterSock; - TString ReqCt; - TString ReqState; - TWaitCallback ReqCallback; - - std::function<void(const TString &error, int ret)> FatalCallback; - bool FatalError = false; - - void MainCallback(const TWaitResponse &event); - inline TWaitCallback GetMainCallback() { - return [this](const TWaitResponse &event) { - MainCallback(event); - }; - } - - int Repair(); - void WatchDog(); - - void SendInt(int fd, int value); - int RecvInt(int fd); - - void HandleAddRequest(); - void HandleDelRequest(); - - void Fatal(const TString &error, int ret) { - FatalError = true; - FatalCallback(error, ret); - } - - public: - TAsyncWaiter(std::function<void(const TString &error, int ret)> fatalCallback); - ~TAsyncWaiter(); - - int Add(const TString &ct, const TString &state, TWaitCallback callback); - int Remove(const TString &ct); - uint64_t InvocationCount() const { - return CallbacksCount; - } -}; -#endif - -} /* namespace Porto */ diff --git a/library/cpp/porto/libporto_ut.cpp b/library/cpp/porto/libporto_ut.cpp deleted file mode 100644 index 9d78397fb8..0000000000 --- a/library/cpp/porto/libporto_ut.cpp +++ /dev/null @@ -1,226 +0,0 @@ -#include <library/cpp/testing/unittest/registar.h> -#include <libporto.hpp> - -#include <signal.h> -#include <cassert> - -#define Expect(a) assert(a) -#define ExpectEq(a, b) assert((a) == (b)) -#define ExpectNeq(a, b) assert((a) != (b)) -#define ExpectSuccess(ret) assert((ret) == Porto::EError::Success) - -const TString CT_NAME = "test-a"; - -void test_porto() { - TVector<TString> list; - TString str, path; - - signal(SIGPIPE, SIG_IGN); - - Porto::TPortoApi api; - - Expect(!api.Connected()); - Expect(api.GetFd() < 0); - - // Connect - ExpectSuccess(api.Connect()); - - Expect(api.Connected()); - Expect(api.GetFd() >= 0); - - // Disconnect - api.Disconnect(); - - Expect(!api.Connected()); - Expect(api.GetFd() < 0); - - // Auto connect - ExpectSuccess(api.GetVersion(str, str)); - Expect(api.Connected()); - - // Auto reconnect - api.Disconnect(); - ExpectSuccess(api.GetVersion(str, str)); - Expect(api.Connected()); - - // No auto reconnect - api.Disconnect(); - api.SetAutoReconnect(false); - ExpectEq(api.GetVersion(str, str), Porto::EError::SocketError); - api.SetAutoReconnect(true); - - uint64_t val = api.GetTimeout(); - ExpectNeq(val, 0); - ExpectSuccess(api.SetTimeout(5)); - - ExpectSuccess(api.List(list)); - - ExpectSuccess(api.ListProperties(list)); - - ExpectSuccess(api.ListVolumes(list)); - - ExpectSuccess(api.ListVolumeProperties(list)); - - ExpectSuccess(api.ListLayers(list)); - - ExpectSuccess(api.ListStorages(list)); - - ExpectSuccess(api.Call("Version {}", str)); - - ExpectSuccess(api.GetProperty("/", "state", str)); - ExpectEq(str, "meta"); - - ExpectSuccess(api.GetProperty("/", "controllers", "memory", str)); - ExpectEq(str, "true"); - - ExpectSuccess(api.GetProperty("/", "memory_usage", str)); - ExpectNeq(str, "0"); - - val = 0; - ExpectSuccess(api.GetInt("/", "memory_usage", val)); - ExpectNeq(val, 0); - - Porto::TContainer ct; - ExpectSuccess(api.GetContainerSpec("/", ct)); - ExpectEq(ct.spec().name(), "/"); - - ExpectEq(api.GetInt("/", "__wrong__", val), Porto::EError::InvalidProperty); - ExpectEq(api.Error(), Porto::EError::InvalidProperty); - ExpectEq(api.GetLastError(str), Porto::EError::InvalidProperty); - - ExpectSuccess(api.GetContainerSpec(CT_NAME, ct)); - ExpectEq(ct.error().error(), Porto::EError::ContainerDoesNotExist); - - ExpectSuccess(api.CreateWeakContainer(CT_NAME)); - - ExpectSuccess(api.SetProperty(CT_NAME, "memory_limit", "20M")); - ExpectSuccess(api.GetProperty(CT_NAME, "memory_limit", str)); - ExpectEq(str, "20971520"); - - ExpectSuccess(api.SetInt(CT_NAME, "memory_limit", 10<<20)); - ExpectSuccess(api.GetInt(CT_NAME, "memory_limit", val)); - ExpectEq(val, 10485760); - - ExpectSuccess(api.SetLabel(CT_NAME, "TEST.a", ".")); - - ExpectSuccess(api.GetContainerSpec(CT_NAME, ct)); - ExpectEq(ct.status().state(), "stopped"); - ExpectEq(ct.spec().memory_limit(), 10 << 20); - - ExpectSuccess(api.WaitContainer(CT_NAME, str)); - ExpectEq(str, "stopped"); - - ExpectSuccess(api.CreateVolume(path, { - {"containers", CT_NAME}, - {"backend", "native"}, - {"space_limit", "1G"}})); - ExpectNeq(path, ""); - - [[maybe_unused]] auto vd = api.GetVolumeDesc(path); - Expect(vd != nullptr); - ExpectEq(vd->path(), path); - - [[maybe_unused]] auto vs = api.GetVolume(path); - Expect(vs != nullptr); - ExpectEq(vs->path(), path); - - ExpectSuccess(api.SetProperty(CT_NAME, "command", "sleep 1000")); - ExpectSuccess(api.Start(CT_NAME)); - - ExpectSuccess(api.GetProperty(CT_NAME, "state", str)); - ExpectEq(str, "running"); - - ExpectSuccess(api.Destroy(CT_NAME)); - - TMap<TString, uint64_t> values; - auto CT_NAME_0 = CT_NAME + "abcd"; - auto CT_NAME_CHILD = CT_NAME + "/b"; - - ExpectSuccess(api.Create(CT_NAME_0)); - ExpectSuccess(api.SetProperty(CT_NAME_0, "command", "sleep 15")); - ExpectSuccess(api.Start(CT_NAME_0)); - - ExpectSuccess(api.Create(CT_NAME)); - ExpectSuccess(api.SetProperty(CT_NAME, "command", "sleep 10")); - ExpectSuccess(api.GetProcMetric(TVector<TString>{CT_NAME, CT_NAME_0}, "ctxsw", values)); - ExpectEq(values[CT_NAME], 0); - ExpectNeq(values[CT_NAME_0], 0); - - ExpectSuccess(api.Start(CT_NAME)); - ExpectSuccess(api.GetProcMetric(TVector<TString>{CT_NAME}, "ctxsw", values)); - ExpectNeq(values[CT_NAME], 0); - - ExpectSuccess(api.Create(CT_NAME_CHILD)); - ExpectSuccess(api.SetProperty(CT_NAME_CHILD, "command", "sleep 10")); - ExpectSuccess(api.GetProcMetric(TVector<TString>{CT_NAME_CHILD}, "ctxsw", values)); - ExpectEq(values[CT_NAME_CHILD], 0); - - ExpectSuccess(api.Start(CT_NAME_CHILD)); - ExpectSuccess(api.GetProcMetric(TVector<TString>{CT_NAME, CT_NAME_CHILD}, "ctxsw", values)); - ExpectNeq(values[CT_NAME_CHILD], 0); - Expect(values[CT_NAME] > values[CT_NAME_CHILD]); - - ExpectSuccess(api.Stop(CT_NAME_CHILD)); - ExpectSuccess(api.GetProcMetric(TVector<TString>{CT_NAME_CHILD}, "ctxsw", values)); - ExpectEq(values[CT_NAME_CHILD], 0); - - ExpectSuccess(api.Stop(CT_NAME)); - ExpectSuccess(api.GetProcMetric(TVector<TString>{CT_NAME}, "ctxsw", values)); - ExpectEq(values[CT_NAME], 0); - - ExpectSuccess(api.Destroy(CT_NAME_CHILD)); - ExpectSuccess(api.Destroy(CT_NAME)); - ExpectSuccess(api.Destroy(CT_NAME_0)); - -#ifdef __linux__ - // test TAsyncWaiter - Porto::TAsyncWaiter waiter([](const TString &error, int ret) { - Y_UNUSED(error); - Y_UNUSED(ret); - - Expect(false); - }); - - TString result; - waiter.Add("abc", "starting", [&result](const Porto::TWaitResponse &event) { - result += event.name() + "-" + event.state(); - }); - - TString name = "abcdef"; - ExpectSuccess(api.Create(name)); - ExpectSuccess(api.SetProperty(name, "command", "sleep 1")); - ExpectSuccess(api.Start(name)); - ExpectSuccess(api.Destroy(name)); - ExpectEq(result, ""); - - // callback work only once - for (int i = 0; i < 3; i++) { - name = "abc"; - ExpectSuccess(api.Create(name)); - ExpectSuccess(api.SetProperty(name, "command", "sleep 1")); - ExpectSuccess(api.Start(name)); - ExpectSuccess(api.Destroy(name)); - ExpectEq(result, "abc-starting"); - } - - waiter.Add("abc", "starting", [&result](const Porto::TWaitResponse &event) { - result += event.name() + "-" + event.state(); - }); - waiter.Remove("abc"); - - name = "abc"; - ExpectSuccess(api.Create(name)); - ExpectSuccess(api.SetProperty(name, "command", "sleep 1")); - ExpectSuccess(api.Start(name)); - ExpectSuccess(api.Destroy(name)); - ExpectEq(result, "abc-starting"); -#endif - - api.Disconnect(); -} - -Y_UNIT_TEST_SUITE(Porto) { - Y_UNIT_TEST(All) { - test_porto(); - } -} diff --git a/library/cpp/porto/metrics.cpp b/library/cpp/porto/metrics.cpp deleted file mode 100644 index 7d17d0aee4..0000000000 --- a/library/cpp/porto/metrics.cpp +++ /dev/null @@ -1,183 +0,0 @@ -#include "metrics.hpp" - -#include <util/folder/path.h> -#include <util/generic/maybe.h> -#include <util/stream/file.h> - -namespace Porto { - -TMap<TString, TMetric*> ProcMetrics; - -TMetric::TMetric(const TString& name, EMetric metric) { - Name = name; - Metric = metric; - ProcMetrics[name] = this; -} - -void TMetric::ClearValues(const TVector<TString>& names, TMap<TString, uint64_t>& values) const { - values.clear(); - - for (const auto&name : names) - values[name] = 0; -} - -EError TMetric::GetValues(const TVector<TString>& names, TMap<TString, uint64_t>& values, TPortoApi& api) const { - ClearValues(names, values); - - int procFd = open("/proc", O_RDONLY | O_CLOEXEC | O_DIRECTORY | O_NOCTTY); - TFileHandle procFdHandle(procFd); - if (procFd == -1) - return EError::Unknown; - - TVector<TString> tids; - TidSnapshot(tids); - - auto getResponse = api.Get(names, TVector<TString>{"cgroups[freezer]"}); - - if (getResponse == nullptr) - return EError::Unknown; - - const auto containersCgroups = GetCtFreezerCgroups(getResponse); - - for (const auto& tid : tids) { - const TString tidCgroup = GetFreezerCgroup(procFd, tid); - if (tidCgroup == "") - continue; - - TMaybe<uint64_t> metricValue; - - for (const auto& keyval : containersCgroups) { - const TString& containerCgroup = keyval.second; - if (MatchCgroups(tidCgroup, containerCgroup)) { - if (!metricValue) - metricValue = GetMetric(procFd, tid); - values[keyval.first] += *metricValue; - } - } - } - - return EError::Success; -} - -uint64_t TMetric::GetTidSchedMetricValue(int procFd, const TString& tid, const TString& metricName) const { - const TString schedPath = tid + "/sched"; - try { - int fd = openat(procFd, schedPath.c_str(), O_RDONLY | O_CLOEXEC | O_NOCTTY, 0); - TFile file(fd); - if (!file.IsOpen()) - return 0ul; - - TIFStream iStream(file); - TString line; - while (iStream.ReadLine(line)) { - auto metricPos = line.find(metricName); - - if (metricPos != TString::npos) { - auto valuePos = metricPos; - - while (valuePos < line.size() && !::isdigit(line[valuePos])) - ++valuePos; - - TString value = line.substr(valuePos); - if (!value.empty() && IsNumber(value)) - return IntFromString<uint64_t, 10>(value); - } - } - } - catch(...) {} - - return 0ul; -} - -void TMetric::GetPidTasks(const TString& pid, TVector<TString>& tids) const { - TFsPath task("/proc/" + pid + "/task"); - TVector<TString> rawTids; - - try { - task.ListNames(rawTids); - } - catch(...) {} - - for (const auto& tid : rawTids) { - tids.push_back(tid); - } -} - -void TMetric::TidSnapshot(TVector<TString>& tids) const { - TFsPath proc("/proc"); - TVector<TString> rawPids; - - try { - proc.ListNames(rawPids); - } - catch(...) {} - - for (const auto& pid : rawPids) { - if (IsNumber(pid)) - GetPidTasks(pid, tids); - } -} - -TString TMetric::GetFreezerCgroup(int procFd, const TString& tid) const { - const TString cgroupPath = tid + "/cgroup"; - try { - int fd = openat(procFd, cgroupPath.c_str(), O_RDONLY | O_CLOEXEC | O_NOCTTY, 0); - TFile file(fd); - if (!file.IsOpen()) - return TString(); - - TIFStream iStream(file); - TString line; - - while (iStream.ReadLine(line)) { - static const TString freezer = ":freezer:"; - auto freezerPos = line.find(freezer); - - if (freezerPos != TString::npos) { - line = line.substr(freezerPos + freezer.size()); - return line; - } - } - } - catch(...){} - - return TString(); -} - -TMap<TString, TString> TMetric::GetCtFreezerCgroups(const TGetResponse* response) const { - TMap<TString, TString> containersProps; - - for (const auto& ctGetListResponse : response->list()) { - for (const auto& keyval : ctGetListResponse.keyval()) { - if (!keyval.error()) { - TString value = keyval.value(); - static const TString freezerPath = "/sys/fs/cgroup/freezer"; - - if (value.find(freezerPath) != TString::npos) - value = value.substr(freezerPath.size()); - - containersProps[ctGetListResponse.name()] = value; - } - } - } - - return containersProps; -} - -bool TMetric::MatchCgroups(const TString& tidCgroup, const TString& ctCgroup) const { - if (tidCgroup.size() <= ctCgroup.size()) - return tidCgroup == ctCgroup; - return ctCgroup == tidCgroup.substr(0, ctCgroup.size()) && tidCgroup[ctCgroup.size()] == '/'; -} - -class TCtxsw : public TMetric { -public: - TCtxsw() : TMetric(M_CTXSW, EMetric::CTXSW) - {} - - uint64_t GetMetric(int procFd, const TString& tid) const override { - return GetTidSchedMetricValue(procFd, tid, "nr_switches"); - } -} static Ctxsw; - -} /* namespace Porto */ diff --git a/library/cpp/porto/metrics.hpp b/library/cpp/porto/metrics.hpp deleted file mode 100644 index 5b2ffde8d9..0000000000 --- a/library/cpp/porto/metrics.hpp +++ /dev/null @@ -1,50 +0,0 @@ -#pragma once - -#include "libporto.hpp" - -#include <util/generic/map.h> -#include <util/generic/vector.h> -#include <util/string/cast.h> -#include <util/string/type.h> - -#include <library/cpp/porto/proto/rpc.pb.h> -namespace Porto { - -constexpr const char *M_CTXSW = "ctxsw"; - -enum class EMetric { - NONE, - CTXSW, -}; - -class TMetric { -public: - TString Name; - EMetric Metric; - - TMetric(const TString& name, EMetric metric); - - void ClearValues(const TVector<TString>& names, TMap<TString, uint64_t>& values) const; - EError GetValues(const TVector<TString>& names, TMap<TString, uint64_t>& values, TPortoApi& api) const; - - // Returns value of metric from /proc/tid/sched for some tid - uint64_t GetTidSchedMetricValue(int procFd, const TString& tid, const TString& metricName) const; - - void TidSnapshot(TVector<TString>& tids) const; - void GetPidTasks(const TString& pid, TVector<TString>& tids) const; - - // Returns freezer cgroup from /proc/tid/cgroup - TString GetFreezerCgroup(int procFd, const TString& tid) const; - - // Resurns clean cgroup[freezer] for containers names - TMap<TString, TString> GetCtFreezerCgroups(const TGetResponse* response) const; - - // Verify inclusion of container cgroup in process cgroup - bool MatchCgroups(const TString& tidCgroup, const TString& ctCgroup) const; - -private: - virtual uint64_t GetMetric(int procFd, const TString& tid) const = 0; -}; - -extern TMap<TString, TMetric*> ProcMetrics; -} /* namespace Porto */ diff --git a/library/cpp/porto/proto/CMakeLists.darwin-x86_64.txt b/library/cpp/porto/proto/CMakeLists.darwin-x86_64.txt deleted file mode 100644 index 9b8be22fe6..0000000000 --- a/library/cpp/porto/proto/CMakeLists.darwin-x86_64.txt +++ /dev/null @@ -1,43 +0,0 @@ - -# This file was generated by the build system used internally in the Yandex monorepo. -# Only simple modifications are allowed (adding source-files to targets, adding simple properties -# like target_include_directories). These modifications will be ported to original -# ya.make files by maintainers. Any complex modifications which can't be ported back to the -# original buildsystem will not be accepted. - - -get_built_tool_path( - TOOL_protoc_bin - TOOL_protoc_dependency - contrib/tools/protoc/bin - protoc -) -get_built_tool_path( - TOOL_cpp_styleguide_bin - TOOL_cpp_styleguide_dependency - contrib/tools/protoc/plugins/cpp_styleguide - cpp_styleguide -) - -add_library(cpp-porto-proto) -target_link_libraries(cpp-porto-proto PUBLIC - contrib-libs-cxxsupp - yutil - contrib-libs-protobuf -) -target_proto_messages(cpp-porto-proto PRIVATE - ${CMAKE_SOURCE_DIR}/library/cpp/porto/proto/rpc.proto -) -target_proto_addincls(cpp-porto-proto - ./ - ${CMAKE_SOURCE_DIR}/ - ${CMAKE_BINARY_DIR} - ${CMAKE_SOURCE_DIR} - ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src - ${CMAKE_BINARY_DIR} - ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src -) -target_proto_outs(cpp-porto-proto - --cpp_out=${CMAKE_BINARY_DIR}/ - --cpp_styleguide_out=${CMAKE_BINARY_DIR}/ -) diff --git a/library/cpp/porto/proto/CMakeLists.linux-aarch64.txt b/library/cpp/porto/proto/CMakeLists.linux-aarch64.txt deleted file mode 100644 index ba0aa7060d..0000000000 --- a/library/cpp/porto/proto/CMakeLists.linux-aarch64.txt +++ /dev/null @@ -1,44 +0,0 @@ - -# This file was generated by the build system used internally in the Yandex monorepo. -# Only simple modifications are allowed (adding source-files to targets, adding simple properties -# like target_include_directories). These modifications will be ported to original -# ya.make files by maintainers. Any complex modifications which can't be ported back to the -# original buildsystem will not be accepted. - - -get_built_tool_path( - TOOL_protoc_bin - TOOL_protoc_dependency - contrib/tools/protoc/bin - protoc -) -get_built_tool_path( - TOOL_cpp_styleguide_bin - TOOL_cpp_styleguide_dependency - contrib/tools/protoc/plugins/cpp_styleguide - cpp_styleguide -) - -add_library(cpp-porto-proto) -target_link_libraries(cpp-porto-proto PUBLIC - contrib-libs-linux-headers - contrib-libs-cxxsupp - yutil - contrib-libs-protobuf -) -target_proto_messages(cpp-porto-proto PRIVATE - ${CMAKE_SOURCE_DIR}/library/cpp/porto/proto/rpc.proto -) -target_proto_addincls(cpp-porto-proto - ./ - ${CMAKE_SOURCE_DIR}/ - ${CMAKE_BINARY_DIR} - ${CMAKE_SOURCE_DIR} - ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src - ${CMAKE_BINARY_DIR} - ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src -) -target_proto_outs(cpp-porto-proto - --cpp_out=${CMAKE_BINARY_DIR}/ - --cpp_styleguide_out=${CMAKE_BINARY_DIR}/ -) diff --git a/library/cpp/porto/proto/CMakeLists.linux-x86_64.txt b/library/cpp/porto/proto/CMakeLists.linux-x86_64.txt deleted file mode 100644 index ba0aa7060d..0000000000 --- a/library/cpp/porto/proto/CMakeLists.linux-x86_64.txt +++ /dev/null @@ -1,44 +0,0 @@ - -# This file was generated by the build system used internally in the Yandex monorepo. -# Only simple modifications are allowed (adding source-files to targets, adding simple properties -# like target_include_directories). These modifications will be ported to original -# ya.make files by maintainers. Any complex modifications which can't be ported back to the -# original buildsystem will not be accepted. - - -get_built_tool_path( - TOOL_protoc_bin - TOOL_protoc_dependency - contrib/tools/protoc/bin - protoc -) -get_built_tool_path( - TOOL_cpp_styleguide_bin - TOOL_cpp_styleguide_dependency - contrib/tools/protoc/plugins/cpp_styleguide - cpp_styleguide -) - -add_library(cpp-porto-proto) -target_link_libraries(cpp-porto-proto PUBLIC - contrib-libs-linux-headers - contrib-libs-cxxsupp - yutil - contrib-libs-protobuf -) -target_proto_messages(cpp-porto-proto PRIVATE - ${CMAKE_SOURCE_DIR}/library/cpp/porto/proto/rpc.proto -) -target_proto_addincls(cpp-porto-proto - ./ - ${CMAKE_SOURCE_DIR}/ - ${CMAKE_BINARY_DIR} - ${CMAKE_SOURCE_DIR} - ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src - ${CMAKE_BINARY_DIR} - ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src -) -target_proto_outs(cpp-porto-proto - --cpp_out=${CMAKE_BINARY_DIR}/ - --cpp_styleguide_out=${CMAKE_BINARY_DIR}/ -) diff --git a/library/cpp/porto/proto/CMakeLists.txt b/library/cpp/porto/proto/CMakeLists.txt deleted file mode 100644 index f8b31df0c1..0000000000 --- a/library/cpp/porto/proto/CMakeLists.txt +++ /dev/null @@ -1,17 +0,0 @@ - -# This file was generated by the build system used internally in the Yandex monorepo. -# Only simple modifications are allowed (adding source-files to targets, adding simple properties -# like target_include_directories). These modifications will be ported to original -# ya.make files by maintainers. Any complex modifications which can't be ported back to the -# original buildsystem will not be accepted. - - -if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) - include(CMakeLists.linux-aarch64.txt) -elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") - include(CMakeLists.darwin-x86_64.txt) -elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) - include(CMakeLists.windows-x86_64.txt) -elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) - include(CMakeLists.linux-x86_64.txt) -endif() diff --git a/library/cpp/porto/proto/CMakeLists.windows-x86_64.txt b/library/cpp/porto/proto/CMakeLists.windows-x86_64.txt deleted file mode 100644 index 9b8be22fe6..0000000000 --- a/library/cpp/porto/proto/CMakeLists.windows-x86_64.txt +++ /dev/null @@ -1,43 +0,0 @@ - -# This file was generated by the build system used internally in the Yandex monorepo. -# Only simple modifications are allowed (adding source-files to targets, adding simple properties -# like target_include_directories). These modifications will be ported to original -# ya.make files by maintainers. Any complex modifications which can't be ported back to the -# original buildsystem will not be accepted. - - -get_built_tool_path( - TOOL_protoc_bin - TOOL_protoc_dependency - contrib/tools/protoc/bin - protoc -) -get_built_tool_path( - TOOL_cpp_styleguide_bin - TOOL_cpp_styleguide_dependency - contrib/tools/protoc/plugins/cpp_styleguide - cpp_styleguide -) - -add_library(cpp-porto-proto) -target_link_libraries(cpp-porto-proto PUBLIC - contrib-libs-cxxsupp - yutil - contrib-libs-protobuf -) -target_proto_messages(cpp-porto-proto PRIVATE - ${CMAKE_SOURCE_DIR}/library/cpp/porto/proto/rpc.proto -) -target_proto_addincls(cpp-porto-proto - ./ - ${CMAKE_SOURCE_DIR}/ - ${CMAKE_BINARY_DIR} - ${CMAKE_SOURCE_DIR} - ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src - ${CMAKE_BINARY_DIR} - ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src -) -target_proto_outs(cpp-porto-proto - --cpp_out=${CMAKE_BINARY_DIR}/ - --cpp_styleguide_out=${CMAKE_BINARY_DIR}/ -) diff --git a/library/cpp/porto/proto/rpc.proto b/library/cpp/porto/proto/rpc.proto deleted file mode 100644 index abb8c63905..0000000000 --- a/library/cpp/porto/proto/rpc.proto +++ /dev/null @@ -1,1607 +0,0 @@ -syntax = "proto2"; - -option go_package = "github.com/ydb-platform/ydb/library/cpp/porto/proto;myapi"; - -/* - Portod daemon listens on /run/portod.socket unix socket. - - Request: Varint32 length, TPortoRequest request - Response: Varint32 length, TPortoResponse response - - Command is defined by optional nested message field. - Result will be in nested message with the same name. - - Push notification is send as out of order response. - - Access level depends on client container and uid. - - See defails in porto.md or manpage porto - - TContainer, TVolume and related methods are Porto v5 API. -*/ - -package Porto; - -// List of error codes -enum EError { - // No errors occured. - Success = 0; - - // Unclassified error, usually unexpected syscall fail. - Unknown = 1; - - // Unknown method or bad request. - InvalidMethod = 2; - - // Container with specified name already exists. - ContainerAlreadyExists = 3; - - // Container with specified name doesn't exist. - ContainerDoesNotExist = 4; - - // Unknown property specified. - InvalidProperty = 5; - - // Unknown data specified. - InvalidData = 6; - - // Invalid value of property or data. - InvalidValue = 7; - - // Can't perform specified operation in current container state. - InvalidState = 8; - - // Permanent faulure: old kernel version, missing feature, configuration, etc. - NotSupported = 9; - - // Temporary failure: too much objects, not enough memory, etc. - ResourceNotAvailable = 10; - - // Insufficient rights for performing requested operation. - Permission = 11; - - // Can't create new volume with specified name, because there is already one. - VolumeAlreadyExists = 12; - - // Volume with specified name doesn't exist. - VolumeNotFound = 13; - - // Not enough disk space. - NoSpace = 14; - - // Object in use. - Busy = 15; - - // Volume already linked with container. - VolumeAlreadyLinked = 16; - - // Volume not linked with container. - VolumeNotLinked = 17; - - // Layer with this name already exists. - LayerAlreadyExists = 18; - - // Layer with this name not found. - LayerNotFound = 19; - - // Property has no value, data source permanently not available. - NoValue = 20; - - // Volume under construction or destruction. - VolumeNotReady = 21; - - // Cannot parse or execute command. - InvalidCommand = 22; - - // Error code is lost or came from future. - LostError = 23; - - // Device node not found. - DeviceNotFound = 24; - - // Path does not match restricitons or does not exist. - InvalidPath = 25; - - // Wrong or unuseable ip address. - InvalidNetworkAddress = 26; - - // Porto in system maintenance state. - PortoFrozen = 27; - - // Label with this name is not set. - LabelNotFound = 28; - - // Label name does not meet restrictions. - InvalidLabel = 29; - - // Errors in tar, on archive extraction - HelperError = 30; - HelperFatalError = 31; - - // Generic object not found. - NotFound = 404; - - // Reserved error code for client library. - SocketError = 502; - - // Reserved error code for client library. - SocketUnavailable = 503; - - // Reserved error code for client library. - SocketTimeout = 504; - - // Portod close client connections on reload - PortodReloaded = 505; - - // Reserved error code for taints. - Taint = 666; - - // Reserved error codes 700-800 to docker - Docker = 700; - DockerImageNotFound = 701; - - // Internal error code, not for users. - Queued = 1000; -} - - -message TPortoRequest { - - /* System methods */ - - // Get portod version - optional TVersionRequest Version = 14; - - // Get portod statistics - optional TGetSystemRequest GetSystem = 300; - - // Change portod state (for host root user only) - optional TSetSystemRequest SetSystem = 301; - - /* Container methods */ - - // Create new container - optional TCreateRequest Create = 1; - - // Create new contaienr and auto destroy when client disconnects - optional TCreateRequest CreateWeak = 17; - - // Force kill all and destroy container and nested containers - optional TDestroyRequest Destroy = 2; - - // List container names in current namespace - optional TListRequest List = 3; - - // Start contianer and parents if needed - optional TStartRequest Start = 7; - - // Kill all and stop container - optional TStopRequest Stop = 8; - - // Freeze execution - optional TPauseRequest Pause = 9; - - // Resume execution - optional TResumeRequest Resume = 10; - - // Send signal to main process - optional TKillRequest Kill = 13; - - // Restart dead container - optional TRespawnRequest Respawn = 18; - - // Wait for process finish or change of labels - optional TWaitRequest Wait = 16; - - // Subscribe to push notifictaions - optional TWaitRequest AsyncWait = 19; - optional TWaitRequest StopAsyncWait = 128; - - /* Container properties */ - - // List supported container properties - optional TListPropertiesRequest ListProperties = 11; - - // Get one property - optional TGetPropertyRequest GetProperty = 4; - - // Set one property - optional TSetPropertyRequest SetProperty = 5; - - // Deprecated, now data properties are also read-only properties - optional TListDataPropertiesRequest ListDataProperties = 12; - optional TGetDataPropertyRequest GetDataProperty = 6; - - // Get multiple properties for multiple containers - optional TGetRequest Get = 15; - - /* Container API based on TContainer (Porto v5 API) */ - - // Create, configure and start container with volumes - optional TCreateFromSpecRequest CreateFromSpec = 230; - - // Set multiple container properties - optional TUpdateFromSpecRequest UpdateFromSpec = 231; - - // Get multiple properties for multiple containers - optional TListContainersRequest ListContainersBy = 232; - - // Modify symlink in container - optional TSetSymlinkRequest SetSymlink = 125; - - /* Container labels - user defined key-value */ - - // Find containers with labels - optional TFindLabelRequest FindLabel = 20; - - // Atomic compare and set for label - optional TSetLabelRequest SetLabel = 21; - - // Atomic add and return for counter in label - optional TIncLabelRequest IncLabel = 22; - - /* Volume methods */ - - optional TListVolumePropertiesRequest ListVolumeProperties = 103; - - // List layers and their properties - optional TListVolumesRequest ListVolumes = 107; - - // Create, configure and build volume - optional TCreateVolumeRequest CreateVolume = 104; - - // Change volume properties - for now only resize - optional TTuneVolumeRequest TuneVolume = 108; - - // Volume API based on TVolume (Porto v5 API) - optional TNewVolumeRequest NewVolume = 126; - optional TGetVolumeRequest GetVolume = 127; - - // Add link between container and volume - optional TLinkVolumeRequest LinkVolume = 105; - - // Same as LinkVolume but fails if target is not supported - optional TLinkVolumeRequest LinkVolumeTarget = 120; - - // Del link between container and volume - optional TUnlinkVolumeRequest UnlinkVolume = 106; - - // Same as UnlinkVolume but fails if target is not supported - optional TUnlinkVolumeRequest UnlinkVolumeTarget = 121; - - /* Layer methods */ - - // Import layer from tarball - optional TImportLayerRequest ImportLayer = 110; - - // Remove layer - optional TRemoveLayerRequest RemoveLayer = 111; - - // List layers - optional TListLayersRequest ListLayers = 112; - - // Export volume or layer into tarball - optional TExportLayerRequest ExportLayer = 113; - - // Get/set layer private (user defined string) - optional TGetLayerPrivateRequest GetLayerPrivate = 114; - optional TSetLayerPrivateRequest SetLayerPrivate = 115; - - /* Storage methods */ - - // Volume creation creates required storage if missing - - // List storages and meta storages - optional TListStoragesRequest ListStorages = 116; - - optional TRemoveStorageRequest RemoveStorage = 117; - - // Import storage from tarball - optional TImportStorageRequest ImportStorage = 118; - - // Export storage into tarball - optional TExportStorageRequest ExportStorage = 119; - - // Meta storage (bundle for storages and layers) - - optional TMetaStorage CreateMetaStorage = 122; - optional TMetaStorage ResizeMetaStorage = 123; - optional TMetaStorage RemoveMetaStorage = 124; - - // Convert path between containers - optional TConvertPathRequest ConvertPath = 200; - - /* Process methods */ - - // Attach process to nested container - optional TAttachProcessRequest AttachProcess = 201; - - // Find container for process - optional TLocateProcessRequest LocateProcess = 202; - - // Attach one thread to nexted container - optional TAttachProcessRequest AttachThread = 203; - - /* Docker images API */ - - optional TDockerImageStatusRequest dockerImageStatus = 303; - optional TDockerImageListRequest listDockerImages = 304; - optional TDockerImagePullRequest pullDockerImage = 305; - optional TDockerImageRemoveRequest removeDockerImage = 306; -} - - -message TPortoResponse { - // Actually always set, hack for adding new error codes - optional EError error = 1 [ default = LostError ]; - - // Human readable comment - must be shown to user as is - optional string errorMsg = 2; - - optional uint64 timestamp = 1000; // for next changed_since - - /* System methods */ - - optional TVersionResponse Version = 8; - - optional TGetSystemResponse GetSystem = 300; - optional TSetSystemResponse SetSystem = 301; - - /* Container methods */ - - optional TListResponse List = 3; - - optional TWaitResponse Wait = 11; - - optional TWaitResponse AsyncWait = 19; - - /* Container properties */ - - optional TListPropertiesResponse ListProperties = 6; - - optional TGetPropertyResponse GetProperty = 4; - - - // Deprecated - optional TListDataPropertiesResponse ListDataProperties = 7; - optional TGetDataPropertyResponse GetDataProperty = 5; - - optional TGetResponse Get = 10; - - /* Container API based on TContainer (Porto v5 API) */ - - optional TListContainersResponse ListContainersBy = 232; - - /* Container Labels */ - - optional TFindLabelResponse FindLabel = 20; - optional TSetLabelResponse SetLabel = 21; - optional TIncLabelResponse IncLabel = 22; - - /* Volume methods */ - - optional TListVolumePropertiesResponse ListVolumeProperties = 12; - - optional TListVolumesResponse ListVolumes = 9; - - optional TVolumeDescription CreateVolume = 13; - - optional TNewVolumeResponse NewVolume = 126; - - optional TGetVolumeResponse GetVolume = 127; - - optional TListLayersResponse ListLayers = 14; - - optional TGetLayerPrivateResponse GetLayerPrivate = 16; - - // List storages and meta storages - optional TListStoragesResponse ListStorages = 17; - - optional TConvertPathResponse ConvertPath = 15; - - // Process - optional TLocateProcessResponse LocateProcess = 18; - - /* Docker images API */ - - optional TDockerImageStatusResponse dockerImageStatus = 302; - optional TDockerImageListResponse listDockerImages = 303; - optional TDockerImagePullResponse pullDockerImage = 304; -} - - -// Common objects - - -message TStringMap { - message TStringMapEntry { - optional string key = 1; - optional string val = 2; - } - // TODO replace with map - // map<string, string> map = 1; - repeated TStringMapEntry map = 1; - optional bool merge = 2; // in, default: replace -} - - -message TUintMap { - message TUintMapEntry { - optional string key = 1; - optional uint64 val = 2; - } - // TODO replace with map - // map<string, uint64> map = 1; - repeated TUintMapEntry map = 1; - optional bool merge = 2; // in, default: replace -} - - -message TError { - optional EError error = 1 [ default = LostError ]; - optional string msg = 2; -} - - -message TCred { - optional string user = 1; // requires user or uid or both - optional fixed32 uid = 2; - optional string group = 3; - optional fixed32 gid = 4; - repeated fixed32 grp = 5; // out, supplementary groups -} - - -message TCapabilities { - repeated string cap = 1; - optional string hex = 2; // out -} - - -message TContainerCommandArgv { - repeated string argv = 1; -} - - -// Container - - -message TContainerEnvVar { - optional string name = 1; //required - optional string value = 2; - optional bool unset = 3; // out - optional string salt = 4; - optional string hash = 5; -} - -message TContainerEnv { - repeated TContainerEnvVar var = 1; - optional bool merge = 2; // in, default: replace -} - - -message TContainerUlimit { - optional string type = 1; //required - optional bool unlimited = 2; - optional uint64 soft = 3; - optional uint64 hard = 4; - optional bool inherited = 5; // out -} - -message TContainerUlimits { - repeated TContainerUlimit ulimit = 1; - optional bool merge = 2; // in, default: replace -} - - -message TContainerControllers { - repeated string controller = 1; -} - - -message TContainerCgroup { - optional string controller = 1; //required - optional string path = 2; //required - optional bool inherited = 3; -} - -message TContainerCgroups { - repeated TContainerCgroup cgroup = 1; -} - - -message TContainerCpuSet { - optional string policy = 1; // inherit|set|node|reserve|threads|cores - optional uint32 arg = 2; // for node|reserve|threads|cores - optional string list = 3; // for set - repeated uint32 cpu = 4; // for set (used if list isn't set) - optional uint32 count = 5; // out - optional string mems = 6; -} - - -message TContainerBindMount { - optional string source = 1; //required - optional string target = 2; //required - repeated string flag = 3; -} - -message TContainerBindMounts { - repeated TContainerBindMount bind = 1; -} - - -message TContainerVolumeLink { - optional string volume = 1; //required - optional string target = 2; - optional bool required = 3; - optional bool read_only = 4; -} - -message TContainerVolumeLinks { - repeated TContainerVolumeLink link = 1; -} - - -message TContainerVolumes { - repeated string volume = 1; -} - - -message TContainerPlace { - optional string place = 1; //required - optional string alias = 2; -} - -message TContainerPlaceConfig { - repeated TContainerPlace cfg = 1; -} - - -message TContainerDevice { - optional string device = 1; //required - optional string access = 2; //required - optional string path = 3; - optional string mode = 4; - optional string user = 5; - optional string group = 6; -} - -message TContainerDevices { - repeated TContainerDevice device = 1; - optional bool merge = 2; // in, default: replace -} - - -message TContainerNetOption { - optional string opt = 1; //required - repeated string arg = 2; -} - -message TContainerNetConfig { - repeated TContainerNetOption cfg = 1; - optional bool inherited = 2; // out -} - - -message TContainerIpLimit { - optional string policy = 1; //required any|none|some - repeated string ip = 2; -} - - -message TContainerIpConfig { - message TContainerIp { - optional string dev = 1; //required - optional string ip = 2; //required - } - repeated TContainerIp cfg = 1; -} - - -message TVmStat { - optional uint64 count = 1; - optional uint64 size = 2; - optional uint64 max_size = 3; - optional uint64 used = 4; - optional uint64 max_used = 5; - optional uint64 anon = 6; - optional uint64 file = 7; - optional uint64 shmem = 8; - optional uint64 huge = 9; - optional uint64 swap = 10; - optional uint64 data = 11; - optional uint64 stack = 12; - optional uint64 code = 13; - optional uint64 locked = 14; - optional uint64 table = 15; -} - -message TContainerStatus { - optional string absolute_name = 1; // out, "/porto/..." - optional string state = 2; // out - optional uint64 id = 3; // out - optional uint32 level = 4; // out - optional string parent = 5; // out, "/porto/..." - - optional string absolute_namespace = 6; // out - - optional int32 root_pid = 7; // out - optional int32 exit_status = 8; // out - optional int32 exit_code = 9; // out - optional bool core_dumped = 10; // out - optional TError start_error = 11; // out - optional uint64 time = 12; // out - optional uint64 dead_time = 13; // out - - optional TCapabilities capabilities_allowed = 14; // out - optional TCapabilities capabilities_ambient_allowed = 15; // out - optional string root_path = 16; // out, in client namespace - optional uint64 stdout_offset = 17; // out - optional uint64 stderr_offset = 18; // out - optional string std_err = 69; // out - optional string std_out = 70; // out - - optional uint64 creation_time = 19; // out - optional uint64 start_time = 20; // out - optional uint64 death_time = 21; // out - optional uint64 change_time = 22; // out - optional bool no_changes = 23; // out, change_time < changed_since - optional string extra_properties = 73; - - optional TContainerCgroups cgroups = 24; // out - optional TContainerCpuSet cpu_set_affinity = 25; // out - - optional uint64 cpu_usage = 26; // out - optional uint64 cpu_usage_system = 27; // out - optional uint64 cpu_wait = 28; // out - optional uint64 cpu_throttled = 29; // out - - optional uint64 process_count = 30; // out - optional uint64 thread_count = 31; // out - - optional TUintMap io_read = 32; // out, bytes - optional TUintMap io_write = 33; // out, bytes - optional TUintMap io_ops = 34; // out, ops - optional TUintMap io_read_ops = 341; // out, ops - optional TUintMap io_write_ops = 342; // out, ops - optional TUintMap io_time = 35; // out, ns - optional TUintMap io_pressure = 351; // out - - optional TUintMap place_usage = 36; - optional uint64 memory_usage = 37; // out, bytes - - optional uint64 memory_guarantee_total = 38; // out - - optional uint64 memory_limit_total = 39; // out - - optional uint64 anon_limit_total = 40; - optional uint64 anon_usage = 41; // out, bytes - optional double cpu_guarantee_total = 42; - optional double cpu_guarantee_bound = 421; - optional double cpu_limit_total = 422; - optional double cpu_limit_bound = 423; - - optional uint64 cache_usage = 43; // out, bytes - - optional uint64 hugetlb_usage = 44; // out, bytes - optional uint64 hugetlb_limit = 45; - - optional uint64 minor_faults = 46; // out - optional uint64 major_faults = 47; // out - optional uint64 memory_reclaimed = 48; // out - optional TVmStat virtual_memory = 49; // out - - optional uint64 shmem_usage = 71; // out, bytes - optional uint64 mlock_usage = 72; // out, bytes - - optional uint64 oom_kills = 50; // out - optional uint64 oom_kills_total = 51; // out - optional bool oom_killed = 52; // out - - optional TUintMap net_bytes = 54; // out - optional TUintMap net_packets = 55; // out - optional TUintMap net_drops = 56; // out - optional TUintMap net_overlimits = 57; // out - optional TUintMap net_rx_bytes = 58; // out - optional TUintMap net_rx_packets = 59; // out - optional TUintMap net_rx_drops = 60; // out - optional TUintMap net_tx_bytes = 61; // out - optional TUintMap net_tx_packets = 62; // out - optional TUintMap net_tx_drops = 63; // out - - optional TContainerVolumeLinks volumes_linked = 64; // out - optional TContainerVolumes volumes_owned = 65; - - repeated TError error = 66; // out - repeated TError warning = 67; // out - repeated TError taint = 68; // out -} - -message TContainerSpec { - optional string name = 1; // required / in client namespace - optional bool weak = 2; - optional string private = 3; - optional TStringMap labels = 4; - - optional string command = 5; - optional TContainerCommandArgv command_argv = 76; - optional TContainerEnv env = 6; - optional TContainerEnv env_secret = 90; // in, out hides values - optional TContainerUlimits ulimit = 7; - optional string core_command = 8; - - optional bool isolate = 9; - optional string virt_mode = 10; - optional string enable_porto = 11; - optional string porto_namespace = 12; - optional string cgroupfs = 78; - optional bool userns = 79; - - optional uint64 aging_time = 13; - - optional TCred task_cred = 14; - optional string user = 15; - optional string group = 16; - - optional TCred owner_cred = 17; - optional string owner_user = 18; - optional string owner_group = 19; - optional string owner_containers = 77; - - optional TCapabilities capabilities = 20; - optional TCapabilities capabilities_ambient = 21; - - optional string root = 22; // in parent namespace - optional bool root_readonly = 23; - optional TContainerBindMounts bind = 24; - optional TStringMap symlink = 25; - optional TContainerDevices devices = 26; - optional TContainerPlaceConfig place = 27; - optional TUintMap place_limit = 28; - - optional string cwd = 29; - optional string stdin_path = 30; - optional string stdout_path = 31; - optional string stderr_path = 32; - optional uint64 stdout_limit = 33; - optional uint32 umask = 34; - - optional bool respawn = 35; - optional uint64 respawn_count = 36; - optional int64 max_respawns = 37; - optional uint64 respawn_delay = 38; - - optional TContainerControllers controllers = 39; - - optional string cpu_policy = 40; // normal|idle|batch|high|rt - optional double cpu_weight = 41; // 0.01 .. 100 - - optional double cpu_guarantee = 42; // in cores - optional double cpu_limit = 43; // in cores - optional double cpu_limit_total = 44; // deprecated (value moved to TContainerStatus) - optional uint64 cpu_period = 45; // ns - - optional TContainerCpuSet cpu_set = 46; - - optional uint64 thread_limit = 47; - - optional string io_policy = 48; // none|rt|high|normal|batch|idle - optional double io_weight = 49; // 0.01 .. 100 - - optional TUintMap io_limit = 50; // bps - optional TUintMap io_guarantee = 84; // bps - optional TUintMap io_ops_limit = 51; // iops - optional TUintMap io_ops_guarantee = 85; // iops - - optional uint64 memory_guarantee = 52; // bytes - - optional uint64 memory_limit = 53; // bytes - - optional uint64 anon_limit = 54; - optional uint64 anon_max_usage = 55; - - optional uint64 dirty_limit = 56; - - optional uint64 hugetlb_limit = 57; - - optional bool recharge_on_pgfault = 58; - optional bool pressurize_on_death = 59; - optional bool anon_only = 60; - - optional int32 oom_score_adj = 61; // -1000 .. +1000 - optional bool oom_is_fatal = 62; - - optional TContainerNetConfig net = 63; - optional TContainerIpLimit ip_limit = 64; - optional TContainerIpConfig ip = 65; - optional TContainerIpConfig default_gw = 66; - optional string hostname = 67; - optional string resolv_conf = 68; - optional string etc_hosts = 69; - optional TStringMap sysctl = 70; - optional TUintMap net_guarantee = 71; // bytes per second - optional TUintMap net_limit = 72; // bytes per second - optional TUintMap net_rx_limit = 73; // bytes per second - - optional TContainerVolumes volumes_required = 75; -} - -message TContainer { - optional TContainerSpec spec = 1; //required - optional TContainerStatus status = 2; - optional TError error = 3; -} - - -// Volumes - -message TVolumeDescription { - required string path = 1; // path in client namespace - map<string, string> properties = 2; - repeated string containers = 3; // linked containers (legacy) - repeated TVolumeLink links = 4; // linked containers with details - - optional uint64 change_time = 5; // sec since epoch - optional bool no_changes = 6; // change_time < changed_since -} - - -message TVolumeLink { - optional string container = 1; - optional string target = 2; // absolute path in container, default: anon - optional bool required = 3; // container cannot work without it - optional bool read_only = 4; - optional string host_target = 5; // out, absolute path in host - optional bool container_root = 6; // in, set container root - optional bool container_cwd = 7; // in, set container cwd -} - -message TVolumeResource { - optional uint64 limit = 1; // bytes or inodes - optional uint64 guarantee = 2; // bytes or inodes - optional uint64 usage = 3; // out, bytes or inodes - optional uint64 available = 4; // out, bytes or inodes -} - -message TVolumeDirectory { - optional string path = 1; // relative path in volume - optional TCred cred = 2; // default: volume cred - optional fixed32 permissions = 3; // default: volume permissions -} - -message TVolumeSymlink { - optional string path = 1; // relative path in volume - optional string target_path = 2; -} - -message TVolumeShare { - optional string path = 1; // relative path in volume - optional string origin_path = 2; // absolute path to origin - optional bool cow = 3; // default: mutable share -} - -// Structured Volume description (Porto V5 API) - -message TVolumeSpec { - optional string path = 1; // path in container, default: auto - optional string container = 2; // defines root for paths, default: self (client container) - repeated TVolumeLink links = 3; // initial links, default: anon link to self - - optional string id = 4; // out - optional string state = 5; // out - - optional string private_value = 6; // at most 4096 bytes - - optional string device_name = 7; // out - - optional string backend = 10; // default: auto - optional string place = 11; // path in host or alias, default from client container - optional string storage = 12; // persistent storage, path or name, default: non-persistent - optional string image = 52; - repeated string layers = 13; // name or path - optional bool read_only = 14; - - // defines root directory user, group and permissions - optional TCred cred = 20; // default: self task cred - optional fixed32 permissions = 21; // default: 0775 - - optional TVolumeResource space = 22; - optional TVolumeResource inodes = 23; - - optional TCred owner = 30; // default: self owner - optional string owner_container = 31; // default: self - optional string place_key = 32; // out, key for place_limit - optional string creator = 33; // out - optional bool auto_path = 34; // out - optional uint32 device_index = 35; // out - optional uint64 build_time = 37; // out, sec since epoch - - // customization at creation - repeated TVolumeDirectory directories = 40; // in - repeated TVolumeSymlink symlinks = 41; // in - repeated TVolumeShare shares = 42; // in - - optional uint64 change_time = 50; // out, sec since epoch - optional bool no_changes = 51; // out, change_time < changed_since -} - - -message TLayer { - optional string name = 1; // name or meta/name - optional string owner_user = 2; - optional string owner_group = 3; - optional uint64 last_usage = 4; // out, sec since last usage - optional string private_value = 5; -} - - -message TStorage { - optional string name = 1; // name or meta/name - optional string owner_user = 2; - optional string owner_group = 3; - optional uint64 last_usage = 4; // out, sec since last usage - optional string private_value = 5; -} - - -message TMetaStorage { - optional string name = 1; - optional string place = 2; - optional string private_value = 3; - optional uint64 space_limit = 4; // bytes - optional uint64 inode_limit = 5; // inodes - - optional uint64 space_used = 6; // out, bytes - optional uint64 space_available = 7; // out, bytes - optional uint64 inode_used = 8; // out, inodes - optional uint64 inode_available = 9; // out, inodes - optional string owner_user = 10; // out - optional string owner_group = 11; // out - optional uint64 last_usage = 12; // out, sec since last usage -} - - -// COMMANDS - -// System - -// Get porto version -message TVersionRequest { -} - -message TVersionResponse { - optional string tag = 1; - optional string revision = 2; -} - - -// Get porto statistics -message TGetSystemRequest { -} - -message TGetSystemResponse { - optional string porto_version = 1; - optional string porto_revision = 2; - optional string kernel_version = 3; - - optional fixed64 errors = 4; - optional fixed64 warnings = 5; - optional fixed64 porto_starts = 6; - optional fixed64 porto_uptime = 7; - optional fixed64 master_uptime = 8; - optional fixed64 taints = 9; - - optional bool frozen = 10; - optional bool verbose = 100; - optional bool debug = 101; - optional fixed64 log_lines = 102; - optional fixed64 log_bytes = 103; - - optional fixed64 stream_rotate_bytes = 104; - optional fixed64 stream_rotate_errors = 105; - - optional fixed64 log_lines_lost = 106; - optional fixed64 log_bytes_lost = 107; - optional fixed64 log_open = 108; - - optional fixed64 container_count = 200; - optional fixed64 container_limit = 201; - optional fixed64 container_running = 202; - optional fixed64 container_created = 203; - optional fixed64 container_started = 204; - optional fixed64 container_start_failed = 205; - optional fixed64 container_oom = 206; - optional fixed64 container_buried = 207; - optional fixed64 container_lost = 208; - optional fixed64 container_tainted = 209; - - optional fixed64 volume_count = 300; - optional fixed64 volume_limit = 301; - optional fixed64 volume_created = 303; - optional fixed64 volume_failed = 304; - optional fixed64 volume_links = 305; - optional fixed64 volume_links_mounted = 306; - optional fixed64 volume_lost = 307; - - optional fixed64 layer_import = 390; - optional fixed64 layer_export = 391; - optional fixed64 layer_remove = 392; - - optional fixed64 client_count = 400; - optional fixed64 client_max = 401; - optional fixed64 client_connected = 402; - - optional fixed64 request_queued = 500; - optional fixed64 request_completed = 501; - optional fixed64 request_failed = 502; - optional fixed64 request_threads = 503; - optional fixed64 request_longer_1s = 504; - optional fixed64 request_longer_3s = 505; - optional fixed64 request_longer_30s = 506; - optional fixed64 request_longer_5m = 507; - - optional fixed64 fail_system = 600; - optional fixed64 fail_invalid_value = 601; - optional fixed64 fail_invalid_command = 602; - optional fixed64 fail_memory_guarantee = 603; - optional fixed64 fail_invalid_netaddr = 604; - - optional fixed64 porto_crash = 666; - - optional fixed64 network_count = 700; - optional fixed64 network_created = 701; - optional fixed64 network_problems = 702; - optional fixed64 network_repairs = 703; -} - - -// Change porto state -message TSetSystemRequest { - optional bool frozen = 10; - optional bool verbose = 100; - optional bool debug = 101; -} - -message TSetSystemResponse { -} - -message TCreateFromSpecRequest { - optional TContainerSpec container = 1; //required - repeated TVolumeSpec volumes = 2; - optional bool start = 3; -} - -message TUpdateFromSpecRequest { - optional TContainerSpec container = 1; //required - optional bool start = 2; -} - -message TListContainersFilter { - optional string name = 1; // name or wildcards, default: all - optional TStringMap labels = 2; - optional uint64 changed_since = 3; // change_time >= changed_since -} - -message TStreamDumpOptions { - optional uint64 stdstream_offset = 2; // default: 0 - optional uint64 stdstream_limit = 3; // default: 8Mb -} - -message TListContainersFieldOptions { - repeated string properties = 1; // property names, default: all - optional TStreamDumpOptions stdout_options = 2; // for GetIndexed stdout - optional TStreamDumpOptions stderr_options = 3; // for GetIndexed stderr -} - -message TListContainersRequest { - repeated TListContainersFilter filters = 1; - optional TListContainersFieldOptions field_options = 2; -} - -message TListContainersResponse { - repeated TContainer containers = 1; -} - -// List available properties -message TListPropertiesRequest { -} - -message TListPropertiesResponse { - message TContainerPropertyListEntry { - optional string name = 1; - optional string desc = 2; - optional bool read_only = 3; - optional bool dynamic = 4; - } - repeated TContainerPropertyListEntry list = 1; -} - - -// deprecated, use ListProperties -message TListDataPropertiesRequest { -} - -message TListDataPropertiesResponse { - message TContainerDataListEntry { - optional string name = 1; - optional string desc = 2; - } - repeated TContainerDataListEntry list = 1; -} - - -// Create stopped container -message TCreateRequest { - optional string name = 1; -} - - -// Stop and destroy container -message TDestroyRequest { - optional string name = 1; -} - - -// List container names -message TListRequest { - optional string mask = 1; - optional uint64 changed_since = 2; // change_time >= changed_since -} - -message TListResponse { - repeated string name = 1; - optional string absolute_namespace = 2; -} - - -// Read one property -message TGetPropertyRequest { - optional string name = 1; - optional string property = 2; - // update cached counters - optional bool sync = 3; - optional bool real = 4; -} - -message TGetPropertyResponse { - optional string value = 1; -} - - -// Alias for GetProperty, deprecated -message TGetDataPropertyRequest { - optional string name = 1; - optional string data = 2; - // update cached counters - optional bool sync = 3; - optional bool real = 4; -} - -message TGetDataPropertyResponse { - optional string value = 1; -} - - -// Change one property -message TSetPropertyRequest { - optional string name = 1; - optional string property = 2; - optional string value = 3; -} - - -// Get multiple properties/data of many containers with one request -message TGetRequest { - // list of containers or wildcards, "***" - all - repeated string name = 1; - - // list of properties/data - repeated string variable = 2; - - // do not wait busy containers - optional bool nonblock = 3; - - // update cached counters - optional bool sync = 4; - optional bool real = 5; - - // change_time >= changed_since - optional uint64 changed_since = 6; -} - -message TGetResponse { - message TContainerGetValueResponse { - optional string variable = 1; - optional EError error = 2; - optional string errorMsg = 3; - optional string value = 4; - } - - message TContainerGetListResponse { - optional string name = 1; - repeated TContainerGetValueResponse keyval = 2; - - optional uint64 change_time = 3; - optional bool no_changes = 4; // change_time < changed_since - } - - repeated TContainerGetListResponse list = 1; -} - - -// Start stopped container -message TStartRequest { - optional string name = 1; -} - - -// Restart dead container -message TRespawnRequest { - optional string name = 1; -} - - -// Stop dead or running container -message TStopRequest { - optional string name = 1; - // Timeout in 1/1000 seconds between SIGTERM and SIGKILL, default 30s - optional uint32 timeout_ms = 2; -} - - -// Freeze running container -message TPauseRequest { - optional string name = 1; -} - - -// Unfreeze paused container -message TResumeRequest { - optional string name = 1; -} - - -// Translate filesystem path between containers -message TConvertPathRequest { - optional string path = 1; - optional string source = 2; - optional string destination = 3; -} - -message TConvertPathResponse { - optional string path = 1; -} - - -// Wait while container(s) is/are in running state -message TWaitRequest { - // list of containers or wildcards, "***" - all - repeated string name = 1; - - // timeout in 1/1000 seconds, 0 - nonblock - optional uint32 timeout_ms = 2; - - // list of label names or wildcards - repeated string label = 3; - - // async wait with target_state works only once - optional string target_state = 4; -} - -message TWaitResponse { - optional string name = 1; // container name - optional string state = 2; // container state or "timeout" - optional uint64 when = 3; // unix time stamp in seconds - optional string label = 4; - optional string value = 5; -} - - -// Send signal main process in container -message TKillRequest { - optional string name = 1; - optional int32 sig = 2; -} - - -// Move process into container -message TAttachProcessRequest { - optional string name = 1; - optional uint32 pid = 2; - optional string comm = 3; // ignored if empty -} - - -// Determine container by pid -message TLocateProcessRequest { - optional uint32 pid = 1; - optional string comm = 2; // ignored if empty -} - -message TLocateProcessResponse { - optional string name = 1; -} - - -// Labels - - -message TFindLabelRequest { - optional string mask = 1; // containers name or wildcard - optional string state = 2; // filter by container state - optional string label = 3; // label name or wildcard - optional string value = 4; // filter by label value -} - -message TFindLabelResponse { - message TFindLabelEntry { - optional string name = 1; - optional string state = 2; - optional string label = 3; - optional string value = 4; - } - repeated TFindLabelEntry list = 1; -} - - -message TSetLabelRequest { - optional string name = 1; - optional string label = 2; - optional string value = 3; - optional string prev_value = 4; // fail with Busy if does not match - optional string state = 5; // fail with InvalidState if not match -} - -message TSetLabelResponse { - optional string prev_value = 1; - optional string state = 2; -} - - -message TIncLabelRequest { - optional string name = 1; - optional string label = 2; // missing label starts from 0 - optional int64 add = 3 [ default = 1]; -} - -message TIncLabelResponse { - optional int64 result = 1; -} - - -message TSetSymlinkRequest { - optional string container = 1; - optional string symlink = 2; - optional string target = 3; -} - - -// Volumes - - -message TNewVolumeRequest { - optional TVolumeSpec volume = 1; -} - -message TNewVolumeResponse { - optional TVolumeSpec volume = 1; -} - - -message TGetVolumeRequest { - optional string container = 1; // get paths in container, default: self (client container) - repeated string path = 2; // volume path in container, default: all - optional uint64 changed_since = 3; // change_time >= changed_since - repeated string label = 4; // labels or wildcards -} - -message TGetVolumeResponse { - repeated TVolumeSpec volume = 1; -} - - -// List available volume properties -message TListVolumePropertiesRequest { -} - -message TListVolumePropertiesResponse { - message TVolumePropertyDescription { - optional string name = 1; - optional string desc = 2; - } - repeated TVolumePropertyDescription list = 1; -} - - -// Create new volume -// "createVolume" returns TVolumeDescription in "volume" -message TCreateVolumeRequest { - optional string path = 1; - map<string, string> properties = 2; -} - - -message TLinkVolumeRequest { - optional string path = 1; - optional string container = 2; // default - self (client container) - optional string target = 3; // path in container, "" - anon - optional bool required = 4; // stop container at fail - optional bool read_only = 5; -} - - -message TUnlinkVolumeRequest { - optional string path = 1; - optional string container = 2; // default - self, "***" - all - optional bool strict = 3; // non-lazy umount - optional string target = 4; // path in container, "" - anon, default - "***" - all -} - - -message TListVolumesRequest { - optional string path = 1; // volume path or wildcard - optional string container = 2; - optional uint64 changed_since = 3; // change_time >= changed_since -} - -message TListVolumesResponse { - repeated TVolumeDescription volumes = 1; -} - - -message TTuneVolumeRequest { - optional string path = 1; - map<string, string> properties = 2; -} - -// Layers - - -message TListLayersRequest { - optional string place = 1; // default from client container - optional string mask = 2; -} - -message TListLayersResponse { - repeated string layer = 1; // layer names (legacy) - repeated TLayer layers = 2; // layer with description -} - - -message TImportLayerRequest { - optional string layer = 1; - optional string tarball = 2; - optional bool merge = 3; - optional string place = 4; - optional string private_value = 5; - optional string compress = 6; - optional bool verbose_error = 7; -} - - -message TExportLayerRequest { - optional string volume = 1; - optional string tarball = 2; - optional string layer = 3; - optional string place = 4; - optional string compress = 5; -} - - -message TRemoveLayerRequest { - optional string layer = 1; - optional string place = 2; - optional bool async = 3; -} - - -message TGetLayerPrivateRequest { - optional string layer = 1; - optional string place = 2; -} - -message TGetLayerPrivateResponse { - optional string private_value = 1; -} - - -message TSetLayerPrivateRequest { - optional string layer = 1; - optional string place = 2; - optional string private_value = 3; -} - - -// Storages - - -message TListStoragesRequest { - optional string place = 1; - optional string mask = 2; // "name" - storage, "name/" - meta-storage -} - -message TListStoragesResponse { - repeated TStorage storages = 1; - repeated TMetaStorage meta_storages = 2; -} - - -message TRemoveStorageRequest { - optional string name = 1; - optional string place = 2; -} - - -message TImportStorageRequest { - optional string name = 1; - optional string tarball = 2; - optional string place = 3; - optional string private_value = 5; - optional string compress = 6; -} - - -message TExportStorageRequest { - optional string name = 1; - optional string tarball = 2; - optional string place = 3; - optional string compress = 4; -} - - -// Docker images API - - -message TDockerImageConfig { - repeated string cmd = 1; - repeated string env = 2; -} - -message TDockerImage { - required string id = 1; - repeated string tags = 2; - repeated string digests = 3; - repeated string layers = 4; - optional uint64 size = 5; - optional TDockerImageConfig config = 6; -} - - -message TDockerImageStatusRequest { - required string name = 1; - optional string place = 2; -} - -message TDockerImageStatusResponse { - optional TDockerImage image = 1; -} - - -message TDockerImageListRequest { - optional string place = 1; - optional string mask = 2; -} - -message TDockerImageListResponse { - repeated TDockerImage images = 1; -} - - -message TDockerImagePullRequest { - required string name = 1; - optional string place = 2; - optional string auth_token = 3; - optional string auth_path = 4; - optional string auth_service = 5; -} - -message TDockerImagePullResponse { - optional TDockerImage image = 1; -} - - -message TDockerImageRemoveRequest { - required string name = 1; - optional string place = 2; -} diff --git a/library/cpp/porto/proto/ya.make b/library/cpp/porto/proto/ya.make deleted file mode 100644 index 525a807ee0..0000000000 --- a/library/cpp/porto/proto/ya.make +++ /dev/null @@ -1,5 +0,0 @@ -PROTO_LIBRARY() -INCLUDE_TAGS(GO_PROTO) -SRCS(rpc.proto) -END() - diff --git a/library/cpp/porto/ut/ya.make b/library/cpp/porto/ut/ya.make deleted file mode 100644 index 766a45eb56..0000000000 --- a/library/cpp/porto/ut/ya.make +++ /dev/null @@ -1,4 +0,0 @@ -UNITTEST_FOR(library/cpp/porto) -TAG(ya:manual sb:portod) -SRCS(libporto_ut.cpp) -END() diff --git a/library/cpp/porto/ya.make b/library/cpp/porto/ya.make deleted file mode 100644 index e1ccbac281..0000000000 --- a/library/cpp/porto/ya.make +++ /dev/null @@ -1,17 +0,0 @@ -LIBRARY() - -BUILD_ONLY_IF(WARNING WARNING LINUX) - -PEERDIR( - library/cpp/porto/proto - contrib/libs/protobuf -) - -SRCS( - libporto.cpp - metrics.cpp -) - -END() - -RECURSE_FOR_TESTS(ut) @@ -34,33 +34,33 @@ REGISTRY_ENDPOINT = os.environ.get("YA_REGISTRY_ENDPOINT", "https://devtools-reg PLATFORM_MAP = { "data": { "darwin": { - "md5": "15db02ad5a0247f689c37c7532077da8", + "md5": "8ad0ba0b103a9a9d9ab672a6049b42e8", "urls": [ - f"{REGISTRY_ENDPOINT}/5386765400" + f"{REGISTRY_ENDPOINT}/5360690390" ] }, "darwin-arm64": { - "md5": "c90e27ab5ebf44b5d404d905eb289faa", + "md5": "e3fc4d5d79739874633cfb99737b8a89", "urls": [ - f"{REGISTRY_ENDPOINT}/5386763663" + f"{REGISTRY_ENDPOINT}/5360689908" ] }, "linux-aarch64": { - "md5": "25dc16ecc480f4496101482a73b3d6fb", + "md5": "b6420de189c44c25ed4b365075bc88df", "urls": [ - f"{REGISTRY_ENDPOINT}/5386762790" + f"{REGISTRY_ENDPOINT}/5360689202" ] }, "win32-clang-cl": { - "md5": "4dde3d2601fa1104f95657456be3a5ba", + "md5": "9a1c6dd3461e5e66021c71086b6205d9", "urls": [ - f"{REGISTRY_ENDPOINT}/5386766276" + f"{REGISTRY_ENDPOINT}/5360690919" ] }, "linux": { - "md5": "673cb4a7484f8b60f8d2e34355007ae6", + "md5": "75838ccb49c923014a8c85f50097cae4", "urls": [ - f"{REGISTRY_ENDPOINT}/5386767097" + f"{REGISTRY_ENDPOINT}/5360691732" ] } } diff --git a/ydb/core/cms/cms.cpp b/ydb/core/cms/cms.cpp index 07d94f0874..30d62a8a2b 100644 --- a/ydb/core/cms/cms.cpp +++ b/ydb/core/cms/cms.cpp @@ -251,8 +251,6 @@ void TCms::AdjustInfo(TClusterInfoPtr &info, const TActorContext &ctx) const { for (const auto &entry : State->Permissions) info->AddLocks(entry.second, &ctx); - for (const auto &entry : State->ScheduledRequests) - info->ScheduleActions(entry.second, &ctx); for (const auto &entry : State->Notifications) info->AddExternalLocks(entry.second, &ctx); } @@ -1730,13 +1728,7 @@ void TCms::Handle(TEvCms::TEvPermissionRequest::TPtr &ev, } } - ClusterInfo->LogManager.PushRollbackPoint(); - for (const auto &scheduled_request : State->ScheduledRequests) { - for (auto &action : scheduled_request.second.Request.GetActions()) - ClusterInfo->LogManager.ApplyAction(action, ClusterInfo); - } bool ok = CheckPermissionRequest(rec, resp->Record, scheduled.Request, ctx); - ClusterInfo->LogManager.RollbackOperations(); // Schedule request if required. if (rec.GetDryRun()) { @@ -1750,7 +1742,6 @@ void TCms::Handle(TEvCms::TEvPermissionRequest::TPtr &ev, scheduled.Owner = user; scheduled.Order = State->NextRequestId - 1; scheduled.RequestId = reqId; - ClusterInfo->ScheduleActions(scheduled, &ctx); copy = new TRequestInfo(scheduled); State->ScheduledRequests.emplace(reqId, std::move(scheduled)); @@ -1801,20 +1792,8 @@ void TCms::Handle(TEvCms::TEvCheckRequest::TPtr &ev, const TActorContext &ctx) auto requestStartTime = TInstant::Now(); - ClusterInfo->LogManager.PushRollbackPoint(); - for (const auto &scheduled_request : State->ScheduledRequests) { - if (scheduled_request.second.Order < request.Order) { - for (auto &action : scheduled_request.second.Request.GetActions()) - ClusterInfo->LogManager.ApplyAction(action, ClusterInfo); - } - } - // Deactivate locks of this and later requests to - // avoid false conflicts. - ClusterInfo->DeactivateScheduledLocks(request.Order); request.Request.SetAvailabilityMode(rec.GetAvailabilityMode()); bool ok = CheckPermissionRequest(request.Request, resp->Record, scheduled.Request, ctx); - ClusterInfo->ReactivateScheduledLocks(); - ClusterInfo->LogManager.RollbackOperations(); // Schedule request if required. if (rec.GetDryRun()) { @@ -1823,7 +1802,6 @@ void TCms::Handle(TEvCms::TEvCheckRequest::TPtr &ev, const TActorContext &ctx) TAutoPtr<TRequestInfo> copy; auto order = request.Order; - ClusterInfo->UnscheduleActions(request.RequestId); State->ScheduledRequests.erase(it); if (scheduled.Request.ActionsSize()) { scheduled.Owner = user; @@ -1831,8 +1809,6 @@ void TCms::Handle(TEvCms::TEvCheckRequest::TPtr &ev, const TActorContext &ctx) scheduled.RequestId = rec.GetRequestId(); resp->Record.SetRequestId(scheduled.RequestId); - ClusterInfo->ScheduleActions(scheduled, &ctx); - copy = new TRequestInfo(scheduled); State->ScheduledRequests.emplace(scheduled.RequestId, std::move(scheduled)); } else { diff --git a/ydb/core/cms/cms_ut.cpp b/ydb/core/cms/cms_ut.cpp index a82ea3f608..77adaf0837 100644 --- a/ydb/core/cms/cms_ut.cpp +++ b/ydb/core/cms/cms_ut.cpp @@ -555,16 +555,17 @@ Y_UNIT_TEST_SUITE(TCmsTest) { // Done with permission-1. env.CheckDonePermission("user", pid1); - // Retry request-2 and get reject (conflict with request-1). - env.CheckRequest("user", rid2, false, TStatus::DISALLOW_TEMP); - // Get permission-2 for request-1. - auto rec4 = env.CheckRequest("user", rid1, false, TStatus::ALLOW_PARTIAL, 1); + // Retry request-2. + auto rec4 = env.CheckRequest("user", rid2, false, TStatus::ALLOW, 2); auto pid2 = rec4.GetPermissions(0).GetId(); + auto pid3 = rec4.GetPermissions(1).GetId(); + // Get permission-2 for request-1. + env.CheckRequest("user", rid1, false, TStatus::DISALLOW_TEMP); - // Get request-1 expecting 2 remaining actions. + // Get request-1 expecting 3 remaining actions. auto rec5 = env.CheckGetRequest("user", rid1); UNIT_ASSERT_VALUES_EQUAL(rec5.RequestsSize(), 1); - UNIT_ASSERT_VALUES_EQUAL(rec5.GetRequests(0).ActionsSize(), 2); + UNIT_ASSERT_VALUES_EQUAL(rec5.GetRequests(0).ActionsSize(), 3); // Schedule request-3. auto rec6 = env.CheckPermissionRequest @@ -575,30 +576,21 @@ Y_UNIT_TEST_SUITE(TCmsTest) { // Schedule request-4. auto rec7 = env.CheckPermissionRequest ("user1", true, false, true, true, TStatus::DISALLOW_TEMP, - MakeAction(TAction::REPLACE_DEVICES, env.GetNodeId(0), 60000000, - env.PDiskName(1, 3))); + MakeAction(TAction::SHUTDOWN_HOST, env.GetNodeId(1), 60000000)); auto rid4 = rec7.GetRequestId(); // List 2 requests of user1. env.CheckListRequests("user1", 2); // Reject request-1. env.CheckRejectRequest("user", rid1); - // Retry request-2 and get reject (conflict with permission-2). - env.CheckRequest("user", rid2, false, TStatus::DISALLOW_TEMP); - // Done with permission-2. + // Done with permissions 2 and 3. env.CheckDonePermission("user", pid2); - // Retry request-3 and get reject (conflict with request-2). - env.CheckRequest("user1", rid3, false, TStatus::DISALLOW_TEMP); + env.CheckDonePermission("user", pid3); + // Retry request-3. + auto rec8 = env.CheckRequest("user1", rid3, false, TStatus::ALLOW, 1); + auto pid4 = rec8.GetPermissions(0).GetId(); // Retry request-4 and get reject (conflict with request-3). env.CheckRequest("user1", rid4, false, TStatus::DISALLOW_TEMP); - // List 1 request of user. - auto rec9 = env.CheckListRequests("user", 1); - UNIT_ASSERT_VALUES_EQUAL(rec9.GetRequests(0).GetRequestId(), rid2); - // Reject request-3. - env.CheckRejectRequest("user1", rid3); - // Retry request-4. - auto rec8 = env.CheckRequest("user1", rid4, false, TStatus::ALLOW, 1); - auto pid3 = rec8.GetPermissions(0).GetId(); // Restart CMS tablet. env.RestartCms(); @@ -606,17 +598,17 @@ Y_UNIT_TEST_SUITE(TCmsTest) { // Check there are no permissions for user. env.CheckListPermissions("user", 0); // Check permission of user1 is restored. - auto rec10 = env.CheckListPermissions("user1", 1); - UNIT_ASSERT_VALUES_EQUAL(rec10.GetPermissions(0).GetId(), pid3); - // Check request for user is restored. - auto rec11 = env.CheckListRequests( "user", 1); - UNIT_ASSERT_VALUES_EQUAL(rec11.GetRequests(0).GetRequestId(), rid2); - // Check there are no requests for user1. - env.CheckListRequests("user1", 0); - // Retry request-2. - env.CheckRequest( "user", rid2, false, TStatus::ALLOW, 2); + auto rec9 = env.CheckListPermissions("user1", 1); + UNIT_ASSERT_VALUES_EQUAL(rec9.GetPermissions(0).GetId(), pid4); + // Done with permissions-4. + env.CheckDonePermission("user1", pid4); + // Check request for user1 is restored. + env.CheckListRequests("user1", 1); + // Retry request-4. + auto rec10 = env.CheckRequest("user1", rid4, false, TStatus::ALLOW, 1); + auto pid5 = rec10.GetPermissions(0).GetId(); // Check there are no requests for user. - env.CheckListRequests( "user", 0); + env.CheckListRequests("user1", 0); } Y_UNIT_TEST(WalleTasks) @@ -664,27 +656,27 @@ Y_UNIT_TEST_SUITE(TCmsTest) { env.CheckWalleRemoveTask("task-5", TStatus::WRONG_REQUEST); // Check modified tasks list. env.CheckWalleListTasks("task-6", "in-process", env.GetNodeId(1)); - // New task should be queued because of task-6. - env.CheckWalleCreateTask("task-8", "change-disk", false, TStatus::DISALLOW_TEMP, + // Lock node-0 for reboot, Task-6 scheduled, but permission was not issued. + env.CheckWalleCreateTask("task-8", "change-disk", false, TStatus::ALLOW, env.GetNodeId(0)); // Check tasks list. env.CheckWalleListTasks(2); - // Task-8 still cannot be executed. - env.CheckWalleCheckTask("task-8", TStatus::DISALLOW_TEMP, env.GetNodeId(0)); - // Task-6 is allowed now. - env.CheckWalleCheckTask("task-6", TStatus::ALLOW, env.GetNodeId(1)); + // Task-6 still cannot be executed. + env.CheckWalleCheckTask("task-6", TStatus::DISALLOW_TEMP, env.GetNodeId(1)); // Kill CMS tablet to check Wall-E tasks are preserved correctly. env.RestartCms(); // Check tasks list. env.CheckWalleListTasks(2); - // Task-6 is allowed. - env.CheckWalleCheckTask("task-6", TStatus::ALLOW, env.GetNodeId(1)); - // Task-8 is waiting. - env.CheckWalleCheckTask("task-8", TStatus::DISALLOW_TEMP, env.GetNodeId(0)); + // Task-8 is allowed. + env.CheckWalleCheckTask("task-8", TStatus::ALLOW, env.GetNodeId(0)); + // Task-6 is waiting. + env.CheckWalleCheckTask("task-6", TStatus::DISALLOW_TEMP, env.GetNodeId(1)); // Remove task-8. env.CheckWalleRemoveTask("task-8"); + // Task-6 now can be executed. + env.CheckWalleCheckTask("task-6", TStatus::ALLOW, env.GetNodeId(1)); // Check modified tasks list. env.CheckWalleListTasks("task-6", "ok", env.GetNodeId(1)); // Remove task-6. diff --git a/ydb/core/grpc_services/CMakeLists.darwin-x86_64.txt b/ydb/core/grpc_services/CMakeLists.darwin-x86_64.txt index e544f10dbb..30f46225c4 100644 --- a/ydb/core/grpc_services/CMakeLists.darwin-x86_64.txt +++ b/ydb/core/grpc_services/CMakeLists.darwin-x86_64.txt @@ -49,7 +49,7 @@ target_link_libraries(ydb-core-grpc_services PUBLIC core-tx-datashard core-tx-sharding tx-long_tx_service-public - core-tx-ev_write + core-tx-data_events ydb-core-ydb_convert ydb-core-security ydb-library-aclib diff --git a/ydb/core/grpc_services/CMakeLists.linux-aarch64.txt b/ydb/core/grpc_services/CMakeLists.linux-aarch64.txt index 176d0a4214..dbeec96adf 100644 --- a/ydb/core/grpc_services/CMakeLists.linux-aarch64.txt +++ b/ydb/core/grpc_services/CMakeLists.linux-aarch64.txt @@ -50,7 +50,7 @@ target_link_libraries(ydb-core-grpc_services PUBLIC core-tx-datashard core-tx-sharding tx-long_tx_service-public - core-tx-ev_write + core-tx-data_events ydb-core-ydb_convert ydb-core-security ydb-library-aclib diff --git a/ydb/core/grpc_services/CMakeLists.linux-x86_64.txt b/ydb/core/grpc_services/CMakeLists.linux-x86_64.txt index 176d0a4214..dbeec96adf 100644 --- a/ydb/core/grpc_services/CMakeLists.linux-x86_64.txt +++ b/ydb/core/grpc_services/CMakeLists.linux-x86_64.txt @@ -50,7 +50,7 @@ target_link_libraries(ydb-core-grpc_services PUBLIC core-tx-datashard core-tx-sharding tx-long_tx_service-public - core-tx-ev_write + core-tx-data_events ydb-core-ydb_convert ydb-core-security ydb-library-aclib diff --git a/ydb/core/grpc_services/CMakeLists.windows-x86_64.txt b/ydb/core/grpc_services/CMakeLists.windows-x86_64.txt index e544f10dbb..30f46225c4 100644 --- a/ydb/core/grpc_services/CMakeLists.windows-x86_64.txt +++ b/ydb/core/grpc_services/CMakeLists.windows-x86_64.txt @@ -49,7 +49,7 @@ target_link_libraries(ydb-core-grpc_services PUBLIC core-tx-datashard core-tx-sharding tx-long_tx_service-public - core-tx-ev_write + core-tx-data_events ydb-core-ydb_convert ydb-core-security ydb-library-aclib diff --git a/ydb/core/grpc_services/rpc_long_tx.cpp b/ydb/core/grpc_services/rpc_long_tx.cpp index cc72ab6174..cda4897e89 100644 --- a/ydb/core/grpc_services/rpc_long_tx.cpp +++ b/ydb/core/grpc_services/rpc_long_tx.cpp @@ -13,7 +13,7 @@ #include <ydb/core/tx/sharding/sharding.h> #include <ydb/core/scheme/scheme_types_proto.h> #include <ydb/core/tx/schemeshard/schemeshard.h> -#include <ydb/core/tx/ev_write/shard_writer.h> +#include <ydb/core/tx/data_events/shard_writer.h> #include <ydb/core/tx/columnshard/columnshard.h> #include <ydb/core/tx/long_tx_service/public/events.h> #include <ydb/services/ext_index/common/service.h> diff --git a/ydb/core/grpc_services/rpc_read_columns.cpp b/ydb/core/grpc_services/rpc_read_columns.cpp index f0e048274f..47e1f86d3f 100644 --- a/ydb/core/grpc_services/rpc_read_columns.cpp +++ b/ydb/core/grpc_services/rpc_read_columns.cpp @@ -382,7 +382,7 @@ private: size_t rowsExtracted = 0; bool skippedBeforeMinKey = false; - if (ev->Get()->GetDataFormat() == NKikimrTxDataShard::ARROW) { + if (ev->Get()->GetDataFormat() == NKikimrDataEvents::FORMAT_ARROW) { return ReplyWithError(Ydb::StatusIds::INTERNAL_ERROR, "Arrow format not supported yet", ctx); } diff --git a/ydb/core/grpc_services/rpc_read_rows.cpp b/ydb/core/grpc_services/rpc_read_rows.cpp index b23bcf6880..46f18a1ca3 100644 --- a/ydb/core/grpc_services/rpc_read_rows.cpp +++ b/ydb/core/grpc_services/rpc_read_rows.cpp @@ -489,7 +489,7 @@ public: record.AddColumns(meta.Id); } - record.SetResultFormat(::NKikimrTxDataShard::EScanDataFormat::CELLVEC); + record.SetResultFormat(::NKikimrDataEvents::FORMAT_CELLVEC); for (auto& key : keys) { request->Keys.emplace_back(TSerializedCellVec::Serialize(key)); diff --git a/ydb/core/grpc_services/rpc_stream_execute_scan_query.cpp b/ydb/core/grpc_services/rpc_stream_execute_scan_query.cpp index afd5d6853c..6f5f30523a 100644 --- a/ydb/core/grpc_services/rpc_stream_execute_scan_query.cpp +++ b/ydb/core/grpc_services/rpc_stream_execute_scan_query.cpp @@ -227,6 +227,8 @@ private: nullptr ); + ev->Record.MutableRequest()->SetCollectDiagnostics(req->Getcollect_full_diagnostics()); + if (!ctx.Send(NKqp::MakeKqpProxyID(ctx.SelfID.NodeId()), ev.Release())) { NYql::TIssues issues; issues.AddIssue(MakeIssue(NKikimrIssues::TIssuesIds::DEFAULT_ERROR, "Internal error")); @@ -313,6 +315,8 @@ private: response.mutable_result()->mutable_query_stats()->set_query_ast(kqpResponse.GetQueryAst()); } + response.mutable_result()->set_query_full_diagnostics(kqpResponse.GetQueryDiagnostics()); + Y_PROTOBUF_SUPPRESS_NODISCARD response.SerializeToString(&out); Request_->SendSerializedResult(std::move(out), record.GetYdbStatus()); } diff --git a/ydb/core/grpc_services/ya.make b/ydb/core/grpc_services/ya.make index fb6c19027a..fe47e03513 100644 --- a/ydb/core/grpc_services/ya.make +++ b/ydb/core/grpc_services/ya.make @@ -114,7 +114,7 @@ PEERDIR( ydb/core/tx/datashard ydb/core/tx/sharding ydb/core/tx/long_tx_service/public - ydb/core/tx/ev_write + ydb/core/tx/data_events ydb/core/ydb_convert ydb/core/security ydb/library/aclib diff --git a/ydb/core/kqp/compute_actor/kqp_compute_events.h b/ydb/core/kqp/compute_actor/kqp_compute_events.h index fbe76e5035..bec341d00e 100644 --- a/ydb/core/kqp/compute_actor/kqp_compute_events.h +++ b/ydb/core/kqp/compute_actor/kqp_compute_events.h @@ -70,11 +70,11 @@ struct TEvKqpCompute { return Remote->SerializeToArcadiaStream(chunker); } - NKikimrTxDataShard::EScanDataFormat GetDataFormat() const { + NKikimrDataEvents::EDataFormat GetDataFormat() const { if (ArrowBatch != nullptr) { - return NKikimrTxDataShard::EScanDataFormat::ARROW; + return NKikimrDataEvents::FORMAT_ARROW; } - return NKikimrTxDataShard::EScanDataFormat::CELLVEC; + return NKikimrDataEvents::FORMAT_CELLVEC; } @@ -128,15 +128,15 @@ struct TEvKqpCompute { } switch (GetDataFormat()) { - case NKikimrTxDataShard::EScanDataFormat::UNSPECIFIED: - case NKikimrTxDataShard::EScanDataFormat::CELLVEC: { + case NKikimrDataEvents::FORMAT_UNSPECIFIED: + case NKikimrDataEvents::FORMAT_CELLVEC: { Remote->Record.MutableRows()->Reserve(Rows.size()); for (const auto& row: Rows) { Remote->Record.AddRows(TSerializedCellVec::Serialize(row)); } break; } - case NKikimrTxDataShard::EScanDataFormat::ARROW: { + case NKikimrDataEvents::FORMAT_ARROW: { Y_DEBUG_ABORT_UNLESS(ArrowBatch != nullptr); auto* protoArrowBatch = Remote->Record.MutableArrowBatch(); protoArrowBatch->SetSchema(NArrow::SerializeSchema(*ArrowBatch->schema())); diff --git a/ydb/core/kqp/compute_actor/kqp_pure_compute_actor.cpp b/ydb/core/kqp/compute_actor/kqp_pure_compute_actor.cpp index 59048148a9..c9f2ffbee8 100644 --- a/ydb/core/kqp/compute_actor/kqp_pure_compute_actor.cpp +++ b/ydb/core/kqp/compute_actor/kqp_pure_compute_actor.cpp @@ -211,15 +211,15 @@ void TKqpComputeActor::HandleExecute(TEvKqpCompute::TEvScanData::TPtr& ev) { { auto guard = TaskRunner->BindAllocator(); switch (msg.GetDataFormat()) { - case NKikimrTxDataShard::EScanDataFormat::UNSPECIFIED: - case NKikimrTxDataShard::EScanDataFormat::CELLVEC: { + case NKikimrDataEvents::FORMAT_UNSPECIFIED: + case NKikimrDataEvents::FORMAT_CELLVEC: { if (!msg.Rows.empty()) { bytes = ScanData->AddData(msg.Rows, {}, TaskRunner->GetHolderFactory()); rowsCount = msg.Rows.size(); } break; } - case NKikimrTxDataShard::EScanDataFormat::ARROW: { + case NKikimrDataEvents::FORMAT_ARROW: { if(msg.ArrowBatch != nullptr) { bytes = ScanData->AddData(*msg.ArrowBatch, {}, TaskRunner->GetHolderFactory()); rowsCount = msg.ArrowBatch->num_rows(); diff --git a/ydb/core/kqp/compute_actor/kqp_scan_events.h b/ydb/core/kqp/compute_actor/kqp_scan_events.h index 8a267df4c7..4e95863e6a 100644 --- a/ydb/core/kqp/compute_actor/kqp_scan_events.h +++ b/ydb/core/kqp/compute_actor/kqp_scan_events.h @@ -46,12 +46,12 @@ struct TEvScanExchange { TEvSendData(TEvKqpCompute::TEvScanData& msg, const ui64 tabletId) : TabletId(tabletId) { switch (msg.GetDataFormat()) { - case NKikimrTxDataShard::EScanDataFormat::CELLVEC: - case NKikimrTxDataShard::EScanDataFormat::UNSPECIFIED: + case NKikimrDataEvents::FORMAT_CELLVEC: + case NKikimrDataEvents::FORMAT_UNSPECIFIED: Rows = std::move(msg.Rows); Y_ABORT_UNLESS(Rows.size()); break; - case NKikimrTxDataShard::EScanDataFormat::ARROW: + case NKikimrDataEvents::FORMAT_ARROW: ArrowBatch = msg.ArrowBatch; Y_ABORT_UNLESS(ArrowBatch); Y_ABORT_UNLESS(ArrowBatch->num_rows()); diff --git a/ydb/core/kqp/executer_actor/kqp_data_executer.cpp b/ydb/core/kqp/executer_actor/kqp_data_executer.cpp index ddc2b6e622..0252f25e14 100644 --- a/ydb/core/kqp/executer_actor/kqp_data_executer.cpp +++ b/ydb/core/kqp/executer_actor/kqp_data_executer.cpp @@ -2058,8 +2058,8 @@ private: bool needCommit = Request.LocksOp == ELocksOp::Commit || VolatileTx; auto locksOp = needCommit - ? NKikimrTxDataShard::TKqpLocks::Commit - : NKikimrTxDataShard::TKqpLocks::Rollback; + ? NKikimrDataEvents::TKqpLocks::Commit + : NKikimrDataEvents::TKqpLocks::Rollback; absl::flat_hash_set<ui64> sendingShardsSet; absl::flat_hash_set<ui64> receivingShardsSet; @@ -2139,13 +2139,13 @@ private: for (auto& [_, tx] : topicTxs) { switch (locksOp) { - case NKikimrTxDataShard::TKqpLocks::Commit: + case NKikimrDataEvents::TKqpLocks::Commit: tx.SetOp(NKikimrPQ::TDataTransaction::Commit); break; - case NKikimrTxDataShard::TKqpLocks::Rollback: + case NKikimrDataEvents::TKqpLocks::Rollback: tx.SetOp(NKikimrPQ::TDataTransaction::Rollback); break; - case NKikimrTxDataShard::TKqpLocks::Unspecified: + case NKikimrDataEvents::TKqpLocks::Unspecified: break; } @@ -2403,7 +2403,7 @@ private: ui64 TxCoordinator = 0; THashMap<ui64, TShardState> ShardStates; - TVector<NKikimrTxDataShard::TLock> Locks; + TVector<NKikimrDataEvents::TLock> Locks; bool ReadOnlyTx = true; bool VolatileTx = false; bool ImmediateTx = false; diff --git a/ydb/core/kqp/executer_actor/kqp_executer_impl.h b/ydb/core/kqp/executer_actor/kqp_executer_impl.h index 786525a941..1f35cd3b6b 100644 --- a/ydb/core/kqp/executer_actor/kqp_executer_impl.h +++ b/ydb/core/kqp/executer_actor/kqp_executer_impl.h @@ -884,9 +884,9 @@ protected: } if (AppData()->FeatureFlags.GetEnableArrowFormatAtDatashard()) { - settings->SetDataFormat(NKikimrTxDataShard::EScanDataFormat::ARROW); + settings->SetDataFormat(NKikimrDataEvents::FORMAT_ARROW); } else { - settings->SetDataFormat(NKikimrTxDataShard::EScanDataFormat::CELLVEC); + settings->SetDataFormat(NKikimrDataEvents::FORMAT_CELLVEC); } if (snapshot.IsValid()) { diff --git a/ydb/core/kqp/executer_actor/kqp_locks_helper.cpp b/ydb/core/kqp/executer_actor/kqp_locks_helper.cpp index 6073aea73f..5ec7be7071 100644 --- a/ydb/core/kqp/executer_actor/kqp_locks_helper.cpp +++ b/ydb/core/kqp/executer_actor/kqp_locks_helper.cpp @@ -6,7 +6,7 @@ namespace NKikimr::NKqp { -void BuildLocks(NKikimrMiniKQL::TResult& result, const TVector<NKikimrTxDataShard::TLock>& locks) { +void BuildLocks(NKikimrMiniKQL::TResult& result, const TVector<NKikimrDataEvents::TLock>& locks) { auto setMemberDataType = [] (NKikimrMiniKQL::TMember& member, const TString& name, ui32 scheme) { member.SetName(name); member.MutableType()->SetKind(NKikimrMiniKQL::ETypeKind::Data); @@ -39,7 +39,7 @@ void BuildLocks(NKikimrMiniKQL::TResult& result, const TVector<NKikimrTxDataShar } } -NKikimrTxDataShard::TLock ExtractLock(const NYql::NDq::TMkqlValueRef& lock) { +NKikimrDataEvents::TLock ExtractLock(const NYql::NDq::TMkqlValueRef& lock) { auto ensureMemberDataType = [] (const NKikimrMiniKQL::TMember& member, const TString& name, ui32 scheme) { YQL_ENSURE(member.GetName() == name); YQL_ENSURE(member.GetType().GetKind() == NKikimrMiniKQL::ETypeKind::Data); @@ -61,7 +61,7 @@ NKikimrTxDataShard::TLock ExtractLock(const NYql::NDq::TMkqlValueRef& lock) { ensureMemberDataType(structType.GetMember(5), "SchemeShard", NKikimr::NUdf::TDataType<ui64>::Id); ensureMemberDataType(structType.GetMember(6), "HasWrites", NKikimr::NUdf::TDataType<bool>::Id); - NKikimrTxDataShard::TLock dsLock; + NKikimrDataEvents::TLock dsLock; dsLock.SetCounter(value.GetStruct(0).GetUint64()); dsLock.SetDataShard(value.GetStruct(1).GetUint64()); dsLock.SetGeneration(value.GetStruct(2).GetUint32()); diff --git a/ydb/core/kqp/executer_actor/kqp_locks_helper.h b/ydb/core/kqp/executer_actor/kqp_locks_helper.h index 9a7f4686a1..3d953acaf3 100644 --- a/ydb/core/kqp/executer_actor/kqp_locks_helper.h +++ b/ydb/core/kqp/executer_actor/kqp_locks_helper.h @@ -7,8 +7,8 @@ namespace NKikimr::NKqp { -void BuildLocks(NKikimrMiniKQL::TResult& result, const TVector<NKikimrTxDataShard::TLock>& locks); +void BuildLocks(NKikimrMiniKQL::TResult& result, const TVector<NKikimrDataEvents::TLock>& locks); -NKikimrTxDataShard::TLock ExtractLock(const NYql::NDq::TMkqlValueRef& lock); +NKikimrDataEvents::TLock ExtractLock(const NYql::NDq::TMkqlValueRef& lock); } // namespace NKikimr::NKqp diff --git a/ydb/core/kqp/executer_actor/kqp_tasks_graph.cpp b/ydb/core/kqp/executer_actor/kqp_tasks_graph.cpp index 0d1c2e2682..fc20975c4e 100644 --- a/ydb/core/kqp/executer_actor/kqp_tasks_graph.cpp +++ b/ydb/core/kqp/executer_actor/kqp_tasks_graph.cpp @@ -860,19 +860,19 @@ void FillTaskMeta(const TStageInfo& stageInfo, const TTask& task, NYql::NDqProto case ETableKind::Unknown: case ETableKind::External: case ETableKind::SysView: { - protoTaskMeta.SetDataFormat(NKikimrTxDataShard::EScanDataFormat::CELLVEC); + protoTaskMeta.SetDataFormat(NKikimrDataEvents::FORMAT_CELLVEC); break; } case ETableKind::Datashard: { if (AppData()->FeatureFlags.GetEnableArrowFormatAtDatashard()) { - protoTaskMeta.SetDataFormat(NKikimrTxDataShard::EScanDataFormat::ARROW); + protoTaskMeta.SetDataFormat(NKikimrDataEvents::FORMAT_ARROW); } else { - protoTaskMeta.SetDataFormat(NKikimrTxDataShard::EScanDataFormat::CELLVEC); + protoTaskMeta.SetDataFormat(NKikimrDataEvents::FORMAT_CELLVEC); } break; } case ETableKind::Olap: { - protoTaskMeta.SetDataFormat(NKikimrTxDataShard::EScanDataFormat::ARROW); + protoTaskMeta.SetDataFormat(NKikimrDataEvents::FORMAT_ARROW); break; } } diff --git a/ydb/core/kqp/gateway/kqp_gateway.h b/ydb/core/kqp/gateway/kqp_gateway.h index a31df57e3f..f5d267d09f 100644 --- a/ydb/core/kqp/gateway/kqp_gateway.h +++ b/ydb/core/kqp/gateway/kqp_gateway.h @@ -127,7 +127,7 @@ public: NKikimr::TControlWrapper PerRequestDataSizeLimit; NKikimr::TControlWrapper MaxShardCount; TVector<TPhysicalTxData> Transactions; - TMap<ui64, TVector<NKikimrTxDataShard::TLock>> DataShardLocks; + TMap<ui64, TVector<NKikimrDataEvents::TLock>> DataShardLocks; NKikimr::NKqp::TTxAllocatorState::TPtr TxAlloc; ELocksOp LocksOp = ELocksOp::Unspecified; TMaybe<ui64> AcquireLocksTxId; diff --git a/ydb/core/kqp/runtime/kqp_read_actor.cpp b/ydb/core/kqp/runtime/kqp_read_actor.cpp index 99aa786772..bb8ed8d046 100644 --- a/ydb/core/kqp/runtime/kqp_read_actor.cpp +++ b/ydb/core/kqp/runtime/kqp_read_actor.cpp @@ -1152,7 +1152,7 @@ public: } TString DebugPrintCells(const TEvDataShard::TEvReadResult* result) { - if (result->Record.GetResultFormat() == NKikimrTxDataShard::EScanDataFormat::ARROW) { + if (result->Record.GetResultFormat() == NKikimrDataEvents::FORMAT_ARROW) { return "{ARROW}"; } TStringBuilder builder; @@ -1244,11 +1244,11 @@ public: if (!batch.Defined()) { batch.ConstructInPlace(); switch (msg.Record.GetResultFormat()) { - case NKikimrTxDataShard::EScanDataFormat::ARROW: + case NKikimrDataEvents::FORMAT_ARROW: BytesStats.AddStatistics(PackArrow(result, freeSpace)); break; - case NKikimrTxDataShard::EScanDataFormat::UNSPECIFIED: - case NKikimrTxDataShard::EScanDataFormat::CELLVEC: + case NKikimrDataEvents::FORMAT_UNSPECIFIED: + case NKikimrDataEvents::FORMAT_CELLVEC: BytesStats.AddStatistics(PackCells(result, freeSpace)); } } @@ -1460,8 +1460,8 @@ private: TQueue<TResult> Results; - TVector<NKikimrTxDataShard::TLock> Locks; - TVector<NKikimrTxDataShard::TLock> BrokenLocks; + TVector<NKikimrDataEvents::TLock> Locks; + TVector<NKikimrDataEvents::TLock> BrokenLocks; IKqpGateway::TKqpSnapshot Snapshot; diff --git a/ydb/core/kqp/runtime/kqp_stream_lookup_actor.cpp b/ydb/core/kqp/runtime/kqp_stream_lookup_actor.cpp index 92bb64e352..3ff0d16d39 100644 --- a/ydb/core/kqp/runtime/kqp_stream_lookup_actor.cpp +++ b/ydb/core/kqp/runtime/kqp_stream_lookup_actor.cpp @@ -388,7 +388,7 @@ private: record.SetMaxRows(Max<ui16>()); record.SetMaxBytes(5_MB); - record.SetResultFormat(NKikimrTxDataShard::EScanDataFormat::CELLVEC); + record.SetResultFormat(NKikimrDataEvents::FORMAT_CELLVEC); Send(MakePipePeNodeCacheID(false), new TEvPipeCache::TEvForward(request.Release(), shardId, true), IEventHandle::FlagTrackDelivery); @@ -484,8 +484,8 @@ private: std::shared_ptr<const TVector<TKeyDesc::TPartitionInfo>> Partitioning; const TDuration SchemeCacheRequestTimeout; NActors::TActorId SchemeCacheRequestTimeoutTimer; - TVector<NKikimrTxDataShard::TLock> Locks; - TVector<NKikimrTxDataShard::TLock> BrokenLocks; + TVector<NKikimrDataEvents::TLock> Locks; + TVector<NKikimrDataEvents::TLock> BrokenLocks; std::unique_ptr<TKqpStreamLookupWorker> StreamLookupWorker; ui64 ReadId = 0; diff --git a/ydb/core/kqp/session_actor/kqp_session_actor.cpp b/ydb/core/kqp/session_actor/kqp_session_actor.cpp index 6566a651c9..2b5c68ffcf 100644 --- a/ydb/core/kqp/session_actor/kqp_session_actor.cpp +++ b/ydb/core/kqp/session_actor/kqp_session_actor.cpp @@ -474,6 +474,7 @@ public: return; } + YQL_ENSURE(QueryState); TTimerGuard timer(this); @@ -1478,6 +1479,8 @@ public: } } + response->SetQueryDiagnostics(QueryState->ReplayMessage); + // Result for scan query is sent directly to target actor. Y_ABORT_UNLESS(response->GetArena()); if (QueryState->PreparedQuery && !QueryState->IsStreamResult()) { diff --git a/ydb/core/kqp/ut/common/CMakeLists.darwin-x86_64.txt b/ydb/core/kqp/ut/common/CMakeLists.darwin-x86_64.txt index 8ebd0100ee..c10448c806 100644 --- a/ydb/core/kqp/ut/common/CMakeLists.darwin-x86_64.txt +++ b/ydb/core/kqp/ut/common/CMakeLists.darwin-x86_64.txt @@ -32,4 +32,5 @@ target_sources(kqp-ut-common PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/kqp/ut/common/kqp_ut_common.cpp ${CMAKE_SOURCE_DIR}/ydb/core/kqp/ut/common/re2_udf.cpp ${CMAKE_SOURCE_DIR}/ydb/core/kqp/ut/common/string_udf.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/kqp/ut/common/columnshard.cpp ) diff --git a/ydb/core/kqp/ut/common/CMakeLists.linux-aarch64.txt b/ydb/core/kqp/ut/common/CMakeLists.linux-aarch64.txt index 091762509e..dcc6b6dfc5 100644 --- a/ydb/core/kqp/ut/common/CMakeLists.linux-aarch64.txt +++ b/ydb/core/kqp/ut/common/CMakeLists.linux-aarch64.txt @@ -33,4 +33,5 @@ target_sources(kqp-ut-common PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/kqp/ut/common/kqp_ut_common.cpp ${CMAKE_SOURCE_DIR}/ydb/core/kqp/ut/common/re2_udf.cpp ${CMAKE_SOURCE_DIR}/ydb/core/kqp/ut/common/string_udf.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/kqp/ut/common/columnshard.cpp ) diff --git a/ydb/core/kqp/ut/common/CMakeLists.linux-x86_64.txt b/ydb/core/kqp/ut/common/CMakeLists.linux-x86_64.txt index 091762509e..dcc6b6dfc5 100644 --- a/ydb/core/kqp/ut/common/CMakeLists.linux-x86_64.txt +++ b/ydb/core/kqp/ut/common/CMakeLists.linux-x86_64.txt @@ -33,4 +33,5 @@ target_sources(kqp-ut-common PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/kqp/ut/common/kqp_ut_common.cpp ${CMAKE_SOURCE_DIR}/ydb/core/kqp/ut/common/re2_udf.cpp ${CMAKE_SOURCE_DIR}/ydb/core/kqp/ut/common/string_udf.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/kqp/ut/common/columnshard.cpp ) diff --git a/ydb/core/kqp/ut/common/CMakeLists.windows-x86_64.txt b/ydb/core/kqp/ut/common/CMakeLists.windows-x86_64.txt index 8ebd0100ee..c10448c806 100644 --- a/ydb/core/kqp/ut/common/CMakeLists.windows-x86_64.txt +++ b/ydb/core/kqp/ut/common/CMakeLists.windows-x86_64.txt @@ -32,4 +32,5 @@ target_sources(kqp-ut-common PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/kqp/ut/common/kqp_ut_common.cpp ${CMAKE_SOURCE_DIR}/ydb/core/kqp/ut/common/re2_udf.cpp ${CMAKE_SOURCE_DIR}/ydb/core/kqp/ut/common/string_udf.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/kqp/ut/common/columnshard.cpp ) diff --git a/ydb/core/kqp/ut/common/columnshard.cpp b/ydb/core/kqp/ut/common/columnshard.cpp new file mode 100644 index 0000000000..cad366fd1a --- /dev/null +++ b/ydb/core/kqp/ut/common/columnshard.cpp @@ -0,0 +1,177 @@ +#include "columnshard.h" +#include <ydb/core/formats/arrow/serializer/full.h> +#include <ydb/core/testlib/cs_helper.h> + +namespace NKikimr { +namespace NKqp { + using namespace NYdb; + + TTestHelper::TTestHelper(const TKikimrSettings& settings) + : Kikimr(settings) + , TableClient(Kikimr.GetTableClient()) + , LongTxClient(Kikimr.GetDriver()) + , Session(TableClient.CreateSession().GetValueSync().GetSession()) + {} + + NKikimr::NKqp::TKikimrRunner& TTestHelper::GetKikimr() { + return Kikimr; + } + + NYdb::NTable::TSession& TTestHelper::GetSession() { + return Session; + } + + void TTestHelper::CreateTable(const TColumnTableBase& table) { + std::cerr << (table.BuildQuery()) << std::endl; + auto result = Session.ExecuteSchemeQuery(table.BuildQuery()).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + + void TTestHelper::InsertData(const TColumnTable& table, TTestHelper::TUpdatesBuilder& updates, const std::function<void()> onBeforeCommit /*= {}*/, const EStatus opStatus /*= EStatus::SUCCESS*/) { + NLongTx::TLongTxBeginResult resBeginTx = LongTxClient.BeginWriteTx().GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(resBeginTx.Status().GetStatus(), EStatus::SUCCESS, resBeginTx.Status().GetIssues().ToString()); + + auto txId = resBeginTx.GetResult().tx_id(); + auto batch = updates.BuildArrow(); + TString data = NArrow::NSerialization::TFullDataSerializer(arrow::ipc::IpcWriteOptions::Defaults()).Serialize(batch); + + NLongTx::TLongTxWriteResult resWrite = + LongTxClient.Write(txId, table.GetName(), txId, data, Ydb::LongTx::Data::APACHE_ARROW).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(resWrite.Status().GetStatus(), opStatus, resWrite.Status().GetIssues().ToString()); + + if (onBeforeCommit) { + onBeforeCommit(); + } + + NLongTx::TLongTxCommitResult resCommitTx = LongTxClient.CommitTx(txId).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(resCommitTx.Status().GetStatus(), EStatus::SUCCESS, resCommitTx.Status().GetIssues().ToString()); + } + + void TTestHelper::BulkUpsert(const TColumnTable& table, TTestHelper::TUpdatesBuilder& updates, const Ydb::StatusIds_StatusCode& opStatus /*= Ydb::StatusIds::SUCCESS*/) { + Y_UNUSED(opStatus); + NKikimr::Tests::NCS::THelper helper(Kikimr.GetTestServer()); + auto batch = updates.BuildArrow(); + helper.SendDataViaActorSystem(table.GetName(), batch, opStatus); + } + + void TTestHelper::ReadData(const TString& query, const TString& expected, const EStatus opStatus /*= EStatus::SUCCESS*/) { + auto it = TableClient.StreamExecuteScanQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(it.GetStatus(), EStatus::SUCCESS, it.GetIssues().ToString()); // Means stream successfully get + TString result = StreamResultToYson(it, false, opStatus); + if (opStatus == EStatus::SUCCESS) { + UNIT_ASSERT_NO_DIFF(ReformatYson(result), ReformatYson(expected)); + } + } + + void TTestHelper::RebootTablets(const TString& tableName) { + auto runtime = Kikimr.GetTestServer().GetRuntime(); + TActorId sender = runtime->AllocateEdgeActor(); + TVector<ui64> shards; + { + auto describeResult = DescribeTable(&Kikimr.GetTestServer(), sender, tableName); + for (auto shard : describeResult.GetPathDescription().GetColumnTableDescription().GetSharding().GetColumnShards()) { + shards.push_back(shard); + } + } + for (auto shard : shards) { + RebootTablet(*runtime, shard, sender); + } + } + + + TString TTestHelper::TColumnSchema::BuildQuery() const { + auto str = TStringBuilder() << Name << " " << NScheme::GetTypeName(Type); + if (!NullableFlag) { + str << " NOT NULL"; + } + return str; + } + + + TString TTestHelper::TColumnTableBase::BuildQuery() const { + auto str = TStringBuilder() << "CREATE " << GetObjectType() << " `" << Name << "`"; + str << " (" << BuildColumnsStr(Schema) << ", PRIMARY KEY (" << JoinStrings(PrimaryKey, ", ") << "))"; + if (!Sharding.empty()) { + str << " PARTITION BY HASH(" << JoinStrings(Sharding, ", ") << ")"; + } + str << " WITH (STORE = COLUMN, AUTO_PARTITIONING_MIN_PARTITIONS_COUNT =" << MinPartitionsCount << ");"; + return str; + } + + + std::shared_ptr<arrow::Schema> TTestHelper::TColumnTableBase::GetArrowSchema(const TVector<TColumnSchema>& columns) { + std::vector<std::shared_ptr<arrow::Field>> result; + for (auto&& col : columns) { + result.push_back(BuildField(col.GetName(), col.GetType(), col.IsNullable())); + } + return std::make_shared<arrow::Schema>(result); + } + + + TString TTestHelper::TColumnTableBase::BuildColumnsStr(const TVector<TColumnSchema>& clumns) const { + TVector<TString> columnStr; + for (auto&& c : clumns) { + columnStr.push_back(c.BuildQuery()); + } + return JoinStrings(columnStr, ", "); + } + + + std::shared_ptr<arrow::Field> TTestHelper::TColumnTableBase::BuildField(const TString name, const NScheme::TTypeId& typeId, bool nullable) const { + switch (typeId) { + case NScheme::NTypeIds::Bool: + return arrow::field(name, arrow::boolean(), nullable); + case NScheme::NTypeIds::Int8: + return arrow::field(name, arrow::int8(), nullable); + case NScheme::NTypeIds::Int16: + return arrow::field(name, arrow::int16(), nullable); + case NScheme::NTypeIds::Int32: + return arrow::field(name, arrow::int32(), nullable); + case NScheme::NTypeIds::Int64: + return arrow::field(name, arrow::int64(), nullable); + case NScheme::NTypeIds::Uint8: + return arrow::field(name, arrow::uint8(), nullable); + case NScheme::NTypeIds::Uint16: + return arrow::field(name, arrow::uint16(), nullable); + case NScheme::NTypeIds::Uint32: + return arrow::field(name, arrow::uint32(), nullable); + case NScheme::NTypeIds::Uint64: + return arrow::field(name, arrow::uint64(), nullable); + case NScheme::NTypeIds::Float: + return arrow::field(name, arrow::float32(), nullable); + case NScheme::NTypeIds::Double: + return arrow::field(name, arrow::float64(), nullable); + case NScheme::NTypeIds::String: + return arrow::field(name, arrow::binary(), nullable); + case NScheme::NTypeIds::Utf8: + return arrow::field(name, arrow::utf8(), nullable); + case NScheme::NTypeIds::Json: + return arrow::field(name, arrow::utf8(), nullable); + case NScheme::NTypeIds::Yson: + return arrow::field(name, arrow::binary(), nullable); + case NScheme::NTypeIds::Date: + return arrow::field(name, arrow::uint16(), nullable); + case NScheme::NTypeIds::Datetime: + return arrow::field(name, arrow::uint32(), nullable); + case NScheme::NTypeIds::Timestamp: + return arrow::field(name, arrow::timestamp(arrow::TimeUnit::TimeUnit::MICRO), nullable); + case NScheme::NTypeIds::Interval: + return arrow::field(name, arrow::duration(arrow::TimeUnit::TimeUnit::MICRO), nullable); + case NScheme::NTypeIds::JsonDocument: + return arrow::field(name, arrow::binary(), nullable); + } + return nullptr; + } + + + TString TTestHelper::TColumnTable::GetObjectType() const { + return "TABLE"; + } + + + TString TTestHelper::TColumnTableStore::GetObjectType() const { + return "TABLESTORE"; + } + +} +} diff --git a/ydb/core/kqp/ut/common/columnshard.h b/ydb/core/kqp/ut/common/columnshard.h new file mode 100644 index 0000000000..13ae578c02 --- /dev/null +++ b/ydb/core/kqp/ut/common/columnshard.h @@ -0,0 +1,70 @@ +#pragma once + +#include "kqp_ut_common.h" +#include <ydb/library/accessor/accessor.h> +#include <ydb/public/lib/scheme_types/scheme_type_id.h> +#include <ydb/public/sdk/cpp/client/draft/ydb_long_tx.h> +#include <ydb/public/sdk/cpp/client/ydb_table/table.h> +#include <ydb/public/sdk/cpp/client/ydb_types/status_codes.h> +#include <ydb/core/tx/columnshard/columnshard_ut_common.h> +#include <contrib/libs/apache/arrow/cpp/src/arrow/type.h> + +namespace NKikimr { +namespace NKqp { + class TTestHelper { + public: + class TColumnSchema { + YDB_ACCESSOR_DEF(TString, Name); + YDB_ACCESSOR_DEF(NScheme::TTypeId, Type); + YDB_FLAG_ACCESSOR(Nullable, true); + public: + TString BuildQuery() const; + }; + + using TUpdatesBuilder = NColumnShard::TTableUpdatesBuilder; + + class TColumnTableBase { + YDB_ACCESSOR_DEF(TString, Name); + YDB_ACCESSOR_DEF(TVector<TColumnSchema>, Schema); + YDB_ACCESSOR_DEF(TVector<TString>, PrimaryKey); + YDB_ACCESSOR_DEF(TVector<TString>, Sharding); + YDB_ACCESSOR(ui32, MinPartitionsCount, 1); + public: + TString BuildQuery() const; + std::shared_ptr<arrow::Schema> GetArrowSchema(const TVector<TColumnSchema>& columns); + + private: + virtual TString GetObjectType() const = 0; + TString BuildColumnsStr(const TVector<TColumnSchema>& clumns) const; + std::shared_ptr<arrow::Field> BuildField(const TString name, const NScheme::TTypeId& typeId, bool nullable) const; + }; + + class TColumnTable : public TColumnTableBase { + private: + TString GetObjectType() const override; + }; + + class TColumnTableStore : public TColumnTableBase { + private: + TString GetObjectType() const override; + }; + + private: + TKikimrRunner Kikimr; + NYdb::NTable::TTableClient TableClient; + NYdb::NLongTx::TClient LongTxClient; + NYdb::NTable::TSession Session; + + public: + TTestHelper(const TKikimrSettings& settings); + TKikimrRunner& GetKikimr(); + NYdb::NTable::TSession& GetSession(); + void CreateTable(const TColumnTableBase& table); + void InsertData(const TColumnTable& table, TTestHelper::TUpdatesBuilder& updates, const std::function<void()> onBeforeCommit = {}, const NYdb::EStatus opStatus = NYdb::EStatus::SUCCESS); + void BulkUpsert(const TColumnTable& table, TTestHelper::TUpdatesBuilder& updates, const Ydb::StatusIds_StatusCode& opStatus = Ydb::StatusIds::SUCCESS); + void ReadData(const TString& query, const TString& expected, const NYdb::EStatus opStatus = NYdb::EStatus::SUCCESS); + void RebootTablets(const TString& tableName); + }; + +} +} diff --git a/ydb/core/kqp/ut/common/ya.make b/ydb/core/kqp/ut/common/ya.make index 30978c9412..2b377f5163 100644 --- a/ydb/core/kqp/ut/common/ya.make +++ b/ydb/core/kqp/ut/common/ya.make @@ -6,6 +6,7 @@ SRCS( kqp_ut_common.h re2_udf.cpp string_udf.cpp + columnshard.cpp ) PEERDIR( diff --git a/ydb/core/kqp/ut/indexes/kqp_indexes_ut.cpp b/ydb/core/kqp/ut/indexes/kqp_indexes_ut.cpp index 56c67a71b8..64ff8b71f4 100644 --- a/ydb/core/kqp/ut/indexes/kqp_indexes_ut.cpp +++ b/ydb/core/kqp/ut/indexes/kqp_indexes_ut.cpp @@ -1791,7 +1791,7 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { UNIT_ASSERT_C(result.GetAst().Contains("'('\"Reverse\")"), result.GetAst()); - UNIT_ASSERT_C(result.GetDiagnostics().empty(), "Query result diagnostics is not empty"); + UNIT_ASSERT_C(result.GetDiagnostics().empty(), "Query result diagnostics should be empty, but it's not"); } } } diff --git a/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp b/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp index bf88da1954..85c77bab71 100644 --- a/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp +++ b/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp @@ -110,11 +110,14 @@ Y_UNIT_TEST_SUITE(KqpOlap) { } } - TVector<THashMap<TString, NYdb::TValue>> CollectRows(NYdb::NTable::TScanQueryPartIterator& it, NJson::TJsonValue* statInfo = nullptr) { + TVector<THashMap<TString, NYdb::TValue>> CollectRows(NYdb::NTable::TScanQueryPartIterator& it, NJson::TJsonValue* statInfo = nullptr, NJson::TJsonValue* diagnostics = nullptr) { TVector<THashMap<TString, NYdb::TValue>> rows; if (statInfo) { *statInfo = NJson::JSON_NULL; } + if (diagnostics) { + *diagnostics = NJson::JSON_NULL; + } for (;;) { auto streamPart = it.ReadNext().GetValueSync(); if (!streamPart.IsSuccess()) { @@ -131,6 +134,13 @@ Y_UNIT_TEST_SUITE(KqpOlap) { UNIT_ASSERT(NJson::ReadJsonFastTree(*plan, statInfo)); } } + + if (streamPart.HasDiagnostics()) { + TString diagnosticsString = streamPart.GetDiagnostics(); + if (!diagnosticsString.empty() && diagnostics) { + UNIT_ASSERT(NJson::ReadJsonFastTree(diagnosticsString, diagnostics)); + } + } if (streamPart.HasResultSet()) { auto resultSet = streamPart.ExtractResultSet(); @@ -820,6 +830,64 @@ Y_UNIT_TEST_SUITE(KqpOlap) { } } + Y_UNIT_TEST(SimpleQueryOlapDiagnostics) { + auto settings = TKikimrSettings() + .SetWithSampleTables(false); + TKikimrRunner kikimr(settings); + + TLocalHelper(kikimr).CreateTestOlapTable(); + + WriteTestData(kikimr, "/Root/olapStore/olapTable", 0, 1000000, 2); + + auto client = kikimr.GetTableClient(); + + { + TStreamExecScanQuerySettings settings; + settings.CollectQueryStats(ECollectQueryStatsMode::Full); + auto it = client.StreamExecuteScanQuery(R"( + --!syntax_v1 + SELECT `resource_id`, `timestamp` + FROM `/Root/olapStore/olapTable` + ORDER BY `resource_id`, `timestamp` + )", settings).GetValueSync(); + + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + NJson::TJsonValue jsonDiagnostics; + CollectRows(it, nullptr, &jsonDiagnostics); + UNIT_ASSERT_C(!jsonDiagnostics.IsDefined(), "Query result diagnostics should be empty, but it's not"); + } + + { + TStreamExecScanQuerySettings settings; + settings.CollectQueryStats(ECollectQueryStatsMode::Full); + settings.CollectFullDiagnostics(true); + auto it = client.StreamExecuteScanQuery(R"( + --!syntax_v1 + SELECT `resource_id`, `timestamp` + FROM `/Root/olapStore/olapTable` + ORDER BY `resource_id`, `timestamp` + )", settings).GetValueSync(); + + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + NJson::TJsonValue jsonDiagnostics; + CollectRows(it, nullptr, &jsonDiagnostics); + UNIT_ASSERT(!jsonDiagnostics.IsNull()); + + UNIT_ASSERT_C(jsonDiagnostics.IsMap(), "Incorrect Diagnostics"); + UNIT_ASSERT_C(jsonDiagnostics.Has("query_id"), "Incorrect Diagnostics"); + UNIT_ASSERT_C(jsonDiagnostics.Has("version"), "Incorrect Diagnostics"); + UNIT_ASSERT_C(jsonDiagnostics.Has("query_text"), "Incorrect Diagnostics"); + UNIT_ASSERT_C(jsonDiagnostics.Has("query_parameter_types"), "Incorrect Diagnostics"); + UNIT_ASSERT_C(jsonDiagnostics.Has("table_metadata"), "Incorrect Diagnostics"); + UNIT_ASSERT_C(jsonDiagnostics.Has("created_at"), "Incorrect Diagnostics"); + UNIT_ASSERT_C(jsonDiagnostics.Has("query_syntax"), "Incorrect Diagnostics"); + UNIT_ASSERT_C(jsonDiagnostics.Has("query_database"), "Incorrect Diagnostics"); + UNIT_ASSERT_C(jsonDiagnostics.Has("query_cluster"), "Incorrect Diagnostics"); + UNIT_ASSERT_C(jsonDiagnostics.Has("query_plan"), "Incorrect Diagnostics"); + UNIT_ASSERT_C(jsonDiagnostics.Has("query_type"), "Incorrect Diagnostics"); + } + } + Y_UNIT_TEST(SimpleLookupOlap) { auto settings = TKikimrSettings() .SetWithSampleTables(false); diff --git a/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp b/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp index f4c84caaab..83946d31e3 100644 --- a/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp +++ b/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp @@ -1,7 +1,7 @@ #include <ydb/core/kqp/ut/common/kqp_ut_common.h> +#include <ydb/core/kqp/ut/common/columnshard.h> #include <ydb/core/formats/arrow/arrow_helpers.h> #include <ydb/core/tx/tx_proxy/proxy.h> -#include <ydb/core/tx/columnshard/columnshard_ut_common.h> #include <ydb/public/sdk/cpp/client/ydb_proto/accessor.h> #include <ydb/public/sdk/cpp/client/ydb_scheme/scheme.h> #include <ydb/public/sdk/cpp/client/ydb_topic/topic.h> @@ -9,7 +9,6 @@ #include <ydb/core/testlib/cs_helper.h> #include <ydb/core/testlib/common_helper.h> #include <ydb/core/formats/arrow/serializer/full.h> - #include <library/cpp/threading/local_executor/local_executor.h> #include <util/generic/serialized_enum.h> @@ -5353,202 +5352,6 @@ Y_UNIT_TEST_SUITE(KqpScheme) { } } -namespace { - class TTestHelper { - public: - class TColumnSchema { - YDB_ACCESSOR_DEF(TString, Name); - YDB_ACCESSOR_DEF(NScheme::TTypeId, Type); - YDB_FLAG_ACCESSOR(Nullable, true); - public: - TString BuildQuery() const { - auto str = TStringBuilder() << Name << " " << NScheme::GetTypeName(Type); - if (!NullableFlag) { - str << " NOT NULL"; - } - return str; - } - }; - - using TUpdatesBuilder = NColumnShard::TTableUpdatesBuilder; - - class TColumnTableBase { - YDB_ACCESSOR_DEF(TString, Name); - YDB_ACCESSOR_DEF(TVector<TColumnSchema>, Schema); - YDB_ACCESSOR_DEF(TVector<TString>, PrimaryKey); - YDB_ACCESSOR_DEF(TVector<TString>, Sharding); - YDB_ACCESSOR(ui32, MinPartitionsCount, 1); - public: - TString BuildQuery() const { - auto str = TStringBuilder() << "CREATE " << GetObjectType() << " `" << Name << "`"; - str << " (" << BuildColumnsStr(Schema) << ", PRIMARY KEY (" << JoinStrings(PrimaryKey, ", ") << "))"; - if (!Sharding.empty()) { - str << " PARTITION BY HASH(" << JoinStrings(Sharding, ", ") << ")"; - } - str << " WITH (STORE = COLUMN, AUTO_PARTITIONING_MIN_PARTITIONS_COUNT =" << MinPartitionsCount << ");"; - return str; - } - - std::shared_ptr<arrow::Schema> GetArrowSchema(const TVector<TColumnSchema>& columns) { - std::vector<std::shared_ptr<arrow::Field>> result; - for (auto&& col : columns) { - result.push_back(BuildField(col.GetName(), col.GetType(), col.IsNullable())); - } - return std::make_shared<arrow::Schema>(result); - } - - private: - virtual TString GetObjectType() const = 0; - TString BuildColumnsStr(const TVector<TColumnSchema>& clumns) const { - TVector<TString> columnStr; - for (auto&& c : clumns) { - columnStr.push_back(c.BuildQuery()); - } - return JoinStrings(columnStr, ", "); - } - - std::shared_ptr<arrow::Field> BuildField(const TString name, const NScheme::TTypeId& typeId, bool nullable) const { - switch(typeId) { - case NScheme::NTypeIds::Bool: - return arrow::field(name, arrow::boolean(), nullable); - case NScheme::NTypeIds::Int8: - return arrow::field(name, arrow::int8(), nullable); - case NScheme::NTypeIds::Int16: - return arrow::field(name, arrow::int16(), nullable); - case NScheme::NTypeIds::Int32: - return arrow::field(name, arrow::int32(), nullable); - case NScheme::NTypeIds::Int64: - return arrow::field(name, arrow::int64(), nullable); - case NScheme::NTypeIds::Uint8: - return arrow::field(name, arrow::uint8(), nullable); - case NScheme::NTypeIds::Uint16: - return arrow::field(name, arrow::uint16(), nullable); - case NScheme::NTypeIds::Uint32: - return arrow::field(name, arrow::uint32(), nullable); - case NScheme::NTypeIds::Uint64: - return arrow::field(name, arrow::uint64(), nullable); - case NScheme::NTypeIds::Float: - return arrow::field(name, arrow::float32(), nullable); - case NScheme::NTypeIds::Double: - return arrow::field(name, arrow::float64(), nullable); - case NScheme::NTypeIds::String: - return arrow::field(name, arrow::binary(), nullable); - case NScheme::NTypeIds::Utf8: - return arrow::field(name, arrow::utf8(), nullable); - case NScheme::NTypeIds::Json: - return arrow::field(name, arrow::utf8(), nullable); - case NScheme::NTypeIds::Yson: - return arrow::field(name, arrow::binary(), nullable); - case NScheme::NTypeIds::Date: - return arrow::field(name, arrow::uint16(), nullable); - case NScheme::NTypeIds::Datetime: - return arrow::field(name, arrow::uint32(), nullable); - case NScheme::NTypeIds::Timestamp: - return arrow::field(name, arrow::timestamp(arrow::TimeUnit::TimeUnit::MICRO), nullable); - case NScheme::NTypeIds::Interval: - return arrow::field(name, arrow::duration(arrow::TimeUnit::TimeUnit::MICRO), nullable); - case NScheme::NTypeIds::JsonDocument: - return arrow::field(name, arrow::binary(), nullable); - } - return nullptr; - } - }; - - class TColumnTable : public TColumnTableBase { - private: - TString GetObjectType() const override { - return "TABLE"; - } - }; - - class TColumnTableStore : public TColumnTableBase { - private: - TString GetObjectType() const override { - return "TABLESTORE"; - } - }; - - private: - TKikimrRunner Kikimr; - NYdb::NTable::TTableClient TableClient; - NYdb::NLongTx::TClient LongTxClient; - NYdb::NTable::TSession Session; - - public: - TTestHelper(const TKikimrSettings& settings) - : Kikimr(settings) - , TableClient(Kikimr.GetTableClient()) - , LongTxClient(Kikimr.GetDriver()) - , Session(TableClient.CreateSession().GetValueSync().GetSession()) - {} - - TKikimrRunner& GetKikimr() { - return Kikimr; - } - - NYdb::NTable::TSession& GetSession() { - return Session; - } - - void CreateTable(const TColumnTableBase& table) { - std::cerr << (table.BuildQuery()) << std::endl; - auto result = Session.ExecuteSchemeQuery(table.BuildQuery()).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - - void InsertData(const TColumnTable& table, TTestHelper::TUpdatesBuilder& updates, const std::function<void()> onBeforeCommit = {}, const EStatus opStatus = EStatus::SUCCESS) { - NLongTx::TLongTxBeginResult resBeginTx = LongTxClient.BeginWriteTx().GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(resBeginTx.Status().GetStatus(), EStatus::SUCCESS, resBeginTx.Status().GetIssues().ToString()); - - auto txId = resBeginTx.GetResult().tx_id(); - auto batch = updates.BuildArrow(); - TString data = NArrow::NSerialization::TFullDataSerializer(arrow::ipc::IpcWriteOptions::Defaults()).Serialize(batch); - - NLongTx::TLongTxWriteResult resWrite = - LongTxClient.Write(txId, table.GetName(), txId, data, Ydb::LongTx::Data::APACHE_ARROW).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(resWrite.Status().GetStatus(), opStatus, resWrite.Status().GetIssues().ToString()); - - if (onBeforeCommit) { - onBeforeCommit(); - } - - NLongTx::TLongTxCommitResult resCommitTx = LongTxClient.CommitTx(txId).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(resCommitTx.Status().GetStatus(), EStatus::SUCCESS, resCommitTx.Status().GetIssues().ToString()); - } - - void BulkUpsert(const TColumnTable& table, TTestHelper::TUpdatesBuilder& updates, const Ydb::StatusIds_StatusCode& opStatus = Ydb::StatusIds::SUCCESS) { - Y_UNUSED(opStatus); - NKikimr::Tests::NCS::THelper helper(Kikimr.GetTestServer()); - auto batch = updates.BuildArrow(); - helper.SendDataViaActorSystem(table.GetName(), batch, opStatus); - } - - void ReadData(const TString& query, const TString& expected, const EStatus opStatus = EStatus::SUCCESS) { - auto it = TableClient.StreamExecuteScanQuery(query).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(it.GetStatus(), EStatus::SUCCESS, it.GetIssues().ToString()); // Means stream successfully get - TString result = StreamResultToYson(it, false, opStatus); - if (opStatus == EStatus::SUCCESS) { - UNIT_ASSERT_NO_DIFF(ReformatYson(result), ReformatYson(expected)); - } - } - - void RebootTablets(const TString& tableName) { - auto runtime = Kikimr.GetTestServer().GetRuntime(); - TActorId sender = runtime->AllocateEdgeActor(); - TVector<ui64> shards; - { - auto describeResult = DescribeTable(&Kikimr.GetTestServer(), sender, tableName); - for (auto shard : describeResult.GetPathDescription().GetColumnTableDescription().GetSharding().GetColumnShards()) { - shards.push_back(shard); - } - } - for (auto shard : shards) { - RebootTablet(*runtime, shard, sender); - } - } - }; -} - Y_UNIT_TEST_SUITE(KqpOlapScheme) { Y_UNIT_TEST(AddColumnLongPk) { diff --git a/ydb/core/load_test/events.h b/ydb/core/load_test/events.h index d698f8b244..d124d3fc01 100644 --- a/ydb/core/load_test/events.h +++ b/ydb/core/load_test/events.h @@ -1,5 +1,6 @@ #include <ydb/core/base/events.h> +#include <ydb/core/protos/kqp.pb.h> #include <ydb/core/protos/load_test.pb.h> #include <library/cpp/monlib/dynamic_counters/percentile/percentile_lg.h> diff --git a/ydb/core/load_test/ut_ycsb.cpp b/ydb/core/load_test/ut_ycsb.cpp index 0cffef3a0f..c8a790f43e 100644 --- a/ydb/core/load_test/ut_ycsb.cpp +++ b/ydb/core/load_test/ut_ycsb.cpp @@ -269,7 +269,7 @@ struct TTestHelper { record.AddColumns(column.GetId()); } - record.SetResultFormat(::NKikimrTxDataShard::EScanDataFormat::CELLVEC); + record.SetResultFormat(::NKikimrDataEvents::FORMAT_CELLVEC); return request; } diff --git a/ydb/core/load_test/ycsb/common.cpp b/ydb/core/load_test/ycsb/common.cpp index 612592ff2e..bdb1cfccad 100644 --- a/ydb/core/load_test/ycsb/common.cpp +++ b/ydb/core/load_test/ycsb/common.cpp @@ -16,7 +16,7 @@ namespace { class TReadIteratorScan : public TActorBootstrapped<TReadIteratorScan> { std::unique_ptr<TEvDataShard::TEvRead> Request; - const NKikimrTxDataShard::EScanDataFormat Format; + const NKikimrDataEvents::EDataFormat Format; const ui64 TabletId; const TActorId Parent; const TSubLoadId Id; @@ -114,7 +114,7 @@ private: return StopWithError(ctx, ss.Str()); } - if (Format != NKikimrTxDataShard::CELLVEC) { + if (Format != NKikimrDataEvents::FORMAT_CELLVEC) { return StopWithError(ctx, "Unsupported format"); } diff --git a/ydb/core/load_test/ycsb/kqp_select.cpp b/ydb/core/load_test/ycsb/kqp_select.cpp index c4c4400ddc..fe59f70966 100644 --- a/ydb/core/load_test/ycsb/kqp_select.cpp +++ b/ydb/core/load_test/ycsb/kqp_select.cpp @@ -390,7 +390,7 @@ private: TVector<TString> to = {TString("zzz")}; AddRangeQuery(*request, from, true, to, true); - record.SetResultFormat(::NKikimrTxDataShard::EScanDataFormat::CELLVEC); + record.SetResultFormat(::NKikimrDataEvents::FORMAT_CELLVEC); TSubLoadId subId(Id.Tag, SelfId(), ++LastSubTag); auto* actor = CreateReadIteratorScan(request.release(), TabletId, SelfId(), subId, sampleKeys); diff --git a/ydb/core/load_test/ycsb/test_load_read_iterator.cpp b/ydb/core/load_test/ycsb/test_load_read_iterator.cpp index 9e1dcfcfce..1916c41e53 100644 --- a/ydb/core/load_test/ycsb/test_load_read_iterator.cpp +++ b/ydb/core/load_test/ycsb/test_load_read_iterator.cpp @@ -31,7 +31,7 @@ const ui64 RECONNECT_LIMIT = 10; class TReadIteratorPoints : public TActorBootstrapped<TReadIteratorPoints> { const std::unique_ptr<const TEvDataShard::TEvRead> BaseRequest; - const NKikimrTxDataShard::EScanDataFormat Format; + const NKikimrDataEvents::EDataFormat Format; const ui64 TabletId; const TActorId Parent; const TSubLoadId Id; @@ -180,7 +180,7 @@ private: return StopWithError(ctx, ss.Str()); } - if (Format != NKikimrTxDataShard::CELLVEC) { + if (Format != NKikimrDataEvents::FORMAT_CELLVEC) { return StopWithError(ctx, "Unsupported format"); } @@ -428,7 +428,7 @@ private: TVector<TString> to = {TString("zzz")}; AddRangeQuery(*request, from, true, to, true); - record.SetResultFormat(::NKikimrTxDataShard::EScanDataFormat::CELLVEC); + record.SetResultFormat(::NKikimrDataEvents::FORMAT_CELLVEC); TSubLoadId subId(Id.Tag, SelfId(), ++LastSubTag); auto* actor = CreateReadIteratorScan(request.release(), TabletId, SelfId(), subId, sampleKeys); @@ -533,7 +533,7 @@ private: record.AddColumns(id); } - record.SetResultFormat(::NKikimrTxDataShard::EScanDataFormat::CELLVEC); + record.SetResultFormat(::NKikimrDataEvents::FORMAT_CELLVEC); TSubLoadId subId(Id.Tag, SelfId(), ++LastSubTag); auto* readActor = new TReadIteratorPoints( diff --git a/ydb/core/protos/CMakeLists.darwin-x86_64.txt b/ydb/core/protos/CMakeLists.darwin-x86_64.txt index 2f2a076636..78d3934e76 100644 --- a/ydb/core/protos/CMakeLists.darwin-x86_64.txt +++ b/ydb/core/protos/CMakeLists.darwin-x86_64.txt @@ -1675,9 +1675,9 @@ target_proto_messages(ydb-core-protos PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/protos/counters_mediator.proto ${CMAKE_SOURCE_DIR}/ydb/core/protos/counters.proto ${CMAKE_SOURCE_DIR}/ydb/core/protos/database_basic_sausage_metainfo.proto + ${CMAKE_SOURCE_DIR}/ydb/core/protos/data_events.proto ${CMAKE_SOURCE_DIR}/ydb/core/protos/datashard_load.proto ${CMAKE_SOURCE_DIR}/ydb/core/protos/drivemodel.proto - ${CMAKE_SOURCE_DIR}/ydb/core/protos/ev_write.proto ${CMAKE_SOURCE_DIR}/ydb/core/protos/export.proto ${CMAKE_SOURCE_DIR}/ydb/core/protos/external_sources.proto ${CMAKE_SOURCE_DIR}/ydb/core/protos/flat_tx_scheme.proto diff --git a/ydb/core/protos/CMakeLists.linux-aarch64.txt b/ydb/core/protos/CMakeLists.linux-aarch64.txt index cff2686486..d4896bb7ee 100644 --- a/ydb/core/protos/CMakeLists.linux-aarch64.txt +++ b/ydb/core/protos/CMakeLists.linux-aarch64.txt @@ -1676,9 +1676,9 @@ target_proto_messages(ydb-core-protos PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/protos/counters_mediator.proto ${CMAKE_SOURCE_DIR}/ydb/core/protos/counters.proto ${CMAKE_SOURCE_DIR}/ydb/core/protos/database_basic_sausage_metainfo.proto + ${CMAKE_SOURCE_DIR}/ydb/core/protos/data_events.proto ${CMAKE_SOURCE_DIR}/ydb/core/protos/datashard_load.proto ${CMAKE_SOURCE_DIR}/ydb/core/protos/drivemodel.proto - ${CMAKE_SOURCE_DIR}/ydb/core/protos/ev_write.proto ${CMAKE_SOURCE_DIR}/ydb/core/protos/export.proto ${CMAKE_SOURCE_DIR}/ydb/core/protos/external_sources.proto ${CMAKE_SOURCE_DIR}/ydb/core/protos/flat_tx_scheme.proto diff --git a/ydb/core/protos/CMakeLists.linux-x86_64.txt b/ydb/core/protos/CMakeLists.linux-x86_64.txt index cff2686486..d4896bb7ee 100644 --- a/ydb/core/protos/CMakeLists.linux-x86_64.txt +++ b/ydb/core/protos/CMakeLists.linux-x86_64.txt @@ -1676,9 +1676,9 @@ target_proto_messages(ydb-core-protos PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/protos/counters_mediator.proto ${CMAKE_SOURCE_DIR}/ydb/core/protos/counters.proto ${CMAKE_SOURCE_DIR}/ydb/core/protos/database_basic_sausage_metainfo.proto + ${CMAKE_SOURCE_DIR}/ydb/core/protos/data_events.proto ${CMAKE_SOURCE_DIR}/ydb/core/protos/datashard_load.proto ${CMAKE_SOURCE_DIR}/ydb/core/protos/drivemodel.proto - ${CMAKE_SOURCE_DIR}/ydb/core/protos/ev_write.proto ${CMAKE_SOURCE_DIR}/ydb/core/protos/export.proto ${CMAKE_SOURCE_DIR}/ydb/core/protos/external_sources.proto ${CMAKE_SOURCE_DIR}/ydb/core/protos/flat_tx_scheme.proto diff --git a/ydb/core/protos/CMakeLists.windows-x86_64.txt b/ydb/core/protos/CMakeLists.windows-x86_64.txt index 2f2a076636..78d3934e76 100644 --- a/ydb/core/protos/CMakeLists.windows-x86_64.txt +++ b/ydb/core/protos/CMakeLists.windows-x86_64.txt @@ -1675,9 +1675,9 @@ target_proto_messages(ydb-core-protos PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/protos/counters_mediator.proto ${CMAKE_SOURCE_DIR}/ydb/core/protos/counters.proto ${CMAKE_SOURCE_DIR}/ydb/core/protos/database_basic_sausage_metainfo.proto + ${CMAKE_SOURCE_DIR}/ydb/core/protos/data_events.proto ${CMAKE_SOURCE_DIR}/ydb/core/protos/datashard_load.proto ${CMAKE_SOURCE_DIR}/ydb/core/protos/drivemodel.proto - ${CMAKE_SOURCE_DIR}/ydb/core/protos/ev_write.proto ${CMAKE_SOURCE_DIR}/ydb/core/protos/export.proto ${CMAKE_SOURCE_DIR}/ydb/core/protos/external_sources.proto ${CMAKE_SOURCE_DIR}/ydb/core/protos/flat_tx_scheme.proto diff --git a/ydb/core/protos/data_events.proto b/ydb/core/protos/data_events.proto new file mode 100644 index 0000000000..09106a8bf6 --- /dev/null +++ b/ydb/core/protos/data_events.proto @@ -0,0 +1,123 @@ +import "library/cpp/actors/protos/actors.proto"; +import "ydb/core/protos/query_stats.proto"; +import "ydb/public/api/protos/ydb_issue_message.proto"; + +package NKikimrDataEvents; +option java_package = "ru.yandex.kikimr.proto"; + +message TLock { + optional fixed64 LockId = 1; + optional fixed64 DataShard = 2; + optional uint32 Generation = 3; + optional uint64 Counter = 4; + optional fixed64 SchemeShard = 5; + optional uint64 PathId = 6; + optional bool HasWrites = 7; +} + +message TKqpLocks { + repeated TLock Locks = 1; + repeated uint64 SendingShards = 2; // empty on Rollback + repeated uint64 ReceivingShards = 3; // empty on Rollback + + enum ELocksOp { + Unspecified = 0; + reserved 1; // Validate op is deprecated + Commit = 2; // Validate locks, commit buffered changes and erase locks + Rollback = 3; // Rollback buffered changes and erase locks + } + optional ELocksOp Op = 4; +} + +message TTableId { + optional uint64 OwnerId = 1; + optional uint64 TableId = 2; + optional uint64 SchemaVersion = 3; +} + +message TTabletInfo { + optional fixed64 TabletId = 1; + optional uint32 Generation = 2; + optional uint64 Step = 3; + optional NActorsProto.TActorId ActorId = 4; + optional bool IsFollower = 5; +} + +enum EDataFormat { + FORMAT_UNSPECIFIED = 0; + FORMAT_CELLVEC = 1; + FORMAT_ARROW = 2; +} + +message TEvWrite { + enum ETxMode { + MODE_UNSPECIFIED = 0; + MODE_PREPARE = 1; + MODE_VOLATILE_PREPARE = 2; + MODE_IMMEDIATE = 3; + } + + message TOperation { + enum EOperationType { + OPERATION_UNSPECIFIED = 0; + OPERATION_INSERT = 1; + OPERATION_DELETE = 2; + OPERATION_UPDATE = 3; + OPERATION_UPSERT = 4; + OPERATION_REPLACE = 5; + } + + optional EOperationType Type = 1; + optional TTableId TableId = 2; + repeated uint32 ColumnIds = 3 [packed = true]; + optional uint64 PayloadIndex = 4; + optional EDataFormat PayloadFormat = 5; + } + + // Transaction operations + repeated TOperation Operations = 1; + + // Transactions params + optional uint64 TxId = 2; + optional ETxMode TxMode = 3; + optional uint64 LockTxId = 4; + optional uint32 LockNodeId = 5; + optional TKqpLocks Locks = 6; + + // Other params + optional uint64 OverloadSubscribe = 7; +} + +message TEvWriteResult { + enum EStatus { + STATUS_UNSPECIFIED = 0; + STATUS_PREPARED = 1; + STATUS_COMPLETED = 2; + STATUS_ABORTED = 3; + STATUS_INTERNAL_ERROR = 4; + STATUS_OVERLOADED = 5; + STATUS_CANCELLED = 6; + STATUS_BAD_REQUEST = 7; + } + + // Status + optional EStatus Status = 1; + repeated Ydb.Issue.IssueMessage Issues = 2; + + // Transactions params + optional uint64 Origin = 3; + optional uint64 TxId = 4; + optional uint64 MinStep = 5; + optional uint64 MaxStep = 6; + repeated fixed64 DomainCoordinators = 7; + optional uint64 Step = 8; + optional uint64 OrderId = 9; + repeated TLock TxLocks = 10; + optional TTabletInfo TabletInfo = 11; + + // Other params + optional uint64 OverloadSubscribed = 12; + + // Statistics + optional NKikimrQueryStats.TTxStats TxStats = 13; +} diff --git a/ydb/core/protos/datashard_load.proto b/ydb/core/protos/datashard_load.proto index c6bdcf0b21..a35ab99364 100644 --- a/ydb/core/protos/datashard_load.proto +++ b/ydb/core/protos/datashard_load.proto @@ -1,6 +1,6 @@ option cc_enable_arenas = true; -import "ydb/core/protos/tx_datashard.proto"; +import "ydb/core/protos/data_events.proto"; package NKikimrDataShardLoad; option java_package = "ru.yandex.kikimr.proto"; @@ -47,7 +47,7 @@ message TEvYCSBTestLoadRequest { repeated uint32 Chunks = 4; // Specifies the format for result data in TEvReadResult - optional NKikimrTxDataShard.EScanDataFormat ResultFormat = 5; + optional NKikimrDataEvents.EDataFormat ResultFormat = 5; // special mode: actor reads RowCount rows again and again // implies no full scan and only first inflight will be used diff --git a/ydb/core/protos/ev_write.proto b/ydb/core/protos/ev_write.proto deleted file mode 100644 index 7f5d6e1e9d..0000000000 --- a/ydb/core/protos/ev_write.proto +++ /dev/null @@ -1,49 +0,0 @@ -import "ydb/core/protos/tx_datashard.proto"; -import "ydb/public/api/protos/ydb_issue_message.proto"; - -package NKikimrDataEvents; -option java_package = "ru.yandex.kikimr.proto"; - - -message TOperationData { - message TArrowData { - optional uint64 PayloadIndex = 1; - } - - repeated uint32 ColumnIds = 1 [packed = true]; - oneof Data { - TArrowData ArrowData = 900; - } -} - -message TEvWrite { - optional NKikimrTxDataShard.TTableId TableId = 1; - optional uint64 TxId = 2; - - oneof Operation { - TOperationData Replace = 900; - } -} - -message TEvWriteResult { - enum EOperationStatus { - UNSPECIFIED = 0; - PREPARED = 1; - COMPLETED = 2; - ABORTED = 3; - ERROR = 4; - OVERLOADED = 6; - BAD_REQUEST = 7; - } - - optional EOperationStatus Status = 1; - optional Ydb.Issue.IssueMessage IssueMessage = 2; - - optional uint64 Origin = 4; - optional uint64 TxId = 5; - - // For Tx planner - optional uint64 MinStep = 6; - optional uint64 MaxStep = 7; - repeated fixed64 DomainCoordinators = 8; -} diff --git a/ydb/core/protos/tx_datashard.proto b/ydb/core/protos/tx_datashard.proto index 8d1ad559b2..390a7914b2 100644 --- a/ydb/core/protos/tx_datashard.proto +++ b/ydb/core/protos/tx_datashard.proto @@ -5,6 +5,7 @@ import "ydb/core/protos/base.proto"; import "ydb/core/scheme/protos/type_info.proto"; import "ydb/core/scheme/protos/key_range.proto"; import "ydb/core/scheme/protos/pathid.proto"; +import "ydb/core/protos/data_events.proto"; import "ydb/core/protos/kqp.proto"; import "ydb/core/protos/ssa.proto"; import "ydb/core/protos/tablet.proto"; @@ -94,12 +95,6 @@ message TRWTransaction { message TROTransaction { } -message TTableId { - optional uint64 OwnerId = 1; - optional uint64 TableId = 2; - optional uint64 SchemaVersion = 3; -} - message TReadTableTransaction { message TColumn { optional uint32 Id = 1; @@ -108,7 +103,7 @@ message TReadTableTransaction { optional NKikimrProto.TTypeInfo TypeInfo = 4; } - optional TTableId TableId = 1; + optional NKikimrDataEvents.TTableId TableId = 1; repeated TColumn Columns = 2; optional NKikimrTx.TKeyRange Range = 3; optional uint32 ApiVersion = 4; @@ -116,23 +111,9 @@ message TReadTableTransaction { optional uint64 SnapshotTxId = 6; } -message TKqpLocks { - repeated TLock Locks = 1; - repeated uint64 SendingShards = 2; // empty on Rollback - repeated uint64 ReceivingShards = 3; // empty on Rollback - - enum ELocksOp { - Unspecified = 0; - reserved 1; // Validate op is deprecated - Commit = 2; // Validate locks, commit buffered changes and erase locks - Rollback = 3; // Rollback buffered changes and erase locks - } - optional ELocksOp Op = 4; -} - message TKqpValidateLocksResult { optional bool Success = 1; - repeated TLock BrokenLocks = 2; + repeated NKikimrDataEvents.TLock BrokenLocks = 2; }; enum EKqpTransactionType { @@ -141,12 +122,6 @@ enum EKqpTransactionType { KQP_TX_TYPE_SCAN = 2; } -enum EScanDataFormat { - UNSPECIFIED = 0; - CELLVEC = 1; - ARROW = 2; -} - message TKqpTransaction { message TColumnMeta { optional uint32 Id = 1; @@ -168,7 +143,7 @@ message TKqpTransaction { } message TTableMeta { - optional TTableId TableId = 1; + optional NKikimrDataEvents.TTableId TableId = 1; optional string TablePath = 2; optional uint64 SchemaVersion = 3; // reserved 4 @@ -230,7 +205,7 @@ message TKqpTransaction { optional uint64 ItemsLimit = 6; optional bool Reverse = 7; reserved 8; // optional bytes ProcessProgram = 8; - optional EScanDataFormat DataFormat = 9; + optional NKikimrDataEvents.EDataFormat DataFormat = 9; optional NKikimrSSA.TOlapProgram OlapProgram = 10; // Currently only for OLAP tables optional bool EnableShardsSequentialScan = 11; repeated TColumnMeta ResultColumns = 12; @@ -240,7 +215,7 @@ message TKqpTransaction { optional EKqpTransactionType Type = 1; repeated NYql.NDqProto.TDqTask Tasks = 2; - optional TKqpLocks Locks = 3; + optional NKikimrDataEvents.TKqpLocks Locks = 3; optional NYql.NDqProto.TComputeRuntimeSettings RuntimeSettings = 4; reserved 5; @@ -252,7 +227,7 @@ message TKqpTransaction { } message TEvKqpInputActorResultInfo { - repeated TLock Locks = 1; + repeated NKikimrDataEvents.TLock Locks = 1; } message TKqpReadRangesSourceSettings { @@ -270,7 +245,7 @@ message TKqpReadRangesSourceSettings { repeated uint32 KeyColumnTypes = 8; repeated NKikimrProto.TTypeInfo KeyColumnTypeInfos = 15; - optional EScanDataFormat DataFormat = 9; + optional NKikimrDataEvents.EDataFormat DataFormat = 9; optional NKikimrProto.TRowVersion Snapshot = 10; optional uint64 ShardIdHint = 11; optional bool Sorted = 12; @@ -361,13 +336,13 @@ message TSnapshotTransferInfo { message TReceiveSnapshot { optional uint64 TableId_Deprecated = 1; repeated TSnapshotTransferInfo ReceiveFrom = 2; - optional TTableId TableId = 3; + optional NKikimrDataEvents.TTableId TableId = 3; } message TSendSnapshot { optional uint64 TableId_Deprecated = 1; repeated TSnapshotTransferInfo SendTo = 2; - optional TTableId TableId = 3; + optional NKikimrDataEvents.TTableId TableId = 3; } message TSchemeOpSeqNo { @@ -529,7 +504,7 @@ message TDistributedEraseRS { message TCommitWritesTransaction { // TODO: add lock checking in the future - optional TTableId TableId = 1; + optional NKikimrDataEvents.TTableId TableId = 1; optional uint64 WriteTxId = 2; } @@ -602,24 +577,6 @@ message TError { optional bytes Reason = 3; } -message TLock { - optional fixed64 LockId = 1; - optional fixed64 DataShard = 2; - optional uint32 Generation = 3; - optional uint64 Counter = 4; - optional fixed64 SchemeShard = 5; - optional uint64 PathId = 6; - optional bool HasWrites = 7; -} - -message TTabletInfo { - optional fixed64 TabletId = 1; - optional uint32 Generation = 2; - optional uint64 Step = 3; - optional NActorsProto.TActorId ActorId = 4; - optional bool IsFollower = 5; -} - message TEvProposeTransactionResult { enum EStatus { PREPARED = 1; @@ -655,11 +612,11 @@ message TEvProposeTransactionResult { optional uint64 PrepareArriveTime = 12; optional uint64 ExecLatency = 13; optional uint64 ProposeLatency = 14; - repeated TLock TxLocks = 15; + repeated NKikimrDataEvents.TLock TxLocks = 15; optional uint64 ReadSize = 16; optional uint64 ReplySize = 17; repeated TReadSetInfo OutgoingReadSetInfo = 18; - optional TTabletInfo TabletInfo = 19; + optional NKikimrDataEvents.TTabletInfo TabletInfo = 19; // For read table tx result holds offsets in TxResult to cut response repeated uint32 RowOffsets = 20; repeated fixed64 DomainCoordinators = 21; @@ -1510,7 +1467,7 @@ message TEvKqpScan { reserved 14; optional uint64 ItemsLimit = 15; optional bool Reverse = 16; - optional EScanDataFormat DataFormat = 17; + optional NKikimrDataEvents.EDataFormat DataFormat = 17; optional NYql.NDqProto.EDqStatsMode StatsMode = 18; optional bytes OlapProgram = 19; optional NKikimrSchemeOp.EOlapProgramType OlapProgramType = 20; @@ -1626,7 +1583,7 @@ message TEvRead { // Only for internal usage we support reading shard system tables: // in this case owner is shard itself, TableId is LocalTid, SchemaVersion // must not be set - optional TTableId TableId = 2; + optional NKikimrDataEvents.TTableId TableId = 2; // A list of column ids that are returned in the result set. Empty columns can be used to Count(*) rows. // * In case of CellVec and empty columns: result will @@ -1648,7 +1605,7 @@ message TEvRead { optional uint32 LockNodeId = 11; // Specifies the format for result data in TEvReadResult - optional EScanDataFormat ResultFormat = 6; + optional NKikimrDataEvents.EDataFormat ResultFormat = 6; // Limits the total number of rows or bytes client currently can handle // MaxRows is hard limit: shard must never violate it @@ -1719,7 +1676,7 @@ message TEvReadResult { optional uint64 SeqNo = 2; optional TStatus Status = 3; - optional EScanDataFormat ResultFormat = 4; + optional NKikimrDataEvents.EDataFormat ResultFormat = 4; optional NKikimrProto.TRowVersion Snapshot = 5; optional bytes ContinuationToken = 6; @@ -1734,8 +1691,8 @@ message TEvReadResult { optional bool Finished = 9; - repeated TLock TxLocks = 10; - repeated TLock BrokenTxLocks = 11; + repeated NKikimrDataEvents.TLock TxLocks = 10; + repeated NKikimrDataEvents.TLock BrokenTxLocks = 11; optional uint64 RowCount = 12; @@ -1797,7 +1754,7 @@ message TEvApplyReplicationChanges { } // Table these changes are applied to - optional TTableId TableId = 1; + optional NKikimrDataEvents.TTableId TableId = 1; // Source of changes for deduplication and exactly-once processing. This // should be a source which guarantees monotonically increasing offsets, @@ -1832,7 +1789,7 @@ message TEvGetReplicationSourceOffsets { // Each read must have a unique (Sender, ReadId) pair for identification optional uint64 ReadId = 1; - optional TTableId TableId = 2; + optional NKikimrDataEvents.TTableId TableId = 2; // The tuple (SourceId, SplitKeyId) where the read should start, inclusive optional uint64 FromSourceId = 3; diff --git a/ydb/core/protos/ya.make b/ydb/core/protos/ya.make index d853fa2096..0e82d394e4 100644 --- a/ydb/core/protos/ya.make +++ b/ydb/core/protos/ya.make @@ -58,9 +58,9 @@ SRCS( counters_mediator.proto counters.proto database_basic_sausage_metainfo.proto + data_events.proto datashard_load.proto drivemodel.proto - ev_write.proto export.proto external_sources.proto flat_tx_scheme.proto diff --git a/ydb/core/tx/CMakeLists.darwin-x86_64.txt b/ydb/core/tx/CMakeLists.darwin-x86_64.txt index 856e2ca8e4..4f64f01014 100644 --- a/ydb/core/tx/CMakeLists.darwin-x86_64.txt +++ b/ydb/core/tx/CMakeLists.darwin-x86_64.txt @@ -10,8 +10,8 @@ add_subdirectory(balance_coverage) add_subdirectory(columnshard) add_subdirectory(conveyor) add_subdirectory(coordinator) +add_subdirectory(data_events) add_subdirectory(datashard) -add_subdirectory(ev_write) add_subdirectory(long_tx_service) add_subdirectory(mediator) add_subdirectory(program) diff --git a/ydb/core/tx/CMakeLists.linux-aarch64.txt b/ydb/core/tx/CMakeLists.linux-aarch64.txt index a3d4f7a61b..cc944bc9e2 100644 --- a/ydb/core/tx/CMakeLists.linux-aarch64.txt +++ b/ydb/core/tx/CMakeLists.linux-aarch64.txt @@ -10,8 +10,8 @@ add_subdirectory(balance_coverage) add_subdirectory(columnshard) add_subdirectory(conveyor) add_subdirectory(coordinator) +add_subdirectory(data_events) add_subdirectory(datashard) -add_subdirectory(ev_write) add_subdirectory(long_tx_service) add_subdirectory(mediator) add_subdirectory(program) diff --git a/ydb/core/tx/CMakeLists.linux-x86_64.txt b/ydb/core/tx/CMakeLists.linux-x86_64.txt index a3d4f7a61b..cc944bc9e2 100644 --- a/ydb/core/tx/CMakeLists.linux-x86_64.txt +++ b/ydb/core/tx/CMakeLists.linux-x86_64.txt @@ -10,8 +10,8 @@ add_subdirectory(balance_coverage) add_subdirectory(columnshard) add_subdirectory(conveyor) add_subdirectory(coordinator) +add_subdirectory(data_events) add_subdirectory(datashard) -add_subdirectory(ev_write) add_subdirectory(long_tx_service) add_subdirectory(mediator) add_subdirectory(program) diff --git a/ydb/core/tx/CMakeLists.windows-x86_64.txt b/ydb/core/tx/CMakeLists.windows-x86_64.txt index 856e2ca8e4..4f64f01014 100644 --- a/ydb/core/tx/CMakeLists.windows-x86_64.txt +++ b/ydb/core/tx/CMakeLists.windows-x86_64.txt @@ -10,8 +10,8 @@ add_subdirectory(balance_coverage) add_subdirectory(columnshard) add_subdirectory(conveyor) add_subdirectory(coordinator) +add_subdirectory(data_events) add_subdirectory(datashard) -add_subdirectory(ev_write) add_subdirectory(long_tx_service) add_subdirectory(mediator) add_subdirectory(program) diff --git a/ydb/core/tx/columnshard/columnshard.h b/ydb/core/tx/columnshard/columnshard.h index e356005d9a..e2b665e436 100644 --- a/ydb/core/tx/columnshard/columnshard.h +++ b/ydb/core/tx/columnshard/columnshard.h @@ -5,7 +5,7 @@ #include <ydb/core/tx/tx.h> #include <ydb/core/tx/message_seqno.h> #include <ydb/core/protos/tx_columnshard.pb.h> -#include <ydb/core/tx/ev_write/write_data.h> +#include <ydb/core/tx/data_events/write_data.h> #include <ydb/core/tx/long_tx_service/public/types.h> diff --git a/ydb/core/tx/columnshard/columnshard__scan.cpp b/ydb/core/tx/columnshard/columnshard__scan.cpp index d9e0cc8b13..93171e27b3 100644 --- a/ydb/core/tx/columnshard/columnshard__scan.cpp +++ b/ydb/core/tx/columnshard/columnshard__scan.cpp @@ -79,7 +79,7 @@ public: const std::shared_ptr<NOlap::IStoragesManager>& storagesManager, ui32 scanId, ui64 txId, ui32 scanGen, ui64 requestCookie, ui64 tabletId, TDuration timeout, std::vector<TTxScan::TReadMetadataPtr>&& readMetadataList, - NKikimrTxDataShard::EScanDataFormat dataFormat, const TScanCounters& scanCountersPool) + NKikimrDataEvents::EDataFormat dataFormat, const TScanCounters& scanCountersPool) : StoragesManager(storagesManager) , ColumnShardActorId(columnShardActorId) , ScanComputeActorId(scanComputeActorId) @@ -219,7 +219,7 @@ private: return false; } - if (ResultYqlSchema.empty() && DataFormat != NKikimrTxDataShard::EScanDataFormat::ARROW) { + if (ResultYqlSchema.empty() && DataFormat != NKikimrDataEvents::FORMAT_ARROW) { ResultYqlSchema = ReadMetadataRanges[ReadMetadataIndex]->GetResultYqlSchema(); } @@ -231,12 +231,12 @@ private: return true; } - ACFL_DEBUG("stage", "ready result")("iterator", ScanIterator->DebugString())("format", NKikimrTxDataShard::EScanDataFormat_Name(DataFormat)) + ACFL_DEBUG("stage", "ready result")("iterator", ScanIterator->DebugString())("format", NKikimrDataEvents::EDataFormat_Name(DataFormat)) ("columns", numColumns)("rows", numRows); switch (DataFormat) { - case NKikimrTxDataShard::EScanDataFormat::UNSPECIFIED: - case NKikimrTxDataShard::EScanDataFormat::CELLVEC: { + case NKikimrDataEvents::FORMAT_UNSPECIFIED: + case NKikimrDataEvents::FORMAT_CELLVEC: { MakeResult(INIT_BATCH_ROWS); NArrow::TArrowToYdbConverter batchConverter(ResultYqlSchema, *this); TString errStr; @@ -244,7 +244,7 @@ private: Y_ABORT_UNLESS(ok, "%s", errStr.c_str()); break; } - case NKikimrTxDataShard::EScanDataFormat::ARROW: { + case NKikimrDataEvents::FORMAT_ARROW: { MakeResult(0); Result->ArrowBatch = batch; Rows += batch->num_rows(); @@ -353,7 +353,7 @@ private: if (!Finished && !Result) { Result = MakeHolder<TEvKqpCompute::TEvScanData>(ScanId, ScanGen); if (reserveRows) { - Y_ABORT_UNLESS(DataFormat != NKikimrTxDataShard::EScanDataFormat::ARROW); + Y_ABORT_UNLESS(DataFormat != NKikimrDataEvents::FORMAT_ARROW); Result->Rows.reserve(reserveRows); } } @@ -528,7 +528,7 @@ private: const ui64 TxId; const ui32 ScanGen; const ui64 RequestCookie; - const NKikimrTxDataShard::EScanDataFormat DataFormat; + const NKikimrDataEvents::EDataFormat DataFormat; const ui64 TabletId; std::vector<NOlap::TReadMetadataBase::TConstPtr> ReadMetadataRanges; diff --git a/ydb/core/tx/columnshard/columnshard__write.cpp b/ydb/core/tx/columnshard/columnshard__write.cpp index acf7fe151a..8da097b232 100644 --- a/ydb/core/tx/columnshard/columnshard__write.cpp +++ b/ydb/core/tx/columnshard/columnshard__write.cpp @@ -5,7 +5,7 @@ #include "operations/write_data.h" #include <ydb/core/tx/conveyor/usage/service.h> -#include <ydb/core/tx/ev_write/events.h> +#include <ydb/core/tx/data_events/events.h> namespace NKikimr::NColumnShard { @@ -52,12 +52,14 @@ TColumnShard::EOverloadStatus TColumnShard::CheckOverloaded(const ui64 tableId) } void TColumnShard::Handle(TEvPrivate::TEvWriteBlobsResult::TPtr& ev, const TActorContext& ctx) { + NActors::TLogContextGuard gLogging = NActors::TLogContextBuilder::Build(NKikimrServices::TX_COLUMNSHARD)("tablet_id", TabletID())("event", "TEvWriteBlobsResult"); + auto& putResult = ev->Get()->GetPutResult(); OnYellowChannels(putResult); const auto& writeMeta = ev->Get()->GetWriteMeta(); if (!TablesManager.IsReadyForWrite(writeMeta.GetTableId())) { - AFL_ERROR(NKikimrServices::TX_COLUMNSHARD)("event", "absent_pathId")("path_id", writeMeta.GetTableId())("has_index", TablesManager.HasPrimaryIndex()); + ACFL_ERROR("event", "absent_pathId")("path_id", writeMeta.GetTableId())("has_index", TablesManager.HasPrimaryIndex()); IncCounter(COUNTER_WRITE_FAIL); auto result = std::make_unique<TEvColumnShard::TEvWriteResult>(TabletID(), writeMeta, NKikimrTxColumnShard::EResultStatus::ERROR); @@ -87,7 +89,7 @@ void TColumnShard::Handle(TEvPrivate::TEvWriteBlobsResult::TPtr& ev, const TActo } else { auto operation = OperationsManager->GetOperation((TWriteId)writeMeta.GetWriteId()); Y_ABORT_UNLESS(operation); - auto result = NEvents::TDataEvents::TEvWriteResult::BuildError(operation->GetTxId(), NKikimrDataEvents::TEvWriteResult::ERROR, "put data fails"); + auto result = NEvents::TDataEvents::TEvWriteResult::BuildError(operation->GetTxId(), NKikimrDataEvents::TEvWriteResult::STATUS_INTERNAL_ERROR, "put data fails"); ctx.Send(writeMeta.GetSource(), result.release()); } CSCounters.OnFailedWriteResponse(); @@ -183,52 +185,72 @@ void TColumnShard::Handle(TEvColumnShard::TEvWrite::TPtr& ev, const TActorContex } void TColumnShard::Handle(NEvents::TDataEvents::TEvWrite::TPtr& ev, const TActorContext& ctx) { + NActors::TLogContextGuard gLogging = NActors::TLogContextBuilder::Build(NKikimrServices::TX_COLUMNSHARD)("tablet_id", TabletID())("event", "TEvWrite"); + const auto& record = ev->Get()->Record; - const auto& tableId = record.GetTableId().GetTableId(); const ui64 txId = ev->Get()->GetTxId(); const auto source = ev->Sender; - if (!record.GetTableId().HasSchemaVersion()) { + if (record.GetOperations().size() != 1) { + IncCounter(COUNTER_WRITE_FAIL); + auto result = NEvents::TDataEvents::TEvWriteResult::BuildError(txId, NKikimrDataEvents::TEvWriteResult::STATUS_BAD_REQUEST, "only single operation is supported"); + ctx.Send(source, result.release()); + return; + } + + const auto& operation = record.GetOperations()[0]; + + if (operation.GetType() != NKikimrDataEvents::TEvWrite::TOperation::OPERATION_REPLACE) { IncCounter(COUNTER_WRITE_FAIL); - auto result = NEvents::TDataEvents::TEvWriteResult::BuildError(txId, NKikimrDataEvents::TEvWriteResult::BAD_REQUEST, "schema version not set"); + auto result = NEvents::TDataEvents::TEvWriteResult::BuildError(txId, NKikimrDataEvents::TEvWriteResult::STATUS_BAD_REQUEST, "only REPLACE operation is supported"); ctx.Send(source, result.release()); return; } - auto schema = TablesManager.GetPrimaryIndex()->GetVersionedIndex().GetSchema(record.GetTableId().GetSchemaVersion()); + if (!operation.GetTableId().HasSchemaVersion()) { + IncCounter(COUNTER_WRITE_FAIL); + auto result = NEvents::TDataEvents::TEvWriteResult::BuildError(txId, NKikimrDataEvents::TEvWriteResult::STATUS_BAD_REQUEST, "schema version not set"); + ctx.Send(source, result.release()); + return; + } + + auto schema = TablesManager.GetPrimaryIndex()->GetVersionedIndex().GetSchema(operation.GetTableId().GetSchemaVersion()); if (!schema) { IncCounter(COUNTER_WRITE_FAIL); - auto result = NEvents::TDataEvents::TEvWriteResult::BuildError(txId, NKikimrDataEvents::TEvWriteResult::BAD_REQUEST, "unknown schema version"); + auto result = NEvents::TDataEvents::TEvWriteResult::BuildError(txId, NKikimrDataEvents::TEvWriteResult::STATUS_BAD_REQUEST, "unknown schema version"); ctx.Send(source, result.release()); return; } + const auto tableId = operation.GetTableId().GetTableId(); + if (!TablesManager.IsReadyForWrite(tableId)) { IncCounter(COUNTER_WRITE_FAIL); - auto result = NEvents::TDataEvents::TEvWriteResult::BuildError(txId, NKikimrDataEvents::TEvWriteResult::ERROR, "table not writable"); + auto result = NEvents::TDataEvents::TEvWriteResult::BuildError(txId, NKikimrDataEvents::TEvWriteResult::STATUS_INTERNAL_ERROR, "table not writable"); ctx.Send(source, result.release()); return; } auto arrowData = std::make_shared<TArrowData>(schema); - if (!arrowData->Parse(record.GetReplace(), TPayloadHelper<NEvents::TDataEvents::TEvWrite>(*ev->Get()))) { + if (!arrowData->Parse(operation, NEvWrite::TPayloadHelper<NEvents::TDataEvents::TEvWrite>(*ev->Get()))) { IncCounter(COUNTER_WRITE_FAIL); - auto result = NEvents::TDataEvents::TEvWriteResult::BuildError(txId, NKikimrDataEvents::TEvWriteResult::BAD_REQUEST, "parsing data error"); + auto result = NEvents::TDataEvents::TEvWriteResult::BuildError(txId, NKikimrDataEvents::TEvWriteResult::STATUS_BAD_REQUEST, "parsing data error"); ctx.Send(source, result.release()); } auto overloadStatus = CheckOverloaded(tableId); if (overloadStatus != EOverloadStatus::None) { NEvWrite::TWriteData writeData(NEvWrite::TWriteMeta(0, tableId, source), arrowData); - std::unique_ptr<NActors::IEventBase> result = NEvents::TDataEvents::TEvWriteResult::BuildError(txId, NKikimrDataEvents::TEvWriteResult::OVERLOADED, "overload data error"); + std::unique_ptr<NActors::IEventBase> result = NEvents::TDataEvents::TEvWriteResult::BuildError(txId, NKikimrDataEvents::TEvWriteResult::STATUS_OVERLOADED, "overload data error"); OverloadWriteFail(overloadStatus, writeData, std::move(result), ctx); return; } auto wg = WritesMonitor.RegisterWrite(arrowData->GetSize()); - auto operation = OperationsManager->RegisterOperation(txId); - Y_ABORT_UNLESS(operation); - operation->Start(*this, tableId, arrowData, source, ctx); + + auto writeOperation = OperationsManager->RegisterOperation(txId); + Y_ABORT_UNLESS(writeOperation); + writeOperation->Start(*this, tableId, arrowData, source, ctx); } } diff --git a/ydb/core/tx/columnshard/columnshard_impl.h b/ydb/core/tx/columnshard/columnshard_impl.h index 68dfb97945..779a37ed01 100644 --- a/ydb/core/tx/columnshard/columnshard_impl.h +++ b/ydb/core/tx/columnshard/columnshard_impl.h @@ -20,7 +20,7 @@ #include <ydb/core/tablet/tablet_pipe_client_cache.h> #include <ydb/core/tablet_flat/flat_cxx_database.h> #include <ydb/core/tablet_flat/tablet_flat_executed.h> -#include <ydb/core/tx/ev_write/events.h> +#include <ydb/core/tx/data_events/events.h> #include <ydb/core/tx/tiering/common.h> #include <ydb/core/tx/tiering/manager.h> #include <ydb/core/tx/time_cast/time_cast.h> diff --git a/ydb/core/tx/columnshard/columnshard_private_events.h b/ydb/core/tx/columnshard/columnshard_private_events.h index 2525c99b35..9161d8ca1c 100644 --- a/ydb/core/tx/columnshard/columnshard_private_events.h +++ b/ydb/core/tx/columnshard/columnshard_private_events.h @@ -8,7 +8,7 @@ #include <ydb/core/tx/columnshard/engines/column_engine.h> #include <ydb/core/tx/columnshard/normalizer/abstract/abstract.h> #include <ydb/core/tx/columnshard/engines/writer/write_controller.h> -#include <ydb/core/tx/ev_write/write_data.h> +#include <ydb/core/tx/data_events/write_data.h> #include <ydb/core/formats/arrow/special_keys.h> namespace NKikimr::NColumnShard { diff --git a/ydb/core/tx/columnshard/columnshard_ut_common.cpp b/ydb/core/tx/columnshard/columnshard_ut_common.cpp index a1d30787f9..9ad4f4b42f 100644 --- a/ydb/core/tx/columnshard/columnshard_ut_common.cpp +++ b/ydb/core/tx/columnshard/columnshard_ut_common.cpp @@ -87,7 +87,7 @@ void PlanWriteTx(TTestBasicRuntime& runtime, TActorId& sender, NOlap::TSnapshot auto ev = runtime.GrabEdgeEvent<NEvents::TDataEvents::TEvWriteResult>(sender); const auto& res = ev->Get()->Record; UNIT_ASSERT_EQUAL(res.GetTxId(), snap.GetTxId()); - UNIT_ASSERT_EQUAL(res.GetStatus(), NKikimrDataEvents::TEvWriteResult::COMPLETED); + UNIT_ASSERT_EQUAL(res.GetStatus(), NKikimrDataEvents::TEvWriteResult::STATUS_COMPLETED); } } @@ -194,7 +194,7 @@ void ScanIndexStats(TTestBasicRuntime& runtime, TActorId& sender, const std::vec record.MutableSnapshot()->SetStep(snap.GetPlanStep()); record.MutableSnapshot()->SetTxId(snap.GetTxId()); - record.SetDataFormat(NKikimrTxDataShard::EScanDataFormat::ARROW); + record.SetDataFormat(NKikimrDataEvents::FORMAT_ARROW); ForwardToTablet(runtime, TTestTxConfig::TxTablet0, sender, scan.release()); } diff --git a/ydb/core/tx/columnshard/columnshard_ut_common.h b/ydb/core/tx/columnshard/columnshard_ut_common.h index b69df6a03e..0ea35595f3 100644 --- a/ydb/core/tx/columnshard/columnshard_ut_common.h +++ b/ydb/core/tx/columnshard/columnshard_ut_common.h @@ -535,29 +535,6 @@ namespace NKikimr::NColumnShard { bool InStore = true; }; - class TArrowDataConstructor : public NKikimr::NEvents::IDataConstructor { - std::vector<std::pair<TString, NScheme::TTypeInfo>> YdbSchema; - ui64 Index; - - public: - TArrowDataConstructor(const std::vector<std::pair<TString, NScheme::TTypeInfo>>& ydbSchema, const ui64 idx) - : YdbSchema(ydbSchema) - , Index(idx) - { - } - - void Serialize(NKikimrDataEvents::TOperationData& proto) const override { - for (ui32 i = 0; i < YdbSchema.size(); ++i) { - proto.AddColumnIds(i + 1); - } - proto.MutableArrowData()->SetPayloadIndex(Index); - } - - ui64 GetSchemaVersion() const override { - return 1; - } - }; - void SetupSchema(TTestBasicRuntime& runtime, TActorId& sender, ui64 pathId, const TestTableDescription& table = {}, TString codec = "none"); diff --git a/ydb/core/tx/columnshard/engines/writer/indexed_blob_constructor.h b/ydb/core/tx/columnshard/engines/writer/indexed_blob_constructor.h index 5261ee7eef..87a165c29d 100644 --- a/ydb/core/tx/columnshard/engines/writer/indexed_blob_constructor.h +++ b/ydb/core/tx/columnshard/engines/writer/indexed_blob_constructor.h @@ -3,7 +3,7 @@ #include "blob_constructor.h" #include "write_controller.h" -#include <ydb/core/tx/ev_write/write_data.h> +#include <ydb/core/tx/data_events/write_data.h> #include <ydb/core/tx/columnshard/blobs_action/abstract/write.h> #include <ydb/core/tx/columnshard/counters/common/object_counter.h> #include <ydb/core/tx/columnshard/engines/portion_info.h> diff --git a/ydb/core/tx/columnshard/operations/CMakeLists.darwin-x86_64.txt b/ydb/core/tx/columnshard/operations/CMakeLists.darwin-x86_64.txt index 7972cccf59..0027f00a3c 100644 --- a/ydb/core/tx/columnshard/operations/CMakeLists.darwin-x86_64.txt +++ b/ydb/core/tx/columnshard/operations/CMakeLists.darwin-x86_64.txt @@ -12,7 +12,7 @@ target_link_libraries(tx-columnshard-operations PUBLIC contrib-libs-cxxsupp yutil ydb-core-protos - core-tx-ev_write + core-tx-data_events ydb-services-metadata ) target_sources(tx-columnshard-operations PRIVATE diff --git a/ydb/core/tx/columnshard/operations/CMakeLists.linux-aarch64.txt b/ydb/core/tx/columnshard/operations/CMakeLists.linux-aarch64.txt index a97ebc42a1..b9ac1e90c3 100644 --- a/ydb/core/tx/columnshard/operations/CMakeLists.linux-aarch64.txt +++ b/ydb/core/tx/columnshard/operations/CMakeLists.linux-aarch64.txt @@ -13,7 +13,7 @@ target_link_libraries(tx-columnshard-operations PUBLIC contrib-libs-cxxsupp yutil ydb-core-protos - core-tx-ev_write + core-tx-data_events ydb-services-metadata ) target_sources(tx-columnshard-operations PRIVATE diff --git a/ydb/core/tx/columnshard/operations/CMakeLists.linux-x86_64.txt b/ydb/core/tx/columnshard/operations/CMakeLists.linux-x86_64.txt index a97ebc42a1..b9ac1e90c3 100644 --- a/ydb/core/tx/columnshard/operations/CMakeLists.linux-x86_64.txt +++ b/ydb/core/tx/columnshard/operations/CMakeLists.linux-x86_64.txt @@ -13,7 +13,7 @@ target_link_libraries(tx-columnshard-operations PUBLIC contrib-libs-cxxsupp yutil ydb-core-protos - core-tx-ev_write + core-tx-data_events ydb-services-metadata ) target_sources(tx-columnshard-operations PRIVATE diff --git a/ydb/core/tx/columnshard/operations/CMakeLists.windows-x86_64.txt b/ydb/core/tx/columnshard/operations/CMakeLists.windows-x86_64.txt index 7972cccf59..0027f00a3c 100644 --- a/ydb/core/tx/columnshard/operations/CMakeLists.windows-x86_64.txt +++ b/ydb/core/tx/columnshard/operations/CMakeLists.windows-x86_64.txt @@ -12,7 +12,7 @@ target_link_libraries(tx-columnshard-operations PUBLIC contrib-libs-cxxsupp yutil ydb-core-protos - core-tx-ev_write + core-tx-data_events ydb-services-metadata ) target_sources(tx-columnshard-operations PRIVATE diff --git a/ydb/core/tx/columnshard/operations/slice_builder.h b/ydb/core/tx/columnshard/operations/slice_builder.h index efd26eacb9..0a45ad16c9 100644 --- a/ydb/core/tx/columnshard/operations/slice_builder.h +++ b/ydb/core/tx/columnshard/operations/slice_builder.h @@ -2,7 +2,7 @@ #include <ydb/core/tx/conveyor/usage/abstract.h> #include <ydb/core/formats/arrow/size_calcer.h> #include <ydb/core/tx/columnshard/blobs_action/abstract/write.h> -#include <ydb/core/tx/ev_write/write_data.h> +#include <ydb/core/tx/data_events/write_data.h> namespace NKikimr::NOlap { diff --git a/ydb/core/tx/columnshard/operations/write.h b/ydb/core/tx/columnshard/operations/write.h index 7d0dd9ade2..4107b7e301 100644 --- a/ydb/core/tx/columnshard/operations/write.h +++ b/ydb/core/tx/columnshard/operations/write.h @@ -1,6 +1,6 @@ #pragma once -#include <ydb/core/tx/ev_write/write_data.h> +#include <ydb/core/tx/data_events/write_data.h> #include <ydb/core/tx/columnshard/common/snapshot.h> #include <ydb/core/tx/columnshard/engines/defs.h> #include <ydb/core/protos/tx_columnshard.pb.h> diff --git a/ydb/core/tx/columnshard/operations/write_data.cpp b/ydb/core/tx/columnshard/operations/write_data.cpp index 151cdcd213..b81b3839d7 100644 --- a/ydb/core/tx/columnshard/operations/write_data.cpp +++ b/ydb/core/tx/columnshard/operations/write_data.cpp @@ -5,8 +5,13 @@ namespace NKikimr::NColumnShard { -bool TArrowData::Parse(const NKikimrDataEvents::TOperationData& proto, const IPayloadData& payload) { - IncomingData = payload.GetDataFromPayload(proto.GetArrowData().GetPayloadIndex()); +bool TArrowData::Parse(const NKikimrDataEvents::TEvWrite_TOperation& proto, const NEvWrite::IPayloadData& payload) { + if(proto.GetPayloadFormat() != NKikimrDataEvents::FORMAT_ARROW) + { + AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "invalid_payload_format")("payload_format", (ui64)proto.GetPayloadFormat()); + return false; + } + IncomingData = payload.GetDataFromPayload(proto.GetPayloadIndex()); std::vector<ui32> columns; for (auto&& columnId : proto.GetColumnIds()) { diff --git a/ydb/core/tx/columnshard/operations/write_data.h b/ydb/core/tx/columnshard/operations/write_data.h index e036747bcd..b4fcb5e68e 100644 --- a/ydb/core/tx/columnshard/operations/write_data.h +++ b/ydb/core/tx/columnshard/operations/write_data.h @@ -1,43 +1,15 @@ #pragma once -#include <ydb/core/tx/ev_write/write_data.h> +#include <ydb/core/tx/data_events/write_data.h> +#include <ydb/core/tx/data_events/payload_helper.h> #include <ydb/core/tx/columnshard/common/snapshot.h> #include <ydb/core/tx/columnshard/engines/scheme/abstract_scheme.h> #include <ydb/core/tx/columnshard/engines/scheme/filtered_scheme.h> -#include <ydb/core/protos/ev_write.pb.h> +#include <ydb/core/protos/data_events.pb.h> namespace NKikimr::NColumnShard { -class IPayloadData { -public: - virtual TString GetDataFromPayload(const ui64 index) const = 0; - virtual ui64 AddDataToPayload(TString&& blobData) = 0; - virtual ~IPayloadData() {} -}; - - -template <class TEvent> -class TPayloadHelper : public IPayloadData { - TEvent& Event; -public: - TPayloadHelper(TEvent& ev) - : Event(ev) {} - - TString GetDataFromPayload(const ui64 index) const override { - TRope rope = Event.GetPayload(index); - TString data = TString::Uninitialized(rope.GetSize()); - rope.Begin().ExtractPlainDataAndAdvance(data.Detach(), data.size()); - return data; - } - - ui64 AddDataToPayload(TString&& blobData) override { - TRope rope; - rope.Insert(rope.End(), TRope(blobData)); - return Event.AddPayload(std::move(rope)); - } -}; - class TArrowData : public NEvWrite::IDataContainer { private: std::optional<ui64> OriginalDataSize; @@ -46,7 +18,7 @@ public: : IndexSchema(schema) {} - bool Parse(const NKikimrDataEvents::TOperationData& proto, const IPayloadData& payload); + bool Parse(const NKikimrDataEvents::TEvWrite::TOperation& proto, const NKikimr::NEvWrite::IPayloadData& payload); virtual std::shared_ptr<arrow::RecordBatch> ExtractBatch() override; ui64 GetSchemaVersion() const override; ui64 GetSize() const override { diff --git a/ydb/core/tx/columnshard/operations/ya.make b/ydb/core/tx/columnshard/operations/ya.make index ebc4e740c1..200a78be15 100644 --- a/ydb/core/tx/columnshard/operations/ya.make +++ b/ydb/core/tx/columnshard/operations/ya.make @@ -8,7 +8,7 @@ SRCS( PEERDIR( ydb/core/protos - ydb/core/tx/ev_write + ydb/core/tx/data_events ydb/services/metadata ) diff --git a/ydb/core/tx/columnshard/tx_controller.h b/ydb/core/tx/columnshard/tx_controller.h index 1f2da419ac..cadeaf7ccb 100644 --- a/ydb/core/tx/columnshard/tx_controller.h +++ b/ydb/core/tx/columnshard/tx_controller.h @@ -3,7 +3,7 @@ #include "columnshard_schema.h" #include <ydb/core/tablet_flat/tablet_flat_executed.h> -#include <ydb/core/tx/ev_write/events.h> +#include <ydb/core/tx/data_events/events.h> namespace NKikimr::NColumnShard { diff --git a/ydb/core/tx/columnshard/ut_rw/ut_columnshard_read_write.cpp b/ydb/core/tx/columnshard/ut_rw/ut_columnshard_read_write.cpp index 1c2b0f39d8..65e4aec39f 100644 --- a/ydb/core/tx/columnshard/ut_rw/ut_columnshard_read_write.cpp +++ b/ydb/core/tx/columnshard/ut_rw/ut_columnshard_read_write.cpp @@ -1872,8 +1872,8 @@ Y_UNIT_TEST_SUITE(EvWrite) { UNIT_ASSERT(blobData.size() < TLimits::GetMaxBlobSize()); auto evWrite = std::make_unique<NKikimr::NEvents::TDataEvents::TEvWrite>(txId); - auto dataPtr = std::make_shared<TArrowDataConstructor>(schema, TPayloadHelper<NKikimr::NEvents::TDataEvents::TEvWrite>(*evWrite).AddDataToPayload(std::move(blobData))); - evWrite->AddReplaceOp(tableId, dataPtr); + ui64 payloadIndex = NEvWrite::TPayloadHelper<NKikimr::NEvents::TDataEvents::TEvWrite>(*evWrite).AddDataToPayload(std::move(blobData)); + evWrite->AddOperation(NKikimrDataEvents::TEvWrite::TOperation::OPERATION_REPLACE, tableId, 1, schema, payloadIndex, NKikimrDataEvents::FORMAT_ARROW); TActorId sender = runtime.AllocateEdgeActor(); ForwardToTablet(runtime, TTestTxConfig::TxTablet0, sender, evWrite.release()); @@ -1882,7 +1882,7 @@ Y_UNIT_TEST_SUITE(EvWrite) { TAutoPtr<NActors::IEventHandle> handle; auto event = runtime.GrabEdgeEvent<NKikimr::NEvents::TDataEvents::TEvWriteResult>(handle); UNIT_ASSERT(event); - UNIT_ASSERT_VALUES_EQUAL((ui64)event->Record.GetStatus(), (ui64)NKikimrDataEvents::TEvWriteResult::PREPARED); + UNIT_ASSERT_VALUES_EQUAL((ui64)event->Record.GetStatus(), (ui64)NKikimrDataEvents::TEvWriteResult::STATUS_PREPARED); auto readResult = ReadAllAsBatch(runtime, tableId, NOlap::TSnapshot(10, txId), schema); UNIT_ASSERT_VALUES_EQUAL(readResult->num_rows(), 0); @@ -1917,8 +1917,8 @@ Y_UNIT_TEST_SUITE(EvWrite) { UNIT_ASSERT(blobData.size() < TLimits::GetMaxBlobSize()); auto evWrite = std::make_unique<NKikimr::NEvents::TDataEvents::TEvWrite>(txId); - auto dataPtr = std::make_shared<TArrowDataConstructor>(schema, TPayloadHelper<NKikimr::NEvents::TDataEvents::TEvWrite>(*evWrite).AddDataToPayload(std::move(blobData))); - evWrite->AddReplaceOp(tableId, dataPtr); + ui64 payloadIndex = NEvWrite::TPayloadHelper<NKikimr::NEvents::TDataEvents::TEvWrite>(*evWrite).AddDataToPayload(std::move(blobData)); + evWrite->AddOperation(NKikimrDataEvents::TEvWrite::TOperation::OPERATION_REPLACE, tableId, 1, schema, payloadIndex, NKikimrDataEvents::FORMAT_ARROW); TActorId sender = runtime.AllocateEdgeActor(); ForwardToTablet(runtime, TTestTxConfig::TxTablet0, sender, evWrite.release()); @@ -1928,7 +1928,7 @@ Y_UNIT_TEST_SUITE(EvWrite) { TAutoPtr<NActors::IEventHandle> handle; auto event = runtime.GrabEdgeEvent<NKikimr::NEvents::TDataEvents::TEvWriteResult>(handle); UNIT_ASSERT(event); - UNIT_ASSERT_VALUES_EQUAL((ui64)event->Record.GetStatus(), (ui64)NKikimrDataEvents::TEvWriteResult::PREPARED); + UNIT_ASSERT_VALUES_EQUAL((ui64)event->Record.GetStatus(), (ui64)NKikimrDataEvents::TEvWriteResult::STATUS_PREPARED); outdatedStep = event->Record.GetMaxStep() + 1; PlanWriteTx(runtime, sender, NOlap::TSnapshot(outdatedStep, txId + 1), false); @@ -1961,8 +1961,8 @@ Y_UNIT_TEST_SUITE(EvWrite) { UNIT_ASSERT(blobData.size() > TLimits::GetMaxBlobSize()); auto evWrite = std::make_unique<NKikimr::NEvents::TDataEvents::TEvWrite>(txId); - auto dataPtr = std::make_shared<TArrowDataConstructor>(schema, TPayloadHelper<NKikimr::NEvents::TDataEvents::TEvWrite>(*evWrite).AddDataToPayload(std::move(blobData))); - evWrite->AddReplaceOp(tableId, dataPtr); + ui64 payloadIndex = NEvWrite::TPayloadHelper<NKikimr::NEvents::TDataEvents::TEvWrite>(*evWrite).AddDataToPayload(std::move(blobData)); + evWrite->AddOperation(NKikimrDataEvents::TEvWrite::TOperation::OPERATION_REPLACE, tableId, 1, schema, payloadIndex, NKikimrDataEvents::FORMAT_ARROW); TActorId sender = runtime.AllocateEdgeActor(); ForwardToTablet(runtime, TTestTxConfig::TxTablet0, sender, evWrite.release()); @@ -1971,7 +1971,7 @@ Y_UNIT_TEST_SUITE(EvWrite) { TAutoPtr<NActors::IEventHandle> handle; auto event = runtime.GrabEdgeEvent<NKikimr::NEvents::TDataEvents::TEvWriteResult>(handle); UNIT_ASSERT(event); - UNIT_ASSERT_VALUES_EQUAL((ui64)event->Record.GetStatus(), (ui64)NKikimrDataEvents::TEvWriteResult::PREPARED); + UNIT_ASSERT_VALUES_EQUAL((ui64)event->Record.GetStatus(), (ui64)NKikimrDataEvents::TEvWriteResult::STATUS_PREPARED); PlanWriteTx(runtime, sender, NOlap::TSnapshot(11, txId)); } diff --git a/ydb/core/tx/columnshard/ut_rw/ut_normalizer.cpp b/ydb/core/tx/columnshard/ut_rw/ut_normalizer.cpp index 4be1264fdb..79407ee1d0 100644 --- a/ydb/core/tx/columnshard/ut_rw/ut_normalizer.cpp +++ b/ydb/core/tx/columnshard/ut_rw/ut_normalizer.cpp @@ -243,8 +243,7 @@ Y_UNIT_TEST_SUITE(Normalizers) { TString blobData = NArrow::SerializeBatchNoCompression(batch); auto evWrite = std::make_unique<NKikimr::NEvents::TDataEvents::TEvWrite>(txId); - auto dataPtr = std::make_shared<TArrowDataConstructor>(schema, TPayloadHelper<NKikimr::NEvents::TDataEvents::TEvWrite>(*evWrite).AddDataToPayload(std::move(blobData))); - evWrite->AddReplaceOp(tableId, dataPtr); + ui64 payloadIndex = NEvWrite::TPayloadHelper<NKikimr::NEvents::TDataEvents::TEvWrite>(*evWrite).AddDataToPayload(std::move(blobData)); evWrite->AddOperation(NKikimrDataEvents::TEvWrite::TOperation::OPERATION_REPLACE, tableId, 1, schema, payloadIndex, NKikimrDataEvents::FORMAT_ARROW); TActorId sender = runtime.AllocateEdgeActor(); ForwardToTablet(runtime, TTestTxConfig::TxTablet0, sender, evWrite.release()); @@ -252,7 +251,7 @@ Y_UNIT_TEST_SUITE(Normalizers) { TAutoPtr<NActors::IEventHandle> handle; auto event = runtime.GrabEdgeEvent<NKikimr::NEvents::TDataEvents::TEvWriteResult>(handle); UNIT_ASSERT(event); - UNIT_ASSERT_VALUES_EQUAL((ui64)event->Record.GetStatus(), (ui64)NKikimrDataEvents::TEvWriteResult::PREPARED); + UNIT_ASSERT_VALUES_EQUAL((ui64)event->Record.GetStatus(), (ui64)NKikimrDataEvents::TEvWriteResult::STATUS_PREPARED); PlanWriteTx(runtime, sender, NOlap::TSnapshot(11, txId)); } diff --git a/ydb/core/tx/ev_write/CMakeLists.darwin-x86_64.txt b/ydb/core/tx/data_events/CMakeLists.darwin-x86_64.txt index 416a6d8b02..70995c1511 100644 --- a/ydb/core/tx/ev_write/CMakeLists.darwin-x86_64.txt +++ b/ydb/core/tx/data_events/CMakeLists.darwin-x86_64.txt @@ -7,8 +7,8 @@ -add_library(core-tx-ev_write) -target_link_libraries(core-tx-ev_write PUBLIC +add_library(core-tx-data_events) +target_link_libraries(core-tx-data_events PUBLIC contrib-libs-cxxsupp yutil ydb-core-tablet @@ -29,8 +29,8 @@ target_link_libraries(core-tx-ev_write PUBLIC ydb-core-tablet_flat blobstorage-vdisk-protos ) -target_sources(core-tx-ev_write PRIVATE - ${CMAKE_SOURCE_DIR}/ydb/core/tx/ev_write/shard_writer.cpp - ${CMAKE_SOURCE_DIR}/ydb/core/tx/ev_write/shards_splitter.cpp - ${CMAKE_SOURCE_DIR}/ydb/core/tx/ev_write/write_data.cpp +target_sources(core-tx-data_events PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/tx/data_events/shard_writer.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/tx/data_events/shards_splitter.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/tx/data_events/write_data.cpp ) diff --git a/ydb/core/tx/ev_write/CMakeLists.linux-aarch64.txt b/ydb/core/tx/data_events/CMakeLists.linux-aarch64.txt index df288b40d6..33f97931c3 100644 --- a/ydb/core/tx/ev_write/CMakeLists.linux-aarch64.txt +++ b/ydb/core/tx/data_events/CMakeLists.linux-aarch64.txt @@ -7,8 +7,8 @@ -add_library(core-tx-ev_write) -target_link_libraries(core-tx-ev_write PUBLIC +add_library(core-tx-data_events) +target_link_libraries(core-tx-data_events PUBLIC contrib-libs-linux-headers contrib-libs-cxxsupp yutil @@ -30,8 +30,8 @@ target_link_libraries(core-tx-ev_write PUBLIC ydb-core-tablet_flat blobstorage-vdisk-protos ) -target_sources(core-tx-ev_write PRIVATE - ${CMAKE_SOURCE_DIR}/ydb/core/tx/ev_write/shard_writer.cpp - ${CMAKE_SOURCE_DIR}/ydb/core/tx/ev_write/shards_splitter.cpp - ${CMAKE_SOURCE_DIR}/ydb/core/tx/ev_write/write_data.cpp +target_sources(core-tx-data_events PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/tx/data_events/shard_writer.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/tx/data_events/shards_splitter.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/tx/data_events/write_data.cpp ) diff --git a/ydb/core/tx/ev_write/CMakeLists.linux-x86_64.txt b/ydb/core/tx/data_events/CMakeLists.linux-x86_64.txt index df288b40d6..33f97931c3 100644 --- a/ydb/core/tx/ev_write/CMakeLists.linux-x86_64.txt +++ b/ydb/core/tx/data_events/CMakeLists.linux-x86_64.txt @@ -7,8 +7,8 @@ -add_library(core-tx-ev_write) -target_link_libraries(core-tx-ev_write PUBLIC +add_library(core-tx-data_events) +target_link_libraries(core-tx-data_events PUBLIC contrib-libs-linux-headers contrib-libs-cxxsupp yutil @@ -30,8 +30,8 @@ target_link_libraries(core-tx-ev_write PUBLIC ydb-core-tablet_flat blobstorage-vdisk-protos ) -target_sources(core-tx-ev_write PRIVATE - ${CMAKE_SOURCE_DIR}/ydb/core/tx/ev_write/shard_writer.cpp - ${CMAKE_SOURCE_DIR}/ydb/core/tx/ev_write/shards_splitter.cpp - ${CMAKE_SOURCE_DIR}/ydb/core/tx/ev_write/write_data.cpp +target_sources(core-tx-data_events PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/tx/data_events/shard_writer.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/tx/data_events/shards_splitter.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/tx/data_events/write_data.cpp ) diff --git a/library/cpp/porto/CMakeLists.txt b/ydb/core/tx/data_events/CMakeLists.txt index f8b31df0c1..f8b31df0c1 100644 --- a/library/cpp/porto/CMakeLists.txt +++ b/ydb/core/tx/data_events/CMakeLists.txt diff --git a/ydb/core/tx/ev_write/CMakeLists.windows-x86_64.txt b/ydb/core/tx/data_events/CMakeLists.windows-x86_64.txt index 416a6d8b02..70995c1511 100644 --- a/ydb/core/tx/ev_write/CMakeLists.windows-x86_64.txt +++ b/ydb/core/tx/data_events/CMakeLists.windows-x86_64.txt @@ -7,8 +7,8 @@ -add_library(core-tx-ev_write) -target_link_libraries(core-tx-ev_write PUBLIC +add_library(core-tx-data_events) +target_link_libraries(core-tx-data_events PUBLIC contrib-libs-cxxsupp yutil ydb-core-tablet @@ -29,8 +29,8 @@ target_link_libraries(core-tx-ev_write PUBLIC ydb-core-tablet_flat blobstorage-vdisk-protos ) -target_sources(core-tx-ev_write PRIVATE - ${CMAKE_SOURCE_DIR}/ydb/core/tx/ev_write/shard_writer.cpp - ${CMAKE_SOURCE_DIR}/ydb/core/tx/ev_write/shards_splitter.cpp - ${CMAKE_SOURCE_DIR}/ydb/core/tx/ev_write/write_data.cpp +target_sources(core-tx-data_events PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/tx/data_events/shard_writer.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/tx/data_events/shards_splitter.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/tx/data_events/write_data.cpp ) diff --git a/ydb/core/tx/ev_write/columnshard_splitter.h b/ydb/core/tx/data_events/columnshard_splitter.h index 264697973a..264697973a 100644 --- a/ydb/core/tx/ev_write/columnshard_splitter.h +++ b/ydb/core/tx/data_events/columnshard_splitter.h diff --git a/ydb/core/tx/ev_write/events.h b/ydb/core/tx/data_events/events.h index 437e7ae528..46f8fa6cb8 100644 --- a/ydb/core/tx/ev_write/events.h +++ b/ydb/core/tx/data_events/events.h @@ -2,7 +2,7 @@ #include "write_data.h" -#include <ydb/core/protos/ev_write.pb.h> +#include <ydb/core/protos/data_events.pb.h> #include <ydb/core/base/events.h> #include <library/cpp/actors/core/event_pb.h> @@ -11,15 +11,6 @@ namespace NKikimr::NEvents { - -class IDataConstructor { -public: - using TPtr = std::shared_ptr<IDataConstructor>; - virtual ~IDataConstructor() {} - virtual void Serialize(NKikimrDataEvents::TOperationData& proto) const = 0; - virtual ui64 GetSchemaVersion() const = 0; -}; - struct TDataEvents { class TCoordinatorInfo { @@ -51,11 +42,21 @@ struct TDataEvents { Record.SetTxId(txId); } - void AddReplaceOp(const ui64 tableId, const IDataConstructor::TPtr& data) { - Record.MutableTableId()->SetTableId(tableId); - Y_ABORT_UNLESS(data); - Record.MutableTableId()->SetSchemaVersion(data->GetSchemaVersion()); - data->Serialize(*Record.MutableReplace()); + void AddOperation(NKikimrDataEvents::TEvWrite_TOperation::EOperationType operationType, ui64 tableId, + ui64 schemaVersion, const std::vector<std::pair<TString, NScheme::TTypeInfo>>& ydbSchema, + ui64 payloadIndex, NKikimrDataEvents::EDataFormat payloadFormat) { + Y_ABORT_UNLESS(operationType != NKikimrDataEvents::TEvWrite::TOperation::OPERATION_UNSPECIFIED); + Y_ABORT_UNLESS(payloadFormat != NKikimrDataEvents::FORMAT_UNSPECIFIED); + + auto operation = Record.AddOperations(); + operation->SetType(operationType); + operation->SetPayloadFormat(payloadFormat); + operation->SetPayloadIndex(payloadIndex); + operation->MutableTableId()->SetTableId(tableId); + operation->MutableTableId()->SetSchemaVersion(schemaVersion); + for (ui32 i = 0; i < ydbSchema.size(); ++i) { + operation->AddColumnIds(i + 1); + } } ui64 GetTxId() const { @@ -68,27 +69,27 @@ struct TDataEvents { TEvWriteResult() = default; - static std::unique_ptr<TEvWriteResult> BuildError(const ui64 txId, const NKikimrDataEvents::TEvWriteResult::EOperationStatus& status, const TString& errorMsg) { + static std::unique_ptr<TEvWriteResult> BuildError(const ui64 txId, const NKikimrDataEvents::TEvWriteResult::EStatus& status, const TString& errorMsg) { auto result = std::make_unique<TEvWriteResult>(); - AFL_ERROR(NKikimrServices::TX_COLUMNSHARD)("event", "ev_write_error")("status", NKikimrDataEvents::TEvWriteResult::EOperationStatus_Name(status)) - ("details", errorMsg)("tx_id", txId); + ACFL_ERROR("event", "ev_write_error")("status", NKikimrDataEvents::TEvWriteResult::EStatus_Name(status))("details", errorMsg)("tx_id", txId); result->Record.SetTxId(txId); result->Record.SetStatus(status); - result->Record.MutableIssueMessage()->set_message(errorMsg); + auto issue = result->Record.AddIssues(); + issue->set_message(errorMsg); return result; } static std::unique_ptr<TEvWriteResult> BuildCommited(const ui64 txId) { auto result = std::make_unique<TEvWriteResult>(); result->Record.SetTxId(txId); - result->Record.SetStatus(NKikimrDataEvents::TEvWriteResult::COMPLETED); + result->Record.SetStatus(NKikimrDataEvents::TEvWriteResult::STATUS_COMPLETED); return result; } static std::unique_ptr<TEvWriteResult> BuildPrepared(const ui64 txId, const TCoordinatorInfo& transactionInfo) { auto result = std::make_unique<TEvWriteResult>(); result->Record.SetTxId(txId); - result->Record.SetStatus(NKikimrDataEvents::TEvWriteResult::PREPARED); + result->Record.SetStatus(NKikimrDataEvents::TEvWriteResult::STATUS_PREPARED); result->Record.SetMinStep(transactionInfo.GetMinStep()); result->Record.SetMaxStep(transactionInfo.GetMaxStep()); diff --git a/ydb/core/tx/data_events/payload_helper.h b/ydb/core/tx/data_events/payload_helper.h new file mode 100644 index 0000000000..7904e0bfb7 --- /dev/null +++ b/ydb/core/tx/data_events/payload_helper.h @@ -0,0 +1,39 @@ +#pragma once + +#include <library/cpp/actors/util/rope.h> + +namespace NKikimr::NEvWrite { + +class IPayloadData { +public: + virtual TString GetDataFromPayload(const ui64 index) const = 0; + virtual ui64 AddDataToPayload(TString&& blobData) = 0; + virtual ~IPayloadData() { + } +}; + +template <class TEvent> +class TPayloadHelper: public IPayloadData { + TEvent& Event; + +public: + TPayloadHelper(TEvent& ev) + : Event(ev) + { + } + + TString GetDataFromPayload(const ui64 index) const override { + TRope rope = Event.GetPayload(index); + TString data = TString::Uninitialized(rope.GetSize()); + rope.Begin().ExtractPlainDataAndAdvance(data.Detach(), data.size()); + return data; + } + + ui64 AddDataToPayload(TString&& blobData) override { + TRope rope; + rope.Insert(rope.End(), TRope(blobData)); + return Event.AddPayload(std::move(rope)); + } +}; + +} diff --git a/ydb/core/tx/ev_write/shard_writer.cpp b/ydb/core/tx/data_events/shard_writer.cpp index b356a70de7..b356a70de7 100644 --- a/ydb/core/tx/ev_write/shard_writer.cpp +++ b/ydb/core/tx/data_events/shard_writer.cpp diff --git a/ydb/core/tx/ev_write/shard_writer.h b/ydb/core/tx/data_events/shard_writer.h index 904f559669..904f559669 100644 --- a/ydb/core/tx/ev_write/shard_writer.h +++ b/ydb/core/tx/data_events/shard_writer.h diff --git a/ydb/core/tx/ev_write/shards_splitter.cpp b/ydb/core/tx/data_events/shards_splitter.cpp index bfab4c8430..bfab4c8430 100644 --- a/ydb/core/tx/ev_write/shards_splitter.cpp +++ b/ydb/core/tx/data_events/shards_splitter.cpp diff --git a/ydb/core/tx/ev_write/shards_splitter.h b/ydb/core/tx/data_events/shards_splitter.h index 7629b520c7..7629b520c7 100644 --- a/ydb/core/tx/ev_write/shards_splitter.h +++ b/ydb/core/tx/data_events/shards_splitter.h diff --git a/ydb/core/tx/ev_write/write_data.cpp b/ydb/core/tx/data_events/write_data.cpp index ae208446ac..ae208446ac 100644 --- a/ydb/core/tx/ev_write/write_data.cpp +++ b/ydb/core/tx/data_events/write_data.cpp diff --git a/ydb/core/tx/ev_write/write_data.h b/ydb/core/tx/data_events/write_data.h index 1b6b7fe159..1b6b7fe159 100644 --- a/ydb/core/tx/ev_write/write_data.h +++ b/ydb/core/tx/data_events/write_data.h diff --git a/ydb/core/tx/ev_write/ya.make b/ydb/core/tx/data_events/ya.make index 21cc23242f..21cc23242f 100644 --- a/ydb/core/tx/ev_write/ya.make +++ b/ydb/core/tx/data_events/ya.make diff --git a/ydb/core/tx/datashard/datashard__kqp_scan.cpp b/ydb/core/tx/datashard/datashard__kqp_scan.cpp index 428c61c058..1aedbc4088 100644 --- a/ydb/core/tx/datashard/datashard__kqp_scan.cpp +++ b/ydb/core/tx/datashard/datashard__kqp_scan.cpp @@ -40,7 +40,7 @@ public: NDataShard::TUserTable::TCPtr tableInfo, const TSmallVec<TSerializedTableRange>&& tableRanges, const TSmallVec<NTable::TTag>&& columnTags, const TSmallVec<bool>&& skipNullKeys, const NYql::NDqProto::EDqStatsMode& statsMode, ui64 timeoutMs, ui32 generation, - NKikimrTxDataShard::EScanDataFormat dataFormat) + NKikimrDataEvents::EDataFormat dataFormat) : TActor(&TKqpScan::StateScan) , ComputeActorId(computeActorId) , DatashardActorId(datashardActorId) @@ -58,7 +58,7 @@ public: , Sleep(true) , IsLocal(computeActorId.NodeId() == datashardActorId.NodeId()) { - if (DataFormat == NKikimrTxDataShard::EScanDataFormat::ARROW) { + if (DataFormat == NKikimrDataEvents::FORMAT_ARROW) { BatchBuilder = MakeHolder<NArrow::TArrowBatchBuilder>(); TVector<std::pair<TString, NScheme::TTypeInfo>> schema; if (!Tags.empty()) { @@ -384,12 +384,12 @@ private: if (!Result) { Result = MakeHolder<TEvKqpCompute::TEvScanData>(ScanId, Generation); switch (DataFormat) { - case NKikimrTxDataShard::EScanDataFormat::UNSPECIFIED: - case NKikimrTxDataShard::EScanDataFormat::CELLVEC: { + case NKikimrDataEvents::FORMAT_UNSPECIFIED: + case NKikimrDataEvents::FORMAT_CELLVEC: { Result->Rows.reserve(INIT_BATCH_ROWS); break; } - case NKikimrTxDataShard::EScanDataFormat::ARROW: { + case NKikimrDataEvents::FORMAT_ARROW: { } } } @@ -405,12 +405,12 @@ private: CellvecBytes += std::max((ui64)8, (ui64)cell.Size()); } switch (DataFormat) { - case NKikimrTxDataShard::EScanDataFormat::UNSPECIFIED: - case NKikimrTxDataShard::EScanDataFormat::CELLVEC: { + case NKikimrDataEvents::FORMAT_UNSPECIFIED: + case NKikimrDataEvents::FORMAT_CELLVEC: { Result->Rows.emplace_back(TOwnedCellVec::Make(*row)); break; } - case NKikimrTxDataShard::EScanDataFormat::ARROW: { + case NKikimrDataEvents::FORMAT_ARROW: { NKikimr::TDbTupleRef key; Y_DEBUG_ABORT_UNLESS((*row).size() == Types.size()); NKikimr::TDbTupleRef value = NKikimr::TDbTupleRef(Types.data(), (*row).data(), Types.size()); @@ -433,7 +433,7 @@ private: } auto sendBytes = CellvecBytes; - if (DataFormat == NKikimrTxDataShard::EScanDataFormat::ARROW) { + if (DataFormat == NKikimrDataEvents::FORMAT_ARROW) { FlushBatchToResult(); sendBytes = NArrow::GetBatchDataSize(Result->ArrowBatch); // Batch is stored inside BatchBuilder until we flush it into Result. So we verify number of rows here. @@ -516,7 +516,7 @@ private: const NYql::NDqProto::EDqStatsMode StatsMode; const TInstant Deadline; const ui32 Generation; - const NKikimrTxDataShard::EScanDataFormat DataFormat; + const NKikimrDataEvents::EDataFormat DataFormat; TChunksLimiter ChunksLimiter; bool Sleep; const bool IsLocal; diff --git a/ydb/core/tx/datashard/datashard__read_iterator.cpp b/ydb/core/tx/datashard/datashard__read_iterator.cpp index 4409add2c3..e63f4c41c1 100644 --- a/ydb/core/tx/datashard/datashard__read_iterator.cpp +++ b/ydb/core/tx/datashard/datashard__read_iterator.cpp @@ -198,10 +198,10 @@ std::pair<std::unique_ptr<IBlockBuilder>, TString> CreateBlockBuilder( } switch (state.Format) { - case NKikimrTxDataShard::ARROW: + case NKikimrDataEvents::FORMAT_ARROW: blockBuilder.reset(new NArrow::TArrowBatchBuilder()); break; - case NKikimrTxDataShard::CELLVEC: + case NKikimrDataEvents::FORMAT_CELLVEC: blockBuilder.reset(new TCellBlockBuilder()); break; default: @@ -681,12 +681,12 @@ public: } switch (State.Format) { - case NKikimrTxDataShard::ARROW: { + case NKikimrDataEvents::FORMAT_ARROW: { auto& arrowBuilder = static_cast<NArrow::TArrowBatchBuilder&>(BlockBuilder); result.SetArrowBatch(arrowBuilder.FlushBatch(false)); break; } - case NKikimrTxDataShard::CELLVEC: { + case NKikimrDataEvents::FORMAT_CELLVEC: { auto& cellBuilder = static_cast<TCellBlockBuilder&>(BlockBuilder); TOwnedCellVecBatch batch; cellBuilder.FlushBatch(batch); @@ -1829,7 +1829,7 @@ private: auto locks = sysLocks.ApplyLocks(); for (auto& lock : locks) { - NKikimrTxDataShard::TLock* addLock; + NKikimrDataEvents::TLock* addLock; if (lock.IsError()) { addLock = Result->Record.AddBrokenTxLocks(); } else { @@ -2063,7 +2063,7 @@ public: ctx.SelfID.NodeId()); return true; } - if (record.GetResultFormat() != NKikimrTxDataShard::CELLVEC) { + if (record.GetResultFormat() != NKikimrDataEvents::FORMAT_CELLVEC) { ReplyError( Ydb::StatusIds::UNSUPPORTED, TStringBuilder() << "Unsupported result format " @@ -2406,7 +2406,7 @@ public: } if (isBroken) { - NKikimrTxDataShard::TLock *addLock = record.AddBrokenTxLocks(); + NKikimrDataEvents::TLock *addLock = record.AddBrokenTxLocks(); addLock->SetLockId(state.Lock->GetLockId()); addLock->SetDataShard(Self->TabletID()); addLock->SetGeneration(state.Lock->GetGeneration()); diff --git a/ydb/core/tx/datashard/datashard_active_transaction.h b/ydb/core/tx/datashard/datashard_active_transaction.h index 1fec5ea4e4..f10462cf7a 100644 --- a/ydb/core/tx/datashard/datashard_active_transaction.h +++ b/ydb/core/tx/datashard/datashard_active_transaction.h @@ -215,7 +215,7 @@ public: return Tx.GetKqpTransaction().GetUseGenericReadSets(); } - inline const ::NKikimrTxDataShard::TKqpLocks& GetKqpLocks() const { + inline const ::NKikimrDataEvents::TKqpLocks& GetKqpLocks() const { Y_ABORT_UNLESS(IsKqpDataTx()); return Tx.GetKqpTransaction().GetLocks(); } diff --git a/ydb/core/tx/datashard/datashard_kqp.cpp b/ydb/core/tx/datashard/datashard_kqp.cpp index 7c63dbd15b..6dc63fd8b1 100644 --- a/ydb/core/tx/datashard/datashard_kqp.cpp +++ b/ydb/core/tx/datashard/datashard_kqp.cpp @@ -98,7 +98,7 @@ NUdf::EFetchStatus FetchOutput(NDq::IDqOutputChannel* channel, NDq::TDqSerialize } NDq::ERunStatus RunKqpTransactionInternal(const TActorContext& ctx, ui64 txId, - const TInputOpData::TInReadSets* inReadSets, const NKikimrTxDataShard::TKqpLocks&, + const TInputOpData::TInReadSets* inReadSets, const NKikimrDataEvents::TKqpLocks&, bool useGenericReadSets, NKqp::TKqpTasksRunner& tasksRunner, bool applyEffects) { @@ -211,40 +211,40 @@ NDq::ERunStatus RunKqpTransactionInternal(const TActorContext& ctx, ui64 txId, return runStatus; } -bool NeedValidateLocks(NKikimrTxDataShard::TKqpLocks_ELocksOp op) { +bool NeedValidateLocks(NKikimrDataEvents::TKqpLocks::ELocksOp op) { switch (op) { - case NKikimrTxDataShard::TKqpLocks::Commit: + case NKikimrDataEvents::TKqpLocks::Commit: return true; - case NKikimrTxDataShard::TKqpLocks::Rollback: - case NKikimrTxDataShard::TKqpLocks::Unspecified: + case NKikimrDataEvents::TKqpLocks::Rollback: + case NKikimrDataEvents::TKqpLocks::Unspecified: return false; } } -bool NeedEraseLocks(NKikimrTxDataShard::TKqpLocks_ELocksOp op) { +bool NeedEraseLocks(NKikimrDataEvents::TKqpLocks::ELocksOp op) { switch (op) { - case NKikimrTxDataShard::TKqpLocks::Commit: - case NKikimrTxDataShard::TKqpLocks::Rollback: + case NKikimrDataEvents::TKqpLocks::Commit: + case NKikimrDataEvents::TKqpLocks::Rollback: return true; - case NKikimrTxDataShard::TKqpLocks::Unspecified: + case NKikimrDataEvents::TKqpLocks::Unspecified: return false; } } -bool NeedCommitLocks(NKikimrTxDataShard::TKqpLocks_ELocksOp op) { +bool NeedCommitLocks(NKikimrDataEvents::TKqpLocks::ELocksOp op) { switch (op) { - case NKikimrTxDataShard::TKqpLocks::Commit: + case NKikimrDataEvents::TKqpLocks::Commit: return true; - case NKikimrTxDataShard::TKqpLocks::Rollback: - case NKikimrTxDataShard::TKqpLocks::Unspecified: + case NKikimrDataEvents::TKqpLocks::Rollback: + case NKikimrDataEvents::TKqpLocks::Unspecified: return false; } } -TVector<TCell> MakeLockKey(const NKikimrTxDataShard::TLock& lockProto) { +TVector<TCell> MakeLockKey(const NKikimrDataEvents::TLock& lockProto) { auto lockId = lockProto.GetLockId(); auto lockDatashard = lockProto.GetDataShard(); auto lockSchemeShard = lockProto.GetSchemeShard(); @@ -265,10 +265,10 @@ TVector<TCell> MakeLockKey(const NKikimrTxDataShard::TLock& lockProto) { } // returns list of broken locks -TVector<NKikimrTxDataShard::TLock> ValidateLocks(const NKikimrTxDataShard::TKqpLocks& txLocks, TSysLocks& sysLocks, +TVector<NKikimrDataEvents::TLock> ValidateLocks(const NKikimrDataEvents::TKqpLocks& txLocks, TSysLocks& sysLocks, ui64 tabletId) { - TVector<NKikimrTxDataShard::TLock> brokenLocks; + TVector<NKikimrDataEvents::TLock> brokenLocks; if (!NeedValidateLocks(txLocks.GetOp())) { return {}; @@ -294,13 +294,13 @@ TVector<NKikimrTxDataShard::TLock> ValidateLocks(const NKikimrTxDataShard::TKqpL return brokenLocks; } -bool SendLocks(const NKikimrTxDataShard::TKqpLocks& locks, ui64 shardId) { +bool SendLocks(const NKikimrDataEvents::TKqpLocks& locks, ui64 shardId) { auto& sendingShards = locks.GetSendingShards(); auto it = std::find(sendingShards.begin(), sendingShards.end(), shardId); return it != sendingShards.end(); } -bool ReceiveLocks(const NKikimrTxDataShard::TKqpLocks& locks, ui64 shardId) { +bool ReceiveLocks(const NKikimrDataEvents::TKqpLocks& locks, ui64 shardId) { auto& receivingShards = locks.GetReceivingShards(); auto it = std::find(receivingShards.begin(), receivingShards.end(), shardId); return it != receivingShards.end(); @@ -475,7 +475,7 @@ void KqpSetTxKeys(ui64 tabletId, ui64 taskId, const TUserTable* tableInfo, } } -void KqpSetTxLocksKeys(const NKikimrTxDataShard::TKqpLocks& locks, const TSysLocks& sysLocks, TEngineBay& engineBay) { +void KqpSetTxLocksKeys(const NKikimrDataEvents::TKqpLocks& locks, const TSysLocks& sysLocks, TEngineBay& engineBay) { if (locks.LocksSize() == 0) { return; } @@ -503,14 +503,14 @@ void KqpSetTxLocksKeys(const NKikimrTxDataShard::TKqpLocks& locks, const TSysLoc } NYql::NDq::ERunStatus KqpRunTransaction(const TActorContext& ctx, ui64 txId, - const NKikimrTxDataShard::TKqpLocks& kqpLocks, bool useGenericReadSets, NKqp::TKqpTasksRunner& tasksRunner) + const NKikimrDataEvents::TKqpLocks& kqpLocks, bool useGenericReadSets, NKqp::TKqpTasksRunner& tasksRunner) { return RunKqpTransactionInternal(ctx, txId, /* inReadSets */ nullptr, kqpLocks, useGenericReadSets, tasksRunner, /* applyEffects */ false); } THolder<TEvDataShard::TEvProposeTransactionResult> KqpCompleteTransaction(const TActorContext& ctx, ui64 origin, ui64 txId, const TInputOpData::TInReadSets* inReadSets, - const NKikimrTxDataShard::TKqpLocks& kqpLocks, bool useGenericReadSets, NKqp::TKqpTasksRunner& tasksRunner, + const NKikimrDataEvents::TKqpLocks& kqpLocks, bool useGenericReadSets, NKqp::TKqpTasksRunner& tasksRunner, const NMiniKQL::TKqpDatashardComputeContext& computeCtx) { auto runStatus = RunKqpTransactionInternal(ctx, txId, inReadSets, kqpLocks, useGenericReadSets, tasksRunner, /* applyEffects */ true); @@ -592,7 +592,7 @@ THolder<TEvDataShard::TEvProposeTransactionResult> KqpCompleteTransaction(const return result; } -void KqpFillOutReadSets(TOutputOpData::TOutReadSets& outReadSets, const NKikimrTxDataShard::TKqpLocks& kqpLocks, +void KqpFillOutReadSets(TOutputOpData::TOutReadSets& outReadSets, const NKikimrDataEvents::TKqpLocks& kqpLocks, bool hasKqpLocks, bool useGenericReadSets, NKqp::TKqpTasksRunner& tasksRunner, TSysLocks& sysLocks, ui64 tabletId) { @@ -949,7 +949,7 @@ void KqpCommitLocks(ui64 origin, TActiveTransaction* tx, const TRowVersion& writ } void KqpPrepareInReadsets(TInputOpData::TInReadSets& inReadSets, - const NKikimrTxDataShard::TKqpLocks& kqpLocks, const NKqp::TKqpTasksRunner& tasksRunner, ui64 tabletId) + const NKikimrDataEvents::TKqpLocks& kqpLocks, const NKqp::TKqpTasksRunner& tasksRunner, ui64 tabletId) { for (auto& [taskId, task] : tasksRunner.GetTasks()) { for (ui32 i = 0; i < task.InputsSize(); ++i) { diff --git a/ydb/core/tx/datashard/datashard_kqp.h b/ydb/core/tx/datashard/datashard_kqp.h index 0a330218cb..2109af8c73 100644 --- a/ydb/core/tx/datashard/datashard_kqp.h +++ b/ydb/core/tx/datashard/datashard_kqp.h @@ -18,22 +18,22 @@ void KqpSetTxKeys(ui64 tabletId, ui64 taskId, const TUserTable* tableInfo, const NKikimrTxDataShard::TKqpTransaction_TDataTaskMeta& meta, const NScheme::TTypeRegistry& typeRegistry, const TActorContext& ctx, TEngineBay& engineBay); -void KqpSetTxLocksKeys(const NKikimrTxDataShard::TKqpLocks& locks, const TSysLocks& sysLocks, TEngineBay& engineBay); +void KqpSetTxLocksKeys(const NKikimrDataEvents::TKqpLocks& locks, const TSysLocks& sysLocks, TEngineBay& engineBay); NYql::NDq::ERunStatus KqpRunTransaction(const TActorContext& ctx, ui64 txId, - const NKikimrTxDataShard::TKqpLocks& kqpLocks, bool useGenericReadSets, NKqp::TKqpTasksRunner& tasksRunner); + const NKikimrDataEvents::TKqpLocks& kqpLocks, bool useGenericReadSets, NKqp::TKqpTasksRunner& tasksRunner); THolder<TEvDataShard::TEvProposeTransactionResult> KqpCompleteTransaction(const TActorContext& ctx, ui64 origin, ui64 txId, const TInputOpData::TInReadSets* inReadSets, - const NKikimrTxDataShard::TKqpLocks& kqpLocks, bool useGenericReadSets, NKqp::TKqpTasksRunner& tasksRunner, + const NKikimrDataEvents::TKqpLocks& kqpLocks, bool useGenericReadSets, NKqp::TKqpTasksRunner& tasksRunner, const NMiniKQL::TKqpDatashardComputeContext& computeCtx); -void KqpFillOutReadSets(TOutputOpData::TOutReadSets& outReadSets, const NKikimrTxDataShard::TKqpLocks& kqpLocks, +void KqpFillOutReadSets(TOutputOpData::TOutReadSets& outReadSets, const NKikimrDataEvents::TKqpLocks& kqpLocks, bool hasKqpLocks, bool useGenericReadSets, NKqp::TKqpTasksRunner& tasksRunner, TSysLocks& sysLocks, ui64 tabletId); void KqpPrepareInReadsets(TInputOpData::TInReadSets& inReadSets, - const NKikimrTxDataShard::TKqpLocks& kqpLocks, const NKqp::TKqpTasksRunner& tasksRunner, ui64 tabletId); + const NKikimrDataEvents::TKqpLocks& kqpLocks, const NKqp::TKqpTasksRunner& tasksRunner, ui64 tabletId); bool KqpValidateLocks(ui64 origin, TActiveTransaction* tx, TSysLocks& sysLocks); bool KqpValidateVolatileTx(ui64 origin, TActiveTransaction* tx, TSysLocks& sysLocks); diff --git a/ydb/core/tx/datashard/datashard_ut_read_iterator.cpp b/ydb/core/tx/datashard/datashard_ut_read_iterator.cpp index f504942869..635f5d422a 100644 --- a/ydb/core/tx/datashard/datashard_ut_read_iterator.cpp +++ b/ydb/core/tx/datashard/datashard_ut_read_iterator.cpp @@ -193,10 +193,10 @@ void CheckResult( UNIT_ASSERT_VALUES_EQUAL(record.GetStatus().GetCode(), Ydb::StatusIds::SUCCESS); if (gold.size()) { switch (record.GetResultFormat()) { - case NKikimrTxDataShard::ARROW: + case NKikimrDataEvents::FORMAT_ARROW: CheckResultArrow(userTable, result, gold, goldTypes, columns); break; - case NKikimrTxDataShard::CELLVEC: + case NKikimrDataEvents::FORMAT_CELLVEC: CheckResultCellVec(userTable, result, gold, goldTypes, columns); break; default: @@ -481,7 +481,7 @@ struct TTestHelper { std::unique_ptr<TEvDataShard::TEvRead> GetBaseReadRequest( const TString& tableName, ui64 readId, - NKikimrTxDataShard::EScanDataFormat format = NKikimrTxDataShard::ARROW, + NKikimrDataEvents::EDataFormat format = NKikimrDataEvents::FORMAT_ARROW, const TRowVersion& snapshot = {}) { const auto& table = Tables[tableName]; @@ -536,7 +536,7 @@ struct TTestHelper { record.AddColumns(1); record.AddColumns(2); - record.SetResultFormat(NKikimrTxDataShard::CELLVEC); + record.SetResultFormat(NKikimrDataEvents::FORMAT_CELLVEC); return request; } @@ -653,14 +653,14 @@ struct TTestHelper { auto readResult = SendRead(tableName, request.release()); - const NKikimrTxDataShard::TLock* prevLock; + const NKikimrDataEvents::TLock* prevLock; if (prevResult.Record.TxLocksSize()) { prevLock = &prevResult.Record.GetTxLocks(0); } else { prevLock = &prevResult.Record.GetBrokenTxLocks(0); } - const NKikimrTxDataShard::TLock* newLock; + const NKikimrDataEvents::TLock* newLock; if (readResult->Record.TxLocksSize()) { newLock = &readResult->Record.GetTxLocks(0); } else { @@ -676,7 +676,7 @@ struct TTestHelper { void TestChunkRead(ui32 chunkSize, ui32 rowCount, ui32 ranges = 1, ui32 limit = Max<ui32>()) { UpsertMany(1, rowCount); - auto request = GetBaseReadRequest("table-1-many", 1, NKikimrTxDataShard::CELLVEC, TRowVersion::Max()); + auto request = GetBaseReadRequest("table-1-many", 1, NKikimrDataEvents::FORMAT_CELLVEC, TRowVersion::Max()); request->Record.ClearSnapshot(); ui32 base = 1; @@ -871,7 +871,7 @@ public: THashMap<TString, TTableInfo> Tables; }; -void TestReadKey(NKikimrTxDataShard::EScanDataFormat format, bool withFollower = false) { +void TestReadKey(NKikimrDataEvents::EDataFormat format, bool withFollower = false) { TTestHelper helper(withFollower); for (ui32 k: {1, 3, 5}) { @@ -883,7 +883,7 @@ void TestReadKey(NKikimrTxDataShard::EScanDataFormat format, bool withFollower = } } -void TestReadRangeInclusiveEnds(NKikimrTxDataShard::EScanDataFormat format) { +void TestReadRangeInclusiveEnds(NKikimrDataEvents::EDataFormat format) { TTestHelper helper; auto request = helper.GetBaseReadRequest("table-1", 1, format); @@ -903,7 +903,7 @@ void TestReadRangeInclusiveEnds(NKikimrTxDataShard::EScanDataFormat format) { }); } -void TestReadRangeMovies(NKikimrTxDataShard::EScanDataFormat format) { +void TestReadRangeMovies(NKikimrDataEvents::EDataFormat format) { // test just to check if non-trivial type like string is properly replied TTestHelper helper; @@ -938,19 +938,19 @@ void TestReadRangeMovies(NKikimrTxDataShard::EScanDataFormat format) { Y_UNIT_TEST_SUITE(DataShardReadIterator) { Y_UNIT_TEST(ShouldReadKeyCellVec) { - TestReadKey(NKikimrTxDataShard::CELLVEC); + TestReadKey(NKikimrDataEvents::FORMAT_CELLVEC); } Y_UNIT_TEST(ShouldReadKeyArrow) { - TestReadKey(NKikimrTxDataShard::ARROW); + TestReadKey(NKikimrDataEvents::FORMAT_ARROW); } Y_UNIT_TEST(ShouldReadRangeCellVec) { - TestReadRangeMovies(NKikimrTxDataShard::CELLVEC); + TestReadRangeMovies(NKikimrDataEvents::FORMAT_CELLVEC); } Y_UNIT_TEST(ShouldReadRangeArrow) { - TestReadRangeMovies(NKikimrTxDataShard::ARROW); + TestReadRangeMovies(NKikimrDataEvents::FORMAT_ARROW); } Y_UNIT_TEST(ShouldReadKeyOnlyValueColumn) { @@ -1017,7 +1017,7 @@ Y_UNIT_TEST_SUITE(DataShardReadIterator) { // KIKIMR-16897: no columns mean we want to calc row count TTestHelper helper; - auto request = helper.GetBaseReadRequest("table-1", 1, NKikimrTxDataShard::CELLVEC); + auto request = helper.GetBaseReadRequest("table-1", 1, NKikimrDataEvents::FORMAT_CELLVEC); request->Record.ClearColumns(); AddKeyQuery(*request, {3, 3, 3}); AddKeyQuery(*request, {1, 1, 1}); @@ -1037,7 +1037,7 @@ Y_UNIT_TEST_SUITE(DataShardReadIterator) { // KIKIMR-16897: no columns mean we want to calc row count TTestHelper helper; - auto request = helper.GetBaseReadRequest("table-1", 1, NKikimrTxDataShard::ARROW); + auto request = helper.GetBaseReadRequest("table-1", 1, NKikimrDataEvents::FORMAT_ARROW); request->Record.ClearColumns(); AddKeyQuery(*request, {3, 3, 3}); AddKeyQuery(*request, {1, 1, 1}); @@ -1057,7 +1057,7 @@ Y_UNIT_TEST_SUITE(DataShardReadIterator) { // KIKIMR-16897: no columns mean we want to calc row count TTestHelper helper; - auto request = helper.GetBaseReadRequest("table-1", 1, NKikimrTxDataShard::CELLVEC); + auto request = helper.GetBaseReadRequest("table-1", 1, NKikimrDataEvents::FORMAT_CELLVEC); request->Record.ClearColumns(); AddRangeQuery<ui32>( *request, @@ -1081,7 +1081,7 @@ Y_UNIT_TEST_SUITE(DataShardReadIterator) { // KIKIMR-16897: no columns mean we want to calc row count TTestHelper helper; - auto request = helper.GetBaseReadRequest("table-1", 1, NKikimrTxDataShard::ARROW); + auto request = helper.GetBaseReadRequest("table-1", 1, NKikimrDataEvents::FORMAT_ARROW); request->Record.ClearColumns(); AddRangeQuery<ui32>( *request, @@ -1718,11 +1718,11 @@ Y_UNIT_TEST_SUITE(DataShardReadIterator) { } Y_UNIT_TEST(ShouldReadRangeInclusiveEndsCellVec) { - TestReadRangeInclusiveEnds(NKikimrTxDataShard::CELLVEC); + TestReadRangeInclusiveEnds(NKikimrDataEvents::FORMAT_CELLVEC); } Y_UNIT_TEST(ShouldReadRangeInclusiveEndsArrow) { - TestReadRangeInclusiveEnds(NKikimrTxDataShard::ARROW); + TestReadRangeInclusiveEnds(NKikimrDataEvents::FORMAT_ARROW); } Y_UNIT_TEST(ShouldReadRangeReverse) { @@ -2372,7 +2372,7 @@ Y_UNIT_TEST_SUITE(DataShardReadIterator) { } Y_UNIT_TEST(ShouldReadFromFollower) { - TestReadKey(NKikimrTxDataShard::CELLVEC, true); + TestReadKey(NKikimrDataEvents::FORMAT_CELLVEC, true); } Y_UNIT_TEST(ShouldNotReadFutureMvccFromFollower) { @@ -2385,7 +2385,7 @@ Y_UNIT_TEST_SUITE(DataShardReadIterator) { TTestHelper helper(serverSettings, shardCount, true); TRowVersion someVersion = TRowVersion(10000, Max<ui64>()); - auto request = helper.GetBaseReadRequest("table-1", 1, NKikimrTxDataShard::ARROW, someVersion); + auto request = helper.GetBaseReadRequest("table-1", 1, NKikimrDataEvents::FORMAT_ARROW, someVersion); AddKeyQuery(*request, {3, 3, 3}); auto readResult = helper.SendRead("table-1", request.release()); const auto& record = readResult->Record; @@ -2401,7 +2401,7 @@ Y_UNIT_TEST_SUITE(DataShardReadIterator) { const ui64 shardCount = 1; TTestHelper helper(serverSettings, shardCount, true); - auto request = helper.GetBaseReadRequest("table-1", 1, NKikimrTxDataShard::ARROW, TRowVersion::Max()); + auto request = helper.GetBaseReadRequest("table-1", 1, NKikimrDataEvents::FORMAT_ARROW, TRowVersion::Max()); request->Record.ClearSnapshot(); AddKeyQuery(*request, {3, 3, 3}); auto readResult = helper.SendRead("table-1", request.release()); @@ -2508,7 +2508,7 @@ Y_UNIT_TEST_SUITE(DataShardReadIterator) { // read from HEAD when there is no conflicting operation TTestHelper helper; - auto request = helper.GetBaseReadRequest("table-1", 1, NKikimrTxDataShard::ARROW, TRowVersion::Max()); + auto request = helper.GetBaseReadRequest("table-1", 1, NKikimrDataEvents::FORMAT_ARROW, TRowVersion::Max()); request->Record.ClearSnapshot(); AddKeyQuery(*request, {3, 3, 3}); @@ -2535,7 +2535,7 @@ Y_UNIT_TEST_SUITE(DataShardReadIterator) { auto hangedInfo = helper.HangWithTransactionWaitingRS(shardCount, false); { - auto request = helper.GetBaseReadRequest("table-1", 1, NKikimrTxDataShard::ARROW, TRowVersion::Max()); + auto request = helper.GetBaseReadRequest("table-1", 1, NKikimrDataEvents::FORMAT_ARROW, TRowVersion::Max()); request->Record.ClearSnapshot(); AddKeyQuery(*request, {3, 3, 3}); AddKeyQuery(*request, {1, 1, 1}); @@ -2590,7 +2590,7 @@ Y_UNIT_TEST_SUITE(DataShardReadIterator) { { // now read HEAD - auto request = helper.GetBaseReadRequest("table-1", 1, NKikimrTxDataShard::ARROW, TRowVersion::Max()); + auto request = helper.GetBaseReadRequest("table-1", 1, NKikimrDataEvents::FORMAT_ARROW, TRowVersion::Max()); request->Record.ClearSnapshot(); AddKeyQuery(*request, {3, 3, 3}); AddKeyQuery(*request, {1, 1, 1}); @@ -2687,7 +2687,7 @@ Y_UNIT_TEST_SUITE(DataShardReadIterator) { // 2a: read prior data { auto oldVersion = TRowVersion(hangedStep - 1, Max<ui64>()); - auto request = helper.GetBaseReadRequest("table-1", 1, NKikimrTxDataShard::ARROW, oldVersion); + auto request = helper.GetBaseReadRequest("table-1", 1, NKikimrDataEvents::FORMAT_ARROW, oldVersion); AddKeyQuery(*request, {3, 3, 3}); auto readResult = helper.SendRead("table-1", request.release()); @@ -2701,7 +2701,7 @@ Y_UNIT_TEST_SUITE(DataShardReadIterator) { // 2b-1 (key): try to read hanged step, note that we have hanged write to the same key { auto oldVersion = TRowVersion(hangedStep, Max<ui64>()); - auto request = helper.GetBaseReadRequest("table-1", 1, NKikimrTxDataShard::ARROW, oldVersion); + auto request = helper.GetBaseReadRequest("table-1", 1, NKikimrDataEvents::FORMAT_ARROW, oldVersion); AddKeyQuery(*request, {3, 3, 3}); auto readResult = helper.SendRead( @@ -2716,7 +2716,7 @@ Y_UNIT_TEST_SUITE(DataShardReadIterator) { // 2b-2 (range): try to read hanged step, note that we have hanged write to the same key { auto oldVersion = TRowVersion(hangedStep, Max<ui64>()); - auto request = helper.GetBaseReadRequest("table-1", 2, NKikimrTxDataShard::ARROW, oldVersion); + auto request = helper.GetBaseReadRequest("table-1", 2, NKikimrDataEvents::FORMAT_ARROW, oldVersion); AddRangeQuery<ui32>( *request, @@ -2738,7 +2738,7 @@ Y_UNIT_TEST_SUITE(DataShardReadIterator) { // 2b-3 (key prefix, equals to range): try to read hanged step, note that we have hanged write to the same key { auto oldVersion = TRowVersion(hangedStep, Max<ui64>()); - auto request = helper.GetBaseReadRequest("table-1", 3, NKikimrTxDataShard::ARROW, oldVersion); + auto request = helper.GetBaseReadRequest("table-1", 3, NKikimrDataEvents::FORMAT_ARROW, oldVersion); AddKeyQuery(*request, {3}); auto readResult = helper.SendRead( @@ -2798,7 +2798,7 @@ Y_UNIT_TEST_SUITE(DataShardReadIterator) { // 4: try to read hanged step again { auto oldVersion = TRowVersion(hangedStep, Max<ui64>()); - auto request = helper.GetBaseReadRequest("table-1", 4, NKikimrTxDataShard::ARROW, oldVersion); + auto request = helper.GetBaseReadRequest("table-1", 4, NKikimrDataEvents::FORMAT_ARROW, oldVersion); AddKeyQuery(*request, {3, 3, 3}); auto readResult = helper.SendRead("table-1", request.release()); @@ -2812,7 +2812,7 @@ Y_UNIT_TEST_SUITE(DataShardReadIterator) { // 5: read prior data again { auto oldVersion = TRowVersion(hangedStep - 1, Max<ui64>()); - auto request = helper.GetBaseReadRequest("table-1", 5, NKikimrTxDataShard::ARROW, oldVersion); + auto request = helper.GetBaseReadRequest("table-1", 5, NKikimrDataEvents::FORMAT_ARROW, oldVersion); AddKeyQuery(*request, {3, 3, 3}); auto readResult = helper.SendRead("table-1", request.release()); @@ -2906,7 +2906,7 @@ Y_UNIT_TEST_SUITE(DataShardReadIterator) { // future snapshot snapshot = TRowVersion(lastStep + 3000, Max<ui64>()); - auto request1 = helper.GetBaseReadRequest("table-1", 1, NKikimrTxDataShard::ARROW, snapshot); + auto request1 = helper.GetBaseReadRequest("table-1", 1, NKikimrDataEvents::FORMAT_ARROW, snapshot); AddKeyQuery(*request1, {3, 3, 3}); AddKeyQuery(*request1, {1, 1, 1}); AddKeyQuery(*request1, {5, 5, 5}); @@ -3042,7 +3042,7 @@ Y_UNIT_TEST_SUITE(DataShardReadIterator) { // future snapshot snapshot = TRowVersion(lastStep + 3000, Max<ui64>()); - auto request1 = helper.GetBaseReadRequest("table-1", 1, NKikimrTxDataShard::ARROW, snapshot); + auto request1 = helper.GetBaseReadRequest("table-1", 1, NKikimrDataEvents::FORMAT_ARROW, snapshot); AddKeyQuery(*request1, {3, 3, 3}); AddKeyQuery(*request1, {1, 1, 1}); AddKeyQuery(*request1, {5, 5, 5}); @@ -3185,7 +3185,7 @@ Y_UNIT_TEST_SUITE(DataShardReadIterator) { const ui64 lockTxId = 1011121314; - auto request1 = helper.GetBaseReadRequest("table-1", 1, NKikimrTxDataShard::ARROW, readVersion); + auto request1 = helper.GetBaseReadRequest("table-1", 1, NKikimrDataEvents::FORMAT_ARROW, readVersion); request1->Record.SetLockTxId(lockTxId); AddRangeQuery<ui32>( @@ -3224,7 +3224,7 @@ Y_UNIT_TEST_SUITE(DataShardReadIterator) { const ui64 lockTxId = 1011121314; - auto request1 = helper.GetBaseReadRequest("table-1", 1, NKikimrTxDataShard::ARROW, readVersion); + auto request1 = helper.GetBaseReadRequest("table-1", 1, NKikimrDataEvents::FORMAT_ARROW, readVersion); request1->Record.SetLockTxId(lockTxId); AddRangeQuery<ui32>( *request1, @@ -3248,7 +3248,7 @@ Y_UNIT_TEST_SUITE(DataShardReadIterator) { (300, 0, 0, 3000); )"); - auto request2 = helper.GetBaseReadRequest("table-1", 2, NKikimrTxDataShard::ARROW, readVersion); + auto request2 = helper.GetBaseReadRequest("table-1", 2, NKikimrDataEvents::FORMAT_ARROW, readVersion); request2->Record.SetLockTxId(lockTxId); AddRangeQuery<ui32>( *request2, @@ -3472,7 +3472,7 @@ Y_UNIT_TEST_SUITE(DataShardReadIterator) { const ui64 lockTxId = 1011121314; - auto request1 = helper.GetBaseReadRequest("table-1", 1, NKikimrTxDataShard::ARROW, readVersion); + auto request1 = helper.GetBaseReadRequest("table-1", 1, NKikimrDataEvents::FORMAT_ARROW, readVersion); request1->Record.SetLockTxId(lockTxId); request1->Record.SetMaxRows(1); // set quota so that DS hangs waiting for ACK @@ -3595,7 +3595,7 @@ Y_UNIT_TEST_SUITE(DataShardReadIteratorSysTables) { true ); - request->Record.SetResultFormat(NKikimrTxDataShard::ARROW); + request->Record.SetResultFormat(NKikimrDataEvents::FORMAT_ARROW); auto readResult = helper.SendRead("table-1", request.release()); const auto& record = readResult->Record; diff --git a/ydb/core/tx/datashard/datashard_ut_snapshot.cpp b/ydb/core/tx/datashard/datashard_ut_snapshot.cpp index 0b6fbcdf15..338ab8660b 100644 --- a/ydb/core/tx/datashard/datashard_ut_snapshot.cpp +++ b/ydb/core/tx/datashard/datashard_ut_snapshot.cpp @@ -1151,7 +1151,7 @@ Y_UNIT_TEST_SUITE(DataShardSnapshots) { }; struct TInjectLocks { - NKikimrTxDataShard::TKqpLocks_ELocksOp Op = NKikimrTxDataShard::TKqpLocks::Commit; + NKikimrDataEvents::TKqpLocks::ELocksOp Op = NKikimrDataEvents::TKqpLocks::Commit; TVector<TLockInfo> Locks; TInjectLocks& AddLocks(const TVector<TLockInfo>& locks) { @@ -1809,7 +1809,7 @@ Y_UNIT_TEST_SUITE(DataShardSnapshots) { for (ui32 columnId : columns) { record.AddColumns(columnId); } - record.SetResultFormat(NKikimrTxDataShard::CELLVEC); + record.SetResultFormat(NKikimrDataEvents::FORMAT_CELLVEC); return request; } diff --git a/ydb/core/tx/datashard/datashard_ut_volatile.cpp b/ydb/core/tx/datashard/datashard_ut_volatile.cpp index 2ca5430456..91e5adec17 100644 --- a/ydb/core/tx/datashard/datashard_ut_volatile.cpp +++ b/ydb/core/tx/datashard/datashard_ut_volatile.cpp @@ -1048,7 +1048,7 @@ Y_UNIT_TEST_SUITE(DataShardVolatile) { msg->Record.MutableSnapshot()->SetTxId(Max<ui64>()); msg->Record.AddColumns(1); msg->Record.AddColumns(2); - msg->Record.SetResultFormat(NKikimrTxDataShard::ARROW); + msg->Record.SetResultFormat(NKikimrDataEvents::FORMAT_ARROW); TVector<TCell> fromKeyCells = { TCell::Make(ui32(0)) }; TVector<TCell> toKeyCells = { TCell::Make(ui32(10)) }; @@ -1178,7 +1178,7 @@ Y_UNIT_TEST_SUITE(DataShardVolatile) { msg->Record.MutableSnapshot()->SetTxId(Max<ui64>()); msg->Record.AddColumns(1); msg->Record.AddColumns(2); - msg->Record.SetResultFormat(NKikimrTxDataShard::ARROW); + msg->Record.SetResultFormat(NKikimrDataEvents::FORMAT_ARROW); msg->Record.SetMaxRowsInResult(1); TVector<TCell> fromKeyCells = { TCell::Make(ui32(0)) }; diff --git a/ydb/core/tx/datashard/read_iterator.h b/ydb/core/tx/datashard/read_iterator.h index 3e1a24ae54..3ce26ec012 100644 --- a/ydb/core/tx/datashard/read_iterator.h +++ b/ydb/core/tx/datashard/read_iterator.h @@ -170,7 +170,7 @@ public: TLockInfo::TPtr Lock; // note that will be always overwritten by values from request - NKikimrTxDataShard::EScanDataFormat Format = NKikimrTxDataShard::EScanDataFormat::CELLVEC; + NKikimrDataEvents::EDataFormat Format = NKikimrDataEvents::FORMAT_CELLVEC; // mainly for tests ui64 MaxRowsInResult = Max<ui64>(); diff --git a/ydb/core/tx/ev_write/CMakeLists.txt b/ydb/core/tx/ev_write/CMakeLists.txt deleted file mode 100644 index f8b31df0c1..0000000000 --- a/ydb/core/tx/ev_write/CMakeLists.txt +++ /dev/null @@ -1,17 +0,0 @@ - -# This file was generated by the build system used internally in the Yandex monorepo. -# Only simple modifications are allowed (adding source-files to targets, adding simple properties -# like target_include_directories). These modifications will be ported to original -# ya.make files by maintainers. Any complex modifications which can't be ported back to the -# original buildsystem will not be accepted. - - -if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) - include(CMakeLists.linux-aarch64.txt) -elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") - include(CMakeLists.darwin-x86_64.txt) -elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) - include(CMakeLists.windows-x86_64.txt) -elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) - include(CMakeLists.linux-x86_64.txt) -endif() diff --git a/ydb/library/yql/ast/yql_constraint.cpp b/ydb/library/yql/ast/yql_constraint.cpp index 08a38a3525..3a342baa8f 100644 --- a/ydb/library/yql/ast/yql_constraint.cpp +++ b/ydb/library/yql/ast/yql_constraint.cpp @@ -1574,7 +1574,7 @@ TPartOfConstraintNode<TOriginalConstraintNode>::GetCommonMapping(const TOriginal template<class TOriginalConstraintNode> void TPartOfConstraintNode<TOriginalConstraintNode>::UniqueMerge(TMapType& output, TMapType&& input) { - output.merge(std::move(input)); + output.merge(input); while (!input.empty()) { const auto exists = input.extract(input.cbegin()); auto& target = output[exists.key()]; diff --git a/ydb/library/yql/dq/actors/compute/dq_async_compute_actor.cpp b/ydb/library/yql/dq/actors/compute/dq_async_compute_actor.cpp index 36795ea1c6..ced976f1fa 100644 --- a/ydb/library/yql/dq/actors/compute/dq_async_compute_actor.cpp +++ b/ydb/library/yql/dq/actors/compute/dq_async_compute_actor.cpp @@ -83,7 +83,7 @@ public: Become(&TDqAsyncComputeActor::StateFuncWrapper<&TDqAsyncComputeActor::StateFuncBody>); - auto wakeup = [this]{ ContinueExecute(); }; + auto wakeup = [this]{ ContinueExecute(EResumeSource::CABootstrapWakeup); }; std::shared_ptr<IDqTaskRunnerExecutionContext> execCtx = std::make_shared<TDqTaskRunnerExecutionContext>( TxId, RuntimeSettings.UseSpilling, std::move(wakeup), TlsActivationContext->AsActorContext()); @@ -454,7 +454,7 @@ private: ForwardToCheckpoints(std::move(DeferredRestoreFromCheckpointEvent)); } - ContinueExecute(); + ContinueExecute(EResumeSource::CATaskRunnerCreated); } bool ReadyToCheckpoint() const override { @@ -690,7 +690,7 @@ private: TakeInputChannelDataRequests.erase(it); - ResumeExecution(); + ResumeExecution(EResumeSource::AsyncPopFinished); } void SinkSend(ui64 index, diff --git a/ydb/library/yql/dq/actors/compute/dq_compute_actor.cpp b/ydb/library/yql/dq/actors/compute/dq_compute_actor.cpp index 4d22704eda..11ab98ba73 100644 --- a/ydb/library/yql/dq/actors/compute/dq_compute_actor.cpp +++ b/ydb/library/yql/dq/actors/compute/dq_compute_actor.cpp @@ -56,11 +56,11 @@ public: auto taskRunner = TaskRunnerFactory(Task, RuntimeSettings.StatsMode, logger); SetTaskRunner(taskRunner); - auto wakeup = [this]{ ContinueExecute(); }; + auto wakeup = [this]{ ContinueExecute(EResumeSource::CABootstrapWakeup); }; TDqTaskRunnerExecutionContext execCtx(TxId, RuntimeSettings.UseSpilling, std::move(wakeup), TlsActivationContext->AsActorContext()); PrepareTaskRunner(execCtx); - ContinueExecute(); + ContinueExecute(EResumeSource::CABootstrap); } void FillExtraStats(NDqProto::TDqComputeActorStats* /* dst */, bool /* last */) { diff --git a/ydb/library/yql/dq/actors/compute/dq_compute_actor.h b/ydb/library/yql/dq/actors/compute/dq_compute_actor.h index fc2cc3daca..c1c0b8429d 100644 --- a/ydb/library/yql/dq/actors/compute/dq_compute_actor.h +++ b/ydb/library/yql/dq/actors/compute/dq_compute_actor.h @@ -21,7 +21,15 @@ struct TEvDqCompute { struct TEvState : public NActors::TEventPB<TEvState, NDqProto::TEvComputeActorState, TDqComputeEvents::EvState> {}; struct TEvStateRequest : public NActors::TEventPB<TEvStateRequest, NDqProto::TEvComputeStateRequest, TDqComputeEvents::EvStateRequest> {}; - struct TEvResumeExecution : public NActors::TEventLocal<TEvResumeExecution, TDqComputeEvents::EvResumeExecution> {}; + struct TEvResumeExecution : public NActors::TEventLocal<TEvResumeExecution, TDqComputeEvents::EvResumeExecution> { + TEvResumeExecution(EResumeSource source) + : Source(source) + { } + + TEvResumeExecution() = default; + + EResumeSource Source = EResumeSource::Default; + }; struct TEvChannelsInfo : public NActors::TEventPB<TEvChannelsInfo, NDqProto::TEvChannelsInfo, TDqComputeEvents::EvChannelsInfo> {}; diff --git a/ydb/library/yql/dq/actors/compute/dq_compute_actor_async_io.h b/ydb/library/yql/dq/actors/compute/dq_compute_actor_async_io.h index b553e7fc20..f1fd8dddf1 100644 --- a/ydb/library/yql/dq/actors/compute/dq_compute_actor_async_io.h +++ b/ydb/library/yql/dq/actors/compute/dq_compute_actor_async_io.h @@ -31,6 +31,31 @@ class TProgramBuilder; namespace NYql::NDq { +enum class EResumeSource : ui32 { + Default, + ChannelsHandleWork, + ChannelsHandleUndeliveredData, + ChannelsHandleUndeliveredAck, + AsyncPopFinished, + CheckpointRegister, + CheckpointInject, + CABootstrap, + CABootstrapWakeup, + CAPendingInput, + CATakeInput, + CASinkFinished, + CATransformFinished, + CAStart, + CAPollAsync, + CAPollAsyncNoSpace, + CANewAsyncInput, + CADataSent, + CAPendingOutput, + CATaskRunnerCreated, + + Last, +}; + struct IMemoryQuotaManager { using TPtr = std::shared_ptr<IMemoryQuotaManager>; using TWeakPtr = std::weak_ptr<IMemoryQuotaManager>; @@ -136,7 +161,7 @@ struct IDqComputeActorAsyncInput { // 8. When checkpoint is written into database, checkpoints actor calls IDqComputeActorAsyncOutput::CommitState() to apply all side effects. struct IDqComputeActorAsyncOutput { struct ICallbacks { // Compute actor - virtual void ResumeExecution() = 0; + virtual void ResumeExecution(EResumeSource source = EResumeSource::Default) = 0; virtual void OnAsyncOutputError(ui64 outputIndex, const TIssues& issues, NYql::NDqProto::StatusIds::StatusCode fatalCode) = 0; // Checkpointing diff --git a/ydb/library/yql/dq/actors/compute/dq_compute_actor_channels.cpp b/ydb/library/yql/dq/actors/compute/dq_compute_actor_channels.cpp index 7d2add26b5..da1fc9371a 100644 --- a/ydb/library/yql/dq/actors/compute/dq_compute_actor_channels.cpp +++ b/ydb/library/yql/dq/actors/compute/dq_compute_actor_channels.cpp @@ -206,7 +206,7 @@ void TDqComputeActorChannels::HandleWork(TEvDqCompute::TEvChannelDataAck::TPtr& } LOG_T("Resume compute actor"); - Cbs->ResumeExecution(); + Cbs->ResumeExecution(EResumeSource::ChannelsHandleWork); } void TDqComputeActorChannels::HandleWork(TEvDqCompute::TEvRetryChannelData::TPtr& ev) { @@ -387,7 +387,7 @@ void TDqComputeActorChannels::HandleUndeliveredEvChannelData(ui64 channelId, NAc if (outputChannel.Finished && outputChannel.EarlyFinish && !SupportCheckpoints) { LOG_I("Ignore undelivered TEvChannelData event due to early finish, channelId: " << channelId); outputChannel.InFlight.clear(); - Cbs->ResumeExecution(); + Cbs->ResumeExecution(EResumeSource::ChannelsHandleUndeliveredData); return; } @@ -418,7 +418,7 @@ void TDqComputeActorChannels::HandleUndeliveredEvChannelDataAck(ui64 channelId, LOG_I("Handle undelivered event: TEvChannelDataAck, channelId: " << channelId << ", reason: " << reason << ". Ignore, channel is finished."); inputChannel.InFlight.clear(); - Cbs->ResumeExecution(); + Cbs->ResumeExecution(EResumeSource::ChannelsHandleUndeliveredAck); return; } diff --git a/ydb/library/yql/dq/actors/compute/dq_compute_actor_channels.h b/ydb/library/yql/dq/actors/compute/dq_compute_actor_channels.h index 078c5bb023..e6ced72de0 100644 --- a/ydb/library/yql/dq/actors/compute/dq_compute_actor_channels.h +++ b/ydb/library/yql/dq/actors/compute/dq_compute_actor_channels.h @@ -66,7 +66,7 @@ public: virtual i64 GetInputChannelFreeSpace(ui64 channelId) const = 0; virtual void TakeInputChannelData(TChannelDataOOB&& channelData, bool ack) = 0; virtual void PeerFinished(ui64 channelId) = 0; - virtual void ResumeExecution() = 0; + virtual void ResumeExecution(EResumeSource source) = 0; virtual ~ICallbacks() = default; }; diff --git a/ydb/library/yql/dq/actors/compute/dq_compute_actor_checkpoints.cpp b/ydb/library/yql/dq/actors/compute/dq_compute_actor_checkpoints.cpp index 04195639d2..c4c3272ef6 100644 --- a/ydb/library/yql/dq/actors/compute/dq_compute_actor_checkpoints.cpp +++ b/ydb/library/yql/dq/actors/compute/dq_compute_actor_checkpoints.cpp @@ -244,7 +244,7 @@ void TDqComputeActorCheckpoints::Handle(TEvDqCompute::TEvInjectCheckpoint::TPtr& StartCheckpoint(ev->Get()->Record.GetCheckpoint()); LOG_PCP_D("TEvInjectCheckpoint"); - ComputeActor->ResumeExecution(); + ComputeActor->ResumeExecution(EResumeSource::CheckpointInject); } void TDqComputeActorCheckpoints::Handle(TEvDqCompute::TEvSaveTaskStateResult::TPtr& ev) { @@ -473,7 +473,7 @@ void TDqComputeActorCheckpoints::RegisterCheckpoint(const NDqProto::TCheckpoint& YQL_ENSURE(PendingCheckpoint.Checkpoint->GetId() == checkpoint.GetId()); } LOG_PCP_D("Got checkpoint barrier from channel " << channelId); - ComputeActor->ResumeExecution(); + ComputeActor->ResumeExecution(EResumeSource::CheckpointRegister); } void TDqComputeActorCheckpoints::StartCheckpoint(const NDqProto::TCheckpoint& checkpoint) { diff --git a/ydb/library/yql/dq/actors/compute/dq_compute_actor_checkpoints.h b/ydb/library/yql/dq/actors/compute/dq_compute_actor_checkpoints.h index 9696f9bc61..2c01975e46 100644 --- a/ydb/library/yql/dq/actors/compute/dq_compute_actor_checkpoints.h +++ b/ydb/library/yql/dq/actors/compute/dq_compute_actor_checkpoints.h @@ -76,7 +76,7 @@ public: virtual void Start() = 0; virtual void Stop() = 0; - virtual void ResumeExecution() = 0; + virtual void ResumeExecution(EResumeSource source) = 0; virtual void LoadState(NDqProto::TComputeActorState&& state) = 0; diff --git a/ydb/library/yql/dq/actors/compute/dq_compute_actor_impl.h b/ydb/library/yql/dq/actors/compute/dq_compute_actor_impl.h index 73d12d8077..8d12ae0c85 100644 --- a/ydb/library/yql/dq/actors/compute/dq_compute_actor_impl.h +++ b/ydb/library/yql/dq/actors/compute/dq_compute_actor_impl.h @@ -256,7 +256,7 @@ protected: } STFUNC(BaseStateFuncBody) { - MetricsReporter.ReportEvent(ev->GetTypeRewrite()); + MetricsReporter.ReportEvent(ev->GetTypeRewrite(), ev); switch (ev->GetTypeRewrite()) { hFunc(TEvDqCompute::TEvResumeExecution, HandleExecuteBase); @@ -426,7 +426,7 @@ protected: // n.b. if c) is not satisfied we will also call ContinueExecute on branch // "status != ERunStatus::Finished -> !pollSent -> ProcessOutputsState.DataWasSent" // but idk what is the logic behind this - ContinueExecute(); + ContinueExecute(EResumeSource::CAPendingInput); return; } @@ -442,7 +442,7 @@ protected: } if (!pollSent) { if (ProcessOutputsState.DataWasSent) { - ContinueExecute(); + ContinueExecute(EResumeSource::CADataSent); } return; } @@ -451,7 +451,7 @@ protected: if (status == ERunStatus::PendingOutput) { if (ProcessOutputsState.DataWasSent) { // we have sent some data, so we have space in output channel(s) - ContinueExecute(); + ContinueExecute(EResumeSource::CAPendingOutput); } return; } @@ -649,10 +649,10 @@ protected: return std::move(log); } - void ContinueExecute() { + void ContinueExecute(EResumeSource source = EResumeSource::Default) { if (!ResumeEventScheduled && Running) { ResumeEventScheduled = true; - this->Send(this->SelfId(), new TEvDqCompute::TEvResumeExecution()); + this->Send(this->SelfId(), new TEvDqCompute::TEvResumeExecution{source}); } } @@ -694,7 +694,7 @@ public: Channels->SendChannelDataAck(channel->GetChannelId(), channel->GetFreeSpace()); } - ContinueExecute(); + ContinueExecute(EResumeSource::CATakeInput); } void PeerFinished(ui64 channelId) override { @@ -718,8 +718,8 @@ public: DoExecute(); } - void ResumeExecution() override { - ContinueExecute(); + void ResumeExecution(EResumeSource source) override { + ContinueExecute(source); } void OnSinkStateSaved(NDqProto::TSinkState&& state, ui64 outputIndex, const NDqProto::TCheckpoint& checkpoint) override { @@ -734,12 +734,12 @@ public: void OnSinkFinished(ui64 outputIndex) override { SinksMap.at(outputIndex).FinishIsAcknowledged = true; - ContinueExecute(); + ContinueExecute(EResumeSource::CASinkFinished); } void OnTransformFinished(ui64 outputIndex) override { OutputTransformsMap.at(outputIndex).FinishIsAcknowledged = true; - ContinueExecute(); + ContinueExecute(EResumeSource::CATransformFinished); } protected: @@ -876,7 +876,7 @@ protected: void Start() override { Running = true; State = NDqProto::COMPUTE_STATE_EXECUTING; - ContinueExecute(); + ContinueExecute(EResumeSource::CAStart); } void Stop() override { @@ -1724,7 +1724,7 @@ protected: // If we have read some data, we must run such reading again // to process the case when async input notified us about new data // but we haven't read all of it. - ContinueExecute(); + ContinueExecute(EResumeSource::CAPollAsync); } MetricsReporter.ReportAsyncInputData(inputIndex, batch.RowCount(), space, watermark); @@ -1743,7 +1743,7 @@ protected: AsyncInputPush(std::move(batch), info, space, finished); } else { CA_LOG_T("Skip polling async input[" << inputIndex << "]: no free space: " << freeSpace); - ContinueExecute(); // If there is no free space in buffer, => we have something to process + ContinueExecute(EResumeSource::CAPollAsyncNoSpace); // If there is no free space in buffer, => we have something to process } } @@ -1772,7 +1772,7 @@ protected: SourceCpuTimeMs->Add(cpuTimeDelta.MilliSeconds()); } CpuTimeSpent += cpuTimeDelta; - ContinueExecute(); + ContinueExecute(EResumeSource::CANewAsyncInput); } void OnAsyncInputError(const IDqComputeActorAsyncInput::TEvAsyncInputError::TPtr& ev) { diff --git a/ydb/library/yql/dq/actors/compute/dq_compute_actor_metrics.cpp b/ydb/library/yql/dq/actors/compute/dq_compute_actor_metrics.cpp index 62315dbbae..8368d4c5ea 100644 --- a/ydb/library/yql/dq/actors/compute/dq_compute_actor_metrics.cpp +++ b/ydb/library/yql/dq/actors/compute/dq_compute_actor_metrics.cpp @@ -32,7 +32,6 @@ TDqComputeActorMetrics::TDqComputeActorMetrics( #define ADD_COUNTER(name) \ name = ComputeActorSubgroup->GetCounter(#name) - ADD_COUNTER(ResumeExecution); ADD_COUNTER(ChannelsInfo); ADD_COUNTER(AbortExecution); ADD_COUNTER(Wakeup); @@ -50,19 +49,27 @@ TDqComputeActorMetrics::TDqComputeActorMetrics( ADD_COUNTER(NewAsyncInputDataArrived); ADD_COUNTER(AsyncInputError); ADD_COUNTER(OtherEvent); + ADD_COUNTER(ResumeExecutionTot); #undef ADD_COUNTER + + for (ui32 i = 0; i < static_cast<ui32>(EResumeSource::Last); ++i) { + ResumeExecution[i] = ComputeActorSubgroup->GetCounter(TStringBuilder() << "ResumeExecution" << i); + } } } -void TDqComputeActorMetrics::ReportEvent(ui32 type) +void TDqComputeActorMetrics::ReportEvent(ui32 type, TAutoPtr<NActors::IEventHandle>& ev) { if (!Enable) { return; } switch (type) { - case TEvDqCompute::TEvResumeExecution::EventType: ResumeExecution->Inc(); break; + case TEvDqCompute::TEvResumeExecution::EventType: + ResumeExecution[static_cast<ui32>((*reinterpret_cast<TEvDqCompute::TEvResumeExecution::TPtr*>(&ev))->Get()->Source)]->Inc(); + ResumeExecutionTot->Inc(); + break; case TEvDqCompute::TEvChannelsInfo::EventType: ChannelsInfo->Inc(); break; case TEvDq::TEvAbortExecution::EventType: AbortExecution->Inc(); break; case NActors::TEvents::TEvWakeup::EventType: Wakeup->Inc(); break; diff --git a/ydb/library/yql/dq/actors/compute/dq_compute_actor_metrics.h b/ydb/library/yql/dq/actors/compute/dq_compute_actor_metrics.h index cd08dd9a4e..6044acd65d 100644 --- a/ydb/library/yql/dq/actors/compute/dq_compute_actor_metrics.h +++ b/ydb/library/yql/dq/actors/compute/dq_compute_actor_metrics.h @@ -4,13 +4,15 @@ #include <util/generic/hash.h> +#include "dq_compute_actor_async_io.h" + namespace NYql::NDq { struct TDqComputeActorMetrics { public: TDqComputeActorMetrics(const NMonitoring::TDynamicCounterPtr& counters); - void ReportEvent(ui32 type); + void ReportEvent(ui32 type, TAutoPtr<NActors::IEventHandle>& ev); void ReportAsyncInputData(ui32 id, ui64 rows, ui64 bytes, TMaybe<TInstant> watermark); void ReportInputChannelWatermark(ui32 id, ui64 dataSize, TMaybe<TInstant> watermark); void ReportInjectedToTaskRunnerWatermark(TInstant watermark); @@ -33,7 +35,8 @@ private: NMonitoring::THistogramPtr InputRows; NMonitoring::THistogramPtr InputBytes; - NMonitoring::TDynamicCounters::TCounterPtr ResumeExecution; + NMonitoring::TDynamicCounters::TCounterPtr ResumeExecutionTot; + NMonitoring::TDynamicCounters::TCounterPtr ResumeExecution[static_cast<ui32>(EResumeSource::Last)]; NMonitoring::TDynamicCounters::TCounterPtr ChannelsInfo; NMonitoring::TDynamicCounters::TCounterPtr AbortExecution; NMonitoring::TDynamicCounters::TCounterPtr Wakeup; diff --git a/ydb/library/yql/providers/common/ut_helpers/dq_fake_ca.h b/ydb/library/yql/providers/common/ut_helpers/dq_fake_ca.h index 8bee231a43..0726b14378 100644 --- a/ydb/library/yql/providers/common/ut_helpers/dq_fake_ca.h +++ b/ydb/library/yql/providers/common/ut_helpers/dq_fake_ca.h @@ -89,7 +89,7 @@ class TFakeActor : public NActors::TActor<TFakeActor> { struct TAsyncOutputCallbacks : public IDqComputeActorAsyncOutput::ICallbacks { explicit TAsyncOutputCallbacks(TFakeActor& parent) : Parent(parent) {} - void ResumeExecution() override { + void ResumeExecution(EResumeSource) override { Parent.AsyncOutputPromises.ResumeExecution.SetValue(); Parent.AsyncOutputPromises.ResumeExecution = NThreading::NewPromise(); }; diff --git a/ydb/library/yql/providers/dq/actors/worker_actor.cpp b/ydb/library/yql/providers/dq/actors/worker_actor.cpp index 1eef8ee582..5b1a7c0bf4 100644 --- a/ydb/library/yql/providers/dq/actors/worker_actor.cpp +++ b/ydb/library/yql/providers/dq/actors/worker_actor.cpp @@ -261,7 +261,7 @@ private: limits.ChannelBufferSize = 20_MB; limits.OutputChunkMaxSize = 2_MB; - auto wakeup = [this]{ ResumeExecution(); }; + auto wakeup = [this]{ ResumeExecution(EResumeSource::Default); }; std::shared_ptr<IDqTaskRunnerExecutionContext> execCtx = std::make_shared<TDqTaskRunnerExecutionContext>( TraceId, UseSpilling, std::move(wakeup), TlsActivationContext->AsActorContext()); @@ -722,7 +722,7 @@ private: } /*_________________________________________________________*/ /*______________________ Sink Events ----__________________*/ - void ResumeExecution() override { + void ResumeExecution(EResumeSource) override { Send(SelfId(), new TEvContinueRun()); } diff --git a/ydb/library/yql/tests/common/test_framework/yqlrun.py b/ydb/library/yql/tests/common/test_framework/yqlrun.py index 70e25efa84..627e056c48 100644 --- a/ydb/library/yql/tests/common/test_framework/yqlrun.py +++ b/ydb/library/yql/tests/common/test_framework/yqlrun.py @@ -85,7 +85,7 @@ class YQLRun(object): def yql_exec(self, program=None, program_file=None, files=None, urls=None, run_sql=False, verbose=False, check_error=True, tables=None, pretty_plan=True, - wait=True, parameters={}, extra_env={}): + wait=True, parameters={}, extra_env={}, require_udf_resolver=False): del pretty_plan res_dir = self.res_dir @@ -235,7 +235,8 @@ class YQLRun(object): else: cmd += '--run ' - if yql_utils.get_param('UDF_RESOLVER'): + if yql_utils.get_param('UDF_RESOLVER') or require_udf_resolver: + assert self.udf_resolver_binary, "Missing udf_resolver binary" cmd += '--udf-resolver=' + self.udf_resolver_binary + ' --scan-udfs ' if not yatest.common.context.sanitize: cmd += ' --udf-resolver-filter-syscalls ' diff --git a/ydb/library/yql/tests/common/udf_test/test.py b/ydb/library/yql/tests/common/udf_test/test.py index 117a31ce51..0487104bb5 100644 --- a/ydb/library/yql/tests/common/udf_test/test.py +++ b/ydb/library/yql/tests/common/udf_test/test.py @@ -88,7 +88,8 @@ def test(case): tables=in_tables, files=files, check_error=not xfail, - extra_env=extra_env + extra_env=extra_env, + require_udf_resolver=False ) if xfail: diff --git a/ydb/library/yql/tools/udf_resolver/udf_resolver.cpp b/ydb/library/yql/tools/udf_resolver/udf_resolver.cpp index 0aeb950d48..c50bc77fb7 100644 --- a/ydb/library/yql/tools/udf_resolver/udf_resolver.cpp +++ b/ydb/library/yql/tools/udf_resolver/udf_resolver.cpp @@ -295,6 +295,7 @@ int main(int argc, char **argv) { Allow(brk), Allow(chdir), Allow(clock_gettime), + Allow(clock_nanosleep), Allow(clone), Allow(close), #ifndef _arm64_ diff --git a/ydb/library/yql/udfs/common/math/test/canondata/result.json b/ydb/library/yql/udfs/common/math/test/canondata/result.json new file mode 100644 index 0000000000..9c2bdb5500 --- /dev/null +++ b/ydb/library/yql/udfs/common/math/test/canondata/result.json @@ -0,0 +1,30 @@ +{ + "test.test[ErfInvNoLLVM]": [ + { + "checksum": "be26c6ffe8018b2afe5c6bd554d4468d", + "size": 6789, + "uri": "https://storage.yandex-team.ru/get-devtools/1937429/e098bd35e45d8b2d18b6958e5e9b0a875d6a03f1/resource.tar.gz#test.test_ErfInvNoLLVM_/results.txt" + } + ], + "test.test[ErfInv]": [ + { + "checksum": "be26c6ffe8018b2afe5c6bd554d4468d", + "size": 6789, + "uri": "https://storage.yandex-team.ru/get-devtools/1937429/e098bd35e45d8b2d18b6958e5e9b0a875d6a03f1/resource.tar.gz#test.test_ErfInv_/results.txt" + } + ], + "test.test[IR]": [ + { + "checksum": "83061f69c401182342478cbc8fd11b63", + "size": 10168, + "uri": "https://storage.yandex-team.ru/get-devtools/1937429/e098bd35e45d8b2d18b6958e5e9b0a875d6a03f1/resource.tar.gz#test.test_IR_/results.txt" + } + ], + "test.test[IR_LLVM_OFF]": [ + { + "checksum": "83061f69c401182342478cbc8fd11b63", + "size": 10168, + "uri": "https://storage.yandex-team.ru/get-devtools/1937429/e098bd35e45d8b2d18b6958e5e9b0a875d6a03f1/resource.tar.gz#test.test_IR_LLVM_OFF_/results.txt" + } + ] +} diff --git a/ydb/library/yql/udfs/common/math/test/cases/ErfInv.sql b/ydb/library/yql/udfs/common/math/test/cases/ErfInv.sql new file mode 100644 index 0000000000..798e0090ad --- /dev/null +++ b/ydb/library/yql/udfs/common/math/test/cases/ErfInv.sql @@ -0,0 +1,24 @@ +pragma config.flags("ValidateUdf", "None"); + +SELECT + Math::ErfInv(1e-8), + Math::ErfInv(1e-4), + Math::ErfInv(0.1), + Math::ErfInv(0.25), + Math::ErfInv(0.5), + Math::ErfInv(0.75), + Math::ErfInv(0.9), + Math::ErfInv(0.99), + Math::ErfInv(0.9999999), + Math::ErfInv(0.99999999), + Math::ErfInv(0.999999999), + Math::ErfInv(0.9999999999), + Math::ErfInv(0), + Math::ErfInv(1 + Math::Eps()), + Math::ErfInv(-1 - Math::Eps()), + Math::ErfInv(1), + Math::ErfInv(-1), + Math::ErfcInv(2), + Math::ErfcInv(0), + Math::ErfcInv(2 + 2 * Math::Eps()), + Math::ErfcInv(-Math::Eps()); diff --git a/ydb/library/yql/udfs/common/math/test/cases/ErfInvNoLLVM.sql b/ydb/library/yql/udfs/common/math/test/cases/ErfInvNoLLVM.sql new file mode 100644 index 0000000000..55e7c60db1 --- /dev/null +++ b/ydb/library/yql/udfs/common/math/test/cases/ErfInvNoLLVM.sql @@ -0,0 +1,25 @@ +pragma config.flags("ValidateUdf", "None"); +pragma config.flags("LLVM_OFF"); + +SELECT + Math::ErfInv(1e-8), + Math::ErfInv(1e-4), + Math::ErfInv(0.1), + Math::ErfInv(0.25), + Math::ErfInv(0.5), + Math::ErfInv(0.75), + Math::ErfInv(0.9), + Math::ErfInv(0.99), + Math::ErfInv(0.9999999), + Math::ErfInv(0.99999999), + Math::ErfInv(0.999999999), + Math::ErfInv(0.9999999999), + Math::ErfInv(0), + Math::ErfInv(1 + Math::Eps()), + Math::ErfInv(-1 - Math::Eps()), + Math::ErfInv(1), + Math::ErfInv(-1), + Math::ErfcInv(2), + Math::ErfcInv(0), + Math::ErfcInv(2 + 2 * Math::Eps()), + Math::ErfcInv(-Math::Eps()); diff --git a/ydb/library/yql/udfs/common/math/test/cases/IR.in b/ydb/library/yql/udfs/common/math/test/cases/IR.in new file mode 100644 index 0000000000..2a8e728cae --- /dev/null +++ b/ydb/library/yql/udfs/common/math/test/cases/IR.in @@ -0,0 +1,4 @@ +{"key"="023";"subkey"="3";"value"="aaa"}; +{"key"="037";"subkey"="5";"value"="ddd"}; +{"key"="075";"subkey"="1";"value"="abc"}; +{"key"="150";"subkey"="1";"value"="aaa"}; diff --git a/ydb/library/yql/udfs/common/math/test/cases/IR.sql b/ydb/library/yql/udfs/common/math/test/cases/IR.sql new file mode 100644 index 0000000000..7dbea3a2dc --- /dev/null +++ b/ydb/library/yql/udfs/common/math/test/cases/IR.sql @@ -0,0 +1,26 @@ +/* syntax version 1 */ +pragma config.flags("ValidateUdf", "None"); + +select + Math::Pi(), + Math::E(), + Math::Eps(), + Math::Abs(-2.34), + Math::Cos(0.234), + Math::IsFinite(0.0), + Math::IsNaN(0.0/0.0), + Math::Pow(2.0, 3.0), + Math::Log(4.0), + Math::Log(-4.0), + Math::Sigmoid(0.5), + Math::FuzzyEquals(1 + 0.0, 1 + 1.0e-200), + Math::FuzzyEquals(1.0001, 1.00012, 0.01 as Epsilon), + Math::Round(34.4564, -2 as Precision), + Math::Exp2(3.4), + Math::Exp(3.4), + Math::Erf(0.4), + Math::Mod(-1, 7), + Math::Mod(-1, 0), + Math::Rem(-1, 7), + Math::Rem(-1, 0) +from Input; diff --git a/ydb/library/yql/udfs/common/math/test/cases/IR_LLVM_OFF.in b/ydb/library/yql/udfs/common/math/test/cases/IR_LLVM_OFF.in new file mode 100644 index 0000000000..2a8e728cae --- /dev/null +++ b/ydb/library/yql/udfs/common/math/test/cases/IR_LLVM_OFF.in @@ -0,0 +1,4 @@ +{"key"="023";"subkey"="3";"value"="aaa"}; +{"key"="037";"subkey"="5";"value"="ddd"}; +{"key"="075";"subkey"="1";"value"="abc"}; +{"key"="150";"subkey"="1";"value"="aaa"}; diff --git a/ydb/library/yql/udfs/common/math/test/cases/IR_LLVM_OFF.sql b/ydb/library/yql/udfs/common/math/test/cases/IR_LLVM_OFF.sql new file mode 100644 index 0000000000..a8c14225ba --- /dev/null +++ b/ydb/library/yql/udfs/common/math/test/cases/IR_LLVM_OFF.sql @@ -0,0 +1,27 @@ +/* syntax version 1 */ +pragma config.flags("ValidateUdf", "None"); +pragma config.flags("LLVM_OFF"); + +select + Math::Pi(), + Math::E(), + Math::Eps(), + Math::Abs(-2.34), + Math::Cos(0.234), + Math::IsFinite(0.0), + Math::IsNaN(0.0/0.0), + Math::Pow(2.0, 3.0), + Math::Log(4.0), + Math::Log(-4.0), + Math::Sigmoid(0.5), + Math::FuzzyEquals(1 + 0.0, 1 + 1.0e-200), + Math::FuzzyEquals(1.0001, 1.00012, 0.01 as Epsilon), + Math::Round(34.4564, -2 as Precision), + Math::Exp2(3.4), + Math::Exp(3.4), + Math::Erf(0.4), + Math::Mod(-1, 7), + Math::Mod(-1, 0), + Math::Rem(-1, 7), + Math::Rem(-1, 0) +from Input; diff --git a/ydb/library/yql/udfs/common/math/test/ya.make b/ydb/library/yql/udfs/common/math/test/ya.make new file mode 100644 index 0000000000..0af0072313 --- /dev/null +++ b/ydb/library/yql/udfs/common/math/test/ya.make @@ -0,0 +1,13 @@ +YQL_UDF_YDB_TEST() + +DEPENDS(ydb/library/yql/udfs/common/math) + +TIMEOUT(300) + +SIZE(MEDIUM) + +IF (SANITIZER_TYPE == "memory") + TAG(ya:not_autocheck) # YQL-15385 +ENDIF() + +END() diff --git a/ydb/library/yql/udfs/common/math/ya.make b/ydb/library/yql/udfs/common/math/ya.make index 49efad201c..bafec1d3ed 100644 --- a/ydb/library/yql/udfs/common/math/ya.make +++ b/ydb/library/yql/udfs/common/math/ya.make @@ -67,3 +67,8 @@ PEERDIR( ) END() + +RECURSE_FOR_TESTS( + test +) + diff --git a/ydb/library/yql/utils/simd/exec/pack_tuple/main.cpp b/ydb/library/yql/utils/simd/exec/pack_tuple/main.cpp index 0c1cba5471..9a58552bea 100644 --- a/ydb/library/yql/utils/simd/exec/pack_tuple/main.cpp +++ b/ydb/library/yql/utils/simd/exec/pack_tuple/main.cpp @@ -21,7 +21,7 @@ struct TPerfomancer { ui8* ShuffleMask(ui32 v[8]) { ui8* det = new ui8[32]; for (size_t i = 0; i < 32; i += 1) { - det[i] = v[i / 4] + i % 4; + det[i] = v[i / 4] == ui32(-1) ? ui8(-1) : 4 * v[i / 4] + i % 4; } return det; } @@ -48,12 +48,12 @@ struct TPerfomancer { TSimd<ui8> permReg11, permReg21; TSimd<ui8> permReg12, permReg22; - TSimd<ui8> permIdx11(ShuffleMask((ui32[8]) {0, 0, 0, 0, 1, 0, 0, 0})); - TSimd<ui8> permIdx12(ShuffleMask((ui32[8]) {0, 0, 0, 0, 3, 0, 0, 2})); - TSimd<ui8> permIdx1f(ShuffleMask((ui32[8]) {7, 7, 7, 7, 7, 6, 5, 4})); + TSimd<ui8> permIdx11(ShuffleMask((ui32[8]) {0, 0, 0, 1, 0, 0, 0, 0})); + TSimd<ui8> permIdx12(ShuffleMask((ui32[8]) {2, 0, 0, 3, 0, 0, 0, 0})); + TSimd<ui8> permIdx1f(ShuffleMask((ui32[8]) {4, 5, 6, 7, 7, 7, 7, 7})); - TSimd<ui8> permIdx21(ShuffleMask((ui32[8]) {0, 0, 3, 2, 0, 1, 0, 0})); - TSimd<ui8> permIdx22(ShuffleMask((ui32[8]) {0, 0, 7, 6, 0, 5, 4, 0})); + TSimd<ui8> permIdx21(ShuffleMask((ui32[8]) {0, 0, 1, 0, 2, 3, 0, 0})); + TSimd<ui8> permIdx22(ShuffleMask((ui32[8]) {0, 4, 5, 0, 6, 7, 0, 0})); ui32 val1[8], val2[8]; // val3[8]; @@ -86,7 +86,7 @@ struct TPerfomancer { readReg2 = TSimd<ui8>((ui8*) addr2); addr2++; permReg21 = readReg2.Shuffle(permIdx21); - blended1 = permReg11.template Blend16<blendMask>(permReg21); + blended1 = permReg11.template Blend32<blendMask>(permReg21); blended1.Store((ui8*) val1); hash1 = TSimd<ui8>::CRC32u32(0, val1[0]); @@ -97,7 +97,7 @@ struct TPerfomancer { permReg12 = readReg1.Shuffle(permIdx12); permReg22 = readReg2.Shuffle(permIdx22); - blended2 = permReg12.template Blend16<blendMask>(permReg12); + blended2 = permReg12.template Blend32<blendMask>(permReg22); blended2.Store((ui8*) val2); hash3 = TSimd<ui8>::CRC32u32(0, val2[0]); @@ -113,6 +113,10 @@ struct TPerfomancer { addr1++; } + + std::chrono::steady_clock::time_point end01 = + std::chrono::steady_clock::now(); + Cerr << "Loaded col1 "; readReg1.template Log<ui32>(Cerr); Cerr << "Loaded col2 "; @@ -124,10 +128,6 @@ struct TPerfomancer { Cerr << "Blended "; blended1.template Log<ui32>(Cerr); - - std::chrono::steady_clock::time_point end01 = - std::chrono::steady_clock::now(); - ui64 microseconds = std::chrono::duration_cast<std::chrono::microseconds>(end01 - begin01).count(); if (log) { diff --git a/ydb/library/yql/utils/simd/exec/tuples_to_bucket/main.cpp b/ydb/library/yql/utils/simd/exec/tuples_to_bucket/main.cpp index 6945465ec9..3204e23492 100644 --- a/ydb/library/yql/utils/simd/exec/tuples_to_bucket/main.cpp +++ b/ydb/library/yql/utils/simd/exec/tuples_to_bucket/main.cpp @@ -85,13 +85,13 @@ struct TPerfomancer { readReg1 = TSimd<ui8>((ui8*) addr1); ui32 currOffset = offsets[bucketNum]; readReg1.Store((ui8*) (bufAddr + currOffset)); - offsets[bucketNum] += TTraits::Size / 8; + offsets[bucketNum] += 4; if (currOffset + 4 >= (BufSize / sizeof(ui64))) { offsets[bucketNum] = 0; bucketAddr = arrBuckets + bucketNum * (BucketSize / sizeof(ui64)); for (ui32 j = 0; j < BufSize / TTraits::Size; j++ ) { - readReg1 = TSimd<ui8>((ui8*) (bufAddr + TTraits::Size / 8 * j)); - readReg1.Load((ui8*) (bucketAddr + bigOffsets[bucketNum] + TTraits::Size / 8 * j)); + readReg1 = TSimd<ui8>((ui8*) (bufAddr + 4 * j)); + readReg1.StoreStream((ui8*) (bucketAddr + bigOffsets[bucketNum] + 4 * j)); } bigOffsets[bucketNum] += (BufSize / sizeof(ui64)); } diff --git a/ydb/library/yql/utils/simd/simd_avx2.h b/ydb/library/yql/utils/simd/simd_avx2.h index 4e03841843..c92ced8d74 100644 --- a/ydb/library/yql/utils/simd/simd_avx2.h +++ b/ydb/library/yql/utils/simd/simd_avx2.h @@ -11,84 +11,54 @@ namespace NSimd { namespace NAVX2 { -template <typename T> -struct TSimd8; - -template<typename Child> -struct TBase { +template<typename T> +struct TSimd8 { __m256i Value; - inline TBase() + static const int SIZE = 32; + + inline TSimd8() : Value{__m256i()} { } - inline TBase(const __m256i value) + inline TSimd8(const __m256i& value) : Value(value) { } - explicit inline operator const __m256i&() const { - return this->Value; - } - explicit inline operator __m256i&() { - return this->Value; + inline TSimd8(T value) { + this->Value = _mm256_set1_epi8(value); } - inline Child operator|(const Child other) const { - return _mm256_or_si256(this->Value, other.Value); - } - inline Child operator&(const Child other) const { - return _mm256_and_si256(this->Value, other.Value); - } - inline Child operator^(const Child other) const { - return _mm256_xor_si256(this->Value, other.Value); - } - inline Child BitAndNot(const Child other) const { - return _mm256_andnot_si256(this->Value, other.Value); - }; - inline Child& operator|=(const Child other) { - auto cast = static_cast<Child*>(*this); - *cast = *cast | other; - return *cast; - } - inline Child& operator&=(const Child other) { - auto cast = static_cast<Child*>(*this); - *cast = *cast & other; - return *cast; + inline TSimd8(const T values[32]) { + this->Value = _mm256_loadu_si256(reinterpret_cast<const __m256i *>(values)); }; - inline Child& operator^=(const Child other) { - auto cast = static_cast<Child*>(*this); - *cast = *cast ^ other; - return *cast; - }; -}; - -template<typename T, typename Mask=TSimd8<bool>> -struct TBase8: TBase<TSimd8<T>> { - inline TBase8() - : TBase<TSimd8<T>>() - { - } - - inline TBase8(const __m256i _value) - : TBase<TSimd8<T>>(_value) + inline TSimd8( + T v0, T v1, T v2, T v3, T v4, T v5, T v6, T v7, + T v8, T v9, T v10, T v11, T v12, T v13, T v14, T v15, + T v16, T v17, T v18, T v19, T v20, T v21, T v22, T v23, + T v24, T v25, T v26, T v27, T v28, T v29, T v30, T v31 + ) : TSimd8(_mm256_setr_epi8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10,v11,v12,v13,v14,v15, + v16,v17,v18,v19,v20,v21,v22,v23, + v24,v25,v26,v27,v28,v29,v30,v31 + )) { } - template<int N> - inline TSimd8<T> Blend16(const TSimd8<T> other) { - return _mm256_blend_epi16(this->Value, other.Value, N); + explicit inline operator const __m256i&() const { + return this->Value; } - - inline TSimd8<T> BlendVar(const TSimd8<T> other, const TSimd8<T> mask) { - return _mm256_blendv_epi8(this->Value, other.Value, mask.Value); + explicit inline operator __m256i&() { + return this->Value; } - static inline ui32 CRC32u8(ui32 crc, ui8 data) { + static inline ui32 CRC32u8(ui32 crc, ui32 data) { return _mm_crc32_u8(crc, data); } - static inline ui32 CRC32u16(ui32 crc, ui16 data) { + static inline ui32 CRC32u16(ui32 crc, ui32 data) { return _mm_crc32_u16(crc, data); } @@ -100,33 +70,12 @@ struct TBase8: TBase<TSimd8<T>> { return _mm_crc32_u64(crc, data); } - friend inline Mask operator==(const TSimd8<T> lhs, const TSimd8<T> rhs) { - return _mm256_cmpeq_epi8(lhs.Value, rhs.Value); - } - - static const int SIZE = sizeof(TBase<T>::Value); -}; - -template<> -struct TSimd8<bool>: TBase8<bool> { - - inline TSimd8<bool>() - : TBase8() - { - } - - inline TSimd8<bool>(const __m256i value) - : TBase8<bool>(value) - { - } - - inline TSimd8<bool>(bool value) - : TBase8<bool>(Set(value)) - { - } - - static inline TSimd8<bool> Set(bool value) { - return _mm256_set1_epi8(ui8(-(!!value))); + inline ui32 CRC32(ui32 crc) { + crc = _mm_crc32_u64(crc, *((ui64*) &this->Value)); + crc = _mm_crc32_u64(crc, *((ui64*) &this->Value + 1)); + crc = _mm_crc32_u64(crc, *((ui64*) &this->Value + 2)); + crc = _mm_crc32_u64(crc, *((ui64*) &this->Value + 3)); + return crc; } inline int ToBitMask() const { @@ -141,31 +90,16 @@ struct TSimd8<bool>: TBase8<bool> { return this->ToBitMask() == i32(0xFFFFFFFF); } - inline TSimd8<bool> operator~() const { - return *this ^ true; - } -}; - -template<typename T> -struct TBase8Numeric: TBase8<T> { - - inline TBase8Numeric() - : TBase8<T>() - { - } - inline TBase8Numeric(const __m256i value) - : TBase8<T>(value) - { - } - static inline TSimd8<T> Set(T value) { - return _mm256_set1_epi8(value); + return TSimd8<T>(value); } + static inline TSimd8<T> Zero() { return _mm256_setzero_si256(); } + static inline TSimd8<T> Load(const T values[32]) { - return _mm256_loadu_si256(reinterpret_cast<const __m256i *>(values)); + return TSimd8<T>(values); } static inline TSimd8<T> LoadAligned(const T values[32]) { @@ -188,7 +122,7 @@ struct TBase8Numeric: TBase8<T> { return _mm256_stream_si256(reinterpret_cast<__m256i *>(dst), this->Value); } - inline TSimd8<T> Shuffle(const TSimd8<T> other) const { + inline TSimd8<T> Shuffle(const TSimd8<T>& other) const { TSimd8<T> mask0(0x70, 0x70, 0x70, 0x70, 0x70, 0x70, 0x70, 0x70, 0x70, 0x70, 0x70, 0x70, 0x70, 0x70, 0x70, 0x70, 0xF0, 0xF0, 0xF0, 0xF0, 0xF0, 0xF0, 0xF0, 0xF0, 0xF0, 0xF0, 0xF0, 0xF0, 0xF0, 0xF0, 0xF0, 0xF0); TSimd8<T> mask1(0xF0, 0xF0, 0xF0, 0xF0, 0xF0, 0xF0, 0xF0, 0xF0, 0xF0, 0xF0, 0xF0, 0xF0, 0xF0, 0xF0, 0xF0, 0xF0, @@ -199,10 +133,24 @@ struct TBase8Numeric: TBase8<T> { return tmp & mask; } - inline TSimd8<T> Shuffle128(const TSimd8<T> other) const { + inline TSimd8<T> Shuffle128(const TSimd8<T>& other) const { return _mm256_shuffle_epi8(this->Value, other.Value); } + template<int N> + inline TSimd8<T> Blend16(const TSimd8<T>& other) { + return _mm256_blend_epi16(this->Value, other.Value, N); + } + + template<int N> + inline TSimd8<T> Blend32(const TSimd8<T>& other) { + return _mm256_blend_epi32(this->Value, other.Value, N); + } + + inline TSimd8<T> BlendVar(const TSimd8<T>& other, const TSimd8<T>& mask) { + return _mm256_blendv_epi8(this->Value, other.Value, mask.Value); + } + static inline TSimd8<T> Repeat16( T v0, T v1, T v2, T v3, T v4, T v5, T v6, T v7, T v8, T v9, T v10, T v11, T v12, T v13, T v14, T v15 @@ -238,174 +186,143 @@ struct TBase8Numeric: TBase8<T> { out << end; } - inline TSimd8<T> operator+(const TSimd8<T> other) const { - return _mm256_add_epi8(this->Value, other.Value); - } - inline TSimd8<T> operator-(const TSimd8<T> other) const { - return _mm256_sub_epi8(this->Value, other.Value); - } - inline TSimd8<T>& operator+=(const TSimd8<T> other) { - *this = *this + other; - return *static_cast<TSimd8<T>*>(this); - } - inline TSimd8<T>& operator-=(const TSimd8<T> other) { - *this = *this - other; - return *static_cast<TSimd8<T>*>(this); + inline TSimd8<T> MaxValue(const TSimd8<T>& other) const { + return _mm256_max_epu8(this->Value, other.Value); } - - // 0xFFu = 11111111 = 2^8 - 1 - inline TSimd8<T> operator~() const { - return *this ^ 0xFFu; + inline TSimd8<T> MinValue(const TSimd8<T>& other) const { + return _mm256_min_epu8(this->Value, other.Value); } -}; -template<> -struct TSimd8<i8> : TBase8Numeric<i8> { - inline TSimd8() - : TBase8Numeric<i8>() - { + inline TSimd8<bool> BitsNotSet() const { + return *this == ui8(0); } - inline TSimd8(const __m256i value) - : TBase8Numeric<i8>(value) - { + inline TSimd8<bool> AnyBitsSet() const { + return ~this->BitsNotSet(); } - inline TSimd8(i8 value) - : TSimd8(Set(value)) - { + inline bool BitsNotSetAnywhere() const { + return _mm256_testz_si256(this->Value, this->Value); } - inline TSimd8(const i8 values[32]) - : TSimd8(Load(values)) - { + inline bool AnyBitsSetAnywhere() const { + return !BitsNotSetAnywhere(); } - inline TSimd8( - i8 v0, i8 v1, i8 v2, i8 v3, i8 v4, i8 v5, i8 v6, i8 v7, - i8 v8, i8 v9, i8 v10, i8 v11, i8 v12, i8 v13, i8 v14, i8 v15, - i8 v16, i8 v17, i8 v18, i8 v19, i8 v20, i8 v21, i8 v22, i8 v23, - i8 v24, i8 v25, i8 v26, i8 v27, i8 v28, i8 v29, i8 v30, i8 v31 - ) : TSimd8(_mm256_setr_epi8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15, - v16,v17,v18,v19,v20,v21,v22,v23, - v24,v25,v26,v27,v28,v29,v30,v31 - )) - { + inline bool BitsNotSetAnywhere(const TSimd8<T>& bits) const { + return _mm256_testz_si256(this->Value, bits.Value); } - - inline static TSimd8<i8> Repeat16( - i8 v0, i8 v1, i8 v2, i8 v3, i8 v4, i8 v5, i8 v6, i8 v7, - i8 v8, i8 v9, i8 v10, i8 v11, i8 v12, i8 v13, i8 v14, i8 v15 - ) { - return TSimd8<i8>( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); + inline bool AnyBitsSetAnywhere(const TSimd8<T>& bits) const { + return !BitsNotSetAnywhere(bits); } - inline TSimd8<i8> MaxValue(const TSimd8<i8> other) const { - return _mm256_max_epi8(this->Value, other.Value); - } - inline TSimd8<i8> MinValue(const TSimd8<i8> other) const { - return _mm256_min_epi8(this->Value, other.Value); - } - inline TSimd8<bool> operator>(const TSimd8<i8> other) const { - return _mm256_cmpgt_epi8(this->Value, other.Value); + template<int N> + inline TSimd8<T> Shr() const { + return TSimd8<T>(_mm256_srli_epi16(this->Value, N)) & T(0xFFu >> N); } - inline TSimd8<bool> operator<(const TSimd8<i8> other) const { - return _mm256_cmpgt_epi8(other.Value, this->Value); + template<int N> + inline TSimd8<T> Shl() const { + return TSimd8<T>(_mm256_slli_epi16(this->Value, N)) & T(0xFFu << N); } -}; -template<> -struct TSimd8<ui8>: TBase8Numeric<ui8> { - inline TSimd8() - : TBase8Numeric<ui8>() - { - } - inline TSimd8(const __m256i _value) - : TBase8Numeric<ui8>(_value) - { - } - inline TSimd8(ui8 _value) - : TSimd8(Set(_value)) - { - } - inline TSimd8(const ui8 values[32]) - : TSimd8(Load(values)) - { + template<int N> + inline int GetBit() const { + return _mm256_movemask_epi8(_mm256_slli_epi16(this->Value, 7-N)); } - inline TSimd8( - ui8 v0, ui8 v1, ui8 v2, ui8 v3, ui8 v4, ui8 v5, ui8 v6, ui8 v7, - ui8 v8, ui8 v9, ui8 v10, ui8 v11, ui8 v12, ui8 v13, ui8 v14, ui8 v15, - ui8 v16, ui8 v17, ui8 v18, ui8 v19, ui8 v20, ui8 v21, ui8 v22, ui8 v23, - ui8 v24, ui8 v25, ui8 v26, ui8 v27, ui8 v28, ui8 v29, ui8 v30, ui8 v31 - ) : TSimd8(_mm256_setr_epi8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15, - v16,v17,v18,v19,v20,v21,v22,v23, - v24,v25,v26,v27,v28,v29,v30,v31 - )) {} - inline static TSimd8<ui8> Repeat16( - ui8 v0, ui8 v1, ui8 v2, ui8 v3, ui8 v4, ui8 v5, ui8 v6, ui8 v7, - ui8 v8, ui8 v9, ui8 v10, ui8 v11, ui8 v12, ui8 v13, ui8 v14, ui8 v15 - ) { - return TSimd8<ui8>( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); + friend inline TSimd8<bool> operator==(const TSimd8<T> &lhs, const TSimd8<T> &rhs) { + return _mm256_cmpeq_epi8(lhs.Value, rhs.Value); } - inline TSimd8<ui8> MaxValue(const TSimd8<ui8> other) const { - return _mm256_max_epu8(this->Value, other.Value); - } - inline TSimd8<ui8> MinValue(const TSimd8<ui8> other) const { - return _mm256_min_epu8(other.Value, this->Value); - } - inline TSimd8<bool> operator<=(const TSimd8<ui8> other) const { + inline TSimd8<bool> operator<=(const TSimd8<T> &other) { return other.MaxValue(*this) == other; } - inline TSimd8<bool> operator>=(const TSimd8<ui8> other) const { + + inline TSimd8<bool> operator>=(const TSimd8<T> &other) { return other.MinValue(*this) == other; } - inline TSimd8<bool> BitsNotSet() const { - return *this == ui8(0); + inline TSimd8<bool> operator>(const TSimd8<T> &other) { + return _mm256_cmpgt_epi8(*this, other); } - inline TSimd8<bool> AnyBitsSet() const { - return ~this->BitsNotSet(); + + inline TSimd8<bool> operator<(const TSimd8<T> &other) { + return _mm256_cmpgt_epi8(other, *this); } - inline bool BitsNotSetAnywhere() const { - return _mm256_testz_si256(this->Value, this->Value); + + inline TSimd8<T> operator|(const TSimd8<T>& other) const { + return _mm256_or_si256(this->Value, other.Value); } - inline bool AnyBitsSetAnywhere() const { - return !BitsNotSetAnywhere(); + inline TSimd8<T> operator&(const TSimd8<T>& other) const { + return _mm256_and_si256(this->Value, other.Value); } - inline bool BitsNotSetAnywhere(TSimd8<ui8> bits) const { - return _mm256_testz_si256(this->Value, bits.Value); + inline TSimd8<T> operator^(const TSimd8<T>& other) const { + return _mm256_xor_si256(this->Value, other.Value); } - inline bool AnyBitsSetAnywhere(TSimd8<ui8> bits) const { - return !BitsNotSetAnywhere(bits); + inline TSimd8<T> BitAndNot(const TSimd8<T>& other) const { + return _mm256_andnot_si256(this->Value, other.Value); + }; + inline TSimd8<T>& operator|=(const TSimd8<T>& other) { + *this = *this | other; + return *this; } + inline TSimd8<T>& operator&=(const TSimd8<T>& other) { + *this = *this & other; + return *this; + }; + inline TSimd8<T>& operator^=(const TSimd8<T>& other) { + *this = *this ^ other; + return *this; + }; - template<int N> - inline TSimd8<ui8> Shr() const { - return TSimd8<ui8>(_mm256_srli_epi16(this->Value, N)) & ui8(0xFFu >> N); + inline TSimd8<T> operator+(const TSimd8<T>& other) const { + return _mm256_add_epi8(this->Value, other.Value); } - template<int N> - inline TSimd8<ui8> Shl() const { - return TSimd8<ui8>(_mm256_slli_epi16(this->Value, N)) & ui8(0xFFu << N); + inline TSimd8<T> operator-(const TSimd8<T>& other) const { + return _mm256_sub_epi8(this->Value, other.Value); + } + inline TSimd8<T>& operator+=(const TSimd8<T>& other) { + *this = *this + other; + return *static_cast<TSimd8<T>*>(this); + } + inline TSimd8<T>& operator-=(const TSimd8<T>& other) { + *this = *this - other; + return *static_cast<TSimd8<T>*>(this); } - template<int N> - inline int GetBit() const { - return _mm256_movemask_epi8(_mm256_slli_epi16(this->Value, 7-N)); + // 0xFFu = 11111111 = 2^8 - 1 + inline TSimd8<T> operator~() const { + return *this ^ 0xFFu; } }; +template<> +inline TSimd8<bool> TSimd8<bool>::Set(bool value) { + return _mm256_set1_epi8(ui8(-value)); +} + +template<> +inline TSimd8<bool>::TSimd8(bool value) { + this->Value = _mm256_set1_epi8(ui8(-value)); +} + +template<> +inline TSimd8<i8> TSimd8<i8>::MaxValue(const TSimd8<i8>& other) const { + return _mm256_max_epi8(this->Value, other.Value); +} + +template<> +inline TSimd8<i8> TSimd8<i8>::MinValue(const TSimd8<i8>& other) const { + return _mm256_min_epi8(this->Value, other.Value); +} + +template<> +inline TSimd8<bool> TSimd8<ui8>::operator>(const TSimd8<ui8> &other) { + return TSimd8<ui8>(_mm256_subs_epu8(this->Value, other.Value)).AnyBitsSet(); +} + +template<> +inline TSimd8<bool> TSimd8<ui8>::operator<(const TSimd8<ui8> &other) { + return TSimd8<ui8>(_mm256_subs_epu8(other.Value, this->Value)).AnyBitsSet(); +} + + } } diff --git a/ydb/library/yql/utils/simd/simd_sse42.h b/ydb/library/yql/utils/simd/simd_sse42.h index a75eadcff9..696708bd2a 100644 --- a/ydb/library/yql/utils/simd/simd_sse42.h +++ b/ydb/library/yql/utils/simd/simd_sse42.h @@ -4,91 +4,58 @@ #include <immintrin.h> #include <util/system/types.h> -#include <util/generic/string.h> #include <util/stream/output.h> +#include <util/generic/string.h> #pragma clang attribute push(__attribute__((target("sse4.2"))), apply_to=function) namespace NSimd { namespace NSSE42 { -template <typename T> -struct TSimd8; - -template<typename Child> -struct TBase { +template<typename T> +struct TSimd8 { __m128i Value; - inline TBase() + static const int SIZE = 32; + + inline TSimd8() : Value{__m128i()} { } - inline TBase(const __m128i value) + inline TSimd8(const __m128i& value) : Value(value) { } - explicit inline operator const __m128i&() const { - return this->Value; - } - explicit inline operator __m128i&() { - return this->Value; + inline TSimd8(T value) { + this->Value = _mm_set1_epi8(value); } - inline Child operator|(const Child other) const { - return _mm_or_si128(this->Value, other.Value); - } - inline Child operator&(const Child other) const { - return _mm_and_si128(this->Value, other.Value); - } - inline Child operator^(const Child other) const { - return _mm_xor_si128(this->Value, other.Value); - } - inline Child BitAndNot(const Child other) const { - return _mm_andnot_si128(this->Value, other.Value); - }; - inline Child& operator|=(const Child other) { - auto cast = static_cast<Child*>(*this); - *cast = *cast | other; - return *cast; - } - inline Child& operator&=(const Child other) { - auto cast = static_cast<Child*>(*this); - *cast = *cast & other; - return *cast; - }; - inline Child& operator^=(const Child other) { - auto cast = static_cast<Child*>(*this); - *cast = *cast ^ other; - return *cast; + inline TSimd8(const T values[16]) { + this->Value = _mm_loadu_si128(reinterpret_cast<const __m128i *>(values)); }; -}; - -template<typename T, typename Mask=TSimd8<bool>> -struct TBase8: TBase<TSimd8<T>> { - inline TBase8() - : TBase<TSimd8<T>>() + inline TSimd8( + T v0, T v1, T v2, T v3, T v4, T v5, T v6, T v7, + T v8, T v9, T v10, T v11, T v12, T v13, T v14, T v15 + ) : TSimd8(_mm_setr_epi8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10,v11,v12,v13,v14,v15 + )) { } - inline TBase8(const __m128i value) - : TBase<TSimd8<T>>(value) - { + explicit inline operator const __m128i&() const { + return this->Value; } - - template<int N> - inline TSimd8<T> Blend16(const TSimd8<T> other) { - return _mm_blend_epi16(this->Value, other.Value, N); + explicit inline operator __m128i&() { + return this->Value; } - inline TSimd8<T> BlendVar(const TSimd8<T> other, const TSimd8<T> mask) { - return _mm_blendv_epi8(this->Value, other.Value, mask.Value); - } - static inline ui32 CRC32u8(ui32 crc, ui8 data) { + static inline ui32 CRC32u8(ui32 crc, ui32 data) { return _mm_crc32_u8(crc, data); } - static inline ui32 CRC32u16(ui32 crc, ui16 data) { + static inline ui32 CRC32u16(ui32 crc, ui32 data) { return _mm_crc32_u16(crc, data); } @@ -100,33 +67,12 @@ struct TBase8: TBase<TSimd8<T>> { return _mm_crc32_u64(crc, data); } - friend inline Mask operator==(const TSimd8<T> lhs, const TSimd8<T> rhs) { - return _mm_cmpeq_epi8(lhs.Value, rhs.Value); - } - - static const int SIZE = sizeof(TBase<T>::Value); -}; - -template<> -struct TSimd8<bool>: TBase8<bool> { - - inline TSimd8<bool>() - : TBase8() - { - } - - inline TSimd8<bool>(const __m128i value) - : TBase8<bool>(value) - { - } - - inline TSimd8<bool>(bool value) - : TBase8<bool>(Set(value)) - { - } - - static inline TSimd8<bool> Set(bool value) { - return _mm_set1_epi8(ui8(-(!!value))); + inline ui32 CRC32(ui32 crc) { + crc = _mm_crc32_u64(crc, *((ui64*) &this->Value)); + crc = _mm_crc32_u64(crc, *((ui64*) &this->Value + 1)); + crc = _mm_crc32_u64(crc, *((ui64*) &this->Value + 2)); + crc = _mm_crc32_u64(crc, *((ui64*) &this->Value + 3)); + return crc; } inline int ToBitMask() const { @@ -141,31 +87,16 @@ struct TSimd8<bool>: TBase8<bool> { return this->ToBitMask() == i32(0xFFFF); } - inline TSimd8<bool> operator~() const { - return *this ^ true; - } -}; - -template<typename T> -struct TBase8Numeric: TBase8<T> { - - inline TBase8Numeric() - : TBase8<T>() - { - } - inline TBase8Numeric(const __m128i value) - : TBase8<T>(value) - { - } - static inline TSimd8<T> Set(T value) { - return _mm_set1_epi8(value); + return TSimd8<T>(value); } + static inline TSimd8<T> Zero() { return _mm_setzero_si128(); } + static inline TSimd8<T> Load(const T values[16]) { - return _mm_loadu_si128(reinterpret_cast<const __m128i *>(values)); + return TSimd8<T>(values); } static inline TSimd8<T> LoadAligned(const T values[16]) { @@ -188,12 +119,27 @@ struct TBase8Numeric: TBase8<T> { return _mm_stream_si128(reinterpret_cast<__m128i *>(dst), this->Value); } - inline TSimd8<T> Shuffle(const TSimd8<T> other) const { + inline TSimd8<T> Shuffle(const TSimd8<T>& other) const { + return Shuffle128(other); + } + + + inline TSimd8<T> Shuffle128(const TSimd8<T>& other) const { return _mm_shuffle_epi8(this->Value, other.Value); } - inline TSimd8<T> Shuffle128(const TSimd8<T> other) const { - return Shuffle(other); + template<int N> + inline TSimd8<T> Blend16(const TSimd8<T>& other) { + return _mm_blend_epi16(this->Value, other.Value, N); + } + + template<int N> + inline TSimd8<T> Blend32(const TSimd8<T>& other) { + return _mm_blend_epi32(this->Value, other.Value, N); + } + + inline TSimd8<T> BlendVar(const TSimd8<T>& other, const TSimd8<T>& mask) { + return _mm_blendv_epi8(this->Value, other.Value, mask.Value); } static inline TSimd8<T> Repeat16( @@ -206,20 +152,16 @@ struct TBase8Numeric: TBase8<T> { ); } - - template<typename TOut> void Log(IOutputStream& out, TString delimeter = " ", TString end = "\n") { const size_t n = sizeof(this->Value) / sizeof(TOut); TOut buf[n]; - Store((T*) buf); + this->Store((T*) buf); if (n == sizeof(this->Value)) { for (size_t i = 0; i < n; i += 1) { out << int(buf[i]); if (i + 1 < n) { out << delimeter; - } else { - out << end; } } } else { @@ -227,172 +169,150 @@ struct TBase8Numeric: TBase8<T> { out << buf[i]; if (i + 1 < n) { out << delimeter; - } else { - out << end; } } } + out << end; } - inline TSimd8<T> operator+(const TSimd8<T> other) const { - return _mm_add_epi8(this->Value, other.Value); - } - inline TSimd8<T> operator-(const TSimd8<T> other) const { - return _mm_sub_epi8(this->Value, other.Value); - } - inline TSimd8<T>& operator+=(const TSimd8<T> other) { - *this = *this + other; - return *static_cast<TSimd8<T>*>(this); + inline TSimd8<T> MaxValue(const TSimd8<T>& other) const { + return _mm_max_epu8(this->Value, other.Value); } - inline TSimd8<T>& operator-=(const TSimd8<T> other) { - *this = *this - other; - return *static_cast<TSimd8<T>*>(this); + inline TSimd8<T> MinValue(const TSimd8<T>& other) const { + return _mm_min_epu8(this->Value, other.Value); } - // 0xFFu = 11111111 = 2^8 - 1 - inline TSimd8<T> operator~() const { - return *this ^ 0xFFu; - } -}; - -template<> -struct TSimd8<i8> : TBase8Numeric<i8> { - inline TSimd8() - : TBase8Numeric<i8>() - { + inline TSimd8<bool> BitsNotSet() const { + return *this == ui8(0); } - inline TSimd8(const __m128i value) - : TBase8Numeric<i8>(value) - { + inline TSimd8<bool> AnyBitsSet() const { + return ~this->BitsNotSet(); } - inline TSimd8(i8 value) - : TSimd8(Set(value)) - { + inline bool BitsNotSetAnywhere() const { + return _mm_testz_si128(this->Value, this->Value); } - inline TSimd8(const i8 values[16]) - : TSimd8(Load(values)) - { + inline bool AnyBitsSetAnywhere() const { + return !BitsNotSetAnywhere(); } - inline TSimd8( - i8 v0, i8 v1, i8 v2, i8 v3, i8 v4, i8 v5, i8 v6, i8 v7, - i8 v8, i8 v9, i8 v10, i8 v11, i8 v12, i8 v13, i8 v14, i8 v15 - ) : TSimd8(_mm_setr_epi8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - )) - { + inline bool BitsNotSetAnywhere(const TSimd8<T>& bits) const { + return _mm_testz_si128(this->Value, bits.Value); } - - inline static TSimd8<i8> Repeat16( - i8 v0, i8 v1, i8 v2, i8 v3, i8 v4, i8 v5, i8 v6, i8 v7, - i8 v8, i8 v9, i8 v10, i8 v11, i8 v12, i8 v13, i8 v14, i8 v15 - ) { - return TSimd8<i8>( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); + inline bool AnyBitsSetAnywhere(const TSimd8<T>& bits) const { + return !BitsNotSetAnywhere(bits); } - inline TSimd8<i8> MaxValue(const TSimd8<i8> other) const { - return _mm_max_epi8(this->Value, other.Value); - } - inline TSimd8<i8> MinValue(const TSimd8<i8> other) const { - return _mm_min_epi8(this->Value, other.Value); - } - inline TSimd8<bool> operator>(const TSimd8<i8> other) const { - return _mm_cmpgt_epi8(this->Value, other.Value); + template<int N> + inline TSimd8<T> Shr() const { + return TSimd8<T>(_mm_srli_epi16(this->Value, N)) & T(0xFFu >> N); } - inline TSimd8<bool> operator<(const TSimd8<i8> other) const { - return _mm_cmpgt_epi8(other.Value, this->Value); + template<int N> + inline TSimd8<T> Shl() const { + return TSimd8<T>(_mm_slli_epi16(this->Value, N)) & T(0xFFu << N); } -}; -template<> -struct TSimd8<ui8>: TBase8Numeric<ui8> { - inline TSimd8() - : TBase8Numeric<ui8>() - { - } - inline TSimd8(const __m128i _value) - : TBase8Numeric<ui8>(_value) - { - } - inline TSimd8(ui8 _value) - : TSimd8(Set(_value)) - { - } - inline TSimd8(const ui8 values[16]) - : TSimd8(Load(values)) - { + template<int N> + inline int GetBit() const { + return _mm_movemask_epi8(_mm_slli_epi16(this->Value, 7-N)); } - inline TSimd8( - ui8 v0, ui8 v1, ui8 v2, ui8 v3, ui8 v4, ui8 v5, ui8 v6, ui8 v7, - ui8 v8, ui8 v9, ui8 v10, ui8 v11, ui8 v12, ui8 v13, ui8 v14, ui8 v15 - ) : TSimd8(_mm_setr_epi8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - )) {} - inline static TSimd8<ui8> Repeat16( - ui8 v0, ui8 v1, ui8 v2, ui8 v3, ui8 v4, ui8 v5, ui8 v6, ui8 v7, - ui8 v8, ui8 v9, ui8 v10, ui8 v11, ui8 v12, ui8 v13, ui8 v14, ui8 v15 - ) { - return TSimd8<ui8>( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); + friend inline TSimd8<bool> operator==(const TSimd8<T> &lhs, const TSimd8<T> &rhs) { + return _mm_cmpeq_epi8(lhs.Value, rhs.Value); } - inline TSimd8<ui8> MaxValue(const TSimd8<ui8> other) const { - return _mm_max_epu8(this->Value, other.Value); - } - inline TSimd8<ui8> MinValue(const TSimd8<ui8> other) const { - return _mm_min_epu8(other.Value, this->Value); - } - inline TSimd8<bool> operator<=(const TSimd8<ui8> other) const { + inline TSimd8<bool> operator<=(const TSimd8<T> &other) { return other.MaxValue(*this) == other; } - inline TSimd8<bool> operator>=(const TSimd8<ui8> other) const { + + inline TSimd8<bool> operator>=(const TSimd8<T> &other) { return other.MinValue(*this) == other; } - inline TSimd8<bool> BitsNotSet() const { - return *this == ui8(0); + inline TSimd8<bool> operator>(const TSimd8<T> &other) { + return _mm_cmpgt_epi8(*this, other); } - inline TSimd8<bool> AnyBitsSet() const { - return ~this->BitsNotSet(); + + inline TSimd8<bool> operator<(const TSimd8<T> &other) { + return _mm_cmpgt_epi8(other, *this); } - inline bool BitsNotSetAnywhere() const { - return _mm_testz_si128(this->Value, this->Value); + + inline TSimd8<T> operator|(const TSimd8<T>& other) const { + return _mm_or_si128(this->Value, other.Value); } - inline bool AnyBitsSetAnywhere() const { - return !BitsNotSetAnywhere(); + inline TSimd8<T> operator&(const TSimd8<T>& other) const { + return _mm_and_si128(this->Value, other.Value); } - inline bool BitsNotSetAnywhere(TSimd8<ui8> bits) const { - return _mm_testz_si128(this->Value, bits.Value); + inline TSimd8<T> operator^(const TSimd8<T>& other) const { + return _mm_xor_si128(this->Value, other.Value); } - inline bool AnyBitsSetAnywhere(TSimd8<ui8> bits) const { - return !BitsNotSetAnywhere(bits); + inline TSimd8<T> BitAndNot(const TSimd8<T>& other) const { + return _mm_andnot_si128(this->Value, other.Value); + }; + inline TSimd8<T>& operator|=(const TSimd8<T>& other) { + *this = *this | other; + return *this; } + inline TSimd8<T>& operator&=(const TSimd8<T>& other) { + *this = *this & other; + return *this; + }; + inline TSimd8<T>& operator^=(const TSimd8<T>& other) { + *this = *this ^ other; + return *this; + }; - template<int N> - inline TSimd8<ui8> Shr() const { - return TSimd8<ui8>(_mm_srli_epi16(this->Value, N)) & ui8(0xFFu >> N); + inline TSimd8<T> operator+(const TSimd8<T>& other) const { + return _mm_add_epi8(this->Value, other.Value); } - template<int N> - inline TSimd8<ui8> Shl() const { - return TSimd8<ui8>(_mm_slli_epi16(this->Value, N)) & ui8(0xFFu << N); + inline TSimd8<T> operator-(const TSimd8<T>& other) const { + return _mm_sub_epi8(this->Value, other.Value); + } + inline TSimd8<T>& operator+=(const TSimd8<T>& other) { + *this = *this + other; + return *static_cast<TSimd8<T>*>(this); + } + inline TSimd8<T>& operator-=(const TSimd8<T>& other) { + *this = *this - other; + return *static_cast<TSimd8<T>*>(this); } - template<int N> - inline int GetBit() const { - return _mm_movemask_epi8(_mm_slli_epi16(this->Value, 7-N)); + // 0xFFu = 11111111 = 2^8 - 1 + inline TSimd8<T> operator~() const { + return *this ^ 0xFFu; } }; +template<> +inline TSimd8<bool> TSimd8<bool>::Set(bool value) { + return _mm_set1_epi8(ui8(-value)); } + +template<> +inline TSimd8<bool>::TSimd8(bool value) { + this->Value = _mm_set1_epi8(ui8(-value)); } +template<> +inline TSimd8<i8> TSimd8<i8>::MaxValue(const TSimd8<i8>& other) const { + return _mm_max_epi8(this->Value, other.Value); +} +template<> +inline TSimd8<i8> TSimd8<i8>::MinValue(const TSimd8<i8>& other) const { + return _mm_min_epi8(this->Value, other.Value); +} + +template<> +inline TSimd8<bool> TSimd8<ui8>::operator>(const TSimd8<ui8> &other) { + return TSimd8<ui8>(_mm_subs_epu8(this->Value, other.Value)).AnyBitsSet(); +} + +template<> +inline TSimd8<bool> TSimd8<ui8>::operator<(const TSimd8<ui8> &other) { + return TSimd8<ui8>(_mm_subs_epu8(other.Value, this->Value)).AnyBitsSet(); +} + + +} +} #pragma clang attribute pop
\ No newline at end of file diff --git a/ydb/public/api/protos/ydb_table.proto b/ydb/public/api/protos/ydb_table.proto index 2ef247f5bc..a5d1abcb48 100644 --- a/ydb/public/api/protos/ydb_table.proto +++ b/ydb/public/api/protos/ydb_table.proto @@ -1152,6 +1152,9 @@ message ExecuteScanQueryRequest { Mode mode = 6; reserved 7; // report_progress QueryStatsCollection.Mode collect_stats = 8; + // works only in mode: MODE_EXPLAIN, + // collects additional diagnostics about query compilation, including query plan and scheme + bool collect_full_diagnostics = 9; } message ExecuteScanQueryPartialResponse { @@ -1167,4 +1170,7 @@ message ExecuteScanQueryPartialResult { reserved 4; // query_progress reserved 5; // query_plan Ydb.TableStats.QueryStats query_stats = 6; + // works only in mode: MODE_EXPLAIN, + // collects additional diagnostics about query compilation, including query plan and scheme + string query_full_diagnostics = 7; } diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_table.cpp b/ydb/public/lib/ydb_cli/commands/ydb_service_table.cpp index e43cc8557d..eee033c9f3 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_table.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_table.cpp @@ -822,6 +822,11 @@ void TCommandExplain::Config(TConfig& config) { config.SetFreeArgsNum(0); } +void TCommandExplain::SaveDiagnosticsToFile(const TString& diagnostics) { + TFileOutput file(TStringBuilder() << "diagnostics_" << TGUID::Create().AsGuidString() << ".txt"); + file << diagnostics; +} + void TCommandExplain::Parse(TConfig& config) { TClientCommand::Parse(config); ParseFormats(); @@ -843,9 +848,15 @@ int TCommandExplain::Run(TConfig& config) { settings.Explain(true); } + if (CollectFullDiagnostics) { + settings.CollectFullDiagnostics(true); + } + auto result = client.StreamExecuteScanQuery(Query, settings).GetValueSync(); ThrowOnError(result); + TString diagnostics; + SetInterruptHandlers(); while (!IsInterrupted()) { auto tablePart = result.ReadNext().GetValueSync(); @@ -860,6 +871,13 @@ int TCommandExplain::Run(TConfig& config) { planJson = proto.query_plan(); ast = proto.query_ast(); } + if (tablePart.HasDiagnostics()) { + diagnostics = tablePart.ExtractDiagnostics(); + } + } + + if (CollectFullDiagnostics) { + SaveDiagnosticsToFile(diagnostics); } if (IsInterrupted()) { @@ -930,8 +948,7 @@ int TCommandExplain::Run(TConfig& config) { ast = result.GetAst(); if (CollectFullDiagnostics) { - TFileOutput file(TStringBuilder() << "diagnostics_" << TGUID::Create().AsGuidString() << ".txt"); - file << result.GetDiagnostics(); + SaveDiagnosticsToFile(result.GetDiagnostics()); } } else { diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_table.h b/ydb/public/lib/ydb_cli/commands/ydb_service_table.h index cd80e7c2be..77950fba03 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_table.h +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_table.h @@ -135,6 +135,7 @@ public: virtual int Run(TConfig& config) override; private: + static void SaveDiagnosticsToFile(const TString& diagnostics); bool PrintAst = false; TString QueryType; diff --git a/ydb/public/sdk/cpp/client/ydb_table/impl/readers.cpp b/ydb/public/sdk/cpp/client/ydb_table/impl/readers.cpp index 721337dbde..11a8d4d0b1 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/impl/readers.cpp +++ b/ydb/public/sdk/cpp/client/ydb_table/impl/readers.cpp @@ -82,16 +82,19 @@ TAsyncScanQueryPart TScanQueryPartIterator::TReaderImpl::ReadNext(std::shared_pt TPlainStatus plainStatus{clientStatus, std::move(issues), self->Endpoint_, {}}; TStatus status{std::move(plainStatus)}; TMaybe<TQueryStats> queryStats; + TMaybe<TString> diagnostics; if (self->Response_.result().has_query_stats()) { queryStats = TQueryStats(self->Response_.result().query_stats()); } + diagnostics = self->Response_.result().query_full_diagnostics(); + if (self->Response_.result().has_result_set()) { promise.SetValue({std::move(status), - TResultSet(std::move(*self->Response_.mutable_result()->mutable_result_set())), queryStats}); + TResultSet(std::move(*self->Response_.mutable_result()->mutable_result_set())), queryStats, diagnostics}); } else { - promise.SetValue({std::move(status), queryStats}); + promise.SetValue({std::move(status), queryStats, diagnostics}); } } }; diff --git a/ydb/public/sdk/cpp/client/ydb_table/impl/table_client.cpp b/ydb/public/sdk/cpp/client/ydb_table/impl/table_client.cpp index bb85e97b78..596e38a858 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/impl/table_client.cpp +++ b/ydb/public/sdk/cpp/client/ydb_table/impl/table_client.cpp @@ -1002,6 +1002,7 @@ TFuture<std::pair<TPlainStatus, TTableClient::TImpl::TScanQueryProcessorPtr>> TT } request.set_collect_stats(GetStatsCollectionMode(settings.CollectQueryStats_)); + request.set_collect_full_diagnostics(settings.CollectFullDiagnostics_); auto promise = NewPromise<std::pair<TPlainStatus, TScanQueryProcessorPtr>>(); diff --git a/ydb/public/sdk/cpp/client/ydb_table/table.h b/ydb/public/sdk/cpp/client/ydb_table/table.h index 452bcb822e..3c5b3db53f 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/table.h +++ b/ydb/public/sdk/cpp/client/ydb_table/table.h @@ -971,6 +971,9 @@ struct TStreamExecScanQuerySettings : public TRequestSettings<TStreamExecScanQue // Collect runtime statistics with a given detalization mode FLUENT_SETTING_DEFAULT(ECollectQueryStatsMode, CollectQueryStats, ECollectQueryStatsMode::None); + + // Collect full query compilation diagnostics + FLUENT_SETTING_DEFAULT(bool, CollectFullDiagnostics, false); }; class TSession; @@ -1900,24 +1903,31 @@ public: const TQueryStats& GetQueryStats() const { return *QueryStats_; } TQueryStats ExtractQueryStats() { return std::move(*QueryStats_); } + bool HasDiagnostics() const { return Diagnostics_.Defined(); } + const TString& GetDiagnostics() const { return *Diagnostics_; } + TString&& ExtractDiagnostics() { return std::move(*Diagnostics_); } + TScanQueryPart(TStatus&& status) : TStreamPartStatus(std::move(status)) {} - TScanQueryPart(TStatus&& status, const TMaybe<TQueryStats> &queryStats) + TScanQueryPart(TStatus&& status, const TMaybe<TQueryStats>& queryStats, const TMaybe<TString>& diagnostics) : TStreamPartStatus(std::move(status)) , QueryStats_(queryStats) + , Diagnostics_(diagnostics) {} - TScanQueryPart(TStatus&& status, TResultSet&& resultSet, const TMaybe<TQueryStats> &queryStats) + TScanQueryPart(TStatus&& status, TResultSet&& resultSet, const TMaybe<TQueryStats>& queryStats, const TMaybe<TString>& diagnostics) : TStreamPartStatus(std::move(status)) , ResultSet_(std::move(resultSet)) , QueryStats_(queryStats) + , Diagnostics_(diagnostics) {} private: TMaybe<TResultSet> ResultSet_; TMaybe<TQueryStats> QueryStats_; + TMaybe<TString> Diagnostics_; }; using TAsyncScanQueryPart = NThreading::TFuture<TScanQueryPart>; diff --git a/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_bs_controller_/flat_bs_controller.schema b/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_bs_controller_/flat_bs_controller.schema index 1a12c9bd11..1d6e4ae03e 100644 --- a/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_bs_controller_/flat_bs_controller.schema +++ b/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_bs_controller_/flat_bs_controller.schema @@ -105,6 +105,11 @@ "ColumnId": 21, "ColumnName": "CompatibilityInfo", "ColumnType": "String" + }, + { + "ColumnId": 22, + "ColumnName": "UseSelfHealLocalPolicy", + "ColumnType": "Bool" } ], "ColumnsDropped": [], @@ -130,7 +135,8 @@ 18, 19, 20, - 21 + 21, + 22 ], "RoomID": 0, "Codec": 0, diff --git a/yt/yt/client/api/rpc_proxy/client_base.cpp b/yt/yt/client/api/rpc_proxy/client_base.cpp index 812b6cd843..17d6fbdf8e 100644 --- a/yt/yt/client/api/rpc_proxy/client_base.cpp +++ b/yt/yt/client/api/rpc_proxy/client_base.cpp @@ -911,11 +911,16 @@ TFuture<std::vector<TUnversionedLookupRowsResult>> TClientBase::MultiLookup( } template<class TRequest> -void FillRequestBySelectRowsOptionsBase(const TSelectRowsOptionsBase& options, TRequest request) +void FillRequestBySelectRowsOptionsBase( + const TSelectRowsOptionsBase& options, + const std::optional<NYPath::TYPath>& defaultUdfRegistryPath, + TRequest request) { request->set_timestamp(options.Timestamp); if (options.UdfRegistryPath) { request->set_udf_registry_path(*options.UdfRegistryPath); + } else if (defaultUdfRegistryPath) { + request->set_udf_registry_path(*options.UdfRegistryPath); } } @@ -929,11 +934,13 @@ TFuture<TSelectRowsResult> TClientBase::SelectRows( req->SetResponseHeavy(true); req->set_query(query); - FillRequestBySelectRowsOptionsBase(options, req); + const auto& config = GetRpcProxyConnection()->GetConfig(); + + FillRequestBySelectRowsOptionsBase(options, config->UdfRegistryPath, req); // TODO(ifsmirnov): retention timestamp in explain_query. req->set_retention_timestamp(options.RetentionTimestamp); // TODO(lukyan): Move to FillRequestBySelectRowsOptionsBase - req->SetTimeout(options.Timeout.value_or(GetRpcProxyConnection()->GetConfig()->DefaultSelectRowsTimeout)); + req->SetTimeout(options.Timeout.value_or(config->DefaultSelectRowsTimeout)); if (options.InputRowLimit) { req->set_input_row_limit(*options.InputRowLimit); @@ -981,7 +988,7 @@ TFuture<TYsonString> TClientBase::ExplainQuery( auto req = proxy.ExplainQuery(); req->set_query(query); - FillRequestBySelectRowsOptionsBase(options, req); + FillRequestBySelectRowsOptionsBase(options, GetRpcProxyConnection()->GetConfig()->UdfRegistryPath, req); return req->Invoke().Apply(BIND([] (const TApiServiceProxy::TRspExplainQueryPtr& rsp) { return TYsonString(rsp->value()); diff --git a/yt/yt/client/api/rpc_proxy/config.cpp b/yt/yt/client/api/rpc_proxy/config.cpp index 7df4d7239f..ced3029902 100644 --- a/yt/yt/client/api/rpc_proxy/config.cpp +++ b/yt/yt/client/api/rpc_proxy/config.cpp @@ -107,6 +107,9 @@ void TConnectionConfig::Register(TRegistrar registrar) registrar.Parameter("clock_cluster_tag", &TThis::ClockClusterTag) .Default(NObjectClient::InvalidCellTag); + registrar.Parameter("udf_registry_path", &TThis::UdfRegistryPath) + .Optional(); + registrar.Postprocessor([] (TThis* config) { if (!config->ProxyEndpoints && !config->ClusterUrl && !config->ProxyAddresses && !config->ProxyUnixDomainSocket) { THROW_ERROR_EXCEPTION("Either \"endpoints\" or \"cluster_url\" or \"proxy_addresses\" or \"proxy_unix_domain_socket\" must be specified"); diff --git a/yt/yt/client/api/rpc_proxy/config.h b/yt/yt/client/api/rpc_proxy/config.h index 244f5cd1d4..3f3de07185 100644 --- a/yt/yt/client/api/rpc_proxy/config.h +++ b/yt/yt/client/api/rpc_proxy/config.h @@ -70,6 +70,9 @@ public: NObjectClient::TCellTag ClockClusterTag; + //! Path in Cypress with UDFs. + std::optional<NYPath::TYPath> UdfRegistryPath; + REGISTER_YSON_STRUCT(TConnectionConfig); static void Register(TRegistrar registrar); diff --git a/yt/yt/client/election/public.h b/yt/yt/client/election/public.h index b48fc712db..7b79682b48 100644 --- a/yt/yt/client/election/public.h +++ b/yt/yt/client/election/public.h @@ -12,8 +12,7 @@ namespace NYT::NElection { using TEpochId = TGuid; using TPeerPriority = std::pair<i64, i64>; -using TPeerId = int; -constexpr TPeerId InvalidPeerId = -1; +constexpr int InvalidPeerId = -1; using TCellId = TGuid; extern const TCellId NullCellId; diff --git a/yt/yt/client/hydra/public.h b/yt/yt/client/hydra/public.h index 5f1e63206b..c2bdbf94cd 100644 --- a/yt/yt/client/hydra/public.h +++ b/yt/yt/client/hydra/public.h @@ -52,7 +52,6 @@ struct TElectionPriority; using NElection::TCellId; using NElection::NullCellId; -using NElection::TPeerId; using NElection::InvalidPeerId; using NElection::TPeerPriority; using NElection::TEpochId; diff --git a/yt/yt/core/yson/protobuf_interop.cpp b/yt/yt/core/yson/protobuf_interop.cpp index 6df2d4854d..6c14f8db3c 100644 --- a/yt/yt/core/yson/protobuf_interop.cpp +++ b/yt/yt/core/yson/protobuf_interop.cpp @@ -69,6 +69,12 @@ static constexpr int ProtobufMapValueFieldNumber = 2; namespace { +//////////////////////////////////////////////////////////////////////////////// + +const NLogging::TLogger Logger("ProtobufInterop"); + +//////////////////////////////////////////////////////////////////////////////// + bool IsSignedIntegralType(FieldDescriptor::Type type) { switch (type) { @@ -850,6 +856,30 @@ protected: } } } + + void ValidateString(TStringBuf data, EUtf8Check checkOption, TString fieldFullName) + { + if (checkOption == EUtf8Check::None || IsUtf(data)) { + return; + } + switch (checkOption) { + case EUtf8Check::None: + break; + case EUtf8Check::Log: + YT_LOG_WARNING("Field %v accepts only valid UTF-8 sequence, but got (%Qv)", + YPathStack_.GetHumanReadablePath(), + data); + break; + case EUtf8Check::Throw: + THROW_ERROR_EXCEPTION("Field %v accepts only valid UTF-8 sequence, but got (%Qv)", + YPathStack_.GetHumanReadablePath(), + data) + << TErrorAttribute("ypath", YPathStack_.GetPath()) + << TErrorAttribute("proto_field", fieldFullName); + break; + } + } + }; //////////////////////////////////////////////////////////////////////////////// @@ -954,12 +984,7 @@ private: const auto* field = FieldStack_.back().Field; switch (field->GetType()) { case FieldDescriptor::TYPE_STRING: - if (Options_.CheckUtf8 && !IsUtf(value)) { - THROW_ERROR_EXCEPTION("Field %v accepts only valid UTF-8 sequence", - YPathStack_.GetHumanReadablePath()) - << TErrorAttribute("ypath", YPathStack_.GetPath()) - << TErrorAttribute("proto_field", field->GetFullName()); - } + ValidateString(value, Options_.CheckUtf8, field->GetFullName()); case FieldDescriptor::TYPE_BYTES: BodyCodedStream_.WriteVarint64(value.length()); BodyCodedStream_.WriteRaw(value.begin(), static_cast<int>(value.length())); @@ -2475,11 +2500,8 @@ private: << TErrorAttribute("proto_field", field->GetFullName()); } TStringBuf data(PooledString_.data(), length); - if (Options_.CheckUtf8 && field->GetType() == FieldDescriptor::TYPE_STRING && !IsUtf(data)) { - THROW_ERROR_EXCEPTION("Field %v expected to contain valid UTF-8 sequence", - YPathStack_.GetHumanReadablePath()) - << TErrorAttribute("ypath", YPathStack_.GetPath()) - << TErrorAttribute("proto_field", field->GetFullName()); + if (field->GetType() == FieldDescriptor::TYPE_STRING) { + ValidateString(data, Options_.CheckUtf8, field->GetFullName()); } ParseScalar([&] { if (field->GetBytesFieldConverter()) { diff --git a/yt/yt/core/yson/protobuf_interop.h b/yt/yt/core/yson/protobuf_interop.h index 48992052eb..3591ef4dab 100644 --- a/yt/yt/core/yson/protobuf_interop.h +++ b/yt/yt/core/yson/protobuf_interop.h @@ -106,11 +106,6 @@ struct TProtobufElementResolveResult TStringBuf TailPath; }; -struct TResolveProtobufElementByYPathOptions -{ - bool AllowUnknownYsonFields = false; -}; - //! Introspects a given #rootType and locates an element (represented //! by TProtobufElement discriminated union) at a given #path. //! Throws if some definite error occurs during resolve (i.e. a malformed @@ -139,20 +134,6 @@ std::unique_ptr<IYsonConsumer> CreateProtobufWriter( //////////////////////////////////////////////////////////////////////////////// -struct TProtobufParserOptions -{ - //! If |true| then fields with numbers not found in protobuf metadata are - //! silently skipped; otherwise an exception is thrown. - bool SkipUnknownFields = false; - - //! If |true| then required fields not found in protobuf metadata are - //! silently skipped; otherwise an exception is thrown. - bool SkipRequiredFields = false; - - // Check if |string| fields contain actual UTF-8 strings. - bool CheckUtf8 = false; -}; - //! Parses a byte sequence and translates it into IYsonConsumer calls. /*! * IMPORTANT! Due to performance reasons the implementation currently assumes diff --git a/yt/yt/core/yson/protobuf_interop_options.h b/yt/yt/core/yson/protobuf_interop_options.h index 254a1faf9f..62173ca268 100644 --- a/yt/yt/core/yson/protobuf_interop_options.h +++ b/yt/yt/core/yson/protobuf_interop_options.h @@ -11,6 +11,17 @@ namespace NYT::NYson { //////////////////////////////////////////////////////////////////////////////// +struct TResolveProtobufElementByYPathOptions +{ + bool AllowUnknownYsonFields = false; +}; + +DEFINE_ENUM(EUtf8Check, + (None) + (Log) + (Throw) +); + struct TProtobufWriterOptions { //! Keep: all unknown fields found during YSON parsing @@ -33,7 +44,21 @@ struct TProtobufWriterOptions bool SkipRequiredFields = false; // Check if |string| fields contain actual UTF-8 strings. - bool CheckUtf8 = false; + EUtf8Check CheckUtf8 = EUtf8Check::None; +}; + +struct TProtobufParserOptions +{ + //! If |true| then fields with numbers not found in protobuf metadata are + //! silently skipped; otherwise an exception is thrown. + bool SkipUnknownFields = false; + + //! If |true| then required fields not found in protobuf metadata are + //! silently skipped; otherwise an exception is thrown. + bool SkipRequiredFields = false; + + // Check if |string| fields contain actual UTF-8 strings. + EUtf8Check CheckUtf8 = EUtf8Check::None; }; //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/core/yson/unittests/protobuf_yson_ut.cpp b/yt/yt/core/yson/unittests/protobuf_yson_ut.cpp index 2719f939a2..111d9cd19e 100644 --- a/yt/yt/core/yson/unittests/protobuf_yson_ut.cpp +++ b/yt/yt/core/yson/unittests/protobuf_yson_ut.cpp @@ -1017,30 +1017,38 @@ TEST(TYsonToProtobufTest, Entities) TEST(TYsonToProtobufTest, ValidUtf8StringCheck) { - TProtobufWriterOptions options{ - .CheckUtf8 = true, - }; - - TString invalidUtf8 = "\xc3\x28"; - - auto check = [&] { - TEST_PROLOGUE_WITH_OPTIONS(TMessage, options) - .BeginMap() - .Item("string_field").Value(invalidUtf8) - .EndMap(); - }; + for (auto option: {EUtf8Check::None, EUtf8Check::Log, EUtf8Check::Throw}) { + TProtobufWriterOptions options{ + .CheckUtf8 = option, + }; - EXPECT_THROW_WITH_SUBSTRING(check(), "valid UTF-8"); + TString invalidUtf8 = "\xc3\x28"; - NProto::TMessage message; - message.set_string_field(invalidUtf8); - TString newYsonString; - TStringOutput newYsonOutputStream(newYsonString); - TYsonWriter ysonWriter(&newYsonOutputStream, EYsonFormat::Pretty); + auto check = [&] { + TEST_PROLOGUE_WITH_OPTIONS(TMessage, options) + .BeginMap() + .Item("string_field").Value(invalidUtf8) + .EndMap(); + }; + if (option == EUtf8Check::Throw) { + EXPECT_THROW_WITH_SUBSTRING(check(), "valid UTF-8"); + } else { + EXPECT_NO_THROW(check()); + } - EXPECT_THROW_WITH_SUBSTRING( - WriteProtobufMessage(&ysonWriter, message, TProtobufParserOptions{.CheckUtf8 = true}), - "valid UTF-8"); + NProto::TMessage message; + message.set_string_field(invalidUtf8); + TString newYsonString; + TStringOutput newYsonOutputStream(newYsonString); + TYsonWriter ysonWriter(&newYsonOutputStream, EYsonFormat::Pretty); + if (option == EUtf8Check::Throw) { + EXPECT_THROW_WITH_SUBSTRING( + WriteProtobufMessage(&ysonWriter, message, TProtobufParserOptions{.CheckUtf8 = option}), + "valid UTF-8"); + } else { + EXPECT_NO_THROW(WriteProtobufMessage(&ysonWriter, message, TProtobufParserOptions{.CheckUtf8 = option})); + } + } } TEST(TYsonToProtobufTest, CustomUnknownFieldsModeResolver) diff --git a/yt/yt/library/CMakeLists.darwin-x86_64.txt b/yt/yt/library/CMakeLists.darwin-x86_64.txt index 2debbb7626..a85455d5cb 100644 --- a/yt/yt/library/CMakeLists.darwin-x86_64.txt +++ b/yt/yt/library/CMakeLists.darwin-x86_64.txt @@ -7,12 +7,10 @@ add_subdirectory(auth) -add_subdirectory(containers) add_subdirectory(decimal) add_subdirectory(erasure) add_subdirectory(monitoring) add_subdirectory(numeric) -add_subdirectory(process) add_subdirectory(profiling) add_subdirectory(program) add_subdirectory(quantile_digest) diff --git a/yt/yt/library/CMakeLists.linux-aarch64.txt b/yt/yt/library/CMakeLists.linux-aarch64.txt index 524ffcf525..07cc57b53c 100644 --- a/yt/yt/library/CMakeLists.linux-aarch64.txt +++ b/yt/yt/library/CMakeLists.linux-aarch64.txt @@ -8,7 +8,6 @@ add_subdirectory(auth) add_subdirectory(backtrace_introspector) -add_subdirectory(containers) add_subdirectory(decimal) add_subdirectory(erasure) add_subdirectory(monitoring) diff --git a/yt/yt/library/CMakeLists.linux-x86_64.txt b/yt/yt/library/CMakeLists.linux-x86_64.txt index 524ffcf525..07cc57b53c 100644 --- a/yt/yt/library/CMakeLists.linux-x86_64.txt +++ b/yt/yt/library/CMakeLists.linux-x86_64.txt @@ -8,7 +8,6 @@ add_subdirectory(auth) add_subdirectory(backtrace_introspector) -add_subdirectory(containers) add_subdirectory(decimal) add_subdirectory(erasure) add_subdirectory(monitoring) diff --git a/yt/yt/library/CMakeLists.windows-x86_64.txt b/yt/yt/library/CMakeLists.windows-x86_64.txt index 4502da4e61..eaf7bb9e34 100644 --- a/yt/yt/library/CMakeLists.windows-x86_64.txt +++ b/yt/yt/library/CMakeLists.windows-x86_64.txt @@ -6,9 +6,7 @@ # original buildsystem will not be accepted. -add_subdirectory(containers) add_subdirectory(monitoring) -add_subdirectory(process) add_subdirectory(profiling) add_subdirectory(program) add_subdirectory(syncmap) diff --git a/yt/yt/library/containers/CMakeLists.darwin-x86_64.txt b/yt/yt/library/containers/CMakeLists.darwin-x86_64.txt deleted file mode 100644 index faab79bbf6..0000000000 --- a/yt/yt/library/containers/CMakeLists.darwin-x86_64.txt +++ /dev/null @@ -1,30 +0,0 @@ - -# This file was generated by the build system used internally in the Yandex monorepo. -# Only simple modifications are allowed (adding source-files to targets, adding simple properties -# like target_include_directories). These modifications will be ported to original -# ya.make files by maintainers. Any complex modifications which can't be ported back to the -# original buildsystem will not be accepted. - - - -add_library(yt-library-containers) -target_compile_options(yt-library-containers PRIVATE - -Wdeprecated-this-capture -) -target_link_libraries(yt-library-containers PUBLIC - contrib-libs-cxxsupp - yutil - cpp-porto-proto - yt-library-process - yt-yt-core -) -target_sources(yt-library-containers PRIVATE - ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/cgroup.cpp - ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/config.cpp - ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/instance.cpp - ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/instance_limits_tracker.cpp - ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/process.cpp - ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/porto_executor.cpp - ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/porto_resource_tracker.cpp - ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/porto_health_checker.cpp -) diff --git a/yt/yt/library/containers/CMakeLists.linux-aarch64.txt b/yt/yt/library/containers/CMakeLists.linux-aarch64.txt deleted file mode 100644 index d3ab3811e0..0000000000 --- a/yt/yt/library/containers/CMakeLists.linux-aarch64.txt +++ /dev/null @@ -1,32 +0,0 @@ - -# This file was generated by the build system used internally in the Yandex monorepo. -# Only simple modifications are allowed (adding source-files to targets, adding simple properties -# like target_include_directories). These modifications will be ported to original -# ya.make files by maintainers. Any complex modifications which can't be ported back to the -# original buildsystem will not be accepted. - - - -add_library(yt-library-containers) -target_compile_options(yt-library-containers PRIVATE - -Wdeprecated-this-capture -) -target_link_libraries(yt-library-containers PUBLIC - contrib-libs-linux-headers - contrib-libs-cxxsupp - yutil - cpp-porto-proto - yt-library-process - yt-yt-core - library-cpp-porto -) -target_sources(yt-library-containers PRIVATE - ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/cgroup.cpp - ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/config.cpp - ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/instance.cpp - ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/instance_limits_tracker.cpp - ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/process.cpp - ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/porto_executor.cpp - ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/porto_resource_tracker.cpp - ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/porto_health_checker.cpp -) diff --git a/yt/yt/library/containers/CMakeLists.linux-x86_64.txt b/yt/yt/library/containers/CMakeLists.linux-x86_64.txt deleted file mode 100644 index d3ab3811e0..0000000000 --- a/yt/yt/library/containers/CMakeLists.linux-x86_64.txt +++ /dev/null @@ -1,32 +0,0 @@ - -# This file was generated by the build system used internally in the Yandex monorepo. -# Only simple modifications are allowed (adding source-files to targets, adding simple properties -# like target_include_directories). These modifications will be ported to original -# ya.make files by maintainers. Any complex modifications which can't be ported back to the -# original buildsystem will not be accepted. - - - -add_library(yt-library-containers) -target_compile_options(yt-library-containers PRIVATE - -Wdeprecated-this-capture -) -target_link_libraries(yt-library-containers PUBLIC - contrib-libs-linux-headers - contrib-libs-cxxsupp - yutil - cpp-porto-proto - yt-library-process - yt-yt-core - library-cpp-porto -) -target_sources(yt-library-containers PRIVATE - ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/cgroup.cpp - ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/config.cpp - ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/instance.cpp - ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/instance_limits_tracker.cpp - ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/process.cpp - ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/porto_executor.cpp - ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/porto_resource_tracker.cpp - ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/porto_health_checker.cpp -) diff --git a/yt/yt/library/containers/CMakeLists.txt b/yt/yt/library/containers/CMakeLists.txt deleted file mode 100644 index f8b31df0c1..0000000000 --- a/yt/yt/library/containers/CMakeLists.txt +++ /dev/null @@ -1,17 +0,0 @@ - -# This file was generated by the build system used internally in the Yandex monorepo. -# Only simple modifications are allowed (adding source-files to targets, adding simple properties -# like target_include_directories). These modifications will be ported to original -# ya.make files by maintainers. Any complex modifications which can't be ported back to the -# original buildsystem will not be accepted. - - -if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) - include(CMakeLists.linux-aarch64.txt) -elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") - include(CMakeLists.darwin-x86_64.txt) -elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) - include(CMakeLists.windows-x86_64.txt) -elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) - include(CMakeLists.linux-x86_64.txt) -endif() diff --git a/yt/yt/library/containers/CMakeLists.windows-x86_64.txt b/yt/yt/library/containers/CMakeLists.windows-x86_64.txt deleted file mode 100644 index 998e1690fa..0000000000 --- a/yt/yt/library/containers/CMakeLists.windows-x86_64.txt +++ /dev/null @@ -1,27 +0,0 @@ - -# This file was generated by the build system used internally in the Yandex monorepo. -# Only simple modifications are allowed (adding source-files to targets, adding simple properties -# like target_include_directories). These modifications will be ported to original -# ya.make files by maintainers. Any complex modifications which can't be ported back to the -# original buildsystem will not be accepted. - - - -add_library(yt-library-containers) -target_link_libraries(yt-library-containers PUBLIC - contrib-libs-cxxsupp - yutil - cpp-porto-proto - yt-library-process - yt-yt-core -) -target_sources(yt-library-containers PRIVATE - ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/cgroup.cpp - ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/config.cpp - ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/instance.cpp - ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/instance_limits_tracker.cpp - ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/process.cpp - ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/porto_executor.cpp - ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/porto_resource_tracker.cpp - ${CMAKE_SOURCE_DIR}/yt/yt/library/containers/porto_health_checker.cpp -) diff --git a/yt/yt/library/containers/cgroup.cpp b/yt/yt/library/containers/cgroup.cpp deleted file mode 100644 index b43ab1e14b..0000000000 --- a/yt/yt/library/containers/cgroup.cpp +++ /dev/null @@ -1,752 +0,0 @@ -#include "cgroup.h" -#include "private.h" - -#include <yt/yt/core/misc/fs.h> -#include <yt/yt/core/misc/proc.h> - -#include <yt/yt/core/ytree/fluent.h> - -#include <util/string/split.h> -#include <util/system/filemap.h> - -#include <util/system/yield.h> - -#ifdef _linux_ - #include <unistd.h> - #include <sys/stat.h> - #include <errno.h> -#endif - -namespace NYT::NContainers { - -using namespace NYTree; - -//////////////////////////////////////////////////////////////////////////////// - -static const auto& Logger = ContainersLogger; -static const TString CGroupRootPath("/sys/fs/cgroup"); -#ifdef _linux_ -static const int ReadByAll = S_IRUSR | S_IRGRP | S_IROTH; -static const int ReadExecuteByAll = ReadByAll | S_IXUSR | S_IXGRP | S_IXOTH; -#endif - -//////////////////////////////////////////////////////////////////////////////// - -namespace { - -TString GetParentFor(const TString& type) -{ -#ifdef _linux_ - auto rawData = TUnbufferedFileInput("/proc/self/cgroup") - .ReadAll(); - auto result = ParseProcessCGroups(rawData); - return result[type]; -#else - Y_UNUSED(type); - return "_parent_"; -#endif -} - -#ifdef _linux_ - -std::vector<TString> ReadAllValues(const TString& fileName) -{ - auto raw = TUnbufferedFileInput(fileName) - .ReadAll(); - - YT_LOG_DEBUG("File %v contains %Qv", - fileName, - raw); - - TVector<TString> values; - StringSplitter(raw.data()) - .SplitBySet(" \n") - .SkipEmpty() - .Collect(&values); - return values; -} - -TDuration FromJiffies(ui64 jiffies) -{ - static const auto TicksPerSecond = sysconf(_SC_CLK_TCK); - return TDuration::MicroSeconds(1000 * 1000 * jiffies / TicksPerSecond); -} - -#endif - -} // namespace - -//////////////////////////////////////////////////////////////////////////////// - -void TKillProcessGroupTool::operator()(const TString& processGroupPath) const -{ - SafeSetUid(0); - TNonOwningCGroup group(processGroupPath); - group.Kill(); -} - -//////////////////////////////////////////////////////////////////////////////// - -TNonOwningCGroup::TNonOwningCGroup(const TString& fullPath) - : FullPath_(fullPath) -{ } - -TNonOwningCGroup::TNonOwningCGroup(const TString& type, const TString& name) - : FullPath_(NFS::CombinePaths({ - CGroupRootPath, - type, - GetParentFor(type), - name - })) -{ } - -TNonOwningCGroup::TNonOwningCGroup(TNonOwningCGroup&& other) - : FullPath_(std::move(other.FullPath_)) -{ } - -void TNonOwningCGroup::AddTask(int pid) const -{ - YT_LOG_INFO( - "Adding task to cgroup (Task: %v, Cgroup: %v)", - pid, - FullPath_); - Append("tasks", ToString(pid)); -} - -void TNonOwningCGroup::AddCurrentTask() const -{ - YT_VERIFY(!IsNull()); -#ifdef _linux_ - auto pid = getpid(); - AddTask(pid); -#endif -} - -TString TNonOwningCGroup::Get(const TString& name) const -{ - YT_VERIFY(!IsNull()); - TString result; -#ifdef _linux_ - const auto path = GetPath(name); - result = TFileInput(path).ReadLine(); -#else - Y_UNUSED(name); -#endif - return result; -} - -void TNonOwningCGroup::Set(const TString& name, const TString& value) const -{ - YT_VERIFY(!IsNull()); -#ifdef _linux_ - auto path = GetPath(name); - TUnbufferedFileOutput output(TFile(path, EOpenModeFlag::WrOnly)); - output << value; -#else - Y_UNUSED(name); - Y_UNUSED(value); -#endif -} - -void TNonOwningCGroup::Append(const TString& name, const TString& value) const -{ - YT_VERIFY(!IsNull()); -#ifdef _linux_ - auto path = GetPath(name); - TUnbufferedFileOutput output(TFile(path, EOpenModeFlag::ForAppend)); - output << value; -#else - Y_UNUSED(name); - Y_UNUSED(value); -#endif -} - -bool TNonOwningCGroup::IsRoot() const -{ - return FullPath_ == CGroupRootPath; -} - -bool TNonOwningCGroup::IsNull() const -{ - return FullPath_.empty(); -} - -bool TNonOwningCGroup::Exists() const -{ - return NFS::Exists(FullPath_); -} - -std::vector<int> TNonOwningCGroup::GetProcesses() const -{ - std::vector<int> results; - if (!IsNull()) { -#ifdef _linux_ - auto values = ReadAllValues(GetPath("cgroup.procs")); - for (const auto& value : values) { - int pid = FromString<int>(value); - results.push_back(pid); - } -#endif - } - return results; -} - -std::vector<int> TNonOwningCGroup::GetTasks() const -{ - std::vector<int> results; - if (!IsNull()) { -#ifdef _linux_ - auto values = ReadAllValues(GetPath("tasks")); - for (const auto& value : values) { - int pid = FromString<int>(value); - results.push_back(pid); - } -#endif - } - return results; -} - -const TString& TNonOwningCGroup::GetFullPath() const -{ - return FullPath_; -} - -std::vector<TNonOwningCGroup> TNonOwningCGroup::GetChildren() const -{ - // We retry enumerating directories, since it may fail with weird diagnostics if - // number of subcgroups changes. - while (true) { - try { - std::vector<TNonOwningCGroup> result; - - if (IsNull()) { - return result; - } - - auto directories = NFS::EnumerateDirectories(FullPath_); - for (const auto& directory : directories) { - result.emplace_back(NFS::CombinePaths(FullPath_, directory)); - } - return result; - } catch (const std::exception& ex) { - YT_LOG_WARNING(ex, "Failed to list subcgroups (Path: %v)", FullPath_); - } - } -} - -void TNonOwningCGroup::EnsureExistence() const -{ - YT_LOG_INFO("Creating cgroup (Cgroup: %v)", FullPath_); - - YT_VERIFY(!IsNull()); - -#ifdef _linux_ - NFS::MakeDirRecursive(FullPath_, 0755); -#endif -} - -void TNonOwningCGroup::Lock() const -{ - Traverse( - BIND([] (const TNonOwningCGroup& group) { group.DoLock(); }), - BIND([] (const TNonOwningCGroup& /*group*/) {})); -} - -void TNonOwningCGroup::Unlock() const -{ - Traverse( - BIND([] (const TNonOwningCGroup& /*group*/) {}), - BIND([] (const TNonOwningCGroup& group) { group.DoUnlock(); })); -} - -void TNonOwningCGroup::Kill() const -{ - YT_VERIFY(!IsRoot()); - - Traverse( - BIND([] (const TNonOwningCGroup& group) { group.DoKill(); }), - BIND([] (const TNonOwningCGroup& /*group*/) {})); -} - -void TNonOwningCGroup::RemoveAllSubcgroups() const -{ - Traverse( - BIND([] (const TNonOwningCGroup& group) { - group.TryUnlock(); - }), - BIND([this_ = this] (const TNonOwningCGroup& group) { - if (this_ != &group) { - group.DoRemove(); - } - })); -} - -void TNonOwningCGroup::RemoveRecursive() const -{ - RemoveAllSubcgroups(); - DoRemove(); -} - -void TNonOwningCGroup::DoLock() const -{ - YT_LOG_INFO("Locking cgroup (Cgroup: %v)", FullPath_); - -#ifdef _linux_ - if (!IsNull()) { - int code = chmod(FullPath_.data(), ReadExecuteByAll); - YT_VERIFY(code == 0); - - code = chmod(GetPath("tasks").data(), ReadByAll); - YT_VERIFY(code == 0); - } -#endif -} - -bool TNonOwningCGroup::TryUnlock() const -{ - YT_LOG_INFO("Unlocking cgroup (Cgroup: %v)", FullPath_); - - if (!Exists()) { - return true; - } - - bool result = true; - -#ifdef _linux_ - if (!IsNull()) { - int code = chmod(GetPath("tasks").data(), ReadByAll | S_IWUSR); - if (code != 0) { - result = false; - } - - code = chmod(FullPath_.data(), ReadExecuteByAll | S_IWUSR); - if (code != 0) { - result = false; - } - } -#endif - - return result; -} - -void TNonOwningCGroup::DoUnlock() const -{ - YT_VERIFY(TryUnlock()); -} - -void TNonOwningCGroup::DoKill() const -{ - YT_LOG_DEBUG("Started killing processes in cgroup (Cgroup: %v)", FullPath_); - -#ifdef _linux_ - while (true) { - auto pids = GetTasks(); - if (pids.empty()) - break; - - YT_LOG_DEBUG("Killing processes (Pids: %v)", pids); - - for (int pid : pids) { - auto result = kill(pid, SIGKILL); - if (result == -1) { - YT_VERIFY(errno == ESRCH); - } - } - - ThreadYield(); - } -#endif - - YT_LOG_DEBUG("Finished killing processes in cgroup (Cgroup: %v)", FullPath_); -} - -void TNonOwningCGroup::DoRemove() const -{ - if (NFS::Exists(FullPath_)) { - NFS::Remove(FullPath_); - } -} - -void TNonOwningCGroup::Traverse( - const TCallback<void(const TNonOwningCGroup&)>& preorderAction, - const TCallback<void(const TNonOwningCGroup&)>& postorderAction) const -{ - preorderAction(*this); - - for (const auto& child : GetChildren()) { - child.Traverse(preorderAction, postorderAction); - } - - postorderAction(*this); -} - -TString TNonOwningCGroup::GetPath(const TString& filename) const -{ - return NFS::CombinePaths(FullPath_, filename); -} - -//////////////////////////////////////////////////////////////////////////////// - -TCGroup::TCGroup(const TString& type, const TString& name) - : TNonOwningCGroup(type, name) -{ } - -TCGroup::TCGroup(TCGroup&& other) - : TNonOwningCGroup(std::move(other)) - , Created_(other.Created_) -{ - other.Created_ = false; -} - -TCGroup::TCGroup(TNonOwningCGroup&& other) - : TNonOwningCGroup(std::move(other)) - , Created_(false) -{ } - -TCGroup::~TCGroup() -{ - if (Created_) { - Destroy(); - } -} - -void TCGroup::Create() -{ - EnsureExistence(); - Created_ = true; -} - -void TCGroup::Destroy() -{ - YT_LOG_INFO("Destroying cgroup (Cgroup: %v)", FullPath_); - YT_VERIFY(Created_); - -#ifdef _linux_ - try { - NFS::Remove(FullPath_); - } catch (const std::exception& ex) { - YT_LOG_FATAL(ex, "Failed to destroy cgroup (Cgroup: %v)", FullPath_); - } -#endif - Created_ = false; -} - -bool TCGroup::IsCreated() const -{ - return Created_; -} - -//////////////////////////////////////////////////////////////////////////////// - -const TString TCpuAccounting::Name = "cpuacct"; - -TCpuAccounting::TStatistics& operator-=(TCpuAccounting::TStatistics& lhs, const TCpuAccounting::TStatistics& rhs) -{ - #define XX(name) lhs.name = lhs.name.ValueOrThrow() - rhs.name.ValueOrThrow(); - XX(UserUsageTime) - XX(SystemUsageTime) - XX(WaitTime) - XX(ThrottledTime) - XX(ContextSwitchesDelta) - XX(PeakThreadCount) - #undef XX - return lhs; -} - -TCpuAccounting::TCpuAccounting(const TString& name) - : TCGroup(Name, name) -{ } - -TCpuAccounting::TCpuAccounting(TNonOwningCGroup&& nonOwningCGroup) - : TCGroup(std::move(nonOwningCGroup)) -{ } - -TCpuAccounting::TStatistics TCpuAccounting::GetStatisticsRecursive() const -{ - TCpuAccounting::TStatistics result; -#ifdef _linux_ - try { - auto path = NFS::CombinePaths(GetFullPath(), "cpuacct.stat"); - auto values = ReadAllValues(path); - YT_VERIFY(values.size() == 4); - - TString type[2]; - ui64 jiffies[2]; - - for (int i = 0; i < 2; ++i) { - type[i] = values[2 * i]; - jiffies[i] = FromString<ui64>(values[2 * i + 1]); - } - - for (int i = 0; i < 2; ++i) { - if (type[i] == "user") { - result.UserUsageTime = FromJiffies(jiffies[i]); - } else if (type[i] == "system") { - result.SystemUsageTime = FromJiffies(jiffies[i]); - } - } - } catch (const std::exception& ex) { - YT_LOG_FATAL( - ex, - "Failed to retrieve CPU statistics from cgroup (Cgroup: %v)", - GetFullPath()); - } -#endif - return result; -} - -TCpuAccounting::TStatistics TCpuAccounting::GetStatistics() const -{ - auto statistics = GetStatisticsRecursive(); - for (auto& cgroup : GetChildren()) { - auto cpuCGroup = TCpuAccounting(std::move(cgroup)); - statistics -= cpuCGroup.GetStatisticsRecursive(); - } - return statistics; -} - - -//////////////////////////////////////////////////////////////////////////////// - -const TString TCpu::Name = "cpu"; - -static const int DefaultCpuShare = 1024; - -TCpu::TCpu(const TString& name) - : TCGroup(Name, name) -{ } - -void TCpu::SetShare(double share) -{ - int cpuShare = static_cast<int>(share * DefaultCpuShare); - Set("cpu.shares", ToString(cpuShare)); -} - -//////////////////////////////////////////////////////////////////////////////// - -const TString TBlockIO::Name = "blkio"; - -TBlockIO::TBlockIO(const TString& name) - : TCGroup(Name, name) -{ } - -// For more information about format of data -// read https://www.kernel.org/doc/Documentation/cgroups/blkio-controller.txt - -TBlockIO::TStatistics TBlockIO::GetStatistics() const -{ - TBlockIO::TStatistics result; -#ifdef _linux_ - auto bytesStats = GetDetailedStatistics("blkio.io_service_bytes"); - for (const auto& item : bytesStats) { - if (item.Type == "Read") { - result.IOReadByte = result.IOReadByte.ValueOrThrow() + item.Value; - } else if (item.Type == "Write") { - result.IOWriteByte = result.IOReadByte.ValueOrThrow() + item.Value; - } - } - - auto ioStats = GetDetailedStatistics("blkio.io_serviced"); - for (const auto& item : ioStats) { - if (item.Type == "Read") { - result.IOReadOps = result.IOReadOps.ValueOrThrow() + item.Value; - result.IOOps = result.IOOps.ValueOrThrow() + item.Value; - } else if (item.Type == "Write") { - result.IOWriteOps = result.IOWriteOps.ValueOrThrow() + item.Value; - result.IOOps = result.IOOps.ValueOrThrow() + item.Value; - } - } -#endif - return result; -} - -std::vector<TBlockIO::TStatisticsItem> TBlockIO::GetIOServiceBytes() const -{ - return GetDetailedStatistics("blkio.io_service_bytes"); -} - -std::vector<TBlockIO::TStatisticsItem> TBlockIO::GetIOServiced() const -{ - return GetDetailedStatistics("blkio.io_serviced"); -} - -std::vector<TBlockIO::TStatisticsItem> TBlockIO::GetDetailedStatistics(const char* filename) const -{ - std::vector<TBlockIO::TStatisticsItem> result; -#ifdef _linux_ - try { - auto path = NFS::CombinePaths(GetFullPath(), filename); - auto values = ReadAllValues(path); - - int lineNumber = 0; - while (3 * lineNumber + 2 < std::ssize(values)) { - TStatisticsItem item; - item.DeviceId = values[3 * lineNumber]; - item.Type = values[3 * lineNumber + 1]; - item.Value = FromString<ui64>(values[3 * lineNumber + 2]); - - { - auto guard = Guard(SpinLock_); - DeviceIds_.insert(item.DeviceId); - } - - if (item.Type == "Read" || item.Type == "Write") { - result.push_back(item); - - YT_LOG_DEBUG("IO operations serviced (OperationCount: %v, OperationType: %v, DeviceId: %v)", - item.Value, - item.Type, - item.DeviceId); - } - ++lineNumber; - } - } catch (const std::exception& ex) { - YT_LOG_FATAL( - ex, - "Failed to retrieve block IO statistics from cgroup (Cgroup: %v)", - GetFullPath()); - } -#else - Y_UNUSED(filename); -#endif - return result; -} - -void TBlockIO::ThrottleOperations(i64 operations) const -{ - auto guard = Guard(SpinLock_); - for (const auto& deviceId : DeviceIds_) { - auto value = Format("%v %v", deviceId, operations); - Append("blkio.throttle.read_iops_device", value); - Append("blkio.throttle.write_iops_device", value); - } -} - -//////////////////////////////////////////////////////////////////////////////// - -const TString TMemory::Name = "memory"; - -TMemory::TMemory(const TString& name) - : TCGroup(Name, name) -{ } - -TMemory::TStatistics TMemory::GetStatistics() const -{ - TMemory::TStatistics result; -#ifdef _linux_ - try { - auto values = ReadAllValues(GetPath("memory.stat")); - int lineNumber = 0; - while (2 * lineNumber + 1 < std::ssize(values)) { - const auto& type = values[2 * lineNumber]; - const auto& unparsedValue = values[2 * lineNumber + 1]; - if (type == "rss") { - result.Rss = FromString<ui64>(unparsedValue); - } - if (type == "mapped_file") { - result.MappedFile = FromString<ui64>(unparsedValue); - } - if (type == "pgmajfault") { - result.MajorPageFaults = FromString<ui64>(unparsedValue); - } - ++lineNumber; - } - } catch (const std::exception& ex) { - YT_LOG_FATAL( - ex, - "Failed to retrieve memory statistics from cgroup (Cgroup: %v)", - GetFullPath()); - } -#endif - return result; -} - -i64 TMemory::GetMaxMemoryUsage() const -{ - return FromString<i64>(Get("memory.max_usage_in_bytes")); -} - -void TMemory::SetLimitInBytes(i64 bytes) const -{ - Set("memory.limit_in_bytes", ToString(bytes)); -} - -void TMemory::ForceEmpty() const -{ - Set("memory.force_empty", "0"); -} - -//////////////////////////////////////////////////////////////////////////////// - -const TString TFreezer::Name = "freezer"; - -TFreezer::TFreezer(const TString& name) - : TCGroup(Name, name) -{ } - -TString TFreezer::GetState() const -{ - return Get("freezer.state"); -} - -void TFreezer::Freeze() const -{ - Set("freezer.state", "FROZEN"); -} - -void TFreezer::Unfreeze() const -{ - Set("freezer.state", "THAWED"); -} - -//////////////////////////////////////////////////////////////////////////////// - -std::map<TString, TString> ParseProcessCGroups(const TString& str) -{ - std::map<TString, TString> result; - - TVector<TString> values; - StringSplitter(str.data()).SplitBySet(":\n").SkipEmpty().Collect(&values); - for (size_t i = 0; i + 2 < values.size(); i += 3) { - // Check format. - FromString<int>(values[i]); - - const auto& subsystemsSet = values[i + 1]; - const auto& name = values[i + 2]; - - TVector<TString> subsystems; - StringSplitter(subsystemsSet.data()).Split(',').SkipEmpty().Collect(&subsystems); - for (const auto& subsystem : subsystems) { - if (!subsystem.StartsWith("name=")) { - int start = 0; - if (name.StartsWith("/")) { - start = 1; - } - result[subsystem] = name.substr(start); - } - } - } - - return result; -} - -std::map<TString, TString> GetProcessCGroups(pid_t pid) -{ - auto cgroupsPath = Format("/proc/%v/cgroup", pid); - auto rawCgroups = TFileInput{cgroupsPath}.ReadAll(); - return ParseProcessCGroups(rawCgroups); -} - -bool IsValidCGroupType(const TString& type) -{ - return - type == TCpuAccounting::Name || - type == TCpu::Name || - type == TBlockIO::Name || - type == TMemory::Name || - type == TFreezer::Name; -} - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NContainers diff --git a/yt/yt/library/containers/cgroup.h b/yt/yt/library/containers/cgroup.h deleted file mode 100644 index a61fbbddc3..0000000000 --- a/yt/yt/library/containers/cgroup.h +++ /dev/null @@ -1,290 +0,0 @@ -#pragma once - -#include "public.h" - -#include <yt/yt/core/actions/public.h> - -#include <yt/yt/core/ytree/yson_struct.h> -#include <yt/yt/core/yson/public.h> - -#include <yt/yt/core/misc/property.h> - -#include <library/cpp/yt/threading/spin_lock.h> - -#include <vector> - -namespace NYT::NContainers { - -//////////////////////////////////////////////////////////////////////////////// - -void RemoveAllSubcgroups(const TString& path); - -//////////////////////////////////////////////////////////////////////////////// - -struct TKillProcessGroupTool -{ - void operator()(const TString& processGroupPath) const; -}; - -//////////////////////////////////////////////////////////////////////////////// - -class TNonOwningCGroup - : private TNonCopyable -{ -public: - DEFINE_BYREF_RO_PROPERTY(TString, FullPath); - -public: - TNonOwningCGroup() = default; - explicit TNonOwningCGroup(const TString& fullPath); - TNonOwningCGroup(const TString& type, const TString& name); - TNonOwningCGroup(TNonOwningCGroup&& other); - - void AddTask(int pid) const; - void AddCurrentTask() const; - - bool IsRoot() const; - bool IsNull() const; - bool Exists() const; - - std::vector<int> GetProcesses() const; - std::vector<int> GetTasks() const; - const TString& GetFullPath() const; - - std::vector<TNonOwningCGroup> GetChildren() const; - - void EnsureExistence() const; - - void Lock() const; - void Unlock() const; - - void Kill() const; - - void RemoveAllSubcgroups() const; - void RemoveRecursive() const; - -protected: - TString Get(const TString& name) const; - void Set(const TString& name, const TString& value) const; - void Append(const TString& name, const TString& value) const; - - void DoLock() const; - void DoUnlock() const; - - bool TryUnlock() const; - - void DoKill() const; - - void DoRemove() const; - - void Traverse( - const TCallback<void(const TNonOwningCGroup&)>& preorderAction, - const TCallback<void(const TNonOwningCGroup&)>& postorderAction) const; - - TString GetPath(const TString& filename) const; -}; - -//////////////////////////////////////////////////////////////////////////////// - -class TCGroup - : public TNonOwningCGroup -{ -protected: - TCGroup(const TString& type, const TString& name); - TCGroup(TNonOwningCGroup&& other); - TCGroup(TCGroup&& other); - -public: - ~TCGroup(); - - void Create(); - void Destroy(); - - bool IsCreated() const; - -private: - bool Created_ = false; -}; - -//////////////////////////////////////////////////////////////////////////////// - -class TCpuAccounting - : public TCGroup -{ -public: - static const TString Name; - - struct TStatistics - { - TErrorOr<TDuration> TotalUsageTime; - TErrorOr<TDuration> UserUsageTime; - TErrorOr<TDuration> SystemUsageTime; - TErrorOr<TDuration> WaitTime; - TErrorOr<TDuration> ThrottledTime; - - TErrorOr<ui64> ThreadCount; - TErrorOr<ui64> ContextSwitches; - TErrorOr<ui64> ContextSwitchesDelta; - TErrorOr<ui64> PeakThreadCount; - - TErrorOr<TDuration> LimitTime; - TErrorOr<TDuration> GuaranteeTime; - }; - - explicit TCpuAccounting(const TString& name); - - TStatistics GetStatisticsRecursive() const; - TStatistics GetStatistics() const; - -private: - explicit TCpuAccounting(TNonOwningCGroup&& nonOwningCGroup); -}; - -void Serialize(const TCpuAccounting::TStatistics& statistics, NYson::IYsonConsumer* consumer); - -//////////////////////////////////////////////////////////////////////////////// - -class TCpu - : public TCGroup -{ -public: - static const TString Name; - - explicit TCpu(const TString& name); - - void SetShare(double share); -}; - -//////////////////////////////////////////////////////////////////////////////// - -class TBlockIO - : public TCGroup -{ -public: - static const TString Name; - - struct TStatistics - { - TErrorOr<ui64> IOReadByte; - TErrorOr<ui64> IOWriteByte; - TErrorOr<ui64> IOBytesLimit; - - TErrorOr<ui64> IOReadOps; - TErrorOr<ui64> IOWriteOps; - TErrorOr<ui64> IOOps; - TErrorOr<ui64> IOOpsLimit; - - TErrorOr<TDuration> IOTotalTime; - TErrorOr<TDuration> IOWaitTime; - }; - - struct TStatisticsItem - { - TString DeviceId; - TString Type; - ui64 Value = 0; - }; - - explicit TBlockIO(const TString& name); - - TStatistics GetStatistics() const; - void ThrottleOperations(i64 iops) const; - -private: - //! Guards device ids. - YT_DECLARE_SPIN_LOCK(NThreading::TSpinLock, SpinLock_); - //! Set of all seen device ids. - mutable THashSet<TString> DeviceIds_; - - std::vector<TBlockIO::TStatisticsItem> GetDetailedStatistics(const char* filename) const; - - std::vector<TStatisticsItem> GetIOServiceBytes() const; - std::vector<TStatisticsItem> GetIOServiced() const; -}; - -void Serialize(const TBlockIO::TStatistics& statistics, NYson::IYsonConsumer* consumer); - -//////////////////////////////////////////////////////////////////////////////// - -class TMemory - : public TCGroup -{ -public: - static const TString Name; - - struct TStatistics - { - TErrorOr<ui64> Rss; - TErrorOr<ui64> MappedFile; - TErrorOr<ui64> MinorPageFaults; - TErrorOr<ui64> MajorPageFaults; - - TErrorOr<ui64> FileCacheUsage; - TErrorOr<ui64> AnonUsage; - TErrorOr<ui64> AnonLimit; - TErrorOr<ui64> MemoryUsage; - TErrorOr<ui64> MemoryGuarantee; - TErrorOr<ui64> MemoryLimit; - TErrorOr<ui64> MaxMemoryUsage; - - TErrorOr<ui64> OomKills; - TErrorOr<ui64> OomKillsTotal; - }; - - explicit TMemory(const TString& name); - - TStatistics GetStatistics() const; - i64 GetMaxMemoryUsage() const; - - void SetLimitInBytes(i64 bytes) const; - - void ForceEmpty() const; -}; - -void Serialize(const TMemory::TStatistics& statistics, NYson::IYsonConsumer* consumer); - -//////////////////////////////////////////////////////////////////////////////// - -class TNetwork -{ -public: - struct TStatistics - { - TErrorOr<ui64> TxBytes; - TErrorOr<ui64> TxPackets; - TErrorOr<ui64> TxDrops; - TErrorOr<ui64> TxLimit; - - TErrorOr<ui64> RxBytes; - TErrorOr<ui64> RxPackets; - TErrorOr<ui64> RxDrops; - TErrorOr<ui64> RxLimit; - }; -}; - -void Serialize(const TNetwork::TStatistics& statistics, NYson::IYsonConsumer* consumer); - -//////////////////////////////////////////////////////////////////////////////// - -class TFreezer - : public TCGroup -{ -public: - static const TString Name; - - explicit TFreezer(const TString& name); - - TString GetState() const; - void Freeze() const; - void Unfreeze() const; -}; - -//////////////////////////////////////////////////////////////////////////////// - -std::map<TString, TString> ParseProcessCGroups(const TString& str); -std::map<TString, TString> GetProcessCGroups(pid_t pid); -bool IsValidCGroupType(const TString& type); - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NContainers diff --git a/yt/yt/library/containers/config.cpp b/yt/yt/library/containers/config.cpp deleted file mode 100644 index 39e46f2372..0000000000 --- a/yt/yt/library/containers/config.cpp +++ /dev/null @@ -1,64 +0,0 @@ -#include "config.h" - -namespace NYT::NContainers { - -//////////////////////////////////////////////////////////////////////////////// - -void TPodSpecConfig::Register(TRegistrar registrar) -{ - registrar.Parameter("cpu_to_vcpu_factor", &TThis::CpuToVCpuFactor) - .Default(); -} - -//////////////////////////////////////////////////////////////////////////////// - -bool TCGroupConfig::IsCGroupSupported(const TString& cgroupType) const -{ - auto it = std::find_if( - SupportedCGroups.begin(), - SupportedCGroups.end(), - [&] (const TString& type) { - return type == cgroupType; - }); - return it != SupportedCGroups.end(); -} - -void TCGroupConfig::Register(TRegistrar registrar) -{ - registrar.Parameter("supported_cgroups", &TThis::SupportedCGroups) - .Default(); - - registrar.Postprocessor([] (TThis* config) { - for (const auto& type : config->SupportedCGroups) { - if (!IsValidCGroupType(type)) { - THROW_ERROR_EXCEPTION("Invalid cgroup type %Qv", type); - } - } - }); -} - -//////////////////////////////////////////////////////////////////////////////// - -void TPortoExecutorDynamicConfig::Register(TRegistrar registrar) -{ - registrar.Parameter("retries_timeout", &TThis::RetriesTimeout) - .Default(TDuration::Seconds(10)); - registrar.Parameter("poll_period", &TThis::PollPeriod) - .Default(TDuration::MilliSeconds(100)); - registrar.Parameter("api_timeout", &TThis::ApiTimeout) - .Default(TDuration::Minutes(5)); - registrar.Parameter("api_disk_timeout", &TThis::ApiDiskTimeout) - .Default(TDuration::Minutes(30)); - registrar.Parameter("enable_network_isolation", &TThis::EnableNetworkIsolation) - .Default(true); - registrar.Parameter("enable_test_porto_failures", &TThis::EnableTestPortoFailures) - .Default(false); - registrar.Parameter("stub_error_code", &TThis::StubErrorCode) - .Default(EPortoErrorCode::SocketError); - registrar.Parameter("enable_test_porto_not_responding", &TThis::EnableTestPortoNotResponding) - .Default(false); -} - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NContainers diff --git a/yt/yt/library/containers/config.h b/yt/yt/library/containers/config.h deleted file mode 100644 index 3639274cff..0000000000 --- a/yt/yt/library/containers/config.h +++ /dev/null @@ -1,64 +0,0 @@ -#pragma once - -#include "public.h" - -#include <yt/yt/core/ytree/yson_struct.h> - -namespace NYT::NContainers { - -//////////////////////////////////////////////////////////////////////////////// - -class TPodSpecConfig - : public virtual NYTree::TYsonStruct -{ -public: - std::optional<double> CpuToVCpuFactor; - - REGISTER_YSON_STRUCT(TPodSpecConfig); - - static void Register(TRegistrar registrar); -}; - -DEFINE_REFCOUNTED_TYPE(TPodSpecConfig) - -//////////////////////////////////////////////////////////////////////////////// - -class TCGroupConfig - : public virtual NYTree::TYsonStruct -{ -public: - std::vector<TString> SupportedCGroups; - - bool IsCGroupSupported(const TString& cgroupType) const; - - REGISTER_YSON_STRUCT(TCGroupConfig); - - static void Register(TRegistrar registrar); -}; - -//////////////////////////////////////////////////////////////////////////////// - -class TPortoExecutorDynamicConfig - : public NYTree::TYsonStruct -{ -public: - TDuration RetriesTimeout; - TDuration PollPeriod; - TDuration ApiTimeout; - TDuration ApiDiskTimeout; - bool EnableNetworkIsolation; - bool EnableTestPortoFailures; - bool EnableTestPortoNotResponding; - - EPortoErrorCode StubErrorCode; - - REGISTER_YSON_STRUCT(TPortoExecutorDynamicConfig); - - static void Register(TRegistrar registrar); -}; - -DEFINE_REFCOUNTED_TYPE(TPortoExecutorDynamicConfig) - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NContainers diff --git a/yt/yt/library/containers/cri/config.cpp b/yt/yt/library/containers/cri/config.cpp deleted file mode 100644 index 5572f4d980..0000000000 --- a/yt/yt/library/containers/cri/config.cpp +++ /dev/null @@ -1,54 +0,0 @@ -#include "config.h" -#include "cri_api.h" - -namespace NYT::NContainers::NCri { - -//////////////////////////////////////////////////////////////////////////////// - -void TCriExecutorConfig::Register(TRegistrar registrar) -{ - registrar.Parameter("runtime_endpoint", &TThis::RuntimeEndpoint) - .Default(TString(DefaultCriEndpoint)); - - registrar.Parameter("image_endpoint", &TThis::ImageEndpoint) - .Default(TString(DefaultCriEndpoint)); - - registrar.Parameter("namespace", &TThis::Namespace) - .NonEmpty(); - - registrar.Parameter("runtime_handler", &TThis::RuntimeHandler) - .Optional(); - - registrar.Parameter("base_cgroup", &TThis::BaseCgroup) - .NonEmpty(); - - registrar.Parameter("cpu_period", &TThis::CpuPeriod) - .Default(TDuration::MilliSeconds(100)); -} - -//////////////////////////////////////////////////////////////////////////////// - -void TCriAuthConfig::Register(TRegistrar registrar) -{ - registrar.Parameter("username", &TThis::Username) - .Optional(); - - registrar.Parameter("password", &TThis::Password) - .Optional(); - - registrar.Parameter("auth", &TThis::Auth) - .Optional(); - - registrar.Parameter("server_address", &TThis::ServerAddress) - .Optional(); - - registrar.Parameter("identity_token", &TThis::IdentityToken) - .Optional(); - - registrar.Parameter("registry_token", &TThis::RegistryToken) - .Optional(); -} - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NContainers::NCri diff --git a/yt/yt/library/containers/cri/config.h b/yt/yt/library/containers/cri/config.h deleted file mode 100644 index 4ea33fd390..0000000000 --- a/yt/yt/library/containers/cri/config.h +++ /dev/null @@ -1,70 +0,0 @@ -#pragma once - -#include "public.h" - -#include <yt/yt/core/rpc/config.h> - -namespace NYT::NContainers::NCri { - -//////////////////////////////////////////////////////////////////////////////// - -class TCriExecutorConfig - : public NRpc::TRetryingChannelConfig -{ -public: - //! gRPC endpoint for CRI container runtime service. - TString RuntimeEndpoint; - - //! gRPC endpoint for CRI image manager service. - TString ImageEndpoint; - - //! CRI namespace where this executor operates. - TString Namespace; - - //! Name of CRI runtime configuration to use. - TString RuntimeHandler; - - //! Common parent cgroup for all pods. - TString BaseCgroup; - - //! Cpu quota period for cpu limits. - TDuration CpuPeriod; - - REGISTER_YSON_STRUCT(TCriExecutorConfig); - - static void Register(TRegistrar registrar); -}; - -DEFINE_REFCOUNTED_TYPE(TCriExecutorConfig) - -//////////////////////////////////////////////////////////////////////////////// - -// TODO(khlebnikov): split docker registry stuff into common "docker" library. - -//! TCriAuthConfig depicts docker registry authentification -class TCriAuthConfig - : public NYTree::TYsonStruct -{ -public: - TString Username; - - TString Password; - - TString Auth; - - TString ServerAddress; - - TString IdentityToken; - - TString RegistryToken; - - REGISTER_YSON_STRUCT(TCriAuthConfig); - - static void Register(TRegistrar registrar); -}; - -DEFINE_REFCOUNTED_TYPE(TCriAuthConfig) - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NContainers::NCri diff --git a/yt/yt/library/containers/cri/cri_api.cpp b/yt/yt/library/containers/cri/cri_api.cpp deleted file mode 100644 index 93457017ba..0000000000 --- a/yt/yt/library/containers/cri/cri_api.cpp +++ /dev/null @@ -1,33 +0,0 @@ -#include "cri_api.h" - -namespace NYT::NContainers::NCri { - -using namespace NRpc; - -//////////////////////////////////////////////////////////////////////////////// - -TCriRuntimeApi::TCriRuntimeApi(IChannelPtr channel) - : TProxyBase(std::move(channel), GetDescriptor()) -{ } - -const TServiceDescriptor& TCriRuntimeApi::GetDescriptor() -{ - static const auto Descriptor = TServiceDescriptor(NProto::RuntimeService::service_full_name()); - return Descriptor; -} - -//////////////////////////////////////////////////////////////////////////////// - -TCriImageApi::TCriImageApi(IChannelPtr channel) - : TProxyBase(std::move(channel), GetDescriptor()) -{ } - -const TServiceDescriptor& TCriImageApi::GetDescriptor() -{ - static const auto Descriptor = TServiceDescriptor(NProto::ImageService::service_full_name()); - return Descriptor; -} - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NContainers::NCri diff --git a/yt/yt/library/containers/cri/cri_api.h b/yt/yt/library/containers/cri/cri_api.h deleted file mode 100644 index 74fe9a64a0..0000000000 --- a/yt/yt/library/containers/cri/cri_api.h +++ /dev/null @@ -1,99 +0,0 @@ -#pragma once - -#include <yt/yt/core/rpc/client.h> - -#include <k8s.io/cri-api/pkg/apis/runtime/v1/api.grpc.pb.h> - -namespace NYT::NContainers::NCri { - -//////////////////////////////////////////////////////////////////////////////// - -namespace NProto = ::runtime::v1; - -//! Reasonable default for CRI gRPC socket address. -constexpr TStringBuf DefaultCriEndpoint = "unix:///run/containerd/containerd.sock"; - -//! RuntimeReady means the runtime is up and ready to accept basic containers. -constexpr TStringBuf RuntimeReady = "RuntimeReady"; - -//! NetworkReady means the runtime network is up and ready to accept containers which require network. -constexpr TStringBuf NetworkReady = "NetworkReady"; - -//! CRI uses cgroupfs notation for systemd slices, but each name must ends with ".slice". -constexpr TStringBuf SystemdSliceSuffix = ".slice"; - -//////////////////////////////////////////////////////////////////////////////// - -//! CRI labels for pods and containers managed by YT -constexpr TStringBuf YTPodNamespaceLabel = "tech.ytsaurus.pod.namespace"; -constexpr TStringBuf YTPodNameLabel = "tech.ytsaurus.pod.name"; -constexpr TStringBuf YTContainerNameLabel = "tech.ytsaurus.container.name"; -constexpr TStringBuf YTJobIdLabel = "tech.ytsaurus.job.id"; - -//////////////////////////////////////////////////////////////////////////////// - -#define DEFINE_CRI_API_METHOD(method, ...) \ - DEFINE_RPC_PROXY_METHOD_GENERIC(method, NProto::method##Request, NProto::method##Response, __VA_ARGS__) - -//! See https://github.com/kubernetes/cri-api -class TCriRuntimeApi - : public NRpc::TProxyBase -{ -public: - explicit TCriRuntimeApi(NRpc::IChannelPtr channel); - - static const NRpc::TServiceDescriptor& GetDescriptor(); - - DEFINE_CRI_API_METHOD(Version); - DEFINE_CRI_API_METHOD(RunPodSandbox); - DEFINE_CRI_API_METHOD(StopPodSandbox); - DEFINE_CRI_API_METHOD(RemovePodSandbox); - DEFINE_CRI_API_METHOD(PodSandboxStatus); - DEFINE_CRI_API_METHOD(ListPodSandbox); - DEFINE_CRI_API_METHOD(CreateContainer); - DEFINE_CRI_API_METHOD(StartContainer); - DEFINE_CRI_API_METHOD(StopContainer); - DEFINE_CRI_API_METHOD(RemoveContainer); - DEFINE_CRI_API_METHOD(ListContainers); - DEFINE_CRI_API_METHOD(ContainerStatus); - DEFINE_CRI_API_METHOD(UpdateContainerResources); - DEFINE_CRI_API_METHOD(ReopenContainerLog); - DEFINE_CRI_API_METHOD(ExecSync); - DEFINE_CRI_API_METHOD(Exec); - DEFINE_CRI_API_METHOD(Attach); - DEFINE_CRI_API_METHOD(PortForward); - DEFINE_CRI_API_METHOD(ContainerStats); - DEFINE_CRI_API_METHOD(ListContainerStats); - DEFINE_CRI_API_METHOD(PodSandboxStats); - DEFINE_CRI_API_METHOD(ListPodSandboxStats); - DEFINE_CRI_API_METHOD(UpdateRuntimeConfig); - DEFINE_CRI_API_METHOD(Status); - DEFINE_CRI_API_METHOD(CheckpointContainer); - DEFINE_CRI_API_METHOD(ListMetricDescriptors); - DEFINE_CRI_API_METHOD(ListPodSandboxMetrics); - - // FIXME(khlebnikov): figure out streaming results - // DEFINE_RPC_PROXY_METHOD_GENERIC(GetContainerEvents, NProto::GetEventsRequest, NProto::ContainerEventResponse, - // .SetStreamingEnabled(true)); -}; - -//////////////////////////////////////////////////////////////////////////////// - -class TCriImageApi - : public NRpc::TProxyBase -{ -public: - explicit TCriImageApi(NRpc::IChannelPtr channel); - - static const NRpc::TServiceDescriptor& GetDescriptor(); - - DEFINE_CRI_API_METHOD(ListImages); - DEFINE_CRI_API_METHOD(ImageStatus); - DEFINE_CRI_API_METHOD(PullImage); - DEFINE_CRI_API_METHOD(RemoveImage); - DEFINE_CRI_API_METHOD(ImageFsInfo); -}; - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NContainers::NCri diff --git a/yt/yt/library/containers/cri/cri_executor.cpp b/yt/yt/library/containers/cri/cri_executor.cpp deleted file mode 100644 index 428fd93165..0000000000 --- a/yt/yt/library/containers/cri/cri_executor.cpp +++ /dev/null @@ -1,666 +0,0 @@ -#include "cri_executor.h" -#include "private.h" - -#include <yt/yt/core/actions/bind.h> - -#include <yt/yt/core/rpc/grpc/channel.h> - -#include <yt/yt/core/rpc/retrying_channel.h> - -#include <yt/yt/core/misc/error.h> -#include <yt/yt/core/misc/proc.h> -#include <yt/yt/core/misc/protobuf_helpers.h> - -#include <yt/yt/core/concurrency/periodic_executor.h> - -namespace NYT::NContainers::NCri { - -using namespace NRpc; -using namespace NRpc::NGrpc; -using namespace NConcurrency; - -//////////////////////////////////////////////////////////////////////////////// - -void FormatValue(TStringBuilderBase* builder, const TCriDescriptor& descriptor, TStringBuf /*spec*/) -{ - builder->AppendFormat("%v (%s)", descriptor.Id.substr(0, 12), descriptor.Name); -} - -void FormatValue(TStringBuilderBase* builder, const TCriPodDescriptor& descriptor, TStringBuf /*spec*/) -{ - builder->AppendFormat("%v (%s)", descriptor.Id.substr(0, 12), descriptor.Name); -} - -void FormatValue(TStringBuilderBase* builder, const TCriImageDescriptor& descriptor, TStringBuf /*spec*/) -{ - builder->AppendString(descriptor.Image); -} - -static TError DecodeExitCode(int exitCode, const TString& reason) -{ - if (exitCode == 0) { - return TError(); - } - - // TODO(khkebnikov) map reason == "OOMKilled" - - // Common bash notation for signals: 128 + signal - if (exitCode > 128) { - int signalNumber = exitCode - 128; - return TError( - EProcessErrorCode::Signal, - "Process terminated by signal %v", - signalNumber) - << TErrorAttribute("signal", signalNumber) - << TErrorAttribute("reason", reason); - } - - // TODO(khkebnikov) check these - // 125 - container failed to run - // 126 - non executable - // 127 - command not found - // 128 - invalid exit code - // 255 - exit code out of range - - return TError( - EProcessErrorCode::NonZeroExitCode, - "Process exited with code %v", - exitCode) - << TErrorAttribute("exit_code", exitCode) - << TErrorAttribute("reason", reason); -} - -//////////////////////////////////////////////////////////////////////////////// - -class TCriProcess - : public TProcessBase -{ -public: - TCriProcess( - const TString& path, - ICriExecutorPtr executor, - TCriContainerSpecPtr containerSpec, - const TCriPodDescriptor& podDescriptor, - TCriPodSpecPtr podSpec, - TDuration pollPeriod = TDuration::MilliSeconds(100)) - : TProcessBase(path) - , Executor_(std::move(executor)) - , ContainerSpec_(std::move(containerSpec)) - , PodDescriptor_(podDescriptor) - , PodSpec_(std::move(podSpec)) - , PollPeriod_(pollPeriod) - { - // Just for symmetry with sibling classes. - AddArgument(Path_); - } - - void Kill(int /*signal*/) override - { - WaitFor(Executor_->StopContainer(ContainerDescriptor_)) - .ThrowOnError(); - } - - NNet::IConnectionWriterPtr GetStdInWriter() override - { - THROW_ERROR_EXCEPTION("Not implemented for CRI process"); - } - - NNet::IConnectionReaderPtr GetStdOutReader() override - { - THROW_ERROR_EXCEPTION("Not implemented for CRI process"); - } - - NNet::IConnectionReaderPtr GetStdErrReader() override - { - THROW_ERROR_EXCEPTION("Not implemented for CRI process"); - } - -private: - const ICriExecutorPtr Executor_; - const TCriContainerSpecPtr ContainerSpec_; - const TCriPodDescriptor PodDescriptor_; - const TCriPodSpecPtr PodSpec_; - const TDuration PollPeriod_; - - TCriDescriptor ContainerDescriptor_; - - TPeriodicExecutorPtr AsyncWaitExecutor_; - - void DoSpawn() override - { - if (ContainerSpec_->Command.empty()) { - ContainerSpec_->Command = {Path_}; - } - ContainerSpec_->Arguments = std::vector<TString>(Args_.begin() + 1, Args_.end()); - ContainerSpec_->WorkingDirectory = WorkingDirectory_; - - ContainerSpec_->BindMounts.emplace_back( - NCri::TCriBindMount { - .ContainerPath = WorkingDirectory_, - .HostPath = WorkingDirectory_, - .ReadOnly = false, - } - ); - - for (const auto& keyVal : Env_) { - TStringBuf key, val; - if (TStringBuf(keyVal).TrySplit('=', key, val)) { - ContainerSpec_->Environment[key] = val; - } - } - - ContainerDescriptor_ = WaitFor(Executor_->CreateContainer(ContainerSpec_, PodDescriptor_, PodSpec_)) - .ValueOrThrow(); - - YT_LOG_DEBUG("Spawning process (Command: %v, Container: %v)", ContainerSpec_->Command[0], ContainerDescriptor_); - WaitFor(Executor_->StartContainer(ContainerDescriptor_)) - .ThrowOnError(); - - // TODO(khkebnikov) replace polling with CRI event - AsyncWaitExecutor_ = New<TPeriodicExecutor>( - GetSyncInvoker(), - BIND(&TCriProcess::PollContainerStatus, MakeStrong(this)), - PollPeriod_); - - AsyncWaitExecutor_->Start(); - } - - void PollContainerStatus() - { - Executor_->GetContainerStatus(ContainerDescriptor_) - .SubscribeUnique(BIND(&TCriProcess::OnContainerStatus, MakeStrong(this))); - } - - void OnContainerStatus(TErrorOr<TCriRuntimeApi::TRspContainerStatusPtr>&& responseOrError) - { - auto response = responseOrError.ValueOrThrow(); - if (!response->has_status()) { - return; - } - auto status = response->status(); - if (status.state() == NProto::CONTAINER_EXITED) { - auto error = DecodeExitCode(status.exit_code(), status.reason()); - YT_LOG_DEBUG(error, "Process finished (Container: %v)", ContainerDescriptor_); - YT_UNUSED_FUTURE(AsyncWaitExecutor_->Stop()); - FinishedPromise_.TrySet(error); - } - } -}; - -DEFINE_REFCOUNTED_TYPE(TCriProcess) - -//////////////////////////////////////////////////////////////////////////////// - -class TCriExecutor - : public ICriExecutor -{ -public: - TCriExecutor( - TCriExecutorConfigPtr config, - IChannelFactoryPtr channelFactory) - : Config_(std::move(config)) - , RuntimeApi_(CreateRetryingChannel(Config_, channelFactory->CreateChannel(Config_->RuntimeEndpoint))) - , ImageApi_(CreateRetryingChannel(Config_, channelFactory->CreateChannel(Config_->ImageEndpoint))) - { } - - TString GetPodCgroup(TString podName) const override - { - TStringBuilder cgroup; - cgroup.AppendString(Config_->BaseCgroup); - cgroup.AppendString("/"); - cgroup.AppendString(podName); - if (Config_->BaseCgroup.EndsWith(SystemdSliceSuffix)) { - cgroup.AppendString(SystemdSliceSuffix); - } - return cgroup.Flush(); - } - - TFuture<TCriRuntimeApi::TRspStatusPtr> GetRuntimeStatus(bool verbose = false) override - { - auto req = RuntimeApi_.Status(); - req->set_verbose(verbose); - return req->Invoke(); - } - - TFuture<TCriRuntimeApi::TRspListPodSandboxPtr> ListPodSandbox( - std::function<void(NProto::PodSandboxFilter&)> initFilter = nullptr) override - { - auto req = RuntimeApi_.ListPodSandbox(); - - { - auto* filter = req->mutable_filter(); - - if (auto namespace_ = Config_->Namespace) { - auto& labels = *filter->mutable_label_selector(); - labels[YTPodNamespaceLabel] = namespace_; - } - - if (initFilter) { - initFilter(*filter); - } - } - - return req->Invoke(); - } - - TFuture<TCriRuntimeApi::TRspListContainersPtr> ListContainers( - std::function<void(NProto::ContainerFilter&)> initFilter = nullptr) override - { - auto req = RuntimeApi_.ListContainers(); - - { - auto* filter = req->mutable_filter(); - - if (auto namespace_ = Config_->Namespace) { - auto& labels = *filter->mutable_label_selector(); - labels[YTPodNamespaceLabel] = namespace_; - } - - if (initFilter) { - initFilter(*filter); - } - } - - return req->Invoke(); - } - - TFuture<void> ForEachPodSandbox( - const TCallback<void(const TCriPodDescriptor&, const NProto::PodSandbox&)>& callback, - std::function<void(NProto::PodSandboxFilter&)> initFilter) override - { - return ListPodSandbox(initFilter).Apply(BIND([=] (const TCriRuntimeApi::TRspListPodSandboxPtr& rsp) { - for (const auto& pod : rsp->items()) { - TCriPodDescriptor descriptor{.Name=pod.metadata().name(), .Id=pod.id()}; - callback(descriptor, pod); - } - })); - } - - TFuture<void> ForEachContainer( - const TCallback<void(const TCriDescriptor&, const NProto::Container&)>& callback, - std::function<void(NProto::ContainerFilter&)> initFilter = nullptr) override - { - return ListContainers(initFilter).Apply(BIND([=] (const TCriRuntimeApi::TRspListContainersPtr& rsp) { - for (const auto& ct : rsp->containers()) { - TCriDescriptor descriptor{.Name=ct.metadata().name(), .Id=ct.id()}; - callback(descriptor, ct); - } - })); - } - - TFuture<TCriRuntimeApi::TRspPodSandboxStatusPtr> GetPodSandboxStatus( - const TCriPodDescriptor& podDescriptor, bool verbose = false) override - { - auto req = RuntimeApi_.PodSandboxStatus(); - req->set_pod_sandbox_id(podDescriptor.Id); - req->set_verbose(verbose); - return req->Invoke(); - } - - TFuture<TCriRuntimeApi::TRspContainerStatusPtr> GetContainerStatus( - const TCriDescriptor& descriptor, bool verbose = false) override - { - auto req = RuntimeApi_.ContainerStatus(); - req->set_container_id(descriptor.Id); - req->set_verbose(verbose); - return req->Invoke(); - } - - TFuture<TCriPodDescriptor> RunPodSandbox(TCriPodSpecPtr podSpec) override - { - auto req = RuntimeApi_.RunPodSandbox(); - - FillPodSandboxConfig(req->mutable_config(), *podSpec); - - if (Config_->RuntimeHandler) { - req->set_runtime_handler(Config_->RuntimeHandler); - } - - return req->Invoke().Apply(BIND([name = podSpec->Name] (const TCriRuntimeApi::TRspRunPodSandboxPtr& rsp) -> TCriPodDescriptor { - return TCriPodDescriptor{.Name = name, .Id = rsp->pod_sandbox_id()}; - })); - } - - TFuture<void> StopPodSandbox(const TCriPodDescriptor& podDescriptor) override - { - auto req = RuntimeApi_.StopPodSandbox(); - req->set_pod_sandbox_id(podDescriptor.Id); - return req->Invoke().AsVoid(); - } - - TFuture<void> RemovePodSandbox(const TCriPodDescriptor& podDescriptor) override - { - auto req = RuntimeApi_.RemovePodSandbox(); - req->set_pod_sandbox_id(podDescriptor.Id); - return req->Invoke().AsVoid(); - } - - TFuture<void> UpdatePodResources( - const TCriPodDescriptor& /*pod*/, - const TCriContainerResources& /*resources*/) override - { - return MakeFuture(TError("Not implemented")); - } - - TFuture<TCriDescriptor> CreateContainer( - TCriContainerSpecPtr ctSpec, - const TCriPodDescriptor& podDescriptor, - TCriPodSpecPtr podSpec) override - { - auto req = RuntimeApi_.CreateContainer(); - req->set_pod_sandbox_id(podDescriptor.Id); - - auto* config = req->mutable_config(); - - { - auto* metadata = config->mutable_metadata(); - metadata->set_name(ctSpec->Name); - } - - { - auto& labels = *config->mutable_labels(); - - for (const auto& [key, val] : ctSpec->Labels) { - labels[key] = val; - } - - labels[YTPodNamespaceLabel] = Config_->Namespace; - labels[YTPodNameLabel] = podSpec->Name; - labels[YTContainerNameLabel] = ctSpec->Name; - } - - FillImageSpec(config->mutable_image(), ctSpec->Image); - - for (const auto& mountSpec : ctSpec->BindMounts) { - auto* mount = config->add_mounts(); - mount->set_container_path(mountSpec.ContainerPath); - mount->set_host_path(mountSpec.HostPath); - mount->set_readonly(mountSpec.ReadOnly); - mount->set_propagation(NProto::PROPAGATION_PRIVATE); - } - - { - ToProto(config->mutable_command(), ctSpec->Command); - ToProto(config->mutable_args(), ctSpec->Arguments); - - config->set_working_dir(ctSpec->WorkingDirectory); - - for (const auto& [key, val] : ctSpec->Environment) { - auto* env = config->add_envs(); - env->set_key(key); - env->set_value(val); - } - } - - { - auto* linux = config->mutable_linux(); - FillLinuxContainerResources(linux->mutable_resources(), ctSpec->Resources); - - auto* security = linux->mutable_security_context(); - - auto* namespaces = security->mutable_namespace_options(); - namespaces->set_network(NProto::NODE); - - security->set_readonly_rootfs(ctSpec->ReadOnlyRootFS); - - if (ctSpec->Credentials.Uid) { - security->mutable_run_as_user()->set_value(*ctSpec->Credentials.Uid); - } - if (ctSpec->Credentials.Gid) { - security->mutable_run_as_group()->set_value(*ctSpec->Credentials.Gid); - } - ToProto(security->mutable_supplemental_groups(), ctSpec->Credentials.Groups); - } - - FillPodSandboxConfig(req->mutable_sandbox_config(), *podSpec); - - return req->Invoke().Apply(BIND([name = ctSpec->Name] (const TCriRuntimeApi::TRspCreateContainerPtr& rsp) -> TCriDescriptor { - return TCriDescriptor{.Name = "", .Id = rsp->container_id()}; - })); - } - - TFuture<void> StartContainer(const TCriDescriptor& descriptor) override - { - auto req = RuntimeApi_.StartContainer(); - req->set_container_id(descriptor.Id); - return req->Invoke().AsVoid(); - } - - TFuture<void> StopContainer(const TCriDescriptor& descriptor, TDuration timeout) override - { - auto req = RuntimeApi_.StopContainer(); - req->set_container_id(descriptor.Id); - req->set_timeout(timeout.Seconds()); - return req->Invoke().AsVoid(); - } - - TFuture<void> RemoveContainer(const TCriDescriptor& descriptor) override - { - auto req = RuntimeApi_.RemoveContainer(); - req->set_container_id(descriptor.Id); - return req->Invoke().AsVoid(); - } - - TFuture<void> UpdateContainerResources(const TCriDescriptor& descriptor, const TCriContainerResources& resources) override - { - auto req = RuntimeApi_.UpdateContainerResources(); - req->set_container_id(descriptor.Id); - FillLinuxContainerResources(req->mutable_linux(), resources); - return req->Invoke().AsVoid(); - } - - void CleanNamespace() override - { - YT_VERIFY(Config_->Namespace); - auto pods = WaitFor(ListPodSandbox()) - .ValueOrThrow(); - - { - std::vector<TFuture<void>> futures; - futures.reserve(pods->items_size()); - for (const auto& pod : pods->items()) { - TCriPodDescriptor podDescriptor{.Name = pod.metadata().name(), .Id = pod.id() }; - futures.push_back(StopPodSandbox(podDescriptor)); - } - WaitFor(AllSucceeded(std::move(futures))) - .ThrowOnError(); - } - - { - std::vector<TFuture<void>> futures; - futures.reserve(pods->items_size()); - for (const auto& pod : pods->items()) { - TCriPodDescriptor podDescriptor{.Name = pod.metadata().name(), .Id = pod.id()}; - futures.push_back(RemovePodSandbox(podDescriptor)); - } - WaitFor(AllSucceeded(std::move(futures))) - .ThrowOnError(); - } - } - - void CleanPodSandbox(const TCriPodDescriptor& podDescriptor) override - { - auto containers = WaitFor(ListContainers([=] (NProto::ContainerFilter& filter) { - filter.set_pod_sandbox_id(podDescriptor.Id); - })) - .ValueOrThrow(); - - { - std::vector<TFuture<void>> futures; - futures.reserve(containers->containers_size()); - for (const auto& ct : containers->containers()) { - TCriDescriptor ctDescriptor{.Name = ct.metadata().name(), .Id = ct.id()}; - futures.push_back(StopContainer(ctDescriptor, TDuration::Zero())); - } - WaitFor(AllSucceeded(std::move(futures))) - .ThrowOnError(); - } - - { - std::vector<TFuture<void>> futures; - futures.reserve(containers->containers_size()); - for (const auto& ct : containers->containers()) { - TCriDescriptor ctDescriptor{.Name = ct.metadata().name(), .Id = ct.id()}; - futures.push_back(RemoveContainer(ctDescriptor)); - } - WaitFor(AllSucceeded(std::move(futures))) - .ThrowOnError(); - } - } - - TFuture<TCriImageApi::TRspListImagesPtr> ListImages( - std::function<void(NProto::ImageFilter&)> initFilter = nullptr) override - { - auto req = ImageApi_.ListImages(); - if (initFilter) { - initFilter(*req->mutable_filter()); - } - return req->Invoke(); - } - - TFuture<TCriImageApi::TRspImageStatusPtr> GetImageStatus( - const TCriImageDescriptor& image, - bool verbose = false) override - { - auto req = ImageApi_.ImageStatus(); - FillImageSpec(req->mutable_image(), image); - req->set_verbose(verbose); - return req->Invoke(); - } - - TFuture<TCriImageDescriptor> PullImage( - const TCriImageDescriptor& image, - bool always, - TCriAuthConfigPtr authConfig, - TCriPodSpecPtr podSpec) override - { - if (!always) { - return GetImageStatus(image) - .Apply(BIND([=, this, this_ = MakeStrong(this)] (const TCriImageApi::TRspImageStatusPtr& imageStatus) { - if (imageStatus->has_image()) { - return MakeFuture(TCriImageDescriptor{.Image = imageStatus->image().id()}); - } - return PullImage(image, /*always*/ true, authConfig, podSpec); - })); - } - - auto req = ImageApi_.PullImage(); - FillImageSpec(req->mutable_image(), image); - if (authConfig) { - FillAuthConfig(req->mutable_auth(), *authConfig); - } - if (podSpec) { - FillPodSandboxConfig(req->mutable_sandbox_config(), *podSpec); - } - return req->Invoke().Apply(BIND([] (const TCriImageApi::TRspPullImagePtr& rsp) -> TCriImageDescriptor { - return TCriImageDescriptor{.Image = rsp->image_ref()}; - })); - } - - TFuture<void> RemoveImage(const TCriImageDescriptor& image) override - { - auto req = ImageApi_.RemoveImage(); - FillImageSpec(req->mutable_image(), image); - return req->Invoke().AsVoid(); - } - - TProcessBasePtr CreateProcess( - const TString& path, - TCriContainerSpecPtr containerSpec, - const TCriPodDescriptor& podDescriptor, - TCriPodSpecPtr podSpec) override - { - return New<TCriProcess>(path, this, std::move(containerSpec), podDescriptor, std::move(podSpec)); - } - -private: - const TCriExecutorConfigPtr Config_; - TCriRuntimeApi RuntimeApi_; - TCriImageApi ImageApi_; - - void FillLinuxContainerResources(NProto::LinuxContainerResources* resources, const TCriContainerResources& spec) - { - auto* unified = resources->mutable_unified(); - - if (spec.CpuLimit) { - i64 period = Config_->CpuPeriod.MicroSeconds(); - i64 quota = period * *spec.CpuLimit; - - resources->set_cpu_period(period); - resources->set_cpu_quota(quota); - } - - if (spec.MemoryLimit) { - resources->set_memory_limit_in_bytes(*spec.MemoryLimit); - } - - if (spec.MemoryRequest) { - (*unified)["memory.low"] = ToString(*spec.MemoryRequest); - } - } - - void FillPodSandboxConfig(NProto::PodSandboxConfig* config, const TCriPodSpec& spec) - { - { - auto* metadata = config->mutable_metadata(); - metadata->set_namespace_(Config_->Namespace); - metadata->set_name(spec.Name); - metadata->set_uid(spec.Name); - } - - { - auto& labels = *config->mutable_labels(); - labels[YTPodNamespaceLabel] = Config_->Namespace; - labels[YTPodNameLabel] = spec.Name; - } - - { - auto* linux = config->mutable_linux(); - linux->set_cgroup_parent(GetPodCgroup(spec.Name)); - - auto* security = linux->mutable_security_context(); - auto* namespaces = security->mutable_namespace_options(); - namespaces->set_network(NProto::NODE); - } - } - - void FillImageSpec(NProto::ImageSpec* spec, const TCriImageDescriptor& image) - { - spec->set_image(image.Image); - } - - void FillAuthConfig(NProto::AuthConfig* auth, const TCriAuthConfig& authConfig) - { - if (!authConfig.Username.empty()) { - auth->set_username(authConfig.Username); - } - if (!authConfig.Password.empty()) { - auth->set_password(authConfig.Password); - } - if (!authConfig.Auth.empty()) { - auth->set_auth(authConfig.Auth); - } - if (!authConfig.ServerAddress.empty()) { - auth->set_server_address(authConfig.ServerAddress); - } - if (!authConfig.IdentityToken.empty()) { - auth->set_identity_token(authConfig.IdentityToken); - } - if (!authConfig.RegistryToken.empty()) { - auth->set_registry_token(authConfig.RegistryToken); - } - } -}; - -//////////////////////////////////////////////////////////////////////////////// - -ICriExecutorPtr CreateCriExecutor(TCriExecutorConfigPtr config) -{ - return New<TCriExecutor>( - std::move(config), - GetGrpcChannelFactory()); -} - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NContainers::NCri diff --git a/yt/yt/library/containers/cri/cri_executor.h b/yt/yt/library/containers/cri/cri_executor.h deleted file mode 100644 index de9741721f..0000000000 --- a/yt/yt/library/containers/cri/cri_executor.h +++ /dev/null @@ -1,207 +0,0 @@ -#pragma once - -#include "public.h" -#include "config.h" -#include "cri_api.h" - -#include <yt/yt/library/process/process.h> - -#include <yt/yt/core/ytree/yson_struct.h> - -namespace NYT::NContainers::NCri { - -//////////////////////////////////////////////////////////////////////////////// - -struct TCriDescriptor -{ - TString Name; - TString Id; -}; - -struct TCriPodDescriptor -{ - TString Name; - TString Id; -}; - -struct TCriImageDescriptor -{ - TString Image; -}; - -void FormatValue(TStringBuilderBase* builder, const TCriDescriptor& descriptor, TStringBuf spec); -void FormatValue(TStringBuilderBase* builder, const TCriPodDescriptor& descriptor, TStringBuf spec); -void FormatValue(TStringBuilderBase* builder, const TCriImageDescriptor& descriptor, TStringBuf spec); - -//////////////////////////////////////////////////////////////////////////////// - -struct TCriContainerResources -{ - std::optional<double> CpuLimit; - std::optional<double> CpuRequest; - std::optional<i64> MemoryLimit; - std::optional<i64> MemoryRequest; -}; - -struct TCriPodSpec - : public TRefCounted -{ - TString Name; - TCriContainerResources Resources; -}; - -DEFINE_REFCOUNTED_TYPE(TCriPodSpec) - -struct TCriBindMount -{ - TString ContainerPath; - TString HostPath; - bool ReadOnly; -}; - -struct TCriCredentials -{ - std::optional<i64> Uid; - std::optional<i64> Gid; - std::vector<i64> Groups; -}; - -struct TCriContainerSpec - : public TRefCounted -{ - TString Name; - - THashMap<TString, TString> Labels; - - TCriImageDescriptor Image; - - bool ReadOnlyRootFS; - - std::vector<TCriBindMount> BindMounts; - - TCriCredentials Credentials; - - TCriContainerResources Resources; - - //! Command to execute (i.e., entrypoint for docker). - std::vector<TString> Command; - - //! Arguments for the Command (i.e., command for docker). - std::vector<TString> Arguments; - - //! Current working directory of the command. - TString WorkingDirectory; - - //! Environment variable to set in the container. - THashMap<TString, TString> Environment; -}; - -DEFINE_REFCOUNTED_TYPE(TCriContainerSpec) - -//////////////////////////////////////////////////////////////////////////////// - -//! Wrapper around CRI gRPC API -//! -//! @see yt/yt/contrib/cri-api/k8s.io/cri-api/pkg/apis/runtime/v1/api.proto -//! @see https://github.com/kubernetes/cri-api -struct ICriExecutor - : public TRefCounted -{ - //! Returns status of the CRI runtime. - //! @param verbose fill field "info" with runtime-specific debug. - virtual TFuture<TCriRuntimeApi::TRspStatusPtr> GetRuntimeStatus(bool verbose = false) = 0; - - // PodSandbox - - virtual TString GetPodCgroup(TString podName) const = 0; - - virtual TFuture<TCriRuntimeApi::TRspListPodSandboxPtr> ListPodSandbox( - std::function<void(NProto::PodSandboxFilter&)> initFilter = nullptr) = 0; - - virtual TFuture<TCriRuntimeApi::TRspListContainersPtr> ListContainers( - std::function<void(NProto::ContainerFilter&)> initFilter = nullptr) = 0; - - virtual TFuture<void> ForEachPodSandbox( - const TCallback<void(const TCriPodDescriptor&, const NProto::PodSandbox&)>& callback, - std::function<void(NProto::PodSandboxFilter&)> initFilter = nullptr) = 0; - - virtual TFuture<void> ForEachContainer( - const TCallback<void(const TCriDescriptor&, const NProto::Container&)>& callback, - std::function<void(NProto::ContainerFilter&)> initFilter = nullptr) = 0; - - //! Returns status of the pod. - //! @param verbose fill field "info" with runtime-specific debug. - virtual TFuture<TCriRuntimeApi::TRspPodSandboxStatusPtr> GetPodSandboxStatus( - const TCriPodDescriptor& pod, bool verbose = false) = 0; - - //! Returns status of the container. - //! @param verbose fill "info" with runtime-specific debug information. - virtual TFuture<TCriRuntimeApi::TRspContainerStatusPtr> GetContainerStatus( - const TCriDescriptor& ct, bool verbose = false) = 0; - - virtual TFuture<TCriPodDescriptor> RunPodSandbox(TCriPodSpecPtr podSpec) = 0; - virtual TFuture<void> StopPodSandbox(const TCriPodDescriptor& pod) = 0; - virtual TFuture<void> RemovePodSandbox(const TCriPodDescriptor& pod) = 0; - virtual TFuture<void> UpdatePodResources( - const TCriPodDescriptor& pod, - const TCriContainerResources& resources) = 0; - - //! Remove all pods and containers in namespace managed by executor. - virtual void CleanNamespace() = 0; - - //! Remove all containers in one pod. - virtual void CleanPodSandbox(const TCriPodDescriptor& pod) = 0; - - virtual TFuture<TCriDescriptor> CreateContainer( - TCriContainerSpecPtr containerSpec, - const TCriPodDescriptor& pod, - TCriPodSpecPtr podSpec) = 0; - - virtual TFuture<void> StartContainer(const TCriDescriptor& ct) = 0; - - //! Stops container if it's running. - //! @param timeout defines timeout for graceful stop, timeout=0 - kill instantly. - virtual TFuture<void> StopContainer( - const TCriDescriptor& ct, - TDuration timeout = TDuration::Zero()) = 0; - - virtual TFuture<void> RemoveContainer(const TCriDescriptor& ct) = 0; - - virtual TFuture<void> UpdateContainerResources( - const TCriDescriptor& ct, - const TCriContainerResources& resources) = 0; - - virtual TFuture<TCriImageApi::TRspListImagesPtr> ListImages( - std::function<void(NProto::ImageFilter&)> initFilter = nullptr) = 0; - - //! Returns status of the image. - //! @param verbose fill field "info" with runtime-specific debug. - virtual TFuture<TCriImageApi::TRspImageStatusPtr> GetImageStatus( - const TCriImageDescriptor& image, - bool verbose = false) = 0; - - virtual TFuture<TCriImageDescriptor> PullImage( - const TCriImageDescriptor& image, - bool always = false, - TCriAuthConfigPtr authConfig = nullptr, - TCriPodSpecPtr podSpec = nullptr) = 0; - - virtual TFuture<void> RemoveImage(const TCriImageDescriptor& image) = 0; - - // FIXME(khlebnikov): temporary compat - virtual TProcessBasePtr CreateProcess( - const TString& path, - TCriContainerSpecPtr containerSpec, - const TCriPodDescriptor& pod, - TCriPodSpecPtr podSpec) = 0; -}; - -DEFINE_REFCOUNTED_TYPE(ICriExecutor) - -//////////////////////////////////////////////////////////////////////////////// - -ICriExecutorPtr CreateCriExecutor(TCriExecutorConfigPtr config); - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NContainers::NCri diff --git a/yt/yt/library/containers/cri/private.h b/yt/yt/library/containers/cri/private.h deleted file mode 100644 index 36fdf194f5..0000000000 --- a/yt/yt/library/containers/cri/private.h +++ /dev/null @@ -1,13 +0,0 @@ -#pragma once - -#include <yt/yt/core/logging/log.h> - -namespace NYT::NContainers::NCri { - -//////////////////////////////////////////////////////////////////////////////// - -inline const NLogging::TLogger Logger("Cri"); - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NContainers::NCri diff --git a/yt/yt/library/containers/cri/public.h b/yt/yt/library/containers/cri/public.h deleted file mode 100644 index a12ee86d57..0000000000 --- a/yt/yt/library/containers/cri/public.h +++ /dev/null @@ -1,17 +0,0 @@ -#pragma once - -#include <yt/yt/core/misc/intrusive_ptr.h> - -namespace NYT::NContainers::NCri { - -//////////////////////////////////////////////////////////////////////////////// - -DECLARE_REFCOUNTED_STRUCT(TCriPodSpec) -DECLARE_REFCOUNTED_STRUCT(TCriContainerSpec) -DECLARE_REFCOUNTED_CLASS(TCriExecutorConfig) -DECLARE_REFCOUNTED_CLASS(TCriAuthConfig) -DECLARE_REFCOUNTED_STRUCT(ICriExecutor) - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NContainers::NCri diff --git a/yt/yt/library/containers/cri/ya.make b/yt/yt/library/containers/cri/ya.make deleted file mode 100644 index dc9dd15a0b..0000000000 --- a/yt/yt/library/containers/cri/ya.make +++ /dev/null @@ -1,22 +0,0 @@ -LIBRARY() - -INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc) - -PEERDIR( - yt/yt/core - yt/yt/core/rpc/grpc - yt/yt/contrib/cri-api -) - -SRCS( - cri_api.cpp - cri_executor.cpp - config.cpp -) - -ADDINCL( - ONE_LEVEL - yt/yt/contrib/cri-api -) - -END() diff --git a/yt/yt/library/containers/disk_manager/config.cpp b/yt/yt/library/containers/disk_manager/config.cpp deleted file mode 100644 index 84484db630..0000000000 --- a/yt/yt/library/containers/disk_manager/config.cpp +++ /dev/null @@ -1,61 +0,0 @@ -#include "config.h" - -namespace NYT::NContainers { - -//////////////////////////////////////////////////////////////////////////////// - -void TMockedDiskConfig::Register(TRegistrar registrar) -{ - registrar.Parameter("disk_id", &TThis::DiskId) - .Default(); - registrar.Parameter("device_path", &TThis::DevicePath) - .Default(); - registrar.Parameter("device_name", &TThis::DeviceName) - .Default(); - registrar.Parameter("disk_model", &TThis::DiskModel) - .Default(); - registrar.Parameter("partition_fs_labels", &TThis::PartitionFsLabels) - .Default(); - registrar.Parameter("state", &TThis::State) - .Default(EDiskState::Ok); -} - -//////////////////////////////////////////////////////////////////////////////// - -void TDiskInfoProviderConfig::Register(TRegistrar registrar) -{ - registrar.Parameter("disk_ids", &TThis::DiskIds) - .Default(); -} - -//////////////////////////////////////////////////////////////////////////////// - -void TDiskManagerProxyConfig::Register(TRegistrar registrar) -{ - registrar.Parameter("disk_manager_address", &TThis::DiskManagerAddress) - .Default("unix:/run/yandex-diskmanager/yandex-diskmanager.sock"); - registrar.Parameter("disk_manager_service_name", &TThis::DiskManagerServiceName) - .Default("diskman.DiskManager"); - - registrar.Parameter("is_mock", &TThis::IsMock) - .Default(false); - registrar.Parameter("mock_disks", &TThis::MockDisks) - .Default(); - registrar.Parameter("mock_yt_paths", &TThis::MockYtPaths) - .Default(); - - registrar.Parameter("request_timeout", &TThis::RequestTimeout) - .Default(TDuration::Seconds(10)); -} - -//////////////////////////////////////////////////////////////////////////////// - -void TDiskManagerProxyDynamicConfig::Register(TRegistrar registrar) -{ - registrar.Parameter("request_timeout", &TThis::RequestTimeout) - .Default(); -} - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NContainers diff --git a/yt/yt/library/containers/disk_manager/config.h b/yt/yt/library/containers/disk_manager/config.h deleted file mode 100644 index 4f01d378b9..0000000000 --- a/yt/yt/library/containers/disk_manager/config.h +++ /dev/null @@ -1,79 +0,0 @@ -#pragma once - -#include "public.h" - -#include <yt/yt/core/ytree/yson_struct.h> - -namespace NYT::NContainers { - -//////////////////////////////////////////////////////////////////////////////// - -struct TMockedDiskConfig - : public NYTree::TYsonStruct -{ - TString DiskId; - TString DevicePath; - TString DeviceName; - TString DiskModel; - std::vector<TString> PartitionFsLabels; - EDiskState State; - - REGISTER_YSON_STRUCT(TMockedDiskConfig); - - static void Register(TRegistrar registrar); -}; - -DEFINE_REFCOUNTED_TYPE(TMockedDiskConfig) - -//////////////////////////////////////////////////////////////////////////////// - -struct TDiskManagerProxyConfig - : public NYTree::TYsonStruct -{ - TString DiskManagerAddress; - TString DiskManagerServiceName; - - bool IsMock; - std::vector<TMockedDiskConfigPtr> MockDisks; - std::vector<TString> MockYtPaths; - - TDuration RequestTimeout; - - REGISTER_YSON_STRUCT(TDiskManagerProxyConfig); - - static void Register(TRegistrar registrar); -}; - -DEFINE_REFCOUNTED_TYPE(TDiskManagerProxyConfig) - -//////////////////////////////////////////////////////////////////////////////// - -struct TDiskInfoProviderConfig - : public NYTree::TYsonStruct -{ - std::vector<TString> DiskIds; - - REGISTER_YSON_STRUCT(TDiskInfoProviderConfig); - - static void Register(TRegistrar registrar); -}; - -DEFINE_REFCOUNTED_TYPE(TDiskInfoProviderConfig) - -//////////////////////////////////////////////////////////////////////////////// - -struct TDiskManagerProxyDynamicConfig - : public NYTree::TYsonStruct -{ - std::optional<TDuration> RequestTimeout; - - REGISTER_YSON_STRUCT(TDiskManagerProxyDynamicConfig); - - static void Register(TRegistrar registrar); -}; - -DEFINE_REFCOUNTED_TYPE(TDiskManagerProxyDynamicConfig) - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NContainers diff --git a/yt/yt/library/containers/disk_manager/disk_info_provider.cpp b/yt/yt/library/containers/disk_manager/disk_info_provider.cpp deleted file mode 100644 index 0ee3a5b6cb..0000000000 --- a/yt/yt/library/containers/disk_manager/disk_info_provider.cpp +++ /dev/null @@ -1,64 +0,0 @@ -#include "disk_info_provider.h" - -#include <yt/yt/library/containers/disk_manager/disk_manager_proxy.h> - -#include <yt/yt/core/actions/future.h> -#include <yt/yt/core/actions/invoker_util.h> - -#include <yt/yt/core/concurrency/public.h> - -namespace NYT::NContainers { - -//////////////////////////////////////////////////////////////////////////////// - -TDiskInfoProvider::TDiskInfoProvider( - IDiskManagerProxyPtr diskManagerProxy, - TDiskInfoProviderConfigPtr config) - : DiskManagerProxy_(std::move(diskManagerProxy)) - , Config_(std::move(config)) -{ } - -const std::vector<TString>& TDiskInfoProvider::GetConfigDiskIds() const -{ - return Config_->DiskIds; -} - -TFuture<std::vector<TDiskInfo>> TDiskInfoProvider::GetYTDiskInfos() -{ - auto diskInfosFuture = DiskManagerProxy_->GetDisks(); - auto ytDiskPathsFuture = DiskManagerProxy_->GetYtDiskMountPaths(); - - // Merge two futures and filter disks placed in /yt. - return diskInfosFuture.Apply(BIND([=] (const std::vector<TDiskInfo>& diskInfos) { - return ytDiskPathsFuture.Apply(BIND([=] (const THashSet<TString>& diskPaths) { - std::vector<TDiskInfo> disks; - - for (const auto& diskInfo : diskInfos) { - for (const auto& partitionFsLabel : diskInfo.PartitionFsLabels) { - if (diskPaths.contains(partitionFsLabel)) { - disks.push_back(diskInfo); - break; - } - } - } - - return disks; - })); - })); -} - -TFuture<void> TDiskInfoProvider::RecoverDisk(const TString& diskId) -{ - return DiskManagerProxy_->RecoverDiskById(diskId, ERecoverPolicy::RecoverAuto); -} - -TFuture<void> TDiskInfoProvider::FailDisk( - const TString& diskId, - const TString& reason) -{ - return DiskManagerProxy_->FailDiskById(diskId, reason); -} - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NContainers diff --git a/yt/yt/library/containers/disk_manager/disk_info_provider.h b/yt/yt/library/containers/disk_manager/disk_info_provider.h deleted file mode 100644 index b8d686438d..0000000000 --- a/yt/yt/library/containers/disk_manager/disk_info_provider.h +++ /dev/null @@ -1,38 +0,0 @@ -#pragma once - -#include "public.h" - -#include <yt/yt/core/actions/future.h> - -namespace NYT::NContainers { - -//////////////////////////////////////////////////////////////////////////////// - -class TDiskInfoProvider - : public TRefCounted -{ -public: - TDiskInfoProvider( - IDiskManagerProxyPtr diskManagerProxy, - TDiskInfoProviderConfigPtr config); - - const std::vector<TString>& GetConfigDiskIds() const; - - TFuture<std::vector<TDiskInfo>> GetYTDiskInfos(); - - TFuture<void> RecoverDisk(const TString& diskId); - - TFuture<void> FailDisk( - const TString& diskId, - const TString& reason); - -private: - const IDiskManagerProxyPtr DiskManagerProxy_; - const TDiskInfoProviderConfigPtr Config_; -}; - -DEFINE_REFCOUNTED_TYPE(TDiskInfoProvider) - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NContainers diff --git a/yt/yt/library/containers/disk_manager/disk_manager_proxy.cpp b/yt/yt/library/containers/disk_manager/disk_manager_proxy.cpp deleted file mode 100644 index 961723c51f..0000000000 --- a/yt/yt/library/containers/disk_manager/disk_manager_proxy.cpp +++ /dev/null @@ -1,49 +0,0 @@ -#include "disk_manager_proxy.h" - -namespace NYT::NContainers { - -//////////////////////////////////////////////////////////////////////////////// - -struct TDiskManagerProxyMock - : public IDiskManagerProxy -{ - virtual TFuture<THashSet<TString>> GetYtDiskMountPaths() - { - THROW_ERROR_EXCEPTION("Disk manager library is not available under this build configuration"); - } - - virtual TFuture<std::vector<TDiskInfo>> GetDisks() - { - THROW_ERROR_EXCEPTION("Disk manager library is not available under this build configuration"); - } - - virtual TFuture<void> RecoverDiskById(const TString& /*diskId*/, ERecoverPolicy /*recoverPolicy*/) - { - THROW_ERROR_EXCEPTION("Disk manager library is not available under this build configuration"); - } - - virtual TFuture<void> FailDiskById(const TString& /*diskId*/, const TString& /*reason*/) - { - THROW_ERROR_EXCEPTION("Disk manager library is not available under this build configuration"); - } - - virtual void OnDynamicConfigChanged(const TDiskManagerProxyDynamicConfigPtr& /*newConfig*/) - { - // Do nothing - } -}; - -DEFINE_REFCOUNTED_TYPE(TDiskManagerProxyMock) - -//////////////////////////////////////////////////////////////////////////////// - -Y_WEAK IDiskManagerProxyPtr CreateDiskManagerProxy(TDiskManagerProxyConfigPtr /*config*/) -{ - // This implementation is used when disk_manager_proxy_impl.cpp is not linked. - - return New<TDiskManagerProxyMock>(); -} - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NContainers diff --git a/yt/yt/library/containers/disk_manager/disk_manager_proxy.h b/yt/yt/library/containers/disk_manager/disk_manager_proxy.h deleted file mode 100644 index d2da5c1873..0000000000 --- a/yt/yt/library/containers/disk_manager/disk_manager_proxy.h +++ /dev/null @@ -1,38 +0,0 @@ -#pragma once - -#include "public.h" - -#include <yt/yt/library/containers/disk_manager/config.h> - -#include <yt/yt/core/misc/atomic_object.h> - -#include <yt/yt/core/rpc/client.h> - -namespace NYT::NContainers { - -//////////////////////////////////////////////////////////////////////////////// - -struct IDiskManagerProxy - : public virtual TRefCounted -{ - virtual TFuture<THashSet<TString>> GetYtDiskMountPaths() = 0; - - virtual TFuture<std::vector<TDiskInfo>> GetDisks() = 0; - - virtual TFuture<void> RecoverDiskById(const TString& diskId, ERecoverPolicy recoverPolicy) = 0; - - virtual TFuture<void> FailDiskById(const TString& diskId, const TString& reason) = 0; - - virtual void OnDynamicConfigChanged(const TDiskManagerProxyDynamicConfigPtr& newConfig) = 0; - -}; - -DEFINE_REFCOUNTED_TYPE(IDiskManagerProxy) - -//////////////////////////////////////////////////////////////////////////////// - -IDiskManagerProxyPtr CreateDiskManagerProxy(TDiskManagerProxyConfigPtr config); - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NContainers diff --git a/yt/yt/library/containers/disk_manager/public.h b/yt/yt/library/containers/disk_manager/public.h deleted file mode 100644 index 8a812638f3..0000000000 --- a/yt/yt/library/containers/disk_manager/public.h +++ /dev/null @@ -1,48 +0,0 @@ -#pragma once - -#include <yt/yt/core/misc/public.h> - -namespace NYT::NContainers { - -//////////////////////////////////////////////////////////////////////////////// - -DEFINE_ENUM(EDiskState, - ((Unknown) (0)) - ((Ok) (1)) - ((Failed) (2)) - ((RecoverWait) (3)) -); - -// 1. Remount all disk volumes to it's default state -// 2. Recreate disk layout, all data on disk will be lost -// 3. Replace phisical disk -DEFINE_ENUM(ERecoverPolicy, - ((RecoverAuto) (0)) - ((RecoverMount) (1)) - ((RecoverLayout) (2)) - ((RecoverDisk) (3)) -); - -struct TDiskInfo -{ - TString DiskId; - TString DevicePath; - TString DeviceName; - TString DiskModel; - THashSet<TString> PartitionFsLabels; - EDiskState State; -}; - -//////////////////////////////////////////////////////////////////////////////// - -DECLARE_REFCOUNTED_STRUCT(TMockedDiskConfig) -DECLARE_REFCOUNTED_STRUCT(TDiskManagerProxyConfig) -DECLARE_REFCOUNTED_STRUCT(TDiskManagerProxyDynamicConfig) -DECLARE_REFCOUNTED_STRUCT(TDiskInfoProviderConfig) - -DECLARE_REFCOUNTED_STRUCT(IDiskManagerProxy) -DECLARE_REFCOUNTED_CLASS(TDiskInfoProvider) - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NContainers diff --git a/yt/yt/library/containers/disk_manager/ya.make b/yt/yt/library/containers/disk_manager/ya.make deleted file mode 100644 index dcb260cf38..0000000000 --- a/yt/yt/library/containers/disk_manager/ya.make +++ /dev/null @@ -1,19 +0,0 @@ -LIBRARY() - -INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc) - -PEERDIR( - yt/yt/core -) - -SRCS( - config.cpp - disk_info_provider.cpp - disk_manager_proxy.cpp -) - -IF (NOT OPENSOURCE) - INCLUDE(ya_non_opensource.inc) -ENDIF() - -END() diff --git a/yt/yt/library/containers/instance.cpp b/yt/yt/library/containers/instance.cpp deleted file mode 100644 index 0a56987e1b..0000000000 --- a/yt/yt/library/containers/instance.cpp +++ /dev/null @@ -1,812 +0,0 @@ -#ifdef __linux__ - -#include "instance.h" - -#include "porto_executor.h" -#include "private.h" - -#include <yt/yt/library/containers/cgroup.h> -#include <yt/yt/library/containers/config.h> - -#include <yt/yt/core/concurrency/scheduler.h> - -#include <yt/yt/core/logging/log.h> - -#include <yt/yt/core/misc/collection_helpers.h> -#include <yt/yt/core/misc/error.h> -#include <yt/yt/core/misc/fs.h> -#include <yt/yt/core/misc/proc.h> - -#include <library/cpp/porto/libporto.hpp> - -#include <util/stream/file.h> - -#include <util/string/cast.h> -#include <util/string/split.h> - -#include <util/system/env.h> - -#include <initializer_list> -#include <string> - -namespace NYT::NContainers { - -using namespace NConcurrency; -using namespace NNet; - -//////////////////////////////////////////////////////////////////////////////// - -namespace NDetail { - -// Porto passes command string to wordexp, where quota (') symbol -// is delimiter. So we must replace it with concatenation ('"'"'). -TString EscapeForWordexp(const char* in) -{ - TString buffer; - while (*in) { - if (*in == '\'') { - buffer.append(R"('"'"')"); - } else { - buffer.append(*in); - } - in++; - } - return buffer; -} - -i64 Extract( - const TString& input, - const TString& pattern, - const TString& terminator = "\n") -{ - auto start = input.find(pattern) + pattern.length(); - auto end = input.find(terminator, start); - return std::stol(input.substr(start, (end == input.npos) ? end : end - start)); -} - -i64 ExtractSum( - const TString& input, - const TString& pattern, - const TString& delimiter, - const TString& terminator = "\n") -{ - i64 sum = 0; - TString::size_type pos = 0; - while (pos < input.length()) { - pos = input.find(pattern, pos); - if (pos == input.npos) { - break; - } - pos += pattern.length(); - - pos = input.find(delimiter, pos); - if (pos == input.npos) { - break; - } - - pos++; - auto end = input.find(terminator, pos); - sum += std::stol(input.substr(pos, (end == input.npos) ? end : end - pos)); - } - return sum; -} - -using TPortoStatRule = std::pair<TString, std::function<i64(const TString& input)>>; - -static const std::function<i64(const TString&)> LongExtractor = [] (const TString& in) { - return std::stol(in); -}; - -static const std::function<i64(const TString&)> CoreNsPerSecondExtractor = [] (const TString& in) { - int pos = in.find("c", 0); - return (std::stod(in.substr(0, pos))) * 1'000'000'000; -}; - -static const std::function<i64(const TString&)> GetIOStatExtractor(const TString& rwMode = "") -{ - return [rwMode] (const TString& in) { - return ExtractSum(in, "hw", rwMode + ":", ";"); - }; -} - -static const std::function<i64(const TString&)> GetStatByKeyExtractor(const TString& statKey) -{ - return [statKey] (const TString& in) { - return Extract(in, statKey); - }; -} - -const THashMap<EStatField, TPortoStatRule> PortoStatRules = { - {EStatField::CpuUsage, {"cpu_usage", LongExtractor}}, - {EStatField::CpuSystemUsage, {"cpu_usage_system", LongExtractor}}, - {EStatField::CpuWait, {"cpu_wait", LongExtractor}}, - {EStatField::CpuThrottled, {"cpu_throttled", LongExtractor}}, - {EStatField::ThreadCount, {"thread_count", LongExtractor}}, - {EStatField::CpuLimit, {"cpu_limit_bound", CoreNsPerSecondExtractor}}, - {EStatField::CpuGuarantee, {"cpu_guarantee_bound", CoreNsPerSecondExtractor}}, - {EStatField::Rss, {"memory.stat", GetStatByKeyExtractor("total_rss")}}, - {EStatField::MappedFile, {"memory.stat", GetStatByKeyExtractor("total_mapped_file")}}, - {EStatField::MinorPageFaults, {"minor_faults", LongExtractor}}, - {EStatField::MajorPageFaults, {"major_faults", LongExtractor}}, - {EStatField::FileCacheUsage, {"cache_usage", LongExtractor}}, - {EStatField::AnonMemoryUsage, {"anon_usage", LongExtractor}}, - {EStatField::AnonMemoryLimit, {"anon_limit_total", LongExtractor}}, - {EStatField::MemoryUsage, {"memory_usage", LongExtractor}}, - {EStatField::MemoryGuarantee, {"memory_guarantee", LongExtractor}}, - {EStatField::MemoryLimit, {"memory_limit_total", LongExtractor}}, - {EStatField::MaxMemoryUsage, {"memory.max_usage_in_bytes", LongExtractor}}, - {EStatField::OomKills, {"oom_kills", LongExtractor}}, - {EStatField::OomKillsTotal, {"oom_kills_total", LongExtractor}}, - - {EStatField::IOReadByte, {"io_read", GetIOStatExtractor()}}, - {EStatField::IOWriteByte, {"io_write", GetIOStatExtractor()}}, - {EStatField::IOBytesLimit, {"io_limit", GetIOStatExtractor()}}, - {EStatField::IOReadOps, {"io_read_ops", GetIOStatExtractor()}}, - {EStatField::IOWriteOps, {"io_write_ops", GetIOStatExtractor()}}, - {EStatField::IOOps, {"io_ops", GetIOStatExtractor()}}, - {EStatField::IOOpsLimit, {"io_ops_limit", GetIOStatExtractor()}}, - {EStatField::IOTotalTime, {"io_time", GetIOStatExtractor()}}, - {EStatField::IOWaitTime, {"io_wait", GetIOStatExtractor()}}, - - {EStatField::NetTxBytes, {"net_tx_bytes[veth]", LongExtractor}}, - {EStatField::NetTxPackets, {"net_tx_packets[veth]", LongExtractor}}, - {EStatField::NetTxDrops, {"net_tx_drops[veth]", LongExtractor}}, - {EStatField::NetTxLimit, {"net_limit[veth]", LongExtractor}}, - {EStatField::NetRxBytes, {"net_rx_bytes[veth]", LongExtractor}}, - {EStatField::NetRxPackets, {"net_rx_packets[veth]", LongExtractor}}, - {EStatField::NetRxDrops, {"net_rx_drops[veth]", LongExtractor}}, - {EStatField::NetRxLimit, {"net_rx_limit[veth]", LongExtractor}}, -}; - -std::optional<TString> GetParentName(const TString& name) -{ - if (name.empty()) { - return std::nullopt; - } - - auto slashPosition = name.rfind('/'); - if (slashPosition == TString::npos) { - return ""; - } - - return name.substr(0, slashPosition); -} - -std::optional<TString> GetRootName(const TString& name) -{ - if (name.empty()) { - return std::nullopt; - } - - if (name == "/") { - return name; - } - - auto slashPosition = name.find('/'); - if (slashPosition == TString::npos) { - return name; - } - - return name.substr(0, slashPosition); -} - -} // namespace NDetail - -//////////////////////////////////////////////////////////////////////////////// - -class TPortoInstanceLauncher - : public IInstanceLauncher -{ -public: - TPortoInstanceLauncher(const TString& name, IPortoExecutorPtr executor) - : Executor_(std::move(executor)) - , Logger(ContainersLogger.WithTag("Container: %v", name)) - { - Spec_.Name = name; - Spec_.CGroupControllers = { - "freezer", - "cpu", - "cpuacct", - "net_cls", - "blkio", - "devices", - "pids" - }; - } - - const TString& GetName() const override - { - return Spec_.Name; - } - - bool HasRoot() const override - { - return static_cast<bool>(Spec_.RootFS); - } - - void SetStdIn(const TString& inputPath) override - { - Spec_.StdinPath = inputPath; - } - - void SetStdOut(const TString& outPath) override - { - Spec_.StdoutPath = outPath; - } - - void SetStdErr(const TString& errorPath) override - { - Spec_.StderrPath = errorPath; - } - - void SetCwd(const TString& pwd) override - { - Spec_.CurrentWorkingDirectory = pwd; - } - - void SetCoreDumpHandler(const std::optional<TString>& handler) override - { - if (handler) { - Spec_.CoreCommand = *handler; - Spec_.EnableCoreDumps = true; - } else { - Spec_.EnableCoreDumps = false; - } - } - - void SetRoot(const TRootFS& rootFS) override - { - Spec_.RootFS = rootFS; - } - - void SetThreadLimit(i64 threadLimit) override - { - Spec_.ThreadLimit = threadLimit; - } - - void SetDevices(const std::vector<TDevice>& devices) override - { - Spec_.Devices = devices; - } - - void SetEnablePorto(EEnablePorto enablePorto) override - { - Spec_.EnablePorto = enablePorto; - } - - void SetIsolate(bool isolate) override - { - Spec_.Isolate = isolate; - } - - void EnableMemoryTracking() override - { - Spec_.CGroupControllers.push_back("memory"); - } - - void SetGroup(int groupId) override - { - Spec_.GroupId = groupId; - } - - void SetUser(const TString& user) override - { - Spec_.User = user; - } - - void SetIPAddresses(const std::vector<NNet::TIP6Address>& addresses, bool enableNat64) override - { - Spec_.IPAddresses = addresses; - Spec_.EnableNat64 = enableNat64; - Spec_.DisableNetwork = false; - } - - void DisableNetwork() override - { - Spec_.DisableNetwork = true; - Spec_.IPAddresses.clear(); - Spec_.EnableNat64 = false; - } - - void SetHostName(const TString& hostName) override - { - Spec_.HostName = hostName; - } - - TFuture<IInstancePtr> Launch( - const TString& path, - const std::vector<TString>& args, - const THashMap<TString, TString>& env) override - { - TStringBuilder commandBuilder; - auto append = [&] (const auto& value) { - commandBuilder.AppendString("'"); - commandBuilder.AppendString(NDetail::EscapeForWordexp(value.c_str())); - commandBuilder.AppendString("' "); - }; - - append(path); - for (const auto& arg : args) { - append(arg); - } - - Spec_.Command = commandBuilder.Flush(); - YT_LOG_DEBUG("Executing Porto container (Name: %v, Command: %v)", - Spec_.Name, - Spec_.Command); - - Spec_.Env = env; - - auto onContainerCreated = [this, this_ = MakeStrong(this)] (const TError& error) -> IInstancePtr { - if (!error.IsOK()) { - THROW_ERROR_EXCEPTION(EErrorCode::FailedToStartContainer, "Unable to start container") - << error; - } - - return GetPortoInstance(Executor_, Spec_.Name); - }; - - return Executor_->CreateContainer(Spec_, /* start */ true) - .Apply(BIND(onContainerCreated)); - } - -private: - IPortoExecutorPtr Executor_; - TRunnableContainerSpec Spec_; - const NLogging::TLogger Logger; -}; - -IInstanceLauncherPtr CreatePortoInstanceLauncher(const TString& name, IPortoExecutorPtr executor) -{ - return New<TPortoInstanceLauncher>(name, executor); -} - -//////////////////////////////////////////////////////////////////////////////// - -class TPortoInstance - : public IInstance -{ -public: - static IInstancePtr GetSelf(IPortoExecutorPtr executor) - { - return New<TPortoInstance>(GetSelfContainerName(executor), executor); - } - - static IInstancePtr GetInstance(IPortoExecutorPtr executor, const TString& name) - { - return New<TPortoInstance>(name, executor); - } - - void Kill(int signal) override - { - auto error = WaitFor(Executor_->KillContainer(Name_, signal)); - // Killing already finished process is not an error. - if (error.FindMatching(EPortoErrorCode::InvalidState)) { - return; - } - if (!error.IsOK()) { - THROW_ERROR_EXCEPTION("Failed to send signal to Porto instance") - << TErrorAttribute("signal", signal) - << TErrorAttribute("container", Name_) - << error; - } - } - - void Destroy() override - { - WaitFor(Executor_->DestroyContainer(Name_)) - .ThrowOnError(); - Destroyed_ = true; - } - - void Stop() override - { - WaitFor(Executor_->StopContainer(Name_)) - .ThrowOnError(); - } - - TErrorOr<ui64> CalculateCpuUserUsage( - TErrorOr<ui64>& cpuUsage, - TErrorOr<ui64>& cpuSystemUsage) const - { - if (cpuUsage.IsOK() && cpuSystemUsage.IsOK()) { - return cpuUsage.Value() > cpuSystemUsage.Value() ? cpuUsage.Value() - cpuSystemUsage.Value() : 0; - } else if (cpuUsage.IsOK()) { - return TError("Missing property %Qlv in Porto response", EStatField::CpuSystemUsage) - << TErrorAttribute("container", Name_); - } else { - return TError("Missing property %Qlv in Porto response", EStatField::CpuUsage) - << TErrorAttribute("container", Name_); - } - } - - TResourceUsage GetResourceUsage( - const std::vector<EStatField>& fields) const override - { - std::vector<TString> properties; - properties.push_back("absolute_name"); - - bool userTimeRequested = false; - bool contextSwitchesRequested = false; - for (auto field : fields) { - if (auto it = NDetail::PortoStatRules.find(field)) { - const auto& rule = it->second; - properties.push_back(rule.first); - } else if (field == EStatField::ContextSwitchesDelta || field == EStatField::ContextSwitches) { - contextSwitchesRequested = true; - } else if (field == EStatField::CpuUserUsage) { - userTimeRequested = true; - } else { - THROW_ERROR_EXCEPTION("Unknown resource field %Qlv requested", field) - << TErrorAttribute("container", Name_); - } - } - - auto propertyMap = WaitFor(Executor_->GetContainerProperties(Name_, properties)) - .ValueOrThrow(); - - TResourceUsage result; - - for (auto field : fields) { - auto ruleIt = NDetail::PortoStatRules.find(field); - if (ruleIt == NDetail::PortoStatRules.end()) { - continue; - } - - const auto& [property, callback] = ruleIt->second; - auto& record = result[field]; - if (auto responseIt = propertyMap.find(property); responseIt != propertyMap.end()) { - const auto& valueOrError = responseIt->second; - if (valueOrError.IsOK()) { - const auto& value = valueOrError.Value(); - - try { - record = callback(value); - } catch (const std::exception& ex) { - record = TError("Error parsing Porto property %Qlv", field) - << TErrorAttribute("container", Name_) - << TErrorAttribute("property_value", value) - << ex; - } - } else { - record = TError("Error getting Porto property %Qlv", field) - << TErrorAttribute("container", Name_) - << valueOrError; - } - } else { - record = TError("Missing property %Qlv in Porto response", field) - << TErrorAttribute("container", Name_); - } - } - - // We should maintain context switch information even if this field - // is not requested since metrics of individual containers can go up and down. - auto subcontainers = WaitFor(Executor_->ListSubcontainers(Name_, /*includeRoot*/ true)) - .ValueOrThrow(); - - auto metricMap = WaitFor(Executor_->GetContainerMetrics(subcontainers, "ctxsw")) - .ValueOrThrow(); - - // TODO(don-dron): remove diff calculation from GetResourceUsage, because GetResourceUsage must return only snapshot stat. - { - auto guard = Guard(ContextSwitchMapLock_); - - for (const auto& [container, newValue] : metricMap) { - auto& prevValue = ContextSwitchMap_[container]; - TotalContextSwitches_ += std::max<i64>(0LL, newValue - prevValue); - prevValue = newValue; - } - - if (contextSwitchesRequested) { - result[EStatField::ContextSwitchesDelta] = TotalContextSwitches_; - } - } - - if (contextSwitchesRequested) { - ui64 totalContextSwitches = 0; - - for (const auto& [container, newValue] : metricMap) { - totalContextSwitches += std::max<ui64>(0UL, newValue); - } - - result[EStatField::ContextSwitches] = totalContextSwitches; - } - - if (userTimeRequested) { - result[EStatField::CpuUserUsage] = CalculateCpuUserUsage( - result[EStatField::CpuUsage], - result[EStatField::CpuSystemUsage]); - } - - return result; - } - - TResourceLimits GetResourceLimits() const override - { - std::vector<TString> properties; - static TString memoryLimitProperty = "memory_limit_total"; - static TString cpuLimitProperty = "cpu_limit_bound"; - static TString cpuGuaranteeProperty = "cpu_guarantee_bound"; - properties.push_back(memoryLimitProperty); - properties.push_back(cpuLimitProperty); - properties.push_back(cpuGuaranteeProperty); - - auto responseOrError = WaitFor(Executor_->GetContainerProperties(Name_, properties)); - THROW_ERROR_EXCEPTION_IF_FAILED(responseOrError, "Failed to get Porto container resource limits"); - - const auto& response = responseOrError.Value(); - - const auto& memoryLimitRsp = response.at(memoryLimitProperty); - THROW_ERROR_EXCEPTION_IF_FAILED(memoryLimitRsp, "Failed to get memory limit from Porto"); - - i64 memoryLimit; - if (!TryFromString<i64>(memoryLimitRsp.Value(), memoryLimit)) { - THROW_ERROR_EXCEPTION("Failed to parse memory limit value from Porto") - << TErrorAttribute(memoryLimitProperty, memoryLimitRsp.Value()); - } - - const auto& cpuLimitRsp = response.at(cpuLimitProperty); - THROW_ERROR_EXCEPTION_IF_FAILED(cpuLimitRsp, "Failed to get CPU limit from Porto"); - - double cpuLimit; - YT_VERIFY(cpuLimitRsp.Value().EndsWith('c')); - auto cpuLimitValue = TStringBuf(cpuLimitRsp.Value().begin(), cpuLimitRsp.Value().size() - 1); - if (!TryFromString<double>(cpuLimitValue, cpuLimit)) { - THROW_ERROR_EXCEPTION("Failed to parse CPU limit value from Porto") - << TErrorAttribute(cpuLimitProperty, cpuLimitRsp.Value()); - } - - const auto& cpuGuaranteeRsp = response.at(cpuGuaranteeProperty); - THROW_ERROR_EXCEPTION_IF_FAILED(cpuGuaranteeRsp, "Failed to get CPU guarantee from Porto"); - - double cpuGuarantee; - if (!cpuGuaranteeRsp.Value()) { - // XXX(ignat): hack for missing response from Porto. - cpuGuarantee = 0.0; - } else { - YT_VERIFY(cpuGuaranteeRsp.Value().EndsWith('c')); - auto cpuGuaranteeValue = TStringBuf(cpuGuaranteeRsp.Value().begin(), cpuGuaranteeRsp.Value().size() - 1); - if (!TryFromString<double>(cpuGuaranteeValue, cpuGuarantee)) { - THROW_ERROR_EXCEPTION("Failed to parse CPU guarantee value from Porto") - << TErrorAttribute(cpuGuaranteeProperty, cpuGuaranteeRsp.Value()); - } - } - - return TResourceLimits{ - .CpuLimit = cpuLimit, - .CpuGuarantee = cpuGuarantee, - .Memory = memoryLimit, - }; - } - - void SetCpuGuarantee(double cores) override - { - SetProperty("cpu_guarantee", ToString(cores) + "c"); - } - - void SetCpuLimit(double cores) override - { - SetProperty("cpu_limit", ToString(cores) + "c"); - } - - void SetCpuWeight(double weight) override - { - SetProperty("cpu_weight", weight); - } - - void SetMemoryGuarantee(i64 memoryGuarantee) override - { - SetProperty("memory_guarantee", memoryGuarantee); - } - - void SetIOWeight(double weight) override - { - SetProperty("io_weight", weight); - } - - void SetIOThrottle(i64 operations) override - { - SetProperty("io_ops_limit", operations); - } - - TString GetStderr() const override - { - return *WaitFor(Executor_->GetContainerProperty(Name_, "stderr")) - .ValueOrThrow(); - } - - TString GetName() const override - { - return Name_; - } - - std::optional<TString> GetParentName() const override - { - return NDetail::GetParentName(Name_); - } - - std::optional<TString> GetRootName() const override - { - return NDetail::GetRootName(Name_); - } - - pid_t GetPid() const override - { - auto pid = *WaitFor(Executor_->GetContainerProperty(Name_, "root_pid")) - .ValueOrThrow(); - return std::stoi(pid); - } - - i64 GetMajorPageFaultCount() const override - { - auto faults = WaitFor(Executor_->GetContainerProperty(Name_, "major_faults")) - .ValueOrThrow(); - return faults - ? std::stoll(*faults) - : 0; - } - - double GetCpuGuarantee() const override - { - auto result = WaitFor(Executor_->GetContainerProperty(Name_, "cpu_guarantee")) - .ValueOrThrow(); - return result - ? std::stod(*result) - : 0; - } - - std::vector<pid_t> GetPids() const override - { - auto getPidCgroup = [&] (const TString& cgroups) { - for (TStringBuf cgroup : StringSplitter(cgroups).SplitByString("; ")) { - if (cgroup.StartsWith("pids:")) { - auto startPosition = cgroup.find('/'); - YT_VERIFY(startPosition != TString::npos); - return cgroup.substr(startPosition); - } - } - THROW_ERROR_EXCEPTION("Pids cgroup not found for container %Qv", GetName()) - << TErrorAttribute("cgroups", cgroups); - }; - - auto cgroups = *WaitFor(Executor_->GetContainerProperty(Name_, "cgroups")) - .ValueOrThrow(); - // Porto returns full cgroup name, with mount prefix, such as "/sys/fs/cgroup/pids". - auto instanceCgroup = getPidCgroup(cgroups); - - std::vector<pid_t> pids; - for (auto pid : ListPids()) { - std::map<TString, TString> cgroups; - try { - cgroups = GetProcessCGroups(pid); - } catch (const std::exception& ex) { - YT_LOG_DEBUG(ex, "Failed to get CGroups for process (Pid: %v)", pid); - continue; - } - - // Pid cgroups are returned in short form. - auto processPidCgroup = cgroups["pids"]; - if (!processPidCgroup.empty() && instanceCgroup.EndsWith(processPidCgroup)) { - pids.push_back(pid); - } - } - - return pids; - } - - TFuture<void> Wait() override - { - return Executor_->PollContainer(Name_) - .Apply(BIND([] (int status) { - StatusToError(status) - .ThrowOnError(); - })); - } - -private: - const TString Name_; - const IPortoExecutorPtr Executor_; - const NLogging::TLogger Logger; - - bool Destroyed_ = false; - - YT_DECLARE_SPIN_LOCK(NThreading::TSpinLock, ContextSwitchMapLock_); - mutable i64 TotalContextSwitches_ = 0; - mutable THashMap<TString, i64> ContextSwitchMap_; - - TPortoInstance(TString name, IPortoExecutorPtr executor) - : Name_(std::move(name)) - , Executor_(std::move(executor)) - , Logger(ContainersLogger.WithTag("Container: %v", Name_)) - { } - - void SetProperty(const TString& key, const TString& value) - { - WaitFor(Executor_->SetContainerProperty(Name_, key, value)) - .ThrowOnError(); - } - - void SetProperty(const TString& key, i64 value) - { - SetProperty(key, ToString(value)); - } - - void SetProperty(const TString& key, double value) - { - SetProperty(key, ToString(value)); - } - - DECLARE_NEW_FRIEND() -}; - -//////////////////////////////////////////////////////////////////////////////// - -TString GetSelfContainerName(const IPortoExecutorPtr& executor) -{ - try { - auto properties = WaitFor(executor->GetContainerProperties( - "self", - std::vector<TString>{"absolute_name", "absolute_namespace"})) - .ValueOrThrow(); - - auto absoluteName = properties.at("absolute_name") - .ValueOrThrow(); - auto absoluteNamespace = properties.at("absolute_namespace") - .ValueOrThrow(); - - if (absoluteName == "/") { - return absoluteName; - } - - if (absoluteName.length() < absoluteNamespace.length()) { - YT_VERIFY(absoluteName + "/" == absoluteNamespace); - return ""; - } else { - YT_VERIFY(absoluteName.StartsWith(absoluteNamespace)); - return absoluteName.substr(absoluteNamespace.length()); - } - } catch (const std::exception& ex) { - THROW_ERROR_EXCEPTION("Failed to get name for container \"self\"") - << ex; - } -} - -IInstancePtr GetSelfPortoInstance(IPortoExecutorPtr executor) -{ - return TPortoInstance::GetSelf(executor); -} - -IInstancePtr GetPortoInstance(IPortoExecutorPtr executor, const TString& name) -{ - return TPortoInstance::GetInstance(executor, name); -} - -IInstancePtr GetRootPortoInstance(IPortoExecutorPtr executor) -{ - auto self = GetSelfPortoInstance(executor); - return TPortoInstance::GetInstance(executor, *self->GetRootName()); -} - -double GetSelfPortoInstanceVCpuFactor() -{ - auto config = New<TPortoExecutorDynamicConfig>(); - auto executorPtr = CreatePortoExecutor(config, ""); - auto currentContainer = GetSelfPortoInstance(executorPtr); - double cpuLimit = currentContainer->GetResourceLimits().CpuLimit; - if (cpuLimit <= 0) { - THROW_ERROR_EXCEPTION("Cpu limit must be greater than 0"); - } - - // DEPLOY_VCPU_LIMIT stores value in millicores - if (TString vcpuLimitStr = GetEnv("DEPLOY_VCPU_LIMIT"); !vcpuLimitStr.Empty()) { - double vcpuLimit = FromString<double>(vcpuLimitStr) / 1000.0; - return vcpuLimit / cpuLimit; - } - THROW_ERROR_EXCEPTION("Failed to get vcpu limit from env variable"); -} - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NContainers - -#endif diff --git a/yt/yt/library/containers/instance.h b/yt/yt/library/containers/instance.h deleted file mode 100644 index ff6e0b3ce1..0000000000 --- a/yt/yt/library/containers/instance.h +++ /dev/null @@ -1,168 +0,0 @@ -#pragma once - -#include "public.h" - -#include <yt/yt/core/actions/future.h> - -#include <yt/yt/core/net/address.h> - -namespace NYT::NContainers { - -//////////////////////////////////////////////////////////////////////////////// - -using TResourceUsage = THashMap<EStatField, TErrorOr<ui64>>; - -const std::vector<EStatField> InstanceStatFields{ - EStatField::CpuUsage, - EStatField::CpuUserUsage, - EStatField::CpuSystemUsage, - EStatField::CpuWait, - EStatField::CpuThrottled, - EStatField::ContextSwitches, - EStatField::ContextSwitchesDelta, - EStatField::ThreadCount, - EStatField::CpuLimit, - EStatField::CpuGuarantee, - - EStatField::Rss, - EStatField::MappedFile, - EStatField::MajorPageFaults, - EStatField::MinorPageFaults, - EStatField::FileCacheUsage, - EStatField::AnonMemoryUsage, - EStatField::AnonMemoryLimit, - EStatField::MemoryUsage, - EStatField::MemoryGuarantee, - EStatField::MemoryLimit, - EStatField::MaxMemoryUsage, - EStatField::OomKills, - EStatField::OomKillsTotal, - - EStatField::IOReadByte, - EStatField::IOWriteByte, - EStatField::IOBytesLimit, - EStatField::IOReadOps, - EStatField::IOWriteOps, - EStatField::IOOps, - EStatField::IOOpsLimit, - EStatField::IOTotalTime, - EStatField::IOWaitTime, - - EStatField::NetTxBytes, - EStatField::NetTxPackets, - EStatField::NetTxDrops, - EStatField::NetTxLimit, - EStatField::NetRxBytes, - EStatField::NetRxPackets, - EStatField::NetRxDrops, - EStatField::NetRxLimit, -}; - -struct TResourceLimits -{ - double CpuLimit; - double CpuGuarantee; - i64 Memory; -}; - -//////////////////////////////////////////////////////////////////////////////// - -struct IInstanceLauncher - : public TRefCounted -{ - virtual bool HasRoot() const = 0; - virtual const TString& GetName() const = 0; - - virtual void SetStdIn(const TString& inputPath) = 0; - virtual void SetStdOut(const TString& outPath) = 0; - virtual void SetStdErr(const TString& errorPath) = 0; - virtual void SetCwd(const TString& pwd) = 0; - - // Null core dump handler implies disabled core dumps. - virtual void SetCoreDumpHandler(const std::optional<TString>& handler) = 0; - virtual void SetRoot(const TRootFS& rootFS) = 0; - - virtual void SetThreadLimit(i64 threadLimit) = 0; - virtual void SetDevices(const std::vector<TDevice>& devices) = 0; - - virtual void SetEnablePorto(EEnablePorto enablePorto) = 0; - virtual void SetIsolate(bool isolate) = 0; - virtual void EnableMemoryTracking() = 0; - virtual void SetGroup(int groupId) = 0; - virtual void SetUser(const TString& user) = 0; - virtual void SetIPAddresses( - const std::vector<NNet::TIP6Address>& addresses, - bool enableNat64 = false) = 0; - virtual void DisableNetwork() = 0; - virtual void SetHostName(const TString& hostName) = 0; - - virtual TFuture<IInstancePtr> Launch( - const TString& path, - const std::vector<TString>& args, - const THashMap<TString, TString>& env) = 0; -}; - -DEFINE_REFCOUNTED_TYPE(IInstanceLauncher) - -#ifdef _linux_ -IInstanceLauncherPtr CreatePortoInstanceLauncher(const TString& name, IPortoExecutorPtr executor); -#endif - -//////////////////////////////////////////////////////////////////////////////// - -struct IInstance - : public TRefCounted -{ - virtual void Kill(int signal) = 0; - virtual void Stop() = 0; - virtual void Destroy() = 0; - - virtual TResourceUsage GetResourceUsage( - const std::vector<EStatField>& fields = InstanceStatFields) const = 0; - virtual TResourceLimits GetResourceLimits() const = 0; - virtual void SetCpuGuarantee(double cores) = 0; - virtual void SetCpuLimit(double cores) = 0; - virtual void SetCpuWeight(double weight) = 0; - virtual void SetIOWeight(double weight) = 0; - virtual void SetIOThrottle(i64 operations) = 0; - virtual void SetMemoryGuarantee(i64 memoryGuarantee) = 0; - - virtual TString GetStderr() const = 0; - - virtual TString GetName() const = 0; - virtual std::optional<TString> GetParentName() const = 0; - virtual std::optional<TString> GetRootName() const = 0; - - //! Returns externally visible pid of the root process inside container. - //! Throws if container is not running. - virtual pid_t GetPid() const = 0; - //! Returns the list of externally visible pids of processes running inside container. - virtual std::vector<pid_t> GetPids() const = 0; - - virtual i64 GetMajorPageFaultCount() const = 0; - virtual double GetCpuGuarantee() const = 0; - - //! Future is set when container reaches terminal state (stopped or dead). - //! Resulting error is OK iff container exited with code 0. - virtual TFuture<void> Wait() = 0; -}; - -DEFINE_REFCOUNTED_TYPE(IInstance) - -//////////////////////////////////////////////////////////////////////////////// - -#ifdef _linux_ -TString GetSelfContainerName(const IPortoExecutorPtr& executor); - -IInstancePtr GetSelfPortoInstance(IPortoExecutorPtr executor); -IInstancePtr GetRootPortoInstance(IPortoExecutorPtr executor); -IInstancePtr GetPortoInstance(IPortoExecutorPtr executor, const TString& name); - -//! Works only in Yandex.Deploy pod environment where env DEPLOY_VCPU_LIMIT is set. -//! Throws if this env is absent. -double GetSelfPortoInstanceVCpuFactor(); -#endif - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NContainers diff --git a/yt/yt/library/containers/instance_limits_tracker.cpp b/yt/yt/library/containers/instance_limits_tracker.cpp deleted file mode 100644 index 55ef7d2d67..0000000000 --- a/yt/yt/library/containers/instance_limits_tracker.cpp +++ /dev/null @@ -1,179 +0,0 @@ -#include "public.h" -#include "instance_limits_tracker.h" -#include "instance.h" -#include "porto_resource_tracker.h" -#include "private.h" - -#include <yt/yt/core/concurrency/periodic_executor.h> - -#include <yt/yt/core/ytree/fluent.h> -#include <yt/yt/core/ytree/ypath_service.h> - -namespace NYT::NContainers { - -using namespace NYTree; - -//////////////////////////////////////////////////////////////////////////////// - -static const auto& Logger = ContainersLogger; - -//////////////////////////////////////////////////////////////////////////////// - -TInstanceLimitsTracker::TInstanceLimitsTracker( - IInstancePtr instance, - IInstancePtr root, - IInvokerPtr invoker, - TDuration updatePeriod) - : Invoker_(std::move(invoker)) - , Executor_(New<NConcurrency::TPeriodicExecutor>( - Invoker_, - BIND(&TInstanceLimitsTracker::DoUpdateLimits, MakeWeak(this)), - updatePeriod)) -{ -#ifdef _linux_ - SelfTracker_ = New<TPortoResourceTracker>(std::move(instance), updatePeriod / 2); - RootTracker_ = New<TPortoResourceTracker>(std::move(root), updatePeriod / 2); -#else - Y_UNUSED(instance); - Y_UNUSED(root); -#endif -} - -void TInstanceLimitsTracker::Start() -{ - if (!Running_) { - Executor_->Start(); - Running_ = true; - YT_LOG_INFO("Instance limits tracker started"); - } -} - -void TInstanceLimitsTracker::Stop() -{ - if (Running_) { - YT_UNUSED_FUTURE(Executor_->Stop()); - Running_ = false; - YT_LOG_INFO("Instance limits tracker stopped"); - } -} - -void TInstanceLimitsTracker::DoUpdateLimits() -{ - VERIFY_INVOKER_AFFINITY(Invoker_); - -#ifdef _linux_ - YT_LOG_DEBUG("Checking for instance limits update"); - - auto setIfOk = [] (auto* destination, const auto& valueOrError, const TString& fieldName, bool alert = true) { - if (valueOrError.IsOK()) { - *destination = valueOrError.Value(); - } else { - YT_LOG_ALERT_IF(alert, valueOrError, "Failed to get container property (Field: %v)", - fieldName); - - YT_LOG_DEBUG(valueOrError, "Failed to get container property (Field: %v)", - fieldName); - } - }; - - try { - auto memoryStatistics = SelfTracker_->GetMemoryStatistics(); - auto netStatistics = RootTracker_->GetNetworkStatistics(); - auto cpuStatistics = SelfTracker_->GetCpuStatistics(); - - setIfOk(&MemoryUsage_, memoryStatistics.Rss, "MemoryRss"); - - TDuration cpuGuarantee; - TDuration cpuLimit; - - if (cpuStatistics.GuaranteeTime.IsOK()) { - setIfOk(&cpuGuarantee, cpuStatistics.GuaranteeTime, "CpuGuarantee"); - } else { - // XXX(don-dron, ignat): do nothing, see NContainers::TPortoInstance::GetResourceLimits, hack for missing response from Porto. - } - - setIfOk(&cpuLimit, cpuStatistics.LimitTime, "CpuLimit"); - - if (CpuGuarantee_ != cpuGuarantee) { - YT_LOG_INFO("Instance CPU guarantee updated (OldCpuGuarantee: %v, NewCpuGuarantee: %v)", - CpuGuarantee_, - cpuGuarantee); - CpuGuarantee_ = cpuGuarantee; - // NB: We do not fire LimitsUpdated since this value used only for diagnostics. - } - - TInstanceLimits limits; - limits.Cpu = cpuLimit.SecondsFloat(); - - if (memoryStatistics.AnonLimit.IsOK() && memoryStatistics.MemoryLimit.IsOK()) { - i64 anonLimit = memoryStatistics.AnonLimit.Value(); - i64 memoryLimit = memoryStatistics.MemoryLimit.Value(); - - if (anonLimit > 0 && memoryLimit > 0) { - limits.Memory = std::min(anonLimit, memoryLimit); - } else if (anonLimit > 0) { - limits.Memory = anonLimit; - } else { - limits.Memory = memoryLimit; - } - } else { - setIfOk(&limits.Memory, memoryStatistics.MemoryLimit, "MemoryLimit"); - } - - static constexpr bool DontFireAlertOnError = {}; - setIfOk(&limits.NetTx, netStatistics.TxLimit, "NetTxLimit", DontFireAlertOnError); - setIfOk(&limits.NetRx, netStatistics.RxLimit, "NetRxLimit", DontFireAlertOnError); - - if (InstanceLimits_ != limits) { - YT_LOG_INFO("Instance limits updated (OldLimits: %v, NewLimits: %v)", - InstanceLimits_, - limits); - InstanceLimits_ = limits; - LimitsUpdated_.Fire(limits); - } - } catch (const std::exception& ex) { - YT_LOG_WARNING(ex, "Failed to get instance limits"); - } -#endif -} - -IYPathServicePtr TInstanceLimitsTracker::GetOrchidService() -{ - return IYPathService::FromProducer(BIND(&TInstanceLimitsTracker::DoBuildOrchid, MakeStrong(this))) - ->Via(Invoker_); -} - -void TInstanceLimitsTracker::DoBuildOrchid(NYson::IYsonConsumer* consumer) const -{ - NYTree::BuildYsonFluently(consumer) - .BeginMap() - .DoIf(static_cast<bool>(InstanceLimits_), [&] (auto fluent) { - fluent.Item("cpu_limit").Value(InstanceLimits_->Cpu); - }) - .DoIf(static_cast<bool>(CpuGuarantee_), [&] (auto fluent) { - fluent.Item("cpu_guarantee").Value(*CpuGuarantee_); - }) - .DoIf(static_cast<bool>(InstanceLimits_), [&] (auto fluent) { - fluent.Item("memory_limit").Value(InstanceLimits_->Memory); - }) - .DoIf(static_cast<bool>(MemoryUsage_), [&] (auto fluent) { - fluent.Item("memory_usage").Value(*MemoryUsage_); - }) - .EndMap(); -} - -//////////////////////////////////////////////////////////////////////////////// - -void FormatValue(TStringBuilderBase* builder, const TInstanceLimits& limits, TStringBuf /*format*/) -{ - builder->AppendFormat( - "{Cpu: %v, Memory: %v, NetTx: %v, NetRx: %v}", - limits.Cpu, - limits.Memory, - limits.NetTx, - limits.NetRx); -} - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NContainers diff --git a/yt/yt/library/containers/instance_limits_tracker.h b/yt/yt/library/containers/instance_limits_tracker.h deleted file mode 100644 index e652fff446..0000000000 --- a/yt/yt/library/containers/instance_limits_tracker.h +++ /dev/null @@ -1,59 +0,0 @@ -#pragma once - -#include "public.h" - -#include <yt/yt/core/actions/signal.h> - -#include <yt/yt/core/concurrency/public.h> - -#include <yt/yt/core/yson/public.h> - -#include <yt/yt/core/ytree/public.h> - -namespace NYT::NContainers { - -//////////////////////////////////////////////////////////////////////////////// - -class TInstanceLimitsTracker - : public TRefCounted -{ -public: - //! Raises when container limits change. - DEFINE_SIGNAL(void(const TInstanceLimits&), LimitsUpdated); - -public: - TInstanceLimitsTracker( - IInstancePtr instance, - IInstancePtr root, - IInvokerPtr invoker, - TDuration updatePeriod); - - void Start(); - void Stop(); - - NYTree::IYPathServicePtr GetOrchidService(); - -private: - void DoUpdateLimits(); - void DoBuildOrchid(NYson::IYsonConsumer* consumer) const; - - TPortoResourceTrackerPtr SelfTracker_; - TPortoResourceTrackerPtr RootTracker_; - const IInvokerPtr Invoker_; - const NConcurrency::TPeriodicExecutorPtr Executor_; - - std::optional<TDuration> CpuGuarantee_; - std::optional<TInstanceLimits> InstanceLimits_; - std::optional<i64> MemoryUsage_; - bool Running_ = false; -}; - -DEFINE_REFCOUNTED_TYPE(TInstanceLimitsTracker) - -//////////////////////////////////////////////////////////////////////////////// - -void FormatValue(TStringBuilderBase* builder, const TInstanceLimits& limits, TStringBuf format); - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NContainers diff --git a/yt/yt/library/containers/porto_executor.cpp b/yt/yt/library/containers/porto_executor.cpp deleted file mode 100644 index a6a44fd20f..0000000000 --- a/yt/yt/library/containers/porto_executor.cpp +++ /dev/null @@ -1,1079 +0,0 @@ -#include "porto_executor.h" -#include "config.h" - -#include "private.h" - -#include <yt/yt/core/concurrency/action_queue.h> -#include <yt/yt/core/concurrency/periodic_executor.h> -#include <yt/yt/core/concurrency/scheduler.h> - -#include <yt/yt/core/logging/log.h> - -#include <yt/yt/core/misc/fs.h> - -#include <yt/yt/core/profiling/timing.h> - -#include <yt/yt/core/ytree/convert.h> - -#include <library/cpp/porto/proto/rpc.pb.h> - -#include <library/cpp/yt/memory/atomic_intrusive_ptr.h> - -#include <string> - -namespace NYT::NContainers { - -using namespace NConcurrency; -using Porto::EError; - -//////////////////////////////////////////////////////////////////////////////// - -#ifdef _linux_ - -static const NLogging::TLogger& Logger = ContainersLogger; -static constexpr auto RetryInterval = TDuration::MilliSeconds(100); - -//////////////////////////////////////////////////////////////////////////////// - -TString PortoErrorCodeFormatter(int code) -{ - return TEnumTraits<EPortoErrorCode>::ToString(static_cast<EPortoErrorCode>(code)); -} - -YT_DEFINE_ERROR_CODE_RANGE(12000, 13999, "NYT::NContainers::EPortoErrorCode", PortoErrorCodeFormatter); - -//////////////////////////////////////////////////////////////////////////////// - -EPortoErrorCode ConvertPortoErrorCode(EError portoError) -{ - return static_cast<EPortoErrorCode>(PortoErrorCodeBase + portoError); -} - -bool IsRetriableErrorCode(EPortoErrorCode error, bool idempotent) -{ - return - error == EPortoErrorCode::Unknown || - // TODO(babenko): it's not obvious that we can always retry SocketError - // but this is how it has used to work for a while. - error == EPortoErrorCode::SocketError || - error == EPortoErrorCode::SocketTimeout && idempotent; -} - -THashMap<TString, TErrorOr<TString>> ParsePortoGetResponse( - const Porto::TGetResponse_TContainerGetListResponse& response) -{ - THashMap<TString, TErrorOr<TString>> result; - for (const auto& property : response.keyval()) { - if (property.error() == EError::Success) { - result[property.variable()] = property.value(); - } else { - result[property.variable()] = TError(ConvertPortoErrorCode(property.error()), property.errormsg()) - << TErrorAttribute("porto_error", ConvertPortoErrorCode(property.error())); - } - } - return result; -} - -THashMap<TString, TErrorOr<TString>> ParseSinglePortoGetResponse( - const TString& name, - const Porto::TGetResponse& getResponse) -{ - for (const auto& container : getResponse.list()) { - if (container.name() == name) { - return ParsePortoGetResponse(container); - } - } - THROW_ERROR_EXCEPTION("Unable to get properties from Porto") - << TErrorAttribute("container", name); -} - -THashMap<TString, THashMap<TString, TErrorOr<TString>>> ParseMultiplePortoGetResponse( - const Porto::TGetResponse& getResponse) -{ - THashMap<TString, THashMap<TString, TErrorOr<TString>>> result; - for (const auto& container : getResponse.list()) { - result[container.name()] = ParsePortoGetResponse(container); - } - return result; -} - -TString FormatEnablePorto(EEnablePorto value) -{ - switch (value) { - case EEnablePorto::None: return "none"; - case EEnablePorto::Isolate: return "isolate"; - case EEnablePorto::Full: return "full"; - default: YT_ABORT(); - } -} - -//////////////////////////////////////////////////////////////////////////////// - -class TPortoExecutor - : public IPortoExecutor -{ -public: - TPortoExecutor( - TPortoExecutorDynamicConfigPtr config, - const TString& threadNameSuffix, - const NProfiling::TProfiler& profiler) - : Config_(std::move(config)) - , Queue_(New<TActionQueue>(Format("Porto:%v", threadNameSuffix))) - , Profiler_(profiler) - , PollExecutor_(New<TPeriodicExecutor>( - Queue_->GetInvoker(), - BIND(&TPortoExecutor::DoPoll, MakeWeak(this)), - Config_->PollPeriod)) - { - DynamicConfig_.Store(New<TPortoExecutorDynamicConfig>()); - - Api_->SetTimeout(Config_->ApiTimeout.Seconds()); - Api_->SetDiskTimeout(Config_->ApiDiskTimeout.Seconds()); - - PollExecutor_->Start(); - } - - void SubscribeFailed(const TCallback<void (const TError&)>& callback) override - { - Failed_.Subscribe(callback); - } - - void UnsubscribeFailed(const TCallback<void (const TError&)>& callback) override - { - Failed_.Unsubscribe(callback); - } - - void OnDynamicConfigChanged(const TPortoExecutorDynamicConfigPtr& newConfig) override - { - DynamicConfig_.Store(newConfig); - } - -private: - template <class T, class... TArgs1, class... TArgs2> - auto ExecutePortoApiAction( - T(TPortoExecutor::*Method)(TArgs1...), - const TString& command, - TArgs2&&... args) - { - YT_LOG_DEBUG("Enqueue Porto API action (Command: %v)", command); - return BIND(Method, MakeStrong(this), std::forward<TArgs2>(args)...) - .AsyncVia(Queue_->GetInvoker()) - .Run(); - }; - -public: - TFuture<void> CreateContainer(const TString& container) override - { - return ExecutePortoApiAction( - &TPortoExecutor::DoCreateContainer, - "CreateContainer", - container); - } - - TFuture<void> CreateContainer(const TRunnableContainerSpec& containerSpec, bool start) override - { - return ExecutePortoApiAction( - &TPortoExecutor::DoCreateContainerFromSpec, - "CreateContainerFromSpec", - containerSpec, - start); - } - - TFuture<std::optional<TString>> GetContainerProperty( - const TString& container, - const TString& property) override - { - return ExecutePortoApiAction( - &TPortoExecutor::DoGetContainerProperty, - "GetContainerProperty", - container, - property); - } - - TFuture<THashMap<TString, TErrorOr<TString>>> GetContainerProperties( - const TString& container, - const std::vector<TString>& properties) override - { - return ExecutePortoApiAction( - &TPortoExecutor::DoGetContainerProperties, - "GetContainerProperty", - container, - properties); - } - - TFuture<THashMap<TString, THashMap<TString, TErrorOr<TString>>>> GetContainerProperties( - const std::vector<TString>& containers, - const std::vector<TString>& properties) override - { - return ExecutePortoApiAction( - &TPortoExecutor::DoGetContainerMultipleProperties, - "GetContainerProperty", - containers, - properties); - } - - TFuture<THashMap<TString, i64>> GetContainerMetrics( - const std::vector<TString>& containers, - const TString& metric) override - { - return ExecutePortoApiAction( - &TPortoExecutor::DoGetContainerMetrics, - "GetContainerMetrics", - containers, - metric); - } - - TFuture<void> SetContainerProperty( - const TString& container, - const TString& property, - const TString& value) override - { - return ExecutePortoApiAction( - &TPortoExecutor::DoSetContainerProperty, - "SetContainerProperty", - container, - property, - value); - } - - TFuture<void> DestroyContainer(const TString& container) override - { - return ExecutePortoApiAction( - &TPortoExecutor::DoDestroyContainer, - "DestroyContainer", - container); - } - - TFuture<void> StopContainer(const TString& container) override - { - return ExecutePortoApiAction( - &TPortoExecutor::DoStopContainer, - "StopContainer", - container); - } - - TFuture<void> StartContainer(const TString& container) override - { - return ExecutePortoApiAction( - &TPortoExecutor::DoStartContainer, - "StartContainer", - container); - } - - TFuture<TString> ConvertPath(const TString& path, const TString& container) override - { - return ExecutePortoApiAction( - &TPortoExecutor::DoConvertPath, - "ConvertPath", - path, - container); - } - - TFuture<void> KillContainer(const TString& container, int signal) override - { - return ExecutePortoApiAction( - &TPortoExecutor::DoKillContainer, - "KillContainer", - container, - signal); - } - - TFuture<std::vector<TString>> ListSubcontainers( - const TString& rootContainer, - bool includeRoot) override - { - return ExecutePortoApiAction( - &TPortoExecutor::DoListSubcontainers, - "ListSubcontainers", - rootContainer, - includeRoot); - } - - TFuture<int> PollContainer(const TString& container) override - { - return ExecutePortoApiAction( - &TPortoExecutor::DoPollContainer, - "PollContainer", - container); - } - - TFuture<int> WaitContainer(const TString& container) override - { - return ExecutePortoApiAction( - &TPortoExecutor::DoWaitContainer, - "WaitContainer", - container); - } - - // This method allocates Porto "resources", so it should be uncancellable. - TFuture<TString> CreateVolume( - const TString& path, - const THashMap<TString, TString>& properties) override - { - return ExecutePortoApiAction( - &TPortoExecutor::DoCreateVolume, - "CreateVolume", - path, - properties) - .ToUncancelable(); - } - - // This method allocates Porto "resources", so it should be uncancellable. - TFuture<void> LinkVolume( - const TString& path, - const TString& name) override - { - return ExecutePortoApiAction( - &TPortoExecutor::DoLinkVolume, - "LinkVolume", - path, - name) - .ToUncancelable(); - } - - // This method deallocates Porto "resources", so it should be uncancellable. - TFuture<void> UnlinkVolume( - const TString& path, - const TString& name) override - { - return ExecutePortoApiAction( - &TPortoExecutor::DoUnlinkVolume, - "UnlinkVolume", - path, - name) - .ToUncancelable(); - } - - TFuture<std::vector<TString>> ListVolumePaths() override - { - return ExecutePortoApiAction( - &TPortoExecutor::DoListVolumePaths, - "ListVolumePaths"); - } - - // This method allocates Porto "resources", so it should be uncancellable. - TFuture<void> ImportLayer(const TString& archivePath, const TString& layerId, const TString& place) override - { - return ExecutePortoApiAction( - &TPortoExecutor::DoImportLayer, - "ImportLayer", - archivePath, - layerId, - place) - .ToUncancelable(); - } - - // This method deallocates Porto "resources", so it should be uncancellable. - TFuture<void> RemoveLayer(const TString& layerId, const TString& place, bool async) override - { - return ExecutePortoApiAction( - &TPortoExecutor::DoRemoveLayer, - "RemoveLayer", - layerId, - place, - async) - .ToUncancelable(); - } - - TFuture<std::vector<TString>> ListLayers(const TString& place) override - { - return ExecutePortoApiAction( - &TPortoExecutor::DoListLayers, - "ListLayers", - place); - } - - IInvokerPtr GetInvoker() const override - { - return Queue_->GetInvoker(); - } - -private: - const TPortoExecutorDynamicConfigPtr Config_; - const TActionQueuePtr Queue_; - const NProfiling::TProfiler Profiler_; - const std::unique_ptr<Porto::TPortoApi> Api_ = std::make_unique<Porto::TPortoApi>(); - const TPeriodicExecutorPtr PollExecutor_; - TAtomicIntrusivePtr<TPortoExecutorDynamicConfig> DynamicConfig_; - - std::vector<TString> Containers_; - THashMap<TString, TPromise<int>> ContainerMap_; - TSingleShotCallbackList<void(const TError&)> Failed_; - - struct TCommandEntry - { - explicit TCommandEntry(const NProfiling::TProfiler& registry) - : TimeGauge(registry.Timer("/command_time")) - , RetryCounter(registry.Counter("/command_retries")) - , SuccessCounter(registry.Counter("/command_successes")) - , FailureCounter(registry.Counter("/command_failures")) - { } - - NProfiling::TEventTimer TimeGauge; - NProfiling::TCounter RetryCounter; - NProfiling::TCounter SuccessCounter; - NProfiling::TCounter FailureCounter; - }; - - YT_DECLARE_SPIN_LOCK(NThreading::TSpinLock, CommandLock_); - THashMap<TString, TCommandEntry> CommandToEntry_; - - static const std::vector<TString> ContainerRequestVars_; - - bool IsTestPortoFailureEnabled() const - { - auto config = DynamicConfig_.Acquire(); - return config->EnableTestPortoFailures; - } - - bool IsTestPortoTimeout() const - { - auto config = DynamicConfig_.Acquire(); - return config->EnableTestPortoNotResponding; - } - - EPortoErrorCode GetFailedStubError() const - { - auto config = DynamicConfig_.Acquire(); - return config->StubErrorCode; - } - - static TError CreatePortoError(EPortoErrorCode errorCode, const TString& message) - { - return TError(errorCode, "Porto API error") - << TErrorAttribute("original_porto_error_code", static_cast<int>(errorCode) - PortoErrorCodeBase) - << TErrorAttribute("porto_error_message", message); - } - - THashMap<TString, TErrorOr<TString>> DoGetContainerProperties( - const TString& container, - const std::vector<TString>& properties) - { - auto response = DoRequestContainerProperties({container}, properties); - return ParseSinglePortoGetResponse(container, response); - } - - THashMap<TString, THashMap<TString, TErrorOr<TString>>> DoGetContainerMultipleProperties( - const std::vector<TString>& containers, - const std::vector<TString>& properties) - { - auto response = DoRequestContainerProperties(containers, properties); - return ParseMultiplePortoGetResponse(response); - } - - std::optional<TString> DoGetContainerProperty( - const TString& container, - const TString& property) - { - auto response = DoRequestContainerProperties({container}, {property}); - auto parsedResponse = ParseSinglePortoGetResponse(container, response); - auto it = parsedResponse.find(property); - if (it == parsedResponse.end()) { - return std::nullopt; - } else { - return it->second.ValueOrThrow(); - } - } - - void DoCreateContainer(const TString& container) - { - ExecuteApiCall( - [&] { return Api_->Create(container); }, - "Create", - /*idempotent*/ false); - } - - void DoCreateContainerFromSpec(const TRunnableContainerSpec& spec, bool start) - { - Porto::TContainerSpec portoSpec; - - // Required properties. - portoSpec.set_name(spec.Name); - portoSpec.set_command(spec.Command); - - portoSpec.set_enable_porto(FormatEnablePorto(spec.EnablePorto)); - portoSpec.set_isolate(spec.Isolate); - - if (spec.StdinPath) { - portoSpec.set_stdin_path(*spec.StdinPath); - } - if (spec.StdoutPath) { - portoSpec.set_stdout_path(*spec.StdoutPath); - } - if (spec.StderrPath) { - portoSpec.set_stderr_path(*spec.StderrPath); - } - - if (spec.CurrentWorkingDirectory) { - portoSpec.set_cwd(*spec.CurrentWorkingDirectory); - } - - if (spec.CoreCommand) { - portoSpec.set_core_command(*spec.CoreCommand); - } - if (spec.User) { - portoSpec.set_user(*spec.User); - } - - // Useful for jobs, where we operate with numeric group ids. - if (spec.GroupId) { - portoSpec.set_group(ToString(*spec.GroupId)); - } - - if (spec.ThreadLimit) { - portoSpec.set_thread_limit(*spec.ThreadLimit); - } - - if (spec.HostName) { - // To get a reasonable and unique host name inside container. - portoSpec.set_hostname(*spec.HostName); - if (!spec.IPAddresses.empty()) { - const auto& address = spec.IPAddresses[0]; - auto etcHosts = Format("%v %v\n", address, *spec.HostName); - // To be able to resolve hostname into IP inside container. - portoSpec.set_etc_hosts(etcHosts); - } - } - - if (spec.DisableNetwork) { - auto* netConfig = portoSpec.mutable_net()->add_cfg(); - netConfig->set_opt("none"); - } else if (!spec.IPAddresses.empty() && Config_->EnableNetworkIsolation) { - // This label is intended for HBF-agent: YT-12512. - auto* label = portoSpec.mutable_labels()->add_map(); - label->set_key("HBF.ignore_address"); - label->set_val("1"); - - auto* netConfig = portoSpec.mutable_net()->add_cfg(); - netConfig->set_opt("L3"); - netConfig->add_arg("veth0"); - - for (const auto& address : spec.IPAddresses) { - auto* ipConfig = portoSpec.mutable_ip()->add_cfg(); - ipConfig->set_dev("veth0"); - ipConfig->set_ip(ToString(address)); - } - - if (spec.EnableNat64) { - // Behave like nanny does. - portoSpec.set_resolv_conf("nameserver fd64::1;nameserver 2a02:6b8:0:3400::5005;options attempts:1 timeout:1"); - } - } - - for (const auto& [key, value] : spec.Labels) { - auto* map = portoSpec.mutable_labels()->add_map(); - map->set_key(key); - map->set_val(value); - } - - for (const auto& [name, value] : spec.Env) { - auto* var = portoSpec.mutable_env()->add_var(); - var->set_name(name); - var->set_value(value); - } - - for (const auto& controller : spec.CGroupControllers) { - portoSpec.mutable_controllers()->add_controller(controller); - } - - for (const auto& device : spec.Devices) { - auto* portoDevice = portoSpec.mutable_devices()->add_device(); - portoDevice->set_device(device.DeviceName); - portoDevice->set_access(device.Enabled ? "rw" : "-"); - } - - auto addBind = [&] (const TBind& bind) { - auto* portoBind = portoSpec.mutable_bind()->add_bind(); - portoBind->set_target(bind.TargetPath); - portoBind->set_source(bind.SourcePath); - portoBind->add_flag(bind.ReadOnly ? "ro" : "rw"); - }; - - if (spec.RootFS) { - portoSpec.set_root_readonly(spec.RootFS->IsRootReadOnly); - portoSpec.set_root(spec.RootFS->RootPath); - - for (const auto& bind : spec.RootFS->Binds) { - addBind(bind); - } - } - - { - auto* ulimit = portoSpec.mutable_ulimit()->add_ulimit(); - ulimit->set_type("core"); - if (spec.EnableCoreDumps) { - ulimit->set_unlimited(true); - } else { - ulimit->set_hard(0); - ulimit->set_soft(0); - } - } - - // Set some universal defaults. - portoSpec.set_oom_is_fatal(false); - - ExecuteApiCall( - [&] { return Api_->CreateFromSpec(portoSpec, {}, start); }, - "CreateFromSpec", - /*idempotent*/ false); - } - - void DoSetContainerProperty(const TString& container, const TString& property, const TString& value) - { - ExecuteApiCall( - [&] { return Api_->SetProperty(container, property, value); }, - "SetProperty", - /*idempotent*/ true); - } - - void DoDestroyContainer(const TString& container) - { - try { - ExecuteApiCall( - [&] { return Api_->Destroy(container); }, - "Destroy", - /*idempotent*/ true); - } catch (const TErrorException& ex) { - if (!ex.Error().FindMatching(EPortoErrorCode::ContainerDoesNotExist)) { - throw; - } - } - } - - void DoStopContainer(const TString& container) - { - ExecuteApiCall( - [&] { return Api_->Stop(container); }, - "Stop", - /*idempotent*/ true); - } - - void DoStartContainer(const TString& container) - { - ExecuteApiCall( - [&] { return Api_->Start(container); }, - "Start", - /*idempotent*/ false); - } - - TString DoConvertPath(const TString& path, const TString& container) - { - TString result; - ExecuteApiCall( - [&] { return Api_->ConvertPath(path, container, "self", result); }, - "ConvertPath", - /*idempotent*/ true); - return result; - } - - void DoKillContainer(const TString& container, int signal) - { - ExecuteApiCall( - [&] { return Api_->Kill(container, signal); }, - "Kill", - /*idempotent*/ false); - } - - std::vector<TString> DoListSubcontainers(const TString& rootContainer, bool includeRoot) - { - Porto::TListContainersRequest req; - auto filter = req.add_filters(); - filter->set_name(rootContainer + "/*"); - if (includeRoot) { - auto rootFilter = req.add_filters(); - rootFilter->set_name(rootContainer); - } - auto fieldOptions = req.mutable_field_options(); - fieldOptions->add_properties("absolute_name"); - TVector<Porto::TContainer> containers; - ExecuteApiCall( - [&] { return Api_->ListContainersBy(req, containers); }, - "ListContainersBy", - /*idempotent*/ true); - - std::vector<TString> containerNames; - containerNames.reserve(containers.size()); - for (const auto& container : containers) { - const auto& absoluteName = container.status().absolute_name(); - if (!absoluteName.empty()) { - containerNames.push_back(absoluteName); - } - } - return containerNames; - } - - TFuture<int> DoWaitContainer(const TString& container) - { - auto result = NewPromise<int>(); - auto waitCallback = [=, this, this_ = MakeStrong(this)] (const Porto::TWaitResponse& rsp) { - return OnContainerTerminated(rsp, result); - }; - - ExecuteApiCall( - [&] { return Api_->AsyncWait({container}, {}, waitCallback); }, - "AsyncWait", - /*idempotent*/ false); - - return result.ToFuture().ToImmediatelyCancelable(); - } - - void OnContainerTerminated(const Porto::TWaitResponse& portoWaitResponse, TPromise<int> result) - { - const auto& container = portoWaitResponse.name(); - const auto& state = portoWaitResponse.state(); - if (state != "dead" && state != "stopped") { - result.TrySet(TError("Container finished with unexpected state") - << TErrorAttribute("container_name", container) - << TErrorAttribute("container_state", state)); - return; - } - - // TODO(max42): switch to Subscribe. - YT_UNUSED_FUTURE(GetContainerProperty(container, "exit_status").Apply(BIND( - [=] (const TErrorOr<std::optional<TString>>& errorOrExitCode) { - if (!errorOrExitCode.IsOK()) { - result.TrySet(TError("Container finished, but exit status is unknown") - << errorOrExitCode); - return; - } - - const auto& optionalExitCode = errorOrExitCode.Value(); - if (!optionalExitCode) { - result.TrySet(TError("Container finished, but exit status is unknown") - << TErrorAttribute("container_name", container) - << TErrorAttribute("container_state", state)); - return; - } - - try { - int exitStatus = FromString<int>(*optionalExitCode); - result.TrySet(exitStatus); - } catch (const std::exception& ex) { - auto error = TError("Failed to parse Porto exit status") - << TErrorAttribute("container_name", container) - << TErrorAttribute("exit_status", optionalExitCode.value()); - error.MutableInnerErrors()->push_back(TError(ex)); - result.TrySet(error); - } - }))); - } - - TFuture<int> DoPollContainer(const TString& container) - { - auto [it, inserted] = ContainerMap_.insert({container, NewPromise<int>()}); - if (!inserted) { - YT_LOG_WARNING("Container already added for polling (Container: %v)", - container); - } else { - Containers_.push_back(container); - } - return it->second.ToFuture(); - } - - Porto::TGetResponse DoRequestContainerProperties( - const std::vector<TString>& containers, - const std::vector<TString>& vars) - { - TVector<TString> containers_(containers.begin(), containers.end()); - TVector<TString> vars_(vars.begin(), vars.end()); - - const Porto::TGetResponse* getResponse; - - ExecuteApiCall( - [&] { - getResponse = Api_->Get(containers_, vars_); - return getResponse ? EError::Success : EError::Unknown; - }, - "Get", - /*idempotent*/ true); - - YT_VERIFY(getResponse); - return *getResponse; - } - - THashMap<TString, i64> DoGetContainerMetrics( - const std::vector<TString>& containers, - const TString& metric) - { - TVector<TString> containers_(containers.begin(), containers.end()); - - TMap<TString, uint64_t> result; - - ExecuteApiCall( - [&] { return Api_->GetProcMetric(containers_, metric, result); }, - "GetProcMetric", - /*idempotent*/ true); - - return {result.begin(), result.end()}; - } - - void DoPoll() - { - try { - if (Containers_.empty()) { - return; - } - - auto getResponse = DoRequestContainerProperties(Containers_, ContainerRequestVars_); - - if (getResponse.list().empty()) { - return; - } - - auto getProperty = [] ( - const Porto::TGetResponse::TContainerGetListResponse& container, - const TString& name) -> Porto::TGetResponse::TContainerGetValueResponse - { - for (const auto& property : container.keyval()) { - if (property.variable() == name) { - return property; - } - } - - return {}; - }; - - for (const auto& container : getResponse.list()) { - auto state = getProperty(container, "state"); - if (state.error() == EError::ContainerDoesNotExist) { - HandleResult(container.name(), state); - } else if (state.value() == "dead" || state.value() == "stopped") { - HandleResult(container.name(), getProperty(container, "exit_status")); - } - //TODO(dcherednik): other states - } - } catch (const std::exception& ex) { - YT_LOG_ERROR(ex, "Fatal exception occurred while polling Porto"); - Failed_.Fire(TError(ex)); - } - } - - TString DoCreateVolume( - const TString& path, - const THashMap<TString, TString>& properties) - { - auto volume = path; - TMap<TString, TString> propertyMap(properties.begin(), properties.end()); - ExecuteApiCall( - [&] { return Api_->CreateVolume(volume, propertyMap); }, - "CreateVolume", - /*idempotent*/ false); - return volume; - } - - void DoLinkVolume(const TString& path, const TString& container) - { - ExecuteApiCall( - [&] { return Api_->LinkVolume(path, container); }, - "LinkVolume", - /*idempotent*/ false); - } - - void DoUnlinkVolume(const TString& path, const TString& container) - { - ExecuteApiCall( - [&] { return Api_->UnlinkVolume(path, container); }, - "UnlinkVolume", - /*idempotent*/ false); - } - - std::vector<TString> DoListVolumePaths() - { - TVector<TString> volumes; - ExecuteApiCall( - [&] { return Api_->ListVolumes(volumes); }, - "ListVolume", - /*idempotent*/ true); - return {volumes.begin(), volumes.end()}; - } - - void DoImportLayer(const TString& archivePath, const TString& layerId, const TString& place) - { - ExecuteApiCall( - [&] { return Api_->ImportLayer(layerId, archivePath, false, place); }, - "ImportLayer", - /*idempotent*/ false); - } - - void DoRemoveLayer(const TString& layerId, const TString& place, bool async) - { - ExecuteApiCall( - [&] { return Api_->RemoveLayer(layerId, place, async); }, - "RemoveLayer", - /*idempotent*/ false); - } - - std::vector<TString> DoListLayers(const TString& place) - { - TVector<TString> layers; - ExecuteApiCall( - [&] { return Api_->ListLayers(layers, place); }, - "ListLayers", - /*idempotent*/ true); - return {layers.begin(), layers.end()}; - } - - TCommandEntry* GetCommandEntry(const TString& command) - { - auto guard = Guard(CommandLock_); - if (auto it = CommandToEntry_.find(command)) { - return &it->second; - } - return &CommandToEntry_.emplace(command, TCommandEntry(Profiler_.WithTag("command", command))).first->second; - } - - void ExecuteApiCall( - std::function<EError()> callback, - const TString& command, - bool idempotent) - { - YT_LOG_DEBUG("Porto API call started (Command: %v)", command); - - if (IsTestPortoTimeout()) { - YT_LOG_DEBUG("Testing Porto timeout (Command: %v)", command); - - auto config = DynamicConfig_.Acquire(); - TDelayedExecutor::WaitForDuration(config->ApiTimeout); - - THROW_ERROR CreatePortoError(GetFailedStubError(), "Porto timeout"); - } - - if (IsTestPortoFailureEnabled()) { - YT_LOG_DEBUG("Testing Porto failure (Command: %v)", command); - THROW_ERROR CreatePortoError(GetFailedStubError(), "Porto stub error"); - } - - auto* entry = GetCommandEntry(command); - auto startTime = NProfiling::GetInstant(); - while (true) { - EError error; - - { - NProfiling::TWallTimer timer; - error = callback(); - entry->TimeGauge.Record(timer.GetElapsedTime()); - } - - if (error == EError::Success) { - entry->SuccessCounter.Increment(); - break; - } - - entry->FailureCounter.Increment(); - HandleApiError(command, startTime, idempotent); - - YT_LOG_DEBUG("Sleeping and retrying Porto API call (Command: %v)", command); - entry->RetryCounter.Increment(); - - TDelayedExecutor::WaitForDuration(RetryInterval); - } - - YT_LOG_DEBUG("Porto API call completed (Command: %v)", command); - } - - void HandleApiError( - const TString& command, - TInstant startTime, - bool idempotent) - { - TString errorMessage; - auto error = ConvertPortoErrorCode(Api_->GetLastError(errorMessage)); - - // These errors are typical during job cleanup: we might try to kill a container that is already stopped. - bool debug = (error == EPortoErrorCode::ContainerDoesNotExist || error == EPortoErrorCode::InvalidState); - YT_LOG_EVENT( - Logger, - debug ? NLogging::ELogLevel::Debug : NLogging::ELogLevel::Error, - "Porto API call error (Error: %v, Command: %v, Message: %v)", - error, - command, - errorMessage); - - if (!IsRetriableErrorCode(error, idempotent) || NProfiling::GetInstant() - startTime > Config_->RetriesTimeout) { - THROW_ERROR CreatePortoError(error, errorMessage); - } - } - - void HandleResult(const TString& container, const Porto::TGetResponse::TContainerGetValueResponse& rsp) - { - auto portoErrorCode = ConvertPortoErrorCode(rsp.error()); - auto it = ContainerMap_.find(container); - if (it == ContainerMap_.end()) { - YT_LOG_ERROR("Got an unexpected container " - "(Container: %v, ResponseError: %v, ErrorMessage: %v, Value: %v)", - container, - portoErrorCode, - rsp.errormsg(), - rsp.value()); - return; - } else { - if (portoErrorCode != EPortoErrorCode::Success) { - YT_LOG_ERROR("Container finished with Porto API error " - "(Container: %v, ResponseError: %v, ErrorMessage: %v, Value: %v)", - container, - portoErrorCode, - rsp.errormsg(), - rsp.value()); - it->second.Set(CreatePortoError(portoErrorCode, rsp.errormsg())); - } else { - try { - int exitStatus = std::stoi(rsp.value()); - YT_LOG_DEBUG("Container finished with exit code (Container: %v, ExitCode: %v)", - container, - exitStatus); - - it->second.Set(exitStatus); - } catch (const std::exception& ex) { - it->second.Set(TError("Failed to parse Porto exit status") << ex); - } - } - } - RemoveFromPoller(container); - } - - void RemoveFromPoller(const TString& container) - { - ContainerMap_.erase(container); - - Containers_.clear(); - for (const auto& [name, pid] : ContainerMap_) { - Containers_.push_back(name); - } - } -}; - -const std::vector<TString> TPortoExecutor::ContainerRequestVars_ = { - "state", - "exit_status" -}; - -//////////////////////////////////////////////////////////////////////////////// - -IPortoExecutorPtr CreatePortoExecutor( - TPortoExecutorDynamicConfigPtr config, - const TString& threadNameSuffix, - const NProfiling::TProfiler& profiler) -{ - return New<TPortoExecutor>( - std::move(config), - threadNameSuffix, - profiler); -} - -//////////////////////////////////////////////////////////////////////////////// - -#else - -IPortoExecutorPtr CreatePortoExecutor( - TPortoExecutorDynamicConfigPtr /* config */, - const TString& /* threadNameSuffix */, - const NProfiling::TProfiler& /* profiler */) -{ - THROW_ERROR_EXCEPTION("Porto executor is not available on this platform"); -} - -#endif - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NContainers diff --git a/yt/yt/library/containers/porto_executor.h b/yt/yt/library/containers/porto_executor.h deleted file mode 100644 index d629ab6275..0000000000 --- a/yt/yt/library/containers/porto_executor.h +++ /dev/null @@ -1,142 +0,0 @@ -#pragma once - -#include "public.h" - -#include <yt/yt/library/profiling/sensor.h> - -#include <yt/yt/core/actions/future.h> -#include <yt/yt/core/actions/signal.h> - -#include <yt/yt/core/net/address.h> - -#include <library/cpp/porto/libporto.hpp> - -namespace NYT::NContainers { - -//////////////////////////////////////////////////////////////////////////////// - -struct TVolumeId -{ - TString Path; -}; - -//////////////////////////////////////////////////////////////////////////////// - -struct TRunnableContainerSpec -{ - TString Name; - TString Command; - - EEnablePorto EnablePorto = EEnablePorto::None; - bool Isolate = true; - - std::optional<TString> StdinPath; - std::optional<TString> StdoutPath; - std::optional<TString> StderrPath; - std::optional<TString> CurrentWorkingDirectory; - std::optional<TString> CoreCommand; - std::optional<TString> User; - std::optional<int> GroupId; - - bool EnableCoreDumps = true; - - std::optional<i64> ThreadLimit; - - std::optional<TString> HostName; - std::vector<NYT::NNet::TIP6Address> IPAddresses; - bool EnableNat64 = false; - bool DisableNetwork = false; - - THashMap<TString, TString> Labels; - THashMap<TString, TString> Env; - std::vector<TString> CGroupControllers; - std::vector<TDevice> Devices; - std::optional<TRootFS> RootFS; -}; - -//////////////////////////////////////////////////////////////////////////////// - -struct IPortoExecutor - : public TRefCounted -{ - virtual void OnDynamicConfigChanged(const TPortoExecutorDynamicConfigPtr& newConfig) = 0; - - virtual TFuture<void> CreateContainer(const TString& container) = 0; - - virtual TFuture<void> CreateContainer(const TRunnableContainerSpec& containerSpec, bool start) = 0; - - virtual TFuture<void> SetContainerProperty( - const TString& container, - const TString& property, - const TString& value) = 0; - - virtual TFuture<std::optional<TString>> GetContainerProperty( - const TString& container, - const TString& property) = 0; - - virtual TFuture<THashMap<TString, TErrorOr<TString>>> GetContainerProperties( - const TString& container, - const std::vector<TString>& properties) = 0; - virtual TFuture<THashMap<TString, THashMap<TString, TErrorOr<TString>>>> GetContainerProperties( - const std::vector<TString>& containers, - const std::vector<TString>& properties) = 0; - - virtual TFuture<THashMap<TString, i64>> GetContainerMetrics( - const std::vector<TString>& containers, - const TString& metric) = 0; - virtual TFuture<void> DestroyContainer(const TString& container) = 0; - virtual TFuture<void> StopContainer(const TString& container) = 0; - virtual TFuture<void> StartContainer(const TString& container) = 0; - virtual TFuture<void> KillContainer(const TString& container, int signal) = 0; - - virtual TFuture<TString> ConvertPath(const TString& path, const TString& container) = 0; - - // Returns absolute names of immediate children only. - virtual TFuture<std::vector<TString>> ListSubcontainers( - const TString& rootContainer, - bool includeRoot) = 0; - // Starts polling a given container, returns future with exit code of finished process. - virtual TFuture<int> PollContainer(const TString& container) = 0; - - // Returns future with exit code of finished process. - // NB: temporarily broken, see https://st.yandex-team.ru/PORTO-846 for details. - virtual TFuture<int> WaitContainer(const TString& container) = 0; - - virtual TFuture<TString> CreateVolume( - const TString& path, - const THashMap<TString, TString>& properties) = 0; - virtual TFuture<void> LinkVolume( - const TString& path, - const TString& name) = 0; - virtual TFuture<void> UnlinkVolume( - const TString& path, - const TString& name) = 0; - virtual TFuture<std::vector<TString>> ListVolumePaths() = 0; - - virtual TFuture<void> ImportLayer( - const TString& archivePath, - const TString& layerId, - const TString& place) = 0; - virtual TFuture<void> RemoveLayer( - const TString& layerId, - const TString& place, - bool async) = 0; - virtual TFuture<std::vector<TString>> ListLayers(const TString& place) = 0; - - virtual IInvokerPtr GetInvoker() const = 0; - - DECLARE_INTERFACE_SIGNAL(void(const TError&), Failed); -}; - -DEFINE_REFCOUNTED_TYPE(IPortoExecutor) - -//////////////////////////////////////////////////////////////////////////////// - -IPortoExecutorPtr CreatePortoExecutor( - TPortoExecutorDynamicConfigPtr config, - const TString& threadNameSuffix, - const NProfiling::TProfiler& profiler = {}); - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NContainers diff --git a/yt/yt/library/containers/porto_health_checker.cpp b/yt/yt/library/containers/porto_health_checker.cpp deleted file mode 100644 index 5a5d358441..0000000000 --- a/yt/yt/library/containers/porto_health_checker.cpp +++ /dev/null @@ -1,69 +0,0 @@ - -#include "porto_health_checker.h" - -#include "porto_executor.h" -#include "private.h" -#include "config.h" - -#include <yt/yt/core/actions/future.h> - -#include <yt/yt/core/misc/fs.h> - -#include <util/random/random.h> - -namespace NYT::NContainers { - -using namespace NConcurrency; -using namespace NLogging; -using namespace NProfiling; - -//////////////////////////////////////////////////////////////////////////////// - -TPortoHealthChecker::TPortoHealthChecker( - TPortoExecutorDynamicConfigPtr config, - IInvokerPtr invoker, - TLogger logger) - : Config_(std::move(config)) - , Logger(std::move(logger)) - , CheckInvoker_(std::move(invoker)) - , Executor_(CreatePortoExecutor( - Config_, - "porto_check")) -{ } - -void TPortoHealthChecker::Start() -{ - YT_LOG_DEBUG("Porto health checker started"); - - PeriodicExecutor_ = New<TPeriodicExecutor>( - CheckInvoker_, - BIND(&TPortoHealthChecker::OnCheck, MakeWeak(this)), - Config_->RetriesTimeout); - PeriodicExecutor_->Start(); -} - -void TPortoHealthChecker::OnDynamicConfigChanged(const TPortoExecutorDynamicConfigPtr& newConfig) -{ - YT_LOG_DEBUG( - "Porto health checker dynamic config changed (EnableTestPortoFailures: %v, StubErrorCode: %v)", - Config_->EnableTestPortoFailures, - Config_->StubErrorCode); - - Executor_->OnDynamicConfigChanged(newConfig); -} - -void TPortoHealthChecker::OnCheck() -{ - YT_LOG_DEBUG("Run Porto health check"); - - auto result = WaitFor(Executor_->ListVolumePaths().AsVoid()); - if (result.IsOK()) { - Success_.Fire(); - } else { - Failed_.Fire(result); - } -} - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NContainers diff --git a/yt/yt/library/containers/porto_health_checker.h b/yt/yt/library/containers/porto_health_checker.h deleted file mode 100644 index f0fb8f0908..0000000000 --- a/yt/yt/library/containers/porto_health_checker.h +++ /dev/null @@ -1,52 +0,0 @@ -#pragma once - -#include "public.h" - -#include <yt/yt/library/profiling/sensor.h> - -#include <yt/yt/core/actions/signal.h> - -#include <yt/yt/core/concurrency/periodic_executor.h> - -#include <yt/yt/core/logging/log.h> - -#include <yt/yt/core/misc/error.h> - -#include <atomic> - -namespace NYT::NContainers { - -//////////////////////////////////////////////////////////////////////////////// - -class TPortoHealthChecker - : public TRefCounted -{ -public: - TPortoHealthChecker( - TPortoExecutorDynamicConfigPtr config, - IInvokerPtr invoker, - NLogging::TLogger logger); - - void Start(); - - void OnDynamicConfigChanged(const TPortoExecutorDynamicConfigPtr& newConfig); - - DEFINE_SIGNAL(void(), Success); - - DEFINE_SIGNAL(void(const TError&), Failed); - -private: - const TPortoExecutorDynamicConfigPtr Config_; - const NLogging::TLogger Logger; - const IInvokerPtr CheckInvoker_; - const IPortoExecutorPtr Executor_; - NConcurrency::TPeriodicExecutorPtr PeriodicExecutor_; - - void OnCheck(); -}; - -DEFINE_REFCOUNTED_TYPE(TPortoHealthChecker) - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NContainers diff --git a/yt/yt/library/containers/porto_resource_tracker.cpp b/yt/yt/library/containers/porto_resource_tracker.cpp deleted file mode 100644 index c1fe48d6af..0000000000 --- a/yt/yt/library/containers/porto_resource_tracker.cpp +++ /dev/null @@ -1,711 +0,0 @@ -#include "porto_resource_tracker.h" -#include "private.h" - -#include <yt/yt/core/logging/log.h> - -#include <yt/yt/core/misc/error.h> - -#include <yt/yt/core/net/address.h> - -#include <yt/yt/core/ytree/public.h> - -#include <yt/yt/library/process/process.h> - -#include <yt/yt/library/containers/cgroup.h> -#include <yt/yt/library/containers/config.h> -#include <yt/yt/library/containers/instance.h> -#include <yt/yt/library/containers/porto_executor.h> -#include <yt/yt/library/containers/public.h> - -namespace NYT::NContainers { - -using namespace NProfiling; - -static const auto& Logger = ContainersLogger; - -#ifdef _linux_ - -//////////////////////////////////////////////////////////////////////////////// - -struct TPortoProfilers - : public TRefCounted -{ - TPortoResourceProfilerPtr DaemonProfiler; - TPortoResourceProfilerPtr ContainerProfiler; - - TPortoProfilers( - TPortoResourceProfilerPtr daemonProfiler, - TPortoResourceProfilerPtr containerProfiler) - : DaemonProfiler(std::move(daemonProfiler)) - , ContainerProfiler(std::move(containerProfiler)) - { } -}; - -DEFINE_REFCOUNTED_TYPE(TPortoProfilers) - -//////////////////////////////////////////////////////////////////////////////// - -static TErrorOr<ui64> GetFieldOrError( - const TResourceUsage& usage, - EStatField field) -{ - auto it = usage.find(field); - if (it == usage.end()) { - return TError("Resource usage is missing %Qlv field", field); - } - const auto& errorOrValue = it->second; - if (errorOrValue.FindMatching(EPortoErrorCode::NotSupported)) { - return TError("Property %Qlv not supported in Porto response", field); - } - return errorOrValue; -} - -//////////////////////////////////////////////////////////////////////////////// - -TPortoResourceTracker::TPortoResourceTracker( - IInstancePtr instance, - TDuration updatePeriod, - bool isDeltaTracker, - bool isForceUpdate) - : Instance_(std::move(instance)) - , UpdatePeriod_(updatePeriod) - , IsDeltaTracker_(isDeltaTracker) - , IsForceUpdate_(isForceUpdate) -{ - ResourceUsage_ = { - {EStatField::IOReadByte, 0}, - {EStatField::IOWriteByte, 0}, - {EStatField::IOBytesLimit, 0}, - {EStatField::IOReadOps, 0}, - {EStatField::IOWriteOps, 0}, - {EStatField::IOOps, 0}, - {EStatField::IOOpsLimit, 0}, - {EStatField::IOTotalTime, 0}, - {EStatField::IOWaitTime, 0} - }; - ResourceUsageDelta_ = ResourceUsage_; -} - -static TErrorOr<TDuration> ExtractDuration(TErrorOr<ui64> timeNs) -{ - if (timeNs.IsOK()) { - return TErrorOr<TDuration>(TDuration::MicroSeconds(timeNs.Value() / 1000)); - } else { - return TError(timeNs); - } -} - -TCpuStatistics TPortoResourceTracker::ExtractCpuStatistics(const TResourceUsage& resourceUsage) const -{ - // NB: Job proxy uses last sample of CPU statistics but we are interested in - // peak thread count value. - auto currentThreadCountPeak = GetFieldOrError(resourceUsage, EStatField::ThreadCount); - - PeakThreadCount_ = currentThreadCountPeak.IsOK() && PeakThreadCount_.IsOK() - ? std::max<ui64>( - PeakThreadCount_.Value(), - currentThreadCountPeak.Value()) - : currentThreadCountPeak.IsOK() ? currentThreadCountPeak : PeakThreadCount_; - - auto totalTimeNs = GetFieldOrError(resourceUsage, EStatField::CpuUsage); - auto systemTimeNs = GetFieldOrError(resourceUsage, EStatField::CpuSystemUsage); - auto userTimeNs = GetFieldOrError(resourceUsage, EStatField::CpuUserUsage); - auto waitTimeNs = GetFieldOrError(resourceUsage, EStatField::CpuWait); - auto throttledNs = GetFieldOrError(resourceUsage, EStatField::CpuThrottled); - auto limitTimeNs = GetFieldOrError(resourceUsage, EStatField::CpuLimit); - auto guaranteeTimeNs = GetFieldOrError(resourceUsage, EStatField::CpuGuarantee); - - return TCpuStatistics{ - .TotalUsageTime = ExtractDuration(totalTimeNs), - .UserUsageTime = ExtractDuration(userTimeNs), - .SystemUsageTime = ExtractDuration(systemTimeNs), - .WaitTime = ExtractDuration(waitTimeNs), - .ThrottledTime = ExtractDuration(throttledNs), - .ThreadCount = GetFieldOrError(resourceUsage, EStatField::ThreadCount), - .ContextSwitches = GetFieldOrError(resourceUsage, EStatField::ContextSwitches), - .ContextSwitchesDelta = GetFieldOrError(resourceUsage, EStatField::ContextSwitchesDelta), - .PeakThreadCount = PeakThreadCount_, - .LimitTime = ExtractDuration(limitTimeNs), - .GuaranteeTime = ExtractDuration(guaranteeTimeNs), - }; -} - -TMemoryStatistics TPortoResourceTracker::ExtractMemoryStatistics(const TResourceUsage& resourceUsage) const -{ - return TMemoryStatistics{ - .Rss = GetFieldOrError(resourceUsage, EStatField::Rss), - .MappedFile = GetFieldOrError(resourceUsage, EStatField::MappedFile), - .MinorPageFaults = GetFieldOrError(resourceUsage, EStatField::MinorPageFaults), - .MajorPageFaults = GetFieldOrError(resourceUsage, EStatField::MajorPageFaults), - .FileCacheUsage = GetFieldOrError(resourceUsage, EStatField::FileCacheUsage), - .AnonUsage = GetFieldOrError(resourceUsage, EStatField::AnonMemoryUsage), - .AnonLimit = GetFieldOrError(resourceUsage, EStatField::AnonMemoryLimit), - .MemoryUsage = GetFieldOrError(resourceUsage, EStatField::MemoryUsage), - .MemoryGuarantee = GetFieldOrError(resourceUsage, EStatField::MemoryGuarantee), - .MemoryLimit = GetFieldOrError(resourceUsage, EStatField::MemoryLimit), - .MaxMemoryUsage = GetFieldOrError(resourceUsage, EStatField::MaxMemoryUsage), - .OomKills = GetFieldOrError(resourceUsage, EStatField::OomKills), - .OomKillsTotal = GetFieldOrError(resourceUsage, EStatField::OomKillsTotal) - }; -} - -TBlockIOStatistics TPortoResourceTracker::ExtractBlockIOStatistics(const TResourceUsage& resourceUsage) const -{ - auto totalTimeNs = GetFieldOrError(resourceUsage, EStatField::IOTotalTime); - auto waitTimeNs = GetFieldOrError(resourceUsage, EStatField::IOWaitTime); - - return TBlockIOStatistics{ - .IOReadByte = GetFieldOrError(resourceUsage, EStatField::IOReadByte), - .IOWriteByte = GetFieldOrError(resourceUsage, EStatField::IOWriteByte), - .IOBytesLimit = GetFieldOrError(resourceUsage, EStatField::IOBytesLimit), - .IOReadOps = GetFieldOrError(resourceUsage, EStatField::IOReadOps), - .IOWriteOps = GetFieldOrError(resourceUsage, EStatField::IOWriteOps), - .IOOps = GetFieldOrError(resourceUsage, EStatField::IOOps), - .IOOpsLimit = GetFieldOrError(resourceUsage, EStatField::IOOpsLimit), - .IOTotalTime = ExtractDuration(totalTimeNs), - .IOWaitTime = ExtractDuration(waitTimeNs) - }; -} - -TNetworkStatistics TPortoResourceTracker::ExtractNetworkStatistics(const TResourceUsage& resourceUsage) const -{ - return TNetworkStatistics{ - .TxBytes = GetFieldOrError(resourceUsage, EStatField::NetTxBytes), - .TxPackets = GetFieldOrError(resourceUsage, EStatField::NetTxPackets), - .TxDrops = GetFieldOrError(resourceUsage, EStatField::NetTxDrops), - .TxLimit = GetFieldOrError(resourceUsage, EStatField::NetTxLimit), - - .RxBytes = GetFieldOrError(resourceUsage, EStatField::NetRxBytes), - .RxPackets = GetFieldOrError(resourceUsage, EStatField::NetRxPackets), - .RxDrops = GetFieldOrError(resourceUsage, EStatField::NetRxDrops), - .RxLimit = GetFieldOrError(resourceUsage, EStatField::NetRxLimit), - }; -} - -TTotalStatistics TPortoResourceTracker::ExtractTotalStatistics(const TResourceUsage& resourceUsage) const -{ - return TTotalStatistics{ - .CpuStatistics = ExtractCpuStatistics(resourceUsage), - .MemoryStatistics = ExtractMemoryStatistics(resourceUsage), - .BlockIOStatistics = ExtractBlockIOStatistics(resourceUsage), - .NetworkStatistics = ExtractNetworkStatistics(resourceUsage), - }; -} - -TCpuStatistics TPortoResourceTracker::GetCpuStatistics() const -{ - return GetStatistics( - CachedCpuStatistics_, - "CPU", - [&] (TResourceUsage& resourceUsage) { - return ExtractCpuStatistics(resourceUsage); - }); -} - -TMemoryStatistics TPortoResourceTracker::GetMemoryStatistics() const -{ - return GetStatistics( - CachedMemoryStatistics_, - "memory", - [&] (TResourceUsage& resourceUsage) { - return ExtractMemoryStatistics(resourceUsage); - }); -} - -TBlockIOStatistics TPortoResourceTracker::GetBlockIOStatistics() const -{ - return GetStatistics( - CachedBlockIOStatistics_, - "block IO", - [&] (TResourceUsage& resourceUsage) { - return ExtractBlockIOStatistics(resourceUsage); - }); -} - -TNetworkStatistics TPortoResourceTracker::GetNetworkStatistics() const -{ - return GetStatistics( - CachedNetworkStatistics_, - "network", - [&] (TResourceUsage& resourceUsage) { - return ExtractNetworkStatistics(resourceUsage); - }); -} - -TTotalStatistics TPortoResourceTracker::GetTotalStatistics() const -{ - return GetStatistics( - CachedTotalStatistics_, - "total", - [&] (TResourceUsage& resourceUsage) { - return ExtractTotalStatistics(resourceUsage); - }); -} - -template <class T, class F> -T TPortoResourceTracker::GetStatistics( - std::optional<T>& cachedStatistics, - const TString& statisticsKind, - F extractor) const -{ - UpdateResourceUsageStatisticsIfExpired(); - - auto guard = Guard(SpinLock_); - try { - auto newStatistics = extractor(IsDeltaTracker_ ? ResourceUsageDelta_ : ResourceUsage_); - cachedStatistics = newStatistics; - return newStatistics; - } catch (const std::exception& ex) { - if (!cachedStatistics) { - THROW_ERROR_EXCEPTION("Unable to get %v statistics", statisticsKind) - << ex; - } - YT_LOG_WARNING(ex, "Unable to get %v statistics; using the last one", statisticsKind); - return *cachedStatistics; - } -} - -bool TPortoResourceTracker::AreResourceUsageStatisticsExpired() const -{ - return TInstant::Now() - LastUpdateTime_.load() > UpdatePeriod_; -} - -TInstant TPortoResourceTracker::GetLastUpdateTime() const -{ - return LastUpdateTime_.load(); -} - -void TPortoResourceTracker::UpdateResourceUsageStatisticsIfExpired() const -{ - if (IsForceUpdate_ || AreResourceUsageStatisticsExpired()) { - DoUpdateResourceUsage(); - } -} - -TErrorOr<ui64> TPortoResourceTracker::CalculateCounterDelta( - const TErrorOr<ui64>& oldValue, - const TErrorOr<ui64>& newValue) const -{ - if (oldValue.IsOK() && newValue.IsOK()) { - return newValue.Value() - oldValue.Value(); - } else if (newValue.IsOK()) { - // It is better to return an error than an incorrect value. - return oldValue; - } else { - return newValue; - } -} - -static bool IsCumulativeStatistics(EStatField statistic) -{ - return - statistic == EStatField::CpuUsage || - statistic == EStatField::CpuUserUsage || - statistic == EStatField::CpuSystemUsage || - statistic == EStatField::CpuWait || - statistic == EStatField::CpuThrottled || - - statistic == EStatField::ContextSwitches || - - statistic == EStatField::MinorPageFaults || - statistic == EStatField::MajorPageFaults || - - statistic == EStatField::IOReadByte || - statistic == EStatField::IOWriteByte || - statistic == EStatField::IOReadOps || - statistic == EStatField::IOWriteOps || - statistic == EStatField::IOOps || - statistic == EStatField::IOTotalTime || - statistic == EStatField::IOWaitTime || - - statistic == EStatField::NetTxBytes || - statistic == EStatField::NetTxPackets || - statistic == EStatField::NetTxDrops || - statistic == EStatField::NetRxBytes || - statistic == EStatField::NetRxPackets || - statistic == EStatField::NetRxDrops; -} - -void TPortoResourceTracker::ReCalculateResourceUsage(const TResourceUsage& newResourceUsage) const -{ - auto guard = Guard(SpinLock_); - - TResourceUsage resourceUsage; - TResourceUsage resourceUsageDelta; - - for (const auto& stat : InstanceStatFields) { - TErrorOr<ui64> oldValue; - TErrorOr<ui64> newValue; - - if (auto newValueIt = newResourceUsage.find(stat); newValueIt.IsEnd()) { - newValue = TError("Missing property %Qlv in Porto response", stat) - << TErrorAttribute("container", Instance_->GetName()); - } else { - newValue = newValueIt->second; - } - - if (auto oldValueIt = ResourceUsage_.find(stat); oldValueIt.IsEnd()) { - oldValue = newValue; - } else { - oldValue = oldValueIt->second; - } - - if (newValue.IsOK()) { - resourceUsage[stat] = newValue; - } else { - resourceUsage[stat] = oldValue; - } - - if (IsCumulativeStatistics(stat)) { - resourceUsageDelta[stat] = CalculateCounterDelta(oldValue, newValue); - } else { - if (newValue.IsOK()) { - resourceUsageDelta[stat] = newValue; - } else { - resourceUsageDelta[stat] = oldValue; - } - } - } - - ResourceUsage_ = resourceUsage; - ResourceUsageDelta_ = resourceUsageDelta; - LastUpdateTime_.store(TInstant::Now()); -} - -void TPortoResourceTracker::DoUpdateResourceUsage() const -{ - try { - ReCalculateResourceUsage(Instance_->GetResourceUsage()); - } catch (const std::exception& ex) { - YT_LOG_ERROR( - ex, - "Couldn't get metrics from Porto"); - } -} - -//////////////////////////////////////////////////////////////////////////////// - -TPortoResourceProfiler::TPortoResourceProfiler( - TPortoResourceTrackerPtr tracker, - TPodSpecConfigPtr podSpec, - const TProfiler& profiler) - : ResourceTracker_(std::move(tracker)) - , PodSpec_(std::move(podSpec)) -{ - profiler.AddProducer("", MakeStrong(this)); -} - -static void WriteGaugeIfOk( - ISensorWriter* writer, - const TString& path, - TErrorOr<ui64> valueOrError) -{ - if (valueOrError.IsOK()) { - i64 value = static_cast<i64>(valueOrError.Value()); - - if (value >= 0) { - writer->AddGauge(path, value); - } - } -} - -static void WriteCumulativeGaugeIfOk( - ISensorWriter* writer, - const TString& path, - TErrorOr<ui64> valueOrError, - i64 timeDeltaUsec) -{ - if (valueOrError.IsOK()) { - i64 value = static_cast<i64>(valueOrError.Value()); - - if (value >= 0) { - writer->AddGauge(path, - 1.0 * value * ResourceUsageUpdatePeriod.MicroSeconds() / timeDeltaUsec); - } - } -} - -void TPortoResourceProfiler::WriteCpuMetrics( - ISensorWriter* writer, - TTotalStatistics& totalStatistics, - i64 timeDeltaUsec) -{ - { - if (totalStatistics.CpuStatistics.UserUsageTime.IsOK()) { - i64 userUsageTimeUs = totalStatistics.CpuStatistics.UserUsageTime.Value().MicroSeconds(); - double userUsagePercent = std::max<double>(0.0, 100. * userUsageTimeUs / timeDeltaUsec); - writer->AddGauge("/cpu/user", userUsagePercent); - } - - if (totalStatistics.CpuStatistics.SystemUsageTime.IsOK()) { - i64 systemUsageTimeUs = totalStatistics.CpuStatistics.SystemUsageTime.Value().MicroSeconds(); - double systemUsagePercent = std::max<double>(0.0, 100. * systemUsageTimeUs / timeDeltaUsec); - writer->AddGauge("/cpu/system", systemUsagePercent); - } - - if (totalStatistics.CpuStatistics.WaitTime.IsOK()) { - i64 waitTimeUs = totalStatistics.CpuStatistics.WaitTime.Value().MicroSeconds(); - double waitPercent = std::max<double>(0.0, 100. * waitTimeUs / timeDeltaUsec); - writer->AddGauge("/cpu/wait", waitPercent); - } - - if (totalStatistics.CpuStatistics.ThrottledTime.IsOK()) { - i64 throttledTimeUs = totalStatistics.CpuStatistics.ThrottledTime.Value().MicroSeconds(); - double throttledPercent = std::max<double>(0.0, 100. * throttledTimeUs / timeDeltaUsec); - writer->AddGauge("/cpu/throttled", throttledPercent); - } - - if (totalStatistics.CpuStatistics.TotalUsageTime.IsOK()) { - i64 totalUsageTimeUs = totalStatistics.CpuStatistics.TotalUsageTime.Value().MicroSeconds(); - double totalUsagePercent = std::max<double>(0.0, 100. * totalUsageTimeUs / timeDeltaUsec); - writer->AddGauge("/cpu/total", totalUsagePercent); - } - - if (totalStatistics.CpuStatistics.GuaranteeTime.IsOK()) { - i64 guaranteeTimeUs = totalStatistics.CpuStatistics.GuaranteeTime.Value().MicroSeconds(); - double guaranteePercent = std::max<double>(0.0, (100. * guaranteeTimeUs) / (1'000'000L)); - writer->AddGauge("/cpu/guarantee", guaranteePercent); - } - - if (totalStatistics.CpuStatistics.LimitTime.IsOK()) { - i64 limitTimeUs = totalStatistics.CpuStatistics.LimitTime.Value().MicroSeconds(); - double limitPercent = std::max<double>(0.0, (100. * limitTimeUs) / (1'000'000L)); - writer->AddGauge("/cpu/limit", limitPercent); - } - } - - if (PodSpec_->CpuToVCpuFactor) { - auto factor = *PodSpec_->CpuToVCpuFactor; - - writer->AddGauge("/cpu_to_vcpu_factor", factor); - - if (totalStatistics.CpuStatistics.UserUsageTime.IsOK()) { - i64 userUsageTimeUs = totalStatistics.CpuStatistics.UserUsageTime.Value().MicroSeconds(); - double userUsagePercent = std::max<double>(0.0, 100. * userUsageTimeUs * factor / timeDeltaUsec); - writer->AddGauge("/vcpu/user", userUsagePercent); - } - - if (totalStatistics.CpuStatistics.SystemUsageTime.IsOK()) { - i64 systemUsageTimeUs = totalStatistics.CpuStatistics.SystemUsageTime.Value().MicroSeconds(); - double systemUsagePercent = std::max<double>(0.0, 100. * systemUsageTimeUs * factor / timeDeltaUsec); - writer->AddGauge("/vcpu/system", systemUsagePercent); - } - - if (totalStatistics.CpuStatistics.WaitTime.IsOK()) { - i64 waitTimeUs = totalStatistics.CpuStatistics.WaitTime.Value().MicroSeconds(); - double waitPercent = std::max<double>(0.0, 100. * waitTimeUs * factor / timeDeltaUsec); - writer->AddGauge("/vcpu/wait", waitPercent); - } - - if (totalStatistics.CpuStatistics.ThrottledTime.IsOK()) { - i64 throttledTimeUs = totalStatistics.CpuStatistics.ThrottledTime.Value().MicroSeconds(); - double throttledPercent = std::max<double>(0.0, 100. * throttledTimeUs * factor / timeDeltaUsec); - writer->AddGauge("/vcpu/throttled", throttledPercent); - } - - if (totalStatistics.CpuStatistics.TotalUsageTime.IsOK()) { - i64 totalUsageTimeUs = totalStatistics.CpuStatistics.TotalUsageTime.Value().MicroSeconds(); - double totalUsagePercent = std::max<double>(0.0, 100. * totalUsageTimeUs * factor / timeDeltaUsec); - writer->AddGauge("/vcpu/total", totalUsagePercent); - } - - if (totalStatistics.CpuStatistics.GuaranteeTime.IsOK()) { - i64 guaranteeTimeUs = totalStatistics.CpuStatistics.GuaranteeTime.Value().MicroSeconds(); - double guaranteePercent = std::max<double>(0.0, 100. * guaranteeTimeUs * factor / 1'000'000L); - writer->AddGauge("/vcpu/guarantee", guaranteePercent); - } - - if (totalStatistics.CpuStatistics.LimitTime.IsOK()) { - i64 limitTimeUs = totalStatistics.CpuStatistics.LimitTime.Value().MicroSeconds(); - double limitPercent = std::max<double>(0.0, 100. * limitTimeUs * factor / 1'000'000L); - writer->AddGauge("/vcpu/limit", limitPercent); - } - } - - WriteGaugeIfOk(writer, "/cpu/thread_count", totalStatistics.CpuStatistics.ThreadCount); - WriteGaugeIfOk(writer, "/cpu/context_switches", totalStatistics.CpuStatistics.ContextSwitches); -} - -void TPortoResourceProfiler::WriteMemoryMetrics( - ISensorWriter* writer, - TTotalStatistics& totalStatistics, - i64 timeDeltaUsec) -{ - WriteCumulativeGaugeIfOk(writer, - "/memory/minor_page_faults", - totalStatistics.MemoryStatistics.MinorPageFaults, - timeDeltaUsec); - WriteCumulativeGaugeIfOk(writer, - "/memory/major_page_faults", - totalStatistics.MemoryStatistics.MajorPageFaults, - timeDeltaUsec); - - WriteGaugeIfOk(writer, "/memory/oom_kills", totalStatistics.MemoryStatistics.OomKills); - WriteGaugeIfOk(writer, "/memory/oom_kills_total", totalStatistics.MemoryStatistics.OomKillsTotal); - - WriteGaugeIfOk(writer, "/memory/file_cache_usage", totalStatistics.MemoryStatistics.FileCacheUsage); - WriteGaugeIfOk(writer, "/memory/anon_usage", totalStatistics.MemoryStatistics.AnonUsage); - WriteGaugeIfOk(writer, "/memory/anon_limit", totalStatistics.MemoryStatistics.AnonLimit); - WriteGaugeIfOk(writer, "/memory/memory_usage", totalStatistics.MemoryStatistics.MemoryUsage); - WriteGaugeIfOk(writer, "/memory/memory_guarantee", totalStatistics.MemoryStatistics.MemoryGuarantee); - WriteGaugeIfOk(writer, "/memory/memory_limit", totalStatistics.MemoryStatistics.MemoryLimit); -} - -void TPortoResourceProfiler::WriteBlockingIOMetrics( - ISensorWriter* writer, - TTotalStatistics& totalStatistics, - i64 timeDeltaUsec) -{ - WriteCumulativeGaugeIfOk(writer, - "/io/read_bytes", - totalStatistics.BlockIOStatistics.IOReadByte, - timeDeltaUsec); - WriteCumulativeGaugeIfOk(writer, - "/io/write_bytes", - totalStatistics.BlockIOStatistics.IOWriteByte, - timeDeltaUsec); - WriteCumulativeGaugeIfOk(writer, - "/io/read_ops", - totalStatistics.BlockIOStatistics.IOReadOps, - timeDeltaUsec); - WriteCumulativeGaugeIfOk(writer, - "/io/write_ops", - totalStatistics.BlockIOStatistics.IOWriteOps, - timeDeltaUsec); - WriteCumulativeGaugeIfOk(writer, - "/io/ops", - totalStatistics.BlockIOStatistics.IOOps, - timeDeltaUsec); - - WriteGaugeIfOk(writer, - "/io/bytes_limit", - totalStatistics.BlockIOStatistics.IOBytesLimit); - WriteGaugeIfOk(writer, - "/io/ops_limit", - totalStatistics.BlockIOStatistics.IOOpsLimit); - - if (totalStatistics.BlockIOStatistics.IOTotalTime.IsOK()) { - i64 totalTimeUs = totalStatistics.BlockIOStatistics.IOTotalTime.Value().MicroSeconds(); - double totalPercent = std::max<double>(0.0, 100. * totalTimeUs / timeDeltaUsec); - writer->AddGauge("/io/total", totalPercent); - } - - if (totalStatistics.BlockIOStatistics.IOWaitTime.IsOK()) { - i64 waitTimeUs = totalStatistics.BlockIOStatistics.IOWaitTime.Value().MicroSeconds(); - double waitPercent = std::max<double>(0.0, 100. * waitTimeUs / timeDeltaUsec); - writer->AddGauge("/io/wait", waitPercent); - } -} - -void TPortoResourceProfiler::WriteNetworkMetrics( - ISensorWriter* writer, - TTotalStatistics& totalStatistics, - i64 timeDeltaUsec) -{ - WriteCumulativeGaugeIfOk( - writer, - "/network/rx_bytes", - totalStatistics.NetworkStatistics.RxBytes, - timeDeltaUsec); - WriteCumulativeGaugeIfOk( - writer, - "/network/rx_drops", - totalStatistics.NetworkStatistics.RxDrops, - timeDeltaUsec); - WriteCumulativeGaugeIfOk( - writer, - "/network/rx_packets", - totalStatistics.NetworkStatistics.RxPackets, - timeDeltaUsec); - WriteGaugeIfOk( - writer, - "/network/rx_limit", - totalStatistics.NetworkStatistics.RxLimit); - - WriteCumulativeGaugeIfOk( - writer, - "/network/tx_bytes", - totalStatistics.NetworkStatistics.TxBytes, - timeDeltaUsec); - WriteCumulativeGaugeIfOk( - writer, - "/network/tx_drops", - totalStatistics.NetworkStatistics.TxDrops, - timeDeltaUsec); - WriteCumulativeGaugeIfOk( - writer, - "/network/tx_packets", - totalStatistics.NetworkStatistics.TxPackets, - timeDeltaUsec); - WriteGaugeIfOk( - writer, - "/network/tx_limit", - totalStatistics.NetworkStatistics.TxLimit); -} - -void TPortoResourceProfiler::CollectSensors(ISensorWriter* writer) -{ - i64 lastUpdate = ResourceTracker_->GetLastUpdateTime().MicroSeconds(); - - auto totalStatistics = ResourceTracker_->GetTotalStatistics(); - i64 timeDeltaUsec = TInstant::Now().MicroSeconds() - lastUpdate; - - WriteCpuMetrics(writer, totalStatistics, timeDeltaUsec); - WriteMemoryMetrics(writer, totalStatistics, timeDeltaUsec); - WriteBlockingIOMetrics(writer, totalStatistics, timeDeltaUsec); - WriteNetworkMetrics(writer, totalStatistics, timeDeltaUsec); -} - -//////////////////////////////////////////////////////////////////////////////// - -TPortoResourceProfilerPtr CreatePortoProfilerWithTags( - const IInstancePtr& instance, - const TString containerCategory, - const TPodSpecConfigPtr& podSpec) -{ - auto portoResourceTracker = New<TPortoResourceTracker>( - instance, - ResourceUsageUpdatePeriod, - true, - true); - - return New<TPortoResourceProfiler>( - portoResourceTracker, - podSpec, - TProfiler("/porto") - .WithTag("container_category", containerCategory)); -} - -//////////////////////////////////////////////////////////////////////////////// - -#endif - -#ifdef __linux__ -void EnablePortoResourceTracker(const TPodSpecConfigPtr& podSpec) -{ - BIND([=] { - auto executor = CreatePortoExecutor(New<TPortoExecutorDynamicConfig>(), "porto-tracker"); - - executor->SubscribeFailed(BIND([=] (const TError& error) { - YT_LOG_ERROR(error, "Fatal error during Porto polling"); - })); - - LeakyRefCountedSingleton<TPortoProfilers>( - CreatePortoProfilerWithTags(GetSelfPortoInstance(executor), "daemon", podSpec), - CreatePortoProfilerWithTags(GetRootPortoInstance(executor), "pod", podSpec)); - }).AsyncVia(GetCurrentInvoker()) - .Run() - .Subscribe(BIND([] (const TError& error) { - YT_LOG_ERROR_IF(!error.IsOK(), error, "Failed to enable Porto profiler"); - })); -} -#else -void EnablePortoResourceTracker(const TPodSpecConfigPtr& /*podSpec*/) -{ - YT_LOG_WARNING("Porto resource tracker not supported"); -} -#endif - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NContainers diff --git a/yt/yt/library/containers/porto_resource_tracker.h b/yt/yt/library/containers/porto_resource_tracker.h deleted file mode 100644 index 8a0f781949..0000000000 --- a/yt/yt/library/containers/porto_resource_tracker.h +++ /dev/null @@ -1,158 +0,0 @@ -#pragma once - -#include <yt/yt/library/containers/instance.h> -#include <yt/yt/library/containers/public.h> - -#include <yt/yt/library/containers/cgroup.h> - -#include <yt/yt/core/misc/singleton.h> -#include <yt/yt/core/net/address.h> -#include <yt/yt/core/ytree/public.h> - -#include <yt/yt/library/process/process.h> -#include <yt/yt/library/profiling/producer.h> - -namespace NYT::NContainers { - -using namespace NProfiling; - -//////////////////////////////////////////////////////////////////////////////// - -static constexpr auto ResourceUsageUpdatePeriod = TDuration::MilliSeconds(1000); - -//////////////////////////////////////////////////////////////////////////////// - -using TCpuStatistics = TCpuAccounting::TStatistics; -using TBlockIOStatistics = TBlockIO::TStatistics; -using TMemoryStatistics = TMemory::TStatistics; -using TNetworkStatistics = TNetwork::TStatistics; - -struct TTotalStatistics -{ -public: - TCpuStatistics CpuStatistics; - TMemoryStatistics MemoryStatistics; - TBlockIOStatistics BlockIOStatistics; - TNetworkStatistics NetworkStatistics; -}; - -#ifdef _linux_ - -//////////////////////////////////////////////////////////////////////////////// - -class TPortoResourceTracker - : public TRefCounted -{ -public: - TPortoResourceTracker( - IInstancePtr instance, - TDuration updatePeriod, - bool isDeltaTracker = false, - bool isForceUpdate = false); - - TCpuStatistics GetCpuStatistics() const; - - TBlockIOStatistics GetBlockIOStatistics() const; - - TMemoryStatistics GetMemoryStatistics() const; - - TNetworkStatistics GetNetworkStatistics() const; - - TTotalStatistics GetTotalStatistics() const; - - bool AreResourceUsageStatisticsExpired() const; - - TInstant GetLastUpdateTime() const; - -private: - const IInstancePtr Instance_; - const TDuration UpdatePeriod_; - const bool IsDeltaTracker_; - const bool IsForceUpdate_; - - mutable std::atomic<TInstant> LastUpdateTime_ = {}; - - YT_DECLARE_SPIN_LOCK(NThreading::TSpinLock, SpinLock_); - mutable TResourceUsage ResourceUsage_; - mutable TResourceUsage ResourceUsageDelta_; - - mutable std::optional<TCpuStatistics> CachedCpuStatistics_; - mutable std::optional<TMemoryStatistics> CachedMemoryStatistics_; - mutable std::optional<TBlockIOStatistics> CachedBlockIOStatistics_; - mutable std::optional<TNetworkStatistics> CachedNetworkStatistics_; - mutable std::optional<TTotalStatistics> CachedTotalStatistics_; - mutable TErrorOr<ui64> PeakThreadCount_ = 0; - - template <class T, class F> - T GetStatistics( - std::optional<T>& cachedStatistics, - const TString& statisticsKind, - F extractor) const; - - TCpuStatistics ExtractCpuStatistics(const TResourceUsage& resourceUsage) const; - TMemoryStatistics ExtractMemoryStatistics(const TResourceUsage& resourceUsage) const; - TBlockIOStatistics ExtractBlockIOStatistics(const TResourceUsage& resourceUsage) const; - TNetworkStatistics ExtractNetworkStatistics(const TResourceUsage& resourceUsage) const; - TTotalStatistics ExtractTotalStatistics(const TResourceUsage& resourceUsage) const; - - TErrorOr<ui64> CalculateCounterDelta( - const TErrorOr<ui64>& oldValue, - const TErrorOr<ui64>& newValue) const; - - void ReCalculateResourceUsage(const TResourceUsage& newResourceUsage) const; - - void UpdateResourceUsageStatisticsIfExpired() const; - - void DoUpdateResourceUsage() const; -}; - -DEFINE_REFCOUNTED_TYPE(TPortoResourceTracker) - -//////////////////////////////////////////////////////////////////////////////// - -class TPortoResourceProfiler - : public ISensorProducer -{ -public: - TPortoResourceProfiler( - TPortoResourceTrackerPtr tracker, - TPodSpecConfigPtr podSpec, - const TProfiler& profiler = TProfiler{"/porto"}); - - void CollectSensors(ISensorWriter* writer) override; - -private: - const TPortoResourceTrackerPtr ResourceTracker_; - const TPodSpecConfigPtr PodSpec_; - - void WriteCpuMetrics( - ISensorWriter* writer, - TTotalStatistics& totalStatistics, - i64 timeDeltaUsec); - - void WriteMemoryMetrics( - ISensorWriter* writer, - TTotalStatistics& totalStatistics, - i64 timeDeltaUsec); - - void WriteBlockingIOMetrics( - ISensorWriter* writer, - TTotalStatistics& totalStatistics, - i64 timeDeltaUsec); - - void WriteNetworkMetrics( - ISensorWriter* writer, - TTotalStatistics& totalStatistics, - i64 timeDeltaUsec); -}; - -DECLARE_REFCOUNTED_TYPE(TPortoResourceProfiler) -DEFINE_REFCOUNTED_TYPE(TPortoResourceProfiler) - -//////////////////////////////////////////////////////////////////////////////// - -#endif - -void EnablePortoResourceTracker(const TPodSpecConfigPtr& podSpec); - -} // namespace NYT::NContainers diff --git a/yt/yt/library/containers/private.h b/yt/yt/library/containers/private.h deleted file mode 100644 index 62682cb364..0000000000 --- a/yt/yt/library/containers/private.h +++ /dev/null @@ -1,13 +0,0 @@ -#pragma once - -#include <yt/yt/core/logging/log.h> - -namespace NYT::NContainers { - -//////////////////////////////////////////////////////////////////////////////// - -inline const NLogging::TLogger ContainersLogger("Containers"); - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NContainers diff --git a/yt/yt/library/containers/process.cpp b/yt/yt/library/containers/process.cpp deleted file mode 100644 index ad1c8d35dc..0000000000 --- a/yt/yt/library/containers/process.cpp +++ /dev/null @@ -1,154 +0,0 @@ -#ifdef __linux__ - -#include "process.h" - -#include <yt/yt/library/containers/instance.h> - -#include <yt/yt/core/misc/proc.h> -#include <yt/yt/core/misc/fs.h> - -namespace NYT::NContainers { - -using namespace NPipes; -using namespace NNet; -using namespace NConcurrency; - -//////////////////////////////////////////////////////////////////////////////// - -static inline const NLogging::TLogger Logger("Process"); - -static constexpr pid_t InvalidProcessId = -1; - -//////////////////////////////////////////////////////////////////////////////// - -TPortoProcess::TPortoProcess( - const TString& path, - IInstanceLauncherPtr containerLauncher, - bool copyEnv) - : TProcessBase(path) - , ContainerLauncher_(std::move(containerLauncher)) -{ - AddArgument(NFS::GetFileName(path)); - if (copyEnv) { - for (char** envIt = environ; *envIt; ++envIt) { - Env_.push_back(Capture(*envIt)); - } - } -} - -void TPortoProcess::Kill(int signal) -{ - if (auto instance = GetInstance()) { - instance->Kill(signal); - } -} - -void TPortoProcess::DoSpawn() -{ - YT_VERIFY(ProcessId_ == InvalidProcessId && !Finished_); - YT_VERIFY(!GetInstance()); - YT_VERIFY(!Started_); - YT_VERIFY(!Args_.empty()); - - if (!WorkingDirectory_.empty()) { - ContainerLauncher_->SetCwd(WorkingDirectory_); - } - - Started_ = true; - - try { - // TPortoProcess doesn't support running processes inside rootFS. - YT_VERIFY(!ContainerLauncher_->HasRoot()); - std::vector<TString> args(Args_.begin() + 1, Args_.end()); - auto instance = WaitFor(ContainerLauncher_->Launch(ResolvedPath_, args, DecomposeEnv())) - .ValueOrThrow(); - ContainerInstance_.Store(instance); - FinishedPromise_.SetFrom(instance->Wait()); - - try { - ProcessId_ = instance->GetPid(); - } catch (const std::exception& ex) { - // This could happen if Porto container has already died or pid namespace of - // parent container is not a parent of pid namespace of child container. - // It's not a problem, since for Porto process pid is used for logging purposes only. - YT_LOG_DEBUG(ex, "Failed to get pid of root process (Container: %v)", - instance->GetName()); - } - - YT_LOG_DEBUG("Process inside Porto spawned successfully (Path: %v, ExternalPid: %v, Container: %v)", - ResolvedPath_, - ProcessId_, - instance->GetName()); - - FinishedPromise_.ToFuture().Subscribe(BIND([=, this, this_ = MakeStrong(this)] (const TError& exitStatus) { - Finished_ = true; - if (exitStatus.IsOK()) { - YT_LOG_DEBUG("Process inside Porto exited gracefully (ExternalPid: %v, Container: %v)", - ProcessId_, - instance->GetName()); - } else { - YT_LOG_DEBUG(exitStatus, "Process inside Porto exited with an error (ExternalPid: %v, Container: %v)", - ProcessId_, - instance->GetName()); - } - })); - } catch (const std::exception& ex) { - Finished_ = true; - THROW_ERROR_EXCEPTION("Failed to start child process inside Porto") - << TErrorAttribute("path", ResolvedPath_) - << TErrorAttribute("container", ContainerLauncher_->GetName()) - << ex; - } -} - -IInstancePtr TPortoProcess::GetInstance() -{ - return ContainerInstance_.Acquire(); -} - -THashMap<TString, TString> TPortoProcess::DecomposeEnv() const -{ - THashMap<TString, TString> result; - for (const auto& env : Env_) { - TStringBuf name, value; - TStringBuf(env).TrySplit('=', name, value); - result[name] = value; - } - return result; -} - -static TString CreateStdIONamedPipePath() -{ - const TString name = ToString(TGuid::Create()); - return NFS::GetRealPath(NFS::CombinePaths("/tmp", name)); -} - -IConnectionWriterPtr TPortoProcess::GetStdInWriter() -{ - auto pipe = TNamedPipe::Create(CreateStdIONamedPipePath()); - ContainerLauncher_->SetStdIn(pipe->GetPath()); - NamedPipes_.push_back(pipe); - return pipe->CreateAsyncWriter(); -} - -IConnectionReaderPtr TPortoProcess::GetStdOutReader() -{ - auto pipe = TNamedPipe::Create(CreateStdIONamedPipePath()); - ContainerLauncher_->SetStdOut(pipe->GetPath()); - NamedPipes_.push_back(pipe); - return pipe->CreateAsyncReader(); -} - -IConnectionReaderPtr TPortoProcess::GetStdErrReader() -{ - auto pipe = TNamedPipe::Create(CreateStdIONamedPipePath()); - ContainerLauncher_->SetStdErr(pipe->GetPath()); - NamedPipes_.push_back(pipe); - return pipe->CreateAsyncReader(); -} - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NContainers - -#endif diff --git a/yt/yt/library/containers/process.h b/yt/yt/library/containers/process.h deleted file mode 100644 index 75255165d8..0000000000 --- a/yt/yt/library/containers/process.h +++ /dev/null @@ -1,46 +0,0 @@ -#pragma once - -#include "public.h" - -#include <yt/yt/library/process/process.h> - -#include <library/cpp/yt/memory/atomic_intrusive_ptr.h> - -#include <library/cpp/porto/libporto.hpp> - -namespace NYT::NContainers { - -//////////////////////////////////////////////////////////////////////////////// - -// NB(psushin): this class is deprecated and only used to run job proxy. -// ToDo(psushin): kill me. -class TPortoProcess - : public TProcessBase -{ -public: - TPortoProcess( - const TString& path, - NContainers::IInstanceLauncherPtr containerLauncher, - bool copyEnv = true); - void Kill(int signal) override; - NNet::IConnectionWriterPtr GetStdInWriter() override; - NNet::IConnectionReaderPtr GetStdOutReader() override; - NNet::IConnectionReaderPtr GetStdErrReader() override; - - NContainers::IInstancePtr GetInstance(); - -private: - const NContainers::IInstanceLauncherPtr ContainerLauncher_; - - TAtomicIntrusivePtr<NContainers::IInstance> ContainerInstance_; - std::vector<NPipes::TNamedPipePtr> NamedPipes_; - - void DoSpawn() override; - THashMap<TString, TString> DecomposeEnv() const; -}; - -DEFINE_REFCOUNTED_TYPE(TPortoProcess) - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NContainers diff --git a/yt/yt/library/containers/public.h b/yt/yt/library/containers/public.h deleted file mode 100644 index d8e3cf3491..0000000000 --- a/yt/yt/library/containers/public.h +++ /dev/null @@ -1,163 +0,0 @@ -#pragma once - -#include <yt/yt/core/misc/public.h> - -#include <library/cpp/porto/proto/rpc.pb.h> -#include <library/cpp/yt/misc/enum.h> - -namespace NYT::NContainers { - -//////////////////////////////////////////////////////////////////////////////// - -const int PortoErrorCodeBase = 12000; - -DEFINE_ENUM(EPortoErrorCode, - ((Success) ((PortoErrorCodeBase + Porto::EError::Success))) - ((Unknown) ((PortoErrorCodeBase + Porto::EError::Unknown))) - ((InvalidMethod) ((PortoErrorCodeBase + Porto::EError::InvalidMethod))) - ((ContainerAlreadyExists) ((PortoErrorCodeBase + Porto::EError::ContainerAlreadyExists))) - ((ContainerDoesNotExist) ((PortoErrorCodeBase + Porto::EError::ContainerDoesNotExist))) - ((InvalidProperty) ((PortoErrorCodeBase + Porto::EError::InvalidProperty))) - ((InvalidData) ((PortoErrorCodeBase + Porto::EError::InvalidData))) - ((InvalidValue) ((PortoErrorCodeBase + Porto::EError::InvalidValue))) - ((InvalidState) ((PortoErrorCodeBase + Porto::EError::InvalidState))) - ((NotSupported) ((PortoErrorCodeBase + Porto::EError::NotSupported))) - ((ResourceNotAvailable) ((PortoErrorCodeBase + Porto::EError::ResourceNotAvailable))) - ((Permission) ((PortoErrorCodeBase + Porto::EError::Permission))) - ((VolumeAlreadyExists) ((PortoErrorCodeBase + Porto::EError::VolumeAlreadyExists))) - ((VolumeNotFound) ((PortoErrorCodeBase + Porto::EError::VolumeNotFound))) - ((NoSpace) ((PortoErrorCodeBase + Porto::EError::NoSpace))) - ((Busy) ((PortoErrorCodeBase + Porto::EError::Busy))) - ((VolumeAlreadyLinked) ((PortoErrorCodeBase + Porto::EError::VolumeAlreadyLinked))) - ((VolumeNotLinked) ((PortoErrorCodeBase + Porto::EError::VolumeNotLinked))) - ((LayerAlreadyExists) ((PortoErrorCodeBase + Porto::EError::LayerAlreadyExists))) - ((LayerNotFound) ((PortoErrorCodeBase + Porto::EError::LayerNotFound))) - ((NoValue) ((PortoErrorCodeBase + Porto::EError::NoValue))) - ((VolumeNotReady) ((PortoErrorCodeBase + Porto::EError::VolumeNotReady))) - ((InvalidCommand) ((PortoErrorCodeBase + Porto::EError::InvalidCommand))) - ((LostError) ((PortoErrorCodeBase + Porto::EError::LostError))) - ((DeviceNotFound) ((PortoErrorCodeBase + Porto::EError::DeviceNotFound))) - ((InvalidPath) ((PortoErrorCodeBase + Porto::EError::InvalidPath))) - ((InvalidNetworkAddress) ((PortoErrorCodeBase + Porto::EError::InvalidNetworkAddress))) - ((PortoFrozen) ((PortoErrorCodeBase + Porto::EError::PortoFrozen))) - ((LabelNotFound) ((PortoErrorCodeBase + Porto::EError::LabelNotFound))) - ((InvalidLabel) ((PortoErrorCodeBase + Porto::EError::InvalidLabel))) - ((NotFound) ((PortoErrorCodeBase + Porto::EError::NotFound))) - ((SocketError) ((PortoErrorCodeBase + Porto::EError::SocketError))) - ((SocketUnavailable) ((PortoErrorCodeBase + Porto::EError::SocketUnavailable))) - ((SocketTimeout) ((PortoErrorCodeBase + Porto::EError::SocketTimeout))) - ((Taint) ((PortoErrorCodeBase + Porto::EError::Taint))) - ((Queued) ((PortoErrorCodeBase + Porto::EError::Queued))) -); - -//////////////////////////////////////////////////////////////////////////////// - -YT_DEFINE_ERROR_ENUM( - ((FailedToStartContainer) (14000)) -); - -DEFINE_ENUM(EStatField, - // CPU - (CpuUsage) - (CpuUserUsage) - (CpuSystemUsage) - (CpuWait) - (CpuThrottled) - (ContextSwitches) - (ContextSwitchesDelta) - (ThreadCount) - (CpuLimit) - (CpuGuarantee) - - // Memory - (Rss) - (MappedFile) - (MajorPageFaults) - (MinorPageFaults) - (FileCacheUsage) - (AnonMemoryUsage) - (AnonMemoryLimit) - (MemoryUsage) - (MemoryGuarantee) - (MemoryLimit) - (MaxMemoryUsage) - (OomKills) - (OomKillsTotal) - - // IO - (IOReadByte) - (IOWriteByte) - (IOBytesLimit) - (IOReadOps) - (IOWriteOps) - (IOOps) - (IOOpsLimit) - (IOTotalTime) - (IOWaitTime) - - // Network - (NetTxBytes) - (NetTxPackets) - (NetTxDrops) - (NetTxLimit) - (NetRxBytes) - (NetRxPackets) - (NetRxDrops) - (NetRxLimit) -); - -DEFINE_ENUM(EEnablePorto, - (None) - (Isolate) - (Full) -); - -struct TBind -{ - TString SourcePath; - TString TargetPath; - bool ReadOnly; -}; - -struct TRootFS -{ - TString RootPath; - bool IsRootReadOnly; - std::vector<TBind> Binds; -}; - -struct TDevice -{ - TString DeviceName; - bool Enabled; -}; - -struct TInstanceLimits -{ - double Cpu = 0; - i64 Memory = 0; - std::optional<i64> NetTx; - std::optional<i64> NetRx; - - bool operator==(const TInstanceLimits&) const = default; -}; - -DECLARE_REFCOUNTED_STRUCT(IContainerManager) -DECLARE_REFCOUNTED_STRUCT(IInstanceLauncher) -DECLARE_REFCOUNTED_STRUCT(IInstance) -DECLARE_REFCOUNTED_STRUCT(IPortoExecutor) - -DECLARE_REFCOUNTED_CLASS(TPortoHealthChecker) -DECLARE_REFCOUNTED_CLASS(TInstanceLimitsTracker) -DECLARE_REFCOUNTED_CLASS(TPortoProcess) -DECLARE_REFCOUNTED_CLASS(TPortoResourceTracker) -DECLARE_REFCOUNTED_CLASS(TPortoExecutorDynamicConfig) -DECLARE_REFCOUNTED_CLASS(TPodSpecConfig) - -//////////////////////////////////////////////////////////////////////////////// - -bool IsValidCGroupType(const TString& type); - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NContainers diff --git a/yt/yt/library/containers/unittests/containers_ut.cpp b/yt/yt/library/containers/unittests/containers_ut.cpp deleted file mode 100644 index 4f1c10a435..0000000000 --- a/yt/yt/library/containers/unittests/containers_ut.cpp +++ /dev/null @@ -1,133 +0,0 @@ -#include <yt/yt/core/test_framework/framework.h> - -#ifdef _linux_ - -#include <yt/yt/library/containers/config.h> -#include <yt/yt/library/containers/porto_executor.h> -#include <yt/yt/library/containers/instance.h> - -#include <util/system/platform.h> -#include <util/system/env.h> - -namespace NYT::NContainers { -namespace { - -using namespace NConcurrency; - -//////////////////////////////////////////////////////////////////////////////// - -class TContainersTest - : public ::testing::Test -{ - void SetUp() override - { - if (GetEnv("SKIP_PORTO_TESTS") != "") { - GTEST_SKIP(); - } - } -}; - -static TString GetUniqueName() -{ - return "yt_ut_" + ToString(TGuid::Create()); -} - -IPortoExecutorPtr CreatePortoExecutor() -{ - return CreatePortoExecutor(New<TPortoExecutorDynamicConfig>(), "default"); -} - -TEST_F(TContainersTest, ListSubcontainers) -{ - auto executor = CreatePortoExecutor(); - auto name = GetUniqueName(); - - WaitFor(executor->CreateContainer(name)) - .ThrowOnError(); - - auto absoluteName = *WaitFor(executor->GetContainerProperty(name, "absolute_name")) - .ValueOrThrow(); - - auto nestedName = absoluteName + "/nested"; - WaitFor(executor->CreateContainer(nestedName)) - .ThrowOnError(); - - auto withRoot = WaitFor(executor->ListSubcontainers(name, true)) - .ValueOrThrow(); - EXPECT_EQ(std::vector<TString>({absoluteName, nestedName}), withRoot); - - auto withoutRoot = WaitFor(executor->ListSubcontainers(name, false)) - .ValueOrThrow(); - EXPECT_EQ(std::vector<TString>({nestedName}), withoutRoot); - - WaitFor(executor->DestroyContainer(absoluteName)) - .ThrowOnError(); -} - -// See https://st.yandex-team.ru/PORTO-846. -TEST_F(TContainersTest, DISABLED_WaitContainer) -{ - auto executor = CreatePortoExecutor(); - auto name = GetUniqueName(); - - WaitFor(executor->CreateContainer(name)) - .ThrowOnError(); - - WaitFor(executor->SetContainerProperty(name, "command", "sleep 10")) - .ThrowOnError(); - - WaitFor(executor->StartContainer(name)) - .ThrowOnError(); - - auto exitCode = WaitFor(executor->WaitContainer(name)) - .ValueOrThrow(); - - EXPECT_EQ(0, exitCode); - - WaitFor(executor->DestroyContainer(name)) - .ThrowOnError(); -} - -TEST_F(TContainersTest, CreateFromSpec) -{ - auto executor = CreatePortoExecutor(); - auto name = GetUniqueName(); - - auto spec = TRunnableContainerSpec { - .Name = name, - .Command = "sleep 2", - }; - - WaitFor(executor->CreateContainer(spec, /*start*/ true)) - .ThrowOnError(); - - auto exitCode = WaitFor(executor->PollContainer(name)) - .ValueOrThrow(); - - EXPECT_EQ(0, exitCode); - - WaitFor(executor->DestroyContainer(name)) - .ThrowOnError(); -} - -TEST_F(TContainersTest, ListPids) -{ - auto launcher = CreatePortoInstanceLauncher( - GetUniqueName(), - CreatePortoExecutor()); - - auto instance = WaitFor(launcher->Launch("sleep", {"5"}, {})) - .ValueOrThrow(); - - auto pids = instance->GetPids(); - EXPECT_LT(0u, pids.size()); - - instance->Destroy(); -} - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace -} // namespace NYT::NContainers - -#endif diff --git a/yt/yt/library/containers/unittests/porto_resource_tracker_ut.cpp b/yt/yt/library/containers/unittests/porto_resource_tracker_ut.cpp deleted file mode 100644 index 04d169ba4e..0000000000 --- a/yt/yt/library/containers/unittests/porto_resource_tracker_ut.cpp +++ /dev/null @@ -1,251 +0,0 @@ -#include <yt/yt/core/test_framework/framework.h> - -#include <yt/yt/core/ytree/convert.h> - -#include <util/system/fs.h> -#include <util/system/tempfile.h> - -#include <yt/yt/library/profiling/producer.h> -#include <yt/yt/library/containers/config.h> -#include <yt/yt/library/containers/porto_executor.h> -#include <yt/yt/library/containers/porto_resource_tracker.h> -#include <yt/yt/library/containers/instance.h> - -#include <util/system/platform.h> -#include <util/system/env.h> - -namespace NYT::NContainers { -namespace { - -using namespace NConcurrency; - -//////////////////////////////////////////////////////////////////////////////// - -static constexpr auto TestUpdatePeriod = TDuration::MilliSeconds(10); - -class TPortoTrackerTest - : public ::testing::Test -{ -public: - IPortoExecutorPtr Executor; - - void SetUp() override - { - if (GetEnv("SKIP_PORTO_TESTS") != "") { - GTEST_SKIP(); - } - - Executor = CreatePortoExecutor(New<TPortoExecutorDynamicConfig>(), "default"); - } -}; - -TString GetUniqueName() -{ - return "yt_porto_ut_" + ToString(TGuid::Create()); -} - -TPortoResourceTrackerPtr CreateSumPortoTracker(IPortoExecutorPtr Executor, const TString& name) -{ - return New<TPortoResourceTracker>( - GetPortoInstance(Executor, name), - TestUpdatePeriod, - false); -} - -TPortoResourceProfilerPtr CreateDeltaPortoProfiler(IPortoExecutorPtr executor, const TString& name) -{ - auto instance = GetPortoInstance(executor, name); - auto portoResourceTracker = New<TPortoResourceTracker>( - instance, - ResourceUsageUpdatePeriod, - true, - true - ); - - // Init metrics for delta tracker. - portoResourceTracker->GetTotalStatistics(); - - return LeakyRefCountedSingleton<TPortoResourceProfiler>( - portoResourceTracker, - New<TPodSpecConfig>(), - TProfiler("/porto") - .WithTag("porto_name", instance->GetName()) - .WithTag("container_category", "yt_daemon")); -} - -void AssertGauges(const std::vector<std::tuple<TString, TTagList, double>>& gauges) { - THashSet<TString> sensors{ - "/cpu/user", - "/cpu/total", - "/cpu/system", - "/cpu/wait", - "/cpu/throttled", - "/cpu/guarantee", - "/cpu/limit", - "/cpu/thread_count", - "/cpu/context_switches", - - "/memory/minor_page_faults", - "/memory/major_page_faults", - "/memory/file_cache_usage", - "/memory/anon_usage", - "/memory/anon_limit", - "/memory/memory_usage", - "/memory/memory_guarantee", - "/memory/memory_limit", - - "/io/read_bytes", - "/io/write_bytes", - "/io/bytes_limit", - - "/io/read_ops", - "/io/write_ops", - "/io/ops", - "/io/ops_limit", - "/io/total", - - "/network/rx_bytes", - "/network/rx_drops", - "/network/rx_packets", - "/network/rx_limit", - "/network/tx_bytes", - "/network/tx_drops", - "/network/tx_packets", - "/network/tx_limit" - }; - - THashSet<TString> mayBeEmpty{ - "/cpu/wait", - "/cpu/throttled", - "/cpu/guarantee", - "/cpu/context_switches", - "/memory/major_page_faults", - "/memory/memory_guarantee", - "/io/ops_limit", - "/io/read_ops", - "/io/write_ops", - "/io/wait", - "/io/bytes_limit", - "/network/rx_bytes", - "/network/rx_drops", - "/network/rx_packets", - "/network/rx_limit", - "/network/tx_bytes", - "/network/tx_drops", - "/network/tx_packets", - "/network/tx_limit" - }; - - for (const auto& [name, tags, value] : gauges) { - EXPECT_TRUE(value >= 0 && sensors.find(name) || mayBeEmpty.find(name)); - } -} - -TEST_F(TPortoTrackerTest, ValidateSummaryPortoTracker) -{ - auto name = GetUniqueName(); - - WaitFor(Executor->CreateContainer( - TRunnableContainerSpec { - .Name = name, - .Command = "sleep .1", - }, true)) - .ThrowOnError(); - - auto tracker = CreateSumPortoTracker(Executor, name); - - auto firstStatistics = tracker->GetTotalStatistics(); - - WaitFor(Executor->StopContainer(name)) - .ThrowOnError(); - WaitFor(Executor->SetContainerProperty( - name, - "command", - "find /")) - .ThrowOnError(); - WaitFor(Executor->StartContainer(name)) - .ThrowOnError(); - Sleep(TDuration::MilliSeconds(500)); - - auto secondStatistics = tracker->GetTotalStatistics(); - - WaitFor(Executor->DestroyContainer(name)) - .ThrowOnError(); -} - -TEST_F(TPortoTrackerTest, ValidateDeltaPortoTracker) -{ - auto name = GetUniqueName(); - - auto spec = TRunnableContainerSpec { - .Name = name, - .Command = "sleep .1", - }; - - WaitFor(Executor->CreateContainer(spec, true)) - .ThrowOnError(); - - auto profiler = CreateDeltaPortoProfiler(Executor, name); - - WaitFor(Executor->StopContainer(name)) - .ThrowOnError(); - WaitFor(Executor->SetContainerProperty( - name, - "command", - "find /")) - .ThrowOnError(); - WaitFor(Executor->StartContainer(name)) - .ThrowOnError(); - - Sleep(TDuration::MilliSeconds(500)); - - auto buffer = New<TSensorBuffer>(); - profiler->CollectSensors(buffer.Get()); - AssertGauges(buffer->GetGauges()); - - WaitFor(Executor->DestroyContainer(name)) - .ThrowOnError(); -} - -TEST_F(TPortoTrackerTest, ValidateDeltaRootPortoTracker) -{ - auto name = GetUniqueName(); - - auto spec = TRunnableContainerSpec { - .Name = name, - .Command = "sleep .1", - }; - - WaitFor(Executor->CreateContainer(spec, true)) - .ThrowOnError(); - - auto profiler = CreateDeltaPortoProfiler( - Executor, - GetPortoInstance( - Executor, - *GetPortoInstance(Executor, name)->GetRootName())->GetName()); - - WaitFor(Executor->StopContainer(name)) - .ThrowOnError(); - WaitFor(Executor->SetContainerProperty( - name, - "command", - "find /")) - .ThrowOnError(); - WaitFor(Executor->StartContainer(name)) - .ThrowOnError(); - - Sleep(TDuration::MilliSeconds(500)); - - auto buffer = New<TSensorBuffer>(); - profiler->CollectSensors(buffer.Get()); - AssertGauges(buffer->GetGauges()); - - WaitFor(Executor->DestroyContainer(name)) - .ThrowOnError(); -} - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace -} // namespace NYT::NContainers diff --git a/yt/yt/library/containers/unittests/process_ut.cpp b/yt/yt/library/containers/unittests/process_ut.cpp deleted file mode 100644 index b9c0d844f4..0000000000 --- a/yt/yt/library/containers/unittests/process_ut.cpp +++ /dev/null @@ -1,302 +0,0 @@ -#include <yt/yt/core/test_framework/framework.h> - -#ifdef _linux_ - -#include <yt/yt/core/actions/bind.h> - -#include <yt/yt/core/concurrency/action_queue.h> -#include <yt/yt/core/concurrency/delayed_executor.h> -#include <yt/yt/core/concurrency/scheduler.h> - -#include <yt/yt/core/misc/guid.h> -#include <yt/yt/core/misc/proc.h> - -#include <yt/yt/core/net/connection.h> - -#include <yt/yt/library/containers/process.h> - -#include <yt/yt/library/containers/config.h> -#include <yt/yt/library/containers/porto_executor.h> -#include <yt/yt/library/containers/instance.h> - -#include <util/system/platform.h> -#include <util/system/env.h> - -namespace NYT::NContainers { -namespace { - -using namespace NConcurrency; - -//////////////////////////////////////////////////////////////////////////////// - -class TPortoProcessTest - : public ::testing::Test -{ - void SetUp() override - { - if (GetEnv("SKIP_PORTO_TESTS") != "") { - GTEST_SKIP(); - } - } -}; - -static TString GetUniqueName() -{ - return "yt_ut_" + ToString(TGuid::Create()); -} - -IPortoExecutorPtr CreatePortoExecutor() -{ - return CreatePortoExecutor(New<TPortoExecutorDynamicConfig>(), "default"); -} - -TEST_F(TPortoProcessTest, Basic) -{ - auto launcher = CreatePortoInstanceLauncher( - GetUniqueName(), - CreatePortoExecutor()); - auto p = New<TPortoProcess>("/bin/ls", launcher, true); - TFuture<void> finished; - ASSERT_NO_THROW(finished = p->Spawn()); - ASSERT_TRUE(p->IsStarted()); - auto error = WaitFor(finished); - EXPECT_TRUE(error.IsOK()) << ToString(error); - EXPECT_TRUE(p->IsFinished()); - p->GetInstance()->Destroy(); -} - -TEST_F(TPortoProcessTest, RunFromPathEnv) -{ - auto launcher = CreatePortoInstanceLauncher( - GetUniqueName(), - CreatePortoExecutor()); - auto p = New<TPortoProcess>("ls", launcher, true); - TFuture<void> finished; - ASSERT_NO_THROW(finished = p->Spawn()); - ASSERT_TRUE(p->IsStarted()); - auto error = WaitFor(finished); - EXPECT_TRUE(error.IsOK()) << ToString(error); - EXPECT_TRUE(p->IsFinished()); - p->GetInstance()->Destroy(); -} - -TEST_F(TPortoProcessTest, MultiBasic) -{ - auto portoExecutor = CreatePortoExecutor(); - auto l1 = CreatePortoInstanceLauncher(GetUniqueName(), portoExecutor); - auto l2 = CreatePortoInstanceLauncher(GetUniqueName(), portoExecutor); - auto p1 = New<TPortoProcess>("/bin/ls", l1, true); - auto p2 = New<TPortoProcess>("/bin/ls", l2, true); - TFuture<void> f1; - TFuture<void> f2; - ASSERT_NO_THROW(f1 = p1->Spawn()); - ASSERT_NO_THROW(f2 = p2->Spawn()); - auto error = WaitFor((AllSucceeded(std::vector<TFuture<void>>{f1, f2}))); - EXPECT_TRUE(error.IsOK()) << ToString(error); - EXPECT_TRUE(p1->IsFinished()); - EXPECT_TRUE(p2->IsFinished()); - p1->GetInstance()->Destroy(); - p2->GetInstance()->Destroy(); -} - -TEST_F(TPortoProcessTest, InvalidPath) -{ - auto portoExecutor = CreatePortoExecutor(); - auto launcher = CreatePortoInstanceLauncher( - GetUniqueName(), - portoExecutor); - auto p = New<TPortoProcess>("/some/bad/path/binary", launcher, true); - TFuture<void> finished; - ASSERT_NO_THROW(finished = p->Spawn()); - ASSERT_FALSE(p->IsStarted()); - auto error = WaitFor(finished); - EXPECT_FALSE(p->IsFinished()); - EXPECT_FALSE(error.IsOK()); - WaitFor(portoExecutor->DestroyContainer(launcher->GetName())) - .ThrowOnError(); -} - -TEST_F(TPortoProcessTest, StdOut) -{ - auto launcher = CreatePortoInstanceLauncher( - GetUniqueName(), - CreatePortoExecutor()); - auto p = New<TPortoProcess>("/bin/date", launcher, true); - - auto outStream = p->GetStdOutReader(); - TFuture<void> finished; - ASSERT_NO_THROW(finished = p->Spawn()); - ASSERT_TRUE(p->IsStarted()); - auto error = WaitFor(finished); - EXPECT_TRUE(error.IsOK()) << ToString(error); - EXPECT_TRUE(p->IsFinished()); - - auto buffer = TSharedMutableRef::Allocate(4_KB, {.InitializeStorage = false}); - auto future = outStream->Read(buffer); - TErrorOr<size_t> result = WaitFor(future); - size_t sz = result.ValueOrThrow(); - EXPECT_TRUE(sz > 0); - p->GetInstance()->Destroy(); -} - -TEST_F(TPortoProcessTest, GetCommandLine) -{ - auto launcher = CreatePortoInstanceLauncher( - GetUniqueName(), - CreatePortoExecutor()); - auto p = New<TPortoProcess>("/bin/bash", launcher, true); - EXPECT_EQ("/bin/bash", p->GetCommandLine()); - p->AddArgument("-c"); - EXPECT_EQ("/bin/bash -c", p->GetCommandLine()); - p->AddArgument("exit 0"); - EXPECT_EQ("/bin/bash -c \"exit 0\"", p->GetCommandLine()); -} - -TEST_F(TPortoProcessTest, ProcessReturnCode0) -{ - auto launcher = CreatePortoInstanceLauncher( - GetUniqueName(), - CreatePortoExecutor()); - auto p = New<TPortoProcess>("/bin/bash", launcher, true); - p->AddArgument("-c"); - p->AddArgument("exit 0"); - - TFuture<void> finished; - ASSERT_NO_THROW(finished = p->Spawn()); - ASSERT_TRUE(p->IsStarted()); - auto error = WaitFor(finished); - EXPECT_TRUE(error.IsOK()) << ToString(error); - EXPECT_TRUE(p->IsFinished()); - p->GetInstance()->Destroy(); -} - -TEST_F(TPortoProcessTest, ProcessReturnCode123) -{ - auto launcher = CreatePortoInstanceLauncher( - GetUniqueName(), - CreatePortoExecutor()); - auto p = New<TPortoProcess>("/bin/bash", launcher, true); - p->AddArgument("-c"); - p->AddArgument("exit 123"); - - TFuture<void> finished; - ASSERT_NO_THROW(finished = p->Spawn()); - ASSERT_TRUE(p->IsStarted()); - auto error = WaitFor(finished); - EXPECT_EQ(EProcessErrorCode::NonZeroExitCode, error.GetCode()); - EXPECT_EQ(123, error.Attributes().Get<int>("exit_code")); - EXPECT_TRUE(p->IsFinished()); - p->GetInstance()->Destroy(); -} - -TEST_F(TPortoProcessTest, Params1) -{ - auto launcher = CreatePortoInstanceLauncher( - GetUniqueName(), - CreatePortoExecutor()); - auto p = New<TPortoProcess>("/bin/bash", launcher, true); - p->AddArgument("-c"); - p->AddArgument("if test 3 -gt 1; then exit 7; fi"); - - auto error = WaitFor(p->Spawn()); - EXPECT_FALSE(error.IsOK()); - EXPECT_TRUE(p->IsFinished()); - p->GetInstance()->Destroy(); -} - -TEST_F(TPortoProcessTest, Params2) -{ - auto launcher = CreatePortoInstanceLauncher( - GetUniqueName(), - CreatePortoExecutor()); - auto p = New<TPortoProcess>("/bin/bash", launcher, true); - p->AddArgument("-c"); - p->AddArgument("if test 1 -gt 3; then exit 7; fi"); - - auto error = WaitFor(p->Spawn()); - EXPECT_TRUE(error.IsOK()) << ToString(error); - EXPECT_TRUE(p->IsFinished()); - p->GetInstance()->Destroy(); -} - -TEST_F(TPortoProcessTest, InheritEnvironment) -{ - const char* name = "SPAWN_TEST_ENV_VAR"; - const char* value = "42"; - setenv(name, value, 1); - - auto launcher = CreatePortoInstanceLauncher( - GetUniqueName(), - CreatePortoExecutor()); - auto p = New<TPortoProcess>("/bin/bash", launcher, true); - p->AddArgument("-c"); - p->AddArgument("if test $SPAWN_TEST_ENV_VAR = 42; then exit 7; fi"); - - auto error = WaitFor(p->Spawn()); - EXPECT_FALSE(error.IsOK()); - EXPECT_TRUE(p->IsFinished()); - - unsetenv(name); - p->GetInstance()->Destroy(); -} - -TEST_F(TPortoProcessTest, Kill) -{ - auto launcher = CreatePortoInstanceLauncher( - GetUniqueName(), - CreatePortoExecutor()); - auto p = New<TPortoProcess>("/bin/sleep", launcher, true); - p->AddArgument("5"); - - auto finished = p->Spawn(); - - NConcurrency::TDelayedExecutor::Submit( - BIND([&] () { - p->Kill(SIGKILL); - }), - TDuration::MilliSeconds(100)); - - auto error = WaitFor(finished); - EXPECT_FALSE(error.IsOK()) << ToString(error); - EXPECT_TRUE(p->IsFinished()); - p->GetInstance()->Destroy(); -} - -TEST_F(TPortoProcessTest, KillFinished) -{ - auto launcher = CreatePortoInstanceLauncher( - GetUniqueName(), - CreatePortoExecutor()); - auto p = New<TPortoProcess>("/bin/bash", launcher, true); - p->AddArgument("-c"); - p->AddArgument("true"); - - auto finished = p->Spawn(); - - auto error = WaitFor(finished); - EXPECT_TRUE(error.IsOK()); - - p->Kill(SIGKILL); - p->GetInstance()->Destroy(); -} - -TEST_F(TPortoProcessTest, PollDuration) -{ - auto launcher = CreatePortoInstanceLauncher( - GetUniqueName(), - CreatePortoExecutor()); - auto p = New<TPortoProcess>("/bin/sleep", launcher, true); - p->AddArgument("1"); - - auto error = WaitFor(p->Spawn()); - EXPECT_TRUE(error.IsOK()) << ToString(error); - EXPECT_TRUE(p->IsFinished()); - p->GetInstance()->Destroy(); -} - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace -} // namespace NYT::NContainers - -#endif diff --git a/yt/yt/library/containers/unittests/ya.make b/yt/yt/library/containers/unittests/ya.make deleted file mode 100644 index 42984e2dc7..0000000000 --- a/yt/yt/library/containers/unittests/ya.make +++ /dev/null @@ -1,35 +0,0 @@ -GTEST(unittester-containers) - -INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc) - -ALLOCATOR(TCMALLOC) - -IF (AUTOCHECK) - ENV(SKIP_PORTO_TESTS=1) -ENDIF() - -IF (DISTBUILD) # TODO(prime@): this is always on - ENV(SKIP_PORTO_TESTS=1) -ENDIF() - -SRCS( - containers_ut.cpp - process_ut.cpp -) - -IF(OS_LINUX) - SRCS( - porto_resource_tracker_ut.cpp - ) -ENDIF() - -INCLUDE(${ARCADIA_ROOT}/yt/opensource_tests.inc) - -PEERDIR( - yt/yt/build - yt/yt/library/containers -) - -SIZE(MEDIUM) - -END() diff --git a/yt/yt/library/containers/ya.make b/yt/yt/library/containers/ya.make deleted file mode 100644 index 499b8d9da8..0000000000 --- a/yt/yt/library/containers/ya.make +++ /dev/null @@ -1,37 +0,0 @@ -LIBRARY() - -INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc) - -SRCS( - cgroup.cpp - config.cpp - instance.cpp - instance_limits_tracker.cpp - process.cpp - porto_executor.cpp - porto_resource_tracker.cpp - porto_health_checker.cpp -) - -PEERDIR( - library/cpp/porto/proto - yt/yt/library/process - yt/yt/core -) - -IF(OS_LINUX) - PEERDIR( - library/cpp/porto - ) -ENDIF() - -END() - -RECURSE( - disk_manager - cri -) - -RECURSE_FOR_TESTS( - unittests -) diff --git a/yt/yt/library/process/CMakeLists.darwin-x86_64.txt b/yt/yt/library/process/CMakeLists.darwin-x86_64.txt deleted file mode 100644 index b66c679390..0000000000 --- a/yt/yt/library/process/CMakeLists.darwin-x86_64.txt +++ /dev/null @@ -1,26 +0,0 @@ - -# This file was generated by the build system used internally in the Yandex monorepo. -# Only simple modifications are allowed (adding source-files to targets, adding simple properties -# like target_include_directories). These modifications will be ported to original -# ya.make files by maintainers. Any complex modifications which can't be ported back to the -# original buildsystem will not be accepted. - - - -add_library(yt-library-process) -target_compile_options(yt-library-process PRIVATE - -Wdeprecated-this-capture -) -target_link_libraries(yt-library-process PUBLIC - contrib-libs-cxxsupp - yutil - yt-yt-core - contrib-libs-re2 -) -target_sources(yt-library-process PRIVATE - ${CMAKE_SOURCE_DIR}/yt/yt/library/process/io_dispatcher.cpp - ${CMAKE_SOURCE_DIR}/yt/yt/library/process/pipe.cpp - ${CMAKE_SOURCE_DIR}/yt/yt/library/process/process.cpp - ${CMAKE_SOURCE_DIR}/yt/yt/library/process/pty.cpp - ${CMAKE_SOURCE_DIR}/yt/yt/library/process/subprocess.cpp -) diff --git a/yt/yt/library/process/CMakeLists.txt b/yt/yt/library/process/CMakeLists.txt index f8b31df0c1..4d48dcdee6 100644 --- a/yt/yt/library/process/CMakeLists.txt +++ b/yt/yt/library/process/CMakeLists.txt @@ -8,10 +8,6 @@ if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) include(CMakeLists.linux-aarch64.txt) -elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") - include(CMakeLists.darwin-x86_64.txt) -elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) - include(CMakeLists.windows-x86_64.txt) elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) include(CMakeLists.linux-x86_64.txt) endif() diff --git a/yt/yt/library/process/CMakeLists.windows-x86_64.txt b/yt/yt/library/process/CMakeLists.windows-x86_64.txt deleted file mode 100644 index 3637ee7dae..0000000000 --- a/yt/yt/library/process/CMakeLists.windows-x86_64.txt +++ /dev/null @@ -1,23 +0,0 @@ - -# This file was generated by the build system used internally in the Yandex monorepo. -# Only simple modifications are allowed (adding source-files to targets, adding simple properties -# like target_include_directories). These modifications will be ported to original -# ya.make files by maintainers. Any complex modifications which can't be ported back to the -# original buildsystem will not be accepted. - - - -add_library(yt-library-process) -target_link_libraries(yt-library-process PUBLIC - contrib-libs-cxxsupp - yutil - yt-yt-core - contrib-libs-re2 -) -target_sources(yt-library-process PRIVATE - ${CMAKE_SOURCE_DIR}/yt/yt/library/process/io_dispatcher.cpp - ${CMAKE_SOURCE_DIR}/yt/yt/library/process/pipe.cpp - ${CMAKE_SOURCE_DIR}/yt/yt/library/process/process.cpp - ${CMAKE_SOURCE_DIR}/yt/yt/library/process/pty.cpp - ${CMAKE_SOURCE_DIR}/yt/yt/library/process/subprocess.cpp -) diff --git a/yt/yt/library/program/CMakeLists.darwin-x86_64.txt b/yt/yt/library/program/CMakeLists.darwin-x86_64.txt index fc3e796e79..e4ea489293 100644 --- a/yt/yt/library/program/CMakeLists.darwin-x86_64.txt +++ b/yt/yt/library/program/CMakeLists.darwin-x86_64.txt @@ -17,7 +17,6 @@ target_link_libraries(yt-library-program PUBLIC yt-yt-core core-service_discovery-yp yt-library-monitoring - yt-library-containers library-profiling-solomon library-profiling-tcmalloc library-profiling-perf diff --git a/yt/yt/library/program/CMakeLists.linux-aarch64.txt b/yt/yt/library/program/CMakeLists.linux-aarch64.txt index be31ba10cf..7c15fb8b7d 100644 --- a/yt/yt/library/program/CMakeLists.linux-aarch64.txt +++ b/yt/yt/library/program/CMakeLists.linux-aarch64.txt @@ -18,7 +18,6 @@ target_link_libraries(yt-library-program PUBLIC yt-yt-core core-service_discovery-yp yt-library-monitoring - yt-library-containers library-profiling-solomon library-profiling-tcmalloc library-profiling-perf diff --git a/yt/yt/library/program/CMakeLists.linux-x86_64.txt b/yt/yt/library/program/CMakeLists.linux-x86_64.txt index be31ba10cf..7c15fb8b7d 100644 --- a/yt/yt/library/program/CMakeLists.linux-x86_64.txt +++ b/yt/yt/library/program/CMakeLists.linux-x86_64.txt @@ -18,7 +18,6 @@ target_link_libraries(yt-library-program PUBLIC yt-yt-core core-service_discovery-yp yt-library-monitoring - yt-library-containers library-profiling-solomon library-profiling-tcmalloc library-profiling-perf diff --git a/yt/yt/library/program/CMakeLists.windows-x86_64.txt b/yt/yt/library/program/CMakeLists.windows-x86_64.txt index 1f2aea4bd0..7f37407e9d 100644 --- a/yt/yt/library/program/CMakeLists.windows-x86_64.txt +++ b/yt/yt/library/program/CMakeLists.windows-x86_64.txt @@ -14,7 +14,6 @@ target_link_libraries(yt-library-program PUBLIC yt-yt-core core-service_discovery-yp yt-library-monitoring - yt-library-containers library-profiling-solomon library-profiling-tcmalloc library-profiling-perf diff --git a/yt/yt/library/program/config.cpp b/yt/yt/library/program/config.cpp index 0705fb48fc..ccc7bb1f2f 100644 --- a/yt/yt/library/program/config.cpp +++ b/yt/yt/library/program/config.cpp @@ -103,12 +103,8 @@ void TSingletonsConfig::Register(TRegistrar registrar) .Default(true); registrar.Parameter("enable_resource_tracker", &TThis::EnableResourceTracker) .Default(true); - registrar.Parameter("enable_porto_resource_tracker", &TThis::EnablePortoResourceTracker) - .Default(false); registrar.Parameter("resource_tracker_vcpu_factor", &TThis::ResourceTrackerVCpuFactor) .Optional(); - registrar.Parameter("pod_spec", &TThis::PodSpec) - .DefaultNew(); registrar.Parameter("heap_profiler", &TThis::HeapProfiler) .DefaultNew(); diff --git a/yt/yt/library/program/config.h b/yt/yt/library/program/config.h index 7d92939f1c..88f649dd8e 100644 --- a/yt/yt/library/program/config.h +++ b/yt/yt/library/program/config.h @@ -22,8 +22,6 @@ #include <yt/yt/library/profiling/solomon/exporter.h> -#include <yt/yt/library/containers/config.h> - #include <yt/yt/library/tracing/jaeger/tracer.h> #include <library/cpp/yt/stockpile/stockpile.h> @@ -149,9 +147,7 @@ public: TStockpileConfigPtr Stockpile; bool EnableRefCountedTrackerProfiling; bool EnableResourceTracker; - bool EnablePortoResourceTracker; std::optional<double> ResourceTrackerVCpuFactor; - NContainers::TPodSpecConfigPtr PodSpec; THeapProfilerConfigPtr HeapProfiler; REGISTER_YSON_STRUCT(TSingletonsConfig); diff --git a/yt/yt/library/program/helpers.cpp b/yt/yt/library/program/helpers.cpp index 5c7ff29db1..fc4a84d828 100644 --- a/yt/yt/library/program/helpers.cpp +++ b/yt/yt/library/program/helpers.cpp @@ -16,19 +16,14 @@ #include <yt/yt/library/profiling/resource_tracker/resource_tracker.h> -#include <yt/yt/library/containers/config.h> -#include <yt/yt/library/containers/porto_resource_tracker.h> - #include <yt/yt/core/logging/log_manager.h> #include <yt/yt/core/concurrency/execution_stack.h> #include <yt/yt/core/concurrency/periodic_executor.h> -#include <yt/yt/core/concurrency/private.h> #include <tcmalloc/malloc_extension.h> #include <yt/yt/core/net/address.h> -#include <yt/yt/core/net/local_address.h> #include <yt/yt/core/rpc/dispatcher.h> #include <yt/yt/core/rpc/grpc/dispatcher.h> @@ -37,8 +32,6 @@ #include <yt/yt/core/threading/spin_wait_slow_path_logger.h> -#include <library/cpp/yt/threading/spin_wait_hook.h> - #include <library/cpp/yt/memory/atomic_intrusive_ptr.h> #include <util/string/split.h> @@ -236,10 +229,6 @@ void ConfigureSingletonsImpl(const TConfig& config) NProfiling::SetVCpuFactor(config->ResourceTrackerVCpuFactor.value()); } } - - if (config->EnablePortoResourceTracker) { - NContainers::EnablePortoResourceTracker(config->PodSpec); - } } void ConfigureSingletons(const TSingletonsConfigPtr& config) diff --git a/yt/yt/library/program/ya.make b/yt/yt/library/program/ya.make index 5742ce9287..1d044b5a8d 100644 --- a/yt/yt/library/program/ya.make +++ b/yt/yt/library/program/ya.make @@ -16,7 +16,6 @@ PEERDIR( yt/yt/core yt/yt/core/service_discovery/yp yt/yt/library/monitoring - yt/yt/library/containers yt/yt/library/profiling/solomon yt/yt/library/profiling/tcmalloc yt/yt/library/profiling/perf |