diff options
| author | robot-piglet <robot-piglet@yandex-team.com> | 2024-02-27 16:27:23 +0300 |
|---|---|---|
| committer | robot-piglet <robot-piglet@yandex-team.com> | 2024-02-27 17:00:48 +0300 |
| commit | 6c35e9daf4dc86464e1a262236b8d3593e690ee5 (patch) | |
| tree | 2727d5da08d12182c1a705ec7213a6b97f5da521 | |
| parent | 6fa64684d15341dd701bf1498ecafdd764d5f097 (diff) | |
| download | ydb-6c35e9daf4dc86464e1a262236b8d3593e690ee5.tar.gz | |
Intermediate changes
74 files changed, 604 insertions, 443 deletions
diff --git a/contrib/python/hypothesis/py3/.dist-info/METADATA b/contrib/python/hypothesis/py3/.dist-info/METADATA index c900020993..0b48218e7a 100644 --- a/contrib/python/hypothesis/py3/.dist-info/METADATA +++ b/contrib/python/hypothesis/py3/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: hypothesis -Version: 6.98.3 +Version: 6.98.4 Summary: A library for property-based testing Home-page: https://hypothesis.works Author: David R. MacIver and Zac Hatfield-Dodds diff --git a/contrib/python/hypothesis/py3/hypothesis/extra/_patching.py b/contrib/python/hypothesis/py3/hypothesis/extra/_patching.py index 2660e04a90..8f53076d72 100644 --- a/contrib/python/hypothesis/py3/hypothesis/extra/_patching.py +++ b/contrib/python/hypothesis/py3/hypothesis/extra/_patching.py @@ -86,7 +86,7 @@ class AddExamplesCodemod(VisitorBasedCodemodCommand): # Codemod the failing examples to Call nodes usable as decorators self.fn_examples = { - k: tuple(self.__call_node_to_example_dec(ex, via) for ex, via in nodes) + k: tuple(d for x in nodes if (d := self.__call_node_to_example_dec(*x))) for k, nodes in fn_examples.items() } @@ -116,10 +116,13 @@ class AddExamplesCodemod(VisitorBasedCodemodCommand): args=[cst.Arg(cst.SimpleString(repr(via)))], ) if black: # pragma: no branch - pretty = black.format_str( - cst.Module([]).code_for_node(via), - mode=black.FileMode(line_length=self.line_length), - ) + try: + pretty = black.format_str( + cst.Module([]).code_for_node(via), + mode=black.FileMode(line_length=self.line_length), + ) + except ImportError: + return None # See https://github.com/psf/black/pull/4224 via = cst.parse_expression(pretty.strip()) return cst.Decorator(via) diff --git a/contrib/python/hypothesis/py3/hypothesis/vendor/tlds-alpha-by-domain.txt b/contrib/python/hypothesis/py3/hypothesis/vendor/tlds-alpha-by-domain.txt index c520fc063e..167e07e0e7 100644 --- a/contrib/python/hypothesis/py3/hypothesis/vendor/tlds-alpha-by-domain.txt +++ b/contrib/python/hypothesis/py3/hypothesis/vendor/tlds-alpha-by-domain.txt @@ -1,4 +1,4 @@ -# Version 2024020300, Last Updated Sat Feb 3 07:07:01 2024 UTC +# Version 2024021000, Last Updated Sat Feb 10 07:07:02 2024 UTC AAA AARP ABB @@ -260,7 +260,6 @@ COFFEE COLLEGE COLOGNE COM -COMCAST COMMBANK COMMUNITY COMPANY @@ -1274,7 +1273,6 @@ WTC WTF XBOX XEROX -XFINITY XIHUAN XIN XN--11B4C3D diff --git a/contrib/python/hypothesis/py3/hypothesis/version.py b/contrib/python/hypothesis/py3/hypothesis/version.py index 2dddb0e5f2..8a33e96d3d 100644 --- a/contrib/python/hypothesis/py3/hypothesis/version.py +++ b/contrib/python/hypothesis/py3/hypothesis/version.py @@ -8,5 +8,5 @@ # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -__version_info__ = (6, 98, 3) +__version_info__ = (6, 98, 4) __version__ = ".".join(map(str, __version_info__)) diff --git a/contrib/python/hypothesis/py3/ya.make b/contrib/python/hypothesis/py3/ya.make index d818e87ba6..bd45605d1c 100644 --- a/contrib/python/hypothesis/py3/ya.make +++ b/contrib/python/hypothesis/py3/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(6.98.3) +VERSION(6.98.4) LICENSE(MPL-2.0) diff --git a/contrib/python/setuptools/py3/.dist-info/METADATA b/contrib/python/setuptools/py3/.dist-info/METADATA index a12855b091..6abcbef24b 100644 --- a/contrib/python/setuptools/py3/.dist-info/METADATA +++ b/contrib/python/setuptools/py3/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: setuptools -Version: 69.0.3 +Version: 69.1.0 Summary: Easily download, build, install, upgrade, and uninstall Python packages Home-page: https://github.com/pypa/setuptools Author: Python Packaging Authority @@ -51,6 +51,7 @@ Requires-Dist: filelock >=3.4.0 ; extra == 'testing' Requires-Dist: ini2toml[lite] >=0.9 ; extra == 'testing' Requires-Dist: tomli-w >=1.0.0 ; extra == 'testing' Requires-Dist: pytest-timeout ; extra == 'testing' +Requires-Dist: pytest-home >=0.5 ; extra == 'testing' Provides-Extra: testing-integration Requires-Dist: pytest ; extra == 'testing-integration' Requires-Dist: pytest-xdist ; extra == 'testing-integration' @@ -63,11 +64,10 @@ Requires-Dist: jaraco.envs >=2.2 ; extra == 'testing-integration' Requires-Dist: build[virtualenv] >=1.0.3 ; extra == 'testing-integration' Requires-Dist: filelock >=3.4.0 ; extra == 'testing-integration' Requires-Dist: packaging >=23.1 ; extra == 'testing-integration' -Requires-Dist: pytest-black >=0.3.7 ; (platform_python_implementation != "PyPy") and extra == 'testing' Requires-Dist: pytest-cov ; (platform_python_implementation != "PyPy") and extra == 'testing' Requires-Dist: pytest-mypy >=0.9.1 ; (platform_python_implementation != "PyPy") and extra == 'testing' Requires-Dist: jaraco.develop >=7.21 ; (python_version >= "3.9" and sys_platform != "cygwin") and extra == 'testing' -Requires-Dist: pytest-ruff ; (sys_platform != "cygwin") and extra == 'testing' +Requires-Dist: pytest-ruff >=0.2.1 ; (sys_platform != "cygwin") and extra == 'testing' Requires-Dist: pytest-perf ; (sys_platform != "cygwin") and extra == 'testing' .. image:: https://img.shields.io/pypi/v/setuptools.svg @@ -83,14 +83,10 @@ Requires-Dist: pytest-perf ; (sys_platform != "cygwin") and extra == 'testing' :target: https://github.com/astral-sh/ruff :alt: Ruff -.. image:: https://img.shields.io/badge/code%20style-black-000000.svg - :target: https://github.com/psf/black - :alt: Code style: Black - .. image:: https://img.shields.io/readthedocs/setuptools/latest.svg :target: https://setuptools.pypa.io -.. image:: https://img.shields.io/badge/skeleton-2023-informational +.. image:: https://img.shields.io/badge/skeleton-2024-informational :target: https://blog.jaraco.com/skeleton .. image:: https://img.shields.io/codecov/c/github/pypa/setuptools/master.svg?logo=codecov&logoColor=white @@ -103,10 +99,9 @@ Requires-Dist: pytest-perf ; (sys_platform != "cygwin") and extra == 'testing' :target: https://discord.com/channels/803025117553754132/815945031150993468 :alt: Discord -See the `Installation Instructions -<https://packaging.python.org/installing/>`_ in the Python Packaging -User's Guide for instructions on installing, upgrading, and uninstalling -Setuptools. +See the `Quickstart <https://setuptools.pypa.io/en/latest/userguide/quickstart.html>`_ +and the `User's Guide <https://setuptools.pypa.io/en/latest/userguide/>`_ for +instructions on how to use Setuptools. Questions and comments should be directed to `GitHub Discussions <https://github.com/pypa/setuptools/discussions>`_. diff --git a/contrib/python/setuptools/py3/README.rst b/contrib/python/setuptools/py3/README.rst index 92c7dc6e87..eec6e35531 100644 --- a/contrib/python/setuptools/py3/README.rst +++ b/contrib/python/setuptools/py3/README.rst @@ -11,14 +11,10 @@ :target: https://github.com/astral-sh/ruff :alt: Ruff -.. image:: https://img.shields.io/badge/code%20style-black-000000.svg - :target: https://github.com/psf/black - :alt: Code style: Black - .. image:: https://img.shields.io/readthedocs/setuptools/latest.svg :target: https://setuptools.pypa.io -.. image:: https://img.shields.io/badge/skeleton-2023-informational +.. image:: https://img.shields.io/badge/skeleton-2024-informational :target: https://blog.jaraco.com/skeleton .. image:: https://img.shields.io/codecov/c/github/pypa/setuptools/master.svg?logo=codecov&logoColor=white @@ -31,10 +27,9 @@ :target: https://discord.com/channels/803025117553754132/815945031150993468 :alt: Discord -See the `Installation Instructions -<https://packaging.python.org/installing/>`_ in the Python Packaging -User's Guide for instructions on installing, upgrading, and uninstalling -Setuptools. +See the `Quickstart <https://setuptools.pypa.io/en/latest/userguide/quickstart.html>`_ +and the `User's Guide <https://setuptools.pypa.io/en/latest/userguide/>`_ for +instructions on how to use Setuptools. Questions and comments should be directed to `GitHub Discussions <https://github.com/pypa/setuptools/discussions>`_. diff --git a/contrib/python/setuptools/py3/_distutils_hack/__init__.py b/contrib/python/setuptools/py3/_distutils_hack/__init__.py index b951c2defd..4d3f09b0ae 100644 --- a/contrib/python/setuptools/py3/_distutils_hack/__init__.py +++ b/contrib/python/setuptools/py3/_distutils_hack/__init__.py @@ -3,16 +3,9 @@ import sys import os -is_pypy = '__pypy__' in sys.builtin_module_names - - def warn_distutils_present(): if 'distutils' not in sys.modules: return - if is_pypy and sys.version_info < (3, 7): - # PyPy for 3.6 unconditionally imports distutils, so bypass the warning - # https://foss.heptapod.net/pypy/pypy/-/blob/be829135bc0d758997b3566062999ee8b23872b4/lib-python/3/site.py#L250 - return import warnings warnings.warn( @@ -90,7 +83,7 @@ class DistutilsMetaFinder: # optimization: only consider top level modules and those # found in the CPython test suite. if path is not None and not fullname.startswith('test.'): - return + return None method_name = 'spec_for_{fullname}'.format(**locals()) method = getattr(self, method_name, lambda: None) @@ -98,7 +91,7 @@ class DistutilsMetaFinder: def spec_for_distutils(self): if self.is_cpython(): - return + return None import importlib import importlib.abc @@ -115,7 +108,7 @@ class DistutilsMetaFinder: # setuptools from the path but only after the hook # has been loaded. Ref #2980. # In either case, fall back to stdlib behavior. - return + return None class DistutilsLoader(importlib.abc.Loader): def create_module(self, spec): diff --git a/contrib/python/setuptools/py3/pkg_resources/__init__.py b/contrib/python/setuptools/py3/pkg_resources/__init__.py index f41d46c634..d83283ff1c 100644 --- a/contrib/python/setuptools/py3/pkg_resources/__init__.py +++ b/contrib/python/setuptools/py3/pkg_resources/__init__.py @@ -18,11 +18,16 @@ This module is deprecated. Users are directed to :mod:`importlib.resources`, """ import sys + +if sys.version_info < (3, 8): + raise RuntimeError("Python 3.8 or later is required") + import os import io import time import re import types +from typing import Protocol import zipfile import zipimport import warnings @@ -41,18 +46,10 @@ import inspect import ntpath import posixpath import importlib +import importlib.machinery from pkgutil import get_importer -try: - import _imp -except ImportError: - # Python 3.2 compatibility - import imp as _imp - -try: - FileExistsError -except NameError: - FileExistsError = OSError +import _imp # capture these to bypass sandboxing from os import utime @@ -68,14 +65,6 @@ except ImportError: from os import open as os_open from os.path import isdir, split -try: - import importlib.machinery as importlib_machinery - - # access attribute to force import under delayed import mechanisms. - importlib_machinery.__name__ -except ImportError: - importlib_machinery = None - from pkg_resources.extern.jaraco.text import ( yield_lines, drop_comment, @@ -91,9 +80,6 @@ __import__('pkg_resources.extern.packaging.requirements') __import__('pkg_resources.extern.packaging.markers') __import__('pkg_resources.extern.packaging.utils') -if sys.version_info < (3, 5): - raise RuntimeError("Python 3.5 or later is required") - # declare some globals that will be defined later to # satisfy the linters. require = None @@ -407,20 +393,18 @@ def get_provider(moduleOrReq): return _find_adapter(_provider_factories, loader)(module) -def _macos_vers(_cache=[]): - if not _cache: - version = platform.mac_ver()[0] - # fallback for MacPorts - if version == '': - plist = '/System/Library/CoreServices/SystemVersion.plist' - if os.path.exists(plist): - if hasattr(plistlib, 'readPlist'): - plist_content = plistlib.readPlist(plist) - if 'ProductVersion' in plist_content: - version = plist_content['ProductVersion'] - - _cache.append(version.split('.')) - return _cache[0] +@functools.lru_cache(maxsize=None) +def _macos_vers(): + version = platform.mac_ver()[0] + # fallback for MacPorts + if version == '': + plist = '/System/Library/CoreServices/SystemVersion.plist' + if os.path.exists(plist): + with open(plist, 'rb') as fh: + plist_content = plistlib.load(fh) + if 'ProductVersion' in plist_content: + version = plist_content['ProductVersion'] + return version.split('.') def _macos_arch(machine): @@ -546,54 +530,54 @@ def get_entry_info(dist, group, name): return get_distribution(dist).get_entry_info(group, name) -class IMetadataProvider: - def has_metadata(name): +class IMetadataProvider(Protocol): + def has_metadata(self, name): """Does the package's distribution contain the named metadata?""" - def get_metadata(name): + def get_metadata(self, name): """The named metadata resource as a string""" - def get_metadata_lines(name): + def get_metadata_lines(self, name): """Yield named metadata resource as list of non-blank non-comment lines Leading and trailing whitespace is stripped from each line, and lines with ``#`` as the first non-blank character are omitted.""" - def metadata_isdir(name): + def metadata_isdir(self, name): """Is the named metadata a directory? (like ``os.path.isdir()``)""" - def metadata_listdir(name): + def metadata_listdir(self, name): """List of metadata names in the directory (like ``os.listdir()``)""" - def run_script(script_name, namespace): + def run_script(self, script_name, namespace): """Execute the named script in the supplied namespace dictionary""" -class IResourceProvider(IMetadataProvider): +class IResourceProvider(IMetadataProvider, Protocol): """An object that provides access to package resources""" - def get_resource_filename(manager, resource_name): + def get_resource_filename(self, manager, resource_name): """Return a true filesystem path for `resource_name` `manager` must be an ``IResourceManager``""" - def get_resource_stream(manager, resource_name): + def get_resource_stream(self, manager, resource_name): """Return a readable file-like object for `resource_name` `manager` must be an ``IResourceManager``""" - def get_resource_string(manager, resource_name): + def get_resource_string(self, manager, resource_name): """Return a string containing the contents of `resource_name` `manager` must be an ``IResourceManager``""" - def has_resource(resource_name): + def has_resource(self, resource_name): """Does the package contain the named resource?""" - def resource_isdir(resource_name): + def resource_isdir(self, resource_name): """Is the named resource a directory? (like ``os.path.isdir()``)""" - def resource_listdir(resource_name): + def resource_listdir(self, resource_name): """List of resource names in the directory (like ``os.listdir()``)""" @@ -1143,8 +1127,7 @@ class Environment: None is returned instead. This method is a hook that allows subclasses to attempt other ways of obtaining a distribution before falling back to the `installer` argument.""" - if installer is not None: - return installer(requirement) + return installer(requirement) if installer else None def __iter__(self): """Yield the unique project names of the available distributions""" @@ -1734,7 +1717,7 @@ class DefaultProvider(EggProvider): 'SourcelessFileLoader', ) for name in loader_names: - loader_cls = getattr(importlib_machinery, name, type(None)) + loader_cls = getattr(importlib.machinery, name, type(None)) register_loader_type(loader_cls, cls) @@ -1895,7 +1878,7 @@ class ZipProvider(EggProvider): try: rename(tmpnam, real_path) - except os.error: + except OSError: if os.path.isfile(real_path): if self._is_current(real_path, zip_path): # the file became current since it was checked above, @@ -1908,7 +1891,7 @@ class ZipProvider(EggProvider): return real_path raise - except os.error: + except OSError: # report a user-friendly error manager.extraction_error() @@ -2229,7 +2212,7 @@ def resolve_egg_link(path): if hasattr(pkgutil, 'ImpImporter'): register_finder(pkgutil.ImpImporter, find_on_path) -register_finder(importlib_machinery.FileFinder, find_on_path) +register_finder(importlib.machinery.FileFinder, find_on_path) _declare_state('dict', _namespace_handlers={}) _declare_state('dict', _namespace_packages={}) @@ -2396,7 +2379,7 @@ if hasattr(pkgutil, 'ImpImporter'): register_namespace_handler(pkgutil.ImpImporter, file_ns_handler) register_namespace_handler(zipimport.zipimporter, file_ns_handler) -register_namespace_handler(importlib_machinery.FileFinder, file_ns_handler) +register_namespace_handler(importlib.machinery.FileFinder, file_ns_handler) def null_ns_handler(importer, path_item, packageName, module): @@ -2422,12 +2405,9 @@ def _cygwin_patch(filename): # pragma: nocover return os.path.abspath(filename) if sys.platform == 'cygwin' else filename -def _normalize_cached(filename, _cache={}): - try: - return _cache[filename] - except KeyError: - _cache[filename] = result = normalize_path(filename) - return result +@functools.lru_cache(maxsize=None) +def _normalize_cached(filename): + return normalize_path(filename) def _is_egg_path(path): @@ -2852,9 +2832,7 @@ class Distribution: def _get_version(self): lines = self._get_metadata(self.PKG_INFO) - version = _version_from_file(lines) - - return version + return _version_from_file(lines) def activate(self, path=None, replace=False): """Ensure distribution is importable on `path` (default=sys.path)""" @@ -2901,7 +2879,7 @@ class Distribution: def __dir__(self): return list( - set(super(Distribution, self).__dir__()) + set(super().__dir__()) | set(attr for attr in self._provider.__dir__() if not attr.startswith('_')) ) @@ -3168,7 +3146,7 @@ class RequirementParseError(packaging.requirements.InvalidRequirement): class Requirement(packaging.requirements.Requirement): def __init__(self, requirement_string): """DO NOT CALL THIS UNDOCUMENTED METHOD; use Requirement.parse()!""" - super(Requirement, self).__init__(requirement_string) + super().__init__(requirement_string) self.unsafe_name = self.name project_name = safe_name(self.name) self.project_name, self.key = project_name, project_name.lower() @@ -3229,6 +3207,7 @@ def _find_adapter(registry, ob): for t in types: if t in registry: return registry[t] + return None def ensure_directory(path): diff --git a/contrib/python/setuptools/py3/setuptools/_core_metadata.py b/contrib/python/setuptools/py3/setuptools/_core_metadata.py index 6c904c3c77..4bf3c7c947 100644 --- a/contrib/python/setuptools/py3/setuptools/_core_metadata.py +++ b/contrib/python/setuptools/py3/setuptools/_core_metadata.py @@ -3,6 +3,7 @@ Handling of Core Metadata for Python packages (including reading and writing). See: https://packaging.python.org/en/latest/specifications/core-metadata/ """ + import os import stat import textwrap diff --git a/contrib/python/setuptools/py3/setuptools/_normalization.py b/contrib/python/setuptools/py3/setuptools/_normalization.py index aa9274f093..8f211b8bfb 100644 --- a/contrib/python/setuptools/py3/setuptools/_normalization.py +++ b/contrib/python/setuptools/py3/setuptools/_normalization.py @@ -2,6 +2,7 @@ Helpers for normalization as expected in wheel/sdist/module file names and core metadata """ + import re from pathlib import Path from typing import Union @@ -119,6 +120,21 @@ def filename_component(value: str) -> str: return value.replace("-", "_").strip("_") +def filename_component_broken(value: str) -> str: + """ + Produce the incorrect filename component for compatibility. + + See pypa/setuptools#4167 for detailed analysis. + + TODO: replace this with filename_component after pip 24 is + nearly-ubiquitous. + + >>> filename_component_broken('foo_bar-baz') + 'foo-bar-baz' + """ + return value.replace('_', '-') + + def safer_name(value: str) -> str: """Like ``safe_name`` but can be used as filename component for wheel""" # See bdist_wheel.safer_name diff --git a/contrib/python/setuptools/py3/setuptools/_reqs.py b/contrib/python/setuptools/py3/setuptools/_reqs.py index 7d7130d50e..9f83437033 100644 --- a/contrib/python/setuptools/py3/setuptools/_reqs.py +++ b/contrib/python/setuptools/py3/setuptools/_reqs.py @@ -24,13 +24,11 @@ def parse_strings(strs: _StrOrIter) -> Iterator[str]: @overload -def parse(strs: _StrOrIter) -> Iterator[Requirement]: - ... +def parse(strs: _StrOrIter) -> Iterator[Requirement]: ... @overload -def parse(strs: _StrOrIter, parser: Callable[[str], _T]) -> Iterator[_T]: - ... +def parse(strs: _StrOrIter, parser: Callable[[str], _T]) -> Iterator[_T]: ... def parse(strs, parser=parse_req): diff --git a/contrib/python/setuptools/py3/setuptools/build_meta.py b/contrib/python/setuptools/py3/setuptools/build_meta.py index 6da80d70b8..0a0abfdae0 100644 --- a/contrib/python/setuptools/py3/setuptools/build_meta.py +++ b/contrib/python/setuptools/py3/setuptools/build_meta.py @@ -121,16 +121,16 @@ def _file_with_extension(directory, extension): raise ValueError( 'No distribution was found. Ensure that `setup.py` ' 'is not empty and that it calls `setup()`.' - ) + ) from None return file def _open_setup_script(setup_script): if not os.path.exists(setup_script): # Supply a default setup.py - return io.StringIO(u"from setuptools import setup; setup()") + return io.StringIO("from setuptools import setup; setup()") - return getattr(tokenize, 'open', open)(setup_script) + return tokenize.open(setup_script) @contextlib.contextmanager @@ -477,7 +477,7 @@ class _BuildMetaLegacyBackend(_BuildMetaBackend): sys.argv[0] = setup_script try: - super(_BuildMetaLegacyBackend, self).run_setup(setup_script=setup_script) + super().run_setup(setup_script=setup_script) finally: # While PEP 517 frontends should be calling each hook in a fresh # subprocess according to the standard (and thus it should not be diff --git a/contrib/python/setuptools/py3/setuptools/command/_requirestxt.py b/contrib/python/setuptools/py3/setuptools/command/_requirestxt.py index 32bae2c4b4..7b732b11ab 100644 --- a/contrib/python/setuptools/py3/setuptools/command/_requirestxt.py +++ b/contrib/python/setuptools/py3/setuptools/command/_requirestxt.py @@ -6,6 +6,7 @@ The ``requires.txt`` file has an specific format: See https://setuptools.pypa.io/en/latest/deprecated/python_eggs.html#requires-txt """ + import io from collections import defaultdict from itertools import filterfalse diff --git a/contrib/python/setuptools/py3/setuptools/command/bdist_egg.py b/contrib/python/setuptools/py3/setuptools/command/bdist_egg.py index bdece56bc9..3687efdf9c 100644 --- a/contrib/python/setuptools/py3/setuptools/command/bdist_egg.py +++ b/contrib/python/setuptools/py3/setuptools/command/bdist_egg.py @@ -232,9 +232,11 @@ class bdist_egg(Command): remove_tree(self.bdist_dir, dry_run=self.dry_run) # Add to 'Distribution.dist_files' so that the "upload" command works - getattr(self.distribution, 'dist_files', []).append( - ('bdist_egg', get_python_version(), self.egg_output) - ) + getattr(self.distribution, 'dist_files', []).append(( + 'bdist_egg', + get_python_version(), + self.egg_output, + )) def zap_pyfiles(self): log.info("Removing .py files from temporary directory") @@ -319,8 +321,7 @@ def walk_egg(egg_dir): if 'EGG-INFO' in dirs: dirs.remove('EGG-INFO') yield base, dirs, files - for bdf in walker: - yield bdf + yield from walker def analyze_egg(egg_dir, stubs): @@ -368,10 +369,7 @@ def scan_module(egg_dir, base, name, stubs): return True # Extension module pkg = base[len(egg_dir) + 1 :].replace(os.sep, '.') module = pkg + (pkg and '.' or '') + os.path.splitext(name)[0] - if sys.version_info < (3, 7): - skip = 12 # skip magic & date & file size - else: - skip = 16 # skip magic & reserved? & date & file size + skip = 16 # skip magic & reserved? & date & file size f = open(filename, 'rb') f.read(skip) code = marshal.load(f) @@ -404,14 +402,12 @@ def scan_module(egg_dir, base, name, stubs): def iter_symbols(code): """Yield names and strings used by `code` and its nested code objects""" - for name in code.co_names: - yield name + yield from code.co_names for const in code.co_consts: if isinstance(const, str): yield const elif isinstance(const, CodeType): - for name in iter_symbols(const): - yield name + yield from iter_symbols(const) def can_scan(): @@ -423,6 +419,7 @@ def can_scan(): "Please ask the author to include a 'zip_safe'" " setting (either True or False) in the package's setup.py" ) + return False # Attribute names of options for commands that might need to be convinced to diff --git a/contrib/python/setuptools/py3/setuptools/command/bdist_rpm.py b/contrib/python/setuptools/py3/setuptools/command/bdist_rpm.py index 30b7c23385..70ed6b6097 100644 --- a/contrib/python/setuptools/py3/setuptools/command/bdist_rpm.py +++ b/contrib/python/setuptools/py3/setuptools/command/bdist_rpm.py @@ -30,11 +30,10 @@ class bdist_rpm(orig.bdist_rpm): def _make_spec_file(self): spec = orig.bdist_rpm._make_spec_file(self) - spec = [ + return [ line.replace( "setup.py install ", "setup.py install --single-version-externally-managed ", ).replace("%setup", "%setup -n %{name}-%{unmangled_version}") for line in spec ] - return spec diff --git a/contrib/python/setuptools/py3/setuptools/command/build.py b/contrib/python/setuptools/py3/setuptools/command/build.py index 0f1d688e17..afda7e3be9 100644 --- a/contrib/python/setuptools/py3/setuptools/command/build.py +++ b/contrib/python/setuptools/py3/setuptools/command/build.py @@ -1,17 +1,8 @@ -import sys -from typing import TYPE_CHECKING, List, Dict +from typing import Dict, List, Protocol from distutils.command.build import build as _build from ..warnings import SetuptoolsDeprecationWarning -if sys.version_info >= (3, 8): - from typing import Protocol -elif TYPE_CHECKING: - from typing_extensions import Protocol -else: - from abc import ABC as Protocol - - _ORIGINAL_SUBCOMMANDS = {"build_py", "build_clib", "build_ext", "build_scripts"} diff --git a/contrib/python/setuptools/py3/setuptools/command/build_ext.py b/contrib/python/setuptools/py3/setuptools/command/build_ext.py index 9a80781cf4..780afe3aec 100644 --- a/contrib/python/setuptools/py3/setuptools/command/build_ext.py +++ b/contrib/python/setuptools/py3/setuptools/command/build_ext.py @@ -37,9 +37,9 @@ def _customize_compiler_for_shlib(compiler): tmp = _CONFIG_VARS.copy() try: # XXX Help! I don't have any idea whether these are right... - _CONFIG_VARS[ - 'LDSHARED' - ] = "gcc -Wl,-x -dynamiclib -undefined dynamic_lookup" + _CONFIG_VARS['LDSHARED'] = ( + "gcc -Wl,-x -dynamiclib -undefined dynamic_lookup" + ) _CONFIG_VARS['CCSHARED'] = " -dynamiclib" _CONFIG_VARS['SO'] = ".dylib" customize_compiler(compiler) @@ -76,6 +76,7 @@ def get_abi3_suffix(): return suffix elif suffix == '.pyd': # Windows return suffix + return None class build_ext(_build_ext): @@ -157,7 +158,7 @@ class build_ext(_build_ext): if fullname in self.ext_map: ext = self.ext_map[fullname] - use_abi3 = getattr(ext, 'py_limited_api') and get_abi3_suffix() + use_abi3 = ext.py_limited_api and get_abi3_suffix() if use_abi3: filename = filename[: -len(so_ext)] so_ext = get_abi3_suffix() @@ -341,33 +342,30 @@ class build_ext(_build_ext): if not self.dry_run: f = open(stub_file, 'w') f.write( - '\n'.join( - [ - "def __bootstrap__():", - " global __bootstrap__, __file__, __loader__", - " import sys, os, pkg_resources, importlib.util" - + if_dl(", dl"), - " __file__ = pkg_resources.resource_filename" - "(__name__,%r)" % os.path.basename(ext._file_name), - " del __bootstrap__", - " if '__loader__' in globals():", - " del __loader__", - if_dl(" old_flags = sys.getdlopenflags()"), - " old_dir = os.getcwd()", - " try:", - " os.chdir(os.path.dirname(__file__))", - if_dl(" sys.setdlopenflags(dl.RTLD_NOW)"), - " spec = importlib.util.spec_from_file_location(", - " __name__, __file__)", - " mod = importlib.util.module_from_spec(spec)", - " spec.loader.exec_module(mod)", - " finally:", - if_dl(" sys.setdlopenflags(old_flags)"), - " os.chdir(old_dir)", - "__bootstrap__()", - "", # terminal \n - ] - ) + '\n'.join([ + "def __bootstrap__():", + " global __bootstrap__, __file__, __loader__", + " import sys, os, pkg_resources, importlib.util" + if_dl(", dl"), + " __file__ = pkg_resources.resource_filename" + "(__name__,%r)" % os.path.basename(ext._file_name), + " del __bootstrap__", + " if '__loader__' in globals():", + " del __loader__", + if_dl(" old_flags = sys.getdlopenflags()"), + " old_dir = os.getcwd()", + " try:", + " os.chdir(os.path.dirname(__file__))", + if_dl(" sys.setdlopenflags(dl.RTLD_NOW)"), + " spec = importlib.util.spec_from_file_location(", + " __name__, __file__)", + " mod = importlib.util.module_from_spec(spec)", + " spec.loader.exec_module(mod)", + " finally:", + if_dl(" sys.setdlopenflags(old_flags)"), + " os.chdir(old_dir)", + "__bootstrap__()", + "", # terminal \n + ]) ) f.close() if compile: diff --git a/contrib/python/setuptools/py3/setuptools/command/build_py.py b/contrib/python/setuptools/py3/setuptools/command/build_py.py index cbdd05aab0..3f40b060b3 100644 --- a/contrib/python/setuptools/py3/setuptools/command/build_py.py +++ b/contrib/python/setuptools/py3/setuptools/command/build_py.py @@ -288,7 +288,7 @@ class build_py(orig.build_py): return list(unique_everseen(keepers)) @staticmethod - def _get_platform_patterns(spec, package, src_dir, extra_patterns=[]): + def _get_platform_patterns(spec, package, src_dir, extra_patterns=()): """ yield platform-specific path patterns (suitable for glob or fn_match) from a glob-based spec (such as diff --git a/contrib/python/setuptools/py3/setuptools/command/develop.py b/contrib/python/setuptools/py3/setuptools/command/develop.py index ea3e48e55c..d8c1b49b3d 100644 --- a/contrib/python/setuptools/py3/setuptools/command/develop.py +++ b/contrib/python/setuptools/py3/setuptools/command/develop.py @@ -5,6 +5,7 @@ import os import glob from setuptools.command.easy_install import easy_install +from setuptools import _normalization from setuptools import _path from setuptools import namespaces import setuptools @@ -52,7 +53,9 @@ class develop(namespaces.DevelopInstaller, easy_install): # pick up setup-dir .egg files only: no .egg-info self.package_index.scan(glob.glob('*.egg')) - egg_link_fn = ei.egg_name + '.egg-link' + egg_link_fn = ( + _normalization.filename_component_broken(ei.egg_name) + '.egg-link' + ) self.egg_link = os.path.join(self.install_dir, egg_link_fn) self.egg_base = ei.egg_base if self.egg_path is None: @@ -157,6 +160,8 @@ class develop(namespaces.DevelopInstaller, easy_install): script_text = strm.read() self.install_script(dist, script_name, script_text, script_path) + return None + def install_wrapper_scripts(self, dist): dist = VersionlessRequirement(dist) return easy_install.install_wrapper_scripts(self, dist) diff --git a/contrib/python/setuptools/py3/setuptools/command/dist_info.py b/contrib/python/setuptools/py3/setuptools/command/dist_info.py index 5ef322168c..f5061afaaf 100644 --- a/contrib/python/setuptools/py3/setuptools/command/dist_info.py +++ b/contrib/python/setuptools/py3/setuptools/command/dist_info.py @@ -5,7 +5,6 @@ As defined in the wheel specification import os import shutil -import sys from contextlib import contextmanager from distutils import log from distutils.core import Command @@ -77,7 +76,7 @@ class dist_info(Command): if requires_bkp: bkp_name = f"{dir_path}.__bkp__" _rm(bkp_name, ignore_errors=True) - _copy(dir_path, bkp_name, dirs_exist_ok=True, symlinks=True) + shutil.copytree(dir_path, bkp_name, dirs_exist_ok=True, symlinks=True) try: yield finally: @@ -103,9 +102,3 @@ class dist_info(Command): def _rm(dir_name, **opts): if os.path.isdir(dir_name): shutil.rmtree(dir_name, **opts) - - -def _copy(src, dst, **opts): - if sys.version_info < (3, 8): - opts.pop("dirs_exist_ok", None) - shutil.copytree(src, dst, **opts) diff --git a/contrib/python/setuptools/py3/setuptools/command/easy_install.py b/contrib/python/setuptools/py3/setuptools/command/easy_install.py index 5d6fd5ca71..cc0c409123 100644 --- a/contrib/python/setuptools/py3/setuptools/command/easy_install.py +++ b/contrib/python/setuptools/py3/setuptools/command/easy_install.py @@ -74,7 +74,7 @@ from pkg_resources import ( DEVELOP_DIST, ) import pkg_resources -from .. import py312compat +from ..compat import py311 from .._path import ensure_directory from ..extern.jaraco.text import yield_lines @@ -245,31 +245,26 @@ class easy_install(Command): self.config_vars = dict(sysconfig.get_config_vars()) - self.config_vars.update( - { - 'dist_name': self.distribution.get_name(), - 'dist_version': self.distribution.get_version(), - 'dist_fullname': self.distribution.get_fullname(), - 'py_version': py_version, - 'py_version_short': ( - f'{sys.version_info.major}.{sys.version_info.minor}' - ), - 'py_version_nodot': f'{sys.version_info.major}{sys.version_info.minor}', - 'sys_prefix': self.config_vars['prefix'], - 'sys_exec_prefix': self.config_vars['exec_prefix'], - # Only python 3.2+ has abiflags - 'abiflags': getattr(sys, 'abiflags', ''), - 'platlibdir': getattr(sys, 'platlibdir', 'lib'), - } - ) + self.config_vars.update({ + 'dist_name': self.distribution.get_name(), + 'dist_version': self.distribution.get_version(), + 'dist_fullname': self.distribution.get_fullname(), + 'py_version': py_version, + 'py_version_short': f'{sys.version_info.major}.{sys.version_info.minor}', + 'py_version_nodot': f'{sys.version_info.major}{sys.version_info.minor}', + 'sys_prefix': self.config_vars['prefix'], + 'sys_exec_prefix': self.config_vars['exec_prefix'], + # Only POSIX systems have abiflags + 'abiflags': getattr(sys, 'abiflags', ''), + # Only python 3.9+ has platlibdir + 'platlibdir': getattr(sys, 'platlibdir', 'lib'), + }) with contextlib.suppress(AttributeError): # only for distutils outside stdlib - self.config_vars.update( - { - 'implementation_lower': install._get_implementation().lower(), - 'implementation': install._get_implementation(), - } - ) + self.config_vars.update({ + 'implementation_lower': install._get_implementation().lower(), + 'implementation': install._get_implementation(), + }) # pypa/distutils#113 Python 3.9 compat self.config_vars.setdefault( @@ -668,7 +663,7 @@ class easy_install(Command): @contextlib.contextmanager def _tmpdir(self): - tmpdir = tempfile.mkdtemp(prefix=u"easy_install-") + tmpdir = tempfile.mkdtemp(prefix="easy_install-") try: # cast to str as workaround for #709 and #710 and #712 yield str(tmpdir) @@ -746,6 +741,7 @@ class easy_install(Command): for dist in dists: if dist in spec: return dist + return None def select_scheme(self, name): try: @@ -1028,9 +1024,9 @@ class easy_install(Command): f.close() script_dir = os.path.join(_egg_info, 'scripts') # delete entry-point scripts to avoid duping - self.delete_blockers( - [os.path.join(script_dir, args[0]) for args in ScriptWriter.get_args(dist)] - ) + self.delete_blockers([ + os.path.join(script_dir, args[0]) for args in ScriptWriter.get_args(dist) + ]) # Build .egg file from tmpdir bdist_egg.make_zipfile( egg_path, @@ -1433,24 +1429,20 @@ def get_site_dirs(): if sys.platform in ('os2emx', 'riscos'): sitedirs.append(os.path.join(prefix, "Lib", "site-packages")) elif os.sep == '/': - sitedirs.extend( - [ - os.path.join( - prefix, - "lib", - "python{}.{}".format(*sys.version_info), - "site-packages", - ), - os.path.join(prefix, "lib", "site-python"), - ] - ) - else: - sitedirs.extend( - [ + sitedirs.extend([ + os.path.join( prefix, - os.path.join(prefix, "lib", "site-packages"), - ] - ) + "lib", + "python{}.{}".format(*sys.version_info), + "site-packages", + ), + os.path.join(prefix, "lib", "site-python"), + ]) + else: + sitedirs.extend([ + prefix, + os.path.join(prefix, "lib", "site-packages"), + ]) if sys.platform != 'darwin': continue @@ -1482,9 +1474,7 @@ def get_site_dirs(): with contextlib.suppress(AttributeError): sitedirs.extend(site.getsitepackages()) - sitedirs = list(map(normalize_path, sitedirs)) - - return sitedirs + return list(map(normalize_path, sitedirs)) def expand_paths(inputs): # noqa: C901 # is too complex (11) # FIXME @@ -1678,9 +1668,11 @@ class PthDistributions(Environment): last_paths.remove(path) # also, re-check that all paths are still valid before saving them for path in self.paths[:]: - if path not in last_paths and not path.startswith( - ('import ', 'from ', '#') - ): + if path not in last_paths and not path.startswith(( + 'import ', + 'from ', + '#', + )): absolute_path = os.path.join(self.basedir, path) if not os.path.exists(absolute_path): self.paths.remove(path) @@ -1751,8 +1743,7 @@ class RewritePthDistributions(PthDistributions): @classmethod def _wrap_lines(cls, lines): yield cls.prelude - for line in lines: - yield line + yield from lines yield cls.postlude prelude = _one_liner( @@ -2032,7 +2023,7 @@ def chmod(path, mode): log.debug("changing mode of %s to %o", path, mode) try: _chmod(path, mode) - except os.error as e: + except OSError as e: log.debug("chmod failed: %s", e) @@ -2188,8 +2179,7 @@ class ScriptWriter: cls._ensure_safe_name(name) script_text = cls.template % locals() args = cls._get_script_args(type_, name, header, script_text) - for res in args: - yield res + yield from args @staticmethod def _ensure_safe_name(name): @@ -2339,7 +2329,7 @@ def load_launcher_manifest(name): def _rmtree(path, ignore_errors=False, onexc=auto_chmod): - return py312compat.shutil_rmtree(path, ignore_errors, onexc) + return py311.shutil_rmtree(path, ignore_errors, onexc) def current_umask(): diff --git a/contrib/python/setuptools/py3/setuptools/command/editable_wheel.py b/contrib/python/setuptools/py3/setuptools/command/editable_wheel.py index 79c839f8f0..8a4ae7928f 100644 --- a/contrib/python/setuptools/py3/setuptools/command/editable_wheel.py +++ b/contrib/python/setuptools/py3/setuptools/command/editable_wheel.py @@ -19,7 +19,7 @@ import traceback from contextlib import suppress from enum import Enum from inspect import cleandoc -from itertools import chain +from itertools import chain, starmap from pathlib import Path from tempfile import TemporaryDirectory from typing import ( @@ -30,6 +30,7 @@ from typing import ( List, Mapping, Optional, + Protocol, Tuple, TypeVar, Union, @@ -54,13 +55,6 @@ from .build_py import build_py as build_py_cls if TYPE_CHECKING: from wheel.wheelfile import WheelFile # noqa -if sys.version_info >= (3, 8): - from typing import Protocol -elif TYPE_CHECKING: - from typing_extensions import Protocol -else: - from abc import ABC as Protocol - _Path = Union[str, Path] _P = TypeVar("_P", bound=_Path) _logger = logging.getLogger(__name__) @@ -384,14 +378,13 @@ class editable_wheel(Command): class EditableStrategy(Protocol): - def __call__(self, wheel: "WheelFile", files: List[str], mapping: Dict[str, str]): - ... + def __call__( + self, wheel: "WheelFile", files: List[str], mapping: Dict[str, str] + ): ... - def __enter__(self): - ... + def __enter__(self): ... - def __exit__(self, _exc_type, _exc_value, _traceback): - ... + def __exit__(self, _exc_type, _exc_value, _traceback): ... class _StaticPth: @@ -401,7 +394,7 @@ class _StaticPth: self.path_entries = path_entries def __call__(self, wheel: "WheelFile", files: List[str], mapping: Dict[str, str]): - entries = "\n".join((str(p.resolve()) for p in self.path_entries)) + entries = "\n".join(str(p.resolve()) for p in self.path_entries) contents = _encode_pth(f"{entries}\n") wheel.writestr(f"__editable__.{self.name}.pth", contents) @@ -413,8 +406,7 @@ class _StaticPth: _logger.warning(msg + _LENIENT_WARNING) return self - def __exit__(self, _exc_type, _exc_value, _traceback): - ... + def __exit__(self, _exc_type, _exc_value, _traceback): ... class _LinkTree(_StaticPth): @@ -608,7 +600,7 @@ def _simple_layout( layout = {pkg: find_package_path(pkg, package_dir, project_dir) for pkg in packages} if not layout: return set(package_dir) in ({}, {""}) - parent = os.path.commonpath([_parent_path(k, v) for k, v in layout.items()]) + parent = os.path.commonpath(starmap(_parent_path, layout.items())) return all( _path.same_path(Path(parent, *key.split('.')), value) for key, value in layout.items() diff --git a/contrib/python/setuptools/py3/setuptools/command/egg_info.py b/contrib/python/setuptools/py3/setuptools/command/egg_info.py index 7c7f57aaf8..62d2feea9b 100644 --- a/contrib/python/setuptools/py3/setuptools/command/egg_info.py +++ b/contrib/python/setuptools/py3/setuptools/command/egg_info.py @@ -385,9 +385,8 @@ class FileList(_FileList): try: process_action = action_map[action] except KeyError: - raise DistutilsInternalError( - "this cannot happen: invalid action '{action!s}'".format(action=action), - ) + msg = f"Invalid MANIFEST.in: unknown action {action!r} in {line!r}" + raise DistutilsInternalError(msg) from None # OK, now we know that the action is valid and we have the # right number of words on the line for that action -- so we @@ -700,9 +699,9 @@ write_setup_requirements = _requirestxt.write_setup_requirements def write_toplevel_names(cmd, basename, filename): - pkgs = dict.fromkeys( - [k.split('.', 1)[0] for k in cmd.distribution.iter_distribution_names()] - ) + pkgs = dict.fromkeys([ + k.split('.', 1)[0] for k in cmd.distribution.iter_distribution_names() + ]) cmd.write_file("top-level names", filename, '\n'.join(sorted(pkgs)) + '\n') diff --git a/contrib/python/setuptools/py3/setuptools/command/install.py b/contrib/python/setuptools/py3/setuptools/command/install.py index 606cce9d89..b97a9b4713 100644 --- a/contrib/python/setuptools/py3/setuptools/command/install.py +++ b/contrib/python/setuptools/py3/setuptools/command/install.py @@ -71,6 +71,7 @@ class install(orig.install): # command without --root or --single-version-externally-managed self.path_file = None self.extra_dirs = '' + return None def run(self): # Explicit request for old-style install? Just do it @@ -83,6 +84,8 @@ class install(orig.install): else: self.do_egg_install() + return None + @staticmethod def _called_from_setup(run_frame): """ @@ -114,6 +117,8 @@ class install(orig.install): return caller_module == 'distutils.dist' and info.function == 'run_commands' + return False + def do_egg_install(self): easy_install = self.distribution.get_command_class('easy_install') diff --git a/contrib/python/setuptools/py3/setuptools/command/sdist.py b/contrib/python/setuptools/py3/setuptools/command/sdist.py index 5f45fb5dee..d455f44c5e 100644 --- a/contrib/python/setuptools/py3/setuptools/command/sdist.py +++ b/contrib/python/setuptools/py3/setuptools/command/sdist.py @@ -1,7 +1,6 @@ from distutils import log import distutils.command.sdist as orig import os -import sys import contextlib from itertools import chain @@ -14,8 +13,7 @@ _default_revctrl = list def walk_revctrl(dirname=''): """Find all files under revision control""" for ep in metadata.entry_points(group='setuptools.file_finders'): - for item in ep.load()(dirname): - yield item + yield from ep.load()(dirname) class sdist(orig.sdist): @@ -72,14 +70,6 @@ class sdist(orig.sdist): def initialize_options(self): orig.sdist.initialize_options(self) - self._default_to_gztar() - - def _default_to_gztar(self): - # only needed on Python prior to 3.6. - if sys.version_info >= (3, 6, 0, 'beta', 1): - return - self.formats = ['gztar'] - def make_distribution(self): """ Workaround for #516 @@ -106,7 +96,7 @@ class sdist(orig.sdist): yield finally: if orig_val is not NoValue: - setattr(os, 'link', orig_val) + os.link = orig_val def add_defaults(self): super().add_defaults() @@ -190,7 +180,7 @@ class sdist(orig.sdist): with open(self.manifest, 'rb') as fp: first_line = fp.readline() - return first_line != '# file GENERATED by distutils, do NOT edit\n'.encode() + return first_line != b'# file GENERATED by distutils, do NOT edit\n' def read_manifest(self): """Read the manifest file (named by 'self.manifest') and use it to diff --git a/contrib/python/setuptools/py3/setuptools/command/test.py b/contrib/python/setuptools/py3/setuptools/command/test.py index 5fce6660c0..0a128f2a7a 100644 --- a/contrib/python/setuptools/py3/setuptools/command/test.py +++ b/contrib/python/setuptools/py3/setuptools/command/test.py @@ -132,7 +132,7 @@ class test(Command): func() @contextlib.contextmanager - def project_on_sys_path(self, include_dists=[]): + def project_on_sys_path(self, include_dists=()): self.run_command('egg_info') # Build extensions in-place diff --git a/contrib/python/setuptools/py3/setuptools/compat/__init__.py b/contrib/python/setuptools/py3/setuptools/compat/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/contrib/python/setuptools/py3/setuptools/compat/__init__.py diff --git a/contrib/python/setuptools/py3/setuptools/compat/py310.py b/contrib/python/setuptools/py3/setuptools/compat/py310.py new file mode 100644 index 0000000000..f7d53d6de9 --- /dev/null +++ b/contrib/python/setuptools/py3/setuptools/compat/py310.py @@ -0,0 +1,10 @@ +import sys + + +__all__ = ['tomllib'] + + +if sys.version_info >= (3, 11): + import tomllib +else: # pragma: no cover + from setuptools.extern import tomli as tomllib diff --git a/contrib/python/setuptools/py3/setuptools/py312compat.py b/contrib/python/setuptools/py3/setuptools/compat/py311.py index 28175b1f75..28175b1f75 100644 --- a/contrib/python/setuptools/py3/setuptools/py312compat.py +++ b/contrib/python/setuptools/py3/setuptools/compat/py311.py diff --git a/contrib/python/setuptools/py3/setuptools/config/__init__.py b/contrib/python/setuptools/py3/setuptools/config/__init__.py index ffea394436..fcc7d008d6 100644 --- a/contrib/python/setuptools/py3/setuptools/config/__init__.py +++ b/contrib/python/setuptools/py3/setuptools/config/__init__.py @@ -1,6 +1,7 @@ """For backward compatibility, expose main functions from ``setuptools.config.setupcfg`` """ + from functools import wraps from typing import Callable, TypeVar, cast diff --git a/contrib/python/setuptools/py3/setuptools/config/_apply_pyprojecttoml.py b/contrib/python/setuptools/py3/setuptools/config/_apply_pyprojecttoml.py index 80318d5d0b..32fb00131e 100644 --- a/contrib/python/setuptools/py3/setuptools/config/_apply_pyprojecttoml.py +++ b/contrib/python/setuptools/py3/setuptools/config/_apply_pyprojecttoml.py @@ -7,6 +7,7 @@ need to be processed before being applied. **PRIVATE MODULE**: API reserved for setuptools internal usage only. """ + import logging import os from collections.abc import Mapping @@ -240,7 +241,7 @@ def _unify_entry_points(project_table: dict): if group # now we can skip empty groups } # Sometimes this will set `project["entry-points"] = {}`, and that is - # intentional (for reseting configurations that are missing `dynamic`). + # intentional (for resetting configurations that are missing `dynamic`). def _copy_command_options(pyproject: dict, dist: "Distribution", filename: _Path): @@ -408,7 +409,7 @@ _RESET_PREVIOUSLY_DEFINED: dict = { "scripts": {}, "gui-scripts": {}, "dependencies": [], - "optional-dependencies": [], + "optional-dependencies": {}, } @@ -423,7 +424,7 @@ class _MissingDynamic(SetuptoolsWarning): According to the spec (see the link below), however, setuptools CANNOT consider this value unless `{field}` is listed as `dynamic`. - https://packaging.python.org/en/latest/specifications/declaring-project-metadata/ + https://packaging.python.org/en/latest/specifications/pyproject-toml/#declaring-project-metadata-the-project-table To prevent this problem, you can list `{field}` under `dynamic` or alternatively remove the `[project]` table from your file and rely entirely on other means of diff --git a/contrib/python/setuptools/py3/setuptools/config/_validate_pyproject/error_reporting.py b/contrib/python/setuptools/py3/setuptools/config/_validate_pyproject/error_reporting.py index f78e4838fb..d44e290e36 100644 --- a/contrib/python/setuptools/py3/setuptools/config/_validate_pyproject/error_reporting.py +++ b/contrib/python/setuptools/py3/setuptools/config/_validate_pyproject/error_reporting.py @@ -24,7 +24,7 @@ _SKIP_DETAILS = ( "must not be there", ) -_NEED_DETAILS = {"anyOf", "oneOf", "anyOf", "contains", "propertyNames", "not", "items"} +_NEED_DETAILS = {"anyOf", "oneOf", "allOf", "contains", "propertyNames", "not", "items"} _CAMEL_CASE_SPLITTER = re.compile(r"\W+|([A-Z][^A-Z\W]*)") _IDENTIFIER = re.compile(r"^[\w_]+$", re.I) diff --git a/contrib/python/setuptools/py3/setuptools/config/expand.py b/contrib/python/setuptools/py3/setuptools/config/expand.py index 1bc71de546..b48fc1187e 100644 --- a/contrib/python/setuptools/py3/setuptools/config/expand.py +++ b/contrib/python/setuptools/py3/setuptools/config/expand.py @@ -17,6 +17,7 @@ functions among several configuration file formats. **PRIVATE MODULE**: API reserved for setuptools internal usage only. """ + import ast import importlib import os diff --git a/contrib/python/setuptools/py3/setuptools/config/pyprojecttoml.py b/contrib/python/setuptools/py3/setuptools/config/pyprojecttoml.py index 379ef222f9..9b9788eff4 100644 --- a/contrib/python/setuptools/py3/setuptools/config/pyprojecttoml.py +++ b/contrib/python/setuptools/py3/setuptools/config/pyprojecttoml.py @@ -8,6 +8,7 @@ To read project metadata, consider using For simple scenarios, you can also try parsing the file directly with the help of ``tomllib`` or ``tomli``. """ + import logging import os from contextlib import contextmanager @@ -28,10 +29,10 @@ _logger = logging.getLogger(__name__) def load_file(filepath: _Path) -> dict: - from setuptools.extern import tomli # type: ignore + from ..compat.py310 import tomllib with open(filepath, "rb") as file: - return tomli.load(file) + return tomllib.load(file) def validate(config: dict, filepath: _Path) -> bool: diff --git a/contrib/python/setuptools/py3/setuptools/config/setupcfg.py b/contrib/python/setuptools/py3/setuptools/config/setupcfg.py index 1a0e4154b9..a7f02714cb 100644 --- a/contrib/python/setuptools/py3/setuptools/config/setupcfg.py +++ b/contrib/python/setuptools/py3/setuptools/config/setupcfg.py @@ -8,6 +8,7 @@ To read project metadata, consider using For simple scenarios, you can also try parsing the file directly with the help of ``configparser``. """ + import contextlib import functools import os @@ -282,8 +283,8 @@ class ConfigHandler(Generic[Target]): try: current_value = getattr(target_obj, option_name) - except AttributeError: - raise KeyError(option_name) + except AttributeError as e: + raise KeyError(option_name) from e if current_value: # Already inhabited. Skipping. @@ -581,11 +582,11 @@ class ConfigMetadataHandler(ConfigHandler["DistributionMetadata"]): # accidentally include newlines and other unintended content try: Version(version) - except InvalidVersion: + except InvalidVersion as e: raise OptionError( f'Version loaded from {value} does not ' f'comply with PEP 440: {version}' - ) + ) from e return version @@ -694,9 +695,9 @@ class ConfigOptionsHandler(ConfigHandler["Distribution"]): valid_keys = ['where', 'include', 'exclude'] - find_kwargs = dict( - [(k, v) for k, v in section_data.items() if k in valid_keys and v] - ) + find_kwargs = dict([ + (k, v) for k, v in section_data.items() if k in valid_keys and v + ]) where = find_kwargs.get('where') if where is not None: diff --git a/contrib/python/setuptools/py3/setuptools/depends.py b/contrib/python/setuptools/py3/setuptools/depends.py index 42907d9bd4..c0ca84d404 100644 --- a/contrib/python/setuptools/py3/setuptools/depends.py +++ b/contrib/python/setuptools/py3/setuptools/depends.py @@ -159,6 +159,8 @@ def extract_constant(code, symbol, default=-1): else: const = default + return None + def _update_globals(): """ diff --git a/contrib/python/setuptools/py3/setuptools/discovery.py b/contrib/python/setuptools/py3/setuptools/discovery.py index 25962863b9..50a948750f 100644 --- a/contrib/python/setuptools/py3/setuptools/discovery.py +++ b/contrib/python/setuptools/py3/setuptools/discovery.py @@ -485,7 +485,7 @@ class ConfigDiscovery: """ if self.dist.metadata.name or self.dist.name: # get_name() is not reliable (can return "UNKNOWN") - return None + return log.debug("No `name` configuration, performing automatic discovery") diff --git a/contrib/python/setuptools/py3/setuptools/dist.py b/contrib/python/setuptools/py3/setuptools/dist.py index 222e8a7623..d5787ed474 100644 --- a/contrib/python/setuptools/py3/setuptools/dist.py +++ b/contrib/python/setuptools/py3/setuptools/dist.py @@ -87,7 +87,7 @@ def check_nsp(dist, attr, value): SetuptoolsDeprecationWarning.emit( "The namespace_packages parameter is deprecated.", "Please replace its usage with implicit namespaces (PEP 420).", - see_docs="references/keywords.html#keyword-namespace-packages" + see_docs="references/keywords.html#keyword-namespace-packages", # TODO: define due_date, it may break old packages that are no longer # maintained (e.g. sphinxcontrib extensions) when installed from source. # Warning officially introduced in May 2022, however the deprecation @@ -778,6 +778,8 @@ class Distribution(_Distribution): if p == package or p.startswith(pfx): return True + return False + def _exclude_misc(self, name, value): """Handle 'exclude()' for list/tuple attrs without a special handler""" if not isinstance(value, sequence): @@ -912,11 +914,9 @@ class Distribution(_Distribution): def iter_distribution_names(self): """Yield all packages, modules, and extension names in distribution""" - for pkg in self.packages or (): - yield pkg + yield from self.packages or () - for module in self.py_modules or (): - yield module + yield from self.py_modules or () for ext in self.ext_modules or (): if isinstance(ext, tuple): diff --git a/contrib/python/setuptools/py3/setuptools/glob.py b/contrib/python/setuptools/py3/setuptools/glob.py index 647b9bc6ed..a184c0b643 100644 --- a/contrib/python/setuptools/py3/setuptools/glob.py +++ b/contrib/python/setuptools/py3/setuptools/glob.py @@ -113,8 +113,7 @@ def glob0(dirname, basename): def glob2(dirname, pattern): assert _isrecursive(pattern) yield pattern[:0] - for x in _rlistdir(dirname): - yield x + yield from _rlistdir(dirname) # Recursively yields relative pathnames inside a literal directory. @@ -126,7 +125,7 @@ def _rlistdir(dirname): dirname = os.curdir try: names = os.listdir(dirname) - except os.error: + except OSError: return for x in names: yield x @@ -160,7 +159,7 @@ def escape(pathname): # Metacharacters do not work in the drive part and shouldn't be escaped. drive, pathname = os.path.splitdrive(pathname) if isinstance(pathname, bytes): - pathname = magic_check_bytes.sub(br'[\1]', pathname) + pathname = magic_check_bytes.sub(rb'[\1]', pathname) else: pathname = magic_check.sub(r'[\1]', pathname) return drive + pathname diff --git a/contrib/python/setuptools/py3/setuptools/installer.py b/contrib/python/setuptools/py3/setuptools/installer.py index e83f959a1b..a6aff723c2 100644 --- a/contrib/python/setuptools/py3/setuptools/installer.py +++ b/contrib/python/setuptools/py3/setuptools/installer.py @@ -107,10 +107,9 @@ def _fetch_build_egg_no_warn(dist, req): # noqa: C901 # is too complex (16) # dist_metadata = pkg_resources.PathMetadata( dist_location, os.path.join(dist_location, 'EGG-INFO') ) - dist = pkg_resources.Distribution.from_filename( + return pkg_resources.Distribution.from_filename( dist_location, metadata=dist_metadata ) - return dist def strip_marker(req): diff --git a/contrib/python/setuptools/py3/setuptools/monkey.py b/contrib/python/setuptools/py3/setuptools/monkey.py index 6c8a2f12f6..da0993506c 100644 --- a/contrib/python/setuptools/py3/setuptools/monkey.py +++ b/contrib/python/setuptools/py3/setuptools/monkey.py @@ -66,21 +66,6 @@ def patch_all(): # we can't patch distutils.cmd, alas distutils.core.Command = setuptools.Command - has_issue_12885 = sys.version_info <= (3, 5, 3) - - if has_issue_12885: - # fix findall bug in distutils (https://bugs.python.org/issue12885) - distutils.filelist.findall = setuptools.findall - - needs_warehouse = (3, 4) < sys.version_info < (3, 4, 6) or ( - 3, - 5, - ) < sys.version_info <= (3, 5, 3) - - if needs_warehouse: - warehouse = 'https://upload.pypi.org/legacy/' - distutils.config.PyPIRCCommand.DEFAULT_REPOSITORY = warehouse - _patch_distribution_metadata() # Install Distribution throughout the distutils @@ -130,7 +115,7 @@ def patch_func(replacement, target_mod, func_name): def get_unpatched_function(candidate): - return getattr(candidate, 'unpatched') + return candidate.unpatched def patch_for_msvc_specialized_compiler(): @@ -138,8 +123,7 @@ def patch_for_msvc_specialized_compiler(): Patch functions in distutils to use standalone Microsoft Visual C++ compilers. """ - # import late to avoid circular imports on Python < 3.5 - msvc = import_module('setuptools.msvc') + from . import msvc if platform.system() != 'Windows': # Compilers only available on Microsoft Windows diff --git a/contrib/python/setuptools/py3/setuptools/msvc.py b/contrib/python/setuptools/py3/setuptools/msvc.py index a910a64b68..53fe7b0de1 100644 --- a/contrib/python/setuptools/py3/setuptools/msvc.py +++ b/contrib/python/setuptools/py3/setuptools/msvc.py @@ -12,7 +12,6 @@ This may also support compilers shipped with compatible Visual Studio versions. """ import json -from io import open from os import listdir, pathsep from os.path import join, isfile, isdir, dirname from subprocess import CalledProcessError @@ -93,21 +92,17 @@ def _msvc14_find_vc2017(): # Workaround for `-requiresAny` (only available on VS 2017 > 15.6) with contextlib.suppress(CalledProcessError, OSError, UnicodeDecodeError): path = ( - subprocess.check_output( - [ - join( - root, "Microsoft Visual Studio", "Installer", "vswhere.exe" - ), - "-latest", - "-prerelease", - "-requires", - component, - "-property", - "installationPath", - "-products", - "*", - ] - ) + subprocess.check_output([ + join(root, "Microsoft Visual Studio", "Installer", "vswhere.exe"), + "-latest", + "-prerelease", + "-requires", + component, + "-property", + "installationPath", + "-products", + "*", + ]) .decode(encoding="mbcs", errors="strict") .strip() ) @@ -582,6 +577,7 @@ class RegistryInfo: finally: if bkey: closekey(bkey) + return None class SystemInfo: @@ -694,9 +690,9 @@ class SystemInfo: listdir(join(vs_path, r'VC\Tools\MSVC')) # Store version and path - vs_versions[ - self._as_float_version(state['installationVersion']) - ] = vs_path + vs_versions[self._as_float_version(state['installationVersion'])] = ( + vs_path + ) except (OSError, KeyError): # Skip if "state.json" file is missing or bad format @@ -828,6 +824,7 @@ class SystemInfo: return '8.1', '8.1a' elif self.vs_ver >= 14.0: return '10.0', '8.1' + return None @property def WindowsSdkLastVersion(self): @@ -919,6 +916,8 @@ class SystemInfo: if execpath: return execpath + return None + @property def FSharpInstallDir(self): """ @@ -951,6 +950,8 @@ class SystemInfo: if sdkdir: return sdkdir or '' + return None + @property def UniversalCRTSdkLastVersion(self): """ diff --git a/contrib/python/setuptools/py3/setuptools/namespaces.py b/contrib/python/setuptools/py3/setuptools/namespaces.py index 3332f864ae..e8f2941d45 100644 --- a/contrib/python/setuptools/py3/setuptools/namespaces.py +++ b/contrib/python/setuptools/py3/setuptools/namespaces.py @@ -42,12 +42,11 @@ class Installer: _nspkg_tmpl = ( "import sys, types, os", - "has_mfs = sys.version_info > (3, 5)", "p = os.path.join(%(root)s, *%(pth)r)", - "importlib = has_mfs and __import__('importlib.util')", - "has_mfs and __import__('importlib.machinery')", + "importlib = __import__('importlib.util')", + "__import__('importlib.machinery')", ( - "m = has_mfs and " + "m = " "sys.modules.setdefault(%(pkg)r, " "importlib.util.module_from_spec(" "importlib.machinery.PathFinder.find_spec(%(pkg)r, " diff --git a/contrib/python/setuptools/py3/setuptools/package_index.py b/contrib/python/setuptools/py3/setuptools/package_index.py index 3cedd5105c..271aa97f71 100644 --- a/contrib/python/setuptools/py3/setuptools/package_index.py +++ b/contrib/python/setuptools/py3/setuptools/package_index.py @@ -112,15 +112,13 @@ def egg_info_for_url(url): def distros_for_url(url, metadata=None): """Yield egg or source distribution objects that might be found at a URL""" base, fragment = egg_info_for_url(url) - for dist in distros_for_location(url, base, metadata): - yield dist + yield from distros_for_location(url, base, metadata) if fragment: match = EGG_FRAGMENT.match(fragment) if match: - for dist in interpret_distro_name( + yield from interpret_distro_name( url, match.group(1), metadata, precedence=CHECKOUT_DIST - ): - yield dist + ) def distros_for_location(location, basename, metadata=None): @@ -321,7 +319,7 @@ class PackageIndex(Environment): try: parse_version(dist.version) except Exception: - return + return None return super().add(dist) # FIXME: 'PackageIndex.process_url' is too complex (14) @@ -408,6 +406,7 @@ class PackageIndex(Environment): raise DistutilsError(msg % url) else: self.warn(msg, url) + return False def scan_egg_links(self, search_path): dirs = filter(os.path.isdir, search_path) @@ -516,7 +515,7 @@ class PackageIndex(Environment): if dist in requirement: return dist self.debug("%s does not match %s", requirement, dist) - return super(PackageIndex, self).obtain(requirement, installer) + return super().obtain(requirement, installer) def check_hash(self, checker, filename, tfp): """ @@ -650,6 +649,8 @@ class PackageIndex(Environment): if os.path.exists(dist.download_location): return dist + return None + if force_scan: self.prescan() self.find_packages(requirement) @@ -673,6 +674,7 @@ class PackageIndex(Environment): (source and "a source distribution of " or ""), requirement, ) + return None else: self.info("Best match: %s", dist) return dist.clone(location=dist.download_location) @@ -1036,6 +1038,7 @@ class PyPIConfig(configparser.RawConfigParser): for repository, cred in self.creds_by_repository.items(): if url.startswith(repository): return cred + return None def open_with_auth(url, opener=urllib.request.urlopen): diff --git a/contrib/python/setuptools/py3/setuptools/sandbox.py b/contrib/python/setuptools/py3/setuptools/sandbox.py index 017c897b86..7634b1320b 100644 --- a/contrib/python/setuptools/py3/setuptools/sandbox.py +++ b/contrib/python/setuptools/py3/setuptools/sandbox.py @@ -115,7 +115,7 @@ class UnpickleableException(Exception): class ExceptionSaver: """ - A Context Manager that will save an exception, serialized, and restore it + A Context Manager that will save an exception, serialize, and restore it later. """ @@ -124,7 +124,7 @@ class ExceptionSaver: def __exit__(self, type, exc, tb): if not exc: - return + return False # dump the exception self._saved = UnpickleableException.dump(type, exc) @@ -408,23 +408,21 @@ else: class DirectorySandbox(AbstractSandbox): """Restrict operations to a single subdirectory - pseudo-chroot""" - write_ops = dict.fromkeys( - [ - "open", - "chmod", - "chown", - "mkdir", - "remove", - "unlink", - "rmdir", - "utime", - "lchown", - "chroot", - "mkfifo", - "mknod", - "tempnam", - ] - ) + write_ops = dict.fromkeys([ + "open", + "chmod", + "chown", + "mkdir", + "remove", + "unlink", + "rmdir", + "utime", + "lchown", + "chroot", + "mkfifo", + "mknod", + "tempnam", + ]) _exception_patterns = [] "exempt writing to paths that match the pattern" diff --git a/contrib/python/setuptools/py3/setuptools/unicode_utils.py b/contrib/python/setuptools/py3/setuptools/unicode_utils.py index e84e65e3e1..d43dcc11f9 100644 --- a/contrib/python/setuptools/py3/setuptools/unicode_utils.py +++ b/contrib/python/setuptools/py3/setuptools/unicode_utils.py @@ -18,7 +18,7 @@ def decompose(path): def filesys_decode(path): """ Ensure that the given path is decoded, - NONE when no expected encoding works + ``None`` when no expected encoding works """ if isinstance(path, str): @@ -33,6 +33,8 @@ def filesys_decode(path): except UnicodeDecodeError: continue + return None + def try_encode(string, enc): "turn unicode encoding into a functional routine" diff --git a/contrib/python/setuptools/py3/setuptools/wheel.py b/contrib/python/setuptools/py3/setuptools/wheel.py index c6eabddc1f..9861b5cf1c 100644 --- a/contrib/python/setuptools/py3/setuptools/wheel.py +++ b/contrib/python/setuptools/py3/setuptools/wheel.py @@ -38,7 +38,7 @@ def _get_supported_tags(): def unpack(src_dir, dst_dir): - '''Move everything under `src_dir` to `dst_dir`, and delete the former.''' + """Move everything under `src_dir` to `dst_dir`, and delete the former.""" for dirpath, dirnames, filenames in os.walk(src_dir): subdir = os.path.relpath(dirpath, src_dir) for f in filenames: @@ -83,7 +83,7 @@ class Wheel: setattr(self, k, v) def tags(self): - '''List tags (py_version, abi, platform) supported by this wheel.''' + """List tags (py_version, abi, platform) supported by this wheel.""" return itertools.product( self.py_version.split('.'), self.abi.split('.'), @@ -91,7 +91,7 @@ class Wheel: ) def is_compatible(self): - '''Is the wheel compatible with the current platform?''' + """Is the wheel compatible with the current platform?""" return next((True for t in self.tags() if t in _get_supported_tags()), False) def egg_name(self): @@ -115,7 +115,7 @@ class Wheel: raise ValueError("unsupported wheel format. .dist-info not found") def install_as_egg(self, destination_eggdir): - '''Install wheel as an egg directory.''' + """Install wheel as an egg directory.""" with zipfile.ZipFile(self.filename) as zf: self._install_as_egg(destination_eggdir, zf) diff --git a/contrib/python/setuptools/py3/ya.make b/contrib/python/setuptools/py3/ya.make index 2cb2a8b26c..564d268875 100644 --- a/contrib/python/setuptools/py3/ya.make +++ b/contrib/python/setuptools/py3/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(69.0.3) +VERSION(69.1.0) LICENSE(MIT) @@ -213,6 +213,9 @@ PY_SRCS( setuptools/command/test.py setuptools/command/upload.py setuptools/command/upload_docs.py + setuptools/compat/__init__.py + setuptools/compat/py310.py + setuptools/compat/py311.py setuptools/config/__init__.py setuptools/config/_apply_pyprojecttoml.py setuptools/config/_validate_pyproject/__init__.py @@ -240,7 +243,6 @@ PY_SRCS( setuptools/msvc.py setuptools/namespaces.py setuptools/package_index.py - setuptools/py312compat.py setuptools/sandbox.py setuptools/unicode_utils.py setuptools/version.py diff --git a/yt/cpp/mapreduce/client/client.cpp b/yt/cpp/mapreduce/client/client.cpp index 22f1253e2e..4548577834 100644 --- a/yt/cpp/mapreduce/client/client.cpp +++ b/yt/cpp/mapreduce/client/client.cpp @@ -44,6 +44,8 @@ #include <yt/cpp/mapreduce/raw_client/raw_requests.h> #include <yt/cpp/mapreduce/raw_client/rpc_parameters_serialization.h> +#include <yt/yt/core/ytree/fluent.h> + #include <library/cpp/json/json_reader.h> #include <util/generic/algorithm.h> @@ -62,6 +64,32 @@ namespace NDetail { //////////////////////////////////////////////////////////////////////////////// +namespace { + +//////////////////////////////////////////////////////////////////////////////// + +THashMap<TString, TString> ParseProxyUrlAliasingRules(TString envConfig) +{ + if (envConfig.empty()) { + return {}; + } + return NYTree::ConvertTo<THashMap<TString, TString>>(NYson::TYsonString(envConfig)); +} + +void ApplyProxyUrlAliasingRules(TString& url) +{ + static auto rules = ParseProxyUrlAliasingRules(GetEnv("YT_PROXY_URL_ALIASING_CONFIG")); + if (auto ruleIt = rules.find(url); ruleIt != rules.end()) { + url = ruleIt->second; + } +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace + +//////////////////////////////////////////////////////////////////////////////// + TClientBase::TClientBase( const TClientContext& context, const TTransactionId& transactionId, @@ -1333,8 +1361,10 @@ TClientPtr CreateClientImpl( context.ProxyAddress = options.ProxyAddress_; context.ServerName = serverName; + if (context.ServerName.find('.') == TString::npos && - context.ServerName.find(':') == TString::npos) + context.ServerName.find(':') == TString::npos && + context.ServerName.find("localhost") == TString::npos) { context.ServerName += ".yt.yandex.net"; } @@ -1419,6 +1449,8 @@ IClientPtr CreateClientFromEnv(const TCreateClientOptions& options) ythrow yexception() << "YT_PROXY is not set"; } + NDetail::ApplyProxyUrlAliasingRules(serverName); + return NDetail::CreateClientImpl(serverName, options); } diff --git a/yt/yt/client/api/internal_client.cpp b/yt/yt/client/api/internal_client.cpp index 2fb7da7506..8c0e2f4ee5 100644 --- a/yt/yt/client/api/internal_client.cpp +++ b/yt/yt/client/api/internal_client.cpp @@ -8,18 +8,26 @@ void TSerializableHunkDescriptor::Register(TRegistrar registrar) { registrar.BaseClassParameter("chunk_id", &TThis::ChunkId); registrar.BaseClassParameter("erasure_codec", &TThis::ErasureCodec) - .Optional(); + .Default(NErasure::ECodec::None); registrar.BaseClassParameter("block_index", &TThis::BlockIndex); registrar.BaseClassParameter("block_offset", &TThis::BlockOffset); registrar.BaseClassParameter("block_size", &TThis::BlockSize) - .Optional(); + .Default(std::nullopt); registrar.BaseClassParameter("length", &TThis::Length); } -TSerializableHunkDescriptor::TSerializableHunkDescriptor(const THunkDescriptor& descriptor) - : THunkDescriptor(descriptor) +TSerializableHunkDescriptorPtr CreateSerializableHunkDescriptor(const THunkDescriptor& descriptor) { - ::NYT::NYTree::TYsonStructRegistry::Get()->InitializeStruct(this); + auto serializableDescriptor = New<TSerializableHunkDescriptor>(); + serializableDescriptor->ChunkId = descriptor.ChunkId; + serializableDescriptor->ErasureCodec = descriptor.ErasureCodec; + serializableDescriptor->BlockIndex = descriptor.BlockIndex; + serializableDescriptor->BlockOffset = descriptor.BlockOffset; + serializableDescriptor->BlockSize = descriptor.BlockSize; + serializableDescriptor->Length = descriptor.Length; + serializableDescriptor->Postprocess(); + + return serializableDescriptor; } //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/client/api/internal_client.h b/yt/yt/client/api/internal_client.h index 01fa29340b..61a2f300ab 100644 --- a/yt/yt/client/api/internal_client.h +++ b/yt/yt/client/api/internal_client.h @@ -29,8 +29,6 @@ class TSerializableHunkDescriptor , public NYTree::TYsonStruct { public: - TSerializableHunkDescriptor(const THunkDescriptor& descriptor); - REGISTER_YSON_STRUCT(TSerializableHunkDescriptor); static void Register(TRegistrar registrar); @@ -38,6 +36,8 @@ public: using TSerializableHunkDescriptorPtr = TIntrusivePtr<TSerializableHunkDescriptor>; +TSerializableHunkDescriptorPtr CreateSerializableHunkDescriptor(const THunkDescriptor& descriptor); + //////////////////////////////////////////////////////////////////////////////// struct TReadHunksOptions diff --git a/yt/yt/client/driver/internal_commands.cpp b/yt/yt/client/driver/internal_commands.cpp index 889737f91a..a69011714e 100644 --- a/yt/yt/client/driver/internal_commands.cpp +++ b/yt/yt/client/driver/internal_commands.cpp @@ -75,7 +75,7 @@ void TWriteHunksCommand::DoExecute(ICommandContextPtr context) std::vector<NApi::TSerializableHunkDescriptorPtr> serializableDescriptors; serializableDescriptors.reserve(descriptors.size()); for (const auto& descriptor : descriptors) { - serializableDescriptors.push_back(New<NApi::TSerializableHunkDescriptor>(descriptor)); + serializableDescriptors.push_back(CreateSerializableHunkDescriptor(descriptor)); } context->ProduceOutputValue(BuildYsonStringFluently() diff --git a/yt/yt/core/bus/tcp/client.cpp b/yt/yt/core/bus/tcp/client.cpp index 049727b860..6fe8a08856 100644 --- a/yt/yt/core/bus/tcp/client.cpp +++ b/yt/yt/core/bus/tcp/client.cpp @@ -175,16 +175,18 @@ public: auto id = TConnectionId::Create(); - YT_LOG_DEBUG("Connecting to server (Address: %v, ConnectionId: %v, MultiplexingBand: %v, EncryptionMode: %v)", + YT_LOG_DEBUG("Connecting to server (Address: %v, ConnectionId: %v, MultiplexingBand: %v, EncryptionMode: %v, VerificationMode: %v)", EndpointDescription_, id, options.MultiplexingBand, - Config_->EncryptionMode); + Config_->EncryptionMode, + Config_->VerificationMode); auto endpointAttributes = ConvertToAttributes(BuildYsonStringFluently() .BeginMap() .Items(*EndpointAttributes_) .Item("connection_id").Value(id) + .Item("connection_type").Value(EConnectionType::Client) .EndMap()); auto poller = TTcpDispatcher::TImpl::Get()->GetXferPoller(); diff --git a/yt/yt/core/bus/tcp/connection.cpp b/yt/yt/core/bus/tcp/connection.cpp index 84e0f050e7..a235ec6290 100644 --- a/yt/yt/core/bus/tcp/connection.cpp +++ b/yt/yt/core/bus/tcp/connection.cpp @@ -121,11 +121,12 @@ TTcpConnection::TTcpConnection( , UnixDomainSocketPath_(unixDomainSocketPath) , Handler_(std::move(handler)) , Poller_(std::move(poller)) - , LoggingTag_(Format("ConnectionId: %v, ConnectionType: %v, RemoteAddress: %v, EncryptionMode: %v", + , LoggingTag_(Format("ConnectionId: %v, ConnectionType: %v, RemoteAddress: %v, EncryptionMode: %v, VerificationMode: %v", Id_, ConnectionType_, EndpointDescription_, - Config_->EncryptionMode)) + Config_->EncryptionMode, + Config_->VerificationMode)) , Logger(BusLogger.WithTag(LoggingTag_.c_str())) , GenerateChecksums_(Config_->GenerateChecksums) , Socket_(socket) @@ -287,11 +288,12 @@ void TTcpConnection::TryEnqueueHandshake() } NProto::THandshake handshake; - ToProto(handshake.mutable_foreign_connection_id(), Id_); + ToProto(handshake.mutable_connection_id(), Id_); if (ConnectionType_ == EConnectionType::Client) { handshake.set_multiplexing_band(ToProto<int>(MultiplexingBand_.load())); } - handshake.set_encryption_mode(static_cast<int>(EncryptionMode_)); + handshake.set_encryption_mode(ToProto<int>(EncryptionMode_)); + handshake.set_verification_mode(ToProto<int>(VerificationMode_)); auto message = MakeHandshakeMessage(handshake); auto messageSize = GetByteSize(message); @@ -500,6 +502,9 @@ void TTcpConnection::Abort(const TError& error) // Construct a detailed error. YT_VERIFY(!error.IsOK()); auto detailedError = error << *EndpointAttributes_; + if (PeerAttributes_) { + detailedError <<= *PeerAttributes_; + } { auto guard = Guard(Lock_); @@ -752,6 +757,9 @@ void TTcpConnection::Terminate(const TError& error) // Construct a detailed error. YT_VERIFY(!error.IsOK()); auto detailedError = error << *EndpointAttributes_; + if (PeerAttributes_) { + detailedError <<= *PeerAttributes_; + } auto guard = Guard(Lock_); @@ -1005,7 +1013,7 @@ ssize_t TTcpConnection::DoReadSocket(char* buffer, size_t size) case ESslState::Established: { auto result = SSL_read(Ssl_.get(), buffer, size); if (PendingSslHandshake_ && result > 0) { - YT_LOG_DEBUG("TLS/SSL connection has been established by SSL_read (VerificationMode: %v)", VerificationMode_); + YT_LOG_DEBUG("TLS/SSL connection has been established by SSL_read"); PendingSslHandshake_ = false; ReadyPromise_.TrySet(); } @@ -1178,10 +1186,18 @@ bool TTcpConnection::OnHandshakePacketReceived() ? std::make_optional(FromProto<EMultiplexingBand>(handshake.multiplexing_band())) : std::nullopt; - YT_LOG_DEBUG("Handshake received (ForeignConnectionId: %v, MultiplexingBand: %v, ForeignEncryptionMode: %v)", - FromProto<TConnectionId>(handshake.foreign_connection_id()), - optionalMultiplexingBand, - static_cast<EEncryptionMode>(handshake.encryption_mode())); + PeerAttributes_ = ConvertToAttributes(BuildYsonStringFluently() + .BeginMap() + .Item("peer_connection_id").Value(FromProto<TConnectionId>(handshake.connection_id())) + .Item("peer_encryption_mode").Value(FromProto<EEncryptionMode>(handshake.encryption_mode())) + .Item("peer_verification_mode").Value(FromProto<EVerificationMode>(handshake.verification_mode())) + .EndMap()); + + YT_LOG_DEBUG("Handshake received (PeerConnectionId: %v, PeerEncryptionMode: %v, PeerVerificationMode: %v, MultiplexingBand: %v)", + PeerAttributes_->Get<TString>("peer_connection_id"), + PeerAttributes_->Get<TString>("peer_encryption_mode"), + PeerAttributes_->Get<TString>("peer_verification_mode"), + optionalMultiplexingBand); if (ConnectionType_ == EConnectionType::Server && optionalMultiplexingBand) { auto guard = Guard(Lock_); @@ -1197,7 +1213,7 @@ bool TTcpConnection::OnHandshakePacketReceived() TryEnqueueHandshake(); } - auto otherEncryptionMode = handshake.has_encryption_mode() ? static_cast<EEncryptionMode>(handshake.encryption_mode()) : EEncryptionMode::Disabled; + auto otherEncryptionMode = handshake.has_encryption_mode() ? FromProto<EEncryptionMode>(handshake.encryption_mode()) : EEncryptionMode::Disabled; if (EncryptionMode_ == EEncryptionMode::Required || otherEncryptionMode == EEncryptionMode::Required) { if (EncryptionMode_ == EEncryptionMode::Disabled || otherEncryptionMode == EEncryptionMode::Disabled) { @@ -1299,7 +1315,7 @@ ssize_t TTcpConnection::DoWriteFragments(const std::vector<struct iovec>& vec) YT_ASSERT(vec.size() == 1); auto result = SSL_write(Ssl_.get(), vec[0].iov_base, vec[0].iov_len); if (PendingSslHandshake_ && result > 0) { - YT_LOG_DEBUG("TLS/SSL connection has been established by SSL_write (VerificationMode: %v)", VerificationMode_); + YT_LOG_DEBUG("TLS/SSL connection has been established by SSL_write"); PendingSslHandshake_ = false; ReadyPromise_.TrySet(); } @@ -1857,7 +1873,7 @@ bool TTcpConnection::DoSslHandshake() auto result = SSL_do_handshake(Ssl_.get()); switch (SSL_get_error(Ssl_.get(), result)) { case SSL_ERROR_NONE: - YT_LOG_DEBUG("TLS/SSL connection has been established by SSL_do_handshake (VerificationMode %v)", VerificationMode_); + YT_LOG_DEBUG("TLS/SSL connection has been established by SSL_do_handshake"); MaxFragmentsPerWrite_ = 1; SslState_ = ESslState::Established; ReadyPromise_.TrySet(); @@ -1918,7 +1934,7 @@ void TTcpConnection::TryEstablishSslSession() return; } - YT_LOG_DEBUG("Starting TLS/SSL connection (VerificationMode: %v)", VerificationMode_); + YT_LOG_DEBUG("Starting TLS/SSL connection"); if (Config_->LoadCertsFromBusCertsDirectory && !TTcpDispatcher::TImpl::Get()->GetBusCertsDirectoryPath()) { Abort(TError(NBus::EErrorCode::SslError, "bus_certs_directory_path is not set in tcp_dispatcher config")); diff --git a/yt/yt/core/bus/tcp/connection.h b/yt/yt/core/bus/tcp/connection.h index 39c5dea4d2..0f5d758152 100644 --- a/yt/yt/core/bus/tcp/connection.h +++ b/yt/yt/core/bus/tcp/connection.h @@ -277,6 +277,8 @@ private: const EEncryptionMode EncryptionMode_; const EVerificationMode VerificationMode_; + NYTree::IAttributeDictionaryPtr PeerAttributes_; + size_t MaxFragmentsPerWrite_ = 256; void Open(); diff --git a/yt/yt/core/crypto/tls.cpp b/yt/yt/core/crypto/tls.cpp index a79a0c9458..f2e9ae036a 100644 --- a/yt/yt/core/crypto/tls.cpp +++ b/yt/yt/core/crypto/tls.cpp @@ -54,6 +54,24 @@ struct TSslContextImpl TSslContextImpl() { + Reset(); + } + + ~TSslContextImpl() + { + if (Ctx) { + SSL_CTX_free(Ctx); + } + if (ActiveCtx_) { + SSL_CTX_free(ActiveCtx_); + } + } + + void Reset() + { + if (Ctx) { + SSL_CTX_free(Ctx); + } #if OPENSSL_VERSION_NUMBER >= 0x10100000L Ctx = SSL_CTX_new(TLS_method()); if (!Ctx) { @@ -77,12 +95,37 @@ struct TSslContextImpl #endif } - ~TSslContextImpl() + void Commit() { - if (Ctx) { - SSL_CTX_free(Ctx); + SSL_CTX* oldCtx; + YT_ASSERT(Ctx); + { + auto guard = WriterGuard(Lock_); + oldCtx = ActiveCtx_; + ActiveCtx_ = Ctx; + Ctx = nullptr; + } + if (oldCtx) { + SSL_CTX_free(oldCtx); } } + + SSL* NewSsl() + { + auto guard = ReaderGuard(Lock_); + YT_ASSERT(ActiveCtx_); + return SSL_new(ActiveCtx_); + } + + bool IsActive(const SSL* ssl) + { + auto guard = ReaderGuard(Lock_); + return SSL_get_SSL_CTX(ssl) == ActiveCtx_; + } + +private: + YT_DECLARE_SPIN_LOCK(NThreading::TReaderWriterSpinLock, Lock_); + SSL_CTX* ActiveCtx_ = nullptr; }; DEFINE_REFCOUNTED_TYPE(TSslContextImpl) @@ -104,7 +147,7 @@ public: , Invoker_(CreateSerializedInvoker(poller->GetInvoker(), "crypto_tls_connection")) , Underlying_(std::move(connection)) { - Ssl_ = SSL_new(Ctx_->Ctx); + Ssl_ = Ctx_->NewSsl(); if (!Ssl_) { THROW_ERROR_EXCEPTION("SSL_new failed") << GetLastSslError(); @@ -124,6 +167,11 @@ public: OutputBuffer_ = TSharedMutableRef::Allocate<TTlsBufferTag>(TlsBufferSize); } + void SetHost(const TString& host) + { + SSL_set_tlsext_host_name(Ssl_, host.c_str()); + } + ~TTlsConnection() { SSL_free(Ssl_); @@ -496,12 +544,16 @@ public: , Poller_(std::move(poller)) { } - TFuture<IConnectionPtr> Dial(const TNetworkAddress& remote) override + TFuture<IConnectionPtr> Dial(const TNetworkAddress& remote, TRemoteContextPtr context) override { - return Underlying_->Dial(remote).Apply(BIND([ctx = Ctx_, poller = Poller_] (const IConnectionPtr& underlying) -> IConnectionPtr { - auto connection = New<TTlsConnection>(ctx, poller, underlying); - connection->StartClient(); - return connection; + return Underlying_->Dial(remote) + .Apply(BIND([ctx = Ctx_, poller = Poller_, context = std::move(context)](const IConnectionPtr& underlying) -> IConnectionPtr { + auto connection = New<TTlsConnection>(ctx, poller, underlying); + if (context != nullptr && context->Host != std::nullopt) { + connection->SetHost(*(context->Host)); + } + connection->StartClient(); + return connection; })); } @@ -558,6 +610,22 @@ TSslContext::TSslContext() : Impl_(New<TSslContextImpl>()) { } +void TSslContext::Reset() +{ + Impl_->Reset(); +} + +void TSslContext::Commit(TInstant time) +{ + CommitTime_ = time; + Impl_->Commit(); +} + +TInstant TSslContext::GetCommitTime() +{ + return CommitTime_; +} + void TSslContext::UseBuiltinOpenSslX509Store() { SSL_CTX_set_cert_store(Impl_->Ctx, GetBuiltinOpenSslX509Store().Release()); diff --git a/yt/yt/core/crypto/tls.h b/yt/yt/core/crypto/tls.h index 5844d3f021..bb9f85503f 100644 --- a/yt/yt/core/crypto/tls.h +++ b/yt/yt/core/crypto/tls.h @@ -20,6 +20,10 @@ class TSslContext public: TSslContext(); + void Reset(); + void Commit(TInstant time = TInstant::Zero()); + TInstant GetCommitTime(); + void UseBuiltinOpenSslX509Store(); void SetCipherList(const TString& list); @@ -48,6 +52,7 @@ public: private: const TIntrusivePtr<TSslContextImpl> Impl_; + TInstant CommitTime_; }; DEFINE_REFCOUNTED_TYPE(TSslContext) diff --git a/yt/yt/core/crypto/unittests/tls_ut.cpp b/yt/yt/core/crypto/unittests/tls_ut.cpp index 2981b24350..288994abaf 100644 --- a/yt/yt/core/crypto/unittests/tls_ut.cpp +++ b/yt/yt/core/crypto/unittests/tls_ut.cpp @@ -37,6 +37,7 @@ public: Context->AddCertificate(TestCertificate); Context->AddPrivateKey(TestCertificate); + Context->Commit(); Poller = CreateThreadPoolPoller(2, "TlsTest"); } @@ -78,7 +79,10 @@ TEST_F(TTlsTest, SimplePingPong) config->SetDefaults(); auto dialer = Context->CreateDialer(config, Poller, NetLogger); - auto asyncFirstSide = dialer->Dial(listener->GetAddress()); + auto context = New<TRemoteContext>(); + context->Host = "localhost"; + + auto asyncFirstSide = dialer->Dial(listener->GetAddress(), context); auto asyncSecondSide = listener->Accept(); auto firstSide = asyncFirstSide.Get().ValueOrThrow(); @@ -106,6 +110,7 @@ TEST(TTlsTestWithoutFixture, LoadCertificateChain) auto grpcLock = NRpc::NGrpc::TDispatcher::Get()->GetLibraryLock(); auto context = New<TSslContext>(); context->AddCertificateChain(TestCertificateChain); + context->Commit(); } //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/core/http/client.cpp b/yt/yt/core/http/client.cpp index 49ab4286f7..b6eb5181be 100644 --- a/yt/yt/core/http/client.cpp +++ b/yt/yt/core/http/client.cpp @@ -126,11 +126,15 @@ private: return TNetworkAddress(address, parsedUrl.Port.value_or(GetDefaultPort(parsedUrl))); } - std::pair<THttpOutputPtr, THttpInputPtr> OpenHttp(const TNetworkAddress& address) + std::pair<THttpOutputPtr, THttpInputPtr> OpenHttp(const TUrlRef& urlRef) { + auto context = New<TRemoteContext>(); + context->Host = urlRef.Host; + auto address = GetAddress(urlRef); + // TODO(aleexfi): Enable connection pool by default if (Config_->MaxIdleConnections == 0) { - auto connection = WaitFor(Dialer_->Dial(address)).ValueOrThrow(); + auto connection = WaitFor(Dialer_->Dial(address, std::move(context))).ValueOrThrow(); auto input = New<THttpInput>( connection, @@ -146,7 +150,7 @@ private: return {std::move(output), std::move(input)}; } else { - auto connection = WaitFor(ConnectionPool_->Connect(address)).ValueOrThrow(); + auto connection = WaitFor(ConnectionPool_->Connect(address, std::move(context))).ValueOrThrow(); auto reuseSharedState = New<NDetail::TReusableConnectionState>(connection, ConnectionPool_); @@ -239,8 +243,8 @@ private: THttpInputPtr response; auto urlRef = ParseUrl(url); - auto address = GetAddress(urlRef); - std::tie(request, response) = OpenHttp(address); + + std::tie(request, response) = OpenHttp(urlRef); request->SetHost(urlRef.Host, urlRef.PortStr); if (headers) { diff --git a/yt/yt/core/http/connection_pool.cpp b/yt/yt/core/http/connection_pool.cpp index 9cb971b8ab..3684bda591 100644 --- a/yt/yt/core/http/connection_pool.cpp +++ b/yt/yt/core/http/connection_pool.cpp @@ -52,7 +52,9 @@ TConnectionPool::~TConnectionPool() YT_UNUSED_FUTURE(ExpiredConnectionsCollector_->Stop()); } -TFuture<IConnectionPtr> TConnectionPool::Connect(const TNetworkAddress& address) +TFuture<IConnectionPtr> TConnectionPool::Connect( + const TNetworkAddress& address, + TRemoteContextPtr context) { { auto guard = Guard(SpinLock_); @@ -64,7 +66,7 @@ TFuture<IConnectionPtr> TConnectionPool::Connect(const TNetworkAddress& address) } } - return Dialer_->Dial(address); + return Dialer_->Dial(address, std::move(context)); } void TConnectionPool::Release(const IConnectionPtr& connection) diff --git a/yt/yt/core/http/connection_pool.h b/yt/yt/core/http/connection_pool.h index b7c06245bd..026626967f 100644 --- a/yt/yt/core/http/connection_pool.h +++ b/yt/yt/core/http/connection_pool.h @@ -38,7 +38,9 @@ public: ~TConnectionPool(); - TFuture<NNet::IConnectionPtr> Connect(const NNet::TNetworkAddress& address); + TFuture<NNet::IConnectionPtr> Connect( + const NNet::TNetworkAddress& address, + NNet::TRemoteContextPtr context = nullptr); void Release(const NNet::IConnectionPtr& connection); diff --git a/yt/yt/core/https/client.cpp b/yt/yt/core/https/client.cpp index 2f4b415b89..8ce559a583 100644 --- a/yt/yt/core/https/client.cpp +++ b/yt/yt/core/https/client.cpp @@ -120,6 +120,7 @@ IClientPtr CreateClient( } else { sslContext->UseBuiltinOpenSslX509Store(); } + sslContext->Commit(); auto tlsDialer = sslContext->CreateDialer( New<TDialerConfig>(), diff --git a/yt/yt/core/https/config.cpp b/yt/yt/core/https/config.cpp index 49c59d5258..c41fde9624 100644 --- a/yt/yt/core/https/config.cpp +++ b/yt/yt/core/https/config.cpp @@ -10,6 +10,8 @@ void TServerCredentialsConfig::Register(TRegistrar registrar) .Optional(); registrar.Parameter("cert_chain", &TThis::CertChain) .Optional(); + registrar.Parameter("update_period", &TThis::UpdatePeriod) + .Optional(); } //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/core/https/config.h b/yt/yt/core/https/config.h index fe4decc26e..eb6f010008 100644 --- a/yt/yt/core/https/config.h +++ b/yt/yt/core/https/config.h @@ -16,6 +16,7 @@ class TServerCredentialsConfig public: NCrypto::TPemBlobConfigPtr PrivateKey; NCrypto::TPemBlobConfigPtr CertChain; + TDuration UpdatePeriod; REGISTER_YSON_STRUCT(TServerCredentialsConfig); diff --git a/yt/yt/core/https/server.cpp b/yt/yt/core/https/server.cpp index e53f132c04..96fd155390 100644 --- a/yt/yt/core/https/server.cpp +++ b/yt/yt/core/https/server.cpp @@ -2,15 +2,23 @@ #include "config.h" #include <yt/yt/core/http/server.h> +#include <yt/yt/core/http/private.h> #include <yt/yt/core/crypto/tls.h> +#include <yt/yt/core/logging/log.h> + +#include <yt/yt/core/misc/fs.h> + #include <yt/yt/core/net/address.h> #include <yt/yt/core/concurrency/poller.h> +#include <yt/yt/core/concurrency/periodic_executor.h> namespace NYT::NHttps { +static const auto& Logger = NHttp::HttpLogger; + using namespace NNet; using namespace NHttp; using namespace NCrypto; @@ -22,8 +30,9 @@ class TServer : public IServer { public: - explicit TServer(IServerPtr underlying) + explicit TServer(IServerPtr underlying, TPeriodicExecutorPtr certificateUpdater) : Underlying_(std::move(underlying)) + , CertificateUpdater_(certificateUpdater) { } void AddHandler( @@ -42,12 +51,18 @@ public: void Start() override { Underlying_->Start(); + if (CertificateUpdater_) { + CertificateUpdater_->Start(); + } } //! Stops the server. void Stop() override { Underlying_->Stop(); + if (CertificateUpdater_) { + YT_UNUSED_FUTURE(CertificateUpdater_->Stop()); + } } void SetPathMatcher(const IRequestPathMatcherPtr& matcher) override @@ -62,28 +77,66 @@ public: private: const IServerPtr Underlying_; + const TPeriodicExecutorPtr CertificateUpdater_; }; -IServerPtr CreateServer( - const TServerConfigPtr& config, - const IPollerPtr& poller, - const IPollerPtr& acceptor) +static void ApplySslConfig(const TSslContextPtr& sslContext, const TServerCredentialsConfigPtr& sslConfig) { - auto sslContext = New<TSslContext>(); - if (config->Credentials->CertChain->FileName) { - sslContext->AddCertificateChainFromFile(*config->Credentials->CertChain->FileName); - } else if (config->Credentials->CertChain->Value) { - sslContext->AddCertificateChain(*config->Credentials->CertChain->Value); + if (sslConfig->CertChain->FileName) { + sslContext->AddCertificateChainFromFile(*sslConfig->CertChain->FileName); + } else if (sslConfig->CertChain->Value) { + sslContext->AddCertificateChain(*sslConfig->CertChain->Value); } else { YT_ABORT(); } - if (config->Credentials->PrivateKey->FileName) { - sslContext->AddPrivateKeyFromFile(*config->Credentials->PrivateKey->FileName); - } else if (config->Credentials->PrivateKey->Value) { - sslContext->AddPrivateKey(*config->Credentials->PrivateKey->Value); + if (sslConfig->PrivateKey->FileName) { + sslContext->AddPrivateKeyFromFile(*sslConfig->PrivateKey->FileName); + } else if (sslConfig->PrivateKey->Value) { + sslContext->AddPrivateKey(*sslConfig->PrivateKey->Value); } else { YT_ABORT(); } +} + +IServerPtr CreateServer( + const TServerConfigPtr& config, + const IPollerPtr& poller, + const IPollerPtr& acceptor) +{ + auto sslContext = New<TSslContext>(); + ApplySslConfig(sslContext, config->Credentials); + sslContext->Commit(); + + auto sslConfig = config->Credentials; + TPeriodicExecutorPtr certificateUpdater; + if (sslConfig->UpdatePeriod && + sslConfig->CertChain->FileName && + sslConfig->PrivateKey->FileName) + { + certificateUpdater = New<TPeriodicExecutor>( + poller->GetInvoker(), + BIND([=, serverName = config->ServerName] { + try { + auto modificationTime = Max( + NFS::GetPathStatistics(*sslConfig->CertChain->FileName).ModificationTime, + NFS::GetPathStatistics(*sslConfig->PrivateKey->FileName).ModificationTime); + + // Detect fresh and stable updates. + if (modificationTime > sslContext->GetCommitTime() && + modificationTime + sslConfig->UpdatePeriod <= TInstant::Now()) + { + YT_LOG_INFO("Updating TLS certificates (ServerName: %v, ModificationTime: %v)", serverName, modificationTime); + sslContext->Reset(); + ApplySslConfig(sslContext, sslConfig); + sslContext->Commit(modificationTime); + YT_LOG_INFO("TLS certificates updated (ServerName: %v)", serverName); + } + } catch (const std::exception& ex) { + YT_LOG_WARNING(ex, "Unexpected exception while updating TLS certificates (ServerName: %v)", serverName); + } + }), + sslConfig->UpdatePeriod); + } auto address = TNetworkAddress::CreateIPv6Any(config->Port); auto tlsListener = sslContext->CreateListener(address, poller, acceptor); @@ -92,7 +145,7 @@ IServerPtr CreateServer( configCopy->IsHttps = true; auto httpServer = NHttp::CreateServer(configCopy, tlsListener, poller, acceptor); - return New<TServer>(std::move(httpServer)); + return New<TServer>(std::move(httpServer), std::move(certificateUpdater)); } IServerPtr CreateServer(const TServerConfigPtr& config, const IPollerPtr& poller) diff --git a/yt/yt/core/net/dialer.cpp b/yt/yt/core/net/dialer.cpp index 1a23091b91..c27489d23b 100644 --- a/yt/yt/core/net/dialer.cpp +++ b/yt/yt/core/net/dialer.cpp @@ -85,7 +85,9 @@ public: , Poller_(std::move(poller)) { } - TFuture<IConnectionPtr> Dial(const TNetworkAddress& remote) override + TFuture<IConnectionPtr> Dial( + const TNetworkAddress& remote, + TRemoteContextPtr /*context*/) override { auto session = New<TDialSession>( remote, diff --git a/yt/yt/core/net/dialer.h b/yt/yt/core/net/dialer.h index b5a6ce3b2a..8dbd67cc43 100644 --- a/yt/yt/core/net/dialer.h +++ b/yt/yt/core/net/dialer.h @@ -14,12 +14,25 @@ namespace NYT::NNet { //////////////////////////////////////////////////////////////////////////////// +//! Сontext that is passed to the Dialer. + +struct TRemoteContext + : public TRefCounted +{ + //! Host is used for TlsDialer. + std::optional<TString> Host; +}; + +DEFINE_REFCOUNTED_TYPE(TRemoteContext) + //! Dialer establishes connection to a (resolved) network address. struct IDialer : public virtual TRefCounted { - virtual TFuture<IConnectionPtr> Dial(const TNetworkAddress& remote) = 0; + virtual TFuture<IConnectionPtr> Dial( + const TNetworkAddress& remote, + TRemoteContextPtr context = nullptr) = 0; }; DEFINE_REFCOUNTED_TYPE(IDialer) diff --git a/yt/yt/core/net/mock/dialer.cpp b/yt/yt/core/net/mock/dialer.cpp index 6e30a087a5..4b3582b1b5 100644 --- a/yt/yt/core/net/mock/dialer.cpp +++ b/yt/yt/core/net/mock/dialer.cpp @@ -7,7 +7,7 @@ namespace NYT::NNet { TDialerMock::TDialerMock(IDialerPtr underlying) : Underlying_(std::move(underlying)) { - ON_CALL(*this, Dial).WillByDefault([this] (const TNetworkAddress& address) { + ON_CALL(*this, Dial).WillByDefault([this] (const TNetworkAddress& address, TRemoteContextPtr /*context*/) { return Underlying_->Dial(address); }); } diff --git a/yt/yt/core/net/mock/dialer.h b/yt/yt/core/net/mock/dialer.h index 707a7b3dad..10a9237f09 100644 --- a/yt/yt/core/net/mock/dialer.h +++ b/yt/yt/core/net/mock/dialer.h @@ -14,7 +14,7 @@ class TDialerMock public: explicit TDialerMock(IDialerPtr underlying); - MOCK_METHOD(TFuture<IConnectionPtr>, Dial, (const TNetworkAddress& remote), (override)); + MOCK_METHOD(TFuture<IConnectionPtr>, Dial, (const TNetworkAddress& remote, TRemoteContextPtr context), (override)); private: const IDialerPtr Underlying_; diff --git a/yt/yt/core/net/public.h b/yt/yt/core/net/public.h index 05d3ec03de..42566ad7fa 100644 --- a/yt/yt/core/net/public.h +++ b/yt/yt/core/net/public.h @@ -17,6 +17,7 @@ DECLARE_REFCOUNTED_STRUCT(IPacketConnection) DECLARE_REFCOUNTED_STRUCT(IConnectionReader) DECLARE_REFCOUNTED_STRUCT(IConnectionWriter) DECLARE_REFCOUNTED_STRUCT(IListener) +DECLARE_REFCOUNTED_STRUCT(TRemoteContext) DECLARE_REFCOUNTED_STRUCT(IDialer) DECLARE_REFCOUNTED_STRUCT(IAsyncDialer) DECLARE_REFCOUNTED_STRUCT(IAsyncDialerSession) diff --git a/yt/yt/library/program/config.h b/yt/yt/library/program/config.h index 10b6fd5b06..64723698f2 100644 --- a/yt/yt/library/program/config.h +++ b/yt/yt/library/program/config.h @@ -29,7 +29,6 @@ #include <library/cpp/yt/stockpile/stockpile.h> - namespace NYT { //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt_proto/yt/client/chunk_client/proto/chunk_meta.proto b/yt/yt_proto/yt/client/chunk_client/proto/chunk_meta.proto index 874ec93151..e6aad0b3c0 100644 --- a/yt/yt_proto/yt/client/chunk_client/proto/chunk_meta.proto +++ b/yt/yt_proto/yt/client/chunk_client/proto/chunk_meta.proto @@ -146,11 +146,10 @@ message TStripedErasurePlacementExt // Sizes of the original blocks. repeated int64 block_sizes = 3; - // Some of the input blocks can be padded to fit into the segment. - repeated int64 block_padding_sizes = 5; - // Checksums of the original blocks. repeated fixed64 block_checksums = 4; + + reserved 5; } // TChunkMeta is stored in *.meta files on data nodes diff --git a/yt/yt_proto/yt/core/bus/proto/bus.proto b/yt/yt_proto/yt/core/bus/proto/bus.proto index daf9d5f083..96638368f5 100644 --- a/yt/yt_proto/yt/core/bus/proto/bus.proto +++ b/yt/yt_proto/yt/core/bus/proto/bus.proto @@ -4,10 +4,12 @@ import "yt_proto/yt/core/misc/proto/guid.proto"; message THandshake { - required NYT.NProto.TGuid foreign_connection_id = 1; + required NYT.NProto.TGuid connection_id = 1; // Only passed from client to server. optional int32 multiplexing_band = 2; // EMultiplexingBand optional int32 encryption_mode = 3; // EEncryptionMode + + optional int32 verification_mode = 4; // EVerificationMode } |